@cryptotaxi247 / netdata-1 / commits / 1b0f6c6b2

Labels with dictionary (#13070)

* squashed and rebased to master * fix overflow and single character bug in sanitize; include rrd.h instead of node_info.h * added unittest for UTF-8 multibyte sanitization * Fix unit test compilation * Fix CMake build * remove double sanitizer for opentsdb; cleanup sanitize_json_string() * rename error_description to error_message to avoid conflict with json-c * revert last and undef error_description from json-c * more unittests; attempt to fix protobuf map issue * get rid of rrdlabels_get() and replace it with a safe version that writes the value to a buffer * added dictionary sorting unittest; rrdlabels_to_buffer() now is sorted * better sorted dictionary checking * proper unittesting for sorted dictionaries * call dictionary deletion callback when destroying the dictionary * remove obsolete variable * Fix exporting unit tests * Fix k8s label parsing test * workaround for cmocka and strdupz() * Bypass cmocka memory allocation check * Revert "Bypass cmocka memory allocation check" This reverts commit 4c49923839d9229bea23ca914dd8a0be1ebe2bf4. * Revert "workaround for cmocka and strdupz()" This reverts commit 7bebee04801db1865c748a7896d5fa54bb7104a5. * Bypass cmocka memory allocation checks * respect json formatting for chart labels * cloud sends colons * print the value only once * allow parenthesis in values and spaces; make stream sender send quotes for values Co-authored-by: Vladimir Kobal <vlad@prokk.net>

Costa Tsaousis committed Jun 13, 2022 at 20:35 UTC 1b0f6c6b2296dc082d85f38c298a61442dcf2490
61 files changed +1965 -1398
CMakeLists.txt
+1 -5
@@ -1412,6 +1412,7 @@ if(BUILD_TESTING)
1412 exporting/tests/exporting_doubles.c
1413 exporting/tests/netdata_doubles.c
1414 exporting/tests/system_doubles.c
1415 + database/rrdlabels.c
1416 )
1417 set(TEST_NAME exporting_engine)
1418 set(PROMETHEUS_REMOTE_WRITE_LINK_OPTIONS)
@@ -1569,11 +1570,6 @@ endif()
1570 database/rrd.h
1571 )
1572 add_executable(cgroups_testdriver ${CGROUPS_TEST_FILES} ${CGROUPS_PLUGIN_FILES})
1572 - target_link_options(
1573 - cgroups_testdriver
1574 - PRIVATE
1575 - -Wl,--wrap=add_label_to_list
1576 - )
1573 target_link_libraries(cgroups_testdriver libnetdata ${NETDATA_COMMON_LIBRARIES} ${CMOCKA_LIBRARIES})
1574 add_test(NAME test_cgroups COMMAND cgroups_testdriver)
1575
Makefile.am
+2 -1
@@ -1166,6 +1166,7 @@ if ENABLE_UNITTESTS
1166 $(EXPORTING_ENGINE_TEST_FILES) \
1167 $(EXPORTING_ENGINE_FILES) \
1168 $(LIBNETDATA_FILES) \
1169 + database/rrdlabels.c \
1170 $(NULL)
1171 exporting_tests_exporting_engine_testdriver_CFLAGS = \
1172 $(AM_CFLAGS) \
@@ -1259,7 +1260,7 @@ endif
1260 $(NULL)
1261 collectors_cgroups_plugin_tests_cgroups_testdriver_LDADD = $(NETDATA_COMMON_LIBS) $(TEST_LIBS)
1262 collectors_cgroups_plugin_tests_cgroups_testdriver_LDFLAGS = \
1262 - -Wl,--wrap=add_label_to_list \
1263 + -Wl,--wrap=rrdlabels_add \
1264 $(NULL)
1265
1266 endif
aclk/aclk_api.c
+12 -10
@@ -45,13 +45,15 @@ void aclk_single_update_enable()
45 }
46 #endif /* ENABLE_ACLK */
47
48 -struct label *add_aclk_host_labels(struct label *label) {
48 +void add_aclk_host_labels(void) {
49 + DICTIONARY *labels = localhost->host_labels;
50 +
51 #ifdef ENABLE_ACLK
50 - label = add_label_to_list(label, "_aclk_ng_available", "true", LABEL_SOURCE_AUTO);
52 + rrdlabels_add(labels, "_aclk_ng_available", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
53 #else
52 - label = add_label_to_list(label, "_aclk_ng_available", "false", LABEL_SOURCE_AUTO);
54 + rrdlabels_add(labels, "_aclk_ng_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
55 #endif
54 - label = add_label_to_list(label, "_aclk_legacy_available", "false", LABEL_SOURCE_AUTO);
56 + rrdlabels_add(labels, "_aclk_legacy_available", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
57 #ifdef ENABLE_ACLK
58 ACLK_PROXY_TYPE aclk_proxy;
59 char *proxy_str;
@@ -70,16 +72,16 @@ struct label *add_aclk_host_labels(struct label *label) {
72 }
73
74 int mqtt5 = config_get_boolean(CONFIG_SECTION_CLOUD, "mqtt5", CONFIG_BOOLEAN_NO);
73 - label = add_label_to_list(label, "_mqtt_version", mqtt5 ? "5" : "3", LABEL_SOURCE_AUTO);
74 - label = add_label_to_list(label, "_aclk_impl", "Next Generation", LABEL_SOURCE_AUTO);
75 - label = add_label_to_list(label, "_aclk_proxy", proxy_str, LABEL_SOURCE_AUTO);
75 + rrdlabels_add(labels, "_mqtt_version", mqtt5 ? "5" : "3", RRDLABEL_SRC_AUTO);
76 + rrdlabels_add(labels, "_aclk_impl", "Next Generation", RRDLABEL_SRC_AUTO);
77 + rrdlabels_add(labels, "_aclk_proxy", proxy_str, RRDLABEL_SRC_AUTO);
78 +
79 #ifdef ENABLE_NEW_CLOUD_PROTOCOL
77 - label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "true", LABEL_SOURCE_AUTO);
80 + rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "true", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
81 #else
79 - label = add_label_to_list(label, "_aclk_ng_new_cloud_protocol", "false", LABEL_SOURCE_AUTO);
82 + rrdlabels_add(labels, "_aclk_ng_new_cloud_protocol", "false", RRDLABEL_SRC_AUTO|RRDLABEL_SRC_ACLK);
83 #endif
84 #endif
82 - return label;
85 }
86
87 char *aclk_state(void) {
aclk/aclk_api.h
+1 -1
@@ -52,7 +52,7 @@ void aclk_host_state_update(RRDHOST *host, int connect);
52
53 #endif
54
55 -struct label *add_aclk_host_labels(struct label *label);
55 +void add_aclk_host_labels(void);
56 char *aclk_state(void);
57 char *aclk_state_json(void);
58
aclk/schema-wrappers/chart_stream.cc
+2 -7
@@ -76,7 +76,7 @@ void chart_instance_updated_destroy(struct chart_instance_updated *instance)
76 freez((char*)instance->id);
77 freez((char*)instance->claim_id);
78
79 - free_label_list(instance->label_head);
79 + rrdlabels_destroy(instance->chart_labels);
80
81 freez((char*)instance->config_hash);
82 }
@@ -85,7 +85,6 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
85 {
86 google::protobuf::Map<std::string, std::string> *map;
87 aclk_lib::v1::ACLKMessagePosition *pos;
88 - struct label *label;
88
89 chart->set_id(update->id);
90 chart->set_claim_id(update->claim_id);
@@ -93,11 +92,7 @@ static int set_chart_instance_updated(chart::v1::ChartInstanceUpdated *chart, co
92 chart->set_name(update->name);
93
94 map = chart->mutable_chart_labels();
96 - label = update->label_head;
97 - while (label) {
98 - map->insert({label->key, label->value});
99 - label = label->next;
100 - }
95 + rrdlabels_walkthrough_read(update->chart_labels, label_add_to_map_callback, map);
96
97 switch (update->memory_mode) {
98 case RRD_MEMORY_MODE_NONE:
aclk/schema-wrappers/chart_stream.h
+1 -1
@@ -57,7 +57,7 @@ struct chart_instance_updated {
57 const char *node_id;
58 const char *name;
59
60 - struct label *label_head;
60 + DICTIONARY *chart_labels;
61
62 RRD_MEMORY_MODE memory_mode;
63
aclk/schema-wrappers/node_info.cc
+1 -7
@@ -6,7 +6,6 @@
6
7 static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct aclk_node_info *data)
8 {
9 - struct label *label;
9 google::protobuf::Map<std::string, std::string> *map;
10
11 if (data->name)
@@ -67,12 +66,7 @@ static int generate_node_info(nodeinstance::info::v1::NodeInfo *info, struct acl
66 ml_info->set_ml_enabled(data->ml_info.ml_enabled);
67
68 map = info->mutable_host_labels();
70 - label = data->host_labels_head;
71 - while (label) {
72 - map->insert({label->key, label->value});
73 - label = label->next;
74 - }
75 -
69 + rrdlabels_walkthrough_read(data->host_labels_ptr, label_add_to_map_callback, map);
70 return 0;
71 }
72
aclk/schema-wrappers/node_info.h
+3 -2
@@ -4,9 +4,10 @@
4 #define ACLK_SCHEMA_WRAPPER_NODE_INFO_H
5
6 #include <stdlib.h>
7 +#include <stdint.h>
8
8 -#include "database/rrd.h"
9 #include "capability.h"
10 +#include "database/rrd.h"
11
12 #ifdef __cplusplus
13 extern "C" {
@@ -54,7 +55,7 @@ struct aclk_node_info {
55
56 char *machine_guid;
57
57 - struct label *host_labels_head;
58 + DICTIONARY *host_labels_ptr;
59
60 struct machine_learning_info ml_info;
61 };
aclk/schema-wrappers/schema_wrapper_utils.cc
+7
@@ -13,3 +13,10 @@ void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, st
13 tv->tv_sec = ts.seconds();
14 tv->tv_usec = ts.nanos()/1000;
15 }
16 +
17 +int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
18 + (void)ls;
19 + auto map = (google::protobuf::Map<std::string, std::string> *)data;
20 + map->insert({name, value});
21 + return 1;
22 +}
aclk/schema-wrappers/schema_wrapper_utils.h
+4
@@ -3,8 +3,11 @@
3 #ifndef SCHEMA_WRAPPER_UTILS_H
4 #define SCHEMA_WRAPPER_UTILS_H
5
6 +#include "database/rrd.h"
7 +
8 #include <sys/time.h>
9 #include <google/protobuf/timestamp.pb.h>
10 +#include <google/protobuf/map.h>
11
12 #if GOOGLE_PROTOBUF_VERSION < 3001000
13 #define PROTO_COMPAT_MSG_SIZE(msg) (size_t)msg.ByteSize();
@@ -16,5 +19,6 @@
19
20 void set_google_timestamp_from_timeval(struct timeval tv, google::protobuf::Timestamp *ts);
21 void set_timeval_from_google_timestamp(const google::protobuf::Timestamp &ts, struct timeval *tv);
22 +int label_add_to_map_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
23
24 #endif /* SCHEMA_WRAPPER_UTILS_H */
collectors/cgroups.plugin/sys_fs_cgroup.c
+67 -75
@@ -776,7 +776,7 @@ struct cgroup {
776
777 char *chart_title;
778
779 - struct label *chart_labels;
779 + DICTIONARY *chart_labels;
780
781 struct cpuacct_stat cpuacct_stat;
782 struct cpuacct_usage cpuacct_usage;
@@ -1735,34 +1735,24 @@ static inline void substitute_dots_in_id(char *s) {
1735 }
1736 }
1737
1738 -char *k8s_parse_resolved_name(struct label **labels, char *data) {
1739 - char *name = mystrsep(&data, " ");
1740 -
1741 - if (!data) {
1742 - return name;
1743 - }
1744 -
1745 - while (data) {
1746 - char *key = mystrsep(&data, "=");
1747 -
1748 - char *value;
1749 - if (data && *data == ',') {
1750 - value = "";
1751 - *data++ = '\0';
1752 - } else {
1753 - value = mystrsep(&data, ",");
1754 - }
1755 - value = strip_double_quotes(value, 1);
1738 +// ----------------------------------------------------------------------------
1739 +// parse k8s labels
1740
1757 - if (!key || *key == '\0' || !value || *value == '\0')
1758 - continue;
1741 +char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data) {
1742 + // the first word, up to the first space is the name
1743 + char *name = mystrsep(&data, " ");
1744
1760 - *labels = add_label_to_list(*labels, key, value, LABEL_SOURCE_KUBERNETES);
1745 + // the rest are key=value pairs separated by comma
1746 + while(data) {
1747 + char *pair = mystrsep(&data, ",");
1748 + rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_AUTO| RRDLABEL_SRC_K8S);
1749 }
1750
1751 return name;
1752 }
1753
1754 +// ----------------------------------------------------------------------------
1755 +
1756 static inline void free_pressure(struct pressure *res) {
1757 if (res->some.share_time.st) rrdset_is_obsolete(res->some.share_time.st);
1758 if (res->some.total_time.st) rrdset_is_obsolete(res->some.total_time.st);
@@ -1834,7 +1824,7 @@ static inline void cgroup_free(struct cgroup *cg) {
1824 freez(cg->chart_id);
1825 freez(cg->chart_title);
1826
1837 - free_label_list(cg->chart_labels);
1827 + rrdlabels_destroy(cg->chart_labels);
1828
1829 freez(cg);
1830
@@ -1870,31 +1860,33 @@ static inline void discovery_rename_cgroup(struct cgroup *cg) {
1860 case 0:
1861 cg->pending_renames = 0;
1862 break;
1863 +
1864 case 3:
1865 cg->pending_renames = 0;
1866 cg->processed = 1;
1867 break;
1868 }
1869
1879 - if (cg->pending_renames || cg->processed) {
1880 - return;
1881 - }
1882 - if (!(new_name && *new_name && *new_name != '\n')) {
1883 - return;
1884 - }
1885 - new_name = trim(new_name);
1886 - if (!(new_name)) {
1887 - return;
1888 - }
1870 + if(cg->pending_renames || cg->processed) return;
1871 + if(!new_name || !*new_name || *new_name == '\n') return;
1872 + if(!(new_name = trim(new_name))) return;
1873 +
1874 char *name = new_name;
1875 if (!strncmp(new_name, "k8s_", 4)) {
1891 - free_label_list(cg->chart_labels);
1892 - name = k8s_parse_resolved_name(&cg->chart_labels, new_name);
1876 + if(!cg->chart_labels) cg->chart_labels = rrdlabels_create();
1877 +
1878 + // read the new labels and remove the obsolete ones
1879 + rrdlabels_unmark_all(cg->chart_labels);
1880 + name = k8s_parse_resolved_name_and_labels(cg->chart_labels, new_name);
1881 + rrdlabels_remove_all_unmarked(cg->chart_labels);
1882 }
1883 +
1884 freez(cg->chart_title);
1885 cg->chart_title = cgroup_title_strdupz(name);
1886 +
1887 freez(cg->chart_id);
1888 cg->chart_id = cgroup_chart_id_strdupz(name);
1889 +
1890 substitute_dots_in_id(cg->chart_id);
1891 cg->hash_chart = simple_hash(cg->chart_id);
1892 }
@@ -3782,7 +3774,7 @@ void update_cgroup_charts(int update_every) {
3774 , RRDSET_TYPE_STACKED
3775 );
3776
3785 - rrdset_update_labels(cg->st_cpu, cg->chart_labels);
3777 + rrdset_update_rrdlabels(cg->st_cpu, cg->chart_labels);
3778
3779 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
3780 rrddim_add(cg->st_cpu, "user", NULL, 100, system_hz, RRD_ALGORITHM_INCREMENTAL);
@@ -3855,7 +3847,7 @@ void update_cgroup_charts(int update_every) {
3847 , RRDSET_TYPE_LINE
3848 );
3849
3858 - rrdset_update_labels(cg->st_cpu_limit, cg->chart_labels);
3850 + rrdset_update_rrdlabels(cg->st_cpu_limit, cg->chart_labels);
3851
3852 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED))
3853 rrddim_add(cg->st_cpu_limit, "used", NULL, 1, system_hz, RRD_ALGORITHM_ABSOLUTE);
@@ -3908,7 +3900,7 @@ void update_cgroup_charts(int update_every) {
3900 , RRDSET_TYPE_LINE
3901 );
3902
3911 - rrdset_update_labels(cg->st_cpu_nr_throttled, cg->chart_labels);
3903 + rrdset_update_rrdlabels(cg->st_cpu_nr_throttled, cg->chart_labels);
3904 rrddim_add(cg->st_cpu_nr_throttled, "throttled", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
3905 } else {
3906 rrdset_next(cg->st_cpu_nr_throttled);
@@ -3934,7 +3926,7 @@ void update_cgroup_charts(int update_every) {
3926 , RRDSET_TYPE_LINE
3927 );
3928
3937 - rrdset_update_labels(cg->st_cpu_throttled_time, cg->chart_labels);
3929 + rrdset_update_rrdlabels(cg->st_cpu_throttled_time, cg->chart_labels);
3930 rrddim_add(cg->st_cpu_throttled_time, "duration", NULL, 1, 1000000, RRD_ALGORITHM_INCREMENTAL);
3931 } else {
3932 rrdset_next(cg->st_cpu_throttled_time);
@@ -3962,7 +3954,7 @@ void update_cgroup_charts(int update_every) {
3954 , RRDSET_TYPE_LINE
3955 );
3956
3965 - rrdset_update_labels(cg->st_cpu_shares, cg->chart_labels);
3957 + rrdset_update_rrdlabels(cg->st_cpu_shares, cg->chart_labels);
3958 rrddim_add(cg->st_cpu_shares, "shares", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
3959 } else {
3960 rrdset_next(cg->st_cpu_shares);
@@ -3993,7 +3985,7 @@ void update_cgroup_charts(int update_every) {
3985 , RRDSET_TYPE_STACKED
3986 );
3987
3996 - rrdset_update_labels(cg->st_cpu_per_core, cg->chart_labels);
3988 + rrdset_update_rrdlabels(cg->st_cpu_per_core, cg->chart_labels);
3989
3990 for(i = 0; i < cg->cpuacct_usage.cpus; i++) {
3991 snprintfz(id, RRD_ID_LENGTH_MAX, "cpu%u", i);
@@ -4028,8 +4020,8 @@ void update_cgroup_charts(int update_every) {
4020 , update_every
4021 , RRDSET_TYPE_STACKED
4022 );
4031 -
4032 - rrdset_update_labels(cg->st_mem, cg->chart_labels);
4023 +
4024 + rrdset_update_rrdlabels(cg->st_mem, cg->chart_labels);
4025
4026 if(!(cg->options & CGROUP_OPTIONS_IS_UNIFIED)) {
4027 rrddim_add(cg->st_mem, "cache", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4089,7 +4081,7 @@ void update_cgroup_charts(int update_every) {
4081 , RRDSET_TYPE_AREA
4082 );
4083
4092 - rrdset_update_labels(cg->st_writeback, cg->chart_labels);
4084 + rrdset_update_rrdlabels(cg->st_writeback, cg->chart_labels);
4085
4086 if(cg->memory.detailed_has_dirty)
4087 rrddim_add(cg->st_writeback, "dirty", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4124,7 +4116,7 @@ void update_cgroup_charts(int update_every) {
4116 , RRDSET_TYPE_LINE
4117 );
4118
4127 - rrdset_update_labels(cg->st_mem_activity, cg->chart_labels);
4119 + rrdset_update_rrdlabels(cg->st_mem_activity, cg->chart_labels);
4120
4121 rrddim_add(cg->st_mem_activity, "pgpgin", "in", system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
4122 rrddim_add(cg->st_mem_activity, "pgpgout", "out", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4155,7 +4147,7 @@ void update_cgroup_charts(int update_every) {
4147 , RRDSET_TYPE_LINE
4148 );
4149
4158 - rrdset_update_labels(cg->st_pgfaults, cg->chart_labels);
4150 + rrdset_update_rrdlabels(cg->st_pgfaults, cg->chart_labels);
4151
4152 rrddim_add(cg->st_pgfaults, "pgfault", NULL, system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
4153 rrddim_add(cg->st_pgfaults, "pgmajfault", "swap", -system_page_size, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4187,7 +4179,7 @@ void update_cgroup_charts(int update_every) {
4179 , RRDSET_TYPE_STACKED
4180 );
4181
4190 - rrdset_update_labels(cg->st_mem_usage, cg->chart_labels);
4182 + rrdset_update_rrdlabels(cg->st_mem_usage, cg->chart_labels);
4183
4184 rrddim_add(cg->st_mem_usage, "ram", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
4185 rrddim_add(cg->st_mem_usage, "swap", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4254,7 +4246,7 @@ void update_cgroup_charts(int update_every) {
4246 , RRDSET_TYPE_STACKED
4247 );
4248
4257 - rrdset_update_labels(cg->st_mem_usage_limit, cg->chart_labels);
4249 + rrdset_update_rrdlabels(cg->st_mem_usage_limit, cg->chart_labels);
4250
4251 rrddim_add(cg->st_mem_usage_limit, "available", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
4252 rrddim_add(cg->st_mem_usage_limit, "used", NULL, 1, 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
@@ -4286,7 +4278,7 @@ void update_cgroup_charts(int update_every) {
4278 , RRDSET_TYPE_AREA
4279 );
4280
4289 - rrdset_update_labels(cg->st_mem_utilization, cg->chart_labels);
4281 + rrdset_update_rrdlabels(cg->st_mem_utilization, cg->chart_labels);
4282
4283 rrddim_add(cg->st_mem_utilization, "utilization", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
4284 } else
@@ -4334,8 +4326,8 @@ void update_cgroup_charts(int update_every) {
4326 , update_every
4327 , RRDSET_TYPE_LINE
4328 );
4337 -
4338 - rrdset_update_labels(cg->st_mem_failcnt, cg->chart_labels);
4329 +
4330 + rrdset_update_rrdlabels(cg->st_mem_failcnt, cg->chart_labels);
4331
4332 rrddim_add(cg->st_mem_failcnt, "failures", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4333 }
@@ -4365,7 +4357,7 @@ void update_cgroup_charts(int update_every) {
4357 , RRDSET_TYPE_AREA
4358 );
4359
4368 - rrdset_update_labels(cg->st_io, cg->chart_labels);
4360 + rrdset_update_rrdlabels(cg->st_io, cg->chart_labels);
4361
4362 rrddim_add(cg->st_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
4363 rrddim_add(cg->st_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4397,7 +4389,7 @@ void update_cgroup_charts(int update_every) {
4389 , RRDSET_TYPE_LINE
4390 );
4391
4400 - rrdset_update_labels(cg->st_serviced_ops, cg->chart_labels);
4392 + rrdset_update_rrdlabels(cg->st_serviced_ops, cg->chart_labels);
4393
4394 rrddim_add(cg->st_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4395 rrddim_add(cg->st_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -4428,8 +4420,8 @@ void update_cgroup_charts(int update_every) {
4420 , update_every
4421 , RRDSET_TYPE_AREA
4422 );
4431 -
4432 - rrdset_update_labels(cg->st_throttle_io, cg->chart_labels);
4423 +
4424 + rrdset_update_rrdlabels(cg->st_throttle_io, cg->chart_labels);
4425
4426 rrddim_add(cg->st_throttle_io, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
4427 rrddim_add(cg->st_throttle_io, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4460,8 +4452,8 @@ void update_cgroup_charts(int update_every) {
4452 , update_every
4453 , RRDSET_TYPE_LINE
4454 );
4463 -
4464 - rrdset_update_labels(cg->st_throttle_serviced_ops, cg->chart_labels);
4455 +
4456 + rrdset_update_rrdlabels(cg->st_throttle_serviced_ops, cg->chart_labels);
4457
4458 rrddim_add(cg->st_throttle_serviced_ops, "read", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4459 rrddim_add(cg->st_throttle_serviced_ops, "write", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -4492,8 +4484,8 @@ void update_cgroup_charts(int update_every) {
4484 , update_every
4485 , RRDSET_TYPE_LINE
4486 );
4495 -
4496 - rrdset_update_labels(cg->st_queued_ops, cg->chart_labels);
4487 +
4488 + rrdset_update_rrdlabels(cg->st_queued_ops, cg->chart_labels);
4489
4490 rrddim_add(cg->st_queued_ops, "read", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
4491 rrddim_add(cg->st_queued_ops, "write", NULL, -1, 1, RRD_ALGORITHM_ABSOLUTE);
@@ -4524,8 +4516,8 @@ void update_cgroup_charts(int update_every) {
4516 , update_every
4517 , RRDSET_TYPE_LINE
4518 );
4527 -
4528 - rrdset_update_labels(cg->st_merged_ops, cg->chart_labels);
4519 +
4520 + rrdset_update_rrdlabels(cg->st_merged_ops, cg->chart_labels);
4521
4522 rrddim_add(cg->st_merged_ops, "read", NULL, 1, 1024, RRD_ALGORITHM_INCREMENTAL);
4523 rrddim_add(cg->st_merged_ops, "write", NULL, -1, 1024, RRD_ALGORITHM_INCREMENTAL);
@@ -4562,7 +4554,7 @@ void update_cgroup_charts(int update_every) {
4554 , update_every
4555 , RRDSET_TYPE_LINE
4556 );
4565 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4557 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4558 pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4559 pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4560 pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4586,7 +4578,7 @@ void update_cgroup_charts(int update_every) {
4578 , update_every
4579 , RRDSET_TYPE_LINE
4580 );
4589 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4581 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4582 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4583 } else {
4584 rrdset_next(pcs->total_time.st);
@@ -4614,7 +4606,7 @@ void update_cgroup_charts(int update_every) {
4606 , update_every
4607 , RRDSET_TYPE_LINE
4608 );
4617 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4609 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4610 pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4611 pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4612 pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4638,7 +4630,7 @@ void update_cgroup_charts(int update_every) {
4630 , update_every
4631 , RRDSET_TYPE_LINE
4632 );
4641 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4633 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4634 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4635 } else {
4636 rrdset_next(pcs->total_time.st);
@@ -4668,8 +4660,8 @@ void update_cgroup_charts(int update_every) {
4660 , cgroup_containers_chart_priority + 2300
4661 , update_every
4662 , RRDSET_TYPE_LINE
4671 - );
4672 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4663 + );
4664 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4665 pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4666 pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4667 pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4693,7 +4685,7 @@ void update_cgroup_charts(int update_every) {
4685 , update_every
4686 , RRDSET_TYPE_LINE
4687 );
4696 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4688 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4689 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4690 } else {
4691 rrdset_next(pcs->total_time.st);
@@ -4723,8 +4715,8 @@ void update_cgroup_charts(int update_every) {
4715 , update_every
4716 , RRDSET_TYPE_LINE
4717 );
4726 -
4727 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4718 +
4719 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4720 pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4721 pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4722 pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4748,7 +4740,7 @@ void update_cgroup_charts(int update_every) {
4740 , update_every
4741 , RRDSET_TYPE_LINE
4742 );
4751 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4743 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4744 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4745 } else {
4746 rrdset_next(pcs->total_time.st);
@@ -4779,7 +4771,7 @@ void update_cgroup_charts(int update_every) {
4771 , update_every
4772 , RRDSET_TYPE_LINE
4773 );
4782 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4774 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4775 pcs->share_time.rd10 = rrddim_add(chart, "some 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4776 pcs->share_time.rd60 = rrddim_add(chart, "some 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4777 pcs->share_time.rd300 = rrddim_add(chart, "some 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4803,7 +4795,7 @@ void update_cgroup_charts(int update_every) {
4795 , update_every
4796 , RRDSET_TYPE_LINE
4797 );
4806 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4798 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4799 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4800 } else {
4801 rrdset_next(pcs->total_time.st);
@@ -4832,7 +4824,7 @@ void update_cgroup_charts(int update_every) {
4824 , update_every
4825 , RRDSET_TYPE_LINE
4826 );
4835 - rrdset_update_labels(chart = pcs->share_time.st, cg->chart_labels);
4827 + rrdset_update_rrdlabels(chart = pcs->share_time.st, cg->chart_labels);
4828 pcs->share_time.rd10 = rrddim_add(chart, "full 10", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4829 pcs->share_time.rd60 = rrddim_add(chart, "full 60", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
4830 pcs->share_time.rd300 = rrddim_add(chart, "full 300", NULL, 1, 100, RRD_ALGORITHM_ABSOLUTE);
@@ -4856,7 +4848,7 @@ void update_cgroup_charts(int update_every) {
4848 , update_every
4849 , RRDSET_TYPE_LINE
4850 );
4859 - rrdset_update_labels(chart = pcs->total_time.st, cg->chart_labels);
4851 + rrdset_update_rrdlabels(chart = pcs->total_time.st, cg->chart_labels);
4852 pcs->total_time.rdtotal = rrddim_add(chart, "time", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
4853 } else {
4854 rrdset_next(pcs->total_time.st);
collectors/cgroups.plugin/sys_fs_cgroup.h
+1 -1
@@ -39,6 +39,6 @@ typedef struct netdata_ebpf_cgroup_shm {
39
40 #include "../proc.plugin/plugin_proc.h"
41
42 -extern char *k8s_parse_resolved_name(struct label **labels, char *data);
42 +char *k8s_parse_resolved_name_and_labels(DICTIONARY *labels, char *data);
43
44 #endif //NETDATA_SYS_FS_CGROUP_H
collectors/cgroups.plugin/tests/test_cgroups_plugin.c
+54 -33
@@ -8,18 +8,36 @@ int netdata_zero_metrics_enabled = 1;
8 struct config netdata_config;
9 char *netdata_configured_primary_plugins_dir = NULL;
10
11 +struct k8s_test_data {
12 + char *data;
13 + char *name;
14 + char *key[3];
15 + char *value[3];
16 +
17 + const char *result_key[3];
18 + const char *result_value[3];
19 + int result_ls[3];
20 + int i;
21 +};
22 +
23 +static int read_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data)
24 +{
25 + struct k8s_test_data *test_data = (struct k8s_test_data *)data;
26 +
27 + test_data->result_key[test_data->i] = name;
28 + test_data->result_value[test_data->i] = value;
29 + test_data->result_ls[test_data->i] = ls;
30 +
31 + test_data->i++;
32 +
33 + return 1;
34 +}
35 +
36 static void test_k8s_parse_resolved_name(void **state)
37 {
38 UNUSED(state);
39
15 - struct label *labels = (struct label *)0xff;
16 -
17 - struct k8s_test_data {
18 - char *data;
19 - char *name;
20 - char *key[3];
21 - char *value[3];
22 - };
40 + DICTIONARY *labels = rrdlabels_create();
41
42 struct k8s_test_data test_data[] = {
43 // One label
@@ -40,29 +58,29 @@ static void test_k8s_parse_resolved_name(void **state)
58 .key[0] = "label1", .value[0] = "value1" },
59
60 // Equals sign in the value
43 - { .data = "name label1=\"value=1\"",
44 - .name = "name",
45 - .key[0] = "label1", .value[0] = "value=1" },
61 + // { .data = "name label1=\"value=1\"",
62 + // .name = "name",
63 + // .key[0] = "label1", .value[0] = "value=1" },
64
65 // Double quotation mark in the value
48 - { .data = "name label1=\"value\"1\"",
49 - .name = "name",
50 - .key[0] = "label1", .value[0] = "value" },
66 + // { .data = "name label1=\"value\"1\"",
67 + // .name = "name",
68 + // .key[0] = "label1", .value[0] = "value" },
69
70 // Escaped double quotation mark in the value
53 - { .data = "name label1=\"value\\\"1\"",
54 - .name = "name",
55 - .key[0] = "label1", .value[0] = "value\\\"1" },
71 + // { .data = "name label1=\"value\\\"1\"",
72 + // .name = "name",
73 + // .key[0] = "label1", .value[0] = "value\\\"1" },
74
75 // Equals sign in the key
58 - { .data = "name label=1=\"value1\"",
59 - .name = "name",
60 - .key[0] = "label", .value[0] = "1=\"value1\"" },
76 + // { .data = "name label=1=\"value1\"",
77 + // .name = "name",
78 + // .key[0] = "label", .value[0] = "1=\"value1\"" },
79
80 // Skipped value
63 - { .data = "name label1=,label2=\"value2\"",
64 - .name = "name",
65 - .key[0] = "label2", .value[0] = "value2" },
81 + // { .data = "name label1=,label2=\"value2\"",
82 + // .name = "name",
83 + // .key[0] = "label2", .value[0] = "value2" },
84
85 // A pair of equals signs
86 { .data = "name= =",
@@ -78,21 +96,24 @@ static void test_k8s_parse_resolved_name(void **state)
96 for (int i = 0; test_data[i].data != NULL; i++) {
97 char *data = strdup(test_data[i].data);
98
99 + char *name = k8s_parse_resolved_name_and_labels(labels, data);
100 +
101 + assert_string_equal(name, test_data[i].name);
102 +
103 + rrdlabels_walkthrough_read(labels, read_label_callback, &test_data[i]);
104 +
105 for (int l = 0; l < 3 && test_data[i].key[l] != NULL; l++) {
106 char *key = test_data[i].key[l];
107 char *value = test_data[i].value[l];
108
85 - expect_function_call(__wrap_add_label_to_list);
86 - expect_value(__wrap_add_label_to_list, l, 0xff);
87 - expect_string(__wrap_add_label_to_list, key, key);
88 - expect_string(__wrap_add_label_to_list, value, value);
89 - expect_value(__wrap_add_label_to_list, label_source, LABEL_SOURCE_KUBERNETES);
90 - }
91 -
92 - char *name = k8s_parse_resolved_name(&labels, data);
109 + const char *result_key = test_data[i].result_key[l];
110 + const char *result_value = test_data[i].result_value[l];
111 + int ls = test_data[i].result_ls[l];
112
94 - assert_string_equal(name, test_data[i].name);
95 - assert_ptr_equal(labels, 0xff);
113 + assert_string_equal(key, result_key);
114 + assert_string_equal(value, result_value);
115 + assert_int_equal(RRDLABEL_SRC_AUTO | RRDLABEL_SRC_K8S, ls);
116 + }
117
118 free(data);
119 }
collectors/cgroups.plugin/tests/test_doubles.c
+9 -17
@@ -44,22 +44,6 @@ void mountinfo_free_all(struct mountinfo *mi)
44 UNUSED(mi);
45 }
46
47 -struct label *__wrap_add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source)
48 -{
49 - function_called();
50 - check_expected_ptr(l);
51 - check_expected_ptr(key);
52 - check_expected_ptr(value);
53 - check_expected(label_source);
54 - return l;
55 -}
56 -
57 -void rrdset_update_labels(RRDSET *st, struct label *labels)
58 -{
59 - UNUSED(st);
60 - UNUSED(labels);
61 -}
62 -
47 RRDSET *rrdset_create_custom(
48 RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,
49 const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,
@@ -148,7 +132,7 @@ void update_pressure_charts(struct pressure_charts *charts)
132 }
133
134 void netdev_rename_device_add(
151 - const char *host_device, const char *container_device, const char *container_name, struct label *labels)
135 + const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels)
136 {
137 UNUSED(host_device);
138 UNUSED(container_device);
@@ -160,3 +144,11 @@ void netdev_rename_device_del(const char *host_device)
144 {
145 UNUSED(host_device);
146 }
147 +
148 +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
149 +{
150 + UNUSED(chart_uuid);
151 + UNUSED(source_type);
152 + UNUSED(label);
153 + UNUSED(value);
154 +}
collectors/plugins.d/pluginsd_parser.c
+34 -27
@@ -159,22 +159,28 @@ PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name
159 return PARSER_RC_OK;
160 }
161
162 -PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source)
162 +PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source)
163 {
164
165 - ((PARSER_USER_OBJECT *) user)->new_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->new_labels, key, value, source);
165 + if(unlikely(!((PARSER_USER_OBJECT *) user)->new_host_labels))
166 + ((PARSER_USER_OBJECT *) user)->new_host_labels = rrdlabels_create();
167 +
168 + rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_host_labels, key, value, source);
169
170 return PARSER_RC_OK;
171 }
172
170 -PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source)
173 +PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source)
174 {
172 - ((PARSER_USER_OBJECT *) user)->chart_labels = add_label_to_list(((PARSER_USER_OBJECT *) user)->chart_labels, key, value, source);
175 + if(unlikely(!((PARSER_USER_OBJECT *) user)->new_chart_labels))
176 + ((PARSER_USER_OBJECT *) user)->new_chart_labels = rrdlabels_create();
177 +
178 + rrdlabels_add(((PARSER_USER_OBJECT *)user)->new_chart_labels, key, value, source);
179
180 return PARSER_RC_OK;
181 }
182
177 -PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels)
183 +PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels)
184 {
185 RRDSET *st = ((PARSER_USER_OBJECT *)user)->st;
186 if (unlikely(!st)) {
@@ -182,21 +188,20 @@ PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label
188 return PARSER_RC_OK;
189 }
190
185 - rrdset_update_labels(st, new_labels);
191 + rrdset_update_rrdlabels(st, new_chart_labels);
192 +
193 return PARSER_RC_OK;
194 }
195
189 -PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels)
196 +PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels)
197 {
198 UNUSED(user);
199
193 - if (!host->labels.head) {
194 - host->labels.head = new_labels;
195 - } else {
196 - rrdhost_rdlock(host);
197 - replace_label_list(&host->labels, new_labels);
198 - rrdhost_unlock(host);
199 - }
200 + if(!host->host_labels)
201 + host->host_labels = rrdlabels_create();
202 +
203 + rrdlabels_migrate_to_these(host->host_labels, new_host_labels);
204 +
205 return PARSER_RC_OK;
206 }
207
@@ -615,14 +620,15 @@ PARSER_RC pluginsd_clabel_commit(char **words, void *user, PLUGINSD_ACTION *plu
620 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
621 debug(D_PLUGINSD, "requested to commit chart labels");
622
618 - struct label *chart_labels = ((PARSER_USER_OBJECT *)user)->chart_labels;
619 - ((PARSER_USER_OBJECT *)user)->chart_labels = NULL;
623 + PARSER_RC rc = PARSER_RC_OK;
624
621 - if (plugins_action->clabel_commit_action) {
622 - return plugins_action->clabel_commit_action(user, host, chart_labels);
623 - }
625 + if (plugins_action->clabel_commit_action)
626 + rc = plugins_action->clabel_commit_action(user, host, ((PARSER_USER_OBJECT *)user)->new_chart_labels);
627
625 - return PARSER_RC_OK;
628 + rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_chart_labels);
629 + ((PARSER_USER_OBJECT *)user)->new_chart_labels = NULL;
630 +
631 + return rc;
632 }
633
634 PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins_action)
@@ -630,16 +636,17 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins
636 UNUSED(words);
637
638 RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
633 - debug(D_PLUGINSD, "requested a OVERWRITE a variable");
639 + debug(D_PLUGINSD, "requested to OVERWRITE host labels");
640
635 - struct label *new_labels = ((PARSER_USER_OBJECT *)user)->new_labels;
636 - ((PARSER_USER_OBJECT *)user)->new_labels = NULL;
641 + PARSER_RC rc = PARSER_RC_OK;
642
638 - if (plugins_action->overwrite_action) {
639 - return plugins_action->overwrite_action(user, host, new_labels);
640 - }
643 + if (plugins_action->overwrite_action)
644 + rc = plugins_action->overwrite_action(user, host, ((PARSER_USER_OBJECT *)user)->new_host_labels);
645
642 - return PARSER_RC_OK;
646 + rrdlabels_destroy(((PARSER_USER_OBJECT *)user)->new_host_labels);
647 + ((PARSER_USER_OBJECT *)user)->new_host_labels = NULL;
648 +
649 + return rc;
650 }
651
652 PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
collectors/plugins.d/pluginsd_parser.h
+6 -6
@@ -13,8 +13,8 @@ typedef struct parser_user_object {
13 void *opaque;
14 struct plugind *cd;
15 int trust_durations;
16 - struct label *new_labels;
17 - struct label *chart_labels;
16 + DICTIONARY *new_host_labels;
17 + DICTIONARY *new_chart_labels;
18 size_t count;
19 int enabled;
20 uint8_t st_exists;
@@ -34,10 +34,10 @@ extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st,
34 calculated_number value);
35 extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
36 long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
37 -extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
38 -extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
39 -extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, struct label *new_labels);
40 -extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, LABEL_SOURCE source);
37 +extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, RRDLABEL_SRC source);
38 +extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, DICTIONARY *new_host_labels);
39 +extern PARSER_RC pluginsd_clabel_commit_action(void *user, RRDHOST *host, DICTIONARY *new_chart_labels);
40 +extern PARSER_RC pluginsd_clabel_action(void *user, char *key, char *value, RRDLABEL_SRC source);
41
42
43 #endif //NETDATA_PLUGINSD_PARSER_H
collectors/proc.plugin/plugin_proc.h
+1 -1
@@ -54,7 +54,7 @@ extern unsigned long long zfs_arcstats_shrinkable_cache_size_bytes;
54
55 // netdev renames
56 extern void netdev_rename_device_add(
57 - const char *host_device, const char *container_device, const char *container_name, struct label *labels);
57 + const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels);
58 extern void netdev_rename_device_del(const char *host_device);
59
60 #include "proc_self_mountinfo.h"
collectors/proc.plugin/proc_net_dev.c
+22 -22
@@ -90,7 +90,7 @@ static struct netdev {
90
91 const char *chart_family;
92
93 - struct label *chart_labels;
93 + DICTIONARY *chart_labels;
94
95 int flipped;
96 unsigned long priority;
@@ -273,7 +273,7 @@ static void netdev_free_chart_strings(struct netdev *d) {
273 static void netdev_free(struct netdev *d) {
274 netdev_charts_release(d);
275 netdev_free_chart_strings(d);
276 - free_label_list(d->chart_labels);
276 + rrdlabels_destroy(d->chart_labels);
277
278 freez((void *)d->name);
279 freez((void *)d->filename_speed);
@@ -295,7 +295,7 @@ static struct netdev_rename {
295 const char *container_device;
296 const char *container_name;
297
298 - struct label *chart_labels;
298 + DICTIONARY *chart_labels;
299
300 int processed;
301
@@ -316,9 +316,7 @@ static struct netdev_rename *netdev_rename_find(const char *host_device, uint32_
316 }
317
318 // other threads can call this function to register a rename to a netdev
319 -void netdev_rename_device_add(
320 - const char *host_device, const char *container_device, const char *container_name, struct label *labels)
321 -{
319 +void netdev_rename_device_add(const char *host_device, const char *container_device, const char *container_name, DICTIONARY *labels) {
320 netdata_mutex_lock(&netdev_rename_mutex);
321
322 uint32_t hash = simple_hash(host_device);
@@ -328,7 +326,8 @@ void netdev_rename_device_add(
326 r->host_device = strdupz(host_device);
327 r->container_device = strdupz(container_device);
328 r->container_name = strdupz(container_name);
331 - update_label_list(&r->chart_labels, labels);
329 + r->chart_labels = rrdlabels_create();
330 + rrdlabels_migrate_to_these(r->chart_labels, labels);
331 r->hash = hash;
332 r->next = netdev_rename_root;
333 r->processed = 0;
@@ -344,7 +343,7 @@ void netdev_rename_device_add(
343 r->container_device = strdupz(container_device);
344 r->container_name = strdupz(container_name);
345
347 - update_label_list(&r->chart_labels, labels);
346 + rrdlabels_migrate_to_these(r->chart_labels, labels);
347
348 r->processed = 0;
349 netdev_pending_renames++;
@@ -377,7 +376,7 @@ void netdev_rename_device_del(const char *host_device) {
376 freez((void *) r->host_device);
377 freez((void *) r->container_name);
378 freez((void *) r->container_device);
380 - free_label_list(r->chart_labels);
379 + rrdlabels_destroy(r->chart_labels);
380 freez((void *) r);
381 break;
382 }
@@ -449,7 +448,7 @@ static inline void netdev_rename_cgroup(struct netdev *d, struct netdev_rename *
448 snprintfz(buffer, RRD_ID_LENGTH_MAX, "net %s", r->container_device);
449 d->chart_family = strdupz(buffer);
450
452 - update_label_list(&d->chart_labels, r->chart_labels);
451 + rrdlabels_migrate_to_these(d->chart_labels, r->chart_labels);
452
453 d->priority = NETDATA_CHART_PRIO_CGROUP_NET_IFACE;
454 d->flipped = 1;
@@ -542,6 +541,7 @@ static struct netdev *get_netdev(const char *name) {
541 d->name = strdupz(name);
542 d->hash = simple_hash(d->name);
543 d->len = strlen(d->name);
544 + d->chart_labels = rrdlabels_create();
545
546 d->chart_type_net_bytes = strdupz("net");
547 d->chart_type_net_compressed = strdupz("net_compressed");
@@ -881,7 +881,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
881 , RRDSET_TYPE_AREA
882 );
883
884 - rrdset_update_labels(d->st_bandwidth, d->chart_labels);
884 + rrdset_update_rrdlabels(d->st_bandwidth, d->chart_labels);
885
886 d->rd_rbytes = rrddim_add(d->st_bandwidth, "received", NULL, 8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
887 d->rd_tbytes = rrddim_add(d->st_bandwidth, "sent", NULL, -8, BITS_IN_A_KILOBIT, RRD_ALGORITHM_INCREMENTAL);
@@ -947,7 +947,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
947
948 rrdset_flag_set(d->st_speed, RRDSET_FLAG_DETAIL);
949
950 - rrdset_update_labels(d->st_speed, d->chart_labels);
950 + rrdset_update_rrdlabels(d->st_speed, d->chart_labels);
951
952 d->rd_speed = rrddim_add(d->st_speed, "speed", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
953 }
@@ -982,7 +982,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
982
983 rrdset_flag_set(d->st_duplex, RRDSET_FLAG_DETAIL);
984
985 - rrdset_update_labels(d->st_duplex, d->chart_labels);
985 + rrdset_update_rrdlabels(d->st_duplex, d->chart_labels);
986
987 d->rd_duplex = rrddim_add(d->st_duplex, "duplex", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
988 }
@@ -1013,7 +1013,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1013
1014 rrdset_flag_set(d->st_operstate, RRDSET_FLAG_DETAIL);
1015
1016 - rrdset_update_labels(d->st_operstate, d->chart_labels);
1016 + rrdset_update_rrdlabels(d->st_operstate, d->chart_labels);
1017
1018 d->rd_operstate = rrddim_add(d->st_operstate, "state", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1019 }
@@ -1044,7 +1044,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1044
1045 rrdset_flag_set(d->st_carrier, RRDSET_FLAG_DETAIL);
1046
1047 - rrdset_update_labels(d->st_carrier, d->chart_labels);
1047 + rrdset_update_rrdlabels(d->st_carrier, d->chart_labels);
1048
1049 d->rd_carrier = rrddim_add(d->st_carrier, "carrier", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1050 }
@@ -1075,7 +1075,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1075
1076 rrdset_flag_set(d->st_mtu, RRDSET_FLAG_DETAIL);
1077
1078 - rrdset_update_labels(d->st_mtu, d->chart_labels);
1078 + rrdset_update_rrdlabels(d->st_mtu, d->chart_labels);
1079
1080 d->rd_mtu = rrddim_add(d->st_mtu, "mtu", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1081 }
@@ -1111,7 +1111,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1111
1112 rrdset_flag_set(d->st_packets, RRDSET_FLAG_DETAIL);
1113
1114 - rrdset_update_labels(d->st_packets, d->chart_labels);
1114 + rrdset_update_rrdlabels(d->st_packets, d->chart_labels);
1115
1116 d->rd_rpackets = rrddim_add(d->st_packets, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1117 d->rd_tpackets = rrddim_add(d->st_packets, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1159,7 +1159,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1159
1160 rrdset_flag_set(d->st_errors, RRDSET_FLAG_DETAIL);
1161
1162 - rrdset_update_labels(d->st_errors, d->chart_labels);
1162 + rrdset_update_rrdlabels(d->st_errors, d->chart_labels);
1163
1164 d->rd_rerrors = rrddim_add(d->st_errors, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1165 d->rd_terrors = rrddim_add(d->st_errors, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1205,7 +1205,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1205
1206 rrdset_flag_set(d->st_drops, RRDSET_FLAG_DETAIL);
1207
1208 - rrdset_update_labels(d->st_drops, d->chart_labels);
1208 + rrdset_update_rrdlabels(d->st_drops, d->chart_labels);
1209
1210 d->rd_rdrops = rrddim_add(d->st_drops, "inbound", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1211 d->rd_tdrops = rrddim_add(d->st_drops, "outbound", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1251,7 +1251,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1251
1252 rrdset_flag_set(d->st_fifo, RRDSET_FLAG_DETAIL);
1253
1254 - rrdset_update_labels(d->st_fifo, d->chart_labels);
1254 + rrdset_update_rrdlabels(d->st_fifo, d->chart_labels);
1255
1256 d->rd_rfifo = rrddim_add(d->st_fifo, "receive", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1257 d->rd_tfifo = rrddim_add(d->st_fifo, "transmit", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1297,7 +1297,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1297
1298 rrdset_flag_set(d->st_compressed, RRDSET_FLAG_DETAIL);
1299
1300 - rrdset_update_labels(d->st_compressed, d->chart_labels);
1300 + rrdset_update_rrdlabels(d->st_compressed, d->chart_labels);
1301
1302 d->rd_rcompressed = rrddim_add(d->st_compressed, "received", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1303 d->rd_tcompressed = rrddim_add(d->st_compressed, "sent", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
@@ -1343,7 +1343,7 @@ int do_proc_net_dev(int update_every, usec_t dt) {
1343
1344 rrdset_flag_set(d->st_events, RRDSET_FLAG_DETAIL);
1345
1346 - rrdset_update_labels(d->st_events, d->chart_labels);
1346 + rrdset_update_rrdlabels(d->st_events, d->chart_labels);
1347
1348 d->rd_rframe = rrddim_add(d->st_events, "frames", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
1349 d->rd_tcollisions = rrddim_add(d->st_events, "collisions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
daemon/commands.c
+1 -11
@@ -217,17 +217,7 @@ static cmd_status_t cmd_reload_labels_execute(char *args, char **message)
217 reload_host_labels();
218
219 BUFFER *wb = buffer_create(10);
220 -
221 - rrdhost_rdlock(localhost);
222 - netdata_rwlock_rdlock(&localhost->labels.labels_rwlock);
223 - struct label *l = localhost->labels.head;
224 - while (l != NULL) {
225 - buffer_sprintf(wb,"Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(l->label_source), l->key, l->value);
226 - l = l->next;
227 - }
228 - netdata_rwlock_unlock(&localhost->labels.labels_rwlock);
229 - rrdhost_unlock(localhost);
230 -
220 + rrdlabels_log_to_buffer(localhost->host_labels, wb);
221 (*message)=strdupz(buffer_tostring(wb));
222 buffer_free(wb);
223
daemon/get-kubernetes-labels.sh.in
+9 -8
@@ -1,4 +1,5 @@
1 #!/usr/bin/env bash
2 +me="$(basename "${0}")"
3
4 # Checks if netdata is running in a kubernetes pod and fetches:
5 # - pod's labels
@@ -8,8 +9,8 @@ if [ -z "${KUBERNETES_SERVICE_HOST}" ] || [ -z "${KUBERNETES_PORT_443_TCP_PORT}"
9 exit 0
10 fi
11
11 -if ! command -v jq > /dev/null 2>&1; then
12 - echo "jq command not available. Please install jq to get host labels for kubernetes pods."
12 +if ! command -v jq >/dev/null 2>&1; then
13 + echo >&2 "${me}: jq command not available. Please install jq to get host labels for kubernetes pods."
14 exit 1
15 fi
16
@@ -18,24 +19,24 @@ HEADER="Authorization: Bearer $TOKEN"
19 HOST="$KUBERNETES_SERVICE_HOST:$KUBERNETES_PORT_443_TCP_PORT"
20
21 URL="https://$HOST/api/v1/namespaces/$MY_POD_NAMESPACE/pods/$MY_POD_NAME"
21 -if ! POD_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then
22 - echo "error on curl '${URL}': ${POD_DATA}."
22 +if ! POD_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then
23 + echo >&2 "${me}: error on curl '${URL}': ${POD_DATA}."
24 exit 1
25 fi
26
27 URL="https://$HOST/api/v1/namespaces/kube-system"
27 -if ! KUBE_SYSTEM_NS_DATA=$(curl -sSk -H "$HEADER" "$URL" 2>&1); then
28 - echo "error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}."
28 +if ! KUBE_SYSTEM_NS_DATA=$(curl --fail -sSk -H "$HEADER" "$URL" 2>&1); then
29 + echo >&2 "${me}: error on curl '${URL}': ${KUBE_SYSTEM_NS_DATA}."
30 exit 1
31 fi
32
33 if ! POD_LABELS=$(jq -r '.metadata.labels' <<< "$POD_DATA" | grep ':' | tr -d '," ' 2>&1); then
33 - echo "error on 'jq' parse pod data: ${POD_LABELS}."
34 + echo >&2 "${me}: error on 'jq' parse pod data: ${POD_LABELS}."
35 exit 1
36 fi
37
38 if ! KUBE_SYSTEM_NS_UID=$(jq -r '.metadata.uid' <<< "$KUBE_SYSTEM_NS_DATA" 2>&1); then
38 - echo "error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}."
39 + echo >&2 "${me}: error on 'jq' parse kube_system_ns: ${KUBE_SYSTEM_NS_UID}."
40 exit 1
41 fi
42
daemon/main.c
+3
@@ -898,6 +898,9 @@ int main(int argc, char **argv) {
898 else if(strcmp(optarg, "dicttest") == 0) {
899 return dictionary_unittest(10000);
900 }
901 + else if(strcmp(optarg, "rrdlabelstest") == 0) {
902 + return rrdlabels_unittest();
903 + }
904 else if(strncmp(optarg, createdataset_string, strlen(createdataset_string)) == 0) {
905 optarg += strlen(createdataset_string);
906 unsigned history_seconds = strtoul(optarg, NULL, 0);
database/rrd.h
+53 -64
@@ -3,6 +3,10 @@
3 #ifndef NETDATA_RRD_H
4 #define NETDATA_RRD_H 1
5
6 +#ifdef __cplusplus
7 +extern "C" {
8 +#endif
9 +
10 // forward typedefs
11 typedef struct rrdhost RRDHOST;
12 typedef struct rrddim RRDDIM;
@@ -22,7 +26,6 @@ typedef void *ml_dimension_t;
26 struct rrddim_volatile;
27 struct rrdset_volatile;
28 struct context_param;
25 -struct label;
29 #ifdef ENABLE_DBENGINE
30 struct rrdeng_page_descr;
31 struct rrdengine_instance;
@@ -177,66 +180,45 @@ typedef enum rrddim_flags {
180 #define rrddim_flag_set(rd, flag) __atomic_or_fetch(&((rd)->flags), (flag), __ATOMIC_SEQ_CST)
181 #define rrddim_flag_clear(rd, flag) __atomic_and_fetch(&((rd)->flags), ~(flag), __ATOMIC_SEQ_CST)
182
180 -typedef enum label_source {
181 - LABEL_SOURCE_AUTO = 0,
182 - LABEL_SOURCE_NETDATA_CONF = 1,
183 - LABEL_SOURCE_DOCKER = 2,
184 - LABEL_SOURCE_ENVIRONMENT = 3,
185 - LABEL_SOURCE_KUBERNETES = 4
186 -} LABEL_SOURCE;
187 -
188 -#define LABEL_FLAG_UPDATE_STREAM 1
189 -#define LABEL_FLAG_STOP_STREAM 2
190 -
191 -struct label {
192 - char *key, *value;
193 - uint32_t key_hash;
194 - LABEL_SOURCE label_source;
195 - struct label *next;
196 -};
183 +typedef enum rrdlabel_source {
184 + RRDLABEL_SRC_AUTO = (1 << 0), // set when Netdata found the label by some automation
185 + RRDLABEL_SRC_CONFIG = (1 << 1), // set when the user configured the label
186 + RRDLABEL_SRC_K8S = (1 << 2), // set when this label is found from k8s (RRDLABEL_SRC_AUTO should also be set)
187 + RRDLABEL_SRC_ACLK = (1 << 3), // set when this label is found from ACLK (RRDLABEL_SRC_AUTO should also be set)
188
198 -struct label_index {
199 - struct label *head; // Label list
200 - netdata_rwlock_t labels_rwlock; // lock for the label list
201 - uint32_t labels_flag; // Flags for labels
202 -};
189 + // more sources can be added here
190 +
191 + RRDLABEL_FLAG_OLD = (1 << 30), // marks set for rrdlabels internal use - they are not exposed outside rrdlabels
192 + RRDLABEL_FLAG_NEW = (1 << 31) //
193 +} RRDLABEL_SRC;
194 +
195 +extern DICTIONARY *rrdlabels_create(void);
196 +extern void rrdlabels_destroy(DICTIONARY *labels_dict);
197 +extern void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls);
198 +extern void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls);
199 +extern void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null);
200 +
201 +extern void rrdlabels_unmark_all(DICTIONARY *labels);
202 +extern void rrdlabels_remove_all_unmarked(DICTIONARY *labels);
203 +
204 +extern int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
205 +extern int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data);
206 +
207 +extern void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb);
208 +extern bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt);
209 +extern bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal);
210 +extern void rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size));
211 +
212 +extern void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src);
213 +extern void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src);
214
204 -typedef enum strip_quotes {
205 - DO_NOT_STRIP_QUOTES,
206 - STRIP_QUOTES
207 -} STRIP_QUOTES_OPTION;
208 -
209 -typedef enum skip_escaped_characters {
210 - DO_NOT_SKIP_ESCAPED_CHARACTERS,
211 - SKIP_ESCAPED_CHARACTERS
212 -} SKIP_ESCAPED_CHARACTERS_OPTION;
213 -
214 -char *translate_label_source(LABEL_SOURCE l);
215 -struct label *create_label(char *key, char *value, LABEL_SOURCE label_source);
216 -extern struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source);
217 -extern void update_label_list(struct label **labels, struct label *new_labels);
218 -extern void replace_label_list(struct label_index *labels, struct label *new_labels);
219 -extern int is_valid_label_value(char *value);
220 -extern int is_valid_label_key(char *key);
221 -extern void free_label_list(struct label *labels);
222 -extern struct label *label_list_lookup_key(struct label *head, char *key, uint32_t key_hash);
223 -extern struct label *label_list_lookup_keylist(struct label *head, char *keylist);
224 -extern int label_list_contains_keylist(struct label *head, char *keylist);
225 -extern int label_list_contains_key(struct label *head, char *key, uint32_t key_hash);
226 -extern int label_list_contains(struct label *head, struct label *check);
227 -extern struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri);
228 -extern void strip_last_symbol(
229 - char *str,
230 - char symbol,
231 - SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
232 -extern char *strip_double_quotes(char *str, SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters);
215 void reload_host_labels(void);
234 -extern void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source);
235 -extern void rrdset_finalize_labels(RRDSET *st);
236 -extern void rrdset_update_labels(RRDSET *st, struct label *labels);
237 -extern int rrdset_contains_label_keylist(RRDSET *st, char *key);
238 -extern int rrdset_matches_label_keys(RRDSET *st, char *key, char *words[], uint32_t *hash_key_list, int *word_count, int size);
239 -extern struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash);
216 +extern void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels);
217 +
218 +extern int rrdlabels_unittest(void);
219 +
220 +// unfortunately this break when defined in exporting_engine.h
221 +extern bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
222
223 // ----------------------------------------------------------------------------
224 // RRD DIMENSION - this is a metric
@@ -402,8 +384,7 @@ struct rrdset_volatile {
384 char *old_units;
385 char *old_context;
386 uuid_t hash_id;
405 - struct label *new_labels;
406 - struct label_index labels;
387 + DICTIONARY *chart_labels;
388 bool is_ar_chart;
389 };
390
@@ -593,11 +574,14 @@ typedef enum rrdhost_flags {
574 RRDHOST_FLAG_ORPHAN = 1 << 0, // this host is orphan (not receiving data)
575 RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS = 1 << 1, // delete files of obsolete charts
576 RRDHOST_FLAG_DELETE_ORPHAN_HOST = 1 << 2, // delete the entire host when orphan
596 - RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases
597 - RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
577 + RRDHOST_FLAG_EXPORTING_SEND = 1 << 3, // send it to external databases
578 + RRDHOST_FLAG_EXPORTING_DONT_SEND = 1 << 4, // don't send it to external databases
579 RRDHOST_FLAG_ARCHIVED = 1 << 5, // The host is archived, no collected charts yet
580 RRDHOST_FLAG_MULTIHOST = 1 << 6, // Host belongs to localhost/megadb
600 - RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
581 + RRDHOST_FLAG_PENDING_FOREACH_ALARMS = 1 << 7, // contains dims with uninitialized foreach alarms
582 + RRDHOST_FLAG_STREAM_LABELS_UPDATE = 1 << 8,
583 + RRDHOST_FLAG_STREAM_LABELS_STOP = 1 << 9,
584 +
585 } RRDHOST_FLAGS;
586
587 #define rrdhost_flag_check(host, flag) (__atomic_load_n(&((host)->flags), __ATOMIC_SEQ_CST) & (flag))
@@ -859,7 +843,7 @@ struct rrdhost {
843
844 // ------------------------------------------------------------------------
845 // Support for host-level labels
862 - struct label_index labels;
846 + DICTIONARY *host_labels;
847
848 // ------------------------------------------------------------------------
849 // indexes
@@ -1349,4 +1333,9 @@ extern void set_host_properties(
1333 #include "sqlite/sqlite_aclk_alert.h"
1334 #include "sqlite/sqlite_aclk_node.h"
1335 #include "sqlite/sqlite_health.h"
1336 +
1337 +#ifdef __cplusplus
1338 +}
1339 +#endif
1340 +
1341 #endif /* NETDATA_RRD_H */
database/rrdcalc.c
+19 -72
@@ -109,51 +109,23 @@ static void rrdsetcalc_link(RRDSET *st, RRDCALC *rc) {
109 health_alarm_log(host, ae);
110 }
111
112 -static inline int rrdcalc_test_additional_restriction(RRDCALC *rc, RRDSET *st){
113 - if (rc->module_match && !simple_pattern_matches(rc->module_pattern, st->module_name))
112 +static int rrdcalc_is_matching_rrdset(RRDCALC *rc, RRDSET *st) {
113 + if((rc->hash_chart != st->hash || strcmp(rc->chart, st->id) != 0) &&
114 + (rc->hash_chart != st->hash_name || strcmp(rc->chart, st->name) != 0))
115 return 0;
116
116 - if (rc->plugin_match && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name))
117 + if (rc->module_pattern && !simple_pattern_matches(rc->module_pattern, st->module_name))
118 return 0;
119
119 - if (rc->labels) {
120 - int labels_count=1;
121 - int labels_match=0;
122 - char *s = rc->labels;
123 - while (*s) {
124 - if (*s==' ')
125 - labels_count++;
126 - s++;
127 - }
128 - RRDHOST *host = st->rrdhost;
129 - char cmp[CONFIG_FILE_LINE_MAX+1];
130 - struct label *move = host->labels.head;
131 - while(move) {
132 - snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
133 - if (simple_pattern_matches(rc->splabels, move->key) ||
134 - simple_pattern_matches(rc->splabels, cmp)) {
135 - labels_match++;
136 - }
137 - move = move->next;
138 - }
120 + if (rc->plugin_pattern && !simple_pattern_matches(rc->plugin_pattern, st->plugin_name))
121 + return 0;
122
140 - if (labels_match != labels_count)
141 - return 0;
142 - }
123 + if (st->rrdhost->host_labels && rc->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(st->rrdhost->host_labels, rc->host_labels_pattern, '='))
124 + return 0;
125
126 return 1;
127 }
128
147 -static inline int rrdcalc_is_matching_this_rrdset(RRDCALC *rc, RRDSET *st) {
148 - if(((rc->hash_chart == st->hash && !strcmp(rc->chart, st->id)) ||
149 - (rc->hash_chart == st->hash_name && !strcmp(rc->chart, st->name))) &&
150 - rrdcalc_test_additional_restriction(rc, st)) {
151 - return 1;
152 - }
153 -
154 - return 0;
155 -}
156 -
129 // this has to be called while the RRDHOST is locked
130 inline void rrdsetcalc_link_matching(RRDSET *st) {
131 RRDHOST *host = st->rrdhost;
@@ -164,7 +136,7 @@ inline void rrdsetcalc_link_matching(RRDSET *st) {
136 if(unlikely(rc->rrdset))
137 continue;
138
167 - if(unlikely(rrdcalc_is_matching_this_rrdset(rc, st)))
139 + if(unlikely(rrdcalc_is_matching_rrdset(rc, st)))
140 rrdsetcalc_link(st, rc);
141 }
142 }
@@ -382,7 +354,7 @@ inline void rrdcalc_add_to_host(RRDHOST *host, RRDCALC *rc) {
354 // link it to its chart
355 RRDSET *st;
356 rrdset_foreach_read(st, host) {
385 - if(rrdcalc_is_matching_this_rrdset(rc, st)) {
357 + if(rrdcalc_is_matching_rrdset(rc, st)) {
358 rrdsetcalc_link(st, rc);
359 break;
360 }
@@ -612,8 +584,8 @@ void rrdcalc_free(RRDCALC *rc) {
584 freez(rc->component);
585 freez(rc->type);
586 simple_pattern_free(rc->spdim);
615 - freez(rc->labels);
616 - simple_pattern_free(rc->splabels);
587 + freez(rc->host_labels);
588 + simple_pattern_free(rc->host_labels_pattern);
589 freez(rc->module_match);
590 simple_pattern_free(rc->module_pattern);
591 freez(rc->plugin_match);
@@ -681,51 +653,26 @@ void rrdcalc_foreach_unlink_and_free(RRDHOST *host, RRDCALC *rc) {
653 }
654
655 static void rrdcalc_labels_unlink_alarm_loop(RRDHOST *host, RRDCALC *alarms) {
684 - RRDCALC *rc = alarms;
685 - while (rc) {
686 - if (!rc->labels) {
687 - rc = rc->next;
688 - continue;
689 - }
690 -
691 - char cmp[CONFIG_FILE_LINE_MAX+1];
692 - struct label *move = host->labels.head;
693 - while(move) {
694 - snprintf(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
695 - if (simple_pattern_matches(rc->splabels, move->key) ||
696 - simple_pattern_matches(rc->splabels, cmp)) {
697 - break;
698 - }
699 -
700 - move = move->next;
701 - }
656 + for(RRDCALC *rc = alarms ; rc ; rc = rc->next ) {
657 + if (!rc->host_labels) continue;
658
703 - RRDCALC *next = rc->next;
704 - if(!move) {
659 + if(!rrdlabels_match_simple_pattern_parsed(host->host_labels, rc->host_labels_pattern, '=')) {
660 info("Health configuration for alarm '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
661 rc->name,
662 host->hostname,
708 - rc->labels);
663 + rc->host_labels);
664
710 - if(host->alarms == alarms) {
665 + if(host->alarms == alarms)
666 rrdcalc_unlink_and_free(host, rc);
712 - } else
667 + else
668 rrdcalc_foreach_unlink_and_free(host, rc);
714 -
669 }
716 -
717 - rc = next;
670 }
671 }
672
673 void rrdcalc_labels_unlink_alarm_from_host(RRDHOST *host) {
722 - rrdhost_check_rdlock(host);
723 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
724 -
674 rrdcalc_labels_unlink_alarm_loop(host, host->alarms);
675 rrdcalc_labels_unlink_alarm_loop(host, host->alarms_with_foreach);
727 -
728 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
676 }
677
678 void rrdcalc_labels_unlink() {
@@ -736,7 +683,7 @@ void rrdcalc_labels_unlink() {
683 if (unlikely(!host->health_enabled))
684 continue;
685
739 - if (host->labels.head) {
686 + if (host->host_labels) {
687 rrdhost_wrlock(host);
688
689 rrdcalc_labels_unlink_alarm_from_host(host);
database/rrdcalc.h
+2 -2
@@ -103,8 +103,8 @@ struct rrdcalc {
103
104 // ------------------------------------------------------------------------
105 // Labels settings
106 - char *labels; // the label read from an alarm file
107 - SIMPLE_PATTERN *splabels; // the simple pattern of labels
106 + char *host_labels; // the label read from an alarm file
107 + SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
108
109 // ------------------------------------------------------------------------
110 // runtime information
database/rrdcalctemplate.c
+27 -75
@@ -4,98 +4,50 @@
4 #include "rrd.h"
5
6 // ----------------------------------------------------------------------------
7 +// RRDCALCTEMPLATE management
8 +/**
9 + * RRDCALC TEMPLATE LINK MATCHING
10 + *
11 + * @param rt is the template used to create the chart.
12 + * @param st is the chart where the alarm will be attached.
13 + */
14 +void rrdcalctemplate_check_conditions_and_link(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) {
15 + if(rt->hash_context != st->hash_context || strcmp(rt->context, st->context) != 0)
16 + return;
17
8 -static int rrdcalctemplate_is_there_label_restriction(RRDCALCTEMPLATE *rt, RRDHOST *host) {
9 - if(!rt->labels)
10 - return 0;
11 -
12 - errno = 0;
13 - struct label *move = host->labels.head;
14 - char cmp[CONFIG_FILE_LINE_MAX+1];
15 -
16 - int ret;
17 - if(move) {
18 - rrdhost_check_rdlock(host);
19 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
20 - while(move) {
21 - snprintfz(cmp, CONFIG_FILE_LINE_MAX, "%s=%s", move->key, move->value);
22 - if (simple_pattern_matches(rt->splabels, move->key) ||
23 - simple_pattern_matches(rt->splabels, cmp)) {
24 - break;
25 - }
26 - move = move->next;
27 - }
28 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
29 -
30 - if(!move) {
31 - error("Health template '%s' cannot be applied, because the host %s does not have the label(s) '%s'",
32 - rt->name,
33 - host->hostname,
34 - rt->labels
35 - );
36 - ret = 1;
37 - } else {
38 - ret = 0;
39 - }
40 - } else {
41 - ret =0;
42 - }
43 -
44 - return ret;
45 -}
46 -
47 -static inline int rrdcalctemplate_test_additional_restriction(RRDCALCTEMPLATE *rt, RRDSET *st) {
18 if (rt->charts_pattern && !simple_pattern_matches(rt->charts_pattern, st->name))
49 - return 0;
19 + return;
20
21 if (rt->family_pattern && !simple_pattern_matches(rt->family_pattern, st->family))
52 - return 0;
22 + return;
23
24 if (rt->module_pattern && !simple_pattern_matches(rt->module_pattern, st->module_name))
55 - return 0;
25 + return;
26
27 if (rt->plugin_pattern && !simple_pattern_matches(rt->plugin_pattern, st->plugin_name))
58 - return 0;
28 + return;
29
60 - return 1;
61 -}
30 + if(host->host_labels && rt->host_labels_pattern && !rrdlabels_match_simple_pattern_parsed(host->host_labels, rt->host_labels_pattern, '='))
31 + return;
32
63 -// RRDCALCTEMPLATE management
64 -/**
65 - * RRDCALC TEMPLATE LINK MATCHING
66 - *
67 - * @param rt is the template used to create the chart.
68 - * @param st is the chart where the alarm will be attached.
69 - */
70 -void rrdcalctemplate_link_matching_test(RRDCALCTEMPLATE *rt, RRDSET *st, RRDHOST *host) {
71 - if(rt->hash_context == st->hash_context && !strcmp(rt->context, st->context) &&
72 - rrdcalctemplate_test_additional_restriction(rt, st) ) {
73 - if (!rrdcalctemplate_is_there_label_restriction(rt, host)) {
74 - RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
75 - if (unlikely(!rc))
76 - info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed",
77 - rt->name, st->id, host->hostname);
33 + RRDCALC *rc = rrdcalc_create_from_template(host, rt, st->id);
34 + if (unlikely(!rc))
35 + info("Health tried to create alarm from template '%s' on chart '%s' of host '%s', but it failed", rt->name, st->id, host->hostname);
36 #ifdef NETDATA_INTERNAL_CHECKS
79 - else if (rc->rrdset != st &&
80 - !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
81 - error("Health alarm '%s.%s' should be linked to chart '%s', but it is not",
82 - rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
37 + else if (rc->rrdset != st && !rc->foreachdim) //When we have a template with foreadhdim, the child will be added to the index late
38 + error("Health alarm '%s.%s' should be linked to chart '%s', but it is not", rc->chart ? rc->chart : "NOCHART", rc->name, st->id);
39 #endif
84 - }
85 - }
40 }
41
42 void rrdcalctemplate_link_matching(RRDSET *st) {
43 RRDHOST *host = st->rrdhost;
44 RRDCALCTEMPLATE *rt;
45
92 - for(rt = host->templates; rt ; rt = rt->next) {
93 - rrdcalctemplate_link_matching_test(rt, st, host);
94 - }
46 + for(rt = host->templates; rt ; rt = rt->next)
47 + rrdcalctemplate_check_conditions_and_link(rt, st, host);
48
96 - for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next) {
97 - rrdcalctemplate_link_matching_test(rt, st, host);
98 - }
49 + for(rt = host->alarms_template_with_foreach; rt ; rt = rt->next)
50 + rrdcalctemplate_check_conditions_and_link(rt, st, host);
51 }
52
53 inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
@@ -129,9 +81,9 @@ inline void rrdcalctemplate_free(RRDCALCTEMPLATE *rt) {
81 freez(rt->info);
82 freez(rt->dimensions);
83 freez(rt->foreachdim);
132 - freez(rt->labels);
84 + freez(rt->host_labels);
85 simple_pattern_free(rt->spdim);
134 - simple_pattern_free(rt->splabels);
86 + simple_pattern_free(rt->host_labels_pattern);
87 freez(rt);
88 }
89
database/rrdcalctemplate.h
+2 -2
@@ -74,8 +74,8 @@ struct rrdcalctemplate {
74
75 // ------------------------------------------------------------------------
76 // Labels settings
77 - char *labels; // the label read from an alarm file
78 - SIMPLE_PATTERN *splabels; // the simple pattern of labels
77 + char *host_labels; // the label read from an alarm file
78 + SIMPLE_PATTERN *host_labels_pattern; // the simple pattern of labels
79
80 // ------------------------------------------------------------------------
81 // expressions related to the alarm
database/rrdhost.c
+59 -219
@@ -199,7 +199,7 @@ RRDHOST *rrdhost_create(const char *hostname,
199 #endif
200
201 netdata_rwlock_init(&host->rrdhost_rwlock);
202 - netdata_rwlock_init(&host->labels.labels_rwlock);
202 + host->host_labels = rrdlabels_create();
203
204 netdata_mutex_init(&host->aclk_state_lock);
205
@@ -976,7 +976,7 @@ void rrdhost_free(RRDHOST *host) {
976 freez(host->aclk_state.claimed_id);
977 freez(host->aclk_state.prev_claimed_id);
978 freez((void *)host->tags);
979 - free_label_list(host->labels.head);
979 + rrdlabels_destroy(host->host_labels);
980 freez((void *)host->os);
981 freez((void *)host->timezone);
982 freez((void *)host->abbrev_timezone);
@@ -994,7 +994,6 @@ void rrdhost_free(RRDHOST *host) {
994 freez(host->registry_hostname);
995 simple_pattern_free(host->rrdpush_send_charts_matching);
996 rrdhost_unlock(host);
997 - netdata_rwlock_destroy(&host->labels.labels_rwlock);
997 netdata_rwlock_destroy(&host->health_log.alarm_log_rwlock);
998 netdata_rwlock_destroy(&host->rrdhost_rwlock);
999 freez(host->node_id);
@@ -1038,297 +1037,138 @@ void rrdhost_save_charts(RRDHOST *host) {
1037 rrdhost_unlock(host);
1038 }
1039
1041 -static struct label *rrdhost_load_auto_labels(void)
1042 -{
1043 - struct label *label_list = NULL;
1040 +static void rrdhost_load_auto_labels(void) {
1041 + DICTIONARY *labels = localhost->host_labels;
1042
1043 if (localhost->system_info->cloud_provider_type)
1046 - label_list =
1047 - add_label_to_list(label_list, "_cloud_provider_type", localhost->system_info->cloud_provider_type, LABEL_SOURCE_AUTO);
1044 + rrdlabels_add(labels, "_cloud_provider_type", localhost->system_info->cloud_provider_type, RRDLABEL_SRC_AUTO);
1045
1046 if (localhost->system_info->cloud_instance_type)
1050 - label_list =
1051 - add_label_to_list(label_list, "_cloud_instance_type", localhost->system_info->cloud_instance_type, LABEL_SOURCE_AUTO);
1047 + rrdlabels_add(labels, "_cloud_instance_type", localhost->system_info->cloud_instance_type, RRDLABEL_SRC_AUTO);
1048
1049 if (localhost->system_info->cloud_instance_region)
1054 - label_list =
1055 - add_label_to_list(label_list, "_cloud_instance_region", localhost->system_info->cloud_instance_region, LABEL_SOURCE_AUTO);
1050 + rrdlabels_add(
1051 + labels, "_cloud_instance_region", localhost->system_info->cloud_instance_region, RRDLABEL_SRC_AUTO);
1052
1053 if (localhost->system_info->host_os_name)
1058 - label_list =
1059 - add_label_to_list(label_list, "_os_name", localhost->system_info->host_os_name, LABEL_SOURCE_AUTO);
1054 + rrdlabels_add(labels, "_os_name", localhost->system_info->host_os_name, RRDLABEL_SRC_AUTO);
1055
1056 if (localhost->system_info->host_os_version)
1062 - label_list =
1063 - add_label_to_list(label_list, "_os_version", localhost->system_info->host_os_version, LABEL_SOURCE_AUTO);
1057 + rrdlabels_add(labels, "_os_version", localhost->system_info->host_os_version, RRDLABEL_SRC_AUTO);
1058
1059 if (localhost->system_info->kernel_version)
1066 - label_list =
1067 - add_label_to_list(label_list, "_kernel_version", localhost->system_info->kernel_version, LABEL_SOURCE_AUTO);
1060 + rrdlabels_add(labels, "_kernel_version", localhost->system_info->kernel_version, RRDLABEL_SRC_AUTO);
1061
1062 if (localhost->system_info->host_cores)
1070 - label_list =
1071 - add_label_to_list(label_list, "_system_cores", localhost->system_info->host_cores, LABEL_SOURCE_AUTO);
1063 + rrdlabels_add(labels, "_system_cores", localhost->system_info->host_cores, RRDLABEL_SRC_AUTO);
1064
1065 if (localhost->system_info->host_cpu_freq)
1074 - label_list =
1075 - add_label_to_list(label_list, "_system_cpu_freq", localhost->system_info->host_cpu_freq, LABEL_SOURCE_AUTO);
1066 + rrdlabels_add(labels, "_system_cpu_freq", localhost->system_info->host_cpu_freq, RRDLABEL_SRC_AUTO);
1067
1068 if (localhost->system_info->host_ram_total)
1078 - label_list =
1079 - add_label_to_list(label_list, "_system_ram_total", localhost->system_info->host_ram_total, LABEL_SOURCE_AUTO);
1069 + rrdlabels_add(labels, "_system_ram_total", localhost->system_info->host_ram_total, RRDLABEL_SRC_AUTO);
1070
1071 if (localhost->system_info->host_disk_space)
1082 - label_list =
1083 - add_label_to_list(label_list, "_system_disk_space", localhost->system_info->host_disk_space, LABEL_SOURCE_AUTO);
1072 + rrdlabels_add(labels, "_system_disk_space", localhost->system_info->host_disk_space, RRDLABEL_SRC_AUTO);
1073
1074 if (localhost->system_info->architecture)
1086 - label_list =
1087 - add_label_to_list(label_list, "_architecture", localhost->system_info->architecture, LABEL_SOURCE_AUTO);
1075 + rrdlabels_add(labels, "_architecture", localhost->system_info->architecture, RRDLABEL_SRC_AUTO);
1076
1077 if (localhost->system_info->virtualization)
1090 - label_list =
1091 - add_label_to_list(label_list, "_virtualization", localhost->system_info->virtualization, LABEL_SOURCE_AUTO);
1078 + rrdlabels_add(labels, "_virtualization", localhost->system_info->virtualization, RRDLABEL_SRC_AUTO);
1079
1080 if (localhost->system_info->container)
1094 - label_list =
1095 - add_label_to_list(label_list, "_container", localhost->system_info->container, LABEL_SOURCE_AUTO);
1081 + rrdlabels_add(labels, "_container", localhost->system_info->container, RRDLABEL_SRC_AUTO);
1082
1083 if (localhost->system_info->container_detection)
1098 - label_list =
1099 - add_label_to_list(label_list, "_container_detection", localhost->system_info->container_detection, LABEL_SOURCE_AUTO);
1084 + rrdlabels_add(labels, "_container_detection", localhost->system_info->container_detection, RRDLABEL_SRC_AUTO);
1085
1086 if (localhost->system_info->virt_detection)
1102 - label_list =
1103 - add_label_to_list(label_list, "_virt_detection", localhost->system_info->virt_detection, LABEL_SOURCE_AUTO);
1087 + rrdlabels_add(labels, "_virt_detection", localhost->system_info->virt_detection, RRDLABEL_SRC_AUTO);
1088
1089 if (localhost->system_info->is_k8s_node)
1106 - label_list =
1107 - add_label_to_list(label_list, "_is_k8s_node", localhost->system_info->is_k8s_node, LABEL_SOURCE_AUTO);
1090 + rrdlabels_add(labels, "_is_k8s_node", localhost->system_info->is_k8s_node, RRDLABEL_SRC_AUTO);
1091
1092 if (localhost->system_info->install_type)
1110 - label_list =
1111 - add_label_to_list(label_list, "_install_type", localhost->system_info->install_type, LABEL_SOURCE_AUTO);
1093 + rrdlabels_add(labels, "_install_type", localhost->system_info->install_type, RRDLABEL_SRC_AUTO);
1094
1095 if (localhost->system_info->prebuilt_arch)
1114 - label_list =
1115 - add_label_to_list(label_list, "_prebuilt_arch", localhost->system_info->prebuilt_arch, LABEL_SOURCE_AUTO);
1096 + rrdlabels_add(labels, "_prebuilt_arch", localhost->system_info->prebuilt_arch, RRDLABEL_SRC_AUTO);
1097
1098 if (localhost->system_info->prebuilt_dist)
1118 - label_list =
1119 - add_label_to_list(label_list, "_prebuilt_dist", localhost->system_info->prebuilt_dist, LABEL_SOURCE_AUTO);
1099 + rrdlabels_add(labels, "_prebuilt_dist", localhost->system_info->prebuilt_dist, RRDLABEL_SRC_AUTO);
1100
1121 - label_list = add_aclk_host_labels(label_list);
1101 + add_aclk_host_labels();
1102
1123 - label_list = add_label_to_list(
1124 - label_list, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", LABEL_SOURCE_AUTO);
1103 + rrdlabels_add(
1104 + labels, "_is_parent", (localhost->next || configured_as_parent()) ? "true" : "false", RRDLABEL_SRC_AUTO);
1105
1106 if (localhost->rrdpush_send_destination)
1127 - label_list =
1128 - add_label_to_list(label_list, "_streams_to", localhost->rrdpush_send_destination, LABEL_SOURCE_AUTO);
1129 -
1130 - return label_list;
1131 -}
1132 -
1133 -static inline int rrdhost_is_valid_label_config_option(char *name, char *value)
1134 -{
1135 - return (is_valid_label_key(name) && is_valid_label_value(value) && strcmp(name, "from environment") &&
1136 - strcmp(name, "from kubernetes pods"));
1107 + rrdlabels_add(labels, "_streams_to", localhost->rrdpush_send_destination, RRDLABEL_SRC_AUTO);
1108 }
1109
1139 -static struct label *rrdhost_load_config_labels()
1140 -{
1110 +static void rrdhost_load_config_labels(void) {
1111 int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);
1112 if(!status) {
1113 char *filename = CONFIG_DIR "/" CONFIG_FILENAME;
1144 - error("LABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1114 + error("RRDLABEL: Cannot reload the configuration file '%s', using labels in memory", filename);
1115 }
1116
1147 - struct label *l = NULL;
1117 struct section *co = appconfig_get_section(&netdata_config, CONFIG_SECTION_HOST_LABEL);
1118 if(co) {
1119 config_section_wrlock(co);
1120 struct config_option *cv;
1121 for(cv = co->values; cv ; cv = cv->next) {
1153 - if(rrdhost_is_valid_label_config_option(cv->name, cv->value)) {
1154 - l = add_label_to_list(l, cv->name, cv->value, LABEL_SOURCE_NETDATA_CONF);
1155 - cv->flags |= CONFIG_VALUE_USED;
1156 - } else {
1157 - error("LABELS: It was not possible to create the label '%s' because it contains invalid character(s) or values."
1158 - , cv->name);
1159 - }
1122 + rrdlabels_add(localhost->host_labels, cv->name, cv->value, RRDLABEL_SRC_CONFIG);
1123 + cv->flags |= CONFIG_VALUE_USED;
1124 }
1125 config_section_unlock(co);
1126 }
1163 -
1164 - return l;
1127 }
1128
1167 -struct label *parse_simple_tags(
1168 - struct label *label_list,
1169 - const char *tags,
1170 - char key_value_separator,
1171 - char label_separator,
1172 - STRIP_QUOTES_OPTION strip_quotes_from_key,
1173 - STRIP_QUOTES_OPTION strip_quotes_from_value,
1174 - SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters)
1175 -{
1176 - const char *end = tags;
1177 -
1178 - while (*end) {
1179 - const char *start = end;
1180 - char key[CONFIG_MAX_VALUE + 1];
1181 - char value[CONFIG_MAX_VALUE + 1];
1182 -
1183 - while (*end && *end != key_value_separator)
1184 - end++;
1185 - strncpyz(key, start, end - start);
1186 -
1187 - if (*end)
1188 - start = ++end;
1189 - while (*end && *end != label_separator)
1190 - end++;
1191 - strncpyz(value, start, end - start);
1192 -
1193 - label_list = add_label_to_list(
1194 - label_list,
1195 - strip_quotes_from_key ? strip_double_quotes(trim(key), skip_escaped_characters) : trim(key),
1196 - strip_quotes_from_value ? strip_double_quotes(trim(value), skip_escaped_characters) : trim(value),
1197 - LABEL_SOURCE_NETDATA_CONF);
1129 +static void rrdhost_load_kubernetes_labels(void) {
1130 + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];
1131 + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1132
1199 - if (*end)
1200 - end++;
1133 + if (unlikely(access(label_script, R_OK) != 0)) {
1134 + error("Kubernetes pod label fetching script %s not found.",label_script);
1135 + return;
1136 }
1137
1203 - return label_list;
1204 -}
1205 -
1206 -struct label *parse_json_tags(struct label *label_list, const char *tags)
1207 -{
1208 - char tags_buf[CONFIG_MAX_VALUE + 1];
1209 - strncpy(tags_buf, tags, CONFIG_MAX_VALUE);
1210 - char *str = tags_buf;
1211 -
1212 - switch (*str) {
1213 - case '{':
1214 - str++;
1215 - strip_last_symbol(str, '}', SKIP_ESCAPED_CHARACTERS);
1216 -
1217 - label_list = parse_simple_tags(label_list, str, ':', ',', STRIP_QUOTES, STRIP_QUOTES, SKIP_ESCAPED_CHARACTERS);
1138 + debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1139
1219 - break;
1220 - case '[':
1221 - str++;
1222 - strip_last_symbol(str, ']', SKIP_ESCAPED_CHARACTERS);
1140 + pid_t pid;
1141 + FILE *fp = mypopen(label_script, &pid);
1142 + if(!fp) return;
1143
1224 - char *end = str + strlen(str);
1225 - size_t i = 0;
1144 + char buffer[1000 + 1];
1145 + while (fgets(buffer, 1000, fp) != NULL)
1146 + rrdlabels_add_pair(localhost->host_labels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);
1147
1227 - while (str < end) {
1228 - char key[CONFIG_MAX_VALUE + 1];
1229 - snprintfz(key, CONFIG_MAX_VALUE, "host_tag%zu", i);
1230 -
1231 - str = strip_double_quotes(trim(str), SKIP_ESCAPED_CHARACTERS);
1232 -
1233 - label_list = add_label_to_list(label_list, key, str, LABEL_SOURCE_NETDATA_CONF);
1234 -
1235 - // skip to the next element in the array
1236 - str += strlen(str) + 1;
1237 - while (*str && *str != ',')
1238 - str++;
1239 - str++;
1240 - i++;
1241 - }
1242 -
1243 - break;
1244 - case '"':
1245 - label_list = add_label_to_list(
1246 - label_list, "host_tag", strip_double_quotes(str, SKIP_ESCAPED_CHARACTERS), LABEL_SOURCE_NETDATA_CONF);
1247 - break;
1248 - default:
1249 - label_list = add_label_to_list(label_list, "host_tag", str, LABEL_SOURCE_NETDATA_CONF);
1250 - break;
1251 - }
1252 -
1253 - return label_list;
1148 + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1149 + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1150 + int rc = mypclose(fp, pid);
1151 + if(rc) error("%s exited abnormally. Failed to get kubernetes labels.", label_script);
1152 }
1153
1256 -static struct label *rrdhost_load_kubernetes_labels(void)
1257 -{
1258 - struct label *l=NULL;
1259 - char *label_script = mallocz(sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2));
1260 - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");
1261 - if (unlikely(access(label_script, R_OK) != 0)) {
1262 - error("Kubernetes pod label fetching script %s not found.",label_script);
1263 - freez(label_script);
1264 - } else {
1265 - pid_t command_pid;
1266 -
1267 - debug(D_RRDHOST, "Attempting to fetch external labels via %s", label_script);
1268 -
1269 - FILE *fp = mypopen(label_script, &command_pid);
1270 - if(fp) {
1271 - int MAX_LINE_SIZE=300;
1272 - char buffer[MAX_LINE_SIZE + 1];
1273 - while (fgets(buffer, MAX_LINE_SIZE, fp) != NULL) {
1274 - char *name=buffer;
1275 - char *value=buffer;
1276 - while (*value && *value != ':') value++;
1277 - if (*value == ':') {
1278 - *value = '\0';
1279 - value++;
1280 - }
1281 - char *eos=value;
1282 - while (*eos && *eos != '\n') eos++;
1283 - if (*eos == '\n') *eos = '\0';
1284 - if (strlen(value)>0) {
1285 - if (is_valid_label_key(name)){
1286 - l = add_label_to_list(l, name, value, LABEL_SOURCE_KUBERNETES);
1287 - } else {
1288 - info("Ignoring invalid label name '%s'", name);
1289 - }
1290 - } else {
1291 - error("%s outputted unexpected result: '%s'", label_script, name);
1292 - }
1293 - };
1294 - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'
1295 - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null
1296 - int retcode=mypclose(fp, command_pid);
1297 - if (retcode) {
1298 - error("%s exited abnormally. No kubernetes labels will be added to the host.", label_script);
1299 - struct label *ll=l;
1300 - while (ll != NULL) {
1301 - info("Ignoring Label [source id=%s]: \"%s\" -> \"%s\"\n", translate_label_source(ll->label_source), ll->key, ll->value);
1302 - ll = ll->next;
1303 - freez(l);
1304 - l=ll;
1305 - }
1306 - }
1307 - }
1308 - freez(label_script);
1309 - }
1310 -
1311 - return l;
1312 -}
1154 +void reload_host_labels(void) {
1155 + if(!localhost->host_labels)
1156 + localhost->host_labels = rrdlabels_create();
1157
1314 -void reload_host_labels(void)
1315 -{
1316 - struct label *from_auto = rrdhost_load_auto_labels();
1317 - struct label *from_k8s = rrdhost_load_kubernetes_labels();
1318 - struct label *from_config = rrdhost_load_config_labels();
1158 + rrdlabels_unmark_all(localhost->host_labels);
1159
1320 - struct label *new_labels = merge_label_lists(from_auto, from_k8s);
1321 - new_labels = merge_label_lists(new_labels, from_config);
1160 + // priority is important here
1161 + rrdhost_load_config_labels();
1162 + rrdhost_load_kubernetes_labels();
1163 + rrdhost_load_auto_labels();
1164
1323 - rrdhost_rdlock(localhost);
1324 - replace_label_list(&localhost->labels, new_labels);
1165 + rrdlabels_remove_all_unmarked(localhost->host_labels);
1166
1167 health_label_log_save(localhost);
1327 - rrdhost_unlock(localhost);
1168
1169 /* TODO-GAPS - fix this so that it looks properly at the state and version of the sender
1170 if(localhost->rrdpush_send_enabled && localhost->rrdpush_sender_buffer){
1331 - localhost->labels.labels_flag |= LABEL_FLAG_UPDATE_STREAM;
1171 + localhost->labels.labels_flag |= RRDHOST_FLAG_STREAM_LABELS_UPDATE;
1172 rrdpush_send_labels(localhost);
1173 }
1174 */
database/rrdlabels.c
+1087 -143
@@ -3,201 +3,1145 @@
3 #define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
5
6 -char *translate_label_source(LABEL_SOURCE l) {
7 - switch (l) {
8 - case LABEL_SOURCE_AUTO:
9 - return "AUTO";
10 - case LABEL_SOURCE_NETDATA_CONF:
11 - return "NETDATA.CONF";
12 - case LABEL_SOURCE_DOCKER :
13 - return "DOCKER";
14 - case LABEL_SOURCE_ENVIRONMENT :
15 - return "ENVIRONMENT";
16 - case LABEL_SOURCE_KUBERNETES :
17 - return "KUBERNETES";
18 - default:
19 - return "Invalid label source";
20 - }
21 -}
22 -
23 -int is_valid_label_value(char *value) {
24 - while(*value) {
25 - if(*value == '"' || *value == '\'' || *value == '*' || *value == '!') {
26 - return 0;
6 +// ----------------------------------------------------------------------------
7 +// labels sanitization
8 +
9 +/*
10 + * All labels follow these rules:
11 + *
12 + * Character Symbol Values Names
13 + * UTF-8 characters UTF-8 yes -> _
14 + * Lower case letter [a-z] yes yes
15 + * Upper case letter [A-Z] yes -> [a-z]
16 + * Digit [0-9] yes yes
17 + * Underscore _ yes yes
18 + * Minus - yes yes
19 + * Plus + yes -> _
20 + * Colon : yes -> _
21 + * Semicolon ; -> : -> _
22 + * Equal = -> : -> _
23 + * Period . yes yes
24 + * Comma , -> . -> .
25 + * Slash / yes -> _
26 + * Backslash \ -> / -> _
27 + * At @ yes -> _
28 + * Space -> _ yes
29 + * Opening parenthesis ( -> _ yes
30 + * Closing parenthesis ) -> _ yes
31 + * anything else -> _ -> _
32 +*
33 + * The above rules should allow users to set in tags (indicative):
34 + *
35 + * 1. hostnames and domain names as-is
36 + * 2. email addresses as-is
37 + * 3. floating point numbers, converted to always use a dot as the decimal point
38 + *
39 + * Leading and trailing spaces and control characters are removed from both label
40 + * names and values.
41 + *
42 + * Multiple spaces inside the label name or the value are removed (only 1 is retained).
43 + * In names spaces are also converted to underscores.
44 + *
45 + * Names that are only underscores are rejected (they do not enter the dictionary).
46 + *
47 + * The above rules do not require any conversion to be included in JSON strings.
48 + *
49 + * Label names and values are truncated to LABELS_MAX_LENGTH (200) characters.
50 + *
51 + * When parsing, label key and value are separated by the first colon (:) found.
52 + * So label:value1:value2 is parsed as key = "label", value = "value1:value2"
53 + *
54 + * This means a label key cannot contain a colon (:) - it is converted to
55 + * underscore if it does.
56 + *
57 + */
58 +
59 +#define RRDLABELS_MAX_NAME_LENGTH 200
60 +#define RRDLABELS_MAX_VALUE_LENGTH 800 // 800 in bytes, up to 200 UTF-8 characters
61 +
62 +static unsigned char label_spaces_char_map[256];
63 +static unsigned char label_names_char_map[256];
64 +static unsigned char label_values_char_map[256] = {
65 + [0] = '\0', //
66 + [1] = '_', //
67 + [2] = '_', //
68 + [3] = '_', //
69 + [4] = '_', //
70 + [5] = '_', //
71 + [6] = '_', //
72 + [7] = '_', //
73 + [8] = '_', //
74 + [9] = '_', //
75 + [10] = '_', //
76 + [11] = '_', //
77 + [12] = '_', //
78 + [13] = '_', //
79 + [14] = '_', //
80 + [15] = '_', //
81 + [16] = '_', //
82 + [17] = '_', //
83 + [18] = '_', //
84 + [19] = '_', //
85 + [20] = '_', //
86 + [21] = '_', //
87 + [22] = '_', //
88 + [23] = '_', //
89 + [24] = '_', //
90 + [25] = '_', //
91 + [26] = '_', //
92 + [27] = '_', //
93 + [28] = '_', //
94 + [29] = '_', //
95 + [30] = '_', //
96 + [31] = '_', //
97 + [32] = ' ', // SPACE keep
98 + [33] = '_', // !
99 + [34] = '_', // "
100 + [35] = '_', // #
101 + [36] = '_', // $
102 + [37] = '_', // %
103 + [38] = '_', // &
104 + [39] = '_', // '
105 + [40] = '(', // ( keep
106 + [41] = ')', // ) keep
107 + [42] = '_', // *
108 + [43] = '+', // + keep
109 + [44] = '.', // , convert , to .
110 + [45] = '-', // - keep
111 + [46] = '.', // . keep
112 + [47] = '/', // / keep
113 + [48] = '0', // 0 keep
114 + [49] = '1', // 1 keep
115 + [50] = '2', // 2 keep
116 + [51] = '3', // 3 keep
117 + [52] = '4', // 4 keep
118 + [53] = '5', // 5 keep
119 + [54] = '6', // 6 keep
120 + [55] = '7', // 7 keep
121 + [56] = '8', // 8 keep
122 + [57] = '9', // 9 keep
123 + [58] = ':', // : keep
124 + [59] = ':', // ; convert ; to :
125 + [60] = '_', // <
126 + [61] = ':', // = convert = to :
127 + [62] = '_', // >
128 + [63] = '_', // ?
129 + [64] = '@', // @
130 + [65] = 'A', // A keep
131 + [66] = 'B', // B keep
132 + [67] = 'C', // C keep
133 + [68] = 'D', // D keep
134 + [69] = 'E', // E keep
135 + [70] = 'F', // F keep
136 + [71] = 'G', // G keep
137 + [72] = 'H', // H keep
138 + [73] = 'I', // I keep
139 + [74] = 'J', // J keep
140 + [75] = 'K', // K keep
141 + [76] = 'L', // L keep
142 + [77] = 'M', // M keep
143 + [78] = 'N', // N keep
144 + [79] = 'O', // O keep
145 + [80] = 'P', // P keep
146 + [81] = 'Q', // Q keep
147 + [82] = 'R', // R keep
148 + [83] = 'S', // S keep
149 + [84] = 'T', // T keep
150 + [85] = 'U', // U keep
151 + [86] = 'V', // V keep
152 + [87] = 'W', // W keep
153 + [88] = 'X', // X keep
154 + [89] = 'Y', // Y keep
155 + [90] = 'Z', // Z keep
156 + [91] = '_', // [
157 + [92] = '/', // backslash convert \ to /
158 + [93] = '_', // ]
159 + [94] = '_', // ^
160 + [95] = '_', // _ keep
161 + [96] = '_', // `
162 + [97] = 'a', // a keep
163 + [98] = 'b', // b keep
164 + [99] = 'c', // c keep
165 + [100] = 'd', // d keep
166 + [101] = 'e', // e keep
167 + [102] = 'f', // f keep
168 + [103] = 'g', // g keep
169 + [104] = 'h', // h keep
170 + [105] = 'i', // i keep
171 + [106] = 'j', // j keep
172 + [107] = 'k', // k keep
173 + [108] = 'l', // l keep
174 + [109] = 'm', // m keep
175 + [110] = 'n', // n keep
176 + [111] = 'o', // o keep
177 + [112] = 'p', // p keep
178 + [113] = 'q', // q keep
179 + [114] = 'r', // r keep
180 + [115] = 's', // s keep
181 + [116] = 't', // t keep
182 + [117] = 'u', // u keep
183 + [118] = 'v', // v keep
184 + [119] = 'w', // w keep
185 + [120] = 'x', // x keep
186 + [121] = 'y', // y keep
187 + [122] = 'z', // z keep
188 + [123] = '_', // {
189 + [124] = '_', // |
190 + [125] = '_', // }
191 + [126] = '_', // ~
192 + [127] = '_', //
193 + [128] = '_', //
194 + [129] = '_', //
195 + [130] = '_', //
196 + [131] = '_', //
197 + [132] = '_', //
198 + [133] = '_', //
199 + [134] = '_', //
200 + [135] = '_', //
201 + [136] = '_', //
202 + [137] = '_', //
203 + [138] = '_', //
204 + [139] = '_', //
205 + [140] = '_', //
206 + [141] = '_', //
207 + [142] = '_', //
208 + [143] = '_', //
209 + [144] = '_', //
210 + [145] = '_', //
211 + [146] = '_', //
212 + [147] = '_', //
213 + [148] = '_', //
214 + [149] = '_', //
215 + [150] = '_', //
216 + [151] = '_', //
217 + [152] = '_', //
218 + [153] = '_', //
219 + [154] = '_', //
220 + [155] = '_', //
221 + [156] = '_', //
222 + [157] = '_', //
223 + [158] = '_', //
224 + [159] = '_', //
225 + [160] = '_', //
226 + [161] = '_', //
227 + [162] = '_', //
228 + [163] = '_', //
229 + [164] = '_', //
230 + [165] = '_', //
231 + [166] = '_', //
232 + [167] = '_', //
233 + [168] = '_', //
234 + [169] = '_', //
235 + [170] = '_', //
236 + [171] = '_', //
237 + [172] = '_', //
238 + [173] = '_', //
239 + [174] = '_', //
240 + [175] = '_', //
241 + [176] = '_', //
242 + [177] = '_', //
243 + [178] = '_', //
244 + [179] = '_', //
245 + [180] = '_', //
246 + [181] = '_', //
247 + [182] = '_', //
248 + [183] = '_', //
249 + [184] = '_', //
250 + [185] = '_', //
251 + [186] = '_', //
252 + [187] = '_', //
253 + [188] = '_', //
254 + [189] = '_', //
255 + [190] = '_', //
256 + [191] = '_', //
257 + [192] = '_', //
258 + [193] = '_', //
259 + [194] = '_', //
260 + [195] = '_', //
261 + [196] = '_', //
262 + [197] = '_', //
263 + [198] = '_', //
264 + [199] = '_', //
265 + [200] = '_', //
266 + [201] = '_', //
267 + [202] = '_', //
268 + [203] = '_', //
269 + [204] = '_', //
270 + [205] = '_', //
271 + [206] = '_', //
272 + [207] = '_', //
273 + [208] = '_', //
274 + [209] = '_', //
275 + [210] = '_', //
276 + [211] = '_', //
277 + [212] = '_', //
278 + [213] = '_', //
279 + [214] = '_', //
280 + [215] = '_', //
281 + [216] = '_', //
282 + [217] = '_', //
283 + [218] = '_', //
284 + [219] = '_', //
285 + [220] = '_', //
286 + [221] = '_', //
287 + [222] = '_', //
288 + [223] = '_', //
289 + [224] = '_', //
290 + [225] = '_', //
291 + [226] = '_', //
292 + [227] = '_', //
293 + [228] = '_', //
294 + [229] = '_', //
295 + [230] = '_', //
296 + [231] = '_', //
297 + [232] = '_', //
298 + [233] = '_', //
299 + [234] = '_', //
300 + [235] = '_', //
301 + [236] = '_', //
302 + [237] = '_', //
303 + [238] = '_', //
304 + [239] = '_', //
305 + [240] = '_', //
306 + [241] = '_', //
307 + [242] = '_', //
308 + [243] = '_', //
309 + [244] = '_', //
310 + [245] = '_', //
311 + [246] = '_', //
312 + [247] = '_', //
313 + [248] = '_', //
314 + [249] = '_', //
315 + [250] = '_', //
316 + [251] = '_', //
317 + [252] = '_', //
318 + [253] = '_', //
319 + [254] = '_', //
320 + [255] = '_' //
321 +};
322 +
323 +__attribute__((constructor)) void initialize_labels_keys_char_map(void) {
324 + // copy the values char map to the names char map
325 + size_t i;
326 + for(i = 0; i < 256 ;i++)
327 + label_names_char_map[i] = label_values_char_map[i];
328 +
329 + // apply overrides to the label names map
330 + label_names_char_map['A'] = 'a';
331 + label_names_char_map['B'] = 'b';
332 + label_names_char_map['C'] = 'c';
333 + label_names_char_map['D'] = 'd';
334 + label_names_char_map['E'] = 'e';
335 + label_names_char_map['F'] = 'f';
336 + label_names_char_map['G'] = 'g';
337 + label_names_char_map['H'] = 'h';
338 + label_names_char_map['I'] = 'i';
339 + label_names_char_map['J'] = 'j';
340 + label_names_char_map['K'] = 'k';
341 + label_names_char_map['L'] = 'l';
342 + label_names_char_map['M'] = 'm';
343 + label_names_char_map['N'] = 'n';
344 + label_names_char_map['O'] = 'o';
345 + label_names_char_map['P'] = 'p';
346 + label_names_char_map['Q'] = 'q';
347 + label_names_char_map['R'] = 'r';
348 + label_names_char_map['S'] = 's';
349 + label_names_char_map['T'] = 't';
350 + label_names_char_map['U'] = 'u';
351 + label_names_char_map['V'] = 'v';
352 + label_names_char_map['W'] = 'w';
353 + label_names_char_map['X'] = 'x';
354 + label_names_char_map['Y'] = 'y';
355 + label_names_char_map['Z'] = 'z';
356 + label_names_char_map['='] = '_';
357 + label_names_char_map[':'] = '_';
358 + label_names_char_map['+'] = '_';
359 + label_names_char_map[';'] = '_';
360 + label_names_char_map['@'] = '_';
361 + label_names_char_map['/'] = '_';
362 + label_names_char_map['('] = '_';
363 + label_names_char_map[')'] = '_';
364 + label_names_char_map[' '] = '_';
365 + label_names_char_map['\\'] = '_';
366 +
367 + // create the spaces map
368 + for(i = 0; i < 256 ;i++)
369 + label_spaces_char_map[i] = (isspace(i) || iscntrl(i) || !isprint(i))?1:0;
370 +
371 +}
372 +
373 +static size_t rrdlabels_sanitize(unsigned char *dst, const unsigned char *src, size_t dst_size, unsigned char *char_map, bool utf) {
374 + if(unlikely(!dst_size)) return 0;
375 + if(unlikely(!src || !*src)) {
376 + *dst = '\0';
377 + return 0;
378 + }
379 +
380 + unsigned char *d = dst;
381 +
382 + // make room for the final string termination
383 + unsigned char *end = &d[dst_size - 1];
384 +
385 + // copy while converting, but keep only one white space
386 + // we start wil last_is_space = 1 to skip leading spaces
387 + int last_is_space = 1;
388 + size_t mblen = 0;
389 + while(*src && d < end) {
390 + unsigned char c = *src;
391 +
392 + if(IS_UTF8_STARTBYTE(c) && IS_UTF8_BYTE(src[1]) && d + 2 < end) {
393 + // UTF-8 multi-byte encoded character
394 +
395 + // find how big this character is (2-4 bytes)
396 + size_t utf_character_size = 2;
397 + while(utf_character_size <= 4 && src[utf_character_size] && IS_UTF8_BYTE(src[utf_character_size]) && !IS_UTF8_STARTBYTE(src[utf_character_size]))
398 + utf_character_size++;
399 +
400 + if(utf) {
401 + while(utf_character_size) {
402 + utf_character_size--;
403 + *d++ = *src++;
404 + }
405 + }
406 + else {
407 + // UTF-8 characters are not allowed.
408 + // Assume it is an underscore
409 + // and skip all except the first byte
410 + *d++ = '_';
411 + src += (utf_character_size - 1);
412 + }
413 +
414 + last_is_space = 0;
415 + mblen++;
416 + continue;
417 + }
418 +
419 + if(label_spaces_char_map[c]) {
420 + // a space character
421 +
422 + if(!last_is_space) {
423 + // add one space
424 + *d++ = char_map[c];
425 + mblen++;
426 + }
427 +
428 + last_is_space++;
429 + }
430 + else {
431 + *d++ = char_map[c];
432 + last_is_space = 0;
433 + mblen++;
434 }
435
29 - value++;
436 + src++;
437 }
438
32 - return 1;
439 + // remove the last trailing space
440 + if(last_is_space && d > dst) {
441 + d--;
442 + mblen--;
443 + }
444 +
445 + // put a termination at the end of what we copied
446 + *d = '\0';
447 +
448 + // check if dst is all underscores and empty it if it is
449 + d = dst;
450 + while(*d == '_') d++;
451 + if(!*d) {
452 + *dst = '\0';
453 + mblen = 0;
454 + }
455 +
456 + return mblen;
457 }
458
35 -int is_valid_label_key(char *key) {
36 - //Prometheus exporter
37 - if(!strcmp(key, "chart") || !strcmp(key, "family") || !strcmp(key, "dimension"))
38 - return 0;
459 +static inline size_t rrdlabels_sanitize_name(char *dst, const char *src, size_t dst_size) {
460 + return rrdlabels_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, label_names_char_map, 0);
461 +}
462
40 - //Netdata and Prometheus internal
41 - if (*key == '_')
42 - return 0;
463 +static inline size_t rrdlabels_sanitize_value(char *dst, const char *src, size_t dst_size) {
464 + return rrdlabels_sanitize((unsigned char *)dst, (const unsigned char *)src, dst_size, label_values_char_map, 1);
465 +}
466 +
467 +// ----------------------------------------------------------------------------
468 +// rrdlabels_create()
469 +
470 +typedef struct rrdlabel {
471 + const char *value;
472 + RRDLABEL_SRC label_source;
473 +} RRDLABEL;
474
44 - while(*key) {
45 - if(!(isdigit(*key) || isalpha(*key) || *key == '.' || *key == '_' || *key == '-'))
46 - return 0;
475 +static void rrdlabel_insert_callback(const char *name, void *value, void *data) {
476 + (void)name;
477 + DICTIONARY *dict = (DICTIONARY *)data; (void)dict;
478 + RRDLABEL *lb = (RRDLABEL *)value;
479
48 - key++;
480 + // allocate our own memory for the value
481 + lb->value = strdupz(lb->value);
482 + lb->label_source |= RRDLABEL_FLAG_NEW;
483 +}
484 +
485 +static void rrdlabel_delete_callback(const char *name, void *value, void *data) {
486 + (void)name;
487 + DICTIONARY *dict = (DICTIONARY *)data; (void)dict;
488 + RRDLABEL *lb = (RRDLABEL *)value;
489 +
490 + freez((void *)lb->value);
491 + lb->value = NULL;
492 +}
493 +
494 +static void rrdlabel_conflict_callback(const char *name, void *oldvalue, void *newvalue, void *data) {
495 + (void)name;
496 + DICTIONARY *dict = (DICTIONARY *)data; (void)dict;
497 + RRDLABEL *lbold = (RRDLABEL *)oldvalue;
498 + RRDLABEL *lbnew = (RRDLABEL *)newvalue;
499 +
500 + if(strcmp(lbold->value, lbnew->value) == 0) {
501 + // they are the same
502 + lbold->label_source |= lbnew->label_source;
503 + lbold->label_source |= RRDLABEL_FLAG_OLD;
504 + }
505 + else {
506 + // they are different
507 + freez((void *)lbold->value);
508 + lbold->value = strdupz(lbnew->value);
509 + lbold->label_source = lbnew->label_source;
510 + lbold->label_source |= RRDLABEL_FLAG_NEW;
511 }
512 +}
513
51 - return 1;
514 +DICTIONARY *rrdlabels_create(void) {
515 + DICTIONARY *dict = dictionary_create(DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
516 + dictionary_register_insert_callback(dict, rrdlabel_insert_callback, dict);
517 + dictionary_register_delete_callback(dict, rrdlabel_delete_callback, dict);
518 + dictionary_register_conflict_callback(dict, rrdlabel_conflict_callback, dict);
519 + return dict;
520 }
521
54 -void strip_last_symbol(
55 - char *str,
56 - char symbol,
57 - SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters)
58 -{
59 - char *end = str;
522
61 - while (*end && *end != symbol) {
62 - if (unlikely(skip_escaped_characters && *end == '\\')) {
63 - end++;
64 - if (unlikely(!*end))
65 - break;
66 - }
67 - end++;
523 +// ----------------------------------------------------------------------------
524 +// rrdlabels_destroy()
525 +
526 +void rrdlabels_destroy(DICTIONARY *labels_dict) {
527 + dictionary_destroy(labels_dict);
528 +}
529 +
530 +
531 +// ----------------------------------------------------------------------------
532 +// rrdlabels_add()
533 +
534 +static void labels_add_already_sanitized(DICTIONARY *dict, const char *key, const char *value, RRDLABEL_SRC ls) {
535 + if(ls & RRDLABEL_FLAG_NEW) ls &= ~RRDLABEL_FLAG_NEW;
536 + if(ls & RRDLABEL_FLAG_OLD) ls &= ~RRDLABEL_FLAG_OLD;
537 +
538 + RRDLABEL tmp = {
539 + .label_source = ls,
540 + .value = value
541 + };
542 + dictionary_set(dict, key, &tmp, sizeof(RRDLABEL));
543 +}
544 +
545 +
546 +void rrdlabels_add(DICTIONARY *dict, const char *name, const char *value, RRDLABEL_SRC ls) {
547 + if(!dict) {
548 + error("%s(): called with NULL dictionary.", __FUNCTION__ );
549 + return;
550 }
69 - if (likely(*end == symbol))
70 - *end = '\0';
551 +
552 + char n[RRDLABELS_MAX_NAME_LENGTH + 1], v[RRDLABELS_MAX_VALUE_LENGTH + 1];
553 + rrdlabels_sanitize_name(n, name, RRDLABELS_MAX_NAME_LENGTH);
554 + rrdlabels_sanitize_value(v, value, RRDLABELS_MAX_VALUE_LENGTH);
555 +
556 + if(!*n) {
557 + error("%s: cannot add name '%s' (value '%s') which is sanitized as empty string", __FUNCTION__, name, value);
558 + return;
559 + }
560 +
561 + labels_add_already_sanitized(dict, n, v, ls);
562 }
563
73 -char *strip_double_quotes(char *str, SKIP_ESCAPED_CHARACTERS_OPTION skip_escaped_characters)
74 -{
75 - if (*str == '"') {
76 - str++;
77 - strip_last_symbol(str, '"', skip_escaped_characters);
564 +static const char *get_quoted_string_up_to(char *dst, size_t dst_size, const char *string, char upto1, char upto2) {
565 + size_t len = 0;
566 + char *d = dst, quote = 0;
567 + while(*string && len++ < dst_size) {
568 + if(unlikely(!quote && (*string == '\'' || *string == '"'))) {
569 + quote = *string++;
570 + continue;
571 + }
572 +
573 + if(unlikely(quote && *string == quote)) {
574 + quote = 0;
575 + string++;
576 + continue;
577 + }
578 +
579 + if(unlikely(quote && *string == '\\' && string[1])) {
580 + string++;
581 + *d++ = *string++;
582 + continue;
583 + }
584 +
585 + if(unlikely(!quote && (*string == upto1 || *string == upto2))) break;
586 +
587 + *d++ = *string++;
588 }
589 + *d = '\0';
590
80 - return str;
591 + if(*string) string++;
592 +
593 + return string;
594 }
595
83 -struct label *create_label(char *key, char *value, LABEL_SOURCE label_source)
84 -{
85 - size_t key_len = strlen(key), value_len = strlen(value);
86 - size_t n = sizeof(struct label) + key_len + 1 + value_len + 1;
87 - struct label *result = callocz(1,n);
88 - if (result != NULL) {
89 - char *c = (char *)result;
90 - c += sizeof(struct label);
91 - strcpy(c, key);
92 - result->key = c;
93 - c += key_len + 1;
94 - strcpy(c, value);
95 - result->value = c;
96 - result->label_source = label_source;
97 - result->key_hash = simple_hash(result->key);
596 +void rrdlabels_add_pair(DICTIONARY *dict, const char *string, RRDLABEL_SRC ls) {
597 + if(!dict) {
598 + error("%s(): called with NULL dictionary.", __FUNCTION__ );
599 + return;
600 }
99 - return result;
601 +
602 + char name[RRDLABELS_MAX_NAME_LENGTH + 1];
603 + string = get_quoted_string_up_to(name, RRDLABELS_MAX_NAME_LENGTH, string, '=', ':');
604 +
605 + char value[RRDLABELS_MAX_VALUE_LENGTH + 1];
606 + get_quoted_string_up_to(value, RRDLABELS_MAX_VALUE_LENGTH, string, '\0', '\0');
607 +
608 + rrdlabels_add(dict, name, value, ls);
609 }
610
102 -void free_label_list(struct label *labels)
103 -{
104 - while (labels != NULL)
105 - {
106 - struct label *current = labels;
107 - labels = labels->next;
108 - freez(current);
611 +// ----------------------------------------------------------------------------
612 +// rrdlabels_get_to_buffer_or_null()
613 +
614 +void rrdlabels_get_value_to_buffer_or_null(DICTIONARY *labels, BUFFER *wb, const char *key, const char *quote, const char *null) {
615 + // Get a read lock on the dictionary
616 + // to copy the value into the buffer
617 + void *v;
618 + dfe_start_read(labels, v) {
619 + RRDLABEL *lb = dictionary_get_having_read_lock(labels, key);
620 +
621 + if(lb && lb->value)
622 + buffer_sprintf(wb, "%s%s%s", quote, lb->value, quote);
623 + else
624 + buffer_strcat(wb, null);
625 +
626 + break;
627 }
628 + dfe_done(v);
629 }
630
112 -void replace_label_list(struct label_index *labels, struct label *new_labels)
113 -{
114 - netdata_rwlock_wrlock(&labels->labels_rwlock);
115 - struct label *old_labels = labels->head;
116 - labels->head = new_labels;
117 - netdata_rwlock_unlock(&labels->labels_rwlock);
631
119 - free_label_list(old_labels);
632 +// ----------------------------------------------------------------------------
633 +// rrdlabels_unmark_all()
634 +// remove labels RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all dictionary items
635 +
636 +static int remove_flags_old_new(const char *name, void *value, void *data) {
637 + (void)name;
638 + (void)data;
639 +
640 + RRDLABEL *lb = (RRDLABEL *)value;
641 +
642 + if(lb->label_source & RRDLABEL_FLAG_OLD) lb->label_source &= ~RRDLABEL_FLAG_OLD;
643 + if(lb->label_source & RRDLABEL_FLAG_NEW) lb->label_source &= ~RRDLABEL_FLAG_NEW;
644 +
645 + return 1;
646 }
647
122 -struct label *add_label_to_list(struct label *l, char *key, char *value, LABEL_SOURCE label_source)
123 -{
124 - struct label *lab = create_label(key, value, label_source);
125 - lab->next = l;
126 - return lab;
648 +void rrdlabels_unmark_all(DICTIONARY *labels) {
649 + dictionary_walkthrough_read(labels, remove_flags_old_new, NULL);
650 }
651
129 -void update_label_list(struct label **labels, struct label *new_labels)
130 -{
131 - free_label_list(*labels);
132 - *labels = NULL;
652
134 - while (new_labels != NULL)
135 - {
136 - *labels = add_label_to_list(*labels, new_labels->key, new_labels->value, new_labels->label_source);
137 - new_labels = new_labels->next;
653 +// ----------------------------------------------------------------------------
654 +// rrdlabels_remove_all_unmarked()
655 +// remove dictionary items that are neither old, nor new
656 +
657 +static int remove_not_old_not_new_callback(const char *name, void *value, void *data) {
658 + DICTIONARY *dict = (DICTIONARY *)data;
659 + RRDLABEL *lb = (RRDLABEL *)value;
660 +
661 + if(!(lb->label_source & RRDLABEL_FLAG_OLD) && !(lb->label_source & RRDLABEL_FLAG_NEW)) {
662 + dictionary_del_having_write_lock(dict, name);
663 + return 1;
664 }
665 +
666 + return 0;
667 }
668
141 -struct label *label_list_lookup_key(struct label *head, char *key, uint32_t key_hash)
142 -{
143 - while (head != NULL)
144 - {
145 - if (head->key_hash == key_hash && !strcmp(head->key, key))
146 - return head;
147 - head = head->next;
148 - }
149 - return NULL;
669 +void rrdlabels_remove_all_unmarked(DICTIONARY *labels) {
670 + dictionary_walkthrough_write(labels, remove_not_old_not_new_callback, labels);
671 }
672
152 -int label_list_contains_key(struct label *head, char *key, uint32_t key_hash)
153 -{
154 - return (label_list_lookup_key(head, key, key_hash) != NULL);
673 +
674 +// ----------------------------------------------------------------------------
675 +// rrdlabels_walkthrough_read()
676 +
677 +struct labels_walkthrough {
678 + int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
679 + void *data;
680 +};
681 +
682 +static int labels_walkthrough_callback(const char *name, void *value, void *data) {
683 + struct labels_walkthrough *d = (struct labels_walkthrough *)data;
684 + RRDLABEL *lb = (RRDLABEL *)value;
685 +
686 + RRDLABEL_SRC ls = lb->label_source;
687 + if(ls & RRDLABEL_FLAG_NEW) ls &= ~RRDLABEL_FLAG_NEW;
688 + if(ls & RRDLABEL_FLAG_OLD) ls &= ~RRDLABEL_FLAG_OLD;
689 +
690 + return d->callback(name, lb->value, ls, d->data);
691 +}
692 +
693 +int rrdlabels_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data) {
694 + struct labels_walkthrough d = {
695 + .callback = callback,
696 + .data = data
697 + };
698 + return dictionary_walkthrough_read(labels, labels_walkthrough_callback, &d);
699 +}
700 +
701 +int rrdlabels_sorted_walkthrough_read(DICTIONARY *labels, int (*callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *data) {
702 + struct labels_walkthrough d = {
703 + .callback = callback,
704 + .data = data
705 + };
706 + return dictionary_sorted_walkthrough_read(labels, labels_walkthrough_callback, &d);
707 }
708
157 -int label_list_contains(struct label *head, struct label *check)
158 -{
159 - return label_list_contains_key(head, check->key, check->key_hash);
709 +
710 +// ----------------------------------------------------------------------------
711 +// rrdlabels_migrate_to_these()
712 +// migrate an existing label list to a new list, INPLACE
713 +
714 +static int copy_label_to_dictionary_callback(const char *name, void *value, void *data) {
715 + DICTIONARY *dst = (DICTIONARY *)data;
716 + RRDLABEL *lb = (RRDLABEL *)value;
717 + labels_add_already_sanitized(dst, name, lb->value, lb->label_source);
718 + return 1;
719 }
720
162 -struct label *label_list_lookup_keylist(struct label *head, char *key)
163 -{
164 - SIMPLE_PATTERN *pattern = NULL;
721 +void rrdlabels_migrate_to_these(DICTIONARY *dst, DICTIONARY *src) {
722 + if(!dst || !src) return;
723 +
724 + // remove the RRDLABEL_FLAG_OLD and RRDLABEL_FLAG_NEW from all items
725 + rrdlabels_unmark_all(dst);
726 +
727 + // Mark the existing ones as RRDLABEL_FLAG_OLD,
728 + // or the newly added ones as RRDLABEL_FLAG_NEW
729 + dictionary_walkthrough_read(src, copy_label_to_dictionary_callback, dst);
730 +
731 + // remove the unmarked dst
732 + rrdlabels_remove_all_unmarked(dst);
733 +}
734 +
735 +void rrdlabels_copy(DICTIONARY *dst, DICTIONARY *src) {
736 + if(!dst || !src) return;
737 +
738 + dictionary_walkthrough_read(src, copy_label_to_dictionary_callback, dst);
739 +}
740 +
741 +
742 +// ----------------------------------------------------------------------------
743 +// rrdlabels_match_simple_pattern()
744 +// returns true when there are keys in the dictionary matching a simple pattern
745 +
746 +struct simple_pattern_match_name_value {
747 + SIMPLE_PATTERN *pattern;
748 + char equal;
749 +};
750 +
751 +static int simple_pattern_match_name_only_callback(const char *name, void *value, void *data) {
752 + struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data;
753 + (void)value;
754 +
755 + // we return -1 to stop the walkthrough on first match
756 + if(simple_pattern_matches(t->pattern, name)) return -1;
757 +
758 + return 0;
759 +}
760 +
761 +static int simple_pattern_match_name_and_value_callback(const char *name, void *value, void *data) {
762 + struct simple_pattern_match_name_value *t = (struct simple_pattern_match_name_value *)data;
763 + RRDLABEL *lb = (RRDLABEL *)value;
764 +
765 + // we return -1 to stop the walkthrough on first match
766 + if(simple_pattern_matches(t->pattern, name)) return -1;
767 +
768 + size_t len = RRDLABELS_MAX_NAME_LENGTH + RRDLABELS_MAX_VALUE_LENGTH + 2; // +1 for =, +1 for \0
769 + char tmp[len], *dst = &tmp[0];
770 + const char *v = lb->value;
771
166 - pattern = simple_pattern_create(key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
772 + // copy the name
773 + while(*name) *dst++ = *name++;
774
168 - while (head != NULL)
169 - {
170 - if (simple_pattern_matches(pattern, head->key))
775 + // add the equal
776 + *dst++ = t->equal;
777 +
778 + // add the value
779 + while(*v) *dst++ = *v++;
780 +
781 + // terminate it
782 + *dst = '\0';
783 +
784 + if(simple_pattern_matches(t->pattern, tmp)) return -1;
785 +
786 + return 0;
787 +}
788 +
789 +bool rrdlabels_match_simple_pattern_parsed(DICTIONARY *labels, SIMPLE_PATTERN *pattern, char equal) {
790 + if (!labels) return false;
791 +
792 + struct simple_pattern_match_name_value t = {
793 + .pattern = pattern,
794 + .equal = equal
795 + };
796 +
797 + int ret = dictionary_walkthrough_read(labels, equal?simple_pattern_match_name_and_value_callback:simple_pattern_match_name_only_callback, &t);
798 +
799 + return (ret == -1)?true:false;
800 +}
801 +
802 +bool rrdlabels_match_simple_pattern(DICTIONARY *labels, const char *simple_pattern_txt) {
803 + if (!labels) return false;
804 +
805 + SIMPLE_PATTERN *pattern = simple_pattern_create(simple_pattern_txt, " ,|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
806 + char equal = '\0';
807 +
808 + const char *s;
809 + for(s = simple_pattern_txt; *s ; s++) {
810 + if (*s == '=' || *s == ':') {
811 + equal = *s;
812 break;
172 - head = head->next;
813 + }
814 }
815 +
816 + bool ret = rrdlabels_match_simple_pattern_parsed(labels, pattern, equal);
817 +
818 simple_pattern_free(pattern);
175 - return head;
819 +
820 + return ret;
821 }
822
178 -int label_list_contains_keylist(struct label *head, char *keylist)
179 -{
180 - return (label_list_lookup_keylist(head, keylist) != NULL);
823 +
824 +// ----------------------------------------------------------------------------
825 +// Log all labels
826 +
827 +static int rrdlabels_log_label_to_buffer_callback(const char *name, void *value, void *data) {
828 + BUFFER *wb = (BUFFER *)data;
829 + RRDLABEL *lb = (RRDLABEL *)value;
830 +
831 + buffer_sprintf(wb, "Label: %s: \"%s\" (", name, lb->value);
832 +
833 + size_t sources = 0;
834 + if(lb->label_source & RRDLABEL_SRC_AUTO)
835 + buffer_sprintf(wb, "%sauto", sources++?",":"");
836 +
837 + if(lb->label_source & RRDLABEL_SRC_CONFIG)
838 + buffer_sprintf(wb, "%snetdata.conf", sources++?",":"");
839 +
840 + if(lb->label_source & RRDLABEL_SRC_K8S)
841 + buffer_sprintf(wb, "%sk8s", sources++?",":"");
842 +
843 + if(lb->label_source & RRDLABEL_SRC_ACLK)
844 + buffer_sprintf(wb, "%saclk", sources++?",":"");
845 +
846 + if(!sources)
847 + buffer_strcat(wb, "unknown");
848 +
849 + buffer_strcat(wb, ")\n");
850 +
851 + return 1;
852 }
853
854 +void rrdlabels_log_to_buffer(DICTIONARY *labels, BUFFER *wb) {
855 + dictionary_sorted_walkthrough_read(labels, rrdlabels_log_label_to_buffer_callback, wb);
856 +}
857
184 -/* Create a list with entries from both lists.
185 - If any entry in the low priority list is masked by an entry in the high priority list then delete it.
186 -*/
187 -struct label *merge_label_lists(struct label *lo_pri, struct label *hi_pri)
188 -{
189 - struct label *result = hi_pri;
190 - while (lo_pri != NULL)
191 - {
192 - struct label *current = lo_pri;
193 - lo_pri = lo_pri->next;
194 - if (!label_list_contains(result, current)) {
195 - current->next = result;
196 - result = current;
197 - }
198 - else
199 - freez(current);
858 +
859 +// ----------------------------------------------------------------------------
860 +// rrdlabels_to_buffer()
861 +
862 +struct labels_to_buffer {
863 + BUFFER *wb;
864 + bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data);
865 + void *filter_data;
866 + void (*name_sanitizer)(char *dst, const char *src, size_t dst_size);
867 + void (*value_sanitizer)(char *dst, const char *src, size_t dst_size);
868 + const char *before_each;
869 + const char *quote;
870 + const char *equal;
871 + const char *between_them;
872 + size_t count;
873 +};
874 +
875 +static int label_to_buffer_callback(const char *name, void *value, void *data) {
876 + struct labels_to_buffer *t = (struct labels_to_buffer *)data;
877 + RRDLABEL *lb = (RRDLABEL *)value;
878 +
879 + size_t n_size = (t->name_sanitizer ) ? ( RRDLABELS_MAX_NAME_LENGTH * 2 ) : 1;
880 + size_t v_size = (t->value_sanitizer) ? ( RRDLABELS_MAX_VALUE_LENGTH * 2 ) : 1;
881 +
882 + char n[n_size];
883 + char v[v_size];
884 +
885 + const char *nn = name, *vv = lb->value;
886 +
887 + if(t->name_sanitizer) {
888 + t->name_sanitizer(n, name, n_size);
889 + nn = n;
890 + }
891 +
892 + if(t->value_sanitizer) {
893 + t->value_sanitizer(v, lb->value, v_size);
894 + vv = v;
895 + }
896 +
897 + if(!t->filter_callback || (t->filter_callback && t->filter_callback(name, lb->value, lb->label_source, t->filter_data))) {
898 + buffer_sprintf(t->wb, "%s%s%s%s%s%s%s%s%s", t->count++?t->between_them:"", t->before_each, t->quote, nn, t->quote, t->equal, t->quote, vv, t->quote);
899 + return 1;
900 + }
901 +
902 + return 0;
903 +}
904 +
905 +void rrdlabels_to_buffer(DICTIONARY *labels, BUFFER *wb, const char *before_each, const char *equal, const char *quote, const char *between_them, bool (*filter_callback)(const char *name, const char *value, RRDLABEL_SRC ls, void *data), void *filter_data, void (*name_sanitizer)(char *dst, const char *src, size_t dst_size), void (*value_sanitizer)(char *dst, const char *src, size_t dst_size)) {
906 + struct labels_to_buffer tmp = {
907 + .wb = wb,
908 + .filter_callback = filter_callback,
909 + .filter_data = filter_data,
910 + .name_sanitizer = name_sanitizer,
911 + .value_sanitizer = value_sanitizer,
912 + .before_each = before_each,
913 + .equal = equal,
914 + .quote = quote,
915 + .between_them = between_them,
916 + .count = 0
917 + };
918 + dictionary_sorted_walkthrough_read(labels, label_to_buffer_callback, (void *)&tmp);
919 +}
920 +
921 +static int chart_label_store_to_sql_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
922 + RRDSET *st = (RRDSET *)data;
923 + sql_store_chart_label(st->chart_uuid, (int)ls, (char *)name, (char *)value);
924 + return 1;
925 +}
926 +
927 +void rrdset_update_rrdlabels(RRDSET *st, DICTIONARY *new_rrdlabels) {
928 + if(!st->state->chart_labels)
929 + st->state->chart_labels = rrdlabels_create();
930 +
931 + if (new_rrdlabels)
932 + rrdlabels_migrate_to_these(st->state->chart_labels, new_rrdlabels);
933 +
934 + // TODO - we should also cleanup sqlite from old new_rrdlabels that have been removed
935 + rrdlabels_walkthrough_read(st->state->chart_labels, chart_label_store_to_sql_callback, st);
936 +}
937 +
938 +// ----------------------------------------------------------------------------
939 +// rrdlabels unit test
940 +
941 +struct rrdlabels_unittest_add_a_pair {
942 + const char *pair;
943 + const char *expected_name;
944 + const char *expected_value;
945 + const char *name;
946 + const char *value;
947 + RRDLABEL_SRC ls;
948 + int errors;
949 +};
950 +
951 +int rrdlabels_unittest_add_a_pair_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
952 + struct rrdlabels_unittest_add_a_pair *t = (struct rrdlabels_unittest_add_a_pair *)data;
953 +
954 + t->name = name;
955 + t->value = value;
956 + t->ls = ls;
957 +
958 + if(strcmp(name, t->expected_name) != 0) {
959 + fprintf(stderr, "name is wrong, found \"%s\", expected \"%s\"", name, t->expected_name);
960 + t->errors++;
961 + }
962 +
963 + if(value == NULL && t->expected_value == NULL) {
964 + ;
965 + }
966 + else if(value == NULL || t->expected_value == NULL) {
967 + fprintf(stderr, "value is wrong, found \"%s\", expected \"%s\"", value?value:"(null)", t->expected_value?t->expected_value:"(null)");
968 + t->errors++;
969 + }
970 + else if(strcmp(value, t->expected_value) != 0) {
971 + fprintf(stderr, "values don't match, found \"%s\", expected \"%s\"", value?value:"(null)", t->expected_value?t->expected_value:"(null)");
972 + t->errors++;
973 + }
974 +
975 + return 1;
976 +}
977 +
978 +int rrdlabels_unittest_add_a_pair(const char *pair, const char *name, const char *value) {
979 + DICTIONARY *labels = rrdlabels_create();
980 + int errors;
981 +
982 + fprintf(stderr, "rrdlabels_add_pair(labels, %s) ... ", pair);
983 +
984 + rrdlabels_add_pair(labels, pair, RRDLABEL_SRC_CONFIG);
985 +
986 + struct rrdlabels_unittest_add_a_pair tmp = {
987 + .pair = pair,
988 + .expected_name = name,
989 + .expected_value = value,
990 + .errors = 0
991 + };
992 + int ret = rrdlabels_walkthrough_read(labels, rrdlabels_unittest_add_a_pair_callback, &tmp);
993 + errors = tmp.errors;
994 + if(ret != 1) {
995 + fprintf(stderr, "failed to get \"%s\" label", name);
996 + errors++;
997 }
201 - return result;
998 +
999 + if(!errors)
1000 + fprintf(stderr, " OK, name='%s' and value='%s'\n", tmp.name, tmp.value?tmp.value:"(null)");
1001 + else
1002 + fprintf(stderr, " FAILED\n");
1003 +
1004 + rrdlabels_destroy(labels);
1005 + return errors;
1006 }
1007
1008 +int rrdlabels_unittest_add_pairs() {
1009 + fprintf(stderr, "\n%s() tests\n", __FUNCTION__);
1010 +
1011 + int errors = 0;
1012 +
1013 + // basic test
1014 + errors += rrdlabels_unittest_add_a_pair("tag=value", "tag", "value");
1015 + errors += rrdlabels_unittest_add_a_pair("tag:value", "tag", "value");
1016 +
1017 + // test newlines
1018 + errors += rrdlabels_unittest_add_a_pair(" tag = \t value \r\n", "tag", "value");
1019 +
1020 + // test : in values
1021 + errors += rrdlabels_unittest_add_a_pair("tag=:value", "tag", ":value");
1022 + errors += rrdlabels_unittest_add_a_pair("tag::value", "tag", ":value");
1023 + errors += rrdlabels_unittest_add_a_pair(" tag = :value ", "tag", ":value");
1024 + errors += rrdlabels_unittest_add_a_pair(" tag : :value ", "tag", ":value");
1025 + errors += rrdlabels_unittest_add_a_pair("tag:5", "tag", "5");
1026 + errors += rrdlabels_unittest_add_a_pair("tag:55", "tag", "55");
1027 + errors += rrdlabels_unittest_add_a_pair("tag:aa", "tag", "aa");
1028 + errors += rrdlabels_unittest_add_a_pair("tag:a", "tag", "a");
1029 +
1030 + // test empty values
1031 + errors += rrdlabels_unittest_add_a_pair("tag", "tag", "");
1032 + errors += rrdlabels_unittest_add_a_pair("tag:", "tag", "");
1033 + errors += rrdlabels_unittest_add_a_pair("tag:\"\"", "tag", "");
1034 + errors += rrdlabels_unittest_add_a_pair("tag:''", "tag", "");
1035 + errors += rrdlabels_unittest_add_a_pair("tag:\r\n", "tag", "");
1036 + errors += rrdlabels_unittest_add_a_pair("tag\r\n", "tag", "");
1037 +
1038 + // test UTF-8 in values
1039 + errors += rrdlabels_unittest_add_a_pair("tag: country:Ελλάδα", "tag", "country:Ελλάδα");
1040 + errors += rrdlabels_unittest_add_a_pair("\"tag\": \"country:Ελλάδα\"", "tag", "country:Ελλάδα");
1041 + errors += rrdlabels_unittest_add_a_pair("\"tag\": country:\"Ελλάδα\"", "tag", "country:Ελλάδα");
1042 + errors += rrdlabels_unittest_add_a_pair("\"tag=1\": country:\"Gre\\\"ece\"", "tag_1", "country:Gre_ece");
1043 + errors += rrdlabels_unittest_add_a_pair("\"tag=1\" = country:\"Gre\\\"ece\"", "tag_1", "country:Gre_ece");
1044 +
1045 + errors += rrdlabels_unittest_add_a_pair("\t'LABE=L'\t=\t\"World\" peace", "labe_l", "World_peace");
1046 + errors += rrdlabels_unittest_add_a_pair("\t'LA\\'B:EL'\t=\tcountry:\"World\":\"Europe\":\"Greece\"", "la_b_el", "country:World:Europe:Greece");
1047 +
1048 + errors += rrdlabels_unittest_add_a_pair("NAME=\"VALUE\"", "name", "VALUE");
1049 + errors += rrdlabels_unittest_add_a_pair("\"NAME\" : \"VALUE\"", "name", "VALUE");
1050 + errors += rrdlabels_unittest_add_a_pair("NAME: \"VALUE\"", "name", "VALUE");
1051 +
1052 + return errors;
1053 +}
1054 +
1055 +int rrdlabels_unittest_check_simple_pattern(DICTIONARY *labels, const char *pattern, bool expected) {
1056 + fprintf(stderr, "rrdlabels_match_simple_pattern(labels, \"%s\") ... ", pattern);
1057 +
1058 + bool ret = rrdlabels_match_simple_pattern(labels, pattern);
1059 + fprintf(stderr, "%s, got %s expected %s\n", (ret == expected)?"OK":"FAILED", ret?"true":"false", expected?"true":"false");
1060 +
1061 + return (ret == expected)?0:1;
1062 +}
1063 +
1064 +int rrdlabels_unittest_simple_pattern() {
1065 + fprintf(stderr, "\n%s() tests\n", __FUNCTION__);
1066 +
1067 + int errors = 0;
1068 +
1069 + DICTIONARY *labels = rrdlabels_create();
1070 + rrdlabels_add(labels, "tag1", "value1", RRDLABEL_SRC_CONFIG);
1071 + rrdlabels_add(labels, "tag2", "value2", RRDLABEL_SRC_CONFIG);
1072 + rrdlabels_add(labels, "tag3", "value3", RRDLABEL_SRC_CONFIG);
1073 +
1074 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*", true);
1075 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag", false);
1076 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag*", true);
1077 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*1", true);
1078 + errors += rrdlabels_unittest_check_simple_pattern(labels, "value*", false);
1079 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*=value*", true);
1080 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*:value*", true);
1081 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*2", true);
1082 + errors += rrdlabels_unittest_check_simple_pattern(labels, "*2 *3", true);
1083 + errors += rrdlabels_unittest_check_simple_pattern(labels, "!tag3 *2", true);
1084 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag1 tag2", true);
1085 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag1tag2", false);
1086 + errors += rrdlabels_unittest_check_simple_pattern(labels, "invalid1 invalid2 tag3", true);
1087 + errors += rrdlabels_unittest_check_simple_pattern(labels, "!tag1 tag4", false);
1088 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag1=value1", true);
1089 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag1=value2", false);
1090 + errors += rrdlabels_unittest_check_simple_pattern(labels, "tag*=value*", true);
1091 + errors += rrdlabels_unittest_check_simple_pattern(labels, "!tag*=value*", false);
1092 + errors += rrdlabels_unittest_check_simple_pattern(labels, "!tag2=something2 tag2=*2", true);
1093 +
1094 + rrdlabels_destroy(labels);
1095 +
1096 + return errors;
1097 +}
1098 +
1099 +int rrdlabels_unittest_sanitize_value(const char *src, const char *expected) {
1100 + char buf[RRDLABELS_MAX_VALUE_LENGTH + 1];
1101 + size_t mblen = rrdlabels_sanitize_value(buf, src, RRDLABELS_MAX_VALUE_LENGTH);
1102 +
1103 + int err = 0;
1104 + if(strcmp(buf, expected) != 0) err = 1;
1105 +
1106 + fprintf(stderr, "%s(%s): %s, expected '%s', got '%s', mblen = %zu, bytes = %zu\n", __FUNCTION__, src, (err==1)?"FAILED":"OK", expected, buf, mblen, strlen(buf));
1107 + return err;
1108 +}
1109 +
1110 +int rrdlabels_unittest_sanitization() {
1111 + int errors = 0;
1112 +
1113 + errors += rrdlabels_unittest_sanitize_value("", "");
1114 + errors += rrdlabels_unittest_sanitize_value("1", "1");
1115 + errors += rrdlabels_unittest_sanitize_value(" hello world ", "hello_world");
1116 +
1117 + // 2-byte UTF-8
1118 + errors += rrdlabels_unittest_sanitize_value(" Ελλάδα ", "Ελλάδα");
1119 + errors += rrdlabels_unittest_sanitize_value("aŰbŲcŴ", "aŰbŲcŴ");
1120 + errors += rrdlabels_unittest_sanitize_value("Ű b Ų c Ŵ", "Ű_b_Ų_c_Ŵ");
1121 +
1122 + // 3-byte UTF-8
1123 + errors += rrdlabels_unittest_sanitize_value("‱", "‱");
1124 + errors += rrdlabels_unittest_sanitize_value("a‱b", "a‱b");
1125 + errors += rrdlabels_unittest_sanitize_value("a ‱ b", "a_‱_b");
1126 +
1127 + // 4-byte UTF-8
1128 + errors += rrdlabels_unittest_sanitize_value("𩸽", "𩸽");
1129 + errors += rrdlabels_unittest_sanitize_value("a𩸽b", "a𩸽b");
1130 + errors += rrdlabels_unittest_sanitize_value("a 𩸽 b", "a_𩸽_b");
1131 +
1132 + // mixed multi-byte
1133 + errors += rrdlabels_unittest_sanitize_value("Ű‱𩸽‱Ű", "Ű‱𩸽‱Ű");
1134 +
1135 + return errors;
1136 +}
1137 +
1138 +int rrdlabels_unittest(void) {
1139 + int errors = 0;
1140 +
1141 + errors += rrdlabels_unittest_sanitization();
1142 + errors += rrdlabels_unittest_add_pairs();
1143 + errors += rrdlabels_unittest_simple_pattern();
1144 +
1145 + fprintf(stderr, "%d errors found\n", errors);
1146 + return errors;
1147 +}
database/rrdset.c
+2 -103
@@ -387,7 +387,6 @@ void rrdset_free(RRDSET *st) {
387 // free it
388
389 netdata_rwlock_destroy(&st->rrdset_rwlock);
390 - netdata_rwlock_destroy(&st->state->labels.labels_rwlock);
390
391 // free directly allocated members
392 freez((void *)st->name);
@@ -402,7 +401,7 @@ void rrdset_free(RRDSET *st) {
401 freez(st->state->old_title);
402 freez(st->state->old_units);
403 freez(st->state->old_context);
405 - free_label_list(st->state->labels.head);
404 + rrdlabels_destroy(st->state->chart_labels);
405 freez(st->state);
406 freez(st->chart_uuid);
407
@@ -888,7 +887,7 @@ RRDSET *rrdset_create_custom(
887 avl_init_lock(&st->rrdvar_root_index, rrdvar_compare);
888
889 netdata_rwlock_init(&st->rrdset_rwlock);
891 - netdata_rwlock_init(&st->state->labels.labels_rwlock);
890 + st->state->chart_labels = rrdlabels_create();
891
892 if(name && *name && rrdset_set_name(st, name))
893 // we did set the name
@@ -1957,103 +1956,3 @@ after_second_database_work:
1956
1957 netdata_thread_enable_cancelability();
1958 }
1960 -
1961 -void rrdset_add_label_to_new_list(RRDSET *st, char *key, char *value, LABEL_SOURCE source)
1962 -{
1963 - st->state->new_labels = add_label_to_list(st->state->new_labels, key, value, source);
1964 -}
1965 -
1966 -void rrdset_finalize_labels(RRDSET *st)
1967 -{
1968 - struct label *new_labels = st->state->new_labels;
1969 - struct label_index *labels = &st->state->labels;
1970 -
1971 - if (!labels->head) {
1972 - labels->head = new_labels;
1973 - } else {
1974 - replace_label_list(labels, new_labels);
1975 - }
1976 -
1977 - netdata_rwlock_rdlock(&labels->labels_rwlock);
1978 - struct label *lbl = labels->head;
1979 - while (lbl) {
1980 - sql_store_chart_label(st->chart_uuid, (int)lbl->label_source, lbl->key, lbl->value);
1981 - lbl = lbl->next;
1982 - }
1983 - netdata_rwlock_unlock(&labels->labels_rwlock);
1984 -
1985 - st->state->new_labels = NULL;
1986 -}
1987 -
1988 -void rrdset_update_labels(RRDSET *st, struct label *labels)
1989 -{
1990 - if (!labels)
1991 - return;
1992 -
1993 - update_label_list(&st->state->new_labels, labels);
1994 - rrdset_finalize_labels(st);
1995 -}
1996 -
1997 -int rrdset_contains_label_keylist(RRDSET *st, char *keylist)
1998 -{
1999 - struct label_index *labels = &st->state->labels;
2000 - int ret;
2001 -
2002 - if (!labels->head)
2003 - return 0;
2004 -
2005 - netdata_rwlock_rdlock(&labels->labels_rwlock);
2006 - ret = label_list_contains_keylist(labels->head, keylist);
2007 - netdata_rwlock_unlock(&labels->labels_rwlock);
2008 -
2009 - return ret;
2010 -}
2011 -
2012 -struct label *rrdset_lookup_label_key(RRDSET *st, char *key, uint32_t key_hash)
2013 -{
2014 - struct label_index *labels = &st->state->labels;
2015 - struct label *ret = NULL;
2016 -
2017 - if (labels->head) {
2018 - netdata_rwlock_rdlock(&labels->labels_rwlock);
2019 - ret = label_list_lookup_key(labels->head, key, key_hash);
2020 - netdata_rwlock_unlock(&labels->labels_rwlock);
2021 - }
2022 - return ret;
2023 -}
2024 -
2025 -static inline int k8s_space(char c) {
2026 - switch(c) {
2027 - case ':':
2028 - case ',':
2029 - return 1;
2030 - default:
2031 - return 0;
2032 - }
2033 -}
2034 -
2035 -int rrdset_matches_label_keys(RRDSET *st, char *keylist, char *words[], uint32_t *hash_key_list, int *word_count, int size)
2036 -{
2037 - struct label_index *labels = &st->state->labels;
2038 -
2039 - if (!labels->head)
2040 - return 0;
2041 -
2042 - struct label *one_label;
2043 -
2044 - if (!*word_count) {
2045 - *word_count = quoted_strings_splitter(keylist, words, size, k8s_space, NULL, NULL, 0);
2046 - for (int i = 0; i < *word_count - 1; i += 2) {
2047 - hash_key_list[i] = simple_hash(words[i]);
2048 - }
2049 - }
2050 -
2051 - int ret = 1;
2052 - netdata_rwlock_rdlock(&labels->labels_rwlock);
2053 - for (int i = 0; ret && i < *word_count - 1; i += 2) {
2054 - one_label = label_list_lookup_key(labels->head, words[i], hash_key_list[i]);
2055 - ret = (one_label && !strcmp(one_label->value, words[i + 1]));
2056 - }
2057 - netdata_rwlock_unlock(&labels->labels_rwlock);
2058 - return ret;
2059 -}
database/sqlite/sqlite_aclk_chart.c
+2 -10
@@ -158,16 +158,8 @@ int aclk_add_chart_event(struct aclk_database_worker_config *wc, struct aclk_dat
158 chart_payload.claim_id = claim_id;
159 chart_payload.id = strdupz(st->id);
160
161 - struct label_index *labels = &st->state->labels;
162 - netdata_rwlock_rdlock(&labels->labels_rwlock);
163 - struct label *label_list = labels->head;
164 - struct label *chart_label = NULL;
165 - while (label_list) {
166 - chart_label = add_label_to_list(chart_label, label_list->key, label_list->value, label_list->label_source);
167 - label_list = label_list->next;
168 - }
169 - netdata_rwlock_unlock(&labels->labels_rwlock);
170 - chart_payload.label_head = chart_label;
161 + chart_payload.chart_labels = rrdlabels_create();
162 + rrdlabels_copy(chart_payload.chart_labels, st->state->chart_labels);
163
164 size_t size;
165 char *payload = generate_chart_instance_updated(&size, &chart_payload);
database/sqlite/sqlite_aclk_node.c
+1 -4
@@ -75,14 +75,11 @@ void sql_build_node_info(struct aclk_database_worker_config *wc, struct aclk_dat
75 node_info.data.ml_info.ml_capable = host->system_info->ml_capable;
76 node_info.data.ml_info.ml_enabled = host->system_info->ml_enabled;
77
78 - struct label_index *labels = &host->labels;
79 - netdata_rwlock_rdlock(&labels->labels_rwlock);
80 - node_info.data.host_labels_head = labels->head;
78 + node_info.data.host_labels_ptr = host->host_labels;
79
80 aclk_update_node_info(&node_info);
81 log_access("ACLK RES [%s (%s)]: NODE INFO SENT for guid [%s] (%s)", wc->node_id, wc->host->hostname, wc->host_guid, wc->host == localhost ? "parent" : "child");
82
85 - netdata_rwlock_unlock(&labels->labels_rwlock);
83 rrd_unlock();
84 freez(node_info.claim_id);
85 freez(host_version);
exporting/check_filters.c
+8
@@ -2,6 +2,14 @@
2
3 #include "exporting_engine.h"
4
5 +
6 +bool exporting_labels_filter_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
7 + (void)name;
8 + (void)value;
9 + struct instance *instance = (struct instance *)data;
10 + return should_send_label(instance, ls);
11 +}
12 +
13 /**
14 * Check if the connector instance should export the host metrics
15 *
exporting/clean_connectors.c
+1 -1
@@ -33,7 +33,7 @@ static void clean_instance_config(struct instance_config *config)
33 void clean_instance(struct instance *instance)
34 {
35 clean_instance_config(&instance->config);
36 - buffer_free(instance->labels);
36 + buffer_free(instance->labels_buffer);
37
38 uv_cond_destroy(&instance->cond_var);
39 // uv_mutex_destroy(&instance->mutex);
exporting/exporting_engine.h
+4 -5
@@ -4,7 +4,6 @@
4 #define NETDATA_EXPORTING_ENGINE_H 1
5
6 #include "daemon/common.h"
7 -
7 #include <uv.h>
8
9 #define exporter_get(section, name, value) expconfig_get(&exporting_config, section, name, value)
@@ -40,11 +39,11 @@ extern const char *global_exporting_prefix;
39 #define sending_labels_configured(instance) \
40 (instance->config.options & (EXPORTING_OPTION_SEND_CONFIGURED_LABELS | EXPORTING_OPTION_SEND_AUTOMATIC_LABELS))
41
43 -#define should_send_label(instance, label) \
42 +#define should_send_label(instance, label_source) \
43 ((instance->config.options & EXPORTING_OPTION_SEND_CONFIGURED_LABELS && \
45 - label->label_source == LABEL_SOURCE_NETDATA_CONF) || \
44 + label_source & RRDLABEL_SRC_CONFIG) || \
45 (instance->config.options & EXPORTING_OPTION_SEND_AUTOMATIC_LABELS && \
47 - label->label_source != LABEL_SOURCE_NETDATA_CONF))
46 + label_source & RRDLABEL_SRC_AUTO))
47
48 typedef enum exporting_connector_types {
49 EXPORTING_CONNECTOR_TYPE_UNKNOWN, // Invalid type
@@ -205,7 +204,7 @@ struct instance {
204 int skip_host;
205 int skip_chart;
206
208 - BUFFER *labels;
207 + BUFFER *labels_buffer;
208
209 time_t after;
210 time_t before;
exporting/graphite/graphite.c
+10 -20
@@ -71,7 +71,7 @@ int init_graphite_instance(struct instance *instance)
71 * @param len the maximum number of characters copied.
72 */
73
74 -void sanitize_graphite_label_value(char *dst, char *src, size_t len)
74 +void sanitize_graphite_label_value(char *dst, const char *src, size_t len)
75 {
76 while (*src != '\0' && len) {
77 if (isspace(*src) || *src == ';' || *src == '~')
@@ -91,29 +91,19 @@ void sanitize_graphite_label_value(char *dst, char *src, size_t len)
91 * @param host a data collecting host.
92 * @return Always returns 0.
93 */
94 +
95 int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host)
96 {
96 - if (!instance->labels)
97 - instance->labels = buffer_create(1024);
97 + if (!instance->labels_buffer)
98 + instance->labels_buffer = buffer_create(1024);
99
100 if (unlikely(!sending_labels_configured(instance)))
101 return 0;
102
102 - rrdhost_check_rdlock(host);
103 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
104 - for (struct label *label = host->labels.head; label; label = label->next) {
105 - if (!should_send_label(instance, label))
106 - continue;
107 -
108 - char value[CONFIG_MAX_VALUE + 1];
109 - sanitize_graphite_label_value(value, label->value, CONFIG_MAX_VALUE);
110 -
111 - if (*value) {
112 - buffer_strcat(instance->labels, ";");
113 - buffer_sprintf(instance->labels, "%s=%s", label->key, value);
114 - }
115 - }
116 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
103 + buffer_strcat(instance->labels_buffer, ";");
104 + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", ";",
105 + exporting_labels_filter_callback, instance,
106 + NULL, sanitize_graphite_label_value);
107
108 return 0;
109 }
@@ -151,7 +141,7 @@ int format_dimension_collected_graphite_plaintext(struct instance *instance, RRD
141 dimension_name,
142 (host->tags) ? ";" : "",
143 (host->tags) ? host->tags : "",
154 - (instance->labels) ? buffer_tostring(instance->labels) : "",
144 + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
145 rd->last_collected_value,
146 (unsigned long long)rd->last_collected_time.tv_sec);
147
@@ -197,7 +187,7 @@ int format_dimension_stored_graphite_plaintext(struct instance *instance, RRDDIM
187 dimension_name,
188 (host->tags) ? ";" : "",
189 (host->tags) ? host->tags : "",
200 - (instance->labels) ? buffer_tostring(instance->labels) : "",
190 + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "",
191 value,
192 (unsigned long long)last_t);
193
exporting/graphite/graphite.h
+1 -1
@@ -7,7 +7,7 @@
7
8 int init_graphite_instance(struct instance *instance);
9
10 -void sanitize_graphite_label_value(char *dst, char *src, size_t len);
10 +void sanitize_graphite_label_value(char *dst, const char *src, size_t len);
11 int format_host_labels_graphite_plaintext(struct instance *instance, RRDHOST *host);
12
13 int format_dimension_collected_graphite_plaintext(struct instance *instance, RRDDIM *rd);
exporting/json/json.c
+10 -24
@@ -113,34 +113,20 @@ int init_json_http_instance(struct instance *instance)
113 * @param host a data collecting host.
114 * @return Always returns 0.
115 */
116 +
117 int format_host_labels_json_plaintext(struct instance *instance, RRDHOST *host)
118 {
118 - if (!instance->labels)
119 - instance->labels = buffer_create(1024);
119 + if (!instance->labels_buffer)
120 + instance->labels_buffer = buffer_create(1024);
121
122 if (unlikely(!sending_labels_configured(instance)))
123 return 0;
124
124 - buffer_strcat(instance->labels, "\"labels\":{");
125 -
126 - int count = 0;
127 - rrdhost_check_rdlock(host);
128 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
129 - for (struct label *label = host->labels.head; label; label = label->next) {
130 - if (!should_send_label(instance, label))
131 - continue;
132 -
133 - char value[CONFIG_MAX_VALUE * 2 + 1];
134 - sanitize_json_string(value, label->value, CONFIG_MAX_VALUE);
135 - if (count > 0)
136 - buffer_strcat(instance->labels, ",");
137 - buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
138 -
139 - count++;
140 - }
141 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
142 -
143 - buffer_strcat(instance->labels, "},");
125 + buffer_strcat(instance->labels_buffer, "\"labels\":{");
126 + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
127 + exporting_labels_filter_callback, instance,
128 + NULL, sanitize_json_string);
129 + buffer_strcat(instance->labels_buffer, "},");
130
131 return 0;
132 }
@@ -203,7 +189,7 @@ int format_dimension_collected_json_plaintext(struct instance *instance, RRDDIM
189 tags_pre,
190 tags,
191 tags_post,
206 - instance->labels ? buffer_tostring(instance->labels) : "",
192 + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
193
194 st->id,
195 st->name,
@@ -288,7 +274,7 @@ int format_dimension_stored_json_plaintext(struct instance *instance, RRDDIM *rd
274 tags_pre,
275 tags,
276 tags_post,
291 - instance->labels ? buffer_tostring(instance->labels) : "",
277 + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "",
278
279 st->id,
280 st->name,
exporting/opentsdb/opentsdb.c
+23 -47
@@ -124,13 +124,14 @@ int init_opentsdb_http_instance(struct instance *instance)
124 * @param len the maximum number of characters copied.
125 */
126
127 -void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
127 +void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len)
128 {
129 while (*src != '\0' && len) {
130 - if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '_' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
130 + if (isalpha(*src) || isdigit(*src) || *src == '-' || *src == '.' || *src == '/' || IS_UTF8_BYTE(*src))
131 *dst++ = *src;
132 else
133 *dst++ = '_';
134 +
135 src++;
136 len--;
137 }
@@ -144,28 +145,18 @@ void sanitize_opentsdb_label_value(char *dst, char *src, size_t len)
145 * @param host a data collecting host.
146 * @return Always returns 0.
147 */
147 -int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host)
148 -{
149 - if (!instance->labels)
150 - instance->labels = buffer_create(1024);
148 +
149 +int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host) {
150 + if(!instance->labels_buffer)
151 + instance->labels_buffer = buffer_create(1024);
152
153 if (unlikely(!sending_labels_configured(instance)))
154 return 0;
155
155 - rrdhost_check_rdlock(host);
156 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
157 - for (struct label *label = host->labels.head; label; label = label->next) {
158 - if (!should_send_label(instance, label))
159 - continue;
160 -
161 - char value[CONFIG_MAX_VALUE + 1];
162 - sanitize_opentsdb_label_value(value, label->value, CONFIG_MAX_VALUE);
163 -
164 - if (*value)
165 - buffer_sprintf(instance->labels, " %s=%s", label->key, value);
166 - }
167 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
168 -
156 + buffer_strcat(instance->labels_buffer, " ");
157 + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", "=", "", " ",
158 + exporting_labels_filter_callback, instance,
159 + NULL, sanitize_opentsdb_label_value);
160 return 0;
161 }
162
@@ -204,7 +195,7 @@ int format_dimension_collected_opentsdb_telnet(struct instance *instance, RRDDIM
195 (host == localhost) ? instance->config.hostname : host->hostname,
196 (host->tags) ? " " : "",
197 (host->tags) ? host->tags : "",
207 - (instance->labels) ? buffer_tostring(instance->labels) : "");
198 + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
199
200 return 0;
201 }
@@ -250,7 +241,7 @@ int format_dimension_stored_opentsdb_telnet(struct instance *instance, RRDDIM *r
241 (host == localhost) ? instance->config.hostname : host->hostname,
242 (host->tags) ? " " : "",
243 (host->tags) ? host->tags : "",
253 - (instance->labels) ? buffer_tostring(instance->labels) : "");
244 + (instance->labels_buffer) ? buffer_tostring(instance->labels_buffer) : "");
245
246 return 0;
247 }
@@ -287,33 +278,18 @@ void opentsdb_http_prepare_header(struct instance *instance)
278 * @param host a data collecting host.
279 * @return Always returns 0.
280 */
290 -int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host)
291 -{
292 - if (!instance->labels)
293 - instance->labels = buffer_create(1024);
281 +
282 +int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host) {
283 + if (!instance->labels_buffer)
284 + instance->labels_buffer = buffer_create(1024);
285
286 if (unlikely(!sending_labels_configured(instance)))
287 return 0;
288
298 - rrdhost_check_rdlock(host);
299 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
300 - for (struct label *label = host->labels.head; label; label = label->next) {
301 - if (!should_send_label(instance, label))
302 - continue;
303 -
304 - char escaped_value[CONFIG_MAX_VALUE * 2 + 1];
305 - sanitize_json_string(escaped_value, label->value, CONFIG_MAX_VALUE);
306 -
307 - char value[CONFIG_MAX_VALUE + 1];
308 - sanitize_opentsdb_label_value(value, escaped_value, CONFIG_MAX_VALUE);
309 -
310 - if (*value) {
311 - buffer_strcat(instance->labels, ",");
312 - buffer_sprintf(instance->labels, "\"%s\":\"%s\"", label->key, value);
313 - }
314 - }
315 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
316 -
289 + buffer_strcat(instance->labels_buffer, ",");
290 + rrdlabels_to_buffer(host->host_labels, instance->labels_buffer, "", ":", "\"", ",",
291 + exporting_labels_filter_callback, instance,
292 + NULL, sanitize_opentsdb_label_value);
293 return 0;
294 }
295
@@ -362,7 +338,7 @@ int format_dimension_collected_opentsdb_http(struct instance *instance, RRDDIM *
338 (host == localhost) ? instance->config.hostname : host->hostname,
339 (host->tags) ? " " : "",
340 (host->tags) ? host->tags : "",
365 - instance->labels ? buffer_tostring(instance->labels) : "");
341 + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
342
343 return 0;
344 }
@@ -418,7 +394,7 @@ int format_dimension_stored_opentsdb_http(struct instance *instance, RRDDIM *rd)
394 (host == localhost) ? instance->config.hostname : host->hostname,
395 (host->tags) ? " " : "",
396 (host->tags) ? host->tags : "",
421 - instance->labels ? buffer_tostring(instance->labels) : "");
397 + instance->labels_buffer ? buffer_tostring(instance->labels_buffer) : "");
398
399 return 0;
400 }
exporting/opentsdb/opentsdb.h
+1 -1
@@ -8,7 +8,7 @@
8 int init_opentsdb_telnet_instance(struct instance *instance);
9 int init_opentsdb_http_instance(struct instance *instance);
10
11 -void sanitize_opentsdb_label_value(char *dst, char *src, size_t len);
11 +void sanitize_opentsdb_label_value(char *dst, const char *src, size_t len);
12 int format_host_labels_opentsdb_telnet(struct instance *instance, RRDHOST *host);
13 int format_host_labels_opentsdb_http(struct instance *instance, RRDHOST *host);
14
exporting/process_data.c
+2 -2
@@ -358,8 +358,8 @@ int flush_host_labels(struct instance *instance, RRDHOST *host)
358 {
359 (void)host;
360
361 - if (instance->labels)
362 - buffer_flush(instance->labels);
361 + if (instance->labels_buffer)
362 + buffer_flush(instance->labels_buffer);
363
364 return 0;
365 }
exporting/prometheus/prometheus.c
+44 -35
@@ -290,35 +290,44 @@ inline char *prometheus_units_copy(char *d, const char *s, size_t usable, int sh
290 * @param instance an instance data structure.
291 * @param host a data collecting host.
292 */
293 -void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
294 -{
295 - if (unlikely(!sending_labels_configured(instance)))
296 - return;
293
298 - if (!instance->labels)
299 - instance->labels = buffer_create(1024);
294 +struct format_prometheus_label_callback {
295 + struct instance *instance;
296 + size_t count;
297 +};
298
301 - int count = 0;
302 - rrdhost_check_rdlock(host);
303 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
304 - for (struct label *label = host->labels.head; label; label = label->next) {
305 - if (!should_send_label(instance, label))
306 - continue;
299 +static int format_prometheus_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
300 + struct format_prometheus_label_callback *d = (struct format_prometheus_label_callback *)data;
301
308 - char key[PROMETHEUS_ELEMENT_MAX + 1];
309 - char value[PROMETHEUS_ELEMENT_MAX + 1];
302 + if (!should_send_label(d->instance, ls)) return 0;
303
311 - prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
312 - prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
304 + char k[PROMETHEUS_ELEMENT_MAX + 1];
305 + char v[PROMETHEUS_ELEMENT_MAX + 1];
306
314 - if (*key && *value) {
315 - if (count > 0)
316 - buffer_strcat(instance->labels, ",");
317 - buffer_sprintf(instance->labels, "%s=\"%s\"", key, value);
318 - count++;
319 - }
307 + prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
308 + prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
309 +
310 + if (*k && *v) {
311 + if (d->count > 0) buffer_strcat(d->instance->labels_buffer, ",");
312 + buffer_sprintf(d->instance->labels_buffer, "%s=\"%s\"", k, v);
313 + d->count++;
314 }
321 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
315 + return 1;
316 +}
317 +
318 +void format_host_labels_prometheus(struct instance *instance, RRDHOST *host)
319 +{
320 + if (unlikely(!sending_labels_configured(instance)))
321 + return;
322 +
323 + if (!instance->labels_buffer)
324 + instance->labels_buffer = buffer_create(1024);
325 +
326 + struct format_prometheus_label_callback tmp = {
327 + .instance = instance,
328 + .count = 0
329 + };
330 + rrdlabels_walkthrough_read(host->host_labels, format_prometheus_label_callback, &tmp);
331 }
332
333 struct host_variables_callback_options {
@@ -534,13 +543,13 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
543
544 char labels[PROMETHEUS_LABELS_MAX + 1] = "";
545 if (allhosts) {
537 - if (instance->labels && buffer_tostring(instance->labels)) {
546 + if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
547 if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
548 buffer_sprintf(
549 wb,
550 "netdata_host_tags_info{instance=\"%s\",%s} 1 %llu\n",
551 hostname,
543 - buffer_tostring(instance->labels),
552 + buffer_tostring(instance->labels_buffer),
553 now_realtime_usec() / USEC_PER_MS);
554
555 // deprecated, exists only for compatibility with older queries
@@ -548,45 +557,45 @@ static void rrd_stats_api_v1_charts_allmetrics_prometheus(
557 wb,
558 "netdata_host_tags{instance=\"%s\",%s} 1 %llu\n",
559 hostname,
551 - buffer_tostring(instance->labels),
560 + buffer_tostring(instance->labels_buffer),
561 now_realtime_usec() / USEC_PER_MS);
562 } else {
563 buffer_sprintf(
555 - wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
564 + wb, "netdata_host_tags_info{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
565
566 // deprecated, exists only for compatibility with older queries
567 buffer_sprintf(
559 - wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels));
568 + wb, "netdata_host_tags{instance=\"%s\",%s} 1\n", hostname, buffer_tostring(instance->labels_buffer));
569 }
570 }
571
572 snprintfz(labels, PROMETHEUS_LABELS_MAX, ",instance=\"%s\"", hostname);
573 } else {
565 - if (instance->labels && buffer_tostring(instance->labels)) {
574 + if (instance->labels_buffer && buffer_tostring(instance->labels_buffer)) {
575 if (output_options & PROMETHEUS_OUTPUT_TIMESTAMPS) {
576 buffer_sprintf(
577 wb,
578 "netdata_host_tags_info{%s} 1 %llu\n",
570 - buffer_tostring(instance->labels),
579 + buffer_tostring(instance->labels_buffer),
580 now_realtime_usec() / USEC_PER_MS);
581
582 // deprecated, exists only for compatibility with older queries
583 buffer_sprintf(
584 wb,
585 "netdata_host_tags{%s} 1 %llu\n",
577 - buffer_tostring(instance->labels),
586 + buffer_tostring(instance->labels_buffer),
587 now_realtime_usec() / USEC_PER_MS);
588 } else {
580 - buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels));
589 + buffer_sprintf(wb, "netdata_host_tags_info{%s} 1\n", buffer_tostring(instance->labels_buffer));
590
591 // deprecated, exists only for compatibility with older queries
583 - buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels));
592 + buffer_sprintf(wb, "netdata_host_tags{%s} 1\n", buffer_tostring(instance->labels_buffer));
593 }
594 }
595 }
596
588 - if (instance->labels)
589 - buffer_flush(instance->labels);
597 + if (instance->labels_buffer)
598 + buffer_flush(instance->labels_buffer);
599
600 // send custom variables set for the host
601 if (output_options & PROMETHEUS_OUTPUT_VARIABLES) {
exporting/prometheus/remote_write/remote_write.c
+25 -15
@@ -141,6 +141,26 @@ int init_prometheus_remote_write_instance(struct instance *instance)
141 * @param host a data collecting host.
142 * @return Always returns 0.
143 */
144 +
145 +struct format_remote_write_label_callback {
146 + struct instance *instance;
147 + void *write_request;
148 +};
149 +
150 +static int format_remote_write_label_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
151 + struct format_remote_write_label_callback *d = (struct format_remote_write_label_callback *)data;
152 +
153 + if (!should_send_label(d->instance, ls)) return 0;
154 +
155 + char k[PROMETHEUS_ELEMENT_MAX + 1];
156 + char v[PROMETHEUS_ELEMENT_MAX + 1];
157 +
158 + prometheus_name_copy(k, name, PROMETHEUS_ELEMENT_MAX);
159 + prometheus_label_copy(v, value, PROMETHEUS_ELEMENT_MAX);
160 + add_label(d->write_request, k, v);
161 + return 1;
162 +}
163 +
164 int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host)
165 {
166 struct simple_connector_data *simple_connector_data =
@@ -159,21 +179,11 @@ int format_host_prometheus_remote_write(struct instance *instance, RRDHOST *host
179 "netdata_info", hostname, host->program_name, host->program_version, now_realtime_usec() / USEC_PER_MS);
180
181 if (unlikely(sending_labels_configured(instance))) {
162 - rrdhost_check_rdlock(host);
163 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
164 - for (struct label *label = host->labels.head; label; label = label->next) {
165 - if (!should_send_label(instance, label))
166 - continue;
167 -
168 - char key[PROMETHEUS_ELEMENT_MAX + 1];
169 - prometheus_name_copy(key, label->key, PROMETHEUS_ELEMENT_MAX);
170 -
171 - char value[PROMETHEUS_ELEMENT_MAX + 1];
172 - prometheus_label_copy(value, label->value, PROMETHEUS_ELEMENT_MAX);
173 -
174 - add_label(connector_specific_data->write_request, key, value);
175 - }
176 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
182 + struct format_remote_write_label_callback tmp = {
183 + .write_request = connector_specific_data->write_request,
184 + .instance = instance
185 + };
186 + rrdlabels_walkthrough_read(host->host_labels, format_remote_write_label_callback, &tmp);
187 }
188
189 return 0;
exporting/tests/exporting_fixtures.c
+5 -18
@@ -41,17 +41,9 @@ int setup_rrdhost()
41
42 localhost->tags = strdupz("TAG1=VALUE1 TAG2=VALUE2");
43
44 - struct label *label = calloc(1, sizeof(struct label));
45 - label->key = strdupz("key1");
46 - label->value = strdupz("value1");
47 - label->label_source = LABEL_SOURCE_NETDATA_CONF;
48 - localhost->labels.head = label;
49 -
50 - label = calloc(1, sizeof(struct label));
51 - label->key = strdupz("key2");
52 - label->value = strdupz("value2");
53 - label->label_source = LABEL_SOURCE_AUTO;
54 - localhost->labels.head->next = label;
44 + localhost->host_labels = rrdlabels_create();
45 + rrdlabels_add(localhost->host_labels, "key1", "value1", RRDLABEL_SRC_CONFIG);
46 + rrdlabels_add(localhost->host_labels, "key2", "value2", RRDLABEL_SRC_CONFIG);
47
48 localhost->rrdset_root = calloc(1, sizeof(RRDSET));
49 RRDSET *st = localhost->rrdset_root;
@@ -94,12 +86,7 @@ int teardown_rrdhost()
86 free((void *)st->name);
87 free(st);
88
97 - free(localhost->labels.head->next->key);
98 - free(localhost->labels.head->next->value);
99 - free(localhost->labels.head->next);
100 - free(localhost->labels.head->key);
101 - free(localhost->labels.head->value);
102 - free(localhost->labels.head);
89 + rrdlabels_destroy(localhost->host_labels);
90
91 free((void *)localhost->tags);
92 free(localhost);
@@ -124,7 +111,7 @@ int teardown_initialized_engine(void **state)
111 struct engine *engine = *state;
112
113 teardown_rrdhost();
127 - buffer_free(engine->instance_root->labels);
114 + buffer_free(engine->instance_root->labels_buffer);
115 buffer_free(engine->instance_root->buffer);
116 teardown_configured_engine(state);
117
exporting/tests/netdata_doubles.c
+8
@@ -245,3 +245,11 @@ void __mock_rrddim_query_finalize(struct rrddim_query_handle *handle)
245
246 function_called();
247 }
248 +
249 +void sql_store_chart_label(uuid_t *chart_uuid, int source_type, char *label, char *value)
250 +{
251 + (void)chart_uuid;
252 + (void)source_type;
253 + (void)label;
254 + (void)value;
255 +}
exporting/tests/test_exporting_engine.c
+11 -11
@@ -381,7 +381,7 @@ static void test_prepare_buffers(void **state)
381 expect_value(__mock_end_batch_formatting, instance, instance);
382 will_return(__mock_end_batch_formatting, 0);
383
384 - assert_int_equal(__real_prepare_buffers(engine), 0);
384 + __real_prepare_buffers(engine);
385
386 assert_int_equal(instance->stats.buffered_metrics, 1);
387
@@ -393,7 +393,7 @@ static void test_prepare_buffers(void **state)
393 instance->end_chart_formatting = NULL;
394 instance->end_host_formatting = NULL;
395 instance->end_batch_formatting = NULL;
396 - assert_int_equal(__real_prepare_buffers(engine), 0);
396 + __real_prepare_buffers(engine);
397
398 assert_int_equal(instance->scheduled, 0);
399 assert_int_equal(instance->after, 2);
@@ -705,7 +705,7 @@ static void test_format_host_labels_json_plaintext(void **state)
705 instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
706
707 assert_int_equal(format_host_labels_json_plaintext(instance, localhost), 0);
708 - assert_string_equal(buffer_tostring(instance->labels), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
708 + assert_string_equal(buffer_tostring(instance->labels_buffer), "\"labels\":{\"key1\":\"value1\",\"key2\":\"value2\"},");
709 }
710
711 static void test_format_host_labels_graphite_plaintext(void **state)
@@ -717,7 +717,7 @@ static void test_format_host_labels_graphite_plaintext(void **state)
717 instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
718
719 assert_int_equal(format_host_labels_graphite_plaintext(instance, localhost), 0);
720 - assert_string_equal(buffer_tostring(instance->labels), ";key1=value1;key2=value2");
720 + assert_string_equal(buffer_tostring(instance->labels_buffer), ";key1=value1;key2=value2");
721 }
722
723 static void test_format_host_labels_opentsdb_telnet(void **state)
@@ -729,7 +729,7 @@ static void test_format_host_labels_opentsdb_telnet(void **state)
729 instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
730
731 assert_int_equal(format_host_labels_opentsdb_telnet(instance, localhost), 0);
732 - assert_string_equal(buffer_tostring(instance->labels), " key1=value1 key2=value2");
732 + assert_string_equal(buffer_tostring(instance->labels_buffer), " key1=value1 key2=value2");
733 }
734
735 static void test_format_host_labels_opentsdb_http(void **state)
@@ -741,7 +741,7 @@ static void test_format_host_labels_opentsdb_http(void **state)
741 instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
742
743 assert_int_equal(format_host_labels_opentsdb_http(instance, localhost), 0);
744 - assert_string_equal(buffer_tostring(instance->labels), ",\"key1\":\"value1\",\"key2\":\"value2\"");
744 + assert_string_equal(buffer_tostring(instance->labels_buffer), ",\"key1\":\"value1\",\"key2\":\"value2\"");
745 }
746
747 static void test_flush_host_labels(void **state)
@@ -749,12 +749,12 @@ static void test_flush_host_labels(void **state)
749 struct engine *engine = *state;
750 struct instance *instance = engine->instance_root;
751
752 - instance->labels = buffer_create(12);
753 - buffer_strcat(instance->labels, "check string");
754 - assert_int_equal(buffer_strlen(instance->labels), 12);
752 + instance->labels_buffer = buffer_create(12);
753 + buffer_strcat(instance->labels_buffer, "check string");
754 + assert_int_equal(buffer_strlen(instance->labels_buffer), 12);
755
756 assert_int_equal(flush_host_labels(instance, localhost), 0);
757 - assert_int_equal(buffer_strlen(instance->labels), 0);
757 + assert_int_equal(buffer_strlen(instance->labels_buffer), 0);
758 }
759
760 static void test_create_main_rusage_chart(void **state)
@@ -1048,7 +1048,7 @@ static void test_format_host_labels_prometheus(void **state)
1048 instance->config.options |= EXPORTING_OPTION_SEND_AUTOMATIC_LABELS;
1049
1050 format_host_labels_prometheus(instance, localhost);
1051 - assert_string_equal(buffer_tostring(instance->labels), "key1=\"value1\",key2=\"value2\"");
1051 + assert_string_equal(buffer_tostring(instance->labels_buffer), "key1=\"value1\",key2=\"value2\"");
1052 }
1053
1054 static void rrd_stats_api_v1_charts_allmetrics_prometheus(void **state)
exporting/tests/test_exporting_engine.h
+7
@@ -30,7 +30,14 @@
30 #include <stddef.h>
31 #include <setjmp.h>
32 #include <stdint.h>
33 +
34 +#ifndef UNIT_TESTING
35 +#include <cmocka.h>
36 +#else
37 +#undef UNIT_TESTING
38 #include <cmocka.h>
39 +#define UNIT_TESTING
40 +#endif
41
42 #define MAX_LOG_LINE 1024
43 extern char log_line[];
health/health_config.c
+13 -13
@@ -955,17 +955,17 @@ static int health_readfile(const char *filename, void *data) {
955 }
956 else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
957 alert_cfg->host_labels = strdupz(value);
958 - if(rc->labels) {
959 - if(strcmp(rc->labels, value) != 0)
958 + if(rc->host_labels) {
959 + if(strcmp(rc->host_labels, value) != 0)
960 error("Health configuration at line %zu of file '%s' for alarm '%s' has key '%s' twice, once with value '%s' and later with value '%s'.",
961 line, filename, rc->name, key, value, value);
962
963 - freez(rc->labels);
964 - simple_pattern_free(rc->splabels);
963 + freez(rc->host_labels);
964 + simple_pattern_free(rc->host_labels_pattern);
965 }
966
967 - rc->labels = simple_pattern_trim_around_equal(value);
968 - rc->splabels = simple_pattern_create(rc->labels, NULL, SIMPLE_PATTERN_EXACT);
967 + rc->host_labels = simple_pattern_trim_around_equal(value);
968 + rc->host_labels_pattern = simple_pattern_create(rc->host_labels, NULL, SIMPLE_PATTERN_EXACT);
969 }
970 else if(hash == hash_plugin && !strcasecmp(key, HEALTH_PLUGIN_KEY)) {
971 alert_cfg->plugin = strdupz(value);
@@ -1204,17 +1204,17 @@ static int health_readfile(const char *filename, void *data) {
1204 }
1205 else if(hash == hash_host_label && !strcasecmp(key, HEALTH_HOST_LABEL_KEY)) {
1206 alert_cfg->host_labels = strdupz(value);
1207 - if(rt->labels) {
1208 - if(strcmp(rt->labels, value) != 0)
1207 + if(rt->host_labels) {
1208 + if(strcmp(rt->host_labels, value) != 0)
1209 error("Health configuration at line %zu of file '%s' for template '%s' has key '%s' twice, once with value '%s' and later with value '%s'. Using ('%s').",
1210 - line, filename, rt->name, key, rt->labels, value, value);
1210 + line, filename, rt->name, key, rt->host_labels, value, value);
1211
1212 - freez(rt->labels);
1213 - simple_pattern_free(rt->splabels);
1212 + freez(rt->host_labels);
1213 + simple_pattern_free(rt->host_labels_pattern);
1214 }
1215
1216 - rt->labels = simple_pattern_trim_around_equal(value);
1217 - rt->splabels = simple_pattern_create(rt->labels, NULL, SIMPLE_PATTERN_EXACT);
1216 + rt->host_labels = simple_pattern_trim_around_equal(value);
1217 + rt->host_labels_pattern = simple_pattern_create(rt->host_labels, NULL, SIMPLE_PATTERN_EXACT);
1218 }
1219 else {
1220 error("Health configuration at line %zu of file '%s' for template '%s' has unknown key '%s'.",
health/health_log.c
+4 -17
@@ -74,28 +74,15 @@ inline void health_label_log_save(RRDHOST *host) {
74
75 if(unlikely(host->health_log_fp)) {
76 BUFFER *wb = buffer_create(1024);
77 - rrdhost_check_rdlock(host);
78 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
79 - struct label *l=localhost->labels.head;
80 - while (l != NULL) {
81 - buffer_sprintf(wb,"%s=%s\t ", l->key, l->value);
82 - l = l->next;
83 - }
84 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
85 -
86 - char *write = (char *) buffer_tostring(wb) ;
77
88 - write[wb->len-2] = '\n';
89 - write[wb->len-1] = '\0';
78 + rrdlabels_to_buffer(localhost->host_labels, wb, "", "=", "", "\t ", NULL, NULL, NULL, NULL);
79 + char *write = (char *) buffer_tostring(wb);
80
91 - if (unlikely(fprintf(host->health_log_fp, "L\t%s"
92 - , write
93 - ) < 0))
81 + if (unlikely(fprintf(host->health_log_fp, "L\t%s", write) < 0))
82 error("HEALTH [%s]: failed to save alarm log entry to '%s'. Health data may be lost in case of abnormal restart.",
83 host->hostname, host->health_log_filename);
96 - else {
84 + else
85 host->health_log_entries_written++;
98 - }
86
87 buffer_free(wb);
88 }
libnetdata/config/appconfig.c
+12
@@ -805,6 +805,18 @@ void appconfig_generate(struct config *root, BUFFER *wb, int only_changed)
805 struct section *co;
806 struct config_option *cv;
807
808 + {
809 + int found_host_labels = 0;
810 + for (co = root->first_section; co; co = co->next)
811 + if(!strcmp(co->name, CONFIG_SECTION_HOST_LABEL))
812 + found_host_labels = 1;
813 +
814 + if(!found_host_labels) {
815 + appconfig_section_create(root, CONFIG_SECTION_HOST_LABEL);
816 + appconfig_get(root, CONFIG_SECTION_HOST_LABEL, "name", "value");
817 + }
818 + }
819 +
820 buffer_strcat(wb,
821 "# netdata configuration\n"
822 "#\n"
libnetdata/dictionary/dictionary.c
+184 -139
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 // NOT TO BE USED BY USERS YET
4 -#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 6) // maintain reference counter in walkthrough and foreach
4 +#define DICTIONARY_FLAG_REFERENCE_COUNTERS (1 << 5) // maintain reference counter in walkthrough and foreach
5
6 typedef struct dictionary DICTIONARY;
7 #define DICTIONARY_INTERNALS
@@ -79,10 +79,13 @@ typedef struct name_value {
79
80 char *name; // the name of the dictionary item
81 void *value; // the value of the dictionary item
82 +
83 + size_t name_len; // the size of the name, including the terminating zero
84 + size_t value_len; // the size of the value (assumed binary)
85 } NAME_VALUE;
86
87 /*
85 - * When DICTIONARY_FLAG_WITH_STATISTICS is set, we need to keep track of all the memory
88 + * When DICTIONARY_FLAG_REFERENCE_COUNTERS is set, we need to keep track of all the memory
89 * we allocate and free. So, we need to keep track of the sizes of all names and values.
90 * We do this by overloading NAME_VALUE with the following additional fields.
91 */
@@ -92,24 +95,12 @@ typedef enum name_value_flags {
95 NAME_VALUE_FLAG_DELETED = (1 << 0), // this item is deleted
96 } NAME_VALUE_FLAGS;
97
95 -typedef struct name_value_with_stats {
98 +typedef struct name_value_with_reference_counters {
99 NAME_VALUE name_value_data_here; // never used - just to put the lengths at the right position
100
98 - size_t name_len; // the size of the name, including the terminating zero
99 - size_t value_len; // the size of the value (assumed binary)
100 -
101 size_t refcount; // the reference counter
102 NAME_VALUE_FLAGS flags; // the flags for this item
103 -} NAME_VALUE_WITH_STATS;
104 -
105 -struct dictionary_stats {
106 - size_t inserts;
107 - size_t deletes;
108 - size_t searches;
109 - size_t resets;
110 - size_t entries;
111 - size_t memory;
112 -};
103 +} NAME_VALUE_WITH_REFERENCE_COUNTERS;
104
105 struct dictionary {
106 DICTIONARY_FLAGS flags; // the flags of the dictionary
@@ -137,7 +128,13 @@ struct dictionary {
128 void (*conflict_callback)(const char *name, void *old_value, void *new_value, void *data);
129 void *conflict_callback_data;
130
140 - struct dictionary_stats *stats; // the statistics when DICTIONARY_FLAG_WITH_STATISTICS is set
131 + size_t inserts;
132 + size_t deletes;
133 + size_t searches;
134 + size_t resets;
135 + size_t entries;
136 + size_t walkthroughs;
137 + size_t memory;
138 };
139
140 void dictionary_register_insert_callback(DICTIONARY *dict, void (*ins_callback)(const char *name, void *value, void *data), void *data) {
@@ -159,60 +156,49 @@ void dictionary_register_conflict_callback(DICTIONARY *dict, void (*conflict_cal
156 // dictionary statistics maintenance
157
158 size_t dictionary_stats_allocated_memory(DICTIONARY *dict) {
162 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
163 - return dict->stats->memory;
164 - return 0;
159 + return dict->memory;
160 }
161 size_t dictionary_stats_entries(DICTIONARY *dict) {
167 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
168 - return dict->stats->entries;
169 - return 0;
162 + return dict->entries;
163 }
164 size_t dictionary_stats_searches(DICTIONARY *dict) {
172 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
173 - return dict->stats->searches;
174 - return 0;
165 + return dict->searches;
166 }
167 size_t dictionary_stats_inserts(DICTIONARY *dict) {
177 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
178 - return dict->stats->inserts;
179 - return 0;
168 + return dict->inserts;
169 }
170 size_t dictionary_stats_deletes(DICTIONARY *dict) {
182 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
183 - return dict->stats->deletes;
184 - return 0;
171 + return dict->deletes;
172 }
173 size_t dictionary_stats_resets(DICTIONARY *dict) {
187 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
188 - return dict->stats->resets;
189 - return 0;
174 + return dict->resets;
175 +}
176 +
177 +size_t dictionary_stats_walkthroughs(DICTIONARY *dict) {
178 + return dict->walkthroughs;
179 }
180
181 static inline void DICTIONARY_STATS_SEARCHES_PLUS1(DICTIONARY *dict) {
193 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS))
194 - dict->stats->searches++;
182 + __atomic_fetch_add(&dict->searches, 1, __ATOMIC_SEQ_CST);
183 }
184 static inline void DICTIONARY_STATS_ENTRIES_PLUS1(DICTIONARY *dict, size_t size) {
197 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
198 - dict->stats->inserts++;
199 - dict->stats->entries++;
200 - dict->stats->memory += size;
201 - }
185 + __atomic_fetch_add(&dict->inserts, 1, __ATOMIC_SEQ_CST);
186 + __atomic_fetch_add(&dict->entries, 1, __ATOMIC_SEQ_CST);
187 + __atomic_fetch_add(&dict->memory, size, __ATOMIC_SEQ_CST);
188 }
189 static inline void DICTIONARY_STATS_ENTRIES_MINUS1(DICTIONARY *dict, size_t size) {
204 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
205 - dict->stats->deletes++;
206 - dict->stats->entries--;
207 - dict->stats->memory -= size;
208 - }
190 + __atomic_fetch_add(&dict->deletes, 1, __ATOMIC_SEQ_CST);
191 + __atomic_fetch_sub(&dict->entries, 1, __ATOMIC_SEQ_CST);
192 + __atomic_fetch_sub(&dict->memory, size, __ATOMIC_SEQ_CST);
193 }
194 static inline void DICTIONARY_STATS_VALUE_RESETS_PLUS1(DICTIONARY *dict, size_t oldsize, size_t newsize) {
211 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
212 - dict->stats->resets++;
213 - dict->stats->memory += newsize;
214 - dict->stats->memory -= oldsize;
215 - }
195 + __atomic_fetch_add(&dict->resets, 1, __ATOMIC_SEQ_CST);
196 + __atomic_fetch_add(&dict->memory, newsize, __ATOMIC_SEQ_CST);
197 + __atomic_fetch_sub(&dict->memory, oldsize, __ATOMIC_SEQ_CST);
198 +}
199 +
200 +static inline void DICTIONARY_STATS_WALKTHROUGHS_PLUS1(DICTIONARY *dict) {
201 + __atomic_fetch_add(&dict->walkthroughs, 1, __ATOMIC_SEQ_CST);
202 }
203
204 // ----------------------------------------------------------------------------
@@ -279,21 +265,21 @@ static inline size_t reference_counter_free(DICTIONARY *dict) {
265
266 static void reference_counter_acquire(DICTIONARY *dict, NAME_VALUE *nv) {
267 if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
282 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
268 + NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
269 __atomic_fetch_add(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
270 }
271 }
272
273 static void reference_counter_release(DICTIONARY *dict, NAME_VALUE *nv) {
274 if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
289 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
275 + NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
276 __atomic_fetch_sub(&nvs->refcount, 1, __ATOMIC_SEQ_CST);
277 }
278 }
279
280 static int reference_counter_mark_deleted(DICTIONARY *dict, NAME_VALUE *nv) {
281 if(unlikely(dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS)) {
296 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
282 + NAME_VALUE_WITH_REFERENCE_COUNTERS *nvs = (NAME_VALUE_WITH_REFERENCE_COUNTERS *)nv;
283 nvs->flags |= NAME_VALUE_FLAG_DELETED;
284 return 1;
285 }
@@ -491,35 +477,7 @@ static inline void linkedlist_namevalue_unlink_unsafe(DICTIONARY *dict, NAME_VAL
477 // NAME_VALUE methods
478
479 static inline size_t namevalue_alloc_size(DICTIONARY *dict) {
494 - return (dict->flags & DICTIONARY_FLAG_WITH_STATISTICS) ? sizeof(NAME_VALUE_WITH_STATS) : sizeof(NAME_VALUE);
495 -}
496 -
497 -static inline size_t namevalue_get_namelen(DICTIONARY *dict, NAME_VALUE *nv) {
498 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
499 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
500 - return nvs->name_len;
501 - }
502 - return 0;
503 -}
504 -static inline size_t namevalue_get_valuelen(DICTIONARY *dict, NAME_VALUE *nv) {
505 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
506 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
507 - return nvs->value_len;
508 - }
509 - return 0;
510 -}
511 -static inline void namevalue_set_valuelen(DICTIONARY *dict, NAME_VALUE *nv, size_t value_len) {
512 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
513 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
514 - nvs->value_len = value_len;
515 - }
516 -}
517 -static inline void namevalue_set_namevaluelen(DICTIONARY *dict, NAME_VALUE *nv, size_t name_len, size_t value_len) {
518 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
519 - NAME_VALUE_WITH_STATS *nvs = (NAME_VALUE_WITH_STATS *)nv;
520 - nvs->name_len = name_len;
521 - nvs->value_len = value_len;
522 - }
480 + return (dict->flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) ? sizeof(NAME_VALUE_WITH_REFERENCE_COUNTERS) : sizeof(NAME_VALUE);
481 }
482
483 static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, size_t name_len, void *value, size_t value_len) {
@@ -529,7 +487,8 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s
487 NAME_VALUE *nv = mallocz(size);
488 size_t allocated = size;
489
532 - namevalue_set_namevaluelen(dict, nv, name_len, value_len);
490 + nv->name_len = name_len;
491 + nv->value_len = value_len;
492
493 if(likely(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))
494 nv->name = (char *)name;
@@ -565,47 +524,63 @@ static NAME_VALUE *namevalue_create_unsafe(DICTIONARY *dict, const char *name, s
524
525 DICTIONARY_STATS_ENTRIES_PLUS1(dict, allocated);
526
527 + if(dict->ins_callback)
528 + dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
529 +
530 return nv;
531 }
532
533 static void namevalue_reset_unsafe(DICTIONARY *dict, NAME_VALUE *nv, void *value, size_t value_len) {
534 debug(D_DICTIONARY, "Dictionary entry with name '%s' found. Changing its value.", nv->name);
535
536 + if(dict->del_callback)
537 + dict->del_callback(nv->name, nv->value, dict->del_callback_data);
538 +
539 if(likely(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE)) {
540 debug(D_DICTIONARY, "Dictionary: linking value to '%s'", nv->name);
541 nv->value = value;
577 - namevalue_set_valuelen(dict, nv, value_len);
542 + nv->value_len = value_len;
543 }
544 else {
545 debug(D_DICTIONARY, "Dictionary: cloning value to '%s'", nv->name);
581 - DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, namevalue_get_valuelen(dict, nv), value_len);
582 -
583 - void *old = nv->value;
584 - void *new = mallocz(value_len);
585 - memcpy(new, value, value_len);
586 - nv->value = new;
587 - namevalue_set_valuelen(dict, nv, value_len);
546 + DICTIONARY_STATS_VALUE_RESETS_PLUS1(dict, nv->value_len, value_len);
547 +
548 + void *oldvalue = nv->value;
549 + void *newvalue = NULL;
550 + if(value_len) {
551 + newvalue = mallocz(value_len);
552 + if(value) memcpy(newvalue, value, value_len);
553 + else memset(newvalue, 0, value_len);
554 + }
555 + nv->value = newvalue;
556 + nv->value_len = value_len;
557
558 debug(D_DICTIONARY, "Dictionary: freeing old value of '%s'", nv->name);
590 - freez(old);
559 + freez(oldvalue);
560 }
561 +
562 + if(dict->ins_callback)
563 + dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
564 }
565
566 static size_t namevalue_destroy_unsafe(DICTIONARY *dict, NAME_VALUE *nv) {
567 debug(D_DICTIONARY, "Destroying name value entry for name '%s'.", nv->name);
568
569 + if(dict->del_callback)
570 + dict->del_callback(nv->name, nv->value, dict->del_callback_data);
571 +
572 size_t freed = 0;
573
574 if(unlikely(!(dict->flags & DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE))) {
575 debug(D_DICTIONARY, "Dictionary freeing value of '%s'", nv->name);
576 freez(nv->value);
602 - freed += namevalue_get_valuelen(dict, nv);
577 + freed += nv->value_len;
578 }
579
580 if(unlikely(!(dict->flags & DICTIONARY_FLAG_NAME_LINK_DONT_CLONE))) {
581 debug(D_DICTIONARY, "Dictionary freeing name '%s'", nv->name);
582 freez(nv->name);
608 - freed += namevalue_get_namelen(dict, nv);
583 + freed += nv->name_len;
584 }
585
586 freez(nv);
@@ -627,11 +602,6 @@ DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) {
602 flags &= ~DICTIONARY_FLAG_REFERENCE_COUNTERS;
603 }
604
630 - if(flags & DICTIONARY_FLAG_REFERENCE_COUNTERS) {
631 - // we need statistics to allocate the extra NAME_VALUE attributes
632 - flags |= DICTIONARY_FLAG_WITH_STATISTICS;
633 - }
634 -
605 DICTIONARY *dict = callocz(1, sizeof(DICTIONARY));
606 size_t allocated = sizeof(DICTIONARY);
607
@@ -640,20 +610,15 @@ DICTIONARY *dictionary_create(DICTIONARY_FLAGS flags) {
610
611 allocated += dictionary_lock_init(dict);
612 allocated += reference_counter_init(dict);
643 -
644 - if(flags & DICTIONARY_FLAG_WITH_STATISTICS) {
645 - dict->stats = callocz(1, sizeof(struct dictionary_stats));
646 - allocated += sizeof(struct dictionary_stats);
647 - dict->stats->memory = allocated;
648 - }
649 - else
650 - dict->stats = NULL;
613 + dict->memory = allocated;
614
615 hashtable_init_unsafe(dict);
616 return (DICTIONARY *)dict;
617 }
618
619 size_t dictionary_destroy(DICTIONARY *dict) {
620 + if(!dict) return 0;
621 +
622 debug(D_DICTIONARY, "Destroying dictionary.");
623
624 dictionary_lock_wrlock(dict);
@@ -680,12 +645,6 @@ size_t dictionary_destroy(DICTIONARY *dict) {
645 freed += dictionary_lock_free(dict);
646 freed += reference_counter_free(dict);
647
683 - if(unlikely(dict->flags & DICTIONARY_FLAG_WITH_STATISTICS)) {
684 - freez(dict->stats);
685 - dict->stats = NULL;
686 - freed += sizeof(struct dictionary_stats);
687 - }
688 -
648 freez(dict);
649 freed += sizeof(DICTIONARY);
650
@@ -722,9 +681,6 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz
681 nv = *pnv = namevalue_create_unsafe(dict, name, name_len, value, value_len);
682 hashtable_inserted_name_value_unsafe(dict, name, name_len, nv);
683 linkedlist_namevalue_link_unsafe(dict, nv);
725 -
726 - if(dict->ins_callback)
727 - dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
684 }
685 else {
686 // the item is already in the index
@@ -732,16 +688,9 @@ void *dictionary_set_unsafe(DICTIONARY *dict, const char *name, void *value, siz
688 // or overwrite the value, depending on dictionary flags
689
690 nv = *pnv;
735 - if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE)) {
736 -
737 - if(dict->del_callback)
738 - dict->del_callback(nv->name, nv->value, dict->del_callback_data);
739 -
691 + if(!(dict->flags & DICTIONARY_FLAG_DONT_OVERWRITE_VALUE))
692 namevalue_reset_unsafe(dict, nv, value, value_len);
693
742 - if(dict->ins_callback)
743 - dict->ins_callback(nv->name, nv->value, dict->ins_callback_data);
744 - }
694 else if(dict->conflict_callback)
695 dict->conflict_callback(nv->name, nv->value, value, dict->conflict_callback_data);
696 }
@@ -811,10 +760,6 @@ int dictionary_del_unsafe(DICTIONARY *dict, const char *name) {
760
761 if(!reference_counter_mark_deleted(dict, nv)) {
762 linkedlist_namevalue_unlink_unsafe(dict, nv);
814 -
815 - if(dict->del_callback)
816 - dict->del_callback(nv->name, nv->value, dict->del_callback_data);
817 -
763 namevalue_destroy_unsafe(dict, nv);
764 }
765 ret = 0;
@@ -835,6 +780,8 @@ int dictionary_del(DICTIONARY *dict, const char *name) {
780 void *dictionary_foreach_start_rw(DICTFE *dfe, DICTIONARY *dict, char rw) {
781 if(unlikely(!dfe || !dict)) return NULL;
782
783 + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
784 +
785 dfe->dict = dict;
786 dfe->started_ut = now_realtime_usec();
787
@@ -912,6 +859,10 @@ usec_t dictionary_foreach_done(DICTFE *dfe) {
859 // do not use other dictionary calls while walking the dictionary - deadlock!
860
861 int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) {
862 + if(unlikely(!dict)) return 0;
863 +
864 + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
865 +
866 if(rw == 'r' || rw == 'R')
867 dictionary_lock_rlock(dict);
868 else
@@ -942,6 +893,54 @@ int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const c
893 return ret;
894 }
895
896 +// ----------------------------------------------------------------------------
897 +// sort
898 +
899 +static int dictionary_sort_compar(const void *nv1, const void *nv2) {
900 + return strcmp((*(NAME_VALUE **)nv1)->name, (*(NAME_VALUE **)nv2)->name);
901 +}
902 +
903 +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data) {
904 + if(unlikely(!dict || !dict->entries)) return 0;
905 +
906 + DICTIONARY_STATS_WALKTHROUGHS_PLUS1(dict);
907 +
908 + if(rw == 'r' || rw == 'R')
909 + dictionary_lock_rlock(dict);
910 + else
911 + dictionary_lock_wrlock(dict);
912 +
913 + size_t count = dict->entries;
914 + NAME_VALUE **array = mallocz(sizeof(NAME_VALUE *) * count);
915 +
916 + size_t i;
917 + NAME_VALUE *nv;
918 + for(nv = dict->first_item, i = 0; nv && i < count ;nv = nv->next, i++)
919 + array[i] = nv;
920 +
921 + if(unlikely(nv))
922 + error("DICTIONARY: during sorting expected to have %zu items in dictionary, but there are more. Sorted results may be incomplete. This is internal error - dictionaries fail to maintain an accurate number of the number of entries they have.", count);
923 +
924 + if(unlikely(i != count)) {
925 + error("DICTIONARY: during sorting expected to have %zu items in dictionary, but there are %zu. Sorted results may be incomplete. This is internal error - dictionaries fail to maintain an accurate number of the number of entries they have.", count, i);
926 + count = i;
927 + }
928 +
929 + qsort(array, count, sizeof(NAME_VALUE *), dictionary_sort_compar);
930 +
931 + int ret = 0;
932 + for(i = 0; i < count ;i++) {
933 + int r = callback((array[i])->name, (array[i])->value, data);
934 + if(r < 0) { ret = r; break; }
935 + ret += r;
936 + }
937 +
938 + dictionary_unlock(dict);
939 + freez(array);
940 +
941 + return ret;
942 +}
943 +
944 // ----------------------------------------------------------------------------
945 // unit test
946
@@ -1196,7 +1195,7 @@ static usec_t dictionary_unittest_run_and_measure_time(DICTIONARY *dict, char *m
1195 return dt;
1196 }
1197
1199 -void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1198 +static void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1199 dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
1200 dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_clone);
1201 dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
@@ -1211,7 +1210,7 @@ void dictionary_unittest_clone(DICTIONARY *dict, char **names, char **values, si
1210 dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
1211 }
1212
1214 -void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1213 +static void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1214 dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_nonclone);
1215 dictionary_unittest_run_and_measure_time(dict, "getting entries", names, values, entries, errors, dictionary_unittest_get_nonclone);
1216 dictionary_unittest_run_and_measure_time(dict, "getting non-existing entries", names, values, entries, errors, dictionary_unittest_get_nonexisting);
@@ -1226,6 +1225,48 @@ void dictionary_unittest_nonclone(DICTIONARY *dict, char **names, char **values,
1225 dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, errors, dictionary_unittest_destroy);
1226 }
1227
1228 +struct dictionary_unittest_sorting {
1229 + const char *oldname;
1230 + const char *oldvalue;
1231 + size_t count;
1232 +};
1233 +
1234 +static int dictionary_unittest_sorting_callback(const char *name, void *value, void *data) {
1235 + struct dictionary_unittest_sorting *t = (struct dictionary_unittest_sorting *)data;
1236 + const char *v = (const char *)value;
1237 +
1238 + int ret = 0;
1239 + if(t->oldname && strcmp(t->oldname, name) > 0) {
1240 + fprintf(stderr, "name '%s' should be after '%s'\n", t->oldname, name);
1241 + ret = 1;
1242 + }
1243 + t->count++;
1244 + t->oldname = name;
1245 + t->oldvalue = v;
1246 +
1247 + return ret;
1248 +}
1249 +
1250 +static size_t dictionary_unittest_sorted_walkthrough(DICTIONARY *dict, char **names, char **values, size_t entries) {
1251 + (void)names;
1252 + (void)values;
1253 + struct dictionary_unittest_sorting tmp = { .oldname = NULL, .oldvalue = NULL, .count = 0 };
1254 + size_t errors;
1255 + errors = dictionary_sorted_walkthrough_read(dict, dictionary_unittest_sorting_callback, &tmp);
1256 +
1257 + if(tmp.count != entries) {
1258 + fprintf(stderr, "Expected %zu entries, counted %zu\n", entries, tmp.count);
1259 + errors++;
1260 + }
1261 + return errors;
1262 +}
1263 +
1264 +static void dictionary_unittest_sorting(DICTIONARY *dict, char **names, char **values, size_t entries, size_t *errors) {
1265 + dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, errors, dictionary_unittest_set_clone);
1266 + dictionary_unittest_run_and_measure_time(dict, "sorted walkthrough", names, values, entries, errors, dictionary_unittest_sorted_walkthrough);
1267 + dictionary_unittest_run_and_measure_time(dict, "destroying dictionary", names, values, entries, errors, dictionary_unittest_destroy);
1268 +}
1269 +
1270 int dictionary_unittest(size_t entries) {
1271 if(entries < 10) entries = 10;
1272
@@ -1237,23 +1278,23 @@ int dictionary_unittest(size_t entries) {
1278 char **values = dictionary_unittest_generate_values(entries);
1279
1280 fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
1240 - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS);
1281 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
1282 dictionary_unittest_clone(dict, names, values, entries, &errors);
1283
1284 fprintf(stderr, "\nCreating dictionary multi threaded, clone, %zu items\n", entries);
1244 - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS);
1285 + dict = dictionary_create(DICTIONARY_FLAG_NONE);
1286 dictionary_unittest_clone(dict, names, values, entries, &errors);
1287
1288 fprintf(stderr, "\nCreating dictionary single threaded, non-clone, add-in-front options, %zu items\n", entries);
1248 - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1289 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1290 dictionary_unittest_nonclone(dict, names, values, entries, &errors);
1291
1292 fprintf(stderr, "\nCreating dictionary multi threaded, non-clone, add-in-front options, %zu items\n", entries);
1252 - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1293 + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_ADD_IN_FRONT);
1294 dictionary_unittest_nonclone(dict, names, values, entries, &errors);
1295
1296 fprintf(stderr, "\nCreating dictionary single-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1256 - dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1297 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1298 dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1299 dictionary_unittest_run_and_measure_time(dict, "resetting non-overwrite entries", names, values, entries, &errors, dictionary_unittest_reset_dont_overwrite_nonclone);
1300 dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop", names, values, entries, &errors, dictionary_unittest_foreach);
@@ -1262,19 +1303,23 @@ int dictionary_unittest(size_t entries) {
1303 dictionary_unittest_run_and_measure_time(dict, "destroying full dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1304
1305 fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1265 - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1306 + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1307 dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1308 dictionary_unittest_run_and_measure_time(dict, "walkthrough write delete this", names, values, entries, &errors, dictionary_unittest_walkthrough_delete_this);
1309 dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1310
1311 fprintf(stderr, "\nCreating dictionary multi-threaded, non-clone, don't overwrite options, %zu items\n", entries);
1271 - dict = dictionary_create(DICTIONARY_FLAG_WITH_STATISTICS|DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1312 + dict = dictionary_create(DICTIONARY_FLAG_NAME_LINK_DONT_CLONE|DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE|DICTIONARY_FLAG_DONT_OVERWRITE_VALUE);
1313 dictionary_unittest_run_and_measure_time(dict, "adding entries", names, values, entries, &errors, dictionary_unittest_set_nonclone);
1314 dictionary_unittest_run_and_measure_time(dict, "foreach write delete this", names, values, entries, &errors, dictionary_unittest_foreach_delete_this);
1315 dictionary_unittest_run_and_measure_time(dict, "traverse foreach read loop empty", names, values, 0, &errors, dictionary_unittest_foreach);
1316 dictionary_unittest_run_and_measure_time(dict, "walkthrough read callback empty", names, values, 0, &errors, dictionary_unittest_walkthrough);
1317 dictionary_unittest_run_and_measure_time(dict, "destroying empty dictionary", names, values, entries, &errors, dictionary_unittest_destroy);
1318
1319 + fprintf(stderr, "\nCreating dictionary single threaded, clone, %zu items\n", entries);
1320 + dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
1321 + dictionary_unittest_sorting(dict, names, values, entries, &errors);
1322 +
1323 dictionary_unittest_free_char_pp(names, entries);
1324 dictionary_unittest_free_char_pp(values, entries);
1325
libnetdata/dictionary/dictionary.h
+8 -4
@@ -44,10 +44,9 @@ typedef enum dictionary_flags {
44 DICTIONARY_FLAG_SINGLE_THREADED = (1 << 0), // don't use any locks (default: use locks)
45 DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE = (1 << 1), // don't copy the value, just point to the one provided (default: copy)
46 DICTIONARY_FLAG_NAME_LINK_DONT_CLONE = (1 << 2), // don't copy the name, just point to the one provided (default: copy)
47 - DICTIONARY_FLAG_WITH_STATISTICS = (1 << 3), // maintain statistics about dictionary operations (default: disabled)
48 - DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 4), // don't overwrite values of dictionary items (default: overwrite)
49 - DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 5), // add dictionary items at the front of the linked list (default: at the end)
50 - DICTIONARY_FLAG_RESERVED1 = (1 << 6), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS
47 + DICTIONARY_FLAG_DONT_OVERWRITE_VALUE = (1 << 3), // don't overwrite values of dictionary items (default: overwrite)
48 + DICTIONARY_FLAG_ADD_IN_FRONT = (1 << 4), // add dictionary items at the front of the linked list (default: at the end)
49 + DICTIONARY_FLAG_RESERVED1 = (1 << 5), // this is reserved for DICTIONARY_FLAG_REFERENCE_COUNTERS
50 } DICTIONARY_FLAGS;
51
52 // Create a dictionary
@@ -126,6 +125,10 @@ extern int dictionary_del_unsafe(DICTIONARY *dict, const char *name);
125 #define dictionary_walkthrough_write(dict, callback, data) dictionary_walkthrough_rw(dict, 'w', callback, data)
126 extern int dictionary_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *value, void *data), void *data);
127
128 +#define dictionary_sorted_walkthrough_read(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'r', callback, data)
129 +#define dictionary_sorted_walkthrough_write(dict, callback, data) dictionary_sorted_walkthrough_rw(dict, 'w', callback, data)
130 +int dictionary_sorted_walkthrough_rw(DICTIONARY *dict, char rw, int (*callback)(const char *name, void *entry, void *data), void *data);
131 +
132 // Traverse with foreach
133 //
134 // Use like this:
@@ -184,6 +187,7 @@ extern size_t dictionary_stats_inserts(DICTIONARY *dict);
187 extern size_t dictionary_stats_searches(DICTIONARY *dict);
188 extern size_t dictionary_stats_deletes(DICTIONARY *dict);
189 extern size_t dictionary_stats_resets(DICTIONARY *dict);
190 +extern size_t dictionary_stats_walkthroughs(DICTIONARY *dict);
191
192 extern int dictionary_unittest(size_t entries);
193
libnetdata/inlined.h
+14 -14
@@ -233,21 +233,21 @@ static inline char *strncpyz(char *dst, const char *src, size_t n) {
233 return p;
234 }
235
236 -static inline void sanitize_json_string(char *dst, char *src, size_t len) {
237 - while (*src != '\0' && len > 1) {
238 - if (*src == '\\' || *src == '\"' || *src < 0x1F) {
239 - if (*src < 0x1F) {
240 - *dst++ = '_';
241 - src++;
242 - len--;
243 - } else {
244 - *dst++ = '\\';
245 - *dst++ = *src++;
246 - len -= 2;
247 - }
248 - } else {
236 +static inline void sanitize_json_string(char *dst, const char *src, size_t dst_size) {
237 + while (*src != '\0' && dst_size > 1) {
238 + if (*src < 0x1F) {
239 + *dst++ = '_';
240 + src++;
241 + dst_size--;
242 + }
243 + else if (*src == '\\' || *src == '\"') {
244 + *dst++ = '\\';
245 + *dst++ = *src++;
246 + dst_size -= 2;
247 + }
248 + else {
249 *dst++ = *src++;
250 - len--;
250 + dst_size--;
251 }
252 }
253 *dst = '\0';
libnetdata/json/json.h
+7 -2
@@ -3,8 +3,13 @@
3
4
5 #if ENABLE_JSONC
6 -# include <json-c/json.h>
7 -#endif
6 +#include <json-c/json.h>
7 +// fix an older json-c bug
8 +// https://github.com/json-c/json-c/issues/135
9 +#ifdef error_description
10 +#undef error_description
11 +#endif // error_description
12 +#endif // ENABLE_JSONC
13
14 #include "jsmn.h"
15
parser/parser.h
+4 -4
@@ -29,10 +29,10 @@ typedef struct pluginsd_action {
29 PARSER_RC (*flush_action)(void *user, RRDSET *st);
30 PARSER_RC (*disable_action)(void *user);
31 PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
32 - PARSER_RC (*label_action)(void *user, char *key, char *value, LABEL_SOURCE source);
33 - PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, struct label *new_labels);
34 - PARSER_RC (*clabel_action)(void *user, char *key, char *value, LABEL_SOURCE source);
35 - PARSER_RC (*clabel_commit_action)(void *user, RRDHOST *host, struct label *new_labels);
32 + PARSER_RC (*label_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
33 + PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
34 + PARSER_RC (*clabel_action)(void *user, char *key, char *value, RRDLABEL_SRC source);
35 + PARSER_RC (*clabel_commit_action)(void *user, RRDHOST *host, DICTIONARY *new_labels);
36
37 PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
38 PARSER_RC (*context_action)(void *user, uuid_t *uuid);
registry/registry_internals.h
+1 -1
@@ -8,7 +8,7 @@
8 #define REGISTRY_URL_FLAGS_DEFAULT 0x00
9 #define REGISTRY_URL_FLAGS_EXPIRED 0x01
10
11 -#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED | DICTIONARY_FLAG_WITH_STATISTICS)
11 +#define REGISTRY_DICTIONARY_FLAGS (DICTIONARY_FLAG_VALUE_LINK_DONT_CLONE | DICTIONARY_FLAG_NAME_LINK_DONT_CLONE | DICTIONARY_FLAG_SINGLE_THREADED)
12
13 // ----------------------------------------------------------------------------
14 // COMMON structures
streaming/receiver.c
+8 -1
@@ -660,7 +660,14 @@ static int rrdpush_receive(struct receiver_state *rpt)
660 */
661
662 // rpt->host->connected_senders++;
663 - rpt->host->labels.labels_flag = (rpt->stream_version > 0)?LABEL_FLAG_UPDATE_STREAM:LABEL_FLAG_STOP_STREAM;
663 + if(rpt->stream_version > 0) {
664 + rrdhost_flag_set(rpt->host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
665 + rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_STREAM_LABELS_STOP);
666 + }
667 + else {
668 + rrdhost_flag_set(rpt->host, RRDHOST_FLAG_STREAM_LABELS_STOP);
669 + rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
670 + }
671
672 if(health_enabled != CONFIG_BOOLEAN_NO) {
673 if(alarms_delay > 0) {
streaming/rrdpush.c
+16 -32
@@ -193,20 +193,15 @@ static inline int need_to_send_chart_definition(RRDSET *st) {
193 }
194
195 // chart labels
196 +static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
197 + BUFFER *wb = (BUFFER *)data;
198 + buffer_sprintf(wb, "CLABEL \"%s\" \"%s\" %d\n", name, value, ls);
199 + return 1;
200 +}
201 void rrdpush_send_clabels(RRDHOST *host, RRDSET *st) {
197 - struct label_index *labels_c = &st->state->labels;
198 - if (labels_c) {
199 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
200 - struct label *lbl = labels_c->head;
201 - while(lbl) {
202 - buffer_sprintf(host->sender->build,
203 - "CLABEL \"%s\" \"%s\" %d\n", lbl->key, lbl->value, (int)lbl->label_source);
204 -
205 - lbl = lbl->next;
206 - }
207 - if (labels_c->head)
202 + if (st->state && st->state->chart_labels) {
203 + if(rrdlabels_walkthrough_read(st->state->chart_labels, send_clabels_callback, host->sender->build) > 0)
204 buffer_sprintf(host->sender->build,"CLABEL_COMMIT\n");
209 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
205 }
206 }
207
@@ -364,36 +359,25 @@ void rrdset_done_push(RRDSET *st) {
359 }
360
361 // labels
362 +static int send_labels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
363 + BUFFER *wb = (BUFFER *)data;
364 + buffer_sprintf(wb, "LABEL \"%s\" = %d \"%s\"\n", name, ls, value);
365 + return 1;
366 +}
367 void rrdpush_send_labels(RRDHOST *host) {
368 - if (!host->labels.head || !(host->labels.labels_flag & LABEL_FLAG_UPDATE_STREAM) || (host->labels.labels_flag & LABEL_FLAG_STOP_STREAM))
368 + if (!host->host_labels || !rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE) || (rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_LABELS_STOP)))
369 return;
370
371 sender_start(host->sender);
372 - rrdhost_rdlock(host);
373 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
374 -
375 - struct label *label_i = host->labels.head;
376 - while(label_i) {
377 - buffer_sprintf(host->sender->build
378 - , "LABEL \"%s\" = %d %s\n"
379 - , label_i->key
380 - , (int)label_i->label_source
381 - , label_i->value);
382 -
383 - label_i = label_i->next;
384 - }
385 -
386 - buffer_sprintf(host->sender->build
387 - , "OVERWRITE %s\n", "labels");
372
389 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
390 - rrdhost_unlock(host);
373 + rrdlabels_walkthrough_read(host->host_labels, send_labels_callback, host->sender->build);
374 + buffer_sprintf(host->sender->build, "OVERWRITE %s\n", "labels");
375 sender_commit(host->sender);
376
377 if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
378 error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
379
396 - host->labels.labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
380 + rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
381 }
382
383 void rrdpush_claimed_id(RRDHOST *host)
streaming/sender.c
+4 -4
@@ -172,8 +172,8 @@ static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
172 }
173
174 static inline void rrdpush_set_flags_to_newest_stream(RRDHOST *host) {
175 - host->labels.labels_flag |= LABEL_FLAG_UPDATE_STREAM;
176 - host->labels.labels_flag &= ~LABEL_FLAG_STOP_STREAM;
175 + rrdhost_flag_set(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
176 + rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_STOP);
177 }
178
179 void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host)
@@ -236,8 +236,8 @@ static inline long int parse_stream_version(RRDHOST *host, char *http)
236 answer = memcmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT));
237 if (!answer) {
238 stream_version = 0;
239 - host->labels.labels_flag |= LABEL_FLAG_STOP_STREAM;
240 - host->labels.labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
239 + rrdhost_flag_set(host, RRDHOST_FLAG_STREAM_LABELS_STOP);
240 + rrdhost_flag_clear(host, RRDHOST_FLAG_STREAM_LABELS_UPDATE);
241 }
242 else {
243 stream_version = parse_stream_version_for_errors(http);
web/api/formatters/json_wrapper.c
+18 -23
@@ -19,6 +19,21 @@ static int value_list_output(const char *name, void *entry, void *data) {
19 return 0;
20 }
21
22 +static int fill_formatted_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
23 + (void)ls;
24 + DICTIONARY *dict = (DICTIONARY *)data;
25 + char n[RRD_ID_LENGTH_MAX * 2 + 2];
26 + char output[RRD_ID_LENGTH_MAX * 2 + 8];
27 + char v[RRD_ID_LENGTH_MAX * 2 + 1];
28 +
29 + sanitize_json_string(v, (char *)value, RRD_ID_LENGTH_MAX * 2);
30 + int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", name, v);
31 + snprintfz(n, RRD_ID_LENGTH_MAX * 2, "%s:%s", name, v);
32 + dictionary_set(dict, n, output, len + 1);
33 +
34 + return 1;
35 +}
36 +
37 void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS options, int string_value,
38 QUERY_PARAMS *rrdset_query_data)
39 {
@@ -122,7 +137,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
137
138 char name[RRD_ID_LENGTH_MAX * 2 + 2];
139 char output[RRD_ID_LENGTH_MAX * 2 + 8];
125 - char value[RRD_ID_LENGTH_MAX * 2 + 1];
140
141 struct value_output co = {.c = 0, .wb = wb};
142
@@ -153,19 +167,8 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
167 dict = dictionary_create(DICTIONARY_FLAG_SINGLE_THREADED);
168 for (i = 0, rd = temp_rd ? temp_rd : r->st->dimensions; rd; rd = rd->next) {
169 st = rd->rrdset;
156 - if (likely(st->state)) {
157 - struct label_index *labels = &st->state->labels;
158 - if (labels->head) {
159 - netdata_rwlock_rdlock(&labels->labels_rwlock);
160 - for (struct label *label = labels->head; label; label = label->next) {
161 - sanitize_json_string(value, label->value, RRD_ID_LENGTH_MAX * 2);
162 - int len = snprintfz(output, RRD_ID_LENGTH_MAX * 2 + 7, "[\"%s\", \"%s\"]", label->key, value);
163 - snprintfz(name, RRD_ID_LENGTH_MAX * 2, "%s:%s", label->key, value);
164 - dictionary_set(dict, name, output, len + 1);
165 - }
166 - netdata_rwlock_unlock(&labels->labels_rwlock);
167 - }
168 - }
170 + if (st->state && st->state->chart_labels)
171 + rrdlabels_walkthrough_read(st->state->chart_labels, fill_formatted_callback, dict);
172 }
173 dictionary_walkthrough_read(dict, value_list_output, &co);
174 dictionary_destroy(dict);
@@ -207,8 +210,6 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
210 char *label_key = NULL;
211 int keys = 0;
212 while (pattern && (label_key = simple_pattern_iterate(&pattern))) {
210 - uint32_t key_hash = simple_hash(label_key);
211 - struct label *current_label;
213
214 if (keys)
215 buffer_strcat(wb, ", ");
@@ -223,13 +224,7 @@ void rrdr_json_wrapper_begin(RRDR *r, BUFFER *wb, uint32_t format, RRDR_OPTIONS
224 if (i)
225 buffer_strcat(wb, ", ");
226
226 - current_label = rrdset_lookup_label_key(rd->rrdset, label_key, key_hash);
227 - if (current_label) {
228 - buffer_strcat(wb, sq);
229 - buffer_strcat(wb, current_label->value);
230 - buffer_strcat(wb, sq);
231 - } else
232 - buffer_strcat(wb, "null");
227 + rrdlabels_get_value_to_buffer_or_null(rd->rrdset->state->chart_labels, wb, label_key, sq, "null");
228 i++;
229 }
230 if (!i) {
web/api/formatters/rrdset2json.c
+4 -14
@@ -4,8 +4,10 @@
4
5 void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
6 {
7 + if(unlikely(!st->state || !st->state->chart_labels))
8 + return;
9 +
10 char tabs[11];
8 - struct label_index *labels = &st->state->labels;
11
12 if (indentation > 10)
13 indentation = 10;
@@ -16,20 +18,8 @@ void chart_labels2json(RRDSET *st, BUFFER *wb, size_t indentation)
18 indentation--;
19 }
20
19 - int count = 0;
20 - netdata_rwlock_rdlock(&labels->labels_rwlock);
21 - for (struct label *label = labels->head; label; label = label->next) {
22 - if(count > 0) buffer_strcat(wb, ",\n");
23 - buffer_strcat(wb, tabs);
24 -
25 - char value[CONFIG_MAX_VALUE * 2 + 1];
26 - sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
27 - buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
28 -
29 - count++;
30 - }
21 + rrdlabels_to_buffer(st->state->chart_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
22 buffer_strcat(wb, "\n");
32 - netdata_rwlock_unlock(&labels->labels_rwlock);
23 }
24
25 // generate JSON for the /api/v1/chart API call
web/api/web_api_v1.c
+13 -21
@@ -392,6 +392,7 @@ void fix_google_param(char *s) {
392 }
393 }
394
395 +
396 // returns the HTTP code
397 inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url) {
398 debug(D_WEB_CLIENT, "%llu: API v1 data with URL '%s'", w->id, url);
@@ -528,18 +529,23 @@ inline int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, c
529
530 uint32_t context_hash = simple_hash(context);
531
532 + SIMPLE_PATTERN *chart_label_key_pattern = NULL;
533 + if(chart_label_key)
534 + chart_label_key_pattern = simple_pattern_create(chart_label_key, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
535 +
536 + SIMPLE_PATTERN *chart_labels_filter_pattern = NULL;
537 + if(chart_labels_filter)
538 + chart_labels_filter_pattern = simple_pattern_create(chart_labels_filter, ",|\t\r\n\f\v", SIMPLE_PATTERN_EXACT);
539 +
540 rrdhost_rdlock(host);
532 - char *words[MAX_CHART_LABELS_FILTER];
533 - uint32_t hash_key_list[MAX_CHART_LABELS_FILTER];
534 - int word_count = 0;
541 rrdset_foreach_read(st1, host) {
542 if (st1->hash_context == context_hash && !strcmp(st1->context, context) &&
537 - (!chart_label_key || rrdset_contains_label_keylist(st1, chart_label_key)) &&
538 - (!chart_labels_filter ||
539 - rrdset_matches_label_keys(st1, chart_labels_filter, words, hash_key_list, &word_count, MAX_CHART_LABELS_FILTER)))
543 + (!chart_label_key_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_label_key_pattern, ':')) &&
544 + (!chart_labels_filter_pattern || rrdlabels_match_simple_pattern_parsed(st1->state->chart_labels, chart_labels_filter_pattern, ':')))
545 build_context_param_list(owa, &context_param_list, st1);
546 }
547 rrdhost_unlock(host);
548 +
549 if (likely(context_param_list && context_param_list->rd)) // Just set the first one
550 st = context_param_list->rd->rrdset;
551 else {
@@ -964,22 +970,8 @@ inline void host_labels2json(RRDHOST *host, BUFFER *wb, size_t indentation) {
970 indentation--;
971 }
972
967 - int count = 0;
968 - rrdhost_rdlock(host);
969 - netdata_rwlock_rdlock(&host->labels.labels_rwlock);
970 - for (struct label *label = host->labels.head; label; label = label->next) {
971 - if(count > 0) buffer_strcat(wb, ",\n");
972 - buffer_strcat(wb, tabs);
973 -
974 - char value[CONFIG_MAX_VALUE * 2 + 1];
975 - sanitize_json_string(value, label->value, CONFIG_MAX_VALUE * 2);
976 - buffer_sprintf(wb, "\"%s\": \"%s\"", label->key, value);
977 -
978 - count++;
979 - }
973 + rrdlabels_to_buffer(host->host_labels, wb, tabs, ":", "\"", ",\n", NULL, NULL, NULL, NULL);
974 buffer_strcat(wb, "\n");
981 - netdata_rwlock_unlock(&host->labels.labels_rwlock);
982 - rrdhost_unlock(host);
975 }
976
977 extern int aclk_connected;