@cryptotaxi247 / netdata-1 / commits / 1bd8a2554

Add support for persistent metadata (#9324)

* Implemented collector metadata logging * Added persistent GUIDs for charts and dimensions * Added metadata log replay and automatic compaction * Added detection of charts with no active collector (archived) * Added new endpoint to report archived charts via `/api/v1/archivedcharts` * Added support for collector metadata update Co-authored-by: Markos Fountoulakis <44345837+mfundul@users.noreply.github.com>

Stelios Fragkakis committed Jun 12, 2020 at 10:35 UTC 1bd8a255441de6056a0d51a7bb787f76b590ffb6
57 files changed +3871 -199
CMakeLists.txt
+13
@@ -556,6 +556,19 @@ set(RRD_PLUGIN_FILES
556 database/engine/pagecache.h
557 database/engine/rrdenglocking.c
558 database/engine/rrdenglocking.h
559 + database/engine/metadata_log/metadatalog.c
560 + database/engine/metadata_log/metadatalog.h
561 + database/engine/metadata_log/metadatalogapi.c
562 + database/engine/metadata_log/metadatalogapi.h
563 + database/engine/metadata_log/logfile.h
564 + database/engine/metadata_log/logfile.c
565 + database/engine/metadata_log/metadatalogprotocol.h
566 + database/engine/metadata_log/metalogpluginsd.c
567 + database/engine/metadata_log/metalogpluginsd.h
568 + database/engine/metadata_log/compaction.c
569 + database/engine/metadata_log/compaction.h
570 + database/engine/global_uuid_map/global_uuid_map.c
571 + database/engine/global_uuid_map/global_uuid_map.h
572 )
573
574 set(WEB_PLUGIN_FILES
Makefile.am
+13
@@ -375,6 +375,19 @@ if ENABLE_DBENGINE
375 database/engine/pagecache.h \
376 database/engine/rrdenglocking.c \
377 database/engine/rrdenglocking.h \
378 + database/engine/metadata_log/metadatalog.c \
379 + database/engine/metadata_log/metadatalog.h \
380 + database/engine/metadata_log/metadatalogapi.c \
381 + database/engine/metadata_log/metadatalogapi.h \
382 + database/engine/metadata_log/logfile.h \
383 + database/engine/metadata_log/logfile.c \
384 + database/engine/metadata_log/metadatalogprotocol.h \
385 + database/engine/metadata_log/metalogpluginsd.c \
386 + database/engine/metadata_log/metalogpluginsd.h \
387 + database/engine/metadata_log/compaction.c \
388 + database/engine/metadata_log/compaction.h \
389 + database/engine/global_uuid_map/global_uuid_map.c \
390 + database/engine/global_uuid_map/global_uuid_map.h \
391 $(NULL)
392 endif
393
aclk/agent_cloud_link.c
+13 -1
@@ -680,6 +680,9 @@ static struct _collector *_add_collector(const char *hostname, const char *plugi
680 void aclk_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
681 {
682 struct _collector *tmp_collector;
683 + if (unlikely(!netdata_ready)) {
684 + return;
685 + }
686
687 COLLECTOR_LOCK;
688
@@ -711,6 +714,9 @@ void aclk_add_collector(const char *hostname, const char *plugin_name, const cha
714 void aclk_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
715 {
716 struct _collector *tmp_collector;
717 + if (unlikely(!netdata_ready)) {
718 + return;
719 + }
720
721 COLLECTOR_LOCK;
722
@@ -1752,7 +1758,7 @@ int aclk_send_info_metadata()
1758 debug(D_ACLK, "Metadata %s with info has %zu bytes", msg_id, local_buffer->len);
1759
1760 buffer_sprintf(local_buffer, ", \n\t \"charts\" : ");
1755 - charts2json(localhost, local_buffer, 1);
1761 + charts2json(localhost, local_buffer, 1, 0);
1762 buffer_sprintf(local_buffer, "\n}\n}");
1763 debug(D_ACLK, "Metadata %s with chart has %zu bytes", msg_id, local_buffer->len);
1764
@@ -1859,6 +1865,9 @@ int aclk_update_chart(RRDHOST *host, char *chart_name, ACLK_CMD aclk_cmd)
1865 UNUSED(chart_name);
1866 return 0;
1867 #else
1868 + if (unlikely(!netdata_ready))
1869 + return 0;
1870 +
1871 if (!netdata_cloud_setting)
1872 return 0;
1873
@@ -1886,6 +1895,9 @@ int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
1895 {
1896 BUFFER *local_buffer = NULL;
1897
1898 + if (unlikely(!netdata_ready))
1899 + return 0;
1900 +
1901 if (host != localhost)
1902 return 0;
1903
collectors/diskspace.plugin/plugin_diskspace.c
+2 -2
@@ -254,7 +254,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
254 netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
255 if(unlikely(!m->st_space)) {
256 m->do_space = CONFIG_BOOLEAN_YES;
257 - m->st_space = rrdset_find_bytype_localhost("disk_space", disk);
257 + m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
258 if(unlikely(!m->st_space)) {
259 char title[4096 + 1];
260 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", family, mi->mount_source);
@@ -296,7 +296,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
296 netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
297 if(unlikely(!m->st_inodes)) {
298 m->do_inodes = CONFIG_BOOLEAN_YES;
299 - m->st_inodes = rrdset_find_bytype_localhost("disk_inodes", disk);
299 + m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
300 if(unlikely(!m->st_inodes)) {
301 char title[4096 + 1];
302 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", family, mi->mount_source);
collectors/freebsd.plugin/plugin_freebsd.c
+1 -1
@@ -129,7 +129,7 @@ void *freebsd_main(void *ptr) {
129 static RRDSET *st = NULL;
130
131 if(unlikely(!st)) {
132 - st = rrdset_find_bytype_localhost("netdata", "plugin_freebsd_modules");
132 + st = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
133
134 if(!st) {
135 st = rrdset_create_localhost(
collectors/macos.plugin/macos_fw.c
+15 -15
@@ -145,7 +145,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
145 total_disk_writes += diskstat.bytes_write;
146 }
147
148 - st = rrdset_find_bytype_localhost("disk", diskstat.name);
148 + st = rrdset_find_active_bytype_localhost("disk", diskstat.name);
149 if (unlikely(!st)) {
150 st = rrdset_create_localhost(
151 "disk"
@@ -183,7 +183,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
183 CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.writes);
184 }
185
186 - st = rrdset_find_bytype_localhost("disk_ops", diskstat.name);
186 + st = rrdset_find_active_bytype_localhost("disk_ops", diskstat.name);
187 if (unlikely(!st)) {
188 st = rrdset_create_localhost(
189 "disk_ops"
@@ -222,7 +222,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
222 CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.time_write);
223 }
224
225 - st = rrdset_find_bytype_localhost("disk_util", diskstat.name);
225 + st = rrdset_find_active_bytype_localhost("disk_util", diskstat.name);
226 if (unlikely(!st)) {
227 st = rrdset_create_localhost(
228 "disk_util"
@@ -260,7 +260,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
260 CFNumberGetValue(number, kCFNumberSInt64Type, &diskstat.latency_write);
261 }
262
263 - st = rrdset_find_bytype_localhost("disk_iotime", diskstat.name);
263 + st = rrdset_find_active_bytype_localhost("disk_iotime", diskstat.name);
264 if (unlikely(!st)) {
265 st = rrdset_create_localhost(
266 "disk_iotime"
@@ -297,7 +297,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
297
298 // --------------------------------------------------------------------
299
300 - st = rrdset_find_bytype_localhost("disk_await", diskstat.name);
300 + st = rrdset_find_active_bytype_localhost("disk_await", diskstat.name);
301 if (unlikely(!st)) {
302 st = rrdset_create_localhost(
303 "disk_await"
@@ -328,7 +328,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
328
329 // --------------------------------------------------------------------
330
331 - st = rrdset_find_bytype_localhost("disk_avgsz", diskstat.name);
331 + st = rrdset_find_active_bytype_localhost("disk_avgsz", diskstat.name);
332 if (unlikely(!st)) {
333 st = rrdset_create_localhost(
334 "disk_avgsz"
@@ -359,7 +359,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
359
360 // --------------------------------------------------------------------
361
362 - st = rrdset_find_bytype_localhost("disk_svctm", diskstat.name);
362 + st = rrdset_find_active_bytype_localhost("disk_svctm", diskstat.name);
363 if (unlikely(!st)) {
364 st = rrdset_create_localhost(
365 "disk_svctm"
@@ -401,7 +401,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
401 }
402
403 if (likely(do_io)) {
404 - st = rrdset_find_bytype_localhost("system", "io");
404 + st = rrdset_find_active_bytype_localhost("system", "io");
405 if (unlikely(!st)) {
406 st = rrdset_create_localhost(
407 "system"
@@ -453,7 +453,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
453 // --------------------------------------------------------------------------
454
455 if (likely(do_space)) {
456 - st = rrdset_find_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
456 + st = rrdset_find_active_bytype_localhost("disk_space", mntbuf[i].f_mntonname);
457 if (unlikely(!st)) {
458 snprintfz(title, 4096, "Disk Space Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
459 st = rrdset_create_localhost(
@@ -486,7 +486,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
486 // --------------------------------------------------------------------------
487
488 if (likely(do_inodes)) {
489 - st = rrdset_find_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
489 + st = rrdset_find_active_bytype_localhost("disk_inodes", mntbuf[i].f_mntonname);
490 if (unlikely(!st)) {
491 snprintfz(title, 4096, "Disk Files (inodes) Usage for %s [%s]", mntbuf[i].f_mntonname, mntbuf[i].f_mntfromname);
492 st = rrdset_create_localhost(
@@ -533,7 +533,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
533
534 // --------------------------------------------------------------------
535
536 - st = rrdset_find_bytype_localhost("net", ifa->ifa_name);
536 + st = rrdset_find_active_bytype_localhost("net", ifa->ifa_name);
537 if (unlikely(!st)) {
538 st = rrdset_create_localhost(
539 "net"
@@ -561,7 +561,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
561
562 // --------------------------------------------------------------------
563
564 - st = rrdset_find_bytype_localhost("net_packets", ifa->ifa_name);
564 + st = rrdset_find_active_bytype_localhost("net_packets", ifa->ifa_name);
565 if (unlikely(!st)) {
566 st = rrdset_create_localhost(
567 "net_packets"
@@ -594,7 +594,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
594
595 // --------------------------------------------------------------------
596
597 - st = rrdset_find_bytype_localhost("net_errors", ifa->ifa_name);
597 + st = rrdset_find_active_bytype_localhost("net_errors", ifa->ifa_name);
598 if (unlikely(!st)) {
599 st = rrdset_create_localhost(
600 "net_errors"
@@ -623,7 +623,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
623
624 // --------------------------------------------------------------------
625
626 - st = rrdset_find_bytype_localhost("net_drops", ifa->ifa_name);
626 + st = rrdset_find_active_bytype_localhost("net_drops", ifa->ifa_name);
627 if (unlikely(!st)) {
628 st = rrdset_create_localhost(
629 "net_drops"
@@ -650,7 +650,7 @@ int do_macos_iokit(int update_every, usec_t dt) {
650
651 // --------------------------------------------------------------------
652
653 - st = rrdset_find_bytype_localhost("net_events", ifa->ifa_name);
653 + st = rrdset_find_active_bytype_localhost("net_events", ifa->ifa_name);
654 if (unlikely(!st)) {
655 st = rrdset_create_localhost(
656 "net_events"
collectors/macos.plugin/macos_mach_smi.c
+1 -1
@@ -55,7 +55,7 @@ int do_macos_mach_smi(int update_every, usec_t dt) {
55 error("DISABLED: system.cpu");
56 } else {
57
58 - st = rrdset_find_bytype_localhost("system", "cpu");
58 + st = rrdset_find_active_bytype_localhost("system", "cpu");
59 if (unlikely(!st)) {
60 st = rrdset_create_localhost(
61 "system"
collectors/macos.plugin/macos_sysctl.c
+1 -1
@@ -230,7 +230,7 @@ int do_macos_sysctl(int update_every, usec_t dt) {
230 error("DISABLED: system.load");
231 } else {
232
233 - st = rrdset_find_bytype_localhost("system", "load");
233 + st = rrdset_find_active_bytype_localhost("system", "load");
234 if (unlikely(!st)) {
235 st = rrdset_create_localhost(
236 "system"
collectors/plugins.d/plugins_d.h
+2 -2
@@ -31,10 +31,10 @@
31 #define PLUGINSD_KEYWORD_VARIABLE "VARIABLE"
32 #define PLUGINSD_KEYWORD_LABEL "LABEL"
33 #define PLUGINSD_KEYWORD_OVERWRITE "OVERWRITE"
34 -#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
34 #define PLUGINSD_KEYWORD_GUID "GUID"
36 -#define PLUGINSD_KEYWORD_HOST "HOST"
35 +#define PLUGINSD_KEYWORD_CONTEXT "CONTEXT"
36 #define PLUGINSD_KEYWORD_TOMBSTONE "TOMBSTONE"
37 +#define PLUGINSD_KEYWORD_HOST "HOST"
38
39
40 #define PLUGINSD_LINE_MAX 1024
collectors/plugins.d/pluginsd_parser.c
+65
@@ -561,6 +561,71 @@ PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins
561 return PARSER_RC_OK;
562 }
563
564 +PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action)
565 +{
566 + char *uuid_str = words[1];
567 + uuid_t uuid;
568 +
569 + if (unlikely(!uuid_str)) {
570 + error("requested a GUID, without a uuid.");
571 + return PARSER_RC_ERROR;
572 + }
573 + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
574 + error("requested a GUID, without a valid uuid string.");
575 + return PARSER_RC_ERROR;
576 + }
577 +
578 + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
579 + if (plugins_action->guid_action) {
580 + return plugins_action->guid_action(user, &uuid);
581 + }
582 +
583 + return PARSER_RC_OK;
584 +}
585 +
586 +PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action)
587 +{
588 + char *uuid_str = words[1];
589 + uuid_t uuid;
590 +
591 + if (unlikely(!uuid_str)) {
592 + error("requested a CONTEXT, without a uuid.");
593 + return PARSER_RC_ERROR;
594 + }
595 + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
596 + error("requested a CONTEXT, without a valid uuid string.");
597 + return PARSER_RC_ERROR;
598 + }
599 +
600 + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
601 + if (plugins_action->context_action) {
602 + return plugins_action->context_action(user, &uuid);
603 + }
604 +
605 + return PARSER_RC_OK;
606 +}
607 +
608 +PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action)
609 +{
610 + char *uuid_str = words[1];
611 + uuid_t uuid;
612 +
613 + if (unlikely(!uuid_str)) {
614 + error("requested a TOMBSTONE, without a uuid.");
615 + return PARSER_RC_ERROR;
616 + }
617 + if (unlikely(strlen(uuid_str) != GUID_LEN || uuid_parse(uuid_str, uuid) == -1)) {
618 + error("requested a TOMBSTONE, without a valid uuid string.");
619 + return PARSER_RC_ERROR;
620 + }
621 +
622 + debug(D_PLUGINSD, "Parsed uuid=%s", uuid_str);
623 + if (plugins_action->tombstone_action) {
624 + return plugins_action->tombstone_action(user, &uuid);
625 + }
626 +
627 + return PARSER_RC_OK;
628 +}
629
630 // New plugins.d parser
631
collectors/plugins.d/pluginsd_parser.h
+16 -12
@@ -16,19 +16,23 @@ typedef struct parser_user_object {
16 struct label *new_labels;
17 size_t count;
18 int enabled;
19 + void *private; // the user can set this for private use
20 } PARSER_USER_OBJECT;
21
21 -PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
22 -PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
23 -PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
24 -PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
25 -PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
26 - char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
27 -PARSER_RC pluginsd_disable_action(void *user);
28 -PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
29 -PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
30 - RRD_ALGORITHM algorithm_type);
31 -PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
32 -PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
22 +extern PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
23 +extern PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
24 +extern PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
25 +extern PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
26 +extern PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
27 + char *title, char *units, char *plugin, char *module, int priority,
28 + int update_every, RRDSET_TYPE chart_type, char *options);
29 +extern PARSER_RC pluginsd_disable_action(void *user);
30 +extern PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global,
31 + calculated_number value);
32 +extern PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
33 + long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type);
34 +extern PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
35 +extern PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
36 +
37
38 #endif //NETDATA_PLUGINSD_PARSER_H
collectors/proc.plugin/plugin_proc.c
+1 -1
@@ -148,7 +148,7 @@ void *proc_main(void *ptr) {
148 static RRDSET *st = NULL;
149
150 if(unlikely(!st)) {
151 - st = rrdset_find_bytype_localhost("netdata", "plugin_proc_modules");
151 + st = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
152
153 if(!st) {
154 st = rrdset_create_localhost(
collectors/proc.plugin/proc_net_softnet_stat.c
+2 -2
@@ -81,7 +81,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
81
82 // --------------------------------------------------------------------
83
84 - st = rrdset_find_bytype_localhost("system", "softnet_stat");
84 + st = rrdset_find_active_bytype_localhost("system", "softnet_stat");
85 if(unlikely(!st)) {
86 st = rrdset_create_localhost(
87 "system"
@@ -114,7 +114,7 @@ int do_proc_net_softnet_stat(int update_every, usec_t dt) {
114 char id[50+1];
115 snprintfz(id, 50, "cpu%zu_softnet_stat", l);
116
117 - st = rrdset_find_bytype_localhost("cpu", id);
117 + st = rrdset_find_active_bytype_localhost("cpu", id);
118 if(unlikely(!st)) {
119 char title[100+1];
120 snprintfz(title, 100, "CPU%zu softnet_stat", l);
collectors/statsd.plugin/statsd.c
+4
@@ -1461,6 +1461,8 @@ static inline RRDSET *statsd_private_rrdset_create(
1461 , chart_type // chart type
1462 , memory_mode // memory mode
1463 , history // history
1464 + , 0 // not archived
1465 + , NULL // no known UUID
1466 );
1467 rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
1468
@@ -1999,6 +2001,8 @@ static inline void statsd_update_app_chart(STATSD_APP *app, STATSD_APP_CHART *ch
2001 , chart->chart_type // chart type
2002 , app->rrd_memory_mode // memory mode
2003 , app->rrd_history_entries // history
2004 + , 0 // not archived
2005 + , NULL // no known UUID
2006 );
2007
2008 rrdset_flag_set(chart->st, RRDSET_FLAG_STORE_FIRST);
configure.ac
+2
@@ -1437,6 +1437,8 @@ AC_CONFIG_FILES([
1437 daemon/Makefile
1438 database/Makefile
1439 database/engine/Makefile
1440 + database/engine/metadata_log/Makefile
1441 + database/engine/global_uuid_map/Makefile
1442 diagrams/Makefile
1443 exporting/Makefile
1444 exporting/graphite/Makefile
daemon/common.h
+6
@@ -68,9 +68,15 @@
68 // netdata agent cloud link
69 #include "aclk/agent_cloud_link.h"
70
71 +// global GUID map functions
72 +
73 // netdata agent spawn server
74 #include "spawn/spawn.h"
75
76 +#ifdef ENABLE_DBENGINE
77 +#include "database/engine/global_uuid_map/global_uuid_map.h"
78 +#endif
79 +
80 // the netdata deamon
81 #include "daemon.h"
82 #include "main.h"
daemon/main.c
+6
@@ -1400,6 +1400,9 @@ int main(int argc, char **argv) {
1400 struct rrdhost_system_info *system_info = calloc(1, sizeof(struct rrdhost_system_info));
1401 get_system_info(system_info);
1402
1403 +#ifdef ENABLE_DBENGINE
1404 + init_global_guid_map();
1405 +#endif
1406 if(rrd_init(netdata_configured_hostname, system_info))
1407 fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);
1408
@@ -1417,6 +1420,9 @@ int main(int argc, char **argv) {
1420
1421 // Load host labels
1422 reload_host_labels();
1423 +#ifdef ENABLE_DBENGINE
1424 + metalog_commit_update_host(localhost);
1425 +#endif
1426
1427 // ------------------------------------------------------------------------
1428 // spawn the threads
daemon/unit_test.c
+4 -2
@@ -1833,9 +1833,10 @@ int test_dbengine(void)
1833 }
1834 }
1835 error_out:
1836 - rrdeng_exit(host->rrdeng_ctx);
1836 rrd_wrlock();
1837 + rrdeng_prepare_exit(host->rrdeng_ctx);
1838 rrdhost_delete_charts(host);
1839 + rrdeng_exit(host->rrdeng_ctx);
1840 rrd_unlock();
1841
1842 return errors;
@@ -2222,9 +2223,10 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi
2223 freez(query_threads[i]);
2224 }
2225 freez(query_threads);
2225 - rrdeng_exit(host->rrdeng_ctx);
2226 rrd_wrlock();
2227 + rrdeng_prepare_exit(host->rrdeng_ctx);
2228 rrdhost_delete_charts(host);
2229 + rrdeng_exit(host->rrdeng_ctx);
2230 rrd_unlock();
2231 }
2232
database/engine/Makefile.am
+5
@@ -3,6 +3,11 @@
3 AUTOMAKE_OPTIONS = subdir-objects
4 MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5
6 +SUBDIRS = \
7 + metadata_log \
8 + global_uuid_map \
9 + $(NULL)
10 +
11 dist_noinst_DATA = \
12 README.md \
13 $(NULL)
database/engine/global_uuid_map/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
database/engine/global_uuid_map/README.md
database/engine/global_uuid_map/global_uuid_map.c new
+269
@@ -0,0 +1,269 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "global_uuid_map.h"
4 +
5 +static Pvoid_t JGUID_map = (Pvoid_t) NULL;
6 +static Pvoid_t JGUID_object_map = (Pvoid_t) NULL;
7 +static uv_rwlock_t guid_lock;
8 +static uv_rwlock_t object_lock;
9 +static uv_rwlock_t global_lock;
10 +
11 +
12 +void dump_object(uuid_t *index, void *object)
13 +{
14 + char uuid_s[36 + 1];
15 + uuid_unparse_lower(*index, uuid_s);
16 + char local_object[3 * 36 + 2 + 1];
17 +
18 + switch (*(char *) object) {
19 + case GUID_TYPE_CHAR:
20 + debug(D_GUIDLOG, "OBJECT GUID %s on [%s]", uuid_s, (char *)object + 1);
21 + break;
22 + case GUID_TYPE_CHART:
23 + uuid_unparse_lower((const unsigned char *)object + 1, local_object);
24 + uuid_unparse_lower((const unsigned char *)object + 17, local_object+37);
25 + local_object[36] = ':';
26 + local_object[74] = '\0';
27 + debug(D_GUIDLOG, "CHART GUID %s on [%s]", uuid_s, local_object);
28 + break;
29 + case GUID_TYPE_DIMENSION:
30 + uuid_unparse_lower((const unsigned char *)object + 1, local_object);
31 + uuid_unparse_lower((const unsigned char *)object + 17, local_object + 37);
32 + uuid_unparse_lower((const unsigned char *)object + 33, local_object + 74);
33 + local_object[36] = ':';
34 + local_object[73] = ':';
35 + local_object[110] = '\0';
36 + debug(D_GUIDLOG, "DIM GUID %s on [%s]", uuid_s, local_object);
37 + break;
38 + default:
39 + debug(D_GUIDLOG, "Unknown object");
40 + }
41 +}
42 +
43 +/* Returns 0 if it successfully stores the uuid-object mapping or if an identical mapping already exists */
44 +static inline int guid_store_nolock(uuid_t *uuid, void *object, GUID_TYPE object_type)
45 +{
46 + char *existing_object;
47 + GUID_TYPE existing_object_type;
48 +
49 + if (unlikely(!object) || uuid == NULL)
50 + return 0;
51 +
52 + Pvoid_t *PValue;
53 +
54 + PValue = JudyHSIns(&JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t), PJE0);
55 + if (PPJERR == PValue)
56 + fatal("JudyHSIns() fatal error.");
57 + if (*PValue) {
58 + existing_object = *PValue;
59 + existing_object_type = existing_object[0];
60 + if (existing_object_type != object_type)
61 + return 1;
62 + switch (existing_object_type) {
63 + case GUID_TYPE_DIMENSION:
64 + if (memcmp(existing_object, object, 1 + 16 + 16 + 16))
65 + return 1;
66 + break;
67 + case GUID_TYPE_CHART:
68 + if (memcmp(existing_object, object, 1 + 16 + 16))
69 + return 1;
70 + break;
71 + case GUID_TYPE_CHAR:
72 + if (strcmp(existing_object + 1, (char *)object))
73 + return 1;
74 + break;
75 + default:
76 + return 1;
77 + }
78 + freez(existing_object);
79 + }
80 +
81 + *PValue = (Pvoid_t *) object;
82 +
83 + PValue = JudyHSIns(&JGUID_object_map, (void *)object, (Word_t) object_type?(object_type * 16)+1:strlen((char *) object+1)+2, PJE0);
84 + if (PPJERR == PValue)
85 + fatal("JudyHSIns() fatal error.");
86 + if (*PValue == NULL) {
87 + uuid_t *value = (uuid_t *) mallocz(sizeof(uuid_t));
88 + uuid_copy(*value, *uuid);
89 + *PValue = value;
90 + }
91 +
92 +#ifdef NETDATA_INTERNAL_CHECKS
93 + static uint32_t count = 0;
94 + count++;
95 + char uuid_s[36 + 1];
96 + uuid_unparse_lower(*uuid, uuid_s);
97 + debug(D_GUIDLOG,"GUID added item %" PRIu32" [%s] as:", count, uuid_s);
98 + dump_object(uuid, object);
99 +#endif
100 + return 0;
101 +}
102 +
103 +
104 +inline int guid_store(uuid_t *uuid, char *object, GUID_TYPE object_type)
105 +{
106 + uv_rwlock_wrlock(&global_lock);
107 + int rc = guid_store_nolock(uuid, object, object_type);
108 + uv_rwlock_wrunlock(&global_lock);
109 + return rc;
110 +}
111 +
112 +/*
113 + * This can be used to bulk load entries into the global map
114 + *
115 + * A lock must be aquired since it will call guid_store_nolock
116 + * with a "no lock" parameter.
117 + *
118 + * Note: object memory must be allocated by caller and not released
119 + */
120 +int guid_bulk_load(char *uuid, char *object)
121 +{
122 + uuid_t target_uuid;
123 + if (likely(!uuid_parse(uuid, target_uuid))) {
124 +#ifdef NETDATA_INTERNAL_CHECKS
125 + debug(D_GUIDLOG,"Mapping GUID [%s] on [%s]", uuid, object);
126 +#endif
127 + return guid_store_nolock(&target_uuid, object, GUID_TYPE_CHAR);
128 + }
129 + return 1;
130 +}
131 +
132 +/*
133 + * Given a GUID, find if an object is stored
134 + * - Optionally return the object
135 + */
136 +
137 +GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes)
138 +{
139 + Pvoid_t *PValue;
140 + GUID_TYPE value_type;
141 +
142 + uv_rwlock_rdlock(&global_lock);
143 + PValue = JudyHSGet(JGUID_map, (void *) uuid, (Word_t) sizeof(uuid_t));
144 + if (unlikely(!PValue)) {
145 + uv_rwlock_rdunlock(&global_lock);
146 + return GUID_TYPE_NOTFOUND;
147 + }
148 +
149 + value_type = *(char *) *PValue;
150 +
151 + if (likely(object && max_bytes)) {
152 + switch (value_type) {
153 + case GUID_TYPE_CHAR:
154 + if (unlikely(max_bytes - 1 < strlen((char *) *PValue+1)))
155 + return GUID_TYPE_NOSPACE;
156 + strncpyz(object, (char *) *PValue+1, max_bytes - 1);
157 + break;
158 + case GUID_TYPE_CHART:
159 + case GUID_TYPE_DIMENSION:
160 + if (unlikely(max_bytes < (size_t) value_type * 16))
161 + return GUID_TYPE_NOSPACE;
162 + memcpy(object, *PValue+1, value_type * 16);
163 + break;
164 + default:
165 + uv_rwlock_rdunlock(&global_lock);
166 + return GUID_TYPE_NOTFOUND;
167 + }
168 + }
169 +
170 +#ifdef NETDATA_INTERNAL_CHECKS
171 + dump_object(uuid, *PValue);
172 +#endif
173 + uv_rwlock_rdunlock(&global_lock);
174 + return value_type;
175 +}
176 +
177 +/*
178 + * Find a GUID of an object
179 + * - Optionally return the GUID
180 + *
181 + */
182 +
183 +int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE object_type)
184 +{
185 + Pvoid_t *PValue;
186 +
187 + uv_rwlock_rdlock(&global_lock);
188 + PValue = JudyHSGet(JGUID_object_map, (void *)object, (Word_t)object_type?object_type*16+1:strlen(object+1)+2);
189 + if (unlikely(!PValue)) {
190 + uv_rwlock_rdunlock(&global_lock);
191 + return 1;
192 + }
193 +
194 + if (likely(uuid))
195 + uuid_copy(*uuid, *PValue);
196 + uv_rwlock_rdunlock(&global_lock);
197 + return 0;
198 +}
199 +
200 +int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate)
201 +{
202 + char *target_object;
203 + uuid_t temp_uuid;
204 + int rc;
205 +
206 + switch (object_type) {
207 + case GUID_TYPE_DIMENSION:
208 + if (unlikely(find_or_generate_guid((void *) ((RRDDIM *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
209 + return 1;
210 + target_object = mallocz(49);
211 + target_object[0] = object_type;
212 + memcpy(target_object + 1, ((RRDDIM *)object)->rrdset->rrdhost->host_uuid, 16);
213 + memcpy(target_object + 17, ((RRDDIM *)object)->rrdset->chart_uuid, 16);
214 + memcpy(target_object + 33, temp_uuid, 16);
215 + break;
216 + case GUID_TYPE_CHART:
217 + if (unlikely(find_or_generate_guid((void *) ((RRDSET *)object)->id, &temp_uuid, GUID_TYPE_CHAR, 0)))
218 + return 1;
219 + target_object = mallocz(33);
220 + target_object[0] = object_type;
221 + memcpy(target_object + 1, (((RRDSET *)object))->rrdhost->host_uuid, 16);
222 + memcpy(target_object + 17, temp_uuid, 16);
223 + break;
224 + case GUID_TYPE_CHAR:
225 + target_object = mallocz(strlen((char *) object)+2);
226 + target_object[0] = object_type;
227 + strcpy(target_object+1, (char *) object);
228 + break;
229 + default:
230 + return 1;
231 + }
232 + rc = find_guid_by_object(target_object, uuid, object_type);
233 + if (rc) {
234 + if (!replace_instead_of_generate) /* else take *uuid as user input */
235 + uuid_generate(*uuid);
236 + uv_rwlock_wrlock(&global_lock);
237 + int rc = guid_store_nolock(uuid, target_object, object_type);
238 + uv_rwlock_wrunlock(&global_lock);
239 + return rc;
240 + }
241 + //uv_rwlock_wrunlock(&global_lock);
242 +#ifdef NETDATA_INTERNAL_CHECKS
243 + dump_object(uuid, target_object);
244 +#endif
245 + return 0;
246 +}
247 +
248 +void init_global_guid_map()
249 +{
250 + static int init = 0;
251 +
252 + if (init)
253 + return;
254 +
255 + init = 1;
256 + info("Configuring locking mechanism for global GUID map");
257 + assert(0 == uv_rwlock_init(&guid_lock));
258 + assert(0 == uv_rwlock_init(&object_lock));
259 + assert(0 == uv_rwlock_init(&global_lock));
260 +
261 +// int rc = guid_bulk_load("6fc56a64-05d7-47a7-bc82-7f3235d8cbda","d6b37186-74db-11ea-88b2-0bf5095b1f9e/cgroup_qemu_ubuntu18.04.cpu_per_core/cpu3");
262 +// rc = guid_bulk_load("75c6fa02-97cc-40c1-aacd-a0132190472e","d6b37186-74db-11ea-88b2-0bf5095b1f9e/services.throttle_io_ops_write/system.slice_setvtrgb.service");
263 +// if (rc == 0)
264 +// info("BULK GUID load successful");
265 +
266 + return;
267 +}
268 +
269 +
database/engine/global_uuid_map/global_uuid_map.h new
+26
@@ -0,0 +1,26 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_GLOBAL_UUID_MAP_H
4 +#define NETDATA_GLOBAL_UUID_MAP_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include <Judy.h>
8 +#include "../../rrd.h"
9 +
10 +typedef enum guid_type {
11 + GUID_TYPE_CHAR,
12 + GUID_TYPE_HOST,
13 + GUID_TYPE_CHART,
14 + GUID_TYPE_DIMENSION,
15 + GUID_TYPE_NOTFOUND,
16 + GUID_TYPE_NOSPACE
17 +} GUID_TYPE;
18 +
19 +extern int guid_store(uuid_t *uuid, char *object, GUID_TYPE);
20 +extern GUID_TYPE find_object_by_guid(uuid_t *uuid, char *object, size_t max_bytes);
21 +extern int find_guid_by_object(char *object, uuid_t *uuid, GUID_TYPE);
22 +extern void init_global_guid_map();
23 +extern int find_or_generate_guid(void *object, uuid_t *uuid, GUID_TYPE object_type, int replace_instead_of_generate);
24 +
25 +
26 +#endif //NETDATA_GLOBAL_UUID_MAP_H
database/engine/metadata_log/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
database/engine/metadata_log/README.md
database/engine/metadata_log/compaction.c new
+381
@@ -0,0 +1,381 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3 +
4 +#include "metadatalog.h"
5 +
6 +void after_compact_old_records(struct metalog_worker_config* wc)
7 +{
8 + struct metalog_instance *ctx = wc->ctx;
9 + int error;
10 +
11 + mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log, &ctx->compaction_state.new_metadata_logfiles);
12 + uv_run(wc->loop, UV_RUN_DEFAULT);
13 +
14 + error = uv_thread_join(wc->now_compacting_files);
15 + if (error) {
16 + error("uv_thread_join(): %s", uv_strerror(error));
17 + }
18 + freez(wc->now_compacting_files);
19 + /* unfreeze command processing */
20 + wc->now_compacting_files = NULL;
21 +
22 + wc->cleanup_thread_compacting_files = 0;
23 +
24 + /* interrupt event loop */
25 + uv_stop(wc->loop);
26 +
27 + info("Finished metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
28 +}
29 +
30 +static void metalog_flush_compaction_records(struct metalog_instance *ctx)
31 +{
32 + struct metalog_cmd cmd;
33 + struct completion compaction_completion;
34 +
35 + init_completion(&compaction_completion);
36 +
37 + cmd.opcode = METALOG_COMPACTION_FLUSH;
38 + cmd.record_io_descr.completion = &compaction_completion;
39 + metalog_enq_cmd(&ctx->worker_config, &cmd);
40 +
41 + wait_for_completion(&compaction_completion);
42 + destroy_completion(&compaction_completion);
43 +}
44 +
45 +/* The caller must have called metalog_flush_compaction_records() before to synchronize and quiesce the event loop. */
46 +static void compaction_test_quota(struct metalog_worker_config *wc)
47 +{
48 + struct metalog_instance *ctx = wc->ctx;
49 + struct logfile_compaction_state *compaction_state;
50 + struct metadata_logfile *oldmetalogfile, *newmetalogfile;
51 + unsigned current_size;
52 + int ret;
53 +
54 + compaction_state = &ctx->compaction_state;
55 + newmetalogfile = compaction_state->new_metadata_logfiles.last;
56 +
57 + oldmetalogfile = ctx->metadata_logfiles.first;
58 +
59 + current_size = newmetalogfile->pos;
60 + if (unlikely(current_size >= MAX_METALOGFILE_SIZE && newmetalogfile->starting_fileno < oldmetalogfile->fileno)) {
61 + /* It's safe to finalize the compacted metadata log file and create a new one since it has already replaced
62 + * an older one. */
63 +
64 + /* Finalize as the immediately previous file than the currently compacted one. */
65 + ret = rename_metadata_logfile(newmetalogfile, 0, newmetalogfile->fileno - 1);
66 + if (ret < 0)
67 + return;
68 +
69 + ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles,
70 + ctx->metadata_logfiles.first->fileno, ctx->metadata_logfiles.first->fileno);
71 +
72 + if (likely(!ret)) {
73 + compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
74 + }
75 + }
76 +}
77 +
78 +
79 +static void compact_record_by_uuid(struct metalog_instance *ctx, uuid_t *uuid)
80 +{
81 + GUID_TYPE ret;
82 + RRDHOST *host = ctx->rrdeng_ctx->host;
83 + RRDSET *st;
84 + RRDDIM *rd;
85 + BUFFER *buffer;
86 +
87 + ret = find_object_by_guid(uuid, NULL, 0);
88 + switch (ret) {
89 + case GUID_TYPE_CHAR:
90 + assert(0);
91 + break;
92 + case GUID_TYPE_CHART:
93 + st = metalog_get_chart_from_uuid(ctx, uuid);
94 + if (st) {
95 + if (ctx->current_compaction_id > st->compaction_id) {
96 + st->compaction_id = ctx->current_compaction_id;
97 + buffer = metalog_update_chart_buffer(st, ctx->current_compaction_id);
98 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
99 + } else {
100 + debug(D_METADATALOG, "Chart has already been compacted, ignoring record.");
101 + }
102 + } else {
103 + debug(D_METADATALOG, "Ignoring nonexistent chart metadata record.");
104 + }
105 + break;
106 + case GUID_TYPE_DIMENSION:
107 + rd = metalog_get_dimension_from_uuid(ctx, uuid);
108 + if (rd) {
109 + if (ctx->current_compaction_id > rd->state->compaction_id) {
110 + rd->state->compaction_id = ctx->current_compaction_id;
111 + buffer = metalog_update_dimension_buffer(rd);
112 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
113 + } else {
114 + debug(D_METADATALOG, "Dimension has already been compacted, ignoring record.");
115 + }
116 + } else {
117 + debug(D_METADATALOG, "Ignoring nonexistent dimension metadata record.");
118 + }
119 + break;
120 + case GUID_TYPE_HOST:
121 + if (ctx->current_compaction_id > host->compaction_id) {
122 + host->compaction_id = ctx->current_compaction_id;
123 + buffer = metalog_update_host_buffer(host);
124 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 1);
125 + } else {
126 + debug(D_METADATALOG, "Host has already been compacted, ignoring record.");
127 + }
128 + break;
129 + case GUID_TYPE_NOTFOUND:
130 + debug(D_METADATALOG, "Ignoring nonexistent metadata record.");
131 + break;
132 + default:
133 + assert(0);
134 + break;
135 + }
136 +}
137 +
138 +/* Returns 0 on success. */
139 +static int compact_metadata_logfile_records(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
140 +{
141 + struct metalog_worker_config* wc = &ctx->worker_config;
142 + struct logfile_compaction_state *compaction_state;
143 + struct metalog_record *record;
144 + struct metalog_record_block *record_block, *prev_record_block;
145 + int ret;
146 + unsigned iterated_records;
147 +#define METADATA_LOG_RECORD_BATCH 128 /* Flush I/O and check sizes whenever this many records have been iterated */
148 +
149 + info("Compacting metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
150 + ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
151 +
152 + compaction_state = &ctx->compaction_state;
153 + record_block = prev_record_block = NULL;
154 + iterated_records = 0;
155 + for (record = mlf_record_get_first(metalogfile) ; record != NULL ; record = mlf_record_get_next(metalogfile)) {
156 + if ((record_block = metalogfile->records.iterator.current) != prev_record_block) {
157 + if (prev_record_block) { /* Deallocate iterated record blocks */
158 + rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
159 + freez(prev_record_block);
160 + }
161 + prev_record_block = record_block;
162 + }
163 + compact_record_by_uuid(ctx, &record->uuid);
164 + if (0 == ++iterated_records % METADATA_LOG_RECORD_BATCH) {
165 + metalog_flush_compaction_records(ctx);
166 + if (compaction_state->throttle) {
167 + (void)sleep_usec(10000); /* 10 msec throttle compaction */
168 + }
169 + compaction_test_quota(wc);
170 + }
171 + }
172 + if (prev_record_block) { /* Deallocate iterated record blocks */
173 + rrd_atomic_fetch_add(&ctx->records_nr, -prev_record_block->records_nr);
174 + freez(prev_record_block);
175 + }
176 +
177 + info("Compacted metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
178 + ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
179 +
180 + metadata_logfile_list_delete(&ctx->metadata_logfiles, metalogfile);
181 + ret = destroy_metadata_logfile(metalogfile);
182 + if (!ret) {
183 + info("Deleted file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
184 + ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
185 + rrd_atomic_fetch_add(&ctx->disk_space, -metalogfile->pos);
186 + } else {
187 + error("Failed to delete file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
188 + ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
189 + }
190 + freez(metalogfile);
191 +
192 + return ret;
193 +}
194 +
195 +static void compact_old_records(void *arg)
196 +{
197 + struct metalog_instance *ctx = arg;
198 + struct metalog_worker_config* wc = &ctx->worker_config;
199 + struct logfile_compaction_state *compaction_state;
200 + struct metadata_logfile *metalogfile, *nextmetalogfile, *newmetalogfile;
201 + int ret;
202 +
203 + compaction_state = &ctx->compaction_state;
204 +
205 + nextmetalogfile = NULL;
206 + for (metalogfile = ctx->metadata_logfiles.first ;
207 + metalogfile != compaction_state->last_original_logfile ;
208 + metalogfile = nextmetalogfile) {
209 + nextmetalogfile = metalogfile->next;
210 +
211 + newmetalogfile = compaction_state->new_metadata_logfiles.last;
212 + ret = rename_metadata_logfile(newmetalogfile, newmetalogfile->starting_fileno, metalogfile->fileno);
213 + if (ret < 0) {
214 + error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
215 + ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
216 + }
217 +
218 + ret = compact_metadata_logfile_records(ctx, metalogfile);
219 + if (ret) {
220 + error("Metadata log compaction failed, cancelling.");
221 + break;
222 + }
223 + }
224 + assert(nextmetalogfile); /* There are always more than 1 metadata log files during compaction */
225 +
226 + newmetalogfile = compaction_state->new_metadata_logfiles.last;
227 + if (newmetalogfile->starting_fileno != 0) { /* Must rename the last compacted file */
228 + ret = rename_metadata_logfile(newmetalogfile, 0, nextmetalogfile->fileno - 1);
229 + if (ret < 0) {
230 + error("Failed to rename file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION"\".",
231 + ctx->rrdeng_ctx->dbfiles_path, newmetalogfile->starting_fileno, newmetalogfile->fileno);
232 + }
233 + }
234 + /* Connect the compacted files to the metadata log */
235 + newmetalogfile->next = nextmetalogfile;
236 + ctx->metadata_logfiles.first = compaction_state->new_metadata_logfiles.first;
237 +
238 + wc->cleanup_thread_compacting_files = 1;
239 + /* wake up event loop */
240 + assert(0 == uv_async_send(&wc->async));
241 +}
242 +
243 +/* Returns 0 on success. */
244 +static int init_compaction_state(struct metalog_instance *ctx)
245 +{
246 + struct metadata_logfile *newmetalogfile;
247 + struct logfile_compaction_state *compaction_state;
248 + int ret;
249 +
250 + compaction_state = &ctx->compaction_state;
251 + compaction_state->new_metadata_logfiles.first = NULL;
252 + compaction_state->new_metadata_logfiles.last = NULL;
253 + compaction_state->starting_fileno = ctx->metadata_logfiles.first->fileno;
254 + compaction_state->fileno = ctx->metadata_logfiles.first->fileno;
255 + compaction_state->last_original_logfile = ctx->metadata_logfiles.last;
256 + compaction_state->throttle = 0;
257 +
258 + ret = add_new_metadata_logfile(ctx, &compaction_state->new_metadata_logfiles, compaction_state->starting_fileno,
259 + compaction_state->fileno);
260 + if (unlikely(ret)) {
261 + error("Cannot create new metadata log files, compaction aborted.");
262 + return ret;
263 + }
264 + newmetalogfile = compaction_state->new_metadata_logfiles.first;
265 + assert(newmetalogfile == compaction_state->new_metadata_logfiles.last);
266 + init_metadata_record_log(&compaction_state->records_log);
267 +
268 + return 0;
269 +}
270 +
271 +void metalog_do_compaction(struct metalog_worker_config *wc)
272 +{
273 + struct metalog_instance *ctx = wc->ctx;
274 + int error;
275 +
276 + if (wc->now_compacting_files) {
277 + /* already compacting metadata log files */
278 + return;
279 + }
280 + wc->now_compacting_files = mallocz(sizeof(*wc->now_compacting_files));
281 + wc->cleanup_thread_compacting_files = 0;
282 + metalog_try_link_new_metadata_logfile(wc);
283 +
284 + error = init_compaction_state(ctx);
285 + if (unlikely(error)) {
286 + error("Cannot create new metadata log files, compaction aborted.");
287 + return;
288 + }
289 + ++ctx->current_compaction_id; /* Signify a new compaction */
290 +
291 + info("Starting metadata log compaction (id:%"PRIu32").", ctx->current_compaction_id);
292 + error = uv_thread_create(wc->now_compacting_files, compact_old_records, ctx);
293 + if (error) {
294 + error("uv_thread_create(): %s", uv_strerror(error));
295 + freez(wc->now_compacting_files);
296 + wc->now_compacting_files = NULL;
297 + }
298 +
299 +}
300 +
301 +/* Return 0 on success. */
302 +int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
303 + unsigned *matched_files)
304 +{
305 + int ret;
306 + unsigned starting_fileno, fileno, i, j, recovered_files;
307 + struct metadata_logfile *metalogfile, *compactionfile, **tmp_metalogfiles;
308 + char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
309 +
310 + for (i = 0 ; i < *matched_files ; ++i) {
311 + metalogfile = metalogfiles[i];
312 + if (0 == metalogfile->starting_fileno)
313 + continue; /* skip standard metadata log files */
314 + break; /* this is a compaction temporary file */
315 + }
316 + if (i == *matched_files) /* no recovery needed */
317 + return 0;
318 + info("Starting metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
319 +
320 + if (*matched_files - i > 1) { /* Can't have more than 1 temporary compaction files */
321 + error("Metadata log files are in an invalid state. Cannot proceed.");
322 + return 1;
323 + }
324 + compactionfile = metalogfile;
325 + starting_fileno = compactionfile->starting_fileno;
326 + fileno = compactionfile->fileno;
327 + /* scratchpad space to move file pointers around */
328 + tmp_metalogfiles = callocz(*matched_files, sizeof(*tmp_metalogfiles));
329 +
330 + for (j = 0, recovered_files = 0 ; j < i ; ++j) {
331 + metalogfile = metalogfiles[j];
332 + assert(0 == metalogfile->starting_fileno);
333 + if (metalogfile->fileno < starting_fileno) {
334 + tmp_metalogfiles[recovered_files++] = metalogfile;
335 + continue;
336 + }
337 + break; /* reached compaction file serial number */
338 + }
339 +
340 + if ((j == i) /* Shouldn't be possible, invalid compaction temporary file */ ||
341 + (metalogfile->fileno == starting_fileno && metalogfile->fileno == fileno)) {
342 + error("Deleting invalid compaction temporary file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
343 + METALOG_EXTENSION"\"", dbfiles_path, starting_fileno, fileno);
344 + unlink_metadata_logfile(compactionfile);
345 + freez(compactionfile);
346 + freez(tmp_metalogfiles);
347 + --*matched_files; /* delete the last one */
348 +
349 + info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
350 + return 0;
351 + }
352 +
353 + for ( ; j < i ; ++j) { /* continue iterating through normal metadata log files */
354 + metalogfile = metalogfiles[j];
355 + assert(0 == metalogfile->starting_fileno);
356 + if (metalogfile->fileno < fileno) { /* It has already been compacted */
357 + error("Deleting invalid metadata log file \"%s/"METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL
358 + METALOG_EXTENSION"\"", dbfiles_path, 0U, metalogfile->fileno);
359 + unlink_metadata_logfile(metalogfile);
360 + freez(metalogfile);
361 + continue;
362 + }
363 + tmp_metalogfiles[recovered_files++] = metalogfile;
364 + }
365 +
366 + /* compaction temporary file is valid */
367 + tmp_metalogfiles[recovered_files++] = compactionfile;
368 + ret = rename_metadata_logfile(compactionfile, 0, starting_fileno);
369 + if (ret < 0) {
370 + error("Cannot rename temporary compaction files. Cannot proceed.");
371 + freez(tmp_metalogfiles);
372 + return 1;
373 + }
374 +
375 + memcpy(metalogfiles, tmp_metalogfiles, recovered_files * sizeof(*metalogfiles));
376 + *matched_files = recovered_files;
377 + freez(tmp_metalogfiles);
378 +
379 + info("Finished metadata log file failure recovery procedure in \"%s\".", dbfiles_path);
380 + return 0;
381 +}
database/engine/metadata_log/compaction.h new
+26
@@ -0,0 +1,26 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_COMPACTION_H
4 +#define NETDATA_COMPACTION_H
5 +
6 +#ifndef _GNU_SOURCE
7 +#define _GNU_SOURCE
8 +#endif
9 +#include "../rrdengine.h"
10 +
11 +struct logfile_compaction_state {
12 + unsigned fileno; /* Starts at 1 */
13 + unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
14 +
15 + struct metadata_record_commit_log records_log;
16 + struct metadata_logfile_list new_metadata_logfiles;
17 + struct metadata_logfile *last_original_logfile; /* Marks the end of compaction */
18 + uint8_t throttle; /* set non-zero to throttle compaction */
19 +};
20 +
21 +extern int compaction_failure_recovery(struct metalog_instance *ctx, struct metadata_logfile **metalogfiles,
22 + unsigned *matched_files);
23 +extern void metalog_do_compaction(struct metalog_worker_config *wc);
24 +extern void after_compact_old_records(struct metalog_worker_config* wc);
25 +
26 +#endif /* NETDATA_COMPACTION_H */
database/engine/metadata_log/logfile.c new
+790
@@ -0,0 +1,790 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +#include "metadatalog.h"
3 +#include "metalogpluginsd.h"
4 +
5 +static void mlf_record_block_insert(struct metadata_logfile *metalogfile, struct metalog_record_block *record_block)
6 +{
7 +
8 + if (likely(NULL != metalogfile->records.last)) {
9 + metalogfile->records.last->next = record_block;
10 + }
11 + if (unlikely(NULL == metalogfile->records.first)) {
12 + metalogfile->records.first = record_block;
13 + }
14 + metalogfile->records.last = record_block;
15 +}
16 +
17 +void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record)
18 +{
19 + struct metalog_record_block *record_block;
20 + struct metalog_instance *ctx = metalogfile->ctx;
21 +
22 + record_block = metalogfile->records.last;
23 + if (likely(NULL != record_block && record_block->records_nr < MAX_METALOG_RECORDS_PER_BLOCK)) {
24 + record_block->record_array[record_block->records_nr++] = *record;
25 + } else { /* Create new record block, the last one filled up */
26 + record_block = mallocz(sizeof(*record_block));
27 + record_block->records_nr = 1;
28 + record_block->record_array[0] = *record;
29 + record_block->next = NULL;
30 +
31 + mlf_record_block_insert(metalogfile, record_block);
32 + }
33 + rrd_atomic_fetch_add(&ctx->records_nr, 1);
34 +}
35 +
36 +struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile)
37 +{
38 + struct metalog_records *records = &metalogfile->records;
39 + struct metalog_record_block *record_block = metalogfile->records.first;
40 +
41 + records->iterator.current = record_block;
42 + records->iterator.record_i = 0;
43 +
44 + if (unlikely(NULL == record_block || !record_block->records_nr)) {
45 + error("Cannot iterate empty metadata log file %u-%u.", metalogfile->starting_fileno, metalogfile->fileno);
46 + return NULL;
47 + }
48 +
49 + return &record_block->record_array[0];
50 +}
51 +
52 +/* Must have called mlf_record_get_first before calling this function. */
53 +struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile)
54 +{
55 + struct metalog_records *records = &metalogfile->records;
56 + struct metalog_record_block *record_block = records->iterator.current;
57 +
58 + if (unlikely(NULL == record_block)) {
59 + return NULL;
60 + }
61 + if (++records->iterator.record_i >= record_block->records_nr) {
62 + record_block = record_block->next;
63 + if (unlikely(NULL == record_block || !record_block->records_nr)) {
64 + return NULL;
65 + }
66 + records->iterator.current = record_block;
67 + records->iterator.record_i = 0;
68 + return &record_block->record_array[0];
69 + }
70 + return &record_block->record_array[records->iterator.record_i];
71 +}
72 +
73 +static void flush_records_buffer_cb(uv_fs_t* req)
74 +{
75 + struct generic_io_descriptor *io_descr = req->data;
76 + struct metalog_worker_config *wc = req->loop->data;
77 + struct metalog_instance *ctx = wc->ctx;
78 +
79 + debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
80 + if (req->result < 0) {
81 + ++ctx->stats.io_errors;
82 + rrd_stat_atomic_add(&global_io_errors, 1);
83 + error("%s: uv_fs_write: %s", __func__, uv_strerror((int)req->result));
84 + } else {
85 + debug(D_METADATALOG, "%s: Metadata log file block was written to disk.", __func__);
86 + }
87 +
88 + uv_fs_req_cleanup(req);
89 + free(io_descr->buf);
90 + freez(io_descr);
91 +}
92 +
93 +/* Careful to always call this before creating a new metadata log file to finish writing the old one */
94 +void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
95 + struct metadata_logfile_list *metadata_logfiles)
96 +{
97 + struct metalog_instance *ctx = wc->ctx;
98 + int ret;
99 + struct generic_io_descriptor *io_descr;
100 + unsigned pos, size;
101 + struct metadata_logfile *metalogfile;
102 +
103 + if (unlikely(NULL == records_log->buf || 0 == records_log->buf_pos)) {
104 + return;
105 + }
106 + /* care with outstanding records when switching metadata log files */
107 + metalogfile = metadata_logfiles->last;
108 +
109 + io_descr = mallocz(sizeof(*io_descr));
110 + pos = records_log->buf_pos;
111 + size = pos; /* no need to align the I/O when doing buffered writes */
112 + io_descr->buf = records_log->buf;
113 + io_descr->bytes = size;
114 + io_descr->pos = metalogfile->pos;
115 + io_descr->req.data = io_descr;
116 + io_descr->completion = NULL;
117 +
118 + io_descr->iov = uv_buf_init((void *)io_descr->buf, size);
119 + ret = uv_fs_write(wc->loop, &io_descr->req, metalogfile->file, &io_descr->iov, 1,
120 + metalogfile->pos, flush_records_buffer_cb);
121 + assert (-1 != ret);
122 + metalogfile->pos += size;
123 + rrd_atomic_fetch_add(&ctx->disk_space, size);
124 + records_log->buf = NULL;
125 + ctx->stats.io_write_bytes += size;
126 + ++ctx->stats.io_write_requests;
127 +}
128 +
129 +void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
130 + struct metadata_logfile_list *metadata_logfiles, unsigned size)
131 +{
132 + int ret;
133 + unsigned buf_pos, buf_size;
134 +
135 + assert(size);
136 + if (records_log->buf) {
137 + unsigned remaining;
138 +
139 + buf_pos = records_log->buf_pos;
140 + buf_size = records_log->buf_size;
141 + remaining = buf_size - buf_pos;
142 + if (size > remaining) {
143 + /* we need a new buffer */
144 + mlf_flush_records_buffer(wc, records_log, metadata_logfiles);
145 + }
146 + }
147 + if (NULL == records_log->buf) {
148 + buf_size = ALIGN_BYTES_CEILING(size);
149 + ret = posix_memalign((void *)&records_log->buf, RRDFILE_ALIGNMENT, buf_size);
150 + if (unlikely(ret)) {
151 + fatal("posix_memalign:%s", strerror(ret));
152 + }
153 + buf_pos = records_log->buf_pos = 0;
154 + records_log->buf_size = buf_size;
155 + }
156 + records_log->buf_pos += size;
157 +
158 + return records_log->buf + buf_pos;
159 +}
160 +
161 +
162 +void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
163 +{
164 + if (likely(NULL != metadata_logfiles->last)) {
165 + metadata_logfiles->last->next = metalogfile;
166 + }
167 + if (unlikely(NULL == metadata_logfiles->first)) {
168 + metadata_logfiles->first = metalogfile;
169 + }
170 + metadata_logfiles->last = metalogfile;
171 +}
172 +
173 +void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles, struct metadata_logfile *metalogfile)
174 +{
175 + struct metadata_logfile *next;
176 +
177 + next = metalogfile->next;
178 + assert((NULL != next) && (metadata_logfiles->first == metalogfile) &&
179 + (metadata_logfiles->last != metalogfile));
180 + metadata_logfiles->first = next;
181 +}
182 +
183 +void generate_metadata_logfile_path(struct metadata_logfile *metalogfile, char *str, size_t maxlen)
184 +{
185 + (void) snprintf(str, maxlen, "%s/" METALOG_PREFIX METALOG_FILE_NUMBER_PRINT_TMPL METALOG_EXTENSION,
186 + metalogfile->ctx->rrdeng_ctx->dbfiles_path, metalogfile->starting_fileno, metalogfile->fileno);
187 +}
188 +
189 +void metadata_logfile_init(struct metadata_logfile *metalogfile, struct metalog_instance *ctx, unsigned starting_fileno,
190 + unsigned fileno)
191 +{
192 + metalogfile->starting_fileno = starting_fileno;
193 + metalogfile->fileno = fileno;
194 + metalogfile->file = (uv_file)0;
195 + metalogfile->pos = 0;
196 + metalogfile->records.first = metalogfile->records.last = NULL;
197 + metalogfile->next = NULL;
198 + metalogfile->ctx = ctx;
199 +}
200 +
201 +int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno, unsigned new_fileno)
202 +{
203 + struct metalog_instance *ctx = metalogfile->ctx;
204 + uv_fs_t req;
205 + int ret;
206 + char oldpath[RRDENG_PATH_MAX], newpath[RRDENG_PATH_MAX];
207 + unsigned backup_starting_fileno, backup_fileno;
208 +
209 + backup_starting_fileno = metalogfile->starting_fileno;
210 + backup_fileno = metalogfile->fileno;
211 + generate_metadata_logfile_path(metalogfile, oldpath, sizeof(oldpath));
212 + metalogfile->starting_fileno = new_starting_fileno;
213 + metalogfile->fileno = new_fileno;
214 + generate_metadata_logfile_path(metalogfile, newpath, sizeof(newpath));
215 +
216 + info("Renaming metadata log file \"%s\" to \"%s\".", oldpath, newpath);
217 + ret = uv_fs_rename(NULL, &req, oldpath, newpath, NULL);
218 + if (ret < 0) {
219 + error("uv_fs_rename(%s): %s", oldpath, uv_strerror(ret));
220 + ++ctx->stats.fs_errors; /* this is racy, may miss some errors */
221 + rrd_stat_atomic_add(&global_fs_errors, 1);
222 + /* restore previous values */
223 + metalogfile->starting_fileno = backup_starting_fileno;
224 + metalogfile->fileno = backup_fileno;
225 + }
226 + uv_fs_req_cleanup(&req);
227 +
228 + return ret;
229 +}
230 +
231 +int close_metadata_logfile(struct metadata_logfile *metalogfile)
232 +{
233 + struct metalog_instance *ctx = metalogfile->ctx;
234 + uv_fs_t req;
235 + int ret;
236 + char path[RRDENG_PATH_MAX];
237 +
238 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
239 +
240 + ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
241 + if (ret < 0) {
242 + error("uv_fs_close(%s): %s", path, uv_strerror(ret));
243 + ++ctx->stats.fs_errors;
244 + rrd_stat_atomic_add(&global_fs_errors, 1);
245 + }
246 + uv_fs_req_cleanup(&req);
247 +
248 + return ret;
249 +}
250 +
251 +int unlink_metadata_logfile(struct metadata_logfile *metalogfile)
252 +{
253 + struct metalog_instance *ctx = metalogfile->ctx;
254 + uv_fs_t req;
255 + int ret;
256 + char path[RRDENG_PATH_MAX];
257 +
258 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
259 +
260 + ret = uv_fs_unlink(NULL, &req, path, NULL);
261 + if (ret < 0) {
262 + error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
263 + ++ctx->stats.fs_errors;
264 + rrd_stat_atomic_add(&global_fs_errors, 1);
265 + }
266 + uv_fs_req_cleanup(&req);
267 +
268 + return ret;
269 +}
270 +
271 +int destroy_metadata_logfile(struct metadata_logfile *metalogfile)
272 +{
273 + struct metalog_instance *ctx = metalogfile->ctx;
274 + uv_fs_t req;
275 + int ret;
276 + char path[RRDENG_PATH_MAX];
277 +
278 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
279 +
280 + ret = uv_fs_ftruncate(NULL, &req, metalogfile->file, 0, NULL);
281 + if (ret < 0) {
282 + error("uv_fs_ftruncate(%s): %s", path, uv_strerror(ret));
283 + ++ctx->stats.fs_errors;
284 + rrd_stat_atomic_add(&global_fs_errors, 1);
285 + }
286 + uv_fs_req_cleanup(&req);
287 +
288 + ret = uv_fs_close(NULL, &req, metalogfile->file, NULL);
289 + if (ret < 0) {
290 + error("uv_fs_close(%s): %s", path, uv_strerror(ret));
291 + ++ctx->stats.fs_errors;
292 + rrd_stat_atomic_add(&global_fs_errors, 1);
293 + }
294 + uv_fs_req_cleanup(&req);
295 +
296 + ret = uv_fs_unlink(NULL, &req, path, NULL);
297 + if (ret < 0) {
298 + error("uv_fs_fsunlink(%s): %s", path, uv_strerror(ret));
299 + ++ctx->stats.fs_errors;
300 + rrd_stat_atomic_add(&global_fs_errors, 1);
301 + }
302 + uv_fs_req_cleanup(&req);
303 +
304 +// ++ctx->stats.metadata_logfile_deletions;
305 +
306 + return ret;
307 +}
308 +
309 +int create_metadata_logfile(struct metadata_logfile *metalogfile)
310 +{
311 + struct metalog_instance *ctx = metalogfile->ctx;
312 + uv_fs_t req;
313 + uv_file file;
314 + int ret, fd;
315 + struct rrdeng_metalog_sb *superblock;
316 + uv_buf_t iov;
317 + char path[RRDENG_PATH_MAX];
318 +
319 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
320 + fd = open_file_buffered_io(path, O_CREAT | O_RDWR | O_TRUNC, &file);
321 + if (fd < 0) {
322 + ++ctx->stats.fs_errors;
323 + rrd_stat_atomic_add(&global_fs_errors, 1);
324 + return fd;
325 + }
326 + metalogfile->file = file;
327 +// ++ctx->stats.metadata_logfile_creations;
328 +
329 + ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
330 + if (unlikely(ret)) {
331 + fatal("posix_memalign:%s", strerror(ret));
332 + }
333 + (void) strncpy(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ);
334 + superblock->version = RRDENG_METALOG_VER;
335 +
336 + iov = uv_buf_init((void *)superblock, sizeof(*superblock));
337 +
338 + ret = uv_fs_write(NULL, &req, file, &iov, 1, 0, NULL);
339 + if (ret < 0) {
340 + assert(req.result < 0);
341 + error("uv_fs_write: %s", uv_strerror(ret));
342 + ++ctx->stats.io_errors;
343 + rrd_stat_atomic_add(&global_io_errors, 1);
344 + }
345 + uv_fs_req_cleanup(&req);
346 + free(superblock);
347 + if (ret < 0) {
348 + destroy_metadata_logfile(metalogfile);
349 + return ret;
350 + }
351 +
352 + metalogfile->pos = sizeof(*superblock);
353 + ctx->stats.io_write_bytes += sizeof(*superblock);
354 + ++ctx->stats.io_write_requests;
355 +
356 + return 0;
357 +}
358 +
359 +static int check_metadata_logfile_superblock(uv_file file)
360 +{
361 + int ret;
362 + struct rrdeng_metalog_sb *superblock;
363 + uv_buf_t iov;
364 + uv_fs_t req;
365 +
366 + ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));
367 + if (unlikely(ret)) {
368 + fatal("posix_memalign:%s", strerror(ret));
369 + }
370 + iov = uv_buf_init((void *)superblock, sizeof(*superblock));
371 +
372 + ret = uv_fs_read(NULL, &req, file, &iov, 1, 0, NULL);
373 + if (ret < 0) {
374 + error("uv_fs_read: %s", uv_strerror(ret));
375 + uv_fs_req_cleanup(&req);
376 + goto error;
377 + }
378 + assert(req.result >= 0);
379 + uv_fs_req_cleanup(&req);
380 +
381 + if (strncmp(superblock->magic_number, RRDENG_METALOG_MAGIC, RRDENG_MAGIC_SZ)) {
382 + error("File has invalid superblock.");
383 + ret = UV_EINVAL;
384 + } else {
385 + ret = 0;
386 + }
387 + if (superblock->version > RRDENG_METALOG_VER) {
388 + error("File has unknown version %"PRIu16". Compatibility is not guaranteed.", superblock->version);
389 + }
390 +error:
391 + free(superblock);
392 + return ret;
393 +}
394 +
395 +void replay_record(struct metadata_logfile *metalogfile, struct rrdeng_metalog_record_header *header, void *payload)
396 +{
397 + struct metalog_instance *ctx = metalogfile->ctx;
398 + char *line, *nextline, *record_end;
399 + int ret;
400 +
401 + debug(D_METADATALOG, "RECORD contents: %.*s", (int)header->payload_length, (char *)payload);
402 + record_end = (char *)payload + header->payload_length - 1;
403 + *record_end = '\0';
404 +
405 + for (line = payload ; line ; line = nextline) {
406 + nextline = strchr(line, '\n');
407 + if (nextline) {
408 + *nextline++ = '\0';
409 + }
410 + ret = parser_action(ctx->metalog_parser_object->parser, line);
411 + debug(D_METADATALOG, "parser_action ret:%d", ret);
412 + if (ret)
413 + return; /* skip record due to error */
414 + };
415 +}
416 +
417 +/* This function only works with buffered I/O */
418 +static inline int metalogfile_read(struct metadata_logfile *metalogfile, void *buf, size_t len, uint64_t offset)
419 +{
420 + struct metalog_instance *ctx;
421 + uv_file file;
422 + uv_buf_t iov;
423 + uv_fs_t req;
424 + int ret;
425 +
426 + ctx = metalogfile->ctx;
427 + file = metalogfile->file;
428 + iov = uv_buf_init(buf, len);
429 + ret = uv_fs_read(NULL, &req, file, &iov, 1, offset, NULL);
430 + if (unlikely(ret < 0 && ret != req.result)) {
431 + fatal("uv_fs_read: %s", uv_strerror(ret));
432 + }
433 + if (req.result < 0) {
434 + ++ctx->stats.io_errors;
435 + rrd_stat_atomic_add(&global_io_errors, 1);
436 + error("%s: uv_fs_read - %s - record at offset %"PRIu64"(%u) in metadata logfile %u-%u.", __func__,
437 + uv_strerror((int)req.result), offset, (unsigned)len, metalogfile->starting_fileno, metalogfile->fileno);
438 + }
439 + uv_fs_req_cleanup(&req);
440 + ctx->stats.io_read_bytes += len;
441 + ++ctx->stats.io_read_requests;
442 +
443 + return ret;
444 +}
445 +
446 +/* Return 0 on success */
447 +static int metadata_record_integrity_check(void *record)
448 +{
449 + int ret;
450 + uint32_t data_size;
451 + struct rrdeng_metalog_record_header *header;
452 + struct rrdeng_metalog_record_trailer *trailer;
453 + uLong crc;
454 +
455 + header = record;
456 + data_size = header->header_length + header->payload_length;
457 + trailer = record + data_size;
458 +
459 + crc = crc32(0L, Z_NULL, 0);
460 + crc = crc32(crc, record, data_size);
461 + ret = crc32cmp(trailer->checksum, crc);
462 +
463 + return ret;
464 +}
465 +
466 +#define MAX_READ_BYTES (RRDENG_BLOCK_SIZE * 32) /* no record should be over 128KiB in this version */
467 +
468 +/*
469 + * Iterates metadata log file records and creates database objects (host/chart/dimension)
470 + */
471 +static void iterate_records(struct metadata_logfile *metalogfile)
472 +{
473 + uint32_t file_size, pos, bytes_remaining, record_size;
474 + void *buf;
475 + struct rrdeng_metalog_record_header *header;
476 + struct metalog_instance *ctx = metalogfile->ctx;
477 + struct metalog_pluginsd_state *state = ctx->metalog_parser_object->private;
478 + const size_t min_header_size = offsetof(struct rrdeng_metalog_record_header, header_length) +
479 + sizeof(header->header_length);
480 +
481 + file_size = metalogfile->pos;
482 + state->metalogfile = metalogfile;
483 +
484 + buf = mallocz(MAX_READ_BYTES);
485 +
486 + for (pos = sizeof(struct rrdeng_metalog_sb) ; pos < file_size ; pos += record_size) {
487 + bytes_remaining = file_size - pos;
488 + if (bytes_remaining < min_header_size) {
489 + error("%s: unexpected end of file in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
490 + metalogfile->fileno);
491 + break;
492 + }
493 + if (metalogfile_read(metalogfile, buf, min_header_size, pos) < 0)
494 + break;
495 + header = (struct rrdeng_metalog_record_header *)buf;
496 + if (METALOG_STORE_PADDING == header->type) {
497 + info("%s: Skipping padding in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
498 + metalogfile->fileno);
499 + record_size = ALIGN_BYTES_FLOOR(pos + RRDENG_BLOCK_SIZE) - pos;
500 + continue;
501 + }
502 + if (metalogfile_read(metalogfile, buf + min_header_size, sizeof(*header) - min_header_size,
503 + pos + min_header_size) < 0)
504 + break;
505 + record_size = header->header_length + header->payload_length + sizeof(struct rrdeng_metalog_record_trailer);
506 + if (header->header_length < min_header_size || record_size > bytes_remaining) {
507 + error("%s: Corrupted record in metadata logfile %u-%u.", __func__, metalogfile->starting_fileno,
508 + metalogfile->fileno);
509 + break;
510 + }
511 + if (record_size > MAX_READ_BYTES) {
512 + error("%s: Record is too long (%u bytes) in metadata logfile %u-%u.", __func__, record_size,
513 + metalogfile->starting_fileno, metalogfile->fileno);
514 + continue;
515 + }
516 + if (metalogfile_read(metalogfile, buf + sizeof(*header), record_size - sizeof(*header),
517 + pos + sizeof(*header)) < 0)
518 + break;
519 + if (metadata_record_integrity_check(buf)) {
520 + error("%s: Record at offset %"PRIu32" was read from disk. CRC32 check: FAILED", __func__, pos);
521 + continue;
522 + }
523 + debug(D_METADATALOG, "%s: Record at offset %"PRIu32" was read from disk. CRC32 check: SUCCEEDED", __func__,
524 + pos);
525 +
526 + replay_record(metalogfile, header, buf + header->header_length);
527 + if (!uuid_is_null(state->uuid)) { /* It's a valid object */
528 + struct metalog_record record;
529 +
530 + uuid_copy(record.uuid, state->uuid);
531 + mlf_record_insert(metalogfile, &record);
532 + uuid_clear(state->uuid); /* Clear state for parsing of next record */
533 + }
534 + }
535 +
536 + freez(buf);
537 +}
538 +
539 +int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *metalogfile)
540 +{
541 + uv_fs_t req;
542 + uv_file file;
543 + int ret, fd, error;
544 + uint64_t file_size;
545 + char path[RRDENG_PATH_MAX];
546 +
547 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
548 + fd = open_file_buffered_io(path, O_RDWR, &file);
549 + if (fd < 0) {
550 + ++ctx->stats.fs_errors;
551 + rrd_stat_atomic_add(&global_fs_errors, 1);
552 + return fd;
553 + }
554 + info("Loading metadata log \"%s\".", path);
555 +
556 + ret = check_file_properties(file, &file_size, sizeof(struct rrdeng_metalog_sb));
557 + if (ret)
558 + goto error;
559 +
560 + ret = check_metadata_logfile_superblock(file);
561 + if (ret)
562 + goto error;
563 + ctx->stats.io_read_bytes += sizeof(struct rrdeng_jf_sb);
564 + ++ctx->stats.io_read_requests;
565 +
566 + metalogfile->file = file;
567 + metalogfile->pos = file_size;
568 +
569 + iterate_records(metalogfile);
570 +
571 + info("Metadata log \"%s\" loaded (size:%"PRIu64").", path, file_size);
572 + return 0;
573 +
574 +error:
575 + error = ret;
576 + ret = uv_fs_close(NULL, &req, file, NULL);
577 + if (ret < 0) {
578 + error("uv_fs_close(%s): %s", path, uv_strerror(ret));
579 + ++ctx->stats.fs_errors;
580 + rrd_stat_atomic_add(&global_fs_errors, 1);
581 + }
582 + uv_fs_req_cleanup(&req);
583 + return error;
584 +}
585 +
586 +void init_metadata_record_log(struct metadata_record_commit_log *records_log)
587 +{
588 + records_log->buf = NULL;
589 + records_log->buf_pos = 0;
590 + records_log->record_id = 1;
591 +}
592 +
593 +static int scan_metalog_files_cmp(const void *a, const void *b)
594 +{
595 + struct metadata_logfile *file1, *file2;
596 + char path1[RRDENG_PATH_MAX], path2[RRDENG_PATH_MAX];
597 +
598 + file1 = *(struct metadata_logfile **)a;
599 + file2 = *(struct metadata_logfile **)b;
600 + generate_metadata_logfile_path(file1, path1, sizeof(path1));
601 + generate_metadata_logfile_path(file2, path2, sizeof(path2));
602 + return strcmp(path1, path2);
603 +}
604 +
605 +/* Returns number of metadata logfiles that were loaded or < 0 on error */
606 +static int scan_metalog_files(struct metalog_instance *ctx)
607 +{
608 + int ret;
609 + unsigned starting_no, no, matched_files, i, failed_to_load;
610 + static uv_fs_t req;
611 + uv_dirent_t dent;
612 + struct metadata_logfile **metalogfiles, *metalogfile;
613 + char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
614 +
615 + ret = uv_fs_scandir(NULL, &req, dbfiles_path, 0, NULL);
616 + if (ret < 0) {
617 + assert(req.result < 0);
618 + uv_fs_req_cleanup(&req);
619 + error("uv_fs_scandir(%s): %s", dbfiles_path, uv_strerror(ret));
620 + ++ctx->stats.fs_errors;
621 + rrd_stat_atomic_add(&global_fs_errors, 1);
622 + return ret;
623 + }
624 + info("Found %d files in path %s", ret, dbfiles_path);
625 +
626 + metalogfiles = callocz(MIN(ret, MAX_DATAFILES), sizeof(*metalogfiles));
627 + for (matched_files = 0 ; UV_EOF != uv_fs_scandir_next(&req, &dent) && matched_files < MAX_DATAFILES ; ) {
628 + info("Scanning file \"%s/%s\"", dbfiles_path, dent.name);
629 + ret = sscanf(dent.name, METALOG_PREFIX METALOG_FILE_NUMBER_SCAN_TMPL METALOG_EXTENSION, &starting_no, &no);
630 + if (2 == ret) {
631 + info("Matched file \"%s/%s\"", dbfiles_path, dent.name);
632 + metalogfile = mallocz(sizeof(*metalogfile));
633 + metadata_logfile_init(metalogfile, ctx, starting_no, no);
634 + metalogfiles[matched_files++] = metalogfile;
635 + }
636 + }
637 + uv_fs_req_cleanup(&req);
638 +
639 + if (0 == matched_files) {
640 + freez(metalogfiles);
641 + return 0;
642 + }
643 + if (matched_files == MAX_DATAFILES) {
644 + error("Warning: hit maximum database engine file limit of %d files", MAX_DATAFILES);
645 + }
646 + qsort(metalogfiles, matched_files, sizeof(*metalogfiles), scan_metalog_files_cmp);
647 + ret = compaction_failure_recovery(ctx, metalogfiles, &matched_files);
648 + if (ret) { /* If the files are corrupted fail */
649 + for (i = 0 ; i < matched_files ; ++i) {
650 + freez(metalogfiles[i]);
651 + }
652 + freez(metalogfiles);
653 + return UV_EINVAL;
654 + }
655 + ctx->last_fileno = metalogfiles[matched_files - 1]->fileno;
656 +
657 + struct plugind cd = {
658 + .enabled = 1,
659 + .update_every = 0,
660 + .pid = 0,
661 + .serial_failures = 0,
662 + .successful_collections = 0,
663 + .obsolete = 0,
664 + .started_t = INVALID_TIME,
665 + .next = NULL,
666 + .version = 0,
667 + };
668 +
669 + struct metalog_pluginsd_state metalog_parser_state;
670 + metalog_pluginsd_state_init(&metalog_parser_state, ctx);
671 +
672 + PARSER_USER_OBJECT metalog_parser_object;
673 + metalog_parser_object.enabled = cd.enabled;
674 + metalog_parser_object.host = ctx->rrdeng_ctx->host;
675 + metalog_parser_object.cd = &cd;
676 + metalog_parser_object.trust_durations = 0;
677 + metalog_parser_object.private = &metalog_parser_state;
678 +
679 + PARSER *parser = parser_init(metalog_parser_object.host, &metalog_parser_object, NULL, PARSER_INPUT_SPLIT);
680 + if (unlikely(!parser)) {
681 + error("Failed to initialize metadata log parser.");
682 + failed_to_load = matched_files;
683 + goto after_failed_to_parse;
684 + }
685 + parser_add_keyword(parser, PLUGINSD_KEYWORD_GUID, pluginsd_guid);
686 + parser_add_keyword(parser, PLUGINSD_KEYWORD_CONTEXT, pluginsd_context);
687 + parser_add_keyword(parser, PLUGINSD_KEYWORD_TOMBSTONE, pluginsd_tombstone);
688 + parser->plugins_action->dimension_action = &metalog_pluginsd_dimension_action;
689 + parser->plugins_action->chart_action = &metalog_pluginsd_chart_action;
690 + parser->plugins_action->guid_action = &metalog_pluginsd_guid_action;
691 + parser->plugins_action->context_action = &metalog_pluginsd_context_action;
692 + parser->plugins_action->tombstone_action = &metalog_pluginsd_tombstone_action;
693 +
694 + metalog_parser_object.parser = parser;
695 + ctx->metalog_parser_object = &metalog_parser_object;
696 +
697 + for (failed_to_load = 0, i = 0 ; i < matched_files ; ++i) {
698 + metalogfile = metalogfiles[i];
699 + ret = load_metadata_logfile(ctx, metalogfile);
700 + if (0 != ret) {
701 + freez(metalogfile);
702 + ++failed_to_load;
703 + break;
704 + }
705 + metadata_logfile_list_insert(&ctx->metadata_logfiles, metalogfile);
706 + rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
707 + }
708 + debug(D_METADATALOG, "PARSER ended");
709 +
710 + parser_destroy(parser);
711 +
712 + size_t count = metalog_parser_object.count;
713 +
714 + debug(D_METADATALOG, "Parsing count=%u", (unsigned)count);
715 +after_failed_to_parse:
716 +
717 + freez(metalogfiles);
718 + if (failed_to_load) {
719 + error("%u metadata log files failed to load.", failed_to_load);
720 + finalize_metalog_files(ctx);
721 + return UV_EIO;
722 + }
723 +
724 + return matched_files;
725 +}
726 +
727 +/* Creates a metadata log file */
728 +int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
729 + unsigned starting_fileno, unsigned fileno)
730 +{
731 + struct metadata_logfile *metalogfile;
732 + int ret;
733 + char path[RRDENG_PATH_MAX];
734 +
735 + info("Creating new metadata log file in path %s", ctx->rrdeng_ctx->dbfiles_path);
736 + metalogfile = mallocz(sizeof(*metalogfile));
737 + metadata_logfile_init(metalogfile, ctx, starting_fileno, fileno);
738 + ret = create_metadata_logfile(metalogfile);
739 + if (!ret) {
740 + generate_metadata_logfile_path(metalogfile, path, sizeof(path));
741 + info("Created metadata log file \"%s\".", path);
742 + } else {
743 + freez(metalogfile);
744 + return ret;
745 + }
746 + metadata_logfile_list_insert(logfile_list, metalogfile);
747 + rrd_atomic_fetch_add(&ctx->disk_space, metalogfile->pos);
748 +
749 + return 0;
750 +}
751 +
752 +/* Return 0 on success. */
753 +int init_metalog_files(struct metalog_instance *ctx)
754 +{
755 + int ret;
756 + char *dbfiles_path = ctx->rrdeng_ctx->dbfiles_path;
757 +
758 + ret = scan_metalog_files(ctx);
759 + if (ret < 0) {
760 + error("Failed to scan path \"%s\".", dbfiles_path);
761 + return ret;
762 + } else if (0 == ret) {
763 + info("Metadata log files not found, creating in path \"%s\".", dbfiles_path);
764 + ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, 1);
765 + if (ret) {
766 + error("Failed to create metadata log file in path \"%s\".", dbfiles_path);
767 + return ret;
768 + }
769 + ctx->last_fileno = 1;
770 + }
771 +
772 + return 0;
773 +}
774 +
775 +void finalize_metalog_files(struct metalog_instance *ctx)
776 +{
777 + struct metadata_logfile *metalogfile, *next_metalogfile;
778 + struct metalog_record_block *record_block, *next_record_block;
779 +
780 + for (metalogfile = ctx->metadata_logfiles.first ; metalogfile != NULL ; metalogfile = next_metalogfile) {
781 + next_metalogfile = metalogfile->next;
782 +
783 + for (record_block = metalogfile->records.first ; record_block != NULL ; record_block = next_record_block) {
784 + next_record_block = record_block->next;
785 + freez(record_block);
786 + }
787 + close_metadata_logfile(metalogfile);
788 + freez(metalogfile);
789 + }
790 +}
database/engine/metadata_log/logfile.h new
+97
@@ -0,0 +1,97 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_LOGFILE_H
4 +#define NETDATA_LOGFILE_H
5 +
6 +#include "metadatalogprotocol.h"
7 +#include "../rrdengine.h"
8 +
9 +/* Forward declarations */
10 +struct metadata_logfile;
11 +struct metalog_worker_config;
12 +
13 +#define METALOG_PREFIX "metadatalog-"
14 +#define METALOG_EXTENSION ".mlf"
15 +
16 +#define MAX_METALOGFILE_SIZE (524288LU)
17 +
18 +/* Deletions are ignored during compaction, so only creation UUIDs are stored */
19 +struct metalog_record {
20 + uuid_t uuid;
21 +};
22 +
23 +#define MAX_METALOG_RECORDS_PER_BLOCK (1024LU)
24 +struct metalog_record_block {
25 + uint64_t file_offset;
26 + uint32_t io_size;
27 +
28 + struct metalog_record record_array[MAX_METALOG_RECORDS_PER_BLOCK];
29 + uint16_t records_nr;
30 +
31 + struct metalog_record_block *next;
32 +};
33 +
34 +struct metalog_records {
35 + /* the record block list is sorted based on disk offset */
36 + struct metalog_record_block *first;
37 + struct metalog_record_block *last;
38 + struct {
39 + struct metalog_record_block *current;
40 + uint16_t record_i;
41 + } iterator;
42 +};
43 +
44 +/* only one event loop is supported for now */
45 +struct metadata_logfile {
46 + unsigned fileno; /* Starts at 1 */
47 + unsigned starting_fileno; /* 0 for normal files, staring number during compaction */
48 + uv_file file;
49 + uint64_t pos;
50 + struct metalog_instance *ctx;
51 + struct metalog_records records;
52 + struct metadata_logfile *next;
53 +};
54 +
55 +struct metadata_logfile_list {
56 + struct metadata_logfile *first; /* oldest */
57 + struct metadata_logfile *last; /* newest */
58 +};
59 +
60 +struct metadata_record_commit_log {
61 + uint64_t record_id;
62 +
63 + /* outstanding record buffer */
64 + void *buf;
65 + unsigned buf_pos;
66 + unsigned buf_size;
67 +};
68 +
69 +extern void mlf_record_insert(struct metadata_logfile *metalogfile, struct metalog_record *record);
70 +extern struct metalog_record *mlf_record_get_first(struct metadata_logfile *metalogfile);
71 +extern struct metalog_record *mlf_record_get_next(struct metadata_logfile *metalogfile);
72 +extern void mlf_flush_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
73 + struct metadata_logfile_list *metadata_logfiles);
74 +extern void *mlf_get_records_buffer(struct metalog_worker_config *wc, struct metadata_record_commit_log *records_log,
75 + struct metadata_logfile_list *metadata_logfiles, unsigned size);
76 +extern void metadata_logfile_list_insert(struct metadata_logfile_list *metadata_logfiles,
77 + struct metadata_logfile *metalogfile);
78 +extern void metadata_logfile_list_delete(struct metadata_logfile_list *metadata_logfiles,
79 + struct metadata_logfile *metalogfile);
80 +extern void generate_metadata_logfile_path(struct metadata_logfile *metadatalog, char *str, size_t maxlen);
81 +extern void metadata_logfile_init(struct metadata_logfile *metadatalog, struct metalog_instance *ctx,
82 + unsigned tier, unsigned fileno);
83 +extern int rename_metadata_logfile(struct metadata_logfile *metalogfile, unsigned new_starting_fileno,
84 + unsigned new_fileno);
85 +extern int close_metadata_logfile(struct metadata_logfile *metadatalog);
86 +extern int unlink_metadata_logfile(struct metadata_logfile *metalogfile);
87 +extern int destroy_metadata_logfile(struct metadata_logfile *metalogfile);
88 +extern int create_metadata_logfile(struct metadata_logfile *metalogfile);
89 +extern int load_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile *logfile);
90 +extern void init_metadata_record_log(struct metadata_record_commit_log *records_log);
91 +extern int add_new_metadata_logfile(struct metalog_instance *ctx, struct metadata_logfile_list *logfile_list,
92 + unsigned tier, unsigned fileno);
93 +extern int init_metalog_files(struct metalog_instance *ctx);
94 +extern void finalize_metalog_files(struct metalog_instance *ctx);
95 +
96 +
97 +#endif /* NETDATA_LOGFILE_H */
database/engine/metadata_log/metadatalog.c new
+407
@@ -0,0 +1,407 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3 +
4 +#include "metadatalog.h"
5 +
6 +static void sanity_check(void)
7 +{
8 + /* Magic numbers must fit in the super-blocks */
9 + BUILD_BUG_ON(strlen(RRDENG_METALOG_MAGIC) > RRDENG_MAGIC_SZ);
10 +
11 + /* Metadata log file super-block cannot be larger than RRDENG_BLOCK_SIZE */
12 + BUILD_BUG_ON(RRDENG_METALOG_SB_PADDING_SZ < 0);
13 +
14 + /* Object duplication factor cannot be less than 1, or too close to 1 */
15 + BUILD_BUG_ON(MAX_DUPLICATION_PERCENTAGE < 110);
16 +}
17 +
18 +char *get_metalog_statistics(struct metalog_instance *ctx, char *str, size_t size)
19 +{
20 + snprintfz(str, size,
21 + "io_write_bytes: %ld\n"
22 + "io_write_requests: %ld\n"
23 + "io_read_bytes: %ld\n"
24 + "io_read_requests: %ld\n"
25 + "io_write_record_bytes: %ld\n"
26 + "io_write_records: %ld\n"
27 + "io_read_record_bytes: %ld\n"
28 + "io_read_records: %ld\n"
29 + "metadata_logfile_creations: %ld\n"
30 + "metadata_logfile_deletions: %ld\n"
31 + "io_errors: %ld\n"
32 + "fs_errors: %ld\n",
33 + (long)ctx->stats.io_write_bytes,
34 + (long)ctx->stats.io_write_requests,
35 + (long)ctx->stats.io_read_bytes,
36 + (long)ctx->stats.io_read_requests,
37 + (long)ctx->stats.io_write_record_bytes,
38 + (long)ctx->stats.io_write_records,
39 + (long)ctx->stats.io_read_record_bytes,
40 + (long)ctx->stats.io_read_records,
41 + (long)ctx->stats.metadata_logfile_creations,
42 + (long)ctx->stats.metadata_logfile_deletions,
43 + (long)ctx->stats.io_errors,
44 + (long)ctx->stats.fs_errors
45 + );
46 + return str;
47 +}
48 +
49 +/* The buffer must not be empty */
50 +void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode, uuid_t *uuid,
51 + int compacting)
52 +{
53 + struct metalog_cmd cmd;
54 +
55 + assert(buffer_strlen(buffer));
56 + assert(opcode == METALOG_COMMIT_CREATION_RECORD || opcode == METALOG_COMMIT_DELETION_RECORD);
57 +
58 + cmd.opcode = opcode;
59 + cmd.record_io_descr.buffer = buffer;
60 + cmd.record_io_descr.compacting = compacting;
61 + if (!uuid)
62 + uuid_clear(cmd.record_io_descr.uuid);
63 + else
64 + uuid_copy(cmd.record_io_descr.uuid, *uuid);
65 + metalog_enq_cmd(&ctx->worker_config, &cmd);
66 +}
67 +
68 +static void commit_record(struct metalog_worker_config* wc, struct metalog_record_io_descr *io_descr, uint8_t type)
69 +{
70 + struct metalog_instance *ctx = wc->ctx;
71 + unsigned payload_length, size_bytes;
72 + void *buf, *mlf_payload;
73 + /* persistent structures */
74 + struct rrdeng_metalog_record_header *mlf_header;
75 + struct rrdeng_metalog_record_trailer *mlf_trailer;
76 + uLong crc;
77 +
78 + payload_length = buffer_strlen(io_descr->buffer);
79 + size_bytes = sizeof(*mlf_header) + payload_length + sizeof(*mlf_trailer);
80 +
81 + if (io_descr->compacting)
82 + buf = mlf_get_records_buffer(wc, &ctx->compaction_state.records_log,
83 + &ctx->compaction_state.new_metadata_logfiles, size_bytes);
84 + else
85 + buf = mlf_get_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles, size_bytes);
86 +
87 + mlf_header = buf;
88 + mlf_header->type = type;
89 + mlf_header->header_length = sizeof(*mlf_header);
90 + mlf_header->payload_length = payload_length;
91 +
92 + mlf_payload = buf + sizeof(*mlf_header);
93 + memcpy(mlf_payload, buffer_tostring(io_descr->buffer), payload_length);
94 +
95 + mlf_trailer = buf + sizeof(*mlf_header) + payload_length;
96 + crc = crc32(0L, Z_NULL, 0);
97 + crc = crc32(crc, buf, sizeof(*mlf_header) + payload_length);
98 + crc32set(mlf_trailer->checksum, crc);
99 +
100 + buffer_free(io_descr->buffer);
101 +}
102 +
103 +static void do_commit_record(struct metalog_worker_config* wc, uint8_t type, void *data)
104 +{
105 + struct metalog_record_io_descr *io_descr = (struct metalog_record_io_descr *)data;
106 + switch (type) {
107 + case METALOG_CREATE_OBJECT:
108 + if (!uuid_is_null(io_descr->uuid)) { /* It's a valid object */
109 + struct metalog_record record;
110 +
111 + uuid_copy(record.uuid, io_descr->uuid);
112 + if (io_descr->compacting)
113 + mlf_record_insert(wc->ctx->compaction_state.new_metadata_logfiles.last, &record);
114 + else
115 + mlf_record_insert(wc->ctx->metadata_logfiles.last, &record);
116 + } /* fall through */
117 + case METALOG_DELETE_OBJECT:
118 + commit_record(wc, (struct metalog_record_io_descr *)data, type);
119 + break;
120 + default:
121 + fatal("Unknown metadata log file record type, possible memory corruption.");
122 + break;
123 + }
124 +}
125 +
126 +/* Only creates a new metadata file and links it to the metadata log if the last one is non empty. */
127 +void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc)
128 +{
129 + struct metalog_instance *ctx = wc->ctx;
130 + struct metadata_logfile *metalogfile;
131 + int ret;
132 +
133 + metalogfile = ctx->metadata_logfiles.last;
134 + if (metalogfile->records.first) { /* it has records */
135 + /* Finalize metadata log file and create a new one */
136 + mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
137 + ret = add_new_metadata_logfile(ctx, &ctx->metadata_logfiles, 0, ctx->last_fileno + 1);
138 + if (likely(!ret)) {
139 + ++ctx->last_fileno;
140 + }
141 + }
142 +}
143 +
144 +void metalog_test_quota(struct metalog_worker_config *wc)
145 +{
146 + struct metalog_instance *ctx = wc->ctx;
147 + struct metadata_logfile *metalogfile;
148 + unsigned current_size;
149 + uint8_t only_one_metalogfile;
150 +
151 + metalogfile = ctx->metadata_logfiles.last;
152 + current_size = metalogfile->pos;
153 + if (unlikely(current_size >= MAX_METALOGFILE_SIZE)) {
154 + metalog_try_link_new_metadata_logfile(wc);
155 + }
156 +
157 + metalogfile = ctx->metadata_logfiles.last;
158 + only_one_metalogfile = (metalogfile == ctx->metadata_logfiles.first) ? 1 : 0;
159 + debug(D_METADATALOG, "records=%lu objects=%lu", (long unsigned)ctx->records_nr,
160 + (long unsigned)ctx->rrdeng_ctx->host->objects_nr);
161 + if (unlikely(!only_one_metalogfile &&
162 + ctx->records_nr > (ctx->rrdeng_ctx->host->objects_nr * (uint64_t)MAX_DUPLICATION_PERCENTAGE) / 100) &&
163 + NO_QUIESCE == ctx->quiesce) {
164 + metalog_do_compaction(wc);
165 + }
166 +}
167 +
168 +static inline int metalog_threads_alive(struct metalog_worker_config* wc)
169 +{
170 + if (wc->cleanup_thread_compacting_files) {
171 + return 1;
172 + }
173 +
174 + return 0;
175 +}
176 +
177 +static void metalog_cleanup_finished_threads(struct metalog_worker_config *wc)
178 +{
179 + struct metalog_instance *ctx = wc->ctx;
180 +
181 + if (unlikely(wc->cleanup_thread_compacting_files)) {
182 + after_compact_old_records(wc);
183 + }
184 + if (unlikely(SET_QUIESCE == ctx->quiesce && !metalog_threads_alive(wc))) {
185 + ctx->quiesce = QUIESCED;
186 + complete(&ctx->metalog_completion);
187 + }
188 +}
189 +
190 +static void metalog_init_cmd_queue(struct metalog_worker_config *wc)
191 +{
192 + wc->cmd_queue.head = wc->cmd_queue.tail = 0;
193 + wc->queue_size = 0;
194 + assert(0 == uv_cond_init(&wc->cmd_cond));
195 + assert(0 == uv_mutex_init(&wc->cmd_mutex));
196 +}
197 +
198 +void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd)
199 +{
200 + unsigned queue_size;
201 +
202 + /* wait for free space in queue */
203 + uv_mutex_lock(&wc->cmd_mutex);
204 + while ((queue_size = wc->queue_size) == METALOG_CMD_Q_MAX_SIZE) {
205 + uv_cond_wait(&wc->cmd_cond, &wc->cmd_mutex);
206 + }
207 + assert(queue_size < METALOG_CMD_Q_MAX_SIZE);
208 + /* enqueue command */
209 + wc->cmd_queue.cmd_array[wc->cmd_queue.tail] = *cmd;
210 + wc->cmd_queue.tail = wc->cmd_queue.tail != METALOG_CMD_Q_MAX_SIZE - 1 ?
211 + wc->cmd_queue.tail + 1 : 0;
212 + wc->queue_size = queue_size + 1;
213 + uv_mutex_unlock(&wc->cmd_mutex);
214 +
215 + /* wake up event loop */
216 + assert(0 == uv_async_send(&wc->async));
217 +}
218 +
219 +struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc)
220 +{
221 + struct metalog_cmd ret;
222 + unsigned queue_size;
223 +
224 + uv_mutex_lock(&wc->cmd_mutex);
225 + queue_size = wc->queue_size;
226 + if (queue_size == 0) {
227 + ret.opcode = METALOG_NOOP;
228 + } else {
229 + /* dequeue command */
230 + ret = wc->cmd_queue.cmd_array[wc->cmd_queue.head];
231 + if (queue_size == 1) {
232 + wc->cmd_queue.head = wc->cmd_queue.tail = 0;
233 + } else {
234 + wc->cmd_queue.head = wc->cmd_queue.head != RRDENG_CMD_Q_MAX_SIZE - 1 ?
235 + wc->cmd_queue.head + 1 : 0;
236 + }
237 + wc->queue_size = queue_size - 1;
238 +
239 + /* wake up producers */
240 + uv_cond_signal(&wc->cmd_cond);
241 + }
242 + uv_mutex_unlock(&wc->cmd_mutex);
243 +
244 + return ret;
245 +}
246 +
247 +static void async_cb(uv_async_t *handle)
248 +{
249 + uv_stop(handle->loop);
250 + uv_update_time(handle->loop);
251 + debug(D_METADATALOG, "%s called, active=%d.", __func__, uv_is_active((uv_handle_t *)handle));
252 +}
253 +
254 +/* Flushes metadata log when timer expires */
255 +#define TIMER_PERIOD_MS (5000)
256 +
257 +static void timer_cb(uv_timer_t* handle)
258 +{
259 + struct metalog_worker_config* wc = handle->data;
260 + struct metalog_instance *ctx = wc->ctx;
261 +
262 + uv_stop(handle->loop);
263 + uv_update_time(handle->loop);
264 + metalog_test_quota(wc);
265 + debug(D_METADATALOG, "%s: timeout reached.", __func__);
266 +#ifdef NETDATA_INTERNAL_CHECKS
267 + {
268 + char buf[4096];
269 + debug(D_METADATALOG, "%s", get_metalog_statistics(wc->ctx, buf, sizeof(buf)));
270 + }
271 +#endif
272 + mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
273 +}
274 +
275 +#define MAX_CMD_BATCH_SIZE (256)
276 +
277 +void metalog_worker(void* arg)
278 +{
279 + struct metalog_worker_config *wc = arg;
280 + struct metalog_instance *ctx = wc->ctx;
281 + uv_loop_t* loop;
282 + int shutdown, ret;
283 + enum metalog_opcode opcode;
284 + uv_timer_t timer_req;
285 + struct metalog_cmd cmd;
286 + unsigned cmd_batch_size;
287 +
288 + metalog_init_cmd_queue(wc);
289 +
290 + loop = wc->loop = mallocz(sizeof(uv_loop_t));
291 + ret = uv_loop_init(loop);
292 + if (ret) {
293 + error("uv_loop_init(): %s", uv_strerror(ret));
294 + goto error_after_loop_init;
295 + }
296 + loop->data = wc;
297 +
298 + ret = uv_async_init(wc->loop, &wc->async, async_cb);
299 + if (ret) {
300 + error("uv_async_init(): %s", uv_strerror(ret));
301 + goto error_after_async_init;
302 + }
303 + wc->async.data = wc;
304 +
305 + wc->now_compacting_files = NULL;
306 + wc->cleanup_thread_compacting_files = 0;
307 +
308 + /* quota check timer */
309 + ret = uv_timer_init(loop, &timer_req);
310 + if (ret) {
311 + error("uv_timer_init(): %s", uv_strerror(ret));
312 + goto error_after_timer_init;
313 + }
314 + timer_req.data = wc;
315 +
316 + wc->error = 0;
317 + /* wake up initialization thread */
318 + complete(&ctx->metalog_completion);
319 +
320 + assert(0 == uv_timer_start(&timer_req, timer_cb, TIMER_PERIOD_MS, TIMER_PERIOD_MS));
321 + shutdown = 0;
322 + while (likely(shutdown == 0 || metalog_threads_alive(wc))) {
323 + uv_run(loop, UV_RUN_DEFAULT);
324 + metalog_cleanup_finished_threads(wc);
325 +
326 + /* wait for commands */
327 + cmd_batch_size = 0;
328 + do {
329 + /*
330 + * Avoid starving the loop when there are too many commands coming in.
331 + * timer_cb will interrupt the loop again to allow serving more commands.
332 + */
333 + if (unlikely(cmd_batch_size >= MAX_CMD_BATCH_SIZE))
334 + break;
335 +
336 + cmd = metalog_deq_cmd(wc);
337 + opcode = cmd.opcode;
338 + ++cmd_batch_size;
339 +
340 + switch (opcode) {
341 + case METALOG_NOOP:
342 + /* the command queue was empty, do nothing */
343 + break;
344 + case METALOG_SHUTDOWN:
345 + shutdown = 1;
346 + break;
347 + case METALOG_QUIESCE:
348 + ctx->quiesce = SET_QUIESCE;
349 + assert(0 == uv_timer_stop(&timer_req));
350 + uv_close((uv_handle_t *)&timer_req, NULL);
351 + mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
352 + if (!metalog_threads_alive(wc)) {
353 + ctx->quiesce = QUIESCED;
354 + complete(&ctx->metalog_completion);
355 + }
356 + break;
357 + case METALOG_COMMIT_CREATION_RECORD:
358 + do_commit_record(wc, METALOG_CREATE_OBJECT, &cmd.record_io_descr);
359 + break;
360 + case METALOG_COMMIT_DELETION_RECORD:
361 + do_commit_record(wc, METALOG_DELETE_OBJECT, &cmd.record_io_descr);
362 + break;
363 + case METALOG_COMPACTION_FLUSH:
364 + mlf_flush_records_buffer(wc, &ctx->compaction_state.records_log,
365 + &ctx->compaction_state.new_metadata_logfiles);
366 + complete(cmd.record_io_descr.completion);
367 + break;
368 + default:
369 + debug(D_METADATALOG, "%s: default.", __func__);
370 + break;
371 + }
372 + } while (opcode != METALOG_NOOP);
373 + }
374 +
375 + /* cleanup operations of the event loop */
376 + info("Shutting down RRD metadata log event loop.");
377 +
378 + /*
379 + * uv_async_send after uv_close does not seem to crash in linux at the moment,
380 + * it is however undocumented behaviour and we need to be aware if this becomes
381 + * an issue in the future.
382 + */
383 + uv_close((uv_handle_t *)&wc->async, NULL);
384 +
385 + mlf_flush_records_buffer(wc, &ctx->records_log, &ctx->metadata_logfiles);
386 + uv_run(loop, UV_RUN_DEFAULT);
387 +
388 + info("Shutting down RRD metadata log loop complete.");
389 + /* TODO: don't let the API block by waiting to enqueue commands */
390 + uv_cond_destroy(&wc->cmd_cond);
391 +/* uv_mutex_destroy(&wc->cmd_mutex); */
392 + assert(0 == uv_loop_close(loop));
393 + freez(loop);
394 +
395 + return;
396 +
397 +error_after_timer_init:
398 + uv_close((uv_handle_t *)&wc->async, NULL);
399 +error_after_async_init:
400 + assert(0 == uv_loop_close(loop));
401 +error_after_loop_init:
402 + freez(loop);
403 +
404 + wc->error = UV_EAGAIN;
405 + /* wake up initialization thread */
406 + complete(&ctx->metalog_completion);
407 +}
database/engine/metadata_log/metadatalog.h new
+136
@@ -0,0 +1,136 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_METADATALOG_H
4 +#define NETDATA_METADATALOG_H
5 +
6 +#ifndef _GNU_SOURCE
7 +#define _GNU_SOURCE
8 +#endif
9 +#include "../rrdengine.h"
10 +#include "metadatalogprotocol.h"
11 +#include "logfile.h"
12 +#include "metadatalogapi.h"
13 +#include "compaction.h"
14 +
15 +/* Forward declerations */
16 +struct metalog_instance;
17 +struct parser_user_object;
18 +
19 +#define MAX_PAGES_PER_EXTENT (64) /* TODO: can go higher only when journal supports bigger than 4KiB transactions */
20 +
21 +#define METALOG_FILE_NUMBER_SCAN_TMPL "%5u-%5u"
22 +#define METALOG_FILE_NUMBER_PRINT_TMPL "%5.5u-%5.5u"
23 +
24 +#define MAX_DUPLICATION_PERCENTAGE 150 /* the maximum duplication factor of objects in metadata log records */
25 +
26 +typedef enum {
27 + METALOG_STATUS_UNINITIALIZED = 0,
28 + METALOG_STATUS_INITIALIZING,
29 + METALOG_STATUS_INITIALIZED
30 +} metalog_state_t;
31 +
32 +struct metalog_record_io_descr {
33 + BUFFER *buffer;
34 + struct completion *completion;
35 + int compacting; /* When 0 append at the end of the metadata log file list.
36 + When 1 append to the temporary compaction metadata log file list. */
37 + uuid_t uuid;
38 +};
39 +
40 +enum metalog_opcode {
41 + /* can be used to return empty status or flush the command queue */
42 + METALOG_NOOP = 0,
43 +
44 + METALOG_SHUTDOWN,
45 + METALOG_COMMIT_CREATION_RECORD,
46 + METALOG_COMMIT_DELETION_RECORD,
47 + METALOG_COMPACTION_FLUSH,
48 + METALOG_QUIESCE,
49 +
50 + METALOG_MAX_OPCODE
51 +};
52 +
53 +struct metalog_cmd {
54 + enum metalog_opcode opcode;
55 + struct metalog_record_io_descr record_io_descr;
56 +};
57 +
58 +#define METALOG_CMD_Q_MAX_SIZE (2048)
59 +
60 +struct metalog_cmdqueue {
61 + unsigned head, tail;
62 + struct metalog_cmd cmd_array[METALOG_CMD_Q_MAX_SIZE];
63 +};
64 +
65 +struct metalog_worker_config {
66 + struct metalog_instance *ctx;
67 +
68 + uv_thread_t thread;
69 + uv_loop_t *loop;
70 + uv_async_t async;
71 +
72 + /* metadata log file comapaction thread */
73 + uv_thread_t *now_compacting_files;
74 + unsigned long cleanup_thread_compacting_files; /* set to 0 when now_compacting_files is still running */
75 +
76 + /* FIFO command queue */
77 + uv_mutex_t cmd_mutex;
78 + uv_cond_t cmd_cond;
79 + volatile unsigned queue_size;
80 + struct metalog_cmdqueue cmd_queue;
81 +
82 + int error;
83 +};
84 +
85 +/*
86 + * Debug statistics not used by code logic.
87 + * They only describe operations since DB engine instance load time.
88 + */
89 +struct metalog_statistics {
90 + rrdeng_stats_t io_write_bytes;
91 + rrdeng_stats_t io_write_requests;
92 + rrdeng_stats_t io_read_bytes;
93 + rrdeng_stats_t io_read_requests;
94 + rrdeng_stats_t io_write_record_bytes;
95 + rrdeng_stats_t io_write_records;
96 + rrdeng_stats_t io_read_record_bytes;
97 + rrdeng_stats_t io_read_records;
98 + rrdeng_stats_t metadata_logfile_creations;
99 + rrdeng_stats_t metadata_logfile_deletions;
100 + rrdeng_stats_t io_errors;
101 + rrdeng_stats_t fs_errors;
102 +};
103 +
104 +struct metalog_instance {
105 + struct rrdengine_instance *rrdeng_ctx;
106 + struct metalog_worker_config worker_config;
107 + struct completion metalog_completion;
108 + struct metadata_record_commit_log records_log;
109 + struct metadata_logfile_list metadata_logfiles;
110 + struct parser_user_object *metalog_parser_object;
111 + struct logfile_compaction_state compaction_state;
112 + uint32_t current_compaction_id; /* Every compaction run increments this by 1 */
113 + unsigned long disk_space;
114 + unsigned long records_nr;
115 + unsigned last_fileno; /* newest index of metadata log file */
116 +
117 + uint8_t quiesce; /*
118 + * 0 initial state when all operations function normally
119 + * 1 set it before shutting down the instance, quiesce long running operations
120 + * 2 is set after all threads have finished running
121 + */
122 +
123 + struct metalog_statistics stats;
124 +};
125 +
126 +extern void metalog_commit_record(struct metalog_instance *ctx, BUFFER *buffer, enum metalog_opcode opcode,
127 + uuid_t *uuid, int compacting);
128 + extern int init_metadata_logfiles(struct metalog_instance *ctx);
129 +extern void finalize_metadata_logfiles(struct metalog_instance *ctx);
130 +extern void metalog_try_link_new_metadata_logfile(struct metalog_worker_config *wc);
131 +extern void metalog_test_quota(struct metalog_worker_config *wc);
132 +extern void metalog_worker(void* arg);
133 +extern void metalog_enq_cmd(struct metalog_worker_config *wc, struct metalog_cmd *cmd);
134 +extern struct metalog_cmd metalog_deq_cmd(struct metalog_worker_config *wc);
135 +
136 +#endif /* NETDATA_METADATALOG_H */
database/engine/metadata_log/metadatalogapi.c new
+444
@@ -0,0 +1,444 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3 +
4 +#include "metadatalog.h"
5 +
6 +static inline int metalog_is_initialized(struct metalog_instance *ctx)
7 +{
8 + return ctx->rrdeng_ctx->metalog_ctx != NULL;
9 +}
10 +
11 +static inline void metalog_commit_creation_record(struct metalog_instance *ctx, BUFFER *buffer, uuid_t *uuid)
12 +{
13 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_CREATION_RECORD, uuid, 0);
14 +}
15 +
16 +static inline void metalog_commit_deletion_record(struct metalog_instance *ctx, BUFFER *buffer)
17 +{
18 + metalog_commit_record(ctx, buffer, METALOG_COMMIT_DELETION_RECORD, NULL, 0);
19 +}
20 +
21 +BUFFER *metalog_update_host_buffer(RRDHOST *host)
22 +{
23 + BUFFER *buffer;
24 + buffer = buffer_create(4096); /* This will be freed after it has been committed to the metadata log buffer */
25 +
26 + rrdhost_rdlock(host);
27 +
28 + buffer_sprintf(buffer,
29 + "HOST \"%s\" \"%s\" \"%s\" %d \"%s\" \"%s\" \"%s\"\n",
30 +// "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"" /* system */
31 +// "\"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\"", /* info */
32 + host->machine_guid,
33 + host->hostname,
34 + host->registry_hostname,
35 + default_rrd_update_every,
36 + host->os,
37 + host->timezone,
38 + (host->tags) ? host->tags : "");
39 +
40 + netdata_rwlock_rdlock(&host->labels_rwlock);
41 + struct label *labels = host->labels;
42 + while (labels) {
43 + buffer_sprintf(buffer
44 + , "LABEL \"%s\" = %d %s\n"
45 + , labels->key
46 + , (int)labels->label_source
47 + , labels->value);
48 +
49 + labels = labels->next;
50 + }
51 + netdata_rwlock_unlock(&host->labels_rwlock);
52 +
53 + buffer_strcat(buffer, "OVERWRITE labels\n");
54 +
55 + rrdhost_unlock(host);
56 + return buffer;
57 +}
58 +
59 +void metalog_commit_update_host(RRDHOST *host)
60 +{
61 + struct metalog_instance *ctx;
62 + BUFFER *buffer;
63 +
64 + /* Metadata are only available with dbengine */
65 + if (!host->rrdeng_ctx)
66 + return;
67 +
68 + ctx = host->rrdeng_ctx->metalog_ctx;
69 + if (!ctx) /* metadata log has not been initialized yet */
70 + return;
71 +
72 + buffer = metalog_update_host_buffer(host);
73 +
74 + metalog_commit_creation_record(ctx, buffer, &host->host_uuid);
75 +}
76 +
77 +/* compaction_id 0 means it was not called by compaction logic */
78 +BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id)
79 +{
80 + BUFFER *buffer;
81 + RRDHOST *host = st->rrdhost;
82 +
83 + buffer = buffer_create(1024); /* This will be freed after it has been committed to the metadata log buffer */
84 +
85 + rrdset_rdlock(st);
86 +
87 + buffer_sprintf(buffer, "CONTEXT %s\n", host->machine_guid);
88 +
89 + char uuid_str[37];
90 + uuid_unparse_lower(*st->chart_uuid, uuid_str);
91 + buffer_sprintf(buffer, "GUID %s\n", uuid_str);
92 +
93 + // properly set the name for the remote end to parse it
94 + char *name = "";
95 + if(likely(st->name)) {
96 + if(unlikely(strcmp(st->id, st->name))) {
97 + // they differ
98 + name = strchr(st->name, '.');
99 + if(name)
100 + name++;
101 + else
102 + name = "";
103 + }
104 + }
105 +
106 + // send the chart
107 + buffer_sprintf(
108 + buffer
109 + , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
110 + , st->id
111 + , name
112 + , st->title
113 + , st->units
114 + , st->family
115 + , st->context
116 + , rrdset_type_name(st->chart_type)
117 + , st->priority
118 + , st->update_every
119 + , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
120 + , rrdset_flag_check(st, RRDSET_FLAG_DETAIL)?"detail":""
121 + , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
122 + , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
123 + , (st->plugin_name)?st->plugin_name:""
124 + , (st->module_name)?st->module_name:""
125 + );
126 +
127 + // send the dimensions
128 + RRDDIM *rd;
129 + rrddim_foreach_read(rd, st) {
130 + char uuid_str[37];
131 +
132 + uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
133 + buffer_sprintf(buffer, "GUID %s\n", uuid_str);
134 +
135 + buffer_sprintf(
136 + buffer
137 + , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
138 + , rd->id
139 + , rd->name
140 + , rrd_algorithm_name(rd->algorithm)
141 + , rd->multiplier
142 + , rd->divisor
143 + , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
144 + , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
145 + , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
146 + );
147 + if (compaction_id && compaction_id > rd->state->compaction_id) {
148 + /* No need to use this dimension again during this compaction cycle */
149 + rd->state->compaction_id = compaction_id;
150 + }
151 + }
152 + rrdset_unlock(st);
153 + return buffer;
154 +}
155 +
156 +void metalog_commit_update_chart(RRDSET *st)
157 +{
158 + struct metalog_instance *ctx;
159 + BUFFER *buffer;
160 + RRDHOST *host = st->rrdhost;
161 +
162 + /* Metadata are only available with dbengine */
163 + if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
164 + return;
165 +
166 + ctx = host->rrdeng_ctx->metalog_ctx;
167 + if (!ctx) /* metadata log has not been initialized yet */
168 + return;
169 +
170 + buffer = metalog_update_chart_buffer(st, 0);
171 +
172 + metalog_commit_creation_record(ctx, buffer, st->chart_uuid);
173 +}
174 +
175 +void metalog_commit_delete_chart(RRDSET *st)
176 +{
177 + struct metalog_instance *ctx;
178 + BUFFER *buffer;
179 + RRDHOST *host = st->rrdhost;
180 + char uuid_str[37];
181 +
182 + /* Metadata are only available with dbengine */
183 + if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
184 + return;
185 +
186 + ctx = host->rrdeng_ctx->metalog_ctx;
187 + if (!ctx) /* metadata log has not been initialized yet */
188 + return;
189 + buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
190 +
191 + uuid_unparse_lower(*st->chart_uuid, uuid_str);
192 + buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
193 +
194 + metalog_commit_deletion_record(ctx, buffer);
195 +}
196 +
197 +BUFFER *metalog_update_dimension_buffer(RRDDIM *rd)
198 +{
199 + BUFFER *buffer;
200 + RRDSET *st = rd->rrdset;
201 + char uuid_str[37];
202 +
203 + buffer = buffer_create(128); /* This will be freed after it has been committed to the metadata log buffer */
204 +
205 + uuid_unparse_lower(*st->chart_uuid, uuid_str);
206 + buffer_sprintf(buffer, "CONTEXT %s\n", uuid_str);
207 + // Activate random GUID
208 + uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
209 + buffer_sprintf(buffer, "GUID %s\n", uuid_str);
210 +
211 + buffer_sprintf(
212 + buffer
213 + , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
214 + , rd->id
215 + , rd->name
216 + , rrd_algorithm_name(rd->algorithm)
217 + , rd->multiplier
218 + , rd->divisor
219 + , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
220 + , rrddim_flag_check(rd, RRDDIM_FLAG_HIDDEN)?"hidden":""
221 + , rrddim_flag_check(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
222 + );
223 + return buffer;
224 +}
225 +
226 +void metalog_commit_update_dimension(RRDDIM *rd)
227 +{
228 + struct metalog_instance *ctx;
229 + BUFFER *buffer;
230 + RRDSET *st = rd->rrdset;
231 + RRDHOST *host = st->rrdhost;
232 +
233 + /* Metadata are only available with dbengine */
234 + if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
235 + return;
236 +
237 + ctx = host->rrdeng_ctx->metalog_ctx;
238 + if (!ctx) /* metadata log has not been initialized yet */
239 + return;
240 +
241 + buffer = metalog_update_dimension_buffer(rd);
242 +
243 + metalog_commit_creation_record(ctx, buffer, rd->state->metric_uuid);
244 +}
245 +
246 +void metalog_commit_delete_dimension(RRDDIM *rd)
247 +{
248 + struct metalog_instance *ctx;
249 + BUFFER *buffer;
250 + RRDSET *st = rd->rrdset;
251 + RRDHOST *host = st->rrdhost;
252 + char uuid_str[37];
253 +
254 + /* Metadata are only available with dbengine */
255 + if (!host->rrdeng_ctx || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)
256 + return;
257 +
258 + ctx = host->rrdeng_ctx->metalog_ctx;
259 + if (!ctx) /* metadata log has not been initialized yet */
260 + return;
261 + buffer = buffer_create(64); /* This will be freed after it has been committed to the metadata log buffer */
262 +
263 + uuid_unparse_lower(*rd->state->metric_uuid, uuid_str);
264 + buffer_sprintf(buffer, "TOMBSTONE %s\n", uuid_str);
265 +
266 + metalog_commit_deletion_record(ctx, buffer);
267 +}
268 +
269 +RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid)
270 +{
271 + GUID_TYPE ret;
272 + char chart_object[33], chart_fullid[RRD_ID_LENGTH_MAX + 1];
273 + uuid_t *machine_guid, *chart_char_guid;
274 +
275 + ret = find_object_by_guid(chart_uuid, chart_object, 33);
276 + assert(GUID_TYPE_CHART == ret);
277 +
278 + machine_guid = (uuid_t *)chart_object;
279 + RRDHOST *host = ctx->rrdeng_ctx->host;
280 + assert(!uuid_compare(host->host_uuid, *machine_guid));
281 +
282 + chart_char_guid = (uuid_t *)(chart_object + 16);
283 +
284 + ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
285 + assert(GUID_TYPE_CHAR == ret);
286 + RRDSET *st = rrdset_find(host, chart_fullid);
287 +
288 + return st;
289 +}
290 +
291 +RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
292 +{
293 + GUID_TYPE ret;
294 + char dim_object[49], chart_object[33], id_str[PLUGINSD_LINE_MAX], chart_fullid[RRD_ID_LENGTH_MAX + 1];
295 + uuid_t *machine_guid, *chart_guid, *chart_char_guid, *dim_char_guid;
296 +
297 + ret = find_object_by_guid(metric_uuid, dim_object, sizeof(dim_object));
298 + if (GUID_TYPE_DIMENSION != ret) /* not found */
299 + return NULL;
300 +
301 + machine_guid = (uuid_t *)dim_object;
302 + RRDHOST *host = ctx->rrdeng_ctx->host;
303 + assert(!uuid_compare(host->host_uuid, *machine_guid));
304 +
305 + chart_guid = (uuid_t *)(dim_object + 16);
306 + dim_char_guid = (uuid_t *)(dim_object + 16 + 16);
307 +
308 + ret = find_object_by_guid(dim_char_guid, id_str, sizeof(id_str));
309 + assert(GUID_TYPE_CHAR == ret);
310 +
311 + ret = find_object_by_guid(chart_guid, chart_object, sizeof(chart_object));
312 + assert(GUID_TYPE_CHART == ret);
313 + chart_char_guid = (uuid_t *)(chart_object + 16);
314 +
315 + ret = find_object_by_guid(chart_char_guid, chart_fullid, RRD_ID_LENGTH_MAX + 1);
316 + assert(GUID_TYPE_CHAR == ret);
317 + RRDSET *st = rrdset_find(host, chart_fullid);
318 + assert(st);
319 +
320 + RRDDIM *rd = rrddim_find(st, id_str);
321 +
322 + return rd;
323 +}
324 +
325 +/* This function is called by dbengine rotation logic when the metric has no writers */
326 +void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid)
327 +{
328 + RRDDIM *rd;
329 + RRDSET *st;
330 + RRDHOST *host;
331 + uint8_t empty_chart;
332 +
333 + rd = metalog_get_dimension_from_uuid(ctx, metric_uuid);
334 + if (!rd) { /* in the case of legacy UUID convert to multihost and try again */
335 + uuid_t multihost_uuid;
336 +
337 + rrdeng_convert_legacy_uuid_to_multihost(ctx->rrdeng_ctx->host->machine_guid, metric_uuid, &multihost_uuid);
338 + rd = metalog_get_dimension_from_uuid(ctx, &multihost_uuid);
339 + }
340 + if(!rd) {
341 + info("Rotated unknown archived metric.");
342 + return;
343 + }
344 + st = rd->rrdset;
345 + host = st->rrdhost;
346 +
347 + /* Since the metric has no writer it will not be commited to the metadata log by rrddim_free_custom().
348 + * It must be commited explicitly before calling rrddim_free_custom(). */
349 + metalog_commit_delete_dimension(rd);
350 +
351 + rrdset_wrlock(st);
352 + rrddim_free_custom(st, rd, 1);
353 + empty_chart = (NULL == st->dimensions);
354 + rrdset_unlock(st);
355 +
356 + if (empty_chart) {
357 + rrdhost_wrlock(host);
358 + rrdset_rdlock(st);
359 + rrdset_delete_custom(st, 1);
360 + rrdset_unlock(st);
361 + rrdset_free(st);
362 + rrdhost_unlock(host);
363 + }
364 +}
365 +
366 +/*
367 + * Returns 0 on success, negative on error
368 + */
369 +int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx)
370 +{
371 + struct metalog_instance *ctx;
372 + int error;
373 +
374 + ctx = callocz(1, sizeof(*ctx));
375 + ctx->records_nr = 0;
376 + ctx->current_compaction_id = 0;
377 + ctx->quiesce = NO_QUIESCE;
378 +
379 + memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
380 + ctx->rrdeng_ctx = rrdeng_parent_ctx;
381 + ctx->worker_config.ctx = ctx;
382 + init_metadata_record_log(&ctx->records_log);
383 + error = init_metalog_files(ctx);
384 + if (error) {
385 + goto error_after_init_rrd_files;
386 + }
387 +
388 + init_completion(&ctx->metalog_completion);
389 + assert(0 == uv_thread_create(&ctx->worker_config.thread, metalog_worker, &ctx->worker_config));
390 + /* wait for worker thread to initialize */
391 + wait_for_completion(&ctx->metalog_completion);
392 + destroy_completion(&ctx->metalog_completion);
393 + uv_thread_set_name_np(ctx->worker_config.thread, "METALOG");
394 + if (ctx->worker_config.error) {
395 + goto error_after_rrdeng_worker;
396 + }
397 + rrdeng_parent_ctx->metalog_ctx = ctx; /* notify dbengine that the metadata log has finished initializing */
398 + return 0;
399 +
400 +error_after_rrdeng_worker:
401 + finalize_metalog_files(ctx);
402 +error_after_init_rrd_files:
403 + freez(ctx);
404 + return UV_EIO;
405 +}
406 +
407 +/*
408 + * Returns 0 on success, 1 on error
409 + */
410 +int metalog_exit(struct metalog_instance *ctx)
411 +{
412 + struct metalog_cmd cmd;
413 +
414 + if (NULL == ctx) {
415 + return 1;
416 + }
417 +
418 + cmd.opcode = METALOG_SHUTDOWN;
419 + metalog_enq_cmd(&ctx->worker_config, &cmd);
420 +
421 + assert(0 == uv_thread_join(&ctx->worker_config.thread));
422 +
423 + finalize_metalog_files(ctx);
424 + freez(ctx);
425 +
426 + return 0;
427 +}
428 +
429 +void metalog_prepare_exit(struct metalog_instance *ctx)
430 +{
431 + struct metalog_cmd cmd;
432 +
433 + if (NULL == ctx) {
434 + return;
435 + }
436 +
437 + init_completion(&ctx->metalog_completion);
438 + cmd.opcode = METALOG_QUIESCE;
439 + metalog_enq_cmd(&ctx->worker_config, &cmd);
440 +
441 + /* wait for metadata log to quiesce */
442 + wait_for_completion(&ctx->metalog_completion);
443 + destroy_completion(&ctx->metalog_completion);
444 +}
database/engine/metadata_log/metadatalogapi.h new
+26
@@ -0,0 +1,26 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_METADATALOGAPI_H
4 +#define NETDATA_METADATALOGAPI_H
5 +
6 +#include "metadatalog.h"
7 +
8 +extern BUFFER *metalog_update_host_buffer(RRDHOST *host);
9 +extern void metalog_commit_update_host(RRDHOST *host);
10 +extern BUFFER *metalog_update_chart_buffer(RRDSET *st, uint32_t compaction_id);
11 +extern void metalog_commit_update_chart(RRDSET *st);
12 +extern void metalog_commit_delete_chart(RRDSET *st);
13 +extern BUFFER *metalog_update_dimension_buffer(RRDDIM *rd);
14 +extern void metalog_commit_update_dimension(RRDDIM *rd);
15 +extern void metalog_commit_delete_dimension(RRDDIM *rd);
16 +
17 +extern RRDSET *metalog_get_chart_from_uuid(struct metalog_instance *ctx, uuid_t *chart_uuid);
18 +extern RRDDIM *metalog_get_dimension_from_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
19 +extern void metalog_delete_dimension_by_uuid(struct metalog_instance *ctx, uuid_t *metric_uuid);
20 +
21 +/* must call once before using anything */
22 +extern int metalog_init(struct rrdengine_instance *rrdeng_parent_ctx);
23 +extern int metalog_exit(struct metalog_instance *ctx);
24 +extern void metalog_prepare_exit(struct metalog_instance *ctx);
25 +
26 +#endif /* NETDATA_METADATALOGAPI_H */
database/engine/metadata_log/metadatalogprotocol.h new
+53
@@ -0,0 +1,53 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_METADATALOGPROTOCOL_H
4 +#define NETDATA_METADATALOGPROTOCOL_H
5 +
6 +#include "../rrddiskprotocol.h"
7 +
8 +#define RRDENG_METALOG_MAGIC "netdata-metadata-log"
9 +
10 +#define RRDENG_METALOG_VER (1)
11 +
12 +#define RRDENG_METALOG_SB_PADDING_SZ (RRDENG_BLOCK_SIZE - (RRDENG_MAGIC_SZ + sizeof(uint16_t)))
13 +/*
14 + * Metadata log persistent super-block
15 + */
16 +struct rrdeng_metalog_sb {
17 + char magic_number[RRDENG_MAGIC_SZ];
18 + uint16_t version;
19 + uint8_t padding[RRDENG_METALOG_SB_PADDING_SZ];
20 +} __attribute__ ((packed));
21 +
22 +/*
23 + * Metadata log record types
24 + */
25 +#define METALOG_STORE_PADDING (0)
26 +#define METALOG_CREATE_OBJECT (1)
27 +#define METALOG_DELETE_OBJECT (2)
28 +#define METALOG_OTHER (3) /* reserved */
29 +
30 +/*
31 + * Metadata log record header
32 + */
33 +struct rrdeng_metalog_record_header {
34 + /* when set to METALOG_STORE_PADDING jump to start of next block */
35 + uint8_t type;
36 +
37 + uint16_t header_length;
38 + uint32_t payload_length;
39 + /******************************************************
40 + * No fields above this point can ever change. *
41 + ******************************************************
42 + * All fields below this point are subject to change. *
43 + ******************************************************/
44 +} __attribute__ ((packed));
45 +
46 +/*
47 + * Metadata log record trailer
48 + */
49 +struct rrdeng_metalog_record_trailer {
50 + uint8_t checksum[CHECKSUM_SZ]; /* CRC32 */
51 +} __attribute__ ((packed));
52 +
53 +#endif /* NETDATA_METADATALOGPROTOCOL_H */
database/engine/metadata_log/metalogpluginsd.c new
+206
@@ -0,0 +1,206 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +#define NETDATA_RRD_INTERNALS
3 +
4 +#include "metadatalog.h"
5 +#include "metalogpluginsd.h"
6 +
7 +PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context,
8 + char *title, char *units, char *plugin, char *module, int priority,
9 + int update_every, RRDSET_TYPE chart_type, char *options)
10 +{
11 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
12 + RRDSET *st = NULL;
13 + RRDHOST *host = ((PARSER_USER_OBJECT *) user)->host;
14 + uuid_t *chart_uuid;
15 +
16 + chart_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
17 + st = rrdset_create_custom(
18 + host, type, id, name, family, context, title, units,
19 + plugin, module, priority, update_every,
20 + chart_type, RRD_MEMORY_MODE_DBENGINE, (host)->rrd_history_entries, 1, chart_uuid);
21 +
22 + if (options && *options) {
23 + if (strstr(options, "obsolete"))
24 + rrdset_is_obsolete(st);
25 + else
26 + rrdset_isnot_obsolete(st);
27 +
28 + if (strstr(options, "detail"))
29 + rrdset_flag_set(st, RRDSET_FLAG_DETAIL);
30 + else
31 + rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
32 +
33 + if (strstr(options, "hidden"))
34 + rrdset_flag_set(st, RRDSET_FLAG_HIDDEN);
35 + else
36 + rrdset_flag_clear(st, RRDSET_FLAG_HIDDEN);
37 +
38 + if (strstr(options, "store_first"))
39 + rrdset_flag_set(st, RRDSET_FLAG_STORE_FIRST);
40 + else
41 + rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
42 + } else {
43 + rrdset_isnot_obsolete(st);
44 + rrdset_flag_clear(st, RRDSET_FLAG_DETAIL);
45 + rrdset_flag_clear(st, RRDSET_FLAG_STORE_FIRST);
46 + }
47 + ((PARSER_USER_OBJECT *)user)->st = st;
48 +
49 + if (chart_uuid) { /* It's a valid object */
50 + struct metalog_record record;
51 + struct metadata_logfile *metalogfile = state->metalogfile;
52 +
53 + uuid_copy(record.uuid, state->uuid);
54 + mlf_record_insert(metalogfile, &record);
55 + uuid_clear(state->uuid); /* Consume UUID */
56 + }
57 + return PARSER_RC_OK;
58 +}
59 +
60 +PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
61 + long multiplier, long divisor, char *options, RRD_ALGORITHM algorithm_type)
62 +{
63 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
64 + UNUSED(user);
65 + UNUSED(algorithm);
66 + uuid_t *dim_uuid;
67 +
68 + dim_uuid = uuid_is_null(state->uuid) ? NULL : &state->uuid;
69 +
70 + RRDDIM *rd = rrddim_add_custom(st, id, name, multiplier, divisor, algorithm_type, RRD_MEMORY_MODE_DBENGINE, 1,
71 + dim_uuid);
72 + rrddim_flag_clear(rd, RRDDIM_FLAG_HIDDEN);
73 + rrddim_flag_clear(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
74 + if (options && *options) {
75 + if (strstr(options, "obsolete") != NULL)
76 + rrddim_is_obsolete(st, rd);
77 + else
78 + rrddim_isnot_obsolete(st, rd);
79 + if (strstr(options, "hidden") != NULL)
80 + rrddim_flag_set(rd, RRDDIM_FLAG_HIDDEN);
81 + if (strstr(options, "noreset") != NULL)
82 + rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
83 + if (strstr(options, "nooverflow") != NULL)
84 + rrddim_flag_set(rd, RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS);
85 + } else {
86 + rrddim_isnot_obsolete(st, rd);
87 + }
88 + if (dim_uuid) { /* It's a valid object */
89 + struct metalog_record record;
90 + struct metadata_logfile *metalogfile = state->metalogfile;
91 +
92 + uuid_copy(record.uuid, state->uuid);
93 + mlf_record_insert(metalogfile, &record);
94 + uuid_clear(state->uuid); /* Consume UUID */
95 + }
96 + return PARSER_RC_OK;
97 +}
98 +
99 +PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid)
100 +{
101 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
102 +
103 + uuid_copy(state->uuid, *uuid);
104 +
105 + return PARSER_RC_OK;
106 +}
107 +
108 +PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid)
109 +{
110 + GUID_TYPE ret;
111 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
112 + struct metalog_instance *ctx = state->ctx;
113 + char object[49], chart_object[33], id_str[1024];
114 + uuid_t *chart_guid, *chart_char_guid;
115 + RRDHOST *host;
116 +
117 + ret = find_object_by_guid(uuid, object, 49);
118 + switch (ret) {
119 + case GUID_TYPE_CHAR:
120 + assert(0);
121 + break;
122 + case GUID_TYPE_CHART:
123 + case GUID_TYPE_DIMENSION:
124 + host = ctx->rrdeng_ctx->host;
125 + switch (ret) {
126 + case GUID_TYPE_CHART:
127 + chart_char_guid = (uuid_t *)(object + 16);
128 +
129 + ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
130 + assert(GUID_TYPE_CHAR == ret);
131 + ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
132 + break;
133 + case GUID_TYPE_DIMENSION:
134 + chart_guid = (uuid_t *)(object + 16);
135 +
136 + ret = find_object_by_guid(chart_guid, chart_object, 33);
137 + assert(GUID_TYPE_CHART == ret);
138 + chart_char_guid = (uuid_t *)(chart_object + 16);
139 +
140 + ret = find_object_by_guid(chart_char_guid, id_str, RRD_ID_LENGTH_MAX + 1);
141 + assert(GUID_TYPE_CHAR == ret);
142 + ((PARSER_USER_OBJECT *) user)->st = rrdset_find(host, id_str);
143 + break;
144 + default:
145 + assert(0);
146 + break;
147 + }
148 + break;
149 + case GUID_TYPE_HOST:
150 + /* Ignore for now */
151 + break;
152 + default:
153 + break;
154 + }
155 +
156 + return PARSER_RC_OK;
157 +}
158 +
159 +PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid)
160 +{
161 + GUID_TYPE ret;
162 + struct metalog_pluginsd_state *state = ((PARSER_USER_OBJECT *)user)->private;
163 + struct metalog_instance *ctx = state->ctx;
164 + RRDHOST *host = ctx->rrdeng_ctx->host;
165 + RRDSET *st;
166 + RRDDIM *rd;
167 +
168 + ret = find_object_by_guid(uuid, NULL, 0);
169 + switch (ret) {
170 + case GUID_TYPE_CHAR:
171 + assert(0);
172 + break;
173 + case GUID_TYPE_CHART:
174 + st = metalog_get_chart_from_uuid(ctx, uuid);
175 + if (st) {
176 + rrdhost_wrlock(host);
177 + rrdset_free(st);
178 + rrdhost_unlock(host);
179 + }
180 + break;
181 + case GUID_TYPE_DIMENSION:
182 + rd = metalog_get_dimension_from_uuid(ctx, uuid);
183 + if (rd) {
184 + st = rd->rrdset;
185 + rrdset_wrlock(st);
186 + rrddim_free_custom(st, rd, 0);
187 + rrdset_unlock(st);
188 + }
189 + break;
190 + case GUID_TYPE_HOST:
191 + /* Ignore for now */
192 + break;
193 + default:
194 + break;
195 + }
196 +
197 + return PARSER_RC_OK;
198 +}
199 +
200 +void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx)
201 +{
202 + state->ctx = ctx;
203 + state->skip_record = 0;
204 + uuid_clear(state->uuid);
205 + state->metalogfile = NULL;
206 +}
\ No newline at end of file
database/engine/metadata_log/metalogpluginsd.h new
+29
@@ -0,0 +1,29 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_METALOGPLUGINSD_H
4 +#define NETDATA_METALOGPLUGINSD_H
5 +
6 +#include "../../../collectors/plugins.d/pluginsd_parser.h"
7 +#include "../../../collectors/plugins.d/plugins_d.h"
8 +#include "../../../parser/parser.h"
9 +
10 +struct metalog_pluginsd_state {
11 + struct metalog_instance *ctx;
12 + uuid_t uuid;
13 + uint8_t skip_record; /* skip this record due to errors in parsing */
14 + struct metadata_logfile *metalogfile; /* current metadata log file being replayed */
15 +};
16 +
17 +extern void metalog_pluginsd_state_init(struct metalog_pluginsd_state *state, struct metalog_instance *ctx);
18 +
19 +extern PARSER_RC metalog_pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family,
20 + char *context, char *title, char *units, char *plugin, char *module,
21 + int priority, int update_every, RRDSET_TYPE chart_type, char *options);
22 +extern PARSER_RC metalog_pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm,
23 + long multiplier, long divisor, char *options,
24 + RRD_ALGORITHM algorithm_type);
25 +extern PARSER_RC metalog_pluginsd_guid_action(void *user, uuid_t *uuid);
26 +extern PARSER_RC metalog_pluginsd_context_action(void *user, uuid_t *uuid);
27 +extern PARSER_RC metalog_pluginsd_tombstone_action(void *user, uuid_t *uuid);
28 +
29 +#endif /* NETDATA_METALOGPLUGINSD_H */
database/engine/pagecache.c
+27 -5
@@ -383,17 +383,26 @@ static int pg_cache_try_evict_one_page_unsafe(struct rrdengine_instance *ctx)
383 return 0;
384 }
385
386 -/*
387 - * Callers of this function need to make sure they're not deleting the same descriptor concurrently
386 +/**
387 + * Deletes a page from the database.
388 + * Callers of this function need to make sure they're not deleting the same descriptor concurrently.
389 + * @param ctx is the database instance.
390 + * @param descr is the page descriptor.
391 + * @param remove_dirty must be non-zero if the page to be deleted is dirty.
392 + * @param is_exclusive_holder must be non-zero if the caller holds an exclusive page reference.
393 + * @param metric_id is set to the metric the page belongs to, if it's safe to delete the metric and metric_id is not
394 + * NULL. Otherwise, metric_id is not set.
395 + * @return 1 if it's safe to delete the metric, 0 otherwise.
396 */
389 -void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
390 - uint8_t is_exclusive_holder)
397 +uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
398 + uint8_t is_exclusive_holder, uuid_t *metric_id)
399 {
400 struct page_cache *pg_cache = &ctx->pg_cache;
401 struct page_cache_descr *pg_cache_descr = NULL;
402 Pvoid_t *PValue;
403 struct pg_cache_page_index *page_index;
404 int ret;
405 + uint8_t can_delete_metric = 0;
406
407 uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
408 PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, descr->id, sizeof(uuid_t));
@@ -403,14 +412,22 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
412
413 uv_rwlock_wrlock(&page_index->lock);
414 ret = JudyLDel(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
406 - uv_rwlock_wrunlock(&page_index->lock);
415 if (unlikely(0 == ret)) {
416 + uv_rwlock_wrunlock(&page_index->lock);
417 error("Page under deletion was not in index.");
418 if (unlikely(debug_flags & D_RRDENGINE)) {
419 print_page_descr(descr);
420 }
421 goto destroy;
422 }
423 + --page_index->page_count;
424 + if (!page_index->writers && !page_index->page_count) {
425 + can_delete_metric = 1;
426 + if (metric_id) {
427 + memcpy(metric_id, page_index->id, sizeof(uuid_t));
428 + }
429 + }
430 + uv_rwlock_wrunlock(&page_index->lock);
431 assert(1 == ret);
432
433 uv_rwlock_wrlock(&pg_cache->pg_cache_rwlock);
@@ -459,6 +476,8 @@ void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_desc
476 destroy:
477 freez(descr);
478 pg_cache_update_metric_times(page_index);
479 +
480 + return can_delete_metric;
481 }
482
483 static inline int is_page_in_time_range(struct rrdeng_page_descr *descr, usec_t start_time, usec_t end_time)
@@ -588,6 +607,7 @@ void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index
607 uv_rwlock_wrlock(&page_index->lock);
608 PValue = JudyLIns(&page_index->JudyL_array, (Word_t)(descr->start_time / USEC_PER_SEC), PJE0);
609 *PValue = descr;
610 + ++page_index->page_count;
611 pg_cache_add_new_metric_time(page_index, descr);
612 uv_rwlock_wrunlock(&page_index->lock);
613
@@ -1032,6 +1052,8 @@ struct pg_cache_page_index *create_page_index(uuid_t *id)
1052 page_index->oldest_time = INVALID_TIME;
1053 page_index->latest_time = INVALID_TIME;
1054 page_index->prev = NULL;
1055 + page_index->page_count = 0;
1056 + page_index->writers = 0;
1057
1058 return page_index;
1059 }
database/engine/pagecache.h
+4 -2
@@ -85,6 +85,8 @@ struct pg_cache_page_index {
85 * TODO: examine if we want to support better granularity than seconds
86 */
87 Pvoid_t JudyL_array;
88 + Word_t page_count;
89 + unsigned short writers;
90 uv_rwlock_t lock;
91
92 /*
@@ -163,8 +165,8 @@ extern void pg_cache_put_unsafe(struct rrdeng_page_descr *descr);
165 extern void pg_cache_put(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr);
166 extern void pg_cache_insert(struct rrdengine_instance *ctx, struct pg_cache_page_index *index,
167 struct rrdeng_page_descr *descr);
166 -extern void pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr, uint8_t remove_dirty,
167 - uint8_t is_exclusive_holder);
168 +extern uint8_t pg_cache_punch_hole(struct rrdengine_instance *ctx, struct rrdeng_page_descr *descr,
169 + uint8_t remove_dirty, uint8_t is_exclusive_holder, uuid_t *metric_id);
170 extern usec_t pg_cache_oldest_time_in_range(struct rrdengine_instance *ctx, uuid_t *id,
171 usec_t start_time, usec_t end_time);
172 extern void pg_cache_get_filtered_info_prev(struct rrdengine_instance *ctx, struct pg_cache_page_index *page_index,
database/engine/rrdengine.c
+38 -7
@@ -304,7 +304,7 @@ static void invalidate_oldest_committed(void *arg)
304
305 goto out;
306 }
307 - pg_cache_punch_hole(ctx, descr, 1, 1);
307 + pg_cache_punch_hole(ctx, descr, 1, 1, NULL);
308
309 uv_rwlock_wrlock(&pg_cache->committed_page_index.lock);
310 nr_committed_pages = --pg_cache->committed_page_index.nr_committed_pages;
@@ -326,6 +326,9 @@ void rrdeng_invalidate_oldest_committed(struct rrdengine_worker_config* wc)
326 unsigned nr_committed_pages;
327 int error;
328
329 + if (unlikely(ctx->quiesce != NO_QUIESCE)) /* Shutting down */
330 + return;
331 +
332 uv_rwlock_rdlock(&pg_cache->committed_page_index.lock);
333 nr_committed_pages = pg_cache->committed_page_index.nr_committed_pages;
334 uv_rwlock_rdunlock(&pg_cache->committed_page_index.lock);
@@ -630,6 +633,8 @@ static void delete_old_data(void *arg)
633 struct extent_info *extent, *next;
634 struct rrdeng_page_descr *descr;
635 unsigned count, i;
636 + uint8_t can_delete_metric;
637 + uuid_t metric_id;
638
639 /* Safe to use since it will be deleted after we are done */
640 datafile = ctx->datafiles.first;
@@ -638,7 +643,14 @@ static void delete_old_data(void *arg)
643 count = extent->number_of_pages;
644 for (i = 0 ; i < count ; ++i) {
645 descr = extent->pages[i];
641 - pg_cache_punch_hole(ctx, descr, 0, 0);
646 + can_delete_metric = pg_cache_punch_hole(ctx, descr, 0, 0, &metric_id);
647 + if (unlikely(can_delete_metric && ctx->metalog_ctx)) {
648 + /*
649 + * If the metric is empty, has no active writers and if the metadata log has been initialized then
650 + * attempt to delete the corresponding netdata dimension.
651 + */
652 + metalog_delete_dimension_by_uuid(ctx->metalog_ctx, &metric_id);
653 + }
654 }
655 next = extent->next;
656 freez(extent);
@@ -674,7 +686,7 @@ void rrdeng_test_quota(struct rrdengine_worker_config* wc)
686 ++ctx->last_fileno;
687 }
688 }
677 - if (unlikely(out_of_space)) {
689 + if (unlikely(out_of_space && NO_QUIESCE == ctx->quiesce)) {
690 /* delete old data */
691 if (wc->now_deleting_files) {
692 /* already deleting data */
@@ -710,12 +722,18 @@ static inline int rrdeng_threads_alive(struct rrdengine_worker_config* wc)
722
723 static void rrdeng_cleanup_finished_threads(struct rrdengine_worker_config* wc)
724 {
725 + struct rrdengine_instance *ctx = wc->ctx;
726 +
727 if (unlikely(wc->cleanup_thread_invalidating_dirty_pages)) {
728 after_invalidate_oldest_committed(wc);
729 }
730 if (unlikely(wc->cleanup_thread_deleting_files)) {
731 after_delete_old_data(wc);
732 }
733 + if (unlikely(SET_QUIESCE == ctx->quiesce && !rrdeng_threads_alive(wc))) {
734 + ctx->quiesce = QUIESCED;
735 + complete(&ctx->rrdengine_completion);
736 + }
737 }
738
739 /* return 0 on success */
@@ -799,14 +817,16 @@ void async_cb(uv_async_t *handle)
817 void timer_cb(uv_timer_t* handle)
818 {
819 struct rrdengine_worker_config* wc = handle->data;
820 + struct rrdengine_instance *ctx = wc->ctx;
821
822 uv_stop(handle->loop);
823 uv_update_time(handle->loop);
824 + if (unlikely(!ctx->metalog_ctx))
825 + return; /* Wait for the metadata log to initialize */
826 rrdeng_test_quota(wc);
827 debug(D_RRDENGINE, "%s: timeout reached.", __func__);
828 if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
829 /* There is free space so we can write to disk and we are not actively deleting dirty buffers */
809 - struct rrdengine_instance *ctx = wc->ctx;
830 struct page_cache *pg_cache = &ctx->pg_cache;
831 unsigned long total_bytes, bytes_written, nr_committed_pages, bytes_to_write = 0, producers, low_watermark,
832 high_watermark;
@@ -920,7 +940,20 @@ void rrdeng_worker(void* arg)
940 break;
941 case RRDENG_SHUTDOWN:
942 shutdown = 1;
943 + break;
944 + case RRDENG_QUIESCE:
945 ctx->drop_metrics_under_page_cache_pressure = 0;
946 + ctx->quiesce = SET_QUIESCE;
947 + assert(0 == uv_timer_stop(&timer_req));
948 + uv_close((uv_handle_t *)&timer_req, NULL);
949 + while (do_flush_pages(wc, 1, NULL)) {
950 + ; /* Force flushing of all committed pages. */
951 + }
952 + wal_flush_transaction_buffer(wc);
953 + if (!rrdeng_threads_alive(wc)) {
954 + ctx->quiesce = QUIESCED;
955 + complete(&ctx->rrdengine_completion);
956 + }
957 break;
958 case RRDENG_READ_PAGE:
959 do_read_extent(wc, &cmd.read_page.page_cache_descr, 1, 0);
@@ -959,8 +992,6 @@ void rrdeng_worker(void* arg)
992 * an issue in the future.
993 */
994 uv_close((uv_handle_t *)&wc->async, NULL);
962 - assert(0 == uv_timer_stop(&timer_req));
963 - uv_close((uv_handle_t *)&timer_req, NULL);
995
996 while (do_flush_pages(wc, 1, NULL)) {
997 ; /* Force flushing of all committed pages. */
@@ -998,7 +1029,7 @@ void rrdengine_main(void)
1029 struct rrdengine_instance *ctx;
1030
1031 sanity_check();
1001 - ret = rrdeng_init(&ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
1032 + ret = rrdeng_init(NULL, &ctx, "/tmp", RRDENG_MIN_PAGE_CACHE_SIZE_MB, RRDENG_MIN_DISK_SPACE_MB);
1033 if (ret) {
1034 exit(ret);
1035 }
database/engine/rrdengine.h
+10
@@ -17,6 +17,7 @@
17 #include "rrdenginelib.h"
18 #include "datafile.h"
19 #include "journalfile.h"
20 +#include "metadata_log/metadatalog.h"
21 #include "rrdengineapi.h"
22 #include "pagecache.h"
23 #include "rrdenglocking.h"
@@ -50,6 +51,7 @@ enum rrdeng_opcode {
51 RRDENG_FLUSH_PAGES,
52 RRDENG_SHUTDOWN,
53 RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE,
54 + RRDENG_QUIESCE,
55
56 RRDENG_MAX_OPCODE
57 };
@@ -169,7 +171,13 @@ extern rrdeng_stats_t rrdeng_reserved_file_descriptors;
171 extern rrdeng_stats_t global_pg_cache_over_half_dirty_events;
172 extern rrdeng_stats_t global_flushing_pressure_page_deletions; /* number of deleted pages */
173
174 +#define NO_QUIESCE (0) /* initial state when all operations function normally */
175 +#define SET_QUIESCE (1) /* set it before shutting down the instance, quiesce long running operations */
176 +#define QUIESCED (2) /* is set after all threads have finished running */
177 +
178 struct rrdengine_instance {
179 + RRDHOST *host;
180 + struct metalog_instance *metalog_ctx;
181 struct rrdengine_worker_config worker_config;
182 struct completion rrdengine_completion;
183 struct page_cache pg_cache;
@@ -185,6 +193,8 @@ struct rrdengine_instance {
193 unsigned long cache_pages_low_watermark;
194 unsigned long metric_API_max_producers;
195
196 + uint8_t quiesce; /* set to SET_QUIESCE before shutdown of the engine */
197 +
198 struct rrdengine_statistics stats;
199 };
200
database/engine/rrdengineapi.c
+161 -47
@@ -9,60 +9,136 @@ int default_rrdeng_disk_quota_mb = 256;
9 /* Default behaviour is to unblock data collection if the page cache is full of dirty pages by dropping metrics */
10 uint8_t rrdeng_drop_metrics_under_page_cache_pressure = 1;
11
12 -/*
13 - * Gets a handle for storing metrics to the database.
14 - * The handle must be released with rrdeng_store_metric_final().
15 - */
16 -void rrdeng_store_metric_init(RRDDIM *rd)
12 +/* This UUID is not unique across hosts */
13 +void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid)
14 {
18 - struct rrdeng_collect_handle *handle;
19 - struct page_cache *pg_cache;
20 - struct rrdengine_instance *ctx;
21 - uuid_t temp_id;
22 - Pvoid_t *PValue;
23 - struct pg_cache_page_index *page_index;
15 EVP_MD_CTX *evpctx;
16 unsigned char hash_value[EVP_MAX_MD_SIZE];
17 unsigned int hash_len;
18
28 - //&default_global_ctx; TODO: test this use case or remove it?
19 + evpctx = EVP_MD_CTX_create();
20 + EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
21 + EVP_DigestUpdate(evpctx, dim_id, strlen(dim_id));
22 + EVP_DigestUpdate(evpctx, chart_id, strlen(chart_id));
23 + EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
24 + EVP_MD_CTX_destroy(evpctx);
25 + assert(hash_len > sizeof(uuid_t));
26 + memcpy(ret_uuid, hash_value, sizeof(uuid_t));
27 +}
28
30 - ctx = rd->rrdset->rrdhost->rrdeng_ctx;
31 - pg_cache = &ctx->pg_cache;
32 - handle = &rd->state->handle.rrdeng;
33 - handle->ctx = ctx;
29 +/* Transform legacy UUID to be unique across hosts deterministacally */
30 +void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid, uuid_t *ret_uuid)
31 +{
32 + EVP_MD_CTX *evpctx;
33 + unsigned char hash_value[EVP_MAX_MD_SIZE];
34 + unsigned int hash_len;
35
36 evpctx = EVP_MD_CTX_create();
37 EVP_DigestInit_ex(evpctx, EVP_sha256(), NULL);
37 - EVP_DigestUpdate(evpctx, rd->id, strlen(rd->id));
38 - EVP_DigestUpdate(evpctx, rd->rrdset->id, strlen(rd->rrdset->id));
38 + EVP_DigestUpdate(evpctx, machine_guid, GUID_LEN);
39 + EVP_DigestUpdate(evpctx, *legacy_uuid, sizeof(uuid_t));
40 EVP_DigestFinal_ex(evpctx, hash_value, &hash_len);
41 EVP_MD_CTX_destroy(evpctx);
41 - assert(hash_len > sizeof(temp_id));
42 - memcpy(&temp_id, hash_value, sizeof(temp_id));
42 + assert(hash_len > sizeof(uuid_t));
43 + memcpy(ret_uuid, hash_value, sizeof(uuid_t));
44 +}
45
44 - handle->descr = NULL;
45 - handle->prev_descr = NULL;
46 - handle->unaligned_page = 0;
46 +void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid)
47 +{
48 + struct page_cache *pg_cache;
49 + struct rrdengine_instance *ctx;
50 + uuid_t legacy_uuid;
51 + Pvoid_t *PValue;
52 + struct pg_cache_page_index *page_index;
53 + int replace_instead_of_generate = 0;
54 +
55 + ctx = rd->rrdset->rrdhost->rrdeng_ctx;
56 + pg_cache = &ctx->pg_cache;
57 +
58 + rrdeng_generate_legacy_uuid(rd->id, rd->rrdset->id, &legacy_uuid);
59 + rd->state->metric_uuid = callocz(1, sizeof(uuid_t));
60
61 uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
49 - PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t));
62 + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, &legacy_uuid, sizeof(uuid_t));
63 if (likely(NULL != PValue)) {
64 page_index = *PValue;
65 }
66 uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
67 if (NULL == PValue) {
55 - /* First time we see the UUID */
56 - uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
57 - PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, &temp_id, sizeof(uuid_t), PJE0);
58 - assert(NULL == *PValue); /* TODO: figure out concurrency model */
59 - *PValue = page_index = create_page_index(&temp_id);
60 - page_index->prev = pg_cache->metrics_index.last_page_index;
61 - pg_cache->metrics_index.last_page_index = page_index;
62 - uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
68 + /* First time we see the legacy UUID, drop legacy support, normal path */
69 +
70 + if (NULL != dim_uuid) {
71 + replace_instead_of_generate = 1;
72 + uuid_copy(*rd->state->metric_uuid, *dim_uuid);
73 + }
74 + if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION,
75 + replace_instead_of_generate))) {
76 + errno = 0;
77 + error("FAILED to generate GUID for %s", rd->id);
78 + freez(rd->state->metric_uuid);
79 + rd->state->metric_uuid = NULL;
80 + assert(0);
81 + }
82 +
83 + uv_rwlock_rdlock(&pg_cache->metrics_index.lock);
84 + PValue = JudyHSGet(pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t));
85 + if (likely(NULL != PValue)) {
86 + page_index = *PValue;
87 + }
88 + uv_rwlock_rdunlock(&pg_cache->metrics_index.lock);
89 + if (NULL == PValue) {
90 + uv_rwlock_wrlock(&pg_cache->metrics_index.lock);
91 + PValue = JudyHSIns(&pg_cache->metrics_index.JudyHS_array, rd->state->metric_uuid, sizeof(uuid_t), PJE0);
92 + assert(NULL == *PValue); /* TODO: figure out concurrency model */
93 + *PValue = page_index = create_page_index(rd->state->metric_uuid);
94 + page_index->prev = pg_cache->metrics_index.last_page_index;
95 + pg_cache->metrics_index.last_page_index = page_index;
96 + uv_rwlock_wrunlock(&pg_cache->metrics_index.lock);
97 + }
98 + } else {
99 + /* There are legacy UUIDs in the database, implement backward compatibility */
100 +
101 +
102 + rrdeng_convert_legacy_uuid_to_multihost(rd->rrdset->rrdhost->machine_guid, &legacy_uuid,
103 + rd->state->metric_uuid);
104 + if (dim_uuid && uuid_compare(*rd->state->metric_uuid, *dim_uuid)) {
105 + error("Mismatch of metadata log DIMENSION GUID with dbengine metric GUID.");
106 + }
107 + if (unlikely(find_or_generate_guid(rd, rd->state->metric_uuid, GUID_TYPE_DIMENSION, 1))) {
108 + errno = 0;
109 + error("FAILED to generate GUID for %s", rd->id);
110 + freez(rd->state->metric_uuid);
111 + rd->state->metric_uuid = NULL;
112 + assert(0);
113 + }
114 }
115 rd->state->rrdeng_uuid = &page_index->id;
65 - handle->page_index = page_index;
116 + rd->state->page_index = page_index;
117 + rd->state->compaction_id = 0;
118 +}
119 +
120 +/*
121 + * Gets a handle for storing metrics to the database.
122 + * The handle must be released with rrdeng_store_metric_final().
123 + */
124 +void rrdeng_store_metric_init(RRDDIM *rd)
125 +{
126 + struct rrdeng_collect_handle *handle;
127 + struct rrdengine_instance *ctx;
128 + struct pg_cache_page_index *page_index;
129 +
130 + ctx = rd->rrdset->rrdhost->rrdeng_ctx;
131 + handle = &rd->state->handle.rrdeng;
132 + handle->ctx = ctx;
133 +
134 + handle->descr = NULL;
135 + handle->prev_descr = NULL;
136 + handle->unaligned_page = 0;
137 +
138 + page_index = rd->state->page_index;
139 + uv_rwlock_wrlock(&page_index->lock);
140 + ++page_index->writers;
141 + uv_rwlock_wrunlock(&page_index->lock);
142 }
143
144 /* The page must be populated and referenced */
@@ -109,7 +185,7 @@ void rrdeng_store_metric_flush_current_page(RRDDIM *rd)
185 if (unlikely(debug_flags & D_RRDENGINE))
186 print_page_cache_descr(descr);
187 pg_cache_put(ctx, descr);
112 - pg_cache_punch_hole(ctx, descr, 1, 0);
188 + pg_cache_punch_hole(ctx, descr, 1, 0, NULL);
189 handle->prev_descr = NULL;
190 } else {
191 /* added 1 extra reference to keep 2 dirty pages pinned per metric, expected refcnt = 2 */
@@ -170,7 +246,7 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
246 must_flush_unaligned_page)) {
247 rrdeng_store_metric_flush_current_page(rd);
248
173 - page = rrdeng_create_page(ctx, &handle->page_index->id, &descr);
249 + page = rrdeng_create_page(ctx, &rd->state->page_index->id, &descr);
250 assert(page);
251
252 handle->descr = descr;
@@ -204,27 +280,38 @@ void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number n
280 }
281 }
282
207 - pg_cache_insert(ctx, handle->page_index, descr);
283 + pg_cache_insert(ctx, rd->state->page_index, descr);
284 } else {
209 - pg_cache_add_new_metric_time(handle->page_index, descr);
285 + pg_cache_add_new_metric_time(rd->state->page_index, descr);
286 }
287 }
288
289 /*
290 * Releases the database reference from the handle for storing metrics.
291 + * Returns 1 if it's safe to delete the dimension.
292 */
216 -void rrdeng_store_metric_finalize(RRDDIM *rd)
293 +int rrdeng_store_metric_finalize(RRDDIM *rd)
294 {
295 struct rrdeng_collect_handle *handle;
296 struct rrdengine_instance *ctx;
297 + struct pg_cache_page_index *page_index;
298 + uint8_t can_delete_metric = 0;
299
300 handle = &rd->state->handle.rrdeng;
301 ctx = handle->ctx;
302 + page_index = rd->state->page_index;
303 rrdeng_store_metric_flush_current_page(rd);
304 if (handle->prev_descr) {
305 /* unpin old second page */
306 pg_cache_put(ctx, handle->prev_descr);
307 }
308 + uv_rwlock_wrlock(&page_index->lock);
309 + if (!--page_index->writers && !page_index->page_count) {
310 + can_delete_metric = 1;
311 + }
312 + uv_rwlock_wrunlock(&page_index->lock);
313 +
314 + return can_delete_metric;
315 }
316
317 /* Returns 1 if the data collection interval is well defined, 0 otherwise */
@@ -577,21 +664,17 @@ void rrdeng_load_metric_finalize(struct rrddim_query_handle *rrdimm_handle)
664
665 time_t rrdeng_metric_latest_time(RRDDIM *rd)
666 {
580 - struct rrdeng_collect_handle *handle;
667 struct pg_cache_page_index *page_index;
668
583 - handle = &rd->state->handle.rrdeng;
584 - page_index = handle->page_index;
669 + page_index = rd->state->page_index;
670
671 return page_index->latest_time / USEC_PER_SEC;
672 }
673 time_t rrdeng_metric_oldest_time(RRDDIM *rd)
674 {
590 - struct rrdeng_collect_handle *handle;
675 struct pg_cache_page_index *page_index;
676
593 - handle = &rd->state->handle.rrdeng;
594 - page_index = handle->page_index;
677 + page_index = rd->state->page_index;
678
679 return page_index->oldest_time / USEC_PER_SEC;
680 }
@@ -765,7 +848,8 @@ void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle)
848 /*
849 * Returns 0 on success, negative on error
850 */
768 -int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb, unsigned disk_space_mb)
851 +int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
852 + unsigned disk_space_mb)
853 {
854 struct rrdengine_instance *ctx;
855 int error;
@@ -776,8 +860,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
860 /* reserve RRDENG_FD_BUDGET_PER_INSTANCE file descriptors for this instance */
861 rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, RRDENG_FD_BUDGET_PER_INSTANCE);
862 if (rrdeng_reserved_file_descriptors > max_open_files) {
779 - error("Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
780 - (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
863 + error(
864 + "Exceeded the budget of available file descriptors (%u/%u), cannot create new dbengine instance.",
865 + (unsigned)rrdeng_reserved_file_descriptors, (unsigned)max_open_files);
866
867 rrd_stat_atomic_add(&global_fs_errors, 1);
868 rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
@@ -804,6 +889,9 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
889 ctx->dbfiles_path[sizeof(ctx->dbfiles_path) - 1] = '\0';
890 ctx->drop_metrics_under_page_cache_pressure = rrdeng_drop_metrics_under_page_cache_pressure;
891 ctx->metric_API_max_producers = 0;
892 + ctx->quiesce = NO_QUIESCE;
893 + ctx->metalog_ctx = NULL; /* only set this after the metadata log has finished initializing */
894 + ctx->host = host;
895
896 memset(&ctx->worker_config, 0, sizeof(ctx->worker_config));
897 ctx->worker_config.ctx = ctx;
@@ -823,6 +911,11 @@ int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned p
911 if (ctx->worker_config.error) {
912 goto error_after_rrdeng_worker;
913 }
914 + error = metalog_init(ctx);
915 + if(error) {
916 + error("Failed to initialize metadata log file event loop.");
917 + goto error_after_rrdeng_worker;
918 + }
919 return 0;
920
921 error_after_rrdeng_worker:
@@ -855,6 +948,7 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
948 assert(0 == uv_thread_join(&ctx->worker_config.thread));
949
950 finalize_rrd_files(ctx);
951 + metalog_exit(ctx->metalog_ctx);
952 free_page_cache(ctx);
953
954 if (ctx != &default_global_ctx) {
@@ -863,3 +957,23 @@ int rrdeng_exit(struct rrdengine_instance *ctx)
957 rrd_stat_atomic_add(&rrdeng_reserved_file_descriptors, -RRDENG_FD_BUDGET_PER_INSTANCE);
958 return 0;
959 }
960 +
961 +void rrdeng_prepare_exit(struct rrdengine_instance *ctx)
962 +{
963 + struct rrdeng_cmd cmd;
964 +
965 + if (NULL == ctx) {
966 + return;
967 + }
968 +
969 + init_completion(&ctx->rrdengine_completion);
970 + cmd.opcode = RRDENG_QUIESCE;
971 + rrdeng_enq_cmd(&ctx->worker_config, &cmd);
972 +
973 + /* wait for dbengine to quiesce */
974 + wait_for_completion(&ctx->rrdengine_completion);
975 + destroy_completion(&ctx->rrdengine_completion);
976 +
977 + metalog_prepare_exit(ctx->metalog_ctx);
978 +}
979 +
database/engine/rrdengineapi.h
+10 -2
@@ -28,10 +28,17 @@ extern void rrdeng_commit_page(struct rrdengine_instance *ctx, struct rrdeng_pag
28 extern void *rrdeng_get_latest_page(struct rrdengine_instance *ctx, uuid_t *id, void **handle);
29 extern void *rrdeng_get_page(struct rrdengine_instance *ctx, uuid_t *id, usec_t point_in_time, void **handle);
30 extern void rrdeng_put_page(struct rrdengine_instance *ctx, void *handle);
31 +
32 +extern void rrdeng_generate_legacy_uuid(const char *dim_id, char *chart_id, uuid_t *ret_uuid);
33 +extern void rrdeng_convert_legacy_uuid_to_multihost(char machine_guid[GUID_LEN + 1], uuid_t *legacy_uuid,
34 + uuid_t *ret_uuid);
35 +
36 +
37 +extern void rrdeng_metric_init(RRDDIM *rd, uuid_t *dim_uuid);
38 extern void rrdeng_store_metric_init(RRDDIM *rd);
39 extern void rrdeng_store_metric_flush_current_page(RRDDIM *rd);
40 extern void rrdeng_store_metric_next(RRDDIM *rd, usec_t point_in_time, storage_number number);
34 -extern void rrdeng_store_metric_finalize(RRDDIM *rd);
41 +extern int rrdeng_store_metric_finalize(RRDDIM *rd);
42 extern unsigned
43 rrdeng_variable_step_boundaries(RRDSET *st, time_t start_time, time_t end_time,
44 struct rrdeng_region_info **region_info_arrayp, unsigned *max_intervalp);
@@ -45,9 +52,10 @@ extern time_t rrdeng_metric_oldest_time(RRDDIM *rd);
52 extern void rrdeng_get_37_statistics(struct rrdengine_instance *ctx, unsigned long long *array);
53
54 /* must call once before using anything */
48 -extern int rrdeng_init(struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
55 +extern int rrdeng_init(RRDHOST *host, struct rrdengine_instance **ctxp, char *dbfiles_path, unsigned page_cache_mb,
56 unsigned disk_space_mb);
57
58 extern int rrdeng_exit(struct rrdengine_instance *ctx);
59 +extern void rrdeng_prepare_exit(struct rrdengine_instance *ctx);
60
61 #endif /* NETDATA_RRDENGINEAPI_H */
\ No newline at end of file
database/engine/rrdenginelib.c
+12 -7
@@ -78,17 +78,22 @@ int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size)
78 return 0;
79 }
80
81 -/*
82 - * Tries to open a file in direct I/O mode, falls back to buffered mode if not possible.
83 - * Returns UV error number that is < 0 on failure.
84 - * On success sets (*file) to be the uv_file that was opened.
81 +/**
82 + * Open file for I/O.
83 + *
84 + * @param path The full path of the file.
85 + * @param flags Same flags as the open() system call uses.
86 + * @param file On success sets (*file) to be the uv_file that was opened.
87 + * @param direct Tries to open a file in direct I/O mode when direct=1, falls back to buffered mode if not possible.
88 + * @return Returns UV error number that is < 0 on failure. 0 on success.
89 */
86 -int open_file_direct_io(char *path, int flags, uv_file *file)
90 +int open_file_for_io(char *path, int flags, uv_file *file, int direct)
91 {
92 uv_fs_t req;
89 - int fd, current_flags, direct;
93 + int fd, current_flags;
94
91 - for (direct = 1 ; direct >= 0 ; --direct) {
95 + assert(0 == direct || 1 == direct);
96 + for ( ; direct >= 0 ; --direct) {
97 #ifdef __APPLE__
98 /* Apple OS does not support O_DIRECT */
99 direct = 0;
database/engine/rrdenginelib.h
+9 -1
@@ -100,7 +100,15 @@ static inline void crc32set(void *crcp, uLong crc)
100 extern void print_page_cache_descr(struct rrdeng_page_descr *page_cache_descr);
101 extern void print_page_descr(struct rrdeng_page_descr *descr);
102 extern int check_file_properties(uv_file file, uint64_t *file_size, size_t min_size);
103 -extern int open_file_direct_io(char *path, int flags, uv_file *file);
103 +extern int open_file_for_io(char *path, int flags, uv_file *file, int direct);
104 +static inline int open_file_direct_io(char *path, int flags, uv_file *file)
105 +{
106 + return open_file_for_io(path, flags, file, 1);
107 +}
108 +static inline int open_file_buffered_io(char *path, int flags, uv_file *file)
109 +{
110 + return open_file_for_io(path, flags, file, 0);
111 +}
112 extern char *get_rrdeng_statistics(struct rrdengine_instance *ctx, char *str, size_t size);
113
114 #endif /* NETDATA_RRDENGINELIB_H */
\ No newline at end of file
database/rrd.h
+89 -14
@@ -16,6 +16,7 @@ typedef struct alarm_entry ALARM_ENTRY;
16
17 // forward declarations
18 struct rrddim_volatile;
19 +struct rrdset_volatile;
20 #ifdef ENABLE_DBENGINE
21 struct rrdeng_page_descr;
22 struct rrdengine_instance;
@@ -31,6 +32,11 @@ struct pg_cache_page_index;
32 #include "rrdcalctemplate.h"
33 #include "../streaming/rrdpush.h"
34
35 +#define META_CHART_UPDATED 1
36 +#define META_PLUGIN_UPDATED 2
37 +#define META_MODULE_UPDATED 4
38 +#define META_CHART_ACTIVATED 8
39 +
40 #define UPDATE_EVERY 1
41 #define UPDATE_EVERY_MAX 3600
42
@@ -136,7 +142,10 @@ typedef enum rrddim_flags {
142 RRDDIM_FLAG_NONE = 0,
143 RRDDIM_FLAG_HIDDEN = (1 << 0), // this dimension will not be offered to callers
144 RRDDIM_FLAG_DONT_DETECT_RESETS_OR_OVERFLOWS = (1 << 1), // do not offer RESET or OVERFLOW info to callers
139 - RRDDIM_FLAG_OBSOLETE = (1 << 2) // this is marked by the collector/module as obsolete
145 + RRDDIM_FLAG_OBSOLETE = (1 << 2), // this is marked by the collector/module as obsolete
146 + // No new values have been collected for this dimension since agent start or it was marked RRDDIM_FLAG_OBSOLETE at
147 + // least rrdset_free_obsolete_time seconds ago.
148 + RRDDIM_FLAG_ARCHIVED = (1 << 3)
149 } RRDDIM_FLAGS;
150
151 #ifdef HAVE_C___ATOMIC
@@ -274,7 +283,6 @@ union rrddim_collect_handle {
283 struct rrdeng_page_descr *descr, *prev_descr;
284 unsigned long page_correlation_id;
285 struct rrdengine_instance *ctx;
277 - struct pg_cache_page_index *page_index;
286 // set to 1 when this dimension is not page aligned with the other dimensions in the chart
287 uint8_t unaligned_page;
288 } rrdeng; // state the database engine uses
@@ -312,6 +320,9 @@ struct rrddim_query_handle {
320 struct rrddim_volatile {
321 #ifdef ENABLE_DBENGINE
322 uuid_t *rrdeng_uuid; // database engine metric UUID
323 + uuid_t *metric_uuid; // global UUID for this metric (unique_across hosts)
324 + struct pg_cache_page_index *page_index;
325 + uint32_t compaction_id; // The last metadata log compaction procedure that has processed this object.
326 #endif
327 union rrddim_collect_handle handle;
328 // ------------------------------------------------------------------------
@@ -324,7 +335,8 @@ struct rrddim_volatile {
335 void (*store_metric)(RRDDIM *rd, usec_t point_in_time, storage_number number);
336
337 // an finalization function to run after collection is over
327 - void (*finalize)(RRDDIM *rd);
338 + // returns 1 if it's safe to delete the dimension
339 + int (*finalize)(RRDDIM *rd);
340 } collect_ops;
341
342 // function pointers that handle database queries
@@ -349,6 +361,14 @@ struct rrddim_volatile {
361 } query_ops;
362 };
363
364 +// ----------------------------------------------------------------------------
365 +// volatile state per chart
366 +struct rrdset_volatile {
367 + char *old_title;
368 + char *old_family;
369 + char *old_context;
370 +};
371 +
372 // ----------------------------------------------------------------------------
373 // these loop macros make sure the linked list is accessed with the right lock
374
@@ -382,7 +402,10 @@ typedef enum rrdset_flags {
402 RRDSET_FLAG_HOMOGENEOUS_CHECK = 1 << 11, // if set, the chart should be checked to determine if the dimensions are homogeneous
403 RRDSET_FLAG_HIDDEN = 1 << 12, // if set, do not show this chart on the dashboard, but use it for backends
404 RRDSET_FLAG_SYNC_CLOCK = 1 << 13, // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
385 - RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14 // this is marked by the collector/module when a chart has obsolete dimensions
405 + RRDSET_FLAG_OBSOLETE_DIMENSIONS = 1 << 14, // this is marked by the collector/module when a chart has obsolete dimensions
406 + // No new values have been collected for this chart since agent start or it was marked RRDSET_FLAG_OBSOLETE at
407 + // least rrdset_free_obsolete_time seconds ago.
408 + RRDSET_FLAG_ARCHIVED = 1 << 15
409 } RRDSET_FLAGS;
410
411 #ifdef HAVE_C___ATOMIC
@@ -459,8 +482,11 @@ struct rrdset {
482
483 char *plugin_name; // the name of the plugin that generated this
484 char *module_name; // the name of the plugin module that generated this
462 -
463 - size_t unused[5];
485 + uuid_t *chart_uuid; // Store the global GUID for this chart
486 + size_t compaction_id; // The last metadata log compaction procedure that has processed
487 + // this object.
488 + struct rrdset_volatile *state; // volatile state that is not persistently stored
489 + size_t unused[2];
490
491 size_t rrddim_page_alignment; // keeps metric pages in alignment when using dbengine
492
@@ -783,6 +809,10 @@ struct rrdhost {
809
810 #ifdef ENABLE_DBENGINE
811 struct rrdengine_instance *rrdeng_ctx; // DB engine instance for this host
812 + uuid_t host_uuid; // Global GUID for this host
813 + unsigned long objects_nr; // Number of charts and dimensions in this host
814 + uint32_t compaction_id; // The last metadata log compaction procedure that has processed
815 + // this object.
816 #endif
817
818 #ifdef ENABLE_HTTPS
@@ -847,6 +877,26 @@ extern RRDHOST *rrdhost_find_or_create(
877 , struct rrdhost_system_info *system_info
878 );
879
880 +extern void rrdhost_update(RRDHOST *host
881 + , const char *hostname
882 + , const char *registry_hostname
883 + , const char *guid
884 + , const char *os
885 + , const char *timezone
886 + , const char *tags
887 + , const char *program_name
888 + , const char *program_version
889 + , int update_every
890 + , long history
891 + , RRD_MEMORY_MODE mode
892 + , unsigned int health_enabled
893 + , unsigned int rrdpush_enabled
894 + , char *rrdpush_destination
895 + , char *rrdpush_api_key
896 + , char *rrdpush_send_charts_matching
897 + , struct rrdhost_system_info *system_info
898 +);
899 +
900 extern int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, char *name, char *value);
901
902 #if defined(NETDATA_INTERNAL_CHECKS) && defined(NETDATA_VERIFY_LOCKS)
@@ -892,10 +942,12 @@ extern RRDSET *rrdset_create_custom(RRDHOST *host
942 , int update_every
943 , RRDSET_TYPE chart_type
944 , RRD_MEMORY_MODE memory_mode
895 - , long history_entries);
945 + , long history_entries
946 + , int is_archived
947 + , uuid_t *chart_uuid);
948
949 #define rrdset_create(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
898 - rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries)
950 + rrdset_create_custom(host, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type, (host)->rrd_memory_mode, (host)->rrd_history_entries, 0, NULL)
951
952 #define rrdset_create_localhost(type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type) \
953 rrdset_create(localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type)
@@ -919,6 +971,14 @@ extern RRDSET *rrdset_find(RRDHOST *host, const char *id);
971
972 extern RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);
973 #define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)
974 +/* This will not return charts that are archived */
975 +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)
976 +{
977 + RRDSET *st = rrdset_find_bytype_localhost(type, id);
978 + if (unlikely(st && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)))
979 + return NULL;
980 + return st;
981 +}
982
983 extern RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);
984 #define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)
@@ -933,8 +993,9 @@ extern void rrdset_is_obsolete(RRDSET *st);
993 extern void rrdset_isnot_obsolete(RRDSET *st);
994
995 // checks if the RRDSET should be offered to viewers
936 -#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
937 -#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && (st)->dimensions)
996 +#define rrdset_is_available_for_viewers(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)
997 +#define rrdset_is_available_for_backends(st) (rrdset_flag_check(st, RRDSET_FLAG_ENABLED) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && !rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
998 +#define rrdset_is_archived(st) (rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED) && (st)->dimensions)
999
1000 // get the total duration in seconds of the round robin database
1001 #define rrdset_duration(st) ((time_t)( (((st)->counter >= ((unsigned long)(st)->entries))?(unsigned long)(st)->entries:(st)->counter) * (st)->update_every ))
@@ -1062,8 +1123,11 @@ static inline time_t rrdset_slot2time(RRDSET *st, size_t slot) {
1123 // RRD DIMENSION functions
1124
1125 extern void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host);
1065 -extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode);
1066 -#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, algorithm, (st)->rrd_memory_mode)
1126 +extern RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
1127 + collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
1128 + int is_archived, uuid_t *dim_uuid);
1129 +#define rrddim_add(st, id, name, multiplier, divisor, algorithm) rrddim_add_custom(st, id, name, multiplier, divisor, \
1130 + algorithm, (st)->rrd_memory_mode, 0, NULL)
1131
1132 extern int rrddim_set_name(RRDSET *st, RRDDIM *rd, const char *name);
1133 extern int rrddim_set_algorithm(RRDSET *st, RRDDIM *rd, RRD_ALGORITHM algorithm);
@@ -1071,6 +1135,15 @@ extern int rrddim_set_multiplier(RRDSET *st, RRDDIM *rd, collected_number multip
1135 extern int rrddim_set_divisor(RRDSET *st, RRDDIM *rd, collected_number divisor);
1136
1137 extern RRDDIM *rrddim_find(RRDSET *st, const char *id);
1138 +/* This will not return dimensions that are archived */
1139 +static inline RRDDIM *rrddim_find_active(RRDSET *st, const char *id)
1140 +{
1141 + RRDDIM *rd = rrddim_find(st, id);
1142 + if (unlikely(rd && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)))
1143 + return NULL;
1144 + return rd;
1145 +}
1146 +
1147
1148 extern int rrddim_hide(RRDSET *st, const char *id);
1149 extern int rrddim_unhide(RRDSET *st, const char *id);
@@ -1099,7 +1172,8 @@ extern avl_tree_lock rrdhost_root_index;
1172 extern char *rrdset_strncpyz_name(char *to, const char *from, size_t length);
1173 extern char *rrdset_cache_dir(RRDHOST *host, const char *id, const char *config_section);
1174
1102 -extern void rrddim_free(RRDSET *st, RRDDIM *rd);
1175 +#define rrddim_free(st, rd) rrddim_free_custom(st, rd, 0)
1176 +extern void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated);
1177
1178 extern int rrddim_compare(void* a, void* b);
1179 extern int rrdset_compare(void* a, void* b);
@@ -1116,7 +1190,8 @@ extern RRDSET *rrdset_index_del_name(RRDHOST *host, RRDSET *st);
1190 extern void rrdset_free(RRDSET *st);
1191 extern void rrdset_reset(RRDSET *st);
1192 extern void rrdset_save(RRDSET *st);
1119 -extern void rrdset_delete(RRDSET *st);
1193 +#define rrdset_delete(st) rrdset_delete_custom(st, 0)
1194 +extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
1195 extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
1196
1197 extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
database/rrddim.c
+59 -21
@@ -100,10 +100,10 @@ static void rrddim_collect_store_metric(RRDDIM *rd, usec_t point_in_time, storag
100
101 rd->values[rd->rrdset->current_entry] = number;
102 }
103 -static void rrddim_collect_finalize(RRDDIM *rd) {
103 +static int rrddim_collect_finalize(RRDDIM *rd) {
104 (void)rd;
105
106 - return;
106 + return 0;
107 }
108
109 // ----------------------------------------------------------------------------
@@ -189,7 +189,9 @@ void rrdcalc_link_to_rrddim(RRDDIM *rd, RRDSET *st, RRDHOST *host) {
189 #endif
190 }
191
192 -RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode) {
192 +RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collected_number multiplier,
193 + collected_number divisor, RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode,
194 + int is_archived, uuid_t *dim_uuid) {
195 RRDHOST *host = st->rrdhost;
196 rrdset_wrlock(st);
197
@@ -200,11 +202,21 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
202 if(unlikely(rd)) {
203 debug(D_RRD_CALLS, "Cannot create rrd dimension '%s/%s', it already exists.", st->id, name?name:"<NONAME>");
204
203 - rrddim_set_name(st, rd, name);
204 - rrddim_set_algorithm(st, rd, algorithm);
205 - rrddim_set_multiplier(st, rd, multiplier);
206 - rrddim_set_divisor(st, rd, divisor);
207 -
205 + int rc = rrddim_set_name(st, rd, name);
206 + rc += rrddim_set_algorithm(st, rd, algorithm);
207 + rc += rrddim_set_multiplier(st, rd, multiplier);
208 + rc += rrddim_set_divisor(st, rd, divisor);
209 + if (!is_archived && rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
210 + rd->state->collect_ops.init(rd);
211 + rrddim_flag_clear(rd, RRDDIM_FLAG_ARCHIVED);
212 + }
213 + // DBENGINE available and activated?
214 +#ifdef ENABLE_DBENGINE
215 + if (likely(!is_archived && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) && unlikely(rc)) {
216 + debug(D_METADATALOG, "DIMENSION [%s] metadata updated", rd->id);
217 + metalog_commit_update_dimension(rd);
218 + }
219 +#endif
220 rrdset_unlock(st);
221 return rd;
222 }
@@ -301,7 +313,6 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
313 else
314 rd->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
315 }
304 -
316 rd->memsize = size;
317
318 strcpy(rd->magic, RRDDIMENSION_MAGIC);
@@ -350,15 +361,16 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
361 rd->state = mallocz(sizeof(*rd->state));
362 if(memory_mode == RRD_MEMORY_MODE_DBENGINE) {
363 #ifdef ENABLE_DBENGINE
353 - rd->state->collect_ops.init = rrdeng_store_metric_init;
364 + rrdeng_metric_init(rd, dim_uuid);
365 + rd->state->collect_ops.init = rrdeng_store_metric_init;
366 rd->state->collect_ops.store_metric = rrdeng_store_metric_next;
355 - rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
356 - rd->state->query_ops.init = rrdeng_load_metric_init;
357 - rd->state->query_ops.next_metric = rrdeng_load_metric_next;
358 - rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
359 - rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
360 - rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
361 - rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
367 + rd->state->collect_ops.finalize = rrdeng_store_metric_finalize;
368 + rd->state->query_ops.init = rrdeng_load_metric_init;
369 + rd->state->query_ops.next_metric = rrdeng_load_metric_next;
370 + rd->state->query_ops.is_finished = rrdeng_load_metric_is_finished;
371 + rd->state->query_ops.finalize = rrdeng_load_metric_finalize;
372 + rd->state->query_ops.latest_time = rrdeng_metric_latest_time;
373 + rd->state->query_ops.oldest_time = rrdeng_metric_oldest_time;
374 #endif
375 } else {
376 rd->state->collect_ops.init = rrddim_collect_init;
@@ -371,7 +383,10 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
383 rd->state->query_ops.latest_time = rrddim_query_latest_time;
384 rd->state->query_ops.oldest_time = rrddim_query_oldest_time;
385 }
374 - rd->state->collect_ops.init(rd);
386 + if (is_archived)
387 + rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
388 + else
389 + rd->state->collect_ops.init(rd); // only initialize if a collector created this dimension
390 // append this dimension
391 if(!st->dimensions)
392 st->dimensions = rd;
@@ -432,17 +447,30 @@ RRDDIM *rrddim_add_custom(RRDSET *st, const char *id, const char *name, collecte
447 if (netdata_cloud_setting)
448 aclk_update_chart(host, st->id, ACLK_CMD_CHART);
449 #endif
450 +#ifdef ENABLE_DBENGINE
451 + rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
452 + metalog_commit_update_dimension(rd);
453 +#endif
454 +
455 return(rd);
456 }
457
458 // ----------------------------------------------------------------------------
459 // RRDDIM remove / free a dimension
460
441 -void rrddim_free(RRDSET *st, RRDDIM *rd)
461 +void rrddim_free_custom(RRDSET *st, RRDDIM *rd, int db_rotated)
462 {
463 debug(D_RRD_CALLS, "rrddim_free() %s.%s", st->name, rd->name);
464
445 - rd->state->collect_ops.finalize(rd);
465 + if (!rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
466 + uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
467 + if (can_delete_metric && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
468 +#ifdef ENABLE_DBENGINE
469 + /* This metric has no data and no references */
470 + metalog_commit_delete_dimension(rd);
471 +#endif
472 + }
473 + }
474 freez(rd->state);
475
476 if(rd == st->dimensions)
@@ -486,9 +514,12 @@ void rrddim_free(RRDSET *st, RRDDIM *rd)
514 break;
515 }
516 #ifdef ENABLE_ACLK
489 - if (netdata_cloud_setting)
517 + if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != rd->rrd_memory_mode))
518 aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
519 #endif
520 +#ifdef ENABLE_DBENGINE
521 + rrd_atomic_fetch_add(&st->rrdhost->objects_nr, -1);
522 +#endif
523 }
524
525
@@ -541,6 +572,10 @@ inline void rrddim_is_obsolete(RRDSET *st, RRDDIM *rd) {
572 if (netdata_cloud_setting)
573 aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
574 #endif
575 +#ifdef ENABLE_DBENGINE
576 + metalog_commit_update_dimension(rd);
577 +#endif
578 +
579 }
580
581 inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
@@ -551,6 +586,9 @@ inline void rrddim_isnot_obsolete(RRDSET *st __maybe_unused, RRDDIM *rd) {
586 if (netdata_cloud_setting)
587 aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHART);
588 #endif
589 +#ifdef ENABLE_DBENGINE
590 + metalog_commit_update_dimension(rd);
591 +#endif
592 }
593
594 // ----------------------------------------------------------------------------
database/rrdhost.c
+118 -39
@@ -245,6 +245,9 @@ RRDHOST *rrdhost_create(const char *hostname,
245 }
246 if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
247 #ifdef ENABLE_DBENGINE
248 + uuid_parse(host->machine_guid, host->host_uuid);
249 + host->objects_nr = 1;
250 + host->compaction_id = 0;
251 char dbenginepath[FILENAME_MAX + 1];
252 int ret;
253
@@ -253,7 +256,7 @@ RRDHOST *rrdhost_create(const char *hostname,
256 if(ret != 0 && errno != EEXIST)
257 error("Host '%s': cannot create directory '%s'", host->hostname, dbenginepath);
258 else
256 - ret = rrdeng_init(&host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
259 + ret = rrdeng_init(host, &host->rrdeng_ctx, dbenginepath, host->page_cache_mb, host->disk_space_mb);
260 if(ret) {
261 error("Host '%s': cannot initialize host with machine guid '%s'. Failed to initialize DB engine at '%s'.",
262 host->hostname, host->machine_guid, host->cache_dir);
@@ -361,9 +364,91 @@ RRDHOST *rrdhost_create(const char *hostname,
364
365 rrd_hosts_available++;
366
367 +#ifdef ENABLE_DBENGINE
368 + if (likely(!is_localhost && host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
369 + metalog_commit_update_host(host);
370 +#endif
371 return host;
372 }
373
374 +void rrdhost_update(RRDHOST *host
375 + , const char *hostname
376 + , const char *registry_hostname
377 + , const char *guid
378 + , const char *os
379 + , const char *timezone
380 + , const char *tags
381 + , const char *program_name
382 + , const char *program_version
383 + , int update_every
384 + , long history
385 + , RRD_MEMORY_MODE mode
386 + , unsigned int health_enabled
387 + , unsigned int rrdpush_enabled
388 + , char *rrdpush_destination
389 + , char *rrdpush_api_key
390 + , char *rrdpush_send_charts_matching
391 + , struct rrdhost_system_info *system_info
392 +)
393 +{
394 + UNUSED(guid);
395 + UNUSED(rrdpush_enabled);
396 + UNUSED(rrdpush_destination);
397 + UNUSED(rrdpush_api_key);
398 + UNUSED(rrdpush_send_charts_matching);
399 +
400 + host->health_enabled = health_enabled;
401 + //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
402 +
403 + rrdhost_system_info_free(host->system_info);
404 + host->system_info = system_info;
405 +
406 + rrdhost_init_os(host, os);
407 + rrdhost_init_timezone(host, timezone);
408 +
409 + freez(host->registry_hostname);
410 + host->registry_hostname = strdupz((registry_hostname && *registry_hostname)?registry_hostname:hostname);
411 +
412 + if(strcmp(host->hostname, hostname) != 0) {
413 + info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
414 + char *t = host->hostname;
415 + host->hostname = strdupz(hostname);
416 + host->hash_hostname = simple_hash(host->hostname);
417 + freez(t);
418 + }
419 +
420 + if(strcmp(host->program_name, program_name) != 0) {
421 + info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
422 + char *t = host->program_name;
423 + host->program_name = strdupz(program_name);
424 + freez(t);
425 + }
426 +
427 + if(strcmp(host->program_version, program_version) != 0) {
428 + info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
429 + char *t = host->program_version;
430 + host->program_version = strdupz(program_version);
431 + freez(t);
432 + }
433 +
434 + if(host->rrd_update_every != update_every)
435 + error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
436 +
437 + if(host->rrd_history_entries < history)
438 + error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
439 +
440 + if(host->rrd_memory_mode != mode)
441 + error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
442 +
443 + // update host tags
444 + rrdhost_init_tags(host, tags);
445 +#ifdef ENABLE_DBENGINE
446 + if (likely(host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE))
447 + metalog_commit_update_host(host);
448 +#endif
449 + return;
450 +}
451 +
452 RRDHOST *rrdhost_find_or_create(
453 const char *hostname
454 , const char *registry_hostname
@@ -410,42 +495,24 @@ RRDHOST *rrdhost_find_or_create(
495 );
496 }
497 else {
413 - host->health_enabled = health_enabled;
414 - //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
415 -
416 - if(strcmp(host->hostname, hostname) != 0) {
417 - info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
418 - char *t = host->hostname;
419 - host->hostname = strdupz(hostname);
420 - host->hash_hostname = simple_hash(host->hostname);
421 - freez(t);
422 - }
423 -
424 - if(strcmp(host->program_name, program_name) != 0) {
425 - info("Host '%s' switched program name from '%s' to '%s'", host->hostname, host->program_name, program_name);
426 - char *t = host->program_name;
427 - host->program_name = strdupz(program_name);
428 - freez(t);
429 - }
430 -
431 - if(strcmp(host->program_version, program_version) != 0) {
432 - info("Host '%s' switched program version from '%s' to '%s'", host->hostname, host->program_version, program_version);
433 - char *t = host->program_version;
434 - host->program_version = strdupz(program_version);
435 - freez(t);
436 - }
437 -
438 - if(host->rrd_update_every != update_every)
439 - error("Host '%s' has an update frequency of %d seconds, but the wanted one is %d seconds. Restart netdata here to apply the new settings.", host->hostname, host->rrd_update_every, update_every);
440 -
441 - if(host->rrd_history_entries < history)
442 - error("Host '%s' has history of %ld entries, but the wanted one is %ld entries. Restart netdata here to apply the new settings.", host->hostname, host->rrd_history_entries, history);
443 -
444 - if(host->rrd_memory_mode != mode)
445 - error("Host '%s' has memory mode '%s', but the wanted one is '%s'. Restart netdata here to apply the new settings.", host->hostname, rrd_memory_mode_name(host->rrd_memory_mode), rrd_memory_mode_name(mode));
446 -
447 - // update host tags
448 - rrdhost_init_tags(host, tags);
498 + rrdhost_update(host
499 + , hostname
500 + , registry_hostname
501 + , guid
502 + , os
503 + , timezone
504 + , tags
505 + , program_name
506 + , program_version
507 + , update_every
508 + , history
509 + , mode
510 + , health_enabled
511 + , rrdpush_enabled
512 + , rrdpush_destination
513 + , rrdpush_api_key
514 + , rrdpush_send_charts_matching
515 + , system_info);
516 }
517
518 rrdhost_cleanup_orphan_hosts_nolock(host);
@@ -454,7 +521,6 @@ RRDHOST *rrdhost_find_or_create(
521
522 return host;
523 }
457 -
524 inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t now) {
525 if(host != protected
526 && host != localhost
@@ -618,6 +684,9 @@ void rrdhost_free(RRDHOST *host) {
684 // ------------------------------------------------------------------------
685 // release its children resources
686
687 +#ifdef ENABLE_DBENGINE
688 + rrdeng_prepare_exit(host->rrdeng_ctx);
689 +#endif
690 while(host->rrdset_root)
691 rrdset_free(host->rrdset_root);
692
@@ -1322,7 +1391,17 @@ restart_after_removal:
1391 && st->last_updated.tv_sec + rrdset_free_obsolete_time < now
1392 && st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now
1393 )) {
1325 -
1394 +#ifdef ENABLE_DBENGINE
1395 + if(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1396 + rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
1397 + rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
1398 + if (st->dimensions) {
1399 + /* If the chart still has dimensions don't delete it from the metadata log */
1400 + continue;
1401 + }
1402 + metalog_commit_delete_chart(st);
1403 + }
1404 +#endif
1405 rrdset_rdlock(st);
1406
1407 if(rrdhost_delete_obsolete_charts)
database/rrdset.c
+203 -9
@@ -2,6 +2,7 @@
2
3 #define NETDATA_RRD_INTERNALS
4 #include "rrd.h"
5 +#include <sched.h>
6
7 void __rrdset_check_rdlock(RRDSET *st, const char *file, const char *function, const unsigned long line) {
8 debug(D_RRD_CALLS, "Checking read lock on chart '%s'", st->id);
@@ -260,7 +261,7 @@ void rrdset_reset(RRDSET *st) {
261 rd->collections_counter = 0;
262 // memset(rd->values, 0, rd->entries * sizeof(storage_number));
263 #ifdef ENABLE_DBENGINE
263 - if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode) {
264 + if (RRD_MEMORY_MODE_DBENGINE == st->rrd_memory_mode && !rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED)) {
265 rrdeng_store_metric_flush_current_page(rd);
266 }
267 #endif
@@ -366,6 +367,10 @@ void rrdset_free(RRDSET *st) {
367 freez(st->config_section);
368 freez(st->plugin_name);
369 freez(st->module_name);
370 + freez(st->state->old_title);
371 + freez(st->state->old_family);
372 + freez(st->state->old_context);
373 + freez(st->state);
374
375 switch(st->rrd_memory_mode) {
376 case RRD_MEMORY_MODE_SAVE:
@@ -381,6 +386,10 @@ void rrdset_free(RRDSET *st) {
386 freez(st);
387 break;
388 }
389 +#ifdef ENABLE_DBENGINE
390 + rrd_atomic_fetch_add(&host->objects_nr, -1);
391 +#endif
392 +
393 }
394
395 void rrdset_save(RRDSET *st) {
@@ -402,7 +411,7 @@ void rrdset_save(RRDSET *st) {
411 }
412 }
413
405 -void rrdset_delete(RRDSET *st) {
414 +void rrdset_delete_custom(RRDSET *st, int db_rotated) {
415 RRDDIM *rd;
416
417 rrdset_check_rdlock(st);
@@ -425,11 +434,12 @@ void rrdset_delete(RRDSET *st) {
434
435 recursively_delete_dir(st->cache_dir, "left-over chart");
436 #ifdef ENABLE_ACLK
428 - if (netdata_cloud_setting) {
437 + if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)) {
438 aclk_del_collector(st->rrdhost->hostname, st->plugin_name, st->module_name);
439 aclk_update_chart(st->rrdhost, st->id, ACLK_CMD_CHARTDEL);
440 }
441 #endif
442 +
443 }
444
445 void rrdset_delete_obsolete_dimensions(RRDSET *st) {
@@ -480,6 +490,8 @@ RRDSET *rrdset_create_custom(
490 , RRDSET_TYPE chart_type
491 , RRD_MEMORY_MODE memory_mode
492 , long history_entries
493 + , int is_archived
494 + , uuid_t *chart_uuid
495 ) {
496 if(!type || !type[0]) {
497 fatal("Cannot create rrd stats without a type: id '%s', name '%s', family '%s', context '%s', title '%s', units '%s', plugin '%s', module '%s'."
@@ -516,15 +528,139 @@ RRDSET *rrdset_create_custom(
528 snprintfz(fullid, RRD_ID_LENGTH_MAX, "%s.%s", type, id);
529
530 RRDSET *st = rrdset_find_on_create(host, fullid);
519 - if(st) {
531 + if (st) {
532 + int mark_rebuild = 0;
533 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
534 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
535 + if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
536 + rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
537 + mark_rebuild |= META_CHART_ACTIVATED;
538 + }
539 + char *old_plugin = NULL, *old_module = NULL, *old_title = NULL, *old_family = NULL, *old_context = NULL,
540 + *old_title_v = NULL, *old_family_v = NULL, *old_context_v = NULL;
541 + const char *new_name = name ? name : id;
542
523 - if(unlikely(name))
524 - rrdset_set_name(st, name);
525 - else
526 - rrdset_set_name(st, id);
543 + if (unlikely((st->name && !strcmp(st->name, new_name)) || !st->name)) {
544 + mark_rebuild |= META_CHART_UPDATED;
545 + rrdset_set_name(st, new_name);
546 + }
547 +
548 + if (unlikely(st->priority != priority)) {
549 + st->priority = priority;
550 + mark_rebuild |= META_CHART_UPDATED;
551 + }
552 + if (unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && st->update_every != update_every)) {
553 + st->update_every = update_every;
554 + mark_rebuild |= META_CHART_UPDATED;
555 + }
556 +
557 + if (plugin && st->plugin_name) {
558 + if (unlikely(strcmp(plugin, st->plugin_name))) {
559 + old_plugin = st->plugin_name;
560 + st->plugin_name = strdupz(plugin);
561 + mark_rebuild |= META_PLUGIN_UPDATED;
562 + }
563 + } else {
564 + if (plugin != st->plugin_name) { // one is NULL?
565 + old_plugin = st->plugin_name;
566 + st->plugin_name = plugin ? strdupz(plugin) : NULL;
567 + mark_rebuild |= META_PLUGIN_UPDATED;
568 + }
569 + }
570 +
571 + if (module && st->module_name) {
572 + if (unlikely(strcmp(module, st->module_name))) {
573 + old_module = st->module_name;
574 + st->module_name = strdupz(module);
575 + mark_rebuild |= META_MODULE_UPDATED;
576 + }
577 + } else {
578 + if (module != st->module_name) {
579 + if (st->module_name && *st->module_name) {
580 + old_module = st->module_name;
581 + st->module_name = module ? strdupz(module) : NULL;
582 + mark_rebuild |= META_MODULE_UPDATED;
583 + }
584 + }
585 + }
586 +
587 + if (unlikely(title && st->state->old_title && strcmp(st->state->old_title, title))) {
588 + char *new_title = strdupz(title);
589 + old_title_v = st->state->old_title;
590 + st->state->old_title = strdupz(title);
591 + json_fix_string(new_title);
592 + old_title = st->title;
593 + st->title = new_title;
594 + mark_rebuild |= META_CHART_UPDATED;
595 + }
596
597 + RRDSET_TYPE new_chart_type =
598 + rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
599 + if (st->chart_type != new_chart_type) {
600 + st->chart_type = new_chart_type;
601 + mark_rebuild |= META_CHART_UPDATED;
602 + }
603 +
604 + if (unlikely(family && st->state->old_family && strcmp(st->state->old_family, family))) {
605 + char *new_family = strdupz(family);
606 + old_family_v = st->state->old_family;
607 + st->state->old_family = strdupz(family);
608 + json_fix_string(new_family);
609 + old_family = st->family;
610 + rrdfamily_free(host, st->rrdfamily);
611 + st->family = new_family;
612 + st->rrdfamily = rrdfamily_create(host, st->family);
613 + mark_rebuild |= META_CHART_UPDATED;
614 + }
615 +
616 + if (unlikely(context && st->state->old_context && strcmp(st->state->old_context, context))) {
617 + char *new_context = strdupz(context);
618 + old_context_v = st->state->old_context;
619 + st->state->old_context = strdupz(context);
620 + json_fix_string(new_context);
621 + old_context = st->context;
622 + st->context = new_context;
623 + st->hash_context = simple_hash(st->context);
624 + mark_rebuild |= META_CHART_UPDATED;
625 + }
626 +
627 + if (mark_rebuild) {
628 +#ifdef ENABLE_ACLK
629 + if (netdata_cloud_setting) {
630 + if (mark_rebuild & META_CHART_ACTIVATED) {
631 + aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
632 + }
633 + else {
634 + if (mark_rebuild & (META_PLUGIN_UPDATED | META_MODULE_UPDATED)) {
635 + aclk_del_collector(
636 + host->hostname, mark_rebuild & META_PLUGIN_UPDATED ? old_plugin : st->plugin_name,
637 + mark_rebuild & META_MODULE_UPDATED ? old_module : st->module_name);
638 + aclk_add_collector(host->hostname, st->plugin_name, st->module_name);
639 + }
640 + }
641 + aclk_update_chart(host, st->id, ACLK_CMD_CHART);
642 + }
643 +#endif
644 + freez(old_plugin);
645 + freez(old_module);
646 + freez(old_title);
647 + freez(old_family);
648 + freez(old_context);
649 + freez(old_title_v);
650 + freez(old_family_v);
651 + freez(old_context_v);
652 + if (mark_rebuild != META_CHART_ACTIVATED) {
653 + info("Collector updated metadata for chart %s", st->id);
654 + sched_yield();
655 + }
656 + }
657 +#ifdef ENABLE_DBENGINE
658 + if (is_archived == 0 && st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE &&
659 + (mark_rebuild & (META_CHART_UPDATED | META_PLUGIN_UPDATED | META_MODULE_UPDATED))) {
660 + debug(D_METADATALOG, "CHART [%s] metadata updated", st->id);
661 + metalog_commit_update_chart(st);
662 + }
663 +#endif
664 return st;
665 }
666
@@ -535,6 +671,9 @@ RRDSET *rrdset_create_custom(
671 rrdhost_unlock(host);
672 rrdset_flag_set(st, RRDSET_FLAG_SYNC_CLOCK);
673 rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
674 + if (!is_archived && rrdset_flag_check(st, RRDSET_FLAG_ARCHIVED)) {
675 + rrdset_flag_clear(st, RRDSET_FLAG_ARCHIVED);
676 + }
677 return st;
678 }
679
@@ -664,6 +803,8 @@ RRDSET *rrdset_create_custom(
803 else
804 st->rrd_memory_mode = (memory_mode == RRD_MEMORY_MODE_NONE) ? RRD_MEMORY_MODE_NONE : RRD_MEMORY_MODE_ALLOC;
805 }
806 + if (is_archived)
807 + rrdset_flag_set(st, RRDSET_FLAG_ARCHIVED);
808
809 st->plugin_name = plugin?strdupz(plugin):NULL;
810 st->module_name = module?strdupz(module):NULL;
@@ -687,13 +828,16 @@ RRDSET *rrdset_create_custom(
828 st->chart_type = rrdset_type_id(config_get(st->config_section, "chart type", rrdset_type_name(chart_type)));
829 st->type = config_get(st->config_section, "type", type);
830
831 + st->state = mallocz(sizeof(*st->state));
832 st->family = config_get(st->config_section, "family", family?family:st->type);
833 + st->state->old_family = strdupz(st->family);
834 json_fix_string(st->family);
835
836 st->units = config_get(st->config_section, "units", units?units:"");
837 json_fix_string(st->units);
838
839 st->context = config_get(st->config_section, "context", context?context:st->id);
840 + st->state->old_context = strdupz(st->context);
841 json_fix_string(st->context);
842 st->hash_context = simple_hash(st->context);
843
@@ -745,6 +889,7 @@ RRDSET *rrdset_create_custom(
889 rrdset_set_name(st, id);
890
891 st->title = config_get(st->config_section, "title", title);
892 + st->state->old_title = strdupz(st->title);
893 json_fix_string(st->title);
894
895 st->rrdfamily = rrdfamily_create(host, st->family);
@@ -765,6 +910,26 @@ RRDSET *rrdset_create_custom(
910
911 rrdsetcalc_link_matching(st);
912 rrdcalctemplate_link_matching(st);
913 +#ifdef ENABLE_DBENGINE
914 + if (st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
915 + int replace_instead_of_generate = 0;
916 +
917 + st->chart_uuid = callocz(1, sizeof(uuid_t));
918 + if (NULL != chart_uuid) {
919 + replace_instead_of_generate = 1;
920 + uuid_copy(*st->chart_uuid, *chart_uuid);
921 + }
922 + if (unlikely(
923 + find_or_generate_guid((void *) st, st->chart_uuid, GUID_TYPE_CHART, replace_instead_of_generate))) {
924 + errno = 0;
925 + error("FAILED to generate GUID for %s", st->id);
926 + freez(st->chart_uuid);
927 + st->chart_uuid = NULL;
928 + assert(0);
929 + }
930 + st->compaction_id = 0;
931 + }
932 +#endif
933
934 rrdhost_cleanup_obsolete_charts(host);
935
@@ -775,6 +940,11 @@ RRDSET *rrdset_create_custom(
940 aclk_update_chart(host, st->id, ACLK_CMD_CHART);
941 }
942 #endif
943 +#ifdef ENABLE_DBENGINE
944 + rrd_atomic_fetch_add(&st->rrdhost->objects_nr, 1);
945 + metalog_commit_update_chart(st);
946 +#endif
947 +
948 return(st);
949 }
950
@@ -1006,6 +1176,9 @@ static inline size_t rrdset_done_interpolate(
1176 last_ut = next_store_ut;
1177
1178 rrddim_foreach_read(rd, st) {
1179 + if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
1180 + continue;
1181 +
1182 calculated_number new_value;
1183
1184 switch(rd->algorithm) {
@@ -1374,6 +1547,8 @@ void rrdset_done(RRDSET *st) {
1547 int dimensions = 0;
1548 st->collected_total = 0;
1549 rrddim_foreach_read(rd, st) {
1550 + if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
1551 + continue;
1552 dimensions++;
1553 if(likely(rd->updated))
1554 st->collected_total += rd->collected_value;
@@ -1385,6 +1560,8 @@ void rrdset_done(RRDSET *st) {
1560 // based on the collected figures only
1561 // at this stage we do not interpolate anything
1562 rrddim_foreach_read(rd, st) {
1563 + if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
1564 + continue;
1565
1566 if(unlikely(!rd->updated)) {
1567 rd->calculated_value = 0;
@@ -1630,6 +1807,8 @@ void rrdset_done(RRDSET *st) {
1807 st->last_collected_total = st->collected_total;
1808
1809 rrddim_foreach_read(rd, st) {
1810 + if (rrddim_flag_check(rd, RRDDIM_FLAG_ARCHIVED))
1811 + continue;
1812 if(unlikely(!rd->updated))
1813 continue;
1814
@@ -1692,7 +1871,7 @@ void rrdset_done(RRDSET *st) {
1871 // find if there are any obsolete dimensions
1872 time_t now = now_realtime_sec();
1873
1695 - if(unlikely(rrddim_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
1874 + if(unlikely(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS))) {
1875 rrddim_foreach_read(rd, st)
1876 if(unlikely(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)))
1877 break;
@@ -1714,6 +1893,21 @@ void rrdset_done(RRDSET *st) {
1893 error("Cannot delete dimension file '%s'", rd->cache_filename);
1894 }
1895
1896 +#ifdef ENABLE_DBENGINE
1897 + if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1898 + rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);
1899 + rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);
1900 + /* only a collector can mark a chart as obsolete, so we must remove the reference */
1901 + uint8_t can_delete_metric = rd->state->collect_ops.finalize(rd);
1902 + if (can_delete_metric) {
1903 + /* This metric has no data and no references */
1904 + metalog_commit_delete_dimension(rd);
1905 + } else {
1906 + /* Do not delete this dimension */
1907 + continue;
1908 + }
1909 + }
1910 +#endif
1911 if(unlikely(!last)) {
1912 rrddim_free(st, rd);
1913 rd = st->dimensions;
exporting/tests/netdata_doubles.c
+5 -1
@@ -100,7 +100,9 @@ RRDSET *rrdset_create_custom(
100 int update_every,
101 RRDSET_TYPE chart_type,
102 RRD_MEMORY_MODE memory_mode,
103 - long history_entries)
103 + long history_entries,
104 + int is_archived,
105 + uuid_t *chart_uuid)
106 {
107 check_expected_ptr(host);
108 check_expected_ptr(type);
@@ -117,6 +119,8 @@ RRDSET *rrdset_create_custom(
119 check_expected(chart_type);
120 UNUSED(memory_mode);
121 UNUSED(history_entries);
122 + UNUSED(is_archived);
123 + UNUSED(chart_uuid);
124
125 function_called();
126
health/health.c
+5
@@ -488,6 +488,11 @@ static inline int rrdcalc_isrunnable(RRDCALC *rc, time_t now, time_t *next_run)
488 return 0;
489 }
490
491 + if(unlikely(rrdset_flag_check(rc->rrdset, RRDSET_FLAG_ARCHIVED))) {
492 + debug(D_HEALTH, "Health not running alarm '%s.%s'. The chart has been marked as archived", rc->chart?rc->chart:"NOCHART", rc->name);
493 + return 0;
494 + }
495 +
496 if(unlikely(!rc->rrdset->last_collected_time.tv_sec || rc->rrdset->counter_done < 2)) {
497 debug(D_HEALTH, "Health not running alarm '%s.%s'. Chart is not fully collected yet.", rc->chart?rc->chart:"NOCHART", rc->name);
498 return 0;
libnetdata/log/log.h
+2
@@ -38,6 +38,8 @@
38 #define D_STREAM 0x0000000040000000
39 #define D_RRDENGINE 0x0000000100000000
40 #define D_ACLK 0x0000000200000000
41 +#define D_METADATALOG 0x0000000400000000
42 +#define D_GUIDLOG 0x0000000800000000
43 #define D_SYSTEM 0x8000000000000000
44
45 //#define DEBUG (D_WEB_CLIENT_ACCESS|D_LISTENER|D_RRD_STATS)
parser/parser.h
+7
@@ -31,6 +31,10 @@ typedef struct pluginsd_action {
31 PARSER_RC (*variable_action)(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
32 PARSER_RC (*label_action)(void *user, char *key, char *value, LABEL_SOURCE source);
33 PARSER_RC (*overwrite_action)(void *user, RRDHOST *host, struct label *new_labels);
34 +
35 + PARSER_RC (*guid_action)(void *user, uuid_t *uuid);
36 + PARSER_RC (*context_action)(void *user, uuid_t *uuid);
37 + PARSER_RC (*tombstone_action)(void *user, uuid_t *uuid);
38 } PLUGINSD_ACTION;
39
40 typedef enum parser_input_type {
@@ -101,5 +105,8 @@ extern PARSER_RC pluginsd_flush(char **words, void *user, PLUGINSD_ACTION *plug
105 extern PARSER_RC pluginsd_disable(char **words, void *user, PLUGINSD_ACTION *plugins_action);
106 extern PARSER_RC pluginsd_label(char **words, void *user, PLUGINSD_ACTION *plugins_action);
107 extern PARSER_RC pluginsd_overwrite(char **words, void *user, PLUGINSD_ACTION *plugins_action);
108 +extern PARSER_RC pluginsd_guid(char **words, void *user, PLUGINSD_ACTION *plugins_action);
109 +extern PARSER_RC pluginsd_context(char **words, void *user, PLUGINSD_ACTION *plugins_action);
110 +extern PARSER_RC pluginsd_tombstone(char **words, void *user, PLUGINSD_ACTION *plugins_action);
111
112 #endif
streaming/receiver.c
+19
@@ -279,6 +279,25 @@ static int rrdpush_receive(struct receiver_state *rpt)
279 }
280 netdata_mutex_unlock(&rpt->host->receiver_lock);
281 }
282 + else rrdhost_update(rpt->host
283 + , rpt->hostname
284 + , rpt->registry_hostname
285 + , rpt->machine_guid
286 + , rpt->os
287 + , rpt->timezone
288 + , rpt->tags
289 + , program_name
290 + , program_version
291 + , rpt->update_every
292 + , history
293 + , mode
294 + , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
295 + , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
296 + , rrdpush_destination
297 + , rrdpush_api_key
298 + , rrdpush_send_charts_matching
299 + , rpt->system_info);
300 +
301
302 int ssl = 0;
303 #ifdef ENABLE_HTTPS
web/api/formatters/charts2json.c
+2 -2
@@ -36,7 +36,7 @@ static inline const char* get_release_channel() {
36 return (use_stable)?"stable":"nightly";
37 }
38
39 -void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
39 +void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived) {
40 static char *custom_dashboard_info_js_filename = NULL;
41 size_t c, dimensions = 0, memory = 0, alarms = 0;
42 RRDSET *st;
@@ -71,7 +71,7 @@ void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile) {
71 c = 0;
72 rrdhost_rdlock(host);
73 rrdset_foreach_read(st, host) {
74 - if(rrdset_is_available_for_viewers(st)) {
74 + if ((!show_archived && rrdset_is_available_for_viewers(st)) || (show_archived && rrdset_is_archived(st))) {
75 if(c) buffer_strcat(wb, ",");
76 buffer_strcat(wb, "\n\t\t\"");
77 buffer_strcat(wb, st->id);
web/api/formatters/charts2json.h
+1 -1
@@ -5,7 +5,7 @@
5
6 #include "rrd2json.h"
7
8 -extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile);
8 +extern void charts2json(RRDHOST *host, BUFFER *wb, int skip_volatile, int show_archived);
9 extern void chartcollectors2json(RRDHOST *host, BUFFER *wb);
10
11 #endif //NETDATA_API_FORMATTER_CHARTS2JSON_H
web/api/web_api_v1.c
+11 -1
@@ -349,7 +349,16 @@ inline int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w,
349
350 buffer_flush(w->response.data);
351 w->response.data->contenttype = CT_APPLICATION_JSON;
352 - charts2json(host, w->response.data, 0);
352 + charts2json(host, w->response.data, 0, 0);
353 + return HTTP_RESP_OK;
354 +}
355 +
356 +inline int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url) {
357 + (void)url;
358 +
359 + buffer_flush(w->response.data);
360 + w->response.data->contenttype = CT_APPLICATION_JSON;
361 + charts2json(host, w->response.data, 0, 1);
362 return HTTP_RESP_OK;
363 }
364
@@ -915,6 +924,7 @@ static struct api_command {
924 { "data", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_data },
925 { "chart", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_chart },
926 { "charts", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_charts },
927 + { "archivedcharts", 0, WEB_CLIENT_ACL_DASHBOARD, web_client_api_request_v1_archivedcharts },
928
929 // registry checks the ACL by itself, so we allow everything
930 { "registry", 0, WEB_CLIENT_ACL_NOCHECK, web_client_api_request_v1_registry },
web/api/web_api_v1.h
+1
@@ -19,6 +19,7 @@ extern int web_client_api_request_single_chart(RRDHOST *host, struct web_client
19 extern int web_client_api_request_v1_alarm_variables(RRDHOST *host, struct web_client *w, char *url);
20 extern int web_client_api_request_v1_alarm_count(RRDHOST *host, struct web_client *w, char *url);
21 extern int web_client_api_request_v1_charts(RRDHOST *host, struct web_client *w, char *url);
22 +extern int web_client_api_request_v1_archivedcharts(RRDHOST *host, struct web_client *w, char *url);
23 extern int web_client_api_request_v1_chart(RRDHOST *host, struct web_client *w, char *url);
24 extern int web_client_api_request_v1_data(RRDHOST *host, struct web_client *w, char *url);
25 extern int web_client_api_request_v1_registry(RRDHOST *host, struct web_client *w, char *url);