78e27a8c4 · cryptotaxi247/netdata-1

CMakeLists.txt

+12 -2

@@ -1473,8 +1473,8 @@ set(RRD_PLUGIN_FILES

1473 src/database/sqlite/vendored/sqlite3recover.c

1474 src/database/sqlite/vendored/sqlite3recover.h

1475 src/database/sqlite/vendored/dbdata.c

1476 - src/database/KolmogorovSmirnovDist.c

1477 - src/database/KolmogorovSmirnovDist.h

1476 + src/web/api/queries/KolmogorovSmirnovDist.c

1477 + src/web/api/queries/KolmogorovSmirnovDist.h

1478 src/database/rrdfunctions-inflight.c

1479 src/database/rrdfunctions-inflight.h

1480 src/database/rrdfunctions-exporters.c

@@ -1503,6 +1503,16 @@ set(RRD_PLUGIN_FILES

1503 src/database/rrddim-backfill.h

1504 src/database/rrddim-collection.c

1505 src/database/rrddim-collection.h

1506 + src/database/rrdset-type.c

1507 + src/database/rrdset-type.h

1508 + src/database/rrdhost-slots.c

1509 + src/database/rrdhost-slots.h

1510 + src/database/rrd-algorithm.c

1511 + src/database/rrd-algorithm.h

1512 + src/database/rrdhost-labels.c

1513 + src/database/rrdhost-labels.h

1514 + src/database/rrdhost-collection.c

1515 + src/database/rrdhost-collection.h

1516 )

1517

1518 if(ENABLE_DBENGINE)

src/collectors/cgroups.plugin/tests/test_doubles.c

+2 -2

@@ -47,7 +47,7 @@ void mountinfo_free_all(struct mountinfo *mi)

47 RRDSET *rrdset_create_custom(

48 RRDHOST *host, const char *type, const char *id, const char *name, const char *family, const char *context,

49 const char *title, const char *units, const char *plugin, const char *module, long priority, int update_every,

50 - RRDSET_TYPE chart_type, RRD_MEMORY_MODE memory_mode, long history_entries)

50 + RRDSET_TYPE chart_type, RRD_DB_MODE memory_mode, long history_entries)

51 {

52 UNUSED(host);

53 UNUSED(type);

@@ -70,7 +70,7 @@ RRDSET *rrdset_create_custom(

70

71 RRDDIM *rrddim_add_custom(

72 RRDSET *st, const char *id, const char *name, collected_number multiplier, collected_number divisor,

73 - RRD_ALGORITHM algorithm, RRD_MEMORY_MODE memory_mode)

73 + RRD_ALGORITHM algorithm, RRD_DB_MODE memory_mode)

74 {

75 UNUSED(st);

76 UNUSED(id);

src/collectors/statsd.plugin/statsd.c

+1 -1

@@ -220,7 +220,7 @@ typedef struct statsd_app {

220 const char *name;

221 SIMPLE_PATTERN *metrics;

222 STATS_METRIC_OPTIONS default_options;

223 - RRD_MEMORY_MODE rrd_memory_mode;

223 + RRD_DB_MODE rrd_memory_mode;

224 int32_t rrd_history_entries;

225 DICTIONARY *dict;

226

src/daemon/config/netdata-conf-db.c

+4 -4

@@ -87,9 +87,9 @@ static void netdata_conf_dbengine_pre_logs(void) {

87 // }

88

89 #else

90 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

91 - error_report("RRD_MEMORY_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");

92 - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;

90 + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

91 + error_report("RRD_DB_MODE_DBENGINE is not supported in this platform. The agent will use db mode 'save' instead.");

92 + default_rrd_memory_mode = RRD_DB_MODE_RAM;

93 }

94 #endif

95 }

@@ -381,7 +381,7 @@ void netdata_conf_section_db(void) {

381 // ------------------------------------------------------------------------

382 // get default database size

383

384 - if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {

384 + if(default_rrd_memory_mode != RRD_DB_MODE_DBENGINE && default_rrd_memory_mode != RRD_DB_MODE_NONE) {

385 default_rrd_history_entries = (int)config_get_duration_seconds(

386 CONFIG_SECTION_DB, "retention",

387 align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));

src/daemon/main.c

+1 -1

@@ -213,7 +213,7 @@ int unittest_prepare_rrd(const char **user) {

213 netdata_conf_section_global_run_as_user(user);

214 netdata_conf_section_global();

215 nd_profile.update_every = 1;

216 - default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;

216 + default_rrd_memory_mode = RRD_DB_MODE_RAM;

217 health_plugin_disable();

218 nd_profile.storage_tiers = 1;

219 registry_init();

src/daemon/pulse/pulse-db-dbengine.c

+1 -1

@@ -1490,7 +1490,7 @@ void pulse_dbengine_do(bool extended) {

1490

1491 /* get localhost's DB engine's statistics for each tier */

1492 for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) {

1493 - if(host->db[tier].mode != RRD_MEMORY_MODE_DBENGINE) continue;

1493 + if(host->db[tier].mode != RRD_DB_MODE_DBENGINE) continue;

1494 if(!host->db[tier].si) continue;

1495

1496 if(counted_multihost_db[tier])

src/daemon/service.c

+1 -1

@@ -34,7 +34,7 @@ static void svc_rrddim_obsolete_to_archive(RRDDIM *rd) {

34 rrddim_flag_set(rd, RRDDIM_FLAG_ARCHIVED);

35 rrddim_flag_clear(rd, RRDDIM_FLAG_OBSOLETE);

36

37 - if (rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

37 + if (rd->rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

38 /* only a collector can mark a chart as obsolete, so we must remove the reference */

39 if (!rrddim_finalize_collection_and_check_retention(rd)) {

40 /* This metric has no data and no references */

src/daemon/unit_test.c

+2 -2

@@ -1268,7 +1268,7 @@ int run_test(struct test *test)

1268 {

1269 fprintf(stderr, "\nRunning test '%s':\n%s\n", test->name, test->description);

1270

1271 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;

1271 + default_rrd_memory_mode = RRD_DB_MODE_ALLOC;

1272 nd_profile.update_every = test->update_every;

1273

1274 char name[101];

@@ -1537,7 +1537,7 @@ int unit_test(long delay, long shift)

1537 snprintfz(name, sizeof(name) - 1, "unittest-%d-%ld-%ld", repeat, delay, shift);

1538

1539 //debug_flags = 0xffffffff;

1540 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;

1540 + default_rrd_memory_mode = RRD_DB_MODE_ALLOC;

1541 nd_profile.update_every = 1;

1542

1543 int do_abs = 1;

src/database/contexts/contexts-loading.c

+1 -1

@@ -121,7 +121,7 @@ void rrdhost_load_rrdcontext_data(RRDHOST *host) {

121 if(host->rrdctx.contexts) return;

122

123 rrdhost_create_rrdcontexts(host);

124 - if (host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)

124 + if (host->rrd_memory_mode != RRD_DB_MODE_DBENGINE)

125 return;

126

127 ignored_metrics = 0;

src/database/contexts/worker.c

+2 -2

@@ -264,7 +264,7 @@ void rrdcontext_delete_from_sql_unsafe(RRDCONTEXT *rc) {

264 rc->hub.units = string2str(rc->units);

265 rc->hub.family = string2str(rc->family);

266

267 - if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)

267 + if (rc->rrdhost->rrd_memory_mode != RRD_DB_MODE_DBENGINE)

268 return;

269

270 // delete it from SQL

@@ -762,7 +762,7 @@ void rrdcontext_message_send_unsafe(RRDCONTEXT *rc, bool snapshot __maybe_unused

762 rrdcontext_delete_from_sql_unsafe(rc);

763

764 else {

765 - if (rc->rrdhost->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)

765 + if (rc->rrdhost->rrd_memory_mode != RRD_DB_MODE_DBENGINE)

766 return;

767 if (ctx_store_context(&rc->rrdhost->host_id.uuid, &rc->hub) != 0)

768 netdata_log_error(

src/database/engine/dbengine-stresstest.c

+3 -3

@@ -20,7 +20,7 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {

20 NETDATA_VERSION,

21 nd_profile.update_every,

22 default_rrd_history_entries,

23 - RRD_MEMORY_MODE_DBENGINE,

23 + RRD_DB_MODE_DBENGINE,

24 health_plugin_enabled(),

25 stream_send.enabled,

26 stream_send.parents.destination,

@@ -140,7 +140,7 @@ void generate_dbengine_dataset(unsigned history_seconds)

140 int i;

141 time_t time_present;

142

143 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

143 + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE;

144 default_rrdeng_page_cache_mb = 128;

145 // Worst case for uncompressible data

146 default_rrdeng_disk_quota_mb = (((uint64_t)DSET_DIMS * DSET_CHARTS) * sizeof(storage_number) * history_seconds) /

@@ -333,7 +333,7 @@ void dbengine_stress_test(unsigned TEST_DURATION_SEC, unsigned DSET_CHARTS, unsi

333 if (PAGE_CACHE_MB < RRDENG_MIN_PAGE_CACHE_SIZE_MB)

334 PAGE_CACHE_MB = RRDENG_MIN_PAGE_CACHE_SIZE_MB;

335

336 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

336 + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE;

337 default_rrdeng_page_cache_mb = PAGE_CACHE_MB;

338 if (DISK_SPACE_MB) {

339 fprintf(stderr, "By setting disk space limit data are allowed to be deleted. "

src/database/engine/dbengine-unittest.c

+2 -2

@@ -106,7 +106,7 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {

106 NETDATA_VERSION,

107 nd_profile.update_every,

108 default_rrd_history_entries,

109 - RRD_MEMORY_MODE_DBENGINE,

109 + RRD_DB_MODE_DBENGINE,

110 health_plugin_enabled(),

111 stream_send.enabled,

112 stream_send.parents.destination,

@@ -362,7 +362,7 @@ int test_dbengine(void) {

362 nd_log_limits_unlimited();

363 fprintf(stderr, "\nRunning DB-engine test\n");

364

365 - default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

365 + default_rrd_memory_mode = RRD_DB_MODE_DBENGINE;

366 fprintf(stderr, "Initializing localhost with hostname 'unittest-dbengine'");

367 RRDHOST *host = dbengine_rrdhost_find_or_create("unittest-dbengine");

368 if(!host)

src/database/rrd-algorithm.c new

+37

@@ -0,0 +1,37 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "rrd-algorithm.h"

4 +

5 +RRD_ALGORITHM rrd_algorithm_id(const char *name) {

6 + if(strcmp(name, RRD_ALGORITHM_INCREMENTAL_NAME) == 0)

7 + return RRD_ALGORITHM_INCREMENTAL;

8 +

9 + else if(strcmp(name, RRD_ALGORITHM_ABSOLUTE_NAME) == 0)

10 + return RRD_ALGORITHM_ABSOLUTE;

11 +

12 + else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME) == 0)

13 + return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL;

14 +

15 + else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME) == 0)

16 + return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL;

17 +

18 + else

19 + return RRD_ALGORITHM_ABSOLUTE;

20 +}

21 +

22 +const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {

23 + switch(algorithm) {

24 + case RRD_ALGORITHM_ABSOLUTE:

25 + default:

26 + return RRD_ALGORITHM_ABSOLUTE_NAME;

27 +

28 + case RRD_ALGORITHM_INCREMENTAL:

29 + return RRD_ALGORITHM_INCREMENTAL_NAME;

30 +

31 + case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:

32 + return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME;

33 +

34 + case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:

35 + return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME;

36 + }

37 +}

src/database/rrd-algorithm.h new

+26

@@ -0,0 +1,26 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RRD_ALGORITHM_H

4 +#define NETDATA_RRD_ALGORITHM_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +typedef enum __attribute__ ((__packed__)) rrd_algorithm {

9 + RRD_ALGORITHM_ABSOLUTE = 0,

10 + RRD_ALGORITHM_INCREMENTAL = 1,

11 + RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2,

12 + RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3,

13 +

14 + // this is 8-bit

15 +} RRD_ALGORITHM;

16 +

17 +#define RRD_ALGORITHM_ABSOLUTE_NAME "absolute"

18 +#define RRD_ALGORITHM_INCREMENTAL_NAME "incremental"

19 +#define RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row"

20 +#define RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row"

21 +

22 +RRD_ALGORITHM rrd_algorithm_id(const char *name);

23 +const char *rrd_algorithm_name(RRD_ALGORITHM algorithm);

24 +

25 +

26 +#endif //NETDATA_RRD_ALGORITHM_H

src/database/rrd-database-mode.c

+12 -12

@@ -1,19 +1,19 @@

1

2 #include "rrd.h"

3

4 -inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {

4 +inline const char *rrd_memory_mode_name(RRD_DB_MODE id) {

5 switch(id) {

6 - case RRD_MEMORY_MODE_RAM:

7 - return RRD_MEMORY_MODE_RAM_NAME;

6 + case RRD_DB_MODE_RAM:

7 + return RRD_DB_MODE_RAM_NAME;

8

9 - case RRD_MEMORY_MODE_NONE:

10 - return RRD_MEMORY_MODE_NONE_NAME;

9 + case RRD_DB_MODE_NONE:

10 + return RRD_DB_MODE_NONE_NAME;

11

12 - case RRD_MEMORY_MODE_ALLOC:

13 - return RRD_MEMORY_MODE_ALLOC_NAME;

12 + case RRD_DB_MODE_ALLOC:

13 + return RRD_DB_MODE_ALLOC_NAME;

14

15 - case RRD_MEMORY_MODE_DBENGINE:

16 - return RRD_MEMORY_MODE_DBENGINE_NAME;

15 + case RRD_DB_MODE_DBENGINE:

16 + return RRD_DB_MODE_DBENGINE_NAME;

17 }

18

19 STORAGE_ENGINE* eng = storage_engine_get(id);

@@ -21,14 +21,14 @@ inline const char *rrd_memory_mode_name(RRD_MEMORY_MODE id) {

21 return eng->name;

22 }

23

24 - return RRD_MEMORY_MODE_RAM_NAME;

24 + return RRD_DB_MODE_RAM_NAME;

25 }

26

27 -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name) {

27 +RRD_DB_MODE rrd_memory_mode_id(const char *name) {

28 STORAGE_ENGINE* eng = storage_engine_find(name);

29 if (eng) {

30 return eng->id;

31 }

32

33 - return RRD_MEMORY_MODE_RAM;

33 + return RRD_DB_MODE_RAM;

34 }

src/database/rrd-database-mode.h

+12 -12

@@ -4,22 +4,22 @@

4 #define NETDATA_RRD_DATABASE_MODE_H

5

6 typedef enum __attribute__ ((__packed__)) rrd_memory_mode {

7 - RRD_MEMORY_MODE_NONE = 0,

8 - RRD_MEMORY_MODE_RAM = 1,

9 - RRD_MEMORY_MODE_ALLOC = 4,

10 - RRD_MEMORY_MODE_DBENGINE = 5,

7 + RRD_DB_MODE_NONE = 0,

8 + RRD_DB_MODE_RAM = 1,

9 + RRD_DB_MODE_ALLOC = 4,

10 + RRD_DB_MODE_DBENGINE = 5,

11

12 // this is 8-bit

13 -} RRD_MEMORY_MODE;

13 +} RRD_DB_MODE;

14

15 -#define RRD_MEMORY_MODE_NONE_NAME "none"

16 -#define RRD_MEMORY_MODE_RAM_NAME "ram"

17 -#define RRD_MEMORY_MODE_ALLOC_NAME "alloc"

18 -#define RRD_MEMORY_MODE_DBENGINE_NAME "dbengine"

15 +#define RRD_DB_MODE_NONE_NAME "none"

16 +#define RRD_DB_MODE_RAM_NAME "ram"

17 +#define RRD_DB_MODE_ALLOC_NAME "alloc"

18 +#define RRD_DB_MODE_DBENGINE_NAME "dbengine"

19

20 -extern RRD_MEMORY_MODE default_rrd_memory_mode;

20 +extern RRD_DB_MODE default_rrd_memory_mode;

21

22 -const char *rrd_memory_mode_name(RRD_MEMORY_MODE id);

23 -RRD_MEMORY_MODE rrd_memory_mode_id(const char *name);

22 +const char *rrd_memory_mode_name(RRD_DB_MODE id);

23 +RRD_DB_MODE rrd_memory_mode_id(const char *name);

24

25 #endif //NETDATA_RRD_DATABASE_MODE_H

src/database/rrd.c

+16 -50

@@ -3,7 +3,7 @@

3 #define RRDHOST_INTERNALS

4 #include "rrd.h"

5

6 -// ----------------------------------------------------------------------------

6 +// --------------------------------------------------------------------------------------------------------------------

7 // globals

8

9 /*

@@ -14,51 +14,14 @@ int rrd_delete_unupdated_dimensions = 0;

14 */

15

16 #ifdef ENABLE_DBENGINE

17 -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_DBENGINE;

17 +RRD_DB_MODE default_rrd_memory_mode = RRD_DB_MODE_DBENGINE;

18 #else

19 -RRD_MEMORY_MODE default_rrd_memory_mode = RRD_MEMORY_MODE_RAM;

19 +RRD_DB_MODE default_rrd_memory_mode = RRD_DB_MODE_RAM;

20 #endif

21 int gap_when_lost_iterations_above = 1;

22

23

24 -// ----------------------------------------------------------------------------

25 -// RRD - algorithms types

26 -

27 -RRD_ALGORITHM rrd_algorithm_id(const char *name) {

28 - if(strcmp(name, RRD_ALGORITHM_INCREMENTAL_NAME) == 0)

29 - return RRD_ALGORITHM_INCREMENTAL;

30 -

31 - else if(strcmp(name, RRD_ALGORITHM_ABSOLUTE_NAME) == 0)

32 - return RRD_ALGORITHM_ABSOLUTE;

33 -

34 - else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME) == 0)

35 - return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL;

36 -

37 - else if(strcmp(name, RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME) == 0)

38 - return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL;

39 -

40 - else

41 - return RRD_ALGORITHM_ABSOLUTE;

42 -}

43 -

44 -const char *rrd_algorithm_name(RRD_ALGORITHM algorithm) {

45 - switch(algorithm) {

46 - case RRD_ALGORITHM_ABSOLUTE:

47 - default:

48 - return RRD_ALGORITHM_ABSOLUTE_NAME;

49 -

50 - case RRD_ALGORITHM_INCREMENTAL:

51 - return RRD_ALGORITHM_INCREMENTAL_NAME;

52 -

53 - case RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL:

54 - return RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME;

55 -

56 - case RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL:

57 - return RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME;

58 - }

59 -}

60 -

61 -// ----------------------------------------------------------------------------

24 +// --------------------------------------------------------------------------------------------------------------------

25 // RRD - string management

26

27 STRING *rrd_string_strdupz(const char *s) {

@@ -71,16 +34,16 @@ STRING *rrd_string_strdupz(const char *s) {

34 return ret;

35 }

36

74 -// ----------------------------------------------------------------------------

37 +// --------------------------------------------------------------------------------------------------------------------

38

76 -inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {

77 - if(mode == RRD_MEMORY_MODE_DBENGINE) return 0;

78 - if(mode == RRD_MEMORY_MODE_NONE) return 5;

39 +inline long align_entries_to_pagesize(RRD_DB_MODE mode, long entries) {

40 + if(mode == RRD_DB_MODE_DBENGINE) return 0;

41 + if(mode == RRD_DB_MODE_NONE) return 5;

42

43 if(entries < 5) entries = 5;

44 if(entries > RRD_HISTORY_ENTRIES_MAX) entries = RRD_HISTORY_ENTRIES_MAX;

45

83 - if(mode == RRD_MEMORY_MODE_RAM) {

46 + if(mode == RRD_DB_MODE_RAM) {

47 long header_size = 0;

48

49 long page = (long)sysconf(_SC_PAGESIZE);

@@ -97,12 +60,15 @@ inline long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries) {

60 return entries;

61 }

62

63 +// --------------------------------------------------------------------------------------------------------------------

64 +

65 void api_v1_management_init(void);

66 +

67 int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool unittest) {

68 rrdhost_init();

69

70 if (unlikely(sql_init_meta_database(DB_CHECK_NONE, system_info ? 0 : 1))) {

105 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

71 + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

72 set_late_analytics_variables(system_info);

73 fatal("Failed to initialize SQLite");

74 }

@@ -119,7 +85,7 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool

85 dbengine_enabled = true;

86 }

87 else {

122 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {

88 + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {

89 nd_log(NDLS_DAEMON, NDLP_DEBUG,

90 "DBENGINE: Initializing ...");

91

@@ -137,12 +103,12 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool

103 nd_profile.storage_tiers = 1;

104 }

105

140 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

106 + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

107 nd_log(NDLS_DAEMON, NDLP_WARNING,

108 "dbengine is not enabled, but it has been given as the default db mode. "

109 "Resetting db mode to alloc");

110

145 - default_rrd_memory_mode = RRD_MEMORY_MODE_ALLOC;

111 + default_rrd_memory_mode = RRD_DB_MODE_ALLOC;

112 }

113 }

114 }

src/database/rrd.h

+1 -21

@@ -31,26 +31,6 @@ typedef enum __attribute__ ((__packed__)) {

31 QUERY_SOURCE_UNITTEST,

32 } QUERY_SOURCE;

33

34 -// --------------------------------------------------------------------------------------------------------------------

35 -// algorithms types

36 -

37 -typedef enum __attribute__ ((__packed__)) rrd_algorithm {

38 - RRD_ALGORITHM_ABSOLUTE = 0,

39 - RRD_ALGORITHM_INCREMENTAL = 1,

40 - RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL = 2,

41 - RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL = 3,

42 -

43 - // this is 8-bit

44 -} RRD_ALGORITHM;

45 -

46 -#define RRD_ALGORITHM_ABSOLUTE_NAME "absolute"

47 -#define RRD_ALGORITHM_INCREMENTAL_NAME "incremental"

48 -#define RRD_ALGORITHM_PCENT_OVER_DIFF_TOTAL_NAME "percentage-of-incremental-row"

49 -#define RRD_ALGORITHM_PCENT_OVER_ROW_TOTAL_NAME "percentage-of-absolute-row"

50 -

51 -RRD_ALGORITHM rrd_algorithm_id(const char *name);

52 -const char *rrd_algorithm_name(RRD_ALGORITHM algorithm);

53 -

34 // --------------------------------------------------------------------------------------------------------------------

35

36 typedef struct rrdcalc RRDCALC;

@@ -101,7 +81,7 @@ extern netdata_rwlock_t rrd_rwlock;

81 STRING *rrd_string_strdupz(const char *s);

82

83 #include "rrd-database-mode.h"

104 -long align_entries_to_pagesize(RRD_MEMORY_MODE mode, long entries);

84 +long align_entries_to_pagesize(RRD_DB_MODE mode, long entries);

85

86 static inline uint32_t get_uint32_id() {

87 return now_realtime_sec() & UINT32_MAX;

src/database/rrddim-backfill.c

-2

@@ -6,8 +6,6 @@

6 // ----------------------------------------------------------------------------

7 // fill the gap of a tier

8

9 -void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);

10 -

9 bool backfill_tier_from_smaller_tiers(RRDDIM *rd, size_t tier, time_t now_s) {

10 if(unlikely(tier >= nd_profile.storage_tiers)) return false;

11 #ifdef ENABLE_DBENGINE

src/database/rrddim-collection.c

+77

@@ -72,3 +72,80 @@ void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAG

72 }

73 }

74 }

75 +

76 +#ifdef NETDATA_LOG_COLLECTION_ERRORS

77 +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) {

78 +#else // !NETDATA_LOG_COLLECTION_ERRORS

79 +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {

80 +#endif // !NETDATA_LOG_COLLECTION_ERRORS

81 +

82 + static __thread struct log_stack_entry lgs[] = {

83 + [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL),

84 + [1] = ND_LOG_FIELD_END(),

85 + };

86 + lgs[0].str = rd->id;

87 + log_stack_push(lgs);

88 +

89 +#ifdef NETDATA_LOG_COLLECTION_ERRORS

90 + rd->rrddim_store_metric_count++;

91 +

92 + if(likely(rd->rrddim_store_metric_count > 1)) {

93 + usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC;

94 +

95 + if(point_end_time_ut != rd->rrddim_store_metric_last_ut) {

96 + internal_error(true,

97 + "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())",

98 + (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP",

99 + rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd),

100 + rd->update_every,

101 + rd->rrddim_store_metric_count,

102 + expected, point_end_time_ut,

103 + (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ",

104 + expected - point_end_time_ut,

105 + function,

106 + rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none");

107 + }

108 + }

109 +

110 + rd->rrddim_store_metric_last_ut = point_end_time_ut;

111 + rd->rrddim_store_metric_last_caller = function;

112 +#endif // NETDATA_LOG_COLLECTION_ERRORS

113 +

114 + // store the metric on tier 0

115 + storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut,

116 + n, 0, 0,

117 + 1, 0, flags);

118 +

119 + rrdset_done_statistics_points_stored_per_tier[0]++;

120 +

121 + time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);

122 +

123 + STORAGE_POINT sp = {

124 + .start_time_s = now_s - rd->rrdset->update_every,

125 + .end_time_s = now_s,

126 + .min = n,

127 + .max = n,

128 + .sum = n,

129 + .count = 1,

130 + .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,

131 + .flags = flags

132 + };

133 +

134 + for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) {

135 + if(unlikely(!rd->tiers[tier].smh)) continue;

136 +

137 + struct rrddim_tier *t = &rd->tiers[tier];

138 +

139 + if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) {

140 + // we have not collected this tier before

141 + // let's fill any gap that may exist

142 + backfill_tier_from_smaller_tiers(rd, tier, now_s);

143 + }

144 +

145 + store_metric_at_tier(rd, tier, t, sp, point_end_time_ut);

146 + }

147 + rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS);

148 +

149 + rrdcontext_collected_rrddim(rd);

150 + log_stack_pop(&lgs);

151 +}

src/database/rrddim-collection.h

+7

@@ -8,4 +8,11 @@

8 void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut);

9 void store_metric_collection_completed(void);

10

11 +#ifdef NETDATA_LOG_COLLECTION_ERRORS

12 +#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)

13 +void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function);

14 +#else

15 +void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);

16 +#endif

17 +

18 #endif //NETDATA_RRDDIM_COLLECTION_H

src/database/rrddim.c

+8 -7

@@ -18,7 +18,7 @@ struct rrddim_constructor {

18 collected_number multiplier;

19 collected_number divisor;

20 RRD_ALGORITHM algorithm;

21 - RRD_MEMORY_MODE memory_mode;

21 + RRD_DB_MODE memory_mode;

22

23 enum {

24 RRDDIM_REACT_NONE = 0,

@@ -57,7 +57,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

57 if(rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST))

58 rd->collector.counter = 1;

59

60 - if(ctr->memory_mode == RRD_MEMORY_MODE_RAM) {

60 + if(ctr->memory_mode == RRD_DB_MODE_RAM) {

61 size_t entries = st->db.entries;

62 if(!entries) entries = 5;

63

@@ -68,11 +68,11 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

68 }

69 else {

70 netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));

71 - ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;

71 + ctr->memory_mode = RRD_DB_MODE_ALLOC;

72 }

73 }

74

75 - if(ctr->memory_mode == RRD_MEMORY_MODE_ALLOC || ctr->memory_mode == RRD_MEMORY_MODE_NONE) {

75 + if(ctr->memory_mode == RRD_DB_MODE_ALLOC || ctr->memory_mode == RRD_DB_MODE_NONE) {

76 size_t entries = st->db.entries;

77 if(entries < 5) entries = 5;

78

@@ -208,7 +208,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

208

209 netdata_log_debug(D_RRD_CALLS, "rrddim_free() %s.%s", rrdset_name(st), rrddim_name(rd));

210

211 - if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

211 + if (!rrddim_finalize_collection_and_check_retention(rd) && rd->rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

212 /* This metric has no data and no references */

213 metaqueue_delete_dimension_uuid(uuidmap_uuid_ptr(rd->uuid));

214 }

@@ -226,7 +226,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

226 if(rd->db.data) {

227 pulse_db_rrd_memory_sub(rd->db.memsize);

228

229 - if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)

229 + if(rd->rrd_memory_mode == RRD_DB_MODE_RAM)

230 nd_munmap(rd->db.data, rd->db.memsize);

231 else

232 freez(rd->db.data);

@@ -454,7 +454,8 @@ RRDDIM *rrddim_add_custom(RRDSET *st

454 , collected_number multiplier

455 , collected_number divisor

456 , RRD_ALGORITHM algorithm

457 - , RRD_MEMORY_MODE memory_mode

457 + ,

458 + RRD_DB_MODE memory_mode

459 ) {

460 struct rrddim_constructor tmp = {

461 .st = st,

src/database/rrddim.h

+3 -12

@@ -4,6 +4,7 @@

4 #define NETDATA_RRDDIM_H

5

6 #include "libnetdata/libnetdata.h"

7 +#include "rrd-algorithm.h"

8

9 typedef struct rrddim RRDDIM;

10 typedef struct rrddim_acquired RRDDIM_ACQUIRED;

@@ -60,7 +61,7 @@ struct rrddim {

61 STRING *name; // the name of this dimension (as presented to user)

62

63 RRD_ALGORITHM algorithm; // the algorithm that is applied to add new collected values

63 - RRD_MEMORY_MODE rrd_memory_mode; // the memory mode for this dimension

64 + RRD_DB_MODE rrd_memory_mode; // the memory mode for this dimension

65 RRDDIM_FLAGS flags; // run time changing status flags

66

67 int32_t multiplier; // the multiplier of the collected values

@@ -195,7 +196,7 @@ RRDDIM *rrddim_add_custom(RRDSET *st

196 , collected_number multiplier

197 , collected_number divisor

198 , RRD_ALGORITHM algorithm

198 - , RRD_MEMORY_MODE memory_mode

199 + , RRD_DB_MODE memory_mode

200 );

201

202 #define rrddim_add(st, id, name, multiplier, divisor, algorithm) \

@@ -223,16 +224,6 @@ collected_number rrddim_set_by_pointer(RRDSET *st, RRDDIM *rd, collected_number

224 collected_number rrddim_set(RRDSET *st, const char *id, collected_number value);

225

226 bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd);

226 -void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);

227 -void rrdhost_finalize_collection(RRDHOST *host);

228 -void rrd_finalize_collection_for_all_hosts(void);

229 -

230 -#ifdef NETDATA_LOG_COLLECTION_ERRORS

231 -#define rrddim_store_metric(rd, point_end_time_ut, n, flags) rrddim_store_metric_with_trace(rd, point_end_time_ut, n, flags, __FUNCTION__)

232 -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function);

233 -#else

234 -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);

235 -#endif

227

228 void rrddim_free(RRDSET *st, RRDDIM *rd);

229

src/database/rrdhost-collection.c new

+35

@@ -0,0 +1,35 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "rrdhost-collection.h"

4 +#include "rrdset.h"

5 +

6 +void rrd_finalize_collection_for_all_hosts(void) {

7 + RRDHOST *host;

8 + dfe_start_reentrant(rrdhost_root_index, host) {

9 + rrdhost_finalize_collection(host);

10 + }

11 + dfe_done(host);

12 +}

13 +

14 +void rrdhost_finalize_collection(RRDHOST *host) {

15 + ND_LOG_STACK lgs[] = {

16 + ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(host)),

17 + ND_LOG_FIELD_END(),

18 + };

19 + ND_LOG_STACK_PUSH(lgs);

20 +

21 + nd_log(NDLS_DAEMON, NDLP_DEBUG,

22 + "RRD: 'host:%s' stopping data collection...",

23 + rrdhost_hostname(host));

24 +

25 + RRDSET *st;

26 + rrdset_foreach_read(st, host)

27 + rrdset_finalize_collection(st, true);

28 + rrdset_foreach_done(st);

29 +}

30 +

31 +bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) {

32 + time_t first_time_s, last_time_s;

33 + rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s);

34 + return query_matches_retention(after, before, first_time_s, last_time_s, 0);

35 +}

src/database/rrdhost-collection.h new

+11

@@ -0,0 +1,11 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RRDHOST_COLLECTION_H

4 +#define NETDATA_RRDHOST_COLLECTION_H

5 +

6 +#include "rrdhost.h"

7 +

8 +void rrdhost_finalize_collection(RRDHOST *host);

9 +void rrd_finalize_collection_for_all_hosts(void);

10 +

11 +#endif //NETDATA_RRDHOST_COLLECTION_H

src/database/rrdhost-labels.c new

+102

@@ -0,0 +1,102 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "rrdhost-labels.h"

4 +#include "rrdhost.h"

5 +#include "streaming/stream.h"

6 +

7 +void rrdhost_set_is_parent_label(void) {

8 + uint32_t count = stream_receivers_currently_connected();

9 +

10 + if (count == 0 || count == 1) {

11 + RRDLABELS *labels = localhost->rrdlabels;

12 + rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);

13 +

14 + // queue a node info

15 + aclk_queue_node_info(localhost, false);

16 + }

17 +}

18 +

19 +static bool config_label_cb(void *data __maybe_unused, const char *name, const char *value) {

20 + rrdlabels_add(localhost->rrdlabels, name, value, RRDLABEL_SRC_CONFIG);

21 + return true;

22 +}

23 +

24 +static void rrdhost_load_config_labels(void) {

25 + int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);

26 + if(!status) {

27 + char *filename = CONFIG_DIR "/" CONFIG_FILENAME;

28 + nd_log(NDLS_DAEMON, NDLP_WARNING,

29 + "RRDLABEL: Cannot reload the configuration file '%s', using labels in memory",

30 + filename);

31 + }

32 +

33 + appconfig_foreach_value_in_section(&netdata_config, CONFIG_SECTION_HOST_LABEL, config_label_cb, NULL);

34 +}

35 +

36 +static void rrdhost_load_kubernetes_labels(void) {

37 + char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];

38 + sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");

39 +

40 + if (unlikely(access(label_script, R_OK) != 0)) {

41 + nd_log(NDLS_DAEMON, NDLP_ERR,

42 + "Kubernetes pod label fetching script %s not found.",

43 + label_script);

44 +

45 + return;

46 + }

47 +

48 + POPEN_INSTANCE *instance = spawn_popen_run(label_script);

49 + if(!instance) return;

50 +

51 + char buffer[1000 + 1];

52 + while (fgets(buffer, 1000, spawn_popen_stdout(instance)) != NULL)

53 + rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);

54 +

55 + // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'

56 + // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null

57 + int rc = spawn_popen_wait(instance);

58 + if(rc)

59 + nd_log(NDLS_DAEMON, NDLP_ERR,

60 + "%s exited abnormally. Failed to get kubernetes labels.",

61 + label_script);

62 +}

63 +

64 +static void rrdhost_load_auto_labels(void) {

65 + RRDLABELS *labels = localhost->rrdlabels;

66 +

67 + rrdhost_system_info_to_rrdlabels(localhost->system_info, labels);

68 + add_aclk_host_labels();

69 +

70 + // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).

71 + // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.

72 +

73 + int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "is ephemeral node", CONFIG_BOOLEAN_NO);

74 + rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO);

75 +

76 + int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "has unstable connection", CONFIG_BOOLEAN_NO);

77 + rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);

78 +

79 + rrdlabels_add(labels, "_is_parent", (stream_receivers_currently_connected() > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);

80 +

81 + rrdlabels_add(labels, "_hostname", string2str(localhost->hostname), RRDLABEL_SRC_AUTO);

82 + rrdlabels_add(labels, "_os", string2str(localhost->os), RRDLABEL_SRC_AUTO);

83 +

84 + if (localhost->stream.snd.destination)

85 + rrdlabels_add(labels, "_streams_to", string2str(localhost->stream.snd.destination), RRDLABEL_SRC_AUTO);

86 +}

87 +

88 +void reload_host_labels(void) {

89 + if(!localhost->rrdlabels)

90 + localhost->rrdlabels = rrdlabels_create();

91 +

92 + rrdlabels_unmark_all(localhost->rrdlabels);

93 +

94 + // priority is important here

95 + rrdhost_load_config_labels();

96 + rrdhost_load_kubernetes_labels();

97 + rrdhost_load_auto_labels();

98 +

99 + rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);

100 +

101 + stream_send_host_labels(localhost);

102 +}

src/database/rrdhost-labels.h new

+11

@@ -0,0 +1,11 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RRDHOST_LABELS_H

4 +#define NETDATA_RRDHOST_LABELS_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +void reload_host_labels(void);

9 +void rrdhost_set_is_parent_label(void);

10 +

11 +#endif //NETDATA_RRDHOST_LABELS_H

src/database/rrdhost-slots.c new

+40

@@ -0,0 +1,40 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "rrdhost-slots.h"

4 +#include "rrdset.h"

5 +

6 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) {

7 + rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t));

8 +

9 + spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);

10 + host->stream.snd.pluginsd_chart_slots.available.ignore = true;

11 + freez(host->stream.snd.pluginsd_chart_slots.available.array);

12 + host->stream.snd.pluginsd_chart_slots.available.array = NULL;

13 + host->stream.snd.pluginsd_chart_slots.available.used = 0;

14 + host->stream.snd.pluginsd_chart_slots.available.size = 0;

15 + spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);

16 +

17 + // zero all the slots on all charts, so that they will not attempt to access the array

18 + RRDSET *st;

19 + rrdset_foreach_read(st, host) {

20 + st->stream.snd.chart_slot = 0;

21 + }

22 + rrdset_foreach_done(st);

23 +}

24 +

25 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) {

26 + rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t));

27 +

28 + spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock);

29 +

30 + if(host->stream.rcv.pluginsd_chart_slots.array) {

31 + for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++)

32 + rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]);

33 +

34 + freez(host->stream.rcv.pluginsd_chart_slots.array);

35 + host->stream.rcv.pluginsd_chart_slots.array = NULL;

36 + host->stream.rcv.pluginsd_chart_slots.size = 0;

37 + }

38 +

39 + spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock);

40 +}

src/database/rrdhost-slots.h new

+12

@@ -0,0 +1,12 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RRDHOST_SLOTS_H

4 +#define NETDATA_RRDHOST_SLOTS_H

5 +

6 +#include "rrdhost.h"

7 +

8 +void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);

9 +void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);

10 +

11 +

12 +#endif //NETDATA_RRDHOST_SLOTS_H

src/database/rrdhost-status.h

+1 -1

@@ -95,7 +95,7 @@ typedef struct rrdhost_status_t {

95 struct {

96 RRDHOST_DB_STATUS status;

97 RRDHOST_DB_LIVENESS liveness;

98 - RRD_MEMORY_MODE mode;

98 + RRD_DB_MODE mode;

99 time_t first_time_s;

100 time_t last_time_s;

101 size_t metrics;

src/database/rrdhost.c

+21 -146

@@ -187,7 +187,8 @@ static inline void rrdhost_init_timezone(RRDHOST *host, const char *timezone, co

187 host->utc_offset = utc_offset;

188 }

189

190 -void set_host_properties(RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode,

190 +void set_host_properties(RRDHOST *host, int update_every,

191 + RRD_DB_MODE memory_mode,

192 const char *registry_hostname, const char *os, const char *tzone,

193 const char *abbrev_tzone, int32_t utc_offset, const char *prog_name,

194 const char *prog_version)

@@ -238,7 +239,7 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host)

239 snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", host->cache_dir);

240

241 if ((initialized = create_dbengine_directory(host, dbenginepath))) {

241 - host->db[0].mode = RRD_MEMORY_MODE_DBENGINE;

242 + host->db[0].mode = RRD_DB_MODE_DBENGINE;

243 host->db[0].eng = storage_engine_get(host->db[0].mode);

244 host->db[0].tier_grouping = get_tier_grouping(0);

245

@@ -274,20 +275,20 @@ static RRDHOST *prepare_host_for_unittest(RRDHOST *host)

275 }

276 #endif

277

277 -static void rrdhost_set_replication_parameters(RRDHOST *host, RRD_MEMORY_MODE memory_mode, time_t period, time_t step) {

278 +static void rrdhost_set_replication_parameters(RRDHOST *host, RRD_DB_MODE memory_mode, time_t period, time_t step) {

279 host->stream.replication.period = period;

280 host->stream.replication.step = step;

281 host->stream.rcv.status.replication.percent = 100.0;

282

283 switch(memory_mode) {

284 default:

284 - case RRD_MEMORY_MODE_ALLOC:

285 - case RRD_MEMORY_MODE_RAM:

285 + case RRD_DB_MODE_ALLOC:

286 + case RRD_DB_MODE_RAM:

287 if(host->stream.replication.period > (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every)

288 host->stream.replication.period = (time_t) host->rrd_history_entries * (time_t) host->rrd_update_every;

289 break;

290

290 - case RRD_MEMORY_MODE_DBENGINE:

291 + case RRD_DB_MODE_DBENGINE:

292 break;

293 }

294 }

@@ -304,7 +305,7 @@ RRDHOST *rrdhost_create(

305 const char *prog_version,

306 int update_every,

307 long entries,

307 - RRD_MEMORY_MODE memory_mode,

308 + RRD_DB_MODE memory_mode,

309 bool health,

310 bool stream,

311 STRING *parents,

@@ -317,12 +318,12 @@ RRDHOST *rrdhost_create(

318 int is_localhost,

319 bool archived

320 ) {

320 - if(memory_mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled) {

321 + if(memory_mode == RRD_DB_MODE_DBENGINE && !dbengine_enabled) {

322 nd_log(NDLS_DAEMON, NDLP_ERR,

323 "memory mode 'dbengine' is not enabled, but host '%s' is configured for it. Falling back to 'alloc'",

324 hostname);

325

325 - memory_mode = RRD_MEMORY_MODE_ALLOC;

326 + memory_mode = RRD_DB_MODE_ALLOC;

327 }

328

329 RRDHOST *host = callocz(1, sizeof(RRDHOST));

@@ -349,7 +350,7 @@ RRDHOST *rrdhost_create(

350 rrdhost_init_hostname(host, hostname, false);

351

352 host->rrd_history_entries = align_entries_to_pagesize(memory_mode, entries);

352 - host->health.enabled = ((memory_mode == RRD_MEMORY_MODE_NONE)) ? 0 : health;

353 + host->health.enabled = ((memory_mode == RRD_DB_MODE_NONE)) ? 0 : health;

354

355 spinlock_init(&host->receiver_lock);

356

@@ -385,7 +386,7 @@ RRDHOST *rrdhost_create(

386

387 rrdcalc_rrdhost_index_init(host);

388

388 - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {

389 + if (host->rrd_memory_mode == RRD_DB_MODE_DBENGINE) {

390 #ifdef ENABLE_DBENGINE

391 if (unittest_running) {

392 host = prepare_host_for_unittest(host);

@@ -394,14 +395,14 @@ RRDHOST *rrdhost_create(

395 }

396 else {

397 for(size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {

397 - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;

398 + host->db[tier].mode = RRD_DB_MODE_DBENGINE;

399 host->db[tier].eng = storage_engine_get(host->db[tier].mode);

400 host->db[tier].si = (STORAGE_INSTANCE *)multidb_ctx[tier];

401 host->db[tier].tier_grouping = get_tier_grouping(tier);

402 }

403 }

404 #else

404 - fatal("RRD_MEMORY_MODE_DBENGINE is not supported in this platform.");

405 + fatal("RRD_DB_MODE_DBENGINE is not supported in this platform.");

406 #endif

407 }

408 else {

@@ -413,7 +414,7 @@ RRDHOST *rrdhost_create(

414 #ifdef ENABLE_DBENGINE

415 // the first tier is reserved for the non-dbengine modes

416 for(size_t tier = 1; tier < nd_profile.storage_tiers; tier++) {

416 - host->db[tier].mode = RRD_MEMORY_MODE_DBENGINE;

417 + host->db[tier].mode = RRD_DB_MODE_DBENGINE;

418 host->db[tier].eng = storage_engine_get(host->db[tier].mode);

419 host->db[tier].si = (STORAGE_INSTANCE *) multidb_ctx[tier];

420 host->db[tier].tier_grouping = get_tier_grouping(tier);

@@ -519,7 +520,8 @@ static void rrdhost_update(RRDHOST *host

520 , const char *prog_version

521 , int update_every

522 , long history

522 - , RRD_MEMORY_MODE mode

523 + ,

524 + RRD_DB_MODE mode

525 , bool health

526 , bool stream

527 , STRING *parents

@@ -535,7 +537,7 @@ static void rrdhost_update(RRDHOST *host

537

538 spinlock_lock(&host->rrdhost_update_lock);

539

538 - host->health.enabled = (mode == RRD_MEMORY_MODE_NONE) ? 0 : health;

540 + host->health.enabled = (mode == RRD_DB_MODE_NONE) ? 0 : health;

541

542 {

543 struct rrdhost_system_info *old = host->system_info;

@@ -596,7 +598,7 @@ static void rrdhost_update(RRDHOST *host

598 rrd_memory_mode_name(host->rrd_memory_mode),

599 rrd_memory_mode_name(mode));

600

599 - else if(host->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && host->rrd_history_entries < history)

601 + else if(host->rrd_memory_mode != RRD_DB_MODE_DBENGINE && host->rrd_history_entries < history)

602 nd_log(NDLS_DAEMON, NDLP_WARNING,

603 "Host '%s' has history of %d entries, but the wanted one is %ld entries. "

604 "Restart netdata here to apply the new settings.",

@@ -654,7 +656,8 @@ RRDHOST *rrdhost_find_or_create(

656 , const char *prog_version

657 , int update_every

658 , long history

657 - , RRD_MEMORY_MODE mode

659 + ,

660 + RRD_DB_MODE mode

661 , bool health

662 , bool stream

663 , STRING *parents

@@ -880,131 +883,3 @@ void rrdhost_free_all(void) {

883

884 rrd_wrunlock();

885 }

883 -

884 -void rrd_finalize_collection_for_all_hosts(void) {

885 - RRDHOST *host;

886 - dfe_start_reentrant(rrdhost_root_index, host) {

887 - rrdhost_finalize_collection(host);

888 - }

889 - dfe_done(host);

890 -}

891 -

892 -void rrdhost_set_is_parent_label(void) {

893 - uint32_t count = stream_receivers_currently_connected();

894 -

895 - if (count == 0 || count == 1) {

896 - RRDLABELS *labels = localhost->rrdlabels;

897 - rrdlabels_add(labels, "_is_parent", (count) ? "true" : "false", RRDLABEL_SRC_AUTO);

898 -

899 - // queue a node info

900 - aclk_queue_node_info(localhost, false);

901 - }

902 -}

903 -

904 -static bool config_label_cb(void *data __maybe_unused, const char *name, const char *value) {

905 - rrdlabels_add(localhost->rrdlabels, name, value, RRDLABEL_SRC_CONFIG);

906 - return true;

907 -}

908 -

909 -static void rrdhost_load_config_labels(void) {

910 - int status = config_load(NULL, 1, CONFIG_SECTION_HOST_LABEL);

911 - if(!status) {

912 - char *filename = CONFIG_DIR "/" CONFIG_FILENAME;

913 - nd_log(NDLS_DAEMON, NDLP_WARNING,

914 - "RRDLABEL: Cannot reload the configuration file '%s', using labels in memory",

915 - filename);

916 - }

917 -

918 - appconfig_foreach_value_in_section(&netdata_config, CONFIG_SECTION_HOST_LABEL, config_label_cb, NULL);

919 -}

920 -

921 -static void rrdhost_load_kubernetes_labels(void) {

922 - char label_script[sizeof(char) * (strlen(netdata_configured_primary_plugins_dir) + strlen("get-kubernetes-labels.sh") + 2)];

923 - sprintf(label_script, "%s/%s", netdata_configured_primary_plugins_dir, "get-kubernetes-labels.sh");

924 -

925 - if (unlikely(access(label_script, R_OK) != 0)) {

926 - nd_log(NDLS_DAEMON, NDLP_ERR,

927 - "Kubernetes pod label fetching script %s not found.",

928 - label_script);

929 -

930 - return;

931 - }

932 -

933 - POPEN_INSTANCE *instance = spawn_popen_run(label_script);

934 - if(!instance) return;

935 -

936 - char buffer[1000 + 1];

937 - while (fgets(buffer, 1000, spawn_popen_stdout(instance)) != NULL)

938 - rrdlabels_add_pair(localhost->rrdlabels, buffer, RRDLABEL_SRC_AUTO|RRDLABEL_SRC_K8S);

939 -

940 - // Non-zero exit code means that all the script output is error messages. We've shown already any message that didn't include a ':'

941 - // Here we'll inform with an ERROR that the script failed, show whatever (if anything) was added to the list of labels, free the memory and set the return to null

942 - int rc = spawn_popen_wait(instance);

943 - if(rc)

944 - nd_log(NDLS_DAEMON, NDLP_ERR,

945 - "%s exited abnormally. Failed to get kubernetes labels.",

946 - label_script);

947 -}

948 -

949 -static void rrdhost_load_auto_labels(void) {

950 - RRDLABELS *labels = localhost->rrdlabels;

951 -

952 - rrdhost_system_info_to_rrdlabels(localhost->system_info, labels);

953 - add_aclk_host_labels();

954 -

955 - // The source should be CONF, but when it is set, these labels are exported by default ('send configured labels' in exporting.conf).

956 - // Their export seems to break exporting to Graphite, see https://github.com/netdata/netdata/issues/14084.

957 -

958 - int is_ephemeral = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "is ephemeral node", CONFIG_BOOLEAN_NO);

959 - rrdlabels_add(labels, "_is_ephemeral", is_ephemeral ? "true" : "false", RRDLABEL_SRC_AUTO);

960 -

961 - int has_unstable_connection = appconfig_get_boolean(&netdata_config, CONFIG_SECTION_GLOBAL, "has unstable connection", CONFIG_BOOLEAN_NO);

962 - rrdlabels_add(labels, "_has_unstable_connection", has_unstable_connection ? "true" : "false", RRDLABEL_SRC_AUTO);

963 -

964 - rrdlabels_add(labels, "_is_parent", (stream_receivers_currently_connected() > 0) ? "true" : "false", RRDLABEL_SRC_AUTO);

965 -

966 - rrdlabels_add(labels, "_hostname", string2str(localhost->hostname), RRDLABEL_SRC_AUTO);

967 - rrdlabels_add(labels, "_os", string2str(localhost->os), RRDLABEL_SRC_AUTO);

968 -

969 - if (localhost->stream.snd.destination)

970 - rrdlabels_add(labels, "_streams_to", string2str(localhost->stream.snd.destination), RRDLABEL_SRC_AUTO);

971 -}

972 -

973 -void reload_host_labels(void) {

974 - if(!localhost->rrdlabels)

975 - localhost->rrdlabels = rrdlabels_create();

976 -

977 - rrdlabels_unmark_all(localhost->rrdlabels);

978 -

979 - // priority is important here

980 - rrdhost_load_config_labels();

981 - rrdhost_load_kubernetes_labels();

982 - rrdhost_load_auto_labels();

983 -

984 - rrdhost_flag_set(localhost,RRDHOST_FLAG_METADATA_LABELS | RRDHOST_FLAG_METADATA_UPDATE);

985 -

986 - stream_send_host_labels(localhost);

987 -}

988 -

989 -void rrdhost_finalize_collection(RRDHOST *host) {

990 - ND_LOG_STACK lgs[] = {

991 - ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(host)),

992 - ND_LOG_FIELD_END(),

993 - };

994 - ND_LOG_STACK_PUSH(lgs);

995 -

996 - nd_log(NDLS_DAEMON, NDLP_DEBUG,

997 - "RRD: 'host:%s' stopping data collection...",

998 - rrdhost_hostname(host));

999 -

1000 - RRDSET *st;

1001 - rrdset_foreach_read(st, host)

1002 - rrdset_finalize_collection(st, true);

1003 - rrdset_foreach_done(st);

1004 -}

1005 -

1006 -bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) {

1007 - time_t first_time_s, last_time_s;

1008 - rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s);

1009 - return query_matches_retention(after, before, first_time_s, last_time_s, 0);

1010 -}

src/database/rrdhost.h

+15 -15

@@ -12,13 +12,14 @@ typedef struct rrdhost RRDHOST;

12 typedef struct ml_host rrd_ml_host_t;

13 typedef struct rrdhost_acquired RRDHOST_ACQUIRED;

14

15 -#include "streaming/stream-traffic-types.h"

15 +//#include "streaming/stream-traffic-types.h"

16 #include "streaming/stream-sender-commit.h"

17 -#include "streaming/stream-replication-tracking.h"

17 +#include "rrd-database-mode.h"

18 +//#include "streaming/stream-replication-tracking.h"

19 #include "streaming/stream-parents.h"

20 #include "streaming/stream-path.h"

21 #include "storage-engine.h"

21 -#include "streaming/stream-traffic-types.h"

22 +//#include "streaming/stream-traffic-types.h"

23 #include "rrdlabels.h"

24 #include "health/health-alert-log.h"

25

@@ -143,13 +144,13 @@ struct rrdhost {

144 int32_t rrd_update_every; // the update frequency of the host

145 int32_t rrd_history_entries; // the number of history entries for the host's charts

146

146 - RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host

147 - // the actual per tier is at .db[tier].mode

147 + RRD_DB_MODE rrd_memory_mode; // the configured memory more for the charts of this host

148 + // the actual per tier is at .db[tier].mode

149

150 char *cache_dir; // the directory to save RRD cache files

151

152 struct {

152 - RRD_MEMORY_MODE mode; // the db mode for this tier

153 + RRD_DB_MODE mode; // the db mode for this tier

154 STORAGE_ENGINE *eng; // the storage engine API for this tier

155 STORAGE_INSTANCE *si; // the db instance for this tier

156 uint32_t tier_grouping; // tier 0 iterations aggregated on this tier

@@ -393,7 +394,7 @@ RRDHOST *rrdhost_create(

394 const char *prog_version,

395 int update_every,

396 long entries,

396 - RRD_MEMORY_MODE memory_mode,

397 + RRD_DB_MODE memory_mode,

398 bool health,

399 bool stream,

400 STRING *parents,

@@ -422,7 +423,7 @@ RRDHOST *rrdhost_find_or_create(

423 const char *prog_version,

424 int update_every,

425 long history,

425 - RRD_MEMORY_MODE mode,

426 + RRD_DB_MODE mode,

427 bool health,

428 bool stream,

429 STRING *parents,

@@ -441,11 +442,9 @@ void rrdhost_free___while_having_rrd_wrlock(RRDHOST *host, bool force);

442 bool rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now_s);

443 bool rrdhost_should_run_health(RRDHOST *host);

444

444 -void reload_host_labels(void);

445 -void rrdhost_set_is_parent_label(void);

446 -

445 void set_host_properties(

448 - RRDHOST *host, int update_every, RRD_MEMORY_MODE memory_mode, const char *registry_hostname,

446 + RRDHOST *host, int update_every,

447 + RRD_DB_MODE memory_mode, const char *registry_hostname,

448 const char *os, const char *tzone, const char *abbrev_tzone, int32_t utc_offset,

449 const char *prog_name, const char *prog_version);

450

@@ -463,10 +462,11 @@ static inline void rrdhost_retention(RRDHOST *host, time_t now, bool online, tim

462 *to = online ? now : last_time_s;

463 }

464

466 -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);

467 -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);

468 -

465 extern time_t rrdhost_free_orphan_time_s;

466 extern time_t rrdhost_free_ephemeral_time_s;

467

468 +#include "rrdhost-collection.h"

469 +#include "rrdhost-slots.h"

470 +#include "rrdhost-labels.h"

471 +

472 #endif //NETDATA_RRDHOST_H

src/database/rrdset-collection.c

+2 -79

@@ -273,83 +273,6 @@ static inline void rrdset_init_last_updated_time(RRDSET *st) {

273

274 __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];

275

276 -#ifdef NETDATA_LOG_COLLECTION_ERRORS

277 -void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) {

278 -#else // !NETDATA_LOG_COLLECTION_ERRORS

279 -void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {

280 -#endif // !NETDATA_LOG_COLLECTION_ERRORS

281 -

282 - static __thread struct log_stack_entry lgs[] = {

283 - [0] = ND_LOG_FIELD_STR(NDF_NIDL_DIMENSION, NULL),

284 - [1] = ND_LOG_FIELD_END(),

285 - };

286 - lgs[0].str = rd->id;

287 - log_stack_push(lgs);

288 -

289 -#ifdef NETDATA_LOG_COLLECTION_ERRORS

290 - rd->rrddim_store_metric_count++;

291 -

292 - if(likely(rd->rrddim_store_metric_count > 1)) {

293 - usec_t expected = rd->rrddim_store_metric_last_ut + rd->update_every * USEC_PER_SEC;

294 -

295 - if(point_end_time_ut != rd->rrddim_store_metric_last_ut) {

296 - internal_error(true,

297 - "%s COLLECTION: 'host:%s/chart:%s/dim:%s' granularity %d, collection %zu, expected to store at tier 0 a value at %llu, but it gave %llu [%s%llu usec] (called from %s(), previously by %s())",

298 - (point_end_time_ut < rd->rrddim_store_metric_last_ut) ? "**PAST**" : "GAP",

299 - rrdhost_hostname(rd->rrdset->rrdhost), rrdset_id(rd->rrdset), rrddim_id(rd),

300 - rd->update_every,

301 - rd->rrddim_store_metric_count,

302 - expected, point_end_time_ut,

303 - (point_end_time_ut < rd->rrddim_store_metric_last_ut)?"by -" : "gap ",

304 - expected - point_end_time_ut,

305 - function,

306 - rd->rrddim_store_metric_last_caller?rd->rrddim_store_metric_last_caller:"none");

307 - }

308 - }

309 -

310 - rd->rrddim_store_metric_last_ut = point_end_time_ut;

311 - rd->rrddim_store_metric_last_caller = function;

312 -#endif // NETDATA_LOG_COLLECTION_ERRORS

313 -

314 - // store the metric on tier 0

315 - storage_engine_store_metric(rd->tiers[0].sch, point_end_time_ut,

316 - n, 0, 0,

317 - 1, 0, flags);

318 -

319 - rrdset_done_statistics_points_stored_per_tier[0]++;

320 -

321 - time_t now_s = (time_t)(point_end_time_ut / USEC_PER_SEC);

322 -

323 - STORAGE_POINT sp = {

324 - .start_time_s = now_s - rd->rrdset->update_every,

325 - .end_time_s = now_s,

326 - .min = n,

327 - .max = n,

328 - .sum = n,

329 - .count = 1,

330 - .anomaly_count = (flags & SN_FLAG_NOT_ANOMALOUS) ? 0 : 1,

331 - .flags = flags

332 - };

333 -

334 - for(size_t tier = 1; tier < nd_profile.storage_tiers;tier++) {

335 - if(unlikely(!rd->tiers[tier].smh)) continue;

336 -

337 - struct rrddim_tier *t = &rd->tiers[tier];

338 -

339 - if(!rrddim_option_check(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS)) {

340 - // we have not collected this tier before

341 - // let's fill any gap that may exist

342 - backfill_tier_from_smaller_tiers(rd, tier, now_s);

343 - }

344 -

345 - store_metric_at_tier(rd, tier, t, sp, point_end_time_ut);

346 - }

347 - rrddim_option_set(rd, RRDDIM_OPTION_BACKFILLED_HIGH_TIERS);

348 -

349 - rrdcontext_collected_rrddim(rd);

350 - log_stack_pop(&lgs);

351 -}

352 -

276 // caching of dimensions rrdset_done() and rrdset_done_interpolate() loop through

277 struct rda_item {

278 const DICTIONARY_ITEM *item;

@@ -360,7 +283,7 @@ struct rda_item {

283 static __thread struct rda_item *thread_rda = NULL;

284 static __thread size_t thread_rda_entries = 0;

285

363 -struct rda_item *rrdset_thread_rda_get(size_t *dimensions) {

286 +static struct rda_item *rrdset_thread_rda_get(size_t *dimensions) {

287

288 if(unlikely(!thread_rda || (*dimensions) > thread_rda_entries)) {

289 size_t old_mem = thread_rda_entries * sizeof(struct rda_item);

@@ -665,7 +588,7 @@ void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next)

588

589 // check if we will re-write the entire data set

590 if(unlikely(dt_usec(&st->last_collected_time, &st->last_updated) > st->db.entries * update_every_ut &&

668 - st->rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE)) {

591 + st->rrd_memory_mode != RRD_DB_MODE_DBENGINE)) {

592 nd_log_daemon(NDLP_DEBUG, "'%s': too old data (last updated at %" PRId64 ".%" PRId64 ", last collected at %" PRId64 ".%" PRId64 "). "

593 "Resetting it. Will not store the next entry.",

594 rrdset_id(st),

src/database/rrdset-collection.h

+8

@@ -7,6 +7,14 @@

7

8 extern __thread size_t rrdset_done_statistics_points_stored_per_tier[RRD_STORAGE_TIERS];

9

10 +void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);

11 +void rrdset_next_usec(RRDSET *st, usec_t microseconds);

12 +void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);

13 +#define rrdset_next(st) rrdset_next_usec(st, 0ULL)

14 +

15 +void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next);

16 +void rrdset_done(RRDSET *st);

17 +

18 void rrdset_finalize_collection(RRDSET *st, bool dimensions_too);

19 time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);

20

src/database/rrdset-index-id.c

+4 -3

@@ -28,7 +28,7 @@ struct rrdset_constructor {

28 long priority;

29 int update_every;

30 RRDSET_TYPE chart_type;

31 - RRD_MEMORY_MODE memory_mode;

31 + RRD_DB_MODE memory_mode;

32 long history_entries;

33

34 enum {

@@ -69,7 +69,7 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

69 st->module_name = rrd_string_strdupz(ctr->module);

70 st->priority = ctr->priority;

71

72 - st->db.entries = (ctr->memory_mode != RRD_MEMORY_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;

72 + st->db.entries = (ctr->memory_mode != RRD_DB_MODE_DBENGINE) ? align_entries_to_pagesize(ctr->memory_mode, ctr->history_entries) : 5;

73 st->update_every = ctr->update_every;

74 st->rrd_memory_mode = ctr->memory_mode;

75

@@ -378,7 +378,8 @@ RRDSET *rrdset_create_custom(

378 , long priority

379 , int update_every

380 , RRDSET_TYPE chart_type

381 - , RRD_MEMORY_MODE memory_mode

381 + ,

382 + RRD_DB_MODE memory_mode

383 , long history_entries

384 ) {

385 if (host != localhost)

src/database/rrdset-index-id.h

+16 -3

@@ -19,7 +19,8 @@ RRDSET *rrdset_create_custom(

19 , long priority

20 , int update_every

21 , RRDSET_TYPE chart_type

22 - , RRD_MEMORY_MODE memory_mode

22 + ,

23 + RRD_DB_MODE memory_mode

24 , long history_entries

25 );

26

@@ -59,8 +60,6 @@ static inline

60 localhost, type, id, name, family, context, title, units, plugin, module, priority, update_every, chart_type);

61 }

62

62 -int rrdset_reset_name(RRDSET *st, const char *name);

63 -

63 void rrdset_index_init(RRDHOST *host);

64 void rrdset_index_destroy(RRDHOST *host);

65

@@ -74,4 +73,18 @@ RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);

73 void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);

74 RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);

75

76 +#define rrdset_find_localhost(id) rrdset_find(localhost, id)

77 +/* This will not return charts that are archived */

78 +static inline RRDSET *rrdset_find_active_localhost(const char *id) {

79 + RRDSET *st = rrdset_find_localhost(id);

80 + return st;

81 +}

82 +

83 +#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)

84 +/* This will not return charts that are archived */

85 +static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id) {

86 + RRDSET *st = rrdset_find_bytype_localhost(type, id);

87 + return st;

88 +}

89 +

90 #endif //NETDATA_RRDSET_INDEX_ID_H

src/database/rrdset-index-name.c

+4 -2

@@ -4,6 +4,8 @@

4 #include "rrdset-index-id.h"

5 #include "rrdset-slots.h"

6

7 +static RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name);

8 +

9 STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name) {

10 if(!name || !*name) return NULL;

11

@@ -74,7 +76,7 @@ static void rrdset_name_delete_callback(const DICTIONARY_ITEM *item __maybe_unus

76 rrdset_flag_clear(st, RRDSET_FLAG_INDEXED_NAME);

77 }

78

77 -RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) {

79 +static RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name) {

80 if (unlikely(!host->rrdset_root_index_name))

81 return NULL;

82 return dictionary_get(host->rrdset_root_index_name, name);

@@ -101,7 +103,7 @@ void rrdset_index_del_name(RRDHOST *host, RRDSET *st) {

103 dictionary_del(host->rrdset_root_index_name, rrdset_name(st));

104 }

105

104 -inline RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {

106 +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name) {

107 netdata_log_debug(D_RRD_CALLS, "rrdset_find_byname() for chart '%s' in host '%s'", name, rrdhost_hostname(host));

108 RRDSET *st = rrdset_index_find_name(host, name);

109 return(st);

src/database/rrdset-index-name.h

+9 -1

@@ -11,7 +11,15 @@ void rrdset_index_byname_init(RRDHOST *host);

11 void rrdset_index_add_name(RRDHOST *host, RRDSET *st);

12 void rrdset_index_del_name(RRDHOST *host, RRDSET *st);

13

14 -RRDSET *rrdset_index_find_name(RRDHOST *host, const char *name);

14 +extern RRDHOST *localhost;

15 +RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);

16 +#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)

17 +

18 +/* This will not return charts that are archived */

19 +static inline RRDSET *rrdset_find_active_byname_localhost(const char *name) {

20 + RRDSET *st = rrdset_find_byname_localhost(name);

21 + return st;

22 +}

23

24 int rrdset_reset_name(RRDSET *st, const char *name);

25 STRING *rrdset_fix_name(RRDHOST *host, const char *chart_full_id, const char *type, const char *current_name, const char *name);

src/database/rrdset-slots.c

-36

@@ -41,25 +41,6 @@ void rrdset_stream_send_chart_slot_release(RRDSET *st) {

41 spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);

42 }

43

44 -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host) {

45 - rrd_slot_memory_removed(host->stream.snd.pluginsd_chart_slots.available.size * sizeof(uint32_t));

46 -

47 - spinlock_lock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);

48 - host->stream.snd.pluginsd_chart_slots.available.ignore = true;

49 - freez(host->stream.snd.pluginsd_chart_slots.available.array);

50 - host->stream.snd.pluginsd_chart_slots.available.array = NULL;

51 - host->stream.snd.pluginsd_chart_slots.available.used = 0;

52 - host->stream.snd.pluginsd_chart_slots.available.size = 0;

53 - spinlock_unlock(&host->stream.snd.pluginsd_chart_slots.available.spinlock);

54 -

55 - // zero all the slots on all charts, so that they will not attempt to access the array

56 - RRDSET *st;

57 - rrdset_foreach_read(st, host) {

58 - st->stream.snd.chart_slot = 0;

59 - }

60 - rrdset_foreach_done(st);

61 -}

62 -

44 void rrdset_pluginsd_receive_unslot(RRDSET *st) {

45 for(size_t i = 0; i < st->pluginsd.size ;i++) {

46 rrddim_acquired_release(st->pluginsd.prd_array[i].rda); // can be NULL

@@ -105,20 +86,3 @@ void rrdset_pluginsd_receive_slots_initialize(RRDSET *st) {

86 spinlock_init(&st->pluginsd.spinlock);

87 st->pluginsd.last_slot = -1;

88 }

108 -

109 -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host) {

110 - rrd_slot_memory_removed(host->stream.rcv.pluginsd_chart_slots.size * sizeof(uint32_t));

111 -

112 - spinlock_lock(&host->stream.rcv.pluginsd_chart_slots.spinlock);

113 -

114 - if(host->stream.rcv.pluginsd_chart_slots.array) {

115 - for (size_t s = 0; s < host->stream.rcv.pluginsd_chart_slots.size; s++)

116 - rrdset_pluginsd_receive_unslot_and_cleanup(host->stream.rcv.pluginsd_chart_slots.array[s]);

117 -

118 - freez(host->stream.rcv.pluginsd_chart_slots.array);

119 - host->stream.rcv.pluginsd_chart_slots.array = NULL;

120 - host->stream.rcv.pluginsd_chart_slots.size = 0;

121 - }

122 -

123 - spinlock_unlock(&host->stream.rcv.pluginsd_chart_slots.spinlock);

124 -}

src/database/rrdset-slots.h

-2

@@ -8,10 +8,8 @@

8 void rrdset_stream_send_chart_slot_assign(RRDSET *st);

9 void rrdset_stream_send_chart_slot_release(RRDSET *st);

10

11 -void rrdhost_pluginsd_send_chart_slots_free(RRDHOST *host);

11 void rrdset_pluginsd_receive_unslot(RRDSET *st);

12 void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);

13 void rrdset_pluginsd_receive_slots_initialize(RRDSET *st);

15 -void rrdhost_pluginsd_receive_chart_slots_free(RRDHOST *host);

14

15 #endif //NETDATA_RRDSET_SLOTS_H

src/database/rrdset-type.c new

+34

@@ -0,0 +1,34 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "rrdset-type.h"

4 +

5 +RRDSET_TYPE rrdset_type_id(const char *name) {

6 + if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))

7 + return RRDSET_TYPE_AREA;

8 +

9 + else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0))

10 + return RRDSET_TYPE_STACKED;

11 +

12 + else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0))

13 + return RRDSET_TYPE_HEATMAP;

14 +

15 + else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0))

16 + return RRDSET_TYPE_LINE;

17 +}

18 +

19 +const char *rrdset_type_name(RRDSET_TYPE chart_type) {

20 + switch(chart_type) {

21 + case RRDSET_TYPE_LINE:

22 + default:

23 + return RRDSET_TYPE_LINE_NAME;

24 +

25 + case RRDSET_TYPE_AREA:

26 + return RRDSET_TYPE_AREA_NAME;

27 +

28 + case RRDSET_TYPE_STACKED:

29 + return RRDSET_TYPE_STACKED_NAME;

30 +

31 + case RRDSET_TYPE_HEATMAP:

32 + return RRDSET_TYPE_HEATMAP_NAME;

33 + }

34 +}

src/database/rrdset-type.h new

+23

@@ -0,0 +1,23 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RRDSET_TYPE_H

4 +#define NETDATA_RRDSET_TYPE_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +typedef enum __attribute__ ((__packed__)) rrdset_type {

9 + RRDSET_TYPE_LINE = 0,

10 + RRDSET_TYPE_AREA = 1,

11 + RRDSET_TYPE_STACKED = 2,

12 + RRDSET_TYPE_HEATMAP = 3,

13 +} RRDSET_TYPE;

14 +

15 +#define RRDSET_TYPE_LINE_NAME "line"

16 +#define RRDSET_TYPE_AREA_NAME "area"

17 +#define RRDSET_TYPE_STACKED_NAME "stacked"

18 +#define RRDSET_TYPE_HEATMAP_NAME "heatmap"

19 +

20 +RRDSET_TYPE rrdset_type_id(const char *name);

21 +const char *rrdset_type_name(RRDSET_TYPE chart_type);

22 +

23 +#endif //NETDATA_RRDSET_TYPE_H

src/database/rrdset.c

+3 -38

@@ -9,41 +9,6 @@ void rrdset_metadata_updated(RRDSET *st) {

9 }

10

11 // ----------------------------------------------------------------------------

12 -// RRD - chart types

13 -

14 -inline RRDSET_TYPE rrdset_type_id(const char *name) {

15 - if(unlikely(strcmp(name, RRDSET_TYPE_AREA_NAME) == 0))

16 - return RRDSET_TYPE_AREA;

17 -

18 - else if(unlikely(strcmp(name, RRDSET_TYPE_STACKED_NAME) == 0))

19 - return RRDSET_TYPE_STACKED;

20 -

21 - else if(unlikely(strcmp(name, RRDSET_TYPE_HEATMAP_NAME) == 0))

22 - return RRDSET_TYPE_HEATMAP;

23 -

24 - else // if(unlikely(strcmp(name, RRDSET_TYPE_LINE_NAME) == 0))

25 - return RRDSET_TYPE_LINE;

26 -}

27 -

28 -const char *rrdset_type_name(RRDSET_TYPE chart_type) {

29 - switch(chart_type) {

30 - case RRDSET_TYPE_LINE:

31 - default:

32 - return RRDSET_TYPE_LINE_NAME;

33 -

34 - case RRDSET_TYPE_AREA:

35 - return RRDSET_TYPE_AREA_NAME;

36 -

37 - case RRDSET_TYPE_STACKED:

38 - return RRDSET_TYPE_STACKED_NAME;

39 -

40 - case RRDSET_TYPE_HEATMAP:

41 - return RRDSET_TYPE_HEATMAP_NAME;

42 - }

43 -}

44 -

45 -// ----------------------------------------------------------------------------

46 -// RRDSET - rename charts

12

13 // get the timestamp of the last entry in the round-robin database

14 time_t rrdset_last_entry_s(RRDSET *st) {

@@ -148,7 +113,7 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_

113 *last_time_s = db_last_entry_s;

114 }

115

151 -inline void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) {

116 +void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) {

117 if(!st) return;

118

119 rrdset_pluginsd_receive_unslot(st);

@@ -171,7 +136,7 @@ inline void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st) {

136 }

137 }

138

174 -inline void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) {

139 +void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) {

140 if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {

141

142 // netdata_log_info("Clearing obsolete flag on chart 'host:%s/chart:%s'",

@@ -188,7 +153,7 @@ inline void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st) {

153 }

154 }

155

191 -inline void rrdset_update_heterogeneous_flag(RRDSET *st) {

156 +void rrdset_update_heterogeneous_flag(RRDSET *st) {

157 RRDHOST *host = st->rrdhost;

158 (void)host;

159

src/database/rrdset.h

+21 -73

@@ -9,28 +9,12 @@ typedef struct rrdset RRDSET;

9 typedef struct rrdset_acquired RRDSET_ACQUIRED;

10 typedef struct ml_chart rrd_ml_chart_t;

11

12 +#include "rrdset-type.h"

13 #include "rrdlabels.h"

14 #include "rrd-database-mode.h"

15

16 // --------------------------------------------------------------------------------------------------------------------

17

17 -typedef enum __attribute__ ((__packed__)) rrdset_type {

18 - RRDSET_TYPE_LINE = 0,

19 - RRDSET_TYPE_AREA = 1,

20 - RRDSET_TYPE_STACKED = 2,

21 - RRDSET_TYPE_HEATMAP = 3,

22 -} RRDSET_TYPE;

23 -

24 -#define RRDSET_TYPE_LINE_NAME "line"

25 -#define RRDSET_TYPE_AREA_NAME "area"

26 -#define RRDSET_TYPE_STACKED_NAME "stacked"

27 -#define RRDSET_TYPE_HEATMAP_NAME "heatmap"

28 -

29 -RRDSET_TYPE rrdset_type_id(const char *name);

30 -const char *rrdset_type_name(RRDSET_TYPE chart_type);

31 -

32 -// --------------------------------------------------------------------------------------------------------------------

33 -

18 struct rrdhost;

19 struct rrdcalc;

20 struct pluginsd_rrddim;

@@ -128,7 +112,7 @@ struct rrdset {

112 // operational state members

113

114 RRDSET_FLAGS flags; // flags

131 - RRD_MEMORY_MODE rrd_memory_mode; // the db mode of this rrdset

115 + RRD_DB_MODE rrd_memory_mode; // the db mode of this rrdset

116

117 DICTIONARY *rrddim_root_index; // dimensions index

118

@@ -295,69 +279,33 @@ void rrdset_metadata_updated(RRDSET *st);

279 // --------------------------------------------------------------------------------------------------------------------

280

281 #ifdef NETDATA_INTERNAL_CHECKS

298 -#define rrdset_debug(st, fmt, args...) do { if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \

299 - netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); } while(0)

282 +#define rrdset_debug(st, fmt, args...) do { \

283 + if(unlikely(debug_flags & D_RRD_STATS && rrdset_flag_check(st, RRDSET_FLAG_DEBUG))) \

284 + netdata_logger(NDLS_DEBUG, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, "%s: " fmt, rrdset_name(st), ##args); \

285 + } while(0)

286 #else

287 #define rrdset_debug(st, fmt, args...) debug_dummy()

288 #endif

289

290 // --------------------------------------------------------------------------------------------------------------------

291

306 -#include "rrdset-index-name.h"

307 -#include "rrdset-index-id.h"

308 -#include "rrdset-slots.h"

309 -#include "rrdset-collection.h"

310 -

292 void rrdset_update_heterogeneous_flag(RRDSET *st);

293

313 -time_t rrdset_set_update_every_s(RRDSET *st, time_t update_every_s);

314 -

315 -RRDSET *rrdset_find(RRDHOST *host, const char *id);

316 -

317 -RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);

318 -RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);

319 -void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);

320 -

321 -#define rrdset_find_localhost(id) rrdset_find(localhost, id)

322 -/* This will not return charts that are archived */

323 -static inline RRDSET *rrdset_find_active_localhost(const char *id)

324 -{

325 - RRDSET *st = rrdset_find_localhost(id);

326 - return st;

327 -}

328 -

329 -RRDSET *rrdset_find_bytype(RRDHOST *host, const char *type, const char *id);

330 -#define rrdset_find_bytype_localhost(type, id) rrdset_find_bytype(localhost, type, id)

331 -/* This will not return charts that are archived */

332 -static inline RRDSET *rrdset_find_active_bytype_localhost(const char *type, const char *id)

333 -{

334 - RRDSET *st = rrdset_find_bytype_localhost(type, id);

335 - return st;

336 -}

337 -

338 -RRDSET *rrdset_find_byname(RRDHOST *host, const char *name);

339 -#define rrdset_find_byname_localhost(name) rrdset_find_byname(localhost, name)

340 -/* This will not return charts that are archived */

341 -static inline RRDSET *rrdset_find_active_byname_localhost(const char *name)

342 -{

343 - RRDSET *st = rrdset_find_byname_localhost(name);

344 - return st;

345 -}

346 -

347 -void rrdset_next_usec_unfiltered(RRDSET *st, usec_t microseconds);

348 -void rrdset_next_usec(RRDSET *st, usec_t microseconds);

349 -void rrdset_timed_next(RRDSET *st, struct timeval now, usec_t microseconds);

350 -#define rrdset_next(st) rrdset_next_usec(st, 0ULL)

351 -

352 -void rrdset_timed_done(RRDSET *st, struct timeval now, bool pending_rrdset_next);

353 -void rrdset_done(RRDSET *st);

354 -

294 void rrdset_is_obsolete___safe_from_collector_thread(RRDSET *st);

295 void rrdset_isnot_obsolete___safe_from_collector_thread(RRDSET *st);

296

297 // checks if the RRDSET should be offered to viewers

359 -#define rrdset_is_available_for_viewers(st) (!rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st) && (st)->rrd_memory_mode != RRD_MEMORY_MODE_NONE)

360 -#define rrdset_is_available_for_exporting_and_alarms(st) (!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && rrdset_number_of_dimensions(st))

298 +#define rrdset_is_available_for_viewers(st) ( \

299 + !rrdset_flag_check(st, RRDSET_FLAG_HIDDEN) && \

300 + !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \

301 + rrdset_number_of_dimensions(st) && \

302 + (st)->rrd_memory_mode != RRD_DB_MODE_NONE \

303 + )

304 +

305 +#define rrdset_is_available_for_exporting_and_alarms(st) ( \

306 + !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE) && \

307 + rrdset_number_of_dimensions(st) \

308 + )

309

310 time_t rrdset_first_entry_s(RRDSET *st);

311 time_t rrdset_first_entry_s_of_tier(RRDSET *st, size_t tier);

@@ -368,9 +316,9 @@ void rrdset_get_retention_of_tier_for_collected_chart(RRDSET *st, time_t *first_

316

317 void rrdset_update_rrdlabels(RRDSET *st, RRDLABELS *new_rrdlabels);

318

371 -void rrdset_free(RRDSET *st);

372 -

373 -void rrdset_pluginsd_receive_unslot_and_cleanup(RRDSET *st);

374 -void rrdset_pluginsd_receive_unslot(RRDSET *st);

319 +#include "rrdset-index-name.h"

320 +#include "rrdset-index-id.h"

321 +#include "rrdset-slots.h"

322 +#include "rrdset-collection.h"

323

324 #endif //NETDATA_RRDSET_H

src/database/sqlite/sqlite_functions.h

+1 -1

@@ -51,7 +51,7 @@ void analytics_set_data_str(char **name, const char *value);

51 #define REQUIRE_DB(db) \

52 ({ \

53 if (unlikely(!(db))) { \

54 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) \

54 + if (default_rrd_memory_mode == RRD_DB_MODE_DBENGINE) \

55 error_report("Database has not been initialized in %s", __FUNCTION__); \

56 } \

57 (db) != NULL; \

src/database/storage-engine.c

+10 -10

@@ -8,8 +8,8 @@

8

9 static STORAGE_ENGINE engines[] = {

10 {

11 - .id = RRD_MEMORY_MODE_NONE,

12 - .name = RRD_MEMORY_MODE_NONE_NAME,

11 + .id = RRD_DB_MODE_NONE,

12 + .name = RRD_DB_MODE_NONE_NAME,

13 .seb = STORAGE_ENGINE_BACKEND_RRDDIM,

14 .api = {

15 .metric_get_by_id = rrddim_metric_get_by_id,

@@ -22,8 +22,8 @@ static STORAGE_ENGINE engines[] = {

22 }

23 },

24 {

25 - .id = RRD_MEMORY_MODE_RAM,

26 - .name = RRD_MEMORY_MODE_RAM_NAME,

25 + .id = RRD_DB_MODE_RAM,

26 + .name = RRD_DB_MODE_RAM_NAME,

27 .seb = STORAGE_ENGINE_BACKEND_RRDDIM,

28 .api = {

29 .metric_get_by_id = rrddim_metric_get_by_id,

@@ -36,8 +36,8 @@ static STORAGE_ENGINE engines[] = {

36 }

37 },

38 {

39 - .id = RRD_MEMORY_MODE_ALLOC,

40 - .name = RRD_MEMORY_MODE_ALLOC_NAME,

39 + .id = RRD_DB_MODE_ALLOC,

40 + .name = RRD_DB_MODE_ALLOC_NAME,

41 .seb = STORAGE_ENGINE_BACKEND_RRDDIM,

42 .api = {

43 .metric_get_by_id = rrddim_metric_get_by_id,

@@ -51,8 +51,8 @@ static STORAGE_ENGINE engines[] = {

51 },

52 #ifdef ENABLE_DBENGINE

53 {

54 - .id = RRD_MEMORY_MODE_DBENGINE,

55 - .name = RRD_MEMORY_MODE_DBENGINE_NAME,

54 + .id = RRD_DB_MODE_DBENGINE,

55 + .name = RRD_DB_MODE_DBENGINE_NAME,

56 .seb = STORAGE_ENGINE_BACKEND_DBENGINE,

57 .api = {

58 .metric_get_by_id = rrdeng_metric_get_by_id,

@@ -65,7 +65,7 @@ static STORAGE_ENGINE engines[] = {

65 }

66 },

67 #endif

68 - { .id = RRD_MEMORY_MODE_NONE, .name = NULL }

68 + { .id = RRD_DB_MODE_NONE, .name = NULL }

69 };

70

71 STORAGE_ENGINE* storage_engine_find(const char* name)

@@ -77,7 +77,7 @@ STORAGE_ENGINE* storage_engine_find(const char* name)

77 return NULL;

78 }

79

80 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode)

80 +STORAGE_ENGINE* storage_engine_get(RRD_DB_MODE mmode)

81 {

82 for (STORAGE_ENGINE* it = engines; it->name; it++) {

83 if (it->id == mmode)

src/database/storage-engine.h

+2 -2

@@ -70,12 +70,12 @@ typedef struct storage_engine_api {

70

71 typedef struct storage {

72 STORAGE_ENGINE_BACKEND seb;

73 - RRD_MEMORY_MODE id;

73 + RRD_DB_MODE id;

74 const char* name;

75 STORAGE_ENGINE_API api;

76 } STORAGE_ENGINE;

77

78 -STORAGE_ENGINE* storage_engine_get(RRD_MEMORY_MODE mmode);

78 +STORAGE_ENGINE* storage_engine_get(RRD_DB_MODE mmode);

79 STORAGE_ENGINE* storage_engine_find(const char* name);

80

81 // Iterator over existing engines

src/exporting/check_filters.c

+1 -1

@@ -79,7 +79,7 @@ int rrdset_is_exportable(struct instance *instance, RRDSET *st)

79 return 0;

80 }

81

82 - if(unlikely(st->rrd_memory_mode == RRD_MEMORY_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {

82 + if(unlikely(st->rrd_memory_mode == RRD_DB_MODE_NONE && !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {

83 netdata_log_debug(D_EXPORTING, "EXPORTING: not sending chart '%s' of host '%s' because its memory mode is '%s' and the exporting engine requires database access.", rrdset_id(st), rrdhost_hostname(host), rrd_memory_mode_name(host->rrd_memory_mode));

84 return 0;

85 }

src/exporting/prometheus/prometheus.c

+1 -1

@@ -66,7 +66,7 @@ inline int can_send_rrdset(struct instance *instance, RRDSET *st, SIMPLE_PATTERN

66 }

67

68 if (unlikely(

69 - st->rrd_memory_mode == RRD_MEMORY_MODE_NONE &&

69 + st->rrd_memory_mode == RRD_DB_MODE_NONE &&

70 !(EXPORTING_OPTIONS_DATA_SOURCE(instance->config.options) == EXPORTING_SOURCE_DATA_AS_COLLECTED))) {

71 netdata_log_debug(

72 D_EXPORTING,

src/ml/ml_config.cc

+1 -1

@@ -144,7 +144,7 @@ void ml_config_load(ml_config_t *cfg) {

144

145 cfg->enable_statistics_charts = enable_statistics_charts;

146

147 - if (cfg->enable_anomaly_detection == CONFIG_BOOLEAN_AUTO && default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE) {

147 + if (cfg->enable_anomaly_detection == CONFIG_BOOLEAN_AUTO && default_rrd_memory_mode != RRD_DB_MODE_DBENGINE) {

148 Cfg.enable_anomaly_detection = 0;

149 config_set_boolean(config_section_ml, "enabled", CONFIG_BOOLEAN_NO);

150 return;

src/streaming/stream-conf.c

+1 -1

@@ -233,7 +233,7 @@ void stream_conf_receiver_config(struct receiver_state *rpt, struct stream_recei

233 appconfig_get(&stream_config, api_key, "db",

234 rrd_memory_mode_name(default_rrd_memory_mode))));

235

236 - if (unlikely(config->mode == RRD_MEMORY_MODE_DBENGINE && !dbengine_enabled)) {

236 + if (unlikely(config->mode == RRD_DB_MODE_DBENGINE && !dbengine_enabled)) {

237 netdata_log_error("STREAM RCV '%s' [from [%s]:%s]: "

238 "dbengine is not enabled, falling back to default."

239 , rpt->hostname

src/streaming/stream-conf.h

+1 -1

@@ -52,7 +52,7 @@ struct _stream_receive {

52 extern struct _stream_receive stream_receive;

53

54 struct stream_receiver_config {

55 - RRD_MEMORY_MODE mode;

55 + RRD_DB_MODE mode;

56 bool ephemeral;

57 int history;

58 int update_every;

src/streaming/stream-parents.h

+6 -7

@@ -17,17 +17,11 @@ typedef struct rrdhost_stream_parents {

17 } RRDHOST_STREAM_PARENTS;

18

19 #include "stream-handshake.h"

20 -#include "database/rrdhost-status.h"

20

21 void rrdhost_stream_parent_ssl_init(struct sender_state *s);

22

23 int stream_info_to_json_v1(BUFFER *wb, const char *machine_guid);

24

26 -void rrdhost_stream_parents_reset(RRDHOST *host, STREAM_HANDSHAKE reason);

27 -

28 -void rrdhost_stream_parents_update_from_destination(RRDHOST *host);

29 -void rrdhost_stream_parents_free(struct rrdhost *host, bool having_write_lock);

30 -

25 bool stream_parent_connect_to_one(

26 ND_SOCK *sender_sock,

27 struct rrdhost *host,

@@ -46,6 +40,11 @@ bool stream_parent_is_ssl(STREAM_PARENT *d);

40

41 usec_t stream_parent_handshake_error_to_json(BUFFER *wb, struct rrdhost *host);

42

49 -void rrdhost_stream_parents_init(RRDHOST *host);

43 +void rrdhost_stream_parents_reset(struct rrdhost *host, STREAM_HANDSHAKE reason);

44 +

45 +void rrdhost_stream_parents_update_from_destination(struct rrdhost *host);

46 +void rrdhost_stream_parents_free(struct rrdhost *host, bool having_write_lock);

47 +

48 +void rrdhost_stream_parents_init(struct rrdhost *host);

49

50 #endif //NETDATA_STREAM_PARENTS_H

src/streaming/stream.h

+1 -1

@@ -21,7 +21,7 @@ void stream_sender_start_host(struct rrdhost *host);

21 void stream_sender_signal_to_stop_and_wait(struct rrdhost *host, STREAM_HANDSHAKE reason, bool wait);

22

23 // managing host sender structures

24 -void stream_sender_structures_init(RRDHOST *host, bool stream, STRING *parents, STRING *api_key, STRING *send_charts_matching);

24 +void stream_sender_structures_init(struct rrdhost *host, bool stream, STRING *parents, STRING *api_key, STRING *send_charts_matching);

25 void stream_sender_structures_free(struct rrdhost *host);

26

27 // querying host sender information

src/web/api/queries/KolmogorovSmirnovDist.c renamed

src/web/api/queries/KolmogorovSmirnovDist.h renamed

src/web/api/queries/weights.c

+1 -1

@@ -1,7 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 #include "daemon/common.h"

4 -#include "database/KolmogorovSmirnovDist.h"

4 +#include "KolmogorovSmirnovDist.h"

5

6 #define MAX_POINTS 10000

7 int metric_correlations_version = 1;

RRD files split, renames, cleanup Part 2 (#19405)