fad05b0b4 · cryptotaxi247/netdata-1

CMakeLists.txt

+6 -2

@@ -1086,8 +1086,8 @@ set(DAEMON_FILES

1086 src/daemon/pulse/pulse-daemon-memory.h

1087 src/daemon/pulse/pulse-sqlite3.c

1088 src/daemon/pulse/pulse-sqlite3.h

1089 - src/daemon/pulse/pulse-dbengine.c

1090 - src/daemon/pulse/pulse-dbengine.h

1089 + src/daemon/pulse/pulse-db-dbengine.c

1090 + src/daemon/pulse/pulse-db-dbengine.h

1091 src/daemon/pulse/pulse-string.c

1092 src/daemon/pulse/pulse-string.h

1093 src/daemon/pulse/pulse-heartbeat.c

@@ -1118,6 +1118,10 @@ set(DAEMON_FILES

1118 src/daemon/daemon-shutdown.h

1119 src/daemon/daemon-service.c

1120 src/daemon/daemon-service.h

1121 + src/daemon/pulse/pulse-db-rrd.c

1122 + src/daemon/pulse/pulse-db-rrd.h

1123 + src/daemon/config/netdata-conf-cloud.c

1124 + src/daemon/config/netdata-conf-cloud.h

1125 )

1126

1127 set(H2O_FILES

src/daemon/analytics.c

+1 -1

@@ -519,7 +519,7 @@ void analytics_gather_mutable_meta_data(void)

519 analytics_alarms_notifications();

520

521 analytics_set_data(

522 - &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || stream_conf_configured_as_parent()) ? "true" : "false");

522 + &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || stream_conf_is_parent(false)) ? "true" : "false");

523

524 analytics_set_data(&analytics_data.netdata_host_agent_claimed, is_agent_claimed() ? "true" : "false");

525

src/daemon/config/netdata-conf-backwards-compatibility.c

+8 -5

@@ -132,11 +132,11 @@ void netdata_conf_backwards_compatibility(void) {

132 config_move(CONFIG_SECTION_DB, "page cache size",

133 CONFIG_SECTION_DB, "dbengine page cache size MB");

134

135 - config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc",

136 - CONFIG_SECTION_DB, "dbengine page cache with malloc");

137 -

138 - config_move(CONFIG_SECTION_DB, "page cache with malloc",

139 - CONFIG_SECTION_DB, "dbengine page cache with malloc");

135 +// config_move(CONFIG_SECTION_GLOBAL, "page cache uses malloc",

136 +// CONFIG_SECTION_DB, "dbengine page cache with malloc");

137 +//

138 +// config_move(CONFIG_SECTION_DB, "page cache with malloc",

139 +// CONFIG_SECTION_DB, "dbengine page cache with malloc");

140

141 config_move(CONFIG_SECTION_GLOBAL, "memory deduplication (ksm)",

142 CONFIG_SECTION_DB, "memory deduplication (ksm)");

@@ -177,6 +177,9 @@ void netdata_conf_backwards_compatibility(void) {

177 config_move(CONFIG_SECTION_GLOBAL, "enable zero metrics",

178 CONFIG_SECTION_DB, "enable zero metrics");

179

180 + config_move(CONFIG_SECTION_CLOUD, "query thread count",

181 + CONFIG_SECTION_CLOUD, "query threads");

182 +

183 // ----------------------------------------------------------------------------------------------------------------

184 // global statistics -> telemetry -> pulse

185

src/daemon/config/netdata-conf-cloud.c new

+18

@@ -0,0 +1,18 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "netdata-conf-cloud.h"

4 +#include "../common.h"

5 +

6 +int netdata_conf_cloud_query_threads(void) {

7 + int cpus = MIN(get_netdata_cpus(), 256); // max 256 cores

8 + int threads = MIN(cpus * (stream_conf_is_parent(false) ? 2 : 1), libuv_worker_threads / 2);

9 + threads = MAX(threads, 6);

10 +

11 + threads = config_get_number(CONFIG_SECTION_CLOUD, "query threads", threads);

12 + if(threads < 1) {

13 + netdata_log_error("[" CONFIG_SECTION_CLOUD "].query threads in netdata.conf needs to be at least 1. Overwriting it.");

14 + threads = 1;

15 + config_set_number(CONFIG_SECTION_CLOUD, "query threads", threads);

16 + }

17 + return threads;

18 +}

src/daemon/config/netdata-conf-cloud.h new

+10

@@ -0,0 +1,10 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_NETDATA_CONF_CLOUD_H

4 +#define NETDATA_NETDATA_CONF_CLOUD_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +int netdata_conf_cloud_query_threads(void);

9 +

10 +#endif //NETDATA_NETDATA_CONF_CLOUD_H

src/daemon/config/netdata-conf-db.c

+2 -2

@@ -373,13 +373,13 @@ void netdata_conf_section_db(void) {

373 // get default database size

374

375 if(default_rrd_memory_mode != RRD_MEMORY_MODE_DBENGINE && default_rrd_memory_mode != RRD_MEMORY_MODE_NONE) {

376 - default_rrd_history_entries = (int)config_get_number(

376 + default_rrd_history_entries = (int)config_get_duration_seconds(

377 CONFIG_SECTION_DB, "retention",

378 align_entries_to_pagesize(default_rrd_memory_mode, RRD_DEFAULT_HISTORY_ENTRIES));

379

380 long h = align_entries_to_pagesize(default_rrd_memory_mode, default_rrd_history_entries);

381 if (h != default_rrd_history_entries) {

382 - config_set_number(CONFIG_SECTION_DB, "retention", h);

382 + config_set_duration_seconds(CONFIG_SECTION_DB, "retention", h);

383 default_rrd_history_entries = (int)h;

384 }

385 }

src/daemon/config/netdata-conf-web.c

+22

@@ -3,6 +3,27 @@

3 #include "netdata-conf-web.h"

4 #include "daemon/static_threads.h"

5

6 +int netdata_conf_web_query_threads(void) {

7 + // See https://github.com/netdata/netdata/issues/11081#issuecomment-831998240 for more details

8 + if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) {

9 + config_set_number(CONFIG_SECTION_WEB, "web server threads", 1);

10 + netdata_log_info("You are running an OpenSSL older than 1.1.0, web server will not enable multithreading.");

11 + return 1;

12 + }

13 +

14 + int cpus = MIN(get_netdata_cpus(), 256); // max 256 cores

15 + int threads = cpus * (stream_conf_is_parent(false) ? 2 : 1);

16 + threads = MAX(threads, 6);

17 +

18 + threads = config_get_number(CONFIG_SECTION_WEB, "web server threads", threads);

19 + if(threads < 1) {

20 + netdata_log_error("[" CONFIG_SECTION_WEB "].web server threads in netdata.conf needs to be at least 1. Overwriting it.");

21 + threads = 1;

22 + config_set_number(CONFIG_SECTION_WEB, "web server threads", threads);

23 + }

24 + return threads;

25 +}

26 +

27 static int make_dns_decision(const char *section_name, const char *config_name, const char *default_value, SIMPLE_PATTERN *p) {

28 const char *value = config_get(section_name,config_name,default_value);

29

@@ -141,3 +162,4 @@ void netdata_conf_web_security_init(void) {

162

163 netdata_ssl_initialize_openssl();

164 }

165 +

src/daemon/config/netdata-conf-web.h

+2

@@ -9,4 +9,6 @@ void netdata_conf_section_web(void);

9 void web_server_threading_selection(void);

10 void netdata_conf_web_security_init(void);

11

12 +int netdata_conf_web_query_threads(void);

13 +

14 #endif //NETDATA_NETDATA_CONF_WEB_H

src/daemon/config/netdata-conf.h

+1

@@ -14,6 +14,7 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user);

14 #include "netdata-conf-global.h"

15 #include "netdata-conf-logs.h"

16 #include "netdata-conf-web.h"

17 +#include "netdata-conf-cloud.h"

18

19 #include "daemon/common.h"

20

src/daemon/pulse/pulse-daemon-memory.c

+15 -10

@@ -12,7 +12,9 @@ struct netdata_buffers_statistics netdata_buffers_statistics = {};

12 void pulse_daemon_memory_do(bool extended) {

13 {

14 static RRDSET *st_memory = NULL;

15 - static RRDDIM *rd_database = NULL;

15 + static RRDDIM *rd_db_dbengine = NULL;

16 + static RRDDIM *rd_db_rrd = NULL;

17 +

18 #ifdef DICT_WITH_STATS

19 static RRDDIM *rd_collectors = NULL;

20 static RRDDIM *rd_rrdhosts = NULL;

@@ -50,7 +52,8 @@ void pulse_daemon_memory_do(bool extended) {

52 localhost->rrd_update_every,

53 RRDSET_TYPE_STACKED);

54

53 - rd_database = rrddim_add(st_memory, "db", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

55 + rd_db_dbengine = rrddim_add(st_memory, "dbengine", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

56 + rd_db_rrd = rrddim_add(st_memory, "rrd", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

57

58 #ifdef DICT_WITH_STATS

59 rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

@@ -75,8 +78,9 @@ void pulse_daemon_memory_do(bool extended) {

78 rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

79 }

80

81 + // each of these should also be analyzed below at the buffers chart

82 size_t buffers =

79 - netdata_buffers_statistics.query_targets_size +

83 + netdata_buffers_statistics.query_targets_size + onewayalloc_allocated_memory() +

84 netdata_buffers_statistics.rrdset_done_rda_size +

85 netdata_buffers_statistics.buffers_aclk +

86 netdata_buffers_statistics.buffers_api +

@@ -92,8 +96,8 @@ void pulse_daemon_memory_do(bool extended) {

96 size_t strings = 0;

97 string_statistics(NULL, NULL, NULL, NULL, NULL, &strings, NULL, NULL);

98

95 - rrddim_set_by_pointer(st_memory, rd_database,

96 - (collected_number)pulse_dbengine_total_memory + (collected_number)rrddim_db_memory_size);

99 + rrddim_set_by_pointer(st_memory, rd_db_dbengine, (collected_number)pulse_dbengine_total_memory);

100 + rrddim_set_by_pointer(st_memory, rd_db_rrd, (collected_number)pulse_rrd_memory_size);

101

102 #ifdef DICT_WITH_STATS

103 rrddim_set_by_pointer(st_memory, rd_collectors,

@@ -157,11 +161,11 @@ void pulse_daemon_memory_do(bool extended) {

161 rrddim_set_by_pointer(st_memory, rd_aral,

162 (collected_number)aral_by_size_structures_bytes());

163

160 - rrddim_set_by_pointer(st_memory,

161 - rd_judy, (collected_number) judy_aral_structures());

164 + rrddim_set_by_pointer(st_memory, rd_judy,

165 + (collected_number) judy_aral_structures());

166

163 - rrddim_set_by_pointer(st_memory,

164 - rd_other, (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other));

167 + rrddim_set_by_pointer(st_memory, rd_other,

168 + (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other));

169

170 rrdset_done(st_memory);

171 }

@@ -221,6 +225,7 @@ void pulse_daemon_memory_do(bool extended) {

225 rd_buffers_judy = rrddim_add(st_memory_buffers, "aral-judy free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);

226 }

227

228 + // the sum of all these needs to be above at the total buffers calculation

229 rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory());

230 rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size);

231 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk);

@@ -231,8 +236,8 @@ void pulse_daemon_memory_do(bool extended) {

236 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health);

237 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming);

238 rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming);

234 - rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers());

239 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web);

240 + rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_allocated_buffers());

241 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)aral_by_size_free_bytes());

242 rrddim_set_by_pointer(st_memory_buffers, rd_buffers_judy, (collected_number)judy_aral_free_bytes());

243

src/daemon/pulse/pulse-db-dbengine.c renamed

+5 -5

@@ -1,7 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 #define PULSE_INTERNALS 1

4 -#include "pulse-dbengine.h"

4 +#include "pulse-db-dbengine.h"

5

6 size_t pulse_dbengine_total_memory = 0;

7

@@ -638,11 +638,7 @@ static void dbengine2_cache_statistics_charts(struct dbengine2_cache_pointers *p

638 }

639 }

640

641 -

641 void pulse_dbengine_do(bool extended) {

643 - if(!main_cache || !main_mrg || !extended)

644 - return;

645 -

642 static struct dbengine2_cache_pointers main_cache_ptrs = {}, open_cache_ptrs = {}, extent_cache_ptrs = {};

643 static struct rrdeng_cache_efficiency_stats cache_efficiency_stats = {}, cache_efficiency_stats_old = {};

644 static struct pgc_statistics pgc_main_stats = {}, pgc_main_stats_old = {}; (void)pgc_main_stats_old;

@@ -686,6 +682,10 @@ void pulse_dbengine_do(bool extended) {

682 mrg_stats.size +

683 buffers_total_size + aral_structures_total_size + aral_padding_total_size + pgd_padding_bytes();

684

685 + // we need all the above for the total dbengine memory as reported by the non-extended netdata memory chart

686 + if(!main_cache || !main_mrg || !extended)

687 + return;

688 +

689 size_t priority = 135000;

690

691 {

src/daemon/pulse/pulse-db-dbengine.h renamed

+3 -3

@@ -1,7 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 -#ifndef NETDATA_PULSE_DBENGINE_H

4 -#define NETDATA_PULSE_DBENGINE_H

3 +#ifndef NETDATA_PULSE_DB_DBENGINE_H

4 +#define NETDATA_PULSE_DB_DBENGINE_H

5

6 #include "daemon/common.h"

7

@@ -14,4 +14,4 @@ void pulse_dbengine_do(bool extended);

14

15 #endif

16

17 -#endif //NETDATA_PULSE_DBENGINE_H

17 +#endif //NETDATA_PULSE_DB_DBENGINE_H

src/daemon/pulse/pulse-db-rrd.c new

+18

@@ -0,0 +1,18 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "pulse-db-rrd.h"

4 +

5 +int64_t pulse_rrd_memory_size = 0;

6 +

7 +void pulse_db_rrd_memory_change(int64_t value) {

8 + __atomic_add_fetch(&pulse_rrd_memory_size, value, __ATOMIC_RELAXED);

9 +}

10 +

11 +void pulse_db_rrd_memory_add(uint64_t value) {

12 + __atomic_add_fetch(&pulse_rrd_memory_size, value, __ATOMIC_RELAXED);

13 +}

14 +

15 +void pulse_db_rrd_memory_sub(uint64_t value) {

16 + __atomic_sub_fetch(&pulse_rrd_memory_size, value, __ATOMIC_RELAXED);

17 +}

18 +

src/daemon/pulse/pulse-db-rrd.h new

+16

@@ -0,0 +1,16 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_PULSE_DB_RRD_H

4 +#define NETDATA_PULSE_DB_RRD_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +void pulse_db_rrd_memory_change(int64_t value);

9 +void pulse_db_rrd_memory_add(uint64_t value);

10 +void pulse_db_rrd_memory_sub(uint64_t value);

11 +

12 +#if defined(PULSE_INTERNALS)

13 +extern int64_t pulse_rrd_memory_size;

14 +#endif

15 +

16 +#endif //NETDATA_PULSE_DB_RRD_H

src/daemon/pulse/pulse-dictionary.c

-2

@@ -13,8 +13,6 @@ struct dictionary_stats dictionary_stats_category_rrdhealth = { .name = "health"

13 struct dictionary_stats dictionary_stats_category_functions = { .name = "functions" };

14 struct dictionary_stats dictionary_stats_category_replication = { .name = "replication" };

15

16 -size_t rrddim_db_memory_size = 0;

17 -

16 #ifdef DICT_WITH_STATS

17 struct dictionary_categories {

18 struct dictionary_stats *stats;

src/daemon/pulse/pulse-dictionary.h

-2

@@ -15,8 +15,6 @@ extern struct dictionary_stats dictionary_stats_category_rrdhealth;

15 extern struct dictionary_stats dictionary_stats_category_functions;

16 extern struct dictionary_stats dictionary_stats_category_replication;

17

18 -extern size_t rrddim_db_memory_size;

19 -

18 #if defined(PULSE_INTERNALS)

19 void pulse_dictionary_do(bool extended);

20 #endif

src/daemon/pulse/pulse.h

+2 -1

@@ -16,7 +16,8 @@ extern bool pulse_extended_enabled;

16 #include "pulse-daemon.h"

17 #include "pulse-daemon-memory.h"

18 #include "pulse-sqlite3.h"

19 -#include "pulse-dbengine.h"

19 +#include "pulse-db-dbengine.h"

20 +#include "pulse-db-rrd.h"

21 #include "pulse-string.h"

22 #include "pulse-heartbeat.h"

23 #include "pulse-dictionary.h"

src/database/ram/rrddim_mem.c

+10 -6

@@ -52,7 +52,9 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si __maybe_unused) {

52 struct mem_metric_handle *mh = (struct mem_metric_handle *)rrddim_metric_get(si, &rd->metric_uuid);

53 while(!mh) {

54 netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);

55 + JudyAllocThreadPulseReset();

56 Pvoid_t *PValue = JudyHSIns(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(nd_uuid_t), PJE0);

57 + int64_t mem = JudyAllocThreadPulseGetAndReset();

58 mh = *PValue;

59 if(!mh) {

60 mh = callocz(1, sizeof(struct mem_metric_handle));

@@ -60,7 +62,7 @@ rrddim_metric_get_or_create(RRDDIM *rd, STORAGE_INSTANCE *si __maybe_unused) {

62 mh->refcount = 1;

63 update_metric_handle_from_rrddim(mh, rd);

64 *PValue = mh;

63 - __atomic_add_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED);

65 + pulse_db_rrd_memory_change(mem + (int64_t)sizeof(struct mem_metric_handle));

66 }

67 else {

68 if(__atomic_add_fetch(&mh->refcount, 1, __ATOMIC_RELAXED) <= 0)

@@ -107,11 +109,13 @@ void rrddim_metric_release(STORAGE_METRIC_HANDLE *smh __maybe_unused) {

109

110 RRDDIM *rd = mh->rd;

111 netdata_rwlock_wrlock(&rrddim_JudyHS_rwlock);

112 + JudyAllocThreadPulseReset();

113 JudyHSDel(&rrddim_JudyHS_array, &rd->metric_uuid, sizeof(nd_uuid_t), PJE0);

114 + int64_t mem = JudyAllocThreadPulseGetAndReset();

115 netdata_rwlock_wrunlock(&rrddim_JudyHS_rwlock);

116

117 freez(mh);

114 - __atomic_sub_fetch(&rrddim_db_memory_size, sizeof(struct mem_metric_handle) + JUDYHS_INDEX_SIZE_ESTIMATE(sizeof(nd_uuid_t)), __ATOMIC_RELAXED);

118 + pulse_db_rrd_memory_change(mem - (int64_t)sizeof(struct mem_metric_handle));

119 }

120 }

121 }

@@ -147,7 +151,7 @@ STORAGE_COLLECT_HANDLE *rrddim_collect_init(STORAGE_METRIC_HANDLE *smh, uint32_t

151 ch->rd = rd;

152 ch->smh = smh;

153

150 - __atomic_add_fetch(&rrddim_db_memory_size, sizeof(struct mem_collect_handle), __ATOMIC_RELAXED);

154 + pulse_db_rrd_memory_add(sizeof(struct mem_collect_handle));

155

156 return (STORAGE_COLLECT_HANDLE *)ch;

157 }

@@ -235,7 +239,7 @@ void rrddim_collect_store_metric(STORAGE_COLLECT_HANDLE *sch,

239

240 int rrddim_collect_finalize(STORAGE_COLLECT_HANDLE *sch) {

241 freez(sch);

238 - __atomic_sub_fetch(&rrddim_db_memory_size, sizeof(struct mem_collect_handle), __ATOMIC_RELAXED);

242 + pulse_db_rrd_memory_sub(sizeof(struct mem_collect_handle));

243 return 0;

244 }

245

@@ -355,7 +359,7 @@ void rrddim_query_init(STORAGE_METRIC_HANDLE *smh, struct storage_engine_query_h

359

360 // netdata_log_info("RRDDIM QUERY INIT: start %ld, end %ld, next %ld, first %ld, last %ld, dt %ld", start_time, end_time, h->next_timestamp, h->slot_timestamp, h->last_timestamp, h->dt);

361

358 - __atomic_add_fetch(&rrddim_db_memory_size, sizeof(struct mem_query_handle), __ATOMIC_RELAXED);

362 + pulse_db_rrd_memory_add(sizeof(struct mem_query_handle));

363 seqh->handle = (STORAGE_QUERY_HANDLE *)h;

364 }

365

@@ -419,7 +423,7 @@ void rrddim_query_finalize(struct storage_engine_query_handle *seqh) {

423

424 #endif

425 freez(seqh->handle);

422 - __atomic_sub_fetch(&rrddim_db_memory_size, sizeof(struct mem_query_handle), __ATOMIC_RELAXED);

426 + pulse_db_rrd_memory_sub(sizeof(struct mem_query_handle));

427 }

428

429 time_t rrddim_query_align_to_optimal_before(struct storage_engine_query_handle *seqh) {

src/database/rrddim.c

+7 -7

@@ -63,13 +63,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

63 if(!entries) entries = 5;

64

65 rd->db.data = netdata_mmap(NULL, entries * sizeof(storage_number), MAP_PRIVATE, 1, false, NULL);

66 - if(!rd->db.data) {

67 - netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));

68 - ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;

66 + if(rd->db.data) {

67 + rd->db.memsize = entries * sizeof(storage_number);

68 + pulse_db_rrd_memory_add(rd->db.memsize);

69 }

70 else {

71 - rd->db.memsize = entries * sizeof(storage_number);

72 - __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

71 + netdata_log_info("Failed to use memory mode ram for chart '%s', dimension '%s', falling back to alloc", rrdset_name(st), rrddim_name(rd));

72 + ctr->memory_mode = RRD_MEMORY_MODE_ALLOC;

73 }

74 }

75

@@ -79,7 +79,7 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

79

80 rd->db.data = rrddim_alloc_db(entries);

81 rd->db.memsize = entries * sizeof(storage_number);

82 - __atomic_add_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

82 + pulse_db_rrd_memory_add(rd->db.memsize);

83 }

84

85 rd->rrd_memory_mode = ctr->memory_mode;

@@ -223,7 +223,7 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v

223 }

224

225 if(rd->db.data) {

226 - __atomic_sub_fetch(&rrddim_db_memory_size, rd->db.memsize, __ATOMIC_RELAXED);

226 + pulse_db_rrd_memory_sub(rd->db.memsize);

227

228 if(rd->rrd_memory_mode == RRD_MEMORY_MODE_RAM)

229 netdata_munmap(rd->db.data, rd->db.memsize);

src/database/sqlite/sqlite_aclk.c

+1 -20

@@ -329,25 +329,6 @@ static void aclk_run_query_job(uv_work_t *req)

329 aclk_run_query(config, query);

330 }

331

332 -static int read_query_thread_count()

333 -{

334 - int threads = MIN(get_netdata_cpus()/2, 6);

335 - threads = MAX(threads, 2);

336 - threads = config_get_number(CONFIG_SECTION_CLOUD, "query thread count", threads);

337 - if(threads < 1) {

338 - netdata_log_error("You need at least one query thread. Overriding configured setting of \"%d\"", threads);

339 - threads = 1;

340 - config_set_number(CONFIG_SECTION_CLOUD, "query thread count", threads);

341 - }

342 - else {

343 - if (threads > libuv_worker_threads / 2) {

344 - threads = MAX(libuv_worker_threads / 2, 2);

345 - config_set_number(CONFIG_SECTION_CLOUD, "query thread count", threads);

346 - }

347 - }

348 - return threads;

349 -}

350 -

332 static void node_update_timer_cb(uv_timer_t *handle)

333 {

334 struct aclk_sync_cfg_t *ahc = handle->data;

@@ -398,7 +379,7 @@ static void aclk_synchronization(void *arg)

379

380 sql_delete_aclk_table_list();

381

401 - int query_thread_count = read_query_thread_count();

382 + int query_thread_count = netdata_conf_cloud_query_threads();

383 netdata_log_info("Starting ACLK synchronization thread with %d parallel query threads", query_thread_count);

384

385 while (likely(service_running(SERVICE_ACLK))) {

src/libnetdata/config/appconfig.c

+17 -6

@@ -62,19 +62,30 @@ bool stream_conf_needs_dbengine(struct config *root) {

62 return ret;

63 }

64

65 -bool stream_conf_has_uuid_section(struct config *root) {

65 +bool stream_conf_has_api_enabled(struct config *root) {

66 struct config_section *sect = NULL;

67 + struct config_option *opt;

68 bool is_parent = false;

69

70 APPCONFIG_LOCK(root);

71 for (sect = root->sections; sect; sect = sect->next) {

72 nd_uuid_t uuid;

73

73 - if (uuid_parse(string2str(sect->name), uuid) != -1 &&

74 - appconfig_get_boolean_by_section(sect, "enabled", 0)) {

75 - is_parent = true;

76 - break;

77 - }

74 + if (uuid_parse(string2str(sect->name), uuid) != 0)

75 + continue;

76 +

77 + opt = appconfig_option_find(sect, "type");

78 + // when the 'type' is missing, we assume it is 'api'

79 + if(opt && string_strcmp(opt->value, "api") != 0)

80 + continue;

81 +

82 + opt = appconfig_option_find(sect, "enabled");

83 + // when the 'enabled' is missing, we assume it is 'false'

84 + if(!opt || !appconfig_test_boolean_value(string2str(opt->value)))

85 + continue;

86 +

87 + is_parent = true;

88 + break;

89 }

90 APPCONFIG_UNLOCK(root);

91

src/libnetdata/config/appconfig.h

+1 -1

@@ -181,7 +181,7 @@ _CONNECTOR_INSTANCE *add_connector_instance(struct config_section *connector, st

181 #define config_section_option_destroy(section, name) appconfig_section_option_destroy_non_loaded(&netdata_config, section, name)

182

183 bool stream_conf_needs_dbengine(struct config *root);

184 -bool stream_conf_has_uuid_section(struct config *root);

184 +bool stream_conf_has_api_enabled(struct config *root);

185

186 void appconfig_foreach_section(struct config *root, void (*cb)(struct config *root, const char *name, void *data), void *data);

187

src/ml/ml_config.cc

+1 -1

@@ -45,7 +45,7 @@ void ml_config_load(ml_config_t *cfg) {

45 std::string anomaly_detection_grouping_method = config_get(config_section_ml, "anomaly detection grouping method", "average");

46 time_t anomaly_detection_query_duration = config_get_duration_seconds(config_section_ml, "anomaly detection grouping duration", 5 * 60);

47

48 - size_t num_worker_threads = stream_conf_configured_as_parent() ? get_netdata_cpus() / 4 : 1;

48 + size_t num_worker_threads = stream_conf_is_parent(false) ? get_netdata_cpus() / 4 : 1;

49 if (num_worker_threads < 1) num_worker_threads = 1;

50 else if (num_worker_threads > 256) num_worker_threads = 256;

51 num_worker_threads = config_get_number(config_section_ml, "num training threads", num_worker_threads);

src/plugins.d/plugins_d.c

+1 -1

@@ -262,7 +262,7 @@ void *pluginsd_main(void *ptr) {

262 CLEANUP_FUNCTION_REGISTER(pluginsd_main_cleanup) cleanup_ptr = ptr;

263

264 int automatic_run = config_get_boolean(CONFIG_SECTION_PLUGINS, "enable running new plugins", 1);

265 - int scan_frequency = (int)config_get_number(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60);

265 + int scan_frequency = (int)config_get_duration_seconds(CONFIG_SECTION_PLUGINS, "check for new plugins every", 60);

266 if (scan_frequency < 1)

267 scan_frequency = 1;

268

src/streaming/replication.c

+1 -1

@@ -1889,7 +1889,7 @@ void *replication_thread_main(void *ptr) {

1889

1890 replication_initialize_workers(true);

1891

1892 - int threads = stream_conf_configured_as_parent() ? (int)(get_netdata_cpus() / 2) : 1;

1892 + int threads = stream_conf_is_parent(false) ? (int)(get_netdata_cpus() / 2) : 1;

1893 if (threads < 1) threads = 1;

1894 else if (threads > MAX_REPLICATION_THREADS) threads = MAX_REPLICATION_THREADS;

1895

src/streaming/stream-conf.c

+12 -4

@@ -82,6 +82,7 @@ static void stream_conf_load_internal() {

82 appconfig_move_everywhere(&stream_config, "default postpone alarms on connect seconds", "postpone alerts on connect");

83 appconfig_move_everywhere(&stream_config, "postpone alarms on connect seconds", "postpone alerts on connect");

84 appconfig_move_everywhere(&stream_config, "health enabled by default", "health enabled");

85 + appconfig_move_everywhere(&stream_config, "buffer size bytes", "buffer size");

86 }

87

88 bool stream_conf_receiver_needs_dbengine(void) {

@@ -119,8 +120,8 @@ void stream_conf_load() {

120 config_get_duration_seconds(CONFIG_SECTION_DB, "replication step",

121 stream_receive.replication.step);

122

122 - stream_send.buffer_max_size = (size_t)appconfig_get_number(

123 - &stream_config, CONFIG_SECTION_STREAM, "buffer size bytes",

123 + stream_send.buffer_max_size = (size_t)appconfig_get_size_bytes(

124 + &stream_config, CONFIG_SECTION_STREAM, "buffer size",

125 stream_send.buffer_max_size);

126

127 stream_send.parents.reconnect_delay_s = (unsigned int)appconfig_get_duration_seconds(

@@ -185,8 +186,15 @@ void stream_conf_load() {

186 }

187 }

188

188 -bool stream_conf_configured_as_parent() {

189 - return stream_conf_has_uuid_section(&stream_config);

189 +bool stream_conf_is_parent(bool recheck) {

190 + static bool rc = false, queried = false;

191 + if(!recheck && queried)

192 + return rc;

193 +

194 + rc = stream_conf_has_api_enabled(&stream_config);

195 + queried = true;

196 +

197 + return rc;

198 }

199

200 void stream_conf_receiver_config(struct receiver_state *rpt, struct stream_receiver_config *config, const char *api_key, const char *machine_guid) {

src/streaming/stream-conf.h

+1 -1

@@ -85,7 +85,7 @@ void stream_conf_receiver_config(struct receiver_state *rpt, struct stream_recei

85

86 void stream_conf_load();

87 bool stream_conf_receiver_needs_dbengine();

88 -bool stream_conf_configured_as_parent();

88 +bool stream_conf_is_parent(bool recheck);

89

90 bool stream_conf_is_key_type(const char *api_key, const char *type);

91 bool stream_conf_api_key_is_enabled(const char *api_key, bool enabled);

src/streaming/stream.conf

+1 -1

@@ -80,7 +80,7 @@

80 # The buffer to use for sending metrics.

81 # 10MB is good for 60 seconds of data, so increase this if you expect latencies.

82 # The buffer is flushed on reconnects (this will not prevent gaps at the charts).

83 - #buffer size bytes = 10485760

83 + #buffer size = 10MiB

84

85 # If the connection fails, or it disconnects,

86 # retry after that many seconds (randomized from 5s to whatever is here).

src/web/api/queries/backfill.c

+1 -1

@@ -228,5 +228,5 @@ void *backfill_thread(void *ptr) {

228 }

229

230 bool backfill_threads_detect_from_stream_conf(void) {

231 - return stream_conf_configured_as_parent();

231 + return stream_conf_is_parent(false);

232 }

src/web/server/static/static-threaded.c

+1 -20

@@ -506,26 +506,7 @@ void *socket_listen_main_static_threaded(void *ptr) {

506

507 netdata_ssl_initialize_ctx(NETDATA_SSL_WEB_SERVER_CTX);

508

509 - // 6 threads is the optimal value

510 - // since 6 are the parallel connections browsers will do

511 - // so, if the machine has more CPUs, avoid using resources unnecessarily

512 - int def_thread_count = (int)get_netdata_cpus();

513 - if(def_thread_count < 6) def_thread_count = 6;

514 -

515 - if (!strcmp(config_get(CONFIG_SECTION_WEB, "mode", ""),"single-threaded")) {

516 - netdata_log_info("Running web server with one thread, because mode is single-threaded");

517 - config_set(CONFIG_SECTION_WEB, "mode", "static-threaded");

518 - def_thread_count = 1;

519 - }

520 - static_threaded_workers_count = config_get_number(CONFIG_SECTION_WEB, "web server threads", def_thread_count);

521 -

522 - if (static_threaded_workers_count < 1) static_threaded_workers_count = 1;

523 -

524 - // See https://github.com/netdata/netdata/issues/11081#issuecomment-831998240 for more details

525 - if (OPENSSL_VERSION_NUMBER < OPENSSL_VERSION_110) {

526 - static_threaded_workers_count = 1;

527 - netdata_log_info("You are running an OpenSSL older than 1.1.0, web server will not enable multithreading.");

528 - }

509 + static_threaded_workers_count = netdata_conf_web_query_threads();

510

511 size_t max_sockets = (size_t)config_get_number(CONFIG_SECTION_WEB, "web server max sockets",

512 (long long int)(rlimit_nofile.rlim_cur / 4));

Streaming improvements No 7 (#19204)