lower compression level to lower cpu resources on parents (#19350)
Costa Tsaousis committed
Jan 8, 2025 at 16:14 UTC
d114830aab578f7d394fa0507b518dba5d9ad3bd
5 files changed
+41
-2
src/daemon/config/netdata-conf-profile.c
+5
@@ -104,6 +104,7 @@ void nd_profile_setup(void) {
104
nd_profile.update_every = 1; // MUST BE 2
105
nd_profile.malloc_arenas = 1;
106
nd_profile.malloc_trim = 32 * 1024;
107
+ nd_profile.stream_sender_compression = ND_COMPRESSION_FASTEST;
108
// web server threads = 6
109
// aclk query threads = 6
110
// backfill threads = 0
@@ -119,6 +120,7 @@ void nd_profile_setup(void) {
120
nd_profile.update_every = 1;
121
nd_profile.malloc_arenas = os_get_system_cpus_cached(true);
122
nd_profile.malloc_trim = 256 * 1024;
123
+ nd_profile.stream_sender_compression = ND_COMPRESSION_FASTEST;
124
// web server threads = dynamic
125
// aclk query threads = dynamic
126
// backfill threads = dynamic
@@ -131,6 +133,7 @@ void nd_profile_setup(void) {
133
nd_profile.update_every = 1;
134
nd_profile.malloc_arenas = 1;
135
nd_profile.malloc_trim = 32 * 1024;
136
+ nd_profile.stream_sender_compression = ND_COMPRESSION_DEFAULT;
137
// web server threads = 6
138
// aclk query threads = 6
139
// backfill threads = 0
@@ -143,6 +146,7 @@ void nd_profile_setup(void) {
146
nd_profile.update_every = 1;
147
nd_profile.malloc_arenas = 1;
148
nd_profile.malloc_trim = 64 * 1024;
149
+ nd_profile.stream_sender_compression = ND_COMPRESSION_DEFAULT;
150
// web server threads = 6
151
// aclk query threads = 6
152
// backfill threads = 0
@@ -152,4 +156,5 @@ void nd_profile_setup(void) {
156
}
157
158
netdata_conf_glibc_malloc_initialize(nd_profile.malloc_arenas, nd_profile.malloc_trim);
159
+ stream_conf_set_sender_compression_levels(nd_profile.stream_sender_compression);
160
}
src/daemon/config/netdata-conf-profile.h
+7
-1
@@ -5,7 +5,7 @@
5
6
#include "libnetdata/libnetdata.h"
7
8
-typedef enum {
8
+typedef enum __attribute__((packed)) {
9
ND_PROFILE_NONE = (0),
10
11
// system profiles
@@ -18,6 +18,11 @@ typedef enum {
18
19
} ND_PROFILE;
20
21
+typedef enum __attribute__((packed)) {
22
+ ND_COMPRESSION_DEFAULT = 0,
23
+ ND_COMPRESSION_FASTEST,
24
+} ND_COMPRESSION_PROFILE;
25
+
26
#define ND_CONF_PROFILES_SYSTEM (ND_PROFILE_STANDALONE | ND_PROFILE_PARENT | ND_PROFILE_CHILD | ND_PROFILE_IOT)
27
28
BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(ND_PROFILE);
@@ -47,6 +52,7 @@ struct nd_profile_t {
52
time_t update_every;
53
size_t malloc_arenas;
54
size_t malloc_trim;
55
+ ND_COMPRESSION_PROFILE stream_sender_compression;
56
};
57
58
extern struct nd_profile_t nd_profile;
src/streaming/stream-capabilities.c
+6
@@ -114,6 +114,12 @@ STREAM_CAPABILITIES stream_our_capabilities(RRDHOST *host, bool sender) {
114
disabled_capabilities |= host->sender->disabled_capabilities;
115
}
116
117
+// if(sender) {
118
+// if(nd_profile.stream_sender_compression == ND_COMPRESSION_FASTEST)
119
+// // lz4 or nothing
120
+// disabled_capabilities |= (STREAM_CAP_COMPRESSIONS_AVAILABLE & ~(STREAM_CAP_LZ4_AVAILABLE));
121
+// }
122
+
123
return (STREAM_CAP_V1 |
124
STREAM_CAP_V2 |
125
STREAM_CAP_VN |
src/streaming/stream-conf.c
+20
-1
@@ -31,7 +31,7 @@ struct _stream_send stream_send = {
31
[COMPRESSION_ALGORITHM_ZSTD] = 3, // 1 (faster) - 22 (smaller)
32
[COMPRESSION_ALGORITHM_LZ4] = 1, // 1 (smaller) - 9 (faster)
33
[COMPRESSION_ALGORITHM_BROTLI] = 3, // 0 (faster) - 11 (smaller)
34
- [COMPRESSION_ALGORITHM_GZIP] = 1, // 1 (faster) - 9 (smaller)
34
+ [COMPRESSION_ALGORITHM_GZIP] = 3, // 1 (faster) - 9 (smaller)
35
}
36
},
37
};
@@ -44,6 +44,25 @@ struct _stream_receive stream_receive = {
44
}
45
};
46
47
+void stream_conf_set_sender_compression_levels(ND_COMPRESSION_PROFILE profile) {
48
+ switch(profile) {
49
+ default:
50
+ case ND_COMPRESSION_DEFAULT:
51
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_ZSTD] = 3;
52
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_LZ4] = 1;
53
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_BROTLI] = 3;
54
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_GZIP] = 3;
55
+ break;
56
+
57
+ case ND_COMPRESSION_FASTEST:
58
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_ZSTD] = 1;
59
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_LZ4] = 9;
60
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_BROTLI] = 1;
61
+ stream_send.compression.levels[COMPRESSION_ALGORITHM_GZIP] = 1;
62
+ break;
63
+ }
64
+}
65
+
66
static void stream_conf_load_internal() {
67
errno_clear();
68
char *filename = filename_from_path_entry_strdupz(netdata_configured_user_config_dir, "stream.conf");
src/streaming/stream-conf.h
+3
@@ -7,6 +7,7 @@
7
#include "stream-compression/compression.h"
8
#include "stream-capabilities.h"
9
#include "database/rrd-database-mode.h"
10
+#include "daemon/config/netdata-conf-profile.h"
11
12
#define SENDER_MIN_RECONNECT_DELAY 5
13
@@ -92,4 +93,6 @@ bool stream_conf_is_key_type(const char *api_key, const char *type);
93
bool stream_conf_api_key_is_enabled(const char *api_key, bool enabled);
94
bool stream_conf_api_key_allows_client(const char *api_key, const char *client_ip);
95
96
+void stream_conf_set_sender_compression_levels(ND_COMPRESSION_PROFILE profile);
97
+
98
#endif //NETDATA_STREAM_CONF_H