| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #define PULSE_INTERNALS 1 |
| 4 | #include "pulse-daemon-memory.h" |
| 5 | #include "streaming/stream-replication-sender.h" |
| 6 | #include "health/health.h" |
| 7 | |
| 8 | static size_t rrd_slot_memory = 0; |
| 9 | |
| 10 | void rrd_slot_memory_added(size_t added) { |
| 11 | __atomic_add_fetch(&rrd_slot_memory, added, __ATOMIC_RELAXED); |
| 12 | } |
| 13 | |
| 14 | void rrd_slot_memory_removed(size_t added) { |
| 15 | __atomic_sub_fetch(&rrd_slot_memory, added, __ATOMIC_RELAXED); |
| 16 | } |
| 17 | |
| 18 | #define dictionary_stats_memory_total(stats) \ |
| 19 | ((stats).memory.dict + (stats).memory.values + (stats).memory.index) |
| 20 | |
| 21 | struct netdata_buffers_statistics netdata_buffers_statistics = { 0 }; |
| 22 | |
| 23 | void pulse_daemon_memory_do(bool extended __maybe_unused) { |
| 24 | |
| 25 | { |
| 26 | static RRDSET *st_memory = NULL; |
| 27 | static RRDDIM *rd_db_dbengine = NULL; |
| 28 | static RRDDIM *rd_db_rrd = NULL; |
| 29 | static RRDDIM *rd_db_sqlite3 = NULL; |
| 30 | |
| 31 | #ifdef DICT_WITH_STATS |
| 32 | static RRDDIM *rd_collectors = NULL; |
| 33 | static RRDDIM *rd_rrdhosts = NULL; |
| 34 | static RRDDIM *rd_rrdsets = NULL; |
| 35 | static RRDDIM *rd_rrddims = NULL; |
| 36 | static RRDDIM *rd_contexts = NULL; |
| 37 | static RRDDIM *rd_health = NULL; |
| 38 | static RRDDIM *rd_functions = NULL; |
| 39 | static RRDDIM *rd_replication = NULL; |
| 40 | #else |
| 41 | static RRDDIM *rd_metadata = NULL; |
| 42 | #endif |
| 43 | static RRDDIM *rd_uuid = NULL; |
| 44 | static RRDDIM *rd_labels = NULL; // labels use dictionary like statistics, but it is not ARAL based dictionary |
| 45 | static RRDDIM *rd_ml = NULL; |
| 46 | static RRDDIM *rd_strings = NULL; |
| 47 | static RRDDIM *rd_streaming = NULL; |
| 48 | static RRDDIM *rd_buffers = NULL; |
| 49 | static RRDDIM *rd_workers = NULL; |
| 50 | static RRDDIM *rd_aral = NULL; |
| 51 | static RRDDIM *rd_judy = NULL; |
| 52 | static RRDDIM *rd_slots = NULL; |
| 53 | static RRDDIM *rd_health_log = NULL; |
| 54 | static RRDDIM *rd_other = NULL; |
| 55 | |
| 56 | if (unlikely(!st_memory)) { |
| 57 | st_memory = rrdset_create_localhost( |
| 58 | "netdata", |
| 59 | "memory", |
| 60 | NULL, |
| 61 | "Memory Usage", |
| 62 | NULL, |
| 63 | "Netdata Memory", |
| 64 | "bytes", |
| 65 | "netdata", |
| 66 | "pulse", |
| 67 | 130100, |
| 68 | localhost->rrd_update_every, |
| 69 | RRDSET_TYPE_STACKED); |
| 70 | |
| 71 | rd_db_dbengine = rrddim_add(st_memory, "dbengine", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 72 | rd_db_rrd = rrddim_add(st_memory, "rrd", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 73 | rd_db_sqlite3 = rrddim_add(st_memory, "sqlite3", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 74 | |
| 75 | #ifdef DICT_WITH_STATS |
| 76 | rd_collectors = rrddim_add(st_memory, "collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 77 | rd_rrdhosts = rrddim_add(st_memory, "hosts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 78 | rd_rrdsets = rrddim_add(st_memory, "rrdset", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 79 | rd_rrddims = rrddim_add(st_memory, "rrddim", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 80 | rd_contexts = rrddim_add(st_memory, "contexts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 81 | rd_health = rrddim_add(st_memory, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 82 | rd_functions = rrddim_add(st_memory, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 83 | rd_replication = rrddim_add(st_memory, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 84 | #else |
| 85 | rd_metadata = rrddim_add(st_memory, "metadata", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 86 | #endif |
| 87 | rd_uuid = rrddim_add(st_memory, "uuid", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 88 | rd_labels = rrddim_add(st_memory, "labels", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 89 | rd_ml = rrddim_add(st_memory, "ML", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 90 | rd_strings = rrddim_add(st_memory, "strings", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 91 | rd_streaming = rrddim_add(st_memory, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 92 | rd_buffers = rrddim_add(st_memory, "buffers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 93 | rd_workers = rrddim_add(st_memory, "workers", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 94 | rd_aral = rrddim_add(st_memory, "aral", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 95 | rd_judy = rrddim_add(st_memory, "judy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 96 | rd_slots = rrddim_add(st_memory, "slots", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 97 | rd_other = rrddim_add(st_memory, "other", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 98 | rd_health_log = rrddim_add(st_memory, "health log", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 99 | } |
| 100 | |
| 101 | // each of these should also be analyzed below at the buffers chart |
| 102 | size_t buffers = |
| 103 | netdata_buffers_statistics.query_targets_size + onewayalloc_allocated_memory() + |
| 104 | netdata_buffers_statistics.rrdset_done_rda_size + |
| 105 | netdata_buffers_statistics.buffers_aclk + |
| 106 | netdata_buffers_statistics.buffers_api + |
| 107 | netdata_buffers_statistics.buffers_functions + |
| 108 | netdata_buffers_statistics.buffers_sqlite + |
| 109 | netdata_buffers_statistics.buffers_exporters + |
| 110 | netdata_buffers_statistics.buffers_health + |
| 111 | netdata_buffers_statistics.buffers_streaming + |
| 112 | netdata_buffers_statistics.cbuffers_streaming + |
| 113 | netdata_buffers_statistics.buffers_web + |
| 114 | replication_sender_allocated_buffers() + |
| 115 | aral_by_size_free_bytes() + |
| 116 | judy_aral_free_bytes() + |
| 117 | uuidmap_free_bytes(); |
| 118 | |
| 119 | sqlite3_int64 sqlite3_memory_used_current = 0, sqlite3_memory_used_highwater = 0; |
| 120 | sqlite3_status64(SQLITE_STATUS_MEMORY_USED, &sqlite3_memory_used_current, &sqlite3_memory_used_highwater, 1); |
| 121 | |
| 122 | size_t strings_memory = 0, strings_index = 0; |
| 123 | string_statistics(NULL, NULL, NULL, NULL, NULL, &strings_memory, &strings_index, NULL, NULL); |
| 124 | |
| 125 | size_t health_log_memory = aral_used_bytes_from_stats(health_alarm_entry_aral_stats()); |
| 126 | |
| 127 | rrddim_set_by_pointer(st_memory, rd_db_dbengine, (collected_number)pulse_dbengine_total_memory); |
| 128 | rrddim_set_by_pointer(st_memory, rd_db_rrd, (collected_number)pulse_rrd_memory_size); |
| 129 | rrddim_set_by_pointer(st_memory, rd_db_sqlite3, (collected_number)sqlite3_memory_used_highwater); |
| 130 | |
| 131 | #ifdef DICT_WITH_STATS |
| 132 | rrddim_set_by_pointer(st_memory, rd_collectors, |
| 133 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_collectors)); |
| 134 | |
| 135 | rrddim_set_by_pointer(st_memory,rd_rrdhosts, |
| 136 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhost) + (collected_number)netdata_buffers_statistics.rrdhost_allocations_size); |
| 137 | |
| 138 | rrddim_set_by_pointer(st_memory, rd_rrdsets, |
| 139 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdset)); |
| 140 | |
| 141 | rrddim_set_by_pointer(st_memory, rd_rrddims, |
| 142 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrddim)); |
| 143 | |
| 144 | rrddim_set_by_pointer(st_memory, rd_contexts, |
| 145 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdcontext)); |
| 146 | |
| 147 | rrddim_set_by_pointer(st_memory, rd_health, |
| 148 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdhealth)); |
| 149 | |
| 150 | rrddim_set_by_pointer(st_memory, rd_functions, |
| 151 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_functions)); |
| 152 | |
| 153 | rrddim_set_by_pointer(st_memory, rd_replication, |
| 154 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_replication) + replication_sender_allocated_memory()); |
| 155 | #else |
| 156 | uint64_t metadata = |
| 157 | aral_by_size_used_bytes() + |
| 158 | dictionary_stats_category_rrdhost.memory.dict + dictionary_stats_category_rrdhost.memory.index + |
| 159 | dictionary_stats_category_rrdset.memory.dict + dictionary_stats_category_rrdset.memory.index + |
| 160 | dictionary_stats_category_rrddim.memory.dict + dictionary_stats_category_rrddim.memory.index + |
| 161 | dictionary_stats_category_rrdcontext.memory.dict + dictionary_stats_category_rrdcontext.memory.index + |
| 162 | dictionary_stats_category_rrdhealth.memory.dict + dictionary_stats_category_rrdhealth.memory.index + |
| 163 | dictionary_stats_category_functions.memory.dict + dictionary_stats_category_functions.memory.index + |
| 164 | dictionary_stats_category_replication.memory.dict + dictionary_stats_category_replication.memory.index + |
| 165 | netdata_buffers_statistics.rrdhost_allocations_size + |
| 166 | replication_sender_allocated_memory(); |
| 167 | |
| 168 | rrddim_set_by_pointer(st_memory, rd_metadata, (collected_number)metadata); |
| 169 | #endif |
| 170 | |
| 171 | rrddim_set_by_pointer(st_memory, rd_uuid, |
| 172 | (collected_number)uuidmap_memory()); |
| 173 | |
| 174 | // labels use dictionary like statistics, but it is not ARAL based dictionary |
| 175 | rrddim_set_by_pointer(st_memory, rd_labels, |
| 176 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_rrdlabels)); |
| 177 | |
| 178 | rrddim_set_by_pointer(st_memory, rd_ml, |
| 179 | (collected_number)pulse_ml_get_current_memory_usage()); |
| 180 | |
| 181 | rrddim_set_by_pointer(st_memory, rd_strings, |
| 182 | (collected_number)(strings_memory + strings_index)); |
| 183 | |
| 184 | rrddim_set_by_pointer(st_memory, rd_streaming, |
| 185 | (collected_number)netdata_buffers_statistics.rrdhost_senders + (collected_number)netdata_buffers_statistics.rrdhost_receivers); |
| 186 | |
| 187 | rrddim_set_by_pointer(st_memory, rd_buffers, |
| 188 | (collected_number)buffers); |
| 189 | |
| 190 | rrddim_set_by_pointer(st_memory, rd_workers, |
| 191 | (collected_number) workers_allocated_memory()); |
| 192 | |
| 193 | rrddim_set_by_pointer(st_memory, rd_aral, |
| 194 | (collected_number)aral_by_size_structures_bytes()); |
| 195 | |
| 196 | rrddim_set_by_pointer(st_memory, rd_judy, |
| 197 | (collected_number) judy_aral_structures()); |
| 198 | |
| 199 | rrddim_set_by_pointer(st_memory, rd_slots, |
| 200 | (collected_number)__atomic_load_n(&rrd_slot_memory, __ATOMIC_RELAXED)); |
| 201 | |
| 202 | |
| 203 | rrddim_set_by_pointer(st_memory, rd_health_log, |
| 204 | (collected_number)health_log_memory); |
| 205 | |
| 206 | rrddim_set_by_pointer(st_memory, rd_other, |
| 207 | (collected_number)dictionary_stats_memory_total(dictionary_stats_category_other)); |
| 208 | |
| 209 | rrdset_done(st_memory); |
| 210 | } |
| 211 | |
| 212 | // ---------------------------------------------------------------------------------------------------------------- |
| 213 | |
| 214 | { |
| 215 | static RRDSET *st_memory_buffers = NULL; |
| 216 | static RRDDIM *rd_queries = NULL; |
| 217 | static RRDDIM *rd_collectors = NULL; |
| 218 | static RRDDIM *rd_buffers_aclk = NULL; |
| 219 | static RRDDIM *rd_buffers_api = NULL; |
| 220 | static RRDDIM *rd_buffers_functions = NULL; |
| 221 | static RRDDIM *rd_buffers_sqlite = NULL; |
| 222 | static RRDDIM *rd_buffers_exporters = NULL; |
| 223 | static RRDDIM *rd_buffers_health = NULL; |
| 224 | static RRDDIM *rd_buffers_streaming = NULL; |
| 225 | static RRDDIM *rd_cbuffers_streaming = NULL; |
| 226 | static RRDDIM *rd_buffers_replication = NULL; |
| 227 | static RRDDIM *rd_buffers_web = NULL; |
| 228 | static RRDDIM *rd_buffers_aral = NULL; |
| 229 | static RRDDIM *rd_buffers_judy = NULL; |
| 230 | static RRDDIM *rd_buffers_uuid = NULL; |
| 231 | |
| 232 | if (unlikely(!st_memory_buffers)) { |
| 233 | st_memory_buffers = rrdset_create_localhost( |
| 234 | "netdata", |
| 235 | "memory_buffers", |
| 236 | NULL, |
| 237 | "Memory Usage", |
| 238 | NULL, |
| 239 | "Netdata Memory Buffers", |
| 240 | "bytes", |
| 241 | "netdata", |
| 242 | "pulse", |
| 243 | 130102, |
| 244 | localhost->rrd_update_every, |
| 245 | RRDSET_TYPE_STACKED); |
| 246 | |
| 247 | rd_queries = rrddim_add(st_memory_buffers, "queries", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 248 | rd_collectors = rrddim_add(st_memory_buffers, "collection", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 249 | rd_buffers_aclk = rrddim_add(st_memory_buffers, "aclk", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 250 | rd_buffers_api = rrddim_add(st_memory_buffers, "api", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 251 | rd_buffers_functions = rrddim_add(st_memory_buffers, "functions", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 252 | rd_buffers_sqlite = rrddim_add(st_memory_buffers, "sqlite", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 253 | rd_buffers_exporters = rrddim_add(st_memory_buffers, "exporters", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 254 | rd_buffers_health = rrddim_add(st_memory_buffers, "health", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 255 | rd_buffers_streaming = rrddim_add(st_memory_buffers, "streaming", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 256 | rd_cbuffers_streaming = rrddim_add(st_memory_buffers, "streaming cbuf", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 257 | rd_buffers_replication = rrddim_add(st_memory_buffers, "replication", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 258 | rd_buffers_web = rrddim_add(st_memory_buffers, "web", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 259 | rd_buffers_aral = rrddim_add(st_memory_buffers, "aral-by-size free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 260 | rd_buffers_judy = rrddim_add(st_memory_buffers, "aral-judy free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 261 | rd_buffers_uuid = rrddim_add(st_memory_buffers, "uuid", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 262 | } |
| 263 | |
| 264 | // the sum of all these needs to be above at the total buffers calculation |
| 265 | rrddim_set_by_pointer(st_memory_buffers, rd_queries, (collected_number)netdata_buffers_statistics.query_targets_size + (collected_number) onewayalloc_allocated_memory()); |
| 266 | rrddim_set_by_pointer(st_memory_buffers, rd_collectors, (collected_number)netdata_buffers_statistics.rrdset_done_rda_size); |
| 267 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aclk, (collected_number)netdata_buffers_statistics.buffers_aclk); |
| 268 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_api, (collected_number)netdata_buffers_statistics.buffers_api); |
| 269 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_functions, (collected_number)netdata_buffers_statistics.buffers_functions); |
| 270 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_sqlite, (collected_number)netdata_buffers_statistics.buffers_sqlite); |
| 271 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_exporters, (collected_number)netdata_buffers_statistics.buffers_exporters); |
| 272 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_health, (collected_number)netdata_buffers_statistics.buffers_health); |
| 273 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_streaming, (collected_number)netdata_buffers_statistics.buffers_streaming); |
| 274 | rrddim_set_by_pointer(st_memory_buffers, rd_cbuffers_streaming, (collected_number)netdata_buffers_statistics.cbuffers_streaming); |
| 275 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_web, (collected_number)netdata_buffers_statistics.buffers_web); |
| 276 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_replication, (collected_number)replication_sender_allocated_buffers()); |
| 277 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_aral, (collected_number)aral_by_size_free_bytes()); |
| 278 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_judy, (collected_number)judy_aral_free_bytes()); |
| 279 | rrddim_set_by_pointer(st_memory_buffers, rd_buffers_uuid, (collected_number)uuidmap_free_bytes()); |
| 280 | |
| 281 | rrdset_done(st_memory_buffers); |
| 282 | } |
| 283 | |
| 284 | // ---------------------------------------------------------------------------------------------------------------- |
| 285 | |
| 286 | #ifdef ENABLE_DBENGINE |
| 287 | OS_SYSTEM_MEMORY sm = os_system_memory(true); |
| 288 | if (OS_SYSTEM_MEMORY_OK(sm) && dbengine_out_of_memory_protection) { |
| 289 | static RRDSET *st_memory_available = NULL; |
| 290 | static RRDDIM *rd_available = NULL; |
| 291 | |
| 292 | if (unlikely(!st_memory_available)) { |
| 293 | st_memory_available = rrdset_create_localhost( |
| 294 | "netdata", |
| 295 | "out_of_memory_protection", |
| 296 | NULL, |
| 297 | "Memory Usage", |
| 298 | NULL, |
| 299 | "Out of Memory Protection", |
| 300 | "bytes", |
| 301 | "netdata", |
| 302 | "pulse", |
| 303 | 130103, |
| 304 | localhost->rrd_update_every, |
| 305 | RRDSET_TYPE_AREA); |
| 306 | |
| 307 | rd_available = rrddim_add(st_memory_available, "available", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE); |
| 308 | } |
| 309 | |
| 310 | // the sum of all these needs to be above at the total buffers calculation |
| 311 | rrddim_set_by_pointer( |
| 312 | st_memory_available, rd_available, |
| 313 | (collected_number)sm.ram_available_bytes); |
| 314 | |
| 315 | rrdset_done(st_memory_available); |
| 316 | } |
| 317 | #endif |
| 318 | // ---------------------------------------------------------------------------------------------------------------- |
| 319 | } |