| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #ifndef NETDATA_LIB_H |
| 4 | #define NETDATA_LIB_H 1 |
| 5 | |
| 6 | # ifdef __cplusplus |
| 7 | extern "C" { |
| 8 | # endif |
| 9 | |
| 10 | #include "common.h" |
| 11 | #include "signals/signals.h" |
| 12 | #include "memory/alignment.h" |
| 13 | #include "memory/nd-mallocz.h" |
| 14 | #include "memory/nd-mmap.h" |
| 15 | #include "libnetdata/exit/exit_initiated.h" |
| 16 | #include "log/nd_log-fatal.h" |
| 17 | #include "atomics/atomics.h" |
| 18 | |
| 19 | // NETDATA_TRACE_ALLOCATIONS does not work under musl libc, so don't enable it |
| 20 | //#if defined(NETDATA_INTERNAL_CHECKS) && !defined(NETDATA_TRACE_ALLOCATIONS) |
| 21 | //#define NETDATA_TRACE_ALLOCATIONS 1 |
| 22 | //#endif |
| 23 | |
| 24 | #include "libjudy/judy-malloc.h" |
| 25 | #include "locks/benchmark.h" |
| 26 | #include "locks/benchmark-rw.h" |
| 27 | #include "object-state/object-state.h" |
| 28 | #include "storage-point.h" |
| 29 | #include "paths/paths.h" |
| 30 | |
| 31 | int vsnprintfz(char *dst, size_t n, const char *fmt, va_list args); |
| 32 | int snprintfz(char *dst, size_t n, const char *fmt, ...) PRINTFLIKE(3, 4); |
| 33 | |
| 34 | void json_escape_string(char *dst, const char *src, size_t size); |
| 35 | |
| 36 | extern struct rlimit rlimit_nofile; |
| 37 | |
| 38 | char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len); |
| 39 | |
| 40 | int verify_netdata_host_prefix(bool log_msg); |
| 41 | |
| 42 | char *read_by_filename(const char *filename, long *file_size); |
| 43 | char *find_and_replace(const char *src, const char *find, const char *replace, const char *where); |
| 44 | |
| 45 | #define BITS_IN_A_KILOBIT 1000 |
| 46 | #define KILOBITS_IN_A_MEGABIT 1000 |
| 47 | |
| 48 | #include "bitmap/bitmap64.h" |
| 49 | |
| 50 | #define COMPRESSION_MAX_CHUNK 0x4000 |
| 51 | #define COMPRESSION_MAX_OVERHEAD 128 |
| 52 | #define COMPRESSION_MAX_MSG_SIZE (COMPRESSION_MAX_CHUNK - COMPRESSION_MAX_OVERHEAD - 1) |
| 53 | #define PLUGINSD_LINE_MAX (COMPRESSION_MAX_MSG_SIZE - 768) |
| 54 | |
| 55 | bool run_command_and_copy_output_to_stdout(const char *command, int max_line_length); |
| 56 | struct web_buffer *run_command_and_get_output_to_buffer(const char *command, int max_line_length); |
| 57 | |
| 58 | #include "runtime-paths/runtime-paths.h" |
| 59 | |
| 60 | // safe includes before O/S specific functions |
| 61 | #include "template-enum.h" |
| 62 | #include "libjudy/vendored/Judy.h" |
| 63 | #include "libjudy/judyl-typed.h" |
| 64 | |
| 65 | #include "string/string.h" |
| 66 | #include "buffer/buffer.h" |
| 67 | |
| 68 | #include "socket/security.h" // must be before windows.h |
| 69 | |
| 70 | // this may include windows.h |
| 71 | #include "os/os.h" |
| 72 | |
| 73 | #include "uuid/uuid.h" |
| 74 | #include "uuid/uuidmap.h" |
| 75 | #include "http/content_type.h" |
| 76 | #include "user-auth/http-access.h" |
| 77 | #include "user-auth/user-auth.h" |
| 78 | |
| 79 | #include "inlined.h" |
| 80 | #include "parsers/parsers.h" |
| 81 | |
| 82 | #include "threads/threads.h" |
| 83 | #include "locks/locks.h" |
| 84 | #include "locks/spinlock.h" |
| 85 | #include "locks/rw-spinlock.h" |
| 86 | #include "completion/completion.h" |
| 87 | #include "libnetdata/locks/waitq.h" |
| 88 | #include "clocks/clocks.h" |
| 89 | #include "simple_pattern/simple_pattern.h" |
| 90 | #include "libnetdata/log/nd_log.h" |
| 91 | |
| 92 | #include "socket/security.h" // must be before windows.h |
| 93 | |
| 94 | // this may include windows.h |
| 95 | #include "os/os.h" |
| 96 | #include "os/ci.h" |
| 97 | |
| 98 | #include "socket/socket.h" |
| 99 | #include "socket/nd-sock.h" |
| 100 | #include "socket/nd-poll.h" |
| 101 | #include "socket/listen-sockets.h" |
| 102 | #include "socket/poll-events.h" |
| 103 | #include "socket/connect-to.h" |
| 104 | #include "socket/socket-peers.h" |
| 105 | #include "avl/avl.h" |
| 106 | |
| 107 | #include "line_splitter/line_splitter.h" |
| 108 | #include "c_rhash/c_rhash.h" |
| 109 | #include "ringbuffer/ringbuffer.h" |
| 110 | #include "circular_buffer/circular_buffer.h" |
| 111 | #include "buffered_reader/buffered_reader.h" |
| 112 | #include "datetime/iso8601.h" |
| 113 | #include "datetime/rfc3339.h" |
| 114 | #include "datetime/rfc7231.h" |
| 115 | #include "sanitizers/sanitizers.h" |
| 116 | |
| 117 | #include "config/config.h" |
| 118 | #include "spawn_server/spawn_server.h" |
| 119 | #include "spawn_server/spawn_popen.h" |
| 120 | #include "procfile/procfile.h" |
| 121 | #include "dictionary/dictionary.h" |
| 122 | #include "dictionary/thread-cache.h" |
| 123 | |
| 124 | #include "log/systemd-journal-helpers.h" |
| 125 | |
| 126 | #include "eval/eval.h" |
| 127 | #include "statistical/statistical.h" |
| 128 | #include "adaptive_resortable_list/adaptive_resortable_list.h" |
| 129 | #include "url/url.h" |
| 130 | #include "json/json.h" |
| 131 | #include "json/json-c-parser-inline.h" |
| 132 | #include "yaml/yaml.h" |
| 133 | #include "string/utf8.h" |
| 134 | #include "libnetdata/aral/aral.h" |
| 135 | #include "onewayalloc/onewayalloc.h" |
| 136 | #include "worker_utilization/worker_utilization.h" |
| 137 | #include "http/http_defs.h" |
| 138 | #include "gorilla/gorilla.h" |
| 139 | #include "facets/facets.h" |
| 140 | #include "functions_evloop/functions_evloop.h" |
| 141 | #include "query_progress/progress.h" |
| 142 | #include "stacktrace/stacktrace.h" |
| 143 | |
| 144 | static ALWAYS_INLINE PPvoid_t JudyLFirstThenNext(Pcvoid_t PArray, Word_t * PIndex, bool *first) { |
| 145 | if(unlikely(*first)) { |
| 146 | *first = false; |
| 147 | return JudyLFirst(PArray, PIndex, PJE0); |
| 148 | } |
| 149 | |
| 150 | return JudyLNext(PArray, PIndex, PJE0); |
| 151 | } |
| 152 | |
| 153 | static ALWAYS_INLINE PPvoid_t JudyLLastThenPrev(Pcvoid_t PArray, Word_t * PIndex, bool *first) { |
| 154 | if(unlikely(*first)) { |
| 155 | *first = false; |
| 156 | return JudyLLast(PArray, PIndex, PJE0); |
| 157 | } |
| 158 | |
| 159 | return JudyLPrev(PArray, PIndex, PJE0); |
| 160 | } |
| 161 | |
| 162 | typedef enum { |
| 163 | TIMING_STEP_INTERNAL = 0, |
| 164 | |
| 165 | TIMING_STEP_BEGIN2_PREPARE, |
| 166 | TIMING_STEP_BEGIN2_FIND_CHART, |
| 167 | TIMING_STEP_BEGIN2_PARSE, |
| 168 | TIMING_STEP_BEGIN2_ML, |
| 169 | TIMING_STEP_BEGIN2_PROPAGATE, |
| 170 | TIMING_STEP_BEGIN2_STORE, |
| 171 | |
| 172 | TIMING_STEP_SET2_PREPARE, |
| 173 | TIMING_STEP_SET2_LOOKUP_DIMENSION, |
| 174 | TIMING_STEP_SET2_PARSE, |
| 175 | TIMING_STEP_SET2_ML, |
| 176 | TIMING_STEP_SET2_PROPAGATE, |
| 177 | TIMING_STEP_RRDSET_STORE_METRIC, |
| 178 | TIMING_STEP_DBENGINE_FIRST_CHECK, |
| 179 | TIMING_STEP_DBENGINE_CHECK_DATA, |
| 180 | TIMING_STEP_DBENGINE_PACK, |
| 181 | TIMING_STEP_DBENGINE_PAGE_FIN, |
| 182 | TIMING_STEP_DBENGINE_MRG_UPDATE, |
| 183 | TIMING_STEP_DBENGINE_PAGE_ALLOC, |
| 184 | TIMING_STEP_DBENGINE_CREATE_NEW_PAGE, |
| 185 | TIMING_STEP_DBENGINE_FLUSH_PAGE, |
| 186 | TIMING_STEP_SET2_STORE, |
| 187 | |
| 188 | TIMING_STEP_END2_PREPARE, |
| 189 | TIMING_STEP_END2_PUSH_V1, |
| 190 | TIMING_STEP_END2_ML, |
| 191 | TIMING_STEP_END2_RRDSET, |
| 192 | TIMING_STEP_END2_PROPAGATE, |
| 193 | TIMING_STEP_END2_STORE, |
| 194 | |
| 195 | TIMING_STEP_FREEIPMI_CTX_CREATE, |
| 196 | TIMING_STEP_FREEIPMI_DSR_CACHE_DIR, |
| 197 | TIMING_STEP_FREEIPMI_SENSOR_CONFIG_FILE, |
| 198 | TIMING_STEP_FREEIPMI_SENSOR_READINGS_BY_X, |
| 199 | TIMING_STEP_FREEIPMI_READ_record_id, |
| 200 | TIMING_STEP_FREEIPMI_READ_sensor_number, |
| 201 | TIMING_STEP_FREEIPMI_READ_sensor_type, |
| 202 | TIMING_STEP_FREEIPMI_READ_sensor_name, |
| 203 | TIMING_STEP_FREEIPMI_READ_sensor_state, |
| 204 | TIMING_STEP_FREEIPMI_READ_sensor_units, |
| 205 | TIMING_STEP_FREEIPMI_READ_sensor_bitmask_type, |
| 206 | TIMING_STEP_FREEIPMI_READ_sensor_bitmask, |
| 207 | TIMING_STEP_FREEIPMI_READ_sensor_bitmask_strings, |
| 208 | TIMING_STEP_FREEIPMI_READ_sensor_reading_type, |
| 209 | TIMING_STEP_FREEIPMI_READ_sensor_reading, |
| 210 | TIMING_STEP_FREEIPMI_READ_event_reading_type_code, |
| 211 | TIMING_STEP_FREEIPMI_READ_record_type, |
| 212 | TIMING_STEP_FREEIPMI_READ_record_type_class, |
| 213 | TIMING_STEP_FREEIPMI_READ_sel_state, |
| 214 | TIMING_STEP_FREEIPMI_READ_event_direction, |
| 215 | TIMING_STEP_FREEIPMI_READ_event_type_code, |
| 216 | TIMING_STEP_FREEIPMI_READ_event_offset_type, |
| 217 | TIMING_STEP_FREEIPMI_READ_event_offset, |
| 218 | TIMING_STEP_FREEIPMI_READ_event_offset_string, |
| 219 | TIMING_STEP_FREEIPMI_READ_manufacturer_id, |
| 220 | |
| 221 | TIMING_STEP_DBENGINE_EVICT_LOCK, |
| 222 | TIMING_STEP_DBENGINE_EVICT_SELECT, |
| 223 | TIMING_STEP_DBENGINE_EVICT_SELECT_PAGE, |
| 224 | TIMING_STEP_DBENGINE_EVICT_RELOCATE_PAGE, |
| 225 | TIMING_STEP_DBENGINE_EVICT_SORT, |
| 226 | TIMING_STEP_DBENGINE_EVICT_DEINDEX, |
| 227 | TIMING_STEP_DBENGINE_EVICT_DEINDEX_PAGE, |
| 228 | TIMING_STEP_DBENGINE_EVICT_FINISHED, |
| 229 | TIMING_STEP_DBENGINE_EVICT_FREE_LOOP, |
| 230 | TIMING_STEP_DBENGINE_EVICT_FREE_PAGE, |
| 231 | TIMING_STEP_DBENGINE_EVICT_FREE_ATOMICS, |
| 232 | TIMING_STEP_DBENGINE_EVICT_FREE_CB, |
| 233 | TIMING_STEP_DBENGINE_EVICT_FREE_ATOMICS2, |
| 234 | TIMING_STEP_DBENGINE_EVICT_FREE_ARAL, |
| 235 | TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_DATA, |
| 236 | TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_ARAL, |
| 237 | TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_TIER1_ARAL, |
| 238 | TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GLIVE, |
| 239 | TIMING_STEP_DBENGINE_EVICT_FREE_MAIN_PGD_GWORKER, |
| 240 | TIMING_STEP_DBENGINE_EVICT_FREE_OPEN, |
| 241 | TIMING_STEP_DBENGINE_EVICT_FREE_EXTENT, |
| 242 | |
| 243 | // terminator |
| 244 | TIMING_STEP_MAX, |
| 245 | } TIMING_STEP; |
| 246 | |
| 247 | typedef enum { |
| 248 | TIMING_ACTION_INIT, |
| 249 | TIMING_ACTION_STEP, |
| 250 | TIMING_ACTION_FINISH, |
| 251 | } TIMING_ACTION; |
| 252 | |
| 253 | #ifdef NETDATA_TIMING_REPORT |
| 254 | #define timing_init() timing_action(TIMING_ACTION_INIT, TIMING_STEP_INTERNAL) |
| 255 | #define timing_step(step) timing_action(TIMING_ACTION_STEP, step) |
| 256 | #define timing_report() timing_action(TIMING_ACTION_FINISH, TIMING_STEP_INTERNAL) |
| 257 | |
| 258 | #define timing_dbengine_evict_init() timing_action(TIMING_ACTION_INIT, TIMING_STEP_INTERNAL) |
| 259 | #define timing_dbengine_evict_step(step) timing_action(TIMING_ACTION_STEP, step) |
| 260 | #define timing_dbengine_evict_report() timing_action(TIMING_ACTION_FINISH, TIMING_STEP_INTERNAL) |
| 261 | #else |
| 262 | #define timing_init() debug_dummy() |
| 263 | #define timing_step(step) debug_dummy() |
| 264 | #define timing_report() debug_dummy() |
| 265 | |
| 266 | #define timing_dbengine_evict_init() debug_dummy() |
| 267 | #define timing_dbengine_evict_step(step) debug_dummy() |
| 268 | #define timing_dbengine_evict_report() debug_dummy() |
| 269 | #endif |
| 270 | |
| 271 | |
| 272 | void timing_action(TIMING_ACTION action, TIMING_STEP step); |
| 273 | |
| 274 | int hash256_string(const unsigned char *string, size_t size, char *hash); |
| 275 | |
| 276 | extern bool unittest_running; |
| 277 | |
| 278 | bool rrdr_relative_window_to_absolute(time_t *after, time_t *before, time_t now); |
| 279 | bool rrdr_relative_window_to_absolute_query(time_t *after, time_t *before, time_t *now_ptr, bool unittest); |
| 280 | |
| 281 | int netdata_base64_decode(unsigned char *out, const unsigned char *in, int in_len); |
| 282 | int netdata_base64_encode(unsigned char *encoded, const unsigned char *input, size_t input_size); |
| 283 | |
| 284 | // -------------------------------------------------------------------------------------------------------------------- |
| 285 | |
| 286 | static inline void freez_charp(char **p) { |
| 287 | freez(*p); |
| 288 | } |
| 289 | |
| 290 | static inline void freez_const_charp(const char **p) { |
| 291 | freez((void *)*p); |
| 292 | } |
| 293 | |
| 294 | #define CLEAN_CONST_CHAR_P _cleanup_(freez_const_charp) const char |
| 295 | #define CLEAN_CHAR_P _cleanup_(freez_charp) char |
| 296 | |
| 297 | // -------------------------------------------------------------------------------------------------------------------- |
| 298 | // automatic cleanup function, instead of pthread pop/push |
| 299 | |
| 300 | // volatile: Tells the compiler that the variable defined might be accessed in unexpected ways |
| 301 | // (e.g., by the cleanup function). This prevents it from being optimized out. |
| 302 | #define CLEANUP_FUNCTION_REGISTER(func) volatile void * __attribute__((cleanup(func))) |
| 303 | |
| 304 | static inline void *CLEANUP_FUNCTION_GET_PTR(void *pptr) { |
| 305 | void *ret; |
| 306 | void **p = (void **)pptr; |
| 307 | if(p) { |
| 308 | ret = *p; |
| 309 | *p = NULL; // use it only once - this will prevent using it again |
| 310 | |
| 311 | if(!ret) |
| 312 | nd_log(NDLS_DAEMON, NDLP_ERR, "cleanup function called multiple times!"); |
| 313 | } |
| 314 | else { |
| 315 | nd_log(NDLS_DAEMON, NDLP_ERR, "cleanup function called with NULL pptr!"); |
| 316 | ret = NULL; |
| 317 | } |
| 318 | |
| 319 | return ret; |
| 320 | } |
| 321 | |
| 322 | // -------------------------------------------------------------------------------------------------------------------- |
| 323 | |
| 324 | # ifdef __cplusplus |
| 325 | } |
| 326 | # endif |
| 327 | |
| 328 | #endif // NETDATA_LIB_H |