d28b61ed5 · cryptotaxi247/netdata-1

CMakeLists.txt

+73

@@ -268,6 +268,41 @@ if(ENABLE_MIMALLOC)

268 netdata_add_mimalloc()

269 endif()

270

271 +option(ENABLE_JEMALLOC "Disable jemalloc allocator" OFF)

272 +

273 +if(ENABLE_JEMALLOC)

274 + pkg_check_modules(JEMALLOC QUIET jemalloc)

275 + if(JEMALLOC_FOUND)

276 + # Check if jemalloc has arena API

277 + set(CMAKE_REQUIRED_INCLUDES ${JEMALLOC_INCLUDE_DIRS})

278 + set(CMAKE_REQUIRED_LIBRARIES ${JEMALLOC_LIBRARIES})

279 + check_c_source_compiles("

280 + #include <jemalloc/jemalloc.h>

281 + int main() {

282 + unsigned narenas;

283 + size_t sz = sizeof(narenas);

284 + mallctl(\"arenas.narenas\", &narenas, &sz, NULL, 0);

285 + return 0;

286 + }

287 + " HAVE_JEMALLOC_ARENA_API)

288 +

289 + if(HAVE_JEMALLOC_ARENA_API)

290 + set(ENABLE_JEMALLOC ON CACHE BOOL "Enable jemalloc allocator" FORCE)

291 + message(STATUS "Jemalloc found with arena API support - enabling")

292 + else()

293 + if(ENABLE_JEMALLOC)

294 + message(FATAL_ERROR "Jemalloc was found but does not have arena API support")

295 + endif()

296 + message(STATUS "Jemalloc found but does not have arena API support - disabling")

297 + endif()

298 + else()

299 + if(ENABLE_JEMALLOC)

300 + message(FATAL_ERROR "Jemalloc support was explicitly enabled but jemalloc was not found")

301 + endif()

302 + message(STATUS "Jemalloc not found - disabling")

303 + endif()

304 +endif()

305 +

306 if(ENABLE_PLUGIN_GO)

307 include(NetdataGoTools)

308

@@ -449,6 +484,8 @@ check_function_exists(arc4random_uniform HAVE_ARC4RANDOM_UNIFORM)

484 check_function_exists(getrandom HAVE_GETRANDOM)

485 check_function_exists(sysinfo HAVE_SYSINFO)

486

487 +check_function_exists(timegm HAVE_TIMEGM)

488 +

489 #

490 # check source compilation

491 #

@@ -456,6 +493,15 @@ check_function_exists(sysinfo HAVE_SYSINFO)

493 include(CheckCSourceCompiles)

494 include(CheckCXXSourceCompiles)

495

496 +check_c_source_compiles("

497 +#include <time.h>

498 +int main(void) {

499 + struct tm t;

500 + (void)t.tm_gmtoff;

501 + return 0;

502 +}

503 +" HAVE_TM_GMTOFF)

504 +

505 set(CMAKE_REQUIRED_LIBRARIES pthread)

506 check_c_source_compiles("

507 #define _GNU_SOURCE

@@ -994,6 +1040,22 @@ set(LIBNETDATA_FILES

1040 src/libnetdata/os/get_system_pagesize.h

1041 src/libnetdata/os/hostname.c

1042 src/libnetdata/os/hostname.h

1043 + src/libnetdata/exit/exit_initiated.c

1044 + src/libnetdata/exit/exit_initiated.h

1045 + src/libnetdata/os/disk_space.c

1046 + src/libnetdata/os/disk_space.h

1047 + src/libnetdata/os/file_metadata.c

1048 + src/libnetdata/os/file_metadata.h

1049 + src/libnetdata/os/process_path.c

1050 + src/libnetdata/os/process_path.h

1051 + src/libnetdata/os/boottime.c

1052 + src/libnetdata/os/boottime.h

1053 + src/libnetdata/os/boot_id.c

1054 + src/libnetdata/os/boot_id.h

1055 + src/libnetdata/os/run_dir.c

1056 + src/libnetdata/os/run_dir.h

1057 + src/libnetdata/os/file_lock.c

1058 + src/libnetdata/os/file_lock.h

1059 )

1060

1061 list(APPEND LIBNETDATA_FILES ${INICFG_FILES})

@@ -1182,8 +1244,12 @@ set(DAEMON_FILES

1244 src/daemon/pulse/pulse-db-dbengine-retention.h

1245 src/daemon/pulse/pulse-parents.c

1246 src/daemon/pulse/pulse-parents.h

1247 + src/daemon/daemon-status-file.c

1248 + src/daemon/daemon-status-file.h

1249 src/daemon/config/netdata-conf-ssl.c

1250 src/daemon/config/netdata-conf-ssl.h

1251 + src/daemon/daemon-systemd-watcher.c

1252 + src/daemon/daemon-systemd-watcher.h

1253 )

1254

1255 set(H2O_FILES

@@ -2170,6 +2236,13 @@ netdata_add_jsonc_to_target(libnetdata)

2236

2237 netdata_add_libyaml_to_target(libnetdata)

2238

2239 +# jemalloc

2240 +if(ENABLE_JEMALLOC)

2241 + target_link_libraries(libnetdata PUBLIC ${JEMALLOC_LIBRARIES})

2242 + target_include_directories(libnetdata PUBLIC ${JEMALLOC_INCLUDE_DIRS})

2243 + target_compile_options(libnetdata PUBLIC ${JEMALLOC_CFLAGS_OTHER})

2244 +endif()

2245 +

2246 # libunwind

2247 if(ENABLE_LIBUNWIND)

2248 pkg_check_modules(LIBUNWIND libunwind IMPORTED_TARGET)

packaging/cmake/config.cmake.h.in

+3

@@ -79,6 +79,8 @@

79 #cmakedefine HAVE_RAND_S

80 #cmakedefine HAVE_GETRANDOM

81 #cmakedefine HAVE_SYSINFO

82 +#cmakedefine HAVE_TIMEGM

83 +#cmakedefine HAVE_TM_GMTOFF

84

85 #cmakedefine HAVE_LIBUNWIND

86 #cmakedefine HAVE_BACKTRACE

@@ -116,6 +118,7 @@

118 #cmakedefine HAVE_FUNC_ATTRIBUTE_NORETURN

119 #cmakedefine HAVE_FUNC_ATTRIBUTE_RETURNS_NONNULL

120 #cmakedefine HAVE_FUNC_ATTRIBUTE_WARN_UNUSED_RESULT

121 +#cmakedefine HAVE_JEMALLOC_ARENA_API

122

123 // enabled features

124

packaging/utils/compile-and-run-windows.sh

+49 -10

@@ -1,4 +1,6 @@

1 -#!/bin/sh

1 +#!/bin/bash

2 +

3 +RUN_AS_SERVICE=0

4

5 # On MSYS2, install these dependencies to build netdata:

6 install_dependencies() {

@@ -21,13 +23,21 @@ install_dependencies() {

23 msys/libcurl msys/libcurl-devel

24 }

25

26 +BUILD_FOR_PACKAGING="Off"

27 +

28 if [ "${1}" = "install" ]

29 then

30 install_dependencies || exit 1

31 exit 0

32 fi

33

30 -BUILD_FOR_PACKAGING="Off"

34 +if [ "${1}" = "service" ]

35 +then

36 + RUN_AS_SERVICE=1

37 + BUILD_FOR_PACKAGING="On"

38 + shift

39 +fi

40 +

41 if [ "${1}" = "package" ]

42 then

43 BUILD_FOR_PACKAGING="On"

@@ -70,12 +80,18 @@ then

80 ${NULL}

81 fi

82

83 +echo "Compiling Netdata..."

84 ninja -v -C "${build}" || ninja -v -C "${build}" -j 1

85

75 -echo "Stopping service Netdata"

76 -sc stop "Netdata" || echo "Failed"

86 +echo "Stopping service Netdata..."

87 +sc stop "Netdata" || echo "stop Failed, ok"

88 +

89 +if [ $RUN_AS_SERVICE -eq 1 ]; then

90 + sc delete "Netdata" || echo "delete Failed, ok"

91 +fi

92

78 -ninja -v -C "${build}" install || ninja -v -C "${build}" -j 1

93 +rm -f /opt/netdata/usr/bin/*.dll || echo "deleting old .dll files failed, ok"

94 +ninja -v -C "${build}" install

95

96 # register the event log publisher

97 cmd.exe //c "$(cygpath -w -a "/opt/netdata/usr/bin/wevt_netdata_install.bat")"

@@ -84,9 +100,32 @@ cmd.exe //c "$(cygpath -w -a "/opt/netdata/usr/bin/wevt_netdata_install.bat")"

100 #echo "Compile with:"

101 #echo "ninja -v -C \"${build}\" install || ninja -v -C \"${build}\" -j 1"

102

87 -echo "starting netdata..."

88 -# enable JIT debug with gdb

89 -export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"

103 +if [ $RUN_AS_SERVICE -eq 1 ]; then

104 + echo

105 + echo "Copying library files to /opt/netdata/usr/bin ..."

106 + ldd /opt/netdata/usr/bin/netdata |\

107 + grep " => /usr/bin/" |\

108 + sed -e 's|\s\+| |g' -e 's|^ ||g' |\

109 + cut -d ' ' -f 3 |\

110 + while read x; do

111 + cp $x /opt/netdata/usr/bin/

112 + done

113

91 -rm -rf /opt/netdata/var/log/netdata/*.log || echo

92 -/opt/netdata/usr/bin/netdata -D

114 + echo

115 + echo "Registering Netdata service..."

116 + sc create "Netdata" binPath= "$(cygpath.exe -w /opt/netdata/usr/bin/netdata.exe)" start= auto

117 +

118 + echo "Starting Netdata service..."

119 + sc start "Netdata"

120 +

121 +else

122 +

123 + echo "Starting netdata..."

124 +

125 + # enable JIT debug with gdb

126 + export MSYS="error_start:$(cygpath -w /usr/bin/gdb)"

127 +

128 + rm -rf /opt/netdata/var/log/netdata/*.log || echo

129 + /opt/netdata/usr/bin/netdata -D

130 +

131 +fi

src/aclk/aclk.c

+7 -7

@@ -199,7 +199,7 @@ static int wait_till_agent_claimed(void)

199 * @param aclk_hostname points to location where string pointer to hostname will be set

200 * @param aclk_port port to int where port will be saved

201 *

202 - * @return If non 0 returned irrecoverable error happened (or netdata_exit) and ACLK should be terminated

202 + * @return If non 0 returned irrecoverable error happened (or exit_initiated) and ACLK should be terminated

203 */

204 static int wait_till_agent_claim_ready()

205 {

@@ -306,7 +306,7 @@ static int handle_connection(mqtt_wss_client client)

306 {

307 while (service_running(SERVICE_ACLK)) {

308 // timeout 1000 to check at least once a second

309 - // for netdata_exit

309 + // for exit_initiated

310 int rc = mqtt_wss_service(client, 1000);

311 if (rc < 0){

312 worker_is_busy(WORKER_ACLK_DISCONNECTED);

@@ -452,9 +452,9 @@ static unsigned long aclk_reconnect_delay() {

452 return aclk_tbeb_delay(0, aclk_env->backoff.base, aclk_env->backoff.min_s, aclk_env->backoff.max_s);

453 }

454

455 -/* Block till aclk_reconnect_delay is satisfied or netdata_exit is signalled

455 +/* Block till aclk_reconnect_delay is satisfied or exit_initiated is signalled

456 * @return 0 - Go ahead and connect (delay expired)

457 - * 1 - netdata_exit

457 + * 1 - exit_initiated

458 */

459 #define NETDATA_EXIT_POLL_MS (MSEC_PER_SEC/4)

460 static int aclk_block_till_recon_allowed() {

@@ -466,7 +466,7 @@ static int aclk_block_till_recon_allowed() {

466 nd_log(NDLS_DAEMON, NDLP_DEBUG,

467 "Wait before attempting to reconnect in %.3f seconds", recon_delay / (float)MSEC_PER_SEC);

468

469 - // we want to wake up from time to time to check netdata_exit

469 + // we want to wake up from time to time to check exit_initiated

470 worker_is_busy(WORKER_ACLK_WAITING_TO_CONNECT);

471 while (recon_delay)

472 {

@@ -602,7 +602,7 @@ const char *aclk_cloud_base_url = NULL;

602 * @param client instance of mqtt_wss_client

603 * @return 0 - Successful Connection,

604 * <0 - Irrecoverable Error -> Kill ACLK,

605 - * >0 - netdata_exit

605 + * >0 - exit_initiated

606 */

607 #define CLOUD_BASE_URL_READ_RETRY 30

608 #ifdef ACLK_SSL_ALLOW_SELF_SIGNED

@@ -865,7 +865,7 @@ void *aclk_main(void *ptr)

865 mqtt_wss_set_max_buf_size(mqttwss_client, 25*1024*1024);

866

867 // Keep reconnecting and talking until our time has come

868 - // and the Grim Reaper (netdata_exit) calls

868 + // and the Grim Reaper (exit_initiated) calls

869 netdata_log_info("ACLK: Starting ACLK query event loop");

870 aclk_query_init(mqttwss_client);

871 do {

src/claim/cloud-conf.c

+2

@@ -54,6 +54,8 @@ static void cloud_conf_load_defaults(void) {

54 }

55

56 void cloud_conf_load(int silent) {

57 + netdata_conf_section_directories();

58 +

59 errno_clear();

60 char *filename = filename_from_path_entry_strdupz(netdata_configured_cloud_dir, "cloud.conf");

61 int ret = inicfg_load(&cloud_config, filename, 1, NULL);

src/cli/cli.c

+1 -1

@@ -133,7 +133,7 @@ static void connect_cb(uv_connect_t* req, int status)

133 (void)req;

134 if (status) {

135 fprintf(stderr, "uv_pipe_connect(): %s\n", uv_strerror(status));

136 - fprintf(stderr, "Make sure the netdata service is running.\n");

136 + fprintf(stderr, "Cannot connect to '%s'.\nMake sure the netdata service is running.\n", daemon_pipename());

137 exit(-1);

138 }

139 if (0 == command_string_size) {

src/collectors/cgroups.plugin/cgroup-name.sh.in

+2 -2

@@ -72,14 +72,14 @@ log() {

72

73 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

74

75 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

75 + systemd-cat-native --log-as-netdata <<EOFLOG

76 INVOCATION_ID=${NETDATA_INVOCATION_ID}

77 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

78 PRIORITY=${level}

79 THREAD_TAG=cgroup-name

80 ND_LOG_SOURCE=collector

81 ND_REQUEST=${cmd_line}

82 -MESSAGE=${*//\\n/--NEWLINE--}

82 +MESSAGE=${*//$'\n'/\\n}

83

84 EOFLOG

85 # AN EMPTY LINE IS NEEDED ABOVE

src/collectors/cgroups.plugin/cgroup-network-helper.sh.in

+2 -2

@@ -93,14 +93,14 @@ log() {

93

94 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

95

96 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

96 + systemd-cat-native --log-as-netdata <<EOFLOG

97 INVOCATION_ID=${NETDATA_INVOCATION_ID}

98 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

99 PRIORITY=${level}

100 THREAD_TAG=cgroup-network-helper

101 ND_LOG_SOURCE=collector

102 ND_REQUEST=${cmd_line}

103 -MESSAGE=${*//\\n/--NEWLINE--}

103 +MESSAGE=${*//$'\n'/\\n}

104

105 EOFLOG

106 # AN EMPTY LINE IS NEEDED ABOVE

src/collectors/charts.d.plugin/charts.d.plugin.in

+2 -2

@@ -76,13 +76,13 @@ log() {

76

77 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

78

79 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

79 + systemd-cat-native --log-as-netdata <<EOFLOG

80 INVOCATION_ID=${NETDATA_INVOCATION_ID}

81 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

82 PRIORITY=${level}

83 THREAD_TAG=charts.d.plugin

84 ND_LOG_SOURCE=collector

85 -MESSAGE=${MODULE_NAME}: ${*//\\n/--NEWLINE--}

85 +MESSAGE=${MODULE_NAME}: ${*//$'\n'/\\n}

86

87 EOFLOG

88 # AN EMPTY LINE IS NEEDED ABOVE

src/collectors/cups.plugin/cups_plugin.c

+3 -3

@@ -243,7 +243,7 @@ int main(int argc, char **argv) {

243 for (iteration = 0; 1; iteration++) {

244 heartbeat_next(&hb);

245

246 - if (unlikely(netdata_exit))

246 + if (unlikely(exit_initiated))

247 break;

248

249 reset_metrics();

@@ -315,7 +315,7 @@ int main(int argc, char **argv) {

315 }

316 cupsFreeDests(num_dest_total, dests);

317

318 - if (unlikely(netdata_exit))

318 + if (unlikely(exit_initiated))

319 break;

320

321 cups_job_t *jobs, *curr_job;

@@ -410,7 +410,7 @@ int main(int argc, char **argv) {

410

411 fflush(stdout);

412

413 - if (unlikely(netdata_exit))

413 + if (unlikely(exit_initiated))

414 break;

415

416 // restart check (14400 seconds)

src/collectors/freebsd.plugin/plugin_freebsd.c

+1 -1

@@ -91,7 +91,7 @@ void *freebsd_main(void *ptr)

91

92 // initialize FreeBSD plugin

93 if (freebsd_plugin_init())

94 - netdata_cleanup_and_exit(1, NULL, NULL, NULL);

94 + netdata_cleanup_and_exit(EXIT_REASON_FATAL, NULL, NULL, NULL);

95

96 // check the enabled status for each module

97 int i;

src/collectors/ioping.plugin/ioping.plugin.in

+2 -2

@@ -149,13 +149,13 @@ log() {

149

150 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

151

152 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

152 + systemd-cat-native --log-as-netdata <<EOFLOG

153 INVOCATION_ID=${NETDATA_INVOCATION_ID}

154 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

155 PRIORITY=${level}

156 THREAD_TAG=ioping.plugin

157 ND_LOG_SOURCE=collector

158 -MESSAGE=${MODULE_NAME}: ${*//\\n/--NEWLINE--}

158 +MESSAGE=${MODULE_NAME}: ${*//$'\n'/\\n}

159

160 EOFLOG

161 # AN EMPTY LINE IS NEEDED ABOVE

src/collectors/log2journal/log2journal-json.c

+69 -67

@@ -167,11 +167,32 @@ static inline bool json_parse_number(LOG_JSON_STATE *js) {

167 }

168 }

169

170 +static inline void copy_newline(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {

171 + if(*remaining > 3) {

172 + *(*d)++ = '\\';

173 + *(*d)++ = 'n';

174 + (*remaining) -= 2;

175 + }

176 +}

177 +

178 +static inline void copy_tab(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {

179 + if(*remaining > 3) {

180 + *(*d)++ = '\\';

181 + *(*d)++ = 't';

182 + (*remaining) -= 2;

183 + }

184 +}

185 +

186 static inline bool encode_utf8(unsigned codepoint, char **d, size_t *remaining) {

187 if (codepoint <= 0x7F) {

188 // 1-byte sequence

189 if (*remaining < 2) return false; // +1 for the null

174 - *(*d)++ = (char)codepoint;

190 + if(codepoint == '\n')

191 + copy_newline(NULL, d, remaining);

192 + else if(codepoint == '\t')

193 + copy_tab(NULL, d, remaining);

194 + else

195 + *(*d)++ = (char)codepoint;

196 (*remaining)--;

197 }

198 else if (codepoint <= 0x7FF) {

@@ -255,22 +276,6 @@ size_t parse_surrogate(const char *s, char *d, size_t *remaining) {

276 }

277 }

278

258 -static inline void copy_newline(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {

259 - if(*remaining > 3) {

260 - *(*d)++ = '\\';

261 - *(*d)++ = 'n';

262 - (*remaining) -= 2;

263 - }

264 -}

265 -

266 -static inline void copy_tab(LOG_JSON_STATE *js __maybe_unused, char **d, size_t *remaining) {

267 - if(*remaining > 3) {

268 - *(*d)++ = '\\';

269 - *(*d)++ = 't';

270 - (*remaining) -= 2;

271 - }

272 -}

273 -

279 static inline bool json_parse_string(LOG_JSON_STATE *js) {

280 static __thread char value[JOURNAL_MAX_VALUE_LEN];

281

@@ -511,35 +516,33 @@ static inline bool json_parse_array(LOG_JSON_STATE *js) {

516

517 json_consume_char(js);

518

514 - size_t index = 0;

515 - do {

516 - const char *s = json_current_pos(js);

517 - if(*s == ']') {

518 - json_consume_char(js);

519 - break;

520 - }

521 -

522 - if(!json_key_index_and_push(js, index))

523 - return false;

519 + const char *s = json_current_pos(js);

520 + if(*s == ']')

521 + json_consume_char(js);

522 + else {

523 + size_t index = 0;

524 + do {

525 + if (!json_key_index_and_push(js, index))

526 + return false;

527

525 - if(!json_parse_value(js))

526 - return false;

528 + if (!json_parse_value(js))

529 + return false;

530

528 - json_key_pop(js);

531 + json_key_pop(js);

532

530 - if(!json_expect_char_after_white_space(js, ",]"))

531 - return false;

533 + if (!json_expect_char_after_white_space(js, ",]"))

534 + return false;

535

533 - s = json_current_pos(js);

534 - json_consume_char(js);

535 - if(*s == ',') {

536 - index++;

537 - continue;

538 - }

539 - else // }

540 - break;

536 + s = json_current_pos(js);

537 + json_consume_char(js);

538 + if (*s == ',') {

539 + index++;

540 + continue;

541 + } else // ']'

542 + break;

543

542 - } while(true);

544 + } while (true);

545 + }

546

547 return true;

548 }

@@ -550,40 +553,39 @@ static inline bool json_parse_object(LOG_JSON_STATE *js) {

553

554 json_consume_char(js);

555

553 - do {

554 - const char *s = json_current_pos(js);

555 - if(*s == '}') {

556 - json_consume_char(js);

557 - break;

558 - }

559 -

560 - if (!json_expect_char_after_white_space(js, "\""))

561 - return false;

556 + const char *s = json_current_pos(js);

557 + if(*s == '}')

558 + json_consume_char(js);

559 + else {

560 + do {

561 + if (!json_expect_char_after_white_space(js, "\""))

562 + return false;

563

563 - if(!json_parse_key_and_push(js))

564 - return false;

564 + if (!json_parse_key_and_push(js))

565 + return false;

566

566 - if(!json_expect_char_after_white_space(js, ":"))

567 - return false;

567 + if (!json_expect_char_after_white_space(js, ":"))

568 + return false;

569

569 - json_consume_char(js);

570 + json_consume_char(js);

571

571 - if(!json_parse_value(js))

572 - return false;

572 + if (!json_parse_value(js))

573 + return false;

574

574 - json_key_pop(js);

575 + json_key_pop(js);

576

576 - if(!json_expect_char_after_white_space(js, ",}"))

577 - return false;

577 + if (!json_expect_char_after_white_space(js, ",}"))

578 + return false;

579

579 - s = json_current_pos(js);

580 - json_consume_char(js);

581 - if(*s == ',')

582 - continue;

583 - else // }

584 - break;

580 + s = json_current_pos(js);

581 + json_consume_char(js);

582 + if (*s == ',')

583 + continue;

584 + else // '}'

585 + break;

586

586 - } while(true);

587 + } while (true);

588 + }

589

590 return true;

591 }

src/collectors/nfacct.plugin/plugin_nfacct.c

+1 -1

@@ -837,7 +837,7 @@ int main(int argc, char **argv) {

837 for(iteration = 0; 1; iteration++) {

838 usec_t dt = heartbeat_next(&hb);

839

840 - if(unlikely(netdata_exit)) break;

840 + if(unlikely(exit_initiated)) break;

841

842 if(debug && iteration)

843 fprintf(stderr, "nfacct.plugin: iteration %zu, dt %"PRIu64" usec\n"

src/collectors/perf.plugin/perf_plugin.c

+1 -1

@@ -1325,7 +1325,7 @@ int main(int argc, char **argv) {

1325 for(iteration = 0; 1; iteration++) {

1326 usec_t dt = heartbeat_next(&hb);

1327

1328 - if (unlikely(netdata_exit))

1328 + if (unlikely(exit_initiated))

1329 break;

1330

1331 if (unlikely(debug && iteration))

src/collectors/systemd-journal.plugin/systemd-journal-annotations.c

+1

@@ -619,6 +619,7 @@ static void netdata_systemd_journal_message_ids_init(void) {

619 msgid_into_dict("ec87a56120d5431bace51e2fb8bba243", "Netdata log flood protection");

620 msgid_into_dict("acb33cb95778476baac702eb7e4e151d", "Netdata Cloud connection");

621 msgid_into_dict("d1f59606dd4d41e3b217a0cfcae8e632", "Netdata extreme cardinality");

622 + msgid_into_dict("02f47d350af5449197bf7a95b605a468", "Netdata exit reason");

623 msgid_into_dict("4fdf40816c124623a032b7fe73beacb8", "Netdata dynamic configuration");

624 }

625

src/collectors/systemd-journal.plugin/systemd-journal-files.c

+6 -6

@@ -344,9 +344,9 @@ void journal_file_update_header(const char *filename, struct journal_file *jf) {

344

345 jf->last_scan_header_vs_last_modified_ut = jf->file_last_modified_ut;

346

347 - nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

348 - "Journal file header updated '%s'",

349 - jf->filename);

347 +// nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

348 +// "Journal file header updated '%s'",

349 +// jf->filename);

350 }

351

352 static STRING *string_strdupz_source(const char *s, const char *e, size_t max_len, const char *prefix) {

@@ -459,9 +459,9 @@ static bool files_registry_conflict_cb(const DICTIONARY_ITEM *item __maybe_unuse

459

460 jf->msg_last_ut = jf->file_last_modified_ut;

461

462 - nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

463 - "Journal file updated to the journal files registry '%s'",

464 - jf->filename);

462 +// nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

463 +// "Journal file updated to the journal files registry '%s'",

464 +// jf->filename);

465 }

466

467 return false;

src/collectors/systemd-journal.plugin/systemd-journal-watcher.c

+4 -4

@@ -296,7 +296,7 @@ void process_event(Watcher *watcher, int inotifyFd, struct inotify_event *event)

296 return;

297 }

298

299 -#ifdef NETDATA_INTERNAL_CHECKS

299 +#if 0

300 {

301 CLEAN_BUFFER *wb = buffer_create(0, NULL);

302 INOTIFY_MASK_2buffer(wb, event->mask, ", ");

@@ -435,9 +435,9 @@ static void process_pending(Watcher *watcher) {

435 dictionary_del(journal_files_registry, fullPath);

436 }

437 else if(S_ISREG(info.st_mode)) {

438 - nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

439 - "JOURNAL WATCHER: file '%s' has been added/updated, updating the registry",

440 - fullPath);

438 +// nd_log(NDLS_COLLECTORS, NDLP_DEBUG,

439 +// "JOURNAL WATCHER: file '%s' has been added/updated, updating the registry",

440 +// fullPath);

441

442 struct journal_file t = {

443 .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC +

src/collectors/tc.plugin/tc-qos-helper.sh.in

+2 -2

@@ -73,14 +73,14 @@ log() {

73

74 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

75

76 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

76 + systemd-cat-native --log-as-netdata <<EOFLOG

77 INVOCATION_ID=${NETDATA_INVOCATION_ID}

78 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

79 PRIORITY=${level}

80 THREAD_TAG=tc-qos-helper

81 ND_LOG_SOURCE=collector

82 ND_REQUEST=${cmd_line}

83 -MESSAGE=${*//\\n/--NEWLINE--}

83 +MESSAGE=${*//$'\n'/\\n}

84

85 EOFLOG

86 # AN EMPTY LINE IS NEEDED ABOVE

src/collectors/windows.plugin/perflib-adcs.c

+2 -5

@@ -662,11 +662,8 @@ static bool do_ADCS(PERF_DATA_BLOCK *pDataBlock, int update_every)

662 netdata_adcs_failed_requets,

663 netdata_adcs_issued_requets,

664 netdata_adcs_pending_requets,

665 -

665 netdata_adcs_challenge_response,

667 -

666 netdata_adcs_retrieval_processing,

669 -

667 netdata_adcs_crypto_singing_time,

668 netdata_adcs_policy_mod_processing_time,

669 netdata_adcs_challenge_response_processing_time,

@@ -691,8 +688,8 @@ static bool do_ADCS(PERF_DATA_BLOCK *pDataBlock, int update_every)

688

689 struct adcs_certificate *ptr = dictionary_set(adcs_certificates, windows_shared_buffer, NULL, sizeof(*ptr));

690

694 - for (int i = 0; doADCS[i]; i++)

695 - doADCS[i](ptr, pDataBlock, pObjectType, update_every);

691 + for (int j = 0; doADCS[j] ;j++)

692 + doADCS[j](ptr, pDataBlock, pObjectType, update_every);

693 }

694

695 return true;

src/collectors/xenstat.plugin/xenstat_plugin.c

+1 -1

@@ -1026,7 +1026,7 @@ int main(int argc, char **argv) {

1026 for(iteration = 0; 1; iteration++) {

1027 usec_t dt = heartbeat_next(&hb);

1028

1029 - if(unlikely(netdata_exit)) break;

1029 + if(unlikely(exit_initiated)) break;

1030

1031 if(unlikely(debug && iteration))

1032 fprintf(stderr, "xenstat.plugin: iteration %zu, dt %lu usec\n", iteration, dt);

src/daemon/analytics.c

+107 -49

@@ -71,45 +71,125 @@ void analytics_log_data(void)

71 void analytics_free_data(void)

72 {

73 freez(analytics_data.netdata_config_stream_enabled);

74 + analytics_data.netdata_config_stream_enabled = NULL;

75 +

76 freez(analytics_data.netdata_config_memory_mode);

77 + analytics_data.netdata_config_memory_mode = NULL;

78 +

79 freez(analytics_data.netdata_config_exporting_enabled);

80 + analytics_data.netdata_config_exporting_enabled = NULL;

81 +

82 freez(analytics_data.netdata_exporting_connectors);

83 + analytics_data.netdata_exporting_connectors = NULL;

84 +

85 freez(analytics_data.netdata_allmetrics_prometheus_used);

86 + analytics_data.netdata_allmetrics_prometheus_used = NULL;

87 +

88 freez(analytics_data.netdata_allmetrics_shell_used);

89 + analytics_data.netdata_allmetrics_shell_used = NULL;

90 +

91 freez(analytics_data.netdata_allmetrics_json_used);

92 + analytics_data.netdata_allmetrics_json_used = NULL;

93 +

94 freez(analytics_data.netdata_dashboard_used);

95 + analytics_data.netdata_dashboard_used = NULL;

96 +

97 freez(analytics_data.netdata_collectors);

98 + analytics_data.netdata_collectors = NULL;

99 +

100 freez(analytics_data.netdata_collectors_count);

101 + analytics_data.netdata_collectors_count = NULL;

102 +

103 freez(analytics_data.netdata_buildinfo);

104 + analytics_data.netdata_buildinfo = NULL;

105 +

106 freez(analytics_data.netdata_config_page_cache_size);

107 + analytics_data.netdata_config_page_cache_size = NULL;

108 +

109 freez(analytics_data.netdata_config_multidb_disk_quota);

110 + analytics_data.netdata_config_multidb_disk_quota = NULL;

111 +

112 freez(analytics_data.netdata_config_https_enabled);

113 + analytics_data.netdata_config_https_enabled = NULL;

114 +

115 freez(analytics_data.netdata_config_web_enabled);

116 + analytics_data.netdata_config_web_enabled = NULL;

117 +

118 freez(analytics_data.netdata_config_release_channel);

119 + analytics_data.netdata_config_release_channel = NULL;

120 +

121 freez(analytics_data.netdata_mirrored_host_count);

122 + analytics_data.netdata_mirrored_host_count = NULL;

123 +

124 freez(analytics_data.netdata_mirrored_hosts_reachable);

125 + analytics_data.netdata_mirrored_hosts_reachable = NULL;

126 +

127 freez(analytics_data.netdata_mirrored_hosts_unreachable);

128 + analytics_data.netdata_mirrored_hosts_unreachable = NULL;

129 +

130 freez(analytics_data.netdata_notification_methods);

131 + analytics_data.netdata_notification_methods = NULL;

132 +

133 freez(analytics_data.netdata_alarms_normal);

134 + analytics_data.netdata_alarms_normal = NULL;

135 +

136 freez(analytics_data.netdata_alarms_warning);

137 + analytics_data.netdata_alarms_warning = NULL;

138 +

139 freez(analytics_data.netdata_alarms_critical);

140 + analytics_data.netdata_alarms_critical = NULL;

141 +

142 freez(analytics_data.netdata_charts_count);

143 + analytics_data.netdata_charts_count = NULL;

144 +

145 freez(analytics_data.netdata_metrics_count);

146 + analytics_data.netdata_metrics_count = NULL;

147 +

148 freez(analytics_data.netdata_config_is_parent);

149 + analytics_data.netdata_config_is_parent = NULL;

150 +

151 freez(analytics_data.netdata_config_hosts_available);

152 + analytics_data.netdata_config_hosts_available = NULL;

153 +

154 freez(analytics_data.netdata_host_cloud_available);

155 + analytics_data.netdata_host_cloud_available = NULL;

156 +

157 freez(analytics_data.netdata_host_aclk_available);

158 + analytics_data.netdata_host_aclk_available = NULL;

159 +

160 freez(analytics_data.netdata_host_aclk_protocol);

161 + analytics_data.netdata_host_aclk_protocol = NULL;

162 +

163 freez(analytics_data.netdata_host_aclk_implementation);

164 + analytics_data.netdata_host_aclk_implementation = NULL;

165 +

166 freez(analytics_data.netdata_host_agent_claimed);

167 + analytics_data.netdata_host_agent_claimed = NULL;

168 +

169 freez(analytics_data.netdata_host_cloud_enabled);

170 + analytics_data.netdata_host_cloud_enabled = NULL;

171 +

172 freez(analytics_data.netdata_config_https_available);

173 + analytics_data.netdata_config_https_available = NULL;

174 +

175 freez(analytics_data.netdata_install_type);

176 + analytics_data.netdata_install_type = NULL;

177 +

178 freez(analytics_data.netdata_config_is_private_registry);

179 + analytics_data.netdata_config_is_private_registry = NULL;

180 +

181 freez(analytics_data.netdata_config_use_private_registry);

182 + analytics_data.netdata_config_use_private_registry = NULL;

183 +

184 freez(analytics_data.netdata_config_oom_score);

185 + analytics_data.netdata_config_oom_score = NULL;

186 +

187 freez(analytics_data.netdata_prebuilt_distro);

188 + analytics_data.netdata_prebuilt_distro = NULL;

189 +

190 freez(analytics_data.netdata_fail_reason);

191 + analytics_data.netdata_fail_reason = NULL;

192 +

193 }

194

195 /*

@@ -150,7 +230,7 @@ void analytics_set_data_str(char **name, const char *value)

230 */

231 void analytics_log_prometheus(void)

232 {

153 - if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.prometheus_hits < ANALYTICS_MAX_PROMETHEUS_HITS)) {

233 + if (netdata_anonymous_statistics_enabled && likely(analytics_data.prometheus_hits < ANALYTICS_MAX_PROMETHEUS_HITS)) {

234 analytics_data.prometheus_hits++;

235 char b[21];

236 snprintfz(b, sizeof(b) - 1, "%zu", analytics_data.prometheus_hits);

@@ -163,7 +243,7 @@ void analytics_log_prometheus(void)

243 */

244 void analytics_log_shell(void)

245 {

166 - if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.shell_hits < ANALYTICS_MAX_SHELL_HITS)) {

246 + if (netdata_anonymous_statistics_enabled && likely(analytics_data.shell_hits < ANALYTICS_MAX_SHELL_HITS)) {

247 analytics_data.shell_hits++;

248 char b[21];

249 snprintfz(b, sizeof(b) - 1, "%zu", analytics_data.shell_hits);

@@ -176,7 +256,7 @@ void analytics_log_shell(void)

256 */

257 void analytics_log_json(void)

258 {

179 - if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.json_hits < ANALYTICS_MAX_JSON_HITS)) {

259 + if (netdata_anonymous_statistics_enabled && likely(analytics_data.json_hits < ANALYTICS_MAX_JSON_HITS)) {

260 analytics_data.json_hits++;

261 char b[21];

262 snprintfz(b, sizeof(b) - 1, "%zu", analytics_data.json_hits);

@@ -189,7 +269,7 @@ void analytics_log_json(void)

269 */

270 void analytics_log_dashboard(void)

271 {

192 - if (netdata_anonymous_statistics_enabled == 1 && likely(analytics_data.dashboard_hits < ANALYTICS_MAX_DASHBOARD_HITS)) {

272 + if (netdata_anonymous_statistics_enabled && likely(analytics_data.dashboard_hits < ANALYTICS_MAX_DASHBOARD_HITS)) {

273 analytics_data.dashboard_hits++;

274 char b[21];

275 snprintfz(b, sizeof(b) - 1, "%zu", analytics_data.dashboard_hits);

@@ -777,47 +857,24 @@ void get_system_timezone(void)

857 }

858 }

859

780 -void analytics_statistic_send(const analytics_statistic_t *statistic) {

781 - if (!statistic)

782 - return;

783 -

784 - static char *as_script;

785 -

786 - if (netdata_anonymous_statistics_enabled == -1) {

787 - char *optout_file = mallocz(

788 - sizeof(char) *

789 - (strlen(netdata_configured_user_config_dir) + strlen(".opt-out-from-anonymous-statistics") + 2));

790 -

791 - sprintf(optout_file, "%s/%s", netdata_configured_user_config_dir, ".opt-out-from-anonymous-statistics");

792 -

793 - if (likely(access(optout_file, R_OK) != 0)) {

794 - as_script = mallocz(

795 - sizeof(char) *

796 - (strlen(netdata_configured_primary_plugins_dir) + strlen("anonymous-statistics.sh") + 2));

797 -

798 - sprintf(as_script, "%s/%s", netdata_configured_primary_plugins_dir, "anonymous-statistics.sh");

799 -

800 - if (unlikely(access(as_script, R_OK) != 0)) {

801 - netdata_anonymous_statistics_enabled = 0;

802 -

803 - nd_log(NDLS_DAEMON, NDLP_DEBUG,

804 - "Statistics script '%s' not found.",

805 - as_script);

860 +static bool analytics_script_exists(void) {

861 + char filename[FILENAME_MAX + 1];

862 + snprintfz(filename, sizeof(filename), "%s/anonymous-statistics.sh", netdata_configured_primary_plugins_dir);

863 + return access(filename, R_OK) == 0;

864 +}

865

807 - freez(as_script);

808 - }

809 - else

810 - netdata_anonymous_statistics_enabled = 1;

811 - }

812 - else {

813 - netdata_anonymous_statistics_enabled = 0;

814 - as_script = NULL;

815 - }

866 +bool analytics_check_enabled(void) {

867 + if(!netdata_anonymous_statistics_enabled)

868 + return false;

869

817 - freez(optout_file);

818 - }

870 + char filename[FILENAME_MAX + 1];

871 + snprintfz(filename, sizeof(filename), "%s/.opt-out-from-anonymous-statistics", netdata_configured_user_config_dir);

872 + netdata_anonymous_statistics_enabled = access(filename, R_OK) != 0;

873 + return netdata_anonymous_statistics_enabled;

874 +}

875

820 - if (!netdata_anonymous_statistics_enabled || !statistic->action)

876 +void analytics_statistic_send(const analytics_statistic_t *statistic) {

877 + if (!statistic || !statistic->action || !analytics_check_enabled() || !analytics_script_exists())

878 return;

879

880 const char *action_result = statistic->result;

@@ -825,16 +882,17 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) {

882

883 if (!statistic->result)

884 action_result = "";

885 +

886 if (!statistic->data)

887 action_data = "";

888

889 char *command_to_run = mallocz(

832 - sizeof(char) * (strlen(statistic->action) + strlen(action_result) + strlen(action_data) + strlen(as_script) +

890 + sizeof(char) * (strlen(statistic->action) + strlen(action_result) + strlen(action_data) + FILENAME_MAX +

891 analytics_data.data_length + (ANALYTICS_NO_OF_ITEMS * 3) + 15));

892 sprintf(

893 command_to_run,

836 - "%s '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ",

837 - as_script,

894 + "%s/anonymous-statistics.sh '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' '%s' ",

895 + netdata_configured_primary_plugins_dir,

896 statistic->action,

897 action_result,

898 action_data,

@@ -880,8 +938,8 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) {

938 analytics_data.netdata_fail_reason);

939

940 nd_log(NDLS_DAEMON, NDLP_DEBUG,

883 - "%s '%s' '%s' '%s'",

884 - as_script, statistic->action, action_result, action_data);

941 + "%s/anonymous-statistics.sh '%s' '%s' '%s'",

942 + netdata_configured_primary_plugins_dir, statistic->action, action_result, action_data);

943

944 POPEN_INSTANCE *instance = spawn_popen_run(command_to_run);

945 if (instance) {

@@ -902,8 +960,8 @@ void analytics_statistic_send(const analytics_statistic_t *statistic) {

960 }

961 else

962 nd_log(NDLS_DAEMON, NDLP_NOTICE,

905 - "Failed to run statistics script: %s.",

906 - as_script);

963 + "Failed to run statistics script: %s/anonymous-statistics.sh",

964 + netdata_configured_primary_plugins_dir);

965

966 freez(command_to_run);

967 }

src/daemon/analytics.h

+1

@@ -96,6 +96,7 @@ typedef struct {

96 } analytics_statistic_t;

97

98 void analytics_statistic_send(const analytics_statistic_t *statistic);

99 +bool analytics_check_enabled(void);

100

101 extern struct analytics_data analytics_data;

102

src/daemon/buildinfo.c

+10 -8

@@ -1512,14 +1512,16 @@ static void populate_packaging_info() {

1512 }

1513

1514 OS_SYSTEM_MEMORY sm = os_system_memory(true);

1515 - char buf[1024];

1516 - snprintfz(buf, sizeof(buf), "%" PRIu64, sm.ram_total_bytes);

1517 - // size_snprintf(buf, sizeof(buf), sm.ram_total_bytes, "B", false);

1518 - build_info_set_value_strdupz(BIB_RUNTIME_MEM_TOTAL, buf);

1519 -

1520 - snprintfz(buf, sizeof(buf), "%" PRIu64, sm.ram_available_bytes);

1521 - // size_snprintf(buf, sizeof(buf), sm.ram_available_bytes, "B", false);

1522 - build_info_set_value_strdupz(BIB_RUNTIME_MEM_AVAIL, buf);

1515 + if(OS_SYSTEM_MEMORY_OK(sm)) {

1516 + char buf[1024];

1517 + snprintfz(buf, sizeof(buf), "%" PRIu64, sm.ram_total_bytes);

1518 + // size_snprintf(buf, sizeof(buf), sm.ram_total_bytes, "B", false);

1519 + build_info_set_value_strdupz(BIB_RUNTIME_MEM_TOTAL, buf);

1520 +

1521 + snprintfz(buf, sizeof(buf), "%" PRIu64, sm.ram_available_bytes);

1522 + // size_snprintf(buf, sizeof(buf), sm.ram_available_bytes, "B", false);

1523 + build_info_set_value_strdupz(BIB_RUNTIME_MEM_AVAIL, buf);

1524 + }

1525 }

1526

1527 // ----------------------------------------------------------------------------

src/daemon/commands.c

+1 -1

@@ -164,7 +164,7 @@ static cmd_status_t cmd_exit_execute(char *args, char **message)

164

165 nd_log_limits_unlimited();

166 netdata_log_info("COMMAND: Cleaning up to exit.");

167 - netdata_cleanup_and_exit(0, NULL, NULL, NULL);

167 + netdata_cleanup_and_exit(EXIT_REASON_CMD_EXIT, NULL, NULL, NULL);

168 exit(0);

169

170 return CMD_STATUS_SUCCESS;

src/daemon/common.h

+1 -1

@@ -90,7 +90,7 @@ extern const char *netdata_configured_host_prefix;

90 extern const char *netdata_configured_timezone;

91 extern const char *netdata_configured_abbrev_timezone;

92 extern int32_t netdata_configured_utc_offset;

93 -extern int netdata_anonymous_statistics_enabled;

93 +extern bool netdata_anonymous_statistics_enabled;

94

95 extern bool netdata_ready;

96 extern time_t netdata_start_time;

src/daemon/config/netdata-conf-backwards-compatibility.c

+2

@@ -4,6 +4,8 @@

4 #include "database/engine/rrdengineapi.h"

5

6 void netdata_conf_backwards_compatibility(void) {

7 + FUNCTION_RUN_ONCE();

8 +

9 // move [global] options to the [web] section

10

11 inicfg_move(&netdata_config, CONFIG_SECTION_GLOBAL, "http port listen backlog",

src/daemon/config/netdata-conf-db.c

+3 -7

@@ -27,9 +27,7 @@ size_t get_tier_grouping(size_t tier) {

27 }

28

29 static void netdata_conf_dbengine_pre_logs(void) {

30 - static bool run = false;

31 - if(run) return;

32 - run = true;

30 + FUNCTION_RUN_ONCE();

31

32 errno_clear();

33

@@ -143,7 +141,7 @@ void netdata_conf_dbengine_init(const char *hostname) {

141

142 dbengine_out_of_memory_protection = 0; // will be calculated below

143 OS_SYSTEM_MEMORY sm = os_system_memory(true);

146 - if(sm.ram_total_bytes && sm.ram_available_bytes && sm.ram_total_bytes > sm.ram_available_bytes) {

144 + if(OS_SYSTEM_MEMORY_OK(sm) && sm.ram_total_bytes > sm.ram_available_bytes) {

145 // calculate the default out of memory protection size

146 uint64_t keep_free = sm.ram_total_bytes / 10;

147 if(keep_free > 5ULL * 1024 * 1024 * 1024)

@@ -340,9 +338,7 @@ void netdata_conf_dbengine_init(const char *hostname) {

338 }

339

340 void netdata_conf_section_db(void) {

343 - static bool run = false;

344 - if(run) return;

345 - run = true;

341 + FUNCTION_RUN_ONCE();

342

343 // ------------------------------------------------------------------------

344 // get default database update frequency

src/daemon/config/netdata-conf-directories.c

+1 -3

@@ -10,9 +10,7 @@ static const char *get_varlib_subdir_from_config(const char *prefix, const char

10 }

11

12 void netdata_conf_section_directories(void) {

13 - static bool run = false;

14 - if(run) return;

15 - run = true;

13 + FUNCTION_RUN_ONCE();

14

15 // ------------------------------------------------------------------------

16 // get system paths

src/daemon/config/netdata-conf-global.c

+4 -2

@@ -86,6 +86,10 @@ void libuv_initialize(void) {

86 }

87

88 void netdata_conf_section_global(void) {

89 + FUNCTION_RUN_ONCE();

90 +

91 + netdata_conf_section_directories();

92 +

93 // ------------------------------------------------------------------------

94 // get the hostname

95

@@ -99,8 +103,6 @@ void netdata_conf_section_global(void) {

103 netdata_configured_hostname = inicfg_get(&netdata_config, CONFIG_SECTION_GLOBAL, "hostname", buf);

104 netdata_log_debug(D_OPTIONS, "hostname set to '%s'", netdata_configured_hostname);

105

102 - netdata_conf_section_directories();

103 -

106 nd_profile_setup(); // required for configuring the database

107 netdata_conf_section_db();

108

src/daemon/config/netdata-conf-logs.c

+3 -3

@@ -25,9 +25,9 @@ static void debug_flags_initialize(void) {

25 }

26

27 void netdata_conf_section_logs(void) {

28 - static bool run = false;

29 - if(run) return;

30 - run = true;

28 + FUNCTION_RUN_ONCE();

29 +

30 + netdata_conf_section_directories();

31

32 nd_log_set_facility(inicfg_get(&netdata_config, CONFIG_SECTION_LOGS, "facility", "daemon"));

33

src/daemon/config/netdata-conf-profile.c

+4 -5

@@ -31,13 +31,14 @@ ND_PROFILE nd_profile_detect_and_configure(bool recheck) {

31

32 // required for detecting the profile

33 stream_conf_load();

34 + netdata_conf_section_directories();

35

36 ND_PROFILE def_profile = ND_PROFILE_NONE;

37

38 OS_SYSTEM_MEMORY mem = os_system_memory(true);

39 size_t cpus = os_get_system_cpus_uncached();

40

40 - if(cpus <= 1 || (mem.ram_total_bytes && mem.ram_total_bytes < 1ULL * 1024 * 1024 * 1024))

41 + if(cpus <= 1 || (OS_SYSTEM_MEMORY_OK(mem) && mem.ram_total_bytes < 1ULL * 1024 * 1024 * 1024))

42 def_profile = ND_PROFILE_IOT;

43

44 else if(stream_conf_is_parent(true))

@@ -94,15 +95,13 @@ ND_PROFILE nd_profile_detect_and_configure(bool recheck) {

95 struct nd_profile_t nd_profile = { 0 };

96

97 void nd_profile_setup(void) {

97 - static bool run = false;

98 - if(run) return;

99 - run = true;

98 + FUNCTION_RUN_ONCE();

99

100 ND_PROFILE profile = nd_profile_detect_and_configure(true); (void)profile;

101 if(netdata_conf_is_iot()) {

102 nd_profile.storage_tiers = 3; // MUST BE 1

103 nd_profile.update_every = 1; // MUST BE 2

105 - nd_profile.malloc_arenas = 1;

104 + nd_profile.malloc_arenas = 4;

105 nd_profile.malloc_trim = 32 * 1024;

106 nd_profile.stream_sender_compression = ND_COMPRESSION_FASTEST;

107 // web server threads = 6

src/daemon/config/netdata-conf-ssl.c

+3 -1

@@ -53,7 +53,7 @@ const char *detect_libcurl_default_ca() {

53 return NULL;

54 }

55

56 -static const char *detect_ca_path(void) {

56 +static inline const char *detect_ca_path(void) {

57 static const char *paths[] = {

58 "/etc/ssl/certs/ca-certificates.crt", // Debian, Ubuntu, Arch

59 "/etc/pki/tls/certs/ca-bundle.crt", // RHEL, CentOS, Fedora

@@ -74,6 +74,8 @@ static const char *detect_ca_path(void) {

74 }

75

76 void netdata_conf_ssl(void) {

77 + FUNCTION_RUN_ONCE();

78 +

79 netdata_ssl_initialize_openssl();

80

81 #if 0

src/daemon/config/netdata-conf-web.c

+3 -9

@@ -42,9 +42,7 @@ static int make_dns_decision(const char *section_name, const char *config_name,

42

43 extern struct netdata_static_thread *static_threads;

44 void web_server_threading_selection(void) {

45 - static bool run = false;

46 - if(run) return;

47 - run = true;

45 + FUNCTION_RUN_ONCE();

46

47 web_server_mode = web_server_mode_id(inicfg_get(&netdata_config, CONFIG_SECTION_WEB, "mode", web_server_mode_name(web_server_mode)));

48

@@ -58,9 +56,7 @@ void web_server_threading_selection(void) {

56 }

57

58 void netdata_conf_section_web(void) {

61 - static bool run = false;

62 - if(run) return;

63 - run = true;

59 + FUNCTION_RUN_ONCE();

60

61 web_client_timeout =

62 (int)inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_WEB, "disconnect idle clients after", web_client_timeout);

@@ -146,9 +142,7 @@ void netdata_conf_section_web(void) {

142 }

143

144 void netdata_conf_web_security_init(void) {

149 - static bool run = false;

150 - if(run) return;

151 - run = true;

145 + FUNCTION_RUN_ONCE();

146

147 char filename[FILENAME_MAX + 1];

148 snprintfz(filename, FILENAME_MAX, "%s/ssl/key.pem",netdata_configured_user_config_dir);

src/daemon/config/netdata-conf.c

+2 -3

@@ -4,9 +4,7 @@

4 #include "daemon/common.h"

5

6 bool netdata_conf_load(char *filename, char overwrite_used, const char **user) {

7 - static bool run = false;

8 - if(run) return false;

9 - run = true;

7 + FUNCTION_RUN_ONCE_RET(false);

8

9 errno_clear();

10

@@ -35,6 +33,7 @@ bool netdata_conf_load(char *filename, char overwrite_used, const char **user) {

33 }

34

35 netdata_conf_backwards_compatibility();

36 + netdata_conf_section_directories();

37 netdata_conf_section_global_run_as_user(user);

38 libuv_initialize();

39 return ret;

src/daemon/daemon-service.c

+1 -1

@@ -90,7 +90,7 @@ bool service_running(SERVICE_TYPE service) {

90 if (sth->type == SERVICE_THREAD_TYPE_NETDATA)

91 cancelled = nd_thread_signaled_to_cancel();

92

93 - return !sth->stop_immediately && !netdata_exit && !cancelled;

93 + return !sth->stop_immediately && !exit_initiated && !cancelled;

94 }

95

96 void service_signal_exit(SERVICE_TYPE service) {

src/daemon/daemon-service.h

+2 -1

@@ -19,7 +19,8 @@ typedef enum {

19 SERVICE_CONTEXT = (1 << 10),

20 SERVICE_ANALYTICS = (1 << 11),

21 SERVICE_EXPORTERS = (1 << 12),

22 - SERVICE_HTTPD = (1 << 13)

22 + SERVICE_HTTPD = (1 << 13),

23 + SERVICE_SYSTEMD = (1 << 14),

24 } SERVICE_TYPE;

25

26 typedef enum {

src/daemon/daemon-shutdown.c

+41 -15

@@ -2,6 +2,7 @@

2

3 #include "daemon-shutdown.h"

4 #include "daemon-service.h"

5 +#include "daemon-status-file.h"

6 #include "daemon/daemon-shutdown-watcher.h"

7 #include "static_threads.h"

8 #include "common.h"

@@ -26,6 +27,21 @@ void web_client_cache_destroy(void);

27

28 extern struct netdata_static_thread *static_threads;

29

30 +void netdata_log_exit_reason(void) {

31 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

32 + EXIT_REASON_2buffer(wb, exit_initiated, ", ");

33 +

34 + ND_LOG_STACK lgs[] = {

35 + ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &netdata_exit_msgid),

36 + ND_LOG_FIELD_END(),

37 + };

38 + ND_LOG_STACK_PUSH(lgs);

39 +

40 + nd_log(NDLS_DAEMON, is_exit_reason_normal(exit_initiated) ? NDLP_NOTICE : NDLP_CRIT,

41 + "NETDATA SHUTDOWN: initializing shutdown with code due to: %s",

42 + buffer_tostring(wb));

43 +}

44 +

45 void cancel_main_threads(void) {

46 nd_log_limits_unlimited();

47

@@ -146,8 +162,25 @@ static void rrdeng_flush_everything_and_wait(bool wait_flush, bool wait_collecto

162 }

163 #endif

164

149 -void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) {

150 - netdata_exit = 1;

165 +void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char *action_result, const char *action_data) {

166 + exit_initiated_set(reason);

167 + int ret = is_exit_reason_normal(exit_initiated) ? 0 : 1;

168 +

169 + // don't recurse (due to a fatal, while exiting)

170 + static bool run = false;

171 + if(run) {

172 + nd_log(NDLS_DAEMON, NDLP_ERR, "EXIT: Recursion detected. Exiting immediately.");

173 + exit(ret);

174 + }

175 + run = true;

176 + daemon_status_file_save(DAEMON_STATUS_EXITING);

177 +

178 + nd_log_limits_unlimited();

179 + netdata_log_exit_reason();

180 +

181 + watcher_thread_start();

182 + usec_t shutdown_start_time = now_monotonic_usec();

183 + watcher_shutdown_begin();

184

185 #ifdef ENABLE_DBENGINE

186 if(!ret && dbengine_enabled)

@@ -155,12 +188,6 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

188 rrdeng_flush_everything_and_wait(false, false);

189 #endif

190

158 - usec_t shutdown_start_time = now_monotonic_usec();

159 - watcher_shutdown_begin();

160 -

161 - nd_log_limits_unlimited();

162 - netdata_log_info("NETDATA SHUTDOWN: initializing shutdown with code %d...", ret);

163 -

191 // send the stat from our caller

192 analytics_statistic_t statistic = { action, action_result, action_data };

193 analytics_statistic_send(&statistic);

@@ -169,11 +196,6 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

196 statistic = (analytics_statistic_t) {"EXIT", ret?"ERROR":"OK","-"};

197 analytics_statistic_send(&statistic);

198

172 - char agent_crash_file[FILENAME_MAX + 1];

173 - char agent_incomplete_shutdown_file[FILENAME_MAX + 1];

174 - snprintfz(agent_crash_file, FILENAME_MAX, "%s/.agent_crash", netdata_configured_varlib_dir);

175 - snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir);

176 - (void) rename(agent_crash_file, agent_incomplete_shutdown_file);

199 watcher_step_complete(WATCHER_STEP_ID_CREATE_SHUTDOWN_FILE);

200

201 netdata_main_spawn_server_cleanup();

@@ -294,13 +316,14 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

316 netdata_ssl_cleanup();

317 watcher_step_complete(WATCHER_STEP_ID_FREE_OPENSSL_STRUCTURES);

318

297 - (void) unlink(agent_incomplete_shutdown_file);

319 watcher_step_complete(WATCHER_STEP_ID_REMOVE_INCOMPLETE_SHUTDOWN_FILE);

320

321 watcher_shutdown_end();

322 watcher_thread_stop();

323 curl_global_cleanup();

324

325 + daemon_status_file_save(DAEMON_STATUS_EXITED);

326 +

327 #ifdef OS_WINDOWS

328 return;

329 #endif

@@ -316,6 +339,9 @@ void netdata_cleanup_and_exit(int ret, const char *action, const char *action_re

339 exit(ret);

340 }

341 #else

319 - exit(ret);

342 + if(ret)

343 + _exit(ret);

344 + else

345 + exit(ret);

346 #endif

347 }

src/daemon/daemon-status-file.c new

+652

@@ -0,0 +1,652 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "common.h"

4 +#include "daemon-status-file.h"

5 +

6 +#include <curl/curl.h>

7 +#include <openssl/evp.h>

8 +#include <openssl/pem.h>

9 +#include <openssl/err.h>

10 +

11 +#define STATUS_FILENAME "status-netdata.json"

12 +#define STATUS_FILENAME_TMP "status-netdata.json.tmp"

13 +

14 +ENUM_STR_MAP_DEFINE(DAEMON_STATUS) = {

15 + { DAEMON_STATUS_NONE, "none"},

16 + { DAEMON_STATUS_INITIALIZING, "initializing"},

17 + { DAEMON_STATUS_RUNNING, "running"},

18 + { DAEMON_STATUS_EXITING, "exiting"},

19 + { DAEMON_STATUS_EXITED, "exited"},

20 +

21 + // terminator

22 + { 0, NULL },

23 +};

24 +ENUM_STR_DEFINE_FUNCTIONS(DAEMON_STATUS, DAEMON_STATUS_NONE, "none");

25 +

26 +ENUM_STR_MAP_DEFINE(DAEMON_OS_TYPE) = {

27 + {DAEMON_OS_TYPE_UNKNOWN, "unknown"},

28 + {DAEMON_OS_TYPE_LINUX, "linux"},

29 + {DAEMON_OS_TYPE_FREEBSD, "freebsd"},

30 + {DAEMON_OS_TYPE_MACOS, "macos"},

31 + {DAEMON_OS_TYPE_WINDOWS, "windows"},

32 +

33 + // terminator

34 + { 0, NULL },

35 +};

36 +ENUM_STR_DEFINE_FUNCTIONS(DAEMON_OS_TYPE, DAEMON_OS_TYPE_UNKNOWN, "unknown");

37 +

38 +static DAEMON_STATUS_FILE last_session_status = { 0 };

39 +static DAEMON_STATUS_FILE session_status = { 0 };

40 +

41 +// --------------------------------------------------------------------------------------------------------------------

42 +// json generation

43 +

44 +static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

45 + buffer_json_member_add_datetime_rfc3339(wb, "@timestamp", ds->timestamp_ut, true); // ECS

46 + buffer_json_member_add_uint64(wb, "version", 1); // custom

47 +

48 + buffer_json_member_add_object(wb, "agent"); // ECS

49 + {

50 + buffer_json_member_add_uuid(wb, "id", ds->host_id.uuid); // ECS

51 + buffer_json_member_add_uuid_compact(wb, "ephemeral_id", ds->invocation.uuid); // ECS

52 + buffer_json_member_add_string(wb, "version", ds->version); // ECS

53 +

54 + buffer_json_member_add_time_t(wb, "uptime", ds->uptime); // custom

55 +

56 + buffer_json_member_add_uuid(wb, "ND_node_id", ds->node_id.uuid); // custom

57 + buffer_json_member_add_uuid(wb, "ND_claim_id", ds->claim_id.uuid); // custom

58 +

59 + ND_PROFILE_2json(wb, "ND_profile", ds->profile); // custom

60 + buffer_json_member_add_string(wb, "ND_status", DAEMON_STATUS_2str(ds->status)); // custom

61 + EXIT_REASON_2json(wb, "ND_exit_reason", ds->exit_reason); // custom

62 +

63 + buffer_json_member_add_object(wb, "ND_timings"); // custom

64 + {

65 + buffer_json_member_add_time_t(wb, "init", ds->timings.init);

66 + buffer_json_member_add_time_t(wb, "exit", ds->timings.exit);

67 + }

68 + buffer_json_object_close(wb);

69 + }

70 + buffer_json_object_close(wb);

71 +

72 + buffer_json_member_add_object(wb, "host"); // ECS

73 + {

74 + buffer_json_member_add_object(wb, "boot"); // ECS

75 + {

76 + buffer_json_member_add_uuid(wb, "id", ds->boot_id.uuid); // ECS

77 + }

78 + buffer_json_object_close(wb);

79 + buffer_json_member_add_time_t(wb, "uptime", ds->boottime); // ECS

80 +

81 + buffer_json_member_add_object(wb, "memory"); // custom

82 + if(OS_SYSTEM_MEMORY_OK(ds->memory)) {

83 + buffer_json_member_add_uint64(wb, "total", ds->memory.ram_total_bytes);

84 + buffer_json_member_add_uint64(wb, "free", ds->memory.ram_available_bytes);

85 + }

86 + buffer_json_object_close(wb);

87 +

88 + buffer_json_member_add_object(wb, "disk"); // ECS

89 + {

90 + buffer_json_member_add_object(wb, "db");

91 + if (OS_SYSTEM_DISK_SPACE_OK(ds->var_cache)) {

92 + buffer_json_member_add_uint64(wb, "total", ds->var_cache.total_bytes);

93 + buffer_json_member_add_uint64(wb, "free", ds->var_cache.free_bytes);

94 + buffer_json_member_add_uint64(wb, "inodes_total", ds->var_cache.total_inodes);

95 + buffer_json_member_add_uint64(wb, "inodes_free", ds->var_cache.free_inodes);

96 + buffer_json_member_add_boolean(wb, "read_only", ds->var_cache.is_read_only);

97 + }

98 + buffer_json_object_close(wb);

99 + }

100 + buffer_json_object_close(wb);

101 + }

102 + buffer_json_object_close(wb);

103 +

104 + buffer_json_member_add_object(wb, "os"); // ECS

105 + {

106 + buffer_json_member_add_string(wb, "type", DAEMON_OS_TYPE_2str(ds->os_type)); // ECS

107 + }

108 + buffer_json_object_close(wb);

109 +

110 + buffer_json_member_add_object(wb, "fatal");

111 + {

112 + buffer_json_member_add_uint64(wb, "line", ds->fatal.line);

113 + buffer_json_member_add_string_or_empty(wb, "filename", ds->fatal.filename);

114 + buffer_json_member_add_string_or_empty(wb, "function", ds->fatal.function);

115 + buffer_json_member_add_string_or_empty(wb, "message", ds->fatal.message);

116 + buffer_json_member_add_string_or_empty(wb, "stack_trace", ds->fatal.stack_trace);

117 + }

118 + buffer_json_object_close(wb);

119 +}

120 +

121 +// --------------------------------------------------------------------------------------------------------------------

122 +// json parsing

123 +

124 +static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *error) {

125 + char path[1024]; path[0] = '\0';

126 +

127 + DAEMON_STATUS_FILE *ds = data;

128 + char datetime[RFC3339_MAX_LENGTH]; datetime[0] = '\0';

129 +

130 + // change management, version to know which fields to expect

131 + uint64_t version = 0;

132 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "version", version, error, true);

133 +

134 + bool required = false; // allow missing fields and values

135 +

136 + // Parse timestamp

137 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required);

138 + if(datetime[0])

139 + ds->timestamp_ut = rfc3339_parse_ut(datetime, NULL);

140 +

141 + // Parse agent object

142 + JSONC_PARSE_SUBOBJECT(jobj, path, "agent", error, required, {

143 + JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "id", ds->host_id.uuid, error, required);

144 + JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "ephemeral_id", ds->invocation.uuid, error, required);

145 + JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->version, error, required);

146 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "uptime", ds->uptime, error, required);

147 + JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, "ND_profile", ND_PROFILE_2id_one, ds->profile, error, required);

148 + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "ND_status", DAEMON_STATUS_2id, ds->status, error, required);

149 + JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, "ND_exit_reason", EXIT_REASON_2id_one, ds->exit_reason, error, required);

150 + JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "ND_node_id", ds->node_id.uuid, error, required);

151 + JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "ND_claim_id", ds->claim_id.uuid, error, required);

152 +

153 + JSONC_PARSE_SUBOBJECT(jobj, path, "ND_timings", error, required, {

154 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "init", ds->timings.init, error, required);

155 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "exit", ds->timings.exit, error, required);

156 + });

157 + });

158 +

159 + // Parse host object

160 + JSONC_PARSE_SUBOBJECT(jobj, path, "host", error, required, {

161 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "uptime", ds->boottime, error, required);

162 +

163 + JSONC_PARSE_SUBOBJECT(jobj, path, "boot", error, required, {

164 + JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "id", ds->boot_id.uuid, error, required);

165 + });

166 +

167 + JSONC_PARSE_SUBOBJECT(jobj, path, "memory", error, required, {

168 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "total", ds->memory.ram_total_bytes, error, required);

169 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "free", ds->memory.ram_available_bytes, error, required);

170 + if(!OS_SYSTEM_MEMORY_OK(ds->memory))

171 + ds->memory = OS_SYSTEM_MEMORY_EMPTY;

172 + });

173 +

174 + JSONC_PARSE_SUBOBJECT(jobj, path, "disk", error, required, {

175 + JSONC_PARSE_SUBOBJECT(jobj, path, "db", error, required, {

176 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "total", ds->var_cache.total_bytes, error, false);

177 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "free", ds->var_cache.free_bytes, error, false);

178 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "inodes_total", ds->var_cache.total_inodes, error, false);

179 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "inodes_free", ds->var_cache.free_inodes, error, false);

180 + JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "read_only", ds->var_cache.is_read_only, error, false);

181 + if(!OS_SYSTEM_DISK_SPACE_OK(ds->var_cache))

182 + ds->var_cache = OS_SYSTEM_DISK_SPACE_EMPTY;

183 + });

184 + });

185 + });

186 +

187 + // Parse os object

188 + JSONC_PARSE_SUBOBJECT(jobj, path, "os", error, required, {

189 + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "type", DAEMON_OS_TYPE_2id, ds->os_type, error, required);

190 + });

191 +

192 + // Parse fatal object

193 + JSONC_PARSE_SUBOBJECT(jobj, path, "fatal", error, required, {

194 + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "filename", ds->fatal.filename, error, required);

195 + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "function", ds->fatal.function, error, required);

196 + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "message", ds->fatal.message, error, required);

197 + JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, "stack_trace", ds->fatal.stack_trace, error, required);

198 + JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "line", ds->fatal.line, error, required);

199 + });

200 +

201 + return true;

202 +}

203 +

204 +// --------------------------------------------------------------------------------------------------------------------

205 +// get the current status

206 +

207 +static DAEMON_STATUS_FILE daemon_status_file_get(DAEMON_STATUS status) {

208 + usec_t now_ut = now_realtime_usec();

209 +

210 +#if defined(OS_LINUX)

211 + session_status.os_type = DAEMON_OS_TYPE_LINUX;

212 +#elif defined(OS_FREEBSD)

213 + session_status.os_type = DAEMON_OS_TYPE_FREEBSD;

214 +#elif defined(OS_MACOS)

215 + session_status.os_type = DAEMON_OS_TYPE_MACOS;

216 +#elif defined(OS_WINDOWS)

217 + session_status.os_type = DAEMON_OS_TYPE_WINDOWS;

218 +#endif

219 +

220 + if(session_status.status == DAEMON_STATUS_INITIALIZING && status == DAEMON_STATUS_RUNNING)

221 + session_status.timings.init = (time_t)((now_ut - session_status.timestamp_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

222 +

223 + if(session_status.status == DAEMON_STATUS_EXITING && status == DAEMON_STATUS_EXITED)

224 + session_status.timings.exit = (time_t)((now_ut - session_status.timestamp_ut + USEC_PER_SEC/2) / USEC_PER_SEC);

225 +

226 + strncpyz(session_status.version, NETDATA_VERSION, sizeof(session_status.version) - 1);

227 +

228 + session_status.boot_id = os_boot_id();

229 + if(!UUIDeq(session_status.boot_id, last_session_status.boot_id) && os_boot_ids_match(session_status.boot_id, last_session_status.boot_id)) {

230 + // there is a slight difference in boot_id, but it is still the same boot

231 + // copy the last boot_id

232 + session_status.boot_id = last_session_status.boot_id;

233 + }

234 +

235 + session_status.boottime = now_boottime_sec();

236 + session_status.uptime = now_realtime_sec() - netdata_start_time;

237 + session_status.timestamp_ut = now_ut;

238 + session_status.invocation = nd_log_get_invocation_id();

239 +

240 + session_status.claim_id = claim_id_get_uuid();

241 +

242 + if(localhost) {

243 + session_status.host_id = localhost->host_id;

244 + session_status.node_id = localhost->node_id;

245 + }

246 + else if(!UUIDiszero(last_session_status.host_id))

247 + session_status.host_id = last_session_status.host_id;

248 + else {

249 + const char *machine_guid = registry_get_this_machine_guid();

250 + if(machine_guid && *machine_guid) {

251 + if (uuid_parse_flexi(machine_guid, session_status.host_id.uuid) != 0)

252 + session_status.host_id = UUID_ZERO;

253 + }

254 + else

255 + session_status.host_id = UUID_ZERO;

256 + }

257 +

258 + if(UUIDiszero(session_status.claim_id))

259 + session_status.claim_id = last_session_status.claim_id;

260 + if(UUIDiszero(session_status.node_id))

261 + session_status.node_id = last_session_status.node_id;

262 + if(UUIDiszero(session_status.host_id))

263 + session_status.host_id = last_session_status.host_id;

264 +

265 + session_status.exit_reason = exit_initiated;

266 + session_status.profile = nd_profile_detect_and_configure(false);

267 +

268 + if(status != DAEMON_STATUS_NONE)

269 + session_status.status = status;

270 +

271 + session_status.memory = os_system_memory(true);

272 + session_status.var_cache = os_disk_space(netdata_configured_cache_dir);

273 +

274 + return session_status;

275 +}

276 +

277 +// --------------------------------------------------------------------------------------------------------------------

278 +// file helpers

279 +

280 +// List of fallback directories to try

281 +static const char *status_file_fallbacks[] = {

282 + "/tmp",

283 + "/run",

284 + "/var/run",

285 +};

286 +

287 +static bool check_status_file(const char *directory, char *filename, size_t filename_size, time_t *mtime) {

288 + if(!directory || !*directory)

289 + return false;

290 +

291 + snprintfz(filename, filename_size, "%s/%s", directory, STATUS_FILENAME);

292 +

293 + // Get file metadata

294 + OS_FILE_METADATA metadata = os_get_file_metadata(filename);

295 + if (!OS_FILE_METADATA_OK(metadata))

296 + return false;

297 +

298 + *mtime = metadata.modified_time;

299 + return true;

300 +}

301 +

302 +// --------------------------------------------------------------------------------------------------------------------

303 +// load a saved status

304 +

305 +static bool load_status_file(const char *filename, DAEMON_STATUS_FILE *status) {

306 + FILE *fp = fopen(filename, "r");

307 + if (!fp)

308 + return false;

309 +

310 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

311 + CLEAN_BUFFER *error = buffer_create(0, NULL);

312 +

313 + // Get file size

314 + fseek(fp, 0, SEEK_END);

315 + long file_size = ftell(fp);

316 + fseek(fp, 0, SEEK_SET);

317 +

318 + // Read the file

319 + buffer_need_bytes(wb, file_size + 1);

320 + size_t read_bytes = fread(wb->buffer, 1, file_size, fp);

321 + fclose(fp);

322 +

323 + if (read_bytes == 0)

324 + return false;

325 +

326 + wb->buffer[read_bytes] = '\0';

327 + wb->len = read_bytes;

328 +

329 + // Parse the JSON

330 + return json_parse_payload_or_error(wb, error, daemon_status_file_from_json, status) == HTTP_RESP_OK;

331 +}

332 +

333 +DAEMON_STATUS_FILE daemon_status_file_load(void) {

334 + DAEMON_STATUS_FILE status = {0};

335 + char newest_filename[FILENAME_MAX] = "";

336 + char current_filename[FILENAME_MAX];

337 + time_t newest_mtime = 0, current_mtime;

338 +

339 + // Check primary directory first

340 + if(check_status_file(netdata_configured_cache_dir, current_filename, sizeof(current_filename), &current_mtime)) {

341 + strncpyz(newest_filename, current_filename, sizeof(newest_filename) - 1);

342 + newest_mtime = current_mtime;

343 + }

344 +

345 + // Check each fallback location

346 + for(size_t i = 0; i < _countof(status_file_fallbacks); i++) {

347 + if(check_status_file(status_file_fallbacks[i], current_filename, sizeof(current_filename), &current_mtime) &&

348 + (!*newest_filename || current_mtime > newest_mtime)) {

349 + strncpyz(newest_filename, current_filename, sizeof(newest_filename) - 1);

350 + newest_mtime = current_mtime;

351 + }

352 + }

353 +

354 + // Load the newest file found

355 + if(*newest_filename) {

356 + if(!load_status_file(newest_filename, &status))

357 + nd_log(NDLS_DAEMON, NDLP_ERR, "Failed to load newest status file: %s", newest_filename);

358 + }

359 + else

360 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot find a status file in any location");

361 +

362 + return status;

363 +}

364 +

365 +// --------------------------------------------------------------------------------------------------------------------

366 +// save the current status

367 +

368 +static bool save_status_file(const char *directory, const char *content, size_t content_size) {

369 + if(!directory || !*directory)

370 + return false;

371 +

372 + char filename[FILENAME_MAX];

373 + char temp_filename[FILENAME_MAX];

374 +

375 + snprintfz(filename, sizeof(filename), "%s/%s", directory, STATUS_FILENAME);

376 + snprintfz(temp_filename, sizeof(temp_filename), "%s/%s", directory, STATUS_FILENAME_TMP);

377 +

378 + FILE *fp = fopen(temp_filename, "w");

379 + if (!fp)

380 + return false;

381 +

382 + bool ok = fwrite(content, 1, content_size, fp) == content_size;

383 + fclose(fp);

384 +

385 + if (!ok) {

386 + unlink(filename);

387 + unlink(temp_filename);

388 + return false;

389 + }

390 +

391 + if (chmod(temp_filename, 0664) != 0) {

392 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot set permissions on status file '%s'", temp_filename);

393 + unlink(temp_filename);

394 + return false;

395 + }

396 +

397 + if (rename(temp_filename, filename) != 0) {

398 + nd_log(NDLS_DAEMON, NDLP_ERR, "Cannot rename status file '%s' to '%s'", temp_filename, filename);

399 + unlink(temp_filename);

400 + return false;

401 + }

402 +

403 + return true;

404 +}

405 +

406 +void daemon_status_file_save(DAEMON_STATUS status) {

407 + static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

408 + spinlock_lock(&spinlock);

409 +

410 + // Get current status

411 + DAEMON_STATUS_FILE ds = daemon_status_file_get(status);

412 +

413 + // Prepare JSON content

414 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

415 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

416 + daemon_status_file_to_json(wb, &ds);

417 + buffer_json_finalize(wb);

418 +

419 + const char *content = buffer_tostring(wb);

420 + size_t content_size = buffer_strlen(wb);

421 +

422 + // Try primary directory first

423 + bool saved = false;

424 + if (save_status_file(netdata_configured_cache_dir, content, content_size))

425 + saved = true;

426 + else {

427 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Failed to save status file in primary directory %s",

428 + netdata_configured_cache_dir);

429 +

430 + // Try each fallback directory until successful

431 + for(size_t i = 0; i < _countof(status_file_fallbacks); i++) {

432 + if(save_status_file(status_file_fallbacks[i], content, content_size)) {

433 + nd_log(NDLS_DAEMON, NDLP_DEBUG, "Saved status file in fallback %s", status_file_fallbacks[i]);

434 + saved = true;

435 + break;

436 + }

437 + }

438 + }

439 +

440 + if (!saved)

441 + nd_log(NDLS_DAEMON, NDLP_ERR, "Failed to save status file in any location");

442 +

443 + spinlock_unlock(&spinlock);

444 +}

445 +

446 +// --------------------------------------------------------------------------------------------------------------------

447 +// POST the last status to agent-events

448 +

449 +struct post_status_file_thread_data {

450 + const char *cause;

451 + const char *msg;

452 + ND_LOG_FIELD_PRIORITY priority;

453 + DAEMON_STATUS_FILE status;

454 +};

455 +

456 +void post_status_file(struct post_status_file_thread_data *d) {

457 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

458 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);

459 + buffer_json_member_add_string(wb, "exit_cause", d->cause); // custom

460 + buffer_json_member_add_string(wb, "message", d->msg); // ECS

461 + buffer_json_member_add_uint64(wb, "priority", d->priority); // custom

462 + daemon_status_file_to_json(wb, &d->status);

463 + buffer_json_finalize(wb);

464 +

465 + const char *json_data = buffer_tostring(wb);

466 +

467 + CURL *curl = curl_easy_init();

468 + if(!curl)

469 + return;

470 +

471 + curl_easy_setopt(curl, CURLOPT_URL, "https://agent-events.netdata.cloud/agent-events");

472 + curl_easy_setopt(curl, CURLOPT_POST, 1L);

473 + curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_data);

474 + struct curl_slist *headers = NULL;

475 + headers = curl_slist_append(headers, "Content-Type: application/json");

476 + curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);

477 +

478 + CURLcode rc = curl_easy_perform(curl);

479 + (void)rc;

480 +

481 + curl_easy_cleanup(curl);

482 + curl_slist_free_all(headers);

483 +}

484 +

485 +void *post_status_file_thread(void *ptr) {

486 + struct post_status_file_thread_data *d = (struct post_status_file_thread_data *)ptr;

487 + post_status_file(d);

488 + freez((void *)d->cause);

489 + freez((void *)d->msg);

490 + freez(d);

491 + return NULL;

492 +}

493 +

494 +// --------------------------------------------------------------------------------------------------------------------

495 +// check last status on startup and post crash report

496 +

497 +void daemon_status_file_check_crash(void) {

498 + last_session_status = daemon_status_file_load();

499 + daemon_status_file_save(DAEMON_STATUS_INITIALIZING);

500 + ND_LOG_FIELD_PRIORITY pri = NDLP_NOTICE;

501 +

502 + bool new_version = strcmp(last_session_status.version, session_status.version) != 0;

503 + bool post_crash_report = false;

504 + bool disable_crash_report = false;

505 + bool dump_json = true;

506 + const char *msg, *cause;

507 + switch(last_session_status.status) {

508 + default:

509 + case DAEMON_STATUS_NONE:

510 + // probably a previous version of netdata was running

511 + cause = "no last status";

512 + msg = "No status found for the previous Netdata session";

513 + disable_crash_report = true;

514 + break;

515 +

516 + case DAEMON_STATUS_EXITED:

517 + if(last_session_status.exit_reason == EXIT_REASON_NONE) {

518 + cause = "exit no reason";

519 + msg = "Netdata was last stopped gracefully (no exit reason set)";

520 + if(!last_session_status.timestamp_ut)

521 + dump_json = false;

522 + }

523 + else if(!is_exit_reason_normal(last_session_status.exit_reason)) {

524 + cause = "exit on fatal";

525 + msg = "Netdata was last stopped gracefully (encountered an error)";

526 + pri = NDLP_ERR;

527 + post_crash_report = true;

528 + }

529 + else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {

530 + cause = "exit on system shutdown";

531 + msg = "Netdata has gracefully stopped due to system shutdown";

532 + }

533 + else if(last_session_status.exit_reason & EXIT_REASON_UPDATE) {

534 + cause = "exit to update";

535 + msg = "Netdata has gracefully restarted to update to a new version";

536 + }

537 + else if(new_version) {

538 + cause = "exit and updated";

539 + msg = "Netdata has gracefully restarted and updated to a new version";

540 + last_session_status.exit_reason |= EXIT_REASON_UPDATE;

541 + }

542 + else {

543 + cause = "exit instructed";

544 + msg = "Netdata was last stopped gracefully (instructed to do so)";

545 + }

546 + break;

547 +

548 + case DAEMON_STATUS_INITIALIZING:

549 + cause = "crashed on start";

550 + msg = "Netdata was last killed/crashed while starting";

551 + pri = NDLP_ERR;

552 + post_crash_report = true;

553 + break;

554 +

555 + case DAEMON_STATUS_EXITING:

556 + if(!is_exit_reason_normal(last_session_status.exit_reason)) {

557 + cause = "crashed on fatal";

558 + msg = "Netdata was last killed/crashed while exiting after encountering an error";

559 + }

560 + else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {

561 + cause = "crashed on system shutdown";

562 + msg = "Netdata was last killed/crashed while exiting due to system shutdown";

563 + }

564 + else if(new_version || (last_session_status.exit_reason & EXIT_REASON_UPDATE)) {

565 + cause = "crashed on update";

566 + msg = "Netdata was last killed/crashed while exiting to update to a new version";

567 + }

568 + else {

569 + cause = "crashed on exit";

570 + msg = "Netdata was last killed/crashed while exiting (instructed to do so)";

571 + }

572 + pri = NDLP_ERR;

573 + post_crash_report = true;

574 + break;

575 +

576 + case DAEMON_STATUS_RUNNING: {

577 + if (!UUIDeq(session_status.boot_id, last_session_status.boot_id)) {

578 + cause = "abnormal power off";

579 + msg = "The system was abnormally powered off while Netdata was running";

580 + pri = NDLP_CRIT;

581 + }

582 + else {

583 + cause = "killed hard";

584 + msg = "Netdata was last killed/crashed while operating normally";

585 + pri = NDLP_CRIT;

586 + post_crash_report = true;

587 + }

588 + break;

589 + }

590 + }

591 +

592 + CLEAN_BUFFER *wb = buffer_create(0, NULL);

593 + buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);

594 + if(dump_json)

595 + daemon_status_file_to_json(wb, &last_session_status);

596 + buffer_json_finalize(wb);

597 +

598 + ND_LOG_STACK lgs[] = {

599 + ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &netdata_startup_msgid),

600 + ND_LOG_FIELD_END(),

601 + };

602 + ND_LOG_STACK_PUSH(lgs);

603 +

604 + nd_log(NDLS_DAEMON, pri,

605 + "Netdata Agent version '%s' is starting...\n"

606 + "Last exit status: %s (%s):\n\n%s",

607 + NETDATA_VERSION, msg, cause, buffer_tostring(wb));

608 +

609 + if(!disable_crash_report && (analytics_check_enabled() || post_crash_report)) {

610 + netdata_conf_ssl();

611 +

612 + struct post_status_file_thread_data *d = calloc(1, sizeof(*d));

613 + d->cause = strdupz(cause);

614 + d->msg = strdupz(msg);

615 + d->status = last_session_status;

616 + d->priority = pri;

617 + nd_thread_create("post_status_file", NETDATA_THREAD_OPTION_DONT_LOG | NETDATA_THREAD_OPTION_DEFAULT, post_status_file_thread, d);

618 + }

619 +}

620 +

621 +bool daemon_status_file_has_last_crashed(void) {

622 + return last_session_status.status != DAEMON_STATUS_EXITED || !is_exit_reason_normal(last_session_status.exit_reason);

623 +}

624 +

625 +bool daemon_status_file_was_incomplete_shutdown(void) {

626 + return last_session_status.status == DAEMON_STATUS_EXITING;

627 +}

628 +

629 +// --------------------------------------------------------------------------------------------------------------------

630 +// ng_log() hook for receiving fatal message information

631 +

632 +void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *stack_trace, long line) {

633 + static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

634 + spinlock_lock(&spinlock);

635 +

636 + if(session_status.fatal.filename || session_status.fatal.function || session_status.fatal.message || session_status.fatal.stack_trace) {

637 + spinlock_unlock(&spinlock);

638 + freez((void *)filename);

639 + freez((void *)function);

640 + freez((void *)message);

641 + freez((void *)stack_trace);

642 + return;

643 + }

644 +

645 + session_status.fatal.filename = filename;

646 + session_status.fatal.function = function;

647 + session_status.fatal.message = message;

648 + session_status.fatal.stack_trace = stack_trace;

649 + session_status.fatal.line = line;

650 +

651 + spinlock_unlock(&spinlock);

652 +}

src/daemon/daemon-status-file.h new

+75

@@ -0,0 +1,75 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_DAEMON_STATUS_FILE_H

4 +#define NETDATA_DAEMON_STATUS_FILE_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +#include "daemon/config/netdata-conf-profile.h"

8 +

9 +typedef enum {

10 + DAEMON_STATUS_NONE,

11 + DAEMON_STATUS_INITIALIZING,

12 + DAEMON_STATUS_RUNNING,

13 + DAEMON_STATUS_EXITING,

14 + DAEMON_STATUS_EXITED,

15 +} DAEMON_STATUS;

16 +ENUM_STR_DEFINE_FUNCTIONS_EXTERN(DAEMON_STATUS);

17 +

18 +typedef enum {

19 + DAEMON_OS_TYPE_UNKNOWN,

20 + DAEMON_OS_TYPE_LINUX,

21 + DAEMON_OS_TYPE_FREEBSD,

22 + DAEMON_OS_TYPE_MACOS,

23 + DAEMON_OS_TYPE_WINDOWS,

24 +} DAEMON_OS_TYPE;

25 +ENUM_STR_DEFINE_FUNCTIONS_EXTERN(DAEMON_OS_TYPE);

26 +

27 +typedef struct daemon_status_file {

28 + char version[32]; // the netdata version

29 + DAEMON_STATUS status; // the daemon status

30 + EXIT_REASON exit_reason; // the exit reason (maybe empty)

31 + ND_PROFILE profile; // the profile of the agent

32 + DAEMON_OS_TYPE os_type;

33 +

34 + time_t boottime; // system boottime

35 + time_t uptime; // netdata uptime

36 + usec_t timestamp_ut; // the timestamp of the status file

37 +

38 + ND_UUID boot_id; // the boot id of the system

39 + ND_UUID invocation; // the netdata invocation id generated the file

40 + ND_UUID host_id; // the machine guid of the agent

41 + ND_UUID node_id; // the Netdata Cloud node id of the agent

42 + ND_UUID claim_id; // the Netdata Cloud claim id of the agent

43 +

44 + struct {

45 + time_t init;

46 + time_t exit;

47 + } timings;

48 +

49 + OS_SYSTEM_MEMORY memory;

50 + OS_SYSTEM_DISK_SPACE var_cache;

51 +

52 + struct {

53 + long line;

54 + const char *filename;

55 + const char *function;

56 + const char *stack_trace;

57 + const char *message;

58 + } fatal;

59 +} DAEMON_STATUS_FILE;

60 +

61 +// loads the last status saved

62 +DAEMON_STATUS_FILE daemon_status_file_load(void);

63 +

64 +// saves the current status

65 +void daemon_status_file_save(DAEMON_STATUS status);

66 +

67 +// check for a crash

68 +void daemon_status_file_check_crash(void);

69 +

70 +bool daemon_status_file_has_last_crashed(void);

71 +bool daemon_status_file_was_incomplete_shutdown(void);

72 +

73 +void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *stack_trace, long line);

74 +

75 +#endif //NETDATA_DAEMON_STATUS_FILE_H

src/daemon/daemon-systemd-watcher.c new

+154

@@ -0,0 +1,154 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "libnetdata/libnetdata.h"

4 +#include "daemon-systemd-watcher.h"

5 +#include "daemon-service.h"

6 +

7 +#ifdef ENABLE_SYSTEMD_DBUS

8 +

9 +#include <systemd/sd-bus.h>

10 +

11 +/* Callback function to handle the PrepareForShutdown signal.

12 + * The signal sends a boolean: true indicates that shutdown is starting,

13 + * false indicates that a previously initiated shutdown was canceled.

14 + */

15 +static int shutdown_event_handler(sd_bus_message *m, void *userdata __maybe_unused, sd_bus_error *ret_error __maybe_unused) {

16 + int shutdown;

17 + int r = sd_bus_message_read(m, "b", &shutdown);

18 + if (r < 0) {

19 + nd_log(NDLS_DAEMON, NDLP_ERR,

20 + "SYSTEMD DBUS: Failed to parse shutdown message: %s",

21 + strerror(-r));

22 + return r;

23 + }

24 +

25 + nd_log(NDLS_DAEMON, NDLP_NOTICE,

26 + "SYSTEMD DBUS: Received PrepareForShutdown signal: shutdown=%s",

27 + shutdown ? "true" : "false");

28 +

29 + if(shutdown)

30 + netdata_cleanup_and_exit(EXIT_REASON_SYSTEM_SHUTDOWN, NULL, NULL, NULL);

31 +

32 + return 0;

33 +}

34 +

35 +/* Callback function to handle the PrepareForSleep signal.

36 + * The signal sends a boolean: true indicates that the system is preparing to suspend,

37 + * false indicates that a previous suspend was canceled (i.e. resuming).

38 + */

39 +static int suspend_event_handler(sd_bus_message *m, void *userdata __maybe_unused, sd_bus_error *ret_error __maybe_unused) {

40 + int suspend;

41 + int r = sd_bus_message_read(m, "b", &suspend);

42 + if (r < 0) {

43 + nd_log(NDLS_DAEMON, NDLP_ERR,

44 + "SYSTEMD DBUS: Failed to parse suspend message: %s",

45 + strerror(-r));

46 + return r;

47 + }

48 +

49 + nd_log(NDLS_DAEMON, NDLP_NOTICE,

50 + "SYSTEMD DBUS: Received PrepareForSleep signal: suspend=%s\n",

51 + suspend ? "true (suspending)" : "false (resuming)");

52 +

53 + // Here you can trigger your suspend/resume logic.

54 + return 0;

55 +}

56 +

57 +/* Function that sets up the sd-bus listener for shutdown and suspend events.

58 + * This function blocks in a loop processing bus events.

59 + */

60 +static void listen_for_systemd_dbus_events(void) {

61 + sd_bus *bus = NULL;

62 + sd_bus_slot *shutdown_slot = NULL;

63 + sd_bus_slot *suspend_slot = NULL;

64 + int r;

65 +

66 + // Connect to the system bus.

67 + r = sd_bus_open_system(&bus);

68 + if (r < 0) {

69 + nd_log(NDLS_DAEMON, NDLP_ERR,

70 + "SYSTEMD DBUS: Failed to connect to system bus: %s",

71 + strerror(-r));

72 + goto finish;

73 + }

74 +

75 + // Add a match rule for the PrepareForShutdown signal on the login1 manager.

76 + r = sd_bus_add_match(

77 + bus,

78 + &shutdown_slot,

79 + "type='signal',"

80 + "sender='org.freedesktop.login1',"

81 + "interface='org.freedesktop.login1.Manager',"

82 + "member='PrepareForShutdown'",

83 + shutdown_event_handler,

84 + NULL);

85 + if (r < 0) {

86 + nd_log(NDLS_DAEMON, NDLP_ERR,

87 + "SYSTEMD DBUS: Failed to add signal match for shutdown: %s",

88 + strerror(-r));

89 + goto finish;

90 + }

91 +

92 + // Add a match rule for the PrepareForSleep signal on the login1 manager.

93 + r = sd_bus_add_match(

94 + bus,

95 + &suspend_slot,

96 + "type='signal',"

97 + "sender='org.freedesktop.login1',"

98 + "interface='org.freedesktop.login1.Manager',"

99 + "member='PrepareForSleep'",

100 + suspend_event_handler,

101 + NULL);

102 + if (r < 0) {

103 + nd_log(NDLS_DAEMON, NDLP_ERR,

104 + "SYSTEMD DBUS: Failed to add signal match for suspend: %s",

105 + strerror(-r));

106 + goto finish;

107 + }

108 +

109 + // Process incoming D-Bus messages.

110 + while (service_running(SERVICE_SYSTEMD)) {

111 + // Process any pending messages.

112 + r = sd_bus_process(bus, NULL);

113 + if (r < 0) {

114 + nd_log(NDLS_DAEMON, NDLP_ERR,

115 + "SYSTEMD DBUS: Failed to process bus: %s",

116 + strerror(-r));

117 + goto finish;

118 + }

119 + if (r > 0) // Message was processed; check for more.

120 + continue;

121 +

122 + // Wait for the next signal.

123 + r = 0;

124 + while(r == 0 && service_running(SERVICE_SYSTEMD))

125 + r = sd_bus_wait(bus, USEC_PER_SEC);

126 +

127 + if (r < 0) {

128 + nd_log(NDLS_DAEMON, NDLP_ERR, "SYSTEMD DBUS: Failed to wait on bus: %s", strerror(-r));

129 + break;

130 + }

131 + }

132 +

133 +finish:

134 + sd_bus_slot_unref(shutdown_slot);

135 + sd_bus_slot_unref(suspend_slot);

136 + sd_bus_unref(bus);

137 +}

138 +

139 +#endif

140 +

141 +void *systemd_watcher_thread(void *arg) {

142 + struct netdata_static_thread *static_thread = arg;

143 +

144 + service_register(SERVICE_THREAD_TYPE_NETDATA, NULL, NULL, NULL, false);

145 +

146 +#ifdef ENABLE_SYSTEMD_DBUS

147 + listen_for_systemd_dbus_events();

148 +#endif

149 +

150 + service_exits();

151 + worker_unregister();

152 + static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;

153 + return NULL;

154 +}

src/daemon/daemon-systemd-watcher.h new

+8

@@ -0,0 +1,8 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_DAEMON_SYSTEMD_WATCHER_H

4 +#define NETDATA_DAEMON_SYSTEMD_WATCHER_H

5 +

6 +void *systemd_watcher_thread(void *arg);

7 +

8 +#endif //NETDATA_DAEMON_SYSTEMD_WATCHER_H

src/daemon/daemon.c

+2 -1

@@ -58,7 +58,7 @@ static void change_dir_ownership(const char *dir, uid_t uid, gid_t gid, bool rec

58 fix_directory_file_permissions(dir, uid, gid, recursive);

59 }

60

61 -static void clean_directory(const char *dirname)

61 +static inline void clean_directory(const char *dirname)

62 {

63 DIR *dir = opendir(dirname);

64 if(!dir) return;

@@ -75,6 +75,7 @@ static void clean_directory(const char *dirname)

75 }

76

77 static void prepare_required_directories(uid_t uid, gid_t gid) {

78 + change_dir_ownership(os_run_dir(true), uid, gid, false);

79 change_dir_ownership(netdata_configured_cache_dir, uid, gid, true);

80 change_dir_ownership(netdata_configured_varlib_dir, uid, gid, false);

81 change_dir_ownership(netdata_configured_log_dir, uid, gid, false);

src/daemon/daemon.h

-2

@@ -5,8 +5,6 @@

5

6 int become_daemon(int dont_fork, const char *user);

7

8 -void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data);

9 -

8 void get_netdata_execution_path(void);

9

10 extern char *pidfile;

src/daemon/main.c

+40 -37

@@ -2,7 +2,8 @@

2

3 #include "common.h"

4 #include "buildinfo.h"

5 -#include "daemon/daemon-shutdown-watcher.h"

5 +#include "daemon-shutdown-watcher.h"

6 +#include "daemon-status-file.h"

7 #include "static_threads.h"

8 #include "web/api/queries/backfill.h"

9

@@ -22,7 +23,7 @@

23 #endif

24

25 bool unittest_running = false;

25 -int netdata_anonymous_statistics_enabled;

26 +bool netdata_anonymous_statistics_enabled = true;

27

28 int libuv_worker_threads = MIN_LIBUV_WORKER_THREADS;

29 bool ieee754_doubles = false;

@@ -228,6 +229,7 @@ int unittest_prepare_rrd(const char **user) {

229 }

230

231 int netdata_main(int argc, char **argv) {

232 + libjudy_malloc_init();

233 string_init();

234 analytics_init();

235

@@ -749,21 +751,40 @@ int netdata_main(int argc, char **argv) {

751

752 // initialize the log files

753 nd_log_initialize();

752 - {

753 - ND_LOG_STACK lgs[] = {

754 - ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &netdata_startup_msgid),

755 - ND_LOG_FIELD_END(),

756 - };

757 - ND_LOG_STACK_PUSH(lgs);

754 + nd_log_register_event_cb(daemon_status_file_register_fatal);

755 +

756 + netdata_conf_section_global(); // get hostname, host prefix, profile, etc

757 + registry_init(); // for machine_guid, must be after netdata_conf_section_global()

758

759 - netdata_log_info("Netdata agent version '%s' is starting", NETDATA_VERSION);

759 + // initialize thread - this is required before the first nd_thread_create()

760 + default_stacksize = netdata_threads_init();

761 + // musl default thread stack size is 128k, let's set it to a higher value to avoid random crashes

762 + if (default_stacksize < 1 * 1024 * 1024)

763 + default_stacksize = 1 * 1024 * 1024;

764 +

765 + // make sure we are the only instance running

766 + {

767 + const char *run_dir = os_run_dir(true);

768 + if(!run_dir) {

769 + netdata_log_error("Cannot get/create a run directory.");

770 + exit(1);

771 + }

772 + netdata_log_info("Netdata run directory is '%s'", run_dir);

773 +

774 + char lock_file[FILENAME_MAX];

775 + snprintfz(lock_file, sizeof(lock_file), "%s/netdata.lock", run_dir);

776 + FILE_LOCK lock = file_lock_get(lock_file);

777 + if(!FILE_LOCK_OK(lock)) {

778 + netdata_log_error("Cannot get exclusive lock on file '%s'. Is Netdata already running?", lock_file);

779 + exit(1);

780 + }

781 }

782

762 - // ----------------------------------------------------------------------------------------------------------------

763 - // global configuration

783 + // status and crash/update/exit detection

784 + exit_initiated_reset();

785 + daemon_status_file_check_crash();

786

787 netdata_conf_ssl();

766 - netdata_conf_section_global();

788

789 // Get execution path before switching user to avoid permission issues

790 get_netdata_execution_path();

@@ -831,12 +852,6 @@ int netdata_main(int argc, char **argv) {

852

853 delta_startup_time("initialize static threads");

854

834 - // setup threads configs

835 - default_stacksize = netdata_threads_init();

836 - // musl default thread stack size is 128k, let's set it to a higher value to avoid random crashes

837 - if (default_stacksize < 1 * 1024 * 1024)

838 - default_stacksize = 1 * 1024 * 1024;

839 -

855 for (i = 0; static_threads[i].name != NULL ; i++) {

856 struct netdata_static_thread *st = &static_threads[i];

857

@@ -909,7 +924,6 @@ int netdata_main(int argc, char **argv) {

924 #endif

925

926 netdata_main_spawn_server_init("plugins", argc, (const char **)argv);

912 - watcher_thread_start();

927

928 // init sentry

929 #ifdef ENABLE_SENTRY

@@ -936,7 +950,7 @@ int netdata_main(int argc, char **argv) {

950

951 // initialize internal registry

952 delta_startup_time("initialize registry");

939 - registry_init();

953 + registry_load();

954 cloud_conf_init_after_registry();

955 netdata_random_session_id_generate();

956

@@ -945,7 +959,6 @@ int netdata_main(int argc, char **argv) {

959

960 delta_startup_time("collecting system info");

961

948 - netdata_anonymous_statistics_enabled=-1;

962 struct rrdhost_system_info *system_info = rrdhost_system_info_create();

963 rrdhost_system_info_detect(system_info);

964

@@ -967,18 +980,6 @@ int netdata_main(int argc, char **argv) {

980 }

981 abort_on_fatal_enable();

982

970 - delta_startup_time("check for incomplete shutdown");

971 -

972 - char agent_crash_file[FILENAME_MAX + 1];

973 - char agent_incomplete_shutdown_file[FILENAME_MAX + 1];

974 - snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir);

975 - int incomplete_shutdown_detected = (unlink(agent_incomplete_shutdown_file) == 0);

976 - snprintfz(agent_crash_file, FILENAME_MAX, "%s/.agent_crash", netdata_configured_varlib_dir);

977 - int crash_detected = (unlink(agent_crash_file) == 0);

978 - int fd = open(agent_crash_file, O_WRONLY | O_CREAT | O_TRUNC | O_CLOEXEC, 444);

979 - if (fd >= 0)

980 - close(fd);

981 -

983 // ------------------------------------------------------------------------

984 // Claim netdata agent to a cloud endpoint

985

@@ -1038,17 +1039,17 @@ int netdata_main(int argc, char **argv) {

1039

1040 analytics_statistic_t start_statistic = { "START", "-", "-" };

1041 analytics_statistic_send(&start_statistic);

1041 - if (crash_detected) {

1042 + if (daemon_status_file_has_last_crashed()) {

1043 analytics_statistic_t crash_statistic = { "CRASH", "-", "-" };

1044 analytics_statistic_send(&crash_statistic);

1045 }

1045 - if (incomplete_shutdown_detected) {

1046 + if (daemon_status_file_was_incomplete_shutdown()) {

1047 analytics_statistic_t incomplete_shutdown_statistic = { "INCOMPLETE_SHUTDOWN", "-", "-" };

1048 analytics_statistic_send(&incomplete_shutdown_statistic);

1049 }

1050

1050 - //check if ANALYTICS needs to start

1051 - if (netdata_anonymous_statistics_enabled == 1) {

1051 + // check if ANALYTICS needs to start

1052 + if (netdata_anonymous_statistics_enabled) {

1053 for (i = 0; static_threads[i].name != NULL; i++) {

1054 if (!strncmp(static_threads[i].name, "ANALYTICS", 9)) {

1055 struct netdata_static_thread *st = &static_threads[i];

@@ -1060,6 +1061,8 @@ int netdata_main(int argc, char **argv) {

1061 }

1062

1063 webrtc_initialize();

1064 +

1065 + daemon_status_file_save(DAEMON_STATUS_RUNNING);

1066 return 10;

1067 }

1068

src/daemon/pipename.c

+18 -7

@@ -2,16 +2,27 @@

2

3 #include "pipename.h"

4

5 -#include <stdlib.h>

5 +#include "libnetdata/libnetdata.h"

6 +

7 +static const char *cached_pipename = NULL;

8

9 const char *daemon_pipename(void) {

10 + if(cached_pipename)

11 + return cached_pipename;

12 +

13 const char *pipename = getenv("NETDATA_PIPENAME");

9 - if (pipename)

14 + if (pipename) {

15 + cached_pipename = strdupz(pipename);

16 return pipename;

17 + }

18

12 -#ifdef _WIN32

13 - return "\\\\?\\pipe\\netdata-cli";

14 -#else

15 - return "/tmp/netdata-ipc";

16 -#endif

19 +//#if defined(OS_WINDOWS)

20 +// cached_pipename = strdupz("\\\\?\\pipe\\netdata-cli");

21 +// return cached_pipename;

22 +//#else

23 + char filename[FILENAME_MAX + 1];

24 + snprintfz(filename, FILENAME_MAX, "%s/netdata.pipe", os_run_dir(false));

25 + cached_pipename = strdupz(filename);

26 + return cached_pipename;

27 +//#endif

28 }

src/daemon/pulse/pulse-daemon-memory-system.c

+1 -1

@@ -113,7 +113,7 @@ void pulse_daemon_memory_system_do(bool extended) {

113 if(!extended) return;

114

115 size_t glibc_mmaps = 0;

116 - bool have_mallinfo = false;

116 + bool have_mallinfo = false; (void)have_mallinfo;

117

118 #ifdef HAVE_C_MALLINFO2

119 struct mallinfo2 mi = mallinfo2();

src/daemon/pulse/pulse-daemon-memory.c

+1 -1

@@ -275,7 +275,7 @@ void pulse_daemon_memory_do(bool extended __maybe_unused) {

275 // ----------------------------------------------------------------------------------------------------------------

276

277 OS_SYSTEM_MEMORY sm = os_system_memory(true);

278 - if (sm.ram_total_bytes && dbengine_out_of_memory_protection) {

278 + if (OS_SYSTEM_MEMORY_OK(sm) && dbengine_out_of_memory_protection) {

279 static RRDSET *st_memory_available = NULL;

280 static RRDDIM *rd_available = NULL;

281

src/daemon/pulse/pulse-workers.c

+92

@@ -88,6 +88,8 @@ struct worker_utilization {

88 double workers_cpu_max;

89 double workers_cpu_total;

90

91 + uint64_t memory_calls[WORKERS_MEMORY_CALL_MAX];

92 +

93 struct worker_thread *threads;

94

95 RRDSET *st_workers_time;

@@ -113,6 +115,9 @@ struct worker_utilization {

115 RRDSET *st_spinlocks_locks;

116 RRDSET *st_spinlocks_spins;

117 SPINLOCKS_JudyLSet spinlocks;

118 +

119 + RRDSET *st_memory_calls;

120 + RRDDIM *rd_memory_calls[WORKERS_MEMORY_CALL_MAX];

121 };

122

123 static struct worker_utilization all_workers_utilization[] = {

@@ -255,6 +260,46 @@ static void workers_total_spinlock_contention_chart(void) {

260 }

261 }

262

263 +static void workers_total_memory_calls_chart(void) {

264 + {

265 + static RRDSET *st = NULL;

266 + static RRDDIM *rd[WORKERS_MEMORY_CALL_MAX] = { NULL };

267 + uint64_t memory_calls[WORKERS_MEMORY_CALL_MAX] = { 0 };

268 +

269 + if(unlikely(!st)) {

270 + st = rrdset_create_localhost(

271 + "netdata"

272 + , "memory_calls_total"

273 + , NULL

274 + , "memory calls"

275 + , "netdata.memory_calls_total"

276 + , "Netdata Total Memory Calls"

277 + , "calls"

278 + , "netdata"

279 + , "pulse"

280 + , 920005

281 + , localhost->rrd_update_every

282 + , RRDSET_TYPE_LINE

283 + );

284 +

285 + for (int j = 0; j < WORKERS_MEMORY_CALL_MAX; ++j)

286 + rd[j] = rrddim_add(st, WORKERS_MEMORY_CALL_2str(j), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

287 + }

288 +

289 + for(size_t i = 0; all_workers_utilization[i].name ;i++) {

290 + struct worker_utilization *wu = &all_workers_utilization[i];

291 +

292 + for (int j = 0; j < WORKERS_MEMORY_CALL_MAX; ++j)

293 + memory_calls[j] += wu->memory_calls[j];

294 + }

295 +

296 + for (int j = 0; j < WORKERS_MEMORY_CALL_MAX; ++j)

297 + rrddim_set_by_pointer(st, rd[j], (collected_number)memory_calls[j]);

298 +

299 + rrdset_done(st);

300 + }

301 +}

302 +

303 static void workers_total_cpu_utilization_chart(void) {

304 size_t i, cpu_enabled = 0;

305 for(i = 0; all_workers_utilization[i].name ;i++)

@@ -653,6 +698,43 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {

698 rrdset_done(wu->st_spinlocks_spins);

699 }

700

701 + // ----------------------------------------------------------------------

702 + // memory calls

703 +

704 + {

705 + if(unlikely(!wu->st_memory_calls)) {

706 + char name[RRD_ID_LENGTH_MAX + 1];

707 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_memory_calls_%s", wu->name_lowercase);

708 +

709 + char context[RRD_ID_LENGTH_MAX + 1];

710 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.memory_calls", wu->name_lowercase);

711 +

712 + wu->st_memory_calls = rrdset_create_localhost(

713 + "netdata"

714 + , name

715 + , NULL

716 + , wu->family

717 + , context

718 + , "Netdata Memory Calls"

719 + , "calls"

720 + , "netdata"

721 + , "pulse"

722 + , wu->priority + 8

723 + , localhost->rrd_update_every

724 + , RRDSET_TYPE_LINE

725 + );

726 + }

727 +

728 + for(size_t i = 0; i < WORKERS_MEMORY_CALL_MAX; i++) {

729 + if(!wu->rd_memory_calls[i])

730 + wu->rd_memory_calls[i] = rrddim_add(wu->st_memory_calls, WORKERS_MEMORY_CALL_2str(i), NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);

731 +

732 + rrddim_set_by_pointer(wu->st_memory_calls, wu->rd_memory_calls[i], (collected_number)wu->memory_calls[i]);

733 + }

734 +

735 + rrdset_done(wu->st_memory_calls);

736 + }

737 +

738 // ----------------------------------------------------------------------

739 // custom metric types WORKER_METRIC_ABSOLUTE

740

@@ -815,6 +897,8 @@ static void workers_utilization_reset_statistics(struct worker_utilization *wu)

897 wt->enabled = false;

898 wt->cpu_enabled = false;

899 }

900 +

901 + memset(wu->memory_calls, 0, sizeof(wu->memory_calls));

902 }

903

904 #define TASK_STAT_PREFIX "/proc/self/task/"

@@ -923,6 +1007,7 @@ static void worker_utilization_charts_callback(void *ptr

1007 , const char *spinlock_functions[]

1008 , size_t *spinlock_locks

1009 , size_t *spinlock_spins

1010 + , uint64_t *memory_calls

1011 ) {

1012 struct worker_utilization *wu = (struct worker_utilization *)ptr;

1013

@@ -1024,6 +1109,12 @@ static void worker_utilization_charts_callback(void *ptr

1109 wusp->locks += spinlock_locks[i];

1110 wusp->spins += spinlock_spins[i];

1111 }

1112 +

1113 + // ----------------------------------------------------------------------------------------------------------------

1114 + // memory calls

1115 +

1116 + for(size_t i = 0; i < WORKERS_MEMORY_CALL_MAX ;i++)

1117 + wu->memory_calls[i] += memory_calls[i];

1118 }

1119

1120 void pulse_workers_cleanup(void) {

@@ -1082,4 +1173,5 @@ void pulse_workers_do(bool extended) {

1173

1174 workers_total_cpu_utilization_chart();

1175 workers_total_spinlock_contention_chart();

1176 + workers_total_memory_calls_chart();

1177 }

src/daemon/signals.c

+18 -11

@@ -1,6 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 #include "common.h"

4 +#include "daemon/daemon-status-file.h"

5

6 typedef enum signal_action {

7 NETDATA_SIGNAL_END_OF_LIST,

@@ -16,17 +17,21 @@ static struct {

17 const char *name; // the name of the signal

18 size_t count; // the number of signals received

19 SIGNAL_ACTION action; // the action to take

20 + EXIT_REASON reason;

21 } signals_waiting[] = {

20 - { SIGPIPE, "SIGPIPE", 0, NETDATA_SIGNAL_IGNORE },

21 - { SIGINT , "SIGINT", 0, NETDATA_SIGNAL_EXIT_CLEANLY },

22 - { SIGQUIT, "SIGQUIT", 0, NETDATA_SIGNAL_EXIT_CLEANLY },

23 - { SIGTERM, "SIGTERM", 0, NETDATA_SIGNAL_EXIT_CLEANLY },

24 - { SIGHUP, "SIGHUP", 0, NETDATA_SIGNAL_REOPEN_LOGS },

25 - { SIGUSR2, "SIGUSR2", 0, NETDATA_SIGNAL_RELOAD_HEALTH },

26 - { SIGBUS, "SIGBUS", 0, NETDATA_SIGNAL_FATAL },

27 -

28 - // terminator

29 - { 0, "NONE", 0, NETDATA_SIGNAL_END_OF_LIST }

22 + { SIGPIPE, "SIGPIPE", 0, NETDATA_SIGNAL_IGNORE, EXIT_REASON_NONE },

23 + { SIGINT , "SIGINT", 0, NETDATA_SIGNAL_EXIT_CLEANLY, EXIT_REASON_SIGINT },

24 + { SIGQUIT, "SIGQUIT", 0, NETDATA_SIGNAL_EXIT_CLEANLY, EXIT_REASON_SIGQUIT },

25 + { SIGTERM, "SIGTERM", 0, NETDATA_SIGNAL_EXIT_CLEANLY, EXIT_REASON_SIGTERM },

26 + { SIGHUP, "SIGHUP", 0, NETDATA_SIGNAL_REOPEN_LOGS, EXIT_REASON_NONE },

27 + { SIGUSR2, "SIGUSR2", 0, NETDATA_SIGNAL_RELOAD_HEALTH, EXIT_REASON_NONE },

28 + { SIGBUS, "SIGBUS", 0, NETDATA_SIGNAL_FATAL, EXIT_REASON_SIGBUS },

29 + { SIGSEGV, "SIGSEGV", 0, NETDATA_SIGNAL_FATAL, EXIT_REASON_SIGSEGV },

30 + { SIGFPE, "SIGFPE", 0, NETDATA_SIGNAL_FATAL, EXIT_REASON_SIGFPE },

31 + { SIGILL, "SIGILL", 0, NETDATA_SIGNAL_FATAL, EXIT_REASON_SIGILL },

32 +

33 + // terminator

34 + { 0, "NONE", 0, NETDATA_SIGNAL_END_OF_LIST, 0 }

35 };

36

37 static void signal_handler(int signo) {

@@ -108,6 +113,7 @@ void nd_process_signals(void) {

113 // is delivered that either terminates the process or causes the invocation

114 // of a signal-catching function.

115 if(pause() == -1 && errno == EINTR) {

116 + daemon_status_file_save(DAEMON_STATUS_NONE);

117 errno_clear();

118

119 // loop once, but keep looping while signals are coming in

@@ -144,11 +150,12 @@ void nd_process_signals(void) {

150 nd_log_limits_unlimited();

151 netdata_log_info("SIGNAL: Received %s. Cleaning up to exit...", name);

152 commands_exit();

147 - netdata_cleanup_and_exit(0, NULL, NULL, NULL);

153 + netdata_cleanup_and_exit(signals_waiting[i].reason, NULL, NULL, NULL);

154 exit(0);

155 break;

156

157 case NETDATA_SIGNAL_FATAL:

158 + exit_initiated_set(signals_waiting[i].reason);

159 fatal("SIGNAL: Received %s. netdata now exits.", name);

160 break;

161

src/daemon/static_threads.c

+11

@@ -2,6 +2,7 @@

2

3 #include "common.h"

4 #include "web/api/queries/backfill.h"

5 +#include "daemon-systemd-watcher.h"

6

7 void *aclk_main(void *ptr);

8 void *analytics_main(void *ptr);

@@ -201,6 +202,16 @@ const struct netdata_static_thread static_threads_common[] = {

202 .init_routine = NULL,

203 .start_routine = backfill_thread

204 },

205 + {

206 + .name = "SDBUSWATCHER",

207 + .config_section = NULL,

208 + .config_name = NULL,

209 + .enable_routine = NULL,

210 + .enabled = 1,

211 + .thread = NULL,

212 + .init_routine = NULL,

213 + .start_routine = systemd_watcher_thread

214 + },

215

216 // terminator

217 {

src/daemon/winsvc.cc

+18 -4

@@ -87,7 +87,7 @@ static HANDLE CreateEventHandle(const char *msg)

87

88 static void *call_netdata_cleanup(void *arg)

89 {

90 - UNUSED(arg);

90 + DWORD controlCode = *((DWORD *)arg);

91

92 // Wait until we have to stop the service

93 netdata_service_log("Cleanup thread waiting for stop event...");

@@ -95,7 +95,20 @@ static void *call_netdata_cleanup(void *arg)

95

96 // Stop the agent

97 netdata_service_log("Running netdata cleanup...");

98 - netdata_cleanup_and_exit(0, NULL, NULL, NULL);

98 + EXIT_REASON reason;

99 + switch(controlCode) {

100 + case SERVICE_CONTROL_SHUTDOWN:

101 + reason = (EXIT_REASON)(EXIT_REASON_SERVICE_STOP|EXIT_REASON_SYSTEM_SHUTDOWN);

102 + break;

103 +

104 + case SERVICE_CONTROL_STOP:

105 + // fall-through

106 +

107 + default:

108 + reason = EXIT_REASON_SERVICE_STOP;

109 + break;

110 + }

111 + netdata_cleanup_and_exit(reason, NULL, NULL, NULL);

112

113 // Close event handle

114 netdata_service_log("Closing stop event handle...");

@@ -112,6 +125,7 @@ static void WINAPI ServiceControlHandler(DWORD controlCode)

125 {

126 switch (controlCode)

127 {

128 + case SERVICE_CONTROL_SHUTDOWN:

129 case SERVICE_CONTROL_STOP:

130 {

131 if (svc_status.dwCurrentState != SERVICE_RUNNING)

@@ -126,7 +140,7 @@ static void WINAPI ServiceControlHandler(DWORD controlCode)

140 netdata_service_log("Creating cleanup thread...");

141 char tag[NETDATA_THREAD_TAG_MAX + 1];

142 snprintfz(tag, NETDATA_THREAD_TAG_MAX, "%s", "CLEANUP");

129 - cleanup_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, call_netdata_cleanup, NULL);

143 + cleanup_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_JOINABLE, call_netdata_cleanup, &controlCode);

144

145 // Signal the stop request

146 netdata_service_log("Signalling the cleanup thread...");

@@ -176,7 +190,7 @@ void WINAPI ServiceMain(DWORD argc, LPSTR* argv)

190

191 // Set status to running

192 netdata_service_log("Setting service status to running...");

179 - if (!ReportSvcStatus(SERVICE_RUNNING, 0, 5000, SERVICE_ACCEPT_STOP))

193 + if (!ReportSvcStatus(SERVICE_RUNNING, 0, 5000, SERVICE_ACCEPT_STOP | SERVICE_ACCEPT_SHUTDOWN))

194 {

195 netdata_service_log("Failed to set service status to running.");

196 return;

src/database/contexts/rrdcontext.c

+1 -1

@@ -93,7 +93,7 @@ ALWAYS_INLINE void rrdcontext_collected_rrdset(RRDSET *st) {

93 }

94

95 ALWAYS_INLINE void rrdcontext_host_child_disconnected(RRDHOST *host) {

96 - rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);

96 + rrdhost_flag_set(host, RRDHOST_FLAG_RRDCONTEXT_GET_RETENTION);

97 }

98

99 ALWAYS_INLINE void rrdcontext_host_child_connected(RRDHOST *host) {

src/database/contexts/worker.c

+5

@@ -1202,6 +1202,11 @@ void *rrdcontext_main(void *ptr) {

1202 if(rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_CONTEXT_LOAD))

1203 continue;

1204

1205 + if(rrdhost_flag_check(host, RRDHOST_FLAG_RRDCONTEXT_GET_RETENTION)) {

1206 + rrdcontext_recalculate_host_retention(host, RRD_FLAG_UPDATE_REASON_DISCONNECTED_CHILD, false);

1207 + rrdhost_flag_clear(host, RRDHOST_FLAG_RRDCONTEXT_GET_RETENTION);

1208 + }

1209 +

1210 worker_is_busy(WORKER_JOB_HOSTS);

1211

1212 if(host->rrdctx.pp_queue) {

src/database/engine/cache.c

+8 -6

@@ -401,7 +401,7 @@ static ssize_t cache_usage_per1000(PGC *cache, int64_t *size_to_evict) {

401 if(cache->config.out_of_memory_protection_bytes) {

402 // out of memory protection

403 OS_SYSTEM_MEMORY sm = os_system_memory(false);

404 - if(sm.ram_total_bytes) {

404 + if(OS_SYSTEM_MEMORY_OK(sm)) {

405 // when the total exists, ram_available_bytes is also right

406

407 const int64_t ram_available_bytes = (int64_t)sm.ram_available_bytes;

@@ -1042,11 +1042,13 @@ static inline void remove_and_free_page_not_in_any_queue_and_acquired_for_deleti

1042 static inline bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache, PGC_PAGE *page) {

1043 pointer_check(cache, page);

1044

1045 + WAITQ_PRIORITY prio = is_page_clean(page) ? PGC_QUEUE_LOCK_PRIO_EVICTORS : PGC_QUEUE_LOCK_PRIO_COLLECTORS;

1046 +

1047 page_transition_lock(cache, page);

1046 - pgc_queue_lock(cache, &cache->clean, PGC_QUEUE_LOCK_PRIO_EVICTORS);

1048 + pgc_queue_lock(cache, &cache->clean, prio);

1049

1050 // make it clean - it does not have any accesses, so it will be prepended

1049 - page_set_clean(cache, page, true, true, PGC_QUEUE_LOCK_PRIO_EVICTORS);

1051 + page_set_clean(cache, page, true, true, prio);

1052

1053 if(!acquired_page_get_for_deletion_or_release_it(cache, page)) {

1054 pgc_queue_unlock(cache, &cache->clean);

@@ -1055,7 +1057,7 @@ static inline bool make_acquired_page_clean_and_evict_or_page_release(PGC *cache

1057 }

1058

1059 // remove it from the linked list

1058 - pgc_queue_del(cache, &cache->clean, page, true, PGC_QUEUE_LOCK_PRIO_EVICTORS);

1060 + pgc_queue_del(cache, &cache->clean, page, true, prio);

1061 pgc_queue_unlock(cache, &cache->clean);

1062 page_transition_unlock(cache, page);

1063

@@ -2006,7 +2008,7 @@ PGC *pgc_create(const char *name,

2008 cache->config.out_of_memory_protection_bytes = (int64_t)dbengine_out_of_memory_protection;

2009

2010 // partitions

2009 - if(partitions == 0) partitions = netdata_conf_cpus();

2011 + if(partitions == 0) partitions = netdata_conf_cpus() * 2;

2012 if(partitions <= 4) partitions = 4;

2013 if(partitions > 256) partitions = 256;

2014 cache->config.partitions = partitions;

@@ -2288,7 +2290,7 @@ bool pgc_flush_pages(PGC *cache) {

2290 }

2291

2292 void pgc_page_hot_set_end_time_s(PGC *cache __maybe_unused, PGC_PAGE *page, time_t end_time_s, size_t additional_bytes) {

2291 - internal_fatal(!is_page_hot(page) && !netdata_exit,

2293 + internal_fatal(!is_page_hot(page) && !exit_initiated,

2294 "DBENGINE CACHE: end_time_s update on non-hot page");

2295

2296 internal_fatal(end_time_s < __atomic_load_n(&page->end_time_s, __ATOMIC_RELAXED),

src/database/engine/datafile.c

+4 -4

@@ -260,7 +260,7 @@ int create_data_file(struct rrdengine_datafile *datafile)

260 datafile->file = file;

261 __atomic_add_fetch(&ctx->stats.datafile_creations, 1, __ATOMIC_RELAXED);

262

263 - ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

263 + ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

264 if (unlikely(ret)) {

265 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

266 }

@@ -278,7 +278,7 @@ int create_data_file(struct rrdengine_datafile *datafile)

278 ctx_io_error(ctx);

279 }

280 uv_fs_req_cleanup(&req);

281 - posix_memfree(superblock);

281 + posix_memalign_freez(superblock);

282 if (ret < 0) {

283 destroy_data_file_unsafe(datafile);

284 return ret;

@@ -297,7 +297,7 @@ static int check_data_file_superblock(uv_file file)

297 uv_buf_t iov;

298 uv_fs_t req;

299

300 - ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

300 + ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

301 if (unlikely(ret)) {

302 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

303 }

@@ -321,7 +321,7 @@ static int check_data_file_superblock(uv_file file)

321 ret = 0;

322 }

323 error:

324 - posix_memfree(superblock);

324 + posix_memalign_freez(superblock);

325 return ret;

326 }

327

src/database/engine/journalfile.c

+6 -6

@@ -580,7 +580,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi

580 journalfile->file = file;

581 __atomic_add_fetch(&ctx->stats.journalfile_creations, 1, __ATOMIC_RELAXED);

582

583 - ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

583 + ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

584 if (unlikely(ret)) {

585 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

586 }

@@ -597,7 +597,7 @@ int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengi

597 ctx_io_error(ctx);

598 }

599 uv_fs_req_cleanup(&req);

600 - posix_memfree(superblock);

600 + posix_memalign_freez(superblock);

601 if (ret < 0) {

602 journalfile_destroy_unsafe(journalfile, datafile);

603 return ret;

@@ -617,7 +617,7 @@ static int journalfile_check_superblock(uv_file file)

617 uv_buf_t iov;

618 uv_fs_t req;

619

620 - ret = posix_memalign((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

620 + ret = posix_memalignz((void *)&superblock, RRDFILE_ALIGNMENT, sizeof(*superblock));

621 if (unlikely(ret)) {

622 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

623 }

@@ -643,7 +643,7 @@ static int journalfile_check_superblock(uv_file file)

643 ret = 0;

644 }

645 error:

646 - posix_memfree(superblock);

646 + posix_memalign_freez(superblock);

647 return ret;

648 }

649

@@ -813,7 +813,7 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,

813 file_size = journalfile->unsafe.pos;

814

815 max_id = 1;

816 - ret = posix_memalign((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);

816 + ret = posix_memalignz((void *)&buf, RRDFILE_ALIGNMENT, READAHEAD_BYTES);

817 if (unlikely(ret))

818 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

819

@@ -844,7 +844,7 @@ static uint64_t journalfile_iterate_transactions(struct rrdengine_instance *ctx,

844 }

845 }

846 skip_file:

847 - posix_memfree(buf);

847 + posix_memalign_freez(buf);

848 return max_id;

849 }

850

src/database/engine/page.c

+6 -2

@@ -899,8 +899,12 @@ size_t pgd_append_point(

899 if (pg->states & PGD_STATE_SCHEDULED_FOR_FLUSHING)

900 pgd_fatal(pg, "Data collection on page already scheduled for flushing");

901

902 - if (!(pg->states & PGD_STATE_CREATED_FROM_COLLECTOR))

903 - pgd_fatal(pg, "DBENGINE: collection on page not created from a collector");

902 + if (!(pg->states & PGD_STATE_CREATED_FROM_COLLECTOR)) {

903 + if(exit_initiated == EXIT_REASON_NONE)

904 + pgd_fatal(pg, "DBENGINE: collection on page not created from a collector");

905 + else

906 + return 0;

907 + }

908

909 if (unlikely(pg->used != expected_slot))

910 pgd_fatal(pg, "DBENGINE: page is not aligned to expected slot (used %u, expected %u)",

src/database/engine/pdc.c

+4 -4

@@ -474,7 +474,7 @@ static ALWAYS_INLINE EPDL_EXTENT *epdl_find_extent_base(EPDL *epdl) {

474 e = callocz(1, sizeof(*e));

475

476 rw_spinlock_write_lock(&epdl->datafile->extent_epdl.spinlock);

477 - Pvoid_t *PValue = JudyLIns(&epdl->datafile->extent_epdl.epdl_per_extent, epdl->extent_offset, PJE0);

477 + PValue = JudyLIns(&epdl->datafile->extent_epdl.epdl_per_extent, epdl->extent_offset, PJE0);

478 internal_fatal(!PValue || PValue == PJERR, "DBENGINE: corrupted pending extent judy");

479 if(!*PValue) {

480 *PValue = e;

@@ -1216,7 +1216,7 @@ static inline void *datafile_extent_read(struct rrdengine_instance *ctx, uv_file

1216 uv_fs_t request;

1217

1218 unsigned real_io_size = ALIGN_BYTES_CEILING(size_bytes);

1219 - int ret = posix_memalign(&buffer, RRDFILE_ALIGNMENT, real_io_size);

1219 + int ret = posix_memalignz(&buffer, RRDFILE_ALIGNMENT, real_io_size);

1220 if (unlikely(ret))

1221 fatal("DBENGINE: posix_memalign(): %s", strerror(ret));

1222

@@ -1224,7 +1224,7 @@ static inline void *datafile_extent_read(struct rrdengine_instance *ctx, uv_file

1224 ret = uv_fs_read(NULL, &request, file, &iov, 1, (int64_t)pos, NULL);

1225 if (unlikely(-1 == ret)) {

1226 ctx_io_error(ctx);

1227 - posix_memfree(buffer);

1227 + posix_memalign_freez(buffer);

1228 buffer = NULL;

1229 }

1230 else

@@ -1236,7 +1236,7 @@ static inline void *datafile_extent_read(struct rrdengine_instance *ctx, uv_file

1236 }

1237

1238 static inline void datafile_extent_read_free(void *buffer) {

1239 - posix_memfree(buffer);

1239 + posix_memalign_freez(buffer);

1240 }

1241

1242 NOT_INLINE_HOT void epdl_find_extent_and_populate_pages(struct rrdengine_instance *ctx, EPDL *epdl, bool worker) {

src/database/engine/rrdengine.c

+4 -4

@@ -367,7 +367,7 @@ static void wal_cleanup1(void) {

367 spinlock_unlock(&wal_globals.protected.spinlock);

368

369 if(wal) {

370 - posix_memfree(wal->buf);

370 + posix_memalign_freez(wal->buf);

371 freez(wal);

372 __atomic_sub_fetch(&wal_globals.atomics.allocated, 1, __ATOMIC_RELAXED);

373 }

@@ -393,7 +393,7 @@ WAL *wal_get(struct rrdengine_instance *ctx, unsigned size) {

393 if(unlikely(!wal)) {

394 wal = mallocz(sizeof(WAL));

395 wal->buf_size = RRDENG_BLOCK_SIZE;

396 - int ret = posix_memalign((void *)&wal->buf, RRDFILE_ALIGNMENT, wal->buf_size);

396 + int ret = posix_memalignz((void *)&wal->buf, RRDFILE_ALIGNMENT, wal->buf_size);

397 if (unlikely(ret))

398 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

399 __atomic_add_fetch(&wal_globals.atomics.allocated, 1, __ATOMIC_RELAXED);

@@ -649,7 +649,7 @@ extent_flush_to_open(struct rrdengine_instance *ctx, struct extent_io_descriptor

649 page_descriptor_release(descr);

650 }

651

652 - posix_memfree(xt_io_descr->buf);

652 + posix_memalign_freez(xt_io_descr->buf);

653 extent_io_descriptor_release(xt_io_descr);

654

655 spinlock_lock(&datafile->writers.spinlock);

@@ -763,7 +763,7 @@ datafile_extent_build(struct rrdengine_instance *ctx, struct page_descr_with_dat

763 payload_offset = sizeof(*header) + count * sizeof(header->descr[0]);

764 max_compressed_size = dbengine_max_compressed_size(uncompressed_payload_length, compression_algorithm);

765 size_bytes = payload_offset + MAX(uncompressed_payload_length, max_compressed_size) + sizeof(*trailer);

766 - ret = posix_memalign((void *)&xt_io_descr->buf, RRDFILE_ALIGNMENT, ALIGN_BYTES_CEILING(size_bytes));

766 + ret = posix_memalignz((void *)&xt_io_descr->buf, RRDFILE_ALIGNMENT, ALIGN_BYTES_CEILING(size_bytes));

767 if (unlikely(ret)) {

768 fatal("DBENGINE: posix_memalign:%s", strerror(ret));

769 /* freez(xt_io_descr);*/

src/database/rrddim-collection.c

+59 -24

@@ -11,37 +11,71 @@ static inline time_t tier_next_point_time_s(RRDDIM *rd, struct rrddim_tier *t, t

11 return now_s + loop - ((now_s + loop) % loop);

12 }

13

14 -ALWAYS_INLINE_HOT void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {

14 +#define LAST_COMPLETED_POINT_EXISTS(t) (t->last_completed_point.end_time_s != 0)

15 +

16 +ALWAYS_INLINE_HOT

17 +void store_metric_at_tier_flush_last_completed(RRDDIM *rd __maybe_unused, size_t tier, struct rrddim_tier *t) {

18 + // when there is no end_time_s we do not have a saved last_completed_point

19 + if(!LAST_COMPLETED_POINT_EXISTS(t)) return;

20 +

21 + STORAGE_POINT *sp = &t->last_completed_point;

22 + if(likely(!storage_point_is_unset(t->last_completed_point))) {

23 + storage_engine_store_metric(

24 + t->sch,

25 + sp->end_time_s * USEC_PER_SEC,

26 + sp->sum,

27 + sp->min,

28 + sp->max,

29 + sp->count,

30 + sp->anomaly_count,

31 + sp->flags);

32 + }

33 + else {

34 + storage_engine_store_metric(

35 + t->sch,

36 + sp->end_time_s * USEC_PER_SEC,

37 + NAN,

38 + NAN,

39 + NAN,

40 + 0,

41 + 0, SN_FLAG_NONE);

42 + }

43 +

44 + rrdset_done_statistics_points_stored_per_tier[tier]++;

45 +

46 + // make the point unset

47 + t->last_completed_point.count = 0; // make it unset

48 + t->last_completed_point.end_time_s = 0; // make it not saved

49 +}

50 +

51 +ALWAYS_INLINE_HOT

52 +void store_metric_at_tier_save_last_completed(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp) {

53 + // make sure the last_completed_point is empty

54 + store_metric_at_tier_flush_last_completed(rd, tier, t);

55 +

56 + // copy the point

57 + t->last_completed_point = sp;

58 +

59 + // set the end_time_s, so that we will know we have saved a last_completed_point

60 + t->last_completed_point.end_time_s = t->next_point_end_time_s;

61 +}

62 +

63 +ALWAYS_INLINE_HOT

64 +void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrddim_tier *t, STORAGE_POINT sp, usec_t now_ut __maybe_unused) {

65 + if(LAST_COMPLETED_POINT_EXISTS(t) && sp.start_time_s % t->last_completed_point_flush_modulo == 0)

66 + store_metric_at_tier_flush_last_completed(rd, tier, t);

67 +

68 if (unlikely(!t->next_point_end_time_s))

69 t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);

70

71 if(unlikely(sp.start_time_s >= t->next_point_end_time_s)) {

72 // flush the virtual point, it is done

73

21 - if (likely(!storage_point_is_unset(t->virtual_point))) {

22 -

23 - storage_engine_store_metric(

24 - t->sch,

25 - t->next_point_end_time_s * USEC_PER_SEC,

26 - t->virtual_point.sum,

27 - t->virtual_point.min,

28 - t->virtual_point.max,

29 - t->virtual_point.count,

30 - t->virtual_point.anomaly_count,

31 - t->virtual_point.flags);

32 - }

33 - else {

34 - storage_engine_store_metric(

35 - t->sch,

36 - t->next_point_end_time_s * USEC_PER_SEC,

37 - NAN,

38 - NAN,

39 - NAN,

40 - 0,

41 - 0, SN_FLAG_NONE);

42 - }

74 + if (likely(!storage_point_is_unset(t->virtual_point)))

75 + store_metric_at_tier_save_last_completed(rd, tier, t, t->virtual_point);

76 + else

77 + store_metric_at_tier_save_last_completed(rd, tier, t, STORAGE_POINT_UNSET);

78

44 - rrdset_done_statistics_points_stored_per_tier[tier]++;

79 t->virtual_point.count = 0; // make the point unset

80 t->next_point_end_time_s = tier_next_point_time_s(rd, t, sp.end_time_s);

81 }

@@ -73,6 +107,7 @@ ALWAYS_INLINE_HOT void store_metric_at_tier(RRDDIM *rd, size_t tier, struct rrdd

107 }

108 }

109

110 +NOT_INLINE_HOT

111 #ifdef NETDATA_LOG_COLLECTION_ERRORS

112 void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags, const char *function) {

113 #else // !NETDATA_LOG_COLLECTION_ERRORS

src/database/rrddim-collection.h

+2

@@ -15,4 +15,6 @@ void rrddim_store_metric_with_trace(RRDDIM *rd, usec_t point_end_time_ut, NETDAT

15 void rrddim_store_metric(RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);

16 #endif

17

18 +void store_metric_at_tier_flush_last_completed(RRDDIM *rd, size_t tier, struct rrddim_tier *t);

19 +

20 #endif //NETDATA_RRDDIM_COLLECTION_H

src/database/rrddim.c

+11 -2

@@ -2,6 +2,7 @@

2

3 #include "rrd.h"

4 #include "storage-engine.h"

5 +#include "rrddim-collection.h"

6

7 void rrddim_metadata_updated(RRDDIM *rd) {

8 rrdcontext_updated_rrddim(rd);

@@ -115,8 +116,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

116 size_t initialized = 0;

117 for (size_t tier = 0; tier < nd_profile.storage_tiers; tier++) {

118 if (rd->tiers[tier].smh) {

119 + uint32_t tier_update_every = st->rrdhost->db[tier].tier_grouping * st->update_every;

120 +

121 + rd->tiers[tier].last_completed_point_flush_modulo = rrddim_collection_modulo(st, tier_update_every);

122 +

123 rd->tiers[tier].sch =

119 - storage_metric_store_init(rd->tiers[tier].seb, rd->tiers[tier].smh, st->rrdhost->db[tier].tier_grouping * st->update_every, rd->rrdset->smg[tier]);

124 + storage_metric_store_init(rd->tiers[tier].seb, rd->tiers[tier].smh, tier_update_every, rd->rrdset->smg[tier]);

125 +

126 initialized++;

127 }

128 }

@@ -175,12 +181,15 @@ bool rrddim_finalize_collection_and_check_retention(RRDDIM *rd) {

181

182 size_t tiers_available = 0, tiers_said_no_retention = 0;

183

178 - for(size_t tier = 0; tier < nd_profile.storage_tiers;tier++) {

184 + for(size_t tier = 0; tier < nd_profile.storage_tiers ;tier++) {

185 spinlock_lock(&rd->tiers[tier].spinlock);

186

187 if(rd->tiers[tier].sch) {

188 tiers_available++;

189

190 + if(tier > 0)

191 + store_metric_at_tier_flush_last_completed(rd, tier, &rd->tiers[tier]);

192 +

193 if (storage_engine_store_finalize(rd->tiers[tier].sch))

194 tiers_said_no_retention++;

195

src/database/rrdhost.h

+1

@@ -84,6 +84,7 @@ typedef enum __attribute__ ((__packed__)) rrdhost_flags {

84 RRDHOST_FLAG_METADATA_CLAIMID = (1 << 27), // metadata needs to be stored in the database

85

86 RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED = (1 << 28), // set when the host has updated global functions

87 + RRDHOST_FLAG_RRDCONTEXT_GET_RETENTION = (1 << 29), // set when rrdcontext needs to update the retention of the host

88 } RRDHOST_FLAGS;

89

90 #define rrdhost_flag_get(host) atomic_flags_get(&((host)->flags))

src/database/rrdset-index-id.c

+19 -1

@@ -4,6 +4,22 @@

4 #include "rrdset-index-name.h"

5 #include "rrdset-slots.h"

6

7 +// --------------------------------------------------------------------------------------------------------------------

8 +// tier1/2 spread over time

9 +

10 +static size_t global_rrdset_counter = 0;

11 +static uint16_t rrdset_collection_modulo_init(void) {

12 + return __atomic_fetch_add(&global_rrdset_counter, 1, __ATOMIC_RELAXED) % 65535;

13 +}

14 +

15 +uint16_t rrddim_collection_modulo(RRDSET *st, uint32_t spread) {

16 + if(!spread) spread = 65535;

17 + spread = MIN(spread, 65535);

18 + return 1 + (st->collection_modulo % spread);

19 +}

20 +

21 +// --------------------------------------------------------------------------------------------------------------------

22 +

23 static inline void rrdset_update_permanent_labels(RRDSET *st) {

24 if(!st->rrdlabels) return;

25

@@ -11,7 +27,7 @@ static inline void rrdset_update_permanent_labels(RRDSET *st) {

27 rrdlabels_add(st->rrdlabels, "_collect_module", rrdset_module_name(st), RRDLABEL_SRC_AUTO | RRDLABEL_FLAG_DONT_DELETE);

28 }

29

14 -// ----------------------------------------------------------------------------

30 +// --------------------------------------------------------------------------------------------------------------------

31 // RRDSET index

32

33 struct rrdset_constructor {

@@ -56,6 +72,8 @@ static void rrdset_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v

72 st->name = rrdset_fix_name(host, chart_full_id, ctr->type, NULL, ctr->id);

73 rrdset_index_add_name(host, st);

74

75 + st->collection_modulo = rrdset_collection_modulo_init();

76 +

77 st->parts.id = string_strdupz(ctr->id);

78 st->parts.type = string_strdupz(ctr->type);

79 st->parts.name = string_strdupz(ctr->name);

src/database/rrdset-index-id.h

+2

@@ -73,6 +73,8 @@ RRDSET_ACQUIRED *rrdset_find_and_acquire(RRDHOST *host, const char *id);

73 void rrdset_acquired_release(RRDSET_ACQUIRED *rsa);

74 RRDSET *rrdset_acquired_to_rrdset(RRDSET_ACQUIRED *rsa);

75

76 +uint16_t rrddim_collection_modulo(RRDSET *st, uint32_t spread);

77 +

78 #define rrdset_find_localhost(id) rrdset_find(localhost, id)

79 /* This will not return charts that are archived */

80 static inline RRDSET *rrdset_find_active_localhost(const char *id) {

src/database/rrdset.h

+2 -1

@@ -112,7 +112,8 @@ struct rrdset {

112 // operational state members

113

114 RRDSET_FLAGS flags; // flags

115 - RRD_DB_MODE rrd_memory_mode; // the db mode of this rrdset

115 + RRD_DB_MODE rrd_memory_mode; // the db mode of this rrdset

116 + uint16_t collection_modulo; // tier1/2 spread over time

117

118 DICTIONARY *rrddim_root_index; // dimensions index

119

src/database/storage-engine.h

+4 -2

@@ -90,11 +90,13 @@ STORAGE_ENGINE* storage_engine_foreach_next(STORAGE_ENGINE* it);

90

91 struct rrddim_tier {

92 STORAGE_POINT virtual_point;

93 - STORAGE_ENGINE_BACKEND seb;

93 + STORAGE_POINT last_completed_point; // tier1/2 spread over time

94 SPINLOCK spinlock;

95 + STORAGE_ENGINE_BACKEND seb;

96 + uint16_t last_completed_point_flush_modulo; // tier1/2 spread over time

97 uint32_t tier_grouping;

98 time_t next_point_end_time_s;

97 - STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database

99 + STORAGE_METRIC_HANDLE *smh; // the metric handle inside the database

100 STORAGE_COLLECT_HANDLE *sch; // the data collection handle

101 };

102

src/health/health_dyncfg.c

+8 -8

@@ -95,7 +95,7 @@ static bool parse_config_value_database_lookup(json_object *jobj, const char *pa

95 }

96

97 static bool parse_config_value(json_object *jobj, const char *path, struct rrd_alert_config *config, BUFFER *error, bool strict) {

98 - JSONC_PARSE_SUBOBJECT(jobj, path, "database_lookup", config, parse_config_value_database_lookup, error, strict);

98 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "database_lookup", config, parse_config_value_database_lookup, error, strict);

99 JSONC_PARSE_TXT2EXPRESSION_OR_ERROR_AND_RETURN(jobj, path, "calculation", config->calculation, error, false);

100 JSONC_PARSE_TXT2STRING_OR_ERROR_AND_RETURN(jobj, path, "units", config->units, error, false);

101 JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, "update_every", config->update_every, error, strict);

@@ -127,8 +127,8 @@ static bool parse_config_action(json_object *jobj, const char *path, struct rrd_

127 JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, "options", alert_action_options_parse_one, config->alert_action_options, error, strict);

128 JSONC_PARSE_TXT2STRING_OR_ERROR_AND_RETURN(jobj, path, "execute", config->exec, error, strict);

129 JSONC_PARSE_TXT2STRING_OR_ERROR_AND_RETURN(jobj, path, "recipient", config->recipient, error, strict);

130 - JSONC_PARSE_SUBOBJECT(jobj, path, "delay", config, parse_config_action_delay, error, strict);

131 - JSONC_PARSE_SUBOBJECT(jobj, path, "repeat", config, parse_config_action_repeat, error, strict);

130 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "delay", config, parse_config_action_delay, error, strict);

131 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "repeat", config, parse_config_action_repeat, error, strict);

132 return true;

133 }

134

@@ -143,10 +143,10 @@ static bool parse_config(json_object *jobj, const char *path, RRD_ALERT_PROTOTYP

143 JSONC_PARSE_TXT2STRING_OR_ERROR_AND_RETURN(jobj, path, "component", ap->config.component, error, false);

144 JSONC_PARSE_TXT2STRING_OR_ERROR_AND_RETURN(jobj, path, "classification", ap->config.classification, error, false);

145

146 - JSONC_PARSE_SUBOBJECT(jobj, path, "value", &ap->config, parse_config_value, error, strict);

147 - JSONC_PARSE_SUBOBJECT(jobj, path, "conditions", &ap->config, parse_config_conditions, error, false);

148 - JSONC_PARSE_SUBOBJECT(jobj, path, "action", &ap->config, parse_config_action, error, false);

149 - JSONC_PARSE_SUBOBJECT(jobj, path, "match", &ap->match, parse_match, error, strict);

146 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "value", &ap->config, parse_config_value, error, strict);

147 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "conditions", &ap->config, parse_config_conditions, error, false);

148 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "action", &ap->config, parse_config_action, error, false);

149 + JSONC_PARSE_SUBOBJECT_CB(jobj, path, "match", &ap->match, parse_match, error, strict);

150

151 return true;

152 }

@@ -194,7 +194,7 @@ static bool parse_prototype(json_object *jobj, const char *path, RRD_ALERT_PROTO

194 return false;

195 }

196

197 - JSONC_PARSE_SUBOBJECT(rule, path, "config", ap, parse_config, error, strict);

197 + JSONC_PARSE_SUBOBJECT_CB(rule, path, "config", ap, parse_config, error, strict);

198

199 ap = NULL; // so that we will create another one, if available

200 }

src/health/notifications/alarm-notify.sh.in

+2 -2

@@ -138,7 +138,7 @@ log() {

138

139 [[ -n "$level" && -n "$LOG_LEVEL" && "$level" -gt "$LOG_LEVEL" ]] && return

140

141 - systemd-cat-native --log-as-netdata --newline="--NEWLINE--" <<EOFLOG

141 + systemd-cat-native --log-as-netdata <<EOFLOG

142 INVOCATION_ID=${NETDATA_INVOCATION_ID}

143 SYSLOG_IDENTIFIER=${PROGRAM_NAME}

144 PRIORITY=${level}

@@ -166,7 +166,7 @@ ND_ALERT_INFO=${info}

166 ND_ALERT_DURATION=${duration}

167 ND_REQUEST=${cmd_line}

168 MESSAGE_ID=6db0018e83e34320ae2a659d78019fb7

169 -MESSAGE=[ALERT NOTIFICATION]: ${*//\\n/--NEWLINE--}

169 +MESSAGE=[ALERT NOTIFICATION]: ${*//$'\n'/\\n}

170

171 EOFLOG

172 # AN EMPTY LINE IS NEEDED ABOVE

src/health/rrdcalc.c

+1 -1

@@ -257,7 +257,7 @@ static void rrdcalc_unlink_from_rrdset(RRDCALC *rc, bool having_ll_wrlock) {

257 return;

258 }

259

260 - if (!netdata_exit) {

260 + if (!exit_initiated) {

261 RRDHOST *host = st->rrdhost;

262

263 time_t now = now_realtime_sec();

src/libnetdata/aral/aral.c

+19 -21

@@ -86,7 +86,7 @@ struct aral_ops {

86 struct {

87 PAD64(size_t) allocators; // the number of threads currently trying to allocate memory

88 PAD64(size_t) deallocators; // the number of threads currently trying to deallocate memory

89 - PAD64(bool) last_allocated_or_deallocated; // stability detector, true when was last allocated

89 + PAD64(bool) last_allocated_page; // stability detector, true when was last allocated

90 } atomic;

91

92 struct {

@@ -97,6 +97,16 @@ struct aral_ops {

97 };

98

99 struct aral {

100 + struct {

101 + SPINLOCK spinlock;

102 + size_t file_number; // for mmap

103 +

104 + ARAL_PAGE *pages_free; // pages with free items

105 + ARAL_PAGE *pages_full; // pages that are completely full

106 +

107 + ARAL_PAGE *pages_marked_free; // pages with marked items and free slots

108 + ARAL_PAGE *pages_marked_full; // pages with marked items completely full

109 + } aral_lock;

110

111 struct {

112 char name[ARAL_MAX_NAME + 1];

@@ -120,22 +130,8 @@ struct aral {

130 } config;

131

132 struct {

123 - SPINLOCK spinlock;

124 - size_t file_number; // for mmap

125 -

126 - ARAL_PAGE *pages_free; // pages with free items

127 - ARAL_PAGE *pages_full; // pages that are completely full

128 -

129 - ARAL_PAGE *pages_marked_free; // pages with marked items and free slots

130 - ARAL_PAGE *pages_marked_full; // pages with marked items completely full

131 -

132 - size_t defragment_operations;

133 - size_t defragment_linked_list_traversals;

134 - } aral_lock;

135 -

136 - struct {

137 - size_t user_malloc_operations;

138 - size_t user_free_operations;

133 + PAD64(size_t) user_malloc_operations;

134 + PAD64(size_t) user_free_operations;

135 } atomic;

136

137 struct aral_ops ops[2];

@@ -497,8 +493,10 @@ static ALWAYS_INLINE size_t aral_next_allocation_size___adders_lock_needed(ARAL

493 size_t idx = mark_to_idx(marked);

494 size_t size = ar->ops[idx].adders.allocation_size;

495

500 - bool last_allocated = __atomic_load_n(&ar->ops[idx].atomic.last_allocated_or_deallocated, __ATOMIC_RELAXED);

501 - if(last_allocated) {

496 + bool last_allocated_page = __atomic_load_n(&ar->ops[idx].atomic.last_allocated_page, __ATOMIC_RELAXED);

497 + if(last_allocated_page) {

498 + // we are growing, double the size

499 +

500 size *= 2;

501 if(size > ar->config.max_allocation_size)

502 size = ar->config.max_allocation_size;

@@ -512,7 +510,7 @@ static ALWAYS_INLINE size_t aral_next_allocation_size___adders_lock_needed(ARAL

510 memory_alignment(sizeof(ARAL_PAGE), SYSTEM_REQUIRED_ALIGNMENT);

511 }

512

515 - __atomic_store_n(&ar->ops[idx].atomic.last_allocated_or_deallocated, true, __ATOMIC_RELAXED);

513 + __atomic_store_n(&ar->ops[idx].atomic.last_allocated_page, true, __ATOMIC_RELAXED);

514

515 return size;

516 }

@@ -615,7 +613,7 @@ static ARAL_PAGE *aral_create_page___no_lock_needed(ARAL *ar, size_t size TRACE_

613

614 static void aral_del_page___no_lock_needed(ARAL *ar, ARAL_PAGE *page TRACE_ALLOCATIONS_FUNCTION_DEFINITION_PARAMS) {

615 size_t idx = mark_to_idx(page->started_marked);

618 - __atomic_store_n(&ar->ops[idx].atomic.last_allocated_or_deallocated, true, __ATOMIC_RELAXED);

616 + __atomic_store_n(&ar->ops[idx].atomic.last_allocated_page, false, __ATOMIC_RELAXED);

617

618 struct aral_page_type_stats *stats;

619 size_t max_elements = page->max_elements;

src/libnetdata/buffer/buffer.c

+7 -2

@@ -224,6 +224,11 @@ BUFFER *buffer_create(size_t size, size_t *statistics)

224 {

225 BUFFER *b;

226

227 + if(!size)

228 + size = 1024 - sizeof(BUFFER_OVERFLOW_EOF) - 2;

229 + else

230 + size++; // make room for the terminator

231 +

232 netdata_log_debug(D_WEB_BUFFER, "Creating new web buffer of size %zu.", size);

233

234 b = callocz(1, sizeof(BUFFER));

@@ -263,10 +268,10 @@ void buffer_increase(BUFFER *b, size_t free_size_required) {

268 if(remaining >= free_size_required) return;

269

270 size_t increase = free_size_required - remaining;

266 - size_t minimum = 128;

271 + size_t minimum = 1024;

272 if(minimum > increase) increase = minimum;

273

269 - size_t optimal = (b->size > 5*1024*1024) ? b->size / 2 : b->size;

274 + size_t optimal = (b->size > 5 * 1024 * 1024) ? b->size / 2 : b->size;

275 if(optimal > increase) increase = optimal;

276

277 netdata_log_debug(D_WEB_BUFFER, "Increasing data buffer from size %zu to %zu.", (size_t)b->size, (size_t)(b->size + increase));

src/libnetdata/buffer/buffer.h

+144 -73

@@ -84,7 +84,8 @@ static inline void _buffer_overflow_check(BUFFER *b __maybe_unused) {

84 "BUFFER: detected overflow.");

85 }

86

87 -static ALWAYS_INLINE void buffer_flush(BUFFER *wb) {

87 +ALWAYS_INLINE

88 +static void buffer_flush(BUFFER *wb) {

89 wb->len = 0;

90

91 wb->json.depth = 0;

@@ -117,7 +118,8 @@ void buffer_char_replace(BUFFER *wb, char from, char to);

118

119 void buffer_print_sn_flags(BUFFER *wb, SN_FLAGS flags, bool send_anomaly_bit);

120

120 -static ALWAYS_INLINE void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {

121 +ALWAYS_INLINE

122 +static void buffer_need_bytes(BUFFER *buffer, size_t needed_free_size) {

123 if(unlikely(buffer->len + needed_free_size >= buffer->size))

124 buffer_increase(buffer, needed_free_size + 1);

125 }

@@ -127,7 +129,8 @@ void buffer_json_initialize(BUFFER *wb, const char *key_quote, const char *value

129

130 void buffer_json_finalize(BUFFER *wb);

131

130 -static ALWAYS_INLINE const char *buffer_tostring(BUFFER *wb)

132 +ALWAYS_INLINE

133 +static const char *buffer_tostring(BUFFER *wb)

134 {

135 if(unlikely(!wb))

136 return NULL;

@@ -140,7 +143,8 @@ static ALWAYS_INLINE const char *buffer_tostring(BUFFER *wb)

143 return(wb->buffer);

144 }

145

143 -static ALWAYS_INLINE void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) {

146 +ALWAYS_INLINE

147 +static void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_TYPE type) {

148 #ifdef NETDATA_INTERNAL_CHECKS

149 assert(wb->json.depth <= BUFFER_JSON_MAX_DEPTH && "BUFFER JSON: max nesting reached");

150 #endif

@@ -152,18 +156,21 @@ static ALWAYS_INLINE void _buffer_json_depth_push(BUFFER *wb, BUFFER_JSON_NODE_T

156 wb->json.stack[wb->json.depth].type = type;

157 }

158

155 -static ALWAYS_INLINE void _buffer_json_depth_pop(BUFFER *wb) {

159 +ALWAYS_INLINE

160 +static void _buffer_json_depth_pop(BUFFER *wb) {

161 wb->json.depth--;

162 }

163

159 -static ALWAYS_INLINE void buffer_putc(BUFFER *wb, char c) {

164 +ALWAYS_INLINE

165 +static void buffer_putc(BUFFER *wb, char c) {

166 buffer_need_bytes(wb, 2);

167 wb->buffer[wb->len++] = c;

168 wb->buffer[wb->len] = '\0';

169 buffer_overflow_check(wb);

170 }

171

166 -static ALWAYS_INLINE void buffer_fast_rawcat(BUFFER *wb, const char *txt, size_t len) {

172 +ALWAYS_INLINE

173 +static void buffer_fast_rawcat(BUFFER *wb, const char *txt, size_t len) {

174 if(unlikely(!txt || !*txt || !len)) return;

175

176 buffer_need_bytes(wb, len + 1);

@@ -182,7 +189,8 @@ static ALWAYS_INLINE void buffer_fast_rawcat(BUFFER *wb, const char *txt, size_t

189 buffer_overflow_check(wb);

190 }

191

185 -static ALWAYS_INLINE void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) {

192 +ALWAYS_INLINE

193 +static void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t len) {

194 if(unlikely(!txt || !*txt || !len)) return;

195

196 buffer_need_bytes(wb, len + 1);

@@ -209,7 +217,8 @@ static ALWAYS_INLINE void buffer_fast_strcat(BUFFER *wb, const char *txt, size_t

217 buffer_overflow_check(wb);

218 }

219

212 -static ALWAYS_INLINE void buffer_strcat(BUFFER *wb, const char *txt) {

220 +ALWAYS_INLINE

221 +static void buffer_strcat(BUFFER *wb, const char *txt) {

222 if(unlikely(!txt || !*txt)) return;

223

224 const char *t = txt;

@@ -231,7 +240,8 @@ static ALWAYS_INLINE void buffer_strcat(BUFFER *wb, const char *txt) {

240 buffer_overflow_check(wb);

241 }

242

234 -static ALWAYS_INLINE void buffer_contents_replace(BUFFER *wb, const char *txt, size_t len) {

243 +ALWAYS_INLINE

244 +static void buffer_contents_replace(BUFFER *wb, const char *txt, size_t len) {

245 wb->len = 0;

246 buffer_need_bytes(wb, len + 1);

247

@@ -242,7 +252,8 @@ static ALWAYS_INLINE void buffer_contents_replace(BUFFER *wb, const char *txt, s

252 buffer_overflow_check(wb);

253 }

254

245 -static ALWAYS_INLINE void buffer_strncat(BUFFER *wb, const char *txt, size_t len) {

255 +ALWAYS_INLINE

256 +static void buffer_strncat(BUFFER *wb, const char *txt, size_t len) {

257 if(unlikely(!txt || !*txt)) return;

258

259 buffer_need_bytes(wb, len + 1);

@@ -255,7 +266,8 @@ static ALWAYS_INLINE void buffer_strncat(BUFFER *wb, const char *txt, size_t len

266 buffer_overflow_check(wb);

267 }

268

258 -static ALWAYS_INLINE void buffer_memcat(BUFFER *wb, const void *mem, size_t bytes) {

269 +ALWAYS_INLINE

270 +static void buffer_memcat(BUFFER *wb, const void *mem, size_t bytes) {

271 if(unlikely(!mem)) return;

272

273 buffer_need_bytes(wb, bytes + 1);

@@ -268,7 +280,8 @@ static ALWAYS_INLINE void buffer_memcat(BUFFER *wb, const void *mem, size_t byte

280 buffer_overflow_check(wb);

281 }

282

271 -static ALWAYS_INLINE void buffer_json_strcat(BUFFER *wb, const char *txt)

283 +ALWAYS_INLINE

284 +static void buffer_json_strcat(BUFFER *wb, const char *txt)

285 {

286 if(unlikely(!txt || !*txt)) return;

287

@@ -310,11 +323,20 @@ static ALWAYS_INLINE void buffer_json_strcat(BUFFER *wb, const char *txt)

323 if(unlikely(*t < ' ')) {

324 uint32_t v = *t++;

325 *d++ = '\\';

313 - *d++ = 'u';

314 - *d++ = hex_digits[(v >> 12) & 0xf];

315 - *d++ = hex_digits[(v >> 8) & 0xf];

316 - *d++ = hex_digits[(v >> 4) & 0xf];

317 - *d++ = hex_digits[v & 0xf];

326 + switch (v) {

327 + case '\n': *d++ = 'n'; break;

328 + case '\r': *d++ = 'r'; break;

329 + case '\t': *d++ = 't'; break;

330 + case '\b': *d++ = 'b'; break;

331 + case '\f': *d++ = 'f'; break;

332 + default:

333 + *d++ = 'u';

334 + *d++ = hex_digits[(v >> 12) & 0xf];

335 + *d++ = hex_digits[(v >> 8) & 0xf];

336 + *d++ = hex_digits[(v >> 4) & 0xf];

337 + *d++ = hex_digits[v & 0xf];

338 + break;

339 + }

340 }

341 else {

342 if (unlikely(*t == '\\' || *t == '\"'))

@@ -333,7 +355,8 @@ static ALWAYS_INLINE void buffer_json_strcat(BUFFER *wb, const char *txt)

355 buffer_overflow_check(wb);

356 }

357

336 -static ALWAYS_INLINE void buffer_json_quoted_strcat(BUFFER *wb, const char *txt) {

358 +ALWAYS_INLINE

359 +static void buffer_json_quoted_strcat(BUFFER *wb, const char *txt) {

360 if(unlikely(!txt || !*txt)) return;

361

362 if(*txt == '"')

@@ -372,13 +395,15 @@ static ALWAYS_INLINE void buffer_json_quoted_strcat(BUFFER *wb, const char *txt)

395 // point the remaining value fits in 32 bits, and then calls

396 // print_number_lu_r() to print the rest with 32 bit arithmetic.

397

375 -static ALWAYS_INLINE char *print_uint32_reversed(char *dst, uint32_t value) {

398 +ALWAYS_INLINE

399 +static char *print_uint32_reversed(char *dst, uint32_t value) {

400 char *d = dst;

401 do *d++ = (char)('0' + (value % 10)); while((value /= 10));

402 return d;

403 }

404

381 -static ALWAYS_INLINE char *print_uint64_reversed(char *dst, uint64_t value) {

405 +ALWAYS_INLINE

406 +static char *print_uint64_reversed(char *dst, uint64_t value) {

407 #ifdef ENV32BIT

408 if(value <= (uint64_t)0xffffffff)

409 return print_uint32_reversed(dst, value);

@@ -394,14 +419,16 @@ static ALWAYS_INLINE char *print_uint64_reversed(char *dst, uint64_t value) {

419 #endif

420 }

421

397 -static ALWAYS_INLINE char *print_uint32_hex_reversed(char *dst, uint32_t value) {

422 +ALWAYS_INLINE

423 +static char *print_uint32_hex_reversed(char *dst, uint32_t value) {

424 static const char *digits = "0123456789ABCDEF";

425 char *d = dst;

426 do *d++ = digits[value & 0xf]; while((value >>= 4));

427 return d;

428 }

429

404 -static ALWAYS_INLINE char *print_uint64_hex_reversed(char *dst, uint64_t value) {

430 +ALWAYS_INLINE

431 +static char *print_uint64_hex_reversed(char *dst, uint64_t value) {

432 #ifdef ENV32BIT

433 if(value <= (uint64_t)0xffffffff)

434 return print_uint32_hex_reversed(dst, value);

@@ -417,7 +444,8 @@ static ALWAYS_INLINE char *print_uint64_hex_reversed(char *dst, uint64_t value)

444 #endif

445 }

446

420 -static ALWAYS_INLINE char *print_uint64_hex_reversed_full(char *dst, uint64_t value) {

447 +ALWAYS_INLINE

448 +static char *print_uint64_hex_reversed_full(char *dst, uint64_t value) {

449 char *d = dst;

450 for(size_t c = 0; c < sizeof(uint64_t) * 2; c++) {

451 *d++ = hex_digits[value & 0xf];

@@ -427,19 +455,22 @@ static ALWAYS_INLINE char *print_uint64_hex_reversed_full(char *dst, uint64_t va

455 return d;

456 }

457

430 -static ALWAYS_INLINE char *print_uint64_base64_reversed(char *dst, uint64_t value) {

458 +ALWAYS_INLINE

459 +static char *print_uint64_base64_reversed(char *dst, uint64_t value) {

460 char *d = dst;

461 do *d++ = base64_digits[value & 63]; while ((value >>= 6));

462 return d;

463 }

464

436 -static ALWAYS_INLINE void char_array_reverse(char *from, char *to) {

465 +ALWAYS_INLINE

466 +static void char_array_reverse(char *from, char *to) {

467 // from and to are inclusive

468 char *begin = from, *end = to, aux;

469 while (end > begin) aux = *end, *end-- = *begin, *begin++ = aux;

470 }

471

442 -static ALWAYS_INLINE int print_netdata_double(char *dst, NETDATA_DOUBLE value) {

472 +ALWAYS_INLINE

473 +static int print_netdata_double(char *dst, NETDATA_DOUBLE value) {

474 char *s = dst;

475

476 if(unlikely(value < 0)) {

@@ -502,7 +533,8 @@ static ALWAYS_INLINE int print_netdata_double(char *dst, NETDATA_DOUBLE value) {

533 return (int)(d - dst);

534 }

535

505 -static ALWAYS_INLINE size_t print_uint64(char *dst, uint64_t value) {

536 +ALWAYS_INLINE

537 +static size_t print_uint64(char *dst, uint64_t value) {

538 char *s = dst;

539 char *d = print_uint64_reversed(s, value);

540 char_array_reverse(s, d - 1);

@@ -510,7 +542,8 @@ static ALWAYS_INLINE size_t print_uint64(char *dst, uint64_t value) {

542 return d - s;

543 }

544

513 -static ALWAYS_INLINE size_t print_int64(char *dst, int64_t value) {

545 +ALWAYS_INLINE

546 +static size_t print_int64(char *dst, int64_t value) {

547 size_t len = 0;

548

549 if(value < 0) {

@@ -523,20 +556,23 @@ static ALWAYS_INLINE size_t print_int64(char *dst, int64_t value) {

556 }

557

558 #define UINT64_MAX_LENGTH (24) // 21 should be enough

526 -static ALWAYS_INLINE void buffer_print_uint64(BUFFER *wb, uint64_t value) {

559 +ALWAYS_INLINE

560 +static void buffer_print_uint64(BUFFER *wb, uint64_t value) {

561 buffer_need_bytes(wb, UINT64_MAX_LENGTH);

562 wb->len += print_uint64(&wb->buffer[wb->len], value);

563 buffer_overflow_check(wb);

564 }

565

532 -static ALWAYS_INLINE void buffer_print_int64(BUFFER *wb, int64_t value) {

566 +ALWAYS_INLINE

567 +static void buffer_print_int64(BUFFER *wb, int64_t value) {

568 buffer_need_bytes(wb, UINT64_MAX_LENGTH);

569 wb->len += print_int64(&wb->buffer[wb->len], value);

570 buffer_overflow_check(wb);

571 }

572

573 #define UINT64_HEX_MAX_LENGTH ((sizeof(HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)

539 -static ALWAYS_INLINE size_t print_uint64_hex(char *dst, uint64_t value) {

574 +ALWAYS_INLINE

575 +static size_t print_uint64_hex(char *dst, uint64_t value) {

576 char *d = dst;

577

578 const char *s = HEX_PREFIX;

@@ -548,7 +584,8 @@ static ALWAYS_INLINE size_t print_uint64_hex(char *dst, uint64_t value) {

584 return e - dst;

585 }

586

551 -static ALWAYS_INLINE size_t print_uint64_hex_full(char *dst, uint64_t value) {

587 +ALWAYS_INLINE

588 +static size_t print_uint64_hex_full(char *dst, uint64_t value) {

589 char *d = dst;

590

591 const char *s = HEX_PREFIX;

@@ -560,20 +597,23 @@ static ALWAYS_INLINE size_t print_uint64_hex_full(char *dst, uint64_t value) {

597 return e - dst;

598 }

599

563 -static ALWAYS_INLINE void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {

600 +ALWAYS_INLINE

601 +static void buffer_print_uint64_hex(BUFFER *wb, uint64_t value) {

602 buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);

603 wb->len += print_uint64_hex(&wb->buffer[wb->len], value);

604 buffer_overflow_check(wb);

605 }

606

569 -static ALWAYS_INLINE void buffer_print_uint64_hex_full(BUFFER *wb, uint64_t value) {

607 +ALWAYS_INLINE

608 +static void buffer_print_uint64_hex_full(BUFFER *wb, uint64_t value) {

609 buffer_need_bytes(wb, UINT64_HEX_MAX_LENGTH);

610 wb->len += print_uint64_hex_full(&wb->buffer[wb->len], value);

611 buffer_overflow_check(wb);

612 }

613

614 #define UINT64_B64_MAX_LENGTH ((sizeof(IEEE754_UINT64_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)

576 -static ALWAYS_INLINE void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) {

615 +ALWAYS_INLINE

616 +static void buffer_print_uint64_base64(BUFFER *wb, uint64_t value) {

617 buffer_need_bytes(wb, UINT64_B64_MAX_LENGTH);

618

619 buffer_fast_strcat(wb, IEEE754_UINT64_B64_PREFIX, sizeof(IEEE754_UINT64_B64_PREFIX) - 1);

@@ -587,11 +627,12 @@ static ALWAYS_INLINE void buffer_print_uint64_base64(BUFFER *wb, uint64_t value)

627 buffer_overflow_check(wb);

628 }

629

590 -static ALWAYS_INLINE void buffer_print_int64_hex(BUFFER *wb, int64_t value) {

630 +ALWAYS_INLINE

631 +static void buffer_print_int64_hex(BUFFER *wb, int64_t value) {

632 buffer_need_bytes(wb, 2);

633

634 if(value < 0) {

594 - buffer_fast_strcat(wb, "-", 1);

635 + buffer_putc(wb, '-');

636 value = -value;

637 }

638

@@ -600,11 +641,12 @@ static ALWAYS_INLINE void buffer_print_int64_hex(BUFFER *wb, int64_t value) {

641 buffer_overflow_check(wb);

642 }

643

603 -static ALWAYS_INLINE void buffer_print_int64_base64(BUFFER *wb, int64_t value) {

644 +ALWAYS_INLINE

645 +static void buffer_print_int64_base64(BUFFER *wb, int64_t value) {

646 buffer_need_bytes(wb, 2);

647

648 if(value < 0) {

607 - buffer_fast_strcat(wb, "-", 1);

649 + buffer_putc(wb, '-');

650 value = -value;

651 }

652

@@ -614,7 +656,9 @@ static ALWAYS_INLINE void buffer_print_int64_base64(BUFFER *wb, int64_t value) {

656 }

657

658 #define DOUBLE_MAX_LENGTH (512) // 318 should be enough, including null

617 -static ALWAYS_INLINE void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) {

659 +

660 +ALWAYS_INLINE

661 +static void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE value) {

662 buffer_need_bytes(wb, DOUBLE_MAX_LENGTH);

663

664 if(isnan(value) || isinf(value)) {

@@ -632,7 +676,8 @@ static ALWAYS_INLINE void buffer_print_netdata_double(BUFFER *wb, NETDATA_DOUBLE

676 }

677

678 #define DOUBLE_HEX_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_HEX_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)

635 -static ALWAYS_INLINE void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) {

679 +ALWAYS_INLINE

680 +static void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DOUBLE value) {

681 buffer_need_bytes(wb, DOUBLE_HEX_MAX_LENGTH);

682

683 uint64_t *ptr = (uint64_t *) (&value);

@@ -648,7 +693,8 @@ static ALWAYS_INLINE void buffer_print_netdata_double_hex(BUFFER *wb, NETDATA_DO

693 }

694

695 #define DOUBLE_B64_MAX_LENGTH ((sizeof(IEEE754_DOUBLE_B64_PREFIX) - 1) + (sizeof(uint64_t) * 2) + 1)

651 -static ALWAYS_INLINE void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) {

696 +ALWAYS_INLINE

697 +static void buffer_print_netdata_double_base64(BUFFER *wb, NETDATA_DOUBLE value) {

698 buffer_need_bytes(wb, DOUBLE_B64_MAX_LENGTH);

699

700 uint64_t *ptr = (uint64_t *) (&value);

@@ -669,7 +715,8 @@ typedef enum {

715 NUMBER_ENCODING_BASE64,

716 } NUMBER_ENCODING;

717

672 -static ALWAYS_INLINE void buffer_print_int64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, int64_t value) {

718 +ALWAYS_INLINE

719 +static void buffer_print_int64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, int64_t value) {

720 if(encoding == NUMBER_ENCODING_BASE64)

721 return buffer_print_int64_base64(wb, value);

722

@@ -679,7 +726,8 @@ static ALWAYS_INLINE void buffer_print_int64_encoded(BUFFER *wb, NUMBER_ENCODING

726 return buffer_print_int64(wb, value);

727 }

728

682 -static ALWAYS_INLINE void buffer_print_uint64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, uint64_t value) {

729 +ALWAYS_INLINE

730 +static void buffer_print_uint64_encoded(BUFFER *wb, NUMBER_ENCODING encoding, uint64_t value) {

731 if(encoding == NUMBER_ENCODING_BASE64)

732 return buffer_print_uint64_base64(wb, value);

733

@@ -689,7 +737,8 @@ static ALWAYS_INLINE void buffer_print_uint64_encoded(BUFFER *wb, NUMBER_ENCODIN

737 return buffer_print_uint64(wb, value);

738 }

739

692 -static ALWAYS_INLINE void buffer_print_netdata_double_encoded(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA_DOUBLE value) {

740 +ALWAYS_INLINE

741 +static void buffer_print_netdata_double_encoded(BUFFER *wb, NUMBER_ENCODING encoding, NETDATA_DOUBLE value) {

742 if(encoding == NUMBER_ENCODING_BASE64)

743 return buffer_print_netdata_double_base64(wb, value);

744

@@ -699,7 +748,8 @@ static ALWAYS_INLINE void buffer_print_netdata_double_encoded(BUFFER *wb, NUMBER

748 return buffer_print_netdata_double(wb, value);

749 }

750

702 -static ALWAYS_INLINE void buffer_print_spaces(BUFFER *wb, size_t spaces) {

751 +ALWAYS_INLINE

752 +static void buffer_print_spaces(BUFFER *wb, size_t spaces) {

753 buffer_need_bytes(wb, spaces * 4 + 1);

754

755 char *d = &wb->buffer[wb->len];

@@ -716,29 +766,33 @@ static ALWAYS_INLINE void buffer_print_spaces(BUFFER *wb, size_t spaces) {

766 buffer_overflow_check(wb);

767 }

768

719 -static ALWAYS_INLINE void buffer_print_json_comma(BUFFER *wb) {

769 +ALWAYS_INLINE

770 +static void buffer_print_json_comma(BUFFER *wb) {

771 if(wb->json.stack[wb->json.depth].count)

721 - buffer_fast_strcat(wb, ",", 1);

772 + buffer_putc(wb, ',');

773 }

774

724 -static ALWAYS_INLINE void buffer_print_json_comma_newline_spacing(BUFFER *wb) {

775 +ALWAYS_INLINE

776 +static void buffer_print_json_comma_newline_spacing(BUFFER *wb) {

777 buffer_print_json_comma(wb);

778

779 if((wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) ||

780 (wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && !(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS)))

781 return;

782

731 - buffer_fast_strcat(wb, "\n", 1);

783 + buffer_putc(wb, '\n');

784 buffer_print_spaces(wb, wb->json.depth + 1);

785 }

786

735 -static ALWAYS_INLINE void buffer_print_json_key(BUFFER *wb, const char *key) {

787 +ALWAYS_INLINE

788 +static void buffer_print_json_key(BUFFER *wb, const char *key) {

789 buffer_strcat(wb, wb->json.key_quote);

790 buffer_json_strcat(wb, key);

791 buffer_strcat(wb, wb->json.key_quote);

792 }

793

741 -static ALWAYS_INLINE void buffer_json_add_string_value(BUFFER *wb, const char *value) {

794 +ALWAYS_INLINE

795 +static void buffer_json_add_string_value(BUFFER *wb, const char *value) {

796 if(value) {

797 buffer_strcat(wb, wb->json.value_quote);

798 buffer_json_strcat(wb, value);

@@ -748,7 +802,8 @@ static ALWAYS_INLINE void buffer_json_add_string_value(BUFFER *wb, const char *v

802 buffer_fast_strcat(wb, "null", 4);

803 }

804

751 -static ALWAYS_INLINE void buffer_json_add_quoted_string_value(BUFFER *wb, const char *value) {

805 +ALWAYS_INLINE

806 +static void buffer_json_add_quoted_string_value(BUFFER *wb, const char *value) {

807 if(value) {

808 buffer_strcat(wb, wb->json.value_quote);

809 buffer_json_quoted_strcat(wb, value);

@@ -773,17 +828,17 @@ static inline void buffer_json_object_close(BUFFER *wb) {

828 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_OBJECT && "BUFFER JSON: an object is not open to close it");

829 #endif

830 if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY)) {

776 - buffer_fast_strcat(wb, "\n", 1);

831 + buffer_putc(wb, '\n');

832 buffer_print_spaces(wb, wb->json.depth);

833 }

779 - buffer_fast_strcat(wb, "}", 1);

834 + buffer_putc(wb, '}');

835 _buffer_json_depth_pop(wb);

836 }

837

838 static inline void buffer_json_member_add_string(BUFFER *wb, const char *key, const char *value) {

839 buffer_print_json_comma_newline_spacing(wb);

840 buffer_print_json_key(wb, key);

786 - buffer_fast_strcat(wb, ":", 1);

841 + buffer_putc(wb, ':');

842 buffer_json_add_string_value(wb, value);

843

844 wb->json.stack[wb->json.depth].count++;

@@ -807,7 +862,7 @@ void buffer_json_member_add_duration_ut(BUFFER *wb, const char *key, int64_t dur

862 static inline void buffer_json_member_add_quoted_string(BUFFER *wb, const char *key, const char *value) {

863 buffer_print_json_comma_newline_spacing(wb);

864 buffer_print_json_key(wb, key);

810 - buffer_fast_strcat(wb, ":", 1);

865 + buffer_putc(wb, ':');

866

867 if(!value || strcmp(value, "null") == 0)

868 buffer_fast_strcat(wb, "null", 4);

@@ -820,7 +875,7 @@ static inline void buffer_json_member_add_quoted_string(BUFFER *wb, const char *

875 static inline void buffer_json_member_add_uuid_ptr(BUFFER *wb, const char *key, nd_uuid_t *value) {

876 buffer_print_json_comma_newline_spacing(wb);

877 buffer_print_json_key(wb, key);

823 - buffer_fast_strcat(wb, ":", 1);

878 + buffer_putc(wb, ':');

879

880 if(value && !uuid_is_null(*value)) {

881 char uuid[GUID_LEN + 1];

@@ -836,10 +891,10 @@ static inline void buffer_json_member_add_uuid_ptr(BUFFER *wb, const char *key,

891 static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_uuid_t value) {

892 buffer_print_json_comma_newline_spacing(wb);

893 buffer_print_json_key(wb, key);

839 - buffer_fast_strcat(wb, ":", 1);

894 + buffer_putc(wb, ':');

895

896 if(!uuid_is_null(value)) {

842 - char uuid[GUID_LEN + 1];

897 + char uuid[UUID_STR_LEN];

898 uuid_unparse_lower(value, uuid);

899 buffer_json_add_string_value(wb, uuid);

900 }

@@ -849,10 +904,26 @@ static inline void buffer_json_member_add_uuid(BUFFER *wb, const char *key, nd_u

904 wb->json.stack[wb->json.depth].count++;

905 }

906

907 +static inline void buffer_json_member_add_uuid_compact(BUFFER *wb, const char *key, nd_uuid_t value) {

908 + buffer_print_json_comma_newline_spacing(wb);

909 + buffer_print_json_key(wb, key);

910 + buffer_putc(wb, ':');

911 +

912 + if(!uuid_is_null(value)) {

913 + char uuid[UUID_COMPACT_STR_LEN];

914 + uuid_unparse_lower_compact(value, uuid);

915 + buffer_json_add_string_value(wb, uuid);

916 + }

917 + else

918 + buffer_json_add_string_value(wb, NULL);

919 +

920 + wb->json.stack[wb->json.depth].count++;

921 +}

922 +

923 static inline void buffer_json_member_add_boolean(BUFFER *wb, const char *key, bool value) {

924 buffer_print_json_comma_newline_spacing(wb);

925 buffer_print_json_key(wb, key);

855 - buffer_fast_strcat(wb, ":", 1);

926 + buffer_putc(wb, ':');

927 buffer_strcat(wb, value?"true":"false");

928

929 wb->json.stack[wb->json.depth].count++;

@@ -865,7 +936,7 @@ static inline void buffer_json_member_add_array(BUFFER *wb, const char *key) {

936 buffer_fast_strcat(wb, ":[", 2);

937 }

938 else

868 - buffer_fast_strcat(wb, "[", 1);

939 + buffer_putc(wb, '[');

940

941 wb->json.stack[wb->json.depth].count++;

942

@@ -876,13 +947,13 @@ static inline void buffer_json_add_array_item_array(BUFFER *wb) {

947 if(!(wb->json.options & BUFFER_JSON_OPTIONS_MINIFY) && wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY) {

948 // an array inside another array

949 buffer_print_json_comma(wb);

879 - buffer_fast_strcat(wb, "\n", 1);

950 + buffer_putc(wb, '\n');

951 buffer_print_spaces(wb, wb->json.depth + 1);

952 }

953 else

954 buffer_print_json_comma_newline_spacing(wb);

955

885 - buffer_fast_strcat(wb, "[", 1);

956 + buffer_putc(wb, '[');

957 wb->json.stack[wb->json.depth].count++;

958

959 _buffer_json_depth_push(wb, BUFFER_JSON_ARRAY);

@@ -969,7 +1040,7 @@ static inline void buffer_json_add_array_item_time_t2ms(BUFFER *wb, time_t value

1040 static inline void buffer_json_add_array_item_object(BUFFER *wb) {

1041 buffer_print_json_comma_newline_spacing(wb);

1042

972 - buffer_fast_strcat(wb, "{", 1);

1043 + buffer_putc(wb, '{');

1044 wb->json.stack[wb->json.depth].count++;

1045

1046 _buffer_json_depth_push(wb, BUFFER_JSON_OBJECT);

@@ -978,7 +1049,7 @@ static inline void buffer_json_add_array_item_object(BUFFER *wb) {

1049 static inline void buffer_json_member_add_time_t(BUFFER *wb, const char *key, time_t value) {

1050 buffer_print_json_comma_newline_spacing(wb);

1051 buffer_print_json_key(wb, key);

981 - buffer_fast_strcat(wb, ":", 1);

1052 + buffer_putc(wb, ':');

1053 buffer_print_int64(wb, value);

1054

1055 wb->json.stack[wb->json.depth].count++;

@@ -987,7 +1058,7 @@ static inline void buffer_json_member_add_time_t(BUFFER *wb, const char *key, ti

1058 static inline void buffer_json_member_add_time_t2ms(BUFFER *wb, const char *key, time_t value) {

1059 buffer_print_json_comma_newline_spacing(wb);

1060 buffer_print_json_key(wb, key);

990 - buffer_fast_strcat(wb, ":", 1);

1061 + buffer_putc(wb, ':');

1062 buffer_print_int64(wb, value);

1063 buffer_fast_strcat(wb, "000", 3);

1064

@@ -997,7 +1068,7 @@ static inline void buffer_json_member_add_time_t2ms(BUFFER *wb, const char *key,

1068 static inline void buffer_json_member_add_uint64(BUFFER *wb, const char *key, uint64_t value) {

1069 buffer_print_json_comma_newline_spacing(wb);

1070 buffer_print_json_key(wb, key);

1000 - buffer_fast_strcat(wb, ":", 1);

1071 + buffer_putc(wb, ':');

1072 buffer_print_uint64(wb, value);

1073

1074 wb->json.stack[wb->json.depth].count++;

@@ -1006,7 +1077,7 @@ static inline void buffer_json_member_add_uint64(BUFFER *wb, const char *key, ui

1077 static inline void buffer_json_member_add_int64(BUFFER *wb, const char *key, int64_t value) {

1078 buffer_print_json_comma_newline_spacing(wb);

1079 buffer_print_json_key(wb, key);

1009 - buffer_fast_strcat(wb, ":", 1);

1080 + buffer_putc(wb, ':');

1081 buffer_print_int64(wb, value);

1082

1083 wb->json.stack[wb->json.depth].count++;

@@ -1015,7 +1086,7 @@ static inline void buffer_json_member_add_int64(BUFFER *wb, const char *key, int

1086 static inline void buffer_json_member_add_double(BUFFER *wb, const char *key, NETDATA_DOUBLE value) {

1087 buffer_print_json_comma_newline_spacing(wb);

1088 buffer_print_json_key(wb, key);

1018 - buffer_fast_strcat(wb, ":", 1);

1089 + buffer_putc(wb, ':');

1090 buffer_print_netdata_double(wb, value);

1091

1092 wb->json.stack[wb->json.depth].count++;

@@ -1027,11 +1098,11 @@ static inline void buffer_json_array_close(BUFFER *wb) {

1098 assert(wb->json.stack[wb->json.depth].type == BUFFER_JSON_ARRAY && "BUFFER JSON: an array is not open to close it");

1099 #endif

1100 if(wb->json.options & BUFFER_JSON_OPTIONS_NEWLINE_ON_ARRAY_ITEMS) {

1030 - buffer_fast_strcat(wb, "\n", 1);

1101 + buffer_putc(wb, '\n');

1102 buffer_print_spaces(wb, wb->json.depth);

1103 }

1104

1034 - buffer_fast_strcat(wb, "]", 1);

1105 + buffer_putc(wb, ']');

1106 _buffer_json_depth_pop(wb);

1107 }

1108

src/libnetdata/common.h

+5

@@ -426,6 +426,11 @@ typedef uint32_t uid_t;

426

427 // --------------------------------------------------------------------------------------------------------------------

428

429 +#define FUNCTION_RUN_ONCE() { static bool __run_once = false; if(__run_once) return; __run_once = true; }

430 +#define FUNCTION_RUN_ONCE_RET(ret) { static bool __run_once = false; if(__run_once) return (ret); __run_once = true; }

431 +

432 +// --------------------------------------------------------------------------------------------------------------------

433 +

434 #ifndef HOST_NAME_MAX

435 #define HOST_NAME_MAX 256

436 #endif

src/libnetdata/datetime/rfc3339.c

+60 -21

@@ -66,7 +66,7 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {

66 char *s;

67 usec_t timestamp, usec = 0;

68

69 - // Use strptime to parse up to seconds

69 + // Parse date and time (up to seconds)

70 s = strptime(rfc3339, "%Y-%m-%dT%H:%M:%S", &tm);

71 if (!s)

72 return 0; // Parsing error

@@ -78,29 +78,33 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {

78 int digits_parsed = (int)(next - (s + 1));

79

80 if (digits_parsed < 1 || digits_parsed > 9)

81 - return 0; // parsing error

81 + return 0; // Parsing error

82

83 static const usec_t fix_usec[] = {

84 - 1000000, // 0 digits (not used)

85 - 100000, // 1 digit

86 - 10000, // 2 digits

87 - 1000, // 3 digits

88 - 100, // 4 digits

89 - 10, // 5 digits

90 - 1, // 6 digits

91 - 10, // 7 digits

92 - 100, // 8 digits

93 - 1000, // 9 digits

84 + 1000000, // 0 digits (not used)

85 + 100000, // 1 digit

86 + 10000, // 2 digits

87 + 1000, // 3 digits

88 + 100, // 4 digits

89 + 10, // 5 digits

90 + 1, // 6 digits

91 + 10, // 7 digits

92 + 100, // 8 digits

93 + 1000 // 9 digits

94 };

95 - usec = digits_parsed <= 6 ? usec * fix_usec[digits_parsed] : usec / fix_usec[digits_parsed];

95 +

96 + if (digits_parsed <= 6)

97 + usec = usec * fix_usec[digits_parsed];

98 + else

99 + usec = usec / fix_usec[digits_parsed];

100

101 s = next;

102 }

103

100 - // Check and parse timezone if present

104 + // Parse timezone specification

105 int tz_offset = 0;

106 if (*s == '+' || *s == '-') {

103 - // Parse the hours:mins part of the timezone

107 + // Ensure format is correct: e.g. +02:00 or -05:30

108

109 if (!isdigit((uint8_t)s[1]) || !isdigit((uint8_t)s[2]) || s[3] != ':' ||

110 !isdigit((uint8_t)s[4]) || !isdigit((uint8_t)s[5]))

@@ -108,8 +112,7 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {

112

113 char tz_sign = *s;

114 tz_hours = (s[1] - '0') * 10 + (s[2] - '0');

111 - tz_mins = (s[4] - '0') * 10 + (s[5] - '0');

112 -

115 + tz_mins = (s[4] - '0') * 10 + (s[5] - '0');

116 tz_offset = tz_hours * 3600 + tz_mins * 60;

117 tz_offset *= (tz_sign == '+' ? 1 : -1);

118

@@ -118,17 +121,53 @@ usec_t rfc3339_parse_ut(const char *rfc3339, char **endptr) {

121 else if (*s == 'Z')

122 s++;

123 else

121 - return 0; // Invalid RFC 3339 format

124 + return 0; // Invalid RFC 3339 timezone specification

125

123 - // Convert to time_t (assuming local time, then adjusting for timezone later)

124 - time_t epoch_s = mktime(&tm);

126 + // Convert struct tm to time_t in UTC

127 + time_t epoch_s;

128 +

129 +#if defined(HAVE_TIMEGM)

130 + // If available, use timegm() which interprets tm as UTC.

131 + epoch_s = timegm(&tm);

132 +#else

133 + // Use mktime(), which assumes tm is local time, then adjust.

134 + epoch_s = mktime(&tm);

135 if (epoch_s == -1)

136 return 0; // Error in time conversion

137

138 +# if defined(HAVE_TM_GMTOFF)

139 + // tm.tm_gmtoff is the offset (in seconds) of local time from UTC.

140 + epoch_s -= tm.tm_gmtoff;

141 +# else

142 + // Fallback: compute the difference between localtime and gmtime.

143 + {

144 + struct tm local_tm, utc_tm;

145 +#if defined(_POSIX_THREAD_SAFE_FUNCTIONS) && !defined(__APPLE__)

146 + localtime_r(&epoch_s, &local_tm);

147 + gmtime_r(&epoch_s, &utc_tm);

148 +#else

149 + // If thread-safe functions are not available, use localtime() and gmtime()

150 + struct tm *lt = localtime(&epoch_s);

151 + struct tm *gt = gmtime(&epoch_s);

152 + if (!lt || !gt)

153 + return 0;

154 + local_tm = *lt;

155 + utc_tm = *gt;

156 +#endif

157 + int local_offset = (local_tm.tm_hour - utc_tm.tm_hour) * 3600 +

158 + (local_tm.tm_min - utc_tm.tm_min) * 60;

159 + int day_diff = local_tm.tm_yday - utc_tm.tm_yday;

160 + local_offset += day_diff * 86400;

161 + epoch_s -= local_offset;

162 + }

163 +# endif

164 +#endif

165 +

166 + // Combine seconds with fractional microseconds, then adjust for the RFC 3339 timezone.

167 timestamp = (usec_t)epoch_s * USEC_PER_SEC + usec;

168 timestamp -= tz_offset * USEC_PER_SEC;

169

131 - if(endptr)

170 + if (endptr)

171 *endptr = s;

172

173 return timestamp;

src/libnetdata/exit/exit_initiated.c new

+112

@@ -0,0 +1,112 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "../libnetdata.h"

4 +

5 +volatile EXIT_REASON exit_initiated = EXIT_REASON_NONE;

6 +

7 +ENUM_STR_MAP_DEFINE(EXIT_REASON) = {

8 + { EXIT_REASON_SIGINT, "signal-interrupt"},

9 + { EXIT_REASON_SIGQUIT, "signal-quit"},

10 + { EXIT_REASON_SIGTERM, "signal-terminate"},

11 + { EXIT_REASON_SIGBUS, "signal-bus-error"},

12 + { EXIT_REASON_SIGSEGV, "signal-segmentation-fault"},

13 + { EXIT_REASON_SIGFPE, "signal-floating-point-exception"},

14 + { EXIT_REASON_SIGILL, "signal-illegal-instruction"},

15 + { EXIT_REASON_API_QUIT, "api-quit"},

16 + { EXIT_REASON_CMD_EXIT, "cmd-exit"},

17 + { EXIT_REASON_FATAL, "fatal"},

18 + { EXIT_REASON_SYSTEM_SHUTDOWN, "system-shutdown"},

19 + { EXIT_REASON_SERVICE_STOP, "service-stop"},

20 + { EXIT_REASON_UPDATE, "update"},

21 +

22 + // terminator

23 + {0, NULL},

24 +};

25 +

26 +BITMAP_STR_DEFINE_FUNCTIONS(EXIT_REASON, EXIT_REASON_NONE, "none");

27 +

28 +#if defined(OS_LINUX)

29 +static bool is_system_shutdown_sysv(void) {

30 + const char *shutdown_files[] = {

31 + "/etc/nologin", // Created during shutdown

32 + "/etc/halt", // SysV shutdown indicator

33 + "/run/nologin", // Modern systems shutdown indicator

34 + NULL

35 + };

36 +

37 + for (const char **file = shutdown_files; *file != NULL; file++) {

38 + if (access(*file, F_OK) == 0)

39 + return true;

40 + }

41 +

42 + return false;

43 +}

44 +

45 +static bool is_system_shutdown(void) {

46 + return is_system_shutdown_sysv();

47 +}

48 +#endif

49 +

50 +#if defined(OS_FREEBSD)

51 +#include <sys/sysctl.h>

52 +static bool is_system_shutdown(void) {

53 + int state = 0;

54 + size_t state_len = sizeof(state);

55 +

56 + if (sysctlbyname("kern.shutdown", &state, &state_len, NULL, 0) == 0)

57 + return state != 0;

58 +

59 + return false;

60 +}

61 +#endif

62 +

63 +#if defined(OS_MACOS)

64 +#include <sys/sysctl.h>

65 +static bool is_system_shutdown(void) {

66 + char buf[1024];

67 + size_t len = sizeof(buf);

68 +

69 + if (sysctlbyname("kern.shutdownstate", buf, &len, NULL, 0) == 0)

70 + return true;

71 +

72 + if (access("/var/db/.SystemShutdown", F_OK) == 0)

73 + return true;

74 +

75 + return false;

76 +}

77 +#endif

78 +

79 +#if defined(OS_WINDOWS)

80 +#include <windows.h>

81 +static bool is_system_shutdown(void) {

82 + return GetSystemMetrics(SM_SHUTTINGDOWN) != 0;

83 +}

84 +#endif

85 +

86 +static const char *self_path = NULL;

87 +static OS_FILE_METADATA self = { 0 };

88 +

89 +void exit_initiated_reset(void) {

90 + exit_initiated = EXIT_REASON_NONE;

91 +

92 + freez((char *)self_path);

93 + self_path = os_get_process_path();

94 + if(self_path)

95 + self = os_get_file_metadata(self_path);

96 +}

97 +

98 +void exit_initiated_set(EXIT_REASON reason) {

99 + if(exit_initiated == EXIT_REASON_NONE && !(reason & EXIT_REASON_SYSTEM_SHUTDOWN) && is_system_shutdown())

100 + reason |= EXIT_REASON_SYSTEM_SHUTDOWN;

101 +

102 + if(exit_initiated == EXIT_REASON_NONE && self_path && OS_FILE_METADATA_OK(self)) {

103 + OS_FILE_METADATA self_now = os_get_file_metadata(self_path);

104 + if(OS_FILE_METADATA_OK(self_now) && (self_now.modified_time != self.modified_time || self_now.size_bytes != self.size_bytes))

105 + reason |= EXIT_REASON_UPDATE;

106 + }

107 +

108 + // we combine all of them together

109 + // so that if this is called multiple times,

110 + // we will have all of them

111 + exit_initiated |= reason;

112 +}

src/libnetdata/exit/exit_initiated.h new

+54

@@ -0,0 +1,54 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_EXIT_INITIATED_H

4 +#define NETDATA_EXIT_INITIATED_H

5 +

6 +#include "../common.h"

7 +#include "../template-enum.h"

8 +

9 +typedef enum {

10 + EXIT_REASON_NONE = 0,

11 +

12 + // automatically detect when exit_initiated_set() is called

13 + // supports Linux, FreeBSD, MacOS, Windows

14 + EXIT_REASON_SYSTEM_SHUTDOWN = (1 << 0), // detected

15 +

16 + // signals - normal termination

17 + EXIT_REASON_SIGQUIT = (1 << 1), // rare, but graceful

18 + EXIT_REASON_SIGTERM = (1 << 2), // received on Linux, FreeBSD, MacOS

19 + EXIT_REASON_SIGINT = (1 << 3), // received on Windows on normal termination

20 +

21 + // signals - abnormal termination

22 + EXIT_REASON_SIGBUS = (1 << 4),

23 + EXIT_REASON_SIGSEGV = (1 << 5),

24 + EXIT_REASON_SIGFPE = (1 << 6),

25 + EXIT_REASON_SIGILL = (1 << 7),

26 +

27 + // normal termination via APIs

28 + EXIT_REASON_API_QUIT = (1 << 7),

29 + EXIT_REASON_CMD_EXIT = (1 << 8),

30 +

31 + // abnormal termination via a fatal message

32 + EXIT_REASON_FATAL = (1 << 9),

33 +

34 + // windows specific, service stop

35 + EXIT_REASON_SERVICE_STOP = (1 << 10),

36 +

37 + // netdata update

38 + EXIT_REASON_UPDATE = (1 << 11),

39 +} EXIT_REASON;

40 +

42 +#define EXIT_REASON_ABNORMAL (EXIT_REASON_SIGBUS|EXIT_REASON_SIGSEGV|EXIT_REASON_SIGFPE|EXIT_REASON_SIGILL|EXIT_REASON_FATAL)

43 +

44 +#define is_exit_reason_normal(reason) (((reason) & EXIT_REASON_NORMAL) && !((reason) & EXIT_REASON_ABNORMAL))

45 +

46 +typedef struct web_buffer BUFFER;

47 +BITMAP_STR_DEFINE_FUNCTIONS_EXTERN(EXIT_REASON);

48 +

49 +extern volatile EXIT_REASON exit_initiated;

50 +

51 +void exit_initiated_reset(void);

52 +void exit_initiated_set(EXIT_REASON reason);

53 +

54 +#endif //NETDATA_EXIT_INITIATED_H

src/libnetdata/facets/logs_query_status.h

+2 -1

@@ -326,7 +326,8 @@ static inline void lqs_function_help(LOGS_QUERY_STATUS *lqs, BUFFER *wb) {

326 );

327 }

328

329 -static inline bool lqs_request_parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {

329 +static inline bool lqs_request_parse_json_payload(json_object *jobj, void *data, BUFFER *error) {

330 + const char *path = "";

331 struct logs_query_data *qd = data;

332 LOGS_QUERY_REQUEST *rq = qd->rq;

333 BUFFER *wb = qd->wb;

src/libnetdata/json/json-c-parser-inline.c

+35 -1

@@ -38,7 +38,7 @@ struct json_object *json_parse_function_payload_or_error(BUFFER *output, BUFFER

38 json_tokener_free(tokener);

39

40 CLEAN_BUFFER *error = buffer_create(0, NULL);

41 - if(!cb(jobj, "", cb_data, error)) {

41 + if(!cb(jobj, cb_data, error)) {

42 char tmp[buffer_strlen(error) + 100];

43 snprintfz(tmp, sizeof(tmp), "JSON parser failed: %s", buffer_tostring(error));

44 *code = rrd_call_function_error(output, tmp, HTTP_RESP_BAD_REQUEST);

@@ -50,3 +50,37 @@ struct json_object *json_parse_function_payload_or_error(BUFFER *output, BUFFER

50

51 return jobj;

52 }

53 +

54 +int json_parse_payload_or_error(BUFFER *payload, BUFFER *error, json_parse_function_payload_t cb, void *cb_data) {

55 + if(!payload || !buffer_strlen(payload)) {

56 + buffer_strcat(error, "No payload given, but a payload is required for this feature.");

57 + return HTTP_RESP_BAD_REQUEST;

58 + }

59 +

60 + struct json_tokener *tokener = json_tokener_new();

61 + if (!tokener) {

62 + buffer_strcat(error, "Failed to initialize json parser.");

63 + return HTTP_RESP_INTERNAL_SERVER_ERROR;

64 + }

65 +

66 + struct json_object *jobj = json_tokener_parse_ex(tokener, buffer_tostring(payload), (int)buffer_strlen(payload));

67 + if (json_tokener_get_error(tokener) != json_tokener_success) {

68 + const char *error_msg = json_tokener_error_desc(json_tokener_get_error(tokener));

69 + char tmp[strlen(error_msg) + 100];

70 + snprintf(tmp, sizeof(tmp), "JSON parser failed: %s", error_msg);

71 + json_tokener_free(tokener);

72 + buffer_strcat(error, tmp);

73 + return HTTP_RESP_BAD_REQUEST;

74 + }

75 + json_tokener_free(tokener);

76 +

77 + if(!cb(jobj, cb_data, error)) {

78 + if(!buffer_strlen(error))

79 + buffer_strcat(error, "Unknown error during parsing");

80 + json_object_put(jobj);

81 + return HTTP_RESP_BAD_REQUEST;

82 + }

83 +

84 + json_object_put(jobj);

85 + return HTTP_RESP_OK;

86 +}

src/libnetdata/json/json-c-parser-inline.h

+64 -3

@@ -25,6 +25,17 @@

25 } \

26 } while(0)

27

28 +#define JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \

29 + json_object *_j; \

30 + if (json_object_object_get_ex(jobj, member, &_j) && json_object_is_type(_j, json_type_string)) { \

31 + strncpyz(dst, json_object_get_string(_j), sizeof(dst) - 1); \

32 + } \

33 + else if(required) { \

34 + buffer_sprintf(error, "missing or invalid type for '%s.%s' string", path, member); \

35 + return false; \

36 + } \

37 +} while(0)

38 +

39 #define JSONC_PARSE_TXT2STRDUPZ_OR_ERROR_AND_RETURN(jobj, path, member, dst, error, required) do { \

40 json_object *_j; \

41 if (json_object_object_get_ex(jobj, member, &_j) && json_object_is_type(_j, json_type_string)) { \

@@ -140,7 +151,7 @@

151 typeof(dst) _bit = converter(_option_str); \

152 if (_bit == 0) { \

153 buffer_sprintf(error, "unknown option '%s' in '%s.%s' at index %zu", _option_str, path, member, _i); \

143 - return false; \

154 + /* return false; */ \

155 } \

156 dst |= _bit; \

157 } \

@@ -217,7 +228,7 @@

228 } \

229 } while(0)

230

220 -#define JSONC_PARSE_SUBOBJECT(jobj, path, member, dst, callback, error, required) do { \

231 +#define JSONC_PARSE_SUBOBJECT_CB(jobj, path, member, dst, callback, error, required) do { \

232 json_object *_j; \

233 if (json_object_object_get_ex(jobj, member, &_j)) { \

234 char _new_path[strlen(path) + strlen(member) + 2]; \

@@ -231,8 +242,58 @@

242 } \

243 } while(0)

244

234 -typedef bool (*json_parse_function_payload_t)(json_object *jobj, const char *path, void *data, BUFFER *error);

245 +#define JSONC_TEMP_VAR(type, line) JSONC_TEMP_VAR_IMPL(type, line)

246 +#define JSONC_TEMP_VAR_IMPL(type, line) _jsonc_temp_##type##line

247 +

248 +#define JSONC_PATH_CONCAT(path, sizeof_path, prefix, member, error) do { \

249 + size_t len = strlen(prefix); \

250 + if(len >= sizeof_path - 1) { \

251 + buffer_sprintf(error, "path too long while adding '%s'", member); \

252 + return false; \

253 + } \

254 + if(len) { \

255 + if(len >= sizeof_path - 2) { \

256 + buffer_sprintf(error, "path too long while adding '.' before '%s'", member); \

257 + return false; \

258 + } \

259 + strncpyz(path + len, ".", sizeof_path - len); \

260 + len++; \

261 + } \

262 + strncpyz(path + len, member, sizeof_path - len); \

263 +} while(0)

264 +

265 +#define JSONC_PARSE_SUBOBJECT(jobj, path, member, error, required, block) do { \

266 + BUILD_BUG_ON(sizeof(path) < 128); /* ensure path is an array of at least 128 bytes */ \

267 + json_object *JSONC_TEMP_VAR(_j, __LINE__); \

268 + if (!json_object_object_get_ex(jobj, member, &JSONC_TEMP_VAR(_j, __LINE__))) { \

269 + if(required) { \

270 + buffer_sprintf(error, "missing '%s.%s' object", *path ? path : "", member); \

271 + return false; \

272 + } \

273 + } \

274 + else { \

275 + if (!json_object_is_type(JSONC_TEMP_VAR(_j, __LINE__), json_type_object)) { \

276 + buffer_sprintf(error, "not an object '%s.%s'", *path ? path : "", member); \

277 + return false; \

278 + } \

279 + json_object *JSONC_TEMP_VAR(saved_jobj, __LINE__) = jobj; \

280 + jobj = JSONC_TEMP_VAR(_j, __LINE__); \

281 + char JSONC_TEMP_VAR(saved_path, __LINE__)[strlen(path) + 1]; \

282 + strncpyz(JSONC_TEMP_VAR(saved_path, __LINE__), path, sizeof(JSONC_TEMP_VAR(saved_path, __LINE__))); \

283 + JSONC_PATH_CONCAT(path, sizeof(path), path, member, error); \

284 + /* Run the user's code block */ \

285 + block \

286 + /* Restore the previous scope's values */ \

287 + jobj = JSONC_TEMP_VAR(saved_jobj, __LINE__); \

288 + strncpyz(path, JSONC_TEMP_VAR(saved_path, __LINE__), sizeof(path)); \

289 + } \

290 +} while(0)

291 +

292 +typedef bool (*json_parse_function_payload_t)(json_object *jobj, void *data, BUFFER *error);

293 int rrd_call_function_error(BUFFER *wb, const char *msg, int code);

294 struct json_object *json_parse_function_payload_or_error(BUFFER *output, BUFFER *payload, int *code, json_parse_function_payload_t cb, void *cb_data);

295

296 +// return HTTP response code

297 +int json_parse_payload_or_error(BUFFER *payload, BUFFER *error, json_parse_function_payload_t cb, void *cb_data);

298 +

299 #endif //NETDATA_JSON_C_PARSER_INLINE_H

src/libnetdata/libjudy/judy-malloc.c

+84 -13

@@ -2,6 +2,9 @@

2

3 #include "judy-malloc.h"

4

5 +// --------------------------------------------------------------------------------------------------------------------

6 +// Judy using ARAL

7 +

8 #define MAX_JUDY_SIZE_TO_ARAL 24

9 static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = {

10 [3] = true,

@@ -15,11 +18,11 @@ static bool judy_sizes_config[MAX_JUDY_SIZE_TO_ARAL + 1] = {

18 [15] = true,

19 [23] = true,

20 };

18 -static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = {};

21 +static ARAL *judy_sizes_aral[MAX_JUDY_SIZE_TO_ARAL + 1] = { 0 };

22

20 -struct aral_statistics judy_sizes_aral_statistics = {};

23 +struct aral_statistics judy_sizes_aral_statistics = { 0 };

24

22 -__attribute__((constructor)) void aral_judy_init(void) {

25 +static void aral_judy_init(void) {

26 for(size_t Words = 0; Words <= MAX_JUDY_SIZE_TO_ARAL; Words++)

27 if(judy_sizes_config[Words]) {

28 char buf[30+1];

@@ -47,12 +50,15 @@ struct aral_statistics *judy_aral_statistics(void) {

50 }

51

52 static ARAL *judy_size_aral(Word_t Words) {

50 - if(Words <= MAX_JUDY_SIZE_TO_ARAL && judy_sizes_aral[Words])

53 + if(Words <= MAX_JUDY_SIZE_TO_ARAL)

54 return judy_sizes_aral[Words];

55

56 return NULL;

57 }

58

59 +// --------------------------------------------------------------------------------------------------------------------

60 +// Judy memory tracking

61 +

62 static __thread int64_t judy_allocated = 0;

63

64 ALWAYS_INLINE void JudyAllocThreadPulseReset(void) {

@@ -65,14 +71,57 @@ ALWAYS_INLINE int64_t JudyAllocThreadPulseGetAndReset(void) {

71 return rc;

72 }

73

68 -inline Word_t JudyMalloc(Word_t Words) {

74 +// --------------------------------------------------------------------------------------------------------------------

75 +// Judy dedicated jemalloc arena

76 +

77 +static unsigned jemalloc_arena_index = 0;

78 +static bool jemalloc_initialized = false;

79 +

80 +#ifdef HAVE_JEMALLOC_ARENA_API

81 +#include <jemalloc/jemalloc.h>

82 +static void jemalloc_init(void) {

83 + // Create shared arena

84 + size_t sz = sizeof(unsigned);

85 + if (mallctl("arenas.create", &jemalloc_arena_index, &sz, NULL, 0) != 0)

86 + return;

87 +

88 + // Disable thread cache for direct arena access

89 + int cache_enabled = 0;

90 + if (mallctl("thread.tcache.enabled", NULL, NULL, &cache_enabled, sizeof(bool)) != 0)

91 + return;

92 +

93 + jemalloc_initialized = true;

94 +}

95 +

96 +static void *jemalloc_malloc(Word_t Words) {

97 + return mallocx(Words * sizeof(Word_t), MALLOCX_ARENA(jemalloc_arena_index));

98 +}

99 +

100 +static void jemalloc_free(void * PWord, Word_t Words __maybe_unused) {

101 + if(PWord)

102 + dallocx(PWord, MALLOCX_ARENA(jemalloc_arena_index));

103 +}

104 +#endif

105 +

106 +// --------------------------------------------------------------------------------------------------------------------

107 +// Judy API

108 +

109 +inline Word_t JudyMalloc(Word_t Words)

110 +{

111 Word_t Addr;

112

71 - ARAL *ar = judy_size_aral(Words);

72 - if(ar)

73 - Addr = (Word_t) aral_mallocz(ar);

113 +#ifdef HAVE_JEMALLOC_ARENA_API

114 + if(jemalloc_initialized)

115 + Addr = (Word_t)jemalloc_malloc(Words);

116 else

75 - Addr = (Word_t) mallocz(Words * sizeof(Word_t));

117 +#endif

118 + {

119 + ARAL *ar = judy_size_aral(Words);

120 + if (ar)

121 + Addr = (Word_t)aral_mallocz(ar);

122 + else

123 + Addr = (Word_t)mallocz(Words * sizeof(Word_t));

124 + }

125

126 judy_allocated += Words * sizeof(Word_t);

127

@@ -80,11 +129,18 @@ inline Word_t JudyMalloc(Word_t Words) {

129 }

130

131 inline void JudyFree(void * PWord, Word_t Words) {

83 - ARAL *ar = judy_size_aral(Words);

84 - if(ar)

85 - aral_freez(ar, PWord);

132 +#ifdef HAVE_JEMALLOC_ARENA_API

133 + if(jemalloc_initialized)

134 + jemalloc_free(PWord, Words);

135 else

87 - freez(PWord);

136 +#endif

137 + {

138 + ARAL *ar = judy_size_aral(Words);

139 + if (ar)

140 + aral_freez(ar, PWord);

141 + else

142 + freez(PWord);

143 + }

144

145 judy_allocated -= Words * sizeof(Word_t);

146 }

@@ -96,3 +152,18 @@ Word_t JudyMallocVirtual(Word_t Words) {

152 void JudyFreeVirtual(void * PWord, Word_t Words) {

153 JudyFree(PWord, Words);

154 }

155 +

156 +// --------------------------------------------------------------------------------------------------------------------

157 +// initialization

158 +

159 +void libjudy_malloc_init(void) {

160 + // IMPORTANT: this is not called on external plugins

161 + // the allocator should run even if this is not called

162 +

163 +#ifdef HAVE_JEMALLOC_ARENA_API

164 + jemalloc_init();

165 + if(!jemalloc_initialized)

166 +#endif

167 + aral_judy_init();

168 +}

169 +

src/libnetdata/libjudy/judy-malloc.h

+2

@@ -12,4 +12,6 @@ struct aral_statistics *judy_aral_statistics(void);

12 void JudyAllocThreadPulseReset(void);

13 int64_t JudyAllocThreadPulseGetAndReset(void);

14

15 +void libjudy_malloc_init(void);

16 +

17 #endif //NETDATA_JUDY_MALLOC_H

src/libnetdata/libnetdata.c

-2

@@ -8,8 +8,6 @@

8

9 struct rlimit rlimit_nofile = { .rlim_cur = 1024, .rlim_max = 1024 };

10

11 -volatile sig_atomic_t netdata_exit = 0;

12 -

11 // --------------------------------------------------------------------------------------------------------------------

12

13 void json_escape_string(char *dst, const char *src, size_t size) {

src/libnetdata/libnetdata.h

+3 -4

@@ -11,6 +11,7 @@ extern "C" {

11 #include "memory/alignment.h"

12 #include "memory/nd-mallocz.h"

13 #include "memory/nd-mmap.h"

14 +#include "libnetdata/exit/exit_initiated.h"

15 #include "log/nd_log-fatal.h"

16 #include "atomics/atomics.h"

17

@@ -39,8 +40,6 @@ char *fgets_trim_len(char *buf, size_t buf_size, FILE *fp, size_t *len);

40

41 int verify_netdata_host_prefix(bool log_msg);

42

42 -extern volatile sig_atomic_t netdata_exit;

43 -

43 char *read_by_filename(const char *filename, long *file_size);

44 char *find_and_replace(const char *src, const char *find, const char *replace, const char *where);

45

@@ -58,9 +57,9 @@ bool run_command_and_copy_output_to_stdout(const char *command, int max_line_len

57 struct web_buffer *run_command_and_get_output_to_buffer(const char *command, int max_line_length);

58

59 #ifdef OS_WINDOWS

61 -void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data);

60 +void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char *action_result, const char *action_data);

61 #else

63 -void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data) NORETURN;

62 +void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char *action_result, const char *action_data) NORETURN;

63 #endif

64

65 extern const char *netdata_configured_host_prefix;

src/libnetdata/log/nd_log-field-formatters.c

+74 -2

@@ -7,7 +7,7 @@ int64_t log_field_to_int64(struct log_field *lf) {

7 // --- FIELD_PARSER_VERSIONS ---

8 //

9 // IMPORTANT:

10 - // THERE ARE 6 VERSIONS OF THIS CODE

10 + // THERE ARE MULTIPLE VERSIONS OF THIS CODE

11 //

12 // 1. journal (direct socket API),

13 // 2. journal (libsystemd API),

@@ -69,7 +69,7 @@ uint64_t log_field_to_uint64(struct log_field *lf) {

69 // --- FIELD_PARSER_VERSIONS ---

70 //

71 // IMPORTANT:

72 - // THERE ARE 6 VERSIONS OF THIS CODE

72 + // THERE ARE MULTIPLE VERSIONS OF THIS CODE

73 //

74 // 1. journal (direct socket API),

75 // 2. journal (libsystemd API),

@@ -125,3 +125,75 @@ uint64_t log_field_to_uint64(struct log_field *lf) {

125

126 return 0;

127 }

128 +

129 +char *log_field_strdupz(struct log_field *lf) {

130 +

131 + // --- FIELD_PARSER_VERSIONS ---

132 + //

133 + // IMPORTANT:

134 + // THERE ARE MULTIPLE VERSIONS OF THIS CODE

135 + //

136 + // 1. journal (direct socket API),

137 + // 2. journal (libsystemd API),

138 + // 3. logfmt,

139 + // 4. json,

140 + // 5. convert to uint64

141 + // 6. convert to int64

142 + //

143 + // UPDATE ALL OF THEM FOR NEW FEATURES OR FIXES

144 +

145 + CLEAN_BUFFER *tmp = NULL;

146 + const char *s = NULL;

147 + char buf[DOUBLE_MAX_LENGTH];

148 +

149 + switch(lf->entry.type) {

150 + default:

151 + case NDFT_UNSET:

152 + return NULL;

153 +

154 + case NDFT_UUID:

155 + uuid_unparse_lower_compact(*lf->entry.uuid, buf);

156 + s = buf;

157 + break;

158 +

159 + case NDFT_TXT:

160 + s = lf->entry.txt;

161 + break;

162 +

163 + case NDFT_STR:

164 + s = string2str(lf->entry.str);

165 + break;

166 +

167 + case NDFT_BFR:

168 + s = buffer_tostring(lf->entry.bfr);

169 + break;

170 +

171 + case NDFT_CALLBACK:

172 + tmp = buffer_create(0, NULL);

173 +

174 + if(lf->entry.cb.formatter(tmp, lf->entry.cb.formatter_data))

175 + s = buffer_tostring(tmp);

176 + else

177 + s = NULL;

178 + break;

179 +

180 + case NDFT_U64:

181 + print_uint64(buf, lf->entry.u64);

182 + s = buf;

183 + break;

184 +

185 + case NDFT_I64:

186 + print_int64(buf, lf->entry.i64);

187 + s = buf;

188 + break;

189 +

190 + case NDFT_DBL:

191 + print_netdata_double(buf, lf->entry.dbl);

192 + break;

193 + }

194 +

195 + if(s && *s)

196 + return strdupz(s);

197 +

198 + return NULL;

199 +}

src/libnetdata/log/nd_log-init.c

+8 -3

@@ -19,6 +19,12 @@ __attribute__((constructor)) void initialize_invocation_id(void) {

19 nd_setenv("NETDATA_INVOCATION_ID", uuid, 1);

20 }

21

22 +ND_UUID nd_log_get_invocation_id(void) {

23 + ND_UUID rc;

24 + uuid_copy(rc.uuid, nd_log.invocation_id);

25 + return rc;

26 +}

27 +

28 // --------------------------------------------------------------------------------------------------------------------

29

30 void nd_log_initialize_for_external_plugins(const char *name) {

@@ -88,8 +94,8 @@ void nd_log_initialize_for_external_plugins(const char *name) {

94

95 switch(method) {

96 case NDLM_JOURNAL:

91 - if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) ||

92 - !nd_log_journal_direct_init(NULL) || !nd_log_journal_systemd_init()) {

97 + if(!nd_log_journal_direct_init(getenv("NETDATA_SYSTEMD_JOURNAL_PATH")) &&

98 + !nd_log_journal_direct_init(NULL) && !nd_log_journal_systemd_init()) {

99 nd_log(NDLS_COLLECTORS, NDLP_WARNING, "Failed to initialize journal. Using stderr.");

100 method = NDLM_STDERR;

101 }

@@ -312,4 +318,3 @@ void nd_log_reopen_log_files_for_spawn_server(const char *name) {

318

319 nd_log_initialize_for_external_plugins(name);

320 }

315 -

src/libnetdata/log/nd_log-internals.h

+4

@@ -125,6 +125,7 @@ struct nd_log {

125 nd_uuid_t invocation_id;

126

127 ND_LOG_SOURCES overwrite_process_source;

128 + log_event_t log_event_cb;

129

130 struct nd_log_source sources[_NDLS_MAX];

131

@@ -148,6 +149,8 @@ struct nd_log {

149 struct {

150 bool etw; // when set use etw, otherwise wel

151 bool initialized;

152 + bool provider_enabled; // track etw provider state

153 + SPINLOCK provider_lock; // Protect etw provider state access

154 } eventlog;

155

156 struct {

@@ -209,6 +212,7 @@ const char *winerror_annotator(struct log_field *lf);

212

213 uint64_t log_field_to_uint64(struct log_field *lf);

214 int64_t log_field_to_int64(struct log_field *lf);

215 +char *log_field_strdupz(struct log_field *lf);

216

217 // --------------------------------------------------------------------------------------------------------------------

218 // common text formatters

src/libnetdata/log/nd_log-libunwind.c

+5 -6

@@ -32,11 +32,11 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

32 unw_getcontext(&context);

33 unw_init_local(&cursor, &context);

34

35 - // Skip first 3 frames (our logging infrastructure)

36 - for (int i = 0; i < 3; i++) {

37 - if (unw_step(&cursor) <= 0)

38 - goto cleanup; // Ensure proper cleanup if unwinding fails early

39 - }

35 +// // Skip first 3 frames (our logging infrastructure)

36 +// for (int i = 0; i < 3; i++) {

37 +// if (unw_step(&cursor) <= 0)

38 +// goto cleanup; // Ensure proper cleanup if unwinding fails early

39 +// }

40

41 while (unw_step(&cursor) > 0) {

42 unw_word_t offset, pc;

@@ -58,7 +58,6 @@ bool stack_trace_formatter(BUFFER *wb, void *data __maybe_unused) {

58 }

59 }

60

61 -cleanup:

61 in_stack_trace = false; // Ensure the flag is reset

62 return true;

63 }

src/libnetdata/log/nd_log-to-windows-events.c

+37 -3

@@ -172,9 +172,27 @@ static void etw_set_source_meta(struct nd_log_source *source, USHORT channelID,

172 source->Keyword = ed->Keyword;

173 }

174

175 +// Callback for provider enable/disable notifications

176 +static void NTAPI ProviderEnableCallback(

177 + LPCGUID SourceId __maybe_unused,

178 + ULONG IsEnabled,

179 + UCHAR Level __maybe_unused,

180 + ULONGLONG MatchAnyKeyword __maybe_unused,

181 + ULONGLONG MatchAllKeyword __maybe_unused,

182 + PEVENT_FILTER_DESCRIPTOR FilterData __maybe_unused,

183 + PVOID CallbackContext __maybe_unused

184 +) {

185 + spinlock_lock(&nd_log.eventlog.provider_lock);

186 + nd_log.eventlog.provider_enabled = IsEnabled ? true : false;

187 + spinlock_unlock(&nd_log.eventlog.provider_lock);

188 +}

189 +

190 static bool etw_register_provider(void) {

191 + spinlock_init(&nd_log.eventlog.provider_lock);

192 + nd_log.eventlog.provider_enabled = false;

193 +

194 // Register the ETW provider

177 - if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, NULL, NULL, &regHandle) != ERROR_SUCCESS)

195 + if (EventRegister(&NETDATA_ETW_PROVIDER_GUID, ProviderEnableCallback, NULL, &regHandle) != ERROR_SUCCESS)

196 return false;

197

198 etw_set_source_meta(&nd_log.sources[NDLS_DAEMON], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);

@@ -185,7 +203,23 @@ static bool etw_register_provider(void) {

203 etw_set_source_meta(&nd_log.sources[NDLS_UNSET], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);

204 etw_set_source_meta(&nd_log.sources[NDLS_DEBUG], CHANNEL_DAEMON, &ED_DAEMON_INFO_MESSAGE_ONLY);

205

188 - return true;

206 + DWORD wait_start = GetTickCount();

207 + while(true) {

208 + spinlock_lock(&nd_log.eventlog.provider_lock);

209 + bool enabled = nd_log.eventlog.provider_enabled;

210 + spinlock_unlock(&nd_log.eventlog.provider_lock);

211 +

212 + if(enabled)

213 + return true;

214 +

215 + // Timeout after 5 seconds

216 + if(GetTickCount() - wait_start > 5000) {

217 + EventUnregister(regHandle);

218 + return false;

219 + }

220 +

221 + Sleep(10); // Short sleep between checks

222 + }

223 }

224 #endif

225

@@ -353,7 +387,7 @@ static const char *get_field_value_unsafe(struct log_field *fields, ND_LOG_FIELD

387 break;

388 case NDFT_UUID:

389 if (!uuid_is_null(*fields[i].entry.uuid)) {

356 - uuid_unparse_lower(*fields[i].entry.uuid, number_str);

390 + uuid_unparse_lower_compact(*fields[i].entry.uuid, number_str);

391 s = number_str;

392 }

393 break;

src/libnetdata/log/nd_log.c

+36 -4

@@ -108,12 +108,41 @@ static ND_LOG_METHOD nd_logger_select_output(ND_LOG_SOURCES source, FILE **fpp,

108 return output;

109 }

110

111 +// --------------------------------------------------------------------------------------------------------------------

112 +

113 +static __thread bool nd_log_event_this = false;

114 +

115 +static void nd_log_event(struct log_field *fields, size_t fields_max __maybe_unused) {

116 + if(!nd_log_event_this)

117 + return;

118 +

119 + nd_log_event_this = false;

120 +

121 + if(!nd_log.log_event_cb)

122 + return;

123 +

124 + const char *filename = log_field_strdupz(&fields[NDF_FILE]);

125 + const char *message = log_field_strdupz(&fields[NDF_MESSAGE]);

126 + const char *function = log_field_strdupz(&fields[NDF_FUNC]);

127 + const char *stack_trace = log_field_strdupz(&fields[NDF_STACK_TRACE]);

128 + long line = log_field_to_int64(&fields[NDF_LINE]);

129 +

130 + nd_log.log_event_cb(filename, function, message, stack_trace, line);

131 +}

132 +

133 +void nd_log_register_event_cb(log_event_t cb) {

134 + nd_log.log_event_cb = cb;

135 +}

136 +

137 // --------------------------------------------------------------------------------------------------------------------

138 // high level logger

139

140 static void nd_logger_log_fields(SPINLOCK *spinlock, FILE *fp, bool limit, ND_LOG_FIELD_PRIORITY priority,

141 ND_LOG_METHOD output, struct nd_log_source *source,

142 struct log_field *fields, size_t fields_max) {

143 +

144 + nd_log_event(fields, fields_max);

145 +

146 if(spinlock)

147 spinlock_lock(spinlock);

148

@@ -437,6 +466,9 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

466 #endif

467 }

468

469 + // send this event to deamon_status_file

470 + nd_log_event_this = true;

471 +

472 int saved_errno = errno;

473 size_t saved_winerror = 0;

474 #if defined(OS_WINDOWS)

@@ -467,7 +499,7 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

499 snprintfz(action_data, 70, "%04lu@%-10.10s:%-15.15s/%d", line, file, function, saved_errno);

500

501 const char *thread_tag = nd_thread_tag();

470 - const char *tag_to_send = thread_tag;

502 + const char *tag_to_send = thread_tag;

503

504 // anonymize thread names

505 if(strncmp(thread_tag, THREAD_TAG_STREAM_RECEIVER, strlen(THREAD_TAG_STREAM_RECEIVER)) == 0)

@@ -475,8 +507,8 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

507 if(strncmp(thread_tag, THREAD_TAG_STREAM_SENDER, strlen(THREAD_TAG_STREAM_SENDER)) == 0)

508 tag_to_send = THREAD_TAG_STREAM_SENDER;

509

478 - char action_result[60+1];

479 - snprintfz(action_result, 60, "%s:%s", program_name, tag_to_send);

510 + char action_result[200+1];

511 + snprintfz(action_result, 60, "%s:%s:%s", program_name, tag_to_send, function);

512

513 #if !defined(ENABLE_SENTRY) && defined(HAVE_BACKTRACE)

514 int fd = nd_log.sources[NDLS_DAEMON].fd;

@@ -495,5 +527,5 @@ void netdata_logger_fatal(const char *file, const char *function, const unsigned

527 abort();

528 #endif

529

498 - netdata_cleanup_and_exit(1, "FATAL", action_result, action_data);

530 + netdata_cleanup_and_exit(EXIT_REASON_FATAL, "FATAL", action_result, action_data);

531 }

src/libnetdata/log/nd_log.h

+4

@@ -31,6 +31,10 @@ ND_LOG_FIELD_ID nd_log_field_id_by_journal_name(const char *field, size_t len);

31 int nd_log_priority2id(const char *priority);

32 const char *nd_log_id2priority(ND_LOG_FIELD_PRIORITY priority);

33 const char *nd_log_method_for_external_plugins(const char *s);

34 +ND_UUID nd_log_get_invocation_id(void);

35 +

36 +typedef void (*log_event_t)(const char *filename, const char *function, const char *message, const char *stack_trace, long line);

37 +void nd_log_register_event_cb(log_event_t cb);

38

39 int nd_log_health_fd(void);

40 int nd_log_collectors_fd(void);

src/libnetdata/log/nd_wevents_manifest.xml deleted

-295

@@ -1,295 +0,0 @@

1 -<?xml version="1.0" encoding="UTF-8"?>

2 -<instrumentationManifest

3 - xmlns="http://schemas.microsoft.com/win/2004/08/events"

4 - xmlns:win="http://manifests.microsoft.com/win/2004/08/windows/events"

5 - xmlns:xs="http://www.w3.org/2001/XMLSchema">

6 - <instrumentation>

7 - <events>

8 -

9 - <provider name="Netdata"

10 - guid="{96c5ca72-9bd8-4634-81e5-000014e7da7a}"

11 - symbol="ND_PROVIDER_NAME"

12 - messageFileName="%SystemRoot%\System32\nd_wevents.dll"

13 - resourceFileName="%SystemRoot%\System32\nd_wevents.dll"

14 - parameterFileName="%SystemRoot%\System32\nd_wevents.dll"

15 - message="$(string.ND_PROVIDER_NAME)">

16 -

17 -

18 - <channels>

19 - <channel name="Netdata/Daemon"

20 - symbol="ND_CHANNEL_DAEMON"

21 - type="Operational"/>

22 -

23 - <channel name="Netdata/Collectors"

24 - symbol="ND_CHANNEL_COLLECTORS"

25 - type="Operational"/>

26 -

27 - <channel name="Netdata/Access"

28 - symbol="ND_CHANNEL_ACCESS"

29 - type="Operational"/>

30 -

31 - <channel symbol="ND_CHANNEL_HEALTH"

32 - name="Netdata/Alerts"

33 - type="Operational"/>

34 -

35 - <channel name="Netdata/ACLK"

36 - symbol="ND_CHANNEL_ACLK"

37 - type="Operational"/>

38 - </channels>

39 -

40 - <levels>

41 - </levels>

42 -

43 - <opcodes>

44 - </opcodes>

45 -

46 - <tasks>

47 - <task name="Daemon" value="1" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Daemon)"/>

48 - <task name="Collector" value="2" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Collector)"/>

49 - <task name="Access" value="3" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Access)"/>

50 - <task name="Health" value="4" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Health)"/>

51 - <task name="Aclk" value="5" eventGUID="{00000000-0000-0000-0000-000000000000}" message="$(string.Task.Aclk)"/>

52 - </tasks>

53 -

54 - <templates>

55 - <template tid="NetdataLogTemplate">

56 -

57 - <data name="Timestamp" inType="win:UnicodeString"/>

58 - <data name="Program" inType="win:UnicodeString"/>

59 - <data name="NetdataLogSource" inType="win:UnicodeString"/>

60 - <data name="Level" inType="win:UnicodeString"/>

61 - <data name="UnixErrno" inType="win:UnicodeString"/>

62 - <data name="WindowsLastError" inType="win:UnicodeString"/>

63 - <data name="InvocationID" inType="win:UnicodeString"/>

64 - <data name="CodeLine" inType="win:UInt32"/>

65 - <data name="CodeFile" inType="win:UnicodeString"/>

66 - <data name="CodeFunction" inType="win:UnicodeString"/>

67 - <data name="ThreadID" inType="win:UInt32"/>

68 - <data name="ThreadName" inType="win:UnicodeString"/>

69 - <data name="MessageID" inType="win:UnicodeString"/>

70 - <data name="Module" inType="win:UnicodeString"/>

71 - <data name="Node" inType="win:UnicodeString"/>

72 - <data name="Instance" inType="win:UnicodeString"/>

73 - <data name="Context" inType="win:UnicodeString"/>

74 - <data name="Dimension" inType="win:UnicodeString"/>

75 - <data name="SourceTransport" inType="win:UnicodeString"/>

76 - <data name="AccountID" inType="win:UnicodeString"/>

77 - <data name="UserName" inType="win:UnicodeString"/>

78 - <data name="UserRole" inType="win:UnicodeString"/>

79 - <data name="UserPermissions" inType="win:UnicodeString"/>

80 - <data name="SourceIP" inType="win:UnicodeString"/>

81 - <data name="SourceForwardedHost" inType="win:UnicodeString"/>

82 - <data name="SourceForwardedFor" inType="win:UnicodeString"/>

83 - <data name="SourcePort" inType="win:UInt32"/>

84 - <data name="SourceCapabilities" inType="win:UnicodeString"/>

85 - <data name="DestinationTransport" inType="win:UnicodeString"/>

86 - <data name="DestinationIP" inType="win:UnicodeString"/>

87 - <data name="DestinationPort" inType="win:UInt32"/>

88 - <data name="DestinationCapabilities" inType="win:UnicodeString"/>

89 - <data name="RequestMethod" inType="win:UnicodeString"/>

90 - <data name="ResponseCode" inType="win:UInt32"/>

91 - <data name="ConnectionID" inType="win:UnicodeString"/>

92 - <data name="TransactionID" inType="win:UnicodeString"/>

93 - <data name="ResponseSentBytes" inType="win:UInt64"/>

94 - <data name="ResponseSizeBytes" inType="win:UInt64"/>

95 - <data name="ResponsePreparationTimeUsec" inType="win:UInt64"/>

96 - <data name="ResponseSentTimeUsec" inType="win:UInt64"/>

97 - <data name="ResponseTotalTimeUsec" inType="win:UInt64"/>

98 - <data name="AlertID" inType="win:UnicodeString"/>

99 - <data name="AlertUniqueID" inType="win:UnicodeString"/>

100 - <data name="AlertTransitionID" inType="win:UnicodeString"/>

101 - <data name="AlertEventID" inType="win:UnicodeString"/>

102 - <data name="AlertConfig" inType="win:UnicodeString"/>

103 - <data name="AlertName" inType="win:UnicodeString"/>

104 - <data name="AlertClass" inType="win:UnicodeString"/>

105 - <data name="AlertComponent" inType="win:UnicodeString"/>

106 - <data name="AlertType" inType="win:UnicodeString"/>

107 - <data name="AlertExec" inType="win:UnicodeString"/>

108 - <data name="AlertRecipient" inType="win:UnicodeString"/>

109 - <data name="AlertDuration" inType="win:UInt64"/>

110 - <data name="AlertValue" inType="win:Double"/>

111 - <data name="AlertOldValue" inType="win:Double"/>

112 - <data name="AlertStatus" inType="win:UnicodeString"/>

113 - <data name="AlertOldStatus" inType="win:UnicodeString"/>

114 - <data name="Source" inType="win:UnicodeString"/>

115 - <data name="AlertUnits" inType="win:UnicodeString"/>

116 - <data name="AlertSummary" inType="win:UnicodeString"/>

117 - <data name="AlertInfo" inType="win:UnicodeString"/>

118 - <data name="AlertNotificationTime" inType="win:UInt64"/>

119 - <data name="Request" inType="win:UnicodeString"/>

120 - <data name="Message" inType="win:UnicodeString"/>

121 - </template>

122 - </templates>

123 -

124 - <events>

125 -

126 - <event symbol="ND_EVENT_DAEMON_INFO"

127 - value="0x1000"

128 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

129 - channel="Netdata/Daemon"

130 - level="win:Informational"

131 - task="Daemon"

132 - opcode="win:Info"

133 - template="NetdataLogTemplate"/>

134 -

135 - <event symbol="ND_EVENT_DAEMON_WARNING"

136 - value="0x1001"

137 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

138 - channel="Netdata/Daemon"

139 - level="win:Warning"

140 - task="Daemon"

141 - opcode="win:Info"

142 - template="NetdataLogTemplate"/>

143 -

144 - <event symbol="ND_EVENT_DAEMON_ERROR"

145 - value="0x1002"

146 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

147 - channel="Netdata/Daemon"

148 - level="win:Error"

149 - task="Daemon"

150 - opcode="win:Info"

151 - template="NetdataLogTemplate"/>

152 -

153 -

154 - <event symbol="ND_EVENT_COLLECTOR_INFO"

155 - value="0x2000"

156 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

157 - channel="Netdata/Collectors"

158 - level="win:Informational"

159 - task="Collector"

160 - opcode="win:Info"

161 - template="NetdataLogTemplate"/>

162 -

163 - <event symbol="ND_EVENT_COLLECTOR_WARNING"

164 - value="0x2001"

165 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

166 - channel="Netdata/Collectors"

167 - level="win:Warning"

168 - task="Collector"

169 - opcode="win:Info"

170 - template="NetdataLogTemplate"/>

171 -

172 - <event symbol="ND_EVENT_COLLECTOR_ERROR"

173 - value="0x2002"

174 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

175 - channel="Netdata/Collectors"

176 - level="win:Error"

177 - task="Collector"

178 - opcode="win:Info"

179 - template="NetdataLogTemplate"/>

180 -

181 -

182 - <event symbol="ND_EVENT_ACCESS_INFO"

183 - value="0x3000"

184 - message="$(string.ND_ACCESS_EVENT_MESSAGE)"

185 - channel="Netdata/Access"

186 - level="win:Informational"

187 - task="Access"

188 - opcode="win:Info"

189 - template="NetdataLogTemplate"/>

190 -

191 - <event symbol="ND_EVENT_ACCESS_WARNING"

192 - value="0x3001"

193 - message="$(string.ND_ACCESS_EVENT_MESSAGE)"

194 - channel="Netdata/Access"

195 - level="win:Warning"

196 - task="Access"

197 - opcode="win:Info"

198 - template="NetdataLogTemplate"/>

199 -

200 - <event symbol="ND_EVENT_ACCESS_ERROR"

201 - value="0x3002"

202 - message="$(string.ND_ACCESS_EVENT_MESSAGE)"

203 - channel="Netdata/Access"

204 - level="win:Error"

205 - task="Access"

206 - opcode="win:Info"

207 - template="NetdataLogTemplate"/>

208 -

209 -

210 - <event symbol="ND_EVENT_HEALTH_INFO"

211 - value="0x4000"

212 - message="$(string.ND_HEALTH_EVENT_MESSAGE)"

213 - channel="Netdata/Alerts"

214 - level="win:Informational"

215 - task="Health"

216 - opcode="win:Info"

217 - template="NetdataLogTemplate"/>

218 -

219 - <event symbol="ND_EVENT_HEALTH_WARNING"

220 - value="0x4001"

221 - message="$(string.ND_HEALTH_EVENT_MESSAGE)"

222 - channel="Netdata/Alerts"

223 - level="win:Warning"

224 - task="Health"

225 - opcode="win:Info"

226 - template="NetdataLogTemplate"/>

227 -

228 - <event symbol="ND_EVENT_HEALTH_ERROR"

229 - value="0x4002"

230 - message="$(string.ND_HEALTH_EVENT_MESSAGE)"

231 - channel="Netdata/Alerts"

232 - level="win:Error"

233 - task="Health"

234 - opcode="win:Info"

235 - template="NetdataLogTemplate"/>

236 -

237 -

238 - <event symbol="ND_EVENT_ACLK_INFO"

239 - value="0x5000"

240 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

241 - channel="Netdata/ACLK"

242 - level="win:Informational"

243 - task="Aclk"

244 - opcode="win:Info"

245 - template="NetdataLogTemplate"/>

246 -

247 - <event symbol="ND_EVENT_ACLK_WARNING"

248 - value="0x5001"

249 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

250 - channel="Netdata/ACLK"

251 - level="win:Warning"

252 - task="Aclk"

253 - opcode="win:Info"

254 - template="NetdataLogTemplate"/>

255 -

256 - <event symbol="ND_EVENT_ACLK_ERROR"

257 - value="0x5002"

258 - message="$(string.ND_GENERIC_LOG_MESSAGE)"

259 - channel="Netdata/ACLK"

260 - level="win:Error"

261 - task="Aclk"

262 - opcode="win:Info"

263 - template="NetdataLogTemplate"/>

264 -

265 - </events>

266 - </provider>

267 - </events>

268 - </instrumentation>

269 -

270 - <localization>

271 - <resources culture="en-US">

272 - <stringTable>

273 - <string id="Task.Daemon" value="ND Daemon Log"/>

274 - <string id="Task.Collector" value="ND Collector Log"/>

275 - <string id="Task.Access" value="ND Access Log"/>

276 - <string id="Task.Health" value="ND Health Log"/>

277 - <string id="Task.Aclk" value="ND ACLK Log"/>

278 -

279 - <string id="ND_PROVIDER_NAME" value="Netdata"/>

280 - <string id="ND_GENERIC_LOG_MESSAGE" value="%64"/>

281 - <string id="ND_ACCESS_EVENT_MESSAGE"

282 - value="Transaction %36, method: %33, path: %63

283 -

284 - Source IP : %24, Forwarded-For: %27

285 - User : %21, role: %22, permissions: %23

286 - Timings (usec): prep %39, sent %40, total %41

287 - Response Size : sent %37, uncompressed %38

288 - Response Code : %34

289 -"/>

290 - <string id="ND_HEALTH_EVENT_MESSAGE"

291 - value="Alert '%47' of instance '%16' on node '%15', transitioned from %57 to %56"/>

292 - </stringTable>

293 - </resources>

294 - </localization>

295 -</instrumentationManifest>

src/libnetdata/log/systemd-cat-native.c

+3 -3

@@ -545,8 +545,8 @@ static int help(void) {

545 " The parameter --newline=STRING allows setting the string to be replaced\n"

546 " with newlines.\n"

547 "\n"

548 - " For example by setting --newline='--NEWLINE--', the program will replace\n"

549 - " all occurrences of --NEWLINE-- with the newline character, within each\n"

548 + " With the default setting of --newline='\\n', the program will replace\n"

549 + " all occurrences of \\n with the newline character, within each\n"

550 " VALUE of the KEY=VALUE lines. Once this this done, the program will\n"

551 " switch the field to the binary Journal Export Format before sending the\n"

552 " log event to systemd-journal.\n"

@@ -741,7 +741,7 @@ int main(int argc, char *argv[]) {

741

742 int timeout_ms = 0; // wait forever

743 bool log_as_netdata = false;

744 - const char *newline = NULL;

744 + const char *newline = "\\n";

745 const char *namespace = NULL;

746 const char *socket = getenv("NETDATA_SYSTEMD_JOURNAL_PATH");

747 #ifdef HAVE_LIBCURL

src/libnetdata/log/systemd-cat-native.md

+12 -13

@@ -34,19 +34,7 @@ printf "MESSAGE=hey, this is error\nPRIORITY=3\n\n" | systemd-cat-native

34 The result:

35 ![image](https://github.com/netdata/netdata/assets/2662304/faf3eaa5-ac56-415b-9de8-16e6ceed9280)

36

37 -Sending multi-line log entries (in this example we replace the text `--NEWLINE--` with a newline in the log entry):

38 -

39 -```bash

40 -printf "MESSAGE=hello--NEWLINE--world\nPRIORITY=6\n\n" | systemd-cat-native --newline='--NEWLINE--'

41 -```

42 -

43 -The result:

44 -

45 -![image](https://github.com/netdata/netdata/assets/2662304/d6037b4a-87da-4693-ae67-e07df0decdd9)

46 -

47 -

48 -Processing the standard `\n` string can be tricky due to shell escaping. This works, but note that

49 -we have to add a lot of backslashes to printf.

37 +The program supports multi-line processing for all fields. The default newline sequence is `\n`.

38

39 ```bash

40 printf "MESSAGE=hello\\\\nworld\nPRIORITY=6\n\n" | systemd-cat-native --newline='\n'

@@ -61,6 +49,17 @@ PRIORITY=6

49

50 ```

51

52 +It also allows changing the newline sequence. In this example we replace the text `--NEWLINE--` with a newline in the log entry:

53 +

54 +```bash

55 +printf "MESSAGE=hello--NEWLINE--world\nPRIORITY=6\n\n" | systemd-cat-native --newline='--NEWLINE--'

56 +```

57 +

58 +The result:

59 +

60 +![image](https://github.com/netdata/netdata/assets/2662304/d6037b4a-87da-4693-ae67-e07df0decdd9)

61 +

62 +

63 ## Best practices

64

65 These are the rules about fields, enforced by `systemd-journald`:

src/libnetdata/log/systemd-journal-helpers.c

+1

@@ -1,6 +1,7 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 #include "systemd-journal-helpers.h"

4 +#include "../libnetdata.h"

5

6 bool is_path_unix_socket(const char *path) {

7 // Check if the path is valid

src/libnetdata/log/systemd-journal-helpers.h

+1 -1

@@ -1,6 +1,6 @@

1 // SPDX-License-Identifier: GPL-3.0-or-later

2

3 -#include "../libnetdata.h"

3 +#include "../common.h"

4

5 #ifndef NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H

6 #define NETDATA_LOG_SYSTEMD_JOURNAL_HELPERS_H

src/libnetdata/log/wevt_netdata_install.bat

+7

@@ -47,6 +47,13 @@ if %errorlevel% neq 0 (

47 exit /b 1

48 )

49

50 +echo.

51 +echo Setting default event sizes...

52 +wevtutil sl "Netdata/Daemon" /ms:104857600

53 +wevtutil sl "Netdata/Collectors" /ms:104857600

54 +wevtutil sl "Netdata/Health" /ms:104857600

55 +wevtutil sl "Netdata/Access" /ms:104857600

56 +

57 echo.

58 echo Netdata Event Tracing for Windows manifest installed successfully.

59 exit /b 0

src/libnetdata/memory/nd-mallocz.c

+18 -2

@@ -387,6 +387,8 @@ void freez_int(void *ptr, const char *file, const char *function, size_t line) {

387 #else

388

389 ALWAYS_INLINE char *strdupz(const char *s) {

390 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRDUP);

391 +

392 char *t = strdup(s);

393 if (unlikely(!t)) {

394 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -396,6 +398,8 @@ ALWAYS_INLINE char *strdupz(const char *s) {

398 }

399

400 ALWAYS_INLINE char *strndupz(const char *s, size_t len) {

401 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_STRNDUP);

402 +

403 char *t = strndup(s, len);

404 if (unlikely(!t)) {

405 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -406,10 +410,14 @@ ALWAYS_INLINE char *strndupz(const char *s, size_t len) {

410

411 // If ptr is NULL, no operation is performed.

412 ALWAYS_INLINE void freez(void *ptr) {

409 - if(likely(ptr)) free(ptr);

413 + if(likely(ptr)) {

414 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

415 + free(ptr);

416 + }

417 }

418

419 ALWAYS_INLINE void *mallocz(size_t size) {

420 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_MALLOC);

421 void *p = malloc(size);

422 if (unlikely(!p)) {

423 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -419,6 +427,7 @@ ALWAYS_INLINE void *mallocz(size_t size) {

427 }

428

429 ALWAYS_INLINE void *callocz(size_t nmemb, size_t size) {

430 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_CALLOC);

431 void *p = calloc(nmemb, size);

432 if (unlikely(!p)) {

433 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -428,6 +437,7 @@ ALWAYS_INLINE void *callocz(size_t nmemb, size_t size) {

437 }

438

439 ALWAYS_INLINE void *reallocz(void *ptr, size_t size) {

440 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_REALLOC);

441 void *p = realloc(ptr, size);

442 if (unlikely(!p)) {

443 OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();

@@ -436,7 +446,13 @@ ALWAYS_INLINE void *reallocz(void *ptr, size_t size) {

446 return p;

447 }

448

439 -ALWAYS_INLINE void posix_memfree(void *ptr) {

449 +ALWAYS_INLINE int posix_memalignz(void **memptr, size_t alignment, size_t size) {

450 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN);

451 + return posix_memalign(memptr, alignment, size);

452 +}

453 +

454 +ALWAYS_INLINE void posix_memalign_freez(void *ptr) {

455 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN_FREE);

456 free(ptr);

457 }

458 #endif

src/libnetdata/memory/nd-mallocz.h

+3 -1

@@ -60,6 +60,8 @@ void freez(void *ptr);

60 #endif // NETDATA_TRACE_ALLOCATIONS

61

62 void mallocz_release_as_much_memory_to_the_system(void);

63 -void posix_memfree(void *ptr);

63 +

64 +int posix_memalignz(void **memptr, size_t alignment, size_t size);

65 +void posix_memalign_freez(void *ptr);

66

67 #endif //NETDATA_ND_MALLOCZ_H

src/libnetdata/memory/nd-mmap.c

+4 -1

@@ -115,7 +115,7 @@ inline int madvise_mergeable(void *mem __maybe_unused, size_t len __maybe_unused

115 #define THP_SIZE (2 * 1024 * 1024) // 2 MiB THP size

116 #define THP_MASK (THP_SIZE - 1) // Mask for alignment check

117

118 -inline int madvise_thp(void *mem, size_t len) {

118 +inline int madvise_thp(void *mem __maybe_unused, size_t len __maybe_unused) {

119 #ifdef MADV_HUGEPAGE

120 // Check if the size is at least THP size and aligned

121 if (len >= THP_SIZE && ((uintptr_t)mem & THP_MASK) == 0) {

@@ -130,6 +130,7 @@ int nd_munmap(void *ptr, size_t size) {

130 malloc_trace_munmap(size);

131 #endif

132

133 + workers_memory_call(WORKERS_MEMORY_CALL_MUNMAP);

134 int rc = munmap(ptr, size);

135

136 if(rc == 0) {

@@ -141,6 +142,8 @@ int nd_munmap(void *ptr, size_t size) {

142 }

143

144 void *nd_mmap(void *addr, size_t len, int prot, int flags, int fd, off_t offset) {

145 + workers_memory_call(WORKERS_MEMORY_CALL_MMAP);

146 +

147 void *rc = mmap(addr, len, prot, flags, fd, offset);

148

149 if(rc != MAP_FAILED) {

src/libnetdata/os/boot_id.c new

+78

@@ -0,0 +1,78 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "boot_id.h"

4 +#include "libnetdata/libnetdata.h"

5 +

6 +static ND_UUID cached_boot_id = { 0 };

7 +static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

8 +

9 +#if defined(OS_LINUX)

10 +

11 +static ND_UUID get_boot_id(void) {

12 + ND_UUID boot_id = { 0 };

13 + char buf[UUID_STR_LEN];

14 +

15 + char filename[FILENAME_MAX + 1];

16 + snprintfz(filename, sizeof(filename), "%s/proc/sys/kernel/random/boot_id",

17 + netdata_configured_host_prefix ? netdata_configured_host_prefix : "");

18 +

19 + // Try reading the official boot_id first

20 + if (read_txt_file(filename, buf, sizeof(buf)) == 0) {

21 + if (uuid_parse(trim(buf), boot_id.uuid) == 0)

22 + return boot_id;

23 + }

24 +

25 + // Fallback to boottime-based ID

26 + time_t boottime = os_boottime();

27 + if(boottime > 0) {

28 + boot_id.parts.low64 = (uint64_t)boottime;

29 + // parts.hig64 remains 0 to indicate this is a synthetic boot_id

30 + }

31 +

32 + return boot_id;

33 +}

34 +

35 +#else // !OS_LINUX

36 +

37 +static ND_UUID get_boot_id(void) {

38 + ND_UUID boot_id = { 0 };

39 +

40 + time_t boottime = os_boottime();

41 + if(boottime > 0) {

42 + boot_id.parts.low64 = (uint64_t)boottime;

43 + // parts.hig64 remains 0 to indicate this is a synthetic boot_id

44 + }

45 +

46 + return boot_id;

47 +}

48 +

49 +#endif // OS_LINUX

50 +

51 +ND_UUID os_boot_id(void) {

52 + // Fast path - return cached value if available

53 + if(!UUIDiszero(cached_boot_id))

54 + return cached_boot_id;

55 +

56 + spinlock_lock(&spinlock);

57 +

58 + // Check again under lock in case another thread set it

59 + if(UUIDiszero(cached_boot_id)) {

60 + cached_boot_id = get_boot_id();

61 + }

62 +

63 + spinlock_unlock(&spinlock);

64 + return cached_boot_id;

65 +}

66 +

67 +bool os_boot_ids_match(ND_UUID a, ND_UUID b) {

68 + if(UUIDeq(a, b))

69 + return true;

70 +

71 + if(a.parts.hig64 == 0 && b.parts.hig64 == 0) {

72 + uint64_t diff = a.parts.low64 > b.parts.low64 ? a.parts.low64 - b.parts.low64 : b.parts.low64 - a.parts.low64;

73 + if(diff <= 3)

74 + return true;

75 + }

76 +

77 + return false;

78 +}

src/libnetdata/os/boot_id.h new

+25

@@ -0,0 +1,25 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_OS_BOOT_ID_H

4 +#define NETDATA_OS_BOOT_ID_H

5 +

6 +#include "libnetdata/common.h"

7 +#include "libnetdata/uuid/uuid.h"

8 +

9 +/**

10 + * Get system boot ID

11 + *

12 + * Returns a UUID that remains constant during system uptime.

13 + * On Linux, this is the systemd boot_id.

14 + * On other systems, this uses the system boot time to generate a unique ID.

15 + *

16 + * The value is cached after first call.

17 + * Returns UUID_ZERO on error.

18 + *

19 + * @return ND_UUID The boot ID

20 + */

21 +ND_UUID os_boot_id(void);

22 +

23 +bool os_boot_ids_match(ND_UUID a, ND_UUID b);

24 +

25 +#endif

src/libnetdata/os/boottime.c new

+125

@@ -0,0 +1,125 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "libnetdata/libnetdata.h"

4 +

5 +static time_t cached_boottime = 0;

6 +static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

7 +

8 +#if defined(OS_LINUX)

9 +

10 +static time_t calculate_boottime(void) {

11 + char buf[8192];

12 +

13 + char filename[FILENAME_MAX + 1];

14 +

15 + // Try to read from /proc/stat first - this provides the absolute timestamp

16 + snprintfz(filename, sizeof(filename), "%s/proc/stat",

17 + netdata_configured_host_prefix ? netdata_configured_host_prefix : "");

18 + if (read_txt_file(filename, buf, sizeof(buf)) == 0) {

19 + char *btime_line = strstr(buf, "btime ");

20 + if (btime_line) {

21 + time_t btime = (time_t)str2ull(btime_line + 6, NULL);

22 + if (btime > 0)

23 + return btime;

24 + }

25 + }

26 +

27 + // If btime is not available, calculate it from uptime

28 + snprintfz(filename, sizeof(filename), "%s/proc/uptime",

29 + netdata_configured_host_prefix ? netdata_configured_host_prefix : "");

30 + if (read_txt_file(filename, buf, sizeof(buf)) == 0) {

31 + double uptime;

32 + if (sscanf(buf, "%lf", &uptime) == 1) {

33 + time_t now = now_realtime_sec();

34 + time_t boottime = now - (time_t)uptime;

35 + if(boottime > 0)

36 + return boottime;

37 + }

38 + }

39 +

40 + return 0;

41 +}

42 +

43 +#elif defined(OS_FREEBSD) || defined(OS_MACOS)

44 +

45 +#include <sys/sysctl.h>

46 +

47 +static time_t calculate_boottime(void) {

48 + struct timeval boottime;

49 + size_t size = sizeof(boottime);

50 +

51 + // kern.boottime provides the absolute timestamp

52 + if (sysctlbyname("kern.boottime", &boottime, &size, NULL, 0) == 0)

53 + return boottime.tv_sec;

54 +

55 + return 0;

56 +}

57 +

58 +#elif defined(OS_WINDOWS)

59 +

60 +#include <windows.h>

61 +

62 +static time_t calculate_boottime(void) {

63 + ULONGLONG uptime_ms = GetTickCount64();

64 + if (uptime_ms > 0) {

65 + FILETIME ft;

66 + ULARGE_INTEGER now;

67 +

68 + GetSystemTimeAsFileTime(&ft);

69 + now.HighPart = ft.dwHighDateTime;

70 + now.LowPart = ft.dwLowDateTime;

71 +

72 + // Convert to Unix epoch (subtract Windows epoch)

73 + ULONGLONG unix_time_ms = (now.QuadPart - 116444736000000000ULL) / 10000;

74 + time_t boottime = (time_t)((unix_time_ms - uptime_ms) / 1000);

75 +

76 + if(boottime > 0)

77 + return boottime;

78 + }

79 +

80 + return 0;

81 +}

82 +

83 +#endif

84 +

85 +static time_t get_stable_boottime(void) {

86 + const int max_attempts = 100;

87 + const int required_matches = 5;

88 + time_t last_boottime = 0;

89 + int matches = 0;

90 +

91 + for(int i = 0; i < max_attempts; i++) {

92 + time_t new_boottime = calculate_boottime();

93 + if(new_boottime == 0)

94 + new_boottime = now_realtime_sec() - now_boottime_sec();

95 +

96 + if(new_boottime == last_boottime)

97 + matches++;

98 + else {

99 + matches = 1;

100 + last_boottime = new_boottime;

101 + }

102 +

103 + if(matches >= required_matches)

104 + return new_boottime;

105 +

106 + microsleep(1000); // 1ms

107 + }

108 +

109 + return 0;

110 +}

111 +

112 +time_t os_boottime(void) {

113 + // Fast path - return cached value if available

114 + if(cached_boottime > 0)

115 + return cached_boottime;

116 +

117 + spinlock_lock(&spinlock);

118 +

119 + // Check again under lock in case another thread set it

120 + if(cached_boottime == 0)

121 + cached_boottime = get_stable_boottime();

122 +

123 + spinlock_unlock(&spinlock);

124 + return cached_boottime;

125 +}

src/libnetdata/os/boottime.h new

+19

@@ -0,0 +1,19 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_BOOTTIME_H

4 +#define NETDATA_BOOTTIME_H

5 +

6 +#include "libnetdata/common.h"

7 +

8 +/**

9 + * Get system boot time

10 + *

11 + * Returns the absolute wallclock timestamp (Unix epoch) of when the system was last booted.

12 + * The value is cached after first successful call.

13 + * Returns 0 on error.

14 + *

15 + * @return time_t The boot timestamp, 0 on error

16 + */

17 +time_t os_boottime(void);

18 +

19 +#endif //NETDATA_BOOTTIME_H

src/libnetdata/os/disk_space.c new

+92

@@ -0,0 +1,92 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "libnetdata/libnetdata.h"

4 +

5 +#if defined(OS_LINUX)

6 +#include <sys/statvfs.h>

7 +

8 +OS_SYSTEM_DISK_SPACE os_disk_space(const char *path) {

9 + OS_SYSTEM_DISK_SPACE space = OS_SYSTEM_DISK_SPACE_EMPTY;

10 + struct statvfs buf;

11 +

12 + if (statvfs(path, &buf) != 0) {

13 + // Error occurred; errno is set

14 + return space;

15 + }

16 +

17 + // Use f_frsize (fragment size) for accurate byte calculations.

18 + space.total_bytes = buf.f_blocks * buf.f_frsize;

19 + space.free_bytes = buf.f_bavail * buf.f_frsize;

20 + space.total_inodes = buf.f_files;

21 + space.free_inodes = buf.f_favail;

22 + space.is_read_only = (buf.f_flag & ST_RDONLY) != 0;

23 + return space;

24 +}

25 +#endif

26 +

27 +#if defined(OS_FREEBSD) || defined(OS_MACOS)

28 +#include <sys/param.h>

29 +#include <sys/mount.h>

30 +

31 +OS_SYSTEM_DISK_SPACE os_disk_space(const char *path) {

32 + OS_SYSTEM_DISK_SPACE space = OS_SYSTEM_DISK_SPACE_EMPTY;

33 + struct statfs buf;

34 +

35 + if (statfs(path, &buf) != 0) {

36 + // Error occurred; errno is set

37 + return space;

38 + }

39 +

40 + space.total_bytes = buf.f_blocks * buf.f_bsize;

41 + space.free_bytes = buf.f_bavail * buf.f_bsize;

42 + space.total_inodes = buf.f_files;

43 + space.free_inodes = buf.f_ffree;

44 + space.is_read_only = (buf.f_flags & MNT_RDONLY) != 0;

45 + return space;

46 +}

47 +#endif

48 +

49 +#if defined(OS_WINDOWS)

50 +#include <windows.h>

51 +

52 +OS_SYSTEM_DISK_SPACE os_disk_space(const char *path_utf8) {

53 + OS_SYSTEM_DISK_SPACE space = OS_SYSTEM_DISK_SPACE_EMPTY;

54 +

55 + // Convert the UTF-8 path to a wide-character string.

56 + int wlen = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, NULL, 0);

57 + if (wlen == 0) {

58 + // Conversion error; optionally, GetLastError() can provide more details.

59 + return space;

60 + }

61 +

62 + wchar_t *wpath = (wchar_t *)mallocz(wlen * sizeof(wchar_t));

63 +

64 + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path_utf8, -1, wpath, wlen) == 0) {

65 + // Conversion error.

66 + freez(wpath);

67 + return space;

68 + }

69 +

70 + // Use the wide-character version of GetDiskFreeSpaceEx.

71 + ULARGE_INTEGER freeBytesAvailable, totalNumberOfBytes, totalNumberOfFreeBytes;

72 + if (!GetDiskFreeSpaceExW(wpath, &freeBytesAvailable, &totalNumberOfBytes, &totalNumberOfFreeBytes)) {

73 + // API call failed; optionally, GetLastError() can provide more details.

74 + freez(wpath);

75 + return space;

76 + }

77 +

78 + // Get the drive type and attributes

79 + DWORD attributes = GetFileAttributesW(wpath);

80 + if (attributes != INVALID_FILE_ATTRIBUTES) {

81 + space.is_read_only = (attributes & FILE_ATTRIBUTE_READONLY) != 0;

82 + }

83 +

84 + freez(wpath);

85 +

86 + space.total_bytes = totalNumberOfBytes.QuadPart;

87 + space.free_bytes = totalNumberOfFreeBytes.QuadPart;

88 + space.total_inodes = 0; // Windows does not have inodes

89 + space.free_inodes = 0; // Windows does not have inodes

90 + return space;

91 +}

92 +#endif

src/libnetdata/os/disk_space.h new

+21

@@ -0,0 +1,21 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_DISK_SPACE_H

4 +#define NETDATA_DISK_SPACE_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +typedef struct {

9 + uint64_t total_bytes; // Total disk size in bytes

10 + uint64_t free_bytes; // Available disk space in bytes

11 + uint64_t total_inodes; // Total number of inodes

12 + uint64_t free_inodes; // Available inodes

13 + bool is_read_only; // True if filesystem is read-only

14 +} OS_SYSTEM_DISK_SPACE;

15 +

16 +#define OS_SYSTEM_DISK_SPACE_OK(space) ((space).total_bytes > 0)

17 +#define OS_SYSTEM_DISK_SPACE_EMPTY (OS_SYSTEM_DISK_SPACE){ 0 }

18 +

19 +OS_SYSTEM_DISK_SPACE os_disk_space(const char *path);

20 +

21 +#endif //NETDATA_DISK_SPACE_H

src/libnetdata/os/file_lock.c new

+112

@@ -0,0 +1,112 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "file_lock.h"

4 +

5 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

6 +#include <sys/file.h>

7 +#include <fcntl.h>

8 +#include <unistd.h>

9 +#endif

10 +

11 +#if defined(OS_WINDOWS)

12 +#include <windows.h>

13 +#endif

14 +

15 +FILE_LOCK file_lock_get(const char *filename) {

16 + if(!filename || !*filename)

17 + return FILE_LOCK_INVALID;

18 +

19 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

20 + // Try to create a new file, or open existing one

21 + int fd = open(filename, O_RDWR | O_CREAT, 0666);

22 + if(fd == -1)

23 + return FILE_LOCK_INVALID;

24 +

25 + // LOCK_NB makes flock() non-blocking

26 + if(flock(fd, LOCK_EX | LOCK_NB) == -1) {

27 + close(fd);

28 + return FILE_LOCK_INVALID;

29 + }

30 +

31 + return (FILE_LOCK){ .fd = fd };

32 +

33 +#elif defined(OS_WINDOWS)

34 + // Convert MSYS2/Cygwin path directly to Windows wide-char path

35 + ssize_t wpath_size = cygwin_conv_path(CCP_POSIX_TO_WIN_W, filename, NULL, 0);

36 + if(wpath_size < 0)

37 + return FILE_LOCK_INVALID;

38 +

39 + wchar_t *wpath = mallocz(wpath_size);

40 + if(!wpath)

41 + return FILE_LOCK_INVALID;

42 +

43 + if(cygwin_conv_path(CCP_POSIX_TO_WIN_W, filename, wpath, wpath_size) != 0) {

44 + freez(wpath);

45 + return FILE_LOCK_INVALID;

46 + }

47 +

48 + // Open existing file or create new one

49 + HANDLE hFile = CreateFileW(

50 + wpath,

51 + GENERIC_READ | GENERIC_WRITE,

52 + FILE_SHARE_READ | FILE_SHARE_WRITE,

53 + NULL,

54 + OPEN_ALWAYS, // Open if exists, create if doesn't

55 + FILE_ATTRIBUTE_NORMAL,

56 + NULL

57 + );

58 +

59 + freez(wpath);

60 +

61 + if(hFile == INVALID_HANDLE_VALUE)

62 + return FILE_LOCK_INVALID;

63 +

64 + // Check if file is empty

65 + LARGE_INTEGER size;

66 + if(!GetFileSizeEx(hFile, &size)) {

67 + CloseHandle(hFile);

68 + return FILE_LOCK_INVALID;

69 + }

70 +

71 + // Write a byte only if file is empty

72 + if(size.QuadPart == 0) {

73 + DWORD written;

74 + if(!WriteFile(hFile, "!", 1, &written, NULL) || written != 1) {

75 + CloseHandle(hFile);

76 + return FILE_LOCK_INVALID;

77 + }

78 + }

79 +

80 + // Try to lock the entire file

81 + OVERLAPPED overlapped = {0};

82 + if(!LockFileEx(

83 + hFile,

84 + LOCKFILE_EXCLUSIVE_LOCK | LOCKFILE_FAIL_IMMEDIATELY,

85 + 0,

86 + MAXDWORD,

87 + MAXDWORD,

88 + &overlapped)) {

89 + CloseHandle(hFile);

90 + return FILE_LOCK_INVALID;

91 + }

92 +

93 + return (FILE_LOCK){ .handle = hFile };

94 +

95 +#else

96 +#error "Unsupported operating system"

97 +#endif

98 +}

99 +

100 +void file_lock_release(FILE_LOCK lock) {

101 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

102 + if(FILE_LOCK_OK(lock)) {

103 + // flock is automatically released when file is closed

104 + close(lock.fd);

105 + }

106 +#elif defined(OS_WINDOWS)

107 + if(FILE_LOCK_OK(lock)) {

108 + // File lock is automatically released when handle is closed

109 + CloseHandle(lock.handle);

110 + }

111 +#endif

112 +}

src/libnetdata/os/file_lock.h new

+49

@@ -0,0 +1,49 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_FILE_LOCK_H

4 +#define NETDATA_FILE_LOCK_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +typedef struct file_lock {

9 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

10 + int fd;

11 +#elif defined(OS_WINDOWS)

12 + HANDLE handle;

13 +#else

14 +#error "Unsupported operating system"

15 +#endif

16 +} FILE_LOCK;

17 +

18 +// Initialize to invalid values

19 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

20 +#define FILE_LOCK_INVALID ((FILE_LOCK){ .fd = -1 })

21 +#define FILE_LOCK_OK(lock) ((lock).fd != -1)

22 +#elif defined(OS_WINDOWS)

23 +#define FILE_LOCK_INVALID ((FILE_LOCK){ .handle = INVALID_HANDLE_VALUE })

24 +#define FILE_LOCK_OK(lock) ((lock).handle != INVALID_HANDLE_VALUE)

25 +#endif

26 +

27 +/**

28 + * Get a file lock

29 + *

30 + * Attempts to acquire an exclusive lock on a file. The lock is automatically released

31 + * when the process exits or if the process crashes. Only one process can hold the lock

32 + * at a time.

33 + *

34 + * @param filename UTF-8 encoded filename (MSYS2 path format on Windows)

35 + * @return FILE_LOCK The lock handle. Use FILE_LOCK_OK() to check if lock was acquired

36 + */

37 +FILE_LOCK file_lock_get(const char *filename);

38 +

39 +/**

40 + * Release a file lock

41 + *

42 + * Releases a previously acquired file lock. After calling this function,

43 + * another process may acquire the lock.

44 + *

45 + * @param lock The lock to release

46 + */

47 +void file_lock_release(FILE_LOCK lock);

48 +

49 +#endif //NETDATA_FILE_LOCK_H

src/libnetdata/os/file_metadata.c new

+72

@@ -0,0 +1,72 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "libnetdata/libnetdata.h"

4 +#include <errno.h>

5 +

6 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

7 +#include <sys/stat.h>

8 +

9 +OS_FILE_METADATA os_get_file_metadata(const char *path) {

10 + OS_FILE_METADATA metadata = {0};

11 + struct stat st;

12 +

13 + if (stat(path, &st) != 0)

14 + return metadata;

15 +

16 + metadata.size_bytes = st.st_size;

17 + metadata.modified_time = st.st_mtime;

18 + return metadata;

19 +}

20 +#endif

21 +

22 +#if defined(OS_WINDOWS)

23 +#include <windows.h>

24 +

25 +OS_FILE_METADATA os_get_file_metadata(const char *path) {

26 + OS_FILE_METADATA metadata = {0};

27 +

28 + // Convert UTF-8 path to wide-character string

29 + int wlen = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, -1, NULL, 0);

30 + if (wlen == 0)

31 + return metadata;

32 +

33 + wchar_t *wpath = (wchar_t *)mallocz(wlen * sizeof(wchar_t));

34 + if (!wpath)

35 + return metadata;

36 +

37 + if (MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, path, -1, wpath, wlen) == 0) {

38 + freez(wpath);

39 + return metadata;

40 + }

41 +

42 + WIN32_FILE_ATTRIBUTE_DATA attr_data;

43 + if (!GetFileAttributesExW(wpath, GetFileExInfoStandard, &attr_data)) {

44 + freez(wpath);

45 + return metadata;

46 + }

47 +

48 + freez(wpath);

49 +

50 + // Combine high and low parts for 64-bit file size

51 + ULARGE_INTEGER file_size;

52 + file_size.HighPart = attr_data.nFileSizeHigh;

53 + file_size.LowPart = attr_data.nFileSizeLow;

54 + metadata.size_bytes = file_size.QuadPart;

55 +

56 + // Convert Windows FILETIME to Unix timestamp

57 + // Windows FILETIME is in 100-nanosecond intervals since January 1, 1601 UTC

58 + // Need to convert to seconds since January 1, 1970 UTC

59 + ULARGE_INTEGER win_time;

60 + win_time.HighPart = attr_data.ftLastWriteTime.dwHighDateTime;

61 + win_time.LowPart = attr_data.ftLastWriteTime.dwLowDateTime;

62 +

63 + // Subtract Windows epoch start (January 1, 1601 UTC)

64 + // Add Unix epoch start (January 1, 1970 UTC)

65 + // Convert from 100-nanosecond intervals to seconds

66 + const uint64_t WINDOWS_TICK = 10000000;

67 + const uint64_t SEC_TO_UNIX_EPOCH = 11644473600LL;

68 + metadata.modified_time = (time_t)((win_time.QuadPart / WINDOWS_TICK) - SEC_TO_UNIX_EPOCH);

69 +

70 + return metadata;

71 +}

72 +#endif

\ No newline at end of file

src/libnetdata/os/file_metadata.h new

+19

@@ -0,0 +1,19 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_FILE_METADATA_H

4 +#define NETDATA_FILE_METADATA_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +#include <stdint.h>

8 +#include <time.h>

9 +

10 +typedef struct {

11 + uint64_t size_bytes; // File size in bytes

12 + time_t modified_time; // Last modification time (Unix timestamp)

13 +} OS_FILE_METADATA;

14 +

15 +OS_FILE_METADATA os_get_file_metadata(const char *path);

16 +

17 +#define OS_FILE_METADATA_OK(metadata) ((metadata).modified_time > 0 && (metadata).size_bytes > 0)

18 +

19 +#endif //NETDATA_FILE_METADATA_H

\ No newline at end of file

src/libnetdata/os/os.h

+7

@@ -32,6 +32,13 @@

32 #include "system-maps/cache-host-users-and-groups.h"

33 #include "system-maps/cached-sid-username.h"

34 #include "windows-perflib/perflib.h"

35 +#include "disk_space.h"

36 +#include "file_metadata.h"

37 +#include "process_path.h"

38 +#include "boottime.h"

39 +#include "boot_id.h"

40 +#include "run_dir.h"

41 +#include "file_lock.h"

42

43 // this includes windows.h to the whole of netdata

44 // so various conflicts arise

src/libnetdata/os/process_path.c new

+90

@@ -0,0 +1,90 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "libnetdata/libnetdata.h"

4 +

5 +#if defined(OS_LINUX)

6 +#include <unistd.h>

7 +

8 +char *os_get_process_path(void) {

9 + char path[PATH_MAX + 1] = "";

10 + ssize_t len = readlink("/proc/self/exe", path, PATH_MAX);

11 +

12 + if (len < 0) {

13 + // Error occurred; errno is set

14 + return NULL;

15 + }

16 +

17 + path[len] = '\0'; // readlink doesn't null terminate

18 + return strdupz(path);

19 +}

20 +#endif

21 +

22 +#if defined(OS_FREEBSD)

23 +#include <sys/sysctl.h>

24 +

25 +char *os_get_process_path(void) {

26 + char path[PATH_MAX + 1] = "";

27 + int mib[4] = { CTL_KERN, KERN_PROC, KERN_PROC_PATHNAME, -1 };

28 + size_t len = sizeof(path);

29 +

30 + if (sysctl(mib, 4, path, &len, NULL, 0) == -1) {

31 + // Error occurred; errno is set

32 + return NULL;

33 + }

34 +

35 + return strdupz(path);

36 +}

37 +#endif

38 +

39 +#if defined(OS_MACOS)

40 +#include <mach-o/dyld.h>

41 +

42 +char *os_get_process_path(void) {

43 + char path[PATH_MAX + 1] = "";

44 + uint32_t size = sizeof(path);

45 +

46 + if (_NSGetExecutablePath(path, &size) != 0) {

47 + // Buffer too small

48 + return NULL;

49 + }

50 +

51 + // Resolve any symlinks to get the real path

52 + char real_path[PATH_MAX + 1] = "";

53 + if (!realpath(path, real_path)) {

54 + // Error occurred; errno is set

55 + return NULL;

56 + }

57 +

58 + return strdupz(real_path);

59 +}

60 +#endif

61 +

62 +#if defined(OS_WINDOWS)

63 +#include <windows.h>

64 +

65 +char *os_get_process_path(void) {

66 + wchar_t wpath[32768] = L""; // Maximum path length in Windows

67 + DWORD length = GetModuleFileNameW(NULL, wpath, sizeof(wpath)/sizeof(wpath[0]));

68 +

69 + if (length == 0) {

70 + // GetModuleFileName failed

71 + return NULL;

72 + }

73 +

74 + // Convert wide string to UTF-8

75 + int utf8_len = WideCharToMultiByte(CP_UTF8, 0, wpath, -1, NULL, 0, NULL, NULL);

76 + if (utf8_len == 0) {

77 + // Conversion error

78 + return NULL;

79 + }

80 +

81 + char *path = mallocz(utf8_len);

82 + if (WideCharToMultiByte(CP_UTF8, 0, wpath, -1, path, utf8_len, NULL, NULL) == 0) {

83 + // Conversion error

84 + freez(path);

85 + return NULL;

86 + }

87 +

88 + return path;

89 +}

90 +#endif

src/libnetdata/os/process_path.h new

+13

@@ -0,0 +1,13 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_PROCESS_PATH_H

4 +#define NETDATA_PROCESS_PATH_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +// Get the full path of the current process executable

9 +// Returns a malloced string that must be freed by the caller

10 +// Returns NULL on error

11 +char *os_get_process_path(void);

12 +

13 +#endif //NETDATA_PROCESS_PATH_H

\ No newline at end of file

src/libnetdata/os/run_dir.c new

+128

@@ -0,0 +1,128 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#include "run_dir.h"

4 +#include "libnetdata/libnetdata.h"

5 +

6 +static char *cached_run_dir = NULL;

7 +static SPINLOCK spinlock = SPINLOCK_INITIALIZER;

8 +

9 +static inline bool is_dir_accessible(const char *dir, bool rw) {

10 + struct stat st;

11 + if (stat(dir, &st) == -1)

12 + return false;

13 +

14 + if (!S_ISDIR(st.st_mode))

15 + return false;

16 +

17 + // Check if we can write to the directory

18 + if (access(dir, rw ? W_OK : R_OK) == -1)

19 + return false;

20 +

21 + return true;

22 +}

23 +

24 +static inline bool netdata_dir_in_parent(const char *parent, char *out_path, size_t out_path_len, bool rw) {

25 + if (!is_dir_accessible(parent, rw))

26 + return false;

27 +

28 + snprintfz(out_path, out_path_len, "%s/netdata", parent);

29 + if (mkdir(out_path, 0755) == -1 && errno != EEXIST)

30 + return false;

31 +

32 + return is_dir_accessible(out_path, rw);

33 +}

34 +

35 +static char *detect_run_dir(bool rw) {

36 + char path[FILENAME_MAX + 1];

37 +

38 + if(!rw) {

39 + // First check for environment variable

40 + const char *env_dir = getenv("NETDATA_RUN_DIR");

41 + if (env_dir && *env_dir) {

42 + if (is_dir_accessible(env_dir, rw))

43 + return strdupz(env_dir);

44 + }

45 + }

46 +

47 +#if defined(OS_LINUX)

48 + // First try /run/netdata

49 + if (netdata_dir_in_parent("/run", path, sizeof(path), rw))

50 + goto success;

51 +#endif

52 +

53 +#if defined(OS_MACOS)

54 + // macOS typically uses /private/var/run

55 + if (netdata_dir_in_parent("/private/var/run", path, sizeof(path), rw))

56 + goto success;

57 +#endif

58 +

59 +#if defined(OS_LINUX) || defined(OS_FREEBSD) || defined(OS_MACOS)

60 + // Then try /var/run/netdata

61 + if (netdata_dir_in_parent("/var/run", path, sizeof(path), rw))

62 + goto success;

63 +#endif

64 +

65 +//#if defined(OS_WINDOWS)

66 +// // On MSYS2/Cygwin get TEMP and convert it properly

67 +// WCHAR temp_pathW[MAX_PATH];

68 +// DWORD len = GetEnvironmentVariableW(L"TEMP", temp_pathW, MAX_PATH);

69 +// if (len > 0 && len < MAX_PATH) {

70 +// // Convert Windows wide path to UTF-8

71 +// int utf8_len = WideCharToMultiByte(CP_UTF8, 0, temp_pathW, -1, NULL, 0, NULL, NULL);

72 +// if (utf8_len > 0 && utf8_len < FILENAME_MAX) {

73 +// char win_path[FILENAME_MAX + 1];

74 +// if (WideCharToMultiByte(CP_UTF8, 0, temp_pathW, -1, win_path, sizeof(win_path), NULL, NULL)) {

75 +// // Convert Windows path to Unix path using Cygwin API

76 +// ssize_t unix_size = cygwin_conv_path(CCP_WIN_A_TO_POSIX, win_path, NULL, 0);

77 +// if (unix_size > 0) {

78 +// char unix_path[FILENAME_MAX + 1];

79 +// if (cygwin_conv_path(CCP_WIN_A_TO_POSIX, win_path, unix_path, sizeof(unix_path)) == 0) {

80 +// if (is_dir_accessible(unix_path, rw)) {

81 +// snprintfz(path, sizeof(path), "%s/netdata", unix_path);

82 +// if (!rw)

83 +// goto success;

84 +//

85 +// if (mkdir(path, 0755) == 0 || errno == EEXIST)

86 +// goto success;

87 +// }

88 +// }

89 +// }

90 +// }

91 +// }

92 +// }

93 +//#endif

94 +

95 + // Fallback to /tmp/netdata - force creation if needed

96 + if (!is_dir_accessible("/tmp", rw)) {

97 + // Try to create /tmp with standard permissions (including sticky bit)

98 + if (rw && mkdir("/tmp", 01777) == -1 && errno != EEXIST)

99 + return NULL;

100 + }

101 +

102 + snprintfz(path, sizeof(path), "/tmp/netdata");

103 + if (rw && mkdir(path, 0755) == -1 && errno != EEXIST)

104 + return NULL;

105 +

106 +success:

107 + // Set the environment variable for child processes

108 + if(rw)

109 + setenv("NETDATA_RUN_DIR", path, 1);

110 +

111 + return strdupz(path);

112 +}

113 +

114 +const char *os_run_dir(bool rw) {

115 + // Fast path - return cached directory if available

116 + if(cached_run_dir)

117 + return cached_run_dir;

118 +

119 + spinlock_lock(&spinlock);

120 +

121 + // Check again under lock in case another thread set it

122 + if(!cached_run_dir)

123 + cached_run_dir = detect_run_dir(rw);

124 +

125 + spinlock_unlock(&spinlock);

126 +

127 + return cached_run_dir;

128 +}

src/libnetdata/os/run_dir.h new

+17

@@ -0,0 +1,17 @@

1 +// SPDX-License-Identifier: GPL-3.0-or-later

2 +

3 +#ifndef NETDATA_RUN_DIR_H

4 +#define NETDATA_RUN_DIR_H

5 +

6 +#include "libnetdata/libnetdata.h"

7 +

8 +/**

9 + * Initialize and get the runtime directory for Netdata

10 + * This function gets or creates the runtime directory based on environment or system defaults

11 + *

12 + * @param rw When true, create the directory if it doesn't exist

13 + * @return const char* The runtime directory path

14 + */

15 +const char *os_run_dir(bool rw);

16 +

17 +#endif //NETDATA_RUN_DIR_H

src/libnetdata/os/system_memory.c

+53 -56

@@ -15,7 +15,7 @@ OS_SYSTEM_MEMORY os_last_reported_system_memory(void) {

15 #include <windows.h>

16

17 OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {

18 - OS_SYSTEM_MEMORY sm = {0, 0};

18 + OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

19

20 MEMORYSTATUSEX statex;

21 statex.dwLength = sizeof(statex);

@@ -29,55 +29,11 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {

29 }

30 #endif

31

32 -// macOS

33 -#if defined(OS_MACOS)

34 -#include <mach/mach.h>

35 -#include <sys/sysctl.h>

36 -

37 -OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {

38 - static uint64_t total_ram = 0;

39 - static uint64_t page_size = 0;

40 -

41 - if (page_size == 0) {

42 - size_t len = sizeof(page_size);

43 - if (sysctlbyname("hw.pagesize", &page_size, &len, NULL, 0) != 0)

44 - return (OS_SYSTEM_MEMORY){ 0, 0 };

45 - }

46 -

47 - if (query_total_ram || total_ram == 0) {

48 - size_t len = sizeof(total_ram);

49 - if (sysctlbyname("hw.memsize", &total_ram, &len, NULL, 0) != 0)

50 - return (OS_SYSTEM_MEMORY){ 0, 0 };

51 - }

52 -

53 - uint64_t ram_available = 0;

54 - if (page_size > 0) {

55 - vm_statistics64_data_t vm_info;

56 - mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;

57 - mach_port_t mach_port = mach_host_self();

58 -

59 - if (host_statistics64(mach_port, HOST_VM_INFO64, (host_info_t)&vm_info, &count) != KERN_SUCCESS) {

60 - mach_port_deallocate(mach_task_self(), mach_port);

61 - return (OS_SYSTEM_MEMORY){0, 0};

62 - }

63 -

64 - ram_available = (vm_info.free_count + vm_info.inactive_count + vm_info.purgeable_count) * page_size;

65 - mach_port_deallocate(mach_task_self(), mach_port);

66 - }

67 -

68 - os_system_memory_last = (OS_SYSTEM_MEMORY){

69 - .ram_total_bytes = total_ram,

70 - .ram_available_bytes = ram_available,

71 - };

72 - return os_system_memory_last;

73 -}

74 -#endif

75 -

32 // Linux

33 #if defined(OS_LINUX)

34

35 static OS_SYSTEM_MEMORY os_system_memory_cgroup_v1(bool query_total_ram __maybe_unused) {

80 - static OS_SYSTEM_MEMORY sm = {0, 0};

36 + static OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

37 char buf[4096];

38 uint64_t used = 0, inactive = 0;

39

@@ -118,13 +74,12 @@ done:

74 return sm;

75

76 failed:

121 - sm.ram_total_bytes = 0;

122 - sm.ram_available_bytes = 0;

77 + sm = OS_SYSTEM_MEMORY_EMPTY;

78 return sm;

79 }

80

81 static OS_SYSTEM_MEMORY os_system_memory_cgroup_v2(bool query_total_ram __maybe_unused) {

127 - static OS_SYSTEM_MEMORY sm = {0, 0};

82 + static OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

83 char buf[4096];

84 uint64_t used = 0, inactive = 0;

85

@@ -169,8 +124,7 @@ done:

124 return sm;

125

126 failed:

172 - sm.ram_total_bytes = 0;

173 - sm.ram_available_bytes = 0;

127 + sm = OS_SYSTEM_MEMORY_EMPTY;

128 return sm;

129 }

130

@@ -178,7 +132,7 @@ failed:

132 #define MEMINFO_MEMAVAILABLE "MemAvailable:"

133

134 static OS_SYSTEM_MEMORY os_system_memory_meminfo(bool query_total_ram __maybe_unused) {

181 - static OS_SYSTEM_MEMORY sm = {0, 0};

135 + static OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

136

137 char buf[4096];

138 if (read_txt_file("/proc/meminfo", buf, sizeof(buf)) != 0)

@@ -212,7 +166,7 @@ typedef enum {

166 } OS_MEM_SRC;

167

168 OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {

215 - static OS_SYSTEM_MEMORY sm = {0, 0};

169 + static OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

170 static usec_t last_ut = 0, last_total_ut = 0;

171 static OS_MEM_SRC src = OS_MEM_SRC_UNKNOWN;

172

@@ -284,7 +238,7 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram __maybe_unused) {

238 #include <sys/sysctl.h>

239

240 OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {

287 - static OS_SYSTEM_MEMORY sm = {0, 0};

241 + static OS_SYSTEM_MEMORY sm = OS_SYSTEM_MEMORY_EMPTY;

242

243 // Query the total RAM only if needed or if it hasn't been cached

244 if (query_total_ram || sm.ram_total_bytes == 0) {

@@ -319,8 +273,51 @@ OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {

273 return sm;

274

275 failed:

322 - sm.ram_total_bytes = 0;

323 - sm.ram_available_bytes = 0;

276 + sm = OS_SYSTEM_MEMORY_EMPTY;

277 return sm;

278 }

279 #endif

280 +

281 +// macOS

282 +#if defined(OS_MACOS)

283 +#include <mach/mach.h>

284 +#include <sys/sysctl.h>

285 +

286 +OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram) {

287 + static uint64_t total_ram = 0;

288 + static uint64_t page_size = 0;

289 +

290 + if (page_size == 0) {

291 + size_t len = sizeof(page_size);

292 + if (sysctlbyname("hw.pagesize", &page_size, &len, NULL, 0) != 0)

293 + return OS_SYSTEM_MEMORY_EMPTY;

294 + }

295 +

296 + if (query_total_ram || total_ram == 0) {

297 + size_t len = sizeof(total_ram);

298 + if (sysctlbyname("hw.memsize", &total_ram, &len, NULL, 0) != 0)

299 + return OS_SYSTEM_MEMORY_EMPTY;

300 + }

301 +

302 + uint64_t ram_available = 0;

303 + if (page_size > 0) {

304 + vm_statistics64_data_t vm_info;

305 + mach_msg_type_number_t count = HOST_VM_INFO64_COUNT;

306 + mach_port_t mach_port = mach_host_self();

307 +

308 + if (host_statistics64(mach_port, HOST_VM_INFO64, (host_info_t)&vm_info, &count) != KERN_SUCCESS) {

309 + mach_port_deallocate(mach_task_self(), mach_port);

310 + return OS_SYSTEM_MEMORY_EMPTY;

311 + }

312 +

313 + ram_available = (vm_info.free_count + vm_info.inactive_count + vm_info.purgeable_count) * page_size;

314 + mach_port_deallocate(mach_task_self(), mach_port);

315 + }

316 +

317 + os_system_memory_last = (OS_SYSTEM_MEMORY){

318 + .ram_total_bytes = total_ram,

319 + .ram_available_bytes = ram_available,

320 + };

321 + return os_system_memory_last;

322 +}

323 +#endif

src/libnetdata/os/system_memory.h

+3

@@ -20,6 +20,9 @@ typedef struct {

20 uint64_t ram_available_bytes;

21 } OS_SYSTEM_MEMORY;

22

23 +#define OS_SYSTEM_MEMORY_OK(mem) ((mem).ram_total_bytes > 0)

24 +#define OS_SYSTEM_MEMORY_EMPTY (OS_SYSTEM_MEMORY){ 0 }

25 +

26 // The function to get current system memory:

27 OS_SYSTEM_MEMORY os_system_memory(bool query_total_ram);

28

src/libnetdata/required_dummies.h

+2 -2

@@ -4,13 +4,13 @@

4 #define NETDATA_LIB_DUMMIES_H 1

5

6 // callback required by fatal()

7 -void netdata_cleanup_and_exit(int ret, const char *action, const char *action_result, const char *action_data)

7 +void netdata_cleanup_and_exit(EXIT_REASON reason, const char *action, const char *action_result, const char *action_data)

8 {

9 (void)action;

10 (void)action_result;

11 (void)action_data;

12

13 - exit(ret);

13 + exit(reason == EXIT_REASON_FATAL ? 1 : 0);

14 }

15

16 void rrdset_thread_rda_free(void){}

src/libnetdata/uuid/uuid.h

+1

@@ -37,6 +37,7 @@ ND_UUID_DEFINE(log_flood_protection_msgid, 0xec, 0x87, 0xa5, 0x61, 0x20, 0xd5, 0

37 ND_UUID_DEFINE(netdata_startup_msgid, 0x1e, 0x60, 0x61, 0xa9, 0xfb, 0xd4, 0x45, 0x01, 0xb3, 0xcc, 0xc3, 0x68, 0x11, 0x9f, 0x2b, 0x69);

38 ND_UUID_DEFINE(aclk_connection_msgid, 0xac, 0xb3, 0x3c, 0xb9, 0x57, 0x78, 0x47, 0x6b, 0xaa, 0xc7, 0x02, 0xeb, 0x7e, 0x4e, 0x15, 0x1d);

39 ND_UUID_DEFINE(extreme_cardinality_msgid, 0xd1, 0xf5, 0x96, 0x06, 0xdd, 0x4d, 0x41, 0xe3, 0xb2, 0x17, 0xa0, 0xcf, 0xca, 0xe8, 0xe6, 0x32);

40 +ND_UUID_DEFINE(netdata_exit_msgid, 0x02, 0xf4, 0x7d, 0x35, 0x0a, 0xf5, 0x44, 0x91, 0x97, 0xbf, 0x7a, 0x95, 0xb6, 0x05, 0xa4, 0x68);

41 ND_UUID_DEFINE(dyncfg_user_action_msgid, 0x4f, 0xdf, 0x40, 0x81, 0x6c, 0x12, 0x46, 0x23, 0xa0, 0x32, 0xb7, 0xfe, 0x73, 0xbe, 0xac, 0xb8);

42

43 ND_UUID UUID_generate_from_hash(const void *payload, size_t payload_len);

src/libnetdata/worker_utilization/worker_utilization.c

+42 -7

@@ -52,6 +52,8 @@ struct worker {

52 size_t spinlocks_used;

53 struct worker_spinlock spinlocks[WORKER_SPINLOCK_CONTENTION_FUNCTIONS];

54

55 + uint64_t memory_calls[WORKERS_MEMORY_CALL_MAX];

56 +

57 struct worker *next;

58 struct worker *prev;

59 };

@@ -61,6 +63,24 @@ struct workers_workname { // this is what we add to Ju

63 struct worker *base;

64 };

65

66 +ENUM_STR_MAP_DEFINE(WORKERS_MEMORY_CALL) = {

67 + {WORKERS_MEMORY_CALL_LIBC_MALLOC, "malloc"},

68 + {WORKERS_MEMORY_CALL_LIBC_CALLOC, "calloc"},

69 + {WORKERS_MEMORY_CALL_LIBC_REALLOC, "realloc"},

70 + {WORKERS_MEMORY_CALL_LIBC_FREE, "free"},

71 + {WORKERS_MEMORY_CALL_LIBC_STRDUP, "strdup"},

72 + {WORKERS_MEMORY_CALL_LIBC_STRNDUP, "strndup"},

73 + {WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN, "posix_memalign"},

74 + {WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN_FREE, "posix_memalign_free"},

75 + {WORKERS_MEMORY_CALL_MMAP, "mmap"},

76 + {WORKERS_MEMORY_CALL_MUNMAP, "munmap"},

77 +

78 + // terminator

79 + {0, NULL},

80 +};

81 +

82 +ENUM_STR_DEFINE_FUNCTIONS(WORKERS_MEMORY_CALL, WORKERS_MEMORY_CALL_LIBC_MALLOC, "other");

83 +

84 static struct workers_globals {

85 bool enabled;

86

@@ -100,7 +120,7 @@ size_t workers_allocated_memory(void) {

120 }

121

122 void worker_register(const char *name) {

103 - if(unlikely(worker || !workers_globals.enabled))

123 + if(likely(worker || !workers_globals.enabled))

124 return;

125

126 worker = callocz(1, sizeof(struct worker));

@@ -140,7 +160,7 @@ void worker_register(const char *name) {

160 }

161

162 void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type) {

143 - if(unlikely(!worker)) return;

163 + if(likely(!worker)) return;

164

165 if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) {

166 netdata_log_error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1));

@@ -166,7 +186,7 @@ void worker_register_job_name(size_t job_id, const char *name) {

186 }

187

188 void worker_unregister(void) {

169 - if(unlikely(!worker)) return;

189 + if(likely(!worker)) return;

190

191 size_t workname_size = strlen(worker->workname) + 1;

192 spinlock_lock(&workers_globals.spinlock);

@@ -214,7 +234,7 @@ static void worker_is_idle_with_time(usec_t now) {

234 }

235

236 ALWAYS_INLINE void worker_is_idle(void) {

217 - if(unlikely(!worker || worker->last_action != WORKER_BUSY)) return;

237 + if(likely(!worker || worker->last_action != WORKER_BUSY)) return;

238

239 worker_is_idle_with_time(worker_now_monotonic_usec());

240 }

@@ -236,7 +256,7 @@ static void worker_is_busy_do(size_t job_id) {

256 }

257

258 ALWAYS_INLINE void worker_is_busy(size_t job_id) {

239 - if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

259 + if(likely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

260 return;

261

262 worker_is_busy_do(job_id);

@@ -257,7 +277,7 @@ static void worker_set_metric_do(size_t job_id, NETDATA_DOUBLE value) {

277 }

278

279 ALWAYS_INLINE void worker_set_metric(size_t job_id, NETDATA_DOUBLE value) {

260 - if(unlikely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

280 + if(likely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

281 return;

282

283 worker_set_metric_do(job_id, value);

@@ -289,12 +309,19 @@ static void worker_spinlock_contention_do(const char *func, size_t spins) {

309 }

310

311 ALWAYS_INLINE void worker_spinlock_contention(const char *func, size_t spins) {

292 - if(unlikely(!worker))

312 + if(likely(!worker))

313 return;

314

315 worker_spinlock_contention_do(func, spins);

316 }

317

318 +ALWAYS_INLINE void workers_memory_call(WORKERS_MEMORY_CALL call) {

319 + if(likely(!worker || call >= WORKERS_MEMORY_CALL_MAX))

320 + return;

321 +

322 + worker->memory_calls[call]++;

323 +}

324 +

325 // statistics interface

326

327 void workers_foreach(const char *name, void (*callback)(

@@ -314,6 +341,7 @@ void workers_foreach(const char *name, void (*callback)(

341 , const char *spinlock_functions[]

342 , size_t *spinlock_locks

343 , size_t *spinlock_spins

344 + , uint64_t *memory_calls

345 )

346 , void *data) {

347 if(!workers_globals.enabled)

@@ -354,6 +382,8 @@ void workers_foreach(const char *name, void (*callback)(

382 size_t spinlock_locks[WORKER_SPINLOCK_CONTENTION_FUNCTIONS];

383 size_t spinlock_spins[WORKER_SPINLOCK_CONTENTION_FUNCTIONS];

384

385 + uint64_t memory_calls[WORKERS_MEMORY_CALL_MAX];

386 +

387 size_t max_job_id = p->worker_max_job_id;

388 for(size_t i = 0; i <= max_job_id ;i++) {

389 per_job_type_name[i] = p->per_job_type[i].name;

@@ -466,6 +496,10 @@ void workers_foreach(const char *name, void (*callback)(

496

497 // ------------------------------------------------------------------------------------------------------------

498

499 + memcpy(memory_calls, p->memory_calls, sizeof(memory_calls));

500 +

501 + // ------------------------------------------------------------------------------------------------------------

502 +

503 callback(data

504 , p->pid

505 , p->tag

@@ -483,6 +517,7 @@ void workers_foreach(const char *name, void (*callback)(

517 , spinlock_functions

518 , spinlock_locks

519 , spinlock_spins

520 + , memory_calls

521 );

522 }

523

src/libnetdata/worker_utilization/worker_utilization.h

+21

@@ -16,6 +16,26 @@ typedef enum __attribute__((packed)) {

16 WORKER_METRIC_INCREMENTAL_TOTAL = 4,

17 } WORKER_METRIC_TYPE;

18

19 +typedef enum {

20 + WORKERS_MEMORY_CALL_LIBC_MALLOC = 0,

21 + WORKERS_MEMORY_CALL_LIBC_CALLOC,

22 + WORKERS_MEMORY_CALL_LIBC_REALLOC,

23 + WORKERS_MEMORY_CALL_LIBC_FREE,

24 + WORKERS_MEMORY_CALL_LIBC_STRDUP,

25 + WORKERS_MEMORY_CALL_LIBC_STRNDUP,

26 + WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN,

27 + WORKERS_MEMORY_CALL_LIBC_POSIX_MEMALIGN_FREE,

28 + WORKERS_MEMORY_CALL_MMAP,

29 + WORKERS_MEMORY_CALL_MUNMAP,

30 +

31 + // terminator

32 + WORKERS_MEMORY_CALL_MAX,

33 +} WORKERS_MEMORY_CALL;

34 +

35 +ENUM_STR_DEFINE_FUNCTIONS_EXTERN(WORKERS_MEMORY_CALL);

36 +

37 +void workers_memory_call(WORKERS_MEMORY_CALL call);

38 +

39 void workers_utilization_enable(void);

40 size_t workers_allocated_memory(void);

41 void worker_register(const char *name);

@@ -48,6 +68,7 @@ void workers_foreach(const char *name, void (*callback)(

68 , const char *spinlock_functions[]

69 , size_t *spinlock_locks

70 , size_t *spinlock_spins

71 + , uint64_t *memory_calls

72 )

73 , void *data);

74

src/ml/ml_memory.cc

+6

@@ -10,6 +10,7 @@ void *operator new(size_t size)

10 throw std::bad_alloc();

11

12 pulse_ml_memory_allocated(size);

13 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_MALLOC);

14 return ptr;

15 }

16

@@ -20,6 +21,7 @@ void *operator new[](size_t size)

21 throw std::bad_alloc();

22

23 pulse_ml_memory_allocated(size);

24 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_MALLOC);

25 return ptr;

26 }

27

@@ -27,6 +29,7 @@ void operator delete(void *ptr, size_t size) noexcept

29 {

30 if (ptr) {

31 pulse_ml_memory_freed(size);

32 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

33 free(ptr);

34 }

35 }

@@ -35,6 +38,7 @@ void operator delete[](void *ptr, size_t size) noexcept

38 {

39 if (ptr) {

40 pulse_ml_memory_freed(size);

41 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

42 free(ptr);

43 }

44 }

@@ -42,6 +46,7 @@ void operator delete[](void *ptr, size_t size) noexcept

46 void operator delete(void *ptr) noexcept

47 {

48 if (ptr) {

49 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

50 free(ptr);

51 }

52 }

@@ -49,6 +54,7 @@ void operator delete(void *ptr) noexcept

54 void operator delete[](void *ptr) noexcept

55 {

56 if (ptr) {

57 + workers_memory_call(WORKERS_MEMORY_CALL_LIBC_FREE);

58 free(ptr);

59 }

60 }

src/registry/registry.h

+2 -1

@@ -55,7 +55,8 @@

55

56 // initialize the registry

57 // should only happen when netdata starts

58 -int registry_init(void);

58 +void registry_init(void);

59 +bool registry_load(void);

60

61 // free all data held by the registry

62 // should only happen when netdata exits

src/registry/registry_init.c

+16 -9

@@ -62,7 +62,11 @@ void registry_generate_curl_urls(void) {

62 fclose(fp);

63 }

64

65 -int registry_init(void) {

65 +void registry_init(void) {

66 + FUNCTION_RUN_ONCE();

67 +

68 + netdata_conf_section_global();

69 +

70 char filename[FILENAME_MAX + 1];

71

72 // registry enabled?

@@ -70,7 +74,7 @@ int registry_init(void) {

74 registry.enabled = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_REGISTRY, "enabled", 0);

75 }

76 else {

73 - netdata_log_info("Registry is disabled - use the central netdata");

77 + netdata_log_info("Registry is disabled");

78 inicfg_set_boolean(&netdata_config, CONFIG_SECTION_REGISTRY, "enabled", 0);

79 registry.enabled = 0;

80 }

@@ -117,8 +121,6 @@ int registry_init(void) {

121 inicfg_set_number(&netdata_config, CONFIG_SECTION_REGISTRY, "max URL name length", (long long)registry.max_name_length);

122 }

123

120 - bool use_mmap = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_REGISTRY, "use mmap", false);

121 -

124 // initialize entries counters

125 registry.persons_count = 0;

126 registry.machines_count = 0;

@@ -128,9 +130,12 @@ int registry_init(void) {

130

131 // initialize locks

132 netdata_mutex_init(&registry.lock);

133 +}

134

132 - // load the registry database

135 +bool registry_load(void) {

136 if(registry.enabled) {

137 + bool use_mmap = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_REGISTRY, "use mmap", false);

138 +

139 // create dictionaries

140 registry.persons = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

141 registry.machines = dictionary_create(REGISTRY_DICTIONARY_OPTIONS);

@@ -180,12 +185,14 @@ int registry_init(void) {

185 if(unlikely(registry_db_should_be_saved()))

186 registry_db_save();

187

183 -// registry_db_stats();

184 -// registry_generate_curl_urls();

185 -// exit(0);

188 + // registry_db_stats();

189 + // registry_generate_curl_urls();

190 + // exit(0);

191 +

192 + return true;

193 }

194

188 - return 0;

195 + return false;

196 }

197

198 static int machine_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, void *entry, void *data __maybe_unused) {

src/streaming/protocol/command-begin-set-end-init.c

+1 -1

@@ -8,7 +8,7 @@ static BUFFER *preferred_sender_buffer(RRDHOST *host) {

8 if(host->stream.snd.commit.receiver_tid == gettid_cached())

9 return sender_host_buffer(host);

10 else

11 - return sender_thread_buffer(host->sender);

11 + return sender_thread_buffer(host->sender, HOST_THREAD_BUFFER_INITIAL_SIZE);

12 }

13

14 ALWAYS_INLINE RRDSET_STREAM_BUFFER stream_send_metrics_init(RRDSET *st, time_t wall_clock_time) {

src/streaming/stream-circular-buffer.h

+2 -1

@@ -12,7 +12,8 @@ extern "C" {

12

13 #define CBUFFER_INITIAL_SIZE (16 * 1024)

14 #define CBUFFER_INITIAL_MAX_SIZE (10 * 1024 * 1024)

15 -#define THREAD_BUFFER_INITIAL_SIZE (8192)

15 +#define HOST_THREAD_BUFFER_INITIAL_SIZE (256 * 1024)

16 +#define REPLICATION_THREAD_BUFFER_INITIAL_SIZE (512 * 1024)

17

18 #define STREAM_CIRCULAR_BUFFER_ADAPT_TO_TIMES_MAX_SIZE 3

19

src/streaming/stream-conf.c

+1 -3

@@ -115,9 +115,7 @@ bool stream_conf_receiver_needs_dbengine(void) {

115 }

116

117 void stream_conf_load() {

118 - static bool run = false;

119 - if(run) return;

120 - run = true;

118 + FUNCTION_RUN_ONCE();

119

120 stream_conf_load_internal();

121 check_local_streaming_capabilities();

src/streaming/stream-receiver.c

+5 -4

@@ -808,8 +808,9 @@ bool stream_receiver_receive_data(struct stream_thread *sth, struct receiver_sta

808 };

809 ND_LOG_STACK_PUSH(lgs);

810

811 + size_t count = 1; // how many reads to do per host, before moving to the next host

812 EVLOOP_STATUS status = EVLOOP_STATUS_CONTINUE;

812 - while(status == EVLOOP_STATUS_CONTINUE) {

813 + while(status == EVLOOP_STATUS_CONTINUE && count-- > 0) {

814 bool removed = false;

815 ssize_t rc = stream_receive_and_process(sth, rpt, parser, now_ut, &removed);

816 if(unlikely(removed))

@@ -963,9 +964,9 @@ void stream_receiver_check_all_nodes_from_poll(struct stream_thread *sth, usec_t

964

965 nd_poll_event_t wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

966 if(unlikely(rpt->thread.wanted != wanted)) {

966 - nd_log(NDLS_DAEMON, NDLP_DEBUG,

967 - "STREAM RCV[%zu] '%s' [from %s]: nd_poll() wanted events mismatch.",

968 - sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

967 +// nd_log(NDLS_DAEMON, NDLP_DEBUG,

968 +// "STREAM RCV[%zu] '%s' [from %s]: nd_poll() wanted events mismatch.",

969 +// sth->id, rrdhost_hostname(rpt->host), rpt->remote_ip);

970

971 rpt->thread.wanted = wanted;

972 if(!nd_poll_upd(sth->run.ndpl, rpt->sock.fd, rpt->thread.wanted))

src/streaming/stream-replication-sender.c

+2 -2

@@ -622,7 +622,7 @@ bool replication_response_execute_finalize_and_send(struct replication_query *q,

622 // we might want to optimize this by filling a temporary buffer

623 // and copying the result to the host's buffer in order to avoid

624 // holding the host's buffer lock for too long

625 - BUFFER *wb = sender_thread_buffer(host->sender);

625 + BUFFER *wb = sender_thread_buffer(host->sender, REPLICATION_THREAD_BUFFER_INITIAL_SIZE);

626

627 buffer_fast_strcat(wb, PLUGINSD_KEYWORD_REPLAY_BEGIN, sizeof(PLUGINSD_KEYWORD_REPLAY_BEGIN) - 1);

628

@@ -1863,7 +1863,7 @@ void *replication_thread_main(void *ptr) {

1863 }

1864

1865 int replication_threads_default(void) {

1866 - int threads = netdata_conf_is_parent() ? (int)MIN(netdata_conf_cpus(), 6) : 1;

1866 + int threads = netdata_conf_is_parent() ? (int)MAX(netdata_conf_cpus() / 3, 4) : 1;

1867 threads = FIT_IN_RANGE(threads, 1, MAX_REPLICATION_THREADS);

1868 return threads;

1869 }

src/streaming/stream-sender-commit.c

+6 -6

@@ -24,7 +24,7 @@ void sender_host_buffer_free(RRDHOST *host) {

24 }

25

26 // Collector thread starting a transmission

27 -BUFFER *sender_commit_start_with_trace(struct sender_state *s, struct sender_buffer *commit, const char *func) {

27 +static BUFFER *sender_commit_start_with_trace(struct sender_state *s, struct sender_buffer *commit, size_t default_size, const char *func) {

28 if(unlikely(commit->used))

29 fatal("STREAM SND '%s' [to %s]: thread buffer is used multiple times concurrently (%u). "

30 "It is already being used by '%s()', and now is called by '%s()'",

@@ -39,14 +39,14 @@ BUFFER *sender_commit_start_with_trace(struct sender_state *s, struct sender_buf

39 commit->receiver_tid, gettid_cached(), func ? func : "(null)");

40

41 if(unlikely(commit->wb &&

42 - commit->wb->size > THREAD_BUFFER_INITIAL_SIZE &&

42 + commit->wb->size > default_size &&

43 commit->our_recreates != commit->sender_recreates)) {

44 buffer_free(commit->wb);

45 commit->wb = NULL;

46 }

47

48 if(unlikely(!commit->wb)) {

49 - commit->wb = buffer_create(THREAD_BUFFER_INITIAL_SIZE, &netdata_buffers_statistics.buffers_streaming);

49 + commit->wb = buffer_create(default_size, &netdata_buffers_statistics.buffers_streaming);

50 commit->our_recreates = commit->sender_recreates;

51 }

52

@@ -58,12 +58,12 @@ BUFFER *sender_commit_start_with_trace(struct sender_state *s, struct sender_buf

58 return commit->wb;

59 }

60

61 -BUFFER *sender_thread_buffer_with_trace(struct sender_state *s, const char *func) {

62 - return sender_commit_start_with_trace(s, &commit___thread, func);

61 +BUFFER *sender_thread_buffer_with_trace(struct sender_state *s, size_t default_size, const char *func) {

62 + return sender_commit_start_with_trace(s, &commit___thread, default_size, func);

63 }

64

65 BUFFER *sender_host_buffer_with_trace(struct rrdhost *host, const char *func) {

66 - return sender_commit_start_with_trace(host->sender, &host->stream.snd.commit, func);

66 + return sender_commit_start_with_trace(host->sender, &host->stream.snd.commit, HOST_THREAD_BUFFER_INITIAL_SIZE, func);

67 }

68

69 // Collector thread finishing a transmission

src/streaming/stream-sender-commit.h

+2 -2

@@ -29,8 +29,8 @@ void sender_host_buffer_free(struct rrdhost *host);

29 // get the thread buffer

30 // this is the preferred buffer for dedicated workers sending a lot of messages (like replication)

31 // these threads need to maintain enough allocation for repeated use of the buffer

32 -BUFFER *sender_thread_buffer_with_trace(struct sender_state *s, const char *func);

33 -#define sender_thread_buffer(s) sender_thread_buffer_with_trace(s, __FUNCTION__)

32 +BUFFER *sender_thread_buffer_with_trace(struct sender_state *s, size_t default_size, const char *func);

33 +#define sender_thread_buffer(s, default_size) sender_thread_buffer_with_trace(s, default_size, __FUNCTION__)

34

35 // get the global host buffer

36 // this is the preferred buffer for stream threads (unified receiver / sender threads)

src/streaming/stream-sender.c

+3 -3

@@ -521,9 +521,9 @@ void stream_sender_check_all_nodes_from_poll(struct stream_thread *sth, usec_t n

521

522 nd_poll_event_t wanted = ND_POLL_READ | (stats.bytes_outstanding ? ND_POLL_WRITE : 0);

523 if(unlikely(s->thread.wanted != wanted)) {

524 - nd_log(NDLS_DAEMON, NDLP_DEBUG,

525 - "STREAM SND[%zu] '%s' [to %s]: nd_poll() wanted events mismatch.",

526 - sth->id, rrdhost_hostname(s->host), s->remote_ip);

524 +// nd_log(NDLS_DAEMON, NDLP_DEBUG,

525 +// "STREAM SND[%zu] '%s' [to %s]: nd_poll() wanted events mismatch.",

526 +// sth->id, rrdhost_hostname(s->host), s->remote_ip);

527

528 s->thread.wanted = wanted;

529 if(!nd_poll_upd(sth->run.ndpl, s->sock.fd, s->thread.wanted))

src/web/api/functions/function-bearer_get_token.c

+2 -1

@@ -13,7 +13,8 @@ struct bearer_token_request {

13 STRING *client_name;

14 };

15

16 -static bool bearer_parse_json_payload(json_object *jobj, const char *path, void *data, BUFFER *error) {

16 +static bool bearer_parse_json_payload(json_object *jobj, void *data, BUFFER *error) {

17 + const char *path = "";

18 struct bearer_token_request *rq = data;

19 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "claim_id", rq->claim_id, error, true);

20 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "machine_guid", rq->machine_guid, error, true);

src/web/server/web_client.c

+2 -2

@@ -1185,13 +1185,13 @@ static inline int web_client_process_url(RRDHOST *host, struct web_client *w, ch

1185 w->response.data->content_type = CT_TEXT_PLAIN;

1186 buffer_flush(w->response.data);

1187

1188 - if(!netdata_exit)

1188 + if(!exit_initiated)

1189 buffer_strcat(w->response.data, "ok, will do...");

1190 else

1191 buffer_strcat(w->response.data, "I am doing it already");

1192

1193 netdata_log_error("web request to exit received.");

1194 - netdata_cleanup_and_exit(0, NULL, NULL, NULL);

1194 + netdata_cleanup_and_exit(EXIT_REASON_API_QUIT, NULL, NULL, NULL);

1195 return HTTP_RESP_OK;

1196 }

1197 else if(unlikely(hash == hash_debug && strcmp(tok, "debug") == 0)) {

src/web/server/web_client.h

+1 -1

@@ -131,7 +131,7 @@ void web_client_set_conn_webrtc(struct web_client *w);

131 #define NETDATA_WEB_RESPONSE_HEADER_INITIAL_SIZE 4096

132 #define NETDATA_WEB_RESPONSE_INITIAL_SIZE 8192

133 #define NETDATA_WEB_REQUEST_INITIAL_SIZE 8192

134 -#define NETDATA_WEB_REQUEST_MAX_SIZE 65536

134 +#define NETDATA_WEB_REQUEST_MAX_SIZE (128 * 1024)

135 #define NETDATA_WEB_DECODED_URL_INITIAL_SIZE 512

136

137 #define CLOUD_CLIENT_NAME_LENGTH 64

detect netdata exit reasons (#19617)