@cryptotaxi247 / netdata-1 / commits / 0e52aff30

Streaming improvements No 2 (#19167)

* fix coverity issue 451681 Read from pointer after free * coverity 451580 Untrusted loop bound * move vendored sqlite3 files to its own directory * move vendored libraries

Costa Tsaousis committed Dec 9, 2024 at 16:12 UTC 0e52aff307cfdd85c6a552eeed433c75f6e807bd
49 files changed +66 -55
CMakeLists.txt
+39 -39
@@ -653,40 +653,40 @@ netdata_detect_protobuf()
653 #
654
655 set(LIBJUDY_PREV_FILES
656 - src/libnetdata/libjudy/src/JudyL/JudyLPrev.c
657 - src/libnetdata/libjudy/src/JudyL/JudyLPrevEmpty.c
656 + src/libnetdata/libjudy/vendored/JudyL/JudyLPrev.c
657 + src/libnetdata/libjudy/vendored/JudyL/JudyLPrevEmpty.c
658 )
659
660 set(LIBJUDY_NEXT_FILES
661 - src/libnetdata/libjudy/src/JudyL/JudyLNext.c
662 - src/libnetdata/libjudy/src/JudyL/JudyLNextEmpty.c
661 + src/libnetdata/libjudy/vendored/JudyL/JudyLNext.c
662 + src/libnetdata/libjudy/vendored/JudyL/JudyLNextEmpty.c
663 )
664
665 set(LIBJUDY_SOURCES
666 - src/libnetdata/libjudy/src/Judy.h
667 - src/libnetdata/libjudy/src/JudyCommon/JudyMalloc.c
668 - src/libnetdata/libjudy/src/JudyCommon/JudyPrivate.h
669 - src/libnetdata/libjudy/src/JudyCommon/JudyPrivate1L.h
670 - src/libnetdata/libjudy/src/JudyCommon/JudyPrivateBranch.h
671 - src/libnetdata/libjudy/src/JudyL/JudyL.h
672 - src/libnetdata/libjudy/src/JudyL/JudyLByCount.c
673 - src/libnetdata/libjudy/src/JudyL/JudyLCascade.c
674 - src/libnetdata/libjudy/src/JudyL/JudyLCount.c
675 - src/libnetdata/libjudy/src/JudyL/JudyLCreateBranch.c
676 - src/libnetdata/libjudy/src/JudyL/JudyLDecascade.c
677 - src/libnetdata/libjudy/src/JudyL/JudyLDel.c
678 - src/libnetdata/libjudy/src/JudyL/JudyLFirst.c
679 - src/libnetdata/libjudy/src/JudyL/JudyLFreeArray.c
680 - src/libnetdata/libjudy/src/JudyL/j__udyLGet.c
681 - src/libnetdata/libjudy/src/JudyL/JudyLGet.c
682 - src/libnetdata/libjudy/src/JudyL/JudyLInsArray.c
683 - src/libnetdata/libjudy/src/JudyL/JudyLIns.c
684 - src/libnetdata/libjudy/src/JudyL/JudyLInsertBranch.c
685 - src/libnetdata/libjudy/src/JudyL/JudyLMallocIF.c
686 - src/libnetdata/libjudy/src/JudyL/JudyLMemActive.c
687 - src/libnetdata/libjudy/src/JudyL/JudyLMemUsed.c
688 - src/libnetdata/libjudy/src/JudyL/JudyLTables.c
689 - src/libnetdata/libjudy/src/JudyHS/JudyHS.c
666 + src/libnetdata/libjudy/vendored/Judy.h
667 + src/libnetdata/libjudy/vendored/JudyCommon/JudyMalloc.c
668 + src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate.h
669 + src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate1L.h
670 + src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivateBranch.h
671 + src/libnetdata/libjudy/vendored/JudyL/JudyL.h
672 + src/libnetdata/libjudy/vendored/JudyL/JudyLByCount.c
673 + src/libnetdata/libjudy/vendored/JudyL/JudyLCascade.c
674 + src/libnetdata/libjudy/vendored/JudyL/JudyLCount.c
675 + src/libnetdata/libjudy/vendored/JudyL/JudyLCreateBranch.c
676 + src/libnetdata/libjudy/vendored/JudyL/JudyLDecascade.c
677 + src/libnetdata/libjudy/vendored/JudyL/JudyLDel.c
678 + src/libnetdata/libjudy/vendored/JudyL/JudyLFirst.c
679 + src/libnetdata/libjudy/vendored/JudyL/JudyLFreeArray.c
680 + src/libnetdata/libjudy/vendored/JudyL/j__udyLGet.c
681 + src/libnetdata/libjudy/vendored/JudyL/JudyLGet.c
682 + src/libnetdata/libjudy/vendored/JudyL/JudyLInsArray.c
683 + src/libnetdata/libjudy/vendored/JudyL/JudyLIns.c
684 + src/libnetdata/libjudy/vendored/JudyL/JudyLInsertBranch.c
685 + src/libnetdata/libjudy/vendored/JudyL/JudyLMallocIF.c
686 + src/libnetdata/libjudy/vendored/JudyL/JudyLMemActive.c
687 + src/libnetdata/libjudy/vendored/JudyL/JudyLMemUsed.c
688 + src/libnetdata/libjudy/vendored/JudyL/JudyLTables.c
689 + src/libnetdata/libjudy/vendored/JudyHS/JudyHS.c
690 ${LIBJUDY_PREV_FILES}
691 ${LIBJUDY_NEXT_FILES}
692 )
@@ -733,8 +733,8 @@ set(LIBNETDATA_FILES
733 src/libnetdata/inlined.h
734 src/libnetdata/json/json.c
735 src/libnetdata/json/json.h
736 - src/libnetdata/json/jsmn.c
737 - src/libnetdata/json/jsmn.h
736 + src/libnetdata/json/vendored/jsmn.c
737 + src/libnetdata/json/vendored/jsmn.h
738 src/libnetdata/libnetdata.c
739 src/libnetdata/libnetdata.h
740 src/libnetdata/line_splitter/line_splitter.c
@@ -1416,11 +1416,11 @@ set(RRD_PLUGIN_FILES
1416 src/database/sqlite/sqlite_aclk_node.h
1417 src/database/sqlite/sqlite_aclk_alert.c
1418 src/database/sqlite/sqlite_aclk_alert.h
1419 - src/database/sqlite/sqlite3.c
1420 - src/database/sqlite/sqlite3.h
1421 - src/database/sqlite/sqlite3recover.c
1422 - src/database/sqlite/sqlite3recover.h
1423 - src/database/sqlite/dbdata.c
1419 + src/database/sqlite/vendored/sqlite3.c
1420 + src/database/sqlite/vendored/sqlite3.h
1421 + src/database/sqlite/vendored/sqlite3recover.c
1422 + src/database/sqlite/vendored/sqlite3recover.h
1423 + src/database/sqlite/vendored/dbdata.c
1424 src/database/KolmogorovSmirnovDist.c
1425 src/database/KolmogorovSmirnovDist.h
1426 src/database/rrdfunctions-inflight.c
@@ -1926,14 +1926,14 @@ target_compile_definitions(judy PRIVATE
1926 )
1927
1928 target_include_directories(judy PUBLIC
1929 - src/libnetdata/libjudy/src
1930 - src/libnetdata/libjudy/src/JudyCommon
1929 + src/libnetdata/libjudy/vendored
1930 + src/libnetdata/libjudy/vendored/JudyCommon
1931 )
1932
1933 set_source_files_properties(${LIBJUDY_PREV_FILES} PROPERTIES COMPILE_OPTIONS "-DJUDYPREV")
1934 set_source_files_properties(${LIBJUDY_NEXT_FILES} PROPERTIES COMPILE_OPTIONS "-DJUDYNEXT")
1935 -set_source_files_properties(src/libnetdata/libjudy/src/JudyL/j__udyLGet.c PROPERTIES COMPILE_OPTIONS "-DJUDYGETINLINE")
1936 -set_source_files_properties(src/libnetdata/libjudy/src/JudyL/JudyLByCount.c PROPERTIES COMPILE_OPTIONS "-DNOSMARTJBB -DNOSMARTJBU -DNOSMARTJLB")
1935 +set_source_files_properties(src/libnetdata/libjudy/vendored/JudyL/j__udyLGet.c PROPERTIES COMPILE_OPTIONS "-DJUDYGETINLINE")
1936 +set_source_files_properties(src/libnetdata/libjudy/vendored/JudyL/JudyLByCount.c PROPERTIES COMPILE_OPTIONS "-DNOSMARTJBB -DNOSMARTJBU -DNOSMARTJLB")
1937 set_source_files_properties(JudyLTables.c PROPERTIES COMPILE_OPTIONS "-I${CMAKE_SOURCE_DIR}/src/libnetdata/libjudy/src/JudyL")
1938
1939 #
src/daemon/config/netdata-conf-global.c
+4 -2
@@ -13,7 +13,9 @@ static int get_hostname(char *buf, size_t buf_size) {
13 }
14 }
15
16 - return gethostname(buf, buf_size);
16 + int rc = gethostname(buf, buf_size);
17 + buf[buf_size - 1] = '\0';
18 + return rc;
19 }
20
21 void netdata_conf_section_global(void) {
@@ -26,7 +28,7 @@ void netdata_conf_section_global(void) {
28 (void) verify_netdata_host_prefix(true);
29
30 char buf[HOSTNAME_MAX + 1];
29 - if (get_hostname(buf, HOSTNAME_MAX))
31 + if (get_hostname(buf, sizeof(buf)))
32 netdata_log_error("Cannot get machine hostname.");
33
34 netdata_configured_hostname = config_get(CONFIG_SECTION_GLOBAL, "hostname", buf);
src/database/sqlite/sqlite_context.h
+1 -1
@@ -4,7 +4,7 @@
4 #define NETDATA_SQLITE_CONTEXT_H
5
6 #include "daemon/common.h"
7 -#include "sqlite3.h"
7 +#include "database/sqlite/vendored/sqlite3.h"
8
9 int sql_context_cache_stats(int op);
10 typedef struct ctx_chart {
src/database/sqlite/sqlite_db_migration.h
+1 -2
@@ -3,8 +3,7 @@
3 #define NETDATA_SQLITE_DB_MIGRATION_H
4
5 #include "daemon/common.h"
6 -#include "sqlite3.h"
7 -
6 +#include "database/sqlite/vendored/sqlite3.h"
7
8 int perform_database_migration(sqlite3 *database, int target_version);
9 int perform_context_database_migration(sqlite3 *database, int target_version);
src/database/sqlite/sqlite_functions.h
+1 -1
@@ -4,7 +4,7 @@
4 #define NETDATA_SQLITE_FUNCTIONS_H
5
6 #include "daemon/common.h"
7 -#include "sqlite3.h"
7 +#include "database/sqlite/vendored/sqlite3.h"
8
9 void analytics_set_data_str(char **name, const char *value);
10
src/database/sqlite/sqlite_health.h
+1 -1
@@ -4,7 +4,7 @@
4 #define NETDATA_SQLITE_HEALTH_H
5
6 #include "daemon/common.h"
7 -#include "sqlite3.h"
7 +#include "database/sqlite/vendored/sqlite3.h"
8
9 #define ALERT_TRANSITION_DELAY_LONG (600)
10 #define ALERT_TRANSITION_DELAY_SHORT (10)
src/database/sqlite/sqlite_metadata.c
+1 -1
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "sqlite_metadata.h"
4 -#include "sqlite3recover.h"
4 +#include "database/sqlite/vendored/sqlite3recover.h"
5 //#include "sqlite_db_migration.h"
6
7 #define DB_METADATA_VERSION 18
src/database/sqlite/sqlite_metadata.h
+1 -1
@@ -3,7 +3,7 @@
3 #ifndef NETDATA_SQLITE_METADATA_H
4 #define NETDATA_SQLITE_METADATA_H
5
6 -#include "sqlite3.h"
6 +#include "database/sqlite/vendored/sqlite3.h"
7 #include "sqlite_functions.h"
8
9 typedef enum event_log_type {
src/database/sqlite/vendored/README.md new
+1
@@ -0,0 +1 @@
1 +Vendored copy of SQLITE3
src/database/sqlite/vendored/dbdata.c renamed
src/database/sqlite/vendored/sqlite3.c renamed
src/database/sqlite/vendored/sqlite3.h renamed
src/database/sqlite/vendored/sqlite3recover.c renamed
src/database/sqlite/vendored/sqlite3recover.h renamed
src/libnetdata/inlined.h
+10 -2
@@ -622,9 +622,14 @@ static inline char *strsep_skip_consecutive_separators(char **ptr, char *s) {
622
623 // remove leading and trailing spaces; may return NULL
624 static inline char *trim(char *s) {
625 + char *buf = s;
626 +
627 // skip leading spaces
628 while (*s && isspace((uint8_t)*s)) s++;
627 - if (!*s) return NULL;
629 + if (!*s) {
630 + *buf = '\0';
631 + return NULL;
632 + }
633
634 // skip tailing spaces
635 // this way is way faster. Writes only one NUL char.
@@ -635,7 +640,10 @@ static inline char *trim(char *s) {
640 *++p = '\0';
641 }
642
638 - if (!*s) return NULL;
643 + if (!*s) {
644 + *buf = '\0';
645 + return NULL;
646 + }
647
648 return s;
649 }
src/libnetdata/json/json.h
+1 -1
@@ -10,7 +10,7 @@
10 #endif // error_description
11 #endif // ENABLE_JSONC
12
13 -#include "jsmn.h"
13 +#include "libnetdata/json/vendored/jsmn.h"
14
15 //https://www.ibm.com/support/knowledgecenter/en/SS9H2Y_7.6.0/com.ibm.dp.doc/json_parserlimits.html
16 #define JSON_NAME_LEN 256
src/libnetdata/json/vendored/jsmn.c renamed
src/libnetdata/json/vendored/jsmn.h renamed
src/libnetdata/libjudy/vendored/Judy.h renamed
src/libnetdata/libjudy/vendored/JudyCommon/JudyMalloc.c renamed
src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate.h renamed
src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivate1L.h renamed
src/libnetdata/libjudy/vendored/JudyCommon/JudyPrivateBranch.h renamed
src/libnetdata/libjudy/vendored/JudyHS/JudyHS.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyL.h renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLByCount.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLCascade.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLCount.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLCreateBranch.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLDecascade.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLDel.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLFirst.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLFreeArray.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLGet.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLIns.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLInsArray.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLInsertBranch.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLMallocIF.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLMemActive.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLMemUsed.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLNext.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLNextEmpty.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLPrev.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLPrevEmpty.c renamed
src/libnetdata/libjudy/vendored/JudyL/JudyLTables.c renamed
src/libnetdata/libjudy/vendored/JudyL/j__udyLGet.c renamed
src/libnetdata/libnetdata.h
+1 -1
@@ -107,7 +107,7 @@ extern const char *netdata_configured_host_prefix;
107
108 // safe includes before O/S specific functions
109 #include "template-enum.h"
110 -#include "libjudy/src/Judy.h"
110 +#include "libjudy/vendored/Judy.h"
111 #include "libjudy/judyl-typed.h"
112 #include "july/july.h"
113
src/ml/ml.cc
+1 -1
@@ -5,7 +5,7 @@
5 #include <array>
6
7 #include "ad_charts.h"
8 -#include "database/sqlite/sqlite3.h"
8 +#include "database/sqlite/vendored/sqlite3.h"
9
10 #define WORKER_TRAIN_QUEUE_POP 0
11 #define WORKER_TRAIN_ACQUIRE_DIMENSION 1
src/streaming/stream-receiver.c
+3 -2
@@ -440,6 +440,7 @@ void stream_receiver_move_queue_to_running_unsafe(struct stream_thread *sth) {
440 static void stream_receiver_remove(struct stream_thread *sth, struct receiver_state *rpt, const char *why) {
441 internal_fatal(sth->tid != gettid_cached(), "Function %s() should only be used by the dispatcher thread", __FUNCTION__ );
442
443 + errno_clear();
444 nd_log(NDLS_DAEMON, NDLP_ERR,
445 "STREAM RECEIVE[%zu] '%s' [from [%s]:%s]: "
446 "receiver disconnected: %s"
@@ -700,8 +701,8 @@ void stream_receive_process_poll_events(struct stream_thread *sth, struct receiv
701 // we can receive data from this socket
702
703 worker_is_busy(WORKER_STREAM_JOB_SOCKET_RECEIVE);
703 - while(true) {
704 - bool removed = false;
704 + bool removed = false;
705 + while(!removed) {
706 ssize_t rc = stream_receive_and_process(sth, rpt, parser, &removed);
707 if (likely(rc > 0)) {
708 rpt->last_msg_t = (time_t)(now_ut / USEC_PER_SEC);