@cryptotaxi247 / netdata-1 / commits / 48b2d1609

logs-management: Disable logs management monitoring section (#16525)

Dimitris P committed Dec 3, 2023 at 14:22 UTC 48b2d1609ce4266f11d84b4a3ae269aad07650b9
9 files changed +19 -746
collectors/plugins.d/plugins_d.c
+1 -1
@@ -240,7 +240,7 @@ void *pluginsd_main(void *ptr)
240 // disable some plugins by default
241 config_get_boolean(CONFIG_SECTION_PLUGINS, "slabinfo", CONFIG_BOOLEAN_NO);
242 config_get_boolean(CONFIG_SECTION_PLUGINS, "logs-management",
243 -#if defined(LOGS_MANAGEMENT_STRESS_TEST)
243 +#if defined(LOGS_MANAGEMENT_DEV_MODE)
244 CONFIG_BOOLEAN_YES
245 #else
246 CONFIG_BOOLEAN_NO
logsmanagement/Makefile.am
-4
@@ -25,8 +25,4 @@ dist_logsmanagconfig_DATA = \
25
26 dist_noinst_DATA = \
27 README.md \
28 - stress_test/logrotate.conf \
29 - stress_test/logs_query.html \
30 - stress_test/run_stress_test.sh \
31 - stress_test/stress_test.c \
28 $(NULL)
logsmanagement/defaults.h
+2 -2
@@ -23,7 +23,7 @@
23
24 #define UPDATE_TIMEOUT_DEFAULT 10 /**< Default timeout to use to update charts if they haven't been updated in the meantime. **/
25
26 -#if !defined(LOGS_MANAGEMENT_STRESS_TEST)
26 +#if !defined(LOGS_MANAGEMENT_DEV_MODE)
27 #define ENABLE_COLLECTED_LOGS_TOTAL_DEFAULT CONFIG_BOOLEAN_NO /**< Default value to enable (or not) metrics of total collected log records **/
28 #else
29 #define ENABLE_COLLECTED_LOGS_TOTAL_DEFAULT CONFIG_BOOLEAN_YES /**< Default value to enable (or not) metrics of total collected log records, if stress tests are enabled **/
@@ -57,7 +57,7 @@ typedef enum {
57
58 #define DISK_SPACE_LIMIT_DEFAULT 500 /**< Global default configuration maximum database disk space limit per log source **/
59
60 -#if !defined(LOGS_MANAGEMENT_STRESS_TEST)
60 +#if !defined(LOGS_MANAGEMENT_DEV_MODE)
61 #define GLOBAL_DB_MODE_DEFAULT_STR "none" /**< db mode string to be used as global default in configuration **/
62 #define GLOBAL_DB_MODE_DEFAULT LOGS_MANAG_DB_MODE_NONE /**< db mode to be used as global default, matching GLOBAL_DB_MODE_DEFAULT_STR **/
63 #else
logsmanagement/helper.h
+3 -3
@@ -16,11 +16,11 @@
16 #define LOGS_MANAG_STR(x) LOGS_MANAG_STR_HELPER(x)
17
18 #ifndef m_assert
19 -#if defined(LOGS_MANAGEMENT_STRESS_TEST)
19 +#if defined(LOGS_MANAGEMENT_DEV_MODE)
20 #define m_assert(expr, msg) assert(((void)(msg), (expr)))
21 #else
22 #define m_assert(expr, msg) do{} while(0)
23 -#endif // LOGS_MANAGEMENT_STRESS_TEST
23 +#endif // LOGS_MANAGEMENT_DEV_MODE
24 #endif // m_assert
25
26 /* Test if a timestamp is within a valid range
@@ -45,7 +45,7 @@
45 #define COMPILE_TIME_ASSERT(X) COMPILE_TIME_ASSERT2(X,__LINE__)
46 #endif // COMPILE_TIME_ASSERT
47
48 -#if defined(NETDATA_INTERNAL_CHECKS) && defined(LOGS_MANAGEMENT_STRESS_TEST)
48 +#if defined(NETDATA_INTERNAL_CHECKS) && defined(LOGS_MANAGEMENT_DEV_MODE)
49 #define debug_log(args...) netdata_logger(NDLS_COLLECTORS, NDLP_DEBUG, __FILE__, __FUNCTION__, __LINE__, ##args)
50 #else
51 #define debug_log(fmt, args...) do {} while(0)
logsmanagement/logsmanagement.c
+13 -14
@@ -23,10 +23,6 @@
23 #include "logsmanagement/unit_test/unit_test.h"
24 #endif
25
26 -#if defined(LOGS_MANAGEMENT_STRESS_TEST) && LOGS_MANAGEMENT_STRESS_TEST == 1
27 -#include "query_test.h"
28 -#endif // defined(LOGS_MANAGEMENT_STRESS_TEST)
29 -
26 netdata_mutex_t stdout_mut = NETDATA_MUTEX_INITIALIZER;
27
28 bool logsmanagement_should_exit = false;
@@ -35,8 +31,6 @@ struct File_infos_arr *p_file_infos_arr = NULL;
31
32 static uv_loop_t *main_loop;
33
38 -static uv_thread_t stats_charts_thread_id;
39 -
34 static struct {
35 uv_signal_t sig;
36 const int signum;
@@ -193,7 +187,15 @@ int main(int argc, char **argv) {
187 exit(1);
188 }
189
196 - fatal_assert(0 == uv_thread_create(&stats_charts_thread_id, stats_charts_init, &stdout_mut));
190 + uv_thread_t *p_stats_charts_thread_id = NULL;
191 + const char *const netdata_internals_monitoring = getenv("NETDATA_INTERNALS_MONITORING");
192 + if( netdata_internals_monitoring &&
193 + *netdata_internals_monitoring &&
194 + strcmp(netdata_internals_monitoring, "YES") == 0){
195 +
196 + p_stats_charts_thread_id = mallocz(sizeof(uv_thread_t));
197 + fatal_assert(0 == uv_thread_create(p_stats_charts_thread_id, stats_charts_init, &stdout_mut));
198 + }
199
200 #if defined(__STDC_VERSION__)
201 debug_log( "__STDC_VERSION__: %ld", __STDC_VERSION__);
@@ -204,12 +206,6 @@ int main(int argc, char **argv) {
206 debug_log( "LZ4 version: %s", LZ4_versionString());
207 debug_log( "SQLITE version: " SQLITE_VERSION);
208
207 -#if defined(LOGS_MANAGEMENT_STRESS_TEST) && LOGS_MANAGEMENT_STRESS_TEST == 1
208 - debug_log( "Running Netdata with logs_management stress test enabled!");
209 - static uv_thread_t run_stress_test_queries_thread_id;
210 - uv_thread_create(&run_stress_test_queries_thread_id, run_stress_test_queries_thread, NULL);
211 -#endif // LOGS_MANAGEMENT_STRESS_TEST
212 -
209 for(int i = 0; i < (int) (sizeof(signals) / sizeof(signals[0])); i++){
210 uv_signal_init(main_loop, &signals[i].sig);
211 uv_signal_start(&signals[i].sig, signal_handler, signals[i].signum);
@@ -229,7 +225,10 @@ int main(int argc, char **argv) {
225 nd_log_limits_unlimited();
226
227 // TODO: Clean up stats charts memory
232 - uv_thread_join(&stats_charts_thread_id);
228 + if(p_stats_charts_thread_id){
229 + uv_thread_join(p_stats_charts_thread_id);
230 + freez(p_stats_charts_thread_id);
231 + }
232
233 uv_stop(main_loop);
234
logsmanagement/stress_test/logrotate.conf deleted
-5
@@ -1,5 +0,0 @@
1 -/tmp/netdata_log_management_stress_test_data/0.log /tmp/netdata_log_management_stress_test_data/1.log {
2 - rotate 1
3 - nocompress
4 - create
5 -}
\ No newline at end of file
logsmanagement/stress_test/logs_query.html deleted
-186
@@ -1,186 +0,0 @@
1 -<!DOCTYPE html>
2 -<html>
3 -<head>
4 - <style>
5 - #wrapper {
6 - width: 600px;
7 - border: 1px solid black;
8 - overflow: hidden;
9 - }
10 - #form {
11 - width: 400px;
12 - padding: 5px;
13 - float:left;
14 - }
15 - #results {
16 - padding: 5px;
17 - overflow: hidden;
18 - }
19 -
20 - table {
21 - table-layout: fixed;
22 - width: 80%;
23 - border-collapse: collapse;
24 - border: 3px solid rgb(0, 0, 0);
25 - }
26 -
27 - thead th:nth-child(1) {
28 - width: 20%;
29 - }
30 -
31 - thead th:nth-child(2) {
32 - width: 80%;
33 - }
34 -
35 - th,
36 - td {
37 - padding: 5px;
38 - }
39 - </style>
40 -</head>
41 -<body>
42 -
43 -<h2>Logs management queries</h2>
44 -<hr>
45 -
46 -
47 -<form id="form" name="query_form" action="javascript:submit_query()"">
48 - <h3>Query parameters</h3>
49 - <hr>
50 - <div id="sources"></div>
51 - <h4>Set query parameters:</h4>
52 - <label for="req_from">Requested from (epoch in ms):</label>
53 - <input type="text" id="req_from" name="req_from"><br><br>
54 - <label for="req_to">Requested to (epoch in ms):</label>
55 - <input type="text" id="req_to" name="req_to" ><br><br>
56 - <label for="req_quota">Quota in bytes:</label>
57 - <input type="text" id="req_quota" name="req_quota" ><br><br>
58 - <label for="keyword">Keyword / regex:</label>
59 - <input type="text" id="keyword" name="keyword" ><br><br>
60 - <label for="keyword_is_regex">Keyword is a regular expression:</label>
61 - <input type="checkbox" id="keyword_is_regex" name="keyword_is_regex" ><br><br>
62 - <label for="keyword_case_sensitive">Case-sensitive keyword / regex search:</label>
63 - <input type="checkbox" id="keyword_case_sensitive" name="keyword_case_sensitive" ><br><br>
64 - <label for="use_functions">Use functions instead of GET API:</label>
65 - <input type="checkbox" id="use_functions" name="use_functions" ><br><br>
66 - <input type="submit" value="Submit">
67 -</form>
68 -
69 -<div id="results"></div>
70 -
71 -<script>
72 -document.getElementById('req_from').value = '1682000000000';
73 -document.getElementById('req_to').value = '1782000000000';
74 -document.getElementById('req_quota').value = '10485760';
75 -document.getElementById('keyword_is_regex').checked = true;
76 -document.getElementById('keyword_case_sensitive').checked = false;
77 -
78 -function populate_sources_checkboxes() {
79 - const xmlhttp = new XMLHttpRequest();
80 - xmlhttp.onload = function() {
81 - results_obj = JSON.parse(this.responseText);
82 - let ele = document.getElementById('sources');
83 - ele.innerHTML += '<h4>Select log sources:</h4>';
84 - for (const [key, value] of Object.entries(results_obj["log sources"])) {
85 - ele.innerHTML += '<label>' + key + '</label>'
86 - if(value["DB dir"]){
87 - ele.innerHTML += '<input type="checkbox" name="' + key +'" class="sourceCheckBox"><br><br>';
88 - } else {
89 - ele.innerHTML += '<input type="checkbox" name="' + key +'" class="sourceCheckBox" disabled="disabled"> <i>(non-queryable)</i><br><br>';
90 - }
91 - }
92 - }
93 - xmlhttp.open("GET", "http://" + location.host + "/api/v1/logsmanagement_sources", true);
94 - xmlhttp.send(null);
95 -}
96 -window.onload = populate_sources_checkboxes;
97 -
98 -function submit_query() {
99 - const xmlhttp = new XMLHttpRequest();
100 - xmlhttp.onload = function() {
101 - results_obj = JSON.parse(this.responseText);
102 - logs_management_meta = document.getElementById('use_functions').checked ? results_obj.logs_management_meta : results_obj;
103 - text = "<h3>Results</h3><hr>" +
104 - "<b>API version:</b> " + logs_management_meta.api_version + "<br>" +
105 - "<b>Requested from (epoch in ms):</b> " + logs_management_meta.requested_from + "<br>" +
106 - "<b>Requested from:</b> " + new Date(logs_management_meta.requested_from) + "<br>" +
107 - "<b>Requested to (epoch in ms):</b> " + logs_management_meta.requested_to + "<br>" +
108 - "<b>Requested to:</b> " + new Date(logs_management_meta.requested_to) + "<br>" +
109 - "<b>Actual from (epoch in ms):</b> " + logs_management_meta.actual_from + "<br>" +
110 - "<b>Actual from:</b> " + new Date(logs_management_meta.actual_from) + "<br>" +
111 - "<b>Actual to (epoch in ms):</b> " + logs_management_meta.actual_to + "<br>" +
112 - "<b>Actual to:</b> " + new Date(logs_management_meta.actual_to) + "<br>" +
113 - "<b>Requested quota:</b> " + logs_management_meta.requested_quota + " KiB<br>" +
114 - "<b>Actual quota:</b> " + logs_management_meta.actual_quota + " KiB<br>" +
115 - "<b>Number of distrinct log records on this page:</b> " + logs_management_meta.num_lines + "<br>" +
116 - "<b>User time:</b> " + logs_management_meta.user_time + "<br>" +
117 - "<b>System time:</b> " + logs_management_meta.system_time + "<br>" +
118 - "<b>Error code:</b> " + logs_management_meta.error_code + "<br>" +
119 - "<b>Error message:</b> " + logs_management_meta.error + "<br>";
120 - text += "<br><table border='1'><thead><tr><th>Timestamp</th><th>Log Record</th></thead>";
121 -
122 - results_obj.data.forEach(function(data_entry) {
123 - datetime = new Date(data_entry[0]); // The 0 there is the key, which sets the date to the epoch
124 - data_entry[1].forEach(function(log_entry) {
125 - text += "<tr><td>" + datetime + "</td><td><pre style=\"white-space: pre-wrap; word-break: keep-all;\">" + log_entry + "</pre></td></tr>";
126 - });
127 - });
128 - text += "</table>";
129 - document.getElementById("results").innerHTML = text;
130 - }
131 - source = document.getElementById("sources");
132 - sources_value = encodeURIComponent(sources.value);
133 - req_from = document.getElementById("req_from");
134 - req_from_value = encodeURIComponent(req_from.value);
135 - req_to = document.getElementById("req_to");
136 - req_to_value = encodeURIComponent(req_to.value);
137 - req_quota = document.getElementById("req_quota");
138 - req_quota_value = encodeURIComponent(req_quota.value);
139 - keyword = document.getElementById("keyword");
140 - keyword_value = encodeURIComponent(keyword.value);
141 - sanitize_keyword = document.getElementById('keyword_is_regex').checked ? "0" : "1";
142 - ignore_case = document.getElementById('keyword_case_sensitive').checked ? "0" : "1";
143 -
144 -
145 - if(document.getElementById('use_functions').checked){
146 - xmlhttp_req = "http://" + location.host + "/api/v1/function?function=logsmanagement" +
147 - " from:" + req_from_value +
148 - " to:" + req_to_value +
149 - " quota:" + req_quota_value +
150 - " keyword:" + keyword_value +
151 - " sanitize_keyword: " + sanitize_keyword +
152 - " ignore_case:" + ignore_case;
153 -
154 - Array.from(document.getElementsByClassName("sourceCheckBox")).forEach(
155 - function(element, index, array) {
156 - if(element.checked){
157 - xmlhttp_req += ' chartname:"' + element.name + '"';
158 - }
159 - }
160 - );
161 - } else {
162 - xmlhttp_req = "http://" + location.host + "/api/v1/logsmanagement?" +
163 - "from=" + req_from_value +
164 - "&to=" + req_to_value +
165 - "&quota=" + req_quota_value +
166 - "&keyword=" + keyword_value +
167 - "&sanitize_keyword=" + sanitize_keyword +
168 - "&ignore_case=" + ignore_case;
169 -
170 - Array.from(document.getElementsByClassName("sourceCheckBox")).forEach(
171 - function(element, index, array) {
172 - if(element.checked){
173 - xmlhttp_req += "&chartname=" + element.name;
174 - }
175 - }
176 - );
177 - }
178 -
179 - console.log("Query:\n" + xmlhttp_req);
180 - xmlhttp.open("GET", xmlhttp_req, true);
181 - xmlhttp.send(null);
182 -}
183 -</script>
184 -
185 -</body>
186 -</html>
logsmanagement/stress_test/run_stress_test.sh deleted
-145
@@ -1,145 +0,0 @@
1 -#!/bin/bash
2 -
3 -# Default configuration options
4 -DEFAULT_BUILD_CLEAN_NETDATA=0
5 -DEFAULT_BUILD_FOR_RELEASE=1
6 -DEFAULT_NUM_LOG_SOURCES=0
7 -DEFAULT_DELAY_BETWEEN_MSG_WRITE=1000000
8 -DEFAULT_TOTAL_MSGS_PER_SOURCE=1000000
9 -DEFAULT_QUERIES_DELAY=3600
10 -DEFAULT_LOG_ROTATE_AFTER_SEC=3600
11 -DEFAULT_DELAY_OPEN_TO_WRITE_SEC=6
12 -DEFAULT_RUN_LOGS_MANAGEMENT_TESTS_ONLY=0
13 -
14 -if [ "$1" == "-h" ] || [ "$1" == "--help" ]; then
15 - echo "Usage: $(basename "$0") [ARGS]..."
16 - echo "Example: $(basename "$0") 0 1 2 1000 1000000 10 6 6 0"
17 - echo "Build, install and run netdata with logs management "
18 - echo "functionality enabled and (optional) stress tests."
19 - echo ""
20 - echo "arg[1]: [build_clean_netdata] Default: $DEFAULT_BUILD_CLEAN_NETDATA"
21 - echo "arg[2]: [build_for_release] Default: $DEFAULT_BUILD_FOR_RELEASE"
22 - echo "arg[3]: [num_log_sources] Default: $DEFAULT_NUM_LOG_SOURCES"
23 - echo "arg[4]: [delay_between_msg_write] Default: $DEFAULT_DELAY_BETWEEN_MSG_WRITE us"
24 - echo "arg[5]: [total_msgs_per_source] Default: $DEFAULT_TOTAL_MSGS_PER_SOURCE"
25 - echo "arg[6]: [queries_delay] Default: $DEFAULT_QUERIES_DELAY s"
26 - echo "arg[7]: [log_rotate_after_sec] Default: $DEFAULT_LOG_ROTATE_AFTER_SEC s"
27 - echo "arg[8]: [delay_open_to_write_sec] Default: $DEFAULT_DELAY_OPEN_TO_WRITE_SEC s"
28 - echo "arg[9]: [run_logs_management_tests_only] Default: $DEFAULT_RUN_LOGS_MANAGEMENT_TESTS_ONLY"
29 - exit 0
30 -fi
31 -
32 -build_clean_netdata="${1:-$DEFAULT_BUILD_CLEAN_NETDATA}"
33 -build_for_release="${2:-$DEFAULT_BUILD_FOR_RELEASE}"
34 -num_log_sources="${3:-$DEFAULT_NUM_LOG_SOURCES}"
35 -delay_between_msg_write="${4:-$DEFAULT_DELAY_BETWEEN_MSG_WRITE}"
36 -total_msgs_per_source="${5:-$DEFAULT_TOTAL_MSGS_PER_SOURCE}"
37 -queries_delay="${6:-$DEFAULT_QUERIES_DELAY}"
38 -log_rotate_after_sec="${7:-$DEFAULT_LOG_ROTATE_AFTER_SEC}"
39 -delay_open_to_write_sec="${8:-$DEFAULT_DELAY_OPEN_TO_WRITE_SEC}"
40 -run_logs_management_tests_only="${9:-$DEFAULT_RUN_LOGS_MANAGEMENT_TESTS_ONLY}"
41 -
42 -if [ "$num_log_sources" -le 0 ]
43 -then
44 - enable_stress_tests=0
45 -else
46 - enable_stress_tests=1
47 -fi
48 -
49 -INSTALL_PATH="/tmp"
50 -
51 -# Terminate running processes
52 -sudo killall -s KILL netdata
53 -sudo killall -s KILL stress_test
54 -sudo killall -s KILL -u netdata
55 -
56 -# Remove potentially persistent directories and files
57 -sudo rm -f $INSTALL_PATH/netdata/var/log/netdata/error.log
58 -sudo rm -rf $INSTALL_PATH/netdata/var/cache/netdata/logs_management_db
59 -sudo rm -rf $INSTALL_PATH/netdata_log_management_stress_test_data
60 -
61 -CPU_CORES=$(grep ^cpu\\scores /proc/cpuinfo | uniq | awk '{print $4}')
62 -
63 -# Build or rebuild Netdata
64 -if [ "$build_clean_netdata" -eq 1 ]
65 -then
66 - cd ../..
67 - sudo $INSTALL_PATH/netdata/usr/libexec/netdata/netdata-uninstaller.sh -y -f -e $INSTALL_PATH/netdata/etc/netdata/.environment
68 - sudo rm -rf $INSTALL_PATH/netdata/etc/netdata # Remove /etc/netdata if it persists for some reason
69 - sudo git clean -dxff && git submodule update --init --recursive --force
70 -
71 - if [ "$build_for_release" -eq 0 ]
72 - then
73 - c_flags="-O1 -ggdb -Wall -Wextra "
74 - c_flags+="-fno-omit-frame-pointer -Wformat-signedness -fstack-protector-all -Wformat-truncation=2 -Wunused-result "
75 - c_flags+="-DNETDATA_INTERNAL_CHECKS=1 -DNETDATA_DEV_MODE=1 -DLOGS_MANAGEMENT_STRESS_TEST=$enable_stress_tests "
76 - # c_flags+="-Wl,--no-as-needed -ldl "
77 - sudo CFLAGS="$c_flags" ./netdata-installer.sh \
78 - --dont-start-it \
79 - --dont-wait \
80 - --disable-lto \
81 - --disable-telemetry \
82 - --disable-go \
83 - --disable-ebpf \
84 - --disable-ml \
85 - --enable-logsmanagement-tests \
86 - --install-prefix $INSTALL_PATH
87 - else
88 - c_flags="-DLOGS_MANAGEMENT_STRESS_TEST=$enable_stress_tests "
89 - # c_flags+="-Wl,--no-as-needed -ldl "
90 - sudo CFLAGS="$c_flags" ./netdata-installer.sh \
91 - --dont-start-it \
92 - --dont-wait \
93 - --disable-telemetry \
94 - --install-prefix $INSTALL_PATH
95 - fi
96 -
97 - sudo cp logsmanagement/stress_test/logs_query.html "$INSTALL_PATH/netdata/usr/share/netdata/web"
98 - sudo chown -R netdata:netdata "$INSTALL_PATH/netdata/usr/share/netdata/web/logs_query.html"
99 -
100 -else
101 - cd ../.. && sudo make -j"$CPU_CORES" || exit 1 && sudo make install
102 - sudo chown -R netdata:netdata "$INSTALL_PATH/netdata/usr/share/netdata/web"
103 -fi
104 -
105 -cd logsmanagement/stress_test || exit
106 -
107 -if [ "$run_logs_management_tests_only" -eq 0 ]
108 -then
109 - # Rebuild and run stress test
110 - if [ "$num_log_sources" -gt 0 ]
111 - then
112 - sudo -u netdata -g netdata mkdir $INSTALL_PATH/netdata_log_management_stress_test_data
113 - gcc stress_test.c -DNUM_LOG_SOURCES="$num_log_sources" \
114 - -DDELAY_BETWEEN_MSG_WRITE="$delay_between_msg_write" \
115 - -DTOTAL_MSGS_PER_SOURCE="$total_msgs_per_source" \
116 - -DQUERIES_DELAY="$queries_delay" \
117 - -DLOG_ROTATE_AFTER_SEC="$log_rotate_after_sec" \
118 - -DDELAY_OPEN_TO_WRITE_SEC="$delay_open_to_write_sec" \
119 - -luv -Og -g -o stress_test
120 - sudo -u netdata -g netdata ./stress_test &
121 - sleep 1
122 - fi
123 -
124 - # Run Netdata
125 - if [ "$build_for_release" -eq 0 ]
126 - then
127 - sudo -u netdata -g netdata -s gdb -ex="set confirm off" -ex=run --args $INSTALL_PATH/netdata/usr/sbin/netdata -D
128 - elif [ "$build_for_release" -eq 2 ]
129 - then
130 - sudo -u netdata -g netdata -s gdb -ex="set confirm off" -ex=run --args $INSTALL_PATH/netdata/usr/libexec/netdata/plugins.d/logs-management.plugin
131 - else
132 - sudo -u netdata -g netdata ASAN_OPTIONS=log_path=stdout $INSTALL_PATH/netdata/usr/sbin/netdata -D
133 - fi
134 -else
135 - if [[ $($INSTALL_PATH/netdata/usr/sbin/netdata -W buildinfo | grep -Fc DLOGS_MANAGEMENT_STRESS_TEST) -eq 1 ]]
136 - then
137 - sudo -u netdata -g netdata ASAN_OPTIONS=log_path=/dev/null $INSTALL_PATH/netdata/usr/libexec/netdata/plugins.d/logs-management.plugin --unittest
138 - else
139 - echo "======================================================================="
140 - echo "run_logs_management_tests_only=1 but logs management tests cannot run."
141 - echo "Netdata must be configured with --enable-logsmanagement-tests."
142 - echo "Please rerun script with build_clean_netdata=1 and build_for_release=0."
143 - echo "======================================================================="
144 - fi
145 -fi
logsmanagement/stress_test/stress_test.c deleted
-386
@@ -1,386 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -/** @file stress_test.c
4 - * @brief Black-box stress testing of Netdata Logs Management
5 - */
6 -
7 -#include <assert.h>
8 -#include <inttypes.h>
9 -#include <stdio.h>
10 -#include <stdlib.h>
11 -#include <string.h>
12 -#include <sys/time.h>
13 -#include <time.h>
14 -#include <unistd.h>
15 -#include <uv.h>
16 -#include "../defaults.h"
17 -
18 -#include "stress_test.h"
19 -
20 -#define SIMULATED_LOGS_DIR "/tmp/netdata_log_management_stress_test_data"
21 -#define LOG_ROTATION_CMD "logrotate --force logrotate.conf -s /tmp/netdata_log_management_stress_test_data/logrotate_status"
22 -#define CSV_DELIMITER " "
23 -#define USE_LTSV_FORMAT 0
24 -
25 -#define MS_IN_S 1000
26 -#define US_IN_S 1000000
27 -
28 -#define NO_OF_FIELDS 10
29 -
30 -#ifdef _WIN32
31 -# define PIPENAME "\\\\?\\pipe\\netdata-logs-stress-test"
32 -#else
33 -# define PIPENAME "/tmp/netdata-logs-stress-test"
34 -#endif // _WIN32
35 -
36 -uv_process_t child_req;
37 -uv_process_options_t options;
38 -size_t max_msg_len;
39 -static int log_files_no;
40 -static volatile int log_rotated = 0;
41 -
42 -static char **all_fields_arr[NO_OF_FIELDS];
43 -static int all_fields_arr_sizes[NO_OF_FIELDS];
44 -
45 -static char *vhosts_ports[] = {
46 - "testhost.host:17",
47 - "invalidhost&%$:80",
48 - "testhost12.host:80",
49 - "testhost57.host:19999",
50 - "testhost111.host:77777",
51 - NULL
52 -};
53 -
54 -static char *vhosts[] = {
55 - "testhost.host",
56 - "invalidhost&%$",
57 - "testhost12.host",
58 - "testhost57.host",
59 - "testhost111.host",
60 - NULL
61 -};
62 -
63 -static char *ports[] = {
64 - "17",
65 - "80",
66 - "123",
67 - "8080",
68 - "19999",
69 - "77777",
70 - NULL
71 -};
72 -
73 -static char *req_clients[] = {
74 - "192.168.15.14",
75 - "192.168.2.1",
76 - "188.133.132.15",
77 - "156.134.132.15",
78 - "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
79 - "8501:0ab8:85a3:0000:0000:4a5d:0370:5213",
80 - "garbageAddress",
81 - NULL
82 -};
83 -
84 -static char *req_methods[] = {
85 - "GET",
86 - "POST",
87 - "UPDATE",
88 - "DELETE",
89 - "PATCH",
90 - "PUT",
91 - "INVALIDMETHOD",
92 - NULL
93 -};
94 -
95 -static char *resp_codes[] = {
96 - "5",
97 - "200",
98 - "202",
99 - "404",
100 - "410",
101 - "1027",
102 - NULL
103 -};
104 -
105 -static char *req_protos[] = {
106 - "HTTP/1",
107 - "HTTP/1.0",
108 - "HTTP/2",
109 - "HTTP/3",
110 - NULL
111 -};
112 -
113 -static char *req_sizes[] = {
114 - "236",
115 - "635",
116 - "954",
117 - "-",
118 - NULL
119 -};
120 -
121 -static char *resp_sizes[] = {
122 - "128",
123 - "452",
124 - "1056",
125 - "-",
126 - NULL
127 -};
128 -
129 -static char *ssl_protos[] = {
130 - "TLSv1",
131 - "TLSv1.1",
132 - "TLSv1.2",
133 - "TLSv1.3",
134 - "SSLv3",
135 - "-",
136 - NULL
137 -};
138 -
139 -static char *ssl_ciphers[] = {
140 - "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
141 - "TLS_PSK_WITH_AES_128_CCM_8",
142 - "ECDHE-RSA-AES128-GCM-SHA256",
143 - "TLS_RSA_WITH_DES_CBC_SHA",
144 - "TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256",
145 - "invalid_SSL_cipher_suite",
146 - "invalidSSLCipher",
147 - NULL
148 -};
149 -
150 -
151 -
152 -// "host:testhost.host\tport:80\treq_client:192.168.15.14\treq_method:\"GET\"\tresp_code:202\treq_proto:HTTP/1\treq_size:635\tresp_size:-\tssl_proto:TLSv1\tssl_cipher:TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256",
153 -
154 -
155 -// TODO: Include query.h instead of copy-pasting
156 -typedef struct db_query_params {
157 - msec_t start_timestamp;
158 - msec_t end_timestamp;
159 - char *filename;
160 - char *keyword;
161 - char *results;
162 - size_t results_size;
163 -} logs_query_params_t;
164 -
165 -size_t get_local_time(char *buf, size_t max_buf_size){
166 - time_t rawtime;
167 - struct tm *info;
168 - time( &rawtime );
169 -#if USE_LTSV_FORMAT
170 - return strftime (buf, max_buf_size, "time:[%d/%b/%Y:%H:%M:%S %z]",localtime( &rawtime ));
171 -#else
172 - return strftime (buf, max_buf_size, "[%d/%b/%Y:%H:%M:%S %z]",localtime( &rawtime ));
173 -#endif
174 -}
175 -
176 -static void produce_logs(void *arg) {
177 - msec_t runtime;
178 - msec_t start_time = now_realtime_msec();
179 - int log_no = *((int *)arg);
180 - int rc = 0;
181 - long int msgs_written = 0;
182 - uv_file file_handle;
183 - uv_buf_t uv_buf;
184 - char *buf = malloc(max_msg_len + 100);
185 -
186 - size_t buf_size;
187 - uv_fs_t write_req;
188 -
189 - uv_loop_t loop;
190 - uv_loop_init(&loop);
191 -
192 - char log_filename[100];
193 - sprintf(log_filename, "%s/%d.log", SIMULATED_LOGS_DIR, log_no);
194 -
195 - uv_fs_t open_req;
196 - rc = uv_fs_open(&loop, &open_req, log_filename, O_WRONLY | O_CREAT | O_TRUNC, 0777, NULL);
197 - if (rc < 0) {
198 - fprintf(stderr, "[STRESS_TEST] file_open() error: %s (%d) %s\n", log_filename, rc, uv_strerror(rc));
199 - } else {
200 - fprintf(stderr, "[STRESS_TEST] Opened file: %s\n", log_filename);
201 - file_handle = open_req.result; // open_req->result of a uv_fs_t is the file descriptor in case of the uv_fs_open
202 - }
203 - uv_fs_req_cleanup(&open_req);
204 -
205 - sleep(DELAY_OPEN_TO_WRITE_SEC);
206 -
207 - fprintf(stderr, "[STRESS_TEST] Start logging: %s\n", log_filename);
208 -
209 - int applied_close_open = 0;
210 - while (msgs_written < TOTAL_MSGS_PER_SOURCE) {
211 -
212 - size_t msg_timestamp_len = 50;
213 - msg_timestamp_len = get_local_time(buf, msg_timestamp_len);
214 - buf_size = msg_timestamp_len;
215 -
216 - for(int i = 0; i < NO_OF_FIELDS; i++){
217 - strcpy(&buf[buf_size++], CSV_DELIMITER);
218 - int arr_item_off = rand() % all_fields_arr_sizes[i];
219 - size_t arr_item_len = strlen(all_fields_arr[i][arr_item_off]);
220 - memcpy(&buf[buf_size], all_fields_arr[i][arr_item_off], arr_item_len);
221 - buf_size += arr_item_len;
222 - }
223 -
224 - buf[buf_size] = '\n';
225 -
226 - uv_buf = uv_buf_init(buf, buf_size + 1);
227 - uv_fs_write(&loop, &write_req, file_handle, &uv_buf, 1, -1, NULL);
228 - msgs_written++;
229 - if(!(msgs_written % 1000000)) fprintf(stderr, "[STRESS_TEST] Wrote %" PRId64 " messages to %s\n", msgs_written, log_filename);
230 - if(log_rotated && !applied_close_open) {
231 - uv_fs_t close_req;
232 - rc = uv_fs_close(&loop, &close_req, file_handle, NULL);
233 - if(rc) {
234 - fprintf(stderr, "[STRESS_TEST] file_close() error: %s (%d) %s\n", log_filename, rc, uv_strerror(rc));
235 - assert(0);
236 - }
237 - uv_fs_req_cleanup(&close_req);
238 -
239 - rc = uv_fs_open(&loop, &open_req, log_filename, O_WRONLY | O_CREAT | O_TRUNC , 0777, NULL);
240 - if (rc < 0) {
241 - fprintf(stderr, "[STRESS_TEST] file_open() error: %s (%d) %s\n", log_filename, rc, uv_strerror(rc));
242 - assert(0);
243 - } else {
244 - fprintf(stderr, "[STRESS_TEST] Rotated file: %s\n", log_filename);
245 - file_handle = open_req.result; // open_req->result of a uv_fs_t is the file descriptor in case of the uv_fs_open
246 - }
247 - uv_fs_req_cleanup(&open_req);
248 -
249 - applied_close_open = 1;
250 - fflush(stderr);
251 - }
252 -#if DELAY_BETWEEN_MSG_WRITE /**< Sleep delay (in us) in between consequent messages writes to a file **/
253 - usleep(DELAY_BETWEEN_MSG_WRITE);
254 -#endif
255 - }
256 -
257 - runtime = now_realtime_msec() - start_time - DELAY_OPEN_TO_WRITE_SEC * MS_IN_S;
258 - fprintf(stderr, "[STRESS_TEST] It took %" PRIu64 "ms to write %" PRId64 " log records in %s (%" PRId64 "k msgs/s))\n. ",
259 - runtime, msgs_written, log_filename, msgs_written / runtime);
260 -}
261 -
262 -static void log_rotate(void *arg){
263 - uv_sleep((DELAY_OPEN_TO_WRITE_SEC + LOG_ROTATE_AFTER_SEC) * MS_IN_S);
264 - assert(system(LOG_ROTATION_CMD) != -1);
265 - log_rotated = 1;
266 - fprintf(stderr, "[STRESS_TEST] Rotate log sources\n");
267 - fflush(stderr);
268 -}
269 -
270 -static void connect_cb(uv_connect_t* req, int status){
271 - int rc = 0;
272 - if(status < 0){
273 - fprintf(stderr, "[STRESS_TEST] Failed to connect to pipe!\n");
274 - exit(-1);
275 - }
276 - else
277 - fprintf(stderr, "[STRESS_TEST] Connection to pipe successful!\n");
278 -
279 - uv_write_t write_req;
280 - write_req.data = req->handle;
281 -
282 - // Serialise logs_query_params_t
283 - char *buf = calloc(100 * log_files_no, sizeof(char));
284 - sprintf(buf, "%d", log_files_no);
285 - for(int i = 0; i < log_files_no ; i++){
286 - sprintf(&buf[strlen(buf)], ",0,2147483646000," SIMULATED_LOGS_DIR "/%d.log,%s,%zu", i, " ", (size_t) MAX_LOG_MSG_SIZE);
287 - }
288 - fprintf(stderr, "[STRESS_TEST] Serialised DB query params: %s\n", buf);
289 -
290 - // Write to pipe
291 - uv_buf_t uv_buf = uv_buf_init(buf, strlen(buf));
292 - rc = uv_write(&write_req, (uv_stream_t *) req->handle, &uv_buf, 1, NULL);
293 - if (rc) {
294 - fprintf(stderr, "[STRESS_TEST] uv_write() error: %s\n", uv_strerror(rc));
295 - uv_close((uv_handle_t *) req->handle, NULL);
296 - exit(-1);
297 - }
298 -
299 -#if 1
300 - uv_shutdown_t shutdown_req;
301 - rc = uv_shutdown(&shutdown_req, (uv_stream_t *) req->handle, NULL);
302 - if (rc) {
303 - fprintf(stderr, "[STRESS_TEST] uv_shutdown() error: %s\n", uv_strerror(rc));
304 - uv_close((uv_handle_t *) req->handle, NULL);
305 - exit(-1);
306 - }
307 -#endif
308 -
309 -}
310 -
311 -int main(int argc, const char *argv[]) {
312 - fprintf(stdout, "*****************************************************************************\n"
313 - "%-15s %40s\n",
314 - "* [STRESS_TEST] Starting stress_test", "*");
315 -
316 - srand(time(NULL));
317 -
318 - all_fields_arr[0] = vhosts;
319 - all_fields_arr[1] = ports;
320 - all_fields_arr[2] = req_clients;
321 - all_fields_arr[3] = req_methods;
322 - all_fields_arr[4] = resp_codes;
323 - all_fields_arr[5] = req_protos;
324 - all_fields_arr[6] = req_sizes;
325 - all_fields_arr[7] = resp_sizes;
326 - all_fields_arr[8] = ssl_protos;
327 - all_fields_arr[9] = ssl_ciphers;
328 -
329 - for (int i = 0; i < NO_OF_FIELDS; i++){
330 - char **arr = all_fields_arr[i];
331 - int arr_size = 0;
332 - size_t max_item_len = 0;
333 - while(arr[arr_size] != NULL){
334 - size_t item_len = strlen(arr[arr_size]);
335 - if(item_len > max_item_len) max_item_len = item_len;
336 - arr_size++;
337 - }
338 - max_msg_len += max_item_len;
339 - all_fields_arr_sizes[i] = arr_size;
340 - }
341 -
342 -
343 - char *ptr;
344 - log_files_no = NUM_LOG_SOURCES;
345 - fprintf(stdout, "*****************************************************************************\n"
346 - "%-15s%42s %-10u%9s\n"
347 - "%-15s%42s %-10u%9s\n"
348 - "%-15s%42s %-10u%9s\n"
349 - "%-15s%42s %-10u%9s\n"
350 - "%-15s%42s %-10u%9s\n"
351 - "%-15s%42s %-10u%9s\n"
352 - "*****************************************************************************\n",
353 - "* [STRESS_TEST]", "Number of log sources to simulate:", log_files_no, "file *",
354 - "* [STRESS_TEST]", "Total log records to produce per source:", TOTAL_MSGS_PER_SOURCE, "records *",
355 - "* [STRESS_TEST]", "Delay between log record write to file:", DELAY_BETWEEN_MSG_WRITE, "us *",
356 - "* [STRESS_TEST]", "Log sources to rotate via create after:", LOG_ROTATE_AFTER_SEC, "s *",
357 - "* [STRESS_TEST]", "Queries to be executed after:", QUERIES_DELAY, "s *",
358 - "* [STRESS_TEST]", "Delay until start writing logs:", DELAY_OPEN_TO_WRITE_SEC, "s *");
359 -
360 - /* Start threads that produce log messages */
361 - uv_thread_t *log_producer_threads = malloc(log_files_no * sizeof(uv_thread_t));
362 - int *log_producer_thread_no = malloc(log_files_no * sizeof(int));
363 - for (int i = 0; i < log_files_no; i++) {
364 - fprintf(stderr, "[STRESS_TEST] Starting up log producer for %d.log\n", i);
365 - log_producer_thread_no[i] = i;
366 - assert(!uv_thread_create(&log_producer_threads[i], produce_logs, &log_producer_thread_no[i]));
367 - }
368 -
369 - uv_thread_t *log_rotate_thread = malloc(sizeof(uv_thread_t));
370 - assert(!uv_thread_create(log_rotate_thread, log_rotate, NULL));
371 -
372 - for (int j = 0; j < log_files_no; j++) {
373 - uv_thread_join(&log_producer_threads[j]);
374 - }
375 -
376 - sleep(QUERIES_DELAY); // Give netdata-logs more than LOG_FILE_READ_INTERVAL to ensure the entire log file has been read.
377 -
378 - uv_pipe_t query_data_pipe;
379 - uv_pipe_init(uv_default_loop(), &query_data_pipe, 1);
380 - uv_connect_t connect_req;
381 - uv_pipe_connect(&connect_req, &query_data_pipe, PIPENAME, connect_cb);
382 -
383 - uv_run(uv_default_loop(), UV_RUN_DEFAULT);
384 -
385 - uv_close((uv_handle_t *) &query_data_pipe, NULL);
386 -}