@cryptotaxi247 / netdata-1 / commits / fd9b49b86

Streaming re-organization (#18941)

split streaming into multiple files

Costa Tsaousis committed Nov 5, 2024 at 15:09 UTC fd9b49b8657438c320ab571a102e867acaa04e1d
57 files changed +2483 -2415
CMakeLists.txt
+36 -22
@@ -973,7 +973,7 @@ set(LIBH2O_FILES
973 set(DAEMON_FILES
974 src/daemon/buildinfo.c
975 src/daemon/buildinfo.h
976 - src/daemon/common.c
976 + src/daemon/h2o-common.c
977 src/daemon/common.h
978 src/daemon/daemon.c
979 src/daemon/daemon.h
@@ -1017,7 +1017,7 @@ set(H2O_FILES
1017 src/web/server/h2o/http_server.h
1018 src/web/server/h2o/h2o_utils.c
1019 src/web/server/h2o/h2o_utils.h
1020 - src/web/server/h2o/streaming.c
1020 + src/web/server/h2o/rrdpush.c
1021 src/web/server/h2o/streaming.h
1022 src/web/server/h2o/connlist.c
1023 src/web/server/h2o/connlist.h
@@ -1371,35 +1371,49 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
1371 )
1372
1373 set(STREAMING_PLUGIN_FILES
1374 - src/streaming/rrdpush.c
1374 src/streaming/rrdpush.h
1376 - src/streaming/compression.c
1377 - src/streaming/compression.h
1378 - src/streaming/compression_brotli.c
1379 - src/streaming/compression_brotli.h
1380 - src/streaming/compression_gzip.c
1381 - src/streaming/compression_gzip.h
1382 - src/streaming/compression_lz4.c
1383 - src/streaming/compression_lz4.h
1384 - src/streaming/compression_zstd.c
1385 - src/streaming/compression_zstd.h
1375 + src/streaming/stream-compression/compression.c
1376 + src/streaming/stream-compression/compression.h
1377 + src/streaming/stream-compression/brotli.c
1378 + src/streaming/stream-compression/brotli.h
1379 + src/streaming/stream-compression/gzip.c
1380 + src/streaming/stream-compression/gzip.h
1381 + src/streaming/stream-compression/lz4.c
1382 + src/streaming/stream-compression/lz4.h
1383 + src/streaming/stream-compression/zstd.c
1384 + src/streaming/stream-compression/zstd.h
1385 src/streaming/receiver.c
1386 src/streaming/sender.c
1387 src/streaming/replication.c
1388 src/streaming/replication.h
1390 - src/streaming/common.h
1389 + src/streaming/h2o-common.h
1390 src/streaming/protocol/command-nodeid.c
1391 src/streaming/protocol/commands.c
1392 src/streaming/protocol/commands.h
1393 src/streaming/protocol/command-claimed_id.c
1395 - src/streaming/stream_path.c
1396 - src/streaming/stream_path.h
1397 - src/streaming/stream_capabilities.c
1398 - src/streaming/stream_capabilities.h
1399 - src/streaming/sender_connect.c
1400 - src/streaming/sender_internals.h
1401 - src/streaming/sender_execute.c
1402 - src/streaming/sender_commit.c
1394 + src/streaming/stream-path.c
1395 + src/streaming/stream-path.h
1396 + src/streaming/stream-capabilities.c
1397 + src/streaming/stream-capabilities.h
1398 + src/streaming/sender-connect.c
1399 + src/streaming/sender-internals.h
1400 + src/streaming/sender-execute.c
1401 + src/streaming/sender-commit.c
1402 + src/streaming/sender-destinations.c
1403 + src/streaming/stream-handshake.c
1404 + src/streaming/protocol/command-function.c
1405 + src/streaming/protocol/command-host-labels.c
1406 + src/streaming/protocol/command-chart-definition.c
1407 + src/streaming/protocol/command-begin-set-end.c
1408 + src/streaming/protocol/command-host-variables.c
1409 + src/streaming/stream-conf.c
1410 + src/streaming/stream-conf.h
1411 + src/streaming/stream-handshake.h
1412 + src/streaming/sender.h
1413 + src/streaming/sender-destinations.h
1414 + src/streaming/rrdhost-status.c
1415 + src/streaming/rrdhost-status.h
1416 + src/streaming/receiver.h
1417 )
1418
1419 set(WEB_PLUGIN_FILES
src/daemon/analytics.c
+2 -2
@@ -522,7 +522,7 @@ void analytics_gather_mutable_meta_data(void)
522 analytics_alarms_notifications();
523
524 analytics_set_data(
525 - &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || configured_as_parent()) ? "true" : "false");
525 + &analytics_data.netdata_config_is_parent, (rrdhost_hosts_available() > 1 || stream_conf_configured_as_parent()) ? "true" : "false");
526
527 analytics_set_data(&analytics_data.netdata_host_agent_claimed, is_agent_claimed() ? "true" : "false");
528
@@ -619,7 +619,7 @@ cleanup:
619 */
620 void set_late_analytics_variables(struct rrdhost_system_info *system_info)
621 {
622 - analytics_set_data(&analytics_data.netdata_config_stream_enabled, default_rrdpush_enabled ? "true" : "false");
622 + analytics_set_data(&analytics_data.netdata_config_stream_enabled, stream_conf_send_enabled ? "true" : "false");
623 analytics_set_data_str(&analytics_data.netdata_config_memory_mode, (char *)rrd_memory_mode_name(default_rrd_memory_mode));
624 analytics_set_data(&analytics_data.netdata_host_cloud_enabled, "true");
625
src/daemon/analytics.h
+1
@@ -76,6 +76,7 @@ struct analytics_data {
76 bool exporting_enabled;
77 };
78
79 +struct rrdhost_system_info;
80 void set_late_analytics_variables(struct rrdhost_system_info *system_info);
81 void analytics_free_data(void);
82 void analytics_log_shell(void);
src/daemon/common.h
-1
@@ -32,7 +32,6 @@
32 // streaming metrics between netdata servers
33 #include "streaming/rrdpush.h"
34
35 -
35 // anomaly detection
36 #include "ml/ml.h"
37
src/daemon/h2o-common.c renamed
src/daemon/main.c
+1 -1
@@ -1503,7 +1503,7 @@ int unittest_prepare_rrd(const char **user) {
1503 fprintf(stderr, "rrd_init failed for unittest\n");
1504 return 1;
1505 }
1506 - default_rrdpush_enabled = 0;
1506 + stream_conf_send_enabled = 0;
1507
1508 return 0;
1509 }
src/database/contexts/api_v2_contexts.c
+1 -1
@@ -422,7 +422,7 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
422 // stale - connected but not having live data
423 // reachable - connected with live data
424 // pruned - not connected for some time and has been removed
425 - buffer_json_member_add_string(wb, "state", rrdhost_state_cloud_emulation(host) ? "reachable" : "stale");
425 + buffer_json_member_add_string(wb, "state", rrdhost_is_online(host) ? "reachable" : "stale");
426
427 rrdhost_health_to_json_v2(wb, "health", &s);
428 agent_capabilities_to_json(wb, host, "capabilities");
src/database/contexts/api_v2_contexts_agents.c
+1 -1
@@ -44,7 +44,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
44 sending++;
45
46 if(host != localhost) {
47 - if (rrdhost_state_cloud_emulation(host))
47 + if (rrdhost_is_online(host))
48 receiving++;
49 else
50 archived++;
src/database/engine/dbengine-stresstest.c
+7 -7
@@ -22,13 +22,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
22 default_rrd_history_entries,
23 RRD_MEMORY_MODE_DBENGINE,
24 health_plugin_enabled(),
25 - default_rrdpush_enabled,
26 - default_rrdpush_destination,
27 - default_rrdpush_api_key,
28 - default_rrdpush_send_charts_matching,
29 - default_rrdpush_enable_replication,
30 - default_rrdpush_seconds_to_replicate,
31 - default_rrdpush_replication_step,
25 + stream_conf_send_enabled,
26 + stream_conf_send_destination,
27 + stream_conf_send_api_key,
28 + stream_conf_send_charts_matching,
29 + stream_conf_replication_enabled,
30 + stream_conf_replication_period,
31 + stream_conf_replication_step,
32 NULL,
33 0
34 );
src/database/engine/dbengine-unittest.c
+7 -7
@@ -108,13 +108,13 @@ static RRDHOST *dbengine_rrdhost_find_or_create(char *name) {
108 default_rrd_history_entries,
109 RRD_MEMORY_MODE_DBENGINE,
110 health_plugin_enabled(),
111 - default_rrdpush_enabled,
112 - default_rrdpush_destination,
113 - default_rrdpush_api_key,
114 - default_rrdpush_send_charts_matching,
115 - default_rrdpush_enable_replication,
116 - default_rrdpush_seconds_to_replicate,
117 - default_rrdpush_replication_step,
111 + stream_conf_send_enabled,
112 + stream_conf_send_destination,
113 + stream_conf_send_api_key,
114 + stream_conf_send_charts_matching,
115 + stream_conf_replication_enabled,
116 + stream_conf_replication_period,
117 + stream_conf_replication_step,
118 NULL,
119 0
120 );
src/database/rrd.h
+41 -41
@@ -98,14 +98,53 @@ struct ml_metrics_statistics {
98 size_t silenced;
99 };
100
101 +
102 +// use this for configuration flags, not for state control
103 +// flags are set/unset in a manner that is not thread safe
104 +// and may lead to missing information.
105 +typedef enum __attribute__ ((__packed__)) rrdset_flags {
106 + RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
107 + RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete
108 + RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases
109 + RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases
110 +
111 + RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
112 + RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
113 +
114 + RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation
115 + RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
116 + RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous
117 + RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting
118 + RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
119 + RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions
120 +
121 + RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored
122 + RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts.
123 + RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id
124 + RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name
125 +
126 + RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18),
127 +
128 + RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress
129 + RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication
130 + RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress
131 + RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication
132 +
133 + RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 23), // a custom variable has been updated and needs to be exposed to parent
134 +
135 + RRDSET_FLAG_COLLECTION_FINISHED = (1 << 24), // when set, data collection is not available for this chart
136 +
137 + RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 25), // this chart has at least one rrdcal linked
138 +} RRDSET_FLAGS;
139 +
140 #include "daemon/common.h"
141 #include "web/api/queries/query.h"
142 #include "web/api/queries/rrdr.h"
143 #include "health/rrdvar.h"
144 #include "health/rrdcalc.h"
145 #include "rrdlabels.h"
107 -#include "streaming/stream_capabilities.h"
108 -#include "streaming/stream_path.h"
146 +#include "streaming/stream-capabilities.h"
147 +#include "streaming/stream-path.h"
148 #include "streaming/rrdpush.h"
149 //#include "aclk/aclk_rrdhost_state.h"
150 #include "sqlite/sqlite_health.h"
@@ -664,45 +703,6 @@ STORAGE_ENGINE* storage_engine_find(const char* name);
703 // ----------------------------------------------------------------------------
704 // RRDSET - this is a chart
705
667 -// use this for configuration flags, not for state control
668 -// flags are set/unset in a manner that is not thread safe
669 -// and may lead to missing information.
670 -
671 -typedef enum __attribute__ ((__packed__)) rrdset_flags {
672 - RRDSET_FLAG_DEBUG = (1 << 2), // enables or disables debugging for a chart
673 - RRDSET_FLAG_OBSOLETE = (1 << 3), // this is marked by the collector/module as obsolete
674 - RRDSET_FLAG_EXPORTING_SEND = (1 << 4), // if set, this chart should be sent to Prometheus web API and external databases
675 - RRDSET_FLAG_EXPORTING_IGNORE = (1 << 5), // if set, this chart should not be sent to Prometheus web API and external databases
676 -
677 - RRDSET_FLAG_UPSTREAM_SEND = (1 << 6), // if set, this chart should be sent upstream (streaming)
678 - RRDSET_FLAG_UPSTREAM_IGNORE = (1 << 7), // if set, this chart should not be sent upstream (streaming)
679 -
680 - RRDSET_FLAG_STORE_FIRST = (1 << 8), // if set, do not eliminate the first collection during interpolation
681 - RRDSET_FLAG_HETEROGENEOUS = (1 << 9), // if set, the chart is not homogeneous (dimensions in it have multiple algorithms, multipliers or dividers)
682 - RRDSET_FLAG_HOMOGENEOUS_CHECK = (1 << 10), // if set, the chart should be checked to determine if the dimensions are homogeneous
683 - RRDSET_FLAG_HIDDEN = (1 << 11), // if set, do not show this chart on the dashboard, but use it for exporting
684 - RRDSET_FLAG_SYNC_CLOCK = (1 << 12), // if set, microseconds on next data collection will be ignored (the chart will be synced to now)
685 - RRDSET_FLAG_OBSOLETE_DIMENSIONS = (1 << 13), // this is marked by the collector/module when a chart has obsolete dimensions
686 -
687 - RRDSET_FLAG_METADATA_UPDATE = (1 << 14), // Mark that metadata needs to be stored
688 - RRDSET_FLAG_ANOMALY_DETECTION = (1 << 15), // flag to identify anomaly detection charts.
689 - RRDSET_FLAG_INDEXED_ID = (1 << 16), // the rrdset is indexed by its id
690 - RRDSET_FLAG_INDEXED_NAME = (1 << 17), // the rrdset is indexed by its name
691 -
692 - RRDSET_FLAG_PENDING_HEALTH_INITIALIZATION = (1 << 18),
693 -
694 - RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS = (1 << 19), // the sending side has replication in progress
695 - RRDSET_FLAG_SENDER_REPLICATION_FINISHED = (1 << 20), // the sending side has completed replication
696 - RRDSET_FLAG_RECEIVER_REPLICATION_IN_PROGRESS = (1 << 21), // the receiving side has replication in progress
697 - RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED = (1 << 22), // the receiving side has completed replication
698 -
699 - RRDSET_FLAG_UPSTREAM_SEND_VARIABLES = (1 << 23), // a custom variable has been updated and needs to be exposed to parent
700 -
701 - RRDSET_FLAG_COLLECTION_FINISHED = (1 << 24), // when set, data collection is not available for this chart
702 -
703 - RRDSET_FLAG_HAS_RRDCALC_LINKED = (1 << 25), // this chart has at least one rrdcal linked
704 -} RRDSET_FLAGS;
705 -
706 #define rrdset_flag_get(st) __atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE)
707 #define rrdset_flag_check(st, flag) (__atomic_load_n(&((st)->flags), __ATOMIC_ACQUIRE) & (flag))
708 #define rrdset_flag_set(st, flag) __atomic_or_fetch(&((st)->flags), flag, __ATOMIC_RELEASE)
src/database/rrdhost.c
+11 -238
@@ -1047,9 +1047,9 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool
1047 dbengine_enabled = true;
1048 }
1049 else {
1050 - rrdpush_init();
1050 + stream_conf_init();
1051
1052 - if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || rrdpush_receiver_needs_dbengine()) {
1052 + if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE || stream_conf_receiver_needs_dbengine()) {
1053 nd_log(NDLS_DAEMON, NDLP_DEBUG,
1054 "DBENGINE: Initializing ...");
1055
@@ -1094,14 +1094,14 @@ int rrd_init(const char *hostname, struct rrdhost_system_info *system_info, bool
1094 , default_rrd_history_entries
1095 , default_rrd_memory_mode
1096 , health_plugin_enabled()
1097 - , default_rrdpush_enabled
1098 - , default_rrdpush_destination
1099 - , default_rrdpush_api_key
1100 - , default_rrdpush_send_charts_matching
1101 - , default_rrdpush_enable_replication
1102 - , default_rrdpush_seconds_to_replicate
1103 - , default_rrdpush_replication_step
1104 - , system_info
1097 + ,
1098 + stream_conf_send_enabled,
1099 + stream_conf_send_destination,
1100 + stream_conf_send_api_key,
1101 + stream_conf_send_charts_matching,
1102 + stream_conf_replication_enabled,
1103 + stream_conf_replication_period,
1104 + stream_conf_replication_step, system_info
1105 , 1
1106 , 0
1107 );
@@ -1184,7 +1184,7 @@ static void rrdhost_streaming_sender_structures_init(RRDHOST *host)
1184 host->sender->rrdpush_sender_socket = -1;
1185 host->sender->disabled_capabilities = STREAM_CAP_NONE;
1186
1187 - if(!default_rrdpush_compression_enabled)
1187 + if(!stream_conf_compression_enabled)
1188 host->sender->disabled_capabilities |= STREAM_CAP_COMPRESSIONS_AVAILABLE;
1189
1190 spinlock_init(&host->sender->spinlock);
@@ -1666,235 +1666,8 @@ int rrdhost_set_system_info_variable(struct rrdhost_system_info *system_info, ch
1666 return res;
1667 }
1668
1669 -static NETDATA_DOUBLE rrdhost_sender_replication_completion_unsafe(RRDHOST *host, time_t now, size_t *instances) {
1670 - size_t charts = rrdhost_sender_replicating_charts(host);
1671 - NETDATA_DOUBLE completion;
1672 - if(!charts || !host->sender || !host->sender->replication.oldest_request_after_t)
1673 - completion = 100.0;
1674 - else if(!host->sender->replication.latest_completed_before_t || host->sender->replication.latest_completed_before_t < host->sender->replication.oldest_request_after_t)
1675 - completion = 0.0;
1676 - else {
1677 - time_t total = now - host->sender->replication.oldest_request_after_t;
1678 - time_t current = host->sender->replication.latest_completed_before_t - host->sender->replication.oldest_request_after_t;
1679 - completion = (NETDATA_DOUBLE) current * 100.0 / (NETDATA_DOUBLE) total;
1680 - }
1681 -
1682 - *instances = charts;
1683 -
1684 - return completion;
1685 -}
1686 -
1669 bool rrdhost_matches_window(RRDHOST *host, time_t after, time_t before, time_t now) {
1670 time_t first_time_s, last_time_s;
1671 rrdhost_retention(host, now, rrdhost_is_online(host), &first_time_s, &last_time_s);
1672 return query_matches_retention(after, before, first_time_s, last_time_s, 0);
1673 }
1692 -
1693 -bool rrdhost_state_cloud_emulation(RRDHOST *host) {
1694 - return rrdhost_is_online(host);
1695 -}
1696 -
1697 -void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
1698 - memset(s, 0, sizeof(*s));
1699 -
1700 - s->host = host;
1701 - s->now = now;
1702 -
1703 - RRDHOST_FLAGS flags = __atomic_load_n(&host->flags, __ATOMIC_RELAXED);
1704 -
1705 - // --- dyncfg ---
1706 -
1707 - s->dyncfg.status = dyncfg_available_for_rrdhost(host) ? RRDHOST_DYNCFG_STATUS_AVAILABLE : RRDHOST_DYNCFG_STATUS_UNAVAILABLE;
1708 -
1709 - // --- db ---
1710 -
1711 - bool online = rrdhost_is_online(host);
1712 -
1713 - rrdhost_retention(host, now, online, &s->db.first_time_s, &s->db.last_time_s);
1714 - s->db.metrics = host->rrdctx.metrics;
1715 - s->db.instances = host->rrdctx.instances;
1716 - s->db.contexts = dictionary_entries(host->rrdctx.contexts);
1717 - if(!s->db.first_time_s || !s->db.last_time_s || !s->db.metrics || !s->db.instances || !s->db.contexts ||
1718 - (flags & (RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
1719 - s->db.status = RRDHOST_DB_STATUS_INITIALIZING;
1720 - else
1721 - s->db.status = RRDHOST_DB_STATUS_QUERYABLE;
1722 -
1723 - s->db.mode = host->rrd_memory_mode;
1724 -
1725 - // --- ingest ---
1726 -
1727 - s->ingest.since = MAX(host->child_connect_time, host->child_disconnected_time);
1728 - s->ingest.reason = (online) ? STREAM_HANDSHAKE_NEVER : host->rrdpush_last_receiver_exit_reason;
1729 -
1730 - spinlock_lock(&host->receiver_lock);
1731 - s->ingest.hops = (host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
1732 - bool has_receiver = false;
1733 - if (host->receiver) {
1734 - has_receiver = true;
1735 - s->ingest.replication.instances = rrdhost_receiver_replicating_charts(host);
1736 - s->ingest.replication.completion = host->rrdpush_receiver_replication_percent;
1737 - s->ingest.replication.in_progress = s->ingest.replication.instances > 0;
1738 -
1739 - s->ingest.capabilities = host->receiver->capabilities;
1740 - s->ingest.peers = socket_peers(host->receiver->fd);
1741 - s->ingest.ssl = SSL_connection(&host->receiver->ssl);
1742 - }
1743 - spinlock_unlock(&host->receiver_lock);
1744 -
1745 - if (online) {
1746 - if(s->db.status == RRDHOST_DB_STATUS_INITIALIZING)
1747 - s->ingest.status = RRDHOST_INGEST_STATUS_INITIALIZING;
1748 -
1749 - else if (host == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
1750 - s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
1751 - s->ingest.since = netdata_start_time;
1752 - }
1753 -
1754 - else if (s->ingest.replication.in_progress)
1755 - s->ingest.status = RRDHOST_INGEST_STATUS_REPLICATING;
1756 -
1757 - else
1758 - s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
1759 - }
1760 - else {
1761 - if (!s->ingest.since) {
1762 - s->ingest.status = RRDHOST_INGEST_STATUS_ARCHIVED;
1763 - s->ingest.since = s->db.last_time_s;
1764 - }
1765 -
1766 - else
1767 - s->ingest.status = RRDHOST_INGEST_STATUS_OFFLINE;
1768 - }
1769 -
1770 - if(host == localhost)
1771 - s->ingest.type = RRDHOST_INGEST_TYPE_LOCALHOST;
1772 - else if(has_receiver || rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
1773 - s->ingest.type = RRDHOST_INGEST_TYPE_CHILD;
1774 - else if(rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST))
1775 - s->ingest.type = RRDHOST_INGEST_TYPE_VIRTUAL;
1776 - else
1777 - s->ingest.type = RRDHOST_INGEST_TYPE_ARCHIVED;
1778 -
1779 - s->ingest.id = host->rrdpush_receiver_connection_counter;
1780 -
1781 - if(!s->ingest.since)
1782 - s->ingest.since = netdata_start_time;
1783 -
1784 - if(s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE)
1785 - s->db.liveness = RRDHOST_DB_LIVENESS_LIVE;
1786 - else
1787 - s->db.liveness = RRDHOST_DB_LIVENESS_STALE;
1788 -
1789 - // --- stream ---
1790 -
1791 - if (!host->sender) {
1792 - s->stream.status = RRDHOST_STREAM_STATUS_DISABLED;
1793 - s->stream.hops = s->ingest.hops + 1;
1794 - }
1795 - else {
1796 - sender_lock(host->sender);
1797 -
1798 - s->stream.since = host->sender->last_state_since_t;
1799 - s->stream.peers = socket_peers(host->sender->rrdpush_sender_socket);
1800 - s->stream.ssl = SSL_connection(&host->sender->ssl);
1801 -
1802 - memcpy(s->stream.sent_bytes_on_this_connection_per_type,
1803 - host->sender->sent_bytes_on_this_connection_per_type,
1804 - MIN(sizeof(s->stream.sent_bytes_on_this_connection_per_type),
1805 - sizeof(host->sender->sent_bytes_on_this_connection_per_type)));
1806 -
1807 - if (rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED)) {
1808 - s->stream.hops = host->sender->hops;
1809 - s->stream.reason = STREAM_HANDSHAKE_NEVER;
1810 - s->stream.capabilities = host->sender->capabilities;
1811 -
1812 - s->stream.replication.completion = rrdhost_sender_replication_completion_unsafe(host, now, &s->stream.replication.instances);
1813 - s->stream.replication.in_progress = s->stream.replication.instances > 0;
1814 -
1815 - if(s->stream.replication.in_progress)
1816 - s->stream.status = RRDHOST_STREAM_STATUS_REPLICATING;
1817 - else
1818 - s->stream.status = RRDHOST_STREAM_STATUS_ONLINE;
1819 -
1820 - s->stream.compression = host->sender->compressor.initialized;
1821 - }
1822 - else {
1823 - s->stream.status = RRDHOST_STREAM_STATUS_OFFLINE;
1824 - s->stream.hops = s->ingest.hops + 1;
1825 - s->stream.reason = host->sender->exit.reason;
1826 - }
1827 -
1828 - sender_unlock(host->sender);
1829 - }
1830 -
1831 - s->stream.id = host->rrdpush_sender_connection_counter;
1832 -
1833 - if(!s->stream.since)
1834 - s->stream.since = netdata_start_time;
1835 -
1836 - // --- ml ---
1837 -
1838 - if(ml_host_get_host_status(host, &s->ml.metrics)) {
1839 - s->ml.type = RRDHOST_ML_TYPE_SELF;
1840 -
1841 - if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE || s->ingest.status == RRDHOST_INGEST_STATUS_ARCHIVED)
1842 - s->ml.status = RRDHOST_ML_STATUS_OFFLINE;
1843 - else
1844 - s->ml.status = RRDHOST_ML_STATUS_RUNNING;
1845 - }
1846 - else if(stream_has_capability(&s->ingest, STREAM_CAP_DATA_WITH_ML)) {
1847 - s->ml.type = RRDHOST_ML_TYPE_RECEIVED;
1848 - s->ml.status = RRDHOST_ML_STATUS_RUNNING;
1849 - }
1850 - else {
1851 - // does not receive ML, does not run ML
1852 - s->ml.type = RRDHOST_ML_TYPE_DISABLED;
1853 - s->ml.status = RRDHOST_ML_STATUS_DISABLED;
1854 - }
1855 -
1856 - // --- health ---
1857 -
1858 - if(host->health.health_enabled) {
1859 - if(flags & RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)
1860 - s->health.status = RRDHOST_HEALTH_STATUS_INITIALIZING;
1861 - else {
1862 - s->health.status = RRDHOST_HEALTH_STATUS_RUNNING;
1863 -
1864 - RRDCALC *rc;
1865 - foreach_rrdcalc_in_rrdhost_read(host, rc) {
1866 - if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
1867 - continue;
1868 -
1869 - switch (rc->status) {
1870 - default:
1871 - case RRDCALC_STATUS_REMOVED:
1872 - break;
1873 -
1874 - case RRDCALC_STATUS_CLEAR:
1875 - s->health.alerts.clear++;
1876 - break;
1877 -
1878 - case RRDCALC_STATUS_WARNING:
1879 - s->health.alerts.warning++;
1880 - break;
1881 -
1882 - case RRDCALC_STATUS_CRITICAL:
1883 - s->health.alerts.critical++;
1884 - break;
1885 -
1886 - case RRDCALC_STATUS_UNDEFINED:
1887 - s->health.alerts.undefined++;
1888 - break;
1889 -
1890 - case RRDCALC_STATUS_UNINITIALIZED:
1891 - s->health.alerts.uninitialized++;
1892 - break;
1893 - }
1894 - }
1895 - foreach_rrdcalc_in_rrdhost_done(rc);
1896 - }
1897 - }
1898 - else
1899 - s->health.status = RRDHOST_HEALTH_STATUS_DISABLED;
1900 -}
src/plugins.d/pluginsd_parser.c
+7 -7
@@ -190,13 +190,13 @@ static inline PARSER_RC pluginsd_host_define_end(char **words __maybe_unused, si
190 default_rrd_history_entries,
191 default_rrd_memory_mode,
192 health_plugin_enabled(),
193 - default_rrdpush_enabled,
194 - default_rrdpush_destination,
195 - default_rrdpush_api_key,
196 - default_rrdpush_send_charts_matching,
197 - default_rrdpush_enable_replication,
198 - default_rrdpush_seconds_to_replicate,
199 - default_rrdpush_replication_step,
193 + stream_conf_send_enabled,
194 + stream_conf_send_destination,
195 + stream_conf_send_api_key,
196 + stream_conf_send_charts_matching,
197 + stream_conf_replication_enabled,
198 + stream_conf_replication_period,
199 + stream_conf_replication_step,
200 rrdhost_labels_to_system_info(parser->user.host_define.rrdlabels),
201 false);
202
src/streaming/h2o-common.h renamed
src/streaming/protocol/command-begin-set-end.c new
+126
@@ -0,0 +1,126 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "commands.h"
4 +#include "plugins.d/pluginsd_internals.h"
5 +
6 +static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_state *s __maybe_unused, RRDSET_FLAGS flags) {
7 + buffer_fast_strcat(wb, "BEGIN \"", 7);
8 + buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
9 + buffer_fast_strcat(wb, "\" ", 2);
10 +
11 + if(st->last_collected_time.tv_sec > st->rrdpush.sender.resync_time_s)
12 + buffer_print_uint64(wb, st->usec_since_last_update);
13 + else
14 + buffer_fast_strcat(wb, "0", 1);
15 +
16 + buffer_fast_strcat(wb, "\n", 1);
17 +
18 + RRDDIM *rd;
19 + rrddim_foreach_read(rd, st) {
20 + if(unlikely(!rrddim_check_updated(rd)))
21 + continue;
22 +
23 + if(likely(rrddim_check_upstream_exposed_collector(rd))) {
24 + buffer_fast_strcat(wb, "SET \"", 5);
25 + buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
26 + buffer_fast_strcat(wb, "\" = ", 4);
27 + buffer_print_int64(wb, rd->collector.collected_value);
28 + buffer_fast_strcat(wb, "\n", 1);
29 + }
30 + else {
31 + internal_error(true, "STREAM: 'host:%s/chart:%s/dim:%s' flag 'exposed' is updated but not exposed",
32 + rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
33 + // we will include it in the next iteration
34 + rrddim_metadata_updated(rd);
35 + }
36 + }
37 + rrddim_foreach_done(rd);
38 +
39 + if(unlikely(flags & RRDSET_FLAG_UPSTREAM_SEND_VARIABLES))
40 + rrdvar_print_to_streaming_custom_chart_variables(st, wb);
41 +
42 + buffer_fast_strcat(wb, "END\n", 4);
43 +}
44 +
45 +void rrdset_push_metrics_v1(RRDSET_STREAM_BUFFER *rsb, RRDSET *st) {
46 + RRDHOST *host = st->rrdhost;
47 + rrdpush_send_chart_metrics(rsb->wb, st, host->sender, rsb->rrdset_flags);
48 +}
49 +
50 +void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
51 + if(!rsb->wb || !rsb->v2 || !netdata_double_isnumber(n) || !does_storage_number_exist(flags))
52 + return;
53 +
54 + bool with_slots = stream_has_capability(rsb, STREAM_CAP_SLOTS) ? true : false;
55 + NUMBER_ENCODING integer_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
56 + NUMBER_ENCODING doubles_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
57 + BUFFER *wb = rsb->wb;
58 + time_t point_end_time_s = (time_t)(point_end_time_ut / USEC_PER_SEC);
59 + if(unlikely(rsb->last_point_end_time_s != point_end_time_s)) {
60 +
61 + if(unlikely(rsb->begin_v2_added))
62 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
63 +
64 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
65 +
66 + if(with_slots) {
67 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
68 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->rrdpush.sender.chart_slot);
69 + }
70 +
71 + buffer_fast_strcat(wb, " '", 2);
72 + buffer_fast_strcat(wb, rrdset_id(rd->rrdset), string_strlen(rd->rrdset->id));
73 + buffer_fast_strcat(wb, "' ", 2);
74 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->update_every);
75 + buffer_fast_strcat(wb, " ", 1);
76 + buffer_print_uint64_encoded(wb, integer_encoding, point_end_time_s);
77 + buffer_fast_strcat(wb, " ", 1);
78 + if(point_end_time_s == rsb->wall_clock_time)
79 + buffer_fast_strcat(wb, "#", 1);
80 + else
81 + buffer_print_uint64_encoded(wb, integer_encoding, rsb->wall_clock_time);
82 + buffer_fast_strcat(wb, "\n", 1);
83 +
84 + rsb->last_point_end_time_s = point_end_time_s;
85 + rsb->begin_v2_added = true;
86 + }
87 +
88 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);
89 +
90 + if(with_slots) {
91 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
92 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
93 + }
94 +
95 + buffer_fast_strcat(wb, " '", 2);
96 + buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
97 + buffer_fast_strcat(wb, "' ", 2);
98 + buffer_print_int64_encoded(wb, integer_encoding, rd->collector.last_collected_value);
99 + buffer_fast_strcat(wb, " ", 1);
100 +
101 + if((NETDATA_DOUBLE)rd->collector.last_collected_value == n)
102 + buffer_fast_strcat(wb, "#", 1);
103 + else
104 + buffer_print_netdata_double_encoded(wb, doubles_encoding, n);
105 +
106 + buffer_fast_strcat(wb, " ", 1);
107 + buffer_print_sn_flags(wb, flags, true);
108 + buffer_fast_strcat(wb, "\n", 1);
109 +}
110 +
111 +void rrdset_push_metrics_finished(RRDSET_STREAM_BUFFER *rsb, RRDSET *st) {
112 + if(!rsb->wb)
113 + return;
114 +
115 + if(rsb->v2 && rsb->begin_v2_added) {
116 + if(unlikely(rsb->rrdset_flags & RRDSET_FLAG_UPSTREAM_SEND_VARIABLES))
117 + rrdvar_print_to_streaming_custom_chart_variables(st, rsb->wb);
118 +
119 + buffer_fast_strcat(rsb->wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
120 + }
121 +
122 + sender_commit(st->rrdhost->sender, rsb->wb, STREAM_TRAFFIC_TYPE_DATA);
123 +
124 + *rsb = (RRDSET_STREAM_BUFFER){ .wb = NULL, };
125 +}
126 +
src/streaming/protocol/command-chart-definition.c new
+206
@@ -0,0 +1,206 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "commands.h"
4 +#include "plugins.d/pluginsd_internals.h"
5 +
6 +// chart labels
7 +static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
8 + BUFFER *wb = (BUFFER *)data;
9 + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL " \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL));
10 + return 1;
11 +}
12 +
13 +static void rrdpush_send_clabels(BUFFER *wb, RRDSET *st) {
14 + if (st->rrdlabels) {
15 + if(rrdlabels_walkthrough_read(st->rrdlabels, send_clabels_callback, wb) > 0)
16 + buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL_COMMIT "\n");
17 + }
18 +}
19 +
20 +// Send the current chart definition.
21 +// Assumes that collector thread has already called sender_start for mutex / buffer state.
22 +bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
23 + uint32_t version = rrdset_metadata_version(st);
24 +
25 + RRDHOST *host = st->rrdhost;
26 + NUMBER_ENCODING integer_encoding = stream_has_capability(host->sender, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
27 + bool with_slots = stream_has_capability(host->sender, STREAM_CAP_SLOTS) ? true : false;
28 +
29 + bool replication_progress = false;
30 +
31 + // properly set the name for the remote end to parse it
32 + char *name = "";
33 + if(likely(st->name)) {
34 + if(unlikely(st->id != st->name)) {
35 + // they differ
36 + name = strchr(rrdset_name(st), '.');
37 + if(name)
38 + name++;
39 + else
40 + name = "";
41 + }
42 + }
43 +
44 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_CHART, sizeof(PLUGINSD_KEYWORD_CHART) - 1);
45 +
46 + if(with_slots) {
47 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
48 + buffer_print_uint64_encoded(wb, integer_encoding, st->rrdpush.sender.chart_slot);
49 + }
50 +
51 + // send the chart
52 + buffer_sprintf(
53 + wb
54 + , " \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\" \"%s\" \"%s\"\n"
55 + , rrdset_id(st)
56 + , name
57 + , rrdset_title(st)
58 + , rrdset_units(st)
59 + , rrdset_family(st)
60 + , rrdset_context(st)
61 + , rrdset_type_name(st->chart_type)
62 + , st->priority
63 + , st->update_every
64 + , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
65 + , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
66 + , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
67 + , rrdset_plugin_name(st)
68 + , rrdset_module_name(st)
69 + );
70 +
71 + // send the chart labels
72 + if (stream_has_capability(host->sender, STREAM_CAP_CLABELS))
73 + rrdpush_send_clabels(wb, st);
74 +
75 + // send the dimensions
76 + RRDDIM *rd;
77 + rrddim_foreach_read(rd, st) {
78 + buffer_fast_strcat(wb, PLUGINSD_KEYWORD_DIMENSION, sizeof(PLUGINSD_KEYWORD_DIMENSION) - 1);
79 +
80 + if(with_slots) {
81 + buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
82 + buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
83 + }
84 +
85 + buffer_sprintf(
86 + wb
87 + , " \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\"\n"
88 + , rrddim_id(rd)
89 + , rrddim_name(rd)
90 + , rrd_algorithm_name(rd->algorithm)
91 + , rd->multiplier
92 + , rd->divisor
93 + , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
94 + , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":""
95 + , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
96 + );
97 + }
98 + rrddim_foreach_done(rd);
99 +
100 + // send the chart functions
101 + if(stream_has_capability(host->sender, STREAM_CAP_FUNCTIONS))
102 + rrd_chart_functions_expose_rrdpush(st, wb);
103 +
104 + // send the chart local custom variables
105 + rrdvar_print_to_streaming_custom_chart_variables(st, wb);
106 +
107 + if (stream_has_capability(host->sender, STREAM_CAP_REPLICATION)) {
108 + time_t db_first_time_t, db_last_time_t;
109 +
110 + time_t now = now_realtime_sec();
111 + rrdset_get_retention_of_tier_for_collected_chart(st, &db_first_time_t, &db_last_time_t, now, 0);
112 +
113 + buffer_sprintf(wb, PLUGINSD_KEYWORD_CHART_DEFINITION_END " %llu %llu %llu\n",
114 + (unsigned long long)db_first_time_t,
115 + (unsigned long long)db_last_time_t,
116 + (unsigned long long)now);
117 +
118 + if(!rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS)) {
119 + rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);
120 + rrdset_flag_clear(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
121 + rrdhost_sender_replicating_charts_plus_one(st->rrdhost);
122 + }
123 + replication_progress = true;
124 +
125 +#ifdef NETDATA_LOG_REPLICATION_REQUESTS
126 + internal_error(true, "REPLAY: 'host:%s/chart:%s' replication starts",
127 + rrdhost_hostname(st->rrdhost), rrdset_id(st));
128 +#endif
129 + }
130 +
131 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
132 +
133 + // we can set the exposed flag, after we commit the buffer
134 + // because replication may pick it up prematurely
135 + rrddim_foreach_read(rd, st) {
136 + rrddim_metadata_exposed_upstream(rd, version);
137 + }
138 + rrddim_foreach_done(rd);
139 + rrdset_metadata_exposed_upstream(st, version);
140 +
141 + st->rrdpush.sender.resync_time_s = st->last_collected_time.tv_sec + (stream_conf_initial_clock_resync_iterations * st->update_every);
142 + return replication_progress;
143 +}
144 +
145 +bool should_send_chart_matching(RRDSET *st, RRDSET_FLAGS flags) {
146 + if(!(flags & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
147 + return false;
148 +
149 + if(unlikely(!(flags & (RRDSET_FLAG_UPSTREAM_SEND | RRDSET_FLAG_UPSTREAM_IGNORE)))) {
150 + RRDHOST *host = st->rrdhost;
151 +
152 + if (flags & RRDSET_FLAG_ANOMALY_DETECTION) {
153 + if(ml_streaming_enabled())
154 + rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
155 + else
156 + rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);
157 + }
158 + else {
159 + int negative = 0, positive = 0;
160 + SIMPLE_PATTERN_RESULT r;
161 +
162 + r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->context, NULL, 0);
163 + if(r == SP_MATCHED_POSITIVE) positive++;
164 + else if(r == SP_MATCHED_NEGATIVE) negative++;
165 +
166 + if(!negative) {
167 + r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->name, NULL, 0);
168 + if (r == SP_MATCHED_POSITIVE) positive++;
169 + else if (r == SP_MATCHED_NEGATIVE) negative++;
170 + }
171 +
172 + if(!negative) {
173 + r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->id, NULL, 0);
174 + if (r == SP_MATCHED_POSITIVE) positive++;
175 + else if (r == SP_MATCHED_NEGATIVE) negative++;
176 + }
177 +
178 + if(!negative && positive)
179 + rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
180 + else
181 + rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);
182 + }
183 +
184 + // get the flags again, to know how to respond
185 + flags = rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_SEND|RRDSET_FLAG_UPSTREAM_IGNORE);
186 + }
187 +
188 + return flags & RRDSET_FLAG_UPSTREAM_SEND;
189 +}
190 +
191 +// Called from the internal collectors to mark a chart obsolete.
192 +bool rrdset_push_chart_definition_now(RRDSET *st) {
193 + RRDHOST *host = st->rrdhost;
194 +
195 + if(unlikely(!rrdhost_can_send_definitions_to_parent(host)
196 + || !should_send_chart_matching(st, rrdset_flag_get(st)))) {
197 + return false;
198 + }
199 +
200 + BUFFER *wb = sender_start(host->sender);
201 + rrdpush_send_chart_definition(wb, st);
202 + sender_thread_buffer_free();
203 +
204 + return true;
205 +}
206 +
src/streaming/protocol/command-function.c new
+20
@@ -0,0 +1,20 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "commands.h"
4 +#include "plugins.d/pluginsd_internals.h"
5 +
6 +void rrdpush_send_global_functions(RRDHOST *host) {
7 + if(!stream_has_capability(host->sender, STREAM_CAP_FUNCTIONS))
8 + return;
9 +
10 + if(unlikely(!rrdhost_can_send_definitions_to_parent(host)))
11 + return;
12 +
13 + BUFFER *wb = sender_start(host->sender);
14 +
15 + rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
16 +
17 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
18 +
19 + sender_thread_buffer_free();
20 +}
src/streaming/protocol/command-host-labels.c new
+25
@@ -0,0 +1,25 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "commands.h"
4 +#include "plugins.d/pluginsd_internals.h"
5 +
6 +static int send_labels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
7 + BUFFER *wb = (BUFFER *)data;
8 + buffer_sprintf(wb, "LABEL \"%s\" = %d \"%s\"\n", name, ls, value);
9 + return 1;
10 +}
11 +
12 +void rrdpush_send_host_labels(RRDHOST *host) {
13 + if(unlikely(!rrdhost_can_send_definitions_to_parent(host)
14 + || !stream_has_capability(host->sender, STREAM_CAP_HLABELS)))
15 + return;
16 +
17 + BUFFER *wb = sender_start(host->sender);
18 +
19 + rrdlabels_walkthrough_read(host->rrdlabels, send_labels_callback, wb);
20 + buffer_sprintf(wb, "OVERWRITE %s\n", "labels");
21 +
22 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
23 +
24 + sender_thread_buffer_free();
25 +}
src/streaming/protocol/command-host-variables.c new
+52
@@ -0,0 +1,52 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "commands.h"
4 +#include "plugins.d/pluginsd_internals.h"
5 +
6 +static inline void rrdpush_sender_add_host_variable_to_buffer(BUFFER *wb, const RRDVAR_ACQUIRED *rva) {
7 + buffer_sprintf(
8 + wb
9 + , "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n"
10 + , rrdvar_name(rva)
11 + , rrdvar2number(rva)
12 + );
13 +
14 + netdata_log_debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rva), rrdvar2number(rva));
15 +}
16 +
17 +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva) {
18 + if(rrdhost_can_send_definitions_to_parent(host)) {
19 + BUFFER *wb = sender_start(host->sender);
20 + rrdpush_sender_add_host_variable_to_buffer(wb, rva);
21 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
22 + sender_thread_buffer_free();
23 + }
24 +}
25 +
26 +struct custom_host_variables_callback {
27 + BUFFER *wb;
28 +};
29 +
30 +static int rrdpush_sender_thread_custom_host_variables_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdvar_ptr __maybe_unused, void *struct_ptr) {
31 + const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item;
32 + struct custom_host_variables_callback *tmp = struct_ptr;
33 + BUFFER *wb = tmp->wb;
34 +
35 + rrdpush_sender_add_host_variable_to_buffer(wb, rv);
36 + return 1;
37 +}
38 +
39 +void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
40 + if(rrdhost_can_send_definitions_to_parent(host)) {
41 + BUFFER *wb = sender_start(host->sender);
42 + struct custom_host_variables_callback tmp = {
43 + .wb = wb
44 + };
45 + int ret = rrdvar_walkthrough_read(host->rrdvars, rrdpush_sender_thread_custom_host_variables_callback, &tmp);
46 + (void)ret;
47 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
48 + sender_thread_buffer_free();
49 +
50 + netdata_log_debug(D_STREAM, "RRDVAR sent %d VARIABLES", ret);
51 + }
52 +}
src/streaming/protocol/commands.c
+55
@@ -1,3 +1,58 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "commands.h"
4 +
5 +RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock_time) {
6 + RRDHOST *host = st->rrdhost;
7 +
8 + // fetch the flags we need to check with one atomic operation
9 + RRDHOST_FLAGS host_flags = __atomic_load_n(&host->flags, __ATOMIC_SEQ_CST);
10 +
11 + // check if we are not connected
12 + if(unlikely(!(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS))) {
13 +
14 + if(unlikely(!(host_flags & (RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))))
15 + rrdpush_sender_thread_spawn(host);
16 +
17 + if(unlikely(!(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS))) {
18 + rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS);
19 + nd_log_daemon(NDLP_NOTICE, "STREAM %s [send]: not ready - collected metrics are not sent to parent.", rrdhost_hostname(host));
20 + }
21 +
22 + return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
23 + }
24 + else if(unlikely(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS)) {
25 + nd_log_daemon(NDLP_INFO, "STREAM %s [send]: sending metrics to parent...", rrdhost_hostname(host));
26 + rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS);
27 + }
28 +
29 + if(unlikely(host_flags & RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED)) {
30 + BUFFER *wb = sender_start(host->sender);
31 + rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
32 + sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
33 + }
34 +
35 + bool exposed_upstream = rrdset_check_upstream_exposed(st);
36 + RRDSET_FLAGS rrdset_flags = rrdset_flag_get(st);
37 + bool replication_in_progress = !(rrdset_flags & RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
38 +
39 + if(unlikely((exposed_upstream && replication_in_progress) ||
40 + !should_send_chart_matching(st, rrdset_flags)))
41 + return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
42 +
43 + if(unlikely(!exposed_upstream)) {
44 + BUFFER *wb = sender_start(host->sender);
45 + replication_in_progress = rrdpush_send_chart_definition(wb, st);
46 + }
47 +
48 + if(replication_in_progress)
49 + return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
50 +
51 + return (RRDSET_STREAM_BUFFER) {
52 + .capabilities = host->sender->capabilities,
53 + .v2 = stream_has_capability(host->sender, STREAM_CAP_INTERPOLATED),
54 + .rrdset_flags = rrdset_flags,
55 + .wb = sender_start(host->sender),
56 + .wall_clock_time = wall_clock_time,
57 + };
58 +}
src/streaming/protocol/commands.h
+27
@@ -3,12 +3,39 @@
3 #ifndef NETDATA_STREAMING_PROTCOL_COMMANDS_H
4 #define NETDATA_STREAMING_PROTCOL_COMMANDS_H
5
6 +#include "database/rrd.h"
7 #include "../rrdpush.h"
8
9 +typedef struct rrdset_stream_buffer {
10 + STREAM_CAPABILITIES capabilities;
11 + bool v2;
12 + bool begin_v2_added;
13 + time_t wall_clock_time;
14 + RRDSET_FLAGS rrdset_flags;
15 + time_t last_point_end_time_s;
16 + BUFFER *wb;
17 +} RRDSET_STREAM_BUFFER;
18 +
19 +RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock_time);
20 +
21 void rrdpush_sender_get_node_and_claim_id_from_parent(struct sender_state *s);
22 void rrdpush_receiver_send_node_and_claim_id_to_child(RRDHOST *host);
23 void rrdpush_sender_clear_parent_claim_id(RRDHOST *host);
24
25 void rrdpush_sender_send_claimed_id(RRDHOST *host);
26
27 +void rrdpush_send_global_functions(RRDHOST *host);
28 +void rrdpush_send_host_labels(RRDHOST *host);
29 +
30 +void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host);
31 +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva);
32 +
33 +bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st);
34 +bool rrdset_push_chart_definition_now(RRDSET *st);
35 +bool should_send_chart_matching(RRDSET *st, RRDSET_FLAGS flags);
36 +
37 +void rrdset_push_metrics_v1(RRDSET_STREAM_BUFFER *rsb, RRDSET *st);
38 +void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);
39 +void rrdset_push_metrics_finished(RRDSET_STREAM_BUFFER *rsb, RRDSET *st);
40 +
41 #endif //NETDATA_STREAMING_PROTCOL_COMMANDS_H
src/streaming/receiver.c
+467 -10
@@ -3,7 +3,10 @@
3 #include "rrdpush.h"
4 #include "web/server/h2o/http_server.h"
5
6 -extern struct config stream_config;
6 +// When a child disconnects this is the maximum we will wait
7 +// before we update the cloud that the child is offline
8 +#define MAX_CHILD_DISC_DELAY (30000)
9 +#define MAX_CHILD_DISC_TOLERANCE (125 / 100)
10
11 void receiver_state_free(struct receiver_state *rpt) {
12 netdata_ssl_close(&rpt->ssl);
@@ -553,7 +556,7 @@ static void rrdpush_send_error_on_taken_over_connection(struct receiver_state *r
556 5);
557 }
558
556 -void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, const char *status, ND_LOG_FIELD_PRIORITY priority) {
559 +static void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, const char *status, ND_LOG_FIELD_PRIORITY priority) {
560 // this function may be called BEFORE we spawn the receiver thread
561 // so, we need to add the fields again (it does not harm)
562 ND_LOG_STACK lgs[] = {
@@ -589,14 +592,14 @@ static void rrdpush_receive(struct receiver_state *rpt)
592 rpt->config.alarms_delay = 60;
593 rpt->config.alarms_history = HEALTH_LOG_RETENTION_DEFAULT;
594
592 - rpt->config.rrdpush_enabled = (int)default_rrdpush_enabled;
593 - rpt->config.rrdpush_destination = default_rrdpush_destination;
594 - rpt->config.rrdpush_api_key = default_rrdpush_api_key;
595 - rpt->config.rrdpush_send_charts_matching = default_rrdpush_send_charts_matching;
595 + rpt->config.rrdpush_enabled = (int)stream_conf_send_enabled;
596 + rpt->config.rrdpush_destination = stream_conf_send_destination;
597 + rpt->config.rrdpush_api_key = stream_conf_send_api_key;
598 + rpt->config.rrdpush_send_charts_matching = stream_conf_send_charts_matching;
599
597 - rpt->config.rrdpush_enable_replication = default_rrdpush_enable_replication;
598 - rpt->config.rrdpush_seconds_to_replicate = default_rrdpush_seconds_to_replicate;
599 - rpt->config.rrdpush_replication_step = default_rrdpush_replication_step;
600 + rpt->config.rrdpush_enable_replication = stream_conf_replication_enabled;
601 + rpt->config.rrdpush_seconds_to_replicate = stream_conf_replication_period;
602 + rpt->config.rrdpush_replication_step = stream_conf_replication_step;
603
604 rpt->config.update_every = (int)appconfig_get_duration_seconds(&stream_config, rpt->machine_guid, "update every", rpt->config.update_every);
605 if(rpt->config.update_every < 0) rpt->config.update_every = 1;
@@ -648,7 +651,7 @@ static void rrdpush_receive(struct receiver_state *rpt)
651 rpt->config.rrdpush_replication_step = appconfig_get_number(&stream_config, rpt->key, "replication step", rpt->config.rrdpush_replication_step);
652 rpt->config.rrdpush_replication_step = appconfig_get_number(&stream_config, rpt->machine_guid, "replication step", rpt->config.rrdpush_replication_step);
653
651 - rpt->config.rrdpush_compression = default_rrdpush_compression_enabled;
654 + rpt->config.rrdpush_compression = stream_conf_compression_enabled;
655 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->key, "enable compression", rpt->config.rrdpush_compression);
656 rpt->config.rrdpush_compression = appconfig_get_boolean(&stream_config, rpt->machine_guid, "enable compression", rpt->config.rrdpush_compression);
657
@@ -922,3 +925,457 @@ void *rrdpush_receiver_thread(void *ptr) {
925 receiver_state_free(rpt);
926 return NULL;
927 }
928 +
929 +int rrdpush_receiver_permission_denied(struct web_client *w) {
930 + // we always respond with the same message and error code
931 + // to prevent an attacker from gaining info about the error
932 + buffer_flush(w->response.data);
933 + buffer_strcat(w->response.data, START_STREAMING_ERROR_NOT_PERMITTED);
934 + return HTTP_RESP_UNAUTHORIZED;
935 +}
936 +
937 +int rrdpush_receiver_too_busy_now(struct web_client *w) {
938 + // we always respond with the same message and error code
939 + // to prevent an attacker from gaining info about the error
940 + buffer_flush(w->response.data);
941 + buffer_strcat(w->response.data, START_STREAMING_ERROR_BUSY_TRY_LATER);
942 + return HTTP_RESP_SERVICE_UNAVAILABLE;
943 +}
944 +
945 +static void rrdpush_receiver_takeover_web_connection(struct web_client *w, struct receiver_state *rpt) {
946 + rpt->fd = w->ifd;
947 +
948 + rpt->ssl.conn = w->ssl.conn;
949 + rpt->ssl.state = w->ssl.state;
950 +
951 + w->ssl = NETDATA_SSL_UNSET_CONNECTION;
952 +
953 + WEB_CLIENT_IS_DEAD(w);
954 +
955 + if(web_server_mode == WEB_SERVER_MODE_STATIC_THREADED) {
956 + web_client_flag_set(w, WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET);
957 + }
958 + else {
959 + if(w->ifd == w->ofd)
960 + w->ifd = w->ofd = -1;
961 + else
962 + w->ifd = -1;
963 + }
964 +
965 + buffer_flush(w->response.data);
966 +}
967 +
968 +int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx __maybe_unused) {
969 +
970 + if(!service_running(ABILITY_STREAMING_CONNECTIONS))
971 + return rrdpush_receiver_too_busy_now(w);
972 +
973 + struct receiver_state *rpt = callocz(1, sizeof(*rpt));
974 + rpt->connected_since_s = now_realtime_sec();
975 + rpt->last_msg_t = now_monotonic_sec();
976 + rpt->hops = 1;
977 +
978 + rpt->capabilities = STREAM_CAP_INVALID;
979 +
980 +#ifdef ENABLE_H2O
981 + rpt->h2o_ctx = h2o_ctx;
982 +#endif
983 +
984 + __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_receivers, sizeof(*rpt), __ATOMIC_RELAXED);
985 + __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);
986 +
987 + rpt->system_info = callocz(1, sizeof(struct rrdhost_system_info));
988 + rpt->system_info->hops = rpt->hops;
989 +
990 + rpt->fd = -1;
991 + rpt->client_ip = strdupz(w->client_ip);
992 + rpt->client_port = strdupz(w->client_port);
993 +
994 + rpt->ssl = NETDATA_SSL_UNSET_CONNECTION;
995 +
996 + rpt->config.update_every = default_rrd_update_every;
997 +
998 + // parse the parameters and fill rpt and rpt->system_info
999 +
1000 + while(decoded_query_string) {
1001 + char *value = strsep_skip_consecutive_separators(&decoded_query_string, "&");
1002 + if(!value || !*value) continue;
1003 +
1004 + char *name = strsep_skip_consecutive_separators(&value, "=");
1005 + if(!name || !*name) continue;
1006 + if(!value || !*value) continue;
1007 +
1008 + if(!strcmp(name, "key") && !rpt->key)
1009 + rpt->key = strdupz(value);
1010 +
1011 + else if(!strcmp(name, "hostname") && !rpt->hostname)
1012 + rpt->hostname = strdupz(value);
1013 +
1014 + else if(!strcmp(name, "registry_hostname") && !rpt->registry_hostname)
1015 + rpt->registry_hostname = strdupz(value);
1016 +
1017 + else if(!strcmp(name, "machine_guid") && !rpt->machine_guid)
1018 + rpt->machine_guid = strdupz(value);
1019 +
1020 + else if(!strcmp(name, "update_every"))
1021 + rpt->config.update_every = (int)strtoul(value, NULL, 0);
1022 +
1023 + else if(!strcmp(name, "os") && !rpt->os)
1024 + rpt->os = strdupz(value);
1025 +
1026 + else if(!strcmp(name, "timezone") && !rpt->timezone)
1027 + rpt->timezone = strdupz(value);
1028 +
1029 + else if(!strcmp(name, "abbrev_timezone") && !rpt->abbrev_timezone)
1030 + rpt->abbrev_timezone = strdupz(value);
1031 +
1032 + else if(!strcmp(name, "utc_offset"))
1033 + rpt->utc_offset = (int32_t)strtol(value, NULL, 0);
1034 +
1035 + else if(!strcmp(name, "hops"))
1036 + rpt->hops = rpt->system_info->hops = (uint16_t) strtoul(value, NULL, 0);
1037 +
1038 + else if(!strcmp(name, "ml_capable"))
1039 + rpt->system_info->ml_capable = strtoul(value, NULL, 0);
1040 +
1041 + else if(!strcmp(name, "ml_enabled"))
1042 + rpt->system_info->ml_enabled = strtoul(value, NULL, 0);
1043 +
1044 + else if(!strcmp(name, "mc_version"))
1045 + rpt->system_info->mc_version = strtoul(value, NULL, 0);
1046 +
1047 + else if(!strcmp(name, "ver") && (rpt->capabilities & STREAM_CAP_INVALID))
1048 + rpt->capabilities = convert_stream_version_to_capabilities(strtoul(value, NULL, 0), NULL, false);
1049 +
1050 + else {
1051 + // An old Netdata child does not have a compatible streaming protocol, map to something sane.
1052 + if (!strcmp(name, "NETDATA_SYSTEM_OS_NAME"))
1053 + name = "NETDATA_HOST_OS_NAME";
1054 +
1055 + else if (!strcmp(name, "NETDATA_SYSTEM_OS_ID"))
1056 + name = "NETDATA_HOST_OS_ID";
1057 +
1058 + else if (!strcmp(name, "NETDATA_SYSTEM_OS_ID_LIKE"))
1059 + name = "NETDATA_HOST_OS_ID_LIKE";
1060 +
1061 + else if (!strcmp(name, "NETDATA_SYSTEM_OS_VERSION"))
1062 + name = "NETDATA_HOST_OS_VERSION";
1063 +
1064 + else if (!strcmp(name, "NETDATA_SYSTEM_OS_VERSION_ID"))
1065 + name = "NETDATA_HOST_OS_VERSION_ID";
1066 +
1067 + else if (!strcmp(name, "NETDATA_SYSTEM_OS_DETECTION"))
1068 + name = "NETDATA_HOST_OS_DETECTION";
1069 +
1070 + else if(!strcmp(name, "NETDATA_PROTOCOL_VERSION") && (rpt->capabilities & STREAM_CAP_INVALID))
1071 + rpt->capabilities = convert_stream_version_to_capabilities(1, NULL, false);
1072 +
1073 + if (unlikely(rrdhost_set_system_info_variable(rpt->system_info, name, value))) {
1074 + nd_log_daemon(NDLP_NOTICE, "STREAM '%s' [receive from [%s]:%s]: "
1075 + "request has parameter '%s' = '%s', which is not used."
1076 + , (rpt->hostname && *rpt->hostname) ? rpt->hostname : "-"
1077 + , rpt->client_ip, rpt->client_port
1078 + , name, value);
1079 + }
1080 + }
1081 + }
1082 +
1083 + if (rpt->capabilities & STREAM_CAP_INVALID)
1084 + // no version is supplied, assume version 0;
1085 + rpt->capabilities = convert_stream_version_to_capabilities(0, NULL, false);
1086 +
1087 + // find the program name and version
1088 + if(w->user_agent && w->user_agent[0]) {
1089 + char *t = strchr(w->user_agent, '/');
1090 + if(t && *t) {
1091 + *t = '\0';
1092 + t++;
1093 + }
1094 +
1095 + rpt->program_name = strdupz(w->user_agent);
1096 + if(t && *t) rpt->program_version = strdupz(t);
1097 + }
1098 +
1099 + // check if we should accept this connection
1100 +
1101 + if(!rpt->key || !*rpt->key) {
1102 + rrdpush_receive_log_status(
1103 + rpt, "request without an API key, rejecting connection",
1104 + RRDPUSH_STATUS_NO_API_KEY, NDLP_WARNING);
1105 +
1106 + receiver_state_free(rpt);
1107 + return rrdpush_receiver_permission_denied(w);
1108 + }
1109 +
1110 + if(!rpt->hostname || !*rpt->hostname) {
1111 + rrdpush_receive_log_status(
1112 + rpt, "request without a hostname, rejecting connection",
1113 + RRDPUSH_STATUS_NO_HOSTNAME, NDLP_WARNING);
1114 +
1115 + receiver_state_free(rpt);
1116 + return rrdpush_receiver_permission_denied(w);
1117 + }
1118 +
1119 + if(!rpt->registry_hostname)
1120 + rpt->registry_hostname = strdupz(rpt->hostname);
1121 +
1122 + if(!rpt->machine_guid || !*rpt->machine_guid) {
1123 + rrdpush_receive_log_status(
1124 + rpt, "request without a machine GUID, rejecting connection",
1125 + RRDPUSH_STATUS_NO_MACHINE_GUID, NDLP_WARNING);
1126 +
1127 + receiver_state_free(rpt);
1128 + return rrdpush_receiver_permission_denied(w);
1129 + }
1130 +
1131 + {
1132 + char buf[GUID_LEN + 1];
1133 +
1134 + if (regenerate_guid(rpt->key, buf) == -1) {
1135 + rrdpush_receive_log_status(
1136 + rpt, "API key is not a valid UUID (use the command uuidgen to generate one)",
1137 + RRDPUSH_STATUS_INVALID_API_KEY, NDLP_WARNING);
1138 +
1139 + receiver_state_free(rpt);
1140 + return rrdpush_receiver_permission_denied(w);
1141 + }
1142 +
1143 + if (regenerate_guid(rpt->machine_guid, buf) == -1) {
1144 + rrdpush_receive_log_status(
1145 + rpt, "machine GUID is not a valid UUID",
1146 + RRDPUSH_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
1147 +
1148 + receiver_state_free(rpt);
1149 + return rrdpush_receiver_permission_denied(w);
1150 + }
1151 + }
1152 +
1153 + const char *api_key_type = appconfig_get(&stream_config, rpt->key, "type", "api");
1154 + if(!api_key_type || !*api_key_type) api_key_type = "unknown";
1155 + if(strcmp(api_key_type, "api") != 0) {
1156 + rrdpush_receive_log_status(
1157 + rpt, "API key is a machine GUID",
1158 + RRDPUSH_STATUS_INVALID_API_KEY, NDLP_WARNING);
1159 +
1160 + receiver_state_free(rpt);
1161 + return rrdpush_receiver_permission_denied(w);
1162 + }
1163 +
1164 + if(!appconfig_get_boolean(&stream_config, rpt->key, "enabled", 0)) {
1165 + rrdpush_receive_log_status(
1166 + rpt, "API key is not enabled",
1167 + RRDPUSH_STATUS_API_KEY_DISABLED, NDLP_WARNING);
1168 +
1169 + receiver_state_free(rpt);
1170 + return rrdpush_receiver_permission_denied(w);
1171 + }
1172 +
1173 + {
1174 + SIMPLE_PATTERN *key_allow_from = simple_pattern_create(
1175 + appconfig_get(&stream_config, rpt->key, "allow from", "*"),
1176 + NULL, SIMPLE_PATTERN_EXACT, true);
1177 +
1178 + if(key_allow_from) {
1179 + if(!simple_pattern_matches(key_allow_from, w->client_ip)) {
1180 + simple_pattern_free(key_allow_from);
1181 +
1182 + rrdpush_receive_log_status(
1183 + rpt, "API key is not allowed from this IP",
1184 + RRDPUSH_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
1185 +
1186 + receiver_state_free(rpt);
1187 + return rrdpush_receiver_permission_denied(w);
1188 + }
1189 +
1190 + simple_pattern_free(key_allow_from);
1191 + }
1192 + }
1193 +
1194 + {
1195 + const char *machine_guid_type = appconfig_get(&stream_config, rpt->machine_guid, "type", "machine");
1196 + if (!machine_guid_type || !*machine_guid_type) machine_guid_type = "unknown";
1197 +
1198 + if (strcmp(machine_guid_type, "machine") != 0) {
1199 + rrdpush_receive_log_status(
1200 + rpt, "machine GUID is an API key",
1201 + RRDPUSH_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
1202 +
1203 + receiver_state_free(rpt);
1204 + return rrdpush_receiver_permission_denied(w);
1205 + }
1206 + }
1207 +
1208 + if(!appconfig_get_boolean(&stream_config, rpt->machine_guid, "enabled", 1)) {
1209 + rrdpush_receive_log_status(
1210 + rpt, "machine GUID is not enabled",
1211 + RRDPUSH_STATUS_MACHINE_GUID_DISABLED, NDLP_WARNING);
1212 +
1213 + receiver_state_free(rpt);
1214 + return rrdpush_receiver_permission_denied(w);
1215 + }
1216 +
1217 + {
1218 + SIMPLE_PATTERN *machine_allow_from = simple_pattern_create(
1219 + appconfig_get(&stream_config, rpt->machine_guid, "allow from", "*"),
1220 + NULL, SIMPLE_PATTERN_EXACT, true);
1221 +
1222 + if(machine_allow_from) {
1223 + if(!simple_pattern_matches(machine_allow_from, w->client_ip)) {
1224 + simple_pattern_free(machine_allow_from);
1225 +
1226 + rrdpush_receive_log_status(
1227 + rpt, "machine GUID is not allowed from this IP",
1228 + RRDPUSH_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
1229 +
1230 + receiver_state_free(rpt);
1231 + return rrdpush_receiver_permission_denied(w);
1232 + }
1233 +
1234 + simple_pattern_free(machine_allow_from);
1235 + }
1236 + }
1237 +
1238 + if (strcmp(rpt->machine_guid, localhost->machine_guid) == 0) {
1239 +
1240 + rrdpush_receiver_takeover_web_connection(w, rpt);
1241 +
1242 + rrdpush_receive_log_status(
1243 + rpt, "machine GUID is my own",
1244 + RRDPUSH_STATUS_LOCALHOST, NDLP_DEBUG);
1245 +
1246 + char initial_response[HTTP_HEADER_SIZE + 1];
1247 + snprintfz(initial_response, HTTP_HEADER_SIZE, "%s", START_STREAMING_ERROR_SAME_LOCALHOST);
1248 +
1249 + if(send_timeout(
1250 + &rpt->ssl,
1251 + rpt->fd, initial_response, strlen(initial_response), 0, 60) != (ssize_t)strlen(initial_response)) {
1252 +
1253 + nd_log_daemon(NDLP_ERR, "STREAM '%s' [receive from [%s]:%s]: "
1254 + "failed to reply."
1255 + , rpt->hostname
1256 + , rpt->client_ip, rpt->client_port
1257 + );
1258 + }
1259 +
1260 + receiver_state_free(rpt);
1261 + return HTTP_RESP_OK;
1262 + }
1263 +
1264 + if(unlikely(web_client_streaming_rate_t > 0)) {
1265 + static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
1266 + static time_t last_stream_accepted_t = 0;
1267 +
1268 + time_t now = now_realtime_sec();
1269 + spinlock_lock(&spinlock);
1270 +
1271 + if(unlikely(last_stream_accepted_t == 0))
1272 + last_stream_accepted_t = now;
1273 +
1274 + if(now - last_stream_accepted_t < web_client_streaming_rate_t) {
1275 + spinlock_unlock(&spinlock);
1276 +
1277 + char msg[100 + 1];
1278 + snprintfz(msg, sizeof(msg) - 1,
1279 + "rate limit, will accept new connection in %ld secs",
1280 + (long)(web_client_streaming_rate_t - (now - last_stream_accepted_t)));
1281 +
1282 + rrdpush_receive_log_status(
1283 + rpt, msg,
1284 + RRDPUSH_STATUS_RATE_LIMIT, NDLP_NOTICE);
1285 +
1286 + receiver_state_free(rpt);
1287 + return rrdpush_receiver_too_busy_now(w);
1288 + }
1289 +
1290 + last_stream_accepted_t = now;
1291 + spinlock_unlock(&spinlock);
1292 + }
1293 +
1294 + /*
1295 + * Quick path for rejecting multiple connections. The lock taken is fine-grained - it only protects the receiver
1296 + * pointer within the host (if a host exists). This protects against multiple concurrent web requests hitting
1297 + * separate threads within the web-server and landing here. The lock guards the thread-shutdown sequence that
1298 + * detaches the receiver from the host. If the host is being created (first time-access) then we also use the
1299 + * lock to prevent race-hazard (two threads try to create the host concurrently, one wins and the other does a
1300 + * lookup to the now-attached structure).
1301 + */
1302 +
1303 + {
1304 + time_t age = 0;
1305 + bool receiver_stale = false;
1306 + bool receiver_working = false;
1307 +
1308 + rrd_rdlock();
1309 + RRDHOST *host = rrdhost_find_by_guid(rpt->machine_guid);
1310 + if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */
1311 + host = NULL;
1312 +
1313 + if (host) {
1314 + spinlock_lock(&host->receiver_lock);
1315 + if (host->receiver) {
1316 + age = now_monotonic_sec() - host->receiver->last_msg_t;
1317 +
1318 + if (age < 30)
1319 + receiver_working = true;
1320 + else
1321 + receiver_stale = true;
1322 + }
1323 + spinlock_unlock(&host->receiver_lock);
1324 + }
1325 + rrd_rdunlock();
1326 +
1327 + if (receiver_stale && stop_streaming_receiver(host, STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER)) {
1328 + // we stopped the receiver
1329 + // we can proceed with this connection
1330 + receiver_stale = false;
1331 +
1332 + nd_log_daemon(NDLP_NOTICE, "STREAM '%s' [receive from [%s]:%s]: "
1333 + "stopped previous stale receiver to accept this one."
1334 + , rpt->hostname
1335 + , rpt->client_ip, rpt->client_port
1336 + );
1337 + }
1338 +
1339 + if (receiver_working || receiver_stale) {
1340 + // another receiver is already connected
1341 + // try again later
1342 +
1343 + char msg[200 + 1];
1344 + snprintfz(msg, sizeof(msg) - 1,
1345 + "multiple connections for same host, "
1346 + "old connection was last used %ld secs ago%s",
1347 + age, receiver_stale ? " (signaled old receiver to stop)" : " (new connection not accepted)");
1348 +
1349 + rrdpush_receive_log_status(
1350 + rpt, msg,
1351 + RRDPUSH_STATUS_ALREADY_CONNECTED, NDLP_DEBUG);
1352 +
1353 + // Have not set WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET - caller should clean up
1354 + buffer_flush(w->response.data);
1355 + buffer_strcat(w->response.data, START_STREAMING_ERROR_ALREADY_STREAMING);
1356 + receiver_state_free(rpt);
1357 + return HTTP_RESP_CONFLICT;
1358 + }
1359 + }
1360 +
1361 + rrdpush_receiver_takeover_web_connection(w, rpt);
1362 +
1363 + char tag[NETDATA_THREAD_TAG_MAX + 1];
1364 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_RECEIVER "[%s]", rpt->hostname);
1365 + tag[NETDATA_THREAD_TAG_MAX] = '\0';
1366 +
1367 + rpt->thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt);
1368 + if(!rpt->thread) {
1369 + rrdpush_receive_log_status(
1370 + rpt, "can't create receiver thread",
1371 + RRDPUSH_STATUS_INTERNAL_SERVER_ERROR, NDLP_ERR);
1372 +
1373 + buffer_flush(w->response.data);
1374 + buffer_strcat(w->response.data, "Can't handle this request");
1375 + receiver_state_free(rpt);
1376 + return HTTP_RESP_INTERNAL_SERVER_ERROR;
1377 + }
1378 +
1379 + // prevent the caller from closing the streaming socket
1380 + return HTTP_RESP_OK;
1381 +}
src/streaming/receiver.h new
+93
@@ -0,0 +1,93 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RECEIVER_H
4 +#define NETDATA_RECEIVER_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "database/rrd.h"
8 +
9 +struct parser;
10 +
11 +struct receiver_state {
12 + RRDHOST *host;
13 + pid_t tid;
14 + ND_THREAD *thread;
15 + int fd;
16 + char *key;
17 + char *hostname;
18 + char *registry_hostname;
19 + char *machine_guid;
20 + char *os;
21 + char *timezone; // Unused?
22 + char *abbrev_timezone;
23 + int32_t utc_offset;
24 + char *client_ip; // Duplicated in pluginsd
25 + char *client_port; // Duplicated in pluginsd
26 + char *program_name; // Duplicated in pluginsd
27 + char *program_version;
28 + struct rrdhost_system_info *system_info;
29 + STREAM_CAPABILITIES capabilities;
30 + time_t last_msg_t;
31 + time_t connected_since_s;
32 +
33 + struct buffered_reader reader;
34 +
35 + uint16_t hops;
36 +
37 + struct {
38 + bool shutdown; // signal the streaming parser to exit
39 + STREAM_HANDSHAKE reason;
40 + } exit;
41 +
42 + struct {
43 + RRD_MEMORY_MODE mode;
44 + int history;
45 + int update_every;
46 + int health_enabled; // CONFIG_BOOLEAN_YES, CONFIG_BOOLEAN_NO, CONFIG_BOOLEAN_AUTO
47 + time_t alarms_delay;
48 + uint32_t alarms_history;
49 + int rrdpush_enabled;
50 + const char *rrdpush_api_key; // DONT FREE - it is allocated in appconfig
51 + const char *rrdpush_send_charts_matching; // DONT FREE - it is allocated in appconfig
52 + bool rrdpush_enable_replication;
53 + time_t rrdpush_seconds_to_replicate;
54 + time_t rrdpush_replication_step;
55 + const char *rrdpush_destination; // DONT FREE - it is allocated in appconfig
56 + unsigned int rrdpush_compression;
57 + STREAM_CAPABILITIES compression_priorities[COMPRESSION_ALGORITHM_MAX];
58 + } config;
59 +
60 + NETDATA_SSL ssl;
61 +
62 + time_t replication_first_time_t;
63 +
64 + struct decompressor_state decompressor;
65 + /*
66 + struct {
67 + uint32_t count;
68 + STREAM_NODE_INSTANCE *array;
69 + } instances;
70 +*/
71 +
72 + // The parser pointer is safe to read and use, only when having the host receiver lock.
73 + // Without this lock, the data pointed by the pointer may vanish randomly.
74 + // Also, since the receiver sets it when it starts, it should be read with
75 + // an atomic read.
76 + struct parser *parser;
77 +
78 +#ifdef ENABLE_H2O
79 + void *h2o_ctx;
80 +#endif
81 +};
82 +
83 +#ifdef ENABLE_H2O
84 +#define is_h2o_rrdpush(x) ((x)->h2o_ctx != NULL)
85 +#define unless_h2o_rrdpush(x) if(!is_h2o_rrdpush(x))
86 +#endif
87 +
88 +int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx);
89 +
90 +void receiver_state_free(struct receiver_state *rpt);
91 +bool stop_streaming_receiver(RRDHOST *host, STREAM_HANDSHAKE reason);
92 +
93 +#endif //NETDATA_RECEIVER_H
src/streaming/replication.h
+2
@@ -5,6 +5,8 @@
5
6 #include "daemon/common.h"
7
8 +struct parser;
9 +
10 struct replication_query_statistics {
11 SPINLOCK spinlock;
12 size_t queries_started;
src/streaming/rrdhost-status.c new
+355
@@ -0,0 +1,355 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdhost-status.h"
4 +
5 +const char *rrdhost_db_status_to_string(RRDHOST_DB_STATUS status) {
6 + switch(status) {
7 + default:
8 + case RRDHOST_DB_STATUS_INITIALIZING:
9 + return "initializing";
10 +
11 + case RRDHOST_DB_STATUS_QUERYABLE:
12 + return "online";
13 + }
14 +}
15 +
16 +const char *rrdhost_db_liveness_to_string(RRDHOST_DB_LIVENESS status) {
17 + switch(status) {
18 + default:
19 + case RRDHOST_DB_LIVENESS_STALE:
20 + return "stale";
21 +
22 + case RRDHOST_DB_LIVENESS_LIVE:
23 + return "live";
24 + }
25 +}
26 +
27 +const char *rrdhost_ingest_status_to_string(RRDHOST_INGEST_STATUS status) {
28 + switch(status) {
29 + case RRDHOST_INGEST_STATUS_ARCHIVED:
30 + return "archived";
31 +
32 + case RRDHOST_INGEST_STATUS_INITIALIZING:
33 + return "initializing";
34 +
35 + case RRDHOST_INGEST_STATUS_REPLICATING:
36 + return "replicating";
37 +
38 + case RRDHOST_INGEST_STATUS_ONLINE:
39 + return "online";
40 +
41 + default:
42 + case RRDHOST_INGEST_STATUS_OFFLINE:
43 + return "offline";
44 + }
45 +}
46 +
47 +const char *rrdhost_ingest_type_to_string(RRDHOST_INGEST_TYPE type) {
48 + switch(type) {
49 + case RRDHOST_INGEST_TYPE_LOCALHOST:
50 + return "localhost";
51 +
52 + case RRDHOST_INGEST_TYPE_VIRTUAL:
53 + return "virtual";
54 +
55 + case RRDHOST_INGEST_TYPE_CHILD:
56 + return "child";
57 +
58 + default:
59 + case RRDHOST_INGEST_TYPE_ARCHIVED:
60 + return "archived";
61 + }
62 +}
63 +
64 +const char *rrdhost_streaming_status_to_string(RRDHOST_STREAMING_STATUS status) {
65 + switch(status) {
66 + case RRDHOST_STREAM_STATUS_DISABLED:
67 + return "disabled";
68 +
69 + case RRDHOST_STREAM_STATUS_REPLICATING:
70 + return "replicating";
71 +
72 + case RRDHOST_STREAM_STATUS_ONLINE:
73 + return "online";
74 +
75 + default:
76 + case RRDHOST_STREAM_STATUS_OFFLINE:
77 + return "offline";
78 + }
79 +}
80 +
81 +const char *rrdhost_ml_status_to_string(RRDHOST_ML_STATUS status) {
82 + switch(status) {
83 + case RRDHOST_ML_STATUS_RUNNING:
84 + return "online";
85 +
86 + case RRDHOST_ML_STATUS_OFFLINE:
87 + return "offline";
88 +
89 + default:
90 + case RRDHOST_ML_STATUS_DISABLED:
91 + return "disabled";
92 + }
93 +}
94 +
95 +const char *rrdhost_ml_type_to_string(RRDHOST_ML_TYPE type) {
96 + switch(type) {
97 + case RRDHOST_ML_TYPE_SELF:
98 + return "self";
99 +
100 + case RRDHOST_ML_TYPE_RECEIVED:
101 + return "received";
102 +
103 + default:
104 + case RRDHOST_ML_TYPE_DISABLED:
105 + return "disabled";
106 + }
107 +}
108 +
109 +const char *rrdhost_health_status_to_string(RRDHOST_HEALTH_STATUS status) {
110 + switch(status) {
111 + default:
112 + case RRDHOST_HEALTH_STATUS_DISABLED:
113 + return "disabled";
114 +
115 + case RRDHOST_HEALTH_STATUS_INITIALIZING:
116 + return "initializing";
117 +
118 + case RRDHOST_HEALTH_STATUS_RUNNING:
119 + return "online";
120 + }
121 +}
122 +
123 +const char *rrdhost_dyncfg_status_to_string(RRDHOST_DYNCFG_STATUS status) {
124 + switch(status) {
125 + default:
126 + case RRDHOST_DYNCFG_STATUS_UNAVAILABLE:
127 + return "unavailable";
128 +
129 + case RRDHOST_DYNCFG_STATUS_AVAILABLE:
130 + return "online";
131 + }
132 +}
133 +
134 +static NETDATA_DOUBLE rrdhost_sender_replication_completion_unsafe(RRDHOST *host, time_t now, size_t *instances) {
135 + size_t charts = rrdhost_sender_replicating_charts(host);
136 + NETDATA_DOUBLE completion;
137 + if(!charts || !host->sender || !host->sender->replication.oldest_request_after_t)
138 + completion = 100.0;
139 + else if(!host->sender->replication.latest_completed_before_t || host->sender->replication.latest_completed_before_t < host->sender->replication.oldest_request_after_t)
140 + completion = 0.0;
141 + else {
142 + time_t total = now - host->sender->replication.oldest_request_after_t;
143 + time_t current = host->sender->replication.latest_completed_before_t - host->sender->replication.oldest_request_after_t;
144 + completion = (NETDATA_DOUBLE) current * 100.0 / (NETDATA_DOUBLE) total;
145 + }
146 +
147 + *instances = charts;
148 +
149 + return completion;
150 +}
151 +
152 +void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s) {
153 + memset(s, 0, sizeof(*s));
154 +
155 + s->host = host;
156 + s->now = now;
157 +
158 + RRDHOST_FLAGS flags = __atomic_load_n(&host->flags, __ATOMIC_RELAXED);
159 +
160 + // --- dyncfg ---
161 +
162 + s->dyncfg.status = dyncfg_available_for_rrdhost(host) ? RRDHOST_DYNCFG_STATUS_AVAILABLE : RRDHOST_DYNCFG_STATUS_UNAVAILABLE;
163 +
164 + // --- db ---
165 +
166 + bool online = rrdhost_is_online(host);
167 +
168 + rrdhost_retention(host, now, online, &s->db.first_time_s, &s->db.last_time_s);
169 + s->db.metrics = host->rrdctx.metrics;
170 + s->db.instances = host->rrdctx.instances;
171 + s->db.contexts = dictionary_entries(host->rrdctx.contexts);
172 + if(!s->db.first_time_s || !s->db.last_time_s || !s->db.metrics || !s->db.instances || !s->db.contexts ||
173 + (flags & (RRDHOST_FLAG_PENDING_CONTEXT_LOAD)))
174 + s->db.status = RRDHOST_DB_STATUS_INITIALIZING;
175 + else
176 + s->db.status = RRDHOST_DB_STATUS_QUERYABLE;
177 +
178 + s->db.mode = host->rrd_memory_mode;
179 +
180 + // --- ingest ---
181 +
182 + s->ingest.since = MAX(host->child_connect_time, host->child_disconnected_time);
183 + s->ingest.reason = (online) ? STREAM_HANDSHAKE_NEVER : host->rrdpush_last_receiver_exit_reason;
184 +
185 + spinlock_lock(&host->receiver_lock);
186 + s->ingest.hops = (host->system_info ? host->system_info->hops : (host == localhost) ? 0 : 1);
187 + bool has_receiver = false;
188 + if (host->receiver) {
189 + has_receiver = true;
190 + s->ingest.replication.instances = rrdhost_receiver_replicating_charts(host);
191 + s->ingest.replication.completion = host->rrdpush_receiver_replication_percent;
192 + s->ingest.replication.in_progress = s->ingest.replication.instances > 0;
193 +
194 + s->ingest.capabilities = host->receiver->capabilities;
195 + s->ingest.peers = socket_peers(host->receiver->fd);
196 + s->ingest.ssl = SSL_connection(&host->receiver->ssl);
197 + }
198 + spinlock_unlock(&host->receiver_lock);
199 +
200 + if (online) {
201 + if(s->db.status == RRDHOST_DB_STATUS_INITIALIZING)
202 + s->ingest.status = RRDHOST_INGEST_STATUS_INITIALIZING;
203 +
204 + else if (host == localhost || rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) {
205 + s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
206 + s->ingest.since = netdata_start_time;
207 + }
208 +
209 + else if (s->ingest.replication.in_progress)
210 + s->ingest.status = RRDHOST_INGEST_STATUS_REPLICATING;
211 +
212 + else
213 + s->ingest.status = RRDHOST_INGEST_STATUS_ONLINE;
214 + }
215 + else {
216 + if (!s->ingest.since) {
217 + s->ingest.status = RRDHOST_INGEST_STATUS_ARCHIVED;
218 + s->ingest.since = s->db.last_time_s;
219 + }
220 +
221 + else
222 + s->ingest.status = RRDHOST_INGEST_STATUS_OFFLINE;
223 + }
224 +
225 + if(host == localhost)
226 + s->ingest.type = RRDHOST_INGEST_TYPE_LOCALHOST;
227 + else if(has_receiver || rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))
228 + s->ingest.type = RRDHOST_INGEST_TYPE_CHILD;
229 + else if(rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST))
230 + s->ingest.type = RRDHOST_INGEST_TYPE_VIRTUAL;
231 + else
232 + s->ingest.type = RRDHOST_INGEST_TYPE_ARCHIVED;
233 +
234 + s->ingest.id = host->rrdpush_receiver_connection_counter;
235 +
236 + if(!s->ingest.since)
237 + s->ingest.since = netdata_start_time;
238 +
239 + if(s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE)
240 + s->db.liveness = RRDHOST_DB_LIVENESS_LIVE;
241 + else
242 + s->db.liveness = RRDHOST_DB_LIVENESS_STALE;
243 +
244 + // --- stream ---
245 +
246 + if (!host->sender) {
247 + s->stream.status = RRDHOST_STREAM_STATUS_DISABLED;
248 + s->stream.hops = s->ingest.hops + 1;
249 + }
250 + else {
251 + sender_lock(host->sender);
252 +
253 + s->stream.since = host->sender->last_state_since_t;
254 + s->stream.peers = socket_peers(host->sender->rrdpush_sender_socket);
255 + s->stream.ssl = SSL_connection(&host->sender->ssl);
256 +
257 + memcpy(s->stream.sent_bytes_on_this_connection_per_type,
258 + host->sender->sent_bytes_on_this_connection_per_type,
259 + MIN(sizeof(s->stream.sent_bytes_on_this_connection_per_type),
260 + sizeof(host->sender->sent_bytes_on_this_connection_per_type)));
261 +
262 + if (rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED)) {
263 + s->stream.hops = host->sender->hops;
264 + s->stream.reason = STREAM_HANDSHAKE_NEVER;
265 + s->stream.capabilities = host->sender->capabilities;
266 +
267 + s->stream.replication.completion = rrdhost_sender_replication_completion_unsafe(host, now, &s->stream.replication.instances);
268 + s->stream.replication.in_progress = s->stream.replication.instances > 0;
269 +
270 + if(s->stream.replication.in_progress)
271 + s->stream.status = RRDHOST_STREAM_STATUS_REPLICATING;
272 + else
273 + s->stream.status = RRDHOST_STREAM_STATUS_ONLINE;
274 +
275 + s->stream.compression = host->sender->compressor.initialized;
276 + }
277 + else {
278 + s->stream.status = RRDHOST_STREAM_STATUS_OFFLINE;
279 + s->stream.hops = s->ingest.hops + 1;
280 + s->stream.reason = host->sender->exit.reason;
281 + }
282 +
283 + sender_unlock(host->sender);
284 + }
285 +
286 + s->stream.id = host->rrdpush_sender_connection_counter;
287 +
288 + if(!s->stream.since)
289 + s->stream.since = netdata_start_time;
290 +
291 + // --- ml ---
292 +
293 + if(ml_host_get_host_status(host, &s->ml.metrics)) {
294 + s->ml.type = RRDHOST_ML_TYPE_SELF;
295 +
296 + if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE || s->ingest.status == RRDHOST_INGEST_STATUS_ARCHIVED)
297 + s->ml.status = RRDHOST_ML_STATUS_OFFLINE;
298 + else
299 + s->ml.status = RRDHOST_ML_STATUS_RUNNING;
300 + }
301 + else if(stream_has_capability(&s->ingest, STREAM_CAP_DATA_WITH_ML)) {
302 + s->ml.type = RRDHOST_ML_TYPE_RECEIVED;
303 + s->ml.status = RRDHOST_ML_STATUS_RUNNING;
304 + }
305 + else {
306 + // does not receive ML, does not run ML
307 + s->ml.type = RRDHOST_ML_TYPE_DISABLED;
308 + s->ml.status = RRDHOST_ML_STATUS_DISABLED;
309 + }
310 +
311 + // --- health ---
312 +
313 + if(host->health.health_enabled) {
314 + if(flags & RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION)
315 + s->health.status = RRDHOST_HEALTH_STATUS_INITIALIZING;
316 + else {
317 + s->health.status = RRDHOST_HEALTH_STATUS_RUNNING;
318 +
319 + RRDCALC *rc;
320 + foreach_rrdcalc_in_rrdhost_read(host, rc) {
321 + if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec))
322 + continue;
323 +
324 + switch (rc->status) {
325 + default:
326 + case RRDCALC_STATUS_REMOVED:
327 + break;
328 +
329 + case RRDCALC_STATUS_CLEAR:
330 + s->health.alerts.clear++;
331 + break;
332 +
333 + case RRDCALC_STATUS_WARNING:
334 + s->health.alerts.warning++;
335 + break;
336 +
337 + case RRDCALC_STATUS_CRITICAL:
338 + s->health.alerts.critical++;
339 + break;
340 +
341 + case RRDCALC_STATUS_UNDEFINED:
342 + s->health.alerts.undefined++;
343 + break;
344 +
345 + case RRDCALC_STATUS_UNINITIALIZED:
346 + s->health.alerts.uninitialized++;
347 + break;
348 + }
349 + }
350 + foreach_rrdcalc_in_rrdhost_done(rc);
351 + }
352 + }
353 + else
354 + s->health.status = RRDHOST_HEALTH_STATUS_DISABLED;
355 +}
src/streaming/rrdhost-status.h new
+161
@@ -0,0 +1,161 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_RRDHOST_STATUS_H
4 +#define NETDATA_RRDHOST_STATUS_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "stream-handshake.h"
8 +#include "stream-capabilities.h"
9 +#include "database/rrd.h"
10 +
11 +typedef enum __attribute__((packed)) {
12 + RRDHOST_DB_STATUS_INITIALIZING = 0,
13 + RRDHOST_DB_STATUS_QUERYABLE,
14 +} RRDHOST_DB_STATUS;
15 +
16 +const char *rrdhost_db_status_to_string(RRDHOST_DB_STATUS status);
17 +
18 +typedef enum __attribute__((packed)) {
19 + RRDHOST_DB_LIVENESS_STALE = 0,
20 + RRDHOST_DB_LIVENESS_LIVE,
21 +} RRDHOST_DB_LIVENESS;
22 +
23 +const char *rrdhost_db_liveness_to_string(RRDHOST_DB_LIVENESS status);
24 +
25 +typedef enum __attribute__((packed)) {
26 + RRDHOST_INGEST_STATUS_ARCHIVED = 0,
27 + RRDHOST_INGEST_STATUS_INITIALIZING,
28 + RRDHOST_INGEST_STATUS_REPLICATING,
29 + RRDHOST_INGEST_STATUS_ONLINE,
30 + RRDHOST_INGEST_STATUS_OFFLINE,
31 +} RRDHOST_INGEST_STATUS;
32 +
33 +const char *rrdhost_ingest_status_to_string(RRDHOST_INGEST_STATUS status);
34 +
35 +typedef enum __attribute__((packed)) {
36 + RRDHOST_INGEST_TYPE_LOCALHOST = 0,
37 + RRDHOST_INGEST_TYPE_VIRTUAL,
38 + RRDHOST_INGEST_TYPE_CHILD,
39 + RRDHOST_INGEST_TYPE_ARCHIVED,
40 +} RRDHOST_INGEST_TYPE;
41 +
42 +const char *rrdhost_ingest_type_to_string(RRDHOST_INGEST_TYPE type);
43 +
44 +typedef enum __attribute__((packed)) {
45 + RRDHOST_STREAM_STATUS_DISABLED = 0,
46 + RRDHOST_STREAM_STATUS_REPLICATING,
47 + RRDHOST_STREAM_STATUS_ONLINE,
48 + RRDHOST_STREAM_STATUS_OFFLINE,
49 +} RRDHOST_STREAMING_STATUS;
50 +
51 +const char *rrdhost_streaming_status_to_string(RRDHOST_STREAMING_STATUS status);
52 +
53 +typedef enum __attribute__((packed)) {
54 + RRDHOST_ML_STATUS_DISABLED = 0,
55 + RRDHOST_ML_STATUS_OFFLINE,
56 + RRDHOST_ML_STATUS_RUNNING,
57 +} RRDHOST_ML_STATUS;
58 +
59 +const char *rrdhost_ml_status_to_string(RRDHOST_ML_STATUS status);
60 +
61 +typedef enum __attribute__((packed)) {
62 + RRDHOST_ML_TYPE_DISABLED = 0,
63 + RRDHOST_ML_TYPE_SELF,
64 + RRDHOST_ML_TYPE_RECEIVED,
65 +} RRDHOST_ML_TYPE;
66 +
67 +const char *rrdhost_ml_type_to_string(RRDHOST_ML_TYPE type);
68 +
69 +typedef enum __attribute__((packed)) {
70 + RRDHOST_HEALTH_STATUS_DISABLED = 0,
71 + RRDHOST_HEALTH_STATUS_INITIALIZING,
72 + RRDHOST_HEALTH_STATUS_RUNNING,
73 +} RRDHOST_HEALTH_STATUS;
74 +
75 +const char *rrdhost_health_status_to_string(RRDHOST_HEALTH_STATUS status);
76 +
77 +typedef enum __attribute__((packed)) {
78 + RRDHOST_DYNCFG_STATUS_UNAVAILABLE = 0,
79 + RRDHOST_DYNCFG_STATUS_AVAILABLE,
80 +} RRDHOST_DYNCFG_STATUS;
81 +
82 +const char *rrdhost_dyncfg_status_to_string(RRDHOST_DYNCFG_STATUS status);
83 +
84 +typedef struct {
85 + RRDHOST *host;
86 + time_t now;
87 +
88 + struct {
89 + RRDHOST_DYNCFG_STATUS status;
90 + } dyncfg;
91 +
92 + struct {
93 + RRDHOST_DB_STATUS status;
94 + RRDHOST_DB_LIVENESS liveness;
95 + RRD_MEMORY_MODE mode;
96 + time_t first_time_s;
97 + time_t last_time_s;
98 + size_t metrics;
99 + size_t instances;
100 + size_t contexts;
101 + } db;
102 +
103 + struct {
104 + RRDHOST_ML_STATUS status;
105 + RRDHOST_ML_TYPE type;
106 + struct ml_metrics_statistics metrics;
107 + } ml;
108 +
109 + struct {
110 + size_t hops;
111 + RRDHOST_INGEST_TYPE type;
112 + RRDHOST_INGEST_STATUS status;
113 + SOCKET_PEERS peers;
114 + bool ssl;
115 + STREAM_CAPABILITIES capabilities;
116 + uint32_t id;
117 + time_t since;
118 + STREAM_HANDSHAKE reason;
119 +
120 + struct {
121 + bool in_progress;
122 + NETDATA_DOUBLE completion;
123 + size_t instances;
124 + } replication;
125 + } ingest;
126 +
127 + struct {
128 + size_t hops;
129 + RRDHOST_STREAMING_STATUS status;
130 + SOCKET_PEERS peers;
131 + bool ssl;
132 + bool compression;
133 + STREAM_CAPABILITIES capabilities;
134 + uint32_t id;
135 + time_t since;
136 + STREAM_HANDSHAKE reason;
137 +
138 + struct {
139 + bool in_progress;
140 + NETDATA_DOUBLE completion;
141 + size_t instances;
142 + } replication;
143 +
144 + size_t sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX];
145 + } stream;
146 +
147 + struct {
148 + RRDHOST_HEALTH_STATUS status;
149 + struct {
150 + uint32_t undefined;
151 + uint32_t uninitialized;
152 + uint32_t clear;
153 + uint32_t warning;
154 + uint32_t critical;
155 + } alerts;
156 + } health;
157 +} RRDHOST_STATUS;
158 +
159 +void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s);
160 +
161 +#endif //NETDATA_RRDHOST_STATUS_H
src/streaming/rrdpush.c deleted
-1258
@@ -1,1258 +0,0 @@
1 -// SPDX-License-Identifier: GPL-3.0-or-later
2 -
3 -#include "rrdpush.h"
4 -
5 -/*
6 - * rrdpush
7 - *
8 - * 3 threads are involved for all stream operations
9 - *
10 - * 1. a random data collection thread, calling rrdset_done_push()
11 - * this is called for each chart.
12 - *
13 - * the output of this work is kept in a thread BUFFER
14 - * the sender thread is signalled via a pipe (in RRDHOST)
15 - *
16 - * 2. a sender thread running at the sending netdata
17 - * this is spawned automatically on the first chart to be pushed
18 - *
19 - * It tries to push the metrics to the remote netdata, as fast
20 - * as possible (i.e. immediately after they are collected).
21 - *
22 - * 3. a receiver thread, running at the receiving netdata
23 - * this is spawned automatically when the sender connects to
24 - * the receiver.
25 - *
26 - */
27 -
28 -struct config stream_config = APPCONFIG_INITIALIZER;
29 -
30 -unsigned int default_rrdpush_enabled = 0;
31 -
32 -unsigned int default_rrdpush_compression_enabled = 1;
33 -const char *default_rrdpush_destination = NULL;
34 -const char *default_rrdpush_api_key = NULL;
35 -const char *default_rrdpush_send_charts_matching = "*";
36 -bool default_rrdpush_enable_replication = true;
37 -time_t default_rrdpush_seconds_to_replicate = 86400;
38 -time_t default_rrdpush_replication_step = 600;
39 -const char *netdata_ssl_ca_path = NULL;
40 -const char *netdata_ssl_ca_file = NULL;
41 -
42 -static void load_stream_conf() {
43 - errno_clear();
44 - char *filename = filename_from_path_entry_strdupz(netdata_configured_user_config_dir, "stream.conf");
45 - if(!appconfig_load(&stream_config, filename, 0, NULL)) {
46 - nd_log_daemon(NDLP_NOTICE, "CONFIG: cannot load user config '%s'. Will try stock config.", filename);
47 - freez(filename);
48 -
49 - filename = filename_from_path_entry_strdupz(netdata_configured_stock_config_dir, "stream.conf");
50 - if(!appconfig_load(&stream_config, filename, 0, NULL))
51 - nd_log_daemon(NDLP_NOTICE, "CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
52 - }
53 -
54 - freez(filename);
55 -
56 - appconfig_move(&stream_config,
57 - CONFIG_SECTION_STREAM, "timeout seconds",
58 - CONFIG_SECTION_STREAM, "timeout");
59 -
60 - appconfig_move(&stream_config,
61 - CONFIG_SECTION_STREAM, "reconnect delay seconds",
62 - CONFIG_SECTION_STREAM, "reconnect delay");
63 -
64 - appconfig_move_everywhere(&stream_config, "default memory mode", "db");
65 - appconfig_move_everywhere(&stream_config, "memory mode", "db");
66 - appconfig_move_everywhere(&stream_config, "db mode", "db");
67 - appconfig_move_everywhere(&stream_config, "default history", "retention");
68 - appconfig_move_everywhere(&stream_config, "history", "retention");
69 - appconfig_move_everywhere(&stream_config, "default proxy enabled", "proxy enabled");
70 - appconfig_move_everywhere(&stream_config, "default proxy destination", "proxy destination");
71 - appconfig_move_everywhere(&stream_config, "default proxy api key", "proxy api key");
72 - appconfig_move_everywhere(&stream_config, "default proxy send charts matching", "proxy send charts matching");
73 - appconfig_move_everywhere(&stream_config, "default health log history", "health log retention");
74 - appconfig_move_everywhere(&stream_config, "health log history", "health log retention");
75 - appconfig_move_everywhere(&stream_config, "seconds to replicate", "replication period");
76 - appconfig_move_everywhere(&stream_config, "seconds per replication step", "replication step");
77 - appconfig_move_everywhere(&stream_config, "default postpone alarms on connect seconds", "postpone alerts on connect");
78 - appconfig_move_everywhere(&stream_config, "postpone alarms on connect seconds", "postpone alerts on connect");
79 -}
80 -
81 -bool rrdpush_receiver_needs_dbengine(void) {
82 - return stream_conf_needs_dbengine(&stream_config);
83 -}
84 -
85 -int rrdpush_init() {
86 - // --------------------------------------------------------------------
87 - // load stream.conf
88 - load_stream_conf();
89 -
90 - default_rrdpush_enabled =
91 - (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "enabled", default_rrdpush_enabled);
92 -
93 - default_rrdpush_destination =
94 - appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "destination", "");
95 -
96 - default_rrdpush_api_key =
97 - appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "api key", "");
98 -
99 - default_rrdpush_send_charts_matching =
100 - appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "send charts matching", default_rrdpush_send_charts_matching);
101 -
102 - default_rrdpush_enable_replication =
103 - config_get_boolean(CONFIG_SECTION_DB, "enable replication", default_rrdpush_enable_replication);
104 -
105 - default_rrdpush_seconds_to_replicate =
106 - config_get_duration_seconds(CONFIG_SECTION_DB, "replication period", default_rrdpush_seconds_to_replicate);
107 -
108 - default_rrdpush_replication_step =
109 - config_get_duration_seconds(CONFIG_SECTION_DB, "replication step", default_rrdpush_replication_step);
110 -
111 - rrdhost_free_orphan_time_s =
112 - config_get_duration_seconds(CONFIG_SECTION_DB, "cleanup orphan hosts after", rrdhost_free_orphan_time_s);
113 -
114 - default_rrdpush_compression_enabled =
115 - (unsigned int)appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
116 - "enable compression", default_rrdpush_compression_enabled);
117 -
118 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_BROTLI] = (int)appconfig_get_number(
119 - &stream_config, CONFIG_SECTION_STREAM, "brotli compression level",
120 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_BROTLI]);
121 -
122 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_ZSTD] = (int)appconfig_get_number(
123 - &stream_config, CONFIG_SECTION_STREAM, "zstd compression level",
124 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_ZSTD]);
125 -
126 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_LZ4] = (int)appconfig_get_number(
127 - &stream_config, CONFIG_SECTION_STREAM, "lz4 compression acceleration",
128 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_LZ4]);
129 -
130 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_GZIP] = (int)appconfig_get_number(
131 - &stream_config, CONFIG_SECTION_STREAM, "gzip compression level",
132 - rrdpush_compression_levels[COMPRESSION_ALGORITHM_GZIP]);
133 -
134 - if(default_rrdpush_enabled && (!default_rrdpush_destination || !*default_rrdpush_destination || !default_rrdpush_api_key || !*default_rrdpush_api_key)) {
135 - nd_log_daemon(NDLP_WARNING, "STREAM [send]: cannot enable sending thread - information is missing.");
136 - default_rrdpush_enabled = 0;
137 - }
138 -
139 - netdata_ssl_validate_certificate_sender = !appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", !netdata_ssl_validate_certificate);
140 -
141 - if(!netdata_ssl_validate_certificate_sender)
142 - nd_log_daemon(NDLP_NOTICE, "SSL: streaming senders will skip SSL certificates verification.");
143 -
144 - netdata_ssl_ca_path = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "CApath", NULL);
145 - netdata_ssl_ca_file = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "CAfile", NULL);
146 -
147 - return default_rrdpush_enabled;
148 -}
149 -
150 -// data collection happens from multiple threads
151 -// each of these threads calls rrdset_done()
152 -// which in turn calls rrdset_done_push()
153 -// which uses this pipe to notify the streaming thread
154 -// that there are more data ready to be sent
155 -#define PIPE_READ 0
156 -#define PIPE_WRITE 1
157 -
158 -// to have the remote netdata re-sync the charts
159 -// to its current clock, we send for this many
160 -// iterations a BEGIN line without microseconds
161 -// this is for the first iterations of each chart
162 -unsigned int remote_clock_resync_iterations = 60;
163 -
164 -static inline bool should_send_chart_matching(RRDSET *st, RRDSET_FLAGS flags) {
165 - if(!(flags & RRDSET_FLAG_RECEIVER_REPLICATION_FINISHED))
166 - return false;
167 -
168 - if(unlikely(!(flags & (RRDSET_FLAG_UPSTREAM_SEND | RRDSET_FLAG_UPSTREAM_IGNORE)))) {
169 - RRDHOST *host = st->rrdhost;
170 -
171 - if (flags & RRDSET_FLAG_ANOMALY_DETECTION) {
172 - if(ml_streaming_enabled())
173 - rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
174 - else
175 - rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);
176 - }
177 - else {
178 - int negative = 0, positive = 0;
179 - SIMPLE_PATTERN_RESULT r;
180 -
181 - r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->context, NULL, 0);
182 - if(r == SP_MATCHED_POSITIVE) positive++;
183 - else if(r == SP_MATCHED_NEGATIVE) negative++;
184 -
185 - if(!negative) {
186 - r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->name, NULL, 0);
187 - if (r == SP_MATCHED_POSITIVE) positive++;
188 - else if (r == SP_MATCHED_NEGATIVE) negative++;
189 - }
190 -
191 - if(!negative) {
192 - r = simple_pattern_matches_string_extract(host->rrdpush.send.charts_matching, st->id, NULL, 0);
193 - if (r == SP_MATCHED_POSITIVE) positive++;
194 - else if (r == SP_MATCHED_NEGATIVE) negative++;
195 - }
196 -
197 - if(!negative && positive)
198 - rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_SEND);
199 - else
200 - rrdset_flag_set(st, RRDSET_FLAG_UPSTREAM_IGNORE);
201 - }
202 -
203 - // get the flags again, to know how to respond
204 - flags = rrdset_flag_check(st, RRDSET_FLAG_UPSTREAM_SEND|RRDSET_FLAG_UPSTREAM_IGNORE);
205 - }
206 -
207 - return flags & RRDSET_FLAG_UPSTREAM_SEND;
208 -}
209 -
210 -int configured_as_parent() {
211 - return stream_conf_has_uuid_section(&stream_config);
212 -}
213 -
214 -// chart labels
215 -static int send_clabels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
216 - BUFFER *wb = (BUFFER *)data;
217 - buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL " \"%s\" \"%s\" %d\n", name, value, ls & ~(RRDLABEL_FLAG_INTERNAL));
218 - return 1;
219 -}
220 -
221 -static void rrdpush_send_clabels(BUFFER *wb, RRDSET *st) {
222 - if (st->rrdlabels) {
223 - if(rrdlabels_walkthrough_read(st->rrdlabels, send_clabels_callback, wb) > 0)
224 - buffer_sprintf(wb, PLUGINSD_KEYWORD_CLABEL_COMMIT "\n");
225 - }
226 -}
227 -
228 -// Send the current chart definition.
229 -// Assumes that collector thread has already called sender_start for mutex / buffer state.
230 -static inline bool rrdpush_send_chart_definition(BUFFER *wb, RRDSET *st) {
231 - uint32_t version = rrdset_metadata_version(st);
232 -
233 - RRDHOST *host = st->rrdhost;
234 - NUMBER_ENCODING integer_encoding = stream_has_capability(host->sender, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
235 - bool with_slots = stream_has_capability(host->sender, STREAM_CAP_SLOTS) ? true : false;
236 -
237 - bool replication_progress = false;
238 -
239 - // properly set the name for the remote end to parse it
240 - char *name = "";
241 - if(likely(st->name)) {
242 - if(unlikely(st->id != st->name)) {
243 - // they differ
244 - name = strchr(rrdset_name(st), '.');
245 - if(name)
246 - name++;
247 - else
248 - name = "";
249 - }
250 - }
251 -
252 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_CHART, sizeof(PLUGINSD_KEYWORD_CHART) - 1);
253 -
254 - if(with_slots) {
255 - buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
256 - buffer_print_uint64_encoded(wb, integer_encoding, st->rrdpush.sender.chart_slot);
257 - }
258 -
259 - // send the chart
260 - buffer_sprintf(
261 - wb
262 - , " \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\" \"%s\" \"%s\"\n"
263 - , rrdset_id(st)
264 - , name
265 - , rrdset_title(st)
266 - , rrdset_units(st)
267 - , rrdset_family(st)
268 - , rrdset_context(st)
269 - , rrdset_type_name(st->chart_type)
270 - , st->priority
271 - , st->update_every
272 - , rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)?"obsolete":""
273 - , rrdset_flag_check(st, RRDSET_FLAG_STORE_FIRST)?"store_first":""
274 - , rrdset_flag_check(st, RRDSET_FLAG_HIDDEN)?"hidden":""
275 - , rrdset_plugin_name(st)
276 - , rrdset_module_name(st)
277 - );
278 -
279 - // send the chart labels
280 - if (stream_has_capability(host->sender, STREAM_CAP_CLABELS))
281 - rrdpush_send_clabels(wb, st);
282 -
283 - // send the dimensions
284 - RRDDIM *rd;
285 - rrddim_foreach_read(rd, st) {
286 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_DIMENSION, sizeof(PLUGINSD_KEYWORD_DIMENSION) - 1);
287 -
288 - if(with_slots) {
289 - buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
290 - buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
291 - }
292 -
293 - buffer_sprintf(
294 - wb
295 - , " \"%s\" \"%s\" \"%s\" %d %d \"%s %s %s\"\n"
296 - , rrddim_id(rd)
297 - , rrddim_name(rd)
298 - , rrd_algorithm_name(rd->algorithm)
299 - , rd->multiplier
300 - , rd->divisor
301 - , rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)?"obsolete":""
302 - , rrddim_option_check(rd, RRDDIM_OPTION_HIDDEN)?"hidden":""
303 - , rrddim_option_check(rd, RRDDIM_OPTION_DONT_DETECT_RESETS_OR_OVERFLOWS)?"noreset":""
304 - );
305 - }
306 - rrddim_foreach_done(rd);
307 -
308 - // send the chart functions
309 - if(stream_has_capability(host->sender, STREAM_CAP_FUNCTIONS))
310 - rrd_chart_functions_expose_rrdpush(st, wb);
311 -
312 - // send the chart local custom variables
313 - rrdvar_print_to_streaming_custom_chart_variables(st, wb);
314 -
315 - if (stream_has_capability(host->sender, STREAM_CAP_REPLICATION)) {
316 - time_t db_first_time_t, db_last_time_t;
317 -
318 - time_t now = now_realtime_sec();
319 - rrdset_get_retention_of_tier_for_collected_chart(st, &db_first_time_t, &db_last_time_t, now, 0);
320 -
321 - buffer_sprintf(wb, PLUGINSD_KEYWORD_CHART_DEFINITION_END " %llu %llu %llu\n",
322 - (unsigned long long)db_first_time_t,
323 - (unsigned long long)db_last_time_t,
324 - (unsigned long long)now);
325 -
326 - if(!rrdset_flag_check(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS)) {
327 - rrdset_flag_set(st, RRDSET_FLAG_SENDER_REPLICATION_IN_PROGRESS);
328 - rrdset_flag_clear(st, RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
329 - rrdhost_sender_replicating_charts_plus_one(st->rrdhost);
330 - }
331 - replication_progress = true;
332 -
333 -#ifdef NETDATA_LOG_REPLICATION_REQUESTS
334 - internal_error(true, "REPLAY: 'host:%s/chart:%s' replication starts",
335 - rrdhost_hostname(st->rrdhost), rrdset_id(st));
336 -#endif
337 - }
338 -
339 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
340 -
341 - // we can set the exposed flag, after we commit the buffer
342 - // because replication may pick it up prematurely
343 - rrddim_foreach_read(rd, st) {
344 - rrddim_metadata_exposed_upstream(rd, version);
345 - }
346 - rrddim_foreach_done(rd);
347 - rrdset_metadata_exposed_upstream(st, version);
348 -
349 - st->rrdpush.sender.resync_time_s = st->last_collected_time.tv_sec + (remote_clock_resync_iterations * st->update_every);
350 - return replication_progress;
351 -}
352 -
353 -// sends the current chart dimensions
354 -static void rrdpush_send_chart_metrics(BUFFER *wb, RRDSET *st, struct sender_state *s __maybe_unused, RRDSET_FLAGS flags) {
355 - buffer_fast_strcat(wb, "BEGIN \"", 7);
356 - buffer_fast_strcat(wb, rrdset_id(st), string_strlen(st->id));
357 - buffer_fast_strcat(wb, "\" ", 2);
358 -
359 - if(st->last_collected_time.tv_sec > st->rrdpush.sender.resync_time_s)
360 - buffer_print_uint64(wb, st->usec_since_last_update);
361 - else
362 - buffer_fast_strcat(wb, "0", 1);
363 -
364 - buffer_fast_strcat(wb, "\n", 1);
365 -
366 - RRDDIM *rd;
367 - rrddim_foreach_read(rd, st) {
368 - if(unlikely(!rrddim_check_updated(rd)))
369 - continue;
370 -
371 - if(likely(rrddim_check_upstream_exposed_collector(rd))) {
372 - buffer_fast_strcat(wb, "SET \"", 5);
373 - buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
374 - buffer_fast_strcat(wb, "\" = ", 4);
375 - buffer_print_int64(wb, rd->collector.collected_value);
376 - buffer_fast_strcat(wb, "\n", 1);
377 - }
378 - else {
379 - internal_error(true, "STREAM: 'host:%s/chart:%s/dim:%s' flag 'exposed' is updated but not exposed",
380 - rrdhost_hostname(st->rrdhost), rrdset_id(st), rrddim_id(rd));
381 - // we will include it in the next iteration
382 - rrddim_metadata_updated(rd);
383 - }
384 - }
385 - rrddim_foreach_done(rd);
386 -
387 - if(unlikely(flags & RRDSET_FLAG_UPSTREAM_SEND_VARIABLES))
388 - rrdvar_print_to_streaming_custom_chart_variables(st, wb);
389 -
390 - buffer_fast_strcat(wb, "END\n", 4);
391 -}
392 -
393 -static void rrdpush_sender_thread_spawn(RRDHOST *host);
394 -
395 -// Called from the internal collectors to mark a chart obsolete.
396 -bool rrdset_push_chart_definition_now(RRDSET *st) {
397 - RRDHOST *host = st->rrdhost;
398 -
399 - if(unlikely(!rrdhost_can_send_definitions_to_parent(host)
400 - || !should_send_chart_matching(st, rrdset_flag_get(st)))) {
401 - return false;
402 - }
403 -
404 - BUFFER *wb = sender_start(host->sender);
405 - rrdpush_send_chart_definition(wb, st);
406 - sender_thread_buffer_free();
407 -
408 - return true;
409 -}
410 -
411 -void rrdset_push_metrics_v1(RRDSET_STREAM_BUFFER *rsb, RRDSET *st) {
412 - RRDHOST *host = st->rrdhost;
413 - rrdpush_send_chart_metrics(rsb->wb, st, host->sender, rsb->rrdset_flags);
414 -}
415 -
416 -void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags) {
417 - if(!rsb->wb || !rsb->v2 || !netdata_double_isnumber(n) || !does_storage_number_exist(flags))
418 - return;
419 -
420 - bool with_slots = stream_has_capability(rsb, STREAM_CAP_SLOTS) ? true : false;
421 - NUMBER_ENCODING integer_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_HEX;
422 - NUMBER_ENCODING doubles_encoding = stream_has_capability(rsb, STREAM_CAP_IEEE754) ? NUMBER_ENCODING_BASE64 : NUMBER_ENCODING_DECIMAL;
423 - BUFFER *wb = rsb->wb;
424 - time_t point_end_time_s = (time_t)(point_end_time_ut / USEC_PER_SEC);
425 - if(unlikely(rsb->last_point_end_time_s != point_end_time_s)) {
426 -
427 - if(unlikely(rsb->begin_v2_added))
428 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
429 -
430 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_BEGIN_V2, sizeof(PLUGINSD_KEYWORD_BEGIN_V2) - 1);
431 -
432 - if(with_slots) {
433 - buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
434 - buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->rrdpush.sender.chart_slot);
435 - }
436 -
437 - buffer_fast_strcat(wb, " '", 2);
438 - buffer_fast_strcat(wb, rrdset_id(rd->rrdset), string_strlen(rd->rrdset->id));
439 - buffer_fast_strcat(wb, "' ", 2);
440 - buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdset->update_every);
441 - buffer_fast_strcat(wb, " ", 1);
442 - buffer_print_uint64_encoded(wb, integer_encoding, point_end_time_s);
443 - buffer_fast_strcat(wb, " ", 1);
444 - if(point_end_time_s == rsb->wall_clock_time)
445 - buffer_fast_strcat(wb, "#", 1);
446 - else
447 - buffer_print_uint64_encoded(wb, integer_encoding, rsb->wall_clock_time);
448 - buffer_fast_strcat(wb, "\n", 1);
449 -
450 - rsb->last_point_end_time_s = point_end_time_s;
451 - rsb->begin_v2_added = true;
452 - }
453 -
454 - buffer_fast_strcat(wb, PLUGINSD_KEYWORD_SET_V2, sizeof(PLUGINSD_KEYWORD_SET_V2) - 1);
455 -
456 - if(with_slots) {
457 - buffer_fast_strcat(wb, " "PLUGINSD_KEYWORD_SLOT":", sizeof(PLUGINSD_KEYWORD_SLOT) - 1 + 2);
458 - buffer_print_uint64_encoded(wb, integer_encoding, rd->rrdpush.sender.dim_slot);
459 - }
460 -
461 - buffer_fast_strcat(wb, " '", 2);
462 - buffer_fast_strcat(wb, rrddim_id(rd), string_strlen(rd->id));
463 - buffer_fast_strcat(wb, "' ", 2);
464 - buffer_print_int64_encoded(wb, integer_encoding, rd->collector.last_collected_value);
465 - buffer_fast_strcat(wb, " ", 1);
466 -
467 - if((NETDATA_DOUBLE)rd->collector.last_collected_value == n)
468 - buffer_fast_strcat(wb, "#", 1);
469 - else
470 - buffer_print_netdata_double_encoded(wb, doubles_encoding, n);
471 -
472 - buffer_fast_strcat(wb, " ", 1);
473 - buffer_print_sn_flags(wb, flags, true);
474 - buffer_fast_strcat(wb, "\n", 1);
475 -}
476 -
477 -void rrdset_push_metrics_finished(RRDSET_STREAM_BUFFER *rsb, RRDSET *st) {
478 - if(!rsb->wb)
479 - return;
480 -
481 - if(rsb->v2 && rsb->begin_v2_added) {
482 - if(unlikely(rsb->rrdset_flags & RRDSET_FLAG_UPSTREAM_SEND_VARIABLES))
483 - rrdvar_print_to_streaming_custom_chart_variables(st, rsb->wb);
484 -
485 - buffer_fast_strcat(rsb->wb, PLUGINSD_KEYWORD_END_V2 "\n", sizeof(PLUGINSD_KEYWORD_END_V2) - 1 + 1);
486 - }
487 -
488 - sender_commit(st->rrdhost->sender, rsb->wb, STREAM_TRAFFIC_TYPE_DATA);
489 -
490 - *rsb = (RRDSET_STREAM_BUFFER){ .wb = NULL, };
491 -}
492 -
493 -RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock_time) {
494 - RRDHOST *host = st->rrdhost;
495 -
496 - // fetch the flags we need to check with one atomic operation
497 - RRDHOST_FLAGS host_flags = __atomic_load_n(&host->flags, __ATOMIC_SEQ_CST);
498 -
499 - // check if we are not connected
500 - if(unlikely(!(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS))) {
501 -
502 - if(unlikely(!(host_flags & (RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN | RRDHOST_FLAG_RRDPUSH_RECEIVER_DISCONNECTED))))
503 - rrdpush_sender_thread_spawn(host);
504 -
505 - if(unlikely(!(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS))) {
506 - rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS);
507 - nd_log_daemon(NDLP_NOTICE, "STREAM %s [send]: not ready - collected metrics are not sent to parent.", rrdhost_hostname(host));
508 - }
509 -
510 - return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
511 - }
512 - else if(unlikely(host_flags & RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS)) {
513 - nd_log_daemon(NDLP_INFO, "STREAM %s [send]: sending metrics to parent...", rrdhost_hostname(host));
514 - rrdhost_flag_clear(host, RRDHOST_FLAG_RRDPUSH_SENDER_LOGGED_STATUS);
515 - }
516 -
517 - if(unlikely(host_flags & RRDHOST_FLAG_GLOBAL_FUNCTIONS_UPDATED)) {
518 - BUFFER *wb = sender_start(host->sender);
519 - rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
520 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
521 - }
522 -
523 - bool exposed_upstream = rrdset_check_upstream_exposed(st);
524 - RRDSET_FLAGS rrdset_flags = rrdset_flag_get(st);
525 - bool replication_in_progress = !(rrdset_flags & RRDSET_FLAG_SENDER_REPLICATION_FINISHED);
526 -
527 - if(unlikely((exposed_upstream && replication_in_progress) ||
528 - !should_send_chart_matching(st, rrdset_flags)))
529 - return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
530 -
531 - if(unlikely(!exposed_upstream)) {
532 - BUFFER *wb = sender_start(host->sender);
533 - replication_in_progress = rrdpush_send_chart_definition(wb, st);
534 - }
535 -
536 - if(replication_in_progress)
537 - return (RRDSET_STREAM_BUFFER) { .wb = NULL, };
538 -
539 - return (RRDSET_STREAM_BUFFER) {
540 - .capabilities = host->sender->capabilities,
541 - .v2 = stream_has_capability(host->sender, STREAM_CAP_INTERPOLATED),
542 - .rrdset_flags = rrdset_flags,
543 - .wb = sender_start(host->sender),
544 - .wall_clock_time = wall_clock_time,
545 - };
546 -}
547 -
548 -// labels
549 -static int send_labels_callback(const char *name, const char *value, RRDLABEL_SRC ls, void *data) {
550 - BUFFER *wb = (BUFFER *)data;
551 - buffer_sprintf(wb, "LABEL \"%s\" = %d \"%s\"\n", name, ls, value);
552 - return 1;
553 -}
554 -
555 -void rrdpush_send_host_labels(RRDHOST *host) {
556 - if(unlikely(!rrdhost_can_send_definitions_to_parent(host)
557 - || !stream_has_capability(host->sender, STREAM_CAP_HLABELS)))
558 - return;
559 -
560 - BUFFER *wb = sender_start(host->sender);
561 -
562 - rrdlabels_walkthrough_read(host->rrdlabels, send_labels_callback, wb);
563 - buffer_sprintf(wb, "OVERWRITE %s\n", "labels");
564 -
565 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
566 -
567 - sender_thread_buffer_free();
568 -}
569 -
570 -void rrdpush_send_global_functions(RRDHOST *host) {
571 - if(!stream_has_capability(host->sender, STREAM_CAP_FUNCTIONS))
572 - return;
573 -
574 - if(unlikely(!rrdhost_can_send_definitions_to_parent(host)))
575 - return;
576 -
577 - BUFFER *wb = sender_start(host->sender);
578 -
579 - rrd_global_functions_expose_rrdpush(host, wb, stream_has_capability(host->sender, STREAM_CAP_DYNCFG));
580 -
581 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_FUNCTIONS);
582 -
583 - sender_thread_buffer_free();
584 -}
585 -
586 -int connect_to_one_of_destinations(
587 - RRDHOST *host,
588 - int default_port,
589 - struct timeval *timeout,
590 - size_t *reconnects_counter,
591 - char *connected_to,
592 - size_t connected_to_size,
593 - struct rrdpush_destinations **destination)
594 -{
595 - int sock = -1;
596 -
597 - for (struct rrdpush_destinations *d = host->destinations; d; d = d->next) {
598 - time_t now = now_realtime_sec();
599 -
600 - if(nd_thread_signaled_to_cancel())
601 - return -1;
602 -
603 - if(d->postpone_reconnection_until > now)
604 - continue;
605 -
606 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
607 - "STREAM %s: connecting to '%s' (default port: %d)...",
608 - rrdhost_hostname(host), string2str(d->destination), default_port);
609 -
610 - if (reconnects_counter)
611 - *reconnects_counter += 1;
612 -
613 - d->since = now;
614 - d->attempts++;
615 - sock = connect_to_this(string2str(d->destination), default_port, timeout);
616 -
617 - if (sock != -1) {
618 - if (connected_to && connected_to_size)
619 - strncpyz(connected_to, string2str(d->destination), connected_to_size);
620 -
621 - *destination = d;
622 -
623 - // move the current item to the end of the list
624 - // without this, this destination will break the loop again and again
625 - // not advancing the destinations to find one that may work
626 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, d, prev, next);
627 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(host->destinations, d, prev, next);
628 -
629 - break;
630 - }
631 - }
632 -
633 - return sock;
634 -}
635 -
636 -struct destinations_init_tmp {
637 - RRDHOST *host;
638 - struct rrdpush_destinations *list;
639 - int count;
640 -};
641 -
642 -bool destinations_init_add_one(char *entry, void *data) {
643 - struct destinations_init_tmp *t = data;
644 -
645 - struct rrdpush_destinations *d = callocz(1, sizeof(struct rrdpush_destinations));
646 - char *colon_ssl = strstr(entry, ":SSL");
647 - if(colon_ssl) {
648 - *colon_ssl = '\0';
649 - d->ssl = true;
650 - }
651 - else
652 - d->ssl = false;
653 -
654 - d->destination = string_strdupz(entry);
655 -
656 - __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
657 -
658 - DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(t->list, d, prev, next);
659 -
660 - t->count++;
661 - nd_log_daemon(NDLP_INFO, "STREAM: added streaming destination No %d: '%s' to host '%s'", t->count, string2str(d->destination), rrdhost_hostname(t->host));
662 -
663 - return false; // we return false, so that we will get all defined destinations
664 -}
665 -
666 -void rrdpush_destinations_init(RRDHOST *host) {
667 - if(!host->rrdpush.send.destination) return;
668 -
669 - rrdpush_destinations_free(host);
670 -
671 - struct destinations_init_tmp t = {
672 - .host = host,
673 - .list = NULL,
674 - .count = 0,
675 - };
676 -
677 - foreach_entry_in_connection_string(host->rrdpush.send.destination, destinations_init_add_one, &t);
678 -
679 - host->destinations = t.list;
680 -}
681 -
682 -void rrdpush_destinations_free(RRDHOST *host) {
683 - while (host->destinations) {
684 - struct rrdpush_destinations *tmp = host->destinations;
685 - DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, tmp, prev, next);
686 - string_freez(tmp->destination);
687 - freez(tmp);
688 - __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
689 - }
690 -
691 - host->destinations = NULL;
692 -}
693 -
694 -// ----------------------------------------------------------------------------
695 -// rrdpush sender thread
696 -
697 -// Either the receiver lost the connection or the host is being destroyed.
698 -// The sender mutex guards thread creation, any spurious data is wiped on reconnection.
699 -void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait) {
700 - if (!host->sender)
701 - return;
702 -
703 - sender_lock(host->sender);
704 -
705 - if(rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN)) {
706 -
707 - host->sender->exit.shutdown = true;
708 - host->sender->exit.reason = reason;
709 -
710 - // signal it to cancel
711 - nd_thread_signal_cancel(host->rrdpush_sender_thread);
712 - }
713 -
714 - sender_unlock(host->sender);
715 -
716 - if(wait) {
717 - sender_lock(host->sender);
718 - while(host->sender->tid) {
719 - sender_unlock(host->sender);
720 - sleep_usec(10 * USEC_PER_MS);
721 - sender_lock(host->sender);
722 - }
723 - sender_unlock(host->sender);
724 - }
725 -}
726 -
727 -// ----------------------------------------------------------------------------
728 -// rrdpush receiver thread
729 -
730 -static void rrdpush_sender_thread_spawn(RRDHOST *host) {
731 - sender_lock(host->sender);
732 -
733 - if(!rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN)) {
734 - char tag[NETDATA_THREAD_TAG_MAX + 1];
735 - snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_SENDER "[%s]", rrdhost_hostname(host));
736 -
737 - host->rrdpush_sender_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT,
738 - rrdpush_sender_thread, (void *)host->sender);
739 - if(!host->rrdpush_sender_thread)
740 - nd_log_daemon(NDLP_ERR, "STREAM %s [send]: failed to create new thread for client.", rrdhost_hostname(host));
741 - else
742 - rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN);
743 - }
744 -
745 - sender_unlock(host->sender);
746 -}
747 -
748 -int rrdpush_receiver_permission_denied(struct web_client *w) {
749 - // we always respond with the same message and error code
750 - // to prevent an attacker from gaining info about the error
751 - buffer_flush(w->response.data);
752 - buffer_strcat(w->response.data, START_STREAMING_ERROR_NOT_PERMITTED);
753 - return HTTP_RESP_UNAUTHORIZED;
754 -}
755 -
756 -int rrdpush_receiver_too_busy_now(struct web_client *w) {
757 - // we always respond with the same message and error code
758 - // to prevent an attacker from gaining info about the error
759 - buffer_flush(w->response.data);
760 - buffer_strcat(w->response.data, START_STREAMING_ERROR_BUSY_TRY_LATER);
761 - return HTTP_RESP_SERVICE_UNAVAILABLE;
762 -}
763 -
764 -static void rrdpush_receiver_takeover_web_connection(struct web_client *w, struct receiver_state *rpt) {
765 - rpt->fd = w->ifd;
766 -
767 - rpt->ssl.conn = w->ssl.conn;
768 - rpt->ssl.state = w->ssl.state;
769 -
770 - w->ssl = NETDATA_SSL_UNSET_CONNECTION;
771 -
772 - WEB_CLIENT_IS_DEAD(w);
773 -
774 - if(web_server_mode == WEB_SERVER_MODE_STATIC_THREADED) {
775 - web_client_flag_set(w, WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET);
776 - }
777 - else {
778 - if(w->ifd == w->ofd)
779 - w->ifd = w->ofd = -1;
780 - else
781 - w->ifd = -1;
782 - }
783 -
784 - buffer_flush(w->response.data);
785 -}
786 -
787 -void *rrdpush_receiver_thread(void *ptr);
788 -int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx __maybe_unused) {
789 -
790 - if(!service_running(ABILITY_STREAMING_CONNECTIONS))
791 - return rrdpush_receiver_too_busy_now(w);
792 -
793 - struct receiver_state *rpt = callocz(1, sizeof(*rpt));
794 - rpt->connected_since_s = now_realtime_sec();
795 - rpt->last_msg_t = now_monotonic_sec();
796 - rpt->hops = 1;
797 -
798 - rpt->capabilities = STREAM_CAP_INVALID;
799 -
800 -#ifdef ENABLE_H2O
801 - rpt->h2o_ctx = h2o_ctx;
802 -#endif
803 -
804 - __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_receivers, sizeof(*rpt), __ATOMIC_RELAXED);
805 - __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_allocations_size, sizeof(struct rrdhost_system_info), __ATOMIC_RELAXED);
806 -
807 - rpt->system_info = callocz(1, sizeof(struct rrdhost_system_info));
808 - rpt->system_info->hops = rpt->hops;
809 -
810 - rpt->fd = -1;
811 - rpt->client_ip = strdupz(w->client_ip);
812 - rpt->client_port = strdupz(w->client_port);
813 -
814 - rpt->ssl = NETDATA_SSL_UNSET_CONNECTION;
815 -
816 - rpt->config.update_every = default_rrd_update_every;
817 -
818 - // parse the parameters and fill rpt and rpt->system_info
819 -
820 - while(decoded_query_string) {
821 - char *value = strsep_skip_consecutive_separators(&decoded_query_string, "&");
822 - if(!value || !*value) continue;
823 -
824 - char *name = strsep_skip_consecutive_separators(&value, "=");
825 - if(!name || !*name) continue;
826 - if(!value || !*value) continue;
827 -
828 - if(!strcmp(name, "key") && !rpt->key)
829 - rpt->key = strdupz(value);
830 -
831 - else if(!strcmp(name, "hostname") && !rpt->hostname)
832 - rpt->hostname = strdupz(value);
833 -
834 - else if(!strcmp(name, "registry_hostname") && !rpt->registry_hostname)
835 - rpt->registry_hostname = strdupz(value);
836 -
837 - else if(!strcmp(name, "machine_guid") && !rpt->machine_guid)
838 - rpt->machine_guid = strdupz(value);
839 -
840 - else if(!strcmp(name, "update_every"))
841 - rpt->config.update_every = (int)strtoul(value, NULL, 0);
842 -
843 - else if(!strcmp(name, "os") && !rpt->os)
844 - rpt->os = strdupz(value);
845 -
846 - else if(!strcmp(name, "timezone") && !rpt->timezone)
847 - rpt->timezone = strdupz(value);
848 -
849 - else if(!strcmp(name, "abbrev_timezone") && !rpt->abbrev_timezone)
850 - rpt->abbrev_timezone = strdupz(value);
851 -
852 - else if(!strcmp(name, "utc_offset"))
853 - rpt->utc_offset = (int32_t)strtol(value, NULL, 0);
854 -
855 - else if(!strcmp(name, "hops"))
856 - rpt->hops = rpt->system_info->hops = (uint16_t) strtoul(value, NULL, 0);
857 -
858 - else if(!strcmp(name, "ml_capable"))
859 - rpt->system_info->ml_capable = strtoul(value, NULL, 0);
860 -
861 - else if(!strcmp(name, "ml_enabled"))
862 - rpt->system_info->ml_enabled = strtoul(value, NULL, 0);
863 -
864 - else if(!strcmp(name, "mc_version"))
865 - rpt->system_info->mc_version = strtoul(value, NULL, 0);
866 -
867 - else if(!strcmp(name, "ver") && (rpt->capabilities & STREAM_CAP_INVALID))
868 - rpt->capabilities = convert_stream_version_to_capabilities(strtoul(value, NULL, 0), NULL, false);
869 -
870 - else {
871 - // An old Netdata child does not have a compatible streaming protocol, map to something sane.
872 - if (!strcmp(name, "NETDATA_SYSTEM_OS_NAME"))
873 - name = "NETDATA_HOST_OS_NAME";
874 -
875 - else if (!strcmp(name, "NETDATA_SYSTEM_OS_ID"))
876 - name = "NETDATA_HOST_OS_ID";
877 -
878 - else if (!strcmp(name, "NETDATA_SYSTEM_OS_ID_LIKE"))
879 - name = "NETDATA_HOST_OS_ID_LIKE";
880 -
881 - else if (!strcmp(name, "NETDATA_SYSTEM_OS_VERSION"))
882 - name = "NETDATA_HOST_OS_VERSION";
883 -
884 - else if (!strcmp(name, "NETDATA_SYSTEM_OS_VERSION_ID"))
885 - name = "NETDATA_HOST_OS_VERSION_ID";
886 -
887 - else if (!strcmp(name, "NETDATA_SYSTEM_OS_DETECTION"))
888 - name = "NETDATA_HOST_OS_DETECTION";
889 -
890 - else if(!strcmp(name, "NETDATA_PROTOCOL_VERSION") && (rpt->capabilities & STREAM_CAP_INVALID))
891 - rpt->capabilities = convert_stream_version_to_capabilities(1, NULL, false);
892 -
893 - if (unlikely(rrdhost_set_system_info_variable(rpt->system_info, name, value))) {
894 - nd_log_daemon(NDLP_NOTICE, "STREAM '%s' [receive from [%s]:%s]: "
895 - "request has parameter '%s' = '%s', which is not used."
896 - , (rpt->hostname && *rpt->hostname) ? rpt->hostname : "-"
897 - , rpt->client_ip, rpt->client_port
898 - , name, value);
899 - }
900 - }
901 - }
902 -
903 - if (rpt->capabilities & STREAM_CAP_INVALID)
904 - // no version is supplied, assume version 0;
905 - rpt->capabilities = convert_stream_version_to_capabilities(0, NULL, false);
906 -
907 - // find the program name and version
908 - if(w->user_agent && w->user_agent[0]) {
909 - char *t = strchr(w->user_agent, '/');
910 - if(t && *t) {
911 - *t = '\0';
912 - t++;
913 - }
914 -
915 - rpt->program_name = strdupz(w->user_agent);
916 - if(t && *t) rpt->program_version = strdupz(t);
917 - }
918 -
919 - // check if we should accept this connection
920 -
921 - if(!rpt->key || !*rpt->key) {
922 - rrdpush_receive_log_status(
923 - rpt, "request without an API key, rejecting connection",
924 - RRDPUSH_STATUS_NO_API_KEY, NDLP_WARNING);
925 -
926 - receiver_state_free(rpt);
927 - return rrdpush_receiver_permission_denied(w);
928 - }
929 -
930 - if(!rpt->hostname || !*rpt->hostname) {
931 - rrdpush_receive_log_status(
932 - rpt, "request without a hostname, rejecting connection",
933 - RRDPUSH_STATUS_NO_HOSTNAME, NDLP_WARNING);
934 -
935 - receiver_state_free(rpt);
936 - return rrdpush_receiver_permission_denied(w);
937 - }
938 -
939 - if(!rpt->registry_hostname)
940 - rpt->registry_hostname = strdupz(rpt->hostname);
941 -
942 - if(!rpt->machine_guid || !*rpt->machine_guid) {
943 - rrdpush_receive_log_status(
944 - rpt, "request without a machine GUID, rejecting connection",
945 - RRDPUSH_STATUS_NO_MACHINE_GUID, NDLP_WARNING);
946 -
947 - receiver_state_free(rpt);
948 - return rrdpush_receiver_permission_denied(w);
949 - }
950 -
951 - {
952 - char buf[GUID_LEN + 1];
953 -
954 - if (regenerate_guid(rpt->key, buf) == -1) {
955 - rrdpush_receive_log_status(
956 - rpt, "API key is not a valid UUID (use the command uuidgen to generate one)",
957 - RRDPUSH_STATUS_INVALID_API_KEY, NDLP_WARNING);
958 -
959 - receiver_state_free(rpt);
960 - return rrdpush_receiver_permission_denied(w);
961 - }
962 -
963 - if (regenerate_guid(rpt->machine_guid, buf) == -1) {
964 - rrdpush_receive_log_status(
965 - rpt, "machine GUID is not a valid UUID",
966 - RRDPUSH_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
967 -
968 - receiver_state_free(rpt);
969 - return rrdpush_receiver_permission_denied(w);
970 - }
971 - }
972 -
973 - const char *api_key_type = appconfig_get(&stream_config, rpt->key, "type", "api");
974 - if(!api_key_type || !*api_key_type) api_key_type = "unknown";
975 - if(strcmp(api_key_type, "api") != 0) {
976 - rrdpush_receive_log_status(
977 - rpt, "API key is a machine GUID",
978 - RRDPUSH_STATUS_INVALID_API_KEY, NDLP_WARNING);
979 -
980 - receiver_state_free(rpt);
981 - return rrdpush_receiver_permission_denied(w);
982 - }
983 -
984 - if(!appconfig_get_boolean(&stream_config, rpt->key, "enabled", 0)) {
985 - rrdpush_receive_log_status(
986 - rpt, "API key is not enabled",
987 - RRDPUSH_STATUS_API_KEY_DISABLED, NDLP_WARNING);
988 -
989 - receiver_state_free(rpt);
990 - return rrdpush_receiver_permission_denied(w);
991 - }
992 -
993 - {
994 - SIMPLE_PATTERN *key_allow_from = simple_pattern_create(
995 - appconfig_get(&stream_config, rpt->key, "allow from", "*"),
996 - NULL, SIMPLE_PATTERN_EXACT, true);
997 -
998 - if(key_allow_from) {
999 - if(!simple_pattern_matches(key_allow_from, w->client_ip)) {
1000 - simple_pattern_free(key_allow_from);
1001 -
1002 - rrdpush_receive_log_status(
1003 - rpt, "API key is not allowed from this IP",
1004 - RRDPUSH_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
1005 -
1006 - receiver_state_free(rpt);
1007 - return rrdpush_receiver_permission_denied(w);
1008 - }
1009 -
1010 - simple_pattern_free(key_allow_from);
1011 - }
1012 - }
1013 -
1014 - {
1015 - const char *machine_guid_type = appconfig_get(&stream_config, rpt->machine_guid, "type", "machine");
1016 - if (!machine_guid_type || !*machine_guid_type) machine_guid_type = "unknown";
1017 -
1018 - if (strcmp(machine_guid_type, "machine") != 0) {
1019 - rrdpush_receive_log_status(
1020 - rpt, "machine GUID is an API key",
1021 - RRDPUSH_STATUS_INVALID_MACHINE_GUID, NDLP_WARNING);
1022 -
1023 - receiver_state_free(rpt);
1024 - return rrdpush_receiver_permission_denied(w);
1025 - }
1026 - }
1027 -
1028 - if(!appconfig_get_boolean(&stream_config, rpt->machine_guid, "enabled", 1)) {
1029 - rrdpush_receive_log_status(
1030 - rpt, "machine GUID is not enabled",
1031 - RRDPUSH_STATUS_MACHINE_GUID_DISABLED, NDLP_WARNING);
1032 -
1033 - receiver_state_free(rpt);
1034 - return rrdpush_receiver_permission_denied(w);
1035 - }
1036 -
1037 - {
1038 - SIMPLE_PATTERN *machine_allow_from = simple_pattern_create(
1039 - appconfig_get(&stream_config, rpt->machine_guid, "allow from", "*"),
1040 - NULL, SIMPLE_PATTERN_EXACT, true);
1041 -
1042 - if(machine_allow_from) {
1043 - if(!simple_pattern_matches(machine_allow_from, w->client_ip)) {
1044 - simple_pattern_free(machine_allow_from);
1045 -
1046 - rrdpush_receive_log_status(
1047 - rpt, "machine GUID is not allowed from this IP",
1048 - RRDPUSH_STATUS_NOT_ALLOWED_IP, NDLP_WARNING);
1049 -
1050 - receiver_state_free(rpt);
1051 - return rrdpush_receiver_permission_denied(w);
1052 - }
1053 -
1054 - simple_pattern_free(machine_allow_from);
1055 - }
1056 - }
1057 -
1058 - if (strcmp(rpt->machine_guid, localhost->machine_guid) == 0) {
1059 -
1060 - rrdpush_receiver_takeover_web_connection(w, rpt);
1061 -
1062 - rrdpush_receive_log_status(
1063 - rpt, "machine GUID is my own",
1064 - RRDPUSH_STATUS_LOCALHOST, NDLP_DEBUG);
1065 -
1066 - char initial_response[HTTP_HEADER_SIZE + 1];
1067 - snprintfz(initial_response, HTTP_HEADER_SIZE, "%s", START_STREAMING_ERROR_SAME_LOCALHOST);
1068 -
1069 - if(send_timeout(
1070 - &rpt->ssl,
1071 - rpt->fd, initial_response, strlen(initial_response), 0, 60) != (ssize_t)strlen(initial_response)) {
1072 -
1073 - nd_log_daemon(NDLP_ERR, "STREAM '%s' [receive from [%s]:%s]: "
1074 - "failed to reply."
1075 - , rpt->hostname
1076 - , rpt->client_ip, rpt->client_port
1077 - );
1078 - }
1079 -
1080 - receiver_state_free(rpt);
1081 - return HTTP_RESP_OK;
1082 - }
1083 -
1084 - if(unlikely(web_client_streaming_rate_t > 0)) {
1085 - static SPINLOCK spinlock = NETDATA_SPINLOCK_INITIALIZER;
1086 - static time_t last_stream_accepted_t = 0;
1087 -
1088 - time_t now = now_realtime_sec();
1089 - spinlock_lock(&spinlock);
1090 -
1091 - if(unlikely(last_stream_accepted_t == 0))
1092 - last_stream_accepted_t = now;
1093 -
1094 - if(now - last_stream_accepted_t < web_client_streaming_rate_t) {
1095 - spinlock_unlock(&spinlock);
1096 -
1097 - char msg[100 + 1];
1098 - snprintfz(msg, sizeof(msg) - 1,
1099 - "rate limit, will accept new connection in %ld secs",
1100 - (long)(web_client_streaming_rate_t - (now - last_stream_accepted_t)));
1101 -
1102 - rrdpush_receive_log_status(
1103 - rpt, msg,
1104 - RRDPUSH_STATUS_RATE_LIMIT, NDLP_NOTICE);
1105 -
1106 - receiver_state_free(rpt);
1107 - return rrdpush_receiver_too_busy_now(w);
1108 - }
1109 -
1110 - last_stream_accepted_t = now;
1111 - spinlock_unlock(&spinlock);
1112 - }
1113 -
1114 - /*
1115 - * Quick path for rejecting multiple connections. The lock taken is fine-grained - it only protects the receiver
1116 - * pointer within the host (if a host exists). This protects against multiple concurrent web requests hitting
1117 - * separate threads within the web-server and landing here. The lock guards the thread-shutdown sequence that
1118 - * detaches the receiver from the host. If the host is being created (first time-access) then we also use the
1119 - * lock to prevent race-hazard (two threads try to create the host concurrently, one wins and the other does a
1120 - * lookup to the now-attached structure).
1121 - */
1122 -
1123 - {
1124 - time_t age = 0;
1125 - bool receiver_stale = false;
1126 - bool receiver_working = false;
1127 -
1128 - rrd_rdlock();
1129 - RRDHOST *host = rrdhost_find_by_guid(rpt->machine_guid);
1130 - if (unlikely(host && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED))) /* Ignore archived hosts. */
1131 - host = NULL;
1132 -
1133 - if (host) {
1134 - spinlock_lock(&host->receiver_lock);
1135 - if (host->receiver) {
1136 - age = now_monotonic_sec() - host->receiver->last_msg_t;
1137 -
1138 - if (age < 30)
1139 - receiver_working = true;
1140 - else
1141 - receiver_stale = true;
1142 - }
1143 - spinlock_unlock(&host->receiver_lock);
1144 - }
1145 - rrd_rdunlock();
1146 -
1147 - if (receiver_stale && stop_streaming_receiver(host, STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER)) {
1148 - // we stopped the receiver
1149 - // we can proceed with this connection
1150 - receiver_stale = false;
1151 -
1152 - nd_log_daemon(NDLP_NOTICE, "STREAM '%s' [receive from [%s]:%s]: "
1153 - "stopped previous stale receiver to accept this one."
1154 - , rpt->hostname
1155 - , rpt->client_ip, rpt->client_port
1156 - );
1157 - }
1158 -
1159 - if (receiver_working || receiver_stale) {
1160 - // another receiver is already connected
1161 - // try again later
1162 -
1163 - char msg[200 + 1];
1164 - snprintfz(msg, sizeof(msg) - 1,
1165 - "multiple connections for same host, "
1166 - "old connection was last used %ld secs ago%s",
1167 - age, receiver_stale ? " (signaled old receiver to stop)" : " (new connection not accepted)");
1168 -
1169 - rrdpush_receive_log_status(
1170 - rpt, msg,
1171 - RRDPUSH_STATUS_ALREADY_CONNECTED, NDLP_DEBUG);
1172 -
1173 - // Have not set WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET - caller should clean up
1174 - buffer_flush(w->response.data);
1175 - buffer_strcat(w->response.data, START_STREAMING_ERROR_ALREADY_STREAMING);
1176 - receiver_state_free(rpt);
1177 - return HTTP_RESP_CONFLICT;
1178 - }
1179 - }
1180 -
1181 - rrdpush_receiver_takeover_web_connection(w, rpt);
1182 -
1183 - char tag[NETDATA_THREAD_TAG_MAX + 1];
1184 - snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_RECEIVER "[%s]", rpt->hostname);
1185 - tag[NETDATA_THREAD_TAG_MAX] = '\0';
1186 -
1187 - rpt->thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, rrdpush_receiver_thread, (void *)rpt);
1188 - if(!rpt->thread) {
1189 - rrdpush_receive_log_status(
1190 - rpt, "can't create receiver thread",
1191 - RRDPUSH_STATUS_INTERNAL_SERVER_ERROR, NDLP_ERR);
1192 -
1193 - buffer_flush(w->response.data);
1194 - buffer_strcat(w->response.data, "Can't handle this request");
1195 - receiver_state_free(rpt);
1196 - return HTTP_RESP_INTERNAL_SERVER_ERROR;
1197 - }
1198 -
1199 - // prevent the caller from closing the streaming socket
1200 - return HTTP_RESP_OK;
1201 -}
1202 -
1203 -void rrdpush_reset_destinations_postpone_time(RRDHOST *host) {
1204 - uint32_t wait = (host->sender) ? host->sender->reconnect_delay : 5;
1205 - time_t now = now_realtime_sec();
1206 - for (struct rrdpush_destinations *d = host->destinations; d; d = d->next)
1207 - d->postpone_reconnection_until = now + wait;
1208 -}
1209 -
1210 -static struct {
1211 - STREAM_HANDSHAKE err;
1212 - const char *str;
1213 -} handshake_errors[] = {
1214 - { STREAM_HANDSHAKE_OK_V3, "CONNECTED" },
1215 - { STREAM_HANDSHAKE_OK_V2, "CONNECTED" },
1216 - { STREAM_HANDSHAKE_OK_V1, "CONNECTED" },
1217 - { STREAM_HANDSHAKE_NEVER, "" },
1218 - { STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE, "BAD HANDSHAKE" },
1219 - { STREAM_HANDSHAKE_ERROR_LOCALHOST, "LOCALHOST" },
1220 - { STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED, "ALREADY CONNECTED" },
1221 - { STREAM_HANDSHAKE_ERROR_DENIED, "DENIED" },
1222 - { STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT, "SEND TIMEOUT" },
1223 - { STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT, "RECEIVE TIMEOUT" },
1224 - { STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE, "INVALID CERTIFICATE" },
1225 - { STREAM_HANDSHAKE_ERROR_SSL_ERROR, "SSL ERROR" },
1226 - { STREAM_HANDSHAKE_ERROR_CANT_CONNECT, "CANT CONNECT" },
1227 - { STREAM_HANDSHAKE_BUSY_TRY_LATER, "BUSY TRY LATER" },
1228 - { STREAM_HANDSHAKE_INTERNAL_ERROR, "INTERNAL ERROR" },
1229 - { STREAM_HANDSHAKE_INITIALIZATION, "REMOTE IS INITIALIZING" },
1230 - { STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, "DISCONNECTED HOST CLEANUP" },
1231 - { STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER, "DISCONNECTED STALE RECEIVER" },
1232 - { STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, "DISCONNECTED SHUTDOWN REQUESTED" },
1233 - { STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT, "DISCONNECTED NETDATA EXIT" },
1234 - { STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, "DISCONNECTED PARSE ENDED" },
1235 - {STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR, "DISCONNECTED UNKNOWN SOCKET READ ERROR" },
1236 - { STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, "DISCONNECTED PARSE ERROR" },
1237 - { STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, "DISCONNECTED RECEIVER LEFT" },
1238 - { STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST, "DISCONNECTED ORPHAN HOST" },
1239 - { STREAM_HANDSHAKE_NON_STREAMABLE_HOST, "NON STREAMABLE HOST" },
1240 - { STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER, "DISCONNECTED NOT SUFFICIENT READ BUFFER" },
1241 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF, "DISCONNECTED SOCKET EOF" },
1242 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED, "DISCONNECTED SOCKET READ FAILED" },
1243 - {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_TIMEOUT, "DISCONNECTED SOCKET READ TIMEOUT" },
1244 - { 0, NULL },
1245 -};
1246 -
1247 -const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error) {
1248 - if(handshake_error >= STREAM_HANDSHAKE_OK_V1)
1249 - // handshake_error is the whole version / capabilities number
1250 - return "CONNECTED";
1251 -
1252 - for(size_t i = 0; handshake_errors[i].str ; i++) {
1253 - if(handshake_error == handshake_errors[i].err)
1254 - return handshake_errors[i].str;
1255 - }
1256 -
1257 - return "UNKNOWN";
1258 -}
src/streaming/rrdpush.h
+8 -681
@@ -3,689 +3,16 @@
3 #ifndef NETDATA_RRDPUSH_H
4 #define NETDATA_RRDPUSH_H 1
5
6 -#include "libnetdata/libnetdata.h"
7 -#include "daemon/common.h"
8 -#include "web/server/web_client.h"
9 -#include "database/rrdfunctions.h"
10 -#include "database/rrd.h"
11 -#include "stream_capabilities.h"
6 +#include "stream-handshake.h"
7 +#include "stream-capabilities.h"
8 +#include "stream-conf.h"
9 +#include "stream-compression/compression.h"
10
13 -// When a child disconnects this is the maximum we will wait
14 -// before we update the cloud that the child is offline
15 -#define MAX_CHILD_DISC_DELAY (30000)
16 -#define MAX_CHILD_DISC_TOLERANCE (125 / 100)
17 -
18 -#define CONNECTED_TO_SIZE 100
19 -#define CBUFFER_INITIAL_SIZE (16 * 1024)
20 -#define THREAD_BUFFER_INITIAL_SIZE (CBUFFER_INITIAL_SIZE / 2)
21 -
22 -// ----------------------------------------------------------------------------
23 -// stream handshake
24 -
25 -#define HTTP_HEADER_SIZE 8192
26 -
27 -#define STREAMING_PROTOCOL_VERSION "1.1"
28 -#define START_STREAMING_PROMPT_V1 "Hit me baby, push them over..."
29 -#define START_STREAMING_PROMPT_V2 "Hit me baby, push them over and bring the host labels..."
30 -#define START_STREAMING_PROMPT_VN "Hit me baby, push them over with the version="
31 -
32 -#define START_STREAMING_ERROR_SAME_LOCALHOST "Don't hit me baby, you are trying to stream my localhost back"
33 -#define START_STREAMING_ERROR_ALREADY_STREAMING "This GUID is already streaming to this server"
34 -#define START_STREAMING_ERROR_NOT_PERMITTED "You are not permitted to access this. Check the logs for more info."
35 -#define START_STREAMING_ERROR_BUSY_TRY_LATER "The server is too busy now to accept this request. Try later."
36 -#define START_STREAMING_ERROR_INTERNAL_ERROR "The server encountered an internal error. Try later."
37 -#define START_STREAMING_ERROR_INITIALIZATION "The server is initializing. Try later."
38 -
39 -#define RRDPUSH_STATUS_CONNECTED "CONNECTED"
40 -#define RRDPUSH_STATUS_ALREADY_CONNECTED "ALREADY CONNECTED"
41 -#define RRDPUSH_STATUS_DISCONNECTED "DISCONNECTED"
42 -#define RRDPUSH_STATUS_RATE_LIMIT "RATE LIMIT TRY LATER"
43 -#define RRDPUSH_STATUS_INITIALIZATION_IN_PROGRESS "INITIALIZATION IN PROGRESS RETRY LATER"
44 -#define RRDPUSH_STATUS_INTERNAL_SERVER_ERROR "INTERNAL SERVER ERROR DROPPING CONNECTION"
45 -#define RRDPUSH_STATUS_DUPLICATE_RECEIVER "DUPLICATE RECEIVER DROPPING CONNECTION"
46 -#define RRDPUSH_STATUS_CANT_REPLY "CANT REPLY DROPPING CONNECTION"
47 -#define RRDPUSH_STATUS_NO_HOSTNAME "NO HOSTNAME PERMISSION DENIED"
48 -#define RRDPUSH_STATUS_NO_API_KEY "NO API KEY PERMISSION DENIED"
49 -#define RRDPUSH_STATUS_INVALID_API_KEY "INVALID API KEY PERMISSION DENIED"
50 -#define RRDPUSH_STATUS_NO_MACHINE_GUID "NO MACHINE GUID PERMISSION DENIED"
51 -#define RRDPUSH_STATUS_MACHINE_GUID_DISABLED "MACHINE GUID DISABLED PERMISSION DENIED"
52 -#define RRDPUSH_STATUS_INVALID_MACHINE_GUID "INVALID MACHINE GUID PERMISSION DENIED"
53 -#define RRDPUSH_STATUS_API_KEY_DISABLED "API KEY DISABLED PERMISSION DENIED"
54 -#define RRDPUSH_STATUS_NOT_ALLOWED_IP "NOT ALLOWED IP PERMISSION DENIED"
55 -#define RRDPUSH_STATUS_LOCALHOST "LOCALHOST PERMISSION DENIED"
56 -#define RRDPUSH_STATUS_PERMISSION_DENIED "PERMISSION DENIED"
57 -#define RRDPUSH_STATUS_BAD_HANDSHAKE "BAD HANDSHAKE"
58 -#define RRDPUSH_STATUS_TIMEOUT "TIMEOUT"
59 -#define RRDPUSH_STATUS_CANT_UPGRADE_CONNECTION "CANT UPGRADE CONNECTION"
60 -#define RRDPUSH_STATUS_SSL_ERROR "SSL ERROR"
61 -#define RRDPUSH_STATUS_INVALID_SSL_CERTIFICATE "INVALID SSL CERTIFICATE"
62 -#define RRDPUSH_STATUS_CANT_ESTABLISH_SSL_CONNECTION "CANT ESTABLISH SSL CONNECTION"
63 -
64 -typedef enum {
65 - STREAM_HANDSHAKE_OK_V3 = 3, // v3+
66 - STREAM_HANDSHAKE_OK_V2 = 2, // v2
67 - STREAM_HANDSHAKE_OK_V1 = 1, // v1
68 - STREAM_HANDSHAKE_NEVER = 0, // never tried to connect
69 - STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE = -1,
70 - STREAM_HANDSHAKE_ERROR_LOCALHOST = -2,
71 - STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED = -3,
72 - STREAM_HANDSHAKE_ERROR_DENIED = -4,
73 - STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT = -5,
74 - STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT = -6,
75 - STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE = -7,
76 - STREAM_HANDSHAKE_ERROR_SSL_ERROR = -8,
77 - STREAM_HANDSHAKE_ERROR_CANT_CONNECT = -9,
78 - STREAM_HANDSHAKE_BUSY_TRY_LATER = -10,
79 - STREAM_HANDSHAKE_INTERNAL_ERROR = -11,
80 - STREAM_HANDSHAKE_INITIALIZATION = -12,
81 - STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP = -13,
82 - STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER = -14,
83 - STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN = -15,
84 - STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT = -16,
85 - STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT = -17,
86 - STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR = -18,
87 - STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED = -19,
88 - STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT = -20,
89 - STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST = -21,
90 - STREAM_HANDSHAKE_NON_STREAMABLE_HOST = -22,
91 - STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER = -23,
92 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF = -24,
93 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED = -25,
94 - STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_TIMEOUT = -26,
95 - STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE = -27,
96 -
97 -} STREAM_HANDSHAKE;
98 -
99 -
100 -// ----------------------------------------------------------------------------
101 -
102 -typedef struct {
103 - char *os_name;
104 - char *os_id;
105 - char *os_version;
106 - char *kernel_name;
107 - char *kernel_version;
108 -} stream_encoded_t;
109 -
110 -#include "compression.h"
111 -
112 -// Thread-local storage
113 -// Metric transmission: collector threads asynchronously fill the buffer, sender thread uses it.
114 -
115 -typedef enum __attribute__((packed)) {
116 - STREAM_TRAFFIC_TYPE_REPLICATION = 0,
117 - STREAM_TRAFFIC_TYPE_FUNCTIONS,
118 - STREAM_TRAFFIC_TYPE_METADATA,
119 - STREAM_TRAFFIC_TYPE_DATA,
120 - STREAM_TRAFFIC_TYPE_DYNCFG,
121 -
122 - // terminator
123 - STREAM_TRAFFIC_TYPE_MAX,
124 -} STREAM_TRAFFIC_TYPE;
125 -
126 -typedef enum __attribute__((packed)) {
127 - SENDER_FLAG_OVERFLOW = (1 << 0), // The buffer has been overflown
128 -} SENDER_FLAGS;
129 -
130 -typedef void (*rrdpush_defer_action_t)(struct sender_state *s, void *data);
131 -typedef void (*rrdpush_defer_cleanup_t)(struct sender_state *s, void *data);
132 -
133 -struct sender_state {
134 - RRDHOST *host;
135 - pid_t tid; // the thread id of the sender, from gettid_cached()
136 - SENDER_FLAGS flags;
137 - int timeout;
138 - int default_port;
139 - uint32_t reconnect_delay;
140 - char connected_to[CONNECTED_TO_SIZE + 1]; // We don't know which proxy we connect to, passed back from socket.c
141 - size_t begin;
142 - size_t reconnects_counter;
143 - size_t sent_bytes;
144 - size_t sent_bytes_on_this_connection;
145 - size_t send_attempts;
146 - time_t last_traffic_seen_t;
147 - time_t last_state_since_t; // the timestamp of the last state (online/offline) change
148 - size_t not_connected_loops;
149 - // Metrics are collected asynchronously by collector threads calling rrdset_done_push(). This can also trigger
150 - // the lazy creation of the sender thread - both cases (buffer access and thread creation) are guarded here.
151 - SPINLOCK spinlock;
152 - struct circular_buffer *buffer;
153 - char read_buffer[PLUGINSD_LINE_MAX + 1];
154 - ssize_t read_len;
155 - STREAM_CAPABILITIES capabilities;
156 - STREAM_CAPABILITIES disabled_capabilities;
157 -
158 - size_t sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX];
159 -
160 - int rrdpush_sender_pipe[2]; // collector to sender thread signaling
161 - int rrdpush_sender_socket;
162 -
163 - uint16_t hops;
164 -
165 - struct line_splitter line;
166 - struct compressor_state compressor;
167 -
168 -#ifdef NETDATA_LOG_STREAM_SENDER
169 - FILE *stream_log_fp;
170 -#endif
171 -
172 - NETDATA_SSL ssl; // structure used to encrypt the connection
173 -
174 - struct {
175 - bool shutdown;
176 - STREAM_HANDSHAKE reason;
177 - } exit;
178 -
179 - struct {
180 - DICTIONARY *requests; // de-duplication of replication requests, per chart
181 - time_t oldest_request_after_t; // the timestamp of the oldest replication request
182 - time_t latest_completed_before_t; // the timestamp of the latest replication request
183 -
184 - struct {
185 - size_t pending_requests; // the currently outstanding replication requests
186 - size_t charts_replicating; // the number of unique charts having pending replication requests (on every request one is added and is removed when we finish it - it does not track completion of the replication for this chart)
187 - bool reached_max; // true when the sender buffer should not get more replication responses
188 - } atomic;
189 -
190 - } replication;
191 -
192 - struct {
193 - bool pending_data;
194 - size_t buffer_used_percentage; // the current utilization of the sending buffer
195 - usec_t last_flush_time_ut; // the last time the sender flushed the sending buffer in USEC
196 - time_t last_buffer_recreate_s; // true when the sender buffer should be re-created
197 - } atomic;
198 -
199 - struct {
200 - const char *end_keyword;
201 - BUFFER *payload;
202 - rrdpush_defer_action_t action;
203 - rrdpush_defer_cleanup_t cleanup;
204 - void *action_data;
205 - } defer;
206 -
207 - int parent_using_h2o;
208 -};
209 -
210 -#define sender_lock(sender) spinlock_lock(&(sender)->spinlock)
211 -#define sender_unlock(sender) spinlock_unlock(&(sender)->spinlock)
212 -
213 -#define rrdpush_sender_pipe_has_pending_data(sender) __atomic_load_n(&(sender)->atomic.pending_data, __ATOMIC_RELAXED)
214 -#define rrdpush_sender_pipe_set_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, true, __ATOMIC_RELAXED)
215 -#define rrdpush_sender_pipe_clear_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, false, __ATOMIC_RELAXED)
216 -
217 -#define rrdpush_sender_last_buffer_recreate_get(sender) __atomic_load_n(&(sender)->atomic.last_buffer_recreate_s, __ATOMIC_RELAXED)
218 -#define rrdpush_sender_last_buffer_recreate_set(sender, value) __atomic_store_n(&(sender)->atomic.last_buffer_recreate_s, value, __ATOMIC_RELAXED)
219 -
220 -#define rrdpush_sender_replication_buffer_full_set(sender, value) __atomic_store_n(&((sender)->replication.atomic.reached_max), value, __ATOMIC_SEQ_CST)
221 -#define rrdpush_sender_replication_buffer_full_get(sender) __atomic_load_n(&((sender)->replication.atomic.reached_max), __ATOMIC_SEQ_CST)
222 -
223 -#define rrdpush_sender_set_buffer_used_percent(sender, value) __atomic_store_n(&((sender)->atomic.buffer_used_percentage), value, __ATOMIC_RELAXED)
224 -#define rrdpush_sender_get_buffer_used_percent(sender) __atomic_load_n(&((sender)->atomic.buffer_used_percentage), __ATOMIC_RELAXED)
225 -
226 -#define rrdpush_sender_set_flush_time(sender) __atomic_store_n(&((sender)->atomic.last_flush_time_ut), now_realtime_usec(), __ATOMIC_RELAXED)
227 -#define rrdpush_sender_get_flush_time(sender) __atomic_load_n(&((sender)->atomic.last_flush_time_ut), __ATOMIC_RELAXED)
228 -
229 -#define rrdpush_sender_replicating_charts(sender) __atomic_load_n(&((sender)->replication.atomic.charts_replicating), __ATOMIC_RELAXED)
230 -#define rrdpush_sender_replicating_charts_plus_one(sender) __atomic_add_fetch(&((sender)->replication.atomic.charts_replicating), 1, __ATOMIC_RELAXED)
231 -#define rrdpush_sender_replicating_charts_minus_one(sender) __atomic_sub_fetch(&((sender)->replication.atomic.charts_replicating), 1, __ATOMIC_RELAXED)
232 -#define rrdpush_sender_replicating_charts_zero(sender) __atomic_store_n(&((sender)->replication.atomic.charts_replicating), 0, __ATOMIC_RELAXED)
233 -
234 -#define rrdpush_sender_pending_replication_requests(sender) __atomic_load_n(&((sender)->replication.atomic.pending_requests), __ATOMIC_RELAXED)
235 -#define rrdpush_sender_pending_replication_requests_plus_one(sender) __atomic_add_fetch(&((sender)->replication.atomic.pending_requests), 1, __ATOMIC_RELAXED)
236 -#define rrdpush_sender_pending_replication_requests_minus_one(sender) __atomic_sub_fetch(&((sender)->replication.atomic.pending_requests), 1, __ATOMIC_RELAXED)
237 -#define rrdpush_sender_pending_replication_requests_zero(sender) __atomic_store_n(&((sender)->replication.atomic.pending_requests), 0, __ATOMIC_RELAXED)
238 -
239 -/*
240 -typedef enum {
241 - STREAM_NODE_INSTANCE_FEATURE_CLOUD_ONLINE = (1 << 0),
242 - STREAM_NODE_INSTANCE_FEATURE_VIRTUAL_HOST = (1 << 1),
243 - STREAM_NODE_INSTANCE_FEATURE_HEALTH_ENABLED = (1 << 2),
244 - STREAM_NODE_INSTANCE_FEATURE_ML_SELF = (1 << 3),
245 - STREAM_NODE_INSTANCE_FEATURE_ML_RECEIVED = (1 << 4),
246 - STREAM_NODE_INSTANCE_FEATURE_SSL = (1 << 5),
247 -} STREAM_NODE_INSTANCE_FEATURES;
248 -
249 -typedef struct stream_node_instance {
250 - uuid_t uuid;
251 - STRING *agent;
252 - STREAM_NODE_INSTANCE_FEATURES features;
253 - uint32_t hops;
254 -
255 - // receiver information on that agent
256 - int32_t capabilities;
257 - uint32_t local_port;
258 - uint32_t remote_port;
259 - STRING *local_ip;
260 - STRING *remote_ip;
261 -} STREAM_NODE_INSTANCE;
262 -*/
263 -
264 -struct parser;
265 -
266 -struct receiver_state {
267 - RRDHOST *host;
268 - pid_t tid;
269 - ND_THREAD *thread;
270 - int fd;
271 - char *key;
272 - char *hostname;
273 - char *registry_hostname;
274 - char *machine_guid;
275 - char *os;
276 - char *timezone; // Unused?
277 - char *abbrev_timezone;
278 - int32_t utc_offset;
279 - char *client_ip; // Duplicated in pluginsd
280 - char *client_port; // Duplicated in pluginsd
281 - char *program_name; // Duplicated in pluginsd
282 - char *program_version;
283 - struct rrdhost_system_info *system_info;
284 - STREAM_CAPABILITIES capabilities;
285 - time_t last_msg_t;
286 - time_t connected_since_s;
287 -
288 - struct buffered_reader reader;
289 -
290 - uint16_t hops;
291 -
292 - struct {
293 - bool shutdown; // signal the streaming parser to exit
294 - STREAM_HANDSHAKE reason;
295 - } exit;
296 -
297 - struct {
298 - RRD_MEMORY_MODE mode;
299 - int history;
300 - int update_every;
301 - int health_enabled; // CONFIG_BOOLEAN_YES, CONFIG_BOOLEAN_NO, CONFIG_BOOLEAN_AUTO
302 - time_t alarms_delay;
303 - uint32_t alarms_history;
304 - int rrdpush_enabled;
305 - const char *rrdpush_api_key; // DONT FREE - it is allocated in appconfig
306 - const char *rrdpush_send_charts_matching; // DONT FREE - it is allocated in appconfig
307 - bool rrdpush_enable_replication;
308 - time_t rrdpush_seconds_to_replicate;
309 - time_t rrdpush_replication_step;
310 - const char *rrdpush_destination; // DONT FREE - it is allocated in appconfig
311 - unsigned int rrdpush_compression;
312 - STREAM_CAPABILITIES compression_priorities[COMPRESSION_ALGORITHM_MAX];
313 - } config;
314 -
315 - NETDATA_SSL ssl;
316 -
317 - time_t replication_first_time_t;
318 -
319 - struct decompressor_state decompressor;
320 -/*
321 - struct {
322 - uint32_t count;
323 - STREAM_NODE_INSTANCE *array;
324 - } instances;
325 -*/
326 -
327 - // The parser pointer is safe to read and use, only when having the host receiver lock.
328 - // Without this lock, the data pointed by the pointer may vanish randomly.
329 - // Also, since the receiver sets it when it starts, it should be read with
330 - // an atomic read.
331 - struct parser *parser;
332 -
333 -#ifdef ENABLE_H2O
334 - void *h2o_ctx;
335 -#endif
336 -};
337 -
338 -#ifdef ENABLE_H2O
339 -#define is_h2o_rrdpush(x) ((x)->h2o_ctx != NULL)
340 -#define unless_h2o_rrdpush(x) if(!is_h2o_rrdpush(x))
341 -#endif
342 -
343 -struct rrdpush_destinations {
344 - STRING *destination;
345 - bool ssl;
346 - uint32_t attempts;
347 - time_t since;
348 - time_t postpone_reconnection_until;
349 - STREAM_HANDSHAKE reason;
350 -
351 - struct rrdpush_destinations *prev;
352 - struct rrdpush_destinations *next;
353 -};
354 -
355 -extern unsigned int default_rrdpush_enabled;
356 -extern unsigned int default_rrdpush_compression_enabled;
357 -extern const char *default_rrdpush_destination;
358 -extern const char *default_rrdpush_api_key;
359 -extern const char *default_rrdpush_send_charts_matching;
360 -extern bool default_rrdpush_enable_replication;
361 -extern time_t default_rrdpush_seconds_to_replicate;
362 -extern time_t default_rrdpush_replication_step;
363 -extern unsigned int remote_clock_resync_iterations;
364 -
365 -void rrdpush_destinations_init(RRDHOST *host);
366 -void rrdpush_destinations_free(RRDHOST *host);
367 -
368 -BUFFER *sender_start(struct sender_state *s);
369 -void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type);
370 -int rrdpush_init();
371 -bool rrdpush_receiver_needs_dbengine();
372 -int configured_as_parent();
373 -
374 -typedef struct rrdset_stream_buffer {
375 - STREAM_CAPABILITIES capabilities;
376 - bool v2;
377 - bool begin_v2_added;
378 - time_t wall_clock_time;
379 - uint64_t rrdset_flags; // RRDSET_FLAGS
380 - time_t last_point_end_time_s;
381 - BUFFER *wb;
382 -} RRDSET_STREAM_BUFFER;
383 -
384 -RRDSET_STREAM_BUFFER rrdset_push_metric_initialize(RRDSET *st, time_t wall_clock_time);
385 -void rrdset_push_metrics_v1(RRDSET_STREAM_BUFFER *rsb, RRDSET *st);
386 -void rrdset_push_metrics_finished(RRDSET_STREAM_BUFFER *rsb, RRDSET *st);
387 -void rrddim_push_metrics_v2(RRDSET_STREAM_BUFFER *rsb, RRDDIM *rd, usec_t point_end_time_ut, NETDATA_DOUBLE n, SN_FLAGS flags);
388 -
389 -bool rrdset_push_chart_definition_now(RRDSET *st);
390 -void *rrdpush_sender_thread(void *ptr);
391 -void rrdpush_send_host_labels(RRDHOST *host);
392 -void rrdpush_send_global_functions(RRDHOST *host);
393 -
394 -int rrdpush_receiver_thread_spawn(struct web_client *w, char *decoded_query_string, void *h2o_ctx);
395 -void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait);
396 -
397 -void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva);
398 -int connect_to_one_of_destinations(
399 - RRDHOST *host,
400 - int default_port,
401 - struct timeval *timeout,
402 - size_t *reconnects_counter,
403 - char *connected_to,
404 - size_t connected_to_size,
405 - struct rrdpush_destinations **destination);
406 -
407 -void rrdpush_signal_sender_to_wake_up(struct sender_state *s);
408 -
409 -void rrdpush_reset_destinations_postpone_time(RRDHOST *host);
410 -const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error);
411 -void rrdpush_receive_log_status(struct receiver_state *rpt, const char *msg, const char *status, ND_LOG_FIELD_PRIORITY priority);
412 -
413 -void receiver_state_free(struct receiver_state *rpt);
414 -bool stop_streaming_receiver(RRDHOST *host, STREAM_HANDSHAKE reason);
415 -
416 -void sender_thread_buffer_free(void);
417 -
418 -#include "replication.h"
419 -
420 -typedef enum __attribute__((packed)) {
421 - RRDHOST_DB_STATUS_INITIALIZING = 0,
422 - RRDHOST_DB_STATUS_QUERYABLE,
423 -} RRDHOST_DB_STATUS;
424 -
425 -static inline const char *rrdhost_db_status_to_string(RRDHOST_DB_STATUS status) {
426 - switch(status) {
427 - default:
428 - case RRDHOST_DB_STATUS_INITIALIZING:
429 - return "initializing";
430 -
431 - case RRDHOST_DB_STATUS_QUERYABLE:
432 - return "online";
433 - }
434 -}
435 -
436 -typedef enum __attribute__((packed)) {
437 - RRDHOST_DB_LIVENESS_STALE = 0,
438 - RRDHOST_DB_LIVENESS_LIVE,
439 -} RRDHOST_DB_LIVENESS;
440 -
441 -static inline const char *rrdhost_db_liveness_to_string(RRDHOST_DB_LIVENESS status) {
442 - switch(status) {
443 - default:
444 - case RRDHOST_DB_LIVENESS_STALE:
445 - return "stale";
446 -
447 - case RRDHOST_DB_LIVENESS_LIVE:
448 - return "live";
449 - }
450 -}
451 -
452 -typedef enum __attribute__((packed)) {
453 - RRDHOST_INGEST_STATUS_ARCHIVED = 0,
454 - RRDHOST_INGEST_STATUS_INITIALIZING,
455 - RRDHOST_INGEST_STATUS_REPLICATING,
456 - RRDHOST_INGEST_STATUS_ONLINE,
457 - RRDHOST_INGEST_STATUS_OFFLINE,
458 -} RRDHOST_INGEST_STATUS;
459 -
460 -static inline const char *rrdhost_ingest_status_to_string(RRDHOST_INGEST_STATUS status) {
461 - switch(status) {
462 - case RRDHOST_INGEST_STATUS_ARCHIVED:
463 - return "archived";
464 -
465 - case RRDHOST_INGEST_STATUS_INITIALIZING:
466 - return "initializing";
467 -
468 - case RRDHOST_INGEST_STATUS_REPLICATING:
469 - return "replicating";
470 -
471 - case RRDHOST_INGEST_STATUS_ONLINE:
472 - return "online";
473 -
474 - default:
475 - case RRDHOST_INGEST_STATUS_OFFLINE:
476 - return "offline";
477 - }
478 -}
479 -
480 -typedef enum __attribute__((packed)) {
481 - RRDHOST_INGEST_TYPE_LOCALHOST = 0,
482 - RRDHOST_INGEST_TYPE_VIRTUAL,
483 - RRDHOST_INGEST_TYPE_CHILD,
484 - RRDHOST_INGEST_TYPE_ARCHIVED,
485 -} RRDHOST_INGEST_TYPE;
486 -
487 -static inline const char *rrdhost_ingest_type_to_string(RRDHOST_INGEST_TYPE type) {
488 - switch(type) {
489 - case RRDHOST_INGEST_TYPE_LOCALHOST:
490 - return "localhost";
491 -
492 - case RRDHOST_INGEST_TYPE_VIRTUAL:
493 - return "virtual";
494 -
495 - case RRDHOST_INGEST_TYPE_CHILD:
496 - return "child";
497 -
498 - default:
499 - case RRDHOST_INGEST_TYPE_ARCHIVED:
500 - return "archived";
501 - }
502 -}
503 -
504 -typedef enum __attribute__((packed)) {
505 - RRDHOST_STREAM_STATUS_DISABLED = 0,
506 - RRDHOST_STREAM_STATUS_REPLICATING,
507 - RRDHOST_STREAM_STATUS_ONLINE,
508 - RRDHOST_STREAM_STATUS_OFFLINE,
509 -} RRDHOST_STREAMING_STATUS;
510 -
511 -static inline const char *rrdhost_streaming_status_to_string(RRDHOST_STREAMING_STATUS status) {
512 - switch(status) {
513 - case RRDHOST_STREAM_STATUS_DISABLED:
514 - return "disabled";
515 -
516 - case RRDHOST_STREAM_STATUS_REPLICATING:
517 - return "replicating";
518 -
519 - case RRDHOST_STREAM_STATUS_ONLINE:
520 - return "online";
521 -
522 - default:
523 - case RRDHOST_STREAM_STATUS_OFFLINE:
524 - return "offline";
525 - }
526 -}
527 -
528 -typedef enum __attribute__((packed)) {
529 - RRDHOST_ML_STATUS_DISABLED = 0,
530 - RRDHOST_ML_STATUS_OFFLINE,
531 - RRDHOST_ML_STATUS_RUNNING,
532 -} RRDHOST_ML_STATUS;
533 -
534 -static inline const char *rrdhost_ml_status_to_string(RRDHOST_ML_STATUS status) {
535 - switch(status) {
536 - case RRDHOST_ML_STATUS_RUNNING:
537 - return "online";
538 -
539 - case RRDHOST_ML_STATUS_OFFLINE:
540 - return "offline";
541 -
542 - default:
543 - case RRDHOST_ML_STATUS_DISABLED:
544 - return "disabled";
545 - }
546 -}
547 -
548 -typedef enum __attribute__((packed)) {
549 - RRDHOST_ML_TYPE_DISABLED = 0,
550 - RRDHOST_ML_TYPE_SELF,
551 - RRDHOST_ML_TYPE_RECEIVED,
552 -} RRDHOST_ML_TYPE;
553 -
554 -static inline const char *rrdhost_ml_type_to_string(RRDHOST_ML_TYPE type) {
555 - switch(type) {
556 - case RRDHOST_ML_TYPE_SELF:
557 - return "self";
558 -
559 - case RRDHOST_ML_TYPE_RECEIVED:
560 - return "received";
561 -
562 - default:
563 - case RRDHOST_ML_TYPE_DISABLED:
564 - return "disabled";
565 - }
566 -}
567 -
568 -typedef enum __attribute__((packed)) {
569 - RRDHOST_HEALTH_STATUS_DISABLED = 0,
570 - RRDHOST_HEALTH_STATUS_INITIALIZING,
571 - RRDHOST_HEALTH_STATUS_RUNNING,
572 -} RRDHOST_HEALTH_STATUS;
573 -
574 -static inline const char *rrdhost_health_status_to_string(RRDHOST_HEALTH_STATUS status) {
575 - switch(status) {
576 - default:
577 - case RRDHOST_HEALTH_STATUS_DISABLED:
578 - return "disabled";
579 -
580 - case RRDHOST_HEALTH_STATUS_INITIALIZING:
581 - return "initializing";
582 -
583 - case RRDHOST_HEALTH_STATUS_RUNNING:
584 - return "online";
585 - }
586 -}
587 -
588 -typedef enum __attribute__((packed)) {
589 - RRDHOST_DYNCFG_STATUS_UNAVAILABLE = 0,
590 - RRDHOST_DYNCFG_STATUS_AVAILABLE,
591 -} RRDHOST_DYNCFG_STATUS;
592 -
593 -static inline const char *rrdhost_dyncfg_status_to_string(RRDHOST_DYNCFG_STATUS status) {
594 - switch(status) {
595 - default:
596 - case RRDHOST_DYNCFG_STATUS_UNAVAILABLE:
597 - return "unavailable";
598 -
599 - case RRDHOST_DYNCFG_STATUS_AVAILABLE:
600 - return "online";
601 - }
602 -}
603 -
604 -typedef struct {
605 - RRDHOST *host;
606 - time_t now;
607 -
608 - struct {
609 - RRDHOST_DYNCFG_STATUS status;
610 - } dyncfg;
611 -
612 - struct {
613 - RRDHOST_DB_STATUS status;
614 - RRDHOST_DB_LIVENESS liveness;
615 - RRD_MEMORY_MODE mode;
616 - time_t first_time_s;
617 - time_t last_time_s;
618 - size_t metrics;
619 - size_t instances;
620 - size_t contexts;
621 - } db;
622 -
623 - struct {
624 - RRDHOST_ML_STATUS status;
625 - RRDHOST_ML_TYPE type;
626 - struct ml_metrics_statistics metrics;
627 - } ml;
628 -
629 - struct {
630 - size_t hops;
631 - RRDHOST_INGEST_TYPE type;
632 - RRDHOST_INGEST_STATUS status;
633 - SOCKET_PEERS peers;
634 - bool ssl;
635 - STREAM_CAPABILITIES capabilities;
636 - uint32_t id;
637 - time_t since;
638 - STREAM_HANDSHAKE reason;
639 -
640 - struct {
641 - bool in_progress;
642 - NETDATA_DOUBLE completion;
643 - size_t instances;
644 - } replication;
645 - } ingest;
646 -
647 - struct {
648 - size_t hops;
649 - RRDHOST_STREAMING_STATUS status;
650 - SOCKET_PEERS peers;
651 - bool ssl;
652 - bool compression;
653 - STREAM_CAPABILITIES capabilities;
654 - uint32_t id;
655 - time_t since;
656 - STREAM_HANDSHAKE reason;
657 -
658 - struct {
659 - bool in_progress;
660 - NETDATA_DOUBLE completion;
661 - size_t instances;
662 - } replication;
663 -
664 - size_t sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX];
665 - } stream;
666 -
667 - struct {
668 - RRDHOST_HEALTH_STATUS status;
669 - struct {
670 - uint32_t undefined;
671 - uint32_t uninitialized;
672 - uint32_t clear;
673 - uint32_t warning;
674 - uint32_t critical;
675 - } alerts;
676 - } health;
677 -} RRDHOST_STATUS;
678 -
679 -void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s);
680 -bool rrdhost_state_cloud_emulation(RRDHOST *host);
681 -
682 -bool rrdpush_compression_initialize(struct sender_state *s);
683 -bool rrdpush_decompression_initialize(struct receiver_state *rpt);
684 -void rrdpush_parse_compression_order(struct receiver_state *rpt, const char *order);
685 -void rrdpush_select_receiver_compression_algorithm(struct receiver_state *rpt);
686 -void rrdpush_compression_deactivate(struct sender_state *s);
11 +#include "sender.h"
12 +#include "receiver.h"
13
14 +#include "rrdhost-status.h"
15 #include "protocol/commands.h"
689 -#include "stream_path.h"
16 +#include "stream-path.h"
17
18 #endif //NETDATA_RRDPUSH_H
src/streaming/sender-commit.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "sender_internals.h"
3 +#include "sender-internals.h"
4
5 static __thread BUFFER *sender_thread_buffer = NULL;
6 static __thread bool sender_thread_buffer_used = false;
src/streaming/sender-connect.c renamed
+68 -3
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "sender_internals.h"
3 +#include "sender-internals.h"
4
5 void rrdpush_sender_thread_close_socket(struct sender_state *s) {
6 rrdhost_flag_clear(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_CONNECTED | RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
@@ -387,7 +387,7 @@ err_cleanup:
387 return 1;
388 }
389
390 -static bool rrdpush_sender_thread_connect_to_parent(RRDHOST *host, int default_port, int timeout, struct sender_state *s) {
390 +static bool sender_send_connection_request(RRDHOST *host, int default_port, int timeout, struct sender_state *s) {
391
392 struct timeval tv = {
393 .tv_sec = timeout,
@@ -638,7 +638,7 @@ bool attempt_to_connect(struct sender_state *state) {
638 state->sent_bytes_on_this_connection = 0;
639 memset(state->sent_bytes_on_this_connection_per_type, 0, sizeof(state->sent_bytes_on_this_connection_per_type));
640
641 - if(rrdpush_sender_thread_connect_to_parent(state->host, state->default_port, state->timeout, state)) {
641 + if(sender_send_connection_request(state->host, state->default_port, state->timeout, state)) {
642 // reset the buffer, to properly send charts and metrics
643 rrdpush_sender_on_connect(state->host);
644
@@ -674,3 +674,68 @@ bool attempt_to_connect(struct sender_state *state) {
674
675 return false;
676 }
677 +
678 +bool rrdpush_sender_connect(struct sender_state *s) {
679 + worker_is_busy(WORKER_SENDER_JOB_CONNECT);
680 +
681 + time_t now_s = now_monotonic_sec();
682 + rrdpush_sender_cbuffer_recreate_timed(s, now_s, false, true);
683 + rrdpush_sender_execute_commands_cleanup(s);
684 +
685 + rrdhost_flag_clear(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
686 + s->flags &= ~SENDER_FLAG_OVERFLOW;
687 + s->read_len = 0;
688 + s->buffer->read = 0;
689 + s->buffer->write = 0;
690 +
691 + if(!attempt_to_connect(s))
692 + return false;
693 +
694 + if(rrdhost_sender_should_exit(s))
695 + return false;
696 +
697 + s->last_traffic_seen_t = now_monotonic_sec();
698 + stream_path_send_to_parent(s->host);
699 + rrdpush_sender_send_claimed_id(s->host);
700 + rrdpush_send_host_labels(s->host);
701 + rrdpush_send_global_functions(s->host);
702 + s->replication.oldest_request_after_t = 0;
703 +
704 + rrdhost_flag_set(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
705 +
706 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
707 + "STREAM %s [send to %s]: enabling metrics streaming...",
708 + rrdhost_hostname(s->host), s->connected_to);
709 +
710 + return true;
711 +}
712 +
713 +// Either the receiver lost the connection or the host is being destroyed.
714 +// The sender mutex guards thread creation, any spurious data is wiped on reconnection.
715 +void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait) {
716 + if (!host->sender)
717 + return;
718 +
719 + sender_lock(host->sender);
720 +
721 + if(rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN)) {
722 +
723 + host->sender->exit.shutdown = true;
724 + host->sender->exit.reason = reason;
725 +
726 + // signal it to cancel
727 + nd_thread_signal_cancel(host->rrdpush_sender_thread);
728 + }
729 +
730 + sender_unlock(host->sender);
731 +
732 + if(wait) {
733 + sender_lock(host->sender);
734 + while(host->sender->tid) {
735 + sender_unlock(host->sender);
736 + sleep_usec(10 * USEC_PER_MS);
737 + sender_lock(host->sender);
738 + }
739 + sender_unlock(host->sender);
740 + }
741 +}
src/streaming/sender-destinations.c new
+143
@@ -0,0 +1,143 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "sender-internals.h"
4 +
5 +void rrdpush_reset_destinations_postpone_time(RRDHOST *host) {
6 + uint32_t wait = (host->sender) ? host->sender->reconnect_delay : 5;
7 + time_t now = now_realtime_sec();
8 + for (struct rrdpush_destinations *d = host->destinations; d; d = d->next)
9 + d->postpone_reconnection_until = now + wait;
10 +}
11 +
12 +void rrdpush_sender_ssl_init(RRDHOST *host) {
13 + static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER;
14 + spinlock_lock(&sp);
15 +
16 + if(netdata_ssl_streaming_sender_ctx || !host) {
17 + spinlock_unlock(&sp);
18 + return;
19 + }
20 +
21 + for(struct rrdpush_destinations *d = host->destinations; d ; d = d->next) {
22 + if (d->ssl) {
23 + // we need to initialize SSL
24 +
25 + netdata_ssl_initialize_ctx(NETDATA_SSL_STREAMING_SENDER_CTX);
26 + ssl_security_location_for_context(netdata_ssl_streaming_sender_ctx, stream_conf_ssl_ca_file, stream_conf_ssl_ca_path);
27 +
28 + // stop the loop
29 + break;
30 + }
31 + }
32 +
33 + spinlock_unlock(&sp);
34 +}
35 +
36 +int connect_to_one_of_destinations(
37 + RRDHOST *host,
38 + int default_port,
39 + struct timeval *timeout,
40 + size_t *reconnects_counter,
41 + char *connected_to,
42 + size_t connected_to_size,
43 + struct rrdpush_destinations **destination)
44 +{
45 + int sock = -1;
46 +
47 + for (struct rrdpush_destinations *d = host->destinations; d; d = d->next) {
48 + time_t now = now_realtime_sec();
49 +
50 + if(nd_thread_signaled_to_cancel())
51 + return -1;
52 +
53 + if(d->postpone_reconnection_until > now)
54 + continue;
55 +
56 + nd_log(NDLS_DAEMON, NDLP_DEBUG,
57 + "STREAM %s: connecting to '%s' (default port: %d)...",
58 + rrdhost_hostname(host), string2str(d->destination), default_port);
59 +
60 + if (reconnects_counter)
61 + *reconnects_counter += 1;
62 +
63 + d->since = now;
64 + d->attempts++;
65 + sock = connect_to_this(string2str(d->destination), default_port, timeout);
66 +
67 + if (sock != -1) {
68 + if (connected_to && connected_to_size)
69 + strncpyz(connected_to, string2str(d->destination), connected_to_size);
70 +
71 + *destination = d;
72 +
73 + // move the current item to the end of the list
74 + // without this, this destination will break the loop again and again
75 + // not advancing the destinations to find one that may work
76 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, d, prev, next);
77 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(host->destinations, d, prev, next);
78 +
79 + break;
80 + }
81 + }
82 +
83 + return sock;
84 +}
85 +
86 +struct destinations_init_tmp {
87 + RRDHOST *host;
88 + struct rrdpush_destinations *list;
89 + int count;
90 +};
91 +
92 +static bool destinations_init_add_one(char *entry, void *data) {
93 + struct destinations_init_tmp *t = data;
94 +
95 + struct rrdpush_destinations *d = callocz(1, sizeof(struct rrdpush_destinations));
96 + char *colon_ssl = strstr(entry, ":SSL");
97 + if(colon_ssl) {
98 + *colon_ssl = '\0';
99 + d->ssl = true;
100 + }
101 + else
102 + d->ssl = false;
103 +
104 + d->destination = string_strdupz(entry);
105 +
106 + __atomic_add_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
107 +
108 + DOUBLE_LINKED_LIST_APPEND_ITEM_UNSAFE(t->list, d, prev, next);
109 +
110 + t->count++;
111 + nd_log_daemon(NDLP_INFO, "STREAM: added streaming destination No %d: '%s' to host '%s'", t->count, string2str(d->destination), rrdhost_hostname(t->host));
112 +
113 + return false; // we return false, so that we will get all defined destinations
114 +}
115 +
116 +void rrdpush_destinations_init(RRDHOST *host) {
117 + if(!host->rrdpush.send.destination) return;
118 +
119 + rrdpush_destinations_free(host);
120 +
121 + struct destinations_init_tmp t = {
122 + .host = host,
123 + .list = NULL,
124 + .count = 0,
125 + };
126 +
127 + foreach_entry_in_connection_string(host->rrdpush.send.destination, destinations_init_add_one, &t);
128 +
129 + host->destinations = t.list;
130 +}
131 +
132 +void rrdpush_destinations_free(RRDHOST *host) {
133 + while (host->destinations) {
134 + struct rrdpush_destinations *tmp = host->destinations;
135 + DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(host->destinations, tmp, prev, next);
136 + string_freez(tmp->destination);
137 + freez(tmp);
138 + __atomic_sub_fetch(&netdata_buffers_statistics.rrdhost_senders, sizeof(struct rrdpush_destinations), __ATOMIC_RELAXED);
139 + }
140 +
141 + host->destinations = NULL;
142 +}
143 +
src/streaming/sender-destinations.h new
+38
@@ -0,0 +1,38 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_SENDER_DESTINATIONS_H
4 +#define NETDATA_SENDER_DESTINATIONS_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "stream-handshake.h"
8 +#include "database/rrd.h"
9 +
10 +struct rrdpush_destinations {
11 + STRING *destination;
12 + bool ssl;
13 + uint32_t attempts;
14 + time_t since;
15 + time_t postpone_reconnection_until;
16 + STREAM_HANDSHAKE reason;
17 +
18 + struct rrdpush_destinations *prev;
19 + struct rrdpush_destinations *next;
20 +};
21 +
22 +void rrdpush_sender_ssl_init(RRDHOST *host);
23 +
24 +void rrdpush_reset_destinations_postpone_time(RRDHOST *host);
25 +
26 +void rrdpush_destinations_init(RRDHOST *host);
27 +void rrdpush_destinations_free(RRDHOST *host);
28 +
29 +int connect_to_one_of_destinations(
30 + RRDHOST *host,
31 + int default_port,
32 + struct timeval *timeout,
33 + size_t *reconnects_counter,
34 + char *connected_to,
35 + size_t connected_to_size,
36 + struct rrdpush_destinations **destination);
37 +
38 +#endif //NETDATA_SENDER_DESTINATIONS_H
src/streaming/sender-execute.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "sender_internals.h"
3 +#include "sender-internals.h"
4
5 struct inflight_stream_function {
6 struct sender_state *sender;
src/streaming/sender-internals.h renamed
+1 -5
@@ -4,7 +4,7 @@
4 #define NETDATA_SENDER_INTERNALS_H
5
6 #include "rrdpush.h"
7 -#include "common.h"
7 +#include "h2o-common.h"
8 #include "aclk/https_client.h"
9
10 #define WORKER_SENDER_JOB_CONNECT 0
@@ -37,10 +37,6 @@
37 #error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 25
38 #endif
39
40 -extern struct config stream_config;
41 -extern const char *netdata_ssl_ca_path;
42 -extern const char *netdata_ssl_ca_file;
43 -
40 bool attempt_to_connect(struct sender_state *state);
41 void rrdpush_sender_on_connect(RRDHOST *host);
42 void rrdpush_sender_after_connect(RRDHOST *host);
src/streaming/sender.c
+27 -113
@@ -1,54 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "sender_internals.h"
4 -
5 -static inline void rrdpush_sender_add_host_variable_to_buffer(BUFFER *wb, const RRDVAR_ACQUIRED *rva) {
6 - buffer_sprintf(
7 - wb
8 - , "VARIABLE HOST %s = " NETDATA_DOUBLE_FORMAT "\n"
9 - , rrdvar_name(rva)
10 - , rrdvar2number(rva)
11 - );
12 -
13 - netdata_log_debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " NETDATA_DOUBLE_FORMAT, rrdvar_name(rva), rrdvar2number(rva));
14 -}
15 -
16 -void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, const RRDVAR_ACQUIRED *rva) {
17 - if(rrdhost_can_send_definitions_to_parent(host)) {
18 - BUFFER *wb = sender_start(host->sender);
19 - rrdpush_sender_add_host_variable_to_buffer(wb, rva);
20 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
21 - sender_thread_buffer_free();
22 - }
23 -}
24 -
25 -struct custom_host_variables_callback {
26 - BUFFER *wb;
27 -};
28 -
29 -static int rrdpush_sender_thread_custom_host_variables_callback(const DICTIONARY_ITEM *item __maybe_unused, void *rrdvar_ptr __maybe_unused, void *struct_ptr) {
30 - const RRDVAR_ACQUIRED *rv = (const RRDVAR_ACQUIRED *)item;
31 - struct custom_host_variables_callback *tmp = struct_ptr;
32 - BUFFER *wb = tmp->wb;
33 -
34 - rrdpush_sender_add_host_variable_to_buffer(wb, rv);
35 - return 1;
36 -}
37 -
38 -static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
39 - if(rrdhost_can_send_definitions_to_parent(host)) {
40 - BUFFER *wb = sender_start(host->sender);
41 - struct custom_host_variables_callback tmp = {
42 - .wb = wb
43 - };
44 - int ret = rrdvar_walkthrough_read(host->rrdvars, rrdpush_sender_thread_custom_host_variables_callback, &tmp);
45 - (void)ret;
46 - sender_commit(host->sender, wb, STREAM_TRAFFIC_TYPE_METADATA);
47 - sender_thread_buffer_free();
48 -
49 - netdata_log_debug(D_STREAM, "RRDVAR sent %d VARIABLES", ret);
50 - }
51 -}
3 +#include "sender-internals.h"
4
5 // resets all the chart, so that their definitions
6 // will be resent to the central netdata
@@ -72,7 +24,7 @@ static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
24 rrdhost_sender_replicating_charts_zero(host);
25 }
26
75 -static void rrdpush_sender_cbuffer_recreate_timed(struct sender_state *s, time_t now_s, bool have_mutex, bool force) {
27 +void rrdpush_sender_cbuffer_recreate_timed(struct sender_state *s, time_t now_s, bool have_mutex, bool force) {
28 static __thread time_t last_reset_time_s = 0;
29
30 if(!force && now_s - last_reset_time_s < 300)
@@ -307,7 +259,7 @@ static void rrdhost_clear_sender___while_having_sender_mutex(RRDHOST *host) {
259 rrdpush_reset_destinations_postpone_time(host);
260 }
261
310 -static bool rrdhost_sender_should_exit(struct sender_state *s) {
262 +bool rrdhost_sender_should_exit(struct sender_state *s) {
263 if(unlikely(nd_thread_signaled_to_cancel())) {
264 if(!s->exit.reason)
265 s->exit.reason = STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN;
@@ -341,30 +293,6 @@ static bool rrdhost_sender_should_exit(struct sender_state *s) {
293 return false;
294 }
295
344 -void rrdpush_initialize_ssl_ctx(RRDHOST *host __maybe_unused) {
345 - static SPINLOCK sp = NETDATA_SPINLOCK_INITIALIZER;
346 - spinlock_lock(&sp);
347 -
348 - if(netdata_ssl_streaming_sender_ctx || !host) {
349 - spinlock_unlock(&sp);
350 - return;
351 - }
352 -
353 - for(struct rrdpush_destinations *d = host->destinations; d ; d = d->next) {
354 - if (d->ssl) {
355 - // we need to initialize SSL
356 -
357 - netdata_ssl_initialize_ctx(NETDATA_SSL_STREAMING_SENDER_CTX);
358 - ssl_security_location_for_context(netdata_ssl_streaming_sender_ctx, netdata_ssl_ca_file, netdata_ssl_ca_path);
359 -
360 - // stop the loop
361 - break;
362 - }
363 - }
364 -
365 - spinlock_unlock(&sp);
366 -}
367 -
296 static bool stream_sender_log_capabilities(BUFFER *wb, void *ptr) {
297 struct sender_state *state = ptr;
298 if(!state)
@@ -461,7 +389,7 @@ void *rrdpush_sender_thread(void *ptr) {
389 return NULL;
390 }
391
464 - rrdpush_initialize_ssl_ctx(s->host);
392 + rrdpush_sender_ssl_init(s->host);
393
394 netdata_log_info("STREAM %s [send]: thread created (task id %d)", rrdhost_hostname(s->host), gettid_cached());
395
@@ -477,10 +405,10 @@ void *rrdpush_sender_thread(void *ptr) {
405 s->reconnect_delay = (unsigned int)appconfig_get_duration_seconds(
406 &stream_config, CONFIG_SECTION_STREAM, "reconnect delay", 5);
407
480 - remote_clock_resync_iterations = (unsigned int)appconfig_get_number(
408 + stream_conf_initial_clock_resync_iterations = (unsigned int)appconfig_get_number(
409 &stream_config, CONFIG_SECTION_STREAM,
410 "initial clock resync iterations",
483 - remote_clock_resync_iterations); // TODO: REMOVE FOR SLEW / GAPFILLING
411 + stream_conf_initial_clock_resync_iterations); // TODO: REMOVE FOR SLEW / GAPFILLING
412
413 s->parent_using_h2o = appconfig_get_boolean(
414 &stream_config, CONFIG_SECTION_STREAM, "parent using h2o", false);
@@ -511,43 +439,11 @@ void *rrdpush_sender_thread(void *ptr) {
439
440 // The connection attempt blocks (after which we use the socket in nonblocking)
441 if(unlikely(s->rrdpush_sender_socket == -1)) {
514 - if(was_connected) {
442 + if(was_connected)
443 rrdpush_sender_on_disconnect(s->host);
516 - was_connected = false;
517 - }
518 -
519 - worker_is_busy(WORKER_SENDER_JOB_CONNECT);
520 -
521 - now_s = now_monotonic_sec();
522 - rrdpush_sender_cbuffer_recreate_timed(s, now_s, false, true);
523 - rrdpush_sender_execute_commands_cleanup(s);
524 -
525 - rrdhost_flag_clear(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
526 - s->flags &= ~SENDER_FLAG_OVERFLOW;
527 - s->read_len = 0;
528 - s->buffer->read = 0;
529 - s->buffer->write = 0;
530 -
531 - if(!attempt_to_connect(s))
532 - continue;
533 -
534 - if(rrdhost_sender_should_exit(s))
535 - break;
536 -
537 - now_s = s->last_traffic_seen_t = now_monotonic_sec();
538 - stream_path_send_to_parent(s->host);
539 - rrdpush_sender_send_claimed_id(s->host);
540 - rrdpush_send_host_labels(s->host);
541 - rrdpush_send_global_functions(s->host);
542 - s->replication.oldest_request_after_t = 0;
543 - was_connected = true;
544 -
545 - rrdhost_flag_set(s->host, RRDHOST_FLAG_RRDPUSH_SENDER_READY_4_METRICS);
546 -
547 - nd_log(NDLS_DAEMON, NDLP_DEBUG,
548 - "STREAM %s [send to %s]: enabling metrics streaming...",
549 - rrdhost_hostname(s->host), s->connected_to);
444
445 + was_connected = rrdpush_sender_connect(s);
446 + now_s = s->last_traffic_seen_t;
447 continue;
448 }
449
@@ -755,3 +651,21 @@ void *rrdpush_sender_thread(void *ptr) {
651
652 return NULL;
653 }
654 +
655 +void rrdpush_sender_thread_spawn(RRDHOST *host) {
656 + sender_lock(host->sender);
657 +
658 + if(!rrdhost_flag_check(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN)) {
659 + char tag[NETDATA_THREAD_TAG_MAX + 1];
660 + snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_SENDER "[%s]", rrdhost_hostname(host));
661 +
662 + host->rrdpush_sender_thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT,
663 + rrdpush_sender_thread, (void *)host->sender);
664 + if(!host->rrdpush_sender_thread)
665 + nd_log_daemon(NDLP_ERR, "STREAM %s [send]: failed to create new thread for client.", rrdhost_hostname(host));
666 + else
667 + rrdhost_flag_set(host, RRDHOST_FLAG_RRDPUSH_SENDER_SPAWN);
668 + }
669 +
670 + sender_unlock(host->sender);
671 +}
src/streaming/sender.h new
+169
@@ -0,0 +1,169 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_SENDER_H
4 +#define NETDATA_SENDER_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +
8 +#define CONNECTED_TO_SIZE 100
9 +
10 +#define CBUFFER_INITIAL_SIZE (16 * 1024)
11 +#define THREAD_BUFFER_INITIAL_SIZE (CBUFFER_INITIAL_SIZE / 2)
12 +
13 +typedef enum __attribute__((packed)) {
14 + STREAM_TRAFFIC_TYPE_REPLICATION = 0,
15 + STREAM_TRAFFIC_TYPE_FUNCTIONS,
16 + STREAM_TRAFFIC_TYPE_METADATA,
17 + STREAM_TRAFFIC_TYPE_DATA,
18 + STREAM_TRAFFIC_TYPE_DYNCFG,
19 +
20 + // terminator
21 + STREAM_TRAFFIC_TYPE_MAX,
22 +} STREAM_TRAFFIC_TYPE;
23 +
24 +typedef enum __attribute__((packed)) {
25 + SENDER_FLAG_OVERFLOW = (1 << 0), // The buffer has been overflown
26 +} SENDER_FLAGS;
27 +
28 +typedef struct {
29 + char *os_name;
30 + char *os_id;
31 + char *os_version;
32 + char *kernel_name;
33 + char *kernel_version;
34 +} stream_encoded_t;
35 +
36 +#include "stream-handshake.h"
37 +#include "stream-capabilities.h"
38 +#include "stream-conf.h"
39 +#include "stream-compression/compression.h"
40 +
41 +#include "sender-destinations.h"
42 +
43 +typedef void (*rrdpush_defer_action_t)(struct sender_state *s, void *data);
44 +typedef void (*rrdpush_defer_cleanup_t)(struct sender_state *s, void *data);
45 +
46 +struct sender_state {
47 + RRDHOST *host;
48 + pid_t tid; // the thread id of the sender, from gettid_cached()
49 + SENDER_FLAGS flags;
50 + int timeout;
51 + int default_port;
52 + uint32_t reconnect_delay;
53 + char connected_to[CONNECTED_TO_SIZE + 1]; // We don't know which proxy we connect to, passed back from socket.c
54 + size_t begin;
55 + size_t reconnects_counter;
56 + size_t sent_bytes;
57 + size_t sent_bytes_on_this_connection;
58 + size_t send_attempts;
59 + time_t last_traffic_seen_t;
60 + time_t last_state_since_t; // the timestamp of the last state (online/offline) change
61 + size_t not_connected_loops;
62 + // Metrics are collected asynchronously by collector threads calling rrdset_done_push(). This can also trigger
63 + // the lazy creation of the sender thread - both cases (buffer access and thread creation) are guarded here.
64 + SPINLOCK spinlock;
65 + struct circular_buffer *buffer;
66 + char read_buffer[PLUGINSD_LINE_MAX + 1];
67 + ssize_t read_len;
68 + STREAM_CAPABILITIES capabilities;
69 + STREAM_CAPABILITIES disabled_capabilities;
70 +
71 + size_t sent_bytes_on_this_connection_per_type[STREAM_TRAFFIC_TYPE_MAX];
72 +
73 + int rrdpush_sender_pipe[2]; // collector to sender thread signaling
74 + int rrdpush_sender_socket;
75 +
76 + uint16_t hops;
77 +
78 + struct line_splitter line;
79 + struct compressor_state compressor;
80 +
81 +#ifdef NETDATA_LOG_STREAM_SENDER
82 + FILE *stream_log_fp;
83 +#endif
84 +
85 + NETDATA_SSL ssl; // structure used to encrypt the connection
86 +
87 + struct {
88 + bool shutdown;
89 + STREAM_HANDSHAKE reason;
90 + } exit;
91 +
92 + struct {
93 + DICTIONARY *requests; // de-duplication of replication requests, per chart
94 + time_t oldest_request_after_t; // the timestamp of the oldest replication request
95 + time_t latest_completed_before_t; // the timestamp of the latest replication request
96 +
97 + struct {
98 + size_t pending_requests; // the currently outstanding replication requests
99 + size_t charts_replicating; // the number of unique charts having pending replication requests (on every request one is added and is removed when we finish it - it does not track completion of the replication for this chart)
100 + bool reached_max; // true when the sender buffer should not get more replication responses
101 + } atomic;
102 +
103 + } replication;
104 +
105 + struct {
106 + bool pending_data;
107 + size_t buffer_used_percentage; // the current utilization of the sending buffer
108 + usec_t last_flush_time_ut; // the last time the sender flushed the sending buffer in USEC
109 + time_t last_buffer_recreate_s; // true when the sender buffer should be re-created
110 + } atomic;
111 +
112 + struct {
113 + const char *end_keyword;
114 + BUFFER *payload;
115 + rrdpush_defer_action_t action;
116 + rrdpush_defer_cleanup_t cleanup;
117 + void *action_data;
118 + } defer;
119 +
120 + bool parent_using_h2o;
121 +};
122 +
123 +#define sender_lock(sender) spinlock_lock(&(sender)->spinlock)
124 +#define sender_unlock(sender) spinlock_unlock(&(sender)->spinlock)
125 +
126 +#define rrdpush_sender_pipe_has_pending_data(sender) __atomic_load_n(&(sender)->atomic.pending_data, __ATOMIC_RELAXED)
127 +#define rrdpush_sender_pipe_set_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, true, __ATOMIC_RELAXED)
128 +#define rrdpush_sender_pipe_clear_pending_data(sender) __atomic_store_n(&(sender)->atomic.pending_data, false, __ATOMIC_RELAXED)
129 +
130 +#define rrdpush_sender_last_buffer_recreate_get(sender) __atomic_load_n(&(sender)->atomic.last_buffer_recreate_s, __ATOMIC_RELAXED)
131 +#define rrdpush_sender_last_buffer_recreate_set(sender, value) __atomic_store_n(&(sender)->atomic.last_buffer_recreate_s, value, __ATOMIC_RELAXED)
132 +
133 +#define rrdpush_sender_replication_buffer_full_set(sender, value) __atomic_store_n(&((sender)->replication.atomic.reached_max), value, __ATOMIC_SEQ_CST)
134 +#define rrdpush_sender_replication_buffer_full_get(sender) __atomic_load_n(&((sender)->replication.atomic.reached_max), __ATOMIC_SEQ_CST)
135 +
136 +#define rrdpush_sender_set_buffer_used_percent(sender, value) __atomic_store_n(&((sender)->atomic.buffer_used_percentage), value, __ATOMIC_RELAXED)
137 +#define rrdpush_sender_get_buffer_used_percent(sender) __atomic_load_n(&((sender)->atomic.buffer_used_percentage), __ATOMIC_RELAXED)
138 +
139 +#define rrdpush_sender_set_flush_time(sender) __atomic_store_n(&((sender)->atomic.last_flush_time_ut), now_realtime_usec(), __ATOMIC_RELAXED)
140 +#define rrdpush_sender_get_flush_time(sender) __atomic_load_n(&((sender)->atomic.last_flush_time_ut), __ATOMIC_RELAXED)
141 +
142 +#define rrdpush_sender_replicating_charts(sender) __atomic_load_n(&((sender)->replication.atomic.charts_replicating), __ATOMIC_RELAXED)
143 +#define rrdpush_sender_replicating_charts_plus_one(sender) __atomic_add_fetch(&((sender)->replication.atomic.charts_replicating), 1, __ATOMIC_RELAXED)
144 +#define rrdpush_sender_replicating_charts_minus_one(sender) __atomic_sub_fetch(&((sender)->replication.atomic.charts_replicating), 1, __ATOMIC_RELAXED)
145 +#define rrdpush_sender_replicating_charts_zero(sender) __atomic_store_n(&((sender)->replication.atomic.charts_replicating), 0, __ATOMIC_RELAXED)
146 +
147 +#define rrdpush_sender_pending_replication_requests(sender) __atomic_load_n(&((sender)->replication.atomic.pending_requests), __ATOMIC_RELAXED)
148 +#define rrdpush_sender_pending_replication_requests_plus_one(sender) __atomic_add_fetch(&((sender)->replication.atomic.pending_requests), 1, __ATOMIC_RELAXED)
149 +#define rrdpush_sender_pending_replication_requests_minus_one(sender) __atomic_sub_fetch(&((sender)->replication.atomic.pending_requests), 1, __ATOMIC_RELAXED)
150 +#define rrdpush_sender_pending_replication_requests_zero(sender) __atomic_store_n(&((sender)->replication.atomic.pending_requests), 0, __ATOMIC_RELAXED)
151 +
152 +BUFFER *sender_start(struct sender_state *s);
153 +void sender_commit(struct sender_state *s, BUFFER *wb, STREAM_TRAFFIC_TYPE type);
154 +
155 +void *rrdpush_sender_thread(void *ptr);
156 +void rrdpush_sender_thread_stop(RRDHOST *host, STREAM_HANDSHAKE reason, bool wait);
157 +
158 +void sender_thread_buffer_free(void);
159 +
160 +void rrdpush_signal_sender_to_wake_up(struct sender_state *s);
161 +
162 +bool rrdpush_sender_connect(struct sender_state *s);
163 +void rrdpush_sender_cbuffer_recreate_timed(struct sender_state *s, time_t now_s, bool have_mutex, bool force);
164 +bool rrdhost_sender_should_exit(struct sender_state *s);
165 +void rrdpush_sender_thread_spawn(RRDHOST *host);
166 +
167 +#include "replication.h"
168 +
169 +#endif //NETDATA_SENDER_H
src/streaming/stream-capabilities.c renamed
src/streaming/stream-capabilities.h renamed
src/streaming/stream-compression/brotli.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "compression_brotli.h"
3 +#include "brotli.h"
4
5 #ifdef ENABLE_BROTLI
6 #include <brotli/encode.h>
src/streaming/stream-compression/brotli.h renamed
src/streaming/stream-compression/compression.c renamed
+4 -4
@@ -2,18 +2,18 @@
2
3 #include "compression.h"
4
5 -#include "compression_gzip.h"
5 +#include "gzip.h"
6
7 #ifdef ENABLE_LZ4
8 -#include "compression_lz4.h"
8 +#include "lz4.h"
9 #endif
10
11 #ifdef ENABLE_ZSTD
12 -#include "compression_zstd.h"
12 +#include "zstd.h"
13 #endif
14
15 #ifdef ENABLE_BROTLI
16 -#include "compression_brotli.h"
16 +#include "brotli.h"
17 #endif
18
19 int rrdpush_compression_levels[COMPRESSION_ALGORITHM_MAX] = {
src/streaming/stream-compression/compression.h renamed
+10 -2
@@ -1,10 +1,10 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "rrdpush.h"
4 -
3 #ifndef NETDATA_RRDPUSH_COMPRESSION_H
4 #define NETDATA_RRDPUSH_COMPRESSION_H 1
5
6 +#include "libnetdata/libnetdata.h"
7 +
8 // signature MUST end with a newline
9
10 #if COMPRESSION_MAX_MSG_SIZE >= (COMPRESSION_MAX_CHUNK - COMPRESSION_MAX_OVERHEAD)
@@ -172,4 +172,12 @@ static inline size_t rrdpush_decompressor_get(struct decompressor_state *state,
172
173 // ----------------------------------------------------------------------------
174
175 +#include "../rrdpush.h"
176 +
177 +bool rrdpush_compression_initialize(struct sender_state *s);
178 +bool rrdpush_decompression_initialize(struct receiver_state *rpt);
179 +void rrdpush_parse_compression_order(struct receiver_state *rpt, const char *order);
180 +void rrdpush_select_receiver_compression_algorithm(struct receiver_state *rpt);
181 +void rrdpush_compression_deactivate(struct sender_state *s);
182 +
183 #endif // NETDATA_RRDPUSH_COMPRESSION_H 1
src/streaming/stream-compression/gzip.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "compression_gzip.h"
3 +#include "gzip.h"
4 #include <zlib.h>
5
6 void rrdpush_compressor_init_gzip(struct compressor_state *state) {
src/streaming/stream-compression/gzip.h renamed
src/streaming/stream-compression/lz4.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "compression_lz4.h"
3 +#include "lz4.h"
4
5 #ifdef ENABLE_LZ4
6 #include "lz4.h"
src/streaming/stream-compression/lz4.h renamed
src/streaming/stream-compression/zstd.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "compression_zstd.h"
3 +#include "zstd.h"
4
5 #ifdef ENABLE_ZSTD
6 #include <zstd.h>
src/streaming/stream-compression/zstd.h renamed
src/streaming/stream-conf.c new
+137
@@ -0,0 +1,137 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "stream-conf.h"
4 +
5 +struct config stream_config = APPCONFIG_INITIALIZER;
6 +
7 +bool stream_conf_send_enabled = false;
8 +bool stream_conf_compression_enabled = true;
9 +bool stream_conf_replication_enabled = true;
10 +
11 +const char *stream_conf_send_destination = NULL;
12 +const char *stream_conf_send_api_key = NULL;
13 +const char *stream_conf_send_charts_matching = "*";
14 +
15 +time_t stream_conf_replication_period = 86400;
16 +time_t stream_conf_replication_step = 600;
17 +
18 +const char *stream_conf_ssl_ca_path = NULL;
19 +const char *stream_conf_ssl_ca_file = NULL;
20 +
21 +// to have the remote netdata re-sync the charts
22 +// to its current clock, we send for this many
23 +// iterations a BEGIN line without microseconds
24 +// this is for the first iterations of each chart
25 +unsigned int stream_conf_initial_clock_resync_iterations = 60;
26 +
27 +static void stream_conf_load() {
28 + errno_clear();
29 + char *filename = filename_from_path_entry_strdupz(netdata_configured_user_config_dir, "stream.conf");
30 + if(!appconfig_load(&stream_config, filename, 0, NULL)) {
31 + nd_log_daemon(NDLP_NOTICE, "CONFIG: cannot load user config '%s'. Will try stock config.", filename);
32 + freez(filename);
33 +
34 + filename = filename_from_path_entry_strdupz(netdata_configured_stock_config_dir, "stream.conf");
35 + if(!appconfig_load(&stream_config, filename, 0, NULL))
36 + nd_log_daemon(NDLP_NOTICE, "CONFIG: cannot load stock config '%s'. Running with internal defaults.", filename);
37 + }
38 +
39 + freez(filename);
40 +
41 + appconfig_move(&stream_config,
42 + CONFIG_SECTION_STREAM, "timeout seconds",
43 + CONFIG_SECTION_STREAM, "timeout");
44 +
45 + appconfig_move(&stream_config,
46 + CONFIG_SECTION_STREAM, "reconnect delay seconds",
47 + CONFIG_SECTION_STREAM, "reconnect delay");
48 +
49 + appconfig_move_everywhere(&stream_config, "default memory mode", "db");
50 + appconfig_move_everywhere(&stream_config, "memory mode", "db");
51 + appconfig_move_everywhere(&stream_config, "db mode", "db");
52 + appconfig_move_everywhere(&stream_config, "default history", "retention");
53 + appconfig_move_everywhere(&stream_config, "history", "retention");
54 + appconfig_move_everywhere(&stream_config, "default proxy enabled", "proxy enabled");
55 + appconfig_move_everywhere(&stream_config, "default proxy destination", "proxy destination");
56 + appconfig_move_everywhere(&stream_config, "default proxy api key", "proxy api key");
57 + appconfig_move_everywhere(&stream_config, "default proxy send charts matching", "proxy send charts matching");
58 + appconfig_move_everywhere(&stream_config, "default health log history", "health log retention");
59 + appconfig_move_everywhere(&stream_config, "health log history", "health log retention");
60 + appconfig_move_everywhere(&stream_config, "seconds to replicate", "replication period");
61 + appconfig_move_everywhere(&stream_config, "seconds per replication step", "replication step");
62 + appconfig_move_everywhere(&stream_config, "default postpone alarms on connect seconds", "postpone alerts on connect");
63 + appconfig_move_everywhere(&stream_config, "postpone alarms on connect seconds", "postpone alerts on connect");
64 +}
65 +
66 +bool stream_conf_receiver_needs_dbengine(void) {
67 + return stream_conf_needs_dbengine(&stream_config);
68 +}
69 +
70 +bool stream_conf_init() {
71 + // --------------------------------------------------------------------
72 + // load stream.conf
73 + stream_conf_load();
74 +
75 + stream_conf_send_enabled =
76 + appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "enabled", stream_conf_send_enabled);
77 +
78 + stream_conf_send_destination =
79 + appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "destination", "");
80 +
81 + stream_conf_send_api_key =
82 + appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "api key", "");
83 +
84 + stream_conf_send_charts_matching =
85 + appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "send charts matching", stream_conf_send_charts_matching);
86 +
87 + stream_conf_replication_enabled =
88 + config_get_boolean(CONFIG_SECTION_DB, "enable replication", stream_conf_replication_enabled);
89 +
90 + stream_conf_replication_period =
91 + config_get_duration_seconds(CONFIG_SECTION_DB, "replication period", stream_conf_replication_period);
92 +
93 + stream_conf_replication_step =
94 + config_get_duration_seconds(CONFIG_SECTION_DB, "replication step", stream_conf_replication_step);
95 +
96 + rrdhost_free_orphan_time_s =
97 + config_get_duration_seconds(CONFIG_SECTION_DB, "cleanup orphan hosts after", rrdhost_free_orphan_time_s);
98 +
99 + stream_conf_compression_enabled =
100 + appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM,
101 + "enable compression", stream_conf_compression_enabled);
102 +
103 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_BROTLI] = (int)appconfig_get_number(
104 + &stream_config, CONFIG_SECTION_STREAM, "brotli compression level",
105 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_BROTLI]);
106 +
107 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_ZSTD] = (int)appconfig_get_number(
108 + &stream_config, CONFIG_SECTION_STREAM, "zstd compression level",
109 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_ZSTD]);
110 +
111 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_LZ4] = (int)appconfig_get_number(
112 + &stream_config, CONFIG_SECTION_STREAM, "lz4 compression acceleration",
113 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_LZ4]);
114 +
115 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_GZIP] = (int)appconfig_get_number(
116 + &stream_config, CONFIG_SECTION_STREAM, "gzip compression level",
117 + rrdpush_compression_levels[COMPRESSION_ALGORITHM_GZIP]);
118 +
119 + if(stream_conf_send_enabled && (!stream_conf_send_destination || !*stream_conf_send_destination || !stream_conf_send_api_key || !*stream_conf_send_api_key)) {
120 + nd_log_daemon(NDLP_WARNING, "STREAM [send]: cannot enable sending thread - information is missing.");
121 + stream_conf_send_enabled = false;
122 + }
123 +
124 + netdata_ssl_validate_certificate_sender = !appconfig_get_boolean(&stream_config, CONFIG_SECTION_STREAM, "ssl skip certificate verification", !netdata_ssl_validate_certificate);
125 +
126 + if(!netdata_ssl_validate_certificate_sender)
127 + nd_log_daemon(NDLP_NOTICE, "SSL: streaming senders will skip SSL certificates verification.");
128 +
129 + stream_conf_ssl_ca_path = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "CApath", NULL);
130 + stream_conf_ssl_ca_file = appconfig_get(&stream_config, CONFIG_SECTION_STREAM, "CAfile", NULL);
131 +
132 + return stream_conf_send_enabled;
133 +}
134 +
135 +bool stream_conf_configured_as_parent() {
136 + return stream_conf_has_uuid_section(&stream_config);
137 +}
src/streaming/stream-conf.h new
+28
@@ -0,0 +1,28 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STREAM_CONF_H
4 +#define NETDATA_STREAM_CONF_H
5 +
6 +#include "libnetdata/libnetdata.h"
7 +#include "daemon/common.h"
8 +
9 +extern bool stream_conf_send_enabled;
10 +extern bool stream_conf_compression_enabled;
11 +extern bool stream_conf_replication_enabled;
12 +
13 +extern const char *stream_conf_send_destination;
14 +extern const char *stream_conf_send_api_key;
15 +extern const char *stream_conf_send_charts_matching;
16 +extern time_t stream_conf_replication_period;
17 +extern time_t stream_conf_replication_step;
18 +extern unsigned int stream_conf_initial_clock_resync_iterations;
19 +
20 +extern struct config stream_config;
21 +extern const char *stream_conf_ssl_ca_path;
22 +extern const char *stream_conf_ssl_ca_file;
23 +
24 +bool stream_conf_init();
25 +bool stream_conf_receiver_needs_dbengine();
26 +bool stream_conf_configured_as_parent();
27 +
28 +#endif //NETDATA_STREAM_CONF_H
src/streaming/stream-handshake.c new
+53
@@ -0,0 +1,53 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdpush.h"
4 +
5 +static struct {
6 + STREAM_HANDSHAKE err;
7 + const char *str;
8 +} handshake_errors[] = {
9 + { STREAM_HANDSHAKE_OK_V3, "CONNECTED" },
10 + { STREAM_HANDSHAKE_OK_V2, "CONNECTED" },
11 + { STREAM_HANDSHAKE_OK_V1, "CONNECTED" },
12 + { STREAM_HANDSHAKE_NEVER, "" },
13 + { STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE, "BAD HANDSHAKE" },
14 + { STREAM_HANDSHAKE_ERROR_LOCALHOST, "LOCALHOST" },
15 + { STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED, "ALREADY CONNECTED" },
16 + { STREAM_HANDSHAKE_ERROR_DENIED, "DENIED" },
17 + { STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT, "SEND TIMEOUT" },
18 + { STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT, "RECEIVE TIMEOUT" },
19 + { STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE, "INVALID CERTIFICATE" },
20 + { STREAM_HANDSHAKE_ERROR_SSL_ERROR, "SSL ERROR" },
21 + { STREAM_HANDSHAKE_ERROR_CANT_CONNECT, "CANT CONNECT" },
22 + { STREAM_HANDSHAKE_BUSY_TRY_LATER, "BUSY TRY LATER" },
23 + { STREAM_HANDSHAKE_INTERNAL_ERROR, "INTERNAL ERROR" },
24 + { STREAM_HANDSHAKE_INITIALIZATION, "REMOTE IS INITIALIZING" },
25 + { STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP, "DISCONNECTED HOST CLEANUP" },
26 + { STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER, "DISCONNECTED STALE RECEIVER" },
27 + { STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN, "DISCONNECTED SHUTDOWN REQUESTED" },
28 + { STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT, "DISCONNECTED NETDATA EXIT" },
29 + { STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT, "DISCONNECTED PARSE ENDED" },
30 + {STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR, "DISCONNECTED UNKNOWN SOCKET READ ERROR" },
31 + { STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED, "DISCONNECTED PARSE ERROR" },
32 + { STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT, "DISCONNECTED RECEIVER LEFT" },
33 + { STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST, "DISCONNECTED ORPHAN HOST" },
34 + { STREAM_HANDSHAKE_NON_STREAMABLE_HOST, "NON STREAMABLE HOST" },
35 + { STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER, "DISCONNECTED NOT SUFFICIENT READ BUFFER" },
36 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF, "DISCONNECTED SOCKET EOF" },
37 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED, "DISCONNECTED SOCKET READ FAILED" },
38 + {STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_TIMEOUT, "DISCONNECTED SOCKET READ TIMEOUT" },
39 + { 0, NULL },
40 +};
41 +
42 +const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error) {
43 + if(handshake_error >= STREAM_HANDSHAKE_OK_V1)
44 + // handshake_error is the whole version / capabilities number
45 + return "CONNECTED";
46 +
47 + for(size_t i = 0; handshake_errors[i].str ; i++) {
48 + if(handshake_error == handshake_errors[i].err)
49 + return handshake_errors[i].str;
50 + }
51 +
52 + return "UNKNOWN";
53 +}
src/streaming/stream-handshake.h new
+82
@@ -0,0 +1,82 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#ifndef NETDATA_STREAM_HANDSHAKE_H
4 +#define NETDATA_STREAM_HANDSHAKE_H
5 +
6 +#define HTTP_HEADER_SIZE 8192
7 +
8 +#define STREAMING_PROTOCOL_VERSION "1.1"
9 +#define START_STREAMING_PROMPT_V1 "Hit me baby, push them over..."
10 +#define START_STREAMING_PROMPT_V2 "Hit me baby, push them over and bring the host labels..."
11 +#define START_STREAMING_PROMPT_VN "Hit me baby, push them over with the version="
12 +
13 +#define START_STREAMING_ERROR_SAME_LOCALHOST "Don't hit me baby, you are trying to stream my localhost back"
14 +#define START_STREAMING_ERROR_ALREADY_STREAMING "This GUID is already streaming to this server"
15 +#define START_STREAMING_ERROR_NOT_PERMITTED "You are not permitted to access this. Check the logs for more info."
16 +#define START_STREAMING_ERROR_BUSY_TRY_LATER "The server is too busy now to accept this request. Try later."
17 +#define START_STREAMING_ERROR_INTERNAL_ERROR "The server encountered an internal error. Try later."
18 +#define START_STREAMING_ERROR_INITIALIZATION "The server is initializing. Try later."
19 +
20 +#define RRDPUSH_STATUS_CONNECTED "CONNECTED"
21 +#define RRDPUSH_STATUS_ALREADY_CONNECTED "ALREADY CONNECTED"
22 +#define RRDPUSH_STATUS_DISCONNECTED "DISCONNECTED"
23 +#define RRDPUSH_STATUS_RATE_LIMIT "RATE LIMIT TRY LATER"
24 +#define RRDPUSH_STATUS_INITIALIZATION_IN_PROGRESS "INITIALIZATION IN PROGRESS RETRY LATER"
25 +#define RRDPUSH_STATUS_INTERNAL_SERVER_ERROR "INTERNAL SERVER ERROR DROPPING CONNECTION"
26 +#define RRDPUSH_STATUS_DUPLICATE_RECEIVER "DUPLICATE RECEIVER DROPPING CONNECTION"
27 +#define RRDPUSH_STATUS_CANT_REPLY "CANT REPLY DROPPING CONNECTION"
28 +#define RRDPUSH_STATUS_NO_HOSTNAME "NO HOSTNAME PERMISSION DENIED"
29 +#define RRDPUSH_STATUS_NO_API_KEY "NO API KEY PERMISSION DENIED"
30 +#define RRDPUSH_STATUS_INVALID_API_KEY "INVALID API KEY PERMISSION DENIED"
31 +#define RRDPUSH_STATUS_NO_MACHINE_GUID "NO MACHINE GUID PERMISSION DENIED"
32 +#define RRDPUSH_STATUS_MACHINE_GUID_DISABLED "MACHINE GUID DISABLED PERMISSION DENIED"
33 +#define RRDPUSH_STATUS_INVALID_MACHINE_GUID "INVALID MACHINE GUID PERMISSION DENIED"
34 +#define RRDPUSH_STATUS_API_KEY_DISABLED "API KEY DISABLED PERMISSION DENIED"
35 +#define RRDPUSH_STATUS_NOT_ALLOWED_IP "NOT ALLOWED IP PERMISSION DENIED"
36 +#define RRDPUSH_STATUS_LOCALHOST "LOCALHOST PERMISSION DENIED"
37 +#define RRDPUSH_STATUS_PERMISSION_DENIED "PERMISSION DENIED"
38 +#define RRDPUSH_STATUS_BAD_HANDSHAKE "BAD HANDSHAKE"
39 +#define RRDPUSH_STATUS_TIMEOUT "TIMEOUT"
40 +#define RRDPUSH_STATUS_CANT_UPGRADE_CONNECTION "CANT UPGRADE CONNECTION"
41 +#define RRDPUSH_STATUS_SSL_ERROR "SSL ERROR"
42 +#define RRDPUSH_STATUS_INVALID_SSL_CERTIFICATE "INVALID SSL CERTIFICATE"
43 +#define RRDPUSH_STATUS_CANT_ESTABLISH_SSL_CONNECTION "CANT ESTABLISH SSL CONNECTION"
44 +
45 +typedef enum {
46 + STREAM_HANDSHAKE_OK_V3 = 3, // v3+
47 + STREAM_HANDSHAKE_OK_V2 = 2, // v2
48 + STREAM_HANDSHAKE_OK_V1 = 1, // v1
49 + STREAM_HANDSHAKE_NEVER = 0, // never tried to connect
50 + STREAM_HANDSHAKE_ERROR_BAD_HANDSHAKE = -1,
51 + STREAM_HANDSHAKE_ERROR_LOCALHOST = -2,
52 + STREAM_HANDSHAKE_ERROR_ALREADY_CONNECTED = -3,
53 + STREAM_HANDSHAKE_ERROR_DENIED = -4,
54 + STREAM_HANDSHAKE_ERROR_SEND_TIMEOUT = -5,
55 + STREAM_HANDSHAKE_ERROR_RECEIVE_TIMEOUT = -6,
56 + STREAM_HANDSHAKE_ERROR_INVALID_CERTIFICATE = -7,
57 + STREAM_HANDSHAKE_ERROR_SSL_ERROR = -8,
58 + STREAM_HANDSHAKE_ERROR_CANT_CONNECT = -9,
59 + STREAM_HANDSHAKE_BUSY_TRY_LATER = -10,
60 + STREAM_HANDSHAKE_INTERNAL_ERROR = -11,
61 + STREAM_HANDSHAKE_INITIALIZATION = -12,
62 + STREAM_HANDSHAKE_DISCONNECT_HOST_CLEANUP = -13,
63 + STREAM_HANDSHAKE_DISCONNECT_STALE_RECEIVER = -14,
64 + STREAM_HANDSHAKE_DISCONNECT_SHUTDOWN = -15,
65 + STREAM_HANDSHAKE_DISCONNECT_NETDATA_EXIT = -16,
66 + STREAM_HANDSHAKE_DISCONNECT_PARSER_EXIT = -17,
67 + STREAM_HANDSHAKE_DISCONNECT_UNKNOWN_SOCKET_READ_ERROR = -18,
68 + STREAM_HANDSHAKE_DISCONNECT_PARSER_FAILED = -19,
69 + STREAM_HANDSHAKE_DISCONNECT_RECEIVER_LEFT = -20,
70 + STREAM_HANDSHAKE_DISCONNECT_ORPHAN_HOST = -21,
71 + STREAM_HANDSHAKE_NON_STREAMABLE_HOST = -22,
72 + STREAM_HANDSHAKE_DISCONNECT_NOT_SUFFICIENT_READ_BUFFER = -23,
73 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_EOF = -24,
74 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_FAILED = -25,
75 + STREAM_HANDSHAKE_DISCONNECT_SOCKET_READ_TIMEOUT = -26,
76 + STREAM_HANDSHAKE_ERROR_HTTP_UPGRADE = -27,
77 +
78 +} STREAM_HANDSHAKE;
79 +
80 +const char *stream_handshake_error_to_string(STREAM_HANDSHAKE handshake_error);
81 +
82 +#endif //NETDATA_STREAM_HANDSHAKE_H
src/streaming/stream-path.c renamed
+1 -1
@@ -1,6 +1,6 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 -#include "stream_path.h"
3 +#include "stream-path.h"
4 #include "rrdpush.h"
5 #include "plugins.d/pluginsd_internals.h"
6
src/streaming/stream-path.h renamed
+1 -1
@@ -3,7 +3,7 @@
3 #ifndef NETDATA_STREAM_PATH_H
4 #define NETDATA_STREAM_PATH_H
5
6 -#include "stream_capabilities.h"
6 +#include "stream-capabilities.h"
7
8 #define STREAM_PATH_JSON_MEMBER "streaming_path"
9
src/web/api/v1/api_v1_info.c
+1 -1
@@ -161,7 +161,7 @@ static int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
161 buffer_json_member_add_uint64(wb, "page-cache-size", default_rrdeng_page_cache_mb);
162 #endif // ENABLE_DBENGINE
163 buffer_json_member_add_boolean(wb, "web-enabled", web_server_mode != WEB_SERVER_MODE_NONE);
164 - buffer_json_member_add_boolean(wb, "stream-enabled", default_rrdpush_enabled);
164 + buffer_json_member_add_boolean(wb, "stream-enabled", stream_conf_send_enabled);
165
166 buffer_json_member_add_boolean(wb, "stream-compression",
167 host->sender && host->sender->compressor.initialized);
src/web/server/h2o/http_server.c
+1 -1
@@ -1,7 +1,7 @@
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "daemon/common.h"
4 -#include "streaming/common.h"
4 +#include "streaming/h2o-common.h"
5 #include "http_server.h"
6
7 #pragma GCC diagnostic push
src/web/server/h2o/rrdpush.c renamed
+1 -1
@@ -4,7 +4,7 @@
4 #include "streaming.h"
5 #include "connlist.h"
6 #include "h2o_utils.h"
7 -#include "streaming/common.h"
7 +#include "streaming/h2o-common.h"
8
9 static int pending_write_reqs = 0;
10