Adds ACLK-NG as fallback(#10315)
* adds a new implementation of ACLK written almost from scratch * external dependencies only OpenSSL and JSON-C * fallback for systems where ACLK Legacy can't build (for technical or philosophical reasons) * can be forced to build by giving "--aclk-ng" to the installer
Timotej S committed
Mar 16, 2021 at 12:38 UTC
e7e5d0c37242d8457e4b2610a95effe0db5ca1b1
42 files changed
+3884
-20
.github/workflows/build-and-install.yml
+6
@@ -12,6 +12,8 @@ jobs:
12
steps:
13
- name: Git clone repository
14
uses: actions/checkout@v2
15
+ with:
16
+ submodules: recursive
17
- run: |
18
git fetch --prune --unshallow --tags
19
- name: Build
@@ -98,6 +100,8 @@ jobs:
100
steps:
101
- name: Git clone repository
102
uses: actions/checkout@v2
103
+ with:
104
+ submodules: recursive
105
- name: install-required-packages.sh on ${{ matrix.distro }}
106
env:
107
PRE: ${{ matrix.pre }}
@@ -183,6 +187,8 @@ jobs:
187
steps:
188
- name: Git clone repository
189
uses: actions/checkout@v2
190
+ with:
191
+ submodules: recursive
192
- name: install-required-packages.sh on ${{ matrix.distro }}
193
env:
194
PRE: ${{ matrix.pre }}
.github/workflows/checks.yml
+10
@@ -12,6 +12,8 @@ jobs:
12
steps:
13
- name: Git clone repository
14
uses: actions/checkout@v2
15
+ with:
16
+ submodules: recursive
17
- name: Run checksum checks on kickstart files
18
env:
19
LOCAL_ONLY: "true"
@@ -23,6 +25,8 @@ jobs:
25
steps:
26
- name: Git clone repository
27
uses: actions/checkout@v2
28
+ with:
29
+ submodules: recursive
30
- name: Install required packages
31
run: |
32
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -43,6 +47,8 @@ jobs:
47
steps:
48
- name: Checkout
49
uses: actions/checkout@v2
50
+ with:
51
+ submodules: recursive
52
- name: Build
53
run: >
54
docker run -v "$PWD":/netdata -w /netdata alpine:latest /bin/sh -c
@@ -68,6 +74,8 @@ jobs:
74
steps:
75
- name: Checkout
76
uses: actions/checkout@v2
77
+ with:
78
+ submodules: recursive
79
- name: Prepare environment
80
run: |
81
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
@@ -98,6 +106,8 @@ jobs:
106
steps:
107
- name: Checkout
108
uses: actions/checkout@v2
109
+ with:
110
+ submodules: recursive
111
- name: Prepare environment
112
run: ./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata
113
- name: Build netdata
.github/workflows/coverity.yml
+2
@@ -15,6 +15,8 @@ jobs:
15
steps:
16
- name: Checkout
17
uses: actions/checkout@v2
18
+ with:
19
+ submodules: recursive
20
- name: Prepare environment
21
env:
22
DEBIAN_FRONTEND: 'noninteractive'
.github/workflows/docker.yml
+2
@@ -26,6 +26,8 @@ jobs:
26
steps:
27
- name: Checkout
28
uses: actions/checkout@v2
29
+ with:
30
+ submodules: recursive
31
- name: Determine if we should push changes and which tags to use
32
if: github.event_name == 'workflow_dispatch' && github.event.inputs.version != 'nightly'
33
run: |
.github/workflows/docs.yml
+2
@@ -16,6 +16,8 @@ jobs:
16
steps:
17
- name: Checkout
18
uses: actions/checkout@v2
19
+ with:
20
+ submodules: recursive
21
- name: Run link check
22
uses: gaurav-nelson/github-action-markdown-link-check@v1
23
with:
.github/workflows/review.yml
+3
@@ -16,6 +16,7 @@ jobs:
16
- name: Git clone repository
17
uses: actions/checkout@v2
18
with:
19
+ submodules: recursive
20
fetch-depth: 0
21
- name: Check files
22
run: |
@@ -57,6 +58,7 @@ jobs:
58
- name: Git clone repository
59
uses: actions/checkout@v2
60
with:
61
+ submodules: recursive
62
fetch-depth: 0
63
- name: Check files
64
run: |
@@ -80,6 +82,7 @@ jobs:
82
- name: Git clone repository
83
uses: actions/checkout@v2
84
with:
85
+ submodules: recursive
86
fetch-depth: 0
87
- name: Check files
88
run: |
.github/workflows/tests.yml
+5
-1
@@ -21,6 +21,8 @@ jobs:
21
steps:
22
- name: Checkout
23
uses: actions/checkout@v2
24
+ with:
25
+ submodules: recursive
26
- name: Prepare environment
27
run: |
28
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -39,6 +41,8 @@ jobs:
41
steps:
42
- name: Checkout
43
uses: actions/checkout@v2
44
+ with:
45
+ submodules: recursive
46
- name: Prepare environment
47
run: |
48
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
@@ -48,7 +52,7 @@ jobs:
52
- name: Configure
53
run: |
54
autoreconf -ivf
51
- ./configure
55
+ ./configure --without-aclk-ng
56
# XXX: Work-around for bug with libbson-1.0 in Ubuntu 18.04
57
# See: https://bugs.launchpad.net/ubuntu/+source/libmongoc/+bug/1790771
58
# https://jira.mongodb.org/browse/CDRIVER-2818
.github/workflows/updater.yml
+2
@@ -34,6 +34,8 @@ jobs:
34
steps:
35
- name: Git clone repository
36
uses: actions/checkout@v2
37
+ with:
38
+ submodules: recursive
39
- name: Install required packages & build tarball
40
run: |
41
./packaging/installer/install-required-packages.sh --dont-wait --non-interactive netdata-all
.gitmodules
new
+3
@@ -0,0 +1,3 @@
1
+[submodule "mqtt_websockets"]
2
+ path = mqtt_websockets
3
+ url = https://github.com/underhood/mqtt_websockets.git
Makefile.am
+42
-6
@@ -116,10 +116,18 @@ SUBDIRS += \
116
web \
117
claim \
118
parser \
119
- aclk/legacy \
119
spawn \
120
$(NULL)
121
122
+if ACLK_NG
123
+SUBDIRS += \
124
+ mqtt_websockets \
125
+ $(NULL)
126
+else
127
+SUBDIRS += \
128
+ aclk/legacy \
129
+ $(NULL)
130
+endif
131
132
AM_CFLAGS = \
133
$(OPTIONAL_MATH_CFLAGS) \
@@ -525,6 +533,28 @@ PARSER_FILES = \
533
parser/parser.h \
534
$(NULL)
535
536
+if ACLK_NG
537
+ACLK_FILES = \
538
+ aclk/aclk.c \
539
+ aclk/aclk.h \
540
+ aclk/aclk_util.c \
541
+ aclk/aclk_util.h \
542
+ aclk/aclk_stats.c \
543
+ aclk/aclk_stats.h \
544
+ aclk/aclk_query.c \
545
+ aclk/aclk_query.h \
546
+ aclk/aclk_query_queue.c \
547
+ aclk/aclk_query_queue.h \
548
+ aclk/aclk_collector_list.c \
549
+ aclk/aclk_collector_list.h \
550
+ aclk/aclk_otp.c \
551
+ aclk/aclk_otp.h \
552
+ aclk/aclk_tx_msgs.c \
553
+ aclk/aclk_tx_msgs.h \
554
+ aclk/aclk_rx_msgs.c \
555
+ aclk/aclk_rx_msgs.h \
556
+ $(NULL)
557
+else #ACLK_NG
558
ACLK_FILES = \
559
aclk/legacy/aclk_rrdhost_state.h \
560
aclk/legacy/aclk_common.c \
@@ -548,9 +578,8 @@ ACLK_FILES += \
578
aclk/legacy/aclk_lws_https_client.c \
579
aclk/legacy/aclk_lws_https_client.h \
580
$(NULL)
551
-endif
552
-
553
-
581
+endif #ENABLE_ACLK
582
+endif #ACLK_NG
583
584
SPAWN_PLUGIN_FILES = \
585
spawn/spawn.c \
@@ -714,6 +743,12 @@ NETDATACLI_FILES = \
743
sbin_PROGRAMS += netdata
744
netdata_SOURCES = $(NETDATA_FILES)
745
746
+if ACLK_NG
747
+netdata_LDADD = \
748
+ mqtt_websockets/libmqttwebsockets.a \
749
+ $(NETDATA_COMMON_LIBS) \
750
+ $(NULL)
751
+else #ACLK_NG
752
if ENABLE_ACLK
753
netdata_LDADD = \
754
externaldeps/mosquitto/libmosquitto.a \
@@ -721,11 +756,12 @@ netdata_LDADD = \
756
$(OPTIONAL_LWS_LIBS) \
757
$(NETDATA_COMMON_LIBS) \
758
$(NULL)
724
-else
759
+else #ENABLE_ACLK
760
netdata_LDADD = \
761
$(NETDATA_COMMON_LIBS) \
762
$(NULL)
728
-endif
763
+endif #ENABLE_ACLK
764
+endif #ACLK_NG
765
766
if ENABLE_CXX_LINKER
767
netdata_LINK = $(CXXLD) $(CXXFLAGS) $(LDFLAGS) -o $@
aclk/aclk.c
new
+820
@@ -0,0 +1,820 @@
1
+#include "aclk.h"
2
+
3
+#include "aclk_stats.h"
4
+#include "mqtt_wss_client.h"
5
+#include "aclk_otp.h"
6
+#include "aclk_tx_msgs.h"
7
+#include "aclk_query.h"
8
+#include "aclk_query_queue.h"
9
+#include "aclk_util.h"
10
+#include "aclk_rx_msgs.h"
11
+#include "aclk_collector_list.h"
12
+
13
+#ifdef ACLK_LOG_CONVERSATION_DIR
14
+#include <sys/types.h>
15
+#include <sys/stat.h>
16
+#include <fcntl.h>
17
+#endif
18
+
19
+#define ACLK_STABLE_TIMEOUT 3 // Minimum delay to mark AGENT as stable
20
+
21
+//TODO remove most (as in 99.999999999%) of this crap
22
+int aclk_connected = 0;
23
+int aclk_disable_runtime = 0;
24
+int aclk_disable_single_updates = 0;
25
+int aclk_kill_link = 0;
26
+
27
+int aclk_pubacks_per_conn = 0; // How many PubAcks we got since MQTT conn est.
28
+
29
+usec_t aclk_session_us = 0; // Used by the mqtt layer
30
+time_t aclk_session_sec = 0; // Used by the mqtt layer
31
+
32
+mqtt_wss_client mqttwss_client;
33
+
34
+netdata_mutex_t aclk_shared_state_mutex = NETDATA_MUTEX_INITIALIZER;
35
+#define ACLK_SHARED_STATE_LOCK netdata_mutex_lock(&aclk_shared_state_mutex)
36
+#define ACLK_SHARED_STATE_UNLOCK netdata_mutex_unlock(&aclk_shared_state_mutex)
37
+
38
+struct aclk_shared_state aclk_shared_state = {
39
+ .agent_state = AGENT_INITIALIZING,
40
+ .last_popcorn_interrupt = 0,
41
+ .version_neg = 0,
42
+ .version_neg_wait_till = 0,
43
+ .mqtt_shutdown_msg_id = -1,
44
+ .mqtt_shutdown_msg_rcvd = 0
45
+};
46
+
47
+void aclk_single_update_disable()
48
+{
49
+ aclk_disable_single_updates = 1;
50
+}
51
+
52
+void aclk_single_update_enable()
53
+{
54
+ aclk_disable_single_updates = 0;
55
+}
56
+
57
+//ENDTODO
58
+
59
+static RSA *aclk_private_key = NULL;
60
+static int load_private_key()
61
+{
62
+ if (aclk_private_key != NULL)
63
+ RSA_free(aclk_private_key);
64
+ aclk_private_key = NULL;
65
+ char filename[FILENAME_MAX + 1];
66
+ snprintfz(filename, FILENAME_MAX, "%s/cloud.d/private.pem", netdata_configured_varlib_dir);
67
+
68
+ long bytes_read;
69
+ char *private_key = read_by_filename(filename, &bytes_read);
70
+ if (!private_key) {
71
+ error("Claimed agent cannot establish ACLK - unable to load private key '%s' failed.", filename);
72
+ return 1;
73
+ }
74
+ debug(D_ACLK, "Claimed agent loaded private key len=%ld bytes", bytes_read);
75
+
76
+ BIO *key_bio = BIO_new_mem_buf(private_key, -1);
77
+ if (key_bio==NULL) {
78
+ error("Claimed agent cannot establish ACLK - failed to create BIO for key");
79
+ goto biofailed;
80
+ }
81
+
82
+ aclk_private_key = PEM_read_bio_RSAPrivateKey(key_bio, NULL, NULL, NULL);
83
+ BIO_free(key_bio);
84
+ if (aclk_private_key!=NULL)
85
+ {
86
+ freez(private_key);
87
+ return 0;
88
+ }
89
+ char err[512];
90
+ ERR_error_string_n(ERR_get_error(), err, sizeof(err));
91
+ error("Claimed agent cannot establish ACLK - cannot create private key: %s", err);
92
+
93
+biofailed:
94
+ freez(private_key);
95
+ return 1;
96
+}
97
+
98
+static int wait_till_cloud_enabled()
99
+{
100
+ info("Waiting for Cloud to be enabled");
101
+ while (!netdata_cloud_setting) {
102
+ sleep_usec(USEC_PER_SEC * 1);
103
+ if (netdata_exit)
104
+ return 1;
105
+ }
106
+ return 0;
107
+}
108
+
109
+/**
110
+ * Will block until agent is claimed. Returns only if agent claimed
111
+ * or if agent needs to shutdown.
112
+ *
113
+ * @return `0` if agent has been claimed,
114
+ * `1` if interrupted due to agent shutting down
115
+ */
116
+static int wait_till_agent_claimed(void)
117
+{
118
+ //TODO prevent malloc and freez
119
+ char *agent_id = is_agent_claimed();
120
+ while (likely(!agent_id)) {
121
+ sleep_usec(USEC_PER_SEC * 1);
122
+ if (netdata_exit)
123
+ return 1;
124
+ agent_id = is_agent_claimed();
125
+ }
126
+ freez(agent_id);
127
+ return 0;
128
+}
129
+
130
+/**
131
+ * Checks everything is ready for connection
132
+ * agent claimed, cloud url set and private key available
133
+ *
134
+ * @param aclk_hostname points to location where string pointer to hostname will be set
135
+ * @param ackl_port port to int where port will be saved
136
+ *
137
+ * @return If non 0 returned irrecoverable error happened and ACLK should be terminated
138
+ */
139
+static int wait_till_agent_claim_ready()
140
+{
141
+ int port;
142
+ char *hostname = NULL;
143
+ while (!netdata_exit) {
144
+ if (wait_till_agent_claimed())
145
+ return 1;
146
+
147
+ // The NULL return means the value was never initialised, but this value has been initialized in post_conf_load.
148
+ // We trap the impossible NULL here to keep the linter happy without using a fatal() in the code.
149
+ char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);
150
+ if (cloud_base_url == NULL) {
151
+ error("Do not move the cloud base url out of post_conf_load!!");
152
+ return 1;
153
+ }
154
+
155
+ // We just check configuration is valid here
156
+ // TODO make it without malloc/free
157
+ if (aclk_decode_base_url(cloud_base_url, &hostname, &port)) {
158
+ error("Agent is claimed but the configuration is invalid, please fix");
159
+ freez(hostname);
160
+ hostname = NULL;
161
+ sleep(5);
162
+ continue;
163
+ }
164
+ freez(hostname);
165
+ hostname = NULL;
166
+
167
+ if (!load_private_key()) {
168
+ sleep(5);
169
+ break;
170
+ }
171
+ }
172
+
173
+ return 0;
174
+}
175
+
176
+void aclk_mqtt_wss_log_cb(mqtt_wss_log_type_t log_type, const char* str)
177
+{
178
+ switch(log_type) {
179
+ case MQTT_WSS_LOG_ERROR:
180
+ case MQTT_WSS_LOG_FATAL:
181
+ case MQTT_WSS_LOG_WARN:
182
+ error("%s", str);
183
+ return;
184
+ case MQTT_WSS_LOG_INFO:
185
+ info("%s", str);
186
+ return;
187
+ case MQTT_WSS_LOG_DEBUG:
188
+ debug(D_ACLK, "%s", str);
189
+ return;
190
+ default:
191
+ error("Unknown log type from mqtt_wss");
192
+ }
193
+}
194
+
195
+//TODO prevent big buffer on stack
196
+#define RX_MSGLEN_MAX 4096
197
+static void msg_callback(const char *topic, const void *msg, size_t msglen, int qos)
198
+{
199
+ char cmsg[RX_MSGLEN_MAX];
200
+ size_t len = (msglen < RX_MSGLEN_MAX - 1) ? msglen : (RX_MSGLEN_MAX - 1);
201
+
202
+ if (msglen > RX_MSGLEN_MAX - 1)
203
+ error("Incoming ACLK message was bigger than MAX of %d and got truncated.", RX_MSGLEN_MAX);
204
+
205
+ memcpy(cmsg,
206
+ msg,
207
+ len);
208
+ cmsg[len] = 0;
209
+
210
+#ifdef ACLK_LOG_CONVERSATION_DIR
211
+#define FN_MAX_LEN 512
212
+ char filename[FN_MAX_LEN];
213
+ int logfd;
214
+ snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-rx.json", ACLK_GET_CONV_LOG_NEXT());
215
+ logfd = open(filename, O_CREAT | O_TRUNC | O_WRONLY, S_IRUSR | S_IWUSR );
216
+ if(logfd < 0)
217
+ error("Error opening ACLK Conversation logfile \"%s\" for RX message.", filename);
218
+ write(logfd, msg, msglen);
219
+ close(logfd);
220
+#endif
221
+
222
+ debug(D_ACLK, "Got Message From Broker Topic \"%s\" QOS %d MSG: \"%s\"", topic, qos, cmsg);
223
+
224
+ if (strcmp(aclk_get_topic(ACLK_TOPICID_COMMAND), topic))
225
+ error("Received message on unexpected topic %s", topic);
226
+
227
+ if (aclk_shared_state.mqtt_shutdown_msg_id > 0) {
228
+ error("Link is shutting down. Ignoring message.");
229
+ return;
230
+ }
231
+
232
+ aclk_handle_cloud_message(cmsg);
233
+}
234
+
235
+static void puback_callback(uint16_t packet_id)
236
+{
237
+ if (++aclk_pubacks_per_conn == ACLK_PUBACKS_CONN_STABLE)
238
+ aclk_reconnect_delay(0);
239
+
240
+#ifdef NETDATA_INTERNAL_CHECKS
241
+ aclk_stats_msg_puback(packet_id);
242
+#endif
243
+
244
+ if (aclk_shared_state.mqtt_shutdown_msg_id == (int)packet_id) {
245
+ error("Got PUBACK for shutdown message. Can exit gracefully.");
246
+ aclk_shared_state.mqtt_shutdown_msg_rcvd = 1;
247
+ }
248
+}
249
+
250
+static int read_query_thread_count()
251
+{
252
+ int threads = MIN(processors/2, 6);
253
+ threads = MAX(threads, 2);
254
+ threads = config_get_number(CONFIG_SECTION_CLOUD, "query thread count", threads);
255
+ if(threads < 1) {
256
+ error("You need at least one query thread. Overriding configured setting of \"%d\"", threads);
257
+ threads = 1;
258
+ config_set_number(CONFIG_SECTION_CLOUD, "query thread count", threads);
259
+ }
260
+ return threads;
261
+}
262
+
263
+/* Keeps connection alive and handles all network comms.
264
+ * Returns on error or when netdata is shutting down.
265
+ * @param client instance of mqtt_wss_client
266
+ * @returns 0 - Netdata Exits
267
+ * >0 - Error happened. Reconnect and start over.
268
+ */
269
+static int handle_connection(mqtt_wss_client client)
270
+{
271
+ time_t last_periodic_query_wakeup = now_monotonic_sec();
272
+ while (!netdata_exit) {
273
+ // timeout 1000 to check at least once a second
274
+ // for netdata_exit
275
+ if (mqtt_wss_service(client, 1000) < 0){
276
+ error("Connection Error or Dropped");
277
+ return 1;
278
+ }
279
+
280
+ // mqtt_wss_service will return faster than in one second
281
+ // if there is enough work to do
282
+ time_t now = now_monotonic_sec();
283
+ if (last_periodic_query_wakeup < now) {
284
+ // wake up at least one Query Thread at least
285
+ // once per second
286
+ last_periodic_query_wakeup = now;
287
+ QUERY_THREAD_WAKEUP;
288
+ }
289
+ }
290
+ return 0;
291
+}
292
+
293
+inline static int aclk_popcorn_check_bump()
294
+{
295
+ ACLK_SHARED_STATE_LOCK;
296
+ if (unlikely(aclk_shared_state.agent_state == AGENT_INITIALIZING)) {
297
+ aclk_shared_state.last_popcorn_interrupt = now_realtime_sec();
298
+ ACLK_SHARED_STATE_UNLOCK;
299
+ return 1;
300
+ }
301
+ ACLK_SHARED_STATE_UNLOCK;
302
+ return 0;
303
+}
304
+
305
+static inline void queue_connect_payloads(void)
306
+{
307
+ aclk_query_t query = aclk_query_new(METADATA_INFO);
308
+ query->data.metadata_info.host = localhost;
309
+ query->data.metadata_info.initial_on_connect = 1;
310
+ aclk_queue_query(query);
311
+ query = aclk_query_new(METADATA_ALARMS);
312
+ query->data.metadata_alarms.initial_on_connect = 1;
313
+ aclk_queue_query(query);
314
+}
315
+
316
+static inline void mqtt_connected_actions(mqtt_wss_client client)
317
+{
318
+ // TODO global vars?
319
+ usec_t now = now_realtime_usec();
320
+ aclk_session_sec = now / USEC_PER_SEC;
321
+ aclk_session_us = now % USEC_PER_SEC;
322
+
323
+ mqtt_wss_subscribe(client, aclk_get_topic(ACLK_TOPICID_COMMAND), 1);
324
+
325
+ aclk_stats_upd_online(1);
326
+ aclk_connected = 1;
327
+ aclk_pubacks_per_conn = 0;
328
+ aclk_hello_msg(client);
329
+ ACLK_SHARED_STATE_LOCK;
330
+ if (aclk_shared_state.agent_state != AGENT_INITIALIZING) {
331
+ error("Sending `connect` payload immediatelly as popcorning was finished already.");
332
+ queue_connect_payloads();
333
+ }
334
+ ACLK_SHARED_STATE_UNLOCK;
335
+}
336
+
337
+/* Waits until agent is ready or needs to exit
338
+ * @param client instance of mqtt_wss_client
339
+ * @param query_threads pointer to aclk_query_threads
340
+ * structure where to store data about started query threads
341
+ * @return 0 - Popcorning Finished - Agent STABLE,
342
+ * !0 - netdata_exit
343
+ */
344
+static int wait_popcorning_finishes(mqtt_wss_client client, struct aclk_query_threads *query_threads)
345
+{
346
+ time_t elapsed;
347
+ int need_wait;
348
+ while (!netdata_exit) {
349
+ ACLK_SHARED_STATE_LOCK;
350
+ if (likely(aclk_shared_state.agent_state != AGENT_INITIALIZING)) {
351
+ ACLK_SHARED_STATE_UNLOCK;
352
+ return 0;
353
+ }
354
+ elapsed = now_realtime_sec() - aclk_shared_state.last_popcorn_interrupt;
355
+ if (elapsed >= ACLK_STABLE_TIMEOUT) {
356
+ aclk_shared_state.agent_state = AGENT_STABLE;
357
+ ACLK_SHARED_STATE_UNLOCK;
358
+ error("ACLK localhost popocorn finished");
359
+ if (unlikely(!query_threads->thread_list))
360
+ aclk_query_threads_start(query_threads, client);
361
+ queue_connect_payloads();
362
+ return 0;
363
+ }
364
+ ACLK_SHARED_STATE_UNLOCK;
365
+ need_wait = ACLK_STABLE_TIMEOUT - elapsed;
366
+ error("ACLK localhost popocorn wait %d seconds longer", need_wait);
367
+ sleep(need_wait);
368
+ }
369
+ return 1;
370
+}
371
+
372
+void aclk_graceful_disconnect(mqtt_wss_client client)
373
+{
374
+ error("Preparing to Gracefully Shutdown the ACLK");
375
+ aclk_queue_lock();
376
+ aclk_queue_flush();
377
+ aclk_shared_state.mqtt_shutdown_msg_id = aclk_send_app_layer_disconnect(client, "graceful");
378
+ time_t t = now_monotonic_sec();
379
+ while (!mqtt_wss_service(client, 100)) {
380
+ if (now_monotonic_sec() - t >= 2) {
381
+ error("Wasn't able to gracefully shutdown ACLK in time!");
382
+ break;
383
+ }
384
+ if (aclk_shared_state.mqtt_shutdown_msg_rcvd) {
385
+ error("MQTT App Layer `disconnect` message sent successfully");
386
+ break;
387
+ }
388
+ }
389
+ aclk_stats_upd_online(0);
390
+ aclk_connected = 0;
391
+
392
+ error("Attempting to Gracefully Shutdown MQTT/WSS connection");
393
+ mqtt_wss_disconnect(client, 1000);
394
+}
395
+
396
+/* Block till aclk_reconnect_delay is satisifed or netdata_exit is signalled
397
+ * @return 0 - Go ahead and connect (delay expired)
398
+ * 1 - netdata_exit
399
+ */
400
+#define NETDATA_EXIT_POLL_MS (MSEC_PER_SEC/4)
401
+static int aclk_block_till_recon_allowed() {
402
+ // Handle reconnect exponential backoff
403
+ // fnc aclk_reconnect_delay comes from ACLK Legacy @amoss
404
+ // but has been modifed slightly (more randomness)
405
+ unsigned long recon_delay = aclk_reconnect_delay(1);
406
+ info("Wait before attempting to reconnect in %.3f seconds\n", recon_delay / (float)MSEC_PER_SEC);
407
+ // we want to wake up from time to time to check netdata_exit
408
+ while (recon_delay)
409
+ {
410
+ if (netdata_exit)
411
+ return 1;
412
+ if (recon_delay > NETDATA_EXIT_POLL_MS) {
413
+ sleep_usec(NETDATA_EXIT_POLL_MS * USEC_PER_MS);
414
+ recon_delay -= NETDATA_EXIT_POLL_MS;
415
+ continue;
416
+ }
417
+ sleep_usec(recon_delay * USEC_PER_MS);
418
+ recon_delay = 0;
419
+ }
420
+ return 0;
421
+}
422
+
423
+#define HTTP_PROXY_PREFIX "http://"
424
+static void set_proxy(struct mqtt_wss_proxy *out)
425
+{
426
+ ACLK_PROXY_TYPE pt;
427
+ const char *ptr = aclk_get_proxy(&pt);
428
+ char *tmp;
429
+ char *host;
430
+ if (pt != PROXY_TYPE_HTTP)
431
+ return;
432
+
433
+ out->port = 0;
434
+
435
+ if (!strncmp(ptr, HTTP_PROXY_PREFIX, strlen(HTTP_PROXY_PREFIX)))
436
+ ptr += strlen(HTTP_PROXY_PREFIX);
437
+
438
+ if ((tmp = strchr(ptr, '@')))
439
+ ptr = tmp;
440
+
441
+ if ((tmp = strchr(ptr, '/'))) {
442
+ host = mallocz((tmp - ptr) + 1);
443
+ memcpy(host, ptr, (tmp - ptr));
444
+ host[tmp - ptr] = 0;
445
+ } else
446
+ host = strdupz(ptr);
447
+
448
+ if ((tmp = strchr(host, ':'))) {
449
+ *tmp = 0;
450
+ tmp++;
451
+ out->port = atoi(tmp);
452
+ }
453
+
454
+ if (out->port <= 0 || out->port > 65535)
455
+ out->port = 8080;
456
+
457
+ out->host = host;
458
+
459
+ out->type = MQTT_WSS_PROXY_HTTP;
460
+}
461
+
462
+/* Attempts to make a connection to MQTT broker over WSS
463
+ * @param client instance of mqtt_wss_client
464
+ * @return 0 - Successfull Connection,
465
+ * <0 - Irrecoverable Error -> Kill ACLK,
466
+ * >0 - netdata_exit
467
+ */
468
+#define CLOUD_BASE_URL_READ_RETRY 30
469
+#ifdef ACLK_SSL_ALLOW_SELF_SIGNED
470
+#define ACLK_SSL_FLAGS MQTT_WSS_SSL_ALLOW_SELF_SIGNED
471
+#else
472
+#define ACLK_SSL_FLAGS MQTT_WSS_SSL_CERT_CHECK_FULL
473
+#endif
474
+static int aclk_attempt_to_connect(mqtt_wss_client client)
475
+{
476
+ char *aclk_hostname = NULL;
477
+ int aclk_port;
478
+
479
+#ifndef ACLK_DISABLE_CHALLENGE
480
+ char *mqtt_otp_user = NULL;
481
+ char *mqtt_otp_pass = NULL;
482
+#endif
483
+
484
+ json_object *lwt;
485
+
486
+ while (!netdata_exit) {
487
+ char *cloud_base_url = appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", NULL);
488
+ if (cloud_base_url == NULL) {
489
+ error("Do not move the cloud base url out of post_conf_load!!");
490
+ return -1;
491
+ }
492
+
493
+ if (aclk_block_till_recon_allowed())
494
+ return 1;
495
+
496
+ info("Attempting connection now");
497
+ if (aclk_decode_base_url(cloud_base_url, &aclk_hostname, &aclk_port)) {
498
+ error("ACLK base URL configuration key could not be parsed. Will retry in %d seconds.", CLOUD_BASE_URL_READ_RETRY);
499
+ sleep(CLOUD_BASE_URL_READ_RETRY);
500
+ continue;
501
+ }
502
+
503
+ struct mqtt_wss_proxy proxy_conf;
504
+ proxy_conf.type = MQTT_WSS_DIRECT;
505
+ set_proxy(&proxy_conf);
506
+
507
+ struct mqtt_connect_params mqtt_conn_params = {
508
+ .clientid = "anon",
509
+ .username = "anon",
510
+ .password = "anon",
511
+ .will_topic = aclk_get_topic(ACLK_TOPICID_METADATA),
512
+ .will_msg = NULL,
513
+ .will_flags = MQTT_WSS_PUB_QOS2,
514
+ .keep_alive = 60
515
+ };
516
+#ifndef ACLK_DISABLE_CHALLENGE
517
+ aclk_get_mqtt_otp(aclk_private_key, aclk_hostname, aclk_port, &mqtt_otp_user, &mqtt_otp_pass);
518
+ mqtt_conn_params.clientid = mqtt_otp_user;
519
+ mqtt_conn_params.username = mqtt_otp_user;
520
+ mqtt_conn_params.password = mqtt_otp_pass;
521
+#endif
522
+
523
+ lwt = aclk_generate_disconnect(NULL);
524
+ mqtt_conn_params.will_msg = json_object_to_json_string_ext(lwt, JSON_C_TO_STRING_PLAIN);
525
+
526
+ mqtt_conn_params.will_msg_len = strlen(mqtt_conn_params.will_msg);
527
+ if (!mqtt_wss_connect(client, aclk_hostname, aclk_port, &mqtt_conn_params, ACLK_SSL_FLAGS, &proxy_conf)) {
528
+ json_object_put(lwt);
529
+ freez(aclk_hostname);
530
+ aclk_hostname = NULL;
531
+ info("MQTTWSS connection succeeded");
532
+ mqtt_connected_actions(client);
533
+ return 0;
534
+ }
535
+
536
+ freez(aclk_hostname);
537
+ aclk_hostname = NULL;
538
+ json_object_put(lwt);
539
+ error("Connect failed\n");
540
+ }
541
+
542
+ return 1;
543
+}
544
+
545
+/**
546
+ * Main agent cloud link thread
547
+ *
548
+ * This thread will simply call the main event loop that handles
549
+ * pending requests - both inbound and outbound
550
+ *
551
+ * @param ptr is a pointer to the netdata_static_thread structure.
552
+ *
553
+ * @return It always returns NULL
554
+ */
555
+void *aclk_main(void *ptr)
556
+{
557
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
558
+
559
+ struct aclk_stats_thread *stats_thread = NULL;
560
+
561
+ struct aclk_query_threads query_threads;
562
+ query_threads.thread_list = NULL;
563
+
564
+ ACLK_PROXY_TYPE proxy_type;
565
+ aclk_get_proxy(&proxy_type);
566
+ if (proxy_type == PROXY_TYPE_SOCKS5) {
567
+ error("SOCKS5 proxy is not supported by ACLK-NG yet.");
568
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
569
+ return NULL;
570
+ }
571
+
572
+ // This thread is unusual in that it cannot be cancelled by cancel_main_threads()
573
+ // as it must notify the far end that it shutdown gracefully and avoid the LWT.
574
+ netdata_thread_disable_cancelability();
575
+
576
+#if defined( DISABLE_CLOUD ) || !defined( ENABLE_ACLK )
577
+ info("Killing ACLK thread -> cloud functionality has been disabled");
578
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
579
+ return NULL;
580
+#endif
581
+ aclk_popcorn_check_bump(); // start localhost popcorn timer
582
+ query_threads.count = read_query_thread_count();
583
+
584
+ if (wait_till_cloud_enabled())
585
+ goto exit;
586
+
587
+ if (wait_till_agent_claim_ready())
588
+ goto exit;
589
+
590
+ if (!(mqttwss_client = mqtt_wss_new("mqtt_wss", aclk_mqtt_wss_log_cb, msg_callback, puback_callback))) {
591
+ error("Couldn't initialize MQTT_WSS network library");
592
+ goto exit;
593
+ }
594
+
595
+ aclk_stats_enabled = config_get_boolean(CONFIG_SECTION_CLOUD, "statistics", CONFIG_BOOLEAN_YES);
596
+ if (aclk_stats_enabled) {
597
+ stats_thread = callocz(1, sizeof(struct aclk_stats_thread));
598
+ stats_thread->thread = mallocz(sizeof(netdata_thread_t));
599
+ stats_thread->query_thread_count = query_threads.count;
600
+ netdata_thread_create(
601
+ stats_thread->thread, ACLK_STATS_THREAD_NAME, NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread,
602
+ stats_thread);
603
+ }
604
+
605
+ // Keep reconnecting and talking until our time has come
606
+ // and the Grim Reaper (netdata_exit) calls
607
+ do {
608
+ if (aclk_attempt_to_connect(mqttwss_client))
609
+ goto exit_full;
610
+
611
+ // warning this assumes the popcorning is relative short (3s)
612
+ // if that changes call mqtt_wss_service from within
613
+ // to keep OpenSSL, WSS and MQTT connection alive
614
+ if (wait_popcorning_finishes(mqttwss_client, &query_threads))
615
+ goto exit_full;
616
+
617
+ if (!handle_connection(mqttwss_client)) {
618
+ aclk_stats_upd_online(0);
619
+ aclk_connected = 0;
620
+ }
621
+ } while (!netdata_exit);
622
+
623
+ aclk_graceful_disconnect(mqttwss_client);
624
+
625
+exit_full:
626
+// Tear Down
627
+ QUERY_THREAD_WAKEUP_ALL;
628
+
629
+ aclk_query_threads_cleanup(&query_threads);
630
+
631
+ if (aclk_stats_enabled) {
632
+ netdata_thread_join(*stats_thread->thread, NULL);
633
+ aclk_stats_thread_cleanup();
634
+ freez(stats_thread->thread);
635
+ freez(stats_thread);
636
+ }
637
+ free_topic_cache();
638
+ mqtt_wss_destroy(mqttwss_client);
639
+exit:
640
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
641
+ return NULL;
642
+}
643
+
644
+// TODO this is taken over as workaround from old ACLK
645
+// fix this in both old and new ACLK
646
+extern void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host);
647
+
648
+void aclk_alarm_reload(void)
649
+{
650
+ ACLK_SHARED_STATE_LOCK;
651
+ if (unlikely(aclk_shared_state.agent_state == AGENT_INITIALIZING)) {
652
+ ACLK_SHARED_STATE_UNLOCK;
653
+ return;
654
+ }
655
+ ACLK_SHARED_STATE_UNLOCK;
656
+
657
+ aclk_queue_query(aclk_query_new(METADATA_ALARMS));
658
+}
659
+
660
+int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae)
661
+{
662
+ BUFFER *local_buffer;
663
+ json_object *msg;
664
+
665
+ if (host != localhost)
666
+ return 0;
667
+
668
+ ACLK_SHARED_STATE_LOCK;
669
+ if (unlikely(aclk_shared_state.agent_state == AGENT_INITIALIZING)) {
670
+ ACLK_SHARED_STATE_UNLOCK;
671
+ return 0;
672
+ }
673
+ ACLK_SHARED_STATE_UNLOCK;
674
+
675
+ local_buffer = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
676
+
677
+ netdata_rwlock_rdlock(&host->health_log.alarm_log_rwlock);
678
+ health_alarm_entry2json_nolock(local_buffer, ae, host);
679
+ netdata_rwlock_unlock(&host->health_log.alarm_log_rwlock);
680
+
681
+ msg = json_tokener_parse(local_buffer->buffer);
682
+
683
+ struct aclk_query *query = aclk_query_new(ALARM_STATE_UPDATE);
684
+ query->data.alarm_update = msg;
685
+ aclk_queue_query(query);
686
+
687
+ buffer_free(local_buffer);
688
+ return 0;
689
+}
690
+
691
+int aclk_update_chart(RRDHOST *host, char *chart_name, int create)
692
+{
693
+ struct aclk_query *query;
694
+
695
+ if (aclk_popcorn_check_bump())
696
+ return 0;
697
+
698
+ query = aclk_query_new(create ? CHART_NEW : CHART_DEL);
699
+ if(create) {
700
+ query->data.chart_add_del.host = host;
701
+ query->data.chart_add_del.chart_name = strdupz(chart_name);
702
+ } else {
703
+ query->data.metadata_info.host = host;
704
+ query->data.metadata_info.initial_on_connect = 0;
705
+ }
706
+
707
+ aclk_queue_query(query);
708
+ return 0;
709
+}
710
+
711
+/*
712
+ * Add a new collector to the list
713
+ * If it exists, update the chart count
714
+ */
715
+void aclk_add_collector(RRDHOST *host, const char *plugin_name, const char *module_name)
716
+{
717
+ struct aclk_query *query;
718
+ struct _collector *tmp_collector;
719
+ if (unlikely(!netdata_ready)) {
720
+ return;
721
+ }
722
+
723
+ COLLECTOR_LOCK;
724
+
725
+ tmp_collector = _add_collector(host->machine_guid, plugin_name, module_name);
726
+
727
+ if (unlikely(tmp_collector->count != 1)) {
728
+ COLLECTOR_UNLOCK;
729
+ return;
730
+ }
731
+
732
+ COLLECTOR_UNLOCK;
733
+
734
+ if (aclk_popcorn_check_bump())
735
+ return;
736
+
737
+ if (host != localhost)
738
+ return;
739
+
740
+ query = aclk_query_new(METADATA_INFO);
741
+ query->data.metadata_info.host = localhost; //TODO
742
+ query->data.metadata_info.initial_on_connect = 0;
743
+ aclk_queue_query(query);
744
+
745
+ query = aclk_query_new(METADATA_ALARMS);
746
+ query->data.metadata_alarms.initial_on_connect = 0;
747
+ aclk_queue_query(query);
748
+}
749
+
750
+/*
751
+ * Delete a collector from the list
752
+ * If the chart count reaches zero the collector will be removed
753
+ * from the list by calling del_collector.
754
+ *
755
+ * This function will release the memory used and schedule
756
+ * a cloud update
757
+ */
758
+void aclk_del_collector(RRDHOST *host, const char *plugin_name, const char *module_name)
759
+{
760
+ struct aclk_query *query;
761
+ struct _collector *tmp_collector;
762
+ if (unlikely(!netdata_ready)) {
763
+ return;
764
+ }
765
+
766
+ COLLECTOR_LOCK;
767
+
768
+ tmp_collector = _del_collector(host->machine_guid, plugin_name, module_name);
769
+
770
+ if (unlikely(!tmp_collector || tmp_collector->count)) {
771
+ COLLECTOR_UNLOCK;
772
+ return;
773
+ }
774
+
775
+ debug(
776
+ D_ACLK, "DEL COLLECTOR [%s:%s] -- charts %u", plugin_name ? plugin_name : "*", module_name ? module_name : "*",
777
+ tmp_collector->count);
778
+
779
+ COLLECTOR_UNLOCK;
780
+
781
+ _free_collector(tmp_collector);
782
+
783
+ if (aclk_popcorn_check_bump())
784
+ return;
785
+
786
+ if (host != localhost)
787
+ return;
788
+
789
+ query = aclk_query_new(METADATA_INFO);
790
+ query->data.metadata_info.host = localhost; //TODO
791
+ query->data.metadata_info.initial_on_connect = 0;
792
+ aclk_queue_query(query);
793
+
794
+ query = aclk_query_new(METADATA_ALARMS);
795
+ query->data.metadata_alarms.initial_on_connect = 0;
796
+ aclk_queue_query(query);
797
+}
798
+
799
+struct label *add_aclk_host_labels(struct label *label) {
800
+#ifdef ENABLE_ACLK
801
+ ACLK_PROXY_TYPE aclk_proxy;
802
+ char *proxy_str;
803
+ aclk_get_proxy(&aclk_proxy);
804
+
805
+ switch(aclk_proxy) {
806
+ case PROXY_TYPE_SOCKS5:
807
+ proxy_str = "SOCKS5";
808
+ break;
809
+ case PROXY_TYPE_HTTP:
810
+ proxy_str = "HTTP";
811
+ break;
812
+ default:
813
+ proxy_str = "none";
814
+ break;
815
+ }
816
+ return add_label_to_list(label, "_aclk_proxy", proxy_str, LABEL_SOURCE_AUTO);
817
+#else
818
+ return label;
819
+#endif
820
+}
aclk/aclk.h
new
+100
@@ -0,0 +1,100 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+#ifndef ACLK_H
3
+#define ACLK_H
4
+
5
+typedef struct aclk_rrdhost_state {
6
+ char *claimed_id; // Claimed ID if host has one otherwise NULL
7
+} aclk_rrdhost_state;
8
+
9
+#include "../daemon/common.h"
10
+#include "aclk_util.h"
11
+
12
+// minimum and maximum supported version of ACLK
13
+// in this version of agent
14
+#define ACLK_VERSION_MIN 2
15
+#define ACLK_VERSION_MAX 2
16
+
17
+// Version negotiation messages have they own versioning
18
+// this is also used for LWT message as we set that up
19
+// before version negotiation
20
+#define ACLK_VERSION_NEG_VERSION 1
21
+
22
+// Maximum time to wait for version negotiation before aborting
23
+// and defaulting to oldest supported version
24
+#define VERSION_NEG_TIMEOUT 3
25
+
26
+#if ACLK_VERSION_MIN > ACLK_VERSION_MAX
27
+#error "ACLK_VERSION_MAX must be >= than ACLK_VERSION_MIN"
28
+#endif
29
+
30
+// Define ACLK Feature Version Boundaries Here
31
+#define ACLK_V_COMPRESSION 2
32
+
33
+// How many MQTT PUBACKs we need to get to consider connection
34
+// stable for the purposes of TBEB (truncated binary exponential backoff)
35
+#define ACLK_PUBACKS_CONN_STABLE 3
36
+
37
+// TODO get rid of this shit
38
+extern int aclk_disable_runtime;
39
+extern int aclk_disable_single_updates;
40
+extern int aclk_kill_link;
41
+extern int aclk_connected;
42
+
43
+extern usec_t aclk_session_us;
44
+extern time_t aclk_session_sec;
45
+
46
+void *aclk_main(void *ptr);
47
+void aclk_single_update_disable();
48
+void aclk_single_update_enable();
49
+
50
+#define NETDATA_ACLK_HOOK \
51
+ { .name = "ACLK_Main", \
52
+ .config_section = NULL, \
53
+ .config_name = NULL, \
54
+ .enabled = 1, \
55
+ .thread = NULL, \
56
+ .init_routine = NULL, \
57
+ .start_routine = aclk_main },
58
+
59
+extern netdata_mutex_t aclk_shared_state_mutex;
60
+#define ACLK_SHARED_STATE_LOCK netdata_mutex_lock(&aclk_shared_state_mutex)
61
+#define ACLK_SHARED_STATE_UNLOCK netdata_mutex_unlock(&aclk_shared_state_mutex)
62
+
63
+typedef enum aclk_agent_state {
64
+ AGENT_INITIALIZING,
65
+ AGENT_STABLE
66
+} ACLK_AGENT_STATE;
67
+extern struct aclk_shared_state {
68
+ ACLK_AGENT_STATE agent_state;
69
+ time_t last_popcorn_interrupt;
70
+
71
+ // read only while ACLK connected
72
+ // protect by lock otherwise
73
+ int version_neg;
74
+ usec_t version_neg_wait_till;
75
+
76
+ // To wait for `disconnect` message PUBACK
77
+ // when shuting down
78
+ // at the same time if > 0 we know link is
79
+ // shutting down
80
+ int mqtt_shutdown_msg_id;
81
+ int mqtt_shutdown_msg_rcvd;
82
+} aclk_shared_state;
83
+
84
+void aclk_alarm_reload(void);
85
+int aclk_update_alarm(RRDHOST *host, ALARM_ENTRY *ae);
86
+
87
+// TODO this is for bacward compatibility with ACLK legacy
88
+#define ACLK_CMD_CHART 1
89
+#define ACLK_CMD_CHARTDEL 0
90
+/* Informs ACLK about created/deleted chart
91
+ * @param create 0 - if chart was deleted, other if chart created
92
+ */
93
+int aclk_update_chart(RRDHOST *host, char *chart_name, int create);
94
+
95
+void aclk_add_collector(RRDHOST *host, const char *plugin_name, const char *module_name);
96
+void aclk_del_collector(RRDHOST *host, const char *plugin_name, const char *module_name);
97
+
98
+struct label *add_aclk_host_labels(struct label *label);
99
+
100
+#endif /* ACLK_H */
aclk/aclk_collector_list.c
new
+193
@@ -0,0 +1,193 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+// This is copied from Legacy ACLK, Original Autor: amoss
3
+
4
+// TODO unmess this
5
+
6
+#include "aclk_collector_list.h"
7
+
8
+netdata_mutex_t collector_mutex = NETDATA_MUTEX_INITIALIZER;
9
+
10
+struct _collector *collector_list = NULL;
11
+
12
+/*
13
+ * Free a collector structure
14
+ */
15
+void _free_collector(struct _collector *collector)
16
+{
17
+ if (likely(collector->plugin_name))
18
+ freez(collector->plugin_name);
19
+
20
+ if (likely(collector->module_name))
21
+ freez(collector->module_name);
22
+
23
+ if (likely(collector->hostname))
24
+ freez(collector->hostname);
25
+
26
+ freez(collector);
27
+}
28
+
29
+/*
30
+ * This will report the collector list
31
+ *
32
+ */
33
+#ifdef ACLK_DEBUG
34
+static void _dump_collector_list()
35
+{
36
+ struct _collector *tmp_collector;
37
+
38
+ COLLECTOR_LOCK;
39
+
40
+ info("DUMPING ALL COLLECTORS");
41
+
42
+ if (unlikely(!collector_list || !collector_list->next)) {
43
+ COLLECTOR_UNLOCK;
44
+ info("DUMPING ALL COLLECTORS -- nothing found");
45
+ return;
46
+ }
47
+
48
+ // Note that the first entry is "dummy"
49
+ tmp_collector = collector_list->next;
50
+
51
+ while (tmp_collector) {
52
+ info(
53
+ "COLLECTOR %s : [%s:%s] count = %u", tmp_collector->hostname,
54
+ tmp_collector->plugin_name ? tmp_collector->plugin_name : "",
55
+ tmp_collector->module_name ? tmp_collector->module_name : "", tmp_collector->count);
56
+
57
+ tmp_collector = tmp_collector->next;
58
+ }
59
+ info("DUMPING ALL COLLECTORS DONE");
60
+ COLLECTOR_UNLOCK;
61
+}
62
+#endif
63
+
64
+/*
65
+ * This will cleanup the collector list
66
+ *
67
+ */
68
+void _reset_collector_list()
69
+{
70
+ struct _collector *tmp_collector, *next_collector;
71
+
72
+ COLLECTOR_LOCK;
73
+
74
+ if (unlikely(!collector_list || !collector_list->next)) {
75
+ COLLECTOR_UNLOCK;
76
+ return;
77
+ }
78
+
79
+ // Note that the first entry is "dummy"
80
+ tmp_collector = collector_list->next;
81
+ collector_list->count = 0;
82
+ collector_list->next = NULL;
83
+
84
+ // We broke the link; we can unlock
85
+ COLLECTOR_UNLOCK;
86
+
87
+ while (tmp_collector) {
88
+ next_collector = tmp_collector->next;
89
+ _free_collector(tmp_collector);
90
+ tmp_collector = next_collector;
91
+ }
92
+}
93
+
94
+/*
95
+ * Find a collector (if it exists)
96
+ * Must lock before calling this
97
+ * If last_collector is not null, it will return the previous collector in the linked
98
+ * list (used in collector delete)
99
+ */
100
+static struct _collector *_find_collector(
101
+ const char *hostname, const char *plugin_name, const char *module_name, struct _collector **last_collector)
102
+{
103
+ struct _collector *tmp_collector, *prev_collector;
104
+ uint32_t plugin_hash;
105
+ uint32_t module_hash;
106
+ uint32_t hostname_hash;
107
+
108
+ if (unlikely(!collector_list)) {
109
+ collector_list = callocz(1, sizeof(struct _collector));
110
+ return NULL;
111
+ }
112
+
113
+ if (unlikely(!collector_list->next))
114
+ return NULL;
115
+
116
+ plugin_hash = plugin_name ? simple_hash(plugin_name) : 1;
117
+ module_hash = module_name ? simple_hash(module_name) : 1;
118
+ hostname_hash = simple_hash(hostname);
119
+
120
+ // Note that the first entry is "dummy"
121
+ tmp_collector = collector_list->next;
122
+ prev_collector = collector_list;
123
+ while (tmp_collector) {
124
+ if (plugin_hash == tmp_collector->plugin_hash && module_hash == tmp_collector->module_hash &&
125
+ hostname_hash == tmp_collector->hostname_hash && (!strcmp(hostname, tmp_collector->hostname)) &&
126
+ (!plugin_name || !tmp_collector->plugin_name || !strcmp(plugin_name, tmp_collector->plugin_name)) &&
127
+ (!module_name || !tmp_collector->module_name || !strcmp(module_name, tmp_collector->module_name))) {
128
+ if (unlikely(last_collector))
129
+ *last_collector = prev_collector;
130
+
131
+ return tmp_collector;
132
+ }
133
+
134
+ prev_collector = tmp_collector;
135
+ tmp_collector = tmp_collector->next;
136
+ }
137
+
138
+ return tmp_collector;
139
+}
140
+
141
+/*
142
+ * Called to delete a collector
143
+ * It will reduce the count (chart_count) and will remove it
144
+ * from the linked list if the count reaches zero
145
+ * The structure will be returned to the caller to free
146
+ * the resources
147
+ *
148
+ */
149
+struct _collector *_del_collector(const char *hostname, const char *plugin_name, const char *module_name)
150
+{
151
+ struct _collector *tmp_collector, *prev_collector = NULL;
152
+
153
+ tmp_collector = _find_collector(hostname, plugin_name, module_name, &prev_collector);
154
+
155
+ if (likely(tmp_collector)) {
156
+ --tmp_collector->count;
157
+ if (unlikely(!tmp_collector->count))
158
+ prev_collector->next = tmp_collector->next;
159
+ }
160
+ return tmp_collector;
161
+}
162
+
163
+/*
164
+ * Add a new collector (plugin / module) to the list
165
+ * If it already exists just update the chart count
166
+ *
167
+ * Lock before calling
168
+ */
169
+struct _collector *_add_collector(const char *hostname, const char *plugin_name, const char *module_name)
170
+{
171
+ struct _collector *tmp_collector;
172
+
173
+ tmp_collector = _find_collector(hostname, plugin_name, module_name, NULL);
174
+
175
+ if (unlikely(!tmp_collector)) {
176
+ tmp_collector = callocz(1, sizeof(struct _collector));
177
+ tmp_collector->hostname_hash = simple_hash(hostname);
178
+ tmp_collector->plugin_hash = plugin_name ? simple_hash(plugin_name) : 1;
179
+ tmp_collector->module_hash = module_name ? simple_hash(module_name) : 1;
180
+
181
+ tmp_collector->hostname = strdupz(hostname);
182
+ tmp_collector->plugin_name = plugin_name ? strdupz(plugin_name) : NULL;
183
+ tmp_collector->module_name = module_name ? strdupz(module_name) : NULL;
184
+
185
+ tmp_collector->next = collector_list->next;
186
+ collector_list->next = tmp_collector;
187
+ }
188
+ tmp_collector->count++;
189
+ debug(
190
+ D_ACLK, "ADD COLLECTOR %s [%s:%s] -- chart %u", hostname, plugin_name ? plugin_name : "*",
191
+ module_name ? module_name : "*", tmp_collector->count);
192
+ return tmp_collector;
193
+}
aclk/aclk_collector_list.h
new
+39
@@ -0,0 +1,39 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+// This is copied from Legacy ACLK, Original Autor: amoss
3
+
4
+// TODO unmess this
5
+
6
+#ifndef ACLK_COLLECTOR_LIST_H
7
+#define ACLK_COLLECTOR_LIST_H
8
+
9
+#include "libnetdata/libnetdata.h"
10
+
11
+extern netdata_mutex_t collector_mutex;
12
+
13
+#define COLLECTOR_LOCK netdata_mutex_lock(&collector_mutex)
14
+#define COLLECTOR_UNLOCK netdata_mutex_unlock(&collector_mutex)
15
+
16
+/*
17
+ * Maintain a list of collectors and chart count
18
+ * If all the charts of a collector are deleted
19
+ * then a new metadata dataset must be send to the cloud
20
+ *
21
+ */
22
+struct _collector {
23
+ time_t created;
24
+ uint32_t count; //chart count
25
+ uint32_t hostname_hash;
26
+ uint32_t plugin_hash;
27
+ uint32_t module_hash;
28
+ char *hostname;
29
+ char *plugin_name;
30
+ char *module_name;
31
+ struct _collector *next;
32
+};
33
+
34
+struct _collector *_add_collector(const char *hostname, const char *plugin_name, const char *module_name);
35
+struct _collector *_del_collector(const char *hostname, const char *plugin_name, const char *module_name);
36
+void _reset_collector_list();
37
+void _free_collector(struct _collector *collector);
38
+
39
+#endif /* ACLK_COLLECTOR_LIST_H */
aclk/aclk_otp.c
new
+509
@@ -0,0 +1,509 @@
1
+
2
+// SPDX-License-Identifier: GPL-3.0-or-later
3
+
4
+#include "aclk_otp.h"
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+#include "legacy/aclk_lws_https_client.h"
9
+
10
+#include "../daemon/common.h"
11
+
12
+#include "../mqtt_websockets/c-rbuf/include/ringbuffer.h"
13
+
14
+struct dictionary_singleton {
15
+ char *key;
16
+ char *result;
17
+};
18
+
19
+static int json_extract_singleton(JSON_ENTRY *e)
20
+{
21
+ struct dictionary_singleton *data = e->callback_data;
22
+
23
+ switch (e->type) {
24
+ case JSON_OBJECT:
25
+ case JSON_ARRAY:
26
+ break;
27
+ case JSON_STRING:
28
+ if (!strcmp(e->name, data->key)) {
29
+ data->result = strdupz(e->data.string);
30
+ break;
31
+ }
32
+ break;
33
+ case JSON_NUMBER:
34
+ case JSON_BOOLEAN:
35
+ case JSON_NULL:
36
+ break;
37
+ }
38
+ return 0;
39
+}
40
+
41
+// Base-64 decoder.
42
+// Note: This is non-validating, invalid input will be decoded without an error.
43
+// Challenges are packed into json strings so we don't skip newlines.
44
+// Size errors (i.e. invalid input size or insufficient output space) are caught.
45
+static size_t base64_decode(unsigned char *input, size_t input_size, unsigned char *output, size_t output_size)
46
+{
47
+ static char lookup[256];
48
+ static int first_time=1;
49
+ if (first_time)
50
+ {
51
+ first_time = 0;
52
+ for(int i=0; i<256; i++)
53
+ lookup[i] = -1;
54
+ for(int i='A'; i<='Z'; i++)
55
+ lookup[i] = i-'A';
56
+ for(int i='a'; i<='z'; i++)
57
+ lookup[i] = i-'a' + 26;
58
+ for(int i='0'; i<='9'; i++)
59
+ lookup[i] = i-'0' + 52;
60
+ lookup['+'] = 62;
61
+ lookup['/'] = 63;
62
+ }
63
+ if ((input_size & 3) != 0)
64
+ {
65
+ error("Can't decode base-64 input length %zu", input_size);
66
+ return 0;
67
+ }
68
+ size_t unpadded_size = (input_size/4) * 3;
69
+ if ( unpadded_size > output_size )
70
+ {
71
+ error("Output buffer size %zu is too small to decode %zu into", output_size, input_size);
72
+ return 0;
73
+ }
74
+ // Don't check padding within full quantums
75
+ for (size_t i = 0 ; i < input_size-4 ; i+=4 )
76
+ {
77
+ uint32_t value = (lookup[input[0]] << 18) + (lookup[input[1]] << 12) + (lookup[input[2]] << 6) + lookup[input[3]];
78
+ output[0] = value >> 16;
79
+ output[1] = value >> 8;
80
+ output[2] = value;
81
+ //error("Decoded %c %c %c %c -> %02x %02x %02x", input[0], input[1], input[2], input[3], output[0], output[1], output[2]);
82
+ output += 3;
83
+ input += 4;
84
+ }
85
+ // Handle padding only in last quantum
86
+ if (input[2] == '=') {
87
+ uint32_t value = (lookup[input[0]] << 6) + lookup[input[1]];
88
+ output[0] = value >> 4;
89
+ //error("Decoded %c %c %c %c -> %02x", input[0], input[1], input[2], input[3], output[0]);
90
+ return unpadded_size-2;
91
+ }
92
+ else if (input[3] == '=') {
93
+ uint32_t value = (lookup[input[0]] << 12) + (lookup[input[1]] << 6) + lookup[input[2]];
94
+ output[0] = value >> 10;
95
+ output[1] = value >> 2;
96
+ //error("Decoded %c %c %c %c -> %02x %02x", input[0], input[1], input[2], input[3], output[0], output[1]);
97
+ return unpadded_size-1;
98
+ }
99
+ else
100
+ {
101
+ uint32_t value = (input[0] << 18) + (input[1] << 12) + (input[2]<<6) + input[3];
102
+ output[0] = value >> 16;
103
+ output[1] = value >> 8;
104
+ output[2] = value;
105
+ //error("Decoded %c %c %c %c -> %02x %02x %02x", input[0], input[1], input[2], input[3], output[0], output[1], output[2]);
106
+ return unpadded_size;
107
+ }
108
+}
109
+
110
+static size_t base64_encode(unsigned char *input, size_t input_size, char *output, size_t output_size)
111
+{
112
+ uint32_t value;
113
+ static char lookup[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
114
+ "abcdefghijklmnopqrstuvwxyz"
115
+ "0123456789+/";
116
+ if ((input_size/3+1)*4 >= output_size)
117
+ {
118
+ error("Output buffer for encoding size=%zu is not large enough for %zu-bytes input", output_size, input_size);
119
+ return 0;
120
+ }
121
+ size_t count = 0;
122
+ while (input_size>3)
123
+ {
124
+ value = ((input[0] << 16) + (input[1] << 8) + input[2]) & 0xffffff;
125
+ output[0] = lookup[value >> 18];
126
+ output[1] = lookup[(value >> 12) & 0x3f];
127
+ output[2] = lookup[(value >> 6) & 0x3f];
128
+ output[3] = lookup[value & 0x3f];
129
+ //error("Base-64 encode (%04x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]);
130
+ output += 4;
131
+ input += 3;
132
+ input_size -= 3;
133
+ count += 4;
134
+ }
135
+ switch (input_size)
136
+ {
137
+ case 2:
138
+ value = (input[0] << 10) + (input[1] << 2);
139
+ output[0] = lookup[(value >> 12) & 0x3f];
140
+ output[1] = lookup[(value >> 6) & 0x3f];
141
+ output[2] = lookup[value & 0x3f];
142
+ output[3] = '=';
143
+ //error("Base-64 encode (%06x) -> %c %c %c %c\n", (value>>2)&0xffff, output[0], output[1], output[2], output[3]);
144
+ count += 4;
145
+ break;
146
+ case 1:
147
+ value = input[0] << 4;
148
+ output[0] = lookup[(value >> 6) & 0x3f];
149
+ output[1] = lookup[value & 0x3f];
150
+ output[2] = '=';
151
+ output[3] = '=';
152
+ //error("Base-64 encode (%06x) -> %c %c %c %c\n", value, output[0], output[1], output[2], output[3]);
153
+ count += 4;
154
+ break;
155
+ case 0:
156
+ break;
157
+ }
158
+ return count;
159
+}
160
+
161
+static int private_decrypt(RSA *p_key, unsigned char * enc_data, int data_len, unsigned char *decrypted)
162
+{
163
+ int result = RSA_private_decrypt( data_len, enc_data, decrypted, p_key, RSA_PKCS1_OAEP_PADDING);
164
+ if (result == -1) {
165
+ char err[512];
166
+ ERR_error_string_n(ERR_get_error(), err, sizeof(err));
167
+ error("Decryption of the challenge failed: %s", err);
168
+ }
169
+ return result;
170
+}
171
+
172
+typedef enum http_req_type {
173
+ HTTP_REQ_GET,
174
+ HTTP_REQ_POST
175
+} http_req_type;
176
+
177
+enum http_parse_state {
178
+ HTTP_PARSE_INITIAL = 0,
179
+ HTTP_PARSE_HEADERS,
180
+ HTTP_PARSE_CONTENT
181
+};
182
+
183
+typedef struct {
184
+ enum http_parse_state state;
185
+ int content_length;
186
+ int http_code;
187
+} http_parse_ctx;
188
+
189
+#define HTTP_PARSE_CTX_INITIALIZER { .state = HTTP_PARSE_INITIAL, .content_length = -1, .http_code = 0 }
190
+
191
+#define NEED_MORE_DATA 0
192
+#define PARSE_SUCCESS 1
193
+#define PARSE_ERROR -1
194
+#define HTTP_LINE_TERM "\x0D\x0A"
195
+#define RESP_PROTO "HTTP/1.1 "
196
+#define HTTP_KEYVAL_SEPARATOR ": "
197
+#define HTTP_HDR_BUFFER_SIZE 256
198
+#define PORT_STR_MAX_BYTES 7
199
+
200
+static void process_http_hdr(http_parse_ctx *parse_ctx, const char *key, const char *val)
201
+{
202
+ // currently we care only about content-length
203
+ // but in future the way this is written
204
+ // it can be extended
205
+ if (!strcmp("content-length", key)) {
206
+ parse_ctx->content_length = atoi(val);
207
+ }
208
+}
209
+
210
+static int parse_http_hdr(rbuf_t buf, http_parse_ctx *parse_ctx)
211
+{
212
+ int idx, idx_end;
213
+ char buf_key[HTTP_HDR_BUFFER_SIZE];
214
+ char buf_val[HTTP_HDR_BUFFER_SIZE];
215
+ char *ptr = buf_key;
216
+ if (!rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx_end)) {
217
+ error("CRLF expected");
218
+ return 1;
219
+ }
220
+
221
+ char *separator = rbuf_find_bytes(buf, HTTP_KEYVAL_SEPARATOR, strlen(HTTP_KEYVAL_SEPARATOR), &idx);
222
+ if (!separator) {
223
+ error("Missing Key/Value separator");
224
+ return 1;
225
+ }
226
+ if (idx >= HTTP_HDR_BUFFER_SIZE) {
227
+ error("Key name is too long");
228
+ return 1;
229
+ }
230
+
231
+ rbuf_pop(buf, buf_key, idx);
232
+ buf_key[idx] = 0;
233
+
234
+ rbuf_bump_tail(buf, strlen(HTTP_KEYVAL_SEPARATOR));
235
+ idx_end -= strlen(HTTP_KEYVAL_SEPARATOR) + idx;
236
+ if (idx_end >= HTTP_HDR_BUFFER_SIZE) {
237
+ error("Value of key \"%s\" too long", buf_key);
238
+ return 1;
239
+ }
240
+
241
+ rbuf_pop(buf, buf_val, idx_end);
242
+ buf_val[idx_end] = 0;
243
+
244
+ rbuf_bump_tail(buf, strlen(HTTP_KEYVAL_SEPARATOR));
245
+
246
+ for (ptr = buf_key; *ptr; ptr++)
247
+ *ptr = tolower(*ptr);
248
+
249
+ process_http_hdr(parse_ctx, buf_key, buf_val);
250
+
251
+ return 0;
252
+}
253
+
254
+static int parse_http_response(rbuf_t buf, http_parse_ctx *parse_ctx)
255
+{
256
+ int idx;
257
+ char rc[4];
258
+
259
+ do {
260
+ if (parse_ctx->state != HTTP_PARSE_CONTENT && !rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx))
261
+ return NEED_MORE_DATA;
262
+ switch (parse_ctx->state) {
263
+ case HTTP_PARSE_INITIAL:
264
+ if (rbuf_memcmp_n(buf, RESP_PROTO, strlen(RESP_PROTO))) {
265
+ error("Expected response to start with \"%s\"", RESP_PROTO);
266
+ return PARSE_ERROR;
267
+ }
268
+ rbuf_bump_tail(buf, strlen(RESP_PROTO));
269
+ if (rbuf_pop(buf, rc, 4) != 4) {
270
+ error("Expected HTTP status code");
271
+ return PARSE_ERROR;
272
+ }
273
+ if (rc[3] != ' ') {
274
+ error("Expected space after HTTP return code");
275
+ return PARSE_ERROR;
276
+ }
277
+ rc[3] = 0;
278
+ parse_ctx->http_code = atoi(rc);
279
+ if (parse_ctx->http_code < 100 || parse_ctx->http_code >= 600) {
280
+ error("HTTP code not in range 100 to 599");
281
+ return PARSE_ERROR;
282
+ }
283
+
284
+ rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx);
285
+
286
+ rbuf_bump_tail(buf, idx + strlen(HTTP_LINE_TERM));
287
+
288
+ parse_ctx->state = HTTP_PARSE_HEADERS;
289
+ break;
290
+ case HTTP_PARSE_HEADERS:
291
+ if (!idx) {
292
+ parse_ctx->state = HTTP_PARSE_CONTENT;
293
+ rbuf_bump_tail(buf, strlen(HTTP_LINE_TERM));
294
+ break;
295
+ }
296
+ if (parse_http_hdr(buf, parse_ctx))
297
+ return PARSE_ERROR;
298
+ rbuf_find_bytes(buf, HTTP_LINE_TERM, strlen(HTTP_LINE_TERM), &idx);
299
+ rbuf_bump_tail(buf, idx + strlen(HTTP_LINE_TERM));
300
+ break;
301
+ case HTTP_PARSE_CONTENT:
302
+ if (parse_ctx->content_length < 0) {
303
+ error("content-length missing and http headers ended");
304
+ return PARSE_ERROR;
305
+ }
306
+ if (rbuf_bytes_available(buf) >= (size_t)parse_ctx->content_length)
307
+ return PARSE_SUCCESS;
308
+ return NEED_MORE_DATA;
309
+ }
310
+ } while(1);
311
+}
312
+
313
+static int https_request(http_req_type method, char *host, int port, char *url, char *b, size_t b_size, char *payload)
314
+{
315
+ struct timeval timeout = { .tv_sec = 30, .tv_usec = 0 };
316
+ char sport[PORT_STR_MAX_BYTES];
317
+ size_t len = 0;
318
+ int rc = 1;
319
+ int ret;
320
+ char *ptr;
321
+ http_parse_ctx parse_ctx = HTTP_PARSE_CTX_INITIALIZER;
322
+
323
+ rbuf_t buffer = rbuf_create(b_size);
324
+ if (!buffer)
325
+ return 1;
326
+
327
+ snprintf(sport, PORT_STR_MAX_BYTES, "%d", port);
328
+
329
+ if (payload != NULL)
330
+ len = strlen(payload);
331
+
332
+ snprintf(
333
+ b,
334
+ b_size,
335
+ "%s %s HTTP/1.1\r\nHost: %s\r\nAccept: application/json\r\nContent-length: %zu\r\nAccept-Language: en-us\r\n"
336
+ "User-Agent: Netdata/rocks\r\n\r\n",
337
+ (method == HTTP_REQ_GET ? "GET" : "POST"), url, host, len);
338
+
339
+ if (payload != NULL)
340
+ strncat(b, payload, b_size - len);
341
+
342
+ len = strlen(b);
343
+
344
+ debug(D_ACLK, "Sending HTTPS req (%zu bytes): '%s'", len, b);
345
+ int sock = connect_to_this_ip46(IPPROTO_TCP, SOCK_STREAM, host, 0, sport, &timeout);
346
+
347
+ if (unlikely(sock == -1)) {
348
+ error("Handshake failed");
349
+ goto exit_buf;
350
+ }
351
+
352
+ SSL_CTX *ctx = security_initialize_openssl_client();
353
+ if (ctx==NULL) {
354
+ error("Cannot allocate SSL context");
355
+ goto exit_sock;
356
+ }
357
+ // Certificate chain: not updating the stores - do we need private CA roots?
358
+ // Calls to SSL_CTX_load_verify_locations would go here.
359
+ SSL *ssl = SSL_new(ctx);
360
+ if (ssl==NULL) {
361
+ error("Cannot allocate SSL");
362
+ goto exit_CTX;
363
+ }
364
+ SSL_set_fd(ssl, sock);
365
+ ret = SSL_connect(ssl);
366
+ if (ret != 1) {
367
+ error("SSL_connect() failed with err=%d", ret);
368
+ goto exit_SSL;
369
+ }
370
+
371
+ ret = SSL_write(ssl, b, len);
372
+ if (ret <= 0)
373
+ {
374
+ error("SSL_write() failed with err=%d", ret);
375
+ goto exit_SSL;
376
+ }
377
+
378
+ b[0] = 0;
379
+
380
+ do {
381
+ ptr = rbuf_get_linear_insert_range(buffer, &len);
382
+ ret = SSL_read(ssl, ptr, len - 1);
383
+ if (ret)
384
+ rbuf_bump_head(buffer, ret);
385
+ if (ret <= 0)
386
+ {
387
+ error("No response available - SSL_read()=%d", ret);
388
+ goto exit_FULL;
389
+ }
390
+ } while (!(ret = parse_http_response(buffer, &parse_ctx)));
391
+
392
+ if (ret != PARSE_SUCCESS) {
393
+ error("Error parsing HTTP response");
394
+ goto exit_FULL;
395
+ }
396
+
397
+ if (parse_ctx.http_code < 200 || parse_ctx.http_code >= 300) {
398
+ error("HTTP Response not Success (got %d)", parse_ctx.http_code);
399
+ goto exit_FULL;
400
+ }
401
+
402
+ len = rbuf_pop(buffer, b, b_size);
403
+ b[MIN(len, b_size-1)] = 0;
404
+
405
+ rc = 0;
406
+exit_FULL:
407
+exit_SSL:
408
+ SSL_free(ssl);
409
+exit_CTX:
410
+ SSL_CTX_free(ctx);
411
+exit_sock:
412
+ close(sock);
413
+exit_buf:
414
+ rbuf_free(buffer);
415
+ return rc;
416
+}
417
+
418
+// aclk_get_mqtt_otp is slightly modified original code from @amoss
419
+void aclk_get_mqtt_otp(RSA *p_key, char *aclk_hostname, int port, char **mqtt_usr, char **mqtt_pass)
420
+{
421
+ char *data_buffer = mallocz(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
422
+ debug(D_ACLK, "Performing challenge-response sequence");
423
+ if (*mqtt_pass != NULL)
424
+ {
425
+ freez(*mqtt_pass);
426
+ *mqtt_pass = NULL;
427
+ }
428
+ // curl http://cloud-iam-agent-service:8080/api/v1/auth/node/00000000-0000-0000-0000-000000000000/challenge
429
+ // TODO - target host?
430
+ char *agent_id = is_agent_claimed();
431
+ if (agent_id == NULL)
432
+ {
433
+ error("Agent was not claimed - cannot perform challenge/response");
434
+ goto CLEANUP;
435
+ }
436
+ char url[1024];
437
+ sprintf(url, "/api/v1/auth/node/%s/challenge", agent_id);
438
+ info("Retrieving challenge from cloud: %s %d %s", aclk_hostname, port, url);
439
+ if (https_request(HTTP_REQ_GET, aclk_hostname, port, url, data_buffer, NETDATA_WEB_RESPONSE_INITIAL_SIZE, NULL))
440
+ {
441
+ error("Challenge failed: %s", data_buffer);
442
+ goto CLEANUP;
443
+ }
444
+ struct dictionary_singleton challenge = { .key = "challenge", .result = NULL };
445
+
446
+ debug(D_ACLK, "Challenge response from cloud: %s", data_buffer);
447
+ if (json_parse(data_buffer, &challenge, json_extract_singleton) != JSON_OK)
448
+ {
449
+ freez(challenge.result);
450
+ error("Could not parse the json response with the challenge: %s", data_buffer);
451
+ goto CLEANUP;
452
+ }
453
+ if (challenge.result == NULL) {
454
+ error("Could not retrieve challenge from auth response: %s", data_buffer);
455
+ goto CLEANUP;
456
+ }
457
+
458
+
459
+ size_t challenge_len = strlen(challenge.result);
460
+ unsigned char decoded[512];
461
+ size_t decoded_len = base64_decode((unsigned char*)challenge.result, challenge_len, decoded, sizeof(decoded));
462
+
463
+ unsigned char plaintext[4096]={};
464
+ int decrypted_length = private_decrypt(p_key, decoded, decoded_len, plaintext);
465
+ freez(challenge.result);
466
+ char encoded[512];
467
+ size_t encoded_len = base64_encode(plaintext, decrypted_length, encoded, sizeof(encoded));
468
+ encoded[encoded_len] = 0;
469
+ debug(D_ACLK, "Encoded len=%zu Decryption len=%d: '%s'", encoded_len, decrypted_length, encoded);
470
+
471
+ char response_json[4096]={};
472
+ sprintf(response_json, "{\"response\":\"%s\"}", encoded);
473
+ debug(D_ACLK, "Password phase: %s",response_json);
474
+ // TODO - host
475
+ sprintf(url, "/api/v1/auth/node/%s/password", agent_id);
476
+ if (https_request(HTTP_REQ_POST, aclk_hostname, port, url, data_buffer, NETDATA_WEB_RESPONSE_INITIAL_SIZE, response_json))
477
+ {
478
+ error("Challenge-response failed: %s", data_buffer);
479
+ goto CLEANUP;
480
+ }
481
+
482
+ debug(D_ACLK, "Password response from cloud: %s", data_buffer);
483
+
484
+ struct dictionary_singleton password = { .key = "password", .result = NULL };
485
+ if (json_parse(data_buffer, &password, json_extract_singleton) != JSON_OK)
486
+ {
487
+ freez(password.result);
488
+ error("Could not parse the json response with the password: %s", data_buffer);
489
+ goto CLEANUP;
490
+ }
491
+
492
+ if (password.result == NULL ) {
493
+ error("Could not retrieve password from auth response");
494
+ goto CLEANUP;
495
+ }
496
+ if (*mqtt_pass != NULL )
497
+ freez(*mqtt_pass);
498
+ *mqtt_pass = password.result;
499
+ if (*mqtt_usr != NULL)
500
+ freez(*mqtt_usr);
501
+ *mqtt_usr = agent_id;
502
+ agent_id = NULL;
503
+
504
+CLEANUP:
505
+ if (agent_id != NULL)
506
+ freez(agent_id);
507
+ freez(data_buffer);
508
+ return;
509
+}
aclk/aclk_otp.h
new
+10
@@ -0,0 +1,10 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef ACLK_OTP_H
4
+#define ACLK_OTP_H
5
+
6
+#include "../daemon/common.h"
7
+
8
+void aclk_get_mqtt_otp(RSA *p_key, char *aclk_hostname, int port, char **mqtt_usr, char **mqtt_pass);
9
+
10
+#endif /* ACLK_OTP_H */
aclk/aclk_query.c
new
+295
@@ -0,0 +1,295 @@
1
+#include "aclk_query.h"
2
+#include "aclk_stats.h"
3
+#include "aclk_query_queue.h"
4
+#include "aclk_tx_msgs.h"
5
+
6
+#define ACLK_QUERY_THREAD_NAME "ACLK_Query"
7
+
8
+#define WEB_HDR_ACCEPT_ENC "Accept-Encoding:"
9
+
10
+pthread_cond_t query_cond_wait = PTHREAD_COND_INITIALIZER;
11
+pthread_mutex_t query_lock_wait = PTHREAD_MUTEX_INITIALIZER;
12
+#define QUERY_THREAD_LOCK pthread_mutex_lock(&query_lock_wait)
13
+#define QUERY_THREAD_UNLOCK pthread_mutex_unlock(&query_lock_wait)
14
+
15
+typedef struct aclk_query_handler {
16
+ aclk_query_type_t type;
17
+ char *name; // for logging purposes
18
+ int(*fnc)(mqtt_wss_client client, aclk_query_t query);
19
+} aclk_query_handler;
20
+
21
+static int info_metadata(mqtt_wss_client client, aclk_query_t query)
22
+{
23
+ aclk_send_info_metadata(client,
24
+ !query->data.metadata_info.initial_on_connect,
25
+ query->data.metadata_info.host);
26
+ return 0;
27
+}
28
+
29
+static int alarms_metadata(mqtt_wss_client client, aclk_query_t query)
30
+{
31
+ aclk_send_alarm_metadata(client,
32
+ !query->data.metadata_info.initial_on_connect);
33
+ return 0;
34
+}
35
+
36
+static usec_t aclk_web_api_v1_request(RRDHOST *host, struct web_client *w, char *url)
37
+{
38
+ usec_t t;
39
+
40
+ t = now_monotonic_high_precision_usec();
41
+ w->response.code = web_client_api_request_v1(host, w, url);
42
+ t = now_monotonic_high_precision_usec() - t;
43
+
44
+ if (aclk_stats_enabled) {
45
+ ACLK_STATS_LOCK;
46
+ aclk_metrics_per_sample.cloud_q_process_total += t;
47
+ aclk_metrics_per_sample.cloud_q_process_count++;
48
+ if (aclk_metrics_per_sample.cloud_q_process_max < t)
49
+ aclk_metrics_per_sample.cloud_q_process_max = t;
50
+ ACLK_STATS_UNLOCK;
51
+ }
52
+
53
+ return t;
54
+}
55
+
56
+static int http_api_v2(mqtt_wss_client client, aclk_query_t query)
57
+{
58
+ int retval = 0;
59
+ usec_t t;
60
+ BUFFER *local_buffer = NULL;
61
+
62
+#ifdef NETDATA_WITH_ZLIB
63
+ int z_ret;
64
+ BUFFER *z_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
65
+ char *start, *end;
66
+#endif
67
+
68
+ struct web_client *w = (struct web_client *)callocz(1, sizeof(struct web_client));
69
+ w->response.data = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
70
+ w->response.header = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
71
+ w->response.header_output = buffer_create(NETDATA_WEB_RESPONSE_HEADER_SIZE);
72
+ strcpy(w->origin, "*"); // Simulate web_client_create_on_fd()
73
+ w->cookie1[0] = 0; // Simulate web_client_create_on_fd()
74
+ w->cookie2[0] = 0; // Simulate web_client_create_on_fd()
75
+ w->acl = 0x1f;
76
+
77
+ char *mysep = strchr(query->data.http_api_v2.query, '?');
78
+ if (mysep) {
79
+ url_decode_r(w->decoded_query_string, mysep, NETDATA_WEB_REQUEST_URL_SIZE + 1);
80
+ *mysep = '\0';
81
+ } else
82
+ url_decode_r(w->decoded_query_string, query->data.http_api_v2.query, NETDATA_WEB_REQUEST_URL_SIZE + 1);
83
+
84
+ mysep = strrchr(query->data.http_api_v2.query, '/');
85
+
86
+ // execute the query
87
+ t = aclk_web_api_v1_request(localhost, w, mysep ? mysep + 1 : "noop");
88
+
89
+#ifdef NETDATA_WITH_ZLIB
90
+ // check if gzip encoding can and should be used
91
+ if ((start = strstr((char *)query->data.http_api_v2.payload, WEB_HDR_ACCEPT_ENC))) {
92
+ start += strlen(WEB_HDR_ACCEPT_ENC);
93
+ end = strstr(start, "\x0D\x0A");
94
+ start = strstr(start, "gzip");
95
+
96
+ if (start && start < end) {
97
+ w->response.zstream.zalloc = Z_NULL;
98
+ w->response.zstream.zfree = Z_NULL;
99
+ w->response.zstream.opaque = Z_NULL;
100
+ if(deflateInit2(&w->response.zstream, web_gzip_level, Z_DEFLATED, 15 + 16, 8, web_gzip_strategy) == Z_OK) {
101
+ w->response.zinitialized = 1;
102
+ w->response.zoutput = 1;
103
+ } else
104
+ error("Failed to initialize zlib. Proceeding without compression.");
105
+ }
106
+ }
107
+
108
+ if (w->response.data->len && w->response.zinitialized) {
109
+ w->response.zstream.next_in = (Bytef *)w->response.data->buffer;
110
+ w->response.zstream.avail_in = w->response.data->len;
111
+ do {
112
+ w->response.zstream.avail_out = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE;
113
+ w->response.zstream.next_out = w->response.zbuffer;
114
+ z_ret = deflate(&w->response.zstream, Z_FINISH);
115
+ if(z_ret < 0) {
116
+ if(w->response.zstream.msg)
117
+ error("Error compressing body. ZLIB error: \"%s\"", w->response.zstream.msg);
118
+ else
119
+ error("Unknown error during zlib compression.");
120
+ retval = 1;
121
+ goto cleanup;
122
+ }
123
+ int bytes_to_cpy = NETDATA_WEB_RESPONSE_ZLIB_CHUNK_SIZE - w->response.zstream.avail_out;
124
+ buffer_need_bytes(z_buffer, bytes_to_cpy);
125
+ memcpy(&z_buffer->buffer[z_buffer->len], w->response.zbuffer, bytes_to_cpy);
126
+ z_buffer->len += bytes_to_cpy;
127
+ } while(z_ret != Z_STREAM_END);
128
+ // so that web_client_build_http_header
129
+ // puts correct content lenght into header
130
+ buffer_free(w->response.data);
131
+ w->response.data = z_buffer;
132
+ z_buffer = NULL;
133
+ }
134
+#endif
135
+
136
+ now_realtime_timeval(&w->tv_ready);
137
+ w->response.data->date = w->tv_ready.tv_sec;
138
+ web_client_build_http_header(w);
139
+ local_buffer = buffer_create(NETDATA_WEB_RESPONSE_INITIAL_SIZE);
140
+ local_buffer->contenttype = CT_APPLICATION_JSON;
141
+
142
+ buffer_strcat(local_buffer, w->response.header_output->buffer);
143
+
144
+ if (w->response.data->len) {
145
+#ifdef NETDATA_WITH_ZLIB
146
+ if (w->response.zinitialized) {
147
+ buffer_need_bytes(local_buffer, w->response.data->len);
148
+ memcpy(&local_buffer->buffer[local_buffer->len], w->response.data->buffer, w->response.data->len);
149
+ local_buffer->len += w->response.data->len;
150
+ } else {
151
+#endif
152
+ buffer_strcat(local_buffer, w->response.data->buffer);
153
+#ifdef NETDATA_WITH_ZLIB
154
+ }
155
+#endif
156
+ }
157
+
158
+ aclk_http_msg_v2(client, query->callback_topic, query->msg_id, t, query->created, w->response.code, local_buffer->buffer, local_buffer->len);
159
+
160
+cleanup:
161
+#ifdef NETDATA_WITH_ZLIB
162
+ if(w->response.zinitialized)
163
+ deflateEnd(&w->response.zstream);
164
+ buffer_free(z_buffer);
165
+#endif
166
+ buffer_free(w->response.data);
167
+ buffer_free(w->response.header);
168
+ buffer_free(w->response.header_output);
169
+ freez(w);
170
+ buffer_free(local_buffer);
171
+ return retval;
172
+}
173
+
174
+static int chart_query(mqtt_wss_client client, aclk_query_t query)
175
+{
176
+ aclk_chart_msg(client, query->data.chart_add_del.host, query->data.chart_add_del.chart_name);
177
+ return 0;
178
+}
179
+
180
+static int alarm_state_update_query(mqtt_wss_client client, aclk_query_t query)
181
+{
182
+ aclk_alarm_state_msg(client, query->data.alarm_update);
183
+ // aclk_alarm_state_msg frees the json object including the header it generates
184
+ query->data.alarm_update = NULL;
185
+ return 0;
186
+}
187
+
188
+aclk_query_handler aclk_query_handlers[] = {
189
+ { .type = HTTP_API_V2, .name = "http api request v2", .fnc = http_api_v2 },
190
+ { .type = ALARM_STATE_UPDATE, .name = "alarm state update", .fnc = alarm_state_update_query },
191
+ { .type = METADATA_INFO, .name = "info metadata", .fnc = info_metadata },
192
+ { .type = METADATA_ALARMS, .name = "alarms metadata", .fnc = alarms_metadata },
193
+ { .type = CHART_NEW, .name = "chart new", .fnc = chart_query },
194
+ { .type = CHART_DEL, .name = "chart delete", .fnc = info_metadata },
195
+ { .type = UNKNOWN, .name = NULL, .fnc = NULL }
196
+};
197
+
198
+
199
+static void aclk_query_process_msg(struct aclk_query_thread *info, aclk_query_t query)
200
+{
201
+ for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) {
202
+ if (aclk_query_handlers[i].type == query->type) {
203
+ debug(D_ACLK, "Processing Queued Message of type: \"%s\"", aclk_query_handlers[i].name);
204
+ aclk_query_handlers[i].fnc(info->client, query);
205
+ aclk_query_free(query);
206
+ if (aclk_stats_enabled) {
207
+ ACLK_STATS_LOCK;
208
+ aclk_metrics_per_sample.queries_dispatched++;
209
+ aclk_queries_per_thread[info->idx]++;
210
+ ACLK_STATS_UNLOCK;
211
+ }
212
+ return;
213
+ }
214
+ }
215
+ fatal("Unknown query in query queue. %u", query->type);
216
+}
217
+
218
+/* Processes messages from queue. Compete for work with other threads
219
+ */
220
+int aclk_query_process_msgs(struct aclk_query_thread *info)
221
+{
222
+ aclk_query_t query;
223
+ while ((query = aclk_queue_pop()))
224
+ aclk_query_process_msg(info, query);
225
+
226
+ return 0;
227
+}
228
+
229
+/**
230
+ * Main query processing thread
231
+ */
232
+void *aclk_query_main_thread(void *ptr)
233
+{
234
+ struct aclk_query_thread *info = ptr;
235
+ while (!netdata_exit) {
236
+ ACLK_SHARED_STATE_LOCK;
237
+ if (unlikely(!aclk_shared_state.version_neg)) {
238
+ if (!aclk_shared_state.version_neg_wait_till || aclk_shared_state.version_neg_wait_till > now_monotonic_usec()) {
239
+ ACLK_SHARED_STATE_UNLOCK;
240
+ info("Waiting for ACLK Version Negotiation message from Cloud");
241
+ sleep(1);
242
+ continue;
243
+ }
244
+ errno = 0;
245
+ error("ACLK version negotiation failed. No reply to \"hello\" with \"version\" from cloud in time of %ds."
246
+ " Reverting to default ACLK version of %d.", VERSION_NEG_TIMEOUT, ACLK_VERSION_MIN);
247
+ aclk_shared_state.version_neg = ACLK_VERSION_MIN;
248
+// When ACLK v3 is implemented you will need this
249
+// aclk_set_rx_handlers(aclk_shared_state.version_neg);
250
+ }
251
+ ACLK_SHARED_STATE_UNLOCK;
252
+
253
+ aclk_query_process_msgs(info);
254
+
255
+ QUERY_THREAD_LOCK;
256
+
257
+ if (unlikely(pthread_cond_wait(&query_cond_wait, &query_lock_wait)))
258
+ sleep_usec(USEC_PER_SEC * 1);
259
+
260
+ QUERY_THREAD_UNLOCK;
261
+ }
262
+ return NULL;
263
+}
264
+
265
+#define TASK_LEN_MAX 16
266
+void aclk_query_threads_start(struct aclk_query_threads *query_threads, mqtt_wss_client client)
267
+{
268
+ info("Starting %d query threads.", query_threads->count);
269
+
270
+ char thread_name[TASK_LEN_MAX];
271
+ query_threads->thread_list = callocz(query_threads->count, sizeof(struct aclk_query_thread));
272
+ for (int i = 0; i < query_threads->count; i++) {
273
+ query_threads->thread_list[i].idx = i; //thread needs to know its index for statistics
274
+
275
+ if(unlikely(snprintf(thread_name, TASK_LEN_MAX, "%s_%d", ACLK_QUERY_THREAD_NAME, i) < 0))
276
+ error("snprintf encoding error");
277
+ netdata_thread_create(
278
+ &query_threads->thread_list[i].thread, thread_name, NETDATA_THREAD_OPTION_JOINABLE, aclk_query_main_thread,
279
+ &query_threads->thread_list[i]);
280
+
281
+ query_threads->thread_list[i].client = client;
282
+ }
283
+}
284
+
285
+void aclk_query_threads_cleanup(struct aclk_query_threads *query_threads)
286
+{
287
+ if (query_threads && query_threads->thread_list) {
288
+ for (int i = 0; i < query_threads->count; i++) {
289
+ netdata_thread_join(query_threads->thread_list[i].thread, NULL);
290
+ }
291
+ freez(query_threads->thread_list);
292
+ }
293
+ aclk_queue_lock();
294
+ aclk_queue_flush();
295
+}
aclk/aclk_query.h
new
+32
@@ -0,0 +1,32 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ACLK_QUERY_H
4
+#define NETDATA_ACLK_QUERY_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+
8
+#include "mqtt_wss_client.h"
9
+
10
+extern pthread_cond_t query_cond_wait;
11
+extern pthread_mutex_t query_lock_wait;
12
+#define QUERY_THREAD_WAKEUP pthread_cond_signal(&query_cond_wait)
13
+#define QUERY_THREAD_WAKEUP_ALL pthread_cond_broadcast(&query_cond_wait)
14
+
15
+// TODO
16
+//extern volatile int aclk_connected;
17
+
18
+struct aclk_query_thread {
19
+ netdata_thread_t thread;
20
+ int idx;
21
+ mqtt_wss_client client;
22
+};
23
+
24
+struct aclk_query_threads {
25
+ struct aclk_query_thread *thread_list;
26
+ int count;
27
+};
28
+
29
+void aclk_query_threads_start(struct aclk_query_threads *query_threads, mqtt_wss_client client);
30
+void aclk_query_threads_cleanup(struct aclk_query_threads *query_threads);
31
+
32
+#endif //NETDATA_AGENT_CLOUD_LINK_H
aclk/aclk_query_queue.c
new
+128
@@ -0,0 +1,128 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "aclk_query_queue.h"
4
+#include "aclk_query.h"
5
+#include "aclk_stats.h"
6
+
7
+static netdata_mutex_t aclk_query_queue_mutex = NETDATA_MUTEX_INITIALIZER;
8
+#define ACLK_QUEUE_LOCK netdata_mutex_lock(&aclk_query_queue_mutex)
9
+#define ACLK_QUEUE_UNLOCK netdata_mutex_unlock(&aclk_query_queue_mutex)
10
+
11
+static struct aclk_query_queue {
12
+ aclk_query_t head;
13
+ aclk_query_t tail;
14
+ int block_push;
15
+} aclk_query_queue = {
16
+ .head = NULL,
17
+ .tail = NULL,
18
+ .block_push = 0
19
+};
20
+
21
+static inline int _aclk_queue_query(aclk_query_t query)
22
+{
23
+ query->created = now_realtime_usec();
24
+ ACLK_QUEUE_LOCK;
25
+ if (aclk_query_queue.block_push) {
26
+ ACLK_QUEUE_UNLOCK;
27
+ if(!netdata_exit)
28
+ error("Query Queue is blocked from accepting new requests. This is normally the case when ACLK prepares to shutdown.");
29
+ aclk_query_free(query);
30
+ return 1;
31
+ }
32
+ if (!aclk_query_queue.head) {
33
+ aclk_query_queue.head = query;
34
+ aclk_query_queue.tail = query;
35
+ ACLK_QUEUE_UNLOCK;
36
+ return 0;
37
+ }
38
+ // TODO deduplication
39
+ aclk_query_queue.tail->next = query;
40
+ aclk_query_queue.tail = query;
41
+ ACLK_QUEUE_UNLOCK;
42
+ return 0;
43
+
44
+}
45
+
46
+int aclk_queue_query(aclk_query_t query)
47
+{
48
+ int ret = _aclk_queue_query(query);
49
+ if (!ret) {
50
+ QUERY_THREAD_WAKEUP;
51
+ if (aclk_stats_enabled) {
52
+ ACLK_STATS_LOCK;
53
+ aclk_metrics_per_sample.queries_queued++;
54
+ ACLK_STATS_UNLOCK;
55
+ }
56
+ }
57
+ return ret;
58
+}
59
+
60
+aclk_query_t aclk_queue_pop(void)
61
+{
62
+ aclk_query_t ret;
63
+
64
+ ACLK_QUEUE_LOCK;
65
+ if (aclk_query_queue.block_push) {
66
+ ACLK_QUEUE_UNLOCK;
67
+ if(!netdata_exit)
68
+ error("POP Query Queue is blocked from accepting new requests. This is normally the case when ACLK prepares to shutdown.");
69
+ return NULL;
70
+ }
71
+
72
+ ret = aclk_query_queue.head;
73
+ if (!ret) {
74
+ ACLK_QUEUE_UNLOCK;
75
+ return ret;
76
+ }
77
+
78
+ aclk_query_queue.head = ret->next;
79
+ if (unlikely(!aclk_query_queue.head))
80
+ aclk_query_queue.tail = aclk_query_queue.head;
81
+ ACLK_QUEUE_UNLOCK;
82
+
83
+ ret->next = NULL;
84
+ return ret;
85
+}
86
+
87
+void aclk_queue_flush(void)
88
+{
89
+ aclk_query_t query = aclk_queue_pop();
90
+ while (query) {
91
+ aclk_query_free(query);
92
+ query = aclk_queue_pop();
93
+ };
94
+}
95
+
96
+aclk_query_t aclk_query_new(aclk_query_type_t type)
97
+{
98
+ aclk_query_t query = callocz(1, sizeof(struct aclk_query));
99
+ query->type = type;
100
+ return query;
101
+}
102
+
103
+void aclk_query_free(aclk_query_t query)
104
+{
105
+ if (query->type == HTTP_API_V2) {
106
+ freez(query->data.http_api_v2.payload);
107
+ if (query->data.http_api_v2.query != query->dedup_id)
108
+ freez(query->data.http_api_v2.query);
109
+ }
110
+
111
+ if (query->type == CHART_NEW)
112
+ freez(query->data.chart_add_del.chart_name);
113
+
114
+ if (query->type == ALARM_STATE_UPDATE && query->data.alarm_update)
115
+ json_object_put(query->data.alarm_update);
116
+
117
+ freez(query->dedup_id);
118
+ freez(query->callback_topic);
119
+ freez(query->msg_id);
120
+ freez(query);
121
+}
122
+
123
+void aclk_queue_lock(void)
124
+{
125
+ ACLK_QUEUE_LOCK;
126
+ aclk_query_queue.block_push = 1;
127
+ ACLK_QUEUE_UNLOCK;
128
+}
aclk/aclk_query_queue.h
new
+71
@@ -0,0 +1,71 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ACLK_QUERY_QUEUE_H
4
+#define NETDATA_ACLK_QUERY_QUEUE_H
5
+
6
+#include "libnetdata/libnetdata.h"
7
+#include "../daemon/common.h"
8
+
9
+typedef enum {
10
+ UNKNOWN,
11
+ METADATA_INFO,
12
+ METADATA_ALARMS,
13
+ HTTP_API_V2,
14
+ CHART_NEW,
15
+ CHART_DEL,
16
+ ALARM_STATE_UPDATE
17
+} aclk_query_type_t;
18
+
19
+struct aclk_query_metadata {
20
+ RRDHOST *host;
21
+ int initial_on_connect;
22
+};
23
+
24
+struct aclk_query_chart_add_del {
25
+ RRDHOST *host;
26
+ char* chart_name;
27
+};
28
+
29
+struct aclk_query_http_api_v2 {
30
+ char *payload;
31
+ char *query;
32
+};
33
+
34
+typedef struct aclk_query *aclk_query_t;
35
+struct aclk_query {
36
+ aclk_query_type_t type;
37
+
38
+ // dedup_id is used to deduplicate queries in the list
39
+ // if type and dedup_id is the same message is deduplicated
40
+ // set dedup_id to NULL to never deduplicate the message
41
+ // set dedup_id to constant (e.g. empty string "") to make
42
+ // message of this type ever exist only once in the list
43
+ char *dedup_id;
44
+ char *callback_topic;
45
+ char *msg_id;
46
+
47
+ usec_t created;
48
+
49
+ aclk_query_t next;
50
+
51
+ // TODO maybe remove?
52
+ int version;
53
+ union {
54
+ struct aclk_query_metadata metadata_info;
55
+ struct aclk_query_metadata metadata_alarms;
56
+ struct aclk_query_http_api_v2 http_api_v2;
57
+ struct aclk_query_chart_add_del chart_add_del;
58
+ json_object *alarm_update;
59
+ } data;
60
+};
61
+
62
+aclk_query_t aclk_query_new(aclk_query_type_t type);
63
+void aclk_query_free(aclk_query_t query);
64
+
65
+int aclk_queue_query(aclk_query_t query);
66
+aclk_query_t aclk_queue_pop(void);
67
+void aclk_queue_flush(void);
68
+
69
+void aclk_queue_lock(void);
70
+
71
+#endif /* NETDATA_ACLK_QUERY_QUEUE_H */
aclk/aclk_rx_msgs.c
new
+343
@@ -0,0 +1,343 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "aclk_rx_msgs.h"
4
+
5
+#include "aclk_stats.h"
6
+#include "aclk_query_queue.h"
7
+
8
+#define ACLK_V2_PAYLOAD_SEPARATOR "\x0D\x0A\x0D\x0A"
9
+#define ACLK_CLOUD_REQ_V2_PREFIX "GET /api/v1/"
10
+
11
+struct aclk_request {
12
+ char *type_id;
13
+ char *msg_id;
14
+ char *callback_topic;
15
+ char *payload;
16
+ int version;
17
+ int min_version;
18
+ int max_version;
19
+};
20
+
21
+int cloud_to_agent_parse(JSON_ENTRY *e)
22
+{
23
+ struct aclk_request *data = e->callback_data;
24
+
25
+ switch (e->type) {
26
+ case JSON_OBJECT:
27
+ case JSON_ARRAY:
28
+ break;
29
+ case JSON_STRING:
30
+ if (!strcmp(e->name, "msg-id")) {
31
+ data->msg_id = strdupz(e->data.string);
32
+ break;
33
+ }
34
+ if (!strcmp(e->name, "type")) {
35
+ data->type_id = strdupz(e->data.string);
36
+ break;
37
+ }
38
+ if (!strcmp(e->name, "callback-topic")) {
39
+ data->callback_topic = strdupz(e->data.string);
40
+ break;
41
+ }
42
+ if (!strcmp(e->name, "payload")) {
43
+ if (likely(e->data.string)) {
44
+ size_t len = strlen(e->data.string);
45
+ data->payload = mallocz(len+1);
46
+ if (!url_decode_r(data->payload, e->data.string, len + 1))
47
+ strcpy(data->payload, e->data.string);
48
+ }
49
+ break;
50
+ }
51
+ break;
52
+ case JSON_NUMBER:
53
+ if (!strcmp(e->name, "version")) {
54
+ data->version = e->data.number;
55
+ break;
56
+ }
57
+ if (!strcmp(e->name, "min-version")) {
58
+ data->min_version = e->data.number;
59
+ break;
60
+ }
61
+ if (!strcmp(e->name, "max-version")) {
62
+ data->max_version = e->data.number;
63
+ break;
64
+ }
65
+
66
+ break;
67
+
68
+ case JSON_BOOLEAN:
69
+ break;
70
+
71
+ case JSON_NULL:
72
+ break;
73
+ }
74
+ return 0;
75
+}
76
+
77
+static inline int aclk_extract_v2_data(char *payload, char **data)
78
+{
79
+ char* ptr = strstr(payload, ACLK_V2_PAYLOAD_SEPARATOR);
80
+ if(!ptr)
81
+ return 1;
82
+ ptr += strlen(ACLK_V2_PAYLOAD_SEPARATOR);
83
+ *data = strdupz(ptr);
84
+ return 0;
85
+}
86
+
87
+static inline int aclk_v2_payload_get_query(const char *payload, char **query_url)
88
+{
89
+ const char *start, *end;
90
+
91
+ if(strncmp(payload, ACLK_CLOUD_REQ_V2_PREFIX, strlen(ACLK_CLOUD_REQ_V2_PREFIX))) {
92
+ errno = 0;
93
+ error("Only accepting requests that start with \"%s\" from CLOUD.", ACLK_CLOUD_REQ_V2_PREFIX);
94
+ return 1;
95
+ }
96
+ start = payload + 4;
97
+
98
+ if(!(end = strstr(payload, " HTTP/1.1\x0D\x0A"))) {
99
+ errno = 0;
100
+ error("Doesn't look like HTTP GET request.");
101
+ return 1;
102
+ }
103
+
104
+ *query_url = mallocz((end - start) + 1);
105
+ strncpyz(*query_url, start, end - start);
106
+
107
+ return 0;
108
+}
109
+
110
+#define HTTP_CHECK_AGENT_INITIALIZED() ACLK_SHARED_STATE_LOCK;\
111
+ if (unlikely(aclk_shared_state.agent_state == AGENT_INITIALIZING)) {\
112
+ debug(D_ACLK, "Ignoring \"http\" cloud request; agent not in stable state");\
113
+ ACLK_SHARED_STATE_UNLOCK;\
114
+ return 1;\
115
+ }\
116
+ ACLK_SHARED_STATE_UNLOCK;
117
+
118
+static int aclk_handle_cloud_request_v2(struct aclk_request *cloud_to_agent, char *raw_payload)
119
+{
120
+ HTTP_CHECK_AGENT_INITIALIZED();
121
+
122
+ aclk_query_t query;
123
+
124
+ errno = 0;
125
+ if (cloud_to_agent->version < ACLK_V_COMPRESSION) {
126
+ error(
127
+ "This handler cannot reply to request with version older than %d, received %d.",
128
+ ACLK_V_COMPRESSION,
129
+ cloud_to_agent->version);
130
+ return 1;
131
+ }
132
+
133
+ query = aclk_query_new(HTTP_API_V2);
134
+
135
+ if (unlikely(aclk_extract_v2_data(raw_payload, &query->data.http_api_v2.payload))) {
136
+ error("Error extracting payload expected after the JSON dictionary.");
137
+ goto error;
138
+ }
139
+
140
+ if (unlikely(aclk_v2_payload_get_query(query->data.http_api_v2.payload, &query->dedup_id))) {
141
+ error("Could not extract payload from query");
142
+ goto error;
143
+ }
144
+
145
+ if (unlikely(!cloud_to_agent->callback_topic)) {
146
+ error("Missing callback_topic");
147
+ goto error;
148
+ }
149
+
150
+ if (unlikely(!cloud_to_agent->msg_id)) {
151
+ error("Missing msg_id");
152
+ goto error;
153
+ }
154
+
155
+ // aclk_queue_query takes ownership of data pointer
156
+ query->callback_topic = cloud_to_agent->callback_topic;
157
+ // for clarity and code readability as when we process the request
158
+ // it would be strange to get URL from `dedup_id`
159
+ query->data.http_api_v2.query = query->dedup_id;
160
+ query->msg_id = cloud_to_agent->msg_id;
161
+ aclk_queue_query(query);
162
+ return 0;
163
+
164
+error:
165
+ aclk_query_free(query);
166
+ return 1;
167
+}
168
+
169
+// This handles `version` message from cloud used to negotiate
170
+// protocol version we will use
171
+static int aclk_handle_version_response(struct aclk_request *cloud_to_agent, char *raw_payload)
172
+{
173
+ UNUSED(raw_payload);
174
+ int version = -1;
175
+ errno = 0;
176
+
177
+ if (unlikely(cloud_to_agent->version != ACLK_VERSION_NEG_VERSION)) {
178
+ error(
179
+ "Unsuported version of \"version\" message from cloud. Expected %d, Got %d",
180
+ ACLK_VERSION_NEG_VERSION,
181
+ cloud_to_agent->version);
182
+ return 1;
183
+ }
184
+ if (unlikely(!cloud_to_agent->min_version)) {
185
+ error("Min version missing or 0");
186
+ return 1;
187
+ }
188
+ if (unlikely(!cloud_to_agent->max_version)) {
189
+ error("Max version missing or 0");
190
+ return 1;
191
+ }
192
+ if (unlikely(cloud_to_agent->max_version < cloud_to_agent->min_version)) {
193
+ error(
194
+ "Max version (%d) must be >= than min version (%d)", cloud_to_agent->max_version,
195
+ cloud_to_agent->min_version);
196
+ return 1;
197
+ }
198
+
199
+ if (unlikely(cloud_to_agent->min_version > ACLK_VERSION_MAX)) {
200
+ error(
201
+ "Agent too old for this cloud. Minimum version required by cloud %d."
202
+ " Maximum version supported by this agent %d.",
203
+ cloud_to_agent->min_version, ACLK_VERSION_MAX);
204
+ aclk_kill_link = 1;
205
+ aclk_disable_runtime = 1;
206
+ return 1;
207
+ }
208
+ if (unlikely(cloud_to_agent->max_version < ACLK_VERSION_MIN)) {
209
+ error(
210
+ "Cloud version is too old for this agent. Maximum version supported by cloud %d."
211
+ " Minimum (oldest) version supported by this agent %d.",
212
+ cloud_to_agent->max_version, ACLK_VERSION_MIN);
213
+ aclk_kill_link = 1;
214
+ return 1;
215
+ }
216
+
217
+ version = MIN(cloud_to_agent->max_version, ACLK_VERSION_MAX);
218
+
219
+ ACLK_SHARED_STATE_LOCK;
220
+ if (unlikely(now_monotonic_usec() > aclk_shared_state.version_neg_wait_till)) {
221
+ errno = 0;
222
+ error("The \"version\" message came too late ignoring.");
223
+ goto err_cleanup;
224
+ }
225
+ if (unlikely(aclk_shared_state.version_neg)) {
226
+ errno = 0;
227
+ error("Version has already been set to %d", aclk_shared_state.version_neg);
228
+ goto err_cleanup;
229
+ }
230
+ aclk_shared_state.version_neg = version;
231
+ ACLK_SHARED_STATE_UNLOCK;
232
+
233
+ info("Choosing version %d of ACLK", version);
234
+
235
+ aclk_set_rx_handlers(version);
236
+
237
+ return 0;
238
+
239
+err_cleanup:
240
+ ACLK_SHARED_STATE_UNLOCK;
241
+ return 1;
242
+}
243
+
244
+typedef struct aclk_incoming_msg_type{
245
+ char *name;
246
+ int(*fnc)(struct aclk_request *, char *);
247
+}aclk_incoming_msg_type;
248
+
249
+aclk_incoming_msg_type aclk_incoming_msg_types_compression[] = {
250
+ { .name = "http", .fnc = aclk_handle_cloud_request_v2 },
251
+ { .name = "version", .fnc = aclk_handle_version_response },
252
+ { .name = NULL, .fnc = NULL }
253
+};
254
+
255
+struct aclk_incoming_msg_type *aclk_incoming_msg_types = aclk_incoming_msg_types_compression;
256
+
257
+void aclk_set_rx_handlers(int version)
258
+{
259
+// ACLK_NG ACLK version support starts at 2
260
+// TODO ACLK v3
261
+ UNUSED(version);
262
+ aclk_incoming_msg_types = aclk_incoming_msg_types_compression;
263
+}
264
+
265
+int aclk_handle_cloud_message(char *payload)
266
+{
267
+ struct aclk_request cloud_to_agent;
268
+ memset(&cloud_to_agent, 0, sizeof(struct aclk_request));
269
+
270
+ if (aclk_stats_enabled) {
271
+ ACLK_STATS_LOCK;
272
+ aclk_metrics_per_sample.cloud_req_recvd++;
273
+ ACLK_STATS_UNLOCK;
274
+ }
275
+
276
+ if (unlikely(!payload)) {
277
+ errno = 0;
278
+ error("ACLK incoming message is empty");
279
+ goto err_cleanup_nojson;
280
+ }
281
+
282
+ debug(D_ACLK, "ACLK incoming message (%s)", payload);
283
+
284
+ int rc = json_parse(payload, &cloud_to_agent, cloud_to_agent_parse);
285
+
286
+ if (unlikely(rc != JSON_OK)) {
287
+ errno = 0;
288
+ error("Malformed json request (%s)", payload);
289
+ goto err_cleanup;
290
+ }
291
+
292
+ if (!cloud_to_agent.type_id) {
293
+ errno = 0;
294
+ error("Cloud message is missing compulsory key \"type\"");
295
+ goto err_cleanup;
296
+ }
297
+
298
+ if (!aclk_shared_state.version_neg && strcmp(cloud_to_agent.type_id, "version")) {
299
+ error("Only \"version\" message is allowed before popcorning and version negotiation is finished. Ignoring");
300
+ goto err_cleanup;
301
+ }
302
+
303
+ for (int i = 0; aclk_incoming_msg_types[i].name; i++) {
304
+ if (strcmp(cloud_to_agent.type_id, aclk_incoming_msg_types[i].name) == 0) {
305
+ if (likely(!aclk_incoming_msg_types[i].fnc(&cloud_to_agent, payload))) {
306
+ // in case of success handler is supposed to clean up after itself
307
+ // or as in the case of aclk_handle_cloud_request take
308
+ // ownership of the pointers (done to avoid copying)
309
+ // see what `aclk_queue_query` parameter `internal` does
310
+
311
+ // NEVER CONTINUE THIS LOOP AFTER CALLING FUNCTION!!!
312
+ // msg handlers (namely aclk_handle_version_responce)
313
+ // can freely change what aclk_incoming_msg_types points to
314
+ // so either exit or restart this for loop
315
+ freez(cloud_to_agent.type_id);
316
+ return 0;
317
+ }
318
+ goto err_cleanup;
319
+ }
320
+ }
321
+
322
+ errno = 0;
323
+ error("Unknown message type from Cloud \"%s\"", cloud_to_agent.type_id);
324
+
325
+err_cleanup:
326
+ if (cloud_to_agent.payload)
327
+ freez(cloud_to_agent.payload);
328
+ if (cloud_to_agent.type_id)
329
+ freez(cloud_to_agent.type_id);
330
+ if (cloud_to_agent.msg_id)
331
+ freez(cloud_to_agent.msg_id);
332
+ if (cloud_to_agent.callback_topic)
333
+ freez(cloud_to_agent.callback_topic);
334
+
335
+err_cleanup_nojson:
336
+ if (aclk_stats_enabled) {
337
+ ACLK_STATS_LOCK;
338
+ aclk_metrics_per_sample.cloud_req_err++;
339
+ ACLK_STATS_UNLOCK;
340
+ }
341
+
342
+ return 1;
343
+}
aclk/aclk_rx_msgs.h
new
+14
@@ -0,0 +1,14 @@
1
+
2
+
3
+// SPDX-License-Identifier: GPL-3.0-or-later
4
+
5
+#ifndef ACLK_RX_MSGS_H
6
+#define ACLK_RX_MSGS_H
7
+
8
+#include "../daemon/common.h"
9
+#include "libnetdata/libnetdata.h"
10
+
11
+int aclk_handle_cloud_message(char *payload);
12
+void aclk_set_rx_handlers(int version);
13
+
14
+#endif /* ACLK_RX_MSGS_H */
aclk/aclk_stats.c
new
+274
@@ -0,0 +1,274 @@
1
+#include "aclk_stats.h"
2
+
3
+netdata_mutex_t aclk_stats_mutex = NETDATA_MUTEX_INITIALIZER;
4
+
5
+int aclk_stats_enabled;
6
+
7
+int query_thread_count;
8
+
9
+// data ACLK stats need per query thread
10
+struct aclk_qt_data {
11
+ RRDDIM *dim;
12
+} *aclk_qt_data = NULL;
13
+
14
+uint32_t *aclk_queries_per_thread = NULL;
15
+uint32_t *aclk_queries_per_thread_sample = NULL;
16
+
17
+struct aclk_metrics aclk_metrics = {
18
+ .online = 0,
19
+};
20
+
21
+struct aclk_metrics_per_sample aclk_metrics_per_sample;
22
+
23
+static void aclk_stats_collect(struct aclk_metrics_per_sample *per_sample, struct aclk_metrics *permanent)
24
+{
25
+ static RRDSET *st_aclkstats = NULL;
26
+ static RRDDIM *rd_online_status = NULL;
27
+
28
+ if (unlikely(!st_aclkstats)) {
29
+ st_aclkstats = rrdset_create_localhost(
30
+ "netdata", "aclk_status", NULL, "aclk", NULL, "ACLK/Cloud connection status",
31
+ "connected", "netdata", "stats", 200000, localhost->rrd_update_every, RRDSET_TYPE_LINE);
32
+
33
+ rd_online_status = rrddim_add(st_aclkstats, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
34
+ } else
35
+ rrdset_next(st_aclkstats);
36
+
37
+ rrddim_set_by_pointer(st_aclkstats, rd_online_status, per_sample->offline_during_sample ? 0 : permanent->online);
38
+
39
+ rrdset_done(st_aclkstats);
40
+}
41
+
42
+static void aclk_stats_query_queue(struct aclk_metrics_per_sample *per_sample)
43
+{
44
+ static RRDSET *st_query_thread = NULL;
45
+ static RRDDIM *rd_queued = NULL;
46
+ static RRDDIM *rd_dispatched = NULL;
47
+
48
+ if (unlikely(!st_query_thread)) {
49
+ st_query_thread = rrdset_create_localhost(
50
+ "netdata", "aclk_query_per_second", NULL, "aclk", NULL, "ACLK Queries per second", "queries/s",
51
+ "netdata", "stats", 200001, localhost->rrd_update_every, RRDSET_TYPE_AREA);
52
+
53
+ rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
54
+ rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, -1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
55
+ } else
56
+ rrdset_next(st_query_thread);
57
+
58
+ rrddim_set_by_pointer(st_query_thread, rd_queued, per_sample->queries_queued);
59
+ rrddim_set_by_pointer(st_query_thread, rd_dispatched, per_sample->queries_dispatched);
60
+
61
+ rrdset_done(st_query_thread);
62
+}
63
+
64
+#ifdef NETDATA_INTERNAL_CHECKS
65
+static void aclk_stats_latency(struct aclk_metrics_per_sample *per_sample)
66
+{
67
+ static RRDSET *st = NULL;
68
+ static RRDDIM *rd_avg = NULL;
69
+ static RRDDIM *rd_max = NULL;
70
+
71
+ if (unlikely(!st)) {
72
+ st = rrdset_create_localhost(
73
+ "netdata", "aclk_latency_mqtt", NULL, "aclk", NULL, "ACLK Message Publish Latency", "ms",
74
+ "netdata", "stats", 200002, localhost->rrd_update_every, RRDSET_TYPE_LINE);
75
+
76
+ rd_avg = rrddim_add(st, "avg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
77
+ rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
78
+ } else
79
+ rrdset_next(st);
80
+ if(per_sample->latency_count)
81
+ rrddim_set_by_pointer(st, rd_avg, roundf((float)per_sample->latency_total / per_sample->latency_count));
82
+ else
83
+ rrddim_set_by_pointer(st, rd_avg, 0);
84
+
85
+ rrddim_set_by_pointer(st, rd_max, per_sample->latency_max);
86
+
87
+ rrdset_done(st);
88
+}
89
+#endif
90
+
91
+static void aclk_stats_cloud_req(struct aclk_metrics_per_sample *per_sample)
92
+{
93
+ static RRDSET *st = NULL;
94
+ static RRDDIM *rd_rq_rcvd = NULL;
95
+ static RRDDIM *rd_rq_err = NULL;
96
+
97
+ if (unlikely(!st)) {
98
+ st = rrdset_create_localhost(
99
+ "netdata", "aclk_cloud_req", NULL, "aclk", NULL, "Requests received from cloud", "req/s",
100
+ "netdata", "stats", 200005, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
101
+
102
+ rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
103
+ rd_rq_err = rrddim_add(st, "malformed", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
104
+ } else
105
+ rrdset_next(st);
106
+
107
+ rrddim_set_by_pointer(st, rd_rq_rcvd, per_sample->cloud_req_recvd - per_sample->cloud_req_err);
108
+ rrddim_set_by_pointer(st, rd_rq_err, per_sample->cloud_req_err);
109
+
110
+ rrdset_done(st);
111
+}
112
+
113
+#define MAX_DIM_NAME 16
114
+static void aclk_stats_query_threads(uint32_t *queries_per_thread)
115
+{
116
+ static RRDSET *st = NULL;
117
+
118
+ char dim_name[MAX_DIM_NAME];
119
+
120
+ if (unlikely(!st)) {
121
+ st = rrdset_create_localhost(
122
+ "netdata", "aclk_query_threads", NULL, "aclk", NULL, "Queries Processed Per Thread", "req/s",
123
+ "netdata", "stats", 200007, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
124
+
125
+ for (int i = 0; i < query_thread_count; i++) {
126
+ if (snprintf(dim_name, MAX_DIM_NAME, "Query %d", i) < 0)
127
+ error("snprintf encoding error");
128
+ aclk_qt_data[i].dim = rrddim_add(st, dim_name, NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
129
+ }
130
+ } else
131
+ rrdset_next(st);
132
+
133
+ for (int i = 0; i < query_thread_count; i++) {
134
+ rrddim_set_by_pointer(st, aclk_qt_data[i].dim, queries_per_thread[i]);
135
+ }
136
+
137
+ rrdset_done(st);
138
+}
139
+
140
+static void aclk_stats_query_time(struct aclk_metrics_per_sample *per_sample)
141
+{
142
+ static RRDSET *st = NULL;
143
+ static RRDDIM *rd_rq_avg = NULL;
144
+ static RRDDIM *rd_rq_max = NULL;
145
+ static RRDDIM *rd_rq_total = NULL;
146
+
147
+ if (unlikely(!st)) {
148
+ st = rrdset_create_localhost(
149
+ "netdata", "aclk_query_time", NULL, "aclk", NULL, "Time it took to process cloud requested DB queries", "us",
150
+ "netdata", "stats", 200006, localhost->rrd_update_every, RRDSET_TYPE_LINE);
151
+
152
+ rd_rq_avg = rrddim_add(st, "avg", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
153
+ rd_rq_max = rrddim_add(st, "max", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
154
+ rd_rq_total = rrddim_add(st, "total", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
155
+ } else
156
+ rrdset_next(st);
157
+
158
+ if(per_sample->cloud_q_process_count)
159
+ rrddim_set_by_pointer(st, rd_rq_avg, roundf((float)per_sample->cloud_q_process_total / per_sample->cloud_q_process_count));
160
+ else
161
+ rrddim_set_by_pointer(st, rd_rq_avg, 0);
162
+ rrddim_set_by_pointer(st, rd_rq_max, per_sample->cloud_q_process_max);
163
+ rrddim_set_by_pointer(st, rd_rq_total, per_sample->cloud_q_process_total);
164
+
165
+ rrdset_done(st);
166
+}
167
+
168
+void aclk_stats_thread_cleanup()
169
+{
170
+ freez(aclk_qt_data);
171
+ freez(aclk_queries_per_thread);
172
+ freez(aclk_queries_per_thread_sample);
173
+}
174
+
175
+void *aclk_stats_main_thread(void *ptr)
176
+{
177
+ struct aclk_stats_thread *args = ptr;
178
+
179
+ query_thread_count = args->query_thread_count;
180
+ aclk_qt_data = callocz(query_thread_count, sizeof(struct aclk_qt_data));
181
+ aclk_queries_per_thread = callocz(query_thread_count, sizeof(uint32_t));
182
+ aclk_queries_per_thread_sample = callocz(query_thread_count, sizeof(uint32_t));
183
+
184
+ heartbeat_t hb;
185
+ heartbeat_init(&hb);
186
+ usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
187
+
188
+ memset(&aclk_metrics_per_sample, 0, sizeof(struct aclk_metrics_per_sample));
189
+
190
+ struct aclk_metrics_per_sample per_sample;
191
+ struct aclk_metrics permanent;
192
+
193
+ while (!netdata_exit) {
194
+ netdata_thread_testcancel();
195
+ // ------------------------------------------------------------------------
196
+ // Wait for the next iteration point.
197
+
198
+ heartbeat_next(&hb, step_ut);
199
+ if (netdata_exit) break;
200
+
201
+ ACLK_STATS_LOCK;
202
+ // to not hold lock longer than necessary, especially not to hold it
203
+ // during database rrd* operations
204
+ memcpy(&per_sample, &aclk_metrics_per_sample, sizeof(struct aclk_metrics_per_sample));
205
+ memcpy(&permanent, &aclk_metrics, sizeof(struct aclk_metrics));
206
+ memset(&aclk_metrics_per_sample, 0, sizeof(struct aclk_metrics_per_sample));
207
+
208
+ memcpy(aclk_queries_per_thread_sample, aclk_queries_per_thread, sizeof(uint32_t) * query_thread_count);
209
+ memset(aclk_queries_per_thread, 0, sizeof(uint32_t) * query_thread_count);
210
+ ACLK_STATS_UNLOCK;
211
+
212
+ aclk_stats_collect(&per_sample, &permanent);
213
+ aclk_stats_query_queue(&per_sample);
214
+#ifdef NETDATA_INTERNAL_CHECKS
215
+ aclk_stats_latency(&per_sample);
216
+#endif
217
+
218
+ aclk_stats_cloud_req(&per_sample);
219
+ aclk_stats_query_threads(aclk_queries_per_thread_sample);
220
+
221
+ aclk_stats_query_time(&per_sample);
222
+ }
223
+
224
+ return 0;
225
+}
226
+
227
+void aclk_stats_upd_online(int online) {
228
+ if(!aclk_stats_enabled)
229
+ return;
230
+
231
+ ACLK_STATS_LOCK;
232
+ aclk_metrics.online = online;
233
+
234
+ if(!online)
235
+ aclk_metrics_per_sample.offline_during_sample = 1;
236
+ ACLK_STATS_UNLOCK;
237
+}
238
+
239
+#ifdef NETDATA_INTERNAL_CHECKS
240
+static usec_t pub_time[UINT16_MAX];
241
+void aclk_stats_msg_published(uint16_t id)
242
+{
243
+ ACLK_STATS_LOCK;
244
+ pub_time[id] = now_boottime_usec();
245
+ ACLK_STATS_UNLOCK;
246
+}
247
+
248
+void aclk_stats_msg_puback(uint16_t id)
249
+{
250
+ ACLK_STATS_LOCK;
251
+ usec_t t;
252
+
253
+ if (!aclk_stats_enabled) {
254
+ ACLK_STATS_UNLOCK;
255
+ return;
256
+ }
257
+
258
+ if (unlikely(!pub_time[id])) {
259
+ ACLK_STATS_UNLOCK;
260
+ error("Received PUBACK for unknown message?!");
261
+ return;
262
+ }
263
+
264
+ t = now_boottime_usec() - pub_time[id];
265
+ t /= USEC_PER_MS;
266
+ pub_time[id] = 0;
267
+ if (aclk_metrics_per_sample.latency_max < t)
268
+ aclk_metrics_per_sample.latency_max = t;
269
+
270
+ aclk_metrics_per_sample.latency_total += t;
271
+ aclk_metrics_per_sample.latency_count++;
272
+ ACLK_STATS_UNLOCK;
273
+}
274
+#endif /* NETDATA_INTERNAL_CHECKS */
aclk/aclk_stats.h
new
+64
@@ -0,0 +1,64 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ACLK_STATS_H
4
+#define NETDATA_ACLK_STATS_H
5
+
6
+#include "../daemon/common.h"
7
+#include "libnetdata/libnetdata.h"
8
+
9
+#define ACLK_STATS_THREAD_NAME "ACLK_Stats"
10
+
11
+extern netdata_mutex_t aclk_stats_mutex;
12
+
13
+#define ACLK_STATS_LOCK netdata_mutex_lock(&aclk_stats_mutex)
14
+#define ACLK_STATS_UNLOCK netdata_mutex_unlock(&aclk_stats_mutex)
15
+
16
+extern int aclk_stats_enabled;
17
+
18
+struct aclk_stats_thread {
19
+ netdata_thread_t *thread;
20
+ int query_thread_count;
21
+};
22
+
23
+// preserve between samples
24
+struct aclk_metrics {
25
+ volatile uint8_t online;
26
+};
27
+
28
+// reset to 0 on every sample
29
+extern struct aclk_metrics_per_sample {
30
+ /* in the unlikely event of ACLK disconnecting
31
+ and reconnecting under 1 sampling rate
32
+ we want to make sure we record the disconnection
33
+ despite it being then seemingly longer in graph */
34
+ volatile uint8_t offline_during_sample;
35
+
36
+ volatile uint32_t queries_queued;
37
+ volatile uint32_t queries_dispatched;
38
+
39
+#ifdef NETDATA_INTERNAL_CHECKS
40
+ volatile uint32_t latency_max;
41
+ volatile uint32_t latency_total;
42
+ volatile uint32_t latency_count;
43
+#endif
44
+
45
+ volatile uint32_t cloud_req_recvd;
46
+ volatile uint32_t cloud_req_err;
47
+
48
+ volatile uint32_t cloud_q_process_total;
49
+ volatile uint32_t cloud_q_process_count;
50
+ volatile uint32_t cloud_q_process_max;
51
+} aclk_metrics_per_sample;
52
+
53
+extern uint32_t *aclk_queries_per_thread;
54
+
55
+void *aclk_stats_main_thread(void *ptr);
56
+void aclk_stats_thread_cleanup();
57
+void aclk_stats_upd_online(int online);
58
+
59
+#ifdef NETDATA_INTERNAL_CHECKS
60
+void aclk_stats_msg_published(uint16_t id);
61
+void aclk_stats_msg_puback(uint16_t id);
62
+#endif /* NETDATA_INTERNAL_CHECKS */
63
+
64
+#endif /* NETDATA_ACLK_STATS_H */
aclk/aclk_tx_msgs.c
new
+395
@@ -0,0 +1,395 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "aclk_tx_msgs.h"
4
+#include "../daemon/common.h"
5
+#include "aclk_util.h"
6
+#include "aclk_stats.h"
7
+
8
+#ifndef __GNUC__
9
+#pragma region aclk_tx_msgs helper functions
10
+#endif
11
+
12
+static void aclk_send_message_subtopic(mqtt_wss_client client, json_object *msg, enum aclk_topics subtopic)
13
+{
14
+ uint16_t packet_id;
15
+ const char *str = json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN);
16
+
17
+ mqtt_wss_publish_pid(client, aclk_get_topic(subtopic), str, strlen(str), MQTT_WSS_PUB_QOS1, &packet_id);
18
+#ifdef NETDATA_INTERNAL_CHECKS
19
+ aclk_stats_msg_published(packet_id);
20
+#endif
21
+#ifdef ACLK_LOG_CONVERSATION_DIR
22
+#define FN_MAX_LEN 1024
23
+ char filename[FN_MAX_LEN];
24
+ snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-tx.json", ACLK_GET_CONV_LOG_NEXT());
25
+ json_object_to_file_ext(filename, msg, JSON_C_TO_STRING_PRETTY);
26
+#endif
27
+}
28
+
29
+static uint16_t aclk_send_message_subtopic_pid(mqtt_wss_client client, json_object *msg, enum aclk_topics subtopic)
30
+{
31
+ uint16_t packet_id;
32
+ const char *str = json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN);
33
+
34
+ mqtt_wss_publish_pid(client, aclk_get_topic(subtopic), str, strlen(str), MQTT_WSS_PUB_QOS1, &packet_id);
35
+#ifdef NETDATA_INTERNAL_CHECKS
36
+ aclk_stats_msg_published(packet_id);
37
+#endif
38
+#ifdef ACLK_LOG_CONVERSATION_DIR
39
+#define FN_MAX_LEN 1024
40
+ char filename[FN_MAX_LEN];
41
+ snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-tx.json", ACLK_GET_CONV_LOG_NEXT());
42
+ json_object_to_file_ext(filename, msg, JSON_C_TO_STRING_PRETTY);
43
+#endif
44
+ return packet_id;
45
+}
46
+
47
+/* UNUSED now but can be used soon MVP1?
48
+static void aclk_send_message_topic(mqtt_wss_client client, json_object *msg, const char *topic)
49
+{
50
+ if (unlikely(!topic || topic[0] != '/')) {
51
+ error ("Full topic required!");
52
+ return;
53
+ }
54
+
55
+ const char *str = json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN);
56
+
57
+ mqtt_wss_publish(client, topic, str, strlen(str), MQTT_WSS_PUB_QOS1);
58
+#ifdef NETDATA_INTERNAL_CHECKS
59
+ aclk_stats_msg_published();
60
+#endif
61
+#ifdef ACLK_LOG_CONVERSATION_DIR
62
+#define FN_MAX_LEN 1024
63
+ char filename[FN_MAX_LEN];
64
+ snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-tx.json", ACLK_GET_CONV_LOG_NEXT());
65
+ json_object_to_file_ext(filename, msg, JSON_C_TO_STRING_PRETTY);
66
+#endif
67
+}
68
+*/
69
+
70
+#define TOPIC_MAX_LEN 512
71
+#define V2_BIN_PAYLOAD_SEPARATOR "\x0D\x0A\x0D\x0A"
72
+static void aclk_send_message_with_bin_payload(mqtt_wss_client client, json_object *msg, const char *topic, const void *payload, size_t payload_len)
73
+{
74
+ uint16_t packet_id;
75
+ const char *str;
76
+ char *full_msg;
77
+ int len;
78
+
79
+ if (unlikely(!topic || topic[0] != '/')) {
80
+ error ("Full topic required!");
81
+ return;
82
+ }
83
+
84
+ str = json_object_to_json_string_ext(msg, JSON_C_TO_STRING_PLAIN);
85
+ len = strlen(str);
86
+
87
+ full_msg = mallocz(len + strlen(V2_BIN_PAYLOAD_SEPARATOR) + payload_len);
88
+
89
+ memcpy(full_msg, str, len);
90
+ memcpy(&full_msg[len], V2_BIN_PAYLOAD_SEPARATOR, strlen(V2_BIN_PAYLOAD_SEPARATOR));
91
+ len += strlen(V2_BIN_PAYLOAD_SEPARATOR);
92
+ memcpy(&full_msg[len], payload, payload_len);
93
+ len += payload_len;
94
+
95
+/* TODO
96
+#ifdef ACLK_LOG_CONVERSATION_DIR
97
+#define FN_MAX_LEN 1024
98
+ char filename[FN_MAX_LEN];
99
+ snprintf(filename, FN_MAX_LEN, ACLK_LOG_CONVERSATION_DIR "/%010d-tx.json", ACLK_GET_CONV_LOG_NEXT());
100
+ json_object_to_file_ext(filename, msg, JSON_C_TO_STRING_PRETTY);
101
+#endif */
102
+
103
+ mqtt_wss_publish_pid(client, topic, full_msg, len, MQTT_WSS_PUB_QOS1, &packet_id);
104
+#ifdef NETDATA_INTERNAL_CHECKS
105
+ aclk_stats_msg_published(packet_id);
106
+#endif
107
+ freez(full_msg);
108
+}
109
+
110
+/*
111
+ * Creates universal header common for all ACLK messages. User gets ownership of json object created.
112
+ * Usually this is freed by send function after message has been sent.
113
+ */
114
+static struct json_object *create_hdr(const char *type, const char *msg_id, time_t ts_secs, usec_t ts_us, int version)
115
+{
116
+ uuid_t uuid;
117
+ char uuid_str[36 + 1];
118
+ json_object *tmp;
119
+ json_object *obj = json_object_new_object();
120
+
121
+ tmp = json_object_new_string(type);
122
+ json_object_object_add(obj, "type", tmp);
123
+
124
+ if (unlikely(!msg_id)) {
125
+ uuid_generate(uuid);
126
+ uuid_unparse(uuid, uuid_str);
127
+ msg_id = uuid_str;
128
+ }
129
+
130
+ if (ts_secs == 0) {
131
+ ts_us = now_realtime_usec();
132
+ ts_secs = ts_us / USEC_PER_SEC;
133
+ ts_us = ts_us % USEC_PER_SEC;
134
+ }
135
+
136
+ tmp = json_object_new_string(msg_id);
137
+ json_object_object_add(obj, "msg-id", tmp);
138
+
139
+ tmp = json_object_new_int64(ts_secs);
140
+ json_object_object_add(obj, "timestamp", tmp);
141
+
142
+// TODO handle this somehow on older json-c
143
+// tmp = json_object_new_uint64(ts_us);
144
+// probably jso->_to_json_strinf -> custom function
145
+// jso->o.c_uint64 -> map this with pointer to signed int
146
+// commit that implements json_object_new_uint64 is 3c3b592
147
+// between 0.14 and 0.15
148
+ tmp = json_object_new_int64(ts_us);
149
+ json_object_object_add(obj, "timestamp-offset-usec", tmp);
150
+
151
+ tmp = json_object_new_int64(aclk_session_sec);
152
+ json_object_object_add(obj, "connect", tmp);
153
+
154
+// TODO handle this somehow see above
155
+// tmp = json_object_new_uint64(0 /* TODO aclk_session_us */);
156
+ tmp = json_object_new_int64(aclk_session_us);
157
+ json_object_object_add(obj, "connect-offset-usec", tmp);
158
+
159
+ tmp = json_object_new_int(version);
160
+ json_object_object_add(obj, "version", tmp);
161
+
162
+ return obj;
163
+}
164
+
165
+static char *create_uuid()
166
+{
167
+ uuid_t uuid;
168
+ char *uuid_str = mallocz(36 + 1);
169
+
170
+ uuid_generate(uuid);
171
+ uuid_unparse(uuid, uuid_str);
172
+
173
+ return uuid_str;
174
+}
175
+
176
+#ifndef __GNUC__
177
+#pragma endregion
178
+#endif
179
+
180
+#ifndef __GNUC__
181
+#pragma region aclk_tx_msgs message generators
182
+#endif
183
+
184
+/*
185
+ * This will send the /api/v1/info
186
+ */
187
+#define BUFFER_INITIAL_SIZE (1024 * 16)
188
+void aclk_send_info_metadata(mqtt_wss_client client, int metadata_submitted, RRDHOST *host)
189
+{
190
+ BUFFER *local_buffer = buffer_create(BUFFER_INITIAL_SIZE);
191
+ json_object *msg, *payload, *tmp;
192
+
193
+ char *msg_id = create_uuid();
194
+ buffer_flush(local_buffer);
195
+ local_buffer->contenttype = CT_APPLICATION_JSON;
196
+
197
+ // on_connect messages are sent on a health reload, if the on_connect message is real then we
198
+ // use the session time as the fake timestamp to indicate that it starts the session. If it is
199
+ // a fake on_connect message then use the real timestamp to indicate it is within the existing
200
+ // session.
201
+ if (metadata_submitted)
202
+ msg = create_hdr("update", msg_id, 0, 0, aclk_shared_state.version_neg);
203
+ else
204
+ msg = create_hdr("connect", msg_id, aclk_session_sec, aclk_session_us, aclk_shared_state.version_neg);
205
+
206
+ payload = json_object_new_object();
207
+ json_object_object_add(msg, "payload", payload);
208
+
209
+ web_client_api_request_v1_info_fill_buffer(host, local_buffer);
210
+ tmp = json_tokener_parse(local_buffer->buffer);
211
+ json_object_object_add(payload, "info", tmp);
212
+
213
+ buffer_flush(local_buffer);
214
+
215
+ charts2json(host, local_buffer, 1, 0);
216
+ tmp = json_tokener_parse(local_buffer->buffer);
217
+ json_object_object_add(payload, "charts", tmp);
218
+
219
+ aclk_send_message_subtopic(client, msg, ACLK_TOPICID_METADATA);
220
+
221
+ json_object_put(msg);
222
+ freez(msg_id);
223
+ buffer_free(local_buffer);
224
+}
225
+
226
+// TODO should include header instead
227
+void health_active_log_alarms_2json(RRDHOST *host, BUFFER *wb);
228
+
229
+void aclk_send_alarm_metadata(mqtt_wss_client client, int metadata_submitted)
230
+{
231
+ BUFFER *local_buffer = buffer_create(BUFFER_INITIAL_SIZE);
232
+ json_object *msg, *payload, *tmp;
233
+
234
+ char *msg_id = create_uuid();
235
+ buffer_flush(local_buffer);
236
+ local_buffer->contenttype = CT_APPLICATION_JSON;
237
+
238
+ // on_connect messages are sent on a health reload, if the on_connect message is real then we
239
+ // use the session time as the fake timestamp to indicate that it starts the session. If it is
240
+ // a fake on_connect message then use the real timestamp to indicate it is within the existing
241
+ // session.
242
+
243
+ if (metadata_submitted)
244
+ msg = create_hdr("connect_alarms", msg_id, 0, 0, aclk_shared_state.version_neg);
245
+ else
246
+ msg = create_hdr("connect_alarms", msg_id, aclk_session_sec, aclk_session_us, aclk_shared_state.version_neg);
247
+
248
+ payload = json_object_new_object();
249
+ json_object_object_add(msg, "payload", payload);
250
+
251
+ health_alarms2json(localhost, local_buffer, 1);
252
+ tmp = json_tokener_parse(local_buffer->buffer);
253
+ json_object_object_add(payload, "configured-alarms", tmp);
254
+
255
+ buffer_flush(local_buffer);
256
+
257
+ health_active_log_alarms_2json(localhost, local_buffer);
258
+ tmp = json_tokener_parse(local_buffer->buffer);
259
+ json_object_object_add(payload, "alarms-active", tmp);
260
+
261
+ aclk_send_message_subtopic(client, msg, ACLK_TOPICID_ALARMS);
262
+
263
+ json_object_put(msg);
264
+ freez(msg_id);
265
+ buffer_free(local_buffer);
266
+}
267
+
268
+void aclk_hello_msg(mqtt_wss_client client)
269
+{
270
+ json_object *tmp, *msg;
271
+
272
+ char *msg_id = create_uuid();
273
+
274
+ ACLK_SHARED_STATE_LOCK;
275
+ aclk_shared_state.version_neg = 0;
276
+ aclk_shared_state.version_neg_wait_till = now_monotonic_usec() + USEC_PER_SEC * VERSION_NEG_TIMEOUT;
277
+ ACLK_SHARED_STATE_UNLOCK;
278
+
279
+ //Hello message is versioned separatelly from the rest of the protocol
280
+ msg = create_hdr("hello", msg_id, 0, 0, ACLK_VERSION_NEG_VERSION);
281
+
282
+ tmp = json_object_new_int(ACLK_VERSION_MIN);
283
+ json_object_object_add(msg, "min-version", tmp);
284
+
285
+ tmp = json_object_new_int(ACLK_VERSION_MAX);
286
+ json_object_object_add(msg, "max-version", tmp);
287
+
288
+#ifdef ACLK_NG
289
+ tmp = json_object_new_string("Next Generation");
290
+#else
291
+ tmp = json_object_new_string("Legacy");
292
+#endif
293
+ json_object_object_add(msg, "aclk-implementation", tmp);
294
+
295
+ aclk_send_message_subtopic(client, msg, ACLK_TOPICID_METADATA);
296
+
297
+ json_object_put(msg);
298
+ freez(msg_id);
299
+}
300
+
301
+void aclk_http_msg_v2(mqtt_wss_client client, const char *topic, const char *msg_id, usec_t t_exec, usec_t created, int http_code, const char *payload, size_t payload_len)
302
+{
303
+ json_object *tmp, *msg;
304
+
305
+ msg = create_hdr("http", msg_id, 0, 0, 2);
306
+
307
+ tmp = json_object_new_int64(t_exec);
308
+ json_object_object_add(msg, "t-exec", tmp);
309
+
310
+ tmp = json_object_new_int64(created);
311
+ json_object_object_add(msg, "t-rx", tmp);
312
+
313
+ tmp = json_object_new_int(http_code);
314
+ json_object_object_add(msg, "http-code", tmp);
315
+
316
+ aclk_send_message_with_bin_payload(client, msg, topic, payload, payload_len);
317
+ json_object_put(msg);
318
+}
319
+
320
+void aclk_chart_msg(mqtt_wss_client client, RRDHOST *host, const char *chart)
321
+{
322
+ json_object *msg, *payload;
323
+ BUFFER *tmp_buffer;
324
+ RRDSET *st;
325
+
326
+ st = rrdset_find(host, chart);
327
+ if (!st)
328
+ st = rrdset_find_byname(host, chart);
329
+ if (!st) {
330
+ info("FAILED to find chart %s", chart);
331
+ return;
332
+ }
333
+
334
+ tmp_buffer = buffer_create(BUFFER_INITIAL_SIZE);
335
+ rrdset2json(st, tmp_buffer, NULL, NULL, 1);
336
+ payload = json_tokener_parse(tmp_buffer->buffer);
337
+ if (!payload) {
338
+ error("Failed to parse JSON from rrdset2json");
339
+ buffer_free(tmp_buffer);
340
+ return;
341
+ }
342
+
343
+ msg = create_hdr("chart", NULL, 0, 0, aclk_shared_state.version_neg);
344
+ json_object_object_add(msg, "payload", payload);
345
+
346
+ aclk_send_message_subtopic(client, msg, ACLK_TOPICID_CHART);
347
+
348
+ buffer_free(tmp_buffer);
349
+ json_object_put(msg);
350
+}
351
+
352
+void aclk_alarm_state_msg(mqtt_wss_client client, json_object *msg)
353
+{
354
+ // we create header here on purpose (and not send message with it already as `msg` param)
355
+ // one is version_neg is guaranteed to be done here
356
+ // other are timestamps etc. which in ACLK legacy would be wrong (because ACLK legacy
357
+ // send message with timestamps already to Query Queue they would be incorrect at time
358
+ // when query queue would get to send them)
359
+ json_object *obj = create_hdr("status-change", NULL, 0, 0, aclk_shared_state.version_neg);
360
+ json_object_object_add(obj, "payload", msg);
361
+
362
+ aclk_send_message_subtopic(client, obj, ACLK_TOPICID_ALARMS);
363
+ json_object_put(obj);
364
+}
365
+
366
+/*
367
+ * Will generate disconnect message.
368
+ * @param message if NULL it will generate LWT message (unexpected).
369
+ * Otherwise string pointed to by this parameter will be used as
370
+ * reason.
371
+ */
372
+json_object *aclk_generate_disconnect(const char *message)
373
+{
374
+ json_object *tmp, *msg;
375
+
376
+ msg = create_hdr("disconnect", NULL, 0, 0, 2);
377
+
378
+ tmp = json_object_new_string(message ? message : "unexpected");
379
+ json_object_object_add(msg, "payload", tmp);
380
+
381
+ return msg;
382
+}
383
+
384
+int aclk_send_app_layer_disconnect(mqtt_wss_client client, const char *message)
385
+{
386
+ int pid;
387
+ json_object *msg = aclk_generate_disconnect(message);
388
+ pid = aclk_send_message_subtopic_pid(client, msg, ACLK_TOPICID_METADATA);
389
+ json_object_put(msg);
390
+ return pid;
391
+}
392
+
393
+#ifndef __GNUC__
394
+#pragma endregion
395
+#endif
aclk/aclk_tx_msgs.h
new
+24
@@ -0,0 +1,24 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+#ifndef ACLK_TX_MSGS_H
3
+#define ACLK_TX_MSGS_H
4
+
5
+#include <json-c/json.h>
6
+#include "libnetdata/libnetdata.h"
7
+#include "../daemon/common.h"
8
+#include "mqtt_wss_client.h"
9
+
10
+void aclk_send_info_metadata(mqtt_wss_client client, int metadata_submitted, RRDHOST *host);
11
+void aclk_send_alarm_metadata(mqtt_wss_client client, int metadata_submitted);
12
+
13
+void aclk_hello_msg(mqtt_wss_client client);
14
+
15
+void aclk_http_msg_v2(mqtt_wss_client client, const char *topic, const char *msg_id, usec_t t_exec, usec_t created, int http_code, const char *payload, size_t payload_len);
16
+
17
+void aclk_chart_msg(mqtt_wss_client client, RRDHOST *host, const char *chart);
18
+
19
+void aclk_alarm_state_msg(mqtt_wss_client client, json_object *msg);
20
+
21
+json_object *aclk_generate_disconnect(const char *message);
22
+int aclk_send_app_layer_disconnect(mqtt_wss_client client, const char *message);
23
+
24
+#endif
aclk/aclk_util.c
new
+347
@@ -0,0 +1,347 @@
1
+#include "aclk_util.h"
2
+
3
+#include <stdio.h>
4
+
5
+#include "../daemon/common.h"
6
+
7
+// CentOS 7 has older version that doesn't define this
8
+// same goes for MacOS
9
+#ifndef UUID_STR_LEN
10
+#define UUID_STR_LEN 37
11
+#endif
12
+
13
+#ifdef ACLK_LOG_CONVERSATION_DIR
14
+volatile int aclk_conversation_log_counter = 0;
15
+#if !defined(HAVE_C___ATOMIC) || defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
16
+netdata_mutex_t aclk_conversation_log_mutex = NETDATA_MUTEX_INITIALIZER;
17
+int aclk_get_conv_log_next()
18
+{
19
+ int ret;
20
+ netdata_mutex_lock(&aclk_conversation_log_mutex);
21
+ ret = aclk_conversation_log_counter++;
22
+ netdata_mutex_unlock(&aclk_conversation_log_mutex);
23
+ return ret;
24
+}
25
+#endif
26
+#endif
27
+
28
+#define ACLK_TOPIC_PREFIX "/agent/"
29
+
30
+struct aclk_topic {
31
+ const char *topic_suffix;
32
+ char *topic;
33
+};
34
+
35
+// This helps to cache finalized topics (assembled with claim_id)
36
+// to not have to alloc or create buffer and construct topic every
37
+// time message is sent as in old ACLK
38
+static struct aclk_topic aclk_topic_cache[] = {
39
+ { .topic_suffix = "outbound/meta", .topic = NULL }, // ACLK_TOPICID_CHART
40
+ { .topic_suffix = "outbound/alarms", .topic = NULL }, // ACLK_TOPICID_ALARMS
41
+ { .topic_suffix = "outbound/meta", .topic = NULL }, // ACLK_TOPICID_METADATA
42
+ { .topic_suffix = "inbound/cmd", .topic = NULL }, // ACLK_TOPICID_COMMAND
43
+ { .topic_suffix = NULL, .topic = NULL }
44
+};
45
+
46
+void free_topic_cache(void)
47
+{
48
+ struct aclk_topic *tc = aclk_topic_cache;
49
+ while (tc->topic_suffix) {
50
+ if (tc->topic) {
51
+ freez(tc->topic);
52
+ tc->topic = NULL;
53
+ }
54
+ tc++;
55
+ }
56
+}
57
+
58
+static inline void generate_topic_cache(void)
59
+{
60
+ struct aclk_topic *tc = aclk_topic_cache;
61
+ char *ptr;
62
+ if (unlikely(!tc->topic)) {
63
+ rrdhost_aclk_state_lock(localhost);
64
+ while(tc->topic_suffix) {
65
+ tc->topic = mallocz(strlen(ACLK_TOPIC_PREFIX) + (UUID_STR_LEN - 1) + 2 /* '/' and \0 */ + strlen(tc->topic_suffix));
66
+ ptr = tc->topic;
67
+ strcpy(ptr, ACLK_TOPIC_PREFIX);
68
+ ptr += strlen(ACLK_TOPIC_PREFIX);
69
+ strcpy(ptr, localhost->aclk_state.claimed_id);
70
+ ptr += (UUID_STR_LEN - 1);
71
+ *ptr++ = '/';
72
+ strcpy(ptr, tc->topic_suffix);
73
+ tc++;
74
+ }
75
+ rrdhost_aclk_state_unlock(localhost);
76
+ }
77
+}
78
+
79
+/*
80
+ * Build a topic based on sub_topic and final_topic
81
+ * if the sub topic starts with / assume that is an absolute topic
82
+ *
83
+ */
84
+const char *aclk_get_topic(enum aclk_topics topic)
85
+{
86
+ generate_topic_cache();
87
+
88
+ return aclk_topic_cache[topic].topic;
89
+}
90
+
91
+int aclk_decode_base_url(char *url, char **aclk_hostname, int *aclk_port)
92
+{
93
+ int pos = 0;
94
+ if (!strncmp("https://", url, 8)) {
95
+ pos = 8;
96
+ } else if (!strncmp("http://", url, 7)) {
97
+ error("Cannot connect ACLK over %s -> unencrypted link is not supported", url);
98
+ return 1;
99
+ }
100
+ int host_end = pos;
101
+ while (url[host_end] != 0 && url[host_end] != '/' && url[host_end] != ':')
102
+ host_end++;
103
+ if (url[host_end] == 0) {
104
+ *aclk_hostname = strdupz(url + pos);
105
+ *aclk_port = 443;
106
+ info("Setting ACLK target host=%s port=%d from %s", *aclk_hostname, *aclk_port, url);
107
+ return 0;
108
+ }
109
+ if (url[host_end] == ':') {
110
+ *aclk_hostname = callocz(host_end - pos + 1, 1);
111
+ strncpy(*aclk_hostname, url + pos, host_end - pos);
112
+ int port_end = host_end + 1;
113
+ while (url[port_end] >= '0' && url[port_end] <= '9')
114
+ port_end++;
115
+ if (port_end - host_end > 6) {
116
+ error("Port specified in %s is invalid", url);
117
+ freez(*aclk_hostname);
118
+ *aclk_hostname = NULL;
119
+ return 1;
120
+ }
121
+ *aclk_port = atoi(&url[host_end+1]);
122
+ }
123
+ if (url[host_end] == '/') {
124
+ *aclk_port = 443;
125
+ *aclk_hostname = callocz(1, host_end - pos + 1);
126
+ strncpy(*aclk_hostname, url+pos, host_end - pos);
127
+ }
128
+ info("Setting ACLK target host=%s port=%d from %s", *aclk_hostname, *aclk_port, url);
129
+ return 0;
130
+}
131
+
132
+/*
133
+ * TBEB with randomness
134
+ *
135
+ * @param mode 0 - to reset the delay,
136
+ * 1 - to advance a step and calculate sleep time [0 .. ACLK_MAX_BACKOFF_DELAY * 1000] ms
137
+ * @returns delay in ms
138
+ *
139
+ */
140
+#define ACLK_MAX_BACKOFF_DELAY 1024
141
+unsigned long int aclk_reconnect_delay(int mode)
142
+{
143
+ static int fail = -1;
144
+ unsigned long int delay;
145
+
146
+ if (!mode || fail == -1) {
147
+ srandom(time(NULL));
148
+ fail = mode - 1;
149
+ return 0;
150
+ }
151
+
152
+ delay = (1 << fail);
153
+
154
+ if (delay >= ACLK_MAX_BACKOFF_DELAY) {
155
+ delay = ACLK_MAX_BACKOFF_DELAY * 1000;
156
+ } else {
157
+ fail++;
158
+ delay *= 1000;
159
+ delay += (random() % (MAX(1000, delay/2)));
160
+ }
161
+
162
+ return delay;
163
+}
164
+
165
+#define ACLK_PROXY_PROTO_ADDR_SEPARATOR "://"
166
+#define ACLK_PROXY_ENV "env"
167
+#define ACLK_PROXY_CONFIG_VAR "proxy"
168
+
169
+struct {
170
+ ACLK_PROXY_TYPE type;
171
+ const char *url_str;
172
+} supported_proxy_types[] = {
173
+ { .type = PROXY_TYPE_SOCKS5, .url_str = "socks5" ACLK_PROXY_PROTO_ADDR_SEPARATOR },
174
+ { .type = PROXY_TYPE_SOCKS5, .url_str = "socks5h" ACLK_PROXY_PROTO_ADDR_SEPARATOR },
175
+ { .type = PROXY_TYPE_HTTP, .url_str = "http" ACLK_PROXY_PROTO_ADDR_SEPARATOR },
176
+ { .type = PROXY_TYPE_UNKNOWN, .url_str = NULL },
177
+};
178
+
179
+const char *aclk_proxy_type_to_s(ACLK_PROXY_TYPE *type)
180
+{
181
+ switch (*type) {
182
+ case PROXY_DISABLED:
183
+ return "disabled";
184
+ case PROXY_TYPE_HTTP:
185
+ return "HTTP";
186
+ case PROXY_TYPE_SOCKS5:
187
+ return "SOCKS";
188
+ default:
189
+ return "Unknown";
190
+ }
191
+}
192
+
193
+static inline ACLK_PROXY_TYPE aclk_find_proxy(const char *string)
194
+{
195
+ int i = 0;
196
+ while (supported_proxy_types[i].url_str) {
197
+ if (!strncmp(supported_proxy_types[i].url_str, string, strlen(supported_proxy_types[i].url_str)))
198
+ return supported_proxy_types[i].type;
199
+ i++;
200
+ }
201
+ return PROXY_TYPE_UNKNOWN;
202
+}
203
+
204
+ACLK_PROXY_TYPE aclk_verify_proxy(const char *string)
205
+{
206
+ if (!string)
207
+ return PROXY_TYPE_UNKNOWN;
208
+
209
+ while (*string == 0x20 && *string!=0) // Help coverity (compiler will remove)
210
+ string++;
211
+
212
+ if (!*string)
213
+ return PROXY_TYPE_UNKNOWN;
214
+
215
+ return aclk_find_proxy(string);
216
+}
217
+
218
+// helper function to censor user&password
219
+// for logging purposes
220
+void safe_log_proxy_censor(char *proxy)
221
+{
222
+ size_t length = strlen(proxy);
223
+ char *auth = proxy + length - 1;
224
+ char *cur;
225
+
226
+ while ((auth >= proxy) && (*auth != '@'))
227
+ auth--;
228
+
229
+ //if not found or @ is first char do nothing
230
+ if (auth <= proxy)
231
+ return;
232
+
233
+ cur = strstr(proxy, ACLK_PROXY_PROTO_ADDR_SEPARATOR);
234
+ if (!cur)
235
+ cur = proxy;
236
+ else
237
+ cur += strlen(ACLK_PROXY_PROTO_ADDR_SEPARATOR);
238
+
239
+ while (cur < auth) {
240
+ *cur = 'X';
241
+ cur++;
242
+ }
243
+}
244
+
245
+static inline void safe_log_proxy_error(char *str, const char *proxy)
246
+{
247
+ char *log = strdupz(proxy);
248
+ safe_log_proxy_censor(log);
249
+ error("%s Provided Value:\"%s\"", str, log);
250
+ freez(log);
251
+}
252
+
253
+static inline int check_socks_enviroment(const char **proxy)
254
+{
255
+ char *tmp = getenv("socks_proxy");
256
+
257
+ if (!tmp)
258
+ return 1;
259
+
260
+ if (aclk_verify_proxy(tmp) == PROXY_TYPE_SOCKS5) {
261
+ *proxy = tmp;
262
+ return 0;
263
+ }
264
+
265
+ safe_log_proxy_error(
266
+ "Environment var \"socks_proxy\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".",
267
+ tmp);
268
+ return 1;
269
+}
270
+
271
+static inline int check_http_enviroment(const char **proxy)
272
+{
273
+ char *tmp = getenv("http_proxy");
274
+
275
+ if (!tmp)
276
+ return 1;
277
+
278
+ if (aclk_verify_proxy(tmp) == PROXY_TYPE_HTTP) {
279
+ *proxy = tmp;
280
+ return 0;
281
+ }
282
+
283
+ safe_log_proxy_error(
284
+ "Environment var \"http_proxy\" defined but of unknown format. Supported syntax: \"http[s]://[user:pass@]host:ip\".",
285
+ tmp);
286
+ return 1;
287
+}
288
+
289
+const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type)
290
+{
291
+ const char *proxy = config_get(CONFIG_SECTION_CLOUD, ACLK_PROXY_CONFIG_VAR, ACLK_PROXY_ENV);
292
+ *type = PROXY_DISABLED;
293
+
294
+ if (strcmp(proxy, "none") == 0)
295
+ return proxy;
296
+
297
+ if (strcmp(proxy, ACLK_PROXY_ENV) == 0) {
298
+ if (check_socks_enviroment(&proxy) == 0) {
299
+#ifdef LWS_WITH_SOCKS5
300
+ *type = PROXY_TYPE_SOCKS5;
301
+ return proxy;
302
+#else
303
+ safe_log_proxy_error("socks_proxy environment variable set to use SOCKS5 proxy "
304
+ "but Libwebsockets used doesn't have SOCKS5 support built in. "
305
+ "Ignoring and checking for other options.",
306
+ proxy);
307
+#endif
308
+ }
309
+ if (check_http_enviroment(&proxy) == 0)
310
+ *type = PROXY_TYPE_HTTP;
311
+ return proxy;
312
+ }
313
+
314
+ *type = aclk_verify_proxy(proxy);
315
+#ifndef LWS_WITH_SOCKS5
316
+ if (*type == PROXY_TYPE_SOCKS5) {
317
+ safe_log_proxy_error(
318
+ "Config var \"" ACLK_PROXY_CONFIG_VAR
319
+ "\" set to use SOCKS5 proxy but Libwebsockets used is built without support for SOCKS proxy. ACLK will be disabled.",
320
+ proxy);
321
+ }
322
+#endif
323
+ if (*type == PROXY_TYPE_UNKNOWN) {
324
+ *type = PROXY_DISABLED;
325
+ safe_log_proxy_error(
326
+ "Config var \"" ACLK_PROXY_CONFIG_VAR
327
+ "\" defined but of unknown format. Supported syntax: \"socks5[h]://[user:pass@]host:ip\".",
328
+ proxy);
329
+ }
330
+
331
+ return proxy;
332
+}
333
+
334
+// helper function to read settings only once (static)
335
+// as claiming, challenge/response and ACLK
336
+// read the same thing, no need to parse again
337
+const char *aclk_get_proxy(ACLK_PROXY_TYPE *type)
338
+{
339
+ static const char *proxy = NULL;
340
+ static ACLK_PROXY_TYPE proxy_type = PROXY_NOT_SET;
341
+
342
+ if (proxy_type == PROXY_NOT_SET)
343
+ proxy = aclk_lws_wss_get_proxy_setting(&proxy_type);
344
+
345
+ *type = proxy_type;
346
+ return proxy;
347
+}
aclk/aclk_util.h
new
+52
@@ -0,0 +1,52 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+#ifndef ACLK_UTIL_H
3
+#define ACLK_UTIL_H
4
+
5
+#include "libnetdata/libnetdata.h"
6
+
7
+// Helper stuff which should not have any further inside ACLK dependency
8
+// and are supposed not to be needed outside of ACLK
9
+
10
+int aclk_decode_base_url(char *url, char **aclk_hostname, int *aclk_port);
11
+
12
+enum aclk_topics {
13
+ ACLK_TOPICID_CHART = 0,
14
+ ACLK_TOPICID_ALARMS = 1,
15
+ ACLK_TOPICID_METADATA = 2,
16
+ ACLK_TOPICID_COMMAND = 3,
17
+};
18
+
19
+const char *aclk_get_topic(enum aclk_topics topic);
20
+void free_topic_cache(void);
21
+// TODO
22
+// aclk_topics_reload //when claim id changes
23
+
24
+#ifdef ACLK_LOG_CONVERSATION_DIR
25
+extern volatile int aclk_conversation_log_counter;
26
+#if defined(HAVE_C___ATOMIC) && !defined(NETDATA_NO_ATOMIC_INSTRUCTIONS)
27
+#define ACLK_GET_CONV_LOG_NEXT() __atomic_fetch_add(&aclk_conversation_log_counter, 1, __ATOMIC_SEQ_CST)
28
+#else
29
+extern netdata_mutex_t aclk_conversation_log_mutex;
30
+int aclk_get_conv_log_next();
31
+#define ACLK_GET_CONV_LOG_NEXT() aclk_get_conv_log_next()
32
+#endif
33
+#endif
34
+
35
+unsigned long int aclk_reconnect_delay(int mode);
36
+
37
+typedef enum aclk_proxy_type {
38
+ PROXY_TYPE_UNKNOWN = 0,
39
+ PROXY_TYPE_SOCKS5,
40
+ PROXY_TYPE_HTTP,
41
+ PROXY_DISABLED,
42
+ PROXY_NOT_SET,
43
+} ACLK_PROXY_TYPE;
44
+
45
+const char *aclk_proxy_type_to_s(ACLK_PROXY_TYPE *type);
46
+ACLK_PROXY_TYPE aclk_verify_proxy(const char *string);
47
+const char *aclk_lws_wss_get_proxy_setting(ACLK_PROXY_TYPE *type);
48
+void safe_log_proxy_censor(char *proxy);
49
+int aclk_decode_base_url(char *url, char **aclk_hostname, int *aclk_port);
50
+const char *aclk_get_proxy(ACLK_PROXY_TYPE *type);
51
+
52
+#endif /* ACLK_UTIL_H */
aclk/legacy/aclk_lws_https_client.c
+4
@@ -3,7 +3,11 @@
3
#define ACLK_LWS_HTTPS_CLIENT_INTERNAL
4
#include "aclk_lws_https_client.h"
5
6
+#ifndef ACLK_NG
7
#include "aclk_common.h"
8
+#else
9
+#include "../aclk.h"
10
+#endif
11
12
#include "aclk_lws_wss_client.h"
13
build/subst.inc
+1
-1
@@ -10,7 +10,7 @@
10
-e 's#[@]registrydir_POST@#$(registrydir)#g' \
11
-e 's#[@]varlibdir_POST@#$(varlibdir)#g' \
12
-e 's#[@]webdir_POST@#$(webdir)#g' \
13
- -e 's#[@]can_enable_aclk_POST@#$(can_enable_aclk)#g' \
13
+ -e 's#[@]enable_aclk_POST@#$(enable_aclk)#g' \
14
-e 's#[@]enable_cloud_POST@#$(enable_cloud)#g' \
15
$< > $@.tmp; then \
16
mv "$@.tmp" "$@"; \
claim/claim.c
+4
@@ -2,7 +2,11 @@
2
3
#include "claim.h"
4
#include "../registry/registry_internals.h"
5
+#ifndef ACLK_NG
6
#include "../aclk/legacy/aclk_common.h"
7
+#else
8
+#include "../aclk/aclk.h"
9
+#endif
10
11
char *claiming_pending_arguments = NULL;
12
claim/netdata-claim.sh.in
+1
-1
@@ -123,7 +123,7 @@ if [ "@enable_cloud_POST@" = "no" ]; then
123
exit 3
124
fi
125
# shellcheck disable=SC2050
126
-if [ "@can_enable_aclk_POST@" != "yes" ]; then
126
+if [ "@enable_aclk_POST@" != "yes" ]; then
127
echo >&2 "This agent was built without the dependencies for Cloud and cannot be claimed"
128
exit 3
129
fi
configure.ac
+42
-3
@@ -197,6 +197,14 @@ AC_ARG_ENABLE(
197
[ enable_cloud="detect" ]
198
)
199
200
+AC_ARG_WITH(
201
+ [aclk-ng],
202
+ [AS_HELP_STRING([--with-aclk-ng],
203
+ [Requires ACLK-NG to be used even in case ACLK Legacy can run on this system])],
204
+ [aclk_ng="$withval"],
205
+ [aclk_ng="fallback"]
206
+)
207
+
208
if test "${enable_cloud}" = "no"; then
209
AC_DEFINE([DISABLE_CLOUD], [1], [disable netdata cloud functionality])
210
fi
@@ -632,7 +640,7 @@ AM_CONDITIONAL([ENABLE_CAPABILITY], [test "${with_libcap}" = "yes"])
640
641
AC_MSG_CHECKING([if cloud functionality should be enabled])
642
AC_MSG_RESULT([${enable_cloud}])
635
-if test "$enable_cloud" != "no"; then
643
+if test "$enable_cloud" != "no" -a "$aclk_ng" != "yes"; then
644
# just to have all messages that can fail ACLK build in one place
645
# so it is easier to see why it can't be built
646
if test -n "${SSL_LIBS}"; then
@@ -706,6 +714,7 @@ if test "$enable_cloud" != "no"; then
714
fi
715
AC_MSG_RESULT([${can_enable_aclk}])
716
717
+# TODO fix this (you need to try fallback)
718
test "${enable_cloud}" = "yes" -a "${can_enable_aclk}" = "no" && \
719
AC_MSG_ERROR([User required agent-cloud-link but it can't be built!])
720
@@ -715,7 +724,6 @@ if test "$enable_cloud" != "no"; then
724
else
725
enable_aclk=$enable_cloud
726
fi
718
- AC_SUBST([can_enable_aclk])
727
728
if test "${enable_aclk}" = "yes"; then
729
AC_DEFINE([ENABLE_ACLK], [1], [netdata ACLK])
@@ -723,7 +731,35 @@ if test "$enable_cloud" != "no"; then
731
732
AC_MSG_RESULT([${enable_aclk}])
733
fi
734
+
735
+if test "$enable_cloud" = "no" -a "$aclk_ng" = "yes"; then
736
+ AC_MSG_ERROR([--disable-cloud && --aclk-ng not allowed together (such configuration is self contradicting)])
737
+fi
738
+
739
+if test "$enable_cloud" != "no" -a "$aclk_ng" != "no"; then
740
+ AC_MSG_CHECKING([if JSON-C available for ACLK Next Generation])
741
+ if test "$enable_jsonc" != "yes"; then
742
+ AC_MSG_RESULT([no])
743
+ else
744
+ AC_MSG_RESULT([yes])
745
+ if test "$aclk_ng" != "yes" -a "$enable_aclk" == "no"; then #default "fallback"
746
+ AC_MSG_NOTICE([ACLK Legacy could not be built. Trying ACLK-NG as fallback.])
747
+ aclk_ng="yes"
748
+ fi
749
+ if test "$aclk_ng" = "yes"; then
750
+ #TODO Check OpenSSL and JSON-C
751
+ AC_MSG_CHECKING([if ACLK Next Generation can be built])
752
+ AC_DEFINE([ACLK_NG], [1], [ACLK Next Generation Should be used])
753
+ AC_DEFINE([ENABLE_ACLK], [1], [netdata ACLK])
754
+ enable_aclk="yes"
755
+ AC_MSG_RESULT([yes])
756
+ OPTIONAL_MQTT_WSS_CFLAGS="-Imqtt_websockets/src/include"
757
+ fi
758
+ fi
759
+fi
760
AC_SUBST([enable_cloud])
761
+AC_SUBST([enable_aclk])
762
+AM_CONDITIONAL([ACLK_NG], [test "${aclk_ng}" = "yes"])
763
AM_CONDITIONAL([ENABLE_ACLK], [test "${enable_aclk}" = "yes"])
764
765
# -----------------------------------------------------------------------------
@@ -1420,7 +1456,8 @@ AC_SUBST([webdir])
1456
CFLAGS="${CFLAGS} ${OPTIONAL_MATH_CFLAGS} ${OPTIONAL_NFACCT_CFLAGS} ${OPTIONAL_ZLIB_CFLAGS} ${OPTIONAL_UUID_CFLAGS} \
1457
${OPTIONAL_LIBCAP_CFLAGS} ${OPTIONAL_IPMIMONITORING_CFLAGS} ${OPTIONAL_CUPS_CFLAGS} ${OPTIONAL_XENSTAT_FLAGS} \
1458
${OPTIONAL_KINESIS_CFLAGS} ${OPTIONAL_PUBSUB_CFLAGS} ${OPTIONAL_PROMETHEUS_REMOTE_WRITE_CFLAGS} \
1423
- ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${OPTIONAL_JUDY_CFLAGS}"
1459
+ ${OPTIONAL_MONGOC_CFLAGS} ${LWS_CFLAGS} ${OPTIONAL_JSONC_STATIC_CFLAGS} ${OPTIONAL_BPF_CFLAGS} ${OPTIONAL_JUDY_CFLAGS} \
1460
+ ${OPTIONAL_MQTT_WSS_CFLAGS}"
1461
1462
CXXFLAGS="${CFLAGS} ${CXX11FLAG}"
1463
@@ -1470,6 +1507,7 @@ AC_SUBST([OPTIONAL_PROMETHEUS_REMOTE_WRITE_LIBS])
1507
AC_SUBST([OPTIONAL_MONGOC_CFLAGS])
1508
AC_SUBST([OPTIONAL_MONGOC_LIBS])
1509
AC_SUBST([OPTIONAL_LWS_LIBS])
1510
+AC_SUBST([OPTIONAL_MQTT_WSS_CFLAGS])
1511
1512
# -----------------------------------------------------------------------------
1513
# Check if cmocka is available - needed for unit testing
@@ -1615,6 +1653,7 @@ AC_CONFIG_FILES([
1653
spawn/Makefile
1654
parser/Makefile
1655
])
1656
+
1657
AC_OUTPUT
1658
1659
test "${with_math}" != "yes" && AC_MSG_WARN([You are building without math. math allows accurate calculations. It should be enabled.]) || :
daemon/buildinfo.c
+18
@@ -8,7 +8,13 @@
8
#ifdef ENABLE_ACLK
9
#define FEAT_CLOUD 1
10
#define FEAT_CLOUD_MSG ""
11
+#ifdef ACLK_NG
12
+#define ACLK_IMPL "Next Generation"
13
#else
14
+#define ACLK_IMPL "Legacy"
15
+#endif
16
+#else
17
+#define ACLK_IMPL ""
18
#ifdef DISABLE_CLOUD
19
#define FEAT_CLOUD 0
20
#define FEAT_CLOUD_MSG "(by user request)"
@@ -62,6 +68,7 @@
68
#define FEAT_LIBCAP 0
69
#endif
70
71
+#ifndef ACLK_NG
72
#ifdef ACLK_NO_LIBMOSQ
73
#define FEAT_MOSQUITTO 0
74
#else
@@ -83,6 +90,7 @@
90
#define FEAT_LWS_MSG "shared-lib"
91
#endif
92
#endif
93
+#endif /* ACLK_NG */
94
95
#ifdef NETDATA_WITH_ZLIB
96
#define FEAT_ZLIB 1
@@ -199,6 +207,9 @@ void print_build_info(void) {
207
printf(" dbengine: %s\n", FEAT_YES_NO(FEAT_DBENGINE));
208
printf(" Native HTTPS: %s\n", FEAT_YES_NO(FEAT_NATIVE_HTTPS));
209
printf(" Netdata Cloud: %s %s\n", FEAT_YES_NO(FEAT_CLOUD), FEAT_CLOUD_MSG);
210
+#if FEAT_CLOUD == 1
211
+ printf(" Cloud Implementation: %s\n", ACLK_IMPL);
212
+#endif
213
printf(" TLS Host Verification: %s\n", FEAT_YES_NO(FEAT_TLS_HOST_VERIFY));
214
215
printf("Libraries:\n");
@@ -207,12 +218,14 @@ void print_build_info(void) {
218
printf(" libcap: %s\n", FEAT_YES_NO(FEAT_LIBCAP));
219
printf(" libcrypto: %s\n", FEAT_YES_NO(FEAT_CRYPTO));
220
printf(" libm: %s\n", FEAT_YES_NO(FEAT_LIBM));
221
+#ifndef ACLK_NG
222
#if defined(ENABLE_ACLK)
223
printf(" LWS: %s %s v%d.%d.%d\n", FEAT_YES_NO(FEAT_LWS), FEAT_LWS_MSG, LWS_LIBRARY_VERSION_MAJOR, LWS_LIBRARY_VERSION_MINOR, LWS_LIBRARY_VERSION_PATCH);
224
#else
225
printf(" LWS: %s %s\n", FEAT_YES_NO(FEAT_LWS), FEAT_LWS_MSG);
226
#endif
227
printf(" mosquitto: %s\n", FEAT_YES_NO(FEAT_MOSQUITTO));
228
+#endif
229
printf(" tcalloc: %s\n", FEAT_YES_NO(FEAT_TCMALLOC));
230
printf(" zlib: %s\n", FEAT_YES_NO(FEAT_ZLIB));
231
@@ -251,6 +264,9 @@ void print_build_info_json(void) {
264
printf(" \"cloud-disabled\": true,\n");
265
#else
266
printf(" \"cloud-disabled\": false,\n");
267
+#endif
268
+#if FEAT_CLOUD == 1
269
+ printf(" \"cloud-implementation\": \"%s\",\n", ACLK_IMPL);
270
#endif
271
printf(" \"tls-host-verify\": %s\n", FEAT_JSON_BOOL(FEAT_TLS_HOST_VERIFY));
272
printf(" },\n");
@@ -261,6 +277,7 @@ void print_build_info_json(void) {
277
printf(" \"libcap\": %s,\n", FEAT_JSON_BOOL(FEAT_LIBCAP));
278
printf(" \"libcrypto\": %s,\n", FEAT_JSON_BOOL(FEAT_CRYPTO));
279
printf(" \"libm\": %s,\n", FEAT_JSON_BOOL(FEAT_LIBM));
280
+#ifndef ACLK_NG
281
#if defined(ENABLE_ACLK)
282
printf(" \"lws\": %s,\n", FEAT_JSON_BOOL(FEAT_LWS));
283
printf(" \"lws-version\": \"%d.%d.%d\",\n", LWS_LIBRARY_VERSION_MAJOR, LWS_LIBRARY_VERSION_MINOR, LWS_LIBRARY_VERSION_PATCH);
@@ -269,6 +286,7 @@ void print_build_info_json(void) {
286
printf(" \"lws\": %s,\n", FEAT_JSON_BOOL(FEAT_LWS));
287
#endif
288
printf(" \"mosquitto\": %s,\n", FEAT_JSON_BOOL(FEAT_MOSQUITTO));
289
+#endif
290
printf(" \"tcmalloc\": %s,\n", FEAT_JSON_BOOL(FEAT_TCMALLOC));
291
printf(" \"zlib\": %s\n", FEAT_JSON_BOOL(FEAT_ZLIB));
292
printf(" },\n");
daemon/common.h
+4
@@ -66,7 +66,11 @@
66
#include "claim/claim.h"
67
68
// netdata agent cloud link
69
+#ifndef ACLK_NG
70
#include "aclk/legacy/agent_cloud_link.h"
71
+#else
72
+#include "aclk/aclk.h"
73
+#endif
74
75
// global GUID map functions
76
daemon/main.c
+1
-1
@@ -80,7 +80,7 @@ struct netdata_static_thread static_threads[] = {
80
NETDATA_PLUGIN_HOOK_IDLEJITTER
81
NETDATA_PLUGIN_HOOK_STATSD
82
83
-#ifdef ENABLE_ACLK
83
+#if defined(ENABLE_ACLK) || defined(ACLK_NG)
84
NETDATA_ACLK_HOOK
85
#endif
86
database/rrd.h
+5
@@ -34,7 +34,12 @@ struct pg_cache_page_index;
34
#include "rrdcalc.h"
35
#include "rrdcalctemplate.h"
36
#include "../streaming/rrdpush.h"
37
+
38
+#ifndef ACLK_NG
39
#include "../aclk/legacy/aclk_rrdhost_state.h"
40
+#else
41
+#include "aclk/aclk.h"
42
+#endif
43
44
struct context_param {
45
RRDDIM *rd;
health/health_json.c
+2
-2
@@ -2,7 +2,7 @@
2
3
#include "health.h"
4
5
-static inline void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
5
+void health_string2json(BUFFER *wb, const char *prefix, const char *label, const char *value, const char *suffix) {
6
if(value && *value) {
7
buffer_sprintf(wb, "%s\"%s\":\"", prefix, label);
8
buffer_strcat_htmlescape(wb, value);
@@ -13,7 +13,7 @@ static inline void health_string2json(BUFFER *wb, const char *prefix, const char
13
buffer_sprintf(wb, "%s\"%s\":null%s", prefix, label, suffix);
14
}
15
16
-inline void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
16
+void health_alarm_entry2json_nolock(BUFFER *wb, ALARM_ENTRY *ae, RRDHOST *host) {
17
buffer_sprintf(wb,
18
"\n\t{\n"
19
"\t\t\"hostname\": \"%s\",\n"
mqtt_websockets
new
+1
@@ -0,0 +1 @@
1
+Subproject commit 62a4af9cc858cb68b4c1e87eabe2908644e34a47
netdata-installer.sh
+7
-3
@@ -319,6 +319,10 @@ while [ -n "${1}" ]; do
319
"--disable-go") NETDATA_DISABLE_GO=1 ;;
320
"--enable-ebpf") NETDATA_DISABLE_EBPF=0 ;;
321
"--disable-ebpf") NETDATA_DISABLE_EBPF=1 NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--disable-ebpf/} --disable-ebpf" ;;
322
+ "--aclk-ng")
323
+ NETDATA_ACLK_NG=1
324
+ NETDATA_CONFIGURE_OPTIONS="${NETDATA_CONFIGURE_OPTIONS//--with-aclk-ng/} --with-aclk-ng"
325
+ ;;
326
"--disable-cloud")
327
if [ -n "${NETDATA_REQUIRE_CLOUD}" ]; then
328
echo "Cloud explicitly enabled, ignoring --disable-cloud."
@@ -567,8 +571,8 @@ copy_libmosquitto() {
571
}
572
573
bundle_libmosquitto() {
570
- if [ -n "${NETDATA_DISABLE_CLOUD}" ]; then
571
- echo "Skipping cloud"
574
+ if [ -n "${NETDATA_DISABLE_CLOUD}" ] || [ -n "${NETDATA_ACLK_NG}" ]; then
575
+ echo "Skipping libmosquitto"
576
return 0
577
fi
578
@@ -664,7 +668,7 @@ copy_libwebsockets() {
668
}
669
670
bundle_libwebsockets() {
667
- if [ -n "${NETDATA_DISABLE_CLOUD}" ] || [ -n "${USE_SYSTEM_LWS}" ]; then
671
+ if [ -n "${NETDATA_DISABLE_CLOUD}" ] || [ -n "${USE_SYSTEM_LWS}" ] || [ -n "${NETDATA_ACLK_NG}" ]; then
672
return 0
673
fi
674
streaming/receiver.c
+2
-2
@@ -440,7 +440,7 @@ static int rrdpush_receive(struct receiver_state *rpt)
440
441
cd.version = rpt->stream_version;
442
443
-#ifdef ENABLE_ACLK
443
+#if defined(ENABLE_ACLK) && !defined(ACLK_NG)
444
// in case we have cloud connection we inform cloud
445
// new slave connected
446
if (netdata_cloud_setting)
@@ -454,7 +454,7 @@ static int rrdpush_receive(struct receiver_state *rpt)
454
error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", rpt->hostname, rpt->client_ip,
455
rpt->client_port, count);
456
457
-#ifdef ENABLE_ACLK
457
+#if defined(ENABLE_ACLK) && !defined(ACLK_NG)
458
// in case we have cloud connection we inform cloud
459
// new slave connected
460
if (netdata_cloud_setting)
web/api/web_api_v1.c
+5
@@ -961,6 +961,11 @@ inline int web_client_api_request_v1_info_fill_buffer(RRDHOST *host, BUFFER *wb)
961
962
#ifdef ENABLE_ACLK
963
buffer_strcat(wb, "\t\"cloud-available\": true,\n");
964
+#ifdef ACLK_NG
965
+ buffer_strcat(wb, "\t\"aclk-implementation\": \"Next Generation\",\n");
966
+#else
967
+ buffer_strcat(wb, "\t\"aclk-implementation\": \"legacy\",\n");
968
+#endif
969
#else
970
buffer_strcat(wb, "\t\"cloud-available\": false,\n");
971
#endif