Adds metrics for ACLK performance and status (#9269)
Adds ACLK charts
Timotej S committed
Jun 11, 2020 at 12:21 UTC
ba350b7554d8a245bc488ed40409e594c4dd6f62
8 files changed
+350
-2
CMakeLists.txt
+2
@@ -651,6 +651,8 @@ set(ACLK_PLUGIN_FILES
651
aclk/aclk_lws_https_client.h
652
aclk/mqtt.c
653
aclk/mqtt.h
654
+ aclk/aclk_stats.c
655
+ aclk/aclk_stats.h
656
)
657
658
set(SPAWN_PLUGIN_FILES
Makefile.am
+2
@@ -494,6 +494,8 @@ PARSER_FILES = \
494
ACLK_FILES = \
495
aclk/aclk_common.c \
496
aclk/aclk_common.h \
497
+ aclk/aclk_stats.c \
498
+ aclk/aclk_stats.h
499
$(NULL)
500
501
if ENABLE_ACLK
aclk/aclk_lws_wss_client.c
+25
-1
@@ -5,6 +5,7 @@
5
#include "libnetdata/libnetdata.h"
6
#include "../daemon/common.h"
7
#include "aclk_common.h"
8
+#include "aclk_stats.h"
9
10
extern int aclk_shutting_down;
11
@@ -436,8 +437,14 @@ static int aclk_lws_wss_callback(struct lws *wsi, enum lws_callback_reasons reas
437
if ( bytes_left > FRAGMENT_SIZE)
438
bytes_left = FRAGMENT_SIZE;
439
int n = lws_write(wsi, data->data + LWS_PRE + data->written, bytes_left, LWS_WRITE_BINARY);
439
- if (n>=0)
440
+ if (n>=0) {
441
data->written += n;
442
+ if (aclk_stats_enabled) {
443
+ ACLK_STATS_LOCK;
444
+ aclk_metrics_per_sample.write_q_consumed += n;
445
+ ACLK_STATS_UNLOCK;
446
+ }
447
+ }
448
//error("lws_write(req=%u,written=%u) %zu of %zu",bytes_left, rc, data->written,data->data_size,rc);
449
if (data->written == data->data_size)
450
{
@@ -455,6 +462,11 @@ static int aclk_lws_wss_callback(struct lws *wsi, enum lws_callback_reasons reas
462
if (!received_data_to_ringbuff(engine_instance->read_ringbuffer, in, len))
463
retval = 1;
464
aclk_lws_mutex_unlock(&engine_instance->read_buf_mutex);
465
+ if (aclk_stats_enabled) {
466
+ ACLK_STATS_LOCK;
467
+ aclk_metrics_per_sample.read_q_added += len;
468
+ ACLK_STATS_UNLOCK;
469
+ }
470
471
// to future myself -> do not call this while read lock is active as it will eventually
472
// want to acquire same lock later in aclk_lws_wss_client_read() function
@@ -524,6 +536,12 @@ int aclk_lws_wss_client_write(void *buf, size_t count)
536
lws_wss_packet_buffer_append(&engine_instance->write_buffer_head, lws_wss_packet_buffer_new(buf, count));
537
aclk_lws_mutex_unlock(&engine_instance->write_buf_mutex);
538
539
+ if (aclk_stats_enabled) {
540
+ ACLK_STATS_LOCK;
541
+ aclk_metrics_per_sample.write_q_added += count;
542
+ ACLK_STATS_UNLOCK;
543
+ }
544
+
545
lws_callback_on_writable(engine_instance->lws_wsi);
546
return count;
547
}
@@ -549,6 +567,12 @@ int aclk_lws_wss_client_read(void *buf, size_t count)
567
if (data_to_be_read == readable_byte_count)
568
engine_instance->data_to_read = 0;
569
570
+ if (aclk_stats_enabled) {
571
+ ACLK_STATS_LOCK;
572
+ aclk_metrics_per_sample.read_q_consumed += data_to_be_read;
573
+ ACLK_STATS_UNLOCK;
574
+ }
575
+
576
abort:
577
aclk_lws_mutex_unlock(&engine_instance->read_buf_mutex);
578
return data_to_be_read;
aclk/aclk_stats.c
new
+195
@@ -0,0 +1,195 @@
1
+#include "aclk_stats.h"
2
+
3
+netdata_mutex_t aclk_stats_mutex = NETDATA_MUTEX_INITIALIZER;
4
+
5
+int aclk_stats_enabled;
6
+
7
+struct aclk_metrics aclk_metrics = {
8
+ .online = 0,
9
+};
10
+
11
+struct aclk_metrics_per_sample aclk_metrics_per_sample;
12
+
13
+static void aclk_stats_collect(struct aclk_metrics_per_sample *per_sample, struct aclk_metrics *permanent)
14
+{
15
+ static RRDSET *st_aclkstats = NULL;
16
+ static RRDDIM *rd_online_status = NULL;
17
+
18
+ if (unlikely(!st_aclkstats)) {
19
+ st_aclkstats = rrdset_create_localhost(
20
+ "netdata", "aclk_status", NULL, "aclk_stats", NULL, "ACLK/Cloud connection status",
21
+ "connected", "netdata", "stats", 200000, localhost->rrd_update_every, RRDSET_TYPE_LINE);
22
+
23
+ rd_online_status = rrddim_add(st_aclkstats, "online", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
24
+ } else
25
+ rrdset_next(st_aclkstats);
26
+
27
+ rrddim_set_by_pointer(st_aclkstats, rd_online_status, per_sample->offline_during_sample ? 0 : permanent->online);
28
+
29
+ rrdset_done(st_aclkstats);
30
+}
31
+
32
+static void aclk_stats_query_thread(struct aclk_metrics_per_sample *per_sample)
33
+{
34
+ static RRDSET *st_query_thread = NULL;
35
+ static RRDDIM *rd_queued = NULL;
36
+ static RRDDIM *rd_dispatched = NULL;
37
+
38
+ if (unlikely(!st_query_thread)) {
39
+ st_query_thread = rrdset_create_localhost(
40
+ "netdata", "aclk_query_per_second", NULL, "aclk_stats", NULL, "ACLK Queries per second", "queries/s",
41
+ "netdata", "stats", 200001, localhost->rrd_update_every, RRDSET_TYPE_AREA);
42
+
43
+ rd_queued = rrddim_add(st_query_thread, "added", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
44
+ rd_dispatched = rrddim_add(st_query_thread, "dispatched", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
45
+ } else
46
+ rrdset_next(st_query_thread);
47
+
48
+ rrddim_set_by_pointer(st_query_thread, rd_queued, per_sample->queries_queued);
49
+ rrddim_set_by_pointer(st_query_thread, rd_dispatched, -per_sample->queries_dispatched);
50
+
51
+ rrdset_done(st_query_thread);
52
+}
53
+
54
+#ifdef NETDATA_INTERNAL_CHECKS
55
+static void aclk_stats_latency(struct aclk_metrics_per_sample *per_sample)
56
+{
57
+ static RRDSET *st = NULL;
58
+ static RRDDIM *rd_avg = NULL;
59
+ static RRDDIM *rd_max = NULL;
60
+
61
+ if (unlikely(!st)) {
62
+ st = rrdset_create_localhost(
63
+ "netdata", "aclk_latency_mqtt", NULL, "aclk_stats", NULL, "ACLK Message Publish Latency", "ms",
64
+ "netdata", "stats", 200002, localhost->rrd_update_every, RRDSET_TYPE_LINE);
65
+
66
+ rd_avg = rrddim_add(st, "avg", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
67
+ rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
68
+ } else
69
+ rrdset_next(st);
70
+ if(per_sample->latency_count)
71
+ rrddim_set_by_pointer(st, rd_avg, roundf((float)per_sample->latency_total / per_sample->latency_count));
72
+ else
73
+ rrddim_set_by_pointer(st, rd_avg, 0);
74
+
75
+ rrddim_set_by_pointer(st, rd_max, per_sample->latency_max);
76
+
77
+ rrdset_done(st);
78
+}
79
+#endif
80
+
81
+static void aclk_stats_write_q(struct aclk_metrics_per_sample *per_sample)
82
+{
83
+ static RRDSET *st = NULL;
84
+ static RRDDIM *rd_wq_add = NULL;
85
+ static RRDDIM *rd_wq_consumed = NULL;
86
+
87
+ if (unlikely(!st)) {
88
+ st = rrdset_create_localhost(
89
+ "netdata", "aclk_write_q", NULL, "aclk_stats", NULL, "Write Queue Mosq->Libwebsockets", "kB/s",
90
+ "netdata", "stats", 200003, localhost->rrd_update_every, RRDSET_TYPE_AREA);
91
+
92
+ rd_wq_add = rrddim_add(st, "added", NULL, 1, 1024 * localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
93
+ rd_wq_consumed = rrddim_add(st, "consumed", NULL, 1, -1024 * localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
94
+ } else
95
+ rrdset_next(st);
96
+
97
+ rrddim_set_by_pointer(st, rd_wq_add, per_sample->write_q_added);
98
+ rrddim_set_by_pointer(st, rd_wq_consumed, per_sample->write_q_consumed);
99
+
100
+ rrdset_done(st);
101
+}
102
+
103
+static void aclk_stats_read_q(struct aclk_metrics_per_sample *per_sample)
104
+{
105
+ static RRDSET *st = NULL;
106
+ static RRDDIM *rd_rq_add = NULL;
107
+ static RRDDIM *rd_rq_consumed = NULL;
108
+
109
+ if (unlikely(!st)) {
110
+ st = rrdset_create_localhost(
111
+ "netdata", "aclk_read_q", NULL, "aclk_stats", NULL, "Read Queue Libwebsockets->Mosq", "kB/s",
112
+ "netdata", "stats", 200004, localhost->rrd_update_every, RRDSET_TYPE_AREA);
113
+
114
+ rd_rq_add = rrddim_add(st, "added", NULL, 1, 1024 * localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
115
+ rd_rq_consumed = rrddim_add(st, "consumed", NULL, 1, -1024 * localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
116
+ } else
117
+ rrdset_next(st);
118
+
119
+ rrddim_set_by_pointer(st, rd_rq_add, per_sample->read_q_added);
120
+ rrddim_set_by_pointer(st, rd_rq_consumed, per_sample->read_q_consumed);
121
+
122
+ rrdset_done(st);
123
+}
124
+
125
+static void aclk_stats_cloud_req(struct aclk_metrics_per_sample *per_sample)
126
+{
127
+ static RRDSET *st = NULL;
128
+ static RRDDIM *rd_rq_rcvd = NULL;
129
+ static RRDDIM *rd_rq_err = NULL;
130
+
131
+ if (unlikely(!st)) {
132
+ st = rrdset_create_localhost(
133
+ "netdata", "aclk_cloud_req", NULL, "aclk_stats", NULL, "Requests received from cloud", "req/s",
134
+ "netdata", "stats", 200005, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
135
+
136
+ rd_rq_rcvd = rrddim_add(st, "received", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
137
+ rd_rq_err = rrddim_add(st, "malformed", NULL, 1, localhost->rrd_update_every, RRD_ALGORITHM_ABSOLUTE);
138
+ } else
139
+ rrdset_next(st);
140
+
141
+ rrddim_set_by_pointer(st, rd_rq_rcvd, per_sample->cloud_req_recvd - per_sample->cloud_req_err);
142
+ rrddim_set_by_pointer(st, rd_rq_err, per_sample->cloud_req_err);
143
+
144
+ rrdset_done(st);
145
+}
146
+
147
+void *aclk_stats_main_thread(void *ptr)
148
+{
149
+ UNUSED(ptr);
150
+ heartbeat_t hb;
151
+ heartbeat_init(&hb);
152
+ usec_t step_ut = localhost->rrd_update_every * USEC_PER_SEC;
153
+ memset(&aclk_metrics_per_sample, 0, sizeof(struct aclk_metrics_per_sample));
154
+ struct aclk_metrics_per_sample per_sample;
155
+ struct aclk_metrics permanent;
156
+
157
+ while (!netdata_exit) {
158
+ netdata_thread_testcancel();
159
+ // ------------------------------------------------------------------------
160
+ // Wait for the next iteration point.
161
+
162
+ heartbeat_next(&hb, step_ut);
163
+
164
+ ACLK_STATS_LOCK;
165
+ // to not hold lock longer than necessary, especially not to hold it
166
+ // during database rrd* operations
167
+ memcpy(&per_sample, &aclk_metrics_per_sample, sizeof(struct aclk_metrics_per_sample));
168
+ memcpy(&permanent, &aclk_metrics, sizeof(struct aclk_metrics));
169
+ memset(&aclk_metrics_per_sample, 0, sizeof(struct aclk_metrics_per_sample));
170
+ ACLK_STATS_UNLOCK;
171
+
172
+ aclk_stats_collect(&per_sample, &permanent);
173
+ aclk_stats_query_thread(&per_sample);
174
+#ifdef NETDATA_INTERNAL_CHECKS
175
+ aclk_stats_latency(&per_sample);
176
+#endif
177
+ aclk_stats_write_q(&per_sample);
178
+ aclk_stats_read_q(&per_sample);
179
+
180
+ aclk_stats_cloud_req(&per_sample);
181
+ }
182
+ return 0;
183
+}
184
+
185
+void aclk_stats_upd_online(int online) {
186
+ if(!aclk_stats_enabled)
187
+ return;
188
+
189
+ ACLK_STATS_LOCK;
190
+ aclk_metrics.online = online;
191
+
192
+ if(!online)
193
+ aclk_metrics_per_sample.offline_during_sample = 1;
194
+ ACLK_STATS_UNLOCK;
195
+}
aclk/aclk_stats.h
new
+51
@@ -0,0 +1,51 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_ACLK_STATS_H
4
+#define NETDATA_ACLK_STATS_H
5
+
6
+#include "../daemon/common.h"
7
+#include "libnetdata/libnetdata.h"
8
+
9
+#define ACLK_STATS_THREAD_NAME "ACLK_Stats"
10
+
11
+extern netdata_mutex_t aclk_stats_mutex;
12
+
13
+#define ACLK_STATS_LOCK netdata_mutex_lock(&aclk_stats_mutex)
14
+#define ACLK_STATS_UNLOCK netdata_mutex_unlock(&aclk_stats_mutex)
15
+
16
+extern int aclk_stats_enabled;
17
+
18
+// preserve between samples
19
+struct aclk_metrics {
20
+ volatile uint8_t online;
21
+};
22
+
23
+// reset to 0 on every sample
24
+extern struct aclk_metrics_per_sample {
25
+ /* in the unlikely event of ACLK disconnecting
26
+ and reconnecting under 1 sampling rate
27
+ we want to make sure we record the disconnection
28
+ despite it being then seemingly longer in graph */
29
+ volatile uint8_t offline_during_sample;
30
+
31
+ volatile uint8_t queries_queued;
32
+ volatile uint8_t queries_dispatched;
33
+#ifdef NETDATA_INTERNAL_CHECKS
34
+ volatile uint32_t latency_max;
35
+ volatile uint32_t latency_total;
36
+ volatile uint32_t latency_count;
37
+#endif
38
+ volatile uint32_t write_q_added;
39
+ volatile uint32_t write_q_consumed;
40
+
41
+ volatile uint32_t read_q_added;
42
+ volatile uint32_t read_q_consumed;
43
+
44
+ volatile uint32_t cloud_req_recvd;
45
+ volatile uint32_t cloud_req_err;
46
+} aclk_metrics_per_sample;
47
+
48
+void *aclk_stats_main_thread(void *ptr);
49
+void aclk_stats_upd_online(int online);
50
+
51
+#endif /* NETDATA_ACLK_STATS_H */
aclk/agent_cloud_link.c
+47
@@ -4,6 +4,7 @@
4
#include "agent_cloud_link.h"
5
#include "aclk_lws_https_client.h"
6
#include "aclk_common.h"
7
+#include "aclk_stats.h"
8
9
int aclk_shutting_down = 0;
10
// State-machine for the on-connect metadata transmission.
@@ -324,6 +325,12 @@ int aclk_queue_query(char *topic, char *data, char *msg_id, char *query, int run
325
aclk_queue.count--;
326
}
327
328
+ if (aclk_stats_enabled) {
329
+ ACLK_STATS_LOCK;
330
+ aclk_metrics_per_sample.queries_queued++;
331
+ ACLK_STATS_UNLOCK;
332
+ }
333
+
334
new_query = callocz(1, sizeof(struct aclk_query));
335
new_query->cmd = aclk_cmd;
336
if (internal) {
@@ -894,6 +901,12 @@ int aclk_process_query()
901
902
aclk_query_free(this_query);
903
904
+ if (aclk_stats_enabled) {
905
+ ACLK_STATS_LOCK;
906
+ aclk_metrics_per_sample.queries_dispatched++;
907
+ ACLK_STATS_UNLOCK;
908
+ }
909
+
910
return 1;
911
}
912
@@ -1358,6 +1371,7 @@ void *aclk_main(void *ptr)
1371
{
1372
struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
1373
struct netdata_static_thread *query_thread;
1374
+ struct netdata_static_thread *stats_thread = NULL;
1375
1376
// This thread is unusual in that it cannot be cancelled by cancel_main_threads()
1377
// as it must notify the far end that it shutdown gracefully and avoid the LWT.
@@ -1383,6 +1397,15 @@ void *aclk_main(void *ptr)
1397
}
1398
}
1399
1400
+ aclk_stats_enabled = appconfig_get_boolean(&cloud_config, CONFIG_SECTION_GLOBAL, "statistics", CONFIG_BOOLEAN_YES);
1401
+ if (aclk_stats_enabled) {
1402
+ stats_thread = callocz(1, sizeof(struct netdata_static_thread));
1403
+ stats_thread->thread = mallocz(sizeof(netdata_thread_t));
1404
+ netdata_thread_create(
1405
+ stats_thread->thread, ACLK_STATS_THREAD_NAME, NETDATA_THREAD_OPTION_JOINABLE, aclk_stats_main_thread,
1406
+ stats_thread);
1407
+ }
1408
+
1409
last_init_sequence = now_realtime_sec();
1410
query_thread = NULL;
1411
@@ -1502,6 +1525,13 @@ exited:
1525
RSA_free(aclk_private_key);
1526
1527
aclk_main_cleanup(ptr);
1528
+
1529
+ if(aclk_stats_enabled) {
1530
+ netdata_thread_join(*stats_thread->thread, NULL);
1531
+ freez(stats_thread->thread);
1532
+ freez(stats_thread);
1533
+ }
1534
+
1535
return NULL;
1536
}
1537
@@ -1587,6 +1617,9 @@ int aclk_subscribe(char *sub_topic, int qos)
1617
void aclk_connect()
1618
{
1619
info("Connection detected (%"PRIu64" queued queries)", aclk_queue.count);
1620
+
1621
+ aclk_stats_upd_online(1);
1622
+
1623
aclk_connected = 1;
1624
waiting_init = 0;
1625
aclk_reconnect_delay(0);
@@ -1599,6 +1632,9 @@ void aclk_disconnect()
1632
{
1633
if (likely(aclk_connected))
1634
info("Disconnect detected (%"PRIu64" queued queries)", aclk_queue.count);
1635
+
1636
+ aclk_stats_upd_online(0);
1637
+
1638
aclk_subscribed = 0;
1639
aclk_metadata_submitted = ACLK_METADATA_REQUIRED;
1640
waiting_init = 1;
@@ -1901,6 +1937,11 @@ int aclk_handle_cloud_request(char *payload)
1937
.type_id = NULL, .msg_id = NULL, .callback_topic = NULL, .payload = NULL, .version = 0
1938
};
1939
1940
+ if (aclk_stats_enabled) {
1941
+ ACLK_STATS_LOCK;
1942
+ aclk_metrics_per_sample.cloud_req_recvd++;
1943
+ ACLK_STATS_UNLOCK;
1944
+ }
1945
1946
if (unlikely(agent_state == AGENT_INITIALIZING)) {
1947
debug(D_ACLK, "Ignoring cloud request; agent not in stable state");
@@ -1938,6 +1979,12 @@ int aclk_handle_cloud_request(char *payload)
1979
if (cloud_to_agent.callback_topic)
1980
freez(cloud_to_agent.callback_topic);
1981
1982
+ if (aclk_stats_enabled) {
1983
+ ACLK_STATS_LOCK;
1984
+ aclk_metrics_per_sample.cloud_req_err++;
1985
+ ACLK_STATS_UNLOCK;
1986
+ }
1987
+
1988
return 1;
1989
}
1990
aclk/mqtt.c
+13
-1
@@ -4,6 +4,7 @@
4
#include "../daemon/common.h"
5
#include "mqtt.h"
6
#include "aclk_lws_wss_client.h"
7
+#include "aclk_stats.h"
8
9
extern usec_t aclk_session_us;
10
extern time_t aclk_session_sec;
@@ -38,8 +39,19 @@ void publish_callback(struct mosquitto *mosq, void *obj, int rc)
39
now_realtime_timeval(&now);
40
orig = &sendTimes[ rc & 0x3ff ];
41
int64_t diff = (now.tv_sec - orig->tv_sec) * USEC_PER_SEC + (now.tv_usec - orig->tv_usec);
42
+ diff /= 1000;
43
42
- info("Publish_callback: mid=%d latency=%" PRId64 "ms", rc, diff / 1000);
44
+ info("Publish_callback: mid=%d latency=%" PRId64 "ms", rc, diff);
45
+
46
+ if (aclk_stats_enabled) {
47
+ ACLK_STATS_LOCK;
48
+ if (aclk_metrics_per_sample.latency_max < diff)
49
+ aclk_metrics_per_sample.latency_max = diff;
50
+
51
+ aclk_metrics_per_sample.latency_total += diff;
52
+ aclk_metrics_per_sample.latency_count++;
53
+ ACLK_STATS_UNLOCK;
54
+ }
55
#endif
56
return;
57
}
web/gui/dashboard_info.js
+15
@@ -3132,6 +3132,21 @@ netdataDashboard.context = {
3132
info: 'Difference between the number of process created and the number of threads created per period(<code>process</code> dimension), it also shows the number of possible zombie process running on system.'
3133
},
3134
3135
+ // ------------------------------------------------------------------------
3136
+ // ACLK Internal Stats
3137
+ 'netdata.aclk_status': {
3138
+ valueRange: "[0, 1]",
3139
+ info: 'This chart shows if ACLK was online during entirety of the sample duration.'
3140
+ },
3141
+
3142
+ 'netdata.aclk_query_per_second': {
3143
+ info: 'This chart shows how many queries were added for ACLK_query thread to process and how many it was actually able to process.'
3144
+ },
3145
+
3146
+ 'netdata.aclk_latency_mqtt': {
3147
+ info: 'Measures latency between MQTT publish of the message and it\'s PUB_ACK being received'
3148
+ },
3149
+
3150
// ------------------------------------------------------------------------
3151
// VerneMQ
3152