@cryptotaxi247 / netdata-1 / commits / 49719a961

Fix bugs in streaming and enable support for gap filling (#9214)

This PR adds (inactive) support that we will use to fill the gaps on chart when a receiving agent goes offline and the sender reconnects. The streaming component has been reworked to make the connection bi-directional and fix several outstanding bugs in the area. * Fixed an incorrect case of version negotiation. Removed fatal() on exhaustion of fds. * Fixed cases that fell through to polling the socket after closing. * Fixed locking of data related to sender and receiver in the host structure. * Added fine-grained locks to reduce contention. * Added circular buffer to sender to prevent starvation in high-latency conditions. * Fixed case where agent is a proxy and negotiated different streaming versions with sender and receiver. * Changed interface to new parser to put the buffering code in streaming. * Fixed the bug that stopped senders from reconnecting after their socket times out - this was part of the scaling fixes that provide an early shortcut path for rejecting connections without lock contention. * Uses fine-grained locking and a different approach to thread shutdown instead. * Added liveness detection to connections to allow selection of the best connection.

Andrew Moss committed Jun 3, 2020 at 08:38 UTC 49719a961d6c079004b65458ea8c5e08ada1c44c
55 files changed +2716 -1178
Makefile.am
+4
@@ -138,6 +138,8 @@ LIBNETDATA_FILES = \
138 libnetdata/avl/avl.h \
139 libnetdata/buffer/buffer.c \
140 libnetdata/buffer/buffer.h \
141 + libnetdata/circular_buffer/circular_buffer.c \
142 + libnetdata/circular_buffer/circular_buffer.h \
143 libnetdata/clocks/clocks.c \
144 libnetdata/clocks/clocks.h \
145 libnetdata/dictionary/dictionary.c \
@@ -429,6 +431,8 @@ API_PLUGIN_FILES = \
431
432 STREAMING_PLUGIN_FILES = \
433 streaming/rrdpush.c \
434 + streaming/sender.c \
435 + streaming/receiver.c \
436 streaming/rrdpush.h \
437 $(NULL)
438
build_external/clean-install-arch-debug.Dockerfile new
+62
@@ -0,0 +1,62 @@
1 +FROM archlinux/base:latest
2 +
3 +# There is some redundancy between this file and the archlinux Dockerfile in the helper images
4 +# repo and also with the clean-install.Dockefile. Once the help image is availabled on Docker
5 +# Hub this file can be deleted.
6 +RUN echo sdlsjdkls
7 +RUN pacman -Syyu --noconfirm
8 +RUN pacman --noconfirm --needed -S autoconf \
9 + autoconf-archive \
10 + autogen \
11 + automake \
12 + gcc \
13 + make \
14 + git \
15 + libuv \
16 + lz4 \
17 + netcat \
18 + openssl \
19 + pkgconfig \
20 + python \
21 + libvirt \
22 + cmake \
23 + valgrind \
24 + gdb
25 +
26 +ARG EXTRA_CFLAGS
27 +COPY . /opt/netdata/source
28 +WORKDIR /opt/netdata/source
29 +
30 +RUN git config --global user.email "root@container"
31 +RUN git config --global user.name "Fake root"
32 +
33 +# RUN make distclean -> not safe if tree state changed on host since last config
34 +# Kill everything that is not in .gitignore preserving any fresh changes, i.e. untracked changes will be
35 +# deleted but local changes to tracked files will be preserved.
36 +RUN if git status --porcelain | grep '^[MADRC]'; then \
37 + git stash && git clean -dxf && (git stash apply || true) \
38 + else \
39 + git clean -dxf ; \
40 + fi
41 +
42 +# Not everybody is updating distclean properly - fix.
43 +RUN find . -name '*.Po' -exec rm \{\} \;
44 +RUN rm -rf autom4te.cache
45 +RUN rm -rf .git/
46 +RUN find . -type f >/opt/netdata/manifest
47 +
48 +RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
49 + -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
50 +
51 +RUN ln -sf /dev/stdout /var/log/netdata/access.log
52 +RUN ln -sf /dev/stdout /var/log/netdata/debug.log
53 +RUN ln -sf /dev/stderr /var/log/netdata/error.log
54 +
55 +RUN printf >/opt/netdata/source/gdb_batch '\
56 +set args -D \n\
57 +handle SIG32 nostop \n\
58 +run \n\
59 +bt'
60 +
61 +#CMD ["/usr/sbin/valgrind", "--leak-check=full", "/usr/sbin/netdata", "-D"]
62 +CMD ["/usr/bin/gdb", "-x", "/opt/netdata/source/gdb_batch", "/usr/sbin/netdata"]
build_external/clean-install-arch-extras.Dockerfile
+6 -3
@@ -3,7 +3,7 @@ FROM archlinux/base:latest
3 # There is some redundancy between this file and the archlinux Dockerfile in the helper images
4 # repo and also with the clean-install.Dockefile. Once the help image is availabled on Docker
5 # Hub this file can be deleted.
6 -
6 +RUN echo sdlsjdkls
7 RUN pacman -Syyu --noconfirm
8 RUN pacman --noconfirm --needed -S autoconf \
9 autoconf-archive \
@@ -20,7 +20,8 @@ RUN pacman --noconfirm --needed -S autoconf \
20 python \
21 libvirt \
22 cmake \
23 - valgrind
23 + valgrind \
24 + gdb
25
26 ARG EXTRA_CFLAGS
27 COPY . /opt/netdata/source
@@ -44,12 +45,14 @@ RUN rm -rf autom4te.cache
45 RUN rm -rf .git/
46 RUN find . -type f >/opt/netdata/manifest
47
47 -RUN CFLAGS="-O1 -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
48 +RUN CFLAGS="-Og -g -ggdb -Wall -Wextra -Wformat-signedness -fstack-protector-all -DNETDATA_INTERNAL_CHECKS=1\
49 -D_FORTIFY_SOURCE=2 -DNETDATA_VERIFY_LOCKS=1 ${EXTRA_CFLAGS}" ./netdata-installer.sh --require-cloud --disable-lto
50
51 RUN ln -sf /dev/stdout /var/log/netdata/access.log
52 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
53 RUN ln -sf /dev/stderr /var/log/netdata/error.log
54
55 +RUN rm /var/lib/netdata/registry/netdata.public.unique.id
56 +
57 CMD ["/usr/sbin/valgrind", "--leak-check=full", "/usr/sbin/netdata", "-D"]
58
build_external/clean-install.Dockerfile
+4 -1
@@ -33,4 +33,7 @@ RUN ln -sf /dev/stdout /var/log/netdata/access.log
33 RUN ln -sf /dev/stdout /var/log/netdata/debug.log
34 RUN ln -sf /dev/stderr /var/log/netdata/error.log
35
36 -CMD ["/usr/sbin/netdata","-D"]
\ No newline at end of file
36 +RUN rm /var/lib/netdata/registry/netdata.public.unique.id
37 +
38 +CMD ["/usr/sbin/netdata","-D"]
39 +ENTRYPOINT []
build_external/scenarios/aclk-testing/agent-compose.yml renamed
build_external/scenarios/aclk-testing/agent-valgrind-compose.yml renamed
build_external/scenarios/aclk-testing/agent_netdata.conf renamed
build_external/scenarios/aclk-testing/configureVerneMQ.Dockerfile renamed
build_external/scenarios/aclk-testing/paho-compose.yml renamed
build_external/scenarios/aclk-testing/paho-inspection.py renamed
build_external/scenarios/aclk-testing/paho.Dockerfile renamed
build_external/scenarios/aclk-testing/vernemq-compose.yml renamed
build_external/scenarios/aclk-testing/vernemq.conf renamed
build_external/scenarios/gaps_hi/master-compose.yml new
+13
@@ -0,0 +1,13 @@
1 +version: '3.3'
2 +services:
3 + agent_master:
4 + image: debian_10_dev
5 + command: /usr/sbin/netdata -D
6 + ports:
7 + - 21000:19999
8 + volumes:
9 + - ./master_stream.conf:/etc/netdata/stream.conf:ro
10 + - ./master_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
11 + - ./min.conf:/etc/netdata/netdata.conf:ro
12 + cap_add:
13 + - SYS_PTRACE
build_external/scenarios/gaps_hi/master_guid new
+1
@@ -0,0 +1 @@
1 +00000000-0000-0000-0000-000000000000
\ No newline at end of file
build_external/scenarios/gaps_hi/master_stream.conf new
+11
@@ -0,0 +1,11 @@
1 +[00000000-0000-0000-0000-000000000000]
2 + enabled = yes
3 + allow from = *
4 + default history = 3600
5 + default memory mode = dbengine
6 + health enabled by default = no
7 +
8 + # postpone alarms for a short period after the sender is connected
9 + default postpone alarms on connect seconds = 60
10 + multiple connections = allow
11 +
build_external/scenarios/gaps_hi/middle-compose.yml new
+13
@@ -0,0 +1,13 @@
1 +version: '3.3'
2 +services:
3 + agent_middle:
4 + image: debian_10_dev
5 + command: /usr/sbin/netdata -D
6 + ports:
7 + - 21001:19999
8 + volumes:
9 + - ./middle_stream.conf:/etc/netdata/stream.conf:ro
10 + - ./middle_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
11 + - ./min.conf:/etc/netdata/netdata.conf:ro
12 + cap_add:
13 + - SYS_PTRACE
build_external/scenarios/gaps_hi/middle_guid new
+1
@@ -0,0 +1 @@
1 +11111111-1111-1111-1111-111111111111
\ No newline at end of file
build_external/scenarios/gaps_hi/middle_stream.conf new
+23
@@ -0,0 +1,23 @@
1 +[stream]
2 + enabled = yes
3 + destination = tcp:agent_master
4 + api key = 00000000-0000-0000-0000-000000000000
5 + timeout seconds = 60
6 + default port = 19999
7 +
8 + send charts matching = *
9 + buffer size bytes = 1048576
10 + reconnect delay seconds = 5
11 + initial clock resync iterations = 60
12 +
13 +[00000000-0000-0000-0000-000000000000]
14 + enabled = yes
15 + allow from = *
16 + default history = 3600
17 + # default memory mode = ram
18 +
19 + health enabled by default = auto
20 +
21 + # postpone alarms for a short period after the sender is connected
22 + default postpone alarms on connect seconds = 60
23 + multiple connections = allow
build_external/scenarios/gaps_hi/min.conf new
+6
@@ -0,0 +1,6 @@
1 +[global]
2 + debug flags = 0x0000000040000000
3 + errors flood protection period = 0
4 +[web]
5 + ssl key = /etc/netdata/ssl/key.pem
6 + ssl certificate = /etc/netdata/ssl/cert.pem
build_external/scenarios/gaps_hi/slave-compose.yml new
+14
@@ -0,0 +1,14 @@
1 +version: '3.3'
2 +services:
3 + agent_slave:
4 + image: debian_10_dev
5 + command: /usr/sbin/netdata -D
6 + #ports:
7 + #- 21002+:19999
8 + volumes:
9 + - ./slave_stream.conf:/etc/netdata/stream.conf:ro
10 + #- ./slave_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
11 + - ./min.conf:/etc/netdata/netdata.conf:ro
12 + cap_add:
13 + - SYS_PTRACE
14 +
build_external/scenarios/gaps_hi/slave_guid new
+1
@@ -0,0 +1 @@
1 +22222222-2222-2222-2222-222222222222
\ No newline at end of file
build_external/scenarios/gaps_hi/slave_stream.conf new
+11
@@ -0,0 +1,11 @@
1 +[stream]
2 + enabled = yes
3 +# destination = tcp:agent_middle
4 + destination = tcp:192.168.1.2
5 + api key = 00000000-0000-0000-0000-000000000000
6 + timeout seconds = 60
7 + default port = 19999
8 + send charts matching = *
9 + buffer size bytes = 10485760
10 + reconnect delay seconds = 5
11 + initial clock resync iterations = 60
build_external/scenarios/gaps_lo/master-compose.yml new
+13
@@ -0,0 +1,13 @@
1 +version: '3.3'
2 +services:
3 + agent_master:
4 + image: debian_10_dev
5 + command: /usr/sbin/netdata -D
6 + ports:
7 + - 21000:19999
8 + volumes:
9 + - ./master_stream.conf:/etc/netdata/stream.conf:ro
10 + - ./master_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
11 + - ./mostly_off.conf:/etc/netdata/netdata.conf:ro
12 + cap_add:
13 + - SYS_PTRACE
build_external/scenarios/gaps_lo/master_guid new
+1
@@ -0,0 +1 @@
1 +00000000-0000-0000-0000-000000000000
\ No newline at end of file
build_external/scenarios/gaps_lo/master_stream.conf new
+12
@@ -0,0 +1,12 @@
1 +[00000000-0000-0000-0000-000000000000]
2 + enabled = yes
3 + allow from = *
4 + default history = 3600
5 + # default memory mode = ram
6 +
7 + health enabled by default = auto
8 +
9 + # postpone alarms for a short period after the sender is connected
10 + default postpone alarms on connect seconds = 60
11 + multiple connections = allow
12 +
build_external/scenarios/gaps_lo/middle-compose.yml new
+14
@@ -0,0 +1,14 @@
1 +version: '3.3'
2 +services:
3 + agent_middle:
4 + image: arch_extras_dev
5 + #command: /usr/sbin/valgrind --leak-check=full /usr/sbin/netdata -D
6 + command: /usr/sbin/netdata -D
7 + ports:
8 + - 21001:19999
9 + volumes:
10 + - ./middle_stream.conf:/etc/netdata/stream.conf:ro
11 + - ./middle_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
12 + - ./mostly_off.conf:/etc/netdata/netdata.conf:ro
13 + cap_add:
14 + - SYS_PTRACE
build_external/scenarios/gaps_lo/middle_guid new
+1
@@ -0,0 +1 @@
1 +11111111-1111-1111-1111-111111111111
\ No newline at end of file
build_external/scenarios/gaps_lo/middle_stream.conf new
+20
@@ -0,0 +1,20 @@
1 +[stream]
2 + enabled = yes
3 + destination = tcp:agent_master
4 + api key = 00000000-0000-0000-0000-000000000000
5 + timeout seconds = 60
6 + default port = 19999
7 + send charts matching = *
8 + buffer size bytes = 1048576
9 + reconnect delay seconds = 5
10 + initial clock resync iterations = 60
11 +
12 +[00000000-0000-0000-0000-000000000000]
13 + enabled = yes
14 + allow from = *
15 + default history = 3600
16 + # default memory mode = ram
17 + health enabled by default = auto
18 + # postpone alarms for a short period after the sender is connected
19 + default postpone alarms on connect seconds = 60
20 + multiple connections = allow
build_external/scenarios/gaps_lo/mostly_off.conf new
+965
@@ -0,0 +1,965 @@
1 +# netdata configuration
2 +#
3 +# You can download the latest version of this file, using:
4 +#
5 +# wget -O /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
6 +# or
7 +# curl -o /etc/netdata/netdata.conf http://localhost:19999/netdata.conf
8 +#
9 +# You can uncomment and change any of the options below.
10 +# The value shown in the commented settings, is the default value.
11 +#
12 +
13 +# global netdata configuration
14 +
15 +[global]
16 +debug flags = 0x0000000040000000
17 +errors flood protection period = 0
18 +
19 +[plugins]
20 + diskspace = no
21 + cgroups = no
22 + tc = no
23 + idlejitter = no
24 + ioping = no
25 + apps = no
26 + go.d = no
27 + perf = no
28 + fping = no
29 + python.d = no
30 + charts.d = no
31 + node.d = no
32 + nfacct = no
33 + cups = no
34 + freeipmi = no
35 +
36 +[health]
37 +enabled = no
38 +
39 +[statsd]
40 +enabled = no
41 +
42 +[plugin:proc]
43 + /proc/uptime = yes
44 + /proc/loadavg = no
45 + /proc/sys/kernel/random/entropy_avail = no
46 + /proc/pressure = no
47 + /proc/interrupts = no
48 + /proc/softirqs = no
49 + /proc/vmstat = no
50 + /proc/meminfo = no
51 + /sys/kernel/mm/ksm = no
52 + /sys/block/zram = no
53 + /sys/devices/system/edac/mc = no
54 + /sys/devices/system/node = no
55 + /proc/net/dev = no
56 + /proc/net/sockstat = no
57 + /proc/net/sockstat6 = no
58 + /proc/net/netstat = no
59 + /proc/net/snmp = no
60 + /proc/net/snmp6 = no
61 + /proc/net/sctp/snmp = no
62 + /proc/net/softnet_stat = no
63 + /proc/net/ip_vs/stats = no
64 + /proc/net/stat/conntrack = no
65 + /proc/net/stat/synproxy = no
66 + /proc/diskstats = no
67 + /proc/mdstat = no
68 + /proc/net/rpc/nfsd = no
69 + /proc/net/rpc/nfs = no
70 + /proc/spl/kstat/zfs/arcstats = no
71 + /sys/fs/btrfs = no
72 + ipc = no
73 + /sys/class/power_supply = no
74 +
75 +
76 +[plugin:proc:/proc/net/dev:docker0]
77 +enabled = no
78 +
79 +[plugin:proc:/proc/net/dev:br-b87e56f878f1]
80 +enabled = no
81 +
82 +[plugin:proc:/proc/net/dev:enp4s0]
83 +enabled = no
84 +
85 +[plugin:proc:/proc/net/stat/nf_conntrack]
86 +filename to monitor = /proc/net/stat/nf_conntrack
87 +netfilter new connections = no
88 +netfilter connection changes = no
89 +netfilter connection expectations = no
90 +netfilter connection searches = no
91 +netfilter errors = no
92 +netfilter connections = no
93 +
94 +[system.idlejitter]
95 +enabled = no
96 +
97 +[netdata.statsd_metrics]
98 +enabled = no
99 +
100 +[netdata.statsd_useful_metrics]
101 +enabled = no
102 +
103 +[netdata.statsd_events]
104 +enabled = no
105 +
106 +[netdata.statsd_reads]
107 +enabled = no
108 +
109 +[netdata.statsd_bytes]
110 +enabled = no
111 +
112 +[netdata.statsd_packets]
113 +enabled = no
114 +
115 +[netdata.tcp_connects]
116 +enabled = no
117 +
118 +[netdata.tcp_connected]
119 +enabled = no
120 +
121 +[netdata.private_charts]
122 +enabled = no
123 +
124 +[netdata.plugin_statsd_charting_cpu]
125 +enabled = no
126 +
127 +[netdata.plugin_statsd_collector1_cpu]
128 +enabled = no
129 +
130 +[netdata.plugin_tc_cpu]
131 +enabled = no
132 +
133 +[netdata.plugin_tc_time]
134 +enabled = no
135 +
136 +[netdata.runtime_sensors]
137 +enabled = no
138 +
139 +[sensors.coretemp-isa-0000_temperature]
140 +enabled = no
141 +
142 +[sensors.acpitz-acpi-0_temperature]
143 +enabled = no
144 +
145 +[system.cpu]
146 +enabled = yes
147 +
148 +[disk_space._dev]
149 +enabled = no
150 +
151 +[netdata.plugin_cgroups_cpu]
152 +enabled = no
153 +
154 +[netdata.apps_cpu]
155 +enabled = no
156 +
157 +[netdata.apps_sizes]
158 +enabled = no
159 +
160 +[netdata.apps_fix]
161 +enabled = no
162 +
163 +[netdata.apps_children_fix]
164 +enabled = no
165 +
166 +[apps.cpu]
167 +enabled = no
168 +
169 +[apps.mem]
170 +enabled = no
171 +
172 +[apps.vmem]
173 +enabled = no
174 +
175 +[apps.threads]
176 +enabled = no
177 +
178 +[apps.processes]
179 +enabled = no
180 +
181 +[apps.uptime]
182 +enabled = no
183 +
184 +[apps.uptime_min]
185 +enabled = no
186 +
187 +[apps.uptime_avg]
188 +enabled = no
189 +
190 +[apps.uptime_max]
191 +enabled = no
192 +
193 +[apps.cpu_user]
194 +enabled = no
195 +
196 +[apps.cpu_system]
197 +enabled = no
198 +
199 +[apps.swap]
200 +enabled = no
201 +
202 +[apps.major_faults]
203 +enabled = no
204 +
205 +[apps.minor_faults]
206 +enabled = no
207 +
208 +[apps.preads]
209 +enabled = no
210 +
211 +[apps.pwrites]
212 +enabled = no
213 +
214 +[apps.lreads]
215 +enabled = no
216 +
217 +[apps.lwrites]
218 +enabled = no
219 +
220 +[apps.files]
221 +enabled = no
222 +
223 +[apps.sockets]
224 +enabled = no
225 +
226 +[apps.pipes]
227 +enabled = no
228 +
229 +[users.cpu]
230 +enabled = no
231 +
232 +[users.mem]
233 +enabled = no
234 +
235 +[users.vmem]
236 +enabled = no
237 +
238 +[users.threads]
239 +enabled = no
240 +
241 +[users.processes]
242 +enabled = no
243 +
244 +[users.uptime]
245 +enabled = no
246 +
247 +[users.uptime_min]
248 +enabled = no
249 +
250 +[users.uptime_avg]
251 +enabled = no
252 +
253 +[users.uptime_max]
254 +enabled = no
255 +
256 +[users.cpu_user]
257 +enabled = no
258 +
259 +[users.cpu_system]
260 +enabled = no
261 +
262 +[users.swap]
263 +enabled = no
264 +
265 +[users.major_faults]
266 +enabled = no
267 +
268 +[users.minor_faults]
269 +enabled = no
270 +
271 +[users.preads]
272 +enabled = no
273 +
274 +[users.pwrites]
275 +enabled = no
276 +
277 +[users.lreads]
278 +enabled = no
279 +
280 +[users.lwrites]
281 +enabled = no
282 +
283 +[users.files]
284 +enabled = no
285 +
286 +[users.sockets]
287 +enabled = no
288 +
289 +[users.pipes]
290 +enabled = no
291 +
292 +[groups.cpu]
293 +enabled = no
294 +
295 +[groups.mem]
296 +enabled = no
297 +
298 +[groups.vmem]
299 +enabled = no
300 +
301 +[groups.threads]
302 +enabled = no
303 +
304 +[groups.processes]
305 +enabled = no
306 +
307 +[groups.uptime]
308 +enabled = no
309 +
310 +[groups.uptime_min]
311 +enabled = no
312 +
313 +[groups.uptime_avg]
314 +enabled = no
315 +
316 +[groups.uptime_max]
317 +enabled = no
318 +
319 +[groups.cpu_user]
320 +enabled = no
321 +
322 +[groups.cpu_system]
323 +enabled = no
324 +
325 +[groups.swap]
326 +enabled = no
327 +
328 +[groups.major_faults]
329 +enabled = no
330 +
331 +[groups.minor_faults]
332 +enabled = no
333 +
334 +[groups.preads]
335 +enabled = no
336 +
337 +[groups.pwrites]
338 +enabled = no
339 +
340 +[groups.lreads]
341 +enabled = no
342 +
343 +[groups.lwrites]
344 +enabled = no
345 +
346 +[groups.files]
347 +enabled = no
348 +
349 +[groups.sockets]
350 +enabled = no
351 +
352 +[groups.pipes]
353 +enabled = no
354 +
355 +[netdata.web_thread4_cpu]
356 +enabled = no
357 +
358 +[netdata.web_thread2_cpu]
359 +enabled = no
360 +
361 +[netdata.web_thread5_cpu]
362 +enabled = no
363 +
364 +[netdata.web_thread3_cpu]
365 +enabled = no
366 +
367 +[netdata.web_thread6_cpu]
368 +enabled = no
369 +
370 +[netdata.web_thread1_cpu]
371 +enabled = no
372 +
373 +[disk_inodes._dev]
374 +enabled = no
375 +
376 +[disk_space._run]
377 +enabled = no
378 +
379 +[disk_inodes._run]
380 +enabled = no
381 +
382 +[disk_space._]
383 +enabled = no
384 +
385 +[disk_inodes._]
386 +enabled = no
387 +
388 +[disk_space._dev_shm]
389 +enabled = no
390 +
391 +[disk_inodes._dev_shm]
392 +enabled = no
393 +
394 +[disk_space._run_lock]
395 +enabled = no
396 +
397 +[disk_inodes._run_lock]
398 +enabled = no
399 +
400 +[disk_space._home]
401 +enabled = no
402 +
403 +[disk_inodes._home]
404 +enabled = no
405 +
406 +[disk_space._boot_efi]
407 +enabled = no
408 +
409 +[disk_space._media_amoss_deb10]
410 +enabled = no
411 +
412 +[netdata.plugin_diskspace]
413 +enabled = no
414 +
415 +[netdata.plugin_diskspace_dt]
416 +enabled = no
417 +
418 +[cpu.cpu0]
419 +enabled = no
420 +
421 +[cpu.cpu1]
422 +enabled = no
423 +
424 +[cpu.cpu2]
425 +enabled = no
426 +
427 +[cpu.cpu3]
428 +enabled = no
429 +
430 +[cpu.cpu4]
431 +enabled = no
432 +
433 +[cpu.cpu5]
434 +enabled = no
435 +
436 +[cpu.cpu6]
437 +enabled = no
438 +
439 +[cpu.cpu7]
440 +enabled = no
441 +
442 +[system.intr]
443 +enabled = no
444 +
445 +[system.ctxt]
446 +enabled = no
447 +
448 +[system.forks]
449 +enabled = no
450 +
451 +[system.processes]
452 +enabled = no
453 +
454 +[cpu.core_throttling]
455 +enabled = no
456 +
457 +[cpu.cpufreq]
458 +enabled = no
459 +
460 +[cpu.cpu0_cpuidle]
461 +enabled = no
462 +
463 +[cpu.cpu1_cpuidle]
464 +enabled = no
465 +
466 +[cpu.cpu2_cpuidle]
467 +enabled = no
468 +
469 +[cpu.cpu3_cpuidle]
470 +enabled = no
471 +
472 +[cpu.cpu4_cpuidle]
473 +enabled = no
474 +
475 +[cpu.cpu5_cpuidle]
476 +enabled = no
477 +
478 +[cpu.cpu6_cpuidle]
479 +enabled = no
480 +
481 +[cpu.cpu7_cpuidle]
482 +enabled = no
483 +
484 +[system.uptime]
485 +enabled = yes
486 +
487 +[system.load]
488 +enabled = no
489 +
490 +[system.active_processes]
491 +enabled = no
492 +
493 +[system.entropy]
494 +enabled = no
495 +
496 +[system.interrupts]
497 +enabled = no
498 +
499 +[cpu.cpu0_interrupts]
500 +enabled = no
501 +
502 +[cpu.cpu1_interrupts]
503 +enabled = no
504 +
505 +[cpu.cpu2_interrupts]
506 +enabled = no
507 +
508 +[cpu.cpu3_interrupts]
509 +enabled = no
510 +
511 +[cpu.cpu4_interrupts]
512 +enabled = no
513 +
514 +[cpu.cpu5_interrupts]
515 +enabled = no
516 +
517 +[cpu.cpu6_interrupts]
518 +enabled = no
519 +
520 +[cpu.cpu7_interrupts]
521 +enabled = no
522 +
523 +[system.softirqs]
524 +enabled = no
525 +
526 +[cpu.cpu0_softirqs]
527 +enabled = no
528 +
529 +[cpu.cpu1_softirqs]
530 +enabled = no
531 +
532 +[cpu.cpu2_softirqs]
533 +enabled = no
534 +
535 +[cpu.cpu3_softirqs]
536 +enabled = no
537 +
538 +[cpu.cpu4_softirqs]
539 +enabled = no
540 +
541 +[cpu.cpu5_softirqs]
542 +enabled = no
543 +
544 +[cpu.cpu6_softirqs]
545 +enabled = no
546 +
547 +[cpu.cpu7_softirqs]
548 +enabled = no
549 +
550 +[system.swapio]
551 +enabled = no
552 +
553 +[system.pgpgio]
554 +enabled = no
555 +
556 +[mem.pgfaults]
557 +enabled = no
558 +
559 +[system.ram]
560 +enabled = no
561 +
562 +[mem.available]
563 +enabled = no
564 +
565 +[system.swap]
566 +enabled = no
567 +
568 +[mem.committed]
569 +enabled = no
570 +
571 +[mem.writeback]
572 +enabled = no
573 +
574 +[mem.kernel]
575 +enabled = no
576 +
577 +[mem.slab]
578 +enabled = no
579 +
580 +[mem.transparent_hugepages]
581 +enabled = no
582 +
583 +[net.docker0]
584 +enabled = no
585 +
586 +[net_packets.docker0]
587 +enabled = no
588 +
589 +[net.br-b87e56f878f1]
590 +enabled = no
591 +
592 +[net_packets.br-b87e56f878f1]
593 +enabled = no
594 +
595 +[net.enp4s0]
596 +enabled = no
597 +
598 +[net_packets.enp4s0]
599 +enabled = no
600 +
601 +[system.net]
602 +enabled = no
603 +
604 +[ipv4.sockstat_sockets]
605 +enabled = no
606 +
607 +[ipv4.sockstat_tcp_sockets]
608 +enabled = no
609 +
610 +[ipv4.sockstat_tcp_mem]
611 +enabled = no
612 +
613 +[ipv4.sockstat_udp_sockets]
614 +enabled = no
615 +
616 +[ipv4.sockstat_udp_mem]
617 +enabled = no
618 +
619 +[ipv4.sockstat_raw_sockets]
620 +enabled = no
621 +
622 +[ipv6.sockstat6_tcp_sockets]
623 +enabled = no
624 +
625 +[ipv6.sockstat6_udp_sockets]
626 +enabled = no
627 +
628 +[ip.tcpconnaborts]
629 +enabled = no
630 +
631 +[ip.tcpreorders]
632 +enabled = no
633 +
634 +[ip.tcpofo]
635 +enabled = no
636 +
637 +[system.ip]
638 +enabled = no
639 +
640 +[ip.inerrors]
641 +enabled = no
642 +
643 +[ip.mcast]
644 +enabled = no
645 +
646 +[ip.bcast]
647 +enabled = no
648 +
649 +[ip.mcastpkts]
650 +enabled = no
651 +
652 +[ip.bcastpkts]
653 +enabled = no
654 +
655 +[ip.ecnpkts]
656 +enabled = no
657 +
658 +[ipv4.packets]
659 +enabled = no
660 +
661 +[ipv4.fragsin]
662 +enabled = no
663 +
664 +[ipv4.errors]
665 +enabled = no
666 +
667 +[ipv4.icmp]
668 +enabled = no
669 +
670 +[ipv4.icmp_errors]
671 +enabled = no
672 +
673 +[ipv4.icmpmsg]
674 +enabled = no
675 +
676 +[ipv4.tcpsock]
677 +enabled = no
678 +
679 +[ipv4.tcppackets]
680 +enabled = no
681 +
682 +[ipv4.tcperrors]
683 +enabled = no
684 +
685 +[ipv4.tcpopens]
686 +enabled = no
687 +
688 +[ipv4.tcphandshake]
689 +enabled = no
690 +
691 +[ipv4.udppackets]
692 +enabled = no
693 +
694 +[ipv4.udperrors]
695 +enabled = no
696 +
697 +[system.ipv6]
698 +enabled = no
699 +
700 +[ipv6.packets]
701 +enabled = no
702 +
703 +[ipv6.errors]
704 +enabled = no
705 +
706 +[ipv6.udppackets]
707 +enabled = no
708 +
709 +[ipv6.udperrors]
710 +enabled = no
711 +
712 +[ipv6.mcast]
713 +enabled = no
714 +
715 +[ipv6.mcastpkts]
716 +enabled = no
717 +
718 +[ipv6.icmp]
719 +enabled = no
720 +
721 +[ipv6.icmperrors]
722 +enabled = no
723 +
724 +[ipv6.icmprouter]
725 +enabled = no
726 +
727 +[ipv6.icmpneighbor]
728 +enabled = no
729 +
730 +[ipv6.icmpmldv2]
731 +enabled = no
732 +
733 +[ipv6.icmptypes]
734 +enabled = no
735 +
736 +[ipv6.ect]
737 +enabled = no
738 +
739 +[system.softnet_stat]
740 +enabled = no
741 +
742 +[cpu.cpu0_softnet_stat]
743 +enabled = no
744 +
745 +[cpu.cpu1_softnet_stat]
746 +enabled = no
747 +
748 +[cpu.cpu2_softnet_stat]
749 +enabled = no
750 +
751 +[cpu.cpu3_softnet_stat]
752 +enabled = no
753 +
754 +[cpu.cpu4_softnet_stat]
755 +enabled = no
756 +
757 +[cpu.cpu5_softnet_stat]
758 +enabled = no
759 +
760 +[cpu.cpu6_softnet_stat]
761 +enabled = no
762 +
763 +[cpu.cpu7_softnet_stat]
764 +enabled = no
765 +
766 +[netfilter.conntrack_sockets]
767 +enabled = no
768 +
769 +[netfilter.conntrack_new]
770 +enabled = no
771 +
772 +[netfilter.conntrack_changes]
773 +enabled = no
774 +
775 +[netfilter.conntrack_expect]
776 +enabled = no
777 +
778 +[netfilter.conntrack_search]
779 +enabled = no
780 +
781 +[netfilter.conntrack_errors]
782 +enabled = no
783 +
784 +[disk.sda]
785 +enabled = no
786 +
787 +[disk_ops.sda]
788 +enabled = no
789 +
790 +[disk_backlog.sda]
791 +enabled = no
792 +
793 +[disk_util.sda]
794 +enabled = no
795 +
796 +[disk_iotime.sda]
797 +enabled = no
798 +
799 +[disk.sdb]
800 +enabled = no
801 +
802 +[disk_ops.sdb]
803 +enabled = no
804 +
805 +[disk_backlog.sdb]
806 +enabled = no
807 +
808 +[disk_util.sdb]
809 +enabled = no
810 +
811 +[disk_mops.sdb]
812 +enabled = no
813 +
814 +[disk_iotime.sdb]
815 +enabled = no
816 +
817 +[disk.sdc]
818 +enabled = no
819 +
820 +[disk_ops.sdc]
821 +enabled = no
822 +
823 +[disk_backlog.sdc]
824 +enabled = no
825 +
826 +[disk_util.sdc]
827 +enabled = no
828 +
829 +[disk_mops.sdc]
830 +enabled = no
831 +
832 +[disk_iotime.sdc]
833 +enabled = no
834 +
835 +[system.io]
836 +enabled = no
837 +
838 +[system.ipc_semaphores]
839 +enabled = no
840 +
841 +[system.ipc_semaphore_arrays]
842 +enabled = no
843 +
844 +[system.shared_memory_segments]
845 +enabled = no
846 +
847 +[system.shared_memory_bytes]
848 +enabled = no
849 +
850 +[netdata.plugin_proc_modules]
851 +enabled = no
852 +
853 +[netdata.plugin_proc_cpu]
854 +enabled = no
855 +
856 +[netdata.server_cpu]
857 +enabled = no
858 +
859 +[netdata.clients]
860 +enabled = no
861 +
862 +[netdata.requests]
863 +enabled = no
864 +
865 +[netdata.net]
866 +enabled = no
867 +
868 +[netdata.response_time]
869 +enabled = no
870 +
871 +[netdata.compression_ratio]
872 +enabled = no
873 +
874 +[netdata.dbengine_compression_ratio]
875 +enabled = no
876 +
877 +[netdata.page_cache_hit_ratio]
878 +enabled = no
879 +
880 +[netdata.page_cache_stats]
881 +enabled = no
882 +
883 +[netdata.dbengine_long_term_page_stats]
884 +enabled = no
885 +
886 +[netdata.dbengine_io_throughput]
887 +enabled = no
888 +
889 +[netdata.dbengine_io_operations]
890 +enabled = no
891 +
892 +[netdata.dbengine_global_errors]
893 +enabled = no
894 +
895 +[netdata.dbengine_global_file_descriptors]
896 +enabled = no
897 +
898 +[netdata.dbengine_ram]
899 +enabled = no
900 +
901 +[disk_await.sda]
902 +enabled = no
903 +
904 +[disk_avgsz.sda]
905 +enabled = no
906 +
907 +[disk_svctm.sda]
908 +enabled = no
909 +
910 +[disk_await.sdb]
911 +enabled = no
912 +
913 +[disk_avgsz.sdb]
914 +enabled = no
915 +
916 +[disk_svctm.sdb]
917 +enabled = no
918 +
919 +[disk_await.sdc]
920 +enabled = no
921 +
922 +[disk_avgsz.sdc]
923 +enabled = no
924 +
925 +[disk_svctm.sdc]
926 +enabled = no
927 +
928 +[netdata.queries]
929 +enabled = no
930 +
931 +[netdata.db_points]
932 +enabled = no
933 +
934 +[services.cpu]
935 +enabled = no
936 +
937 +[services.mem_usage]
938 +enabled = no
939 +
940 +[services.throttle_io_read]
941 +enabled = no
942 +
943 +[services.throttle_io_write]
944 +enabled = no
945 +
946 +[services.throttle_io_ops_read]
947 +enabled = no
948 +
949 +[services.throttle_io_ops_write]
950 +enabled = no
951 +
952 +[netfilter.netlink_new]
953 +enabled = no
954 +
955 +[netfilter.netlink_changes]
956 +enabled = no
957 +
958 +[netfilter.netlink_search]
959 +enabled = no
960 +
961 +[netfilter.netlink_errors]
962 +enabled = no
963 +
964 +[netfilter.netlink_expect]
965 +enabled = no
build_external/scenarios/gaps_lo/slave-compose.yml new
+15
@@ -0,0 +1,15 @@
1 +version: '3.3'
2 +services:
3 + agent_slave:
4 + image: arch_extras_dev
5 + #command: /usr/sbin/valgrind --leak-check=full /usr/sbin/netdata -D
6 + command: /usr/sbin/netdata -D # gdb does not like valgrind !
7 + #ports:
8 + #- 21002:19999
9 + volumes:
10 + - ./slave_stream.conf:/etc/netdata/stream.conf:ro
11 +# - ./slave_guid:/var/lib/netdata/registry/netdata.public.unique.id:ro
12 + - ./mostly_off.conf:/etc/netdata/netdata.conf:ro
13 + cap_add:
14 + - SYS_PTRACE
15 +
build_external/scenarios/gaps_lo/slave_guid new
+1
@@ -0,0 +1 @@
1 +22222222-2222-2222-2222-222222222222
\ No newline at end of file
build_external/scenarios/gaps_lo/slave_stream.conf new
+11
@@ -0,0 +1,11 @@
1 +[stream]
2 + # Enable this on slaves, to have them send metrics.
3 + enabled = yes
4 + destination = tcp:192.168.1.2
5 + api key = 00000000-0000-0000-0000-000000000000
6 + timeout seconds = 60
7 + default port = 19999
8 + send charts matching = *
9 + buffer size bytes = 1048576
10 + reconnect delay seconds = 5
11 + initial clock resync iterations = 60
build_external/scenarios/master-slaves/docker-compose.yml renamed
build_external/scenarios/master-slaves/master_stream.conf renamed
build_external/scenarios/master-slaves/slave_stream.conf renamed
build_external/scenarios/only-agent/docker-compose.yml renamed
collectors/plugins.d/plugins_d.c
-98
@@ -139,104 +139,6 @@ inline int pluginsd_split_words(char *str, char **words, int max_words, char *re
139 return quoted_strings_splitter(str, words, max_words, pluginsd_space, recover_input, recover_location, max_recover);
140 }
141
142 -#ifdef ENABLE_HTTPS
143 -/**
144 - * Update Buffer
145 - *
146 - * Update the temporary buffer used to parse data received from slave
147 - *
148 - * @param output is a pointer to the vector where I will store the data
149 - * @param ssl is the connection pointer with the server
150 - *
151 - * @return it returns the total of bytes read on success and a negative number otherwise
152 - */
153 -int pluginsd_update_buffer(char *output, SSL *ssl)
154 -{
155 - ERR_clear_error();
156 - int bytesleft = SSL_read(ssl, output, PLUGINSD_LINE_MAX_SSL_READ);
157 - if (bytesleft <= 0) {
158 - int sslerrno = SSL_get_error(ssl, bytesleft);
159 - switch (sslerrno) {
160 - case SSL_ERROR_WANT_READ:
161 - case SSL_ERROR_WANT_WRITE: {
162 - break;
163 - }
164 - default: {
165 - u_long err;
166 - char buf[256];
167 - int counter = 0;
168 - while ((err = ERR_get_error()) != 0) {
169 - ERR_error_string_n(err, buf, sizeof(buf));
170 - info(
171 - "%d SSL Handshake error (%s) on socket %d ", counter++,
172 - ERR_error_string((long)SSL_get_error(ssl, bytesleft), NULL), SSL_get_fd(ssl));
173 - }
174 - }
175 - }
176 - } else {
177 - output[bytesleft] = '\0';
178 - }
179 -
180 - return bytesleft;
181 -}
182 -
183 -/**
184 - * Get from Buffer
185 - *
186 - * Get data to process from buffer
187 - *
188 - * @param output is the output vector that will be used to parse the string.
189 - * @param bytesread the amount of bytes read in the previous iteration.
190 - * @param input the input vector where there are data to process
191 - * @param ssl a pointer to the connection with the server
192 - * @param src the first address of the input, because sometime will be necessary to restart the addr with it.
193 - *
194 - * @return It returns a pointer for the next iteration on success and NULL otherwise.
195 - */
196 -char *pluginsd_get_from_buffer(char *output, int *bytesread, char *input, SSL *ssl, char *src)
197 -{
198 - int copying = 1;
199 - char *endbuffer;
200 - size_t length;
201 - while (copying) {
202 - if (*bytesread > 0) {
203 - endbuffer = strchr(input, '\n');
204 - if (endbuffer) {
205 - copying = 0;
206 - endbuffer++; //Advance due the fact I wanna copy '\n'
207 - length = endbuffer - input;
208 - *bytesread -= length;
209 -
210 - memcpy(output, input, length);
211 - output += length;
212 - *output = '\0';
213 - input += length;
214 - } else {
215 - length = strlen(input);
216 - memcpy(output, input, length);
217 - output += length;
218 - input = src;
219 -
220 - *bytesread = pluginsd_update_buffer(input, ssl);
221 - if (*bytesread <= 0) {
222 - input = NULL;
223 - copying = 0;
224 - }
225 - }
226 - } else {
227 - //reduce sample of bytes read, print the length
228 - *bytesread = pluginsd_update_buffer(input, ssl);
229 - if (*bytesread <= 0) {
230 - input = NULL;
231 - copying = 0;
232 - }
233 - }
234 - }
235 -
236 - return input;
237 -}
238 -#endif
239 -
142
143 static void pluginsd_worker_thread_cleanup(void *arg)
144 {
collectors/plugins.d/plugins_d.h
-2
@@ -80,7 +80,5 @@ extern int pluginsd_initialize_plugin_directories();
80
81 extern int config_isspace(char c);
82 extern int pluginsd_space(char c);
83 -extern int pluginsd_update_buffer(char *output, SSL *ssl);
84 -extern char * pluginsd_get_from_buffer(char *output, int *bytesread, char *input, SSL *ssl, char *src);
83
84 #endif /* NETDATA_PLUGINS_D_H */
collectors/plugins.d/pluginsd_parser.c
+2 -1
@@ -239,7 +239,8 @@ PARSER_RC pluginsd_begin(char **words, void *user, PLUGINSD_ACTION *plugins_act
239 microseconds = str2ull(microseconds_txt);
240
241 if (plugins_action->begin_action) {
242 - return plugins_action->begin_action(user, st, microseconds, ((PARSER_USER_OBJECT *)user)->trust_durations);
242 + return plugins_action->begin_action(user, st, microseconds,
243 + ((PARSER_USER_OBJECT *)user)->trust_durations);
244 }
245 return PARSER_RC_OK;
246 disable:
collectors/plugins.d/pluginsd_parser.h
+14
@@ -10,6 +10,7 @@ typedef struct parser_user_object {
10 PARSER *parser;
11 RRDSET *st;
12 RRDHOST *host;
13 + void *opaque;
14 struct plugind *cd;
15 int trust_durations;
16 struct label *new_labels;
@@ -17,4 +18,17 @@ typedef struct parser_user_object {
18 int enabled;
19 } PARSER_USER_OBJECT;
20
21 +PARSER_RC pluginsd_set_action(void *user, RRDSET *st, RRDDIM *rd, long long int value);
22 +PARSER_RC pluginsd_flush_action(void *user, RRDSET *st);
23 +PARSER_RC pluginsd_begin_action(void *user, RRDSET *st, usec_t microseconds, int trust_durations);
24 +PARSER_RC pluginsd_end_action(void *user, RRDSET *st);
25 +PARSER_RC pluginsd_chart_action(void *user, char *type, char *id, char *name, char *family, char *context, char *title, char *units, char *plugin,
26 + char *module, int priority, int update_every, RRDSET_TYPE chart_type, char *options);
27 +PARSER_RC pluginsd_disable_action(void *user);
28 +PARSER_RC pluginsd_variable_action(void *user, RRDHOST *host, RRDSET *st, char *name, int global, calculated_number value);
29 +PARSER_RC pluginsd_dimension_action(void *user, RRDSET *st, char *id, char *name, char *algorithm, long multiplier, long divisor, char *options,
30 + RRD_ALGORITHM algorithm_type);
31 +PARSER_RC pluginsd_label_action(void *user, char *key, char *value, LABEL_SOURCE source);
32 +PARSER_RC pluginsd_overwrite_action(void *user, RRDHOST *host, struct label *new_labels);
33 +
34 #endif //NETDATA_PLUGINSD_PARSER_H
database/rrd.h
+9 -5
@@ -29,6 +29,7 @@ struct pg_cache_page_index;
29 #include "rrddimvar.h"
30 #include "rrdcalc.h"
31 #include "rrdcalctemplate.h"
32 +#include "../streaming/rrdpush.h"
33
34 #define UPDATE_EVERY 1
35 #define UPDATE_EVERY_MAX 3600
@@ -692,6 +693,7 @@ struct rrdhost {
693
694 // the following are state information for the threading
695 // streaming metrics from this netdata to an upstream netdata
696 + struct sender_state *sender;
697 volatile unsigned int rrdpush_sender_spawn:1; // 1 when the sender thread has been spawn
698 netdata_thread_t rrdpush_sender_thread; // the sender thread
699
@@ -703,13 +705,10 @@ struct rrdhost {
705
706 SIMPLE_PATTERN *rrdpush_send_charts_matching; // pattern to match the charts to be sent
707
706 - // metrics may be collected asynchronously
707 - // these synchronize all the threads willing the write to our sending buffer
708 - netdata_mutex_t rrdpush_sender_buffer_mutex; // exclusive access to rrdpush_sender_buffer
708 int rrdpush_sender_pipe[2]; // collector to sender thread signaling
710 - BUFFER *rrdpush_sender_buffer; // collector fills it, sender sends it
709 + //BUFFER *rrdpush_sender_buffer; // collector fills it, sender sends it
710
712 - uint32_t stream_version; //Set the current version of the stream.
711 + //uint32_t stream_version; //Set the current version of the stream.
712
713 // ------------------------------------------------------------------------
714 // streaming of data from remote hosts - rrdpush
@@ -719,6 +718,9 @@ struct rrdhost {
718
719 time_t senders_disconnected_time; // the time the last sender was disconnected
720
721 + struct receiver_state *receiver;
722 + netdata_mutex_t receiver_lock;
723 +
724 // ------------------------------------------------------------------------
725 // health monitoring options
726
@@ -974,6 +976,8 @@ static inline time_t rrdset_first_entry_t(RRDSET *st) {
976 }
977 }
978
979 +time_t rrdhost_last_entry_t(RRDHOST *h);
980 +
981 // get the last slot updated in the round robin database
982 #define rrdset_last_slot(st) ((size_t)(((st)->current_entry == 0) ? (st)->entries - 1 : (st)->current_entry - 1))
983
database/rrdhost.c
+35 -8
@@ -139,6 +139,11 @@ RRDHOST *rrdhost_create(const char *hostname,
139 host->disk_space_mb = default_rrdeng_disk_quota_mb;
140 #endif
141 host->health_enabled = (memory_mode == RRD_MEMORY_MODE_NONE)? 0 : health_enabled;
142 +
143 + host->sender = mallocz(sizeof(*host->sender));
144 + sender_init(host->sender, host);
145 + netdata_mutex_init(&host->receiver_lock);
146 +
147 host->rrdpush_send_enabled = (rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key) ? 1 : 0;
148 host->rrdpush_send_destination = (host->rrdpush_send_enabled)?strdupz(rrdpush_destination):NULL;
149 host->rrdpush_send_api_key = (host->rrdpush_send_enabled)?strdupz(rrdpush_api_key):NULL;
@@ -148,7 +153,7 @@ RRDHOST *rrdhost_create(const char *hostname,
153 host->rrdpush_sender_pipe[1] = -1;
154 host->rrdpush_sender_socket = -1;
155
151 - host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION;
156 + //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
157 #ifdef ENABLE_HTTPS
158 host->ssl.conn = NULL;
159 host->ssl.flags = NETDATA_SSL_START;
@@ -156,7 +161,6 @@ RRDHOST *rrdhost_create(const char *hostname,
161 host->stream_ssl.flags = NETDATA_SSL_START;
162 #endif
163
159 - netdata_mutex_init(&host->rrdpush_sender_buffer_mutex);
164 netdata_rwlock_init(&host->rrdhost_rwlock);
165 netdata_rwlock_init(&host->labels_rwlock);
166
@@ -407,7 +411,7 @@ RRDHOST *rrdhost_find_or_create(
411 }
412 else {
413 host->health_enabled = health_enabled;
410 - host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION;
414 + //host->stream_version = STREAMING_PROTOCOL_CURRENT_VERSION; Unused?
415
416 if(strcmp(host->hostname, hostname) != 0) {
417 info("Host '%s' has been renamed to '%s'. If this is not intentional it may mean multiple hosts are using the same machine_guid.", host->hostname, hostname);
@@ -455,7 +459,7 @@ inline int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected, time_t n
459 if(host != protected
460 && host != localhost
461 && rrdhost_flag_check(host, RRDHOST_FLAG_ORPHAN)
458 - && !host->connected_senders
462 + && host->receiver
463 && host->senders_disconnected_time
464 && host->senders_disconnected_time + rrdhost_free_orphan_time < now)
465 return 1;
@@ -601,8 +605,13 @@ void rrdhost_free(RRDHOST *host) {
605
606 rrd_check_wrlock(); // make sure the RRDs are write locked
607
604 - // stop a possibly running thread
605 - rrdpush_sender_thread_stop(host);
608 + // ------------------------------------------------------------------------
609 + // clean up the sender
610 + rrdpush_sender_thread_stop(host); // stop a possibly running thread
611 + cbuffer_free(host->sender->buffer);
612 + buffer_free(host->sender->build);
613 + freez(host->sender);
614 + host->sender = NULL;
615
616 rrdhost_wrlock(host); // lock this RRDHOST
617
@@ -668,6 +677,8 @@ void rrdhost_free(RRDHOST *host) {
677 else error("Request to free RRDHOST '%s': cannot find it", host->hostname);
678 }
679
680 +
681 +
682 // ------------------------------------------------------------------------
683 // free it
684
@@ -1193,11 +1204,12 @@ void reload_host_labels()
1204 health_label_log_save(localhost);
1205 rrdhost_unlock(localhost);
1206
1207 +/* TODO-GAPS - fix this so that it looks properly at the state and version of the sender
1208 if(localhost->rrdpush_send_enabled && localhost->rrdpush_sender_buffer){
1209 localhost->labels_flag |= LABEL_FLAG_UPDATE_STREAM;
1210 rrdpush_send_labels(localhost);
1211 }
1200 -
1212 +*/
1213 health_reload();
1214 }
1215
@@ -1283,7 +1295,7 @@ void rrdhost_cleanup_all(void) {
1295
1296 RRDHOST *host;
1297 rrdhost_foreach_read(host) {
1286 - if(host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS) && !host->connected_senders)
1298 + if(host != localhost && rrdhost_flag_check(host, RRDHOST_FLAG_DELETE_OBSOLETE_CHARTS) && !host->receiver)
1299 rrdhost_delete_charts(host);
1300 else
1301 rrdhost_cleanup_charts(host);
@@ -1482,3 +1494,18 @@ int alarm_compare_name(void *a, void *b) {
1494
1495 return strcmp(in1->name,in2->name);
1496 }
1497 +
1498 +// Added for gap-filling, if this proves to be a bottleneck in large-scale systems then we will need to cache
1499 +// the last entry times as the metric updates, but let's see if it is a problem first.
1500 +time_t rrdhost_last_entry_t(RRDHOST *h) {
1501 + rrdhost_rdlock(h);
1502 + RRDSET *st;
1503 + time_t result = 0;
1504 + rrdset_foreach_read(st, h) {
1505 + time_t st_last = rrdset_last_entry_t(st);
1506 + if (st_last > result)
1507 + result = st_last;
1508 + }
1509 + rrdhost_unlock(h);
1510 + return result;
1511 +}
libnetdata/circular_buffer/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
libnetdata/circular_buffer/README.md new
+13
@@ -0,0 +1,13 @@
1 +<!--
2 +--
3 +title: "circular_buffer"
4 +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/circular_buffer/README.md
5 +---
6 +-->
7 +
8 +# Circular Buffer
9 +
10 +`struct circular_buffer` is an adaptive circular buffer. It will start at an initial size
11 +and grow up to a maximum size as it fills. Two indices within the structure track the current
12 +`read` and `write` position for data.
13 +
libnetdata/circular_buffer/circular_buffer.c new
+85
@@ -0,0 +1,85 @@
1 +#include "../libnetdata.h"
2 +
3 +struct circular_buffer *cbuffer_new(size_t initial, size_t max) {
4 + struct circular_buffer *result = mallocz(sizeof(*result));
5 + result->size = initial;
6 + result->data = mallocz(initial);
7 + result->write = 0;
8 + result->read = 0;
9 + result->max_size = max;
10 + return result;
11 +}
12 +
13 +void cbuffer_free(struct circular_buffer *buf) {
14 + freez(buf->data);
15 + freez(buf);
16 +}
17 +
18 +static int cbuffer_realloc_unsafe(struct circular_buffer *buf) {
19 + // Check that we can grow
20 + if (buf->size >= buf->max_size)
21 + return 1;
22 + size_t new_size = buf->size * 2;
23 + if (new_size > buf->max_size)
24 + new_size = buf->max_size;
25 +
26 + // We know that: size < new_size <= max_size
27 + // For simplicity align the current data at the bottom of the new buffer
28 + char *new_data = mallocz(new_size);
29 + if (buf->read == buf->write)
30 + buf->write = 0; // buffer is empty
31 + else if (buf->read < buf->write) {
32 + memcpy(new_data, buf->data + buf->read, buf->write - buf->read);
33 + buf->write -= buf->read;
34 + } else {
35 + size_t top_part = buf->size - buf->read;
36 + memcpy(new_data, buf->data + buf->read, top_part);
37 + memcpy(new_data + top_part, buf->data, buf->write);
38 + buf->write = top_part + buf->write;
39 + }
40 + buf->read = 0;
41 +
42 + // Switch buffers
43 + freez(buf->data);
44 + buf->data = new_data;
45 + buf->size = new_size;
46 + return 0;
47 +}
48 +
49 +int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len) {
50 + size_t len = (buf->write >= buf->read) ? (buf->write - buf->read) : (buf->size - buf->read + buf->write);
51 + while (d_len + len >= buf->size) {
52 + if (cbuffer_realloc_unsafe(buf)) {
53 + return 1;
54 + }
55 + }
56 + // Guarantee: write + d_len cannot hit read
57 + if (buf->write + d_len < buf->size) {
58 + memcpy(buf->data + buf->write, d, d_len);
59 + buf->write += d_len;
60 + }
61 + else {
62 + size_t top_part = buf->size - buf->write;
63 + memcpy(buf->data + buf->write, d, top_part);
64 + memcpy(buf->data, d + top_part, d_len - top_part);
65 + buf->write = d_len - top_part;
66 + }
67 + return 0;
68 +}
69 +
70 +// Assume caller does not remove too many bytes (i.e. read will jump over write)
71 +void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num) {
72 + buf->read += num;
73 + // Assume num < size (i.e. caller cannot remove more bytes than are in the buffer)
74 + if (buf->read >= buf->size)
75 + buf->read -= buf->size;
76 +}
77 +
78 +size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start) {
79 + if (start != NULL)
80 + *start = buf->data + buf->read;
81 + if (buf->read <= buf->write) {
82 + return buf->write - buf->read; // Includes empty case
83 + }
84 + return buf->size - buf->read;
85 +}
libnetdata/circular_buffer/circular_buffer.h new
+16
@@ -0,0 +1,16 @@
1 +#ifndef CIRCULAR_BUFFER_H
2 +#define CIRCULAR_BUFFER_H 1
3 +
4 +#include <string.h>
5 +
6 +struct circular_buffer {
7 + size_t size, write, read, max_size;
8 + char *data;
9 +};
10 +
11 +extern struct circular_buffer *cbuffer_new(size_t initial, size_t max);
12 +extern void cbuffer_free(struct circular_buffer *buf);
13 +extern int cbuffer_add_unsafe(struct circular_buffer *buf, const char *d, size_t d_len);
14 +extern void cbuffer_remove_unsafe(struct circular_buffer *buf, size_t num);
15 +extern size_t cbuffer_next_unsafe(struct circular_buffer *buf, char **start);
16 +#endif
libnetdata/clocks/clocks.h
+10
@@ -60,13 +60,23 @@ typedef struct heartbeat {
60
61 #endif // CLOCK_BOOTTIME
62
63 +#ifndef NSEC_PER_MSEC
64 #define NSEC_PER_MSEC 1000000ULL
65 +#endif
66
67 +#ifndef NSEC_PER_SEC
68 #define NSEC_PER_SEC 1000000000ULL
69 +#endif
70 +#ifndef NSEC_PER_USEC
71 #define NSEC_PER_USEC 1000ULL
72 +#endif
73
74 +#ifndef USEC_PER_SEC
75 #define USEC_PER_SEC 1000000ULL
76 +#endif
77 +#ifndef MSEC_PER_SEC
78 #define MSEC_PER_SEC 1000ULL
79 +#endif
80
81 #define USEC_PER_MS 1000ULL
82
libnetdata/libnetdata.h
+1
@@ -300,6 +300,7 @@ extern char *netdata_configured_host_prefix;
300 #include "threads/threads.h"
301 #include "buffer/buffer.h"
302 #include "locks/locks.h"
303 +#include "circular_buffer/circular_buffer.h"
304 #include "avl/avl.h"
305 #include "inlined.h"
306 #include "clocks/clocks.h"
parser/parser.c
+1 -33
@@ -209,42 +209,10 @@ int parser_next(PARSER *parser)
209 return 0;
210 }
211
212 -#ifdef ENABLE_HTTPS
213 - int normalread = 1;
214 - if (netdata_srv_ctx) {
215 - if (parser->host->stream_ssl.conn && !parser->host->stream_ssl.flags) {
216 - tmp = parser->buffer;
217 - if (!parser->bytesleft) {
218 - parser->readfrom = parser->tmpbuffer;
219 - parser->bytesleft =
220 - pluginsd_update_buffer(parser->readfrom, parser->host->stream_ssl.conn);
221 - if (parser->bytesleft <= 0) {
222 - return 1;
223 - }
224 - }
225 -
226 - parser->readfrom = pluginsd_get_from_buffer(
227 - parser->buffer, &parser->bytesleft, parser->readfrom,
228 - parser->host->stream_ssl.conn, parser->tmpbuffer);
229 -
230 - if (!parser->readfrom)
231 - tmp = NULL;
232 -
233 - normalread = 0;
234 - }
235 - }
236 - if (normalread) {
237 - if (unlikely(parser->read_function))
238 - tmp = parser->read_function(parser->buffer, PLUGINSD_LINE_MAX, parser->input);
239 - else
240 - tmp = fgets(parser->buffer, PLUGINSD_LINE_MAX, (FILE *)parser->input);
241 - }
242 -#else
212 if (unlikely(parser->read_function))
213 tmp = parser->read_function(parser->buffer, PLUGINSD_LINE_MAX, parser->input);
214 else
215 tmp = fgets(parser->buffer, PLUGINSD_LINE_MAX, (FILE *)parser->input);
247 -#endif
216
217 if (unlikely(!tmp)) {
218 if (unlikely(parser->eof_function)) {
@@ -320,7 +288,7 @@ inline int parser_action(PARSER *parser, char *input)
288 else
289 rc = PARSER_RC_ERROR;
290 #ifdef NETDATA_INTERNAL_CHECKS
323 - error("Unknown keyword [%s]", words[0]);
291 + error("Unknown keyword [%s]", input);
292 #endif
293 }
294 else {
streaming/receiver.c new
+429
@@ -0,0 +1,429 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdpush.h"
4 +
5 +extern struct config stream_config;
6 +
7 +static void rrdpush_receiver_thread_cleanup(void *ptr) {
8 + static __thread int executed = 0;
9 + if(!executed) {
10 + executed = 1;
11 + struct receiver_state *rpt = (struct receiver_state *) ptr;
12 +
13 + // Make sure that we detach this thread and don't kill a freshly arriving receiver
14 + netdata_mutex_lock(&rpt->host->receiver_lock);
15 + if (rpt->host->receiver == rpt)
16 + rpt->host->receiver = NULL;
17 + netdata_mutex_unlock(&rpt->host->receiver_lock);
18 +
19 + info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
20 +
21 + freez(rpt->key);
22 + freez(rpt->hostname);
23 + freez(rpt->registry_hostname);
24 + freez(rpt->machine_guid);
25 + freez(rpt->os);
26 + freez(rpt->timezone);
27 + freez(rpt->tags);
28 + freez(rpt->client_ip);
29 + freez(rpt->client_port);
30 + freez(rpt->program_name);
31 + freez(rpt->program_version);
32 +#ifdef ENABLE_HTTPS
33 + if(rpt->ssl.conn){
34 + SSL_free(rpt->ssl.conn);
35 + }
36 +#endif
37 + freez(rpt);
38 +
39 + }
40 +}
41 +
42 +#include "../collectors/plugins.d/pluginsd_parser.h"
43 +
44 +PARSER_RC streaming_timestamp(char **words, void *user, PLUGINSD_ACTION *plugins_action)
45 +{
46 + UNUSED(plugins_action);
47 + char *remote_time_txt = words[1];
48 + time_t remote_time = 0;
49 + RRDHOST *host = ((PARSER_USER_OBJECT *)user)->host;
50 + struct plugind *cd = ((PARSER_USER_OBJECT *)user)->cd;
51 + if (cd->version < VERSION_GAP_FILLING ) {
52 + error("STREAM %s from %s: Slave negotiated version %u but sent TIMESTAMP!", host->hostname, cd->cmd,
53 + cd->version);
54 + return PARSER_RC_OK; // Ignore error and continue stream
55 + }
56 + if (remote_time_txt && *remote_time_txt) {
57 + remote_time = str2ull(remote_time_txt);
58 + time_t now = now_realtime_sec(), prev = rrdhost_last_entry_t(host);
59 + time_t gap = 0;
60 + if (prev == 0)
61 + info("STREAM %s from %s: Initial connection (no gap to check), remote=%ld local=%ld slew=%ld",
62 + host->hostname, cd->cmd, remote_time, now, now-remote_time);
63 + else {
64 + gap = now - prev;
65 + info("STREAM %s from %s: Checking for gaps... remote=%ld local=%ld..%ld slew=%ld %ld-sec gap",
66 + host->hostname, cd->cmd, remote_time, prev, now, remote_time - now, gap);
67 + }
68 + char message[128];
69 + sprintf(message,"REPLICATE %ld %ld\n", remote_time - gap, remote_time);
70 + int ret;
71 +#ifdef ENABLE_HTTPS
72 + SSL *conn = host->stream_ssl.conn ;
73 + if(conn && !host->stream_ssl.flags) {
74 + ret = SSL_write(conn, message, strlen(message));
75 + } else {
76 + ret = send(host->receiver->fd, message, strlen(message), MSG_DONTWAIT);
77 + }
78 +#else
79 + ret = send(host->receiver->fd, message, strlen(message), MSG_DONTWAIT);
80 +#endif
81 + if (ret != (int)strlen(message))
82 + error("Failed to send initial timestamp - gaps may appear in charts");
83 + return PARSER_RC_OK;
84 + }
85 + return PARSER_RC_ERROR;
86 +}
87 +
88 +/* The receiver socket is blocking, perform a single read into a buffer so that we can reassemble lines for parsing.
89 + */
90 +static int receiver_read(struct receiver_state *r, FILE *fp) {
91 +#ifdef ENABLE_HTTPS
92 + if (r->ssl.conn && !r->ssl.flags) {
93 + ERR_clear_error();
94 + int desired = sizeof(r->read_buffer) - r->read_len - 1;
95 + int ret = SSL_read(r->ssl.conn, r->read_buffer + r->read_len, desired);
96 + if (ret > 0 ) {
97 + r->read_len += ret;
98 + return 0;
99 + }
100 + // Don't treat SSL_ERROR_WANT_READ or SSL_ERROR_WANT_WRITE differently on blocking socket
101 + u_long err;
102 + char buf[256];
103 + while ((err = ERR_get_error()) != 0) {
104 + ERR_error_string_n(err, buf, sizeof(buf));
105 + error("STREAM %s [receive from %s] ssl error: %s", r->hostname, r->client_ip, buf);
106 + }
107 + return 1;
108 + }
109 +#endif
110 + if (!fgets(r->read_buffer, sizeof(r->read_buffer), fp))
111 + return 1;
112 + r->read_len = strlen(r->read_buffer);
113 + return 0;
114 +}
115 +
116 +/* Produce a full line if one exists, statefully return where we start next time.
117 + * When we hit the end of the buffer with a partial line move it to the beginning for the next fill.
118 + */
119 +static char *receiver_next_line(struct receiver_state *r, int *pos) {
120 + int start = *pos, scan = *pos;
121 + if (scan >= r->read_len) {
122 + r->read_len = 0;
123 + return NULL;
124 + }
125 + while (scan < r->read_len && r->read_buffer[scan] != '\n')
126 + scan++;
127 + if (scan < r->read_len && r->read_buffer[scan] == '\n') {
128 + *pos = scan+1;
129 + r->read_buffer[scan] = 0;
130 + return &r->read_buffer[start];
131 + }
132 + memcpy(r->read_buffer, &r->read_buffer[start], r->read_len - start);
133 + r->read_len -= start;
134 + return NULL;
135 +}
136 +
137 +
138 +size_t streaming_parser(struct receiver_state *rpt, struct plugind *cd, FILE *fp) {
139 + size_t result;
140 + PARSER_USER_OBJECT *user = callocz(1, sizeof(*user));
141 + user->enabled = cd->enabled;
142 + user->host = rpt->host;
143 + user->opaque = rpt;
144 + user->cd = cd;
145 + user->trust_durations = 0;
146 +
147 + PARSER *parser = parser_init(rpt->host, user, fp, PARSER_INPUT_SPLIT);
148 + parser_add_keyword(parser, "TIMESTAMP", streaming_timestamp);
149 +
150 + if (unlikely(!parser)) {
151 + error("Failed to initialize parser");
152 + cd->serial_failures++;
153 + freez(user);
154 + return 0;
155 + }
156 +
157 + parser->plugins_action->begin_action = &pluginsd_begin_action;
158 + parser->plugins_action->flush_action = &pluginsd_flush_action;
159 + parser->plugins_action->end_action = &pluginsd_end_action;
160 + parser->plugins_action->disable_action = &pluginsd_disable_action;
161 + parser->plugins_action->variable_action = &pluginsd_variable_action;
162 + parser->plugins_action->dimension_action = &pluginsd_dimension_action;
163 + parser->plugins_action->label_action = &pluginsd_label_action;
164 + parser->plugins_action->overwrite_action = &pluginsd_overwrite_action;
165 + parser->plugins_action->chart_action = &pluginsd_chart_action;
166 + parser->plugins_action->set_action = &pluginsd_set_action;
167 +
168 + user->parser = parser;
169 +
170 + do {
171 + if (receiver_read(rpt, fp))
172 + break;
173 + int pos = 0;
174 + char *line;
175 + while ((line = receiver_next_line(rpt, &pos))) {
176 + if (unlikely(netdata_exit || rpt->shutdown || parser_action(parser, line)))
177 + goto done;
178 + }
179 + rpt->last_msg_t = now_realtime_sec();
180 + }
181 + while(!netdata_exit);
182 +done:
183 + result= user->count;
184 + freez(user);
185 + parser_destroy(parser);
186 + return result;
187 +}
188 +
189 +
190 +static int rrdpush_receive(struct receiver_state *rpt)
191 +{
192 + int history = default_rrd_history_entries;
193 + RRD_MEMORY_MODE mode = default_rrd_memory_mode;
194 + int health_enabled = default_health_enabled;
195 + int rrdpush_enabled = default_rrdpush_enabled;
196 + char *rrdpush_destination = default_rrdpush_destination;
197 + char *rrdpush_api_key = default_rrdpush_api_key;
198 + char *rrdpush_send_charts_matching = default_rrdpush_send_charts_matching;
199 + time_t alarms_delay = 60;
200 +
201 + rpt->update_every = (int)appconfig_get_number(&stream_config, rpt->machine_guid, "update every", rpt->update_every);
202 + if(rpt->update_every < 0) rpt->update_every = 1;
203 +
204 + history = (int)appconfig_get_number(&stream_config, rpt->key, "default history", history);
205 + history = (int)appconfig_get_number(&stream_config, rpt->machine_guid, "history", history);
206 + if(history < 5) history = 5;
207 +
208 + mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->key, "default memory mode", rrd_memory_mode_name(mode)));
209 + mode = rrd_memory_mode_id(appconfig_get(&stream_config, rpt->machine_guid, "memory mode", rrd_memory_mode_name(mode)));
210 +
211 + health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->key, "health enabled by default", health_enabled);
212 + health_enabled = appconfig_get_boolean_ondemand(&stream_config, rpt->machine_guid, "health enabled", health_enabled);
213 +
214 + alarms_delay = appconfig_get_number(&stream_config, rpt->key, "default postpone alarms on connect seconds", alarms_delay);
215 + alarms_delay = appconfig_get_number(&stream_config, rpt->machine_guid, "postpone alarms on connect seconds", alarms_delay);
216 +
217 + rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->key, "default proxy enabled", rrdpush_enabled);
218 + rrdpush_enabled = appconfig_get_boolean(&stream_config, rpt->machine_guid, "proxy enabled", rrdpush_enabled);
219 +
220 + rrdpush_destination = appconfig_get(&stream_config, rpt->key, "default proxy destination", rrdpush_destination);
221 + rrdpush_destination = appconfig_get(&stream_config, rpt->machine_guid, "proxy destination", rrdpush_destination);
222 +
223 + rrdpush_api_key = appconfig_get(&stream_config, rpt->key, "default proxy api key", rrdpush_api_key);
224 + rrdpush_api_key = appconfig_get(&stream_config, rpt->machine_guid, "proxy api key", rrdpush_api_key);
225 +
226 + rrdpush_send_charts_matching = appconfig_get(&stream_config, rpt->key, "default proxy send charts matching", rrdpush_send_charts_matching);
227 + rrdpush_send_charts_matching = appconfig_get(&stream_config, rpt->machine_guid, "proxy send charts matching", rrdpush_send_charts_matching);
228 +
229 + rpt->tags = (char*)appconfig_set_default(&stream_config, rpt->machine_guid, "host tags", (rpt->tags)?rpt->tags:"");
230 + if(rpt->tags && !*rpt->tags) rpt->tags = NULL;
231 +
232 + if (strcmp(rpt->machine_guid, localhost->machine_guid) == 0) {
233 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->machine_guid, rpt->hostname, "DENIED - ATTEMPT TO RECEIVE METRICS FROM MACHINE_GUID IDENTICAL TO MASTER");
234 + error("STREAM %s [receive from %s:%s]: denied to receive metrics, machine GUID [%s] is my own. Did you copy the master/proxy machine guid to a slave?", rpt->hostname, rpt->client_ip, rpt->client_port, rpt->machine_guid);
235 + close(rpt->fd);
236 + return 1;
237 + }
238 +
239 + if (rpt->host==NULL) {
240 +
241 + rpt->host = rrdhost_find_or_create(
242 + rpt->hostname
243 + , rpt->registry_hostname
244 + , rpt->machine_guid
245 + , rpt->os
246 + , rpt->timezone
247 + , rpt->tags
248 + , program_name
249 + , program_version
250 + , rpt->update_every
251 + , history
252 + , mode
253 + , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
254 + , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
255 + , rrdpush_destination
256 + , rrdpush_api_key
257 + , rrdpush_send_charts_matching
258 + , rpt->system_info
259 + );
260 +
261 + if(!rpt->host) {
262 + close(rpt->fd);
263 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->machine_guid, rpt->hostname, "FAILED - CANNOT ACQUIRE HOST");
264 + error("STREAM %s [receive from [%s]:%s]: failed to find/create host structure.", rpt->hostname, rpt->client_ip, rpt->client_port);
265 + return 1;
266 + }
267 +
268 + netdata_mutex_lock(&rpt->host->receiver_lock);
269 + if (rpt->host->receiver == NULL)
270 + rpt->host->receiver = rpt;
271 + else {
272 + error("Multiple receivers connected for %s concurrently, cancelling this one...", rpt->machine_guid);
273 + netdata_mutex_unlock(&rpt->host->receiver_lock);
274 + close(rpt->fd);
275 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->machine_guid, rpt->hostname, "FAILED - BEATEN TO HOST CREATION");
276 + return 1;
277 + }
278 + netdata_mutex_unlock(&rpt->host->receiver_lock);
279 + }
280 +
281 + int ssl = 0;
282 +#ifdef ENABLE_HTTPS
283 + if (rpt->ssl.conn != NULL)
284 + ssl = 1;
285 +#endif
286 +
287 +#ifdef NETDATA_INTERNAL_CHECKS
288 + info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s,%s tags '%s'"
289 + , rpt->hostname
290 + , rpt->client_ip
291 + , rpt->client_port
292 + , rpt->host->hostname
293 + , rpt->host->machine_guid
294 + , rpt->host->rrd_update_every
295 + , rpt->host->rrd_history_entries
296 + , rrd_memory_mode_name(rpt->host->rrd_memory_mode)
297 + , (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
298 + , ssl ? " SSL," : ""
299 + , rpt->host->tags?rpt->host->tags:""
300 + );
301 +#endif // NETDATA_INTERNAL_CHECKS
302 +
303 +
304 + struct plugind cd = {
305 + .enabled = 1,
306 + .update_every = default_rrd_update_every,
307 + .pid = 0,
308 + .serial_failures = 0,
309 + .successful_collections = 0,
310 + .obsolete = 0,
311 + .started_t = now_realtime_sec(),
312 + .next = NULL,
313 + .version = 0,
314 + };
315 +
316 + // put the client IP and port into the buffers used by plugins.d
317 + snprintfz(cd.id, CONFIG_MAX_NAME, "%s:%s", rpt->client_ip, rpt->client_port);
318 + snprintfz(cd.filename, FILENAME_MAX, "%s:%s", rpt->client_ip, rpt->client_port);
319 + snprintfz(cd.fullfilename, FILENAME_MAX, "%s:%s", rpt->client_ip, rpt->client_port);
320 + snprintfz(cd.cmd, PLUGINSD_CMD_MAX, "%s:%s", rpt->client_ip, rpt->client_port);
321 +
322 + info("STREAM %s [receive from [%s]:%s]: initializing communication...", rpt->host->hostname, rpt->client_ip, rpt->client_port);
323 + char initial_response[HTTP_HEADER_SIZE];
324 + if (rpt->stream_version > 1) {
325 + info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version);
326 + sprintf(initial_response, "%s%u", START_STREAMING_PROMPT_VN, rpt->stream_version);
327 + } else if (rpt->stream_version == 1) {
328 + info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->stream_version);
329 + sprintf(initial_response, "%s", START_STREAMING_PROMPT_V2);
330 + } else {
331 + info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
332 + sprintf(initial_response, "%s", START_STREAMING_PROMPT);
333 + }
334 + debug(D_STREAM, "Initial response to %s: %s", rpt->client_ip, initial_response);
335 + #ifdef ENABLE_HTTPS
336 + rpt->host->stream_ssl.conn = rpt->ssl.conn;
337 + rpt->host->stream_ssl.flags = rpt->ssl.flags;
338 + if(send_timeout(&rpt->ssl, rpt->fd, initial_response, strlen(initial_response), 0, 60) != (ssize_t)strlen(initial_response)) {
339 +#else
340 + if(send_timeout(rpt->fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) {
341 +#endif
342 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - CANNOT REPLY");
343 + error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
344 + close(rpt->fd);
345 + return 0;
346 + }
347 +
348 + // remove the non-blocking flag from the socket
349 + if(sock_delnonblock(rpt->fd) < 0)
350 + error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd);
351 +
352 + // convert the socket to a FILE *
353 + FILE *fp = fdopen(rpt->fd, "r");
354 + if(!fp) {
355 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "FAILED - SOCKET ERROR");
356 + error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", rpt->host->hostname, rpt->client_ip, rpt->client_port, rpt->fd);
357 + close(rpt->fd);
358 + return 0;
359 + }
360 +
361 + rrdhost_wrlock(rpt->host);
362 +/* if(rpt->host->connected_senders > 0) {
363 + rrdhost_unlock(rpt->host);
364 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "REJECTED - ALREADY CONNECTED");
365 + info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. Rejecting new connection.", rpt->host->hostname, rpt->client_ip, rpt->client_port);
366 + fclose(fp);
367 + return 0;
368 + }
369 +*/
370 +
371 + rrdhost_flag_clear(rpt->host, RRDHOST_FLAG_ORPHAN);
372 +// rpt->host->connected_senders++;
373 + rpt->host->senders_disconnected_time = 0;
374 + rpt->host->labels_flag = (rpt->stream_version > 0)?LABEL_FLAG_UPDATE_STREAM:LABEL_FLAG_STOP_STREAM;
375 +
376 + if(health_enabled != CONFIG_BOOLEAN_NO) {
377 + if(alarms_delay > 0) {
378 + rpt->host->health_delay_up_to = now_realtime_sec() + alarms_delay;
379 + info("Postponing health checks for %ld seconds, on host '%s', because it was just connected."
380 + , alarms_delay
381 + , rpt->host->hostname
382 + );
383 + }
384 + }
385 + rrdhost_unlock(rpt->host);
386 +
387 + // call the plugins.d processor to receive the metrics
388 + info("STREAM %s [receive from [%s]:%s]: receiving metrics...", rpt->host->hostname, rpt->client_ip, rpt->client_port);
389 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "CONNECTED");
390 +
391 + cd.version = rpt->stream_version;
392 +
393 +
394 + size_t count = streaming_parser(rpt, &cd, fp);
395 + //size_t count = pluginsd_process(host, &cd, fp, 1);
396 +
397 + log_stream_connection(rpt->client_ip, rpt->client_port, rpt->key, rpt->host->machine_guid, rpt->host->hostname, "DISCONNECTED");
398 + error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", rpt->host->hostname, rpt->client_ip, rpt->client_port, count);
399 +
400 + netdata_mutex_lock(&rpt->host->receiver_lock);
401 + if (rpt->host->receiver == rpt) {
402 + rrdhost_wrlock(rpt->host);
403 + rpt->host->senders_disconnected_time = now_realtime_sec();
404 + rrdhost_flag_set(rpt->host, RRDHOST_FLAG_ORPHAN);
405 + if(health_enabled == CONFIG_BOOLEAN_AUTO)
406 + rpt->host->health_enabled = 0;
407 + rrdhost_unlock(rpt->host);
408 + rrdpush_sender_thread_stop(rpt->host);
409 + }
410 + netdata_mutex_unlock(&rpt->host->receiver_lock);
411 +
412 + // cleanup
413 + fclose(fp);
414 +
415 + return (int)count;
416 +}
417 +
418 +void *rrdpush_receiver_thread(void *ptr) {
419 + netdata_thread_cleanup_push(rrdpush_receiver_thread_cleanup, ptr);
420 +
421 + struct receiver_state *rpt = (struct receiver_state *)ptr;
422 + info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
423 +
424 + rrdpush_receive(rpt);
425 +
426 + netdata_thread_cleanup_pop(1);
427 + return NULL;
428 +}
429 +
streaming/rrdpush.c
+74 -1025
@@ -26,25 +26,7 @@
26 *
27 */
28
29 -#define STREAMING_PROTOCOL_VERSION "1.1"
30 -#define START_STREAMING_PROMPT "Hit me baby, push them over..."
31 -#define START_STREAMING_PROMPT_V2 "Hit me baby, push them over and bring the host labels..."
32 -#define START_STREAMING_PROMPT_VN "Hit me baby, push them over with the version="
33 -
34 -typedef enum {
35 - RRDPUSH_MULTIPLE_CONNECTIONS_ALLOW,
36 - RRDPUSH_MULTIPLE_CONNECTIONS_DENY_NEW
37 -} RRDPUSH_MULTIPLE_CONNECTIONS_STRATEGY;
38 -
39 -typedef struct {
40 - char *os_name;
41 - char *os_id;
42 - char *os_version;
43 - char *kernel_name;
44 - char *kernel_version;
45 -} stream_encoded_t;
46 -
47 -static struct config stream_config = {
29 +struct config stream_config = {
30 .first_section = NULL,
31 .last_section = NULL,
32 .mutex = NETDATA_MUTEX_INITIALIZER,
@@ -125,8 +107,6 @@ int rrdpush_init() {
107 return default_rrdpush_enabled;
108 }
109
128 -#define CONNECTED_TO_SIZE 100
129 -
110 // data collection happens from multiple threads
111 // each of these threads calls rrdset_done()
112 // which in turn calls rrdset_done_push()
@@ -141,8 +121,6 @@ int rrdpush_init() {
121 // this is for the first iterations of each chart
122 unsigned int remote_clock_resync_iterations = 60;
123
144 -#define rrdpush_buffer_lock(host) netdata_mutex_lock(&((host)->rrdpush_sender_buffer_mutex))
145 -#define rrdpush_buffer_unlock(host) netdata_mutex_unlock(&((host)->rrdpush_sender_buffer_mutex))
124
125 static inline int should_send_chart_matching(RRDSET *st) {
126 if(unlikely(!rrdset_flag_check(st, RRDSET_FLAG_ENABLED))) {
@@ -205,7 +183,8 @@ static inline int need_to_send_chart_definition(RRDSET *st) {
183 return 0;
184 }
185
208 -// sends the current chart definition
186 +// Send the current chart definition.
187 +// Assumes that collector thread has already called sender_start for mutex / buffer state.
188 static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
189 RRDHOST *host = st->rrdhost;
190
@@ -224,11 +203,9 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
203 }
204 }
205
227 - // info("CHART '%s' '%s'", st->id, name);
228 -
206 // send the chart
207 buffer_sprintf(
231 - host->rrdpush_sender_buffer
208 + host->sender->build
209 , "CHART \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" \"%s\" %ld %d \"%s %s %s %s\" \"%s\" \"%s\"\n"
210 , st->id
211 , name
@@ -251,7 +228,7 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
228 RRDDIM *rd;
229 rrddim_foreach_read(rd, st) {
230 buffer_sprintf(
254 - host->rrdpush_sender_buffer
231 + host->sender->build
232 , "DIMENSION \"%s\" \"%s\" \"%s\" " COLLECTED_NUMBER_FORMAT " " COLLECTED_NUMBER_FORMAT " \"%s %s %s\"\n"
233 , rd->id
234 , rd->name
@@ -272,7 +249,7 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
249 calculated_number *value = (calculated_number *) rs->value;
250
251 buffer_sprintf(
275 - host->rrdpush_sender_buffer
252 + host->sender->build
253 , "VARIABLE CHART %s = " CALCULATED_NUMBER_FORMAT "\n"
254 , rs->variable
255 , *value
@@ -284,25 +261,29 @@ static inline void rrdpush_send_chart_definition_nolock(RRDSET *st) {
261 }
262
263 // sends the current chart dimensions
287 -static inline void rrdpush_send_chart_metrics_nolock(RRDSET *st) {
264 +static inline void rrdpush_send_chart_metrics_nolock(RRDSET *st, struct sender_state *s) {
265 RRDHOST *host = st->rrdhost;
289 - buffer_sprintf(host->rrdpush_sender_buffer, "BEGIN \"%s\" %llu\n", st->id, (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0);
266 + buffer_sprintf(host->sender->build, "BEGIN \"%s\" %llu", st->id, (st->last_collected_time.tv_sec > st->upstream_resync_time)?st->usec_since_last_update:0);
267 + if (s->version >= VERSION_GAP_FILLING)
268 + buffer_sprintf(host->sender->build, " %ld\n", st->last_collected_time.tv_sec);
269 + else
270 + buffer_strcat(host->sender->build, "\n");
271
272 RRDDIM *rd;
273 rrddim_foreach_read(rd, st) {
274 if(rd->updated && rd->exposed)
294 - buffer_sprintf(host->rrdpush_sender_buffer
275 + buffer_sprintf(host->sender->build
276 , "SET \"%s\" = " COLLECTED_NUMBER_FORMAT "\n"
277 , rd->id
278 , rd->collected_value
279 );
280 }
300 -
301 - buffer_strcat(host->rrdpush_sender_buffer, "END\n");
281 + buffer_strcat(host->sender->build, "END\n");
282 }
283
284 static void rrdpush_sender_thread_spawn(RRDHOST *host);
285
286 +// Called from the internal collectors to mark a chart obsolete.
287 void rrdset_push_chart_definition_now(RRDSET *st) {
288 RRDHOST *host = st->rrdhost;
289
@@ -310,9 +291,9 @@ void rrdset_push_chart_definition_now(RRDSET *st) {
291 return;
292
293 rrdset_rdlock(st);
313 - rrdpush_buffer_lock(host);
294 + sender_start(host->sender);
295 rrdpush_send_chart_definition_nolock(st);
315 - rrdpush_buffer_unlock(host);
296 + sender_commit(host->sender);
297 rrdset_unlock(st);
298 }
299
@@ -322,18 +303,14 @@ void rrdset_done_push(RRDSET *st) {
303
304 RRDHOST *host = st->rrdhost;
305
325 - rrdpush_buffer_lock(host);
326 -
306 if(unlikely(host->rrdpush_send_enabled && !host->rrdpush_sender_spawn))
307 rrdpush_sender_thread_spawn(host);
308
330 - if(unlikely(!host->rrdpush_sender_buffer || !host->rrdpush_sender_connected)) {
309 + // Handle non-connected case
310 + if(unlikely(!host->rrdpush_sender_connected)) {
311 if(unlikely(!host->rrdpush_sender_error_shown))
312 error("STREAM %s [send]: not ready - discarding collected metrics.", host->hostname);
333 -
313 host->rrdpush_sender_error_shown = 1;
335 -
336 - rrdpush_buffer_unlock(host);
314 return;
315 }
316 else if(unlikely(host->rrdpush_sender_error_shown)) {
@@ -341,16 +318,18 @@ void rrdset_done_push(RRDSET *st) {
318 host->rrdpush_sender_error_shown = 0;
319 }
320
321 + sender_start(host->sender);
322 +
323 if(need_to_send_chart_definition(st))
324 rrdpush_send_chart_definition_nolock(st);
325
347 - rrdpush_send_chart_metrics_nolock(st);
326 + rrdpush_send_chart_metrics_nolock(st, host->sender);
327
328 // signal the sender there are more data
329 if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
330 error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
331
353 - rrdpush_buffer_unlock(host);
332 + sender_commit(host->sender);
333 }
334
335 // labels
@@ -358,13 +337,13 @@ void rrdpush_send_labels(RRDHOST *host) {
337 if (!host->labels || !(host->labels_flag & LABEL_FLAG_UPDATE_STREAM) || (host->labels_flag & LABEL_FLAG_STOP_STREAM))
338 return;
339
361 - rrdpush_buffer_lock(host);
340 + sender_start(host->sender);
341 rrdhost_rdlock(host);
342 netdata_rwlock_rdlock(&host->labels_rwlock);
343
344 struct label *labels = host->labels;
345 while(labels) {
367 - buffer_sprintf(host->rrdpush_sender_buffer
346 + buffer_sprintf(host->sender->build
347 , "LABEL \"%s\" = %d %s\n"
348 , labels->key
349 , (int)labels->label_source
@@ -373,103 +352,26 @@ void rrdpush_send_labels(RRDHOST *host) {
352 labels = labels->next;
353 }
354
376 - buffer_sprintf(host->rrdpush_sender_buffer
355 + buffer_sprintf(host->sender->build
356 , "OVERWRITE %s\n", "labels");
357
358 netdata_rwlock_unlock(&host->labels_rwlock);
359 rrdhost_unlock(host);
360 + sender_commit(host->sender);
361
362 if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1 && write(host->rrdpush_sender_pipe[PIPE_WRITE], " ", 1) == -1)
363 error("STREAM %s [send]: cannot write to internal pipe", host->hostname);
364
385 - rrdpush_buffer_unlock(host);
365 host->labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
366 }
367 // ----------------------------------------------------------------------------
368 // rrdpush sender thread
369
391 -static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) {
392 - calculated_number *value = (calculated_number *)rv->value;
393 -
394 - buffer_sprintf(
395 - host->rrdpush_sender_buffer
396 - , "VARIABLE HOST %s = " CALCULATED_NUMBER_FORMAT "\n"
397 - , rv->name
398 - , *value
399 - );
400 -
401 - debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " CALCULATED_NUMBER_FORMAT, rv->name, *value);
402 -}
403 -
404 -void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {
405 - if(host->rrdpush_send_enabled && host->rrdpush_sender_spawn && host->rrdpush_sender_connected) {
406 - rrdpush_buffer_lock(host);
407 - rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv);
408 - rrdpush_buffer_unlock(host);
409 - }
410 -}
411 -
412 -static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr, void *host_ptr) {
413 - RRDVAR *rv = (RRDVAR *)rrdvar_ptr;
414 - RRDHOST *host = (RRDHOST *)host_ptr;
415 -
416 - if(unlikely(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR && rv->type == RRDVAR_TYPE_CALCULATED)) {
417 - rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv);
418 -
419 - // return 1, so that the traversal will return the number of variables sent
420 - return 1;
421 - }
422 -
423 - // returning a negative number will break the traversal
424 - return 0;
425 -}
426 -
427 -static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
428 - int ret = rrdvar_callback_for_all_host_variables(host, rrdpush_sender_thread_custom_host_variables_callback, host);
429 - (void)ret;
430 -
431 - debug(D_STREAM, "RRDVAR sent %d VARIABLES", ret);
432 -}
433 -
434 -// resets all the chart, so that their definitions
435 -// will be resent to the central netdata
436 -static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
437 - rrdhost_rdlock(host);
438 -
439 - RRDSET *st;
440 - rrdset_foreach_read(st, host) {
441 - rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
442 -
443 - st->upstream_resync_time = 0;
444 -
445 - rrdset_rdlock(st);
446 -
447 - RRDDIM *rd;
448 - rrddim_foreach_read(rd, st)
449 - rd->exposed = 0;
450 -
451 - rrdset_unlock(st);
452 - }
453 -
454 - rrdhost_unlock(host);
455 -}
456 -
457 -static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
458 - rrdpush_buffer_lock(host);
459 -
460 - if(buffer_strlen(host->rrdpush_sender_buffer))
461 - error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, buffer_strlen(host->rrdpush_sender_buffer));
462 -
463 - buffer_flush(host->rrdpush_sender_buffer);
464 -
465 - rrdpush_sender_thread_reset_all_charts(host);
466 - rrdpush_sender_thread_send_custom_host_variables(host);
467 -
468 - rrdpush_buffer_unlock(host);
469 -}
470 -
370 +// Either the receiver lost the connection or the host is being destroyed.
371 +// Don't lock the sender buffer - doesn't affect consistency in either case.
372 +// TODO-GAPS During the host destruction sequence we should make sure the disconnect happens early enough to lock
373 +// out collectors hitting the sender. Locking the mutex means there may be waiting threads when we free.
374 void rrdpush_sender_thread_stop(RRDHOST *host) {
472 - rrdpush_buffer_lock(host);
375 rrdhost_wrlock(host);
376
377 netdata_thread_t thr = 0;
@@ -489,7 +391,6 @@ void rrdpush_sender_thread_stop(RRDHOST *host) {
391 }
392
393 rrdhost_unlock(host);
492 - rrdpush_buffer_unlock(host);
394
395 if(thr != 0) {
396 info("STREAM %s [send]: waiting for the sending thread to stop...", host->hostname);
@@ -499,903 +400,14 @@ void rrdpush_sender_thread_stop(RRDHOST *host) {
400 }
401 }
402
502 -static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) {
503 - host->rrdpush_sender_connected = 0;
504 -
505 - if(host->rrdpush_sender_socket != -1) {
506 - close(host->rrdpush_sender_socket);
507 - host->rrdpush_sender_socket = -1;
508 - }
509 -}
510 -
511 -static inline void rrdpush_set_flags_to_newest_stream(RRDHOST *host) {
512 - host->labels_flag |= LABEL_FLAG_UPDATE_STREAM;
513 - host->labels_flag &= ~LABEL_FLAG_STOP_STREAM;
514 -}
515 -
516 -void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host)
517 -{
518 - se->os_name = (host->system_info->host_os_name)?url_encode(host->system_info->host_os_name):"";
519 - se->os_id = (host->system_info->host_os_id)?url_encode(host->system_info->host_os_id):"";
520 - se->os_version = (host->system_info->host_os_version)?url_encode(host->system_info->host_os_version):"";
521 - se->kernel_name = (host->system_info->kernel_name)?url_encode(host->system_info->kernel_name):"";
522 - se->kernel_version = (host->system_info->kernel_version)?url_encode(host->system_info->kernel_version):"";
523 -}
524 -
525 -void rrdpush_clean_encoded(stream_encoded_t *se)
526 -{
527 - if (se->os_name)
528 - freez(se->os_name);
529 -
530 - if (se->os_id)
531 - freez(se->os_id);
532 -
533 - if (se->os_version)
534 - freez(se->os_version);
535 -
536 - if (se->kernel_name)
537 - freez(se->kernel_name);
538 -
539 - if (se->kernel_version)
540 - freez(se->kernel_version);
541 -}
542 -
543 -//called from client side
544 -static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_port, int timeout, size_t *reconnects_counter, char *connected_to, size_t connected_to_size) {
545 - struct timeval tv = {
546 - .tv_sec = timeout,
547 - .tv_usec = 0
548 - };
549 -
550 - // make sure the socket is closed
551 - rrdpush_sender_thread_close_socket(host);
552 -
553 - debug(D_STREAM, "STREAM: Attempting to connect...");
554 - info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_send_destination);
555 -
556 - host->rrdpush_sender_socket = connect_to_one_of(
557 - host->rrdpush_send_destination
558 - , default_port
559 - , &tv
560 - , reconnects_counter
561 - , connected_to
562 - , connected_to_size
563 - );
564 -
565 - if(unlikely(host->rrdpush_sender_socket == -1)) {
566 - error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_send_destination);
567 - return 0;
568 - }
569 -
570 - info("STREAM %s [send to %s]: initializing communication...", host->hostname, connected_to);
571 -
572 -#ifdef ENABLE_HTTPS
573 - if( netdata_client_ctx ){
574 - host->ssl.flags = NETDATA_SSL_START;
575 - if (!host->ssl.conn){
576 - host->ssl.conn = SSL_new(netdata_client_ctx);
577 - if(!host->ssl.conn){
578 - error("Failed to allocate SSL structure.");
579 - host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
580 - }
581 - }
582 - else{
583 - SSL_clear(host->ssl.conn);
584 - }
585 -
586 - if (host->ssl.conn)
587 - {
588 - if (SSL_set_fd(host->ssl.conn, host->rrdpush_sender_socket) != 1) {
589 - error("Failed to set the socket to the SSL on socket fd %d.", host->rrdpush_sender_socket);
590 - host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
591 - } else{
592 - host->ssl.flags = NETDATA_SSL_HANDSHAKE_COMPLETE;
593 - }
594 - }
595 - }
596 - else {
597 - host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
598 - }
599 -#endif
600 -
601 - /* TODO: During the implementation of #7265 switch the set of variables to HOST_* and CONTAINER_* if the
602 - version negotiation resulted in a high enough version.
603 - */
604 - stream_encoded_t se;
605 - rrdpush_encode_variable(&se, host);
606 -
607 - #define HTTP_HEADER_SIZE 8192
608 - char http[HTTP_HEADER_SIZE + 1];
609 - int eol = snprintfz(http, HTTP_HEADER_SIZE,
610 - "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s&ver=%u"
611 - "&NETDATA_SYSTEM_OS_NAME=%s"
612 - "&NETDATA_SYSTEM_OS_ID=%s"
613 - "&NETDATA_SYSTEM_OS_ID_LIKE=%s"
614 - "&NETDATA_SYSTEM_OS_VERSION=%s"
615 - "&NETDATA_SYSTEM_OS_VERSION_ID=%s"
616 - "&NETDATA_SYSTEM_OS_DETECTION=%s"
617 - "&NETDATA_SYSTEM_KERNEL_NAME=%s"
618 - "&NETDATA_SYSTEM_KERNEL_VERSION=%s"
619 - "&NETDATA_SYSTEM_ARCHITECTURE=%s"
620 - "&NETDATA_SYSTEM_VIRTUALIZATION=%s"
621 - "&NETDATA_SYSTEM_VIRT_DETECTION=%s"
622 - "&NETDATA_SYSTEM_CONTAINER=%s"
623 - "&NETDATA_SYSTEM_CONTAINER_DETECTION=%s"
624 - "&NETDATA_CONTAINER_OS_NAME=%s"
625 - "&NETDATA_CONTAINER_OS_ID=%s"
626 - "&NETDATA_CONTAINER_OS_ID_LIKE=%s"
627 - "&NETDATA_CONTAINER_OS_VERSION=%s"
628 - "&NETDATA_CONTAINER_OS_VERSION_ID=%s"
629 - "&NETDATA_CONTAINER_OS_DETECTION=%s"
630 - "&NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT=%s"
631 - "&NETDATA_SYSTEM_CPU_FREQ=%s"
632 - "&NETDATA_SYSTEM_TOTAL_RAM=%s"
633 - "&NETDATA_SYSTEM_TOTAL_DISK_SIZE=%s"
634 - "&NETDATA_PROTOCOL_VERSION=%s"
635 - " HTTP/1.1\r\n"
636 - "User-Agent: %s/%s\r\n"
637 - "Accept: */*\r\n\r\n"
638 - , host->rrdpush_send_api_key
639 - , host->hostname
640 - , host->registry_hostname
641 - , host->machine_guid
642 - , default_rrd_update_every
643 - , host->os
644 - , host->timezone
645 - , (host->tags) ? host->tags : ""
646 - , STREAMING_PROTOCOL_CURRENT_VERSION
647 - , se.os_name
648 - , se.os_id
649 - , (host->system_info->host_os_id_like) ? host->system_info->host_os_id_like : ""
650 - , se.os_version
651 - , (host->system_info->host_os_version_id) ? host->system_info->host_os_version_id : ""
652 - , (host->system_info->host_os_detection) ? host->system_info->host_os_detection : ""
653 - , se.kernel_name
654 - , se.kernel_version
655 - , (host->system_info->architecture) ? host->system_info->architecture : ""
656 - , (host->system_info->virtualization) ? host->system_info->virtualization : ""
657 - , (host->system_info->virt_detection) ? host->system_info->virt_detection : ""
658 - , (host->system_info->container) ? host->system_info->container : ""
659 - , (host->system_info->container_detection) ? host->system_info->container_detection : ""
660 - , (host->system_info->container_os_name) ? host->system_info->container_os_name : ""
661 - , (host->system_info->container_os_id) ? host->system_info->container_os_id : ""
662 - , (host->system_info->container_os_id_like) ? host->system_info->container_os_id_like : ""
663 - , (host->system_info->container_os_version) ? host->system_info->container_os_version : ""
664 - , (host->system_info->container_os_version_id) ? host->system_info->container_os_version_id : ""
665 - , (host->system_info->container_os_detection) ? host->system_info->container_os_detection : ""
666 - , (host->system_info->host_cores) ? host->system_info->host_cores : ""
667 - , (host->system_info->host_cpu_freq) ? host->system_info->host_cpu_freq : ""
668 - , (host->system_info->host_ram_total) ? host->system_info->host_ram_total : ""
669 - , (host->system_info->host_disk_space) ? host->system_info->host_disk_space : ""
670 - , STREAMING_PROTOCOL_VERSION
671 - , host->program_name
672 - , host->program_version
673 - );
674 - http[eol] = 0x00;
675 - rrdpush_clean_encoded(&se);
676 -
677 -#ifdef ENABLE_HTTPS
678 - if (!host->ssl.flags) {
679 - ERR_clear_error();
680 - SSL_set_connect_state(host->ssl.conn);
681 - int err = SSL_connect(host->ssl.conn);
682 - if (err != 1){
683 - err = SSL_get_error(host->ssl.conn, err);
684 - error("SSL cannot connect with the server: %s ",ERR_error_string((long)SSL_get_error(host->ssl.conn,err),NULL));
685 - if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
686 - rrdpush_sender_thread_close_socket(host);
687 - return 0;
688 - }else {
689 - host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
690 - }
691 - }
692 - else {
693 - if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
694 - if (netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE) {
695 - if ( security_test_certificate(host->ssl.conn)) {
696 - error("Closing the stream connection, because the server SSL certificate is not valid.");
697 - rrdpush_sender_thread_close_socket(host);
698 - return 0;
699 - }
700 - }
701 - }
702 - }
703 - }
704 - if(send_timeout(&host->ssl,host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
705 -#else
706 - if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
707 -#endif
708 - error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, connected_to);
709 - rrdpush_sender_thread_close_socket(host);
710 - return 0;
711 - }
712 -
713 - info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, connected_to);
714 -
715 - ssize_t received;
716 -#ifdef ENABLE_HTTPS
717 - received = recv_timeout(&host->ssl,host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout);
718 - if(received == -1) {
719 -#else
720 - received = recv_timeout(host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout);
721 - if(received == -1) {
722 -#endif
723 - error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, connected_to);
724 - rrdpush_sender_thread_close_socket(host);
725 - return 0;
726 - }
727 -
728 - http[received] = '\0';
729 - int answer = -1;
730 - char *version_start = strchr(http, '=');
731 - uint32_t version;
732 - if(version_start) {
733 - version_start++;
734 - version = (uint32_t)strtol(version_start, NULL, 10);
735 - answer = memcmp(http, START_STREAMING_PROMPT_VN, (size_t)(version_start - http));
736 - if(!answer) {
737 - rrdpush_set_flags_to_newest_stream(host);
738 - host->stream_version = version;
739 - }
740 - } else {
741 - answer = memcmp(http, START_STREAMING_PROMPT_V2, strlen(START_STREAMING_PROMPT_V2));
742 - if(!answer) {
743 - version = 1;
744 - rrdpush_set_flags_to_newest_stream(host);
745 - }
746 - else {
747 - answer = memcmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT));
748 - if(!answer) {
749 - version = 0;
750 - host->labels_flag |= LABEL_FLAG_STOP_STREAM;
751 - host->labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
752 - }
753 - }
754 - }
755 -
756 - if(answer != 0) {
757 - error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, connected_to);
758 - rrdpush_sender_thread_close_socket(host);
759 - return 0;
760 - }
761 -
762 - info("STREAM %s [send to %s]: established communication with a master using protocol version %u - ready to send metrics..."
763 - , host->hostname
764 - , connected_to
765 - , version);
766 -
767 - if(sock_setnonblock(host->rrdpush_sender_socket) < 0)
768 - error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, connected_to);
769 -
770 - if(sock_enlarge_out(host->rrdpush_sender_socket) < 0)
771 - error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, connected_to);
772 -
773 - debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_sender_socket);
774 -
775 - return 1;
776 -}
777 -
778 -static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
779 - RRDHOST *host = (RRDHOST *)ptr;
780 -
781 - rrdpush_buffer_lock(host);
782 - rrdhost_wrlock(host);
783 -
784 - info("STREAM %s [send]: sending thread cleans up...", host->hostname);
785 -
786 - rrdpush_sender_thread_close_socket(host);
787 -
788 - // close the pipe
789 - if(host->rrdpush_sender_pipe[PIPE_READ] != -1) {
790 - close(host->rrdpush_sender_pipe[PIPE_READ]);
791 - host->rrdpush_sender_pipe[PIPE_READ] = -1;
792 - }
793 -
794 - if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1) {
795 - close(host->rrdpush_sender_pipe[PIPE_WRITE]);
796 - host->rrdpush_sender_pipe[PIPE_WRITE] = -1;
797 - }
798 -
799 - buffer_free(host->rrdpush_sender_buffer);
800 - host->rrdpush_sender_buffer = NULL;
801 -
802 - if(!host->rrdpush_sender_join) {
803 - info("STREAM %s [send]: sending thread detaches itself.", host->hostname);
804 - netdata_thread_detach(netdata_thread_self());
805 - }
806 -
807 - host->rrdpush_sender_spawn = 0;
808 -
809 - info("STREAM %s [send]: sending thread now exits.", host->hostname);
810 -
811 - rrdhost_unlock(host);
812 - rrdpush_buffer_unlock(host);
813 -}
814 -
815 -void *rrdpush_sender_thread(void *ptr) {
816 - RRDHOST *host = (RRDHOST *)ptr;
817 -
818 - if(!host->rrdpush_send_enabled || !host->rrdpush_send_destination || !*host->rrdpush_send_destination || !host->rrdpush_send_api_key || !*host->rrdpush_send_api_key) {
819 - error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.", host->hostname, gettid());
820 - return NULL;
821 - }
822 -
823 -#ifdef ENABLE_HTTPS
824 - if (netdata_use_ssl_on_stream & NETDATA_SSL_FORCE ){
825 - security_start_ssl(NETDATA_SSL_CONTEXT_STREAMING);
826 - security_location_for_context(netdata_client_ctx, netdata_ssl_ca_file, netdata_ssl_ca_path);
827 - }
828 -#endif
829 -
830 - info("STREAM %s [send]: thread created (task id %d)", host->hostname, gettid());
831 -
832 - int timeout = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60);
833 - int default_port = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "default port", 19999);
834 - size_t max_size = (size_t)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "buffer size bytes", 1024 * 1024);
835 - unsigned int reconnect_delay = (unsigned int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "reconnect delay seconds", 5);
836 - remote_clock_resync_iterations = (unsigned int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "initial clock resync iterations", remote_clock_resync_iterations);
837 - char connected_to[CONNECTED_TO_SIZE + 1] = "";
838 -
839 - // initialize rrdpush globals
840 - host->rrdpush_sender_buffer = buffer_create(1);
841 - host->rrdpush_sender_connected = 0;
842 - if(pipe(host->rrdpush_sender_pipe) == -1) fatal("STREAM %s [send]: cannot create required pipe.", host->hostname);
843 -
844 - // initialize local variables
845 - size_t begin = 0;
846 - size_t reconnects_counter = 0;
847 - size_t sent_bytes = 0;
848 - size_t sent_bytes_on_this_connection = 0;
849 - size_t send_attempts = 0;
850 -
851 -
852 - time_t last_sent_t = 0;
853 - struct pollfd fds[2], *ifd, *ofd;
854 - nfds_t fdmax;
855 -
856 - ifd = &fds[0];
857 - ofd = &fds[1];
858 -
859 - size_t not_connected_loops = 0;
860 -
861 - netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, host);
862 -
863 - for(; host->rrdpush_send_enabled && !netdata_exit ;) {
864 - // check for outstanding cancellation requests
865 - netdata_thread_testcancel();
866 -
867 - // if we don't have socket open, lets wait a bit
868 - if(unlikely(host->rrdpush_sender_socket == -1)) {
869 - send_attempts = 0;
870 -
871 - if(not_connected_loops == 0 && sent_bytes_on_this_connection > 0) {
872 - // fast re-connection on first disconnect
873 - sleep_usec(USEC_PER_MS * 500); // milliseconds
874 - }
875 - else {
876 - // slow re-connection on repeating errors
877 - sleep_usec(USEC_PER_SEC * reconnect_delay); // seconds
878 - }
879 -
880 - if(rrdpush_sender_thread_connect_to_master(host, default_port, timeout, &reconnects_counter, connected_to, CONNECTED_TO_SIZE)) {
881 - last_sent_t = now_monotonic_sec();
882 -
883 - // reset the buffer, to properly send charts and metrics
884 - rrdpush_sender_thread_data_flush(host);
885 -
886 - // send from the beginning
887 - begin = 0;
888 -
889 - // make sure the next reconnection will be immediate
890 - not_connected_loops = 0;
891 -
892 - // reset the bytes we have sent for this session
893 - sent_bytes_on_this_connection = 0;
894 -
895 - // let the data collection threads know we are ready
896 - host->rrdpush_sender_connected = 1;
897 - }
898 - else {
899 - // increase the failed connections counter
900 - not_connected_loops++;
901 -
902 - // reset the number of bytes sent
903 - sent_bytes_on_this_connection = 0;
904 - }
905 -
906 - // loop through
907 - continue;
908 - }
909 - else if(unlikely(now_monotonic_sec() - last_sent_t > timeout)) {
910 - error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", host->hostname, connected_to, timeout, sent_bytes_on_this_connection, send_attempts);
911 - rrdpush_sender_thread_close_socket(host);
912 - }
913 -
914 - ifd->fd = host->rrdpush_sender_pipe[PIPE_READ];
915 - ifd->events = POLLIN;
916 - ifd->revents = 0;
917 -
918 - ofd->fd = host->rrdpush_sender_socket;
919 - ofd->revents = 0;
920 - if(ofd->fd != -1 && begin < buffer_strlen(host->rrdpush_sender_buffer)) {
921 - debug(D_STREAM, "STREAM: Requesting data output on streaming socket %d...", ofd->fd);
922 - ofd->events = POLLOUT;
923 - fdmax = 2;
924 - send_attempts++;
925 - }
926 - else {
927 - debug(D_STREAM, "STREAM: Not requesting data output on streaming socket %d (nothing to send now)...", ofd->fd);
928 - ofd->events = 0;
929 - fdmax = 1;
930 - }
931 -
932 - debug(D_STREAM, "STREAM: Waiting for poll() events (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer));
933 - if(unlikely(netdata_exit)) break;
934 - int retval = poll(fds, fdmax, 1000);
935 - if(unlikely(netdata_exit)) break;
936 -
937 - if(unlikely(retval == -1)) {
938 - debug(D_STREAM, "STREAM: poll() failed (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer));
939 -
940 - if(errno == EAGAIN || errno == EINTR) {
941 - debug(D_STREAM, "STREAM: poll() failed with EAGAIN or EINTR...");
942 - }
943 - else {
944 - error("STREAM %s [send to %s]: failed to poll(). Closing socket.", host->hostname, connected_to);
945 - rrdpush_sender_thread_close_socket(host);
946 - }
947 -
948 - continue;
949 - }
950 - else if(likely(retval)) {
951 - if (ifd->revents & POLLIN || ifd->revents & POLLPRI) {
952 - debug(D_STREAM, "STREAM: Data added to send buffer (current buffer length %zu bytes)...", buffer_strlen(host->rrdpush_sender_buffer));
953 -
954 - char buffer[1000 + 1];
955 - if (read(host->rrdpush_sender_pipe[PIPE_READ], buffer, 1000) == -1)
956 - error("STREAM %s [send to %s]: cannot read from internal pipe.", host->hostname, connected_to);
957 - }
958 -
959 - if (ofd->revents & POLLOUT) {
960 - rrdpush_send_labels(host);
961 -
962 - if (begin < buffer_strlen(host->rrdpush_sender_buffer)) {
963 - debug(D_STREAM, "STREAM: Sending data (current buffer length %zu bytes, begin = %zu)...", buffer_strlen(host->rrdpush_sender_buffer), begin);
964 -
965 - // BEGIN RRDPUSH LOCKED SESSION
966 -
967 - // during this session, data collectors
968 - // will not be able to append data to our buffer
969 - // but the socket is in non-blocking mode
970 - // so, we will not block at send()
971 -
972 - netdata_thread_disable_cancelability();
973 -
974 - debug(D_STREAM, "STREAM: Getting exclusive lock on host...");
975 - rrdpush_buffer_lock(host);
976 -
977 - debug(D_STREAM, "STREAM: Sending data, starting from %zu, size %zu...", begin, buffer_strlen(host->rrdpush_sender_buffer));
978 - ssize_t ret;
979 -#ifdef ENABLE_HTTPS
980 - SSL *conn = host->ssl.conn ;
981 - if(conn && !host->ssl.flags) {
982 - ret = SSL_write(conn,&host->rrdpush_sender_buffer->buffer[begin], buffer_strlen(host->rrdpush_sender_buffer) - begin);
983 - } else {
984 - ret = send(host->rrdpush_sender_socket, &host->rrdpush_sender_buffer->buffer[begin], buffer_strlen(host->rrdpush_sender_buffer) - begin, MSG_DONTWAIT);
985 - }
986 -#else
987 - ret = send(host->rrdpush_sender_socket, &host->rrdpush_sender_buffer->buffer[begin], buffer_strlen(host->rrdpush_sender_buffer) - begin, MSG_DONTWAIT);
988 -#endif
989 - if (unlikely(ret == -1)) {
990 - if (errno != EAGAIN && errno != EINTR && errno != EWOULDBLOCK) {
991 - debug(D_STREAM, "STREAM: Send failed - closing socket...");
992 - error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", host->hostname, connected_to, sent_bytes_on_this_connection);
993 - rrdpush_sender_thread_close_socket(host);
994 - }
995 - else {
996 - debug(D_STREAM, "STREAM: Send failed - will retry...");
997 - }
998 - }
999 - else if (likely(ret > 0)) {
1000 - // DEBUG - dump the string to see it
1001 - //char c = host->rrdpush_sender_buffer->buffer[begin + ret];
1002 - //host->rrdpush_sender_buffer->buffer[begin + ret] = '\0';
1003 - //debug(D_STREAM, "STREAM: sent from %zu to %zd:\n%s\n", begin, ret, &host->rrdpush_sender_buffer->buffer[begin]);
1004 - //host->rrdpush_sender_buffer->buffer[begin + ret] = c;
1005 -
1006 - sent_bytes_on_this_connection += ret;
1007 - sent_bytes += ret;
1008 - begin += ret;
1009 -
1010 - if (begin == buffer_strlen(host->rrdpush_sender_buffer)) {
1011 - // we send it all
1012 -
1013 - debug(D_STREAM, "STREAM: Sent %zd bytes (the whole buffer)...", ret);
1014 - buffer_flush(host->rrdpush_sender_buffer);
1015 - begin = 0;
1016 - }
1017 - else {
1018 - debug(D_STREAM, "STREAM: Sent %zd bytes (part of the data buffer)...", ret);
1019 - }
1020 -
1021 - last_sent_t = now_monotonic_sec();
1022 - }
1023 - else {
1024 - debug(D_STREAM, "STREAM: send() returned %zd - closing the socket...", ret);
1025 - error("STREAM %s [send to %s]: failed to send metrics (send() returned %zd) - closing connection - we have sent %zu bytes on this connection.",
1026 - host->hostname, connected_to, ret, sent_bytes_on_this_connection);
1027 - rrdpush_sender_thread_close_socket(host);
1028 - }
1029 -
1030 - debug(D_STREAM, "STREAM: Releasing exclusive lock on host...");
1031 - rrdpush_buffer_unlock(host);
1032 -
1033 - netdata_thread_enable_cancelability();
1034 -
1035 - // END RRDPUSH LOCKED SESSION
1036 - }
1037 - else {
1038 - debug(D_STREAM, "STREAM: we have sent the entire buffer, but we received POLLOUT...");
1039 - }
1040 - }
1041 -
1042 - if(host->rrdpush_sender_socket != -1) {
1043 - char *error = NULL;
1044 -
1045 - if (unlikely(ofd->revents & POLLERR))
1046 - error = "socket reports errors (POLLERR)";
1047 -
1048 - else if (unlikely(ofd->revents & POLLHUP))
1049 - error = "connection closed by remote end (POLLHUP)";
1050 -
1051 - else if (unlikely(ofd->revents & POLLNVAL))
1052 - error = "connection is invalid (POLLNVAL)";
1053 -
1054 - if(unlikely(error)) {
1055 - debug(D_STREAM, "STREAM: %s - closing socket...", error);
1056 - error("STREAM %s [send to %s]: %s - reopening socket - we have sent %zu bytes on this connection.", host->hostname, connected_to, error, sent_bytes_on_this_connection);
1057 - rrdpush_sender_thread_close_socket(host);
1058 - }
1059 - }
1060 - }
1061 - else {
1062 - debug(D_STREAM, "STREAM: poll() timed out.");
1063 - }
1064 -
1065 - // protection from overflow
1066 - if(buffer_strlen(host->rrdpush_sender_buffer) > max_size) {
1067 - debug(D_STREAM, "STREAM: Buffer is too big (%zu bytes), bigger than the max (%zu) - flushing it...", buffer_strlen(host->rrdpush_sender_buffer), max_size);
1068 - errno = 0;
1069 - error("STREAM %s [send to %s]: too many data pending - buffer is %zu bytes long, %zu unsent - we have sent %zu bytes in total, %zu on this connection. Closing connection to flush the data.", host->hostname, connected_to, host->rrdpush_sender_buffer->len, host->rrdpush_sender_buffer->len - begin, sent_bytes, sent_bytes_on_this_connection);
1070 - rrdpush_sender_thread_close_socket(host);
1071 - }
1072 - }
1073 -
1074 - netdata_thread_cleanup_pop(1);
1075 - return NULL;
1076 -}
1077 -
403
404 // ----------------------------------------------------------------------------
405 // rrdpush receiver thread
406
1082 -static void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg) {
407 +void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg) {
408 log_access("STREAM: %d '[%s]:%s' '%s' host '%s' api key '%s' machine guid '%s'", gettid(), client_ip, client_port, msg, host, api_key, machine_guid);
409 }
410
1086 -static int rrdpush_receive(int fd
1087 - , const char *key
1088 - , const char *hostname
1089 - , const char *registry_hostname
1090 - , const char *machine_guid
1091 - , const char *os
1092 - , const char *timezone
1093 - , const char *tags
1094 - , const char *program_name
1095 - , const char *program_version
1096 - , struct rrdhost_system_info *system_info
1097 - , int update_every
1098 - , char *client_ip
1099 - , char *client_port
1100 - , uint32_t stream_version
1101 -#ifdef ENABLE_HTTPS
1102 - , struct netdata_ssl *ssl
1103 -#endif
1104 -) {
1105 - RRDHOST *host;
1106 - int history = default_rrd_history_entries;
1107 - RRD_MEMORY_MODE mode = default_rrd_memory_mode;
1108 - int health_enabled = default_health_enabled;
1109 - int rrdpush_enabled = default_rrdpush_enabled;
1110 - char *rrdpush_destination = default_rrdpush_destination;
1111 - char *rrdpush_api_key = default_rrdpush_api_key;
1112 - char *rrdpush_send_charts_matching = default_rrdpush_send_charts_matching;
1113 - time_t alarms_delay = 60;
1114 -
1115 - update_every = (int)appconfig_get_number(&stream_config, machine_guid, "update every", update_every);
1116 - if(update_every < 0) update_every = 1;
1117 -
1118 - history = (int)appconfig_get_number(&stream_config, key, "default history", history);
1119 - history = (int)appconfig_get_number(&stream_config, machine_guid, "history", history);
1120 - if(history < 5) history = 5;
1121 -
1122 - mode = rrd_memory_mode_id(appconfig_get(&stream_config, key, "default memory mode", rrd_memory_mode_name(mode)));
1123 - mode = rrd_memory_mode_id(appconfig_get(&stream_config, machine_guid, "memory mode", rrd_memory_mode_name(mode)));
1124 -
1125 - health_enabled = appconfig_get_boolean_ondemand(&stream_config, key, "health enabled by default", health_enabled);
1126 - health_enabled = appconfig_get_boolean_ondemand(&stream_config, machine_guid, "health enabled", health_enabled);
1127 -
1128 - alarms_delay = appconfig_get_number(&stream_config, key, "default postpone alarms on connect seconds", alarms_delay);
1129 - alarms_delay = appconfig_get_number(&stream_config, machine_guid, "postpone alarms on connect seconds", alarms_delay);
1130 -
1131 - rrdpush_enabled = appconfig_get_boolean(&stream_config, key, "default proxy enabled", rrdpush_enabled);
1132 - rrdpush_enabled = appconfig_get_boolean(&stream_config, machine_guid, "proxy enabled", rrdpush_enabled);
1133 -
1134 - rrdpush_destination = appconfig_get(&stream_config, key, "default proxy destination", rrdpush_destination);
1135 - rrdpush_destination = appconfig_get(&stream_config, machine_guid, "proxy destination", rrdpush_destination);
1136 -
1137 - rrdpush_api_key = appconfig_get(&stream_config, key, "default proxy api key", rrdpush_api_key);
1138 - rrdpush_api_key = appconfig_get(&stream_config, machine_guid, "proxy api key", rrdpush_api_key);
1139 -
1140 - rrdpush_send_charts_matching = appconfig_get(&stream_config, key, "default proxy send charts matching", rrdpush_send_charts_matching);
1141 - rrdpush_send_charts_matching = appconfig_get(&stream_config, machine_guid, "proxy send charts matching", rrdpush_send_charts_matching);
1142 -
1143 - tags = appconfig_set_default(&stream_config, machine_guid, "host tags", (tags)?tags:"");
1144 - if(tags && !*tags) tags = NULL;
1145 -
1146 - if (strcmp(machine_guid, localhost->machine_guid) == 0) {
1147 - log_stream_connection(client_ip, client_port, key, machine_guid, hostname, "DENIED - ATTEMPT TO RECEIVE METRICS FROM MACHINE_GUID IDENTICAL TO MASTER");
1148 - error("STREAM %s [receive from %s:%s]: denied to receive metrics, machine GUID [%s] is my own. Did you copy the master/proxy machine guid to a slave?", hostname, client_ip, client_port, machine_guid);
1149 - close(fd);
1150 - return 1;
1151 - }
1152 -
1153 - /*
1154 - * Quick path for rejecting multiple connections. Don't take any locks so that progress is made. The same
1155 - * condition will be checked again below, while holding the global and host writer locks. Any potential false
1156 - * positives will not cause harm. Data hazards with host deconstruction will be handled when reference counting
1157 - * is implemented.
1158 - */
1159 - host = rrdhost_find_by_guid(machine_guid, 0);
1160 - if(host && host->connected_senders > 0) {
1161 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "REJECTED - ALREADY CONNECTED");
1162 - info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. Rejecting new connection.", host->hostname, client_ip, client_port);
1163 - close(fd);
1164 - return 0;
1165 - }
1166 -
1167 - host = rrdhost_find_or_create(
1168 - hostname
1169 - , registry_hostname
1170 - , machine_guid
1171 - , os
1172 - , timezone
1173 - , tags
1174 - , program_name
1175 - , program_version
1176 - , update_every
1177 - , history
1178 - , mode
1179 - , (unsigned int)(health_enabled != CONFIG_BOOLEAN_NO)
1180 - , (unsigned int)(rrdpush_enabled && rrdpush_destination && *rrdpush_destination && rrdpush_api_key && *rrdpush_api_key)
1181 - , rrdpush_destination
1182 - , rrdpush_api_key
1183 - , rrdpush_send_charts_matching
1184 - , system_info
1185 - );
1186 -
1187 - if(!host) {
1188 - close(fd);
1189 - log_stream_connection(client_ip, client_port, key, machine_guid, hostname, "FAILED - CANNOT ACQUIRE HOST");
1190 - error("STREAM %s [receive from [%s]:%s]: failed to find/create host structure.", hostname, client_ip, client_port);
1191 - return 1;
1192 - }
1193 -
1194 -#ifdef NETDATA_INTERNAL_CHECKS
1195 - info("STREAM %s [receive from [%s]:%s]: client willing to stream metrics for host '%s' with machine_guid '%s': update every = %d, history = %ld, memory mode = %s, health %s, tags '%s'"
1196 - , hostname
1197 - , client_ip
1198 - , client_port
1199 - , host->hostname
1200 - , host->machine_guid
1201 - , host->rrd_update_every
1202 - , host->rrd_history_entries
1203 - , rrd_memory_mode_name(host->rrd_memory_mode)
1204 - , (health_enabled == CONFIG_BOOLEAN_NO)?"disabled":((health_enabled == CONFIG_BOOLEAN_YES)?"enabled":"auto")
1205 - , host->tags?host->tags:""
1206 - );
1207 -#endif // NETDATA_INTERNAL_CHECKS
1208 -
1209 - struct plugind cd = {
1210 - .enabled = 1,
1211 - .update_every = default_rrd_update_every,
1212 - .pid = 0,
1213 - .serial_failures = 0,
1214 - .successful_collections = 0,
1215 - .obsolete = 0,
1216 - .started_t = now_realtime_sec(),
1217 - .next = NULL,
1218 - .version = 0,
1219 - };
1220 -
1221 - // put the client IP and port into the buffers used by plugins.d
1222 - snprintfz(cd.id, CONFIG_MAX_NAME, "%s:%s", client_ip, client_port);
1223 - snprintfz(cd.filename, FILENAME_MAX, "%s:%s", client_ip, client_port);
1224 - snprintfz(cd.fullfilename, FILENAME_MAX, "%s:%s", client_ip, client_port);
1225 - snprintfz(cd.cmd, PLUGINSD_CMD_MAX, "%s:%s", client_ip, client_port);
1226 -
1227 - info("STREAM %s [receive from [%s]:%s]: initializing communication...", host->hostname, client_ip, client_port);
1228 - char initial_response[HTTP_HEADER_SIZE];
1229 - if (stream_version > 1) {
1230 - info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", host->hostname, client_ip, client_port, stream_version);
1231 - sprintf(initial_response, "%s%u", START_STREAMING_PROMPT_VN, stream_version);
1232 - } else if (stream_version == 1) {
1233 - info("STREAM %s [receive from [%s]:%s]: Netdata is using the stream version %u.", host->hostname, client_ip, client_port, stream_version);
1234 - sprintf(initial_response, "%s", START_STREAMING_PROMPT_V2);
1235 - } else {
1236 - info("STREAM %s [receive from [%s]:%s]: Netdata is using first stream protocol.", host->hostname, client_ip, client_port);
1237 - sprintf(initial_response, "%s", START_STREAMING_PROMPT);
1238 - }
1239 - #ifdef ENABLE_HTTPS
1240 - host->stream_ssl.conn = ssl->conn;
1241 - host->stream_ssl.flags = ssl->flags;
1242 - if(send_timeout(ssl, fd, initial_response, strlen(initial_response), 0, 60) != (ssize_t)strlen(initial_response)) {
1243 -#else
1244 - if(send_timeout(fd, initial_response, strlen(initial_response), 0, 60) != strlen(initial_response)) {
1245 -#endif
1246 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "FAILED - CANNOT REPLY");
1247 - error("STREAM %s [receive from [%s]:%s]: cannot send ready command.", host->hostname, client_ip, client_port);
1248 - close(fd);
1249 - return 0;
1250 - }
1251 -
1252 - // remove the non-blocking flag from the socket
1253 - if(sock_delnonblock(fd) < 0)
1254 - error("STREAM %s [receive from [%s]:%s]: cannot remove the non-blocking flag from socket %d", host->hostname, client_ip, client_port, fd);
1255 -
1256 - // convert the socket to a FILE *
1257 - FILE *fp = fdopen(fd, "r");
1258 - if(!fp) {
1259 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "FAILED - SOCKET ERROR");
1260 - error("STREAM %s [receive from [%s]:%s]: failed to get a FILE for FD %d.", host->hostname, client_ip, client_port, fd);
1261 - close(fd);
1262 - return 0;
1263 - }
1264 -
1265 - rrdhost_wrlock(host);
1266 - if(host->connected_senders > 0) {
1267 - rrdhost_unlock(host);
1268 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "REJECTED - ALREADY CONNECTED");
1269 - info("STREAM %s [receive from [%s]:%s]: multiple streaming connections for the same host detected. Rejecting new connection.", host->hostname, client_ip, client_port);
1270 - fclose(fp);
1271 - return 0;
1272 - }
1273 -
1274 - rrdhost_flag_clear(host, RRDHOST_FLAG_ORPHAN);
1275 - host->connected_senders++;
1276 - host->senders_disconnected_time = 0;
1277 - host->labels_flag = (stream_version > 0)?LABEL_FLAG_UPDATE_STREAM:LABEL_FLAG_STOP_STREAM;
1278 -
1279 - if(health_enabled != CONFIG_BOOLEAN_NO) {
1280 - if(alarms_delay > 0) {
1281 - host->health_delay_up_to = now_realtime_sec() + alarms_delay;
1282 - info("Postponing health checks for %ld seconds, on host '%s', because it was just connected."
1283 - , alarms_delay
1284 - , host->hostname
1285 - );
1286 - }
1287 - }
1288 - rrdhost_unlock(host);
1289 -
1290 - // call the plugins.d processor to receive the metrics
1291 - info("STREAM %s [receive from [%s]:%s]: receiving metrics...", host->hostname, client_ip, client_port);
1292 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "CONNECTED");
1293 -
1294 - cd.version = stream_version;
1295 - size_t count = pluginsd_process(host, &cd, fp, 1);
1296 -
1297 - log_stream_connection(client_ip, client_port, key, host->machine_guid, host->hostname, "DISCONNECTED");
1298 - error("STREAM %s [receive from [%s]:%s]: disconnected (completed %zu updates).", host->hostname, client_ip, client_port, count);
1299 -
1300 - rrdhost_wrlock(host);
1301 - host->senders_disconnected_time = now_realtime_sec();
1302 - host->connected_senders--;
1303 - if(!host->connected_senders) {
1304 - rrdhost_flag_set(host, RRDHOST_FLAG_ORPHAN);
1305 - if(health_enabled == CONFIG_BOOLEAN_AUTO)
1306 - host->health_enabled = 0;
1307 - }
1308 - rrdhost_unlock(host);
1309 -
1310 - if(host->connected_senders == 0)
1311 - rrdpush_sender_thread_stop(host);
1312 -
1313 - // cleanup
1314 - fclose(fp);
1315 -
1316 - return (int)count;
1317 -}
1318 -
1319 -struct rrdpush_thread {
1320 - int fd;
1321 - char *key;
1322 - char *hostname;
1323 - char *registry_hostname;
1324 - char *machine_guid;
1325 - char *os;
1326 - char *timezone;
1327 - char *tags;
1328 - char *client_ip;
1329 - char *client_port;
1330 - char *program_name;
1331 - char *program_version;
1332 - struct rrdhost_system_info *system_info;
1333 - int update_every;
1334 - uint32_t stream_version;
1335 -#ifdef ENABLE_HTTPS
1336 - struct netdata_ssl ssl;
1337 -#endif
1338 -};
1339 -
1340 -static void rrdpush_receiver_thread_cleanup(void *ptr) {
1341 - static __thread int executed = 0;
1342 - if(!executed) {
1343 - executed = 1;
1344 - struct rrdpush_thread *rpt = (struct rrdpush_thread *) ptr;
1345 -
1346 - info("STREAM %s [receive from [%s]:%s]: receive thread ended (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
1347 -
1348 - freez(rpt->key);
1349 - freez(rpt->hostname);
1350 - freez(rpt->registry_hostname);
1351 - freez(rpt->machine_guid);
1352 - freez(rpt->os);
1353 - freez(rpt->timezone);
1354 - freez(rpt->tags);
1355 - freez(rpt->client_ip);
1356 - freez(rpt->client_port);
1357 - freez(rpt->program_name);
1358 - freez(rpt->program_version);
1359 -#ifdef ENABLE_HTTPS
1360 - if(rpt->ssl.conn){
1361 - SSL_free(rpt->ssl.conn);
1362 - }
1363 -#endif
1364 - freez(rpt);
1365 -
1366 - }
1367 -}
1368 -
1369 -static void *rrdpush_receiver_thread(void *ptr) {
1370 - netdata_thread_cleanup_push(rrdpush_receiver_thread_cleanup, ptr);
1371 -
1372 - struct rrdpush_thread *rpt = (struct rrdpush_thread *)ptr;
1373 - info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
1374 -
1375 - rrdpush_receive(
1376 - rpt->fd
1377 - , rpt->key
1378 - , rpt->hostname
1379 - , rpt->registry_hostname
1380 - , rpt->machine_guid
1381 - , rpt->os
1382 - , rpt->timezone
1383 - , rpt->tags
1384 - , rpt->program_name
1385 - , rpt->program_version
1386 - , rpt->system_info
1387 - , rpt->update_every
1388 - , rpt->client_ip
1389 - , rpt->client_port
1390 - , rpt->stream_version
1391 -#ifdef ENABLE_HTTPS
1392 - , &rpt->ssl
1393 -#endif
1394 - );
1395 -
1396 - netdata_thread_cleanup_pop(1);
1397 - return NULL;
1398 -}
411
412 static void rrdpush_sender_thread_spawn(RRDHOST *host) {
413 rrdhost_wrlock(host);
@@ -1404,7 +416,7 @@ static void rrdpush_sender_thread_spawn(RRDHOST *host) {
416 char tag[NETDATA_THREAD_TAG_MAX + 1];
417 snprintfz(tag, NETDATA_THREAD_TAG_MAX, "STREAM_SENDER[%s]", host->hostname);
418
1407 - if(netdata_thread_create(&host->rrdpush_sender_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, rrdpush_sender_thread, (void *) host))
419 + if(netdata_thread_create(&host->rrdpush_sender_thread, tag, NETDATA_THREAD_OPTION_JOINABLE, rrdpush_sender_thread, (void *) host->sender))
420 error("STREAM %s [send]: failed to create new thread for client.", host->hostname);
421 else
422 host->rrdpush_sender_spawn = 1;
@@ -1429,9 +441,8 @@ int rrdpush_receiver_too_busy_now(struct web_client *w) {
441 return 503;
442 }
443
1432 -int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url) {
1433 - (void)host;
1434 -
444 +void *rrdpush_receiver_thread(void *ptr);
445 +int rrdpush_receiver_thread_spawn(struct web_client *w, char *url) {
446 info("clients wants to STREAM metrics.");
447
448 char *key = NULL, *hostname = NULL, *registry_hostname = NULL, *machine_guid = NULL, *os = "unknown", *timezone = "unknown", *tags = NULL;
@@ -1593,7 +604,45 @@ int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url
604 netdata_mutex_unlock(&stream_rate_mutex);
605 }
606
1596 - struct rrdpush_thread *rpt = callocz(1, sizeof(struct rrdpush_thread));
607 + /*
608 + * Quick path for rejecting multiple connections. The lock taken is fine-grained - it only protects the receiver
609 + * pointer within the host (if a host exists). This protects against multiple concurrent web requests hitting
610 + * separate threads within the web-server and landing here. The lock guards the thread-shutdown sequence that
611 + * detaches the receiver from the host. If the host is being created (first time-access) then we also use the
612 + * lock to prevent race-hazard (two threads try to create the host concurrently, one wins and the other does a
613 + * lookup to the now-attached structure).
614 + */
615 + struct receiver_state *rpt = callocz(1, sizeof(*rpt));
616 + RRDHOST *host = rrdhost_find_by_guid(machine_guid, 0);
617 + if (host) {
618 + netdata_mutex_lock(&host->receiver_lock);
619 + if (host->receiver != NULL) {
620 + time_t age = now_realtime_sec() - host->receiver->last_msg_t;
621 + if (age > 30) {
622 + host->receiver->shutdown = 1;
623 + shutdown(host->receiver->fd, SHUT_RDWR);
624 + host->receiver = NULL; // Thread holds reference to structure
625 + info("STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - existing connection is dead (%ld sec), accepting new connection.", host->hostname, w->client_ip, w->client_port, age);
626 + }
627 + else {
628 + netdata_mutex_unlock(&host->receiver_lock);
629 + log_stream_connection(w->client_ip, w->client_port, key, host->machine_guid, host->hostname,
630 + "REJECTED - ALREADY CONNECTED");
631 + info("STREAM %s [receive from [%s]:%s]: multiple connections for same host detected - existing connection is active (within last %ld sec), rejecting new connection.", host->hostname, w->client_ip, w->client_port, age);
632 + // Have not set WEB_CLIENT_FLAG_DONT_CLOSE_SOCKET - caller should clean up
633 + buffer_flush(w->response.data);
634 + buffer_strcat(w->response.data, "This GUID is already streaming to this server");
635 + freez(rpt);
636 + return 409;
637 + }
638 + }
639 + host->receiver = rpt;
640 + netdata_mutex_unlock(&host->receiver_lock);
641 + }
642 +
643 + rpt->last_msg_t = now_realtime_sec();
644 +
645 + rpt->host = host;
646 rpt->fd = w->ifd;
647 rpt->key = strdupz(key);
648 rpt->hostname = strdupz(hostname);
streaming/rrdpush.h
+86 -1
@@ -3,10 +3,91 @@
3 #ifndef NETDATA_RRDPUSH_H
4 #define NETDATA_RRDPUSH_H 1
5
6 +#include "../database/rrd.h"
7 +#include "../libnetdata/libnetdata.h"
8 #include "web/server/web_client.h"
9 #include "daemon/common.h"
10
11 +#define CONNECTED_TO_SIZE 100
12 +
13 +// #define STREAMING_PROTOCOL_CURRENT_VERSION (uint32_t)3 Gap-filling
14 #define STREAMING_PROTOCOL_CURRENT_VERSION (uint32_t)2
15 +#define VERSION_GAP_FILLING 3
16 +
17 +#define STREAMING_PROTOCOL_VERSION "1.1"
18 +#define START_STREAMING_PROMPT "Hit me baby, push them over..."
19 +#define START_STREAMING_PROMPT_V2 "Hit me baby, push them over and bring the host labels..."
20 +#define START_STREAMING_PROMPT_VN "Hit me baby, push them over with the version="
21 +
22 +#define HTTP_HEADER_SIZE 8192
23 +
24 +typedef enum {
25 + RRDPUSH_MULTIPLE_CONNECTIONS_ALLOW,
26 + RRDPUSH_MULTIPLE_CONNECTIONS_DENY_NEW
27 +} RRDPUSH_MULTIPLE_CONNECTIONS_STRATEGY;
28 +
29 +typedef struct {
30 + char *os_name;
31 + char *os_id;
32 + char *os_version;
33 + char *kernel_name;
34 + char *kernel_version;
35 +} stream_encoded_t;
36 +
37 +// Thread-local storage
38 + // Metric transmission: collector threads asynchronously fill the buffer, sender thread uses it.
39 +
40 +struct sender_state {
41 + RRDHOST *host;
42 + pid_t task_id;
43 + unsigned int overflow:1;
44 + int timeout, default_port;
45 + size_t max_size;
46 + usec_t reconnect_delay;
47 + char connected_to[CONNECTED_TO_SIZE + 1]; // We don't know which proxy we connect to, passed back from socket.c
48 + size_t begin;
49 + size_t reconnects_counter;
50 + size_t sent_bytes;
51 + size_t sent_bytes_on_this_connection;
52 + size_t send_attempts;
53 + time_t last_sent_t;
54 + size_t not_connected_loops;
55 + // metrics may be collected asynchronously
56 + // these synchronize all the threads willing the write to our sending buffer
57 + netdata_mutex_t mutex; // Guard access to buffer / build
58 + struct circular_buffer *buffer;
59 + BUFFER *build;
60 + char read_buffer[512];
61 + int read_len;
62 + int32_t version;
63 +};
64 +
65 +struct receiver_state {
66 + RRDHOST *host;
67 + int fd;
68 + char *key;
69 + char *hostname;
70 + char *registry_hostname;
71 + char *machine_guid;
72 + char *os;
73 + char *timezone; // Unused?
74 + char *tags;
75 + char *client_ip; // Duplicated in pluginsd
76 + char *client_port; // Duplicated in pluginsd
77 + char *program_name; // Duplicated in pluginsd
78 + char *program_version;
79 + struct rrdhost_system_info *system_info;
80 + int update_every;
81 + uint32_t stream_version;
82 + time_t last_msg_t;
83 + char read_buffer[512];
84 + int read_len;
85 +#ifdef ENABLE_HTTPS
86 + struct netdata_ssl ssl;
87 +#endif
88 + unsigned int shutdown:1;
89 +};
90 +
91
92 extern unsigned int default_rrdpush_enabled;
93 extern char *default_rrdpush_destination;
@@ -14,6 +95,9 @@ extern char *default_rrdpush_api_key;
95 extern char *default_rrdpush_send_charts_matching;
96 extern unsigned int remote_clock_resync_iterations;
97
98 +extern void sender_init(struct sender_state *s, RRDHOST *parent);
99 +void sender_start(struct sender_state *s);
100 +void sender_commit(struct sender_state *s);
101 extern int rrdpush_init();
102 extern int configured_as_master();
103 extern void rrdset_done_push(RRDSET *st);
@@ -21,9 +105,10 @@ extern void rrdset_push_chart_definition_now(RRDSET *st);
105 extern void *rrdpush_sender_thread(void *ptr);
106 extern void rrdpush_send_labels(RRDHOST *host);
107
24 -extern int rrdpush_receiver_thread_spawn(RRDHOST *host, struct web_client *w, char *url);
108 +extern int rrdpush_receiver_thread_spawn(struct web_client *w, char *url);
109 extern void rrdpush_sender_thread_stop(RRDHOST *host);
110
111 extern void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv);
112 +extern void log_stream_connection(const char *client_ip, const char *client_port, const char *api_key, const char *machine_guid, const char *host, const char *msg);
113
114 #endif //NETDATA_RRDPUSH_H
streaming/sender.c new
+709
@@ -0,0 +1,709 @@
1 +// SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +#include "rrdpush.h"
4 +
5 +extern struct config stream_config;
6 +extern int netdata_use_ssl_on_stream;
7 +extern char *netdata_ssl_ca_path;
8 +extern char *netdata_ssl_ca_file;
9 +
10 +// Collector thread starting a transmission
11 +void sender_start(struct sender_state *s) {
12 + netdata_mutex_lock(&s->mutex);
13 + buffer_flush(s->build);
14 +}
15 +
16 +// Collector thread finishing a transmission
17 +void sender_commit(struct sender_state *s) {
18 + if(cbuffer_add_unsafe(s->host->sender->buffer, buffer_tostring(s->host->sender->build),
19 + s->host->sender->build->len))
20 + s->overflow = 1;
21 + buffer_flush(s->build);
22 + netdata_mutex_unlock(&s->mutex);
23 +}
24 +
25 +
26 +static inline void rrdpush_sender_thread_close_socket(RRDHOST *host) {
27 + host->rrdpush_sender_connected = 0;
28 +
29 + if(host->rrdpush_sender_socket != -1) {
30 + close(host->rrdpush_sender_socket);
31 + host->rrdpush_sender_socket = -1;
32 + }
33 +}
34 +
35 +static inline void rrdpush_sender_add_host_variable_to_buffer_nolock(RRDHOST *host, RRDVAR *rv) {
36 + calculated_number *value = (calculated_number *)rv->value;
37 +
38 + buffer_sprintf(
39 + host->sender->build
40 + , "VARIABLE HOST %s = " CALCULATED_NUMBER_FORMAT "\n"
41 + , rv->name
42 + , *value
43 + );
44 +
45 + debug(D_STREAM, "RRDVAR pushed HOST VARIABLE %s = " CALCULATED_NUMBER_FORMAT, rv->name, *value);
46 +}
47 +
48 +void rrdpush_sender_send_this_host_variable_now(RRDHOST *host, RRDVAR *rv) {
49 + if(host->rrdpush_send_enabled && host->rrdpush_sender_spawn && host->rrdpush_sender_connected) {
50 + sender_start(host->sender);
51 + rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv);
52 + sender_commit(host->sender);
53 + }
54 +}
55 +
56 +
57 +static int rrdpush_sender_thread_custom_host_variables_callback(void *rrdvar_ptr, void *host_ptr) {
58 + RRDVAR *rv = (RRDVAR *)rrdvar_ptr;
59 + RRDHOST *host = (RRDHOST *)host_ptr;
60 +
61 + if(unlikely(rv->options & RRDVAR_OPTION_CUSTOM_HOST_VAR && rv->type == RRDVAR_TYPE_CALCULATED)) {
62 + rrdpush_sender_add_host_variable_to_buffer_nolock(host, rv);
63 +
64 + // return 1, so that the traversal will return the number of variables sent
65 + return 1;
66 + }
67 +
68 + // returning a negative number will break the traversal
69 + return 0;
70 +}
71 +
72 +static void rrdpush_sender_thread_send_custom_host_variables(RRDHOST *host) {
73 + int ret = rrdvar_callback_for_all_host_variables(host, rrdpush_sender_thread_custom_host_variables_callback, host);
74 + (void)ret;
75 +
76 + debug(D_STREAM, "RRDVAR sent %d VARIABLES", ret);
77 +}
78 +
79 +// resets all the chart, so that their definitions
80 +// will be resent to the central netdata
81 +static void rrdpush_sender_thread_reset_all_charts(RRDHOST *host) {
82 + rrdhost_rdlock(host);
83 +
84 + RRDSET *st;
85 + rrdset_foreach_read(st, host) {
86 + rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
87 +
88 + st->upstream_resync_time = 0;
89 +
90 + rrdset_rdlock(st);
91 +
92 + RRDDIM *rd;
93 + rrddim_foreach_read(rd, st)
94 + rd->exposed = 0;
95 +
96 + rrdset_unlock(st);
97 + }
98 +
99 + rrdhost_unlock(host);
100 +}
101 +
102 +static inline void rrdpush_sender_thread_data_flush(RRDHOST *host) {
103 + netdata_mutex_lock(&host->sender->mutex);
104 +
105 + size_t len = cbuffer_next_unsafe(host->sender->buffer, NULL);
106 + if (len)
107 + error("STREAM %s [send]: discarding %zu bytes of metrics already in the buffer.", host->hostname, len);
108 +
109 + cbuffer_remove_unsafe(host->sender->buffer, len);
110 + netdata_mutex_unlock(&host->sender->mutex);
111 +
112 + rrdpush_sender_thread_reset_all_charts(host);
113 + rrdpush_sender_thread_send_custom_host_variables(host);
114 +}
115 +
116 +static inline void rrdpush_set_flags_to_newest_stream(RRDHOST *host) {
117 + host->labels_flag |= LABEL_FLAG_UPDATE_STREAM;
118 + host->labels_flag &= ~LABEL_FLAG_STOP_STREAM;
119 +}
120 +
121 +void rrdpush_encode_variable(stream_encoded_t *se, RRDHOST *host)
122 +{
123 + se->os_name = (host->system_info->host_os_name)?url_encode(host->system_info->host_os_name):"";
124 + se->os_id = (host->system_info->host_os_id)?url_encode(host->system_info->host_os_id):"";
125 + se->os_version = (host->system_info->host_os_version)?url_encode(host->system_info->host_os_version):"";
126 + se->kernel_name = (host->system_info->kernel_name)?url_encode(host->system_info->kernel_name):"";
127 + se->kernel_version = (host->system_info->kernel_version)?url_encode(host->system_info->kernel_version):"";
128 +}
129 +
130 +void rrdpush_clean_encoded(stream_encoded_t *se)
131 +{
132 + if (se->os_name)
133 + freez(se->os_name);
134 +
135 + if (se->os_id)
136 + freez(se->os_id);
137 +
138 + if (se->os_version)
139 + freez(se->os_version);
140 +
141 + if (se->kernel_name)
142 + freez(se->kernel_name);
143 +
144 + if (se->kernel_version)
145 + freez(se->kernel_version);
146 +}
147 +
148 +static int rrdpush_sender_thread_connect_to_master(RRDHOST *host, int default_port, int timeout,
149 + struct sender_state *s) {
150 +
151 + struct timeval tv = {
152 + .tv_sec = timeout,
153 + .tv_usec = 0
154 + };
155 +
156 + // make sure the socket is closed
157 + rrdpush_sender_thread_close_socket(host);
158 +
159 + debug(D_STREAM, "STREAM: Attempting to connect...");
160 + info("STREAM %s [send to %s]: connecting...", host->hostname, host->rrdpush_send_destination);
161 +
162 + host->rrdpush_sender_socket = connect_to_one_of(
163 + host->rrdpush_send_destination
164 + , default_port
165 + , &tv
166 + , &s->reconnects_counter
167 + , s->connected_to
168 + , sizeof(s->connected_to)-1
169 + );
170 +
171 + if(unlikely(host->rrdpush_sender_socket == -1)) {
172 + error("STREAM %s [send to %s]: failed to connect", host->hostname, host->rrdpush_send_destination);
173 + return 0;
174 + }
175 +
176 + info("STREAM %s [send to %s]: initializing communication...", host->hostname, s->connected_to);
177 +
178 +#ifdef ENABLE_HTTPS
179 + if( netdata_client_ctx ){
180 + host->ssl.flags = NETDATA_SSL_START;
181 + if (!host->ssl.conn){
182 + host->ssl.conn = SSL_new(netdata_client_ctx);
183 + if(!host->ssl.conn){
184 + error("Failed to allocate SSL structure.");
185 + host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
186 + }
187 + }
188 + else{
189 + SSL_clear(host->ssl.conn);
190 + }
191 +
192 + if (host->ssl.conn)
193 + {
194 + if (SSL_set_fd(host->ssl.conn, host->rrdpush_sender_socket) != 1) {
195 + error("Failed to set the socket to the SSL on socket fd %d.", host->rrdpush_sender_socket);
196 + host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
197 + } else{
198 + host->ssl.flags = NETDATA_SSL_HANDSHAKE_COMPLETE;
199 + }
200 + }
201 + }
202 + else {
203 + host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
204 + }
205 +#endif
206 +
207 + /* TODO: During the implementation of #7265 switch the set of variables to HOST_* and CONTAINER_* if the
208 + version negotiation resulted in a high enough version.
209 + */
210 + stream_encoded_t se;
211 + rrdpush_encode_variable(&se, host);
212 +
213 + char http[HTTP_HEADER_SIZE + 1];
214 + int eol = snprintfz(http, HTTP_HEADER_SIZE,
215 + "STREAM key=%s&hostname=%s&registry_hostname=%s&machine_guid=%s&update_every=%d&os=%s&timezone=%s&tags=%s&ver=%u"
216 + "&NETDATA_SYSTEM_OS_NAME=%s"
217 + "&NETDATA_SYSTEM_OS_ID=%s"
218 + "&NETDATA_SYSTEM_OS_ID_LIKE=%s"
219 + "&NETDATA_SYSTEM_OS_VERSION=%s"
220 + "&NETDATA_SYSTEM_OS_VERSION_ID=%s"
221 + "&NETDATA_SYSTEM_OS_DETECTION=%s"
222 + "&NETDATA_SYSTEM_KERNEL_NAME=%s"
223 + "&NETDATA_SYSTEM_KERNEL_VERSION=%s"
224 + "&NETDATA_SYSTEM_ARCHITECTURE=%s"
225 + "&NETDATA_SYSTEM_VIRTUALIZATION=%s"
226 + "&NETDATA_SYSTEM_VIRT_DETECTION=%s"
227 + "&NETDATA_SYSTEM_CONTAINER=%s"
228 + "&NETDATA_SYSTEM_CONTAINER_DETECTION=%s"
229 + "&NETDATA_CONTAINER_OS_NAME=%s"
230 + "&NETDATA_CONTAINER_OS_ID=%s"
231 + "&NETDATA_CONTAINER_OS_ID_LIKE=%s"
232 + "&NETDATA_CONTAINER_OS_VERSION=%s"
233 + "&NETDATA_CONTAINER_OS_VERSION_ID=%s"
234 + "&NETDATA_CONTAINER_OS_DETECTION=%s"
235 + "&NETDATA_SYSTEM_CPU_LOGICAL_CPU_COUNT=%s"
236 + "&NETDATA_SYSTEM_CPU_FREQ=%s"
237 + "&NETDATA_SYSTEM_TOTAL_RAM=%s"
238 + "&NETDATA_SYSTEM_TOTAL_DISK_SIZE=%s"
239 + "&NETDATA_PROTOCOL_VERSION=%s"
240 + " HTTP/1.1\r\n"
241 + "User-Agent: %s/%s\r\n"
242 + "Accept: */*\r\n\r\n"
243 + , host->rrdpush_send_api_key
244 + , host->hostname
245 + , host->registry_hostname
246 + , host->machine_guid
247 + , default_rrd_update_every
248 + , host->os
249 + , host->timezone
250 + , (host->tags) ? host->tags : ""
251 + , STREAMING_PROTOCOL_CURRENT_VERSION
252 + , se.os_name
253 + , se.os_id
254 + , (host->system_info->host_os_id_like) ? host->system_info->host_os_id_like : ""
255 + , se.os_version
256 + , (host->system_info->host_os_version_id) ? host->system_info->host_os_version_id : ""
257 + , (host->system_info->host_os_detection) ? host->system_info->host_os_detection : ""
258 + , se.kernel_name
259 + , se.kernel_version
260 + , (host->system_info->architecture) ? host->system_info->architecture : ""
261 + , (host->system_info->virtualization) ? host->system_info->virtualization : ""
262 + , (host->system_info->virt_detection) ? host->system_info->virt_detection : ""
263 + , (host->system_info->container) ? host->system_info->container : ""
264 + , (host->system_info->container_detection) ? host->system_info->container_detection : ""
265 + , (host->system_info->container_os_name) ? host->system_info->container_os_name : ""
266 + , (host->system_info->container_os_id) ? host->system_info->container_os_id : ""
267 + , (host->system_info->container_os_id_like) ? host->system_info->container_os_id_like : ""
268 + , (host->system_info->container_os_version) ? host->system_info->container_os_version : ""
269 + , (host->system_info->container_os_version_id) ? host->system_info->container_os_version_id : ""
270 + , (host->system_info->container_os_detection) ? host->system_info->container_os_detection : ""
271 + , (host->system_info->host_cores) ? host->system_info->host_cores : ""
272 + , (host->system_info->host_cpu_freq) ? host->system_info->host_cpu_freq : ""
273 + , (host->system_info->host_ram_total) ? host->system_info->host_ram_total : ""
274 + , (host->system_info->host_disk_space) ? host->system_info->host_disk_space : ""
275 + , STREAMING_PROTOCOL_VERSION
276 + , host->program_name
277 + , host->program_version
278 + );
279 + http[eol] = 0x00;
280 + rrdpush_clean_encoded(&se);
281 +
282 +#ifdef ENABLE_HTTPS
283 + if (!host->ssl.flags) {
284 + ERR_clear_error();
285 + SSL_set_connect_state(host->ssl.conn);
286 + int err = SSL_connect(host->ssl.conn);
287 + if (err != 1){
288 + err = SSL_get_error(host->ssl.conn, err);
289 + error("SSL cannot connect with the server: %s ",ERR_error_string((long)SSL_get_error(host->ssl.conn,err),NULL));
290 + if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
291 + rrdpush_sender_thread_close_socket(host);
292 + return 0;
293 + }else {
294 + host->ssl.flags = NETDATA_SSL_NO_HANDSHAKE;
295 + }
296 + }
297 + else {
298 + if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
299 + if (netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE) {
300 + if ( security_test_certificate(host->ssl.conn)) {
301 + error("Closing the stream connection, because the server SSL certificate is not valid.");
302 + rrdpush_sender_thread_close_socket(host);
303 + return 0;
304 + }
305 + }
306 + }
307 + }
308 + }
309 + if(send_timeout(&host->ssl,host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
310 +#else
311 + if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
312 +#endif
313 + error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, s->connected_to);
314 + rrdpush_sender_thread_close_socket(host);
315 + return 0;
316 + }
317 +
318 + info("STREAM %s [send to %s]: waiting response from remote netdata...", host->hostname, s->connected_to);
319 +
320 + ssize_t received;
321 +#ifdef ENABLE_HTTPS
322 + received = recv_timeout(&host->ssl,host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout);
323 + if(received == -1) {
324 +#else
325 + received = recv_timeout(host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout);
326 + if(received == -1) {
327 +#endif
328 + error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, s->connected_to);
329 + rrdpush_sender_thread_close_socket(host);
330 + return 0;
331 + }
332 +
333 + http[received] = '\0';
334 + debug(D_STREAM, "Response to sender from far end: %s", http);
335 + int answer = -1;
336 + char *version_start = strchr(http, '=');
337 + int32_t version = -1;
338 + if(version_start) {
339 + version_start++;
340 + version = (int32_t)strtol(version_start, NULL, 10);
341 + answer = memcmp(http, START_STREAMING_PROMPT_VN, (size_t)(version_start - http));
342 + if(!answer) {
343 + rrdpush_set_flags_to_newest_stream(host);
344 + }
345 + } else {
346 + answer = memcmp(http, START_STREAMING_PROMPT_V2, strlen(START_STREAMING_PROMPT_V2));
347 + if(!answer) {
348 + version = 1;
349 + rrdpush_set_flags_to_newest_stream(host);
350 + }
351 + else {
352 + answer = memcmp(http, START_STREAMING_PROMPT, strlen(START_STREAMING_PROMPT));
353 + if(!answer) {
354 + version = 0;
355 + host->labels_flag |= LABEL_FLAG_STOP_STREAM;
356 + host->labels_flag &= ~LABEL_FLAG_UPDATE_STREAM;
357 + }
358 + }
359 + }
360 +
361 + if(version == -1) {
362 + error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, s->connected_to);
363 + rrdpush_sender_thread_close_socket(host);
364 + return 0;
365 + }
366 + s->version = version;
367 +
368 + info("STREAM %s [send to %s]: established communication with a master using protocol version %d - ready to send metrics..."
369 + , host->hostname
370 + , s->connected_to
371 + , version);
372 +
373 + if(sock_setnonblock(host->rrdpush_sender_socket) < 0)
374 + error("STREAM %s [send to %s]: cannot set non-blocking mode for socket.", host->hostname, s->connected_to);
375 +
376 + if(sock_enlarge_out(host->rrdpush_sender_socket) < 0)
377 + error("STREAM %s [send to %s]: cannot enlarge the socket buffer.", host->hostname, s->connected_to);
378 +
379 + debug(D_STREAM, "STREAM: Connected on fd %d...", host->rrdpush_sender_socket);
380 +
381 + return 1;
382 +}
383 +
384 +static void attempt_to_connect(struct sender_state *state)
385 +{
386 + state->send_attempts = 0;
387 +
388 + if(rrdpush_sender_thread_connect_to_master(state->host, state->default_port, state->timeout, state)) {
389 + state->last_sent_t = now_monotonic_sec();
390 +
391 + // reset the buffer, to properly send charts and metrics
392 + rrdpush_sender_thread_data_flush(state->host);
393 +
394 + // send from the beginning
395 + state->begin = 0;
396 +
397 + // make sure the next reconnection will be immediate
398 + state->not_connected_loops = 0;
399 +
400 + // reset the bytes we have sent for this session
401 + state->sent_bytes_on_this_connection = 0;
402 +
403 + // let the data collection threads know we are ready
404 + state->host->rrdpush_sender_connected = 1;
405 + }
406 + else {
407 + // increase the failed connections counter
408 + state->not_connected_loops++;
409 +
410 + // reset the number of bytes sent
411 + state->sent_bytes_on_this_connection = 0;
412 +
413 + // slow re-connection on repeating errors
414 + sleep_usec(USEC_PER_SEC * state->reconnect_delay); // seconds
415 + }
416 +}
417 +
418 +// TCP window is open and we have data to transmit.
419 +void attempt_to_send(struct sender_state *s, char *chunk, size_t outstanding) {
420 + rrdpush_send_labels(s->host);
421 +
422 + struct circular_buffer *cb = s->host->sender->buffer;
423 + debug(D_STREAM, "STREAM: Sending data. Buffer r=%zu w=%zu s=%zu, next chunk=%zu", cb->read, cb->write, cb->size, outstanding);
424 +
425 + netdata_thread_disable_cancelability();
426 + netdata_mutex_lock(&s->host->sender->mutex);
427 +
428 + ssize_t ret;
429 +#ifdef ENABLE_HTTPS
430 + SSL *conn = s->host->ssl.conn ;
431 + if(conn && !s->host->ssl.flags) {
432 + ret = SSL_write(conn, chunk, outstanding);
433 + } else {
434 + ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT);
435 + }
436 +#else
437 + ret = send(s->host->rrdpush_sender_socket, chunk, outstanding, MSG_DONTWAIT);
438 +#endif
439 + if (likely(ret > 0)) {
440 + cbuffer_remove_unsafe(s->host->sender->buffer, ret);
441 + s->sent_bytes_on_this_connection += ret;
442 + s->sent_bytes += ret;
443 + debug(D_STREAM, "STREAM %s [send to %s]: Sent %zd bytes", s->host->hostname, s->connected_to, ret);
444 + s->last_sent_t = now_monotonic_sec();
445 + }
446 + else if (ret == -1 && (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK))
447 + debug(D_STREAM, "STREAM %s [send to %s]: unavailable aftering polling POLLOUT", s->host->hostname,
448 + s->connected_to);
449 + else if (ret == -1) {
450 + debug(D_STREAM, "STREAM: Send failed - closing socket...");
451 + error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", s->host->hostname, s->connected_to, s->sent_bytes_on_this_connection);
452 + rrdpush_sender_thread_close_socket(s->host);
453 + }
454 + else {
455 + debug(D_STREAM, "STREAM: send() returned 0 -> no error but no transmission");
456 + }
457 +
458 + netdata_mutex_unlock(&s->host->sender->mutex);
459 + netdata_thread_enable_cancelability();
460 +}
461 +
462 +void attempt_read(struct sender_state *s) {
463 +int ret;
464 +#ifdef ENABLE_HTTPS
465 + if (s->host->ssl.conn && !s->host->stream_ssl.flags) {
466 + ERR_clear_error();
467 + int desired = sizeof(s->read_buffer) - s->read_len - 1;
468 + ret = SSL_read(s->host->ssl.conn, s->read_buffer, desired);
469 + if (ret > 0 ) {
470 + s->read_len += ret;
471 + return;
472 + }
473 + int sslerrno = SSL_get_error(s->host->ssl.conn, desired);
474 + if (sslerrno == SSL_ERROR_WANT_READ || sslerrno == SSL_ERROR_WANT_WRITE)
475 + return;
476 + u_long err;
477 + char buf[256];
478 + while ((err = ERR_get_error()) != 0) {
479 + ERR_error_string_n(err, buf, sizeof(buf));
480 + error("STREAM %s [send to %s] ssl error: %s", s->host->hostname, s->connected_to, buf);
481 + }
482 + error("Restarting connection");
483 + rrdpush_sender_thread_close_socket(s->host);
484 + return;
485 + }
486 +#endif
487 + ret = recv(s->host->rrdpush_sender_socket, s->read_buffer + s->read_len, sizeof(s->read_buffer) - s->read_len - 1,
488 + MSG_DONTWAIT);
489 + if (ret>0) {
490 + s->read_len += ret;
491 + return;
492 + }
493 + debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %d", sizeof(s->read_buffer) - s->read_len - 1, ret);
494 + if (ret<0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))
495 + return;
496 + if (ret==0)
497 + error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", s->host->hostname, s->connected_to);
498 + else
499 + error("STREAM %s [send to %s]: error during read (%d). Restarting connection", s->host->hostname, s->connected_to,
500 + ret);
501 + rrdpush_sender_thread_close_socket(s->host);
502 +}
503 +
504 +// This is just a placeholder until the gap filling state machine is inserted
505 +void execute_commands(struct sender_state *s) {
506 + char *start = s->read_buffer, *end = &s->read_buffer[s->read_len], *newline;
507 + *end = 0;
508 + while( start<end && (newline=strchr(start, '\n')) ) {
509 + *newline = 0;
510 + info("STREAM %s [send to %s] received command over connection: %s", s->host->hostname, s->connected_to, start);
511 + start = newline+1;
512 + }
513 + if (start<end) {
514 + memcpy( s->read_buffer, start, end-start);
515 + s->read_len = end-start;
516 + }
517 +}
518 +
519 +
520 +static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
521 + RRDHOST *host = (RRDHOST *)ptr;
522 +
523 + rrdhost_wrlock(host);
524 +
525 + info("STREAM %s [send]: sending thread cleans up...", host->hostname);
526 +
527 + rrdpush_sender_thread_close_socket(host);
528 +
529 + // close the pipe
530 + if(host->rrdpush_sender_pipe[PIPE_READ] != -1) {
531 + close(host->rrdpush_sender_pipe[PIPE_READ]);
532 + host->rrdpush_sender_pipe[PIPE_READ] = -1;
533 + }
534 +
535 + if(host->rrdpush_sender_pipe[PIPE_WRITE] != -1) {
536 + close(host->rrdpush_sender_pipe[PIPE_WRITE]);
537 + host->rrdpush_sender_pipe[PIPE_WRITE] = -1;
538 + }
539 +
540 + if(!host->rrdpush_sender_join) {
541 + info("STREAM %s [send]: sending thread detaches itself.", host->hostname);
542 + netdata_thread_detach(netdata_thread_self());
543 + }
544 +
545 + host->rrdpush_sender_spawn = 0;
546 +
547 + info("STREAM %s [send]: sending thread now exits.", host->hostname);
548 +
549 + rrdhost_unlock(host);
550 +}
551 +
552 +void sender_init(struct sender_state *s, RRDHOST *parent) {
553 + memset(s, 0, sizeof(*s));
554 + s->host = parent;
555 + s->buffer = cbuffer_new(1024, 1024*1024);
556 + s->build = buffer_create(1);
557 + netdata_mutex_init(&s->mutex);
558 +}
559 +
560 +void *rrdpush_sender_thread(void *ptr) {
561 + struct sender_state *s = ptr;
562 + s->task_id = gettid();
563 +
564 + if(!s->host->rrdpush_send_enabled || !s->host->rrdpush_send_destination ||
565 + !*s->host->rrdpush_send_destination || !s->host->rrdpush_send_api_key ||
566 + !*s->host->rrdpush_send_api_key) {
567 + error("STREAM %s [send]: thread created (task id %d), but host has streaming disabled.",
568 + s->host->hostname, s->task_id);
569 + return NULL;
570 + }
571 +
572 +#ifdef ENABLE_HTTPS
573 + if (netdata_use_ssl_on_stream & NETDATA_SSL_FORCE ){
574 + security_start_ssl(NETDATA_SSL_CONTEXT_STREAMING);
575 + security_location_for_context(netdata_client_ctx, netdata_ssl_ca_file, netdata_ssl_ca_path);
576 + }
577 +#endif
578 +
579 + info("STREAM %s [send]: thread created (task id %d)", s->host->hostname, s->task_id);
580 +
581 + s->timeout = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "timeout seconds", 60);
582 + s->default_port = (int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "default port", 19999);
583 + s->max_size = (size_t)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "buffer size bytes",
584 + 1024 * 1024);
585 + s->reconnect_delay = (unsigned int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "reconnect delay seconds", 5);
586 + remote_clock_resync_iterations = (unsigned int)appconfig_get_number(&stream_config, CONFIG_SECTION_STREAM, "initial clock resync iterations", remote_clock_resync_iterations); // TODO: REMOVE FOR SLEW / GAPFILLING
587 +
588 + // initialize rrdpush globals
589 + s->host->rrdpush_sender_connected = 0;
590 + if(pipe(s->host->rrdpush_sender_pipe) == -1) {
591 + error("STREAM %s [send]: cannot create required pipe. DISABLING STREAMING THREAD", s->host->hostname);
592 + return NULL;
593 + }
594 +
595 + enum {
596 + Collector,
597 + Socket
598 + };
599 + struct pollfd fds[2];
600 + fds[Collector].fd = s->host->rrdpush_sender_pipe[PIPE_READ];
601 + fds[Collector].events = POLLIN;
602 +
603 + netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, s->host);
604 + for(; s->host->rrdpush_send_enabled && !netdata_exit ;) {
605 + // check for outstanding cancellation requests
606 + netdata_thread_testcancel();
607 +
608 + // The connection attempt blocks (after which we use the socket in nonblocking)
609 + if(unlikely(s->host->rrdpush_sender_socket == -1)) {
610 + s->overflow = 0;
611 + s->read_len = 0;
612 + s->buffer->read = 0;
613 + s->buffer->write = 0;
614 + attempt_to_connect(s);
615 + if (s->version >= VERSION_GAP_FILLING) {
616 + time_t now = now_realtime_sec();
617 + sender_start(s);
618 + buffer_sprintf(s->build, "TIMESTAMP %ld", now);
619 + sender_commit(s);
620 + }
621 + continue;
622 + }
623 +
624 + // If the TCP window never opened then something is wrong, restart connection
625 + if(unlikely(now_monotonic_sec() - s->last_sent_t > s->timeout)) {
626 + error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", s->host->hostname, s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);
627 + rrdpush_sender_thread_close_socket(s->host);
628 + continue;
629 + }
630 +
631 + // Wait until buffer opens in the socket or a rrdset_done_push wakes us
632 + fds[Collector].revents = 0;
633 + fds[Socket].revents = 0;
634 + fds[Socket].fd = s->host->rrdpush_sender_socket;
635 +
636 + char *chunk;
637 + size_t outstanding = cbuffer_next_unsafe(s->host->sender->buffer, &chunk);
638 + if(outstanding) {
639 + s->send_attempts++;
640 + fds[Socket].events = POLLIN | POLLOUT;
641 + }
642 + else {
643 + fds[Socket].events = POLLIN;
644 + }
645 +
646 + int retval = poll(fds, 2, 1000);
647 + debug(D_STREAM, "STREAM: poll() finished collector=%d socket=%d (current chunk %zu bytes)...",
648 + fds[Collector].revents, fds[Socket].revents, outstanding);
649 + if(unlikely(netdata_exit)) break;
650 +
651 + // Spurious wake-ups without error - loop again
652 + if (retval == 0 || ((retval == -1) && (errno == EAGAIN || errno == EINTR)))
653 + {
654 + debug(D_STREAM, "Spurious wakeup");
655 + continue;
656 + }
657 + // Only errors from poll() are internal, but try restarting the connection
658 + if(unlikely(retval == -1)) {
659 + error("STREAM %s [send to %s]: failed to poll(). Closing socket.", s->host->hostname, s->connected_to);
660 + rrdpush_sender_thread_close_socket(s->host);
661 + continue;
662 + }
663 +
664 + // If the collector woke us up then empty the pipe to remove the signal
665 + if (fds[Collector].revents & POLLIN || fds[Collector].revents & POLLPRI) {
666 + debug(D_STREAM, "STREAM: Data added to send buffer (current buffer chunk %zu bytes)...", outstanding);
667 +
668 + char buffer[1000 + 1];
669 + if (read(s->host->rrdpush_sender_pipe[PIPE_READ], buffer, 1000) == -1)
670 + error("STREAM %s [send to %s]: cannot read from internal pipe.", s->host->hostname, s->connected_to);
671 + }
672 +
673 + // Read as much as possible to fill the buffer, split into full lines for execution.
674 + if (fds[Socket].revents & POLLIN)
675 + attempt_read(s);
676 + execute_commands(s);
677 +
678 + // If we have data and have seen the TCP window open then try to close it by a transmission.
679 + if (outstanding && fds[Socket].revents & POLLOUT)
680 + attempt_to_send(s, chunk, outstanding);
681 +
682 + // TODO-GAPS - why do we only check this on the socket, not the pipe?
683 + if (outstanding) {
684 + char *error = NULL;
685 + if (unlikely(fds[Socket].revents & POLLERR))
686 + error = "socket reports errors (POLLERR)";
687 + else if (unlikely(fds[Socket].revents & POLLHUP))
688 + error = "connection closed by remote end (POLLHUP)";
689 + else if (unlikely(fds[Socket].revents & POLLNVAL))
690 + error = "connection is invalid (POLLNVAL)";
691 + if(unlikely(error)) {
692 + error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", s->host->hostname,
693 + s->connected_to, error, s->sent_bytes_on_this_connection);
694 + rrdpush_sender_thread_close_socket(s->host);
695 + }
696 + }
697 +
698 + // protection from overflow
699 + if (s->overflow) {
700 + errno = 0;
701 + error("STREAM %s [send to %s]: buffer full (%zu-bytes) after %zu bytes. Restarting connection",
702 + s->host->hostname, s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);
703 + rrdpush_sender_thread_close_socket(s->host);
704 + }
705 + }
706 +
707 + netdata_thread_cleanup_pop(1);
708 + return NULL;
709 +}
web/server/web_client.c
+1 -1
@@ -1511,7 +1511,7 @@ void web_client_process_request(struct web_client *w) {
1511 return;
1512 }
1513
1514 - w->response.code = rrdpush_receiver_thread_spawn(localhost, w, w->decoded_url);
1514 + w->response.code = rrdpush_receiver_thread_spawn(w, w->decoded_url);
1515 return;
1516
1517 case WEB_CLIENT_MODE_OPTIONS: