| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "stream-sender-internals.h" |
| 4 | |
| 5 | static struct { |
| 6 | const char *response; |
| 7 | size_t length; |
| 8 | int32_t version; |
| 9 | bool dynamic; |
| 10 | const char *error; |
| 11 | int worker_job_id; |
| 12 | int postpone_reconnect_seconds; |
| 13 | ND_LOG_FIELD_PRIORITY priority; |
| 14 | } stream_responses[] = { |
| 15 | { |
| 16 | .response = START_STREAMING_PROMPT_VN, |
| 17 | .length = sizeof(START_STREAMING_PROMPT_VN) - 1, |
| 18 | .version = STREAM_HANDSHAKE_OK_V3, // and above |
| 19 | .dynamic = true, // dynamic = we will parse the version / capabilities |
| 20 | .error = NULL, |
| 21 | .worker_job_id = 0, |
| 22 | .postpone_reconnect_seconds = 0, |
| 23 | .priority = NDLP_INFO, |
| 24 | }, |
| 25 | { |
| 26 | .response = START_STREAMING_PROMPT_V2, |
| 27 | .length = sizeof(START_STREAMING_PROMPT_V2) - 1, |
| 28 | .version = STREAM_HANDSHAKE_OK_V2, |
| 29 | .dynamic = false, |
| 30 | .error = NULL, |
| 31 | .worker_job_id = 0, |
| 32 | .postpone_reconnect_seconds = 0, |
| 33 | .priority = NDLP_INFO, |
| 34 | }, |
| 35 | { |
| 36 | .response = START_STREAMING_PROMPT_V1, |
| 37 | .length = sizeof(START_STREAMING_PROMPT_V1) - 1, |
| 38 | .version = STREAM_HANDSHAKE_OK_V1, |
| 39 | .dynamic = false, |
| 40 | .error = NULL, |
| 41 | .worker_job_id = 0, |
| 42 | .postpone_reconnect_seconds = 0, |
| 43 | .priority = NDLP_INFO, |
| 44 | }, |
| 45 | { |
| 46 | .response = START_STREAMING_ERROR_SAME_LOCALHOST, |
| 47 | .length = sizeof(START_STREAMING_ERROR_SAME_LOCALHOST) - 1, |
| 48 | .version = STREAM_HANDSHAKE_PARENT_IS_LOCALHOST, |
| 49 | .dynamic = false, |
| 50 | .error = "remote server rejected this stream, the host we are trying to stream is its localhost", |
| 51 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 52 | .postpone_reconnect_seconds = 60 * 60, // the IP may change, try it every hour |
| 53 | .priority = NDLP_DEBUG, |
| 54 | }, |
| 55 | { |
| 56 | .response = START_STREAMING_ERROR_LOCAL_VNODE, |
| 57 | .length = sizeof(START_STREAMING_ERROR_LOCAL_VNODE) - 1, |
| 58 | .version = STREAM_HANDSHAKE_PARENT_VNODE_IS_LOCAL, |
| 59 | .dynamic = false, |
| 60 | .error = "remote server rejected this stream, the vnode is collected locally on that server", |
| 61 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 62 | .postpone_reconnect_seconds = 60 * 60, // the vnode may stop being collected, try every hour |
| 63 | .priority = NDLP_DEBUG, |
| 64 | }, |
| 65 | { |
| 66 | .response = START_STREAMING_ERROR_ALREADY_STREAMING, |
| 67 | .length = sizeof(START_STREAMING_ERROR_ALREADY_STREAMING) - 1, |
| 68 | .version = STREAM_HANDSHAKE_PARENT_NODE_ALREADY_CONNECTED, |
| 69 | .dynamic = false, |
| 70 | .error = "remote server rejected this stream, the host we are trying to stream is already streamed to it", |
| 71 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 72 | .postpone_reconnect_seconds = 2 * 60, // 2 minutes |
| 73 | .priority = NDLP_DEBUG, |
| 74 | }, |
| 75 | { |
| 76 | .response = START_STREAMING_ERROR_NOT_PERMITTED, |
| 77 | .length = sizeof(START_STREAMING_ERROR_NOT_PERMITTED) - 1, |
| 78 | .version = STREAM_HANDSHAKE_PARENT_DENIED_ACCESS, |
| 79 | .dynamic = false, |
| 80 | .error = "remote server denied access, probably we don't have the right API key?", |
| 81 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 82 | .postpone_reconnect_seconds = 1 * 60, // 1 minute |
| 83 | .priority = NDLP_ERR, |
| 84 | }, |
| 85 | { |
| 86 | .response = START_STREAMING_ERROR_BUSY_TRY_LATER, |
| 87 | .length = sizeof(START_STREAMING_ERROR_BUSY_TRY_LATER) - 1, |
| 88 | .version = STREAM_HANDSHAKE_PARENT_BUSY_TRY_LATER, |
| 89 | .dynamic = false, |
| 90 | .error = "remote server is currently busy, we should try later", |
| 91 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 92 | .postpone_reconnect_seconds = 2 * 60, // 2 minutes |
| 93 | .priority = NDLP_NOTICE, |
| 94 | }, |
| 95 | { |
| 96 | .response = START_STREAMING_ERROR_INTERNAL_ERROR, |
| 97 | .length = sizeof(START_STREAMING_ERROR_INTERNAL_ERROR) - 1, |
| 98 | .version = STREAM_HANDSHAKE_PARENT_INTERNAL_ERROR, |
| 99 | .dynamic = false, |
| 100 | .error = "remote server is encountered an internal error, we should try later", |
| 101 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 102 | .postpone_reconnect_seconds = 5 * 60, // 5 minutes |
| 103 | .priority = NDLP_CRIT, |
| 104 | }, |
| 105 | { |
| 106 | .response = START_STREAMING_ERROR_INITIALIZATION, |
| 107 | .length = sizeof(START_STREAMING_ERROR_INITIALIZATION) - 1, |
| 108 | .version = STREAM_HANDSHAKE_PARENT_IS_INITIALIZING, |
| 109 | .dynamic = false, |
| 110 | .error = "remote server is initializing, we should try later", |
| 111 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 112 | .postpone_reconnect_seconds = 30, // 30 seconds |
| 113 | .priority = NDLP_NOTICE, |
| 114 | }, |
| 115 | |
| 116 | // terminator |
| 117 | { |
| 118 | .response = NULL, |
| 119 | .length = 0, |
| 120 | .version = STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED, |
| 121 | .dynamic = false, |
| 122 | .error = "remote node response is not understood, is it Netdata?", |
| 123 | .worker_job_id = WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, |
| 124 | .postpone_reconnect_seconds = 1 * 60, // 1 minute |
| 125 | .priority = NDLP_ERR, |
| 126 | } |
| 127 | }; |
| 128 | |
| 129 | #define CONN_UPGRADE_VAL "upgrade" |
| 130 | static int stream_connect_upgrade_prelude(RRDHOST *host __maybe_unused, struct sender_state *s) { |
| 131 | |
| 132 | char http[HTTP_HEADER_SIZE + 1]; |
| 133 | snprintfz(http, HTTP_HEADER_SIZE, |
| 134 | "GET " NETDATA_STREAM_URL HTTP_1_1 HTTP_ENDL |
| 135 | "Upgrade: " NETDATA_STREAM_PROTO_NAME HTTP_ENDL |
| 136 | "Connection: Upgrade" |
| 137 | HTTP_HDR_END); |
| 138 | |
| 139 | ssize_t bytes; |
| 140 | bytes = nd_sock_send_timeout(&s->sock, http, strlen(http), 0, 1000); |
| 141 | if (bytes <= 0) { |
| 142 | error_report("Error writing to remote"); |
| 143 | return 1; |
| 144 | } |
| 145 | |
| 146 | bytes = nd_sock_recv_timeout(&s->sock, http, HTTP_HEADER_SIZE, 0, 1000); |
| 147 | if (bytes <= 0) { |
| 148 | error_report("Error reading from remote"); |
| 149 | return 1; |
| 150 | } |
| 151 | |
| 152 | rbuf_t buf = rbuf_create(bytes); |
| 153 | rbuf_push(buf, http, bytes); |
| 154 | |
| 155 | http_parse_ctx ctx; |
| 156 | http_parse_ctx_create(&ctx, HTTP_PARSE_INITIAL); |
| 157 | ctx.flags |= HTTP_PARSE_FLAG_DONT_WAIT_FOR_CONTENT; |
| 158 | |
| 159 | int rc; |
| 160 | // while((rc = parse_http_response(buf, &ctx)) == HTTP_PARSE_NEED_MORE_DATA); |
| 161 | rc = parse_http_response(buf, &ctx); |
| 162 | |
| 163 | if (rc != HTTP_PARSE_SUCCESS) { |
| 164 | error_report("Failed to parse HTTP response sent. (%d)", rc); |
| 165 | goto err_cleanup; |
| 166 | } |
| 167 | if (ctx.http_code == HTTP_RESP_MOVED_PERM) { |
| 168 | const char *hdr = get_http_header_by_name(&ctx, "location"); |
| 169 | if (hdr) |
| 170 | error_report("HTTP response is %d Moved Permanently (location: \"%s\") instead of expected %d Switching Protocols.", ctx.http_code, hdr, HTTP_RESP_SWITCH_PROTO); |
| 171 | else |
| 172 | error_report("HTTP response is %d instead of expected %d Switching Protocols.", ctx.http_code, HTTP_RESP_SWITCH_PROTO); |
| 173 | goto err_cleanup; |
| 174 | } |
| 175 | if (ctx.http_code == HTTP_RESP_NOT_FOUND) { |
| 176 | error_report("HTTP response is %d instead of expected %d Switching Protocols. Parent version too old.", ctx.http_code, HTTP_RESP_SWITCH_PROTO); |
| 177 | // TODO set some flag here that will signify parent is older version |
| 178 | // and to try connection without rrdpush_http_upgrade_prelude next time |
| 179 | goto err_cleanup; |
| 180 | } |
| 181 | if (ctx.http_code != HTTP_RESP_SWITCH_PROTO) { |
| 182 | error_report("HTTP response is %d instead of expected %d Switching Protocols", ctx.http_code, HTTP_RESP_SWITCH_PROTO); |
| 183 | goto err_cleanup; |
| 184 | } |
| 185 | |
| 186 | const char *hdr = get_http_header_by_name(&ctx, "connection"); |
| 187 | if (!hdr) { |
| 188 | error_report("Missing \"connection\" header in reply"); |
| 189 | goto err_cleanup; |
| 190 | } |
| 191 | if (strncmp(hdr, CONN_UPGRADE_VAL, strlen(CONN_UPGRADE_VAL)) != 0) { |
| 192 | error_report("Expected \"connection: " CONN_UPGRADE_VAL "\""); |
| 193 | goto err_cleanup; |
| 194 | } |
| 195 | |
| 196 | hdr = get_http_header_by_name(&ctx, "upgrade"); |
| 197 | if (!hdr) { |
| 198 | error_report("Missing \"upgrade\" header in reply"); |
| 199 | goto err_cleanup; |
| 200 | } |
| 201 | if (strncmp(hdr, NETDATA_STREAM_PROTO_NAME, strlen(NETDATA_STREAM_PROTO_NAME)) != 0) { |
| 202 | error_report("Expected \"upgrade: " NETDATA_STREAM_PROTO_NAME "\""); |
| 203 | goto err_cleanup; |
| 204 | } |
| 205 | |
| 206 | netdata_log_debug(D_STREAM, "STREAM SNDer upgrade to \"" NETDATA_STREAM_PROTO_NAME "\" successful"); |
| 207 | rbuf_free(buf); |
| 208 | http_parse_ctx_destroy(&ctx); |
| 209 | return 0; |
| 210 | err_cleanup: |
| 211 | rbuf_free(buf); |
| 212 | http_parse_ctx_destroy(&ctx); |
| 213 | return 1; |
| 214 | } |
| 215 | |
| 216 | static bool |
| 217 | stream_connect_validate_first_response(RRDHOST *host, struct sender_state *s, char *http, size_t http_length) { |
| 218 | int32_t version = STREAM_HANDSHAKE_CONNECT_HANDSHAKE_FAILED; |
| 219 | |
| 220 | int i; |
| 221 | for(i = 0; stream_responses[i].response ; i++) { |
| 222 | if(stream_responses[i].dynamic && |
| 223 | http_length > stream_responses[i].length && http_length < (stream_responses[i].length + 30) && |
| 224 | strncmp(http, stream_responses[i].response, stream_responses[i].length) == 0) { |
| 225 | |
| 226 | version = str2i(&http[stream_responses[i].length]); |
| 227 | break; |
| 228 | } |
| 229 | else if(http_length == stream_responses[i].length && strcmp(http, stream_responses[i].response) == 0) { |
| 230 | version = stream_responses[i].version; |
| 231 | |
| 232 | break; |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | if(version >= STREAM_HANDSHAKE_OK_V1) { |
| 237 | stream_parent_set_host_reconnect_delay( |
| 238 | host, STREAM_HANDSHAKE_SP_CONNECTED, stream_send.parents.reconnect_delay_s); |
| 239 | s->capabilities = convert_stream_version_to_capabilities(version, host, true); |
| 240 | s->host->stream.snd.status.reason = (STREAM_HANDSHAKE)s->capabilities; |
| 241 | return true; |
| 242 | } |
| 243 | |
| 244 | ND_LOG_FIELD_PRIORITY priority = stream_responses[i].priority; |
| 245 | const char *error = stream_responses[i].error; |
| 246 | int worker_job_id = stream_responses[i].worker_job_id; |
| 247 | int delay = stream_responses[i].postpone_reconnect_seconds; |
| 248 | |
| 249 | worker_is_busy(worker_job_id); |
| 250 | stream_parent_set_host_connect_failure_reason(host, version, delay); |
| 251 | |
| 252 | ND_LOG_STACK lgs[] = { |
| 253 | ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(version)), |
| 254 | ND_LOG_FIELD_END(), |
| 255 | }; |
| 256 | ND_LOG_STACK_PUSH(lgs); |
| 257 | |
| 258 | char buf[RFC3339_MAX_LENGTH]; |
| 259 | rfc3339_datetime_ut(buf, sizeof(buf), stream_parent_get_reconnection_ut(host->stream.snd.parents.current), 0, false); |
| 260 | |
| 261 | nd_log(NDLS_DAEMON, priority, |
| 262 | "STREAM CONNECT '%s' [to %s]: %s - will retry in %d secs, at %s", |
| 263 | rrdhost_hostname(host), s->remote_ip, error, delay, buf); |
| 264 | |
| 265 | return false; |
| 266 | } |
| 267 | |
| 268 | bool stream_connect(struct sender_state *s, uint16_t default_port, time_t timeout) { |
| 269 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_CONNECTING); |
| 270 | |
| 271 | RRDHOST *host = s->host; |
| 272 | |
| 273 | // make sure the socket is closed |
| 274 | nd_sock_close(&s->sock); |
| 275 | |
| 276 | s->hops = (int16_t)(rrdhost_ingestion_hops(s->host) + 1); |
| 277 | |
| 278 | // reset this to make sure we have its current value |
| 279 | s->sock.verify_certificate = netdata_ssl_validate_certificate_sender; |
| 280 | s->sock.ctx = netdata_ssl_streaming_sender_ctx; |
| 281 | |
| 282 | pulse_host_status(s->host, PULSE_HOST_STATUS_SND_PENDING, 0); |
| 283 | if(!stream_parent_connect_to_one( |
| 284 | &s->sock, host, default_port, timeout, |
| 285 | s->remote_ip, sizeof(s->remote_ip) - 1, |
| 286 | &host->stream.snd.parents.current)) { |
| 287 | |
| 288 | if(s->sock.error != ND_SOCK_ERR_NO_DESTINATION_AVAILABLE) { |
| 289 | nd_log(NDLS_DAEMON, NDLP_WARNING, "can't connect to a parent, last error: %s", |
| 290 | ND_SOCK_ERROR_2str(s->sock.error)); |
| 291 | } |
| 292 | |
| 293 | nd_sock_close(&s->sock); |
| 294 | return false; |
| 295 | } |
| 296 | |
| 297 | // reset our capabilities to default |
| 298 | s->capabilities = stream_our_capabilities(host, true); |
| 299 | |
| 300 | /* TODO: During the implementation of #7265 switch the set of variables to HOST_* and CONTAINER_* if the |
| 301 | version negotiation resulted in a high enough version. |
| 302 | */ |
| 303 | CLEAN_BUFFER *wb = buffer_create(0, NULL); |
| 304 | buffer_strcat(wb, "STREAM "); |
| 305 | buffer_key_value_urlencode(wb, "key", string2str(host->stream.snd.api_key)); |
| 306 | buffer_key_value_urlencode(wb, "&hostname", rrdhost_hostname(host)); |
| 307 | buffer_key_value_urlencode(wb, "®istry_hostname", rrdhost_registry_hostname(host)); |
| 308 | buffer_key_value_urlencode(wb, "&machine_guid", host->machine_guid); |
| 309 | buffer_sprintf(wb, "&update_every=%d", (int)nd_profile.update_every); |
| 310 | buffer_key_value_urlencode(wb, "&os", rrdhost_os(host)); |
| 311 | { |
| 312 | RRDHOST_TZ host_tz = rrdhost_tz_get(host); |
| 313 | buffer_key_value_urlencode(wb, "&timezone", host_tz.timezone); |
| 314 | buffer_key_value_urlencode(wb, "&abbrev_timezone", host_tz.abbrev_timezone); |
| 315 | buffer_sprintf(wb, "&utc_offset=%d", host_tz.utc_offset); |
| 316 | rrdhost_tz_free(&host_tz); |
| 317 | } |
| 318 | buffer_sprintf(wb, "&hops=%d", s->hops); |
| 319 | buffer_sprintf(wb, "&ver=%u", s->capabilities); |
| 320 | rrdhost_system_info_to_url_encode_stream(wb, host->system_info); |
| 321 | buffer_key_value_urlencode(wb, "&NETDATA_PROTOCOL_VERSION", STREAMING_PROTOCOL_VERSION); |
| 322 | buffer_strcat(wb, HTTP_1_1 HTTP_ENDL); |
| 323 | buffer_sprintf(wb, "User-Agent: %s/%s" HTTP_ENDL, rrdhost_program_name(host), rrdhost_program_version(host)); |
| 324 | buffer_strcat(wb, "Accept: */*" HTTP_HDR_END); |
| 325 | |
| 326 | if (s->parent_using_h2o && stream_connect_upgrade_prelude(host, s)) { |
| 327 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_CANT_UPGRADE_CONNECTION); |
| 328 | nd_sock_close(&s->sock); |
| 329 | stream_parent_set_host_connect_failure_reason(host, STREAM_HANDSHAKE_SND_DISCONNECT_HTTP_UPGRADE_FAILED, 60); |
| 330 | return false; |
| 331 | } |
| 332 | |
| 333 | ssize_t len = (ssize_t)buffer_strlen(wb); |
| 334 | ssize_t bytes = nd_sock_send_timeout(&s->sock, (void *)buffer_tostring(wb), len, 0, timeout); |
| 335 | if(bytes <= 0) { // timeout is 0 |
| 336 | ND_LOG_STACK lgs[] = { |
| 337 | ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT)), |
| 338 | ND_LOG_FIELD_END(), |
| 339 | }; |
| 340 | ND_LOG_STACK_PUSH(lgs); |
| 341 | |
| 342 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_TIMEOUT); |
| 343 | nd_sock_close(&s->sock); |
| 344 | |
| 345 | nd_log(NDLS_DAEMON, NDLP_ERR, |
| 346 | "STREAM CONNECT '%s' [to %s]: failed to send HTTP header to remote netdata.", |
| 347 | rrdhost_hostname(host), s->remote_ip); |
| 348 | |
| 349 | stream_parent_set_host_connect_failure_reason(host, STREAM_HANDSHAKE_CONNECT_SEND_TIMEOUT, 60); |
| 350 | return false; |
| 351 | } |
| 352 | |
| 353 | char response[4096]; |
| 354 | bytes = nd_sock_recv_timeout(&s->sock, response, sizeof(response) - 1, 0, timeout); |
| 355 | if(bytes <= 0) { // timeout is 0 |
| 356 | nd_sock_close(&s->sock); |
| 357 | |
| 358 | ND_LOG_STACK lgs[] = { |
| 359 | ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, stream_handshake_error_to_response_code(STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT)), |
| 360 | ND_LOG_FIELD_END(), |
| 361 | }; |
| 362 | ND_LOG_STACK_PUSH(lgs); |
| 363 | |
| 364 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_TIMEOUT); |
| 365 | |
| 366 | nd_log(NDLS_DAEMON, NDLP_ERR, |
| 367 | "STREAM CONNECT '%s' [to %s]: remote netdata does not respond.", |
| 368 | rrdhost_hostname(host), s->remote_ip); |
| 369 | |
| 370 | stream_parent_set_host_connect_failure_reason(host, STREAM_HANDSHAKE_CONNECT_RECEIVE_TIMEOUT, 30); |
| 371 | return false; |
| 372 | } |
| 373 | response[bytes] = '\0'; |
| 374 | |
| 375 | if(!stream_connect_validate_first_response(host, s, response, bytes)) { |
| 376 | nd_sock_close(&s->sock); |
| 377 | return false; |
| 378 | } |
| 379 | |
| 380 | stream_compression_initialize(s); |
| 381 | |
| 382 | log_sender_capabilities(s); |
| 383 | |
| 384 | ND_LOG_STACK lgs[] = { |
| 385 | ND_LOG_FIELD_I64(NDF_RESPONSE_CODE, HTTP_RESP_OK), |
| 386 | ND_LOG_FIELD_END(), |
| 387 | }; |
| 388 | ND_LOG_STACK_PUSH(lgs); |
| 389 | |
| 390 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 391 | "STREAM CONNECT '%s' [to %s]: connected to parent...", |
| 392 | rrdhost_hostname(host), s->remote_ip); |
| 393 | |
| 394 | return true; |
| 395 | } |
| 396 | |
| 397 | #define MAX_CONNECTORS 1 |
| 398 | |
| 399 | struct connector { |
| 400 | int8_t id; |
| 401 | pid_t tid; |
| 402 | ND_THREAD *thread; |
| 403 | struct completion completion; |
| 404 | |
| 405 | Word_t idx; |
| 406 | |
| 407 | size_t nodes; |
| 408 | |
| 409 | struct { |
| 410 | // the incoming queue of the connector thread |
| 411 | // all other threads leave new senders here, to be connected to their parents |
| 412 | SPINLOCK spinlock; |
| 413 | SENDERS_JudyLSet senders; |
| 414 | } queue; |
| 415 | }; |
| 416 | |
| 417 | static inline Word_t get_unique_idx(struct connector *cn, STRCNT_CMD cmd) { |
| 418 | Word_t t = STRCNT_CMD_MAX - 1; |
| 419 | Word_t reserved_bits = (sizeof(Word_t) * 8) - __builtin_clz(t); |
| 420 | return (__atomic_add_fetch(&cn->idx, 1, __ATOMIC_RELAXED) << reserved_bits) | cmd; |
| 421 | } |
| 422 | |
| 423 | static struct { |
| 424 | int id; |
| 425 | struct connector connectors[MAX_CONNECTORS]; |
| 426 | } connector_globals = { 0 }; |
| 427 | |
| 428 | bool stream_connector_is_signaled_to_stop(struct sender_state *s) { |
| 429 | return __atomic_load_n(&s->exit.shutdown, __ATOMIC_RELAXED); |
| 430 | } |
| 431 | |
| 432 | struct connector *stream_connector_get(struct sender_state *s) { |
| 433 | stream_sender_lock(s); |
| 434 | |
| 435 | if(s->connector.id < 0 || s->connector.id >= MAX_CONNECTORS) { |
| 436 | // assign this to the dispatcher with fewer nodes |
| 437 | |
| 438 | static SPINLOCK spinlock = SPINLOCK_INITIALIZER; |
| 439 | spinlock_lock(&spinlock); |
| 440 | int min_slot = 0; |
| 441 | size_t min_nodes = __atomic_load_n(&connector_globals.connectors[0].nodes, __ATOMIC_RELAXED); |
| 442 | for(int i = 1; i < MAX_CONNECTORS ;i++) { |
| 443 | size_t nodes = __atomic_load_n(&connector_globals.connectors[i].nodes, __ATOMIC_RELAXED); |
| 444 | if(nodes < min_nodes) { |
| 445 | min_nodes = nodes; |
| 446 | min_slot = i; |
| 447 | } |
| 448 | } |
| 449 | __atomic_add_fetch(&connector_globals.connectors[min_slot].nodes, 1, __ATOMIC_RELAXED); |
| 450 | s->connector.id = min_slot; |
| 451 | spinlock_unlock(&spinlock); |
| 452 | } |
| 453 | |
| 454 | struct connector *sc = &connector_globals.connectors[s->connector.id]; |
| 455 | stream_sender_unlock(s); |
| 456 | |
| 457 | return sc; |
| 458 | } |
| 459 | |
| 460 | void stream_connector_requeue(struct sender_state *s, STRCNT_CMD cmd) { |
| 461 | struct connector *sc = stream_connector_get(s); |
| 462 | |
| 463 | switch(cmd) { |
| 464 | case STRCNT_CMD_CONNECT: |
| 465 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 466 | "STREAM CONNECT '%s' [to parent]: adding host in connector queue...", |
| 467 | rrdhost_hostname(s->host)); |
| 468 | |
| 469 | pulse_host_status(s->host, PULSE_HOST_STATUS_SND_PENDING, 0); |
| 470 | break; |
| 471 | |
| 472 | case STRCNT_CMD_REMOVE: |
| 473 | break; |
| 474 | |
| 475 | default: |
| 476 | fatal("STREAM CONNECT '%s': invalid cmd %d", rrdhost_hostname(s->host), cmd); |
| 477 | } |
| 478 | |
| 479 | spinlock_lock(&sc->queue.spinlock); |
| 480 | SENDERS_SET(&sc->queue.senders, get_unique_idx(sc, cmd), s); |
| 481 | spinlock_unlock(&sc->queue.spinlock); |
| 482 | |
| 483 | // signal the connector to catch the job |
| 484 | completion_mark_complete_a_job(&sc->completion); |
| 485 | } |
| 486 | |
| 487 | void stream_connector_add(struct sender_state *s) { |
| 488 | // multiple threads may come here - only one should be able to pass through |
| 489 | stream_sender_lock(s); |
| 490 | if(!rrdhost_has_stream_sender_enabled(s->host) || !s->host->stream.snd.destination || !s->host->stream.snd.api_key) { |
| 491 | nd_log(NDLS_DAEMON, NDLP_ERR, "STREAM CONNECT '%s' [disabled]: host has streaming disabled - not sending data to a parent.", |
| 492 | rrdhost_hostname(s->host)); |
| 493 | stream_sender_unlock(s); |
| 494 | return; |
| 495 | } |
| 496 | if(rrdhost_flag_check(s->host, RRDHOST_FLAG_STREAM_SENDER_ADDED)) { |
| 497 | nd_log(NDLS_DAEMON, NDLP_DEBUG, "STREAM CONNECT '%s' [duplicate]: host has already added to sender - ignoring request.", |
| 498 | rrdhost_hostname(s->host)); |
| 499 | stream_sender_unlock(s); |
| 500 | return; |
| 501 | } |
| 502 | rrdhost_flag_set(s->host, RRDHOST_FLAG_STREAM_SENDER_ADDED); |
| 503 | rrdhost_flag_clear(s->host, RRDHOST_FLAG_STREAM_SENDER_CONNECTED | RRDHOST_FLAG_STREAM_SENDER_READY_4_METRICS); |
| 504 | stream_sender_unlock(s); |
| 505 | |
| 506 | nd_sock_close(&s->sock); |
| 507 | s->parent_using_h2o = stream_send.parents.h2o; |
| 508 | |
| 509 | stream_parents_host_reset(s->host, 0); |
| 510 | |
| 511 | // do not call this with any locks held |
| 512 | stream_connector_requeue(s, STRCNT_CMD_CONNECT); |
| 513 | } |
| 514 | |
| 515 | static void stream_connector_remove(struct sender_state *s) { |
| 516 | struct connector *sc = stream_connector_get(s); |
| 517 | __atomic_sub_fetch(&sc->nodes, 1, __ATOMIC_RELAXED); |
| 518 | |
| 519 | nd_log(NDLS_DAEMON, NDLP_NOTICE, |
| 520 | "STREAM CNT '%s' [to %s]: streaming connector removed host: %s (signaled to stop)", |
| 521 | rrdhost_hostname(s->host), s->remote_ip, stream_handshake_error_to_string(s->exit.reason)); |
| 522 | |
| 523 | STREAM_HANDSHAKE reason = s->exit.reason ? s->exit.reason : STREAM_HANDSHAKE_DISCONNECT_SIGNALED_TO_STOP; |
| 524 | pulse_host_status(s->host, PULSE_HOST_STATUS_SND_OFFLINE, reason); |
| 525 | stream_sender_remove(s, reason); |
| 526 | } |
| 527 | |
| 528 | static void stream_connector_thread(void *ptr) { |
| 529 | struct connector *sc = ptr; |
| 530 | sc->tid = gettid_cached(); |
| 531 | |
| 532 | nd_thread_can_run_sql(false); |
| 533 | |
| 534 | worker_register("STREAMCNT"); |
| 535 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_CONNECTING, "connect"); |
| 536 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_CONNECTED, "connected"); |
| 537 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_REMOVED, "removed"); |
| 538 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_BAD_HANDSHAKE, "bad handshake"); |
| 539 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_TIMEOUT, "timeout"); |
| 540 | worker_register_job_name(WORKER_SENDER_CONNECTOR_JOB_DISCONNECT_CANT_UPGRADE_CONNECTION, "cant upgrade"); |
| 541 | |
| 542 | worker_register_job_custom_metric(WORKER_SENDER_CONNECTOR_JOB_QUEUED_NODES, "queued nodes", "nodes", WORKER_METRIC_ABSOLUTE); |
| 543 | worker_register_job_custom_metric(WORKER_SENDER_CONNECTOR_JOB_CONNECTED_NODES, "connected nodes", "nodes", WORKER_METRIC_ABSOLUTE); |
| 544 | worker_register_job_custom_metric(WORKER_SENDER_CONNECTOR_JOB_FAILED_NODES, "failed nodes", "nodes", WORKER_METRIC_ABSOLUTE); |
| 545 | worker_register_job_custom_metric(WORKER_SENDER_CONNECTOR_JOB_CANCELLED_NODES, "cancelled nodes", "nodes", WORKER_METRIC_ABSOLUTE); |
| 546 | |
| 547 | unsigned job_id = 0; |
| 548 | size_t exiting = 0; |
| 549 | while(exiting <= 5) { |
| 550 | worker_is_idle(); |
| 551 | job_id = completion_wait_for_a_job_with_timeout(&sc->completion, job_id, exiting ? 250 : 1000); |
| 552 | size_t nodes = 0, connected_nodes = 0, failed_nodes = 0, cancelled_nodes = 0; |
| 553 | |
| 554 | if(!service_running(SERVICE_STREAMING_CONNECTOR)) |
| 555 | exiting++; |
| 556 | |
| 557 | spinlock_lock(&sc->queue.spinlock); |
| 558 | Word_t idx = 0; |
| 559 | for(struct sender_state *s = SENDERS_FIRST(&sc->queue.senders, &idx); |
| 560 | s; |
| 561 | s = SENDERS_NEXT(&sc->queue.senders, &idx)) { |
| 562 | nodes++; |
| 563 | |
| 564 | ND_LOG_STACK lgs[] = { |
| 565 | ND_LOG_FIELD_STR(NDF_NIDL_NODE, s->host->hostname), |
| 566 | ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_to_parent_msgid), |
| 567 | ND_LOG_FIELD_END(), |
| 568 | }; |
| 569 | ND_LOG_STACK_PUSH(lgs); |
| 570 | |
| 571 | if(stream_connector_is_signaled_to_stop(s)) { |
| 572 | cancelled_nodes++; |
| 573 | SENDERS_DEL(&sc->queue.senders, idx); |
| 574 | spinlock_unlock(&sc->queue.spinlock); |
| 575 | |
| 576 | // do not have the connector lock when calling these |
| 577 | stream_sender_on_disconnect(s); |
| 578 | stream_connector_remove(s); |
| 579 | |
| 580 | spinlock_lock(&sc->queue.spinlock); |
| 581 | continue; |
| 582 | } |
| 583 | |
| 584 | STRCNT_CMD cmd = idx & (STRCNT_CMD_CONNECT| STRCNT_CMD_REMOVE); |
| 585 | if(unlikely(exiting)) |
| 586 | cmd = STRCNT_CMD_REMOVE; |
| 587 | |
| 588 | switch(cmd) { |
| 589 | case STRCNT_CMD_CONNECT: |
| 590 | spinlock_unlock(&sc->queue.spinlock); |
| 591 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_CONNECTING); |
| 592 | |
| 593 | // do not have the connector lock when calling these |
| 594 | bool move_to_sender = |
| 595 | stream_connect(s, stream_send.parents.default_port, stream_send.parents.timeout_s); |
| 596 | |
| 597 | spinlock_lock(&sc->queue.spinlock); |
| 598 | |
| 599 | if (move_to_sender) { |
| 600 | connected_nodes++; |
| 601 | |
| 602 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_CONNECTED); |
| 603 | SENDERS_DEL(&sc->queue.senders, idx); |
| 604 | spinlock_unlock(&sc->queue.spinlock); |
| 605 | |
| 606 | // do not have the connector lock when calling these |
| 607 | stream_sender_on_connect(s); |
| 608 | stream_sender_add_to_queue(s); |
| 609 | |
| 610 | spinlock_lock(&sc->queue.spinlock); |
| 611 | } |
| 612 | else |
| 613 | failed_nodes++; |
| 614 | |
| 615 | break; |
| 616 | |
| 617 | case STRCNT_CMD_REMOVE: |
| 618 | worker_is_busy(WORKER_SENDER_CONNECTOR_JOB_REMOVED); |
| 619 | SENDERS_DEL(&sc->queue.senders, idx); |
| 620 | spinlock_unlock(&sc->queue.spinlock); |
| 621 | |
| 622 | // do not have the connector lock when calling these |
| 623 | stream_sender_on_disconnect(s); |
| 624 | stream_sender_remove(s, s->exit.reason); |
| 625 | |
| 626 | spinlock_lock(&sc->queue.spinlock); |
| 627 | break; |
| 628 | |
| 629 | default: |
| 630 | fatal("STREAM CONNECT '%s': invalid cmd %d", rrdhost_hostname(s->host), cmd); |
| 631 | } |
| 632 | |
| 633 | worker_is_idle(); |
| 634 | } |
| 635 | spinlock_unlock(&sc->queue.spinlock); |
| 636 | |
| 637 | worker_set_metric(WORKER_SENDER_CONNECTOR_JOB_QUEUED_NODES, (NETDATA_DOUBLE)nodes); |
| 638 | worker_set_metric(WORKER_SENDER_CONNECTOR_JOB_CONNECTED_NODES, (NETDATA_DOUBLE)connected_nodes); |
| 639 | worker_set_metric(WORKER_SENDER_CONNECTOR_JOB_FAILED_NODES, (NETDATA_DOUBLE)failed_nodes); |
| 640 | worker_set_metric(WORKER_SENDER_CONNECTOR_JOB_CANCELLED_NODES, (NETDATA_DOUBLE)cancelled_nodes); |
| 641 | } |
| 642 | } |
| 643 | |
| 644 | void stream_connector_remove_host(RRDHOST *host) { |
| 645 | if(!host || !host->sender) return; |
| 646 | |
| 647 | struct connector *sc = stream_connector_get(host->sender); |
| 648 | |
| 649 | spinlock_lock(&sc->queue.spinlock); |
| 650 | Word_t idx = 0; |
| 651 | for(struct sender_state *s = SENDERS_FIRST(&sc->queue.senders, &idx); |
| 652 | s; |
| 653 | s = SENDERS_NEXT(&sc->queue.senders, &idx)) { |
| 654 | |
| 655 | if(s != host->sender) |
| 656 | continue; |
| 657 | |
| 658 | ND_LOG_STACK lgs[] = { |
| 659 | ND_LOG_FIELD_STR(NDF_NIDL_NODE, s->host->hostname), |
| 660 | ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &streaming_to_parent_msgid), |
| 661 | ND_LOG_FIELD_END(), |
| 662 | }; |
| 663 | ND_LOG_STACK_PUSH(lgs); |
| 664 | |
| 665 | SENDERS_DEL(&sc->queue.senders, idx); |
| 666 | spinlock_unlock(&sc->queue.spinlock); |
| 667 | |
| 668 | // do not have the connector lock when calling these |
| 669 | stream_sender_on_disconnect(s); |
| 670 | stream_sender_remove(s, s->exit.reason); |
| 671 | |
| 672 | spinlock_lock(&sc->queue.spinlock); |
| 673 | break; |
| 674 | } |
| 675 | |
| 676 | spinlock_unlock(&sc->queue.spinlock); |
| 677 | } |
| 678 | |
| 679 | bool stream_connector_init(struct sender_state *s) { |
| 680 | static SPINLOCK spinlock = SPINLOCK_INITIALIZER; |
| 681 | if(!s) return false; |
| 682 | |
| 683 | spinlock_lock(&spinlock); |
| 684 | |
| 685 | struct connector *sc = stream_connector_get(s); |
| 686 | |
| 687 | if(!sc->thread) { |
| 688 | sc->id = (int8_t)(sc - connector_globals.connectors); // find the slot number |
| 689 | if(&connector_globals.connectors[sc->id] != sc) |
| 690 | fatal("STREAM CONNECT '%s': connector ID and slot do not match!", rrdhost_hostname(s->host)); |
| 691 | |
| 692 | spinlock_init(&sc->queue.spinlock); |
| 693 | completion_init(&sc->completion); |
| 694 | |
| 695 | char tag[NETDATA_THREAD_TAG_MAX + 1]; |
| 696 | snprintfz(tag, NETDATA_THREAD_TAG_MAX, THREAD_TAG_STREAM_SENDER "-CN" "[%d]", |
| 697 | sc->id); |
| 698 | |
| 699 | sc->thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, stream_connector_thread, sc); |
| 700 | if (!sc->thread) |
| 701 | nd_log_daemon(NDLP_ERR, |
| 702 | "STREAM CONNECT '%s': failed to create new thread for client.", |
| 703 | rrdhost_hostname(s->host)); |
| 704 | } |
| 705 | |
| 706 | spinlock_unlock(&spinlock); |
| 707 | |
| 708 | return sc->thread != NULL; |
| 709 | } |
| 710 | |
| 711 | void stream_connector_cancel_threads(void) { |
| 712 | for(int id = 0; id < MAX_CONNECTORS ; id++) |
| 713 | nd_thread_signal_cancel(connector_globals.connectors[id].thread); |
| 714 | } |