| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "rrdhost-status.h" |
| 4 | #include "streaming/stream-receiver-internals.h" |
| 5 | #include "streaming/stream-sender-internals.h" |
| 6 | |
| 7 | ENUM_STR_MAP_DEFINE(RRDHOST_DB_STATUS) = { |
| 8 | { RRDHOST_DB_STATUS_INITIALIZING, "initializing" }, |
| 9 | { RRDHOST_DB_STATUS_QUERYABLE, "online" }, |
| 10 | { 0, NULL } // Sentinel |
| 11 | }; |
| 12 | |
| 13 | ENUM_STR_MAP_DEFINE(RRDHOST_DB_LIVENESS) = { |
| 14 | { RRDHOST_DB_LIVENESS_STALE, "stale" }, |
| 15 | { RRDHOST_DB_LIVENESS_LIVE, "live" }, |
| 16 | { 0, NULL } // Sentinel |
| 17 | }; |
| 18 | |
| 19 | ENUM_STR_MAP_DEFINE(RRDHOST_INGEST_STATUS) = { |
| 20 | { RRDHOST_INGEST_STATUS_ARCHIVED, "archived" }, |
| 21 | { RRDHOST_INGEST_STATUS_INITIALIZING, "initializing" }, |
| 22 | { RRDHOST_INGEST_STATUS_REPLICATING, "replicating" }, |
| 23 | { RRDHOST_INGEST_STATUS_ONLINE, "online" }, |
| 24 | { RRDHOST_INGEST_STATUS_OFFLINE, "offline" }, |
| 25 | { 0, NULL } // Sentinel |
| 26 | }; |
| 27 | |
| 28 | ENUM_STR_MAP_DEFINE(RRDHOST_INGEST_TYPE) = { |
| 29 | { RRDHOST_INGEST_TYPE_LOCALHOST, "localhost" }, |
| 30 | { RRDHOST_INGEST_TYPE_VIRTUAL, "virtual" }, |
| 31 | { RRDHOST_INGEST_TYPE_CHILD, "child" }, |
| 32 | { RRDHOST_INGEST_TYPE_ARCHIVED, "archived" }, |
| 33 | { 0, NULL } // Sentinel |
| 34 | }; |
| 35 | |
| 36 | ENUM_STR_MAP_DEFINE(RRDHOST_STREAMING_STATUS) = { |
| 37 | { RRDHOST_STREAM_STATUS_DISABLED, "disabled" }, |
| 38 | { RRDHOST_STREAM_STATUS_REPLICATING, "replicating" }, |
| 39 | { RRDHOST_STREAM_STATUS_ONLINE, "online" }, |
| 40 | { RRDHOST_STREAM_STATUS_OFFLINE, "offline" }, |
| 41 | { 0, NULL } // Sentinel |
| 42 | }; |
| 43 | |
| 44 | ENUM_STR_MAP_DEFINE(RRDHOST_ML_STATUS) = { |
| 45 | { RRDHOST_ML_STATUS_DISABLED, "disabled" }, |
| 46 | { RRDHOST_ML_STATUS_OFFLINE, "offline" }, |
| 47 | { RRDHOST_ML_STATUS_RUNNING, "online" }, |
| 48 | { 0, NULL } // Sentinel |
| 49 | }; |
| 50 | |
| 51 | ENUM_STR_MAP_DEFINE(RRDHOST_ML_TYPE) = { |
| 52 | { RRDHOST_ML_TYPE_DISABLED, "disabled" }, |
| 53 | { RRDHOST_ML_TYPE_SELF, "self" }, |
| 54 | { RRDHOST_ML_TYPE_RECEIVED, "received" }, |
| 55 | { 0, NULL } // Sentinel |
| 56 | }; |
| 57 | |
| 58 | ENUM_STR_MAP_DEFINE(RRDHOST_HEALTH_STATUS) = { |
| 59 | { RRDHOST_HEALTH_STATUS_DISABLED, "disabled" }, |
| 60 | { RRDHOST_HEALTH_STATUS_INITIALIZING, "initializing" }, |
| 61 | { RRDHOST_HEALTH_STATUS_RUNNING, "online" }, |
| 62 | { 0, NULL } // Sentinel |
| 63 | }; |
| 64 | |
| 65 | ENUM_STR_MAP_DEFINE(RRDHOST_DYNCFG_STATUS) = { |
| 66 | { RRDHOST_DYNCFG_STATUS_UNAVAILABLE, "unavailable" }, |
| 67 | { RRDHOST_DYNCFG_STATUS_AVAILABLE, "online" }, |
| 68 | { 0, NULL } // Sentinel |
| 69 | }; |
| 70 | |
| 71 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_DB_STATUS, RRDHOST_DB_STATUS_INITIALIZING, "initializing"); |
| 72 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_DB_LIVENESS, RRDHOST_DB_LIVENESS_STALE, "stale"); |
| 73 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_INGEST_STATUS, RRDHOST_INGEST_STATUS_OFFLINE, "offline"); |
| 74 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_INGEST_TYPE, RRDHOST_INGEST_TYPE_ARCHIVED, "archived"); |
| 75 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_STREAMING_STATUS, RRDHOST_STREAM_STATUS_OFFLINE, "offline"); |
| 76 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_ML_STATUS, RRDHOST_ML_STATUS_DISABLED, "disabled"); |
| 77 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_ML_TYPE, RRDHOST_ML_TYPE_DISABLED, "disabled"); |
| 78 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_HEALTH_STATUS, RRDHOST_HEALTH_STATUS_DISABLED, "disabled"); |
| 79 | ENUM_STR_DEFINE_FUNCTIONS(RRDHOST_DYNCFG_STATUS, RRDHOST_DYNCFG_STATUS_UNAVAILABLE, "unavailable"); |
| 80 | |
| 81 | static NETDATA_DOUBLE rrdhost_sender_replication_completion_unsafe(RRDHOST *host, time_t now, size_t *instances) { |
| 82 | size_t charts = rrdhost_sender_replicating_charts(host); |
| 83 | NETDATA_DOUBLE completion; |
| 84 | if(!charts || !host->sender || !host->sender->replication.oldest_request_after_t) |
| 85 | completion = 100.0; |
| 86 | else if(!host->sender->replication.latest_completed_before_t || host->sender->replication.latest_completed_before_t < host->sender->replication.oldest_request_after_t) |
| 87 | completion = 0.0; |
| 88 | else { |
| 89 | time_t total = now - host->sender->replication.oldest_request_after_t; |
| 90 | time_t current = host->sender->replication.latest_completed_before_t - host->sender->replication.oldest_request_after_t; |
| 91 | completion = (NETDATA_DOUBLE) current * 100.0 / (NETDATA_DOUBLE) total; |
| 92 | } |
| 93 | |
| 94 | *instances = charts; |
| 95 | |
| 96 | return completion; |
| 97 | } |
| 98 | |
| 99 | RRDHOST_INGEST_STATUS rrdhost_ingestion_status(RRDHOST *host) { |
| 100 | return rrdhost_get_ingest_status(host, now_realtime_sec()); |
| 101 | } |
| 102 | |
| 103 | int16_t rrdhost_ingestion_hops(RRDHOST *host) { |
| 104 | if(host == localhost) return 0; |
| 105 | if(rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST) || !host->system_info) return 1; |
| 106 | return rrdhost_system_info_hops(host->system_info); |
| 107 | } |
| 108 | |
| 109 | static inline RRDHOST_DB_STATUS rrdhost_status_db(RRDHOST *host, time_t now, RRDHOST_STATUS *s, RRDHOST_FLAGS flags, bool online) { |
| 110 | RRDHOST_DB_STATUS status; |
| 111 | |
| 112 | uint32_t metrics = UINT32_MAX; |
| 113 | uint32_t instances = UINT32_MAX; |
| 114 | uint32_t contexts = UINT32_MAX; |
| 115 | |
| 116 | time_t first_time_s = 0, last_time_s = 0; |
| 117 | rrdhost_retention(host, now, online, &first_time_s, &last_time_s); |
| 118 | |
| 119 | if (!first_time_s || |
| 120 | !last_time_s || |
| 121 | (flags & RRDHOST_FLAG_PENDING_CONTEXT_LOAD) || |
| 122 | !(metrics = __atomic_load_n(&host->rrdctx.metrics_count, __ATOMIC_RELAXED)) || |
| 123 | !(instances = __atomic_load_n(&host->rrdctx.instances_count, __ATOMIC_RELAXED)) || |
| 124 | !(contexts = __atomic_load_n(&host->rrdctx.contexts_count, __ATOMIC_RELAXED))) |
| 125 | status = RRDHOST_DB_STATUS_INITIALIZING; |
| 126 | else |
| 127 | status = RRDHOST_DB_STATUS_QUERYABLE; |
| 128 | |
| 129 | |
| 130 | if(s) { |
| 131 | s->db.status = status; |
| 132 | |
| 133 | s->db.first_time_s = first_time_s; |
| 134 | s->db.last_time_s = last_time_s; |
| 135 | s->db.status = status; |
| 136 | s->db.mode = host->rrd_memory_mode; |
| 137 | |
| 138 | s->db.metrics = (metrics == UINT32_MAX) ? __atomic_load_n(&host->rrdctx.metrics_count, __ATOMIC_RELAXED) : metrics; |
| 139 | s->db.instances = (instances == UINT32_MAX) ? __atomic_load_n(&host->rrdctx.instances_count, __ATOMIC_RELAXED) : instances; |
| 140 | s->db.contexts = (contexts == UINT32_MAX) ? __atomic_load_n(&host->rrdctx.contexts_count, __ATOMIC_RELAXED) : contexts; |
| 141 | } |
| 142 | |
| 143 | return status; |
| 144 | } |
| 145 | |
| 146 | static inline RRDHOST_INGEST_STATUS rrdhost_status_ingest(RRDHOST *host, RRDHOST_STATUS *s, RRDHOST_FLAGS flags, RRDHOST_DB_STATUS db_status, bool online) { |
| 147 | RRDHOST_INGEST_STATUS status; |
| 148 | |
| 149 | uint32_t collected_metrics = UINT32_MAX; |
| 150 | uint32_t replicating_instances = UINT32_MAX; |
| 151 | |
| 152 | time_t last_connected; |
| 153 | time_t last_disconnected; |
| 154 | uint32_t connections; |
| 155 | STREAM_HANDSHAKE reason; |
| 156 | |
| 157 | rrdhost_receiver_lock(host); |
| 158 | last_connected = host->stream.rcv.status.last_connected; |
| 159 | last_disconnected = host->stream.rcv.status.last_disconnected; |
| 160 | connections = host->stream.rcv.status.connections; |
| 161 | reason = host->stream.rcv.status.reason; |
| 162 | rrdhost_receiver_unlock(host); |
| 163 | |
| 164 | time_t since = MAX(last_connected, last_disconnected); |
| 165 | |
| 166 | if (online) { |
| 167 | if (db_status == RRDHOST_DB_STATUS_INITIALIZING) |
| 168 | status = RRDHOST_INGEST_STATUS_INITIALIZING; |
| 169 | |
| 170 | else if (rrdhost_is_local(host)) { |
| 171 | status = RRDHOST_INGEST_STATUS_ONLINE; |
| 172 | since = netdata_start_time; |
| 173 | } |
| 174 | else if ( |
| 175 | (replicating_instances = rrdhost_receiver_replicating_charts(host)) > 0 || |
| 176 | !(collected_metrics = __atomic_load_n(&host->collected.metrics_count, __ATOMIC_RELAXED))) |
| 177 | status = RRDHOST_INGEST_STATUS_REPLICATING; |
| 178 | |
| 179 | else |
| 180 | status = RRDHOST_INGEST_STATUS_ONLINE; |
| 181 | } |
| 182 | else { |
| 183 | if(!connections) |
| 184 | status = RRDHOST_INGEST_STATUS_ARCHIVED; |
| 185 | else |
| 186 | status = RRDHOST_INGEST_STATUS_OFFLINE; |
| 187 | } |
| 188 | |
| 189 | bool has_receiver = false; |
| 190 | |
| 191 | if(s) { |
| 192 | if(status == RRDHOST_INGEST_STATUS_ARCHIVED) |
| 193 | since = s->db.last_time_s; |
| 194 | |
| 195 | s->ingest.status = status; |
| 196 | |
| 197 | s->ingest.since = since ? since : netdata_start_time; |
| 198 | s->ingest.reason = reason; |
| 199 | s->ingest.hops = rrdhost_ingestion_hops(host); |
| 200 | |
| 201 | s->ingest.collected.metrics = collected_metrics == UINT32_MAX ? __atomic_load_n(&host->collected.metrics_count, __ATOMIC_RELAXED) : collected_metrics; |
| 202 | s->ingest.collected.instances = __atomic_load_n(&host->collected.instances_count, __ATOMIC_RELAXED); |
| 203 | s->ingest.collected.contexts = __atomic_load_n(&host->collected.contexts_count, __ATOMIC_RELAXED); |
| 204 | |
| 205 | if(!rrdhost_is_local(host)) { |
| 206 | rrdhost_receiver_lock(host); |
| 207 | if (host->receiver && (flags & RRDHOST_FLAG_COLLECTOR_ONLINE)) { |
| 208 | has_receiver = true; |
| 209 | s->ingest.replication.instances = replicating_instances == UINT32_MAX ? rrdhost_receiver_replicating_charts(host) : replicating_instances; |
| 210 | s->ingest.replication.completion = host->stream.rcv.status.replication.percent; |
| 211 | s->ingest.replication.in_progress = s->ingest.replication.instances > 0; |
| 212 | |
| 213 | s->ingest.capabilities = host->receiver->capabilities; |
| 214 | s->ingest.peers = nd_sock_socket_peers(&host->receiver->sock); |
| 215 | s->ingest.ssl = nd_sock_is_ssl(&host->receiver->sock); |
| 216 | } |
| 217 | rrdhost_receiver_unlock(host); |
| 218 | } |
| 219 | |
| 220 | if(host == localhost) |
| 221 | s->ingest.type = RRDHOST_INGEST_TYPE_LOCALHOST; |
| 222 | else if(has_receiver) |
| 223 | s->ingest.type = RRDHOST_INGEST_TYPE_CHILD; |
| 224 | else if(rrdhost_option_check(host, RRDHOST_OPTION_VIRTUAL_HOST)) |
| 225 | s->ingest.type = RRDHOST_INGEST_TYPE_VIRTUAL; |
| 226 | else |
| 227 | s->ingest.type = RRDHOST_INGEST_TYPE_ARCHIVED; |
| 228 | |
| 229 | s->ingest.id = connections; |
| 230 | } |
| 231 | |
| 232 | return status; |
| 233 | } |
| 234 | |
| 235 | static void rrdhost_status_stream_internal(RRDHOST_STATUS *s) { |
| 236 | RRDHOST *host = s->host; |
| 237 | time_t now = s->now; |
| 238 | |
| 239 | if (!host->sender) { |
| 240 | s->stream.status = RRDHOST_STREAM_STATUS_DISABLED; |
| 241 | s->stream.hops = (int16_t)(s->ingest.hops + 1); |
| 242 | } |
| 243 | else { |
| 244 | stream_sender_lock(host->sender); |
| 245 | |
| 246 | s->stream.since = host->sender->last_state_since_t; |
| 247 | s->stream.peers = nd_sock_socket_peers(&host->sender->sock); |
| 248 | s->stream.ssl = nd_sock_is_ssl(&host->sender->sock); |
| 249 | |
| 250 | { |
| 251 | STREAM_CIRCULAR_BUFFER_STATS *stats = stream_circular_buffer_stats_unsafe(host->sender->scb); |
| 252 | |
| 253 | memcpy( |
| 254 | s->stream.sent_bytes_on_this_connection_per_type, |
| 255 | stats->bytes_sent_by_type, |
| 256 | MIN(sizeof(s->stream.sent_bytes_on_this_connection_per_type), sizeof(stats->bytes_sent_by_type))); |
| 257 | } |
| 258 | |
| 259 | if (rrdhost_flag_check(host, RRDHOST_FLAG_STREAM_SENDER_CONNECTED)) { |
| 260 | s->stream.hops = host->sender->hops; |
| 261 | s->stream.capabilities = host->sender->capabilities; |
| 262 | |
| 263 | s->stream.replication.completion = rrdhost_sender_replication_completion_unsafe(host, now, &s->stream.replication.instances); |
| 264 | s->stream.replication.in_progress = s->stream.replication.instances > 0; |
| 265 | |
| 266 | if(s->stream.replication.in_progress) |
| 267 | s->stream.status = RRDHOST_STREAM_STATUS_REPLICATING; |
| 268 | else |
| 269 | s->stream.status = RRDHOST_STREAM_STATUS_ONLINE; |
| 270 | |
| 271 | s->stream.compression = host->sender->thread.compressor.initialized; |
| 272 | } |
| 273 | else { |
| 274 | s->stream.status = RRDHOST_STREAM_STATUS_OFFLINE; |
| 275 | s->stream.hops = (int16_t)(s->ingest.hops + 1); |
| 276 | } |
| 277 | s->stream.reason = host->stream.snd.status.reason; |
| 278 | |
| 279 | stream_sender_unlock(host->sender); |
| 280 | } |
| 281 | |
| 282 | s->stream.id = host->stream.snd.status.connections; |
| 283 | |
| 284 | if(!s->stream.since) |
| 285 | s->stream.since = netdata_start_time; |
| 286 | } |
| 287 | |
| 288 | static void rrdhost_status_ml_internal(RRDHOST_STATUS *s) { |
| 289 | RRDHOST *host = s->host; |
| 290 | |
| 291 | if(ml_host_get_host_status(host, &s->ml.metrics)) { |
| 292 | if(stream_has_capability(&s->ingest, STREAM_CAP_ML_MODELS)) |
| 293 | s->ml.type = RRDHOST_ML_TYPE_RECEIVED; |
| 294 | else |
| 295 | s->ml.type = RRDHOST_ML_TYPE_SELF; |
| 296 | |
| 297 | if(s->ingest.status == RRDHOST_INGEST_STATUS_OFFLINE || s->ingest.status == RRDHOST_INGEST_STATUS_ARCHIVED) |
| 298 | s->ml.status = RRDHOST_ML_STATUS_OFFLINE; |
| 299 | else |
| 300 | s->ml.status = RRDHOST_ML_STATUS_RUNNING; |
| 301 | } |
| 302 | else { |
| 303 | // does not receive ML, does not run ML |
| 304 | s->ml.type = RRDHOST_ML_TYPE_DISABLED; |
| 305 | s->ml.status = RRDHOST_ML_STATUS_DISABLED; |
| 306 | } |
| 307 | } |
| 308 | |
| 309 | static void rrdhost_status_health_internal(RRDHOST_STATUS *s, RRDHOST_FLAGS flags) { |
| 310 | RRDHOST *host = s->host; |
| 311 | |
| 312 | if(host->health.enabled) { |
| 313 | if(flags & RRDHOST_FLAG_PENDING_HEALTH_INITIALIZATION) |
| 314 | s->health.status = RRDHOST_HEALTH_STATUS_INITIALIZING; |
| 315 | else |
| 316 | s->health.status = RRDHOST_HEALTH_STATUS_RUNNING; |
| 317 | |
| 318 | RRDCALC *rc; |
| 319 | foreach_rrdcalc_in_rrdhost_read(host, rc) { |
| 320 | if (unlikely(!rc->rrdset || !rc->rrdset->last_collected_time.tv_sec)) |
| 321 | continue; |
| 322 | |
| 323 | switch (rc->status) { |
| 324 | default: |
| 325 | case RRDCALC_STATUS_REMOVED: |
| 326 | break; |
| 327 | |
| 328 | case RRDCALC_STATUS_CLEAR: |
| 329 | s->health.alerts.clear++; |
| 330 | break; |
| 331 | |
| 332 | case RRDCALC_STATUS_WARNING: |
| 333 | s->health.alerts.warning++; |
| 334 | break; |
| 335 | |
| 336 | case RRDCALC_STATUS_CRITICAL: |
| 337 | s->health.alerts.critical++; |
| 338 | break; |
| 339 | |
| 340 | case RRDCALC_STATUS_UNDEFINED: |
| 341 | s->health.alerts.undefined++; |
| 342 | break; |
| 343 | |
| 344 | case RRDCALC_STATUS_UNINITIALIZED: |
| 345 | s->health.alerts.uninitialized++; |
| 346 | break; |
| 347 | } |
| 348 | } |
| 349 | foreach_rrdcalc_in_rrdhost_done(rc); |
| 350 | } |
| 351 | else |
| 352 | s->health.status = RRDHOST_HEALTH_STATUS_DISABLED; |
| 353 | } |
| 354 | |
| 355 | void rrdhost_status(RRDHOST *host, time_t now, RRDHOST_STATUS *s, RRDHOST_STATUS_INFO info) { |
| 356 | memset(s, 0, sizeof(*s)); |
| 357 | |
| 358 | s->host = host; |
| 359 | s->now = now; |
| 360 | |
| 361 | RRDHOST_FLAGS flags = __atomic_load_n(&host->flags, __ATOMIC_RELAXED); |
| 362 | bool online = rrdhost_is_local(host) || rrdhost_is_online_flags(flags); |
| 363 | |
| 364 | // --- db --- |
| 365 | |
| 366 | rrdhost_status_db(host, now, s, flags, online); |
| 367 | |
| 368 | // --- ingest --- |
| 369 | |
| 370 | rrdhost_status_ingest(host, s, flags, s->db.status, online); |
| 371 | |
| 372 | // --- db (part 2) --- |
| 373 | |
| 374 | if(s->ingest.status == RRDHOST_INGEST_STATUS_ONLINE) |
| 375 | s->db.liveness = RRDHOST_DB_LIVENESS_LIVE; |
| 376 | else |
| 377 | s->db.liveness = RRDHOST_DB_LIVENESS_STALE; |
| 378 | |
| 379 | // --- stream --- |
| 380 | |
| 381 | if(info & (RRDHOST_STATUS_STREAM | RRDHOST_STATUS_ML)) |
| 382 | rrdhost_status_stream_internal(s); |
| 383 | |
| 384 | // --- ml --- |
| 385 | |
| 386 | if(info & RRDHOST_STATUS_ML) |
| 387 | rrdhost_status_ml_internal(s); |
| 388 | |
| 389 | // --- dyncfg --- |
| 390 | |
| 391 | if(info & RRDHOST_STATUS_DYNCFG) |
| 392 | s->dyncfg.status = dyncfg_available_for_rrdhost(host) ? RRDHOST_DYNCFG_STATUS_AVAILABLE : RRDHOST_DYNCFG_STATUS_UNAVAILABLE; |
| 393 | |
| 394 | // --- health --- |
| 395 | |
| 396 | if(info & RRDHOST_STATUS_HEALTH) |
| 397 | rrdhost_status_health_internal(s, flags); |
| 398 | |
| 399 | } |
| 400 | |
| 401 | // Minimal function to get the ingest status only |
| 402 | RRDHOST_INGEST_STATUS rrdhost_get_ingest_status(RRDHOST *host, time_t now) { |
| 403 | RRDHOST_FLAGS flags = __atomic_load_n(&host->flags, __ATOMIC_RELAXED); |
| 404 | bool online = rrdhost_is_local(host) || rrdhost_is_online_flags(flags); |
| 405 | |
| 406 | RRDHOST_DB_STATUS db_status = rrdhost_status_db(host, now, NULL, flags, online); |
| 407 | return rrdhost_status_ingest(host, NULL, flags, db_status, online); |
| 408 | } |