| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "common.h" |
| 4 | |
| 5 | static bool svc_rrddim_obsolete_to_archive(RRDDIM *rd) { |
| 6 | RRDSET *st = rd->rrdset; |
| 7 | |
| 8 | if(rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE) && spinlock_trylock(&rd->destroy_lock)) { |
| 9 | if(!rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)) { |
| 10 | spinlock_unlock(&rd->destroy_lock); |
| 11 | return false; |
| 12 | } |
| 13 | } |
| 14 | else |
| 15 | return false; |
| 16 | |
| 17 | worker_is_busy(UV_EVENT_ARCHIVE_DIMENSION); |
| 18 | |
| 19 | if (rd->rrd_memory_mode == RRD_DB_MODE_DBENGINE) { |
| 20 | if (!rrddim_finalize_collection_and_check_retention(rd)) { |
| 21 | /* This metric has no data and no references */ |
| 22 | metaqueue_delete_dimension_uuid(uuidmap_uuid_ptr(rd->uuid)); |
| 23 | } |
| 24 | } |
| 25 | |
| 26 | worker_is_busy(UV_EVENT_FREE_DIMENSION); |
| 27 | rrddim_free(st, rd); |
| 28 | return true; |
| 29 | } |
| 30 | |
| 31 | // Returns the number of dimensions actually archived this call. |
| 32 | // |
| 33 | // Two callable shapes: |
| 34 | // 1. all_dimensions == false and RRDSET_FLAG_OBSOLETE_DIMENSIONS unset: |
| 35 | // early-return path. Nothing scanned, flag not touched, returns 0. |
| 36 | // 2. Any other case: scans the dimensions. The flag is cleared up |
| 37 | // front, then re-set at the end iff some candidate could not be |
| 38 | // archived this pass. Callers on this path can detect "all |
| 39 | // candidates archived" by reading the flag after the call. |
| 40 | static inline size_t svc_rrdset_archive_obsolete_dimensions(RRDSET *st, bool all_dimensions) { |
| 41 | if(!all_dimensions && !rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) |
| 42 | return 0; |
| 43 | |
| 44 | worker_is_busy(UV_EVENT_ARCHIVE_CHART_DIMENSIONS); |
| 45 | |
| 46 | rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS); |
| 47 | |
| 48 | RRDDIM *rd; |
| 49 | time_t now = now_realtime_sec(); |
| 50 | |
| 51 | size_t dim_candidates = 0; |
| 52 | size_t dim_archives = 0; |
| 53 | |
| 54 | dfe_start_write(st->rrddim_root_index, rd) { |
| 55 | bool candidate = (all_dimensions || rrddim_flag_check(rd, RRDDIM_FLAG_OBSOLETE)); |
| 56 | |
| 57 | if(candidate) { |
| 58 | dim_candidates++; |
| 59 | |
| 60 | if(rd->collector.last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now) { |
| 61 | size_t references = dictionary_acquired_item_references(rd_dfe.item); |
| 62 | if(references == 1) { |
| 63 | if(svc_rrddim_obsolete_to_archive(rd)) |
| 64 | dim_archives++; |
| 65 | } |
| 66 | } |
| 67 | } |
| 68 | } |
| 69 | dfe_done(rd); |
| 70 | |
| 71 | if(dim_archives != dim_candidates) |
| 72 | rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS); |
| 73 | |
| 74 | return dim_archives; |
| 75 | } |
| 76 | |
| 77 | static bool svc_rrdset_lock_for_deletion(RRDSET *st, time_t now) { |
| 78 | if(st->last_accessed_time_s + rrdset_free_obsolete_time_s < now && |
| 79 | st->last_updated.tv_sec + rrdset_free_obsolete_time_s < now && |
| 80 | st->last_collected_time.tv_sec + rrdset_free_obsolete_time_s < now && |
| 81 | spinlock_trylock(&st->destroy_lock)) { |
| 82 | |
| 83 | if(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)) |
| 84 | return true; |
| 85 | |
| 86 | spinlock_unlock(&st->destroy_lock); |
| 87 | } |
| 88 | |
| 89 | return false; |
| 90 | } |
| 91 | |
| 92 | static inline size_t svc_rrdhost_cleanup_charts_marked_obsolete(RRDHOST *host) { |
| 93 | if(!rrdhost_flag_check(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS)) |
| 94 | return 0; |
| 95 | |
| 96 | worker_is_busy(UV_EVENT_CLEANUP_OBSOLETE_CHARTS); |
| 97 | |
| 98 | rrdhost_flag_clear(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS|RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS); |
| 99 | |
| 100 | size_t full_candidates = 0; |
| 101 | size_t full_archives = 0; |
| 102 | size_t partial_candidates = 0; |
| 103 | size_t partial_archives = 0; |
| 104 | // Total archived metadata items (RRDMETRIC + RRDINSTANCE). Used by the |
| 105 | // caller to decide whether to schedule a deep rrdcontext GC pass. |
| 106 | // Counts every dimension archived (each produces an archived RRDMETRIC) |
| 107 | // plus every chart freed (each produces an archived RRDINSTANCE). |
| 108 | size_t archived_items = 0; |
| 109 | |
| 110 | time_t now = now_realtime_sec(); |
| 111 | RRDSET *st; |
| 112 | rrdset_foreach_reentrant(st, host) { |
| 113 | bool is_replicating = rrdset_is_replicating(st); |
| 114 | RRDSET_FLAGS flags = rrdset_flag_get(st); |
| 115 | |
| 116 | // A replicating chart with pending obsolete work must still be |
| 117 | // counted as a candidate, even though we cannot archive it this |
| 118 | // pass. The PENDING_OBSOLETE_* host flags are cleared up-front; |
| 119 | // if we did not count this chart, candidates == archives == 0 |
| 120 | // for it and the host flag would not be re-armed -- the cleanup |
| 121 | // would never retry once replication finishes. Counting it as a |
| 122 | // candidate without an archive forces archives != candidates at |
| 123 | // end-of-loop, which re-arms the host flag for the next pass. |
| 124 | |
| 125 | if(flags & RRDSET_FLAG_OBSOLETE_DIMENSIONS) { |
| 126 | partial_candidates++; |
| 127 | |
| 128 | if(!is_replicating) { |
| 129 | archived_items += svc_rrdset_archive_obsolete_dimensions(st, false); |
| 130 | |
| 131 | // "all candidates archived" -> flag was not re-set inside. |
| 132 | if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) |
| 133 | partial_archives++; |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | if(flags & RRDSET_FLAG_OBSOLETE) { |
| 138 | full_candidates++; |
| 139 | |
| 140 | if(!is_replicating && svc_rrdset_lock_for_deletion(st, now)) { |
| 141 | archived_items += svc_rrdset_archive_obsolete_dimensions(st, true); |
| 142 | |
| 143 | if(!rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE_DIMENSIONS)) { |
| 144 | full_archives++; |
| 145 | archived_items++; // rrdset_free archives the RRDINSTANCE |
| 146 | |
| 147 | worker_is_busy(UV_EVENT_FREE_CHART); |
| 148 | rrdset_free(st); |
| 149 | } |
| 150 | else |
| 151 | spinlock_unlock(&st->destroy_lock); |
| 152 | } |
| 153 | } |
| 154 | } |
| 155 | rrdset_foreach_done(st); |
| 156 | |
| 157 | dictionary_garbage_collect(host->rrdset_root_index); |
| 158 | |
| 159 | if(partial_archives != partial_candidates) |
| 160 | rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_DIMENSIONS); |
| 161 | |
| 162 | if(full_archives != full_candidates) |
| 163 | rrdhost_flag_set(host, RRDHOST_FLAG_PENDING_OBSOLETE_CHARTS); |
| 164 | |
| 165 | return archived_items; |
| 166 | } |
| 167 | |
| 168 | void svc_rrdhost_obsolete_all_charts(RRDHOST *host) { |
| 169 | ml_host_disconnected(host); |
| 170 | |
| 171 | RRDSET *st; |
| 172 | rrdset_foreach_read(st, host) { |
| 173 | rrdset_is_obsolete___safe_from_collector_thread(st); |
| 174 | } |
| 175 | rrdset_foreach_done(st); |
| 176 | } |
| 177 | |
| 178 | static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() { |
| 179 | worker_is_busy(UV_EVENT_CLEANUP_OBSOLETE_CHARTS_ON_HOSTS); |
| 180 | |
| 181 | rrd_rdlock(); |
| 182 | |
| 183 | size_t archived = 0; |
| 184 | |
| 185 | RRDHOST *host; |
| 186 | rrdhost_foreach_read(host) { |
| 187 | // Per-chart correctness gate is rrdset_is_replicating(st) inside |
| 188 | // svc_rrdhost_cleanup_charts_marked_obsolete. The previous host-level |
| 189 | // gate here (rrdhost_*_replicating_charts(host) > 0) was a defensive |
| 190 | // holdover from before the sender replication counter was accurate; |
| 191 | // on streaming children with continuous chart churn it permanently |
| 192 | // tripped and blocked all obsolete-chart cleanup on the host, piling |
| 193 | // up obsolete-but-still-live charts and their RAM-mode dim mmaps. |
| 194 | archived += svc_rrdhost_cleanup_charts_marked_obsolete(host); |
| 195 | |
| 196 | if (rrdhost_is_local(host) || IS_VIRTUAL_HOST_OS(host)) |
| 197 | continue; |
| 198 | |
| 199 | // Two-phase obsolete-all: decide under receiver_lock (short held), |
| 200 | // run the O(charts) walk + ml_host_disconnected without the lock, |
| 201 | // gated by RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS so a reconnecting |
| 202 | // receiver bails out in rrdhost_set_receiver() instead of overlapping. |
| 203 | bool obsolete_all = false; |
| 204 | |
| 205 | rrdhost_receiver_lock(host); |
| 206 | |
| 207 | time_t now = now_realtime_sec(); |
| 208 | |
| 209 | if (!host->receiver && |
| 210 | host->stream.rcv.status.last_connected == 0 && |
| 211 | (host->stream.rcv.status.last_disconnected + rrdset_free_obsolete_time_s < now) && |
| 212 | !rrdhost_flag_check(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS)) { |
| 213 | rrdhost_flag_set(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS); |
| 214 | obsolete_all = true; |
| 215 | } |
| 216 | |
| 217 | rrdhost_receiver_unlock(host); |
| 218 | |
| 219 | if (obsolete_all) { |
| 220 | svc_rrdhost_obsolete_all_charts(host); |
| 221 | rrdhost_flag_clear(host, RRDHOST_FLAG_OBSOLETE_ALL_IN_PROGRESS); |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | rrd_rdunlock(); |
| 226 | |
| 227 | // If anything was archived (a chart freed, or just dimensions archived |
| 228 | // on a still-live chart), schedule a deep rrdcontext GC pass. On |
| 229 | // non-dbengine hosts, dbengine rotation never triggers it, so archived |
| 230 | // RRDINSTANCE / RRDMETRIC entries would otherwise accumulate forever in |
| 231 | // host->rrdctx.contexts -> rc->rrdinstances / ri->rrdmetrics. |
| 232 | if(archived) |
| 233 | rrdcontext_request_full_gc(); |
| 234 | } |
| 235 | |
| 236 | static void svc_rrdhost_cleanup_orphan_hosts(RRDHOST *protected_host) { |
| 237 | worker_is_busy(UV_EVENT_CLEANUP_ORPHAN_HOSTS); |
| 238 | |
| 239 | time_t now = now_realtime_sec(); |
| 240 | |
| 241 | rrd_wrlock(); |
| 242 | RRDHOST *host, *next = localhost; |
| 243 | while((host = next) != NULL) { |
| 244 | next = host->next; |
| 245 | |
| 246 | if(!rrdhost_should_be_cleaned_up(host, protected_host, now)) |
| 247 | continue; |
| 248 | |
| 249 | bool delete = rrdhost_free_ephemeral_time_s && |
| 250 | now - host->stream.rcv.status.last_disconnected > rrdhost_free_ephemeral_time_s && |
| 251 | rrdhost_option_check(host, RRDHOST_OPTION_EPHEMERAL_HOST); |
| 252 | |
| 253 | if (!delete && rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) { |
| 254 | // the node is archived, so the cleanup has already run |
| 255 | // however, the node may not have any retention now |
| 256 | // so it may still need to be needed |
| 257 | time_t from_s = 0, to_s = 0; |
| 258 | rrdhost_retention(host, now, rrdhost_is_online(host), &from_s, &to_s); |
| 259 | if(!from_s && !to_s) |
| 260 | delete = true; |
| 261 | else |
| 262 | continue; |
| 263 | } |
| 264 | |
| 265 | worker_is_busy(UV_EVENT_FREE_HOST); |
| 266 | |
| 267 | if (delete) { |
| 268 | netdata_log_info("Host '%s' with machine guid '%s' is archived, ephemeral clean up.", rrdhost_hostname(host), host->machine_guid); |
| 269 | |
| 270 | // Save machine_guid before releasing lock - we'll use it to look up fresh pointers |
| 271 | char machine_guid[UUID_STR_LEN]; |
| 272 | strncpyz(machine_guid, host->machine_guid, GUID_LEN); |
| 273 | |
| 274 | // Release lock before synchronous cloud operations to avoid deadlock |
| 275 | // (build_node_info acquires rrd_rdlock which would deadlock with our wrlock) |
| 276 | rrd_wrunlock(); |
| 277 | |
| 278 | // Look up fresh pointer for cloud operations (don't use stale 'host' pointer) |
| 279 | RRDHOST *cloud_host = rrdhost_find_by_guid(machine_guid); |
| 280 | if (cloud_host) { |
| 281 | send_node_info_with_wait(cloud_host); |
| 282 | send_node_update_with_wait(cloud_host, 0, 0); |
| 283 | } |
| 284 | |
| 285 | // Re-acquire lock for cleanup |
| 286 | rrd_wrlock(); |
| 287 | |
| 288 | // Re-validate host still exists for cleanup |
| 289 | RRDHOST *host_check = rrdhost_find_by_guid(machine_guid); |
| 290 | if (host_check) { |
| 291 | unregister_node(host_check->machine_guid); |
| 292 | rrdhost_free___while_having_rrd_wrlock(host_check); |
| 293 | } |
| 294 | |
| 295 | // Restart iteration - the list may have changed while lock was released |
| 296 | next = localhost; |
| 297 | now = now_realtime_sec(); |
| 298 | } |
| 299 | else |
| 300 | rrdhost_cleanup_data_collection_and_health(host); |
| 301 | } |
| 302 | rrd_wrunlock(); |
| 303 | } |
| 304 | |
| 305 | void run_maintenace() { |
| 306 | svc_rrd_cleanup_obsolete_charts_from_all_hosts(); |
| 307 | svc_rrdhost_cleanup_orphan_hosts(localhost); |
| 308 | } |