| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #ifndef NETDATA_MRG_INTERNALS_H |
| 4 | #define NETDATA_MRG_INTERNALS_H |
| 5 | |
| 6 | #include "mrg.h" |
| 7 | #include "cache.h" |
| 8 | #include "libnetdata/locks/locks.h" |
| 9 | #include "rrddiskprotocol.h" |
| 10 | |
| 11 | struct metric { |
| 12 | Word_t section; // never changes |
| 13 | UUIDMAP_ID uuid; // never changes |
| 14 | |
| 15 | REFCOUNT refcount; |
| 16 | uint8_t partition; |
| 17 | bool deleted; |
| 18 | |
| 19 | uint32_t latest_update_every_s; // the latest data collection frequency |
| 20 | |
| 21 | time_t first_time_s; // the timestamp of the oldest point in the database |
| 22 | time_t latest_time_s_clean; // the timestamp of the newest point in the database |
| 23 | time_t latest_time_s_hot; // the timestamp of the latest point that has been collected (not yet stored) |
| 24 | |
| 25 | #ifdef NETDATA_INTERNAL_CHECKS |
| 26 | pid_t writer; |
| 27 | #endif |
| 28 | |
| 29 | // THIS IS allocated with malloc() |
| 30 | // YOU HAVE TO INITIALIZE IT YOURSELF! |
| 31 | }; |
| 32 | |
| 33 | #define set_metric_field_with_condition(field, value, condition) ({ \ |
| 34 | typeof(field) _current = __atomic_load_n(&(field), __ATOMIC_RELAXED); \ |
| 35 | typeof(field) _wanted = value; \ |
| 36 | bool did_it = true; \ |
| 37 | \ |
| 38 | do { \ |
| 39 | if((condition) && (_current != _wanted)) { \ |
| 40 | ; \ |
| 41 | } \ |
| 42 | else { \ |
| 43 | did_it = false; \ |
| 44 | break; \ |
| 45 | } \ |
| 46 | } while(!__atomic_compare_exchange_n(&(field), &_current, _wanted, \ |
| 47 | false, __ATOMIC_RELAXED, __ATOMIC_RELAXED)); \ |
| 48 | \ |
| 49 | did_it; \ |
| 50 | }) |
| 51 | |
| 52 | extern struct aral_statistics mrg_aral_statistics; |
| 53 | |
| 54 | struct mrg { |
| 55 | struct mrg_partition { |
| 56 | ARAL *aral; // not protected by our spinlock - it has its own |
| 57 | |
| 58 | RW_SPINLOCK rw_spinlock; |
| 59 | Pvoid_t uuid_judy; // JudyL: each UUID has a JudyL of sections (tiers) |
| 60 | |
| 61 | struct mrg_statistics stats; |
| 62 | } index[UUIDMAP_PARTITIONS]; |
| 63 | }; |
| 64 | |
| 65 | static inline void MRG_STATS_DUPLICATE_ADD(MRG *mrg, size_t partition) { |
| 66 | mrg->index[partition].stats.additions_duplicate++; |
| 67 | } |
| 68 | |
| 69 | static inline void MRG_STATS_ADDED_METRIC(MRG *mrg, size_t partition, Word_t section) { |
| 70 | mrg->index[partition].stats.entries++; |
| 71 | mrg->index[partition].stats.additions++; |
| 72 | mrg->index[partition].stats.size += sizeof(METRIC); |
| 73 | struct rrdengine_instance *ctx = (struct rrdengine_instance *) section; |
| 74 | __atomic_add_fetch(&ctx->atomic.metrics, 1, __ATOMIC_RELAXED); |
| 75 | } |
| 76 | |
| 77 | static inline void MRG_STATS_DELETED_METRIC(MRG *mrg, size_t partition, Word_t section) { |
| 78 | mrg->index[partition].stats.entries--; |
| 79 | mrg->index[partition].stats.size -= sizeof(METRIC); |
| 80 | mrg->index[partition].stats.deletions++; |
| 81 | struct rrdengine_instance *ctx = (struct rrdengine_instance *) section; |
| 82 | __atomic_sub_fetch(&ctx->atomic.metrics, 1, __ATOMIC_RELAXED); |
| 83 | } |
| 84 | |
| 85 | static inline void MRG_STATS_SEARCH_HIT(MRG *mrg, size_t partition) { |
| 86 | __atomic_add_fetch(&mrg->index[partition].stats.search_hits, 1, __ATOMIC_RELAXED); |
| 87 | } |
| 88 | |
| 89 | static inline void MRG_STATS_SEARCH_MISS(MRG *mrg, size_t partition) { |
| 90 | __atomic_add_fetch(&mrg->index[partition].stats.search_misses, 1, __ATOMIC_RELAXED); |
| 91 | } |
| 92 | |
| 93 | static inline void MRG_STATS_DELETE_MISS(MRG *mrg, size_t partition) { |
| 94 | mrg->index[partition].stats.delete_misses++; |
| 95 | } |
| 96 | |
| 97 | #define mrg_index_read_lock(mrg, partition) rw_spinlock_read_lock(&(mrg)->index[partition].rw_spinlock) |
| 98 | #define mrg_index_read_unlock(mrg, partition) rw_spinlock_read_unlock(&(mrg)->index[partition].rw_spinlock) |
| 99 | #define mrg_index_write_lock(mrg, partition) rw_spinlock_write_lock(&(mrg)->index[partition].rw_spinlock) |
| 100 | #define mrg_index_write_unlock(mrg, partition) rw_spinlock_write_unlock(&(mrg)->index[partition].rw_spinlock) |
| 101 | |
| 102 | static inline void mrg_stats_judy_mem(MRG *mrg, size_t partition, int64_t judy_mem) { |
| 103 | __atomic_add_fetch(&mrg->index[partition].stats.size, judy_mem, __ATOMIC_RELAXED); |
| 104 | } |
| 105 | |
| 106 | static inline void metric_log(MRG *mrg __maybe_unused, METRIC *metric, const char *msg) { |
| 107 | struct rrdengine_instance *ctx = (struct rrdengine_instance *)metric->section; |
| 108 | |
| 109 | nd_uuid_t uuid; |
| 110 | uuidmap_uuid(metric->uuid, uuid); |
| 111 | char uuid_txt[UUID_STR_LEN]; |
| 112 | uuid_unparse_lower(uuid, uuid_txt); |
| 113 | nd_log(NDLS_DAEMON, NDLP_ERR, |
| 114 | "METRIC: %s on %s at tier %d, refcount %d, partition %u, " |
| 115 | "retention [%ld - %ld (hot), %ld (clean)], update every %"PRIu32 |
| 116 | #ifdef NETDATA_INTERNAL_CHECKS |
| 117 | ", writer pid %d " |
| 118 | #endif |
| 119 | " --- PLEASE OPEN A GITHUB ISSUE TO REPORT THIS LOG LINE TO NETDATA --- ", |
| 120 | msg, |
| 121 | uuid_txt, |
| 122 | ctx->config.tier, |
| 123 | metric->refcount, |
| 124 | metric->partition, |
| 125 | metric->first_time_s, |
| 126 | metric->latest_time_s_hot, |
| 127 | metric->latest_time_s_clean, |
| 128 | metric->latest_update_every_s |
| 129 | #ifdef NETDATA_INTERNAL_CHECKS |
| 130 | , (int)metric->writer |
| 131 | #endif |
| 132 | ); |
| 133 | } |
| 134 | |
| 135 | |
| 136 | ALWAYS_INLINE |
| 137 | static time_t mrg_metric_get_first_time_s_smart(MRG *mrg __maybe_unused, METRIC *metric) { |
| 138 | time_t first_time_s = __atomic_load_n(&metric->first_time_s, __ATOMIC_RELAXED); |
| 139 | |
| 140 | if(first_time_s <= 0) { |
| 141 | first_time_s = __atomic_load_n(&metric->latest_time_s_clean, __ATOMIC_RELAXED); |
| 142 | if(first_time_s <= 0) |
| 143 | first_time_s = __atomic_load_n(&metric->latest_time_s_hot, __ATOMIC_RELAXED); |
| 144 | |
| 145 | if(first_time_s <= 0) |
| 146 | first_time_s = 0; |
| 147 | else |
| 148 | __atomic_store_n(&metric->first_time_s, first_time_s, __ATOMIC_RELAXED); |
| 149 | } |
| 150 | |
| 151 | return first_time_s; |
| 152 | } |
| 153 | |
| 154 | ALWAYS_INLINE |
| 155 | static bool acquired_metric_has_retention(MRG *mrg, METRIC *metric) { |
| 156 | time_t first, last; |
| 157 | mrg_metric_get_retention(mrg, metric, &first, &last, NULL); |
| 158 | bool rc = (first != 0 && last != 0 && first <= last); |
| 159 | |
| 160 | if(!rc && __atomic_load_n(&mrg->index[metric->partition].stats.writers, __ATOMIC_RELAXED) > 0) |
| 161 | rc = true; |
| 162 | |
| 163 | return rc; |
| 164 | } |
| 165 | |
| 166 | ALWAYS_INLINE |
| 167 | static void acquired_for_deletion_metric_delete(MRG *mrg, METRIC *metric) { |
| 168 | JudyAllocThreadPulseReset(); |
| 169 | |
| 170 | size_t partition = metric->partition; |
| 171 | |
| 172 | mrg_index_write_lock(mrg, partition); |
| 173 | |
| 174 | Pvoid_t *sections_judy_pptr = JudyLGet(mrg->index[partition].uuid_judy, metric->uuid, PJE0); |
| 175 | if(unlikely(sections_judy_pptr == PJERR)) |
| 176 | fatal("METRIC: corrupted JudyL"); |
| 177 | |
| 178 | if(unlikely(!sections_judy_pptr || !*sections_judy_pptr)) { |
| 179 | MRG_STATS_DELETE_MISS(mrg, partition); |
| 180 | mrg_index_write_unlock(mrg, partition); |
| 181 | return; |
| 182 | } |
| 183 | |
| 184 | int rc = JudyLDel(sections_judy_pptr, metric->section, PJE0); |
| 185 | if(unlikely(!rc)) { |
| 186 | MRG_STATS_DELETE_MISS(mrg, partition); |
| 187 | mrg_index_write_unlock(mrg, partition); |
| 188 | mrg_stats_judy_mem(mrg, partition, JudyAllocThreadPulseGetAndReset()); |
| 189 | return; |
| 190 | } |
| 191 | |
| 192 | if(!*sections_judy_pptr) { |
| 193 | rc = JudyLDel(&mrg->index[partition].uuid_judy, metric->uuid, PJE0); |
| 194 | |
| 195 | if(unlikely(!rc)) |
| 196 | fatal("DBENGINE METRIC: cannot delete UUID from JudyL"); |
| 197 | } |
| 198 | |
| 199 | MRG_STATS_DELETED_METRIC(mrg, partition, metric->section); |
| 200 | |
| 201 | mrg_index_write_unlock(mrg, partition); |
| 202 | |
| 203 | __atomic_store_n(&metric->deleted, true, __ATOMIC_RELEASE); |
| 204 | |
| 205 | mrg_stats_judy_mem(mrg, partition, JudyAllocThreadPulseGetAndReset()); |
| 206 | } |
| 207 | |
| 208 | ALWAYS_INLINE |
| 209 | static bool metric_acquire(MRG *mrg, METRIC *metric) { |
| 210 | REFCOUNT rc = refcount_acquire_advanced(&metric->refcount); |
| 211 | if(!REFCOUNT_ACQUIRED(rc)) |
| 212 | return false; |
| 213 | |
| 214 | if (__atomic_load_n(&metric->deleted, __ATOMIC_ACQUIRE)) { |
| 215 | refcount_release(&metric->refcount); |
| 216 | return false; |
| 217 | } |
| 218 | |
| 219 | size_t partition = metric->partition; |
| 220 | |
| 221 | if(rc == 1) |
| 222 | __atomic_add_fetch(&mrg->index[partition].stats.entries_acquired, 1, __ATOMIC_RELAXED); |
| 223 | |
| 224 | __atomic_add_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED); |
| 225 | |
| 226 | return true; |
| 227 | } |
| 228 | |
| 229 | ALWAYS_INLINE |
| 230 | static bool metric_release(MRG *mrg, METRIC *metric) { |
| 231 | size_t partition = metric->partition; |
| 232 | |
| 233 | if (refcount_release(&metric->refcount) == 0) { |
| 234 | // we are the last user |
| 235 | if (!acquired_metric_has_retention(mrg, metric)) { |
| 236 | // This metric is eligible for deletion. |
| 237 | // Atomically check and set the 'deleted' flag. |
| 238 | // If __atomic_test_and_set returns 'true', it means the flag was already set. |
| 239 | if (!__atomic_test_and_set(&metric->deleted, __ATOMIC_ACQ_REL)) { |
| 240 | // We won the race. The flag was 'false' and we set it to 'true'. |
| 241 | // We are now responsible for deletion. |
| 242 | acquired_for_deletion_metric_delete(mrg, metric); |
| 243 | uuidmap_free(metric->uuid); |
| 244 | aral_freez(mrg->index[partition].aral, metric); |
| 245 | __atomic_sub_fetch(&mrg->index[partition].stats.entries_acquired, 1, __ATOMIC_RELAXED); |
| 246 | __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED); |
| 247 | return true; |
| 248 | } |
| 249 | // Another thread is already deleting it. nothing to do |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | __atomic_sub_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED); |
| 254 | return false; |
| 255 | } |
| 256 | |
| 257 | ALWAYS_INLINE |
| 258 | static METRIC *metric_add_and_acquire(MRG *mrg, MRG_ENTRY *entry, bool *ret) { |
| 259 | JudyAllocThreadPulseReset(); |
| 260 | |
| 261 | UUIDMAP_ID id = uuidmap_create(*entry->uuid); |
| 262 | |
| 263 | size_t partition = uuid_to_uuidmap_partition(*entry->uuid); |
| 264 | |
| 265 | METRIC *allocation = aral_mallocz(mrg->index[partition].aral); |
| 266 | Pvoid_t *PValue; |
| 267 | |
| 268 | while(1) { |
| 269 | mrg_index_write_lock(mrg, partition); |
| 270 | |
| 271 | Pvoid_t *sections_judy_pptr = JudyLIns(&mrg->index[partition].uuid_judy, id, PJE0); |
| 272 | if (unlikely(!sections_judy_pptr || sections_judy_pptr == PJERR)) |
| 273 | fatal("DBENGINE METRIC: corrupted UUIDs JudyL array"); |
| 274 | |
| 275 | PValue = JudyLIns(sections_judy_pptr, entry->section, PJE0); |
| 276 | if (unlikely(!PValue || PValue == PJERR)) |
| 277 | fatal("DBENGINE METRIC: corrupted section JudyL array"); |
| 278 | |
| 279 | if (unlikely(*PValue != NULL)) { |
| 280 | METRIC *metric = *PValue; |
| 281 | |
| 282 | if(!metric_acquire(mrg, metric)) { |
| 283 | mrg_index_write_unlock(mrg, partition); |
| 284 | continue; |
| 285 | } |
| 286 | |
| 287 | MRG_STATS_DUPLICATE_ADD(mrg, partition); |
| 288 | mrg_index_write_unlock(mrg, partition); |
| 289 | |
| 290 | if (ret) |
| 291 | *ret = false; |
| 292 | |
| 293 | uuidmap_free(id); |
| 294 | aral_freez(mrg->index[partition].aral, allocation); |
| 295 | |
| 296 | mrg_stats_judy_mem(mrg, partition, JudyAllocThreadPulseGetAndReset()); |
| 297 | return metric; |
| 298 | } |
| 299 | |
| 300 | break; |
| 301 | } |
| 302 | |
| 303 | METRIC *metric = allocation; |
| 304 | metric->uuid = id; |
| 305 | metric->section = entry->section; |
| 306 | metric->first_time_s = MAX(0, entry->first_time_s); |
| 307 | metric->latest_time_s_clean = MAX(0, entry->last_time_s); |
| 308 | metric->latest_time_s_hot = 0; |
| 309 | metric->latest_update_every_s = entry->latest_update_every_s; |
| 310 | metric->deleted = false; |
| 311 | #ifdef NETDATA_INTERNAL_CHECKS |
| 312 | metric->writer = 0; |
| 313 | #endif |
| 314 | metric->refcount = 1; |
| 315 | metric->partition = partition; |
| 316 | *PValue = metric; |
| 317 | |
| 318 | __atomic_add_fetch(&mrg->index[partition].stats.entries_acquired, 1, __ATOMIC_RELAXED); |
| 319 | __atomic_add_fetch(&mrg->index[partition].stats.current_references, 1, __ATOMIC_RELAXED); |
| 320 | |
| 321 | MRG_STATS_ADDED_METRIC(mrg, partition, metric->section); |
| 322 | |
| 323 | mrg_index_write_unlock(mrg, partition); |
| 324 | |
| 325 | if(ret) |
| 326 | *ret = true; |
| 327 | |
| 328 | mrg_stats_judy_mem(mrg, partition, JudyAllocThreadPulseGetAndReset()); |
| 329 | return metric; |
| 330 | } |
| 331 | |
| 332 | ALWAYS_INLINE |
| 333 | static METRIC *metric_get_and_acquire_by_id(MRG *mrg, UUIDMAP_ID id, Word_t section) { |
| 334 | size_t partition = uuidmap_id_to_partition(id); |
| 335 | |
| 336 | while(1) { |
| 337 | mrg_index_read_lock(mrg, partition); |
| 338 | |
| 339 | Pvoid_t *sections_judy_pptr = JudyLGet(mrg->index[partition].uuid_judy, id, PJE0); |
| 340 | if (unlikely(!sections_judy_pptr)) { |
| 341 | mrg_index_read_unlock(mrg, partition); |
| 342 | MRG_STATS_SEARCH_MISS(mrg, partition); |
| 343 | return NULL; |
| 344 | } |
| 345 | |
| 346 | Pvoid_t *PValue = JudyLGet(*sections_judy_pptr, section, PJE0); |
| 347 | if (unlikely(!PValue)) { |
| 348 | mrg_index_read_unlock(mrg, partition); |
| 349 | MRG_STATS_SEARCH_MISS(mrg, partition); |
| 350 | return NULL; |
| 351 | } |
| 352 | |
| 353 | METRIC *metric = *PValue; |
| 354 | |
| 355 | if(metric && !metric_acquire(mrg, metric)) |
| 356 | metric = NULL; |
| 357 | |
| 358 | mrg_index_read_unlock(mrg, partition); |
| 359 | |
| 360 | if(metric) { |
| 361 | MRG_STATS_SEARCH_HIT(mrg, partition); |
| 362 | return metric; |
| 363 | } |
| 364 | } |
| 365 | } |
| 366 | |
| 367 | #endif //NETDATA_MRG_INTERNALS_H |