Calculate currently collected metrics (#18803)
* Count live metrics * Remove debug messages * Remove debug entry
Stelios Fragkakis committed
Oct 18, 2024 at 08:48 UTC
c39a074dcb1572be0ee11dc7a817dac98e893f4d
4 files changed
+52
-1
src/database/contexts/api_v2_contexts.c
+22
@@ -442,6 +442,28 @@ static void rrdcontext_to_json_v2_rrdhost(BUFFER *wb, RRDHOST *host, struct rrdc
442
buffer_json_member_add_time_t(wb, "first_time", s.db.first_time_s);
443
buffer_json_member_add_time_t(wb, "last_time", s.db.last_time_s);
444
buffer_json_member_add_uint64(wb, "metrics", s.db.metrics);
445
+
446
+ spinlock_lock(&s.host->accounting.spinlock);
447
+ int64_t count = 0;
448
+
449
+ if (s.host->accounting.cache_timestamp &&
450
+ ctl->now - s.host->accounting.cache_timestamp < host->rrd_update_every * 1.5)
451
+ count = s.host->accounting.currently_collected;
452
+ else {
453
+ Pvoid_t *Pvalue;
454
+ bool first = true;
455
+ Word_t dimension_id = 0;
456
+ while ((Pvalue = JudyLFirstThenNext(s.host->accounting.JudyL, &dimension_id, &first))) {
457
+ RRDDIM *rd = *Pvalue;
458
+ if (rd->collector.last_collected_time.tv_sec > ctl->now - (rd->rrdset->update_every * 2))
459
+ count++;
460
+ }
461
+ s.host->accounting.currently_collected = count;
462
+ s.host->accounting.cache_timestamp = ctl->now;
463
+ }
464
+ spinlock_unlock(&s.host->accounting.spinlock);
465
+
466
+ buffer_json_member_add_uint64(wb, "currently_collected_metrics", count);
467
buffer_json_member_add_uint64(wb, "instances", s.db.instances);
468
buffer_json_member_add_uint64(wb, "contexts", s.db.contexts);
469
}
src/database/contexts/api_v2_contexts_agents.c
+4
-1
@@ -31,6 +31,8 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
31
32
buffer_json_cloud_status(wb, now_s);
33
34
+ size_t currently_collected_metrics = 0;
35
+
36
buffer_json_member_add_object(wb, "nodes");
37
{
38
size_t receiving = 0, archived = 0, sending = 0, total = 0;
@@ -47,6 +49,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
49
else
50
archived++;
51
}
52
+ currently_collected_metrics += host->accounting.currently_collected;
53
}
54
dfe_done(host);
55
@@ -82,7 +85,7 @@ void buffer_json_agents_v2(BUFFER *wb, struct query_timings *timings, time_t now
85
}
86
#endif
87
time_t first_time_s = storage_engine_global_first_time_s(eng->seb, localhost->db[tier].si);
85
- size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si);
88
+// size_t currently_collected_metrics = storage_engine_collected_metrics(eng->seb, localhost->db[tier].si);
89
90
NETDATA_DOUBLE percent;
91
if (used && max)
src/database/rrd.h
+9
@@ -284,6 +284,7 @@ struct rrddim {
284
285
int32_t multiplier; // the multiplier of the collected values
286
int32_t divisor; // the divider of the collected values
287
+ int32_t dimension_id; // Dimension id
288
289
// ------------------------------------------------------------------------
290
// operational state members
@@ -1176,6 +1177,14 @@ struct rrdhost {
1177
int32_t rrd_update_every; // the update frequency of the host
1178
int32_t rrd_history_entries; // the number of history entries for the host's charts
1179
1180
+ struct {
1181
+ uint32_t dimension_count; // Dimension count for this host
1182
+ uint32_t currently_collected; // Currectly collected metrics cache
1183
+ time_t cache_timestamp;
1184
+ Pvoid_t JudyL; // Store metrics collected -- link to rrddim
1185
+ SPINLOCK spinlock;
1186
+ } accounting;
1187
+
1188
RRD_MEMORY_MODE rrd_memory_mode; // the configured memory more for the charts of this host
1189
// the actual per tier is at .db[tier].mode
1190
src/database/rrddim.c
+17
@@ -41,6 +41,13 @@ static void rrddim_insert_callback(const DICTIONARY_ITEM *item __maybe_unused, v
41
RRDSET *st = ctr->st;
42
RRDHOST *host = st->rrdhost;
43
44
+ rd->dimension_id = __atomic_add_fetch(&host->accounting.dimension_count, 1, __ATOMIC_RELAXED);
45
+ spinlock_lock(&host->accounting.spinlock);
46
+ Pvoid_t *Pvalue = JudyLIns(&host->accounting.JudyL, rd->dimension_id, PJE0);
47
+ if (Pvalue)
48
+ *Pvalue = rd;
49
+ spinlock_unlock(&host->accounting.spinlock);
50
+
51
rd->flags = RRDDIM_FLAG_NONE;
52
53
rd->id = string_strdupz(ctr->id);
@@ -231,6 +238,10 @@ static void rrddim_delete_callback(const DICTIONARY_ITEM *item __maybe_unused, v
238
freez(rd->db.data);
239
}
240
241
+ spinlock_lock(&host->accounting.spinlock);
242
+ (void) JudyLDel(&host->accounting.JudyL, rd->dimension_id, PJE0);
243
+ spinlock_unlock(&host->accounting.spinlock);
244
+
245
string_freez(rd->id);
246
string_freez(rd->name);
247
}
@@ -557,6 +568,12 @@ collected_number rrddim_timed_set_by_pointer(RRDSET *st __maybe_unused, RRDDIM *
568
rrddim_set_updated(rd);
569
rd->collector.counter++;
570
571
+// spinlock_lock(&st->rrdhost->accounting.spinlock);
572
+// Pvoid_t *Pvalue = JudyLIns(&st->rrdhost->accounting.JudySecL, (Word_t) collected_time.tv_sec, PJE0);
573
+// if (Pvalue)
574
+// *((int64_t *)Pvalue) = *((int64_t *)Pvalue) + 1;
575
+// spinlock_unlock(&st->rrdhost->accounting.spinlock);
576
+
577
collected_number v = (value >= 0) ? value : -value;
578
if (unlikely(v > rd->collector.collected_value_max))
579
rd->collector.collected_value_max = v;