cleanup ML cached pointers on child disconnection (#20195)
Costa Tsaousis committed
Apr 28, 2025 at 13:30 UTC
731913da57a0f1c09d7dd10aae1d91f0e19b96ab
6 files changed
+49
-1
src/daemon/service.c
+4
-1
@@ -160,6 +160,8 @@ static inline void svc_rrdhost_cleanup_charts_marked_obsolete(RRDHOST *host) {
160
}
161
162
void svc_rrdhost_obsolete_all_charts(RRDHOST *host) {
163
+ ml_host_disconnected(host);
164
+
165
RRDSET *st;
166
rrdset_foreach_read(st, host) {
167
rrdset_is_obsolete___safe_from_collector_thread(st);
@@ -190,7 +192,8 @@ static void svc_rrd_cleanup_obsolete_charts_from_all_hosts() {
192
193
time_t now = now_realtime_sec();
194
193
- if (host->stream.rcv.status.last_connected == 0 &&
195
+ if (!host->receiver &&
196
+ host->stream.rcv.status.last_connected == 0 &&
197
(host->stream.rcv.status.last_disconnected + rrdset_free_obsolete_time_s < now)) {
198
svc_rrdhost_obsolete_all_charts(host);
199
}
src/ml/ad_charts.cc
+29
@@ -4,6 +4,35 @@
4
#include "ml_config.h"
5
6
void ml_update_dimensions_chart(ml_host_t *host, const ml_machine_learning_stats_t &mls) {
7
+
8
+ if(__atomic_load_n(&host->reset_pointers, __ATOMIC_RELAXED)) {
9
+ __atomic_store_n(&host->reset_pointers, false, __ATOMIC_RELAXED);
10
+
11
+ host->ml_running_rs = nullptr;
12
+ host->ml_running_rd = nullptr;
13
+ host->machine_learning_status_rs = nullptr;
14
+ host->machine_learning_status_enabled_rd = nullptr;
15
+ host->machine_learning_status_disabled_sp_rd = nullptr;
16
+ host->metric_type_rs = nullptr;
17
+ host->metric_type_constant_rd = nullptr;
18
+ host->metric_type_variable_rd = nullptr;
19
+ host->training_status_rs = nullptr;
20
+ host->training_status_untrained_rd = nullptr;
21
+ host->training_status_pending_without_model_rd = nullptr;
22
+ host->training_status_trained_rd = nullptr;
23
+ host->training_status_pending_with_model_rd = nullptr;
24
+ host->training_status_silenced_rd = nullptr;
25
+ host->dimensions_rs = nullptr;
26
+ host->dimensions_anomalous_rd = nullptr;
27
+ host->dimensions_normal_rd = nullptr;
28
+ host->anomaly_rate_rs = nullptr;
29
+ host->anomaly_rate_rd = nullptr;
30
+ host->detector_events_rs = nullptr;
31
+ host->detector_events_above_threshold_rd = nullptr;
32
+ host->detector_events_new_anomaly_event_rd = nullptr;
33
+ host->context_anomaly_rate_rs = nullptr;
34
+ }
35
+
36
/*
37
* Machine learning status
38
*/
src/ml/ml-dummy.c
+4
@@ -131,4 +131,8 @@ bool ml_model_received_from_child(RRDHOST *host, const char *json) {
131
return false;
132
}
133
134
+ void ml_host_disconnected(RRDHOST *rh) {
135
+ UNUSED(rh);
136
+}
137
+
138
#endif
src/ml/ml_host.h
+2
@@ -84,6 +84,8 @@ typedef struct {
84
RRDSET *context_anomaly_rate_rs;
85
SPINLOCK context_anomaly_rate_spinlock;
86
std::unordered_map<STRING *, ml_context_anomaly_rate_t> context_anomaly_rate;
87
+
88
+ bool reset_pointers;
89
} ml_host_t;
90
91
#endif /* NETDATA_ML_HOST_H */
src/ml/ml_public.cc
+8
@@ -503,3 +503,11 @@ bool ml_model_received_from_child(RRDHOST *host, const char *json)
503
504
return ok;
505
}
506
+
507
+void ml_host_disconnected(RRDHOST *rh) {
508
+ ml_host_t *host = (ml_host_t *) rh->ml_host;
509
+ if (!host)
510
+ return;
511
+
512
+ __atomic_store_n(&host->reset_pointers, true, __ATOMIC_RELAXED);
513
+}
src/ml/ml_public.h
+2
@@ -58,6 +58,8 @@ uint64_t sqlite_get_ml_space(void);
58
59
bool ml_model_received_from_child(RRDHOST *host, const char *json);
60
61
+void ml_host_disconnected(RRDHOST *host);
62
+
63
#ifdef __cplusplus
64
};
65
#endif