Protect type anomaly rate map (#17044)
* Add spinlock API to skip cancelability state changes. * Protect type anomaly rate map
vkalintiris committed
Feb 26, 2024 at 11:16 UTC
ac6a2f6563084948332beba5c71d0204760e77a5
5 files changed
+67
-20
src/libnetdata/locks/locks.c
+42
-7
@@ -297,14 +297,15 @@ void spinlock_init(SPINLOCK *spinlock) {
297
memset(spinlock, 0, sizeof(SPINLOCK));
298
}
299
300
-void spinlock_lock(SPINLOCK *spinlock) {
300
+static inline void spinlock_lock_internal(SPINLOCK *spinlock, bool cancelable) {
301
static const struct timespec ns = { .tv_sec = 0, .tv_nsec = 1 };
302
303
#ifdef NETDATA_INTERNAL_CHECKS
304
size_t spins = 0;
305
#endif
306
307
- netdata_thread_disable_cancelability();
307
+ if (!cancelable)
308
+ netdata_thread_disable_cancelability();
309
310
for(int i = 1;
311
__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) ||
@@ -329,16 +330,19 @@ void spinlock_lock(SPINLOCK *spinlock) {
330
#endif
331
}
332
332
-void spinlock_unlock(SPINLOCK *spinlock) {
333
+static inline void spinlock_unlock_internal(SPINLOCK *spinlock, bool cancelable) {
334
#ifdef NETDATA_INTERNAL_CHECKS
335
spinlock->locker_pid = 0;
336
#endif
337
__atomic_clear(&spinlock->locked, __ATOMIC_RELEASE);
337
- netdata_thread_enable_cancelability();
338
+
339
+ if (!cancelable)
340
+ netdata_thread_enable_cancelability();
341
}
342
340
-bool spinlock_trylock(SPINLOCK *spinlock) {
341
- netdata_thread_disable_cancelability();
343
+static inline bool spinlock_trylock_internal(SPINLOCK *spinlock, bool cancelable) {
344
+ if (!cancelable)
345
+ netdata_thread_disable_cancelability();
346
347
if(!__atomic_load_n(&spinlock->locked, __ATOMIC_RELAXED) &&
348
!__atomic_test_and_set(&spinlock->locked, __ATOMIC_ACQUIRE))
@@ -346,10 +350,41 @@ bool spinlock_trylock(SPINLOCK *spinlock) {
350
return true;
351
352
// we didn't get the lock
349
- netdata_thread_enable_cancelability();
353
+ if (!cancelable)
354
+ netdata_thread_enable_cancelability();
355
return false;
356
}
357
358
+void spinlock_lock(SPINLOCK *spinlock)
359
+{
360
+ spinlock_lock_internal(spinlock, false);
361
+}
362
+
363
+void spinlock_unlock(SPINLOCK *spinlock)
364
+{
365
+ spinlock_unlock_internal(spinlock, false);
366
+}
367
+
368
+bool spinlock_trylock(SPINLOCK *spinlock)
369
+{
370
+ return spinlock_trylock_internal(spinlock, false);
371
+}
372
+
373
+void spinlock_lock_cancelable(SPINLOCK *spinlock)
374
+{
375
+ spinlock_lock_internal(spinlock, true);
376
+}
377
+
378
+void spinlock_unlock_cancelable(SPINLOCK *spinlock)
379
+{
380
+ spinlock_unlock_internal(spinlock, true);
381
+}
382
+
383
+bool spinlock_trylock_cancelable(SPINLOCK *spinlock)
384
+{
385
+ return spinlock_trylock_internal(spinlock, true);
386
+}
387
+
388
// ----------------------------------------------------------------------------
389
// rw_spinlock implementation
390
src/libnetdata/locks/locks.h
+4
@@ -25,6 +25,10 @@ void spinlock_lock(SPINLOCK *spinlock);
25
void spinlock_unlock(SPINLOCK *spinlock);
26
bool spinlock_trylock(SPINLOCK *spinlock);
27
28
+void spinlock_lock_cancelable(SPINLOCK *spinlock);
29
+void spinlock_unlock_cancelable(SPINLOCK *spinlock);
30
+bool spinlock_trylock_cancelable(SPINLOCK *spinlock);
31
+
32
typedef struct netdata_rw_spinlock {
33
int32_t readers;
34
SPINLOCK spinlock;
src/ml/ad_charts.cc
+2
@@ -288,6 +288,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
288
rrdset_flag_set(host->type_anomaly_rate_rs, RRDSET_FLAG_ANOMALY_DETECTION);
289
}
290
291
+ spinlock_lock_cancelable(&host->type_anomaly_rate_spinlock);
292
for (auto &entry : host->type_anomaly_rate) {
293
ml_type_anomaly_rate_t &type_anomaly_rate = entry.second;
294
@@ -304,6 +305,7 @@ void ml_update_host_and_detection_rate_charts(ml_host_t *host, collected_number
305
type_anomaly_rate.anomalous_dimensions = 0;
306
type_anomaly_rate.normal_dimensions = 0;
307
}
308
+ spinlock_unlock_cancelable(&host->type_anomaly_rate_spinlock);
309
310
rrdset_done(host->type_anomaly_rate_rs);
311
}
src/ml/ml-private.h
+1
@@ -264,6 +264,7 @@ typedef struct {
264
RRDDIM *detector_events_new_anomaly_event_rd;
265
266
RRDSET *type_anomaly_rate_rs;
267
+ SPINLOCK type_anomaly_rate_spinlock;
268
std::unordered_map<STRING *, ml_type_anomaly_rate_t> type_anomaly_rate;
269
} ml_host_t;
270
src/ml/ml.cc
+18
-13
@@ -1090,20 +1090,24 @@ ml_host_detect_once(ml_host_t *host)
1090
host->mls.num_anomalous_dimensions += chart_mls.num_anomalous_dimensions;
1091
host->mls.num_normal_dimensions += chart_mls.num_normal_dimensions;
1092
1093
- STRING *key = rs->parts.type;
1094
- auto &um = host->type_anomaly_rate;
1095
- auto it = um.find(key);
1096
- if (it == um.end()) {
1097
- um[key] = ml_type_anomaly_rate_t {
1098
- .rd = NULL,
1099
- .normal_dimensions = 0,
1100
- .anomalous_dimensions = 0
1101
- };
1102
- it = um.find(key);
1103
- }
1093
+ if (spinlock_trylock_cancelable(&host->type_anomaly_rate_spinlock))
1094
+ {
1095
+ STRING *key = rs->parts.type;
1096
+ auto &um = host->type_anomaly_rate;
1097
+ auto it = um.find(key);
1098
+ if (it == um.end()) {
1099
+ um[key] = ml_type_anomaly_rate_t {
1100
+ .rd = NULL,
1101
+ .normal_dimensions = 0,
1102
+ .anomalous_dimensions = 0
1103
+ };
1104
+ it = um.find(key);
1105
+ }
1106
1105
- it->second.anomalous_dimensions += chart_mls.num_anomalous_dimensions;
1106
- it->second.normal_dimensions += chart_mls.num_normal_dimensions;
1107
+ it->second.anomalous_dimensions += chart_mls.num_anomalous_dimensions;
1108
+ it->second.normal_dimensions += chart_mls.num_normal_dimensions;
1109
+ spinlock_unlock_cancelable(&host->type_anomaly_rate_spinlock);
1110
+ }
1111
}
1112
rrdset_foreach_done(rsp);
1113
@@ -1310,6 +1314,7 @@ void ml_host_new(RRDHOST *rh)
1314
host->training_queue = Cfg.training_threads[times_called++ % Cfg.num_training_threads].training_queue;
1315
1316
netdata_mutex_init(&host->mutex);
1317
+ spinlock_init(&host->type_anomaly_rate_spinlock);
1318
1319
host->ml_running = true;
1320
rh->ml_host = (rrd_ml_host_t *) host;