Move cleanup of obsolete charts to a separate thread (#11222)
Vladimir Kobal committed
Jul 19, 2021 at 08:08 UTC
96636b7c6fd8f4e32e733744fecc4a7018c5f198
10 files changed
+130
-12
CMakeLists.txt
+2
@@ -896,6 +896,8 @@ set(DAEMON_FILES
896
daemon/main.h
897
daemon/signals.c
898
daemon/signals.h
899
+ daemon/service.c
900
+ daemon/service.h
901
daemon/commands.c
902
daemon/commands.h
903
daemon/unit_test.c
Makefile.am
+2
@@ -733,6 +733,8 @@ DAEMON_FILES = \
733
daemon/main.h \
734
daemon/signals.c \
735
daemon/signals.h \
736
+ daemon/service.c \
737
+ daemon/service.h \
738
daemon/commands.c \
739
daemon/commands.h \
740
daemon/unit_test.c \
daemon/common.h
+1
@@ -77,6 +77,7 @@
77
#include "daemon.h"
78
#include "main.h"
79
#include "signals.h"
80
+#include "service.h"
81
#include "commands.h"
82
#include "analytics.h"
83
daemon/main.c
+1
@@ -104,6 +104,7 @@ struct netdata_static_thread static_threads[] = {
104
NETDATA_PLUGIN_HOOK_PLUGINSD
105
NETDATA_PLUGIN_HOOK_HEALTH
106
NETDATA_PLUGIN_HOOK_ANALYTICS
107
+ NETDATA_PLUGIN_HOOK_SERVICE
108
109
{NULL, NULL, NULL, 0, NULL, NULL, NULL}
110
};
daemon/service.c
new
+38
@@ -0,0 +1,38 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "common.h"
4
+
5
+/* Run service jobs every X seconds */
6
+#define SERVICE_HEARTBEAT 10
7
+
8
+void service_main_cleanup(void *ptr)
9
+{
10
+ struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
11
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
12
+
13
+ debug(D_SYSTEM, "Cleaning up...");
14
+
15
+ static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
16
+}
17
+
18
+/*
19
+ * The service thread.
20
+ */
21
+void *service_main(void *ptr)
22
+{
23
+ netdata_thread_cleanup_push(service_main_cleanup, ptr);
24
+ heartbeat_t hb;
25
+ heartbeat_init(&hb);
26
+ usec_t step = USEC_PER_SEC * SERVICE_HEARTBEAT;
27
+
28
+ debug(D_SYSTEM, "Service thread starts");
29
+
30
+ while (!netdata_exit) {
31
+ heartbeat_next(&hb, step);
32
+
33
+ rrd_cleanup_obsolete_charts();
34
+ }
35
+
36
+ netdata_thread_cleanup_pop(1);
37
+ return NULL;
38
+}
daemon/service.h
new
+19
@@ -0,0 +1,19 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_SERVICE_H
4
+#define NETDATA_SERVICE_H 1
5
+
6
+#define NETDATA_PLUGIN_HOOK_SERVICE \
7
+ { \
8
+ .name = "SERVICE", \
9
+ .config_section = NULL, \
10
+ .config_name = NULL, \
11
+ .enabled = 1, \
12
+ .thread = NULL, \
13
+ .init_routine = NULL, \
14
+ .start_routine = service_main \
15
+ },
16
+
17
+extern void *service_main(void *ptr);
18
+
19
+#endif //NETDATA_SERVICE_H
database/rrd.h
+4
-2
@@ -762,7 +762,7 @@ struct rrdhost {
762
const char *timezone; // the timezone of the host
763
764
#ifdef ENABLE_ACLK
765
- long obsolete_count;
765
+ long deleted_charts_count;
766
#endif
767
768
const char *abbrev_timezone; // the abbriviated timezone of the host
@@ -859,6 +859,8 @@ struct rrdhost {
859
860
RRDSET *rrdset_root; // the host charts
861
862
+ unsigned int obsolete_charts_count;
863
+
864
865
// ------------------------------------------------------------------------
866
// locks
@@ -1038,6 +1040,7 @@ extern void rrdhost_system_info_free(struct rrdhost_system_info *system_info);
1040
extern void rrdhost_free(RRDHOST *host);
1041
extern void rrdhost_save_charts(RRDHOST *host);
1042
extern void rrdhost_delete_charts(RRDHOST *host);
1043
+extern void rrd_cleanup_obsolete_charts();
1044
1045
extern int rrdhost_should_be_removed(RRDHOST *host, RRDHOST *protected_host, time_t now);
1046
@@ -1326,7 +1329,6 @@ extern void rrdset_save(RRDSET *st);
1329
extern void rrdset_delete_custom(RRDSET *st, int db_rotated);
1330
extern void rrdset_delete_obsolete_dimensions(RRDSET *st);
1331
1329
-extern void rrdhost_cleanup_obsolete_charts(RRDHOST *host);
1332
extern RRDHOST *rrdhost_create(
1333
const char *hostname, const char *registry_hostname, const char *guid, const char *os, const char *timezone,
1334
const char *abbrev_timezone, int32_t utc_offset,const char *tags, const char *program_name, const char *program_version,
database/rrdhost.c
+33
@@ -650,6 +650,13 @@ restart_after_removal:
650
651
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
652
rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_GLOBAL, "cleanup obsolete charts after seconds", rrdset_free_obsolete_time);
653
+ // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentaion faults if a short
654
+ // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
655
+ // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
656
+ if (rrdset_free_obsolete_time < 10) {
657
+ rrdset_free_obsolete_time = 10;
658
+ info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds. A lower delay can potentially cause a segmentaion fault.");
659
+ }
660
gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_GLOBAL, "gap when lost iterations above", gap_when_lost_iterations_above);
661
if (gap_when_lost_iterations_above < 1)
662
gap_when_lost_iterations_above = 1;
@@ -1392,6 +1399,7 @@ restart_after_removal:
1399
&& st->last_updated.tv_sec + rrdset_free_obsolete_time < now
1400
&& st->last_collected_time.tv_sec + rrdset_free_obsolete_time < now
1401
)) {
1402
+ st->rrdhost->obsolete_charts_count--;
1403
#ifdef ENABLE_DBENGINE
1404
if(st->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE) {
1405
RRDDIM *rd, *last;
@@ -1437,6 +1445,7 @@ restart_after_removal:
1445
rrdvar_free_remaining_variables(host, &st->rrdvar_root_index);
1446
1447
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
1448
+
1449
if (st->dimensions) {
1450
/* If the chart still has dimensions don't delete it from the metadata log */
1451
continue;
@@ -1458,6 +1467,30 @@ restart_after_removal:
1467
}
1468
}
1469
1470
+void rrd_cleanup_obsolete_charts()
1471
+{
1472
+ rrd_rdlock();
1473
+
1474
+ RRDHOST *host;
1475
+ rrdhost_foreach_read(host)
1476
+ {
1477
+ if (host->obsolete_charts_count) {
1478
+ rrdhost_wrlock(host);
1479
+#ifdef ENABLE_ACLK
1480
+ host->deleted_charts_count = 0;
1481
+#endif
1482
+ rrdhost_cleanup_obsolete_charts(host);
1483
+#ifdef ENABLE_ACLK
1484
+ if (host->deleted_charts_count)
1485
+ aclk_update_chart(host, "dummy-chart", 0);
1486
+#endif
1487
+ rrdhost_unlock(host);
1488
+ }
1489
+ }
1490
+
1491
+ rrd_unlock();
1492
+}
1493
+
1494
// ----------------------------------------------------------------------------
1495
// RRDHOST - set system info from environment variables
1496
// system_info fields must be heap allocated or NULL
database/rrdset.c
+5
-10
@@ -194,6 +194,8 @@ inline void rrdset_is_obsolete(RRDSET *st) {
194
195
if(unlikely(!(rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
196
rrdset_flag_set(st, RRDSET_FLAG_OBSOLETE);
197
+ st->rrdhost->obsolete_charts_count++;
198
+
199
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
200
201
// the chart will not get more updates (data collection)
@@ -205,6 +207,8 @@ inline void rrdset_is_obsolete(RRDSET *st) {
207
inline void rrdset_isnot_obsolete(RRDSET *st) {
208
if(unlikely((rrdset_flag_check(st, RRDSET_FLAG_OBSOLETE)))) {
209
rrdset_flag_clear(st, RRDSET_FLAG_OBSOLETE);
210
+ st->rrdhost->obsolete_charts_count--;
211
+
212
rrdset_flag_clear(st, RRDSET_FLAG_UPSTREAM_EXPOSED);
213
214
// the chart will be pushed upstream automatically
@@ -452,7 +456,7 @@ void rrdset_delete_custom(RRDSET *st, int db_rotated) {
456
#ifdef ENABLE_ACLK
457
if ((netdata_cloud_setting) && (db_rotated || RRD_MEMORY_MODE_DBENGINE != st->rrd_memory_mode)) {
458
aclk_del_collector(st->rrdhost, st->plugin_name, st->module_name);
455
- st->rrdhost->obsolete_count++;
459
+ st->rrdhost->deleted_charts_count++;
460
}
461
#endif
462
@@ -932,15 +936,6 @@ RRDSET *rrdset_create_custom(
936
937
store_active_chart(st->chart_uuid);
938
935
-#ifdef ENABLE_ACLK
936
- host->obsolete_count = 0;
937
-#endif
938
- rrdhost_cleanup_obsolete_charts(host);
939
-#ifdef ENABLE_ACLK
940
- if (host->obsolete_count)
941
- aclk_update_chart(st->rrdhost, "dummy-chart", 0);
942
-#endif
943
-
939
rrdhost_unlock(host);
940
#ifdef ENABLE_ACLK
941
if (netdata_cloud_setting)
libnetdata/config/appconfig.c
+25
@@ -225,6 +225,31 @@ void appconfig_section_destroy_non_loaded(struct config *root, const char *secti
225
error("Cannot remove section '%s' from config.", section);
226
return;
227
}
228
+
229
+ appconfig_wrlock(root);
230
+
231
+ if (root->first_section == co) {
232
+ root->first_section = co->next;
233
+
234
+ if (root->last_section == co)
235
+ root->last_section = root->first_section;
236
+ } else {
237
+ struct section *co_cur = root->first_section, *co_prev = NULL;
238
+
239
+ while(co_cur && co_cur != co) {
240
+ co_prev = co_cur;
241
+ co_cur = co_cur->next;
242
+ }
243
+
244
+ if (co_cur) {
245
+ co_prev->next = co_cur->next;
246
+
247
+ if (root->last_section == co_cur)
248
+ root->last_section = co_prev;
249
+ }
250
+ }
251
+
252
+ appconfig_unlock(root);
253
254
avl_destroy_lock(&co->values_index);
255
freez(co->name);