Fixes required to make the agent work without crashes on MacOS (#14304)
* Bump the soft limit on open FDs to the max. On systems with a low soft-limit for open file descriptors, the agent would fail to initialize all dbengine tiers. * Iterate the right number of dbengine tiers. For whatever reason, this was causing a crash on MacOS but it was running "correctly" on Linux systems.
vkalintiris committed
Jan 20, 2023 at 18:31 UTC
790ef9dfc8670527724b27d17d8decb8adb88601
4 files changed
+31
-10
daemon/main.c
+27
-6
@@ -591,6 +591,32 @@ int killpid(pid_t pid) {
591
return ret;
592
}
593
594
+static void set_nofile_limit(struct rlimit *rl) {
595
+ // get the num files allowed
596
+ if(getrlimit(RLIMIT_NOFILE, rl) != 0) {
597
+ error("getrlimit(RLIMIT_NOFILE) failed");
598
+ return;
599
+ }
600
+
601
+ info("resources control: allowed file descriptors: soft = %zu, max = %zu",
602
+ (size_t) rl->rlim_cur, (size_t) rl->rlim_max);
603
+
604
+ // make the soft/hard limits equal
605
+ rl->rlim_cur = rl->rlim_max;
606
+ if (setrlimit(RLIMIT_NOFILE, rl) != 0) {
607
+ error("setrlimit(RLIMIT_NOFILE, { %llu, %llu }) failed", rl->rlim_cur, rl->rlim_max);
608
+ }
609
+
610
+ // sanity check to make sure we have enough file descriptors available to open
611
+ if (getrlimit(RLIMIT_NOFILE, rl) != 0) {
612
+ error("getrlimit(RLIMIT_NOFILE) failed");
613
+ return;
614
+ }
615
+
616
+ if (rl->rlim_cur < 1024)
617
+ error("Number of open file descriptors allowed for this process is too low (RLIMIT_NOFILE=%zu)", (size_t) rl->rlim_cur);
618
+}
619
+
620
void cancel_main_threads() {
621
error_log_limit_unlimited();
622
@@ -1883,12 +1909,7 @@ int main(int argc, char **argv) {
1909
}
1910
#endif /* NETDATA_INTERNAL_CHECKS */
1911
1886
- // get the max file limit
1887
- if(getrlimit(RLIMIT_NOFILE, &rlimit_nofile) != 0)
1888
- error("getrlimit(RLIMIT_NOFILE) failed");
1889
- else
1890
- info("resources control: allowed file descriptors: soft = %zu, max = %zu", (size_t)rlimit_nofile.rlim_cur, (size_t)rlimit_nofile.rlim_max);
1891
-
1912
+ set_nofile_limit(&rlimit_nofile);
1913
1914
delta_startup_time("become daemon");
1915
database/engine/journalfile.c
+2
-2
@@ -213,10 +213,10 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo
213
return unmounted;
214
}
215
216
-void journalfile_v2_data_unmount_cleanup(time_t now_s) {
216
+void journalfile_v2_data_unmount_cleanup(time_t now_s, int storage_tiers) {
217
// DO NOT WAIT ON ANY LOCK!!!
218
219
- for(size_t tier = 0; tier < RRD_STORAGE_TIERS ;tier++) {
219
+ for(size_t tier = 0; tier < storage_tiers ;tier++) {
220
struct rrdengine_instance *ctx = multidb_ctx[tier];
221
if(!ctx) continue;
222
database/engine/journalfile.h
+1
-1
@@ -152,6 +152,6 @@ size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile);
152
void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size);
153
struct journal_v2_header *journalfile_v2_data_acquire(struct rrdengine_journalfile *journalfile, size_t *data_size, time_t wanted_first_time_s, time_t wanted_last_time_s);
154
void journalfile_v2_data_release(struct rrdengine_journalfile *journalfile);
155
-void journalfile_v2_data_unmount_cleanup(time_t now_s);
155
+void journalfile_v2_data_unmount_cleanup(time_t now_s, int storage_tiers);
156
157
#endif /* NETDATA_JOURNALFILE_H */
\ No newline at end of file
database/engine/rrdengine.c
+1
-1
@@ -1487,7 +1487,7 @@ void timer_cb(uv_timer_t* handle) {
1487
time_t now_s = now_monotonic_sec();
1488
if(now_s - last_run_s >= 10) {
1489
last_run_s = now_s;
1490
- journalfile_v2_data_unmount_cleanup(now_s);
1490
+ journalfile_v2_data_unmount_cleanup(now_s, storage_tiers);
1491
}
1492
}
1493