fix crash on start on slow disks because ml is initialized before dbengine starts (#13342)
Costa Tsaousis committed
Jul 8, 2022 at 20:18 UTC
329ef5ebef088cdf691c1dcce4931ded96880e83
2 files changed
+66
-72
daemon/main.c
+18
@@ -714,6 +714,24 @@ static void get_netdata_configured_variables() {
714
enable_metric_correlations = config_get_boolean(CONFIG_SECTION_GLOBAL, "enable metric correlations", enable_metric_correlations);
715
default_metric_correlations_method = mc_string_to_method(config_get(CONFIG_SECTION_GLOBAL, "metric correlations method", mc_method_to_string(default_metric_correlations_method)));
716
717
+ // --------------------------------------------------------------------
718
+
719
+ rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
720
+ // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
721
+ // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
722
+ // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
723
+ if (rrdset_free_obsolete_time < 10) {
724
+ rrdset_free_obsolete_time = 10;
725
+ info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
726
+ config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
727
+ }
728
+
729
+ gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
730
+ if (gap_when_lost_iterations_above < 1) {
731
+ gap_when_lost_iterations_above = 1;
732
+ config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
733
+ }
734
+
735
// --------------------------------------------------------------------
736
// get various system parameters
737
database/rrdhost.c
+48
-72
@@ -742,6 +742,12 @@ restart_after_removal:
742
743
int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
744
745
+ if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
746
+ if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
747
+ fatal("Failed to initialize SQLite");
748
+ info("Skipping SQLITE metadata initialization since memory mode is not dbengine");
749
+ }
750
+
751
#ifdef ENABLE_DBENGINE
752
storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
753
if(storage_tiers < 1) {
@@ -773,63 +779,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
779
rrdeng_page_descr_use_mmap();
780
else
781
rrdeng_page_descr_use_malloc();
776
-#endif
777
-
778
- rrdset_free_obsolete_time = config_get_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
779
- // Current chart locking and invalidation scheme doesn't prevent Netdata from segmentation faults if a short
780
- // cleanup delay is set. Extensive stress tests showed that 10 seconds is quite a safe delay. Look at
781
- // https://github.com/netdata/netdata/pull/11222#issuecomment-868367920 for more information.
782
- if (rrdset_free_obsolete_time < 10) {
783
- rrdset_free_obsolete_time = 10;
784
- info("The \"cleanup obsolete charts after seconds\" option was set to 10 seconds.");
785
- config_set_number(CONFIG_SECTION_DB, "cleanup obsolete charts after secs", rrdset_free_obsolete_time);
786
- }
787
-
788
- gap_when_lost_iterations_above = (int)config_get_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
789
- if (gap_when_lost_iterations_above < 1) {
790
- gap_when_lost_iterations_above = 1;
791
- config_set_number(CONFIG_SECTION_DB, "gap when lost iterations above", gap_when_lost_iterations_above);
792
- }
793
-
794
- if (unlikely(sql_init_database(DB_CHECK_NONE, system_info ? 0 : 1))) {
795
- if (default_rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE)
796
- fatal("Failed to initialize SQLite");
797
- info("Skipping SQLITE metadata initialization since memory mode is not db engine");
798
- }
799
-
800
- health_init();
801
- rrdpush_init();
802
-
803
- debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
804
- rrd_wrlock();
805
- localhost = rrdhost_create(
806
- hostname
807
- , registry_get_this_machine_hostname()
808
- , registry_get_this_machine_guid()
809
- , os_type
810
- , netdata_configured_timezone
811
- , netdata_configured_abbrev_timezone
812
- , netdata_configured_utc_offset
813
- , ""
814
- , program_name
815
- , program_version
816
- , default_rrd_update_every
817
- , default_rrd_history_entries
818
- , default_rrd_memory_mode
819
- , default_health_enabled
820
- , default_rrdpush_enabled
821
- , default_rrdpush_destination
822
- , default_rrdpush_api_key
823
- , default_rrdpush_send_charts_matching
824
- , system_info
825
- , 1
826
- );
827
- if (unlikely(!localhost)) {
828
- rrd_unlock();
829
- return 1;
830
- }
782
832
-#ifdef ENABLE_DBENGINE
783
rrdeng_page_descr_aral_go_singlethreaded();
784
785
int created_tiers = 0;
@@ -837,13 +787,13 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
787
char dbengineconfig[200 + 1];
788
for(int tier = 0; tier < storage_tiers ;tier++) {
789
if(tier == 0)
840
- snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", localhost->cache_dir);
790
+ snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine", netdata_configured_cache_dir);
791
else
842
- snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%d", localhost->cache_dir, tier);
792
+ snprintfz(dbenginepath, FILENAME_MAX, "%s/dbengine-tier%d", netdata_configured_cache_dir, tier);
793
794
int ret = mkdir(dbenginepath, 0775);
795
if (ret != 0 && errno != EEXIST) {
846
- error("DBENGINE on '%s': cannot create directory '%s'", localhost->hostname, dbenginepath);
796
+ error("DBENGINE on '%s': cannot create directory '%s'", hostname, dbenginepath);
797
break;
798
}
799
@@ -864,7 +814,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
814
if(grouping_iterations < 2) {
815
grouping_iterations = 2;
816
config_set_number(CONFIG_SECTION_DB, dbengineconfig, grouping_iterations);
867
- error("DBENGINE on '%s': 'dbegnine tier %d update every iterations' cannot be less than 2. Assuming 2.", localhost->hostname, tier);
817
+ error("DBENGINE on '%s': 'dbegnine tier %d update every iterations' cannot be less than 2. Assuming 2.", hostname, tier);
818
}
819
820
snprintfz(dbengineconfig, 200, "dbengine tier %d backfill", tier);
@@ -884,7 +834,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
834
835
if(tier > 0 && get_tier_grouping(tier) > 65535) {
836
storage_tiers_grouping_iterations[tier] = 1;
887
- error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", localhost->hostname, tier, tier);
837
+ error("DBENGINE on '%s': dbengine tier %d gives aggregation of more than 65535 points of tier 0. Disabling tiers above %d", hostname, tier, tier);
838
break;
839
}
840
@@ -892,7 +842,7 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
842
ret = rrdeng_init(NULL, NULL, dbenginepath, page_cache_mb, disk_space_mb, tier);
843
if(ret != 0) {
844
error("DBENGINE on '%s': Failed to initialize multi-host database tier %d on path '%s'",
895
- localhost->hostname, tier, dbenginepath);
845
+ hostname, tier, dbenginepath);
846
break;
847
}
848
else
@@ -901,28 +851,54 @@ int rrd_init(char *hostname, struct rrdhost_system_info *system_info) {
851
852
if(created_tiers && created_tiers < storage_tiers) {
853
error("DBENGINE on '%s': Managed to create %d tiers instead of %d. Continuing with %d available.",
904
- localhost->hostname, created_tiers, storage_tiers, created_tiers);
854
+ hostname, created_tiers, storage_tiers, created_tiers);
855
storage_tiers = created_tiers;
856
}
907
- else if(!created_tiers) {
908
- error("DBENGINE on '%s', with machine guid '%s', failed to initialize databases at '%s'.",
909
- localhost->hostname, localhost->machine_guid, localhost->cache_dir);
910
- rrdhost_free(localhost);
911
- localhost = NULL;
912
- rrd_unlock();
913
- fatal("DBENGINE: Failed to be initialized.");
914
- }
857
+ else if(!created_tiers)
858
+ fatal("DBENGINE on '%s', failed to initialize databases at '%s'.", hostname, netdata_configured_cache_dir);
859
860
rrdeng_page_descr_aral_go_multithreaded();
861
#else
862
storage_tiers = config_get_number(CONFIG_SECTION_DB, "storage tiers", 1);
863
if(storage_tiers != 1) {
920
- error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", localhost->hostname);
864
+ error("DBENGINE is not available on '%s', so only 1 database tier can be supported.", hostname);
865
storage_tiers = 1;
866
config_set_number(CONFIG_SECTION_DB, "storage tiers", storage_tiers);
867
}
868
#endif
869
870
+ health_init();
871
+ rrdpush_init();
872
+
873
+ debug(D_RRDHOST, "Initializing localhost with hostname '%s'", hostname);
874
+ rrd_wrlock();
875
+ localhost = rrdhost_create(
876
+ hostname
877
+ , registry_get_this_machine_hostname()
878
+ , registry_get_this_machine_guid()
879
+ , os_type
880
+ , netdata_configured_timezone
881
+ , netdata_configured_abbrev_timezone
882
+ , netdata_configured_utc_offset
883
+ , ""
884
+ , program_name
885
+ , program_version
886
+ , default_rrd_update_every
887
+ , default_rrd_history_entries
888
+ , default_rrd_memory_mode
889
+ , default_health_enabled
890
+ , default_rrdpush_enabled
891
+ , default_rrdpush_destination
892
+ , default_rrdpush_api_key
893
+ , default_rrdpush_send_charts_matching
894
+ , system_info
895
+ , 1
896
+ );
897
+ if (unlikely(!localhost)) {
898
+ rrd_unlock();
899
+ return 1;
900
+ }
901
+
902
if (likely(system_info))
903
migrate_localhost(&localhost->host_uuid);
904
sql_aclk_sync_init();