3a77de1f5 · cryptotaxi247/netdata-1

@@ -226,7 +226,8 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

226 // signal them to stop

227 size_t last_running = 0;

228 size_t stale_time_ut = 0;

229 - usec_t sleep_ut = 500 * USEC_PER_MS;

229 + usec_t sleep_ut = 50 * USEC_PER_MS;

230 + size_t log_countdown_ut = sleep_ut;

231 do {

232 if(running != last_running)

233 stale_time_ut = 0;

@@ -257,11 +258,17 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

258 netdata_spinlock_unlock(&service_globals.lock);

259

260 if(running) {

260 - buffer_flush(service_list);

261 - service_to_buffer(service_list, running_services);

262 - info("SERVICE CONTROL: waiting for the following %zu services [ %s] to exit: %s",

263 - running, buffer_tostring(service_list),

264 - running <= 10 ? buffer_tostring(thread_list) : "");

261 + log_countdown_ut -= (log_countdown_ut >= sleep_ut) ? sleep_ut : log_countdown_ut;

262 + if(log_countdown_ut == 0 || running != last_running) {

263 + log_countdown_ut = 20 * sleep_ut;

264 +

265 + buffer_flush(service_list);

266 + service_to_buffer(service_list, running_services);

267 + info("SERVICE CONTROL: waiting for the following %zu services [ %s] to exit: %s",

268 + running, buffer_tostring(service_list),

269 + running <= 10 ? buffer_tostring(thread_list) : "");

270 + }

271 +

272 sleep_usec(sleep_ut);

273 stale_time_ut += sleep_ut;

274 }

@@ -285,18 +292,39 @@ static bool service_wait_exit(SERVICE_TYPE service, usec_t timeout_ut) {

292 return (running == 0);

293 }

294

295 +#define delta_shutdown_time(msg) \

296 + { \

297 + usec_t now_ut = now_monotonic_usec(); \

298 + if(prev_msg) \

299 + info("NETDATA SHUTDOWN: in %7llu ms, %s%s - next: %s", (now_ut - last_ut) / USEC_PER_MS, (timeout)?"(TIMEOUT) ":"", prev_msg, msg); \

300 + else \

301 + info("NETDATA SHUTDOWN: next: %s", msg); \

302 + last_ut = now_ut; \

303 + prev_msg = msg; \

304 + timeout = false; \

305 + }

306 +

307 void netdata_cleanup_and_exit(int ret) {

308 + usec_t started_ut = now_monotonic_usec();

309 + usec_t last_ut = started_ut;

310 + const char *prev_msg = NULL;

311 + bool timeout = false;

312 +

313 error_log_limit_unlimited();

290 - info("EXIT: netdata prepares to exit with code %d...", ret);

314 + info("NETDATA SHUTDOWN: initializing shutdown with code %d...", ret);

315

316 send_statistics("EXIT", ret?"ERROR":"OK","-");

317

318 + delta_shutdown_time("create shutdown file");

319 +

320 char agent_crash_file[FILENAME_MAX + 1];

321 char agent_incomplete_shutdown_file[FILENAME_MAX + 1];

322 snprintfz(agent_crash_file, FILENAME_MAX, "%s/.agent_crash", netdata_configured_varlib_dir);

323 snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir);

324 (void) rename(agent_crash_file, agent_incomplete_shutdown_file);

325

326 + delta_shutdown_time("disable maintenance, new queries, new web requests, new streaming connections and aclk");

327 +

328 service_signal_exit(

329 SERVICE_MAINTENANCE

330 | ABILITY_DATA_QUERIES

@@ -305,7 +333,9 @@ void netdata_cleanup_and_exit(int ret) {

333 | SERVICE_ACLK

334 );

335

308 - service_wait_exit(

336 + delta_shutdown_time("stop replication, exporters, ML training, health and web servers threads");

337 +

338 + timeout = !service_wait_exit(

339 SERVICE_REPLICATION

340 | SERVICE_EXPORTERS

341 | SERVICE_ML_TRAINING

@@ -313,38 +343,51 @@ void netdata_cleanup_and_exit(int ret) {

343 | SERVICE_WEB_SERVER

344 , 3 * USEC_PER_SEC);

345

316 - service_wait_exit(

346 + delta_shutdown_time("stop collectors and streaming threads");

347 +

348 + timeout = !service_wait_exit(

349 SERVICE_COLLECTORS

350 | SERVICE_STREAMING

351 , 3 * USEC_PER_SEC);

352

321 - service_wait_exit(

353 + delta_shutdown_time("stop ML prediction and context threads");

354 +

355 + timeout = !service_wait_exit(

356 SERVICE_ML_PREDICTION

357 | SERVICE_CONTEXT

358 , 3 * USEC_PER_SEC);

359

326 - service_wait_exit(

360 + delta_shutdown_time("stop maintenance thread");

361 +

362 + timeout = !service_wait_exit(

363 SERVICE_MAINTENANCE

364 , 3 * USEC_PER_SEC);

365

330 - info("EXIT: cleaning up the database...");

366 + delta_shutdown_time("clean rrdhost database");

367 +

368 rrdhost_cleanup_all();

369

333 - info("EXIT: metasync shutdown prepare...");

370 + delta_shutdown_time("prepare metasync shutdown");

371 +

372 metadata_sync_shutdown_prepare();

373

374 #ifdef ENABLE_ACLK

375 + delta_shutdown_time("signal aclk sync to stop");

376 aclk_sync_exit_all();

377 #endif

378

340 - service_wait_exit(

379 + delta_shutdown_time("stop aclk threads");

380 +

381 + timeout = !service_wait_exit(

382 SERVICE_ACLK

383 , 3 * USEC_PER_SEC);

384

344 - // stop everything else

345 - service_wait_exit(~0, 10 * USEC_PER_SEC);

385 + delta_shutdown_time("stop all remaining worker threads");

386 +

387 + timeout = !service_wait_exit(~0, 10 * USEC_PER_SEC);

388 +

389 + delta_shutdown_time("cancel main threads");

390

347 - info("EXIT: stopping static threads...");

391 cancel_main_threads();

392

393 if(!ret) {

@@ -352,42 +395,59 @@ void netdata_cleanup_and_exit(int ret) {

395

396 #ifdef ENABLE_DBENGINE

397 if(dbengine_enabled) {

355 - info("EXIT: flushing dbengine...");

398 + delta_shutdown_time("flush dbengine tiers");

399 for (size_t tier = 0; tier < storage_tiers; tier++)

400 rrdeng_prepare_exit(multidb_ctx[tier]);

401 }

402 #endif

403

404 // free the database

362 - info("EXIT: freeing database memory...");

405 + delta_shutdown_time("free rrdhost structures");

406 +

407 rrdhost_free_all();

408

409 + delta_shutdown_time("stop metasync threads");

410 +

411 metadata_sync_shutdown();

412

413 #ifdef ENABLE_DBENGINE

414 if(dbengine_enabled) {

369 - info("EXIT: stopping dbengine...");

415 + delta_shutdown_time("stop dbengine tiers");

416 for (size_t tier = 0; tier < storage_tiers; tier++)

417 rrdeng_exit(multidb_ctx[tier]);

418 }

419 #endif

420 }

421

422 + delta_shutdown_time("close SQL context db");

423 +

424 sql_close_context_database();

425 +

426 + delta_shutdown_time("closed SQL main db");

427 +

428 sql_close_database();

429

430 // unlink the pid

431 if(pidfile[0]) {

381 - info("EXIT: removing netdata PID file '%s'...", pidfile);

432 + delta_shutdown_time("remove pid file");

433 +

434 if(unlink(pidfile) != 0)

435 error("EXIT: cannot unlink pidfile '%s'.", pidfile);

436 }

437

438 #ifdef ENABLE_HTTPS

439 + delta_shutdown_time("free openssl structures");

440 security_clean_openssl();

441 #endif

389 - info("EXIT: all done - netdata is now exiting - bye bye...");

442 +

443 + delta_shutdown_time("remove incomplete shutdown file");

444 +

445 (void) unlink(agent_incomplete_shutdown_file);

446 +

447 + delta_shutdown_time("exit");

448 +

449 + usec_t ended_ut = now_monotonic_usec();

450 + info("NETDATA SHUTDOWN: completed in %llu ms - netdata is now exiting - bye bye...", (ended_ut - started_ut) / USEC_PER_MS);

451 exit(ret);

452 }

453

@@ -1178,11 +1238,28 @@ void post_conf_load(char **user)

1238 appconfig_get(&cloud_config, CONFIG_SECTION_GLOBAL, "cloud base url", DEFAULT_CLOUD_BASE_URL);

1239 }

1240

1241 +#define delta_startup_time(msg) \

1242 + { \

1243 + usec_t now_ut = now_monotonic_usec(); \

1244 + if(prev_msg) \

1245 + info("NETDATA STARTUP: in %7llu ms, %s - next: %s", (now_ut - last_ut) / USEC_PER_MS, prev_msg, msg); \

1246 + else \

1247 + info("NETDATA STARTUP: next: %s", msg); \

1248 + last_ut = now_ut; \

1249 + prev_msg = msg; \

1250 + }

1251 +

1252 int pgc_unittest(void);

1253 int mrg_unittest(void);

1254 int julytest(void);

1255

1256 int main(int argc, char **argv) {

1257 + // initialize the system clocks

1258 + clocks_init();

1259 + usec_t started_ut = now_monotonic_usec();

1260 + usec_t last_ut = started_ut;

1261 + const char *prev_msg = NULL;

1262 +

1263 int i;

1264 int config_loaded = 0;

1265 int dont_fork = 0;

@@ -1629,9 +1706,7 @@ int main(int argc, char **argv) {

1706 }

1707 #endif

1708

1632 -

1633 - if(!config_loaded)

1634 - {

1709 + if(!config_loaded) {

1710 load_netdata_conf(NULL, 0);

1711 post_conf_load(&user);

1712 load_cloud_conf(0);

@@ -1642,7 +1717,6 @@ int main(int argc, char **argv) {

1717 appconfig_set(&cloud_config, CONFIG_SECTION_GLOBAL, "enabled", "false");

1718 }

1719

1645 -

1720 // ------------------------------------------------------------------------

1721 // initialize netdata

1722 {

@@ -1662,9 +1736,6 @@ int main(int argc, char **argv) {

1736 #endif

1737 #endif

1738

1665 - // initialize the system clocks

1666 - clocks_init();

1667 -

1739 // set libuv worker threads

1740 libuv_worker_threads = get_system_cpus() * 2;

1741

@@ -1749,6 +1820,7 @@ int main(int argc, char **argv) {

1820 // --------------------------------------------------------------------

1821 // Initialize ML configuration

1822

1823 + delta_startup_time("initialize ML");

1824 ml_init();

1825

1826 // --------------------------------------------------------------------

@@ -1757,18 +1829,18 @@ int main(int argc, char **argv) {

1829 // block signals while initializing threads.

1830 // this causes the threads to block signals.

1831

1832 + delta_startup_time("initialize signals");

1833 signals_block();

1834 + signals_init(); // setup the signals we want to use

1835

1762 - // setup the signals we want to use

1836 + // --------------------------------------------------------------------

1837 + // check which threads are enabled and initialize them

1838

1764 - signals_init();

1839 + delta_startup_time("initialize static threads");

1840

1841 // setup threads configs

1842 default_stacksize = netdata_threads_init();

1843

1769 - // --------------------------------------------------------------------

1770 - // check which threads are enabled and initialize them

1771 -

1844 for (i = 0; static_threads[i].name != NULL ; i++) {

1845 struct netdata_static_thread *st = &static_threads[i];

1846

@@ -1788,14 +1860,17 @@ int main(int argc, char **argv) {

1860 // --------------------------------------------------------------------

1861 // create the listening sockets

1862

1863 + delta_startup_time("initialize web server");

1864 +

1865 web_client_api_v1_init();

1866 web_server_threading_selection();

1867

1868 if(web_server_mode != WEB_SERVER_MODE_NONE)

1869 api_listen_sockets_setup();

1796 -

1870 }

1871

1872 + delta_startup_time("set resource limits");

1873 +

1874 #ifdef NETDATA_INTERNAL_CHECKS

1875 if(debug_flags != 0) {

1876 struct rlimit rl = { RLIM_INFINITY, RLIM_INFINITY };

@@ -1813,18 +1888,27 @@ int main(int argc, char **argv) {

1888 else

1889 info("resources control: allowed file descriptors: soft = %zu, max = %zu", (size_t)rlimit_nofile.rlim_cur, (size_t)rlimit_nofile.rlim_max);

1890

1891 +

1892 + delta_startup_time("become daemon");

1893 +

1894 // fork, switch user, create pid file, set process priority

1895 if(become_daemon(dont_fork, user) == -1)

1896 fatal("Cannot daemonize myself.");

1897

1898 info("netdata started on pid %d.", getpid());

1899

1900 + delta_startup_time("initialize threads after fork");

1901 +

1902 netdata_threads_init_after_fork((size_t)config_get_number(CONFIG_SECTION_GLOBAL, "pthread stack size", (long)default_stacksize));

1903

1904 // initialize internal registry

1905 + delta_startup_time("initialize registry");

1906 registry_init();

1907 +

1908 // fork the spawn server

1909 + delta_startup_time("fork the spawn server");

1910 spawn_init();

1911 +

1912 /*

1913 * Libuv uv_spawn() uses SIGCHLD internally:

1914 * https://github.com/libuv/libuv/blob/cc51217a317e96510fbb284721d5e6bc2af31e33/src/unix/process.c#L485

@@ -1837,15 +1921,21 @@ int main(int argc, char **argv) {

1921 // ------------------------------------------------------------------------

1922 // initialize rrd, registry, health, rrdpush, etc.

1923

1924 + delta_startup_time("collecting system info");

1925 +

1926 netdata_anonymous_statistics_enabled=-1;

1927 struct rrdhost_system_info *system_info = callocz(1, sizeof(struct rrdhost_system_info));

1928 get_system_info(system_info);

1929 system_info->hops = 0;

1930 get_install_type(&system_info->install_type, &system_info->prebuilt_arch, &system_info->prebuilt_dist);

1931

1932 + delta_startup_time("initialize RRD structures");

1933 +

1934 if(rrd_init(netdata_configured_hostname, system_info, false))

1935 fatal("Cannot initialize localhost instance with name '%s'.", netdata_configured_hostname);

1936

1937 + delta_startup_time("check for incomplete shutdown");

1938 +

1939 char agent_crash_file[FILENAME_MAX + 1];

1940 char agent_incomplete_shutdown_file[FILENAME_MAX + 1];

1941 snprintfz(agent_incomplete_shutdown_file, FILENAME_MAX, "%s/.agent_incomplete_shutdown", netdata_configured_varlib_dir);

@@ -1860,6 +1950,8 @@ int main(int argc, char **argv) {

1950 // ------------------------------------------------------------------------

1951 // Claim netdata agent to a cloud endpoint

1952

1953 + delta_startup_time("collect claiming info");

1954 +

1955 if (claiming_pending_arguments)

1956 claim_agent(claiming_pending_arguments);

1957 load_claiming_state();

@@ -1870,11 +1962,14 @@ int main(int argc, char **argv) {

1962 error_log_limit_reset();

1963

1964 // Load host labels

1965 + delta_startup_time("collect host labels");

1966 reload_host_labels();

1967

1968 // ------------------------------------------------------------------------

1969 // spawn the threads

1970

1971 + delta_startup_time("start the static threads");

1972 +

1973 web_server_config_options();

1974

1975 netdata_zero_metrics_enabled = config_get_boolean_ondemand(CONFIG_SECTION_DB, "enable zero metrics", CONFIG_BOOLEAN_NO);

@@ -1895,9 +1990,14 @@ int main(int argc, char **argv) {

1990 // ------------------------------------------------------------------------

1991 // Initialize netdata agent command serving from cli and signals

1992

1993 + delta_startup_time("initialize commands API");

1994 +

1995 commands_init();

1996

1900 - info("netdata initialization completed. Enjoy real-time performance monitoring!");

1997 + delta_startup_time("ready");

1998 +

1999 + usec_t ready_ut = now_monotonic_usec();

2000 + info("NETDATA STARTUP: completed in %llu ms. Enjoy real-time performance monitoring!", (ready_ut - started_ut) / USEC_PER_MS);

2001 netdata_ready = 1;

2002

2003 send_statistics("START", "-", "-");

profile startup and shutdown timings (#14243)