@cryptotaxi247 / netdata-1 / commits / eb216a1f4

Workers utilization charts (#12807)

* initial version of worker utilization * working example * without mutexes * monitoring DBENGINE, ACLKSYNC, WEB workers * added charts to monitor worker usage * fixed charts units * updated contexts * updated priorities * added documentation * converted threads to stacked chart * One query per query thread * Revert "One query per query thread" This reverts commit 6aeb391f5987c3c6ba2864b559fd7f0cd64b14d3. * fixed priority for web charts * read worker cpu utilization from proc * read workers cpu utilization via /proc/self/task/PID/stat, so that we have cpu utilization even when the jobs are too long to finish within our update_every frequency * disabled web server cpu utilization monitoring - it is now monitored by worker utilization * tight integration of worker utilization to web server * monitoring statsd worker threads * code cleanup and renaming of variables * contrained worker and statistics conflict to just one variable * support for rendering jobs per type * better priorities and removed the total jobs chart * added busy time in ms per job type * added proc.plugin monitoring, switch clock to MONOTONIC_RAW if available, global statistics now cleans up old worker threads * isolated worker thread families * added cgroups.plugin workers * remove unneeded dimensions when then expected worker is just one * plugins.d and streaming monitoring * rebased; support worker_is_busy() to be called one after another * added diskspace plugin monitoring * added tc.plugin monitoring * added ML threads monitoring * dont create dimensions and charts that are not needed * fix crash when job types are added on the fly * added timex and idlejitter plugins; collected heartbeat statistics; reworked heartbeat according to the POSIX * the right name is heartbeat for this chart * monitor streaming senders * added streaming senders to global stats * prevent division by zero * added clock_init() to external C plugins * added freebsd and macos plugins * added freebsd and macos to global statistics * dont use new as a variable; address compiler warnings on FreeBSD and MacOS * refactored contexts to be unique; added health threads monitoring Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com>

Costa Tsaousis committed May 9, 2022 at 16:34 UTC eb216a1f4bbb26e1f18537b30d22e8ad8711f42c
42 files changed +2070 -1096
CMakeLists.txt
+2
@@ -410,6 +410,8 @@ set(LIBNETDATA_FILES
410 libnetdata/string/utf8.h
411 libnetdata/socket/security.c
412 libnetdata/socket/security.h
413 + libnetdata/worker_utilization/worker_utilization.c
414 + libnetdata/worker_utilization/worker_utilization.h
415 libnetdata/circular_buffer/circular_buffer.c
416 libnetdata/circular_buffer/circular_buffer.h)
417
Makefile.am
+2
@@ -187,6 +187,8 @@ LIBNETDATA_FILES = \
187 libnetdata/health/health.c \
188 libnetdata/health/health.h \
189 libnetdata/string/utf8.h \
190 + libnetdata/worker_utilization/worker_utilization.c \
191 + libnetdata/worker_utilization/worker_utilization.h \
192 $(NULL)
193
194 if ENABLE_PLUGIN_EBPF
aclk/aclk_query.c
+16
@@ -351,6 +351,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que
351 {
352 for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) {
353 if (aclk_query_handlers[i].type == query->type) {
354 + worker_is_busy(i);
355 +
356 debug(D_ACLK, "Processing Queued Message of type: \"%s\"", aclk_query_handlers[i].name);
357 aclk_query_handlers[i].fnc(query_thr, query);
358 if (aclk_stats_enabled) {
@@ -361,6 +363,8 @@ static void aclk_query_process_msg(struct aclk_query_thread *query_thr, aclk_que
363 ACLK_STATS_UNLOCK;
364 }
365 aclk_query_free(query);
366 +
367 + worker_is_idle();
368 return;
369 }
370 }
@@ -378,21 +382,33 @@ int aclk_query_process_msgs(struct aclk_query_thread *query_thr)
382 return 0;
383 }
384
385 +static void worker_aclk_register(void) {
386 + worker_register("ACLKQUERY");
387 + for (int i = 0; aclk_query_handlers[i].type != UNKNOWN; i++) {
388 + worker_register_job_name(i, aclk_query_handlers[i].name);
389 + }
390 +}
391 +
392 /**
393 * Main query processing thread
394 */
395 void *aclk_query_main_thread(void *ptr)
396 {
397 + worker_aclk_register();
398 +
399 struct aclk_query_thread *query_thr = ptr;
400
401 while (!netdata_exit) {
402 aclk_query_process_msgs(query_thr);
403
404 + worker_is_idle();
405 QUERY_THREAD_LOCK;
406 if (unlikely(pthread_cond_wait(&query_cond_wait, &query_lock_wait)))
407 sleep_usec(USEC_PER_SEC * 1);
408 QUERY_THREAD_UNLOCK;
409 }
410 +
411 + worker_unregister();
412 return NULL;
413 }
414
collectors/all.h
+1 -3
@@ -360,10 +360,8 @@
360
361 #define NETDATA_CHART_PRIO_CHECKS 99999
362
363 -#define NETDATA_CHART_PRIO_NETDATA_DISKSPACE 132020
363 #define NETDATA_CHART_PRIO_NETDATA_TIMEX 132030
365 -#define NETDATA_CHART_PRIO_NETDATA_TC_CPU 135000
366 -#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 135001
364 +#define NETDATA_CHART_PRIO_NETDATA_TC_TIME 1000100
365
366
367 #endif //NETDATA_ALL_H
collectors/apps.plugin/apps_plugin.c
+2
@@ -4124,6 +4124,8 @@ static int check_capabilities() {
4124 int main(int argc, char **argv) {
4125 // debug_flags = D_PROCFILE;
4126
4127 + clocks_init();
4128 +
4129 pagesize = (size_t)sysconf(_SC_PAGESIZE);
4130
4131 // set the name for logging
collectors/cgroups.plugin/sys_fs_cgroup.c
+62 -40
@@ -2646,11 +2646,26 @@ static inline void discovery_process_cgroup(struct cgroup *cg) {
2646 read_cgroup_network_interfaces(cg);
2647 }
2648
2649 +#define WORKER_DISCOVERY_INIT 0
2650 +#define WORKER_DISCOVERY_FIND 1
2651 +#define WORKER_DISCOVERY_PROCESS 2
2652 +#define WORKER_DISCOVERY_UPDATE 3
2653 +#define WORKER_DISCOVERY_CLEANUP 4
2654 +#define WORKER_DISCOVERY_COPY 5
2655 +#define WORKER_DISCOVERY_SHARE 6
2656 +#define WORKER_DISCOVERY_LOCK 7
2657 +
2658 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
2659 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
2660 +#endif
2661 +
2662 static inline void discovery_find_all_cgroups() {
2663 debug(D_CGROUP, "searching for cgroups");
2664
2665 + worker_is_busy(WORKER_DISCOVERY_INIT);
2666 discovery_mark_all_cgroups_as_unavailable();
2667
2668 + worker_is_busy(WORKER_DISCOVERY_FIND);
2669 if (!cgroup_use_unified_cgroups) {
2670 discovery_find_all_cgroups_v1();
2671 } else {
@@ -2659,16 +2674,25 @@ static inline void discovery_find_all_cgroups() {
2674
2675 struct cgroup *cg;
2676 for (cg = discovered_cgroup_root; cg; cg = cg->discovered_next) {
2677 + worker_is_busy(WORKER_DISCOVERY_PROCESS);
2678 discovery_process_cgroup(cg);
2679 }
2680
2681 + worker_is_busy(WORKER_DISCOVERY_UPDATE);
2682 discovery_update_filenames();
2683
2684 + worker_is_busy(WORKER_DISCOVERY_LOCK);
2685 uv_mutex_lock(&cgroup_root_mutex);
2686 +
2687 + worker_is_busy(WORKER_DISCOVERY_CLEANUP);
2688 discovery_cleanup_all_cgroups();
2689 +
2690 + worker_is_busy(WORKER_DISCOVERY_COPY);
2691 discovery_copy_discovered_cgroups_to_reader();
2692 +
2693 uv_mutex_unlock(&cgroup_root_mutex);
2694
2695 + worker_is_busy(WORKER_DISCOVERY_SHARE);
2696 discovery_share_cgroups_with_ebpf();
2697
2698 debug(D_CGROUP, "done searching for cgroups");
@@ -2678,7 +2702,19 @@ void cgroup_discovery_worker(void *ptr)
2702 {
2703 UNUSED(ptr);
2704
2705 + worker_register("CGROUPSDISC");
2706 + worker_register_job_name(WORKER_DISCOVERY_INIT, "init");
2707 + worker_register_job_name(WORKER_DISCOVERY_FIND, "find");
2708 + worker_register_job_name(WORKER_DISCOVERY_PROCESS, "process");
2709 + worker_register_job_name(WORKER_DISCOVERY_UPDATE, "update");
2710 + worker_register_job_name(WORKER_DISCOVERY_CLEANUP, "cleanup");
2711 + worker_register_job_name(WORKER_DISCOVERY_COPY, "copy");
2712 + worker_register_job_name(WORKER_DISCOVERY_SHARE, "share");
2713 + worker_register_job_name(WORKER_DISCOVERY_LOCK, "lock");
2714 +
2715 while (!netdata_exit) {
2716 + worker_is_idle();
2717 +
2718 uv_mutex_lock(&discovery_thread.mutex);
2719 while (!discovery_thread.start_discovery)
2720 uv_cond_wait(&discovery_thread.cond_var, &discovery_thread.mutex);
@@ -2692,6 +2728,7 @@ void cgroup_discovery_worker(void *ptr)
2728 }
2729
2730 discovery_thread.exited = 1;
2731 + worker_unregister();
2732 }
2733
2734 // ----------------------------------------------------------------------------
@@ -4650,6 +4687,8 @@ void update_cgroup_charts(int update_every) {
4687 // cgroups main
4688
4689 static void cgroup_main_cleanup(void *ptr) {
4690 + worker_unregister();
4691 +
4692 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
4693 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
4694
@@ -4687,24 +4726,30 @@ static void cgroup_main_cleanup(void *ptr) {
4726 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
4727 }
4728
4729 +#define WORKER_CGROUPS_LOCK 0
4730 +#define WORKER_CGROUPS_READ 1
4731 +#define WORKER_CGROUPS_CHART 2
4732 +
4733 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
4734 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
4735 +#endif
4736 +
4737 void *cgroups_main(void *ptr) {
4691 - netdata_thread_cleanup_push(cgroup_main_cleanup, ptr);
4738 + worker_register("CGROUPS");
4739 + worker_register_job_name(WORKER_CGROUPS_LOCK, "lock");
4740 + worker_register_job_name(WORKER_CGROUPS_READ, "read");
4741 + worker_register_job_name(WORKER_CGROUPS_READ, "chart");
4742
4693 - struct rusage thread;
4743 + netdata_thread_cleanup_push(cgroup_main_cleanup, ptr);
4744
4745 if (getenv("KUBERNETES_SERVICE_HOST") != NULL && getenv("KUBERNETES_SERVICE_PORT") != NULL) {
4746 is_inside_k8s = 1;
4747 cgroup_enable_cpuacct_cpu_shares = CONFIG_BOOLEAN_YES;
4748 }
4749
4700 - // when ZERO, attempt to do it
4701 - int vdo_cpu_netdata = config_get_boolean("plugin:cgroups", "cgroups plugin resource charts", 1);
4702 -
4750 read_cgroup_plugin_configuration();
4751 netdata_cgroup_ebpf_initialize_shm();
4752
4706 - RRDSET *stcpu_thread = NULL;
4707 -
4753 if (uv_mutex_init(&cgroup_root_mutex)) {
4754 error("CGROUP: cannot initialize mutex for the main cgroup list");
4755 goto exit;
@@ -4736,6 +4781,8 @@ void *cgroups_main(void *ptr) {
4781 usec_t find_every = cgroup_check_for_new_every * USEC_PER_SEC, find_dt = 0;
4782
4783 while(!netdata_exit) {
4784 + worker_is_idle();
4785 +
4786 usec_t hb_dt = heartbeat_next(&hb, step);
4787 if(unlikely(netdata_exit)) break;
4788
@@ -4747,46 +4794,21 @@ void *cgroups_main(void *ptr) {
4794 cgroups_check = 0;
4795 }
4796
4797 + worker_is_busy(WORKER_CGROUPS_LOCK);
4798 uv_mutex_lock(&cgroup_root_mutex);
4751 - read_all_discovered_cgroups(cgroup_root);
4752 - update_cgroup_charts(cgroup_update_every);
4753 - uv_mutex_unlock(&cgroup_root_mutex);
4754 -
4755 - // --------------------------------------------------------------------
4756 -
4757 - if(vdo_cpu_netdata) {
4758 - getrusage(RUSAGE_THREAD, &thread);
4799
4760 - if(unlikely(!stcpu_thread)) {
4761 -
4762 - stcpu_thread = rrdset_create_localhost(
4763 - "netdata"
4764 - , "plugin_cgroups_cpu"
4765 - , NULL
4766 - , "cgroups"
4767 - , NULL
4768 - , "Netdata CGroups Plugin CPU usage"
4769 - , "milliseconds/s"
4770 - , PLUGIN_CGROUPS_NAME
4771 - , "stats"
4772 - , 132000
4773 - , cgroup_update_every
4774 - , RRDSET_TYPE_STACKED
4775 - );
4800 + worker_is_busy(WORKER_CGROUPS_READ);
4801 + read_all_discovered_cgroups(cgroup_root);
4802
4777 - rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
4778 - rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
4779 - }
4780 - else
4781 - rrdset_next(stcpu_thread);
4803 + worker_is_busy(WORKER_CGROUPS_CHART);
4804 + update_cgroup_charts(cgroup_update_every);
4805
4783 - rrddim_set(stcpu_thread, "user" , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
4784 - rrddim_set(stcpu_thread, "system", thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
4785 - rrdset_done(stcpu_thread);
4786 - }
4806 + worker_is_idle();
4807 + uv_mutex_unlock(&cgroup_root_mutex);
4808 }
4809
4810 exit:
4811 + worker_unregister();
4812 netdata_thread_cleanup_pop(1);
4813 return NULL;
4814 }
collectors/cups.plugin/cups_plugin.c
+1
@@ -224,6 +224,7 @@ void reset_metrics() {
224 }
225
226 int main(int argc, char **argv) {
227 + clocks_init();
228
229 // ------------------------------------------------------------------------
230 // initialization of netdata plugin
collectors/diskspace.plugin/plugin_diskspace.c
+22 -72
@@ -365,6 +365,8 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
365 }
366
367 static void diskspace_main_cleanup(void *ptr) {
368 + worker_unregister();
369 +
370 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
371 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
372
@@ -373,10 +375,21 @@ static void diskspace_main_cleanup(void *ptr) {
375 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
376 }
377
378 +#define WORKER_JOB_MOUNTINFO 0
379 +#define WORKER_JOB_MOUNTPOINT 1
380 +#define WORKER_JOB_CLEANUP 2
381 +
382 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
383 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
384 +#endif
385 +
386 void *diskspace_main(void *ptr) {
377 - netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
387 + worker_register("DISKSPACE");
388 + worker_register_job_name(WORKER_JOB_MOUNTINFO, "mountinfo");
389 + worker_register_job_name(WORKER_JOB_MOUNTPOINT, "mountpoint");
390 + worker_register_job_name(WORKER_JOB_CLEANUP, "cleanup");
391
379 - int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", 1);
392 + netdata_thread_cleanup_push(diskspace_main_cleanup, ptr);
393
394 cleanup_mount_points = config_get_boolean(CONFIG_SECTION_DISKSPACE, "remove charts of unmounted disks" , cleanup_mount_points);
395
@@ -388,14 +401,11 @@ void *diskspace_main(void *ptr) {
401 if(check_for_new_mountpoints_every < update_every)
402 check_for_new_mountpoints_every = update_every;
403
391 - struct rusage thread;
392 -
393 - usec_t duration = 0;
404 usec_t step = update_every * USEC_PER_SEC;
405 heartbeat_t hb;
406 heartbeat_init(&hb);
407 while(!netdata_exit) {
398 - duration = heartbeat_monotonic_dt_to_now_usec(&hb);
408 + worker_is_idle();
409 /* usec_t hb_dt = */ heartbeat_next(&hb, step);
410
411 if(unlikely(netdata_exit)) break;
@@ -404,9 +414,9 @@ void *diskspace_main(void *ptr) {
414 // --------------------------------------------------------------------------
415 // this is smart enough not to reload it every time
416
417 + worker_is_busy(WORKER_JOB_MOUNTINFO);
418 mountinfo_reload(0);
419
409 -
420 // --------------------------------------------------------------------------
421 // disk space metrics
422
@@ -420,80 +430,20 @@ void *diskspace_main(void *ptr) {
430 if(mi->flags & MOUNTINFO_READONLY && !strcmp(mi->root, mi->mount_point))
431 continue;
432
433 + worker_is_busy(WORKER_JOB_MOUNTPOINT);
434 do_disk_space_stats(mi, update_every);
435 if(unlikely(netdata_exit)) break;
436 }
437
438 if(unlikely(netdata_exit)) break;
439
429 - if(dict_mountpoints)
440 + if(dict_mountpoints) {
441 + worker_is_busy(WORKER_JOB_CLEANUP);
442 dictionary_get_all(dict_mountpoints, mount_point_cleanup, NULL);
431 -
432 - if(vdo_cpu_netdata) {
433 - static RRDSET *stcpu_thread = NULL, *st_duration = NULL;
434 - static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL;
435 -
436 - // ----------------------------------------------------------------
437 -
438 - getrusage(RUSAGE_THREAD, &thread);
439 -
440 - if(unlikely(!stcpu_thread)) {
441 - stcpu_thread = rrdset_create_localhost(
442 - "netdata"
443 - , "plugin_diskspace"
444 - , NULL
445 - , "diskspace"
446 - , NULL
447 - , "Netdata Disk Space Plugin CPU usage"
448 - , "milliseconds/s"
449 - , PLUGIN_DISKSPACE_NAME
450 - , NULL
451 - , NETDATA_CHART_PRIO_NETDATA_DISKSPACE
452 - , update_every
453 - , RRDSET_TYPE_STACKED
454 - );
455 -
456 - rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
457 - rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
458 - }
459 - else
460 - rrdset_next(stcpu_thread);
461 -
462 - rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
463 - rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
464 - rrdset_done(stcpu_thread);
465 -
466 - // ----------------------------------------------------------------
467 -
468 - if(unlikely(!st_duration)) {
469 - st_duration = rrdset_create_localhost(
470 - "netdata"
471 - , "plugin_diskspace_dt"
472 - , NULL
473 - , "diskspace"
474 - , NULL
475 - , "Netdata Disk Space Plugin Duration"
476 - , "milliseconds/run"
477 - , PLUGIN_DISKSPACE_NAME
478 - , NULL
479 - , 132021
480 - , update_every
481 - , RRDSET_TYPE_AREA
482 - );
483 -
484 - rd_duration = rrddim_add(st_duration, "duration", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
485 - }
486 - else
487 - rrdset_next(st_duration);
488 -
489 - rrddim_set_by_pointer(st_duration, rd_duration, duration);
490 - rrdset_done(st_duration);
491 -
492 - // ----------------------------------------------------------------
493 -
494 - if(unlikely(netdata_exit)) break;
443 }
444 +
445 }
446 + worker_unregister();
447
448 netdata_thread_cleanup_pop(1);
449 return NULL;
collectors/ebpf.plugin/ebpf.c
+2
@@ -1864,6 +1864,8 @@ static void ebpf_manage_pid(pid_t pid)
1864 */
1865 int main(int argc, char **argv)
1866 {
1867 + clocks_init();
1868 +
1869 set_global_variables();
1870 ebpf_parse_args(argc, argv);
1871 ebpf_manage_pid(getpid());
collectors/freebsd.plugin/plugin_freebsd.c
+13 -88
@@ -9,7 +9,6 @@ static struct freebsd_module {
9 int enabled;
10
11 int (*func)(int update_every, usec_t dt);
12 - usec_t duration;
12
13 RRDDIM *rd;
14
@@ -68,8 +67,14 @@ static struct freebsd_module {
67 {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
68 };
69
70 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 33
71 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 33
72 +#endif
73 +
74 static void freebsd_main_cleanup(void *ptr)
75 {
76 + worker_unregister();
77 +
78 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
79 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
80
@@ -80,9 +85,9 @@ static void freebsd_main_cleanup(void *ptr)
85
86 void *freebsd_main(void *ptr)
87 {
83 - netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
88 + worker_register("FREEBSD");
89
85 - int vdo_cpu_netdata = config_get_boolean("plugin:freebsd", "netdata server resources", 1);
90 + netdata_thread_cleanup_push(freebsd_main_cleanup, ptr);
91
92 // initialize FreeBSD plugin
93 if (freebsd_plugin_init())
@@ -94,8 +99,9 @@ void *freebsd_main(void *ptr)
99 struct freebsd_module *pm = &freebsd_modules[i];
100
101 pm->enabled = config_get_boolean("plugin:freebsd", pm->name, pm->enabled);
97 - pm->duration = 0ULL;
102 pm->rd = NULL;
103 +
104 + worker_register_job_name(i, freebsd_modules[i].dim);
105 }
106
107 usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
@@ -103,14 +109,13 @@ void *freebsd_main(void *ptr)
109 heartbeat_init(&hb);
110
111 while (!netdata_exit) {
112 + worker_is_idle();
113 +
114 usec_t hb_dt = heartbeat_next(&hb, step);
107 - usec_t duration = 0ULL;
115
116 if (unlikely(netdata_exit))
117 break;
118
112 - // BEGIN -- the job to be done
113 -
119 for (i = 0; freebsd_modules[i].name; i++) {
120 struct freebsd_module *pm = &freebsd_modules[i];
121 if (unlikely(!pm->enabled))
@@ -118,92 +123,12 @@ void *freebsd_main(void *ptr)
123
124 debug(D_PROCNETDEV_LOOP, "FREEBSD calling %s.", pm->name);
125
126 + worker_is_busy(i);
127 pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
122 - pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
123 - duration += pm->duration;
128
129 if (unlikely(netdata_exit))
130 break;
131 }
128 -
129 - // END -- the job is done
130 -
131 - if (vdo_cpu_netdata) {
132 - static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
133 - static RRDDIM *rd_user = NULL, *rd_system = NULL;
134 -
135 - // ----------------------------------------------------------------
136 -
137 - struct rusage thread;
138 - getrusage(RUSAGE_THREAD, &thread);
139 -
140 - if (unlikely(!st_cpu_thread)) {
141 - st_cpu_thread = rrdset_create_localhost(
142 - "netdata",
143 - "plugin_freebsd_cpu",
144 - NULL,
145 - "freebsd",
146 - NULL,
147 - "Netdata FreeBSD plugin CPU usage",
148 - "milliseconds/s",
149 - "freebsd.plugin",
150 - "stats",
151 - 132000,
152 - localhost->rrd_update_every,
153 - RRDSET_TYPE_STACKED);
154 -
155 - rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
156 - rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
157 - } else {
158 - rrdset_next(st_cpu_thread);
159 - }
160 -
161 - rrddim_set_by_pointer(
162 - st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
163 - rrddim_set_by_pointer(
164 - st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
165 - rrdset_done(st_cpu_thread);
166 -
167 - // ----------------------------------------------------------------
168 -
169 - if (unlikely(!st_duration)) {
170 - st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_freebsd_modules");
171 -
172 - if (!st_duration) {
173 - st_duration = rrdset_create_localhost(
174 - "netdata",
175 - "plugin_freebsd_modules",
176 - NULL,
177 - "freebsd",
178 - NULL,
179 - "Netdata FreeBSD plugin modules durations",
180 - "milliseconds/run",
181 - "freebsd.plugin",
182 - "stats",
183 - 132001,
184 - localhost->rrd_update_every,
185 - RRDSET_TYPE_STACKED);
186 -
187 - for (i = 0; freebsd_modules[i].name; i++) {
188 - struct freebsd_module *pm = &freebsd_modules[i];
189 - if (unlikely(!pm->enabled))
190 - continue;
191 -
192 - pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
193 - }
194 - }
195 - } else
196 - rrdset_next(st_duration);
197 -
198 - for (i = 0; freebsd_modules[i].name; i++) {
199 - struct freebsd_module *pm = &freebsd_modules[i];
200 - if (unlikely(!pm->enabled))
201 - continue;
202 -
203 - rrddim_set_by_pointer(st_duration, pm->rd, pm->duration);
204 - }
205 - rrdset_done(st_duration);
206 - }
132 }
133
134 netdata_thread_cleanup_pop(1);
collectors/freeipmi.plugin/freeipmi_plugin.c
+1
@@ -1596,6 +1596,7 @@ int host_is_local(const char *host)
1596 }
1597
1598 int main (int argc, char **argv) {
1599 + clocks_init();
1600
1601 // ------------------------------------------------------------------------
1602 // initialization of netdata plugin
collectors/idlejitter.plugin/plugin_idlejitter.c
+7
@@ -5,6 +5,8 @@
5 #define CPU_IDLEJITTER_SLEEP_TIME_MS 20
6
7 static void cpuidlejitter_main_cleanup(void *ptr) {
8 + worker_unregister();
9 +
10 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
11 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
12
@@ -14,6 +16,9 @@ static void cpuidlejitter_main_cleanup(void *ptr) {
16 }
17
18 void *cpuidlejitter_main(void *ptr) {
19 + worker_register("IDLEJITTER");
20 + worker_register_job_name(0, "measurements");
21 +
22 netdata_thread_cleanup_push(cpuidlejitter_main_cleanup, ptr);
23
24 usec_t sleep_ut = config_get_number("plugin:idlejitter", "loop time in ms", CPU_IDLEJITTER_SLEEP_TIME_MS) * USEC_PER_MS;
@@ -55,7 +60,9 @@ void *cpuidlejitter_main(void *ptr) {
60
61 while(elapsed < update_every_ut) {
62 now_monotonic_high_precision_timeval(&before);
63 + worker_is_idle();
64 sleep_usec(sleep_ut);
65 + worker_is_busy(0);
66 now_monotonic_high_precision_timeval(&after);
67
68 usec_t dt = dt_usec(&after, &before);
collectors/macos.plugin/plugin_macos.c
+12 -88
@@ -9,7 +9,6 @@ static struct macos_module {
9 int enabled;
10
11 int (*func)(int update_every, usec_t dt);
12 - usec_t duration;
12
13 RRDDIM *rd;
14
@@ -22,8 +21,14 @@ static struct macos_module {
21 {.name = NULL, .dim = NULL, .enabled = 0, .func = NULL}
22 };
23
24 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 3
25 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 3
26 +#endif
27 +
28 static void macos_main_cleanup(void *ptr)
29 {
30 + worker_unregister();
31 +
32 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
33 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
34
@@ -34,17 +39,18 @@ static void macos_main_cleanup(void *ptr)
39
40 void *macos_main(void *ptr)
41 {
37 - netdata_thread_cleanup_push(macos_main_cleanup, ptr);
42 + worker_register("MACOS");
43
39 - int vdo_cpu_netdata = config_get_boolean("plugin:macos", "netdata server resources", CONFIG_BOOLEAN_YES);
44 + netdata_thread_cleanup_push(macos_main_cleanup, ptr);
45
46 // check the enabled status for each module
47 for (int i = 0; macos_modules[i].name; i++) {
48 struct macos_module *pm = &macos_modules[i];
49
50 pm->enabled = config_get_boolean("plugin:macos", pm->name, pm->enabled);
46 - pm->duration = 0ULL;
51 pm->rd = NULL;
52 +
53 + worker_register_job_name(i, macos_modules[i].dim);
54 }
55
56 usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
@@ -52,10 +58,8 @@ void *macos_main(void *ptr)
58 heartbeat_init(&hb);
59
60 while (!netdata_exit) {
61 + worker_is_idle();
62 usec_t hb_dt = heartbeat_next(&hb, step);
56 - usec_t duration = 0ULL;
57 -
58 - // BEGIN -- the job to be done
63
64 for (int i = 0; macos_modules[i].name; i++) {
65 struct macos_module *pm = &macos_modules[i];
@@ -64,92 +68,12 @@ void *macos_main(void *ptr)
68
69 debug(D_PROCNETDEV_LOOP, "macos calling %s.", pm->name);
70
71 + worker_is_busy(i);
72 pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
68 - pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
69 - duration += pm->duration;
73
74 if (unlikely(netdata_exit))
75 break;
76 }
74 -
75 - // END -- the job is done
76 -
77 - if (vdo_cpu_netdata) {
78 - static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
79 - static RRDDIM *rd_user = NULL, *rd_system = NULL;
80 -
81 - // ----------------------------------------------------------------
82 -
83 - struct rusage thread;
84 - getrusage(RUSAGE_THREAD, &thread);
85 -
86 - if (unlikely(!st_cpu_thread)) {
87 - st_cpu_thread = rrdset_create_localhost(
88 - "netdata",
89 - "plugin_macos_cpu",
90 - NULL,
91 - "macos",
92 - NULL,
93 - "Netdata macOS plugin CPU usage",
94 - "milliseconds/s",
95 - "macos.plugin",
96 - "stats",
97 - 132000,
98 - localhost->rrd_update_every,
99 - RRDSET_TYPE_STACKED);
100 -
101 - rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
102 - rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
103 - } else {
104 - rrdset_next(st_cpu_thread);
105 - }
106 -
107 - rrddim_set_by_pointer(
108 - st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
109 - rrddim_set_by_pointer(
110 - st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
111 - rrdset_done(st_cpu_thread);
112 -
113 - // ----------------------------------------------------------------
114 -
115 - if (unlikely(!st_duration)) {
116 - st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_macos_modules");
117 -
118 - if (!st_duration) {
119 - st_duration = rrdset_create_localhost(
120 - "netdata",
121 - "plugin_macos_modules",
122 - NULL,
123 - "macos",
124 - NULL,
125 - "Netdata macOS plugin modules durations",
126 - "milliseconds/run",
127 - "macos.plugin",
128 - "stats",
129 - 132001,
130 - localhost->rrd_update_every,
131 - RRDSET_TYPE_STACKED);
132 -
133 - for (int i = 0; macos_modules[i].name; i++) {
134 - struct macos_module *pm = &macos_modules[i];
135 - if (unlikely(!pm->enabled))
136 - continue;
137 -
138 - pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
139 - }
140 - }
141 - } else
142 - rrdset_next(st_duration);
143 -
144 - for (int i = 0; macos_modules[i].name; i++) {
145 - struct macos_module *pm = &macos_modules[i];
146 - if (unlikely(!pm->enabled))
147 - continue;
148 -
149 - rrddim_set_by_pointer(st_duration, pm->rd, pm->duration);
150 - }
151 - rrdset_done(st_duration);
152 - }
77 }
78
79 netdata_thread_cleanup_pop(1);
collectors/nfacct.plugin/plugin_nfacct.c
+1
@@ -745,6 +745,7 @@ void nfacct_signals()
745 }
746
747 int main(int argc, char **argv) {
748 + clocks_init();
749
750 // ------------------------------------------------------------------------
751 // initialization of netdata plugin
collectors/perf.plugin/perf_plugin.c
+1
@@ -1283,6 +1283,7 @@ void parse_command_line(int argc, char **argv) {
1283 }
1284
1285 int main(int argc, char **argv) {
1286 + clocks_init();
1287
1288 // ------------------------------------------------------------------------
1289 // initialization of netdata plugin
collectors/plugins.d/plugins_d.c
+5
@@ -230,6 +230,8 @@ static void pluginsd_worker_thread_handle_error(struct plugind *cd, int worker_r
230
231 void *pluginsd_worker_thread(void *arg)
232 {
233 + worker_register("PLUGINSD");
234 +
235 netdata_thread_cleanup_push(pluginsd_worker_thread_cleanup, arg);
236
237 struct plugind *cd = (struct plugind *)arg;
@@ -260,6 +262,7 @@ void *pluginsd_worker_thread(void *arg)
262 if (unlikely(!cd->enabled))
263 break;
264 }
265 + worker_unregister();
266
267 netdata_thread_cleanup_pop(1);
268 return NULL;
@@ -281,6 +284,8 @@ static void pluginsd_main_cleanup(void *data)
284
285 info("cleanup completed.");
286 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
287 +
288 + worker_unregister();
289 }
290
291 void *pluginsd_main(void *ptr)
collectors/proc.plugin/plugin_proc.c
+16 -107
@@ -9,7 +9,6 @@ static struct proc_module {
9 int enabled;
10
11 int (*func)(int update_every, usec_t dt);
12 - usec_t duration;
12
13 RRDDIM *rd;
14
@@ -66,9 +65,7 @@ static struct proc_module {
65
66 // ZFS metrics
67 {.name = "/proc/spl/kstat/zfs/arcstats", .dim = "zfs_arcstats", .func = do_proc_spl_kstat_zfs_arcstats},
69 - {.name = "/proc/spl/kstat/zfs/pool/state",
70 - .dim = "zfs_pool_state",
71 - .func = do_proc_spl_kstat_zfs_pool_state},
68 + {.name = "/proc/spl/kstat/zfs/pool/state",.dim = "zfs_pool_state",.func = do_proc_spl_kstat_zfs_pool_state},
69
70 // BTRFS metrics
71 {.name = "/sys/fs/btrfs", .dim = "btrfs", .func = do_sys_fs_btrfs},
@@ -83,6 +80,10 @@ static struct proc_module {
80 {.name = NULL, .dim = NULL, .func = NULL}
81 };
82
83 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 36
84 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 36
85 +#endif
86 +
87 static void proc_main_cleanup(void *ptr)
88 {
89 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
@@ -91,13 +92,15 @@ static void proc_main_cleanup(void *ptr)
92 info("cleaning up...");
93
94 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
95 +
96 + worker_unregister();
97 }
98
99 void *proc_main(void *ptr)
100 {
98 - netdata_thread_cleanup_push(proc_main_cleanup, ptr);
101 + worker_register("PROC");
102
100 - int vdo_cpu_netdata = config_get_boolean("plugin:proc", "netdata server resources", CONFIG_BOOLEAN_YES);
103 + netdata_thread_cleanup_push(proc_main_cleanup, ptr);
104
105 config_get_boolean("plugin:proc", "/proc/pagetypeinfo", CONFIG_BOOLEAN_NO);
106
@@ -107,128 +110,34 @@ void *proc_main(void *ptr)
110 struct proc_module *pm = &proc_modules[i];
111
112 pm->enabled = config_get_boolean("plugin:proc", pm->name, CONFIG_BOOLEAN_YES);
110 - pm->duration = 0ULL;
113 pm->rd = NULL;
114 +
115 + worker_register_job_name(i, proc_modules[i].dim);
116 }
117
118 usec_t step = localhost->rrd_update_every * USEC_PER_SEC;
119 heartbeat_t hb;
120 heartbeat_init(&hb);
117 - size_t iterations = 0;
121
122 while (!netdata_exit) {
120 - iterations++;
121 - (void)iterations;
122 -
123 + worker_is_idle();
124 usec_t hb_dt = heartbeat_next(&hb, step);
124 - usec_t duration = 0ULL;
125
126 if (unlikely(netdata_exit))
127 break;
128
129 - // BEGIN -- the job to be done
130 -
129 for (i = 0; proc_modules[i].name; i++) {
130 + if (unlikely(netdata_exit))
131 + break;
132 +
133 struct proc_module *pm = &proc_modules[i];
134 if (unlikely(!pm->enabled))
135 continue;
136
137 debug(D_PROCNETDEV_LOOP, "PROC calling %s.", pm->name);
138
138 -//#ifdef NETDATA_LOG_ALLOCATIONS
139 -// if(pm->func == do_proc_interrupts)
140 -// log_thread_memory_allocations = iterations;
141 -//#endif
139 + worker_is_busy(i);
140 pm->enabled = !pm->func(localhost->rrd_update_every, hb_dt);
143 - pm->duration = heartbeat_monotonic_dt_to_now_usec(&hb) - duration;
144 - duration += pm->duration;
145 -
146 -//#ifdef NETDATA_LOG_ALLOCATIONS
147 -// if(pm->func == do_proc_interrupts)
148 -// log_thread_memory_allocations = 0;
149 -//#endif
150 -
151 - if (unlikely(netdata_exit))
152 - break;
153 - }
154 -
155 - // END -- the job is done
156 -
157 - if (vdo_cpu_netdata) {
158 - static RRDSET *st_cpu_thread = NULL, *st_duration = NULL;
159 - static RRDDIM *rd_user = NULL, *rd_system = NULL;
160 -
161 - // ----------------------------------------------------------------
162 -
163 - struct rusage thread;
164 - getrusage(RUSAGE_THREAD, &thread);
165 -
166 - if (unlikely(!st_cpu_thread)) {
167 - st_cpu_thread = rrdset_create_localhost(
168 - "netdata",
169 - "plugin_proc_cpu",
170 - NULL,
171 - "proc",
172 - NULL,
173 - "Netdata proc plugin CPU usage",
174 - "milliseconds/s",
175 - "proc",
176 - "stats",
177 - 132000,
178 - localhost->rrd_update_every,
179 - RRDSET_TYPE_STACKED);
180 -
181 - rd_user = rrddim_add(st_cpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
182 - rd_system = rrddim_add(st_cpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
183 - } else {
184 - rrdset_next(st_cpu_thread);
185 - }
186 -
187 - rrddim_set_by_pointer(
188 - st_cpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
189 - rrddim_set_by_pointer(
190 - st_cpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
191 - rrdset_done(st_cpu_thread);
192 -
193 - // ----------------------------------------------------------------
194 -
195 - if (unlikely(!st_duration)) {
196 - st_duration = rrdset_find_active_bytype_localhost("netdata", "plugin_proc_modules");
197 -
198 - if (!st_duration) {
199 - st_duration = rrdset_create_localhost(
200 - "netdata",
201 - "plugin_proc_modules",
202 - NULL,
203 - "proc",
204 - NULL,
205 - "Netdata proc plugin modules durations",
206 - "milliseconds/run",
207 - "proc",
208 - "stats",
209 - 132001,
210 - localhost->rrd_update_every,
211 - RRDSET_TYPE_STACKED);
212 -
213 - for (i = 0; proc_modules[i].name; i++) {
214 - struct proc_module *pm = &proc_modules[i];
215 - if (unlikely(!pm->enabled))
216 - continue;
217 -
218 - pm->rd = rrddim_add(st_duration, pm->dim, NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
219 - }
220 - }
221 - } else
222 - rrdset_next(st_duration);
223 -
224 - for (i = 0; proc_modules[i].name; i++) {
225 - struct proc_module *pm = &proc_modules[i];
226 - if (unlikely(!pm->enabled))
227 - continue;
228 -
229 - rrddim_set_by_pointer(st_duration, pm->rd, pm->duration);
230 - }
231 - rrdset_done(st_duration);
141 }
142 }
143
collectors/slabinfo.plugin/slabinfo.c
+1
@@ -336,6 +336,7 @@ void usage(void) {
336 }
337
338 int main(int argc, char **argv) {
339 + clocks_init();
340
341 program_name = argv[0];
342 program_version = "0.1";
collectors/statsd.plugin/statsd.c
+88 -81
@@ -9,6 +9,15 @@
9 #define STATSD_LISTEN_PORT 8125
10 #define STATSD_LISTEN_BACKLOG 4096
11
12 +#define WORKER_JOB_TYPE_TCP_CONNECTED 0
13 +#define WORKER_JOB_TYPE_TCP_DISCONNECTED 1
14 +#define WORKER_JOB_TYPE_RCV_DATA 2
15 +#define WORKER_JOB_TYPE_SND_DATA 3
16 +
17 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 4
18 +#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least 4
19 +#endif
20 +
21 // --------------------------------------------------------------------------------------
22
23 // #define STATSD_MULTITHREADED 1
@@ -237,10 +246,6 @@ struct collection_thread_status {
246 size_t max_sockets;
247
248 netdata_thread_t thread;
240 - struct rusage rusage;
241 - RRDSET *st_cpu;
242 - RRDDIM *rd_user;
243 - RRDDIM *rd_system;
249 };
250
251 static struct statsd {
@@ -788,6 +793,7 @@ static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
793 (void)pi;
794 (void)data;
795
796 + worker_is_busy(WORKER_JOB_TYPE_TCP_CONNECTED);
797 *events = POLLIN;
798
799 struct statsd_tcp *t = (struct statsd_tcp *)callocz(sizeof(struct statsd_tcp) + STATSD_TCP_BUFFER_SIZE, 1);
@@ -796,11 +802,14 @@ static void *statsd_add_callback(POLLINFO *pi, short int *events, void *data) {
802 statsd.tcp_socket_connects++;
803 statsd.tcp_socket_connected++;
804
805 + worker_is_idle();
806 return t;
807 }
808
809 // TCP client disconnected
810 static void statsd_del_callback(POLLINFO *pi) {
811 + worker_is_busy(WORKER_JOB_TYPE_TCP_DISCONNECTED);
812 +
813 struct statsd_tcp *t = pi->data;
814
815 if(likely(t)) {
@@ -818,10 +827,15 @@ static void statsd_del_callback(POLLINFO *pi) {
827
828 freez(t);
829 }
830 +
831 + worker_is_idle();
832 }
833
834 // Receive data
835 static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
836 + int retval = -1;
837 + worker_is_busy(WORKER_JOB_TYPE_RCV_DATA);
838 +
839 *events = POLLIN;
840
841 int fd = pi->fd;
@@ -832,14 +846,16 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
846 if(unlikely(!d)) {
847 error("STATSD: internal error: expected TCP data pointer is NULL");
848 statsd.socket_errors++;
835 - return -1;
849 + retval = -1;
850 + goto cleanup;
851 }
852
853 #ifdef NETDATA_INTERNAL_CHECKS
854 if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_TCP)) {
855 error("STATSD: internal error: socket data type should be %d, but it is %d", (int)STATSD_SOCKET_DATA_TYPE_TCP, (int)d->type);
856 statsd.socket_errors++;
842 - return -1;
857 + retval = -1;
858 + goto cleanup;
859 }
860 #endif
861
@@ -872,8 +888,10 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
888 d->len = statsd_process(d->buffer, d->len, 1);
889 }
890
875 - if(unlikely(ret == -1))
876 - return -1;
891 + if(unlikely(ret == -1)) {
892 + retval = -1;
893 + goto cleanup;
894 + }
895
896 } while (rc != -1);
897 break;
@@ -884,14 +902,16 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
902 if(unlikely(!d)) {
903 error("STATSD: internal error: expected UDP data pointer is NULL");
904 statsd.socket_errors++;
887 - return -1;
905 + retval = -1;
906 + goto cleanup;
907 }
908
909 #ifdef NETDATA_INTERNAL_CHECKS
910 if(unlikely(d->type != STATSD_SOCKET_DATA_TYPE_UDP)) {
911 error("STATSD: internal error: socket data should be %d, but it is %d", (int)d->type, (int)STATSD_SOCKET_DATA_TYPE_UDP);
912 statsd.socket_errors++;
894 - return -1;
913 + retval = -1;
914 + goto cleanup;
915 }
916 #endif
917
@@ -904,7 +924,8 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
924 if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
925 error("STATSD: recvmmsg() on UDP socket %d failed.", fd);
926 statsd.socket_errors++;
907 - return -1;
927 + retval = -1;
928 + goto cleanup;
929 }
930 } else if (rc) {
931 // data received
@@ -929,7 +950,8 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
950 if (errno != EWOULDBLOCK && errno != EAGAIN && errno != EINTR) {
951 error("STATSD: recv() on UDP socket %d failed.", fd);
952 statsd.socket_errors++;
932 - return -1;
953 + retval = -1;
954 + goto cleanup;
955 }
956 } else if (rc) {
957 // data received
@@ -947,24 +969,26 @@ static int statsd_rcv_callback(POLLINFO *pi, short int *events) {
969 default: {
970 error("STATSD: internal error: unknown socktype %d on socket %d", pi->socktype, fd);
971 statsd.socket_errors++;
950 - return -1;
972 + retval = -1;
973 + goto cleanup;
974 }
975 }
976
954 - return 0;
977 + retval = 0;
978 +cleanup:
979 + worker_is_idle();
980 + return retval;
981 }
982
983 static int statsd_snd_callback(POLLINFO *pi, short int *events) {
984 (void)pi;
985 (void)events;
986
987 + worker_is_busy(WORKER_JOB_TYPE_SND_DATA);
988 error("STATSD: snd_callback() called, but we never requested to send data to statsd clients.");
962 - return -1;
963 -}
989 + worker_is_idle();
990
965 -static void statsd_timer_callback(void *timer_data) {
966 - struct collection_thread_status *status = timer_data;
967 - getrusage(RUSAGE_THREAD, &status->rusage);
991 + return -1;
992 }
993
994 // --------------------------------------------------------------------------------------------------------------------
@@ -986,12 +1010,19 @@ void statsd_collector_thread_cleanup(void *data) {
1010 #endif
1011
1012 freez(d);
1013 + worker_unregister();
1014 }
1015
1016 void *statsd_collector_thread(void *ptr) {
1017 struct collection_thread_status *status = ptr;
1018 status->status = 1;
1019
1020 + worker_register("STATSD");
1021 + worker_register_job_name(WORKER_JOB_TYPE_TCP_CONNECTED, "tcp connect");
1022 + worker_register_job_name(WORKER_JOB_TYPE_TCP_DISCONNECTED, "tcp disconnect");
1023 + worker_register_job_name(WORKER_JOB_TYPE_RCV_DATA, "receive");
1024 + worker_register_job_name(WORKER_JOB_TYPE_SND_DATA, "send");
1025 +
1026 info("STATSD collector thread started with taskid %d", gettid());
1027
1028 struct statsd_udp *d = callocz(sizeof(struct statsd_udp), 1);
@@ -1019,7 +1050,7 @@ void *statsd_collector_thread(void *ptr) {
1050 , statsd_del_callback
1051 , statsd_rcv_callback
1052 , statsd_snd_callback
1022 - , statsd_timer_callback
1053 + , NULL
1054 , NULL // No access control pattern
1055 , 0 // No dns lookups for access control pattern
1056 , (void *)d
@@ -2147,9 +2178,32 @@ static void statsd_main_cleanup(void *data) {
2178
2179 info("STATSD: cleanup completed.");
2180 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
2181 +
2182 + worker_unregister();
2183 }
2184
2185 +#define WORKER_STATSD_FLUSH_GAUGES 0
2186 +#define WORKER_STATSD_FLUSH_COUNTERS 1
2187 +#define WORKER_STATSD_FLUSH_METERS 2
2188 +#define WORKER_STATSD_FLUSH_TIMERS 3
2189 +#define WORKER_STATSD_FLUSH_HISTOGRAMS 4
2190 +#define WORKER_STATSD_FLUSH_SETS 5
2191 +#define WORKER_STATSD_FLUSH_STATS 6
2192 +
2193 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 7
2194 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 6
2195 +#endif
2196 +
2197 void *statsd_main(void *ptr) {
2198 + worker_register("STATSDFLUSH");
2199 + worker_register_job_name(WORKER_STATSD_FLUSH_GAUGES, "gauges");
2200 + worker_register_job_name(WORKER_STATSD_FLUSH_COUNTERS, "counters");
2201 + worker_register_job_name(WORKER_STATSD_FLUSH_METERS, "meters");
2202 + worker_register_job_name(WORKER_STATSD_FLUSH_TIMERS, "timers");
2203 + worker_register_job_name(WORKER_STATSD_FLUSH_HISTOGRAMS, "histograms");
2204 + worker_register_job_name(WORKER_STATSD_FLUSH_SETS, "sets");
2205 + worker_register_job_name(WORKER_STATSD_FLUSH_STATS, "statistics");
2206 +
2207 netdata_thread_cleanup_push(statsd_main_cleanup, ptr);
2208
2209 // ----------------------------------------------------------------------------------------------------------------
@@ -2420,71 +2474,37 @@ void *statsd_main(void *ptr) {
2474 );
2475 RRDDIM *rd_pcharts = rrddim_add(st_pcharts, "charts", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
2476
2423 - RRDSET *stcpu_thread = rrdset_create_localhost(
2424 - "netdata"
2425 - , "plugin_statsd_charting_cpu"
2426 - , NULL
2427 - , "statsd"
2428 - , "netdata.statsd_cpu"
2429 - , "Netdata statsd charting thread CPU usage"
2430 - , "milliseconds/s"
2431 - , PLUGIN_STATSD_NAME
2432 - , "stats"
2433 - , 132001
2434 - , statsd.update_every
2435 - , RRDSET_TYPE_STACKED
2436 - );
2437 -
2438 - RRDDIM *rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
2439 - RRDDIM *rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
2440 - struct rusage thread;
2441 -
2442 - for(i = 0; i < statsd.threads ;i++) {
2443 - char id[100 + 1];
2444 - char title[100 + 1];
2445 -
2446 - snprintfz(id, 100, "plugin_statsd_collector%d_cpu", i + 1);
2447 - snprintfz(title, 100, "Netdata statsd collector thread No %d CPU usage", i + 1);
2448 -
2449 - statsd.collection_threads_status[i].st_cpu = rrdset_create_localhost(
2450 - "netdata"
2451 - , id
2452 - , NULL
2453 - , "statsd"
2454 - , "netdata.statsd_cpu"
2455 - , title
2456 - , "milliseconds/s"
2457 - , PLUGIN_STATSD_NAME
2458 - , "stats"
2459 - , 132002 + i
2460 - , statsd.update_every
2461 - , RRDSET_TYPE_STACKED
2462 - );
2463 -
2464 - statsd.collection_threads_status[i].rd_user = rrddim_add(statsd.collection_threads_status[i].st_cpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
2465 - statsd.collection_threads_status[i].rd_system = rrddim_add(statsd.collection_threads_status[i].st_cpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
2466 - }
2467 -
2468 - // ----------------------------------------------------------------------------------------------------------------
2477 + // ----------------------------------------------------------------------------------------------------------------
2478 // statsd thread to turn metrics into charts
2479
2480 usec_t step = statsd.update_every * USEC_PER_SEC;
2481 heartbeat_t hb;
2482 heartbeat_init(&hb);
2483 while(!netdata_exit) {
2484 + worker_is_idle();
2485 usec_t hb_dt = heartbeat_next(&hb, step);
2486
2487 + worker_is_busy(WORKER_STATSD_FLUSH_GAUGES);
2488 statsd_flush_index_metrics(&statsd.gauges, statsd_flush_gauge);
2489 +
2490 + worker_is_busy(WORKER_STATSD_FLUSH_COUNTERS);
2491 statsd_flush_index_metrics(&statsd.counters, statsd_flush_counter);
2492 +
2493 + worker_is_busy(WORKER_STATSD_FLUSH_METERS);
2494 statsd_flush_index_metrics(&statsd.meters, statsd_flush_meter);
2495 +
2496 + worker_is_busy(WORKER_STATSD_FLUSH_TIMERS);
2497 statsd_flush_index_metrics(&statsd.timers, statsd_flush_timer);
2498 +
2499 + worker_is_busy(WORKER_STATSD_FLUSH_HISTOGRAMS);
2500 statsd_flush_index_metrics(&statsd.histograms, statsd_flush_histogram);
2501 +
2502 + worker_is_busy(WORKER_STATSD_FLUSH_SETS);
2503 statsd_flush_index_metrics(&statsd.sets, statsd_flush_set);
2504
2505 + worker_is_busy(WORKER_STATSD_FLUSH_STATS);
2506 statsd_update_all_app_charts();
2507
2486 - getrusage(RUSAGE_THREAD, &thread);
2487 -
2508 if(unlikely(netdata_exit))
2509 break;
2510
@@ -2498,9 +2518,6 @@ void *statsd_main(void *ptr) {
2518 rrdset_next(st_tcp_connects);
2519 rrdset_next(st_tcp_connected);
2520 rrdset_next(st_pcharts);
2501 - rrdset_next(stcpu_thread);
2502 - for(i = 0; i < statsd.threads ;i++)
2503 - rrdset_next(statsd.collection_threads_status[i].st_cpu);
2521 }
2522
2523 rrddim_set_by_pointer(st_metrics, rd_metrics_gauge, (collected_number)statsd.gauges.metrics);
@@ -2550,16 +2567,6 @@ void *statsd_main(void *ptr) {
2567
2568 rrddim_set_by_pointer(st_pcharts, rd_pcharts, (collected_number)statsd.private_charts);
2569 rrdset_done(st_pcharts);
2553 -
2554 - rrddim_set_by_pointer(stcpu_thread, rd_user, thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
2555 - rrddim_set_by_pointer(stcpu_thread, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
2556 - rrdset_done(stcpu_thread);
2557 -
2558 - for(i = 0; i < statsd.threads ;i++) {
2559 - rrddim_set_by_pointer(statsd.collection_threads_status[i].st_cpu, statsd.collection_threads_status[i].rd_user, statsd.collection_threads_status[i].rusage.ru_utime.tv_sec * 1000000ULL + statsd.collection_threads_status[i].rusage.ru_utime.tv_usec);
2560 - rrddim_set_by_pointer(statsd.collection_threads_status[i].st_cpu, statsd.collection_threads_status[i].rd_system, statsd.collection_threads_status[i].rusage.ru_stime.tv_sec * 1000000ULL + statsd.collection_threads_status[i].rusage.ru_stime.tv_usec);
2561 - rrdset_done(statsd.collection_threads_status[i].st_cpu);
2562 - }
2570 }
2571
2572 cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
collectors/tc.plugin/plugin_tc.c
+56 -33
@@ -844,6 +844,8 @@ static inline void tc_split_words(char *str, char **words, int max_words) {
844 static pid_t tc_child_pid = 0;
845
846 static void tc_main_cleanup(void *ptr) {
847 + worker_unregister();
848 +
849 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
850 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
851
@@ -864,10 +866,35 @@ static void tc_main_cleanup(void *ptr) {
866 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
867 }
868
869 +#define WORKER_TC_CLASS 0
870 +#define WORKER_TC_BEGIN 1
871 +#define WORKER_TC_END 2
872 +#define WORKER_TC_SENT 3
873 +#define WORKER_TC_LENDED 4
874 +#define WORKER_TC_TOKENS 5
875 +#define WORKER_TC_SETDEVICENAME 6
876 +#define WORKER_TC_SETDEVICEGROUP 7
877 +#define WORKER_TC_SETCLASSNAME 8
878 +#define WORKER_TC_WORKTIME 9
879 +
880 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 10
881 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 10
882 +#endif
883 +
884 void *tc_main(void *ptr) {
868 - netdata_thread_cleanup_push(tc_main_cleanup, ptr);
885 + worker_register("TC");
886 + worker_register_job_name(WORKER_TC_CLASS, "class");
887 + worker_register_job_name(WORKER_TC_BEGIN, "begin");
888 + worker_register_job_name(WORKER_TC_END, "end");
889 + worker_register_job_name(WORKER_TC_SENT, "sent");
890 + worker_register_job_name(WORKER_TC_LENDED, "lended");
891 + worker_register_job_name(WORKER_TC_TOKENS, "tokens");
892 + worker_register_job_name(WORKER_TC_SETDEVICENAME, "devicename");
893 + worker_register_job_name(WORKER_TC_SETDEVICEGROUP, "devicegroup");
894 + worker_register_job_name(WORKER_TC_SETCLASSNAME, "classname");
895 + worker_register_job_name(WORKER_TC_WORKTIME, "worktime");
896
870 - struct rusage thread;
897 + netdata_thread_cleanup_push(tc_main_cleanup, ptr);
898
899 char command[FILENAME_MAX + 1];
900 char *words[PLUGINSD_MAX_WORDS] = { NULL };
@@ -913,6 +940,7 @@ void *tc_main(void *ptr) {
940
941 if(unlikely(!words[0] || !*words[0])) {
942 // debug(D_TC_LOOP, "empty line");
943 + worker_is_idle();
944 continue;
945 }
946 // else debug(D_TC_LOOP, "First word is '%s'", words[0]);
@@ -920,6 +948,8 @@ void *tc_main(void *ptr) {
948 first_hash = simple_hash(words[0]);
949
950 if(unlikely(device && ((first_hash == CLASS_HASH && strcmp(words[0], "class") == 0) || (first_hash == QDISC_HASH && strcmp(words[0], "qdisc") == 0)))) {
951 + worker_is_busy(WORKER_TC_CLASS);
952 +
953 // debug(D_TC_LOOP, "CLASS line on class id='%s', parent='%s', parentid='%s', leaf='%s', leafid='%s'", words[2], words[3], words[4], words[5], words[6]);
954
955 char *type = words[1]; // the class/qdisc type: htb, fq_codel, etc
@@ -949,6 +979,7 @@ void *tc_main(void *ptr) {
979 // there should be an IFB interface for this
980
981 class = NULL;
982 + worker_is_idle();
983 continue;
984 }
985
@@ -985,6 +1016,8 @@ void *tc_main(void *ptr) {
1016 }
1017 }
1018 else if(unlikely(first_hash == END_HASH && strcmp(words[0], "END") == 0)) {
1019 + worker_is_busy(WORKER_TC_END);
1020 +
1021 // debug(D_TC_LOOP, "END line");
1022
1023 if(likely(device)) {
@@ -998,6 +1031,8 @@ void *tc_main(void *ptr) {
1031 class = NULL;
1032 }
1033 else if(unlikely(first_hash == BEGIN_HASH && strcmp(words[0], "BEGIN") == 0)) {
1034 + worker_is_busy(WORKER_TC_BEGIN);
1035 +
1036 // debug(D_TC_LOOP, "BEGIN line on device '%s'", words[1]);
1037
1038 if(likely(words[1] && *words[1])) {
@@ -1011,6 +1046,8 @@ void *tc_main(void *ptr) {
1046 class = NULL;
1047 }
1048 else if(unlikely(device && class && first_hash == SENT_HASH && strcmp(words[0], "Sent") == 0)) {
1049 + worker_is_busy(WORKER_TC_SENT);
1050 +
1051 // debug(D_TC_LOOP, "SENT line '%s'", words[1]);
1052 if(likely(words[1] && *words[1])) {
1053 class->bytes = str2ull(words[1]);
@@ -1033,6 +1070,8 @@ void *tc_main(void *ptr) {
1070 class->requeues = str2ull(words[8]);
1071 }
1072 else if(unlikely(device && class && class->updated && first_hash == LENDED_HASH && strcmp(words[0], "lended:") == 0)) {
1073 + worker_is_busy(WORKER_TC_LENDED);
1074 +
1075 // debug(D_TC_LOOP, "LENDED line '%s'", words[1]);
1076 if(likely(words[1] && *words[1]))
1077 class->lended = str2ull(words[1]);
@@ -1044,6 +1083,8 @@ void *tc_main(void *ptr) {
1083 class->giants = str2ull(words[5]);
1084 }
1085 else if(unlikely(device && class && class->updated && first_hash == TOKENS_HASH && strcmp(words[0], "tokens:") == 0)) {
1086 + worker_is_busy(WORKER_TC_TOKENS);
1087 +
1088 // debug(D_TC_LOOP, "TOKENS line '%s'", words[1]);
1089 if(likely(words[1] && *words[1]))
1090 class->tokens = str2ull(words[1]);
@@ -1052,16 +1093,22 @@ void *tc_main(void *ptr) {
1093 class->ctokens = str2ull(words[3]);
1094 }
1095 else if(unlikely(device && first_hash == SETDEVICENAME_HASH && strcmp(words[0], "SETDEVICENAME") == 0)) {
1096 + worker_is_busy(WORKER_TC_SETDEVICENAME);
1097 +
1098 // debug(D_TC_LOOP, "SETDEVICENAME line '%s'", words[1]);
1099 if(likely(words[1] && *words[1]))
1100 tc_device_set_device_name(device, words[1]);
1101 }
1102 else if(unlikely(device && first_hash == SETDEVICEGROUP_HASH && strcmp(words[0], "SETDEVICEGROUP") == 0)) {
1103 + worker_is_busy(WORKER_TC_SETDEVICEGROUP);
1104 +
1105 // debug(D_TC_LOOP, "SETDEVICEGROUP line '%s'", words[1]);
1106 if(likely(words[1] && *words[1]))
1107 tc_device_set_device_family(device, words[1]);
1108 }
1109 else if(unlikely(device && first_hash == SETCLASSNAME_HASH && strcmp(words[0], "SETCLASSNAME") == 0)) {
1110 + worker_is_busy(WORKER_TC_SETCLASSNAME);
1111 +
1112 // debug(D_TC_LOOP, "SETCLASSNAME line '%s' '%s'", words[1], words[2]);
1113 char *id = words[1];
1114 char *path = words[2];
@@ -1069,36 +1116,9 @@ void *tc_main(void *ptr) {
1116 tc_device_set_class_name(device, id, path);
1117 }
1118 else if(unlikely(first_hash == WORKTIME_HASH && strcmp(words[0], "WORKTIME") == 0)) {
1072 - // debug(D_TC_LOOP, "WORKTIME line '%s' '%s'", words[1], words[2]);
1073 - getrusage(RUSAGE_THREAD, &thread);
1074 -
1075 - static RRDSET *stcpu = NULL;
1076 - static RRDDIM *rd_user = NULL, *rd_system = NULL;
1077 -
1078 - if(unlikely(!stcpu)) {
1079 - stcpu = rrdset_create_localhost(
1080 - "netdata"
1081 - , "plugin_tc_cpu"
1082 - , NULL
1083 - , "tc.helper"
1084 - , NULL
1085 - , "Netdata TC CPU usage"
1086 - , "milliseconds/s"
1087 - , PLUGIN_TC_NAME
1088 - , NULL
1089 - , NETDATA_CHART_PRIO_NETDATA_TC_CPU
1090 - , localhost->rrd_update_every
1091 - , RRDSET_TYPE_STACKED
1092 - );
1093 - rd_user = rrddim_add(stcpu, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
1094 - rd_system = rrddim_add(stcpu, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
1095 - }
1096 - else rrdset_next(stcpu);
1097 -
1098 - rrddim_set_by_pointer(stcpu, rd_user , thread.ru_utime.tv_sec * 1000000ULL + thread.ru_utime.tv_usec);
1099 - rrddim_set_by_pointer(stcpu, rd_system, thread.ru_stime.tv_sec * 1000000ULL + thread.ru_stime.tv_usec);
1100 - rrdset_done(stcpu);
1119 + worker_is_busy(WORKER_TC_WORKTIME);
1120
1121 + // debug(D_TC_LOOP, "WORKTIME line '%s' '%s'", words[1], words[2]);
1122 static RRDSET *sttime = NULL;
1123 static RRDDIM *rd_run_time = NULL;
1124
@@ -1107,8 +1127,8 @@ void *tc_main(void *ptr) {
1127 "netdata"
1128 , "plugin_tc_time"
1129 , NULL
1110 - , "tc.helper"
1111 - , NULL
1130 + , "workers plugin tc"
1131 + , "netdata.workers.tc.script_time"
1132 , "Netdata TC script execution"
1133 , "milliseconds/run"
1134 , PLUGIN_TC_NAME
@@ -1128,6 +1148,8 @@ void *tc_main(void *ptr) {
1148 //else {
1149 // debug(D_TC_LOOP, "IGNORED line");
1150 //}
1151 +
1152 + worker_is_idle();
1153 }
1154
1155 // fgets() failed or loop broke
@@ -1158,6 +1180,7 @@ void *tc_main(void *ptr) {
1180 }
1181
1182 cleanup: ; // added semi-colon to prevent older gcc error: label at end of compound statement
1183 + worker_unregister();
1184 netdata_thread_cleanup_pop(1);
1185 return NULL;
1186 }
collectors/timex.plugin/plugin_timex.c
+7 -65
@@ -32,6 +32,8 @@ struct status_codes {
32
33 static void timex_main_cleanup(void *ptr)
34 {
35 + worker_unregister();
36 +
37 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
38 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
39
@@ -42,9 +44,10 @@ static void timex_main_cleanup(void *ptr)
44
45 void *timex_main(void *ptr)
46 {
45 - netdata_thread_cleanup_push(timex_main_cleanup, ptr);
47 + worker_register("TIMEX");
48 + worker_register_job_name(0, "clock check");
49
47 - int vdo_cpu_netdata = config_get_boolean(CONFIG_SECTION_TIMEX, "timex plugin resource charts", CONFIG_BOOLEAN_YES);
50 + netdata_thread_cleanup_push(timex_main_cleanup, ptr);
51
52 int update_every = (int)config_get_number(CONFIG_SECTION_TIMEX, "update every", 10);
53 if (update_every < localhost->rrd_update_every)
@@ -62,8 +65,9 @@ void *timex_main(void *ptr)
65 heartbeat_t hb;
66 heartbeat_init(&hb);
67 while (!netdata_exit) {
65 - usec_t duration = heartbeat_monotonic_dt_to_now_usec(&hb);
68 + worker_is_idle();
69 heartbeat_next(&hb, step);
70 + worker_is_busy(0);
71
72 struct timex timex_buf = {};
73 int sync_state = 0;
@@ -170,68 +174,6 @@ void *timex_main(void *ptr)
174 rrddim_set_by_pointer(st_offset, rd_offset, timex_buf.offset);
175 rrdset_done(st_offset);
176 }
173 -
174 - if (vdo_cpu_netdata) {
175 - static RRDSET *stcpu_thread = NULL, *st_duration = NULL;
176 - static RRDDIM *rd_user = NULL, *rd_system = NULL, *rd_duration = NULL;
177 -
178 - // ----------------------------------------------------------------
179 -
180 - struct rusage thread;
181 - getrusage(RUSAGE_THREAD, &thread);
182 -
183 - if (unlikely(!stcpu_thread)) {
184 - stcpu_thread = rrdset_create_localhost(
185 - "netdata",
186 - "plugin_timex",
187 - NULL,
188 - "timex",
189 - NULL,
190 - "Netdata Timex Plugin CPU usage",
191 - "milliseconds/s",
192 - PLUGIN_TIMEX_NAME,
193 - NULL,
194 - NETDATA_CHART_PRIO_NETDATA_TIMEX,
195 - update_every,
196 - RRDSET_TYPE_STACKED);
197 -
198 - rd_user = rrddim_add(stcpu_thread, "user", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
199 - rd_system = rrddim_add(stcpu_thread, "system", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_INCREMENTAL);
200 - } else {
201 - rrdset_next(stcpu_thread);
202 - }
203 -
204 - rrddim_set_by_pointer(
205 - stcpu_thread, rd_user, thread.ru_utime.tv_sec * USEC_PER_SEC + thread.ru_utime.tv_usec);
206 - rrddim_set_by_pointer(
207 - stcpu_thread, rd_system, thread.ru_stime.tv_sec * USEC_PER_SEC + thread.ru_stime.tv_usec);
208 - rrdset_done(stcpu_thread);
209 -
210 - // ----------------------------------------------------------------
211 -
212 - if (unlikely(!st_duration)) {
213 - st_duration = rrdset_create_localhost(
214 - "netdata",
215 - "plugin_timex_dt",
216 - NULL,
217 - "timex",
218 - NULL,
219 - "Netdata Timex Plugin Duration",
220 - "milliseconds/run",
221 - PLUGIN_TIMEX_NAME,
222 - NULL,
223 - NETDATA_CHART_PRIO_NETDATA_TIMEX + 1,
224 - update_every,
225 - RRDSET_TYPE_AREA);
226 -
227 - rd_duration = rrddim_add(st_duration, "duration", NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
228 - } else {
229 - rrdset_next(st_duration);
230 - }
231 -
232 - rrddim_set_by_pointer(st_duration, rd_duration, duration);
233 - rrdset_done(st_duration);
234 - }
177 }
178
179 exit:
collectors/xenstat.plugin/xenstat_plugin.c
+1
@@ -920,6 +920,7 @@ static void xenstat_send_domain_metrics() {
920 }
921
922 int main(int argc, char **argv) {
923 + clocks_init();
924
925 // ------------------------------------------------------------------------
926 // initialization of netdata plugin
configure.ac
+1
@@ -1767,6 +1767,7 @@ AC_CONFIG_FILES([
1767 libnetdata/url/Makefile
1768 libnetdata/json/Makefile
1769 libnetdata/health/Makefile
1770 + libnetdata/worker_utilization/Makefile
1771 registry/Makefile
1772 streaming/Makefile
1773 system/Makefile
daemon/global_statistics.c
+943 -362
@@ -6,6 +6,16 @@
6
7 #define CONFIG_SECTION_GLOBAL_STATISTICS "global statistics"
8
9 +#define WORKER_JOB_GLOBAL 0
10 +#define WORKER_JOB_REGISTRY 1
11 +#define WORKER_JOB_WORKERS 2
12 +#define WORKER_JOB_DBENGINE 3
13 +#define WORKER_JOB_HEARTBEAT 4
14 +
15 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
16 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
17 +#endif
18 +
19 static struct global_statistics {
20 volatile uint16_t connected_clients;
21
@@ -436,435 +446,993 @@ static void global_statistics_charts(void) {
446 }
447
448 // ----------------------------------------------------------------
449 +}
450
451 +static void dbengine_statistics_charts(void) {
452 #ifdef ENABLE_DBENGINE
441 - RRDHOST *host;
442 - unsigned long long stats_array[RRDENG_NR_STATS] = {0};
443 - unsigned long long local_stats_array[RRDENG_NR_STATS];
444 - unsigned dbengine_contexts = 0, counted_multihost_db = 0, i;
445 -
446 - rrd_rdlock();
447 - rrdhost_foreach_read(host) {
448 - if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
449 - if (&multidb_ctx == host->rrdeng_ctx) {
450 - if (counted_multihost_db)
451 - continue; /* Only count multi-host DB once */
452 - counted_multihost_db = 1;
453 - }
454 - ++dbengine_contexts;
455 - /* get localhost's DB engine's statistics */
456 - rrdeng_get_37_statistics(host->rrdeng_ctx, local_stats_array);
457 - for (i = 0 ; i < RRDENG_NR_STATS ; ++i) {
458 - /* aggregate statistics across hosts */
459 - stats_array[i] += local_stats_array[i];
453 + if(netdata_rwlock_tryrdlock(&rrd_rwlock) == 0) {
454 + RRDHOST *host;
455 + unsigned long long stats_array[RRDENG_NR_STATS] = {0};
456 + unsigned long long local_stats_array[RRDENG_NR_STATS];
457 + unsigned dbengine_contexts = 0, counted_multihost_db = 0, i;
458 +
459 + rrdhost_foreach_read(host) {
460 + if (host->rrd_memory_mode == RRD_MEMORY_MODE_DBENGINE && !rrdhost_flag_check(host, RRDHOST_FLAG_ARCHIVED)) {
461 + if (&multidb_ctx == host->rrdeng_ctx) {
462 + if (counted_multihost_db)
463 + continue; /* Only count multi-host DB once */
464 + counted_multihost_db = 1;
465 + }
466 + ++dbengine_contexts;
467 + /* get localhost's DB engine's statistics */
468 + rrdeng_get_37_statistics(host->rrdeng_ctx, local_stats_array);
469 + for (i = 0; i < RRDENG_NR_STATS; ++i) {
470 + /* aggregate statistics across hosts */
471 + stats_array[i] += local_stats_array[i];
472 + }
473 }
474 }
462 - }
463 - rrd_unlock();
464 -
465 - if (dbengine_contexts) {
466 - /* deduplicate global statistics by getting the ones from the last context */
467 - stats_array[30] = local_stats_array[30];
468 - stats_array[31] = local_stats_array[31];
469 - stats_array[32] = local_stats_array[32];
470 - stats_array[34] = local_stats_array[34];
471 - stats_array[36] = local_stats_array[36];
472 -
473 - // ----------------------------------------------------------------
474 -
475 - {
476 - static RRDSET *st_compression = NULL;
477 - static RRDDIM *rd_savings = NULL;
478 -
479 - if (unlikely(!st_compression)) {
480 - st_compression = rrdset_create_localhost(
481 - "netdata"
482 - , "dbengine_compression_ratio"
483 - , NULL
484 - , "dbengine"
485 - , NULL
486 - , "Netdata DB engine data extents' compression savings ratio"
487 - , "percentage"
488 - , "netdata"
489 - , "stats"
490 - , 130502
491 - , localhost->rrd_update_every
492 - , RRDSET_TYPE_LINE
493 - );
494 -
495 - rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
475 + rrd_unlock();
476 +
477 + if (dbengine_contexts) {
478 + /* deduplicate global statistics by getting the ones from the last context */
479 + stats_array[30] = local_stats_array[30];
480 + stats_array[31] = local_stats_array[31];
481 + stats_array[32] = local_stats_array[32];
482 + stats_array[34] = local_stats_array[34];
483 + stats_array[36] = local_stats_array[36];
484 +
485 + // ----------------------------------------------------------------
486 +
487 + {
488 + static RRDSET *st_compression = NULL;
489 + static RRDDIM *rd_savings = NULL;
490 +
491 + if (unlikely(!st_compression)) {
492 + st_compression = rrdset_create_localhost(
493 + "netdata",
494 + "dbengine_compression_ratio",
495 + NULL,
496 + "dbengine",
497 + NULL,
498 + "Netdata DB engine data extents' compression savings ratio",
499 + "percentage",
500 + "netdata",
501 + "stats",
502 + 130502,
503 + localhost->rrd_update_every,
504 + RRDSET_TYPE_LINE);
505 +
506 + rd_savings = rrddim_add(st_compression, "savings", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
507 + } else
508 + rrdset_next(st_compression);
509 +
510 + unsigned long long ratio;
511 + unsigned long long compressed_content_size = stats_array[12];
512 + unsigned long long content_size = stats_array[11];
513 +
514 + if (content_size) {
515 + // allow negative savings
516 + ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
517 + } else {
518 + ratio = 0;
519 + }
520 + rrddim_set_by_pointer(st_compression, rd_savings, ratio);
521 +
522 + rrdset_done(st_compression);
523 }
497 - else
498 - rrdset_next(st_compression);
499 -
500 - unsigned long long ratio;
501 - unsigned long long compressed_content_size = stats_array[12];
502 - unsigned long long content_size = stats_array[11];
503 -
504 - if (content_size) {
505 - // allow negative savings
506 - ratio = ((content_size - compressed_content_size) * 100 * 1000) / content_size;
507 - } else {
508 - ratio = 0;
509 - }
510 - rrddim_set_by_pointer(st_compression, rd_savings, ratio);
524
512 - rrdset_done(st_compression);
513 - }
525 + // ----------------------------------------------------------------
526 +
527 + {
528 + static RRDSET *st_pg_cache_hit_ratio = NULL;
529 + static RRDDIM *rd_hit_ratio = NULL;
530 +
531 + if (unlikely(!st_pg_cache_hit_ratio)) {
532 + st_pg_cache_hit_ratio = rrdset_create_localhost(
533 + "netdata",
534 + "page_cache_hit_ratio",
535 + NULL,
536 + "dbengine",
537 + NULL,
538 + "Netdata DB engine page cache hit ratio",
539 + "percentage",
540 + "netdata",
541 + "stats",
542 + 130503,
543 + localhost->rrd_update_every,
544 + RRDSET_TYPE_LINE);
545 +
546 + rd_hit_ratio = rrddim_add(st_pg_cache_hit_ratio, "ratio", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
547 + } else
548 + rrdset_next(st_pg_cache_hit_ratio);
549 +
550 + static unsigned long long old_hits = 0;
551 + static unsigned long long old_misses = 0;
552 + unsigned long long hits = stats_array[7];
553 + unsigned long long misses = stats_array[8];
554 + unsigned long long hits_delta;
555 + unsigned long long misses_delta;
556 + unsigned long long ratio;
557 +
558 + hits_delta = hits - old_hits;
559 + misses_delta = misses - old_misses;
560 + old_hits = hits;
561 + old_misses = misses;
562 +
563 + if (hits_delta + misses_delta) {
564 + ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta);
565 + } else {
566 + ratio = 0;
567 + }
568 + rrddim_set_by_pointer(st_pg_cache_hit_ratio, rd_hit_ratio, ratio);
569 +
570 + rrdset_done(st_pg_cache_hit_ratio);
571 + }
572
515 - // ----------------------------------------------------------------
516 -
517 - {
518 - static RRDSET *st_pg_cache_hit_ratio = NULL;
519 - static RRDDIM *rd_hit_ratio = NULL;
520 -
521 - if (unlikely(!st_pg_cache_hit_ratio)) {
522 - st_pg_cache_hit_ratio = rrdset_create_localhost(
523 - "netdata"
524 - , "page_cache_hit_ratio"
525 - , NULL
526 - , "dbengine"
527 - , NULL
528 - , "Netdata DB engine page cache hit ratio"
529 - , "percentage"
530 - , "netdata"
531 - , "stats"
532 - , 130503
533 - , localhost->rrd_update_every
534 - , RRDSET_TYPE_LINE
535 - );
536 -
537 - rd_hit_ratio = rrddim_add(st_pg_cache_hit_ratio, "ratio", NULL, 1, 1000, RRD_ALGORITHM_ABSOLUTE);
573 + // ----------------------------------------------------------------
574 +
575 + {
576 + static RRDSET *st_pg_cache_pages = NULL;
577 + static RRDDIM *rd_descriptors = NULL;
578 + static RRDDIM *rd_populated = NULL;
579 + static RRDDIM *rd_dirty = NULL;
580 + static RRDDIM *rd_backfills = NULL;
581 + static RRDDIM *rd_evictions = NULL;
582 + static RRDDIM *rd_used_by_collectors = NULL;
583 +
584 + if (unlikely(!st_pg_cache_pages)) {
585 + st_pg_cache_pages = rrdset_create_localhost(
586 + "netdata",
587 + "page_cache_stats",
588 + NULL,
589 + "dbengine",
590 + NULL,
591 + "Netdata dbengine page cache statistics",
592 + "pages",
593 + "netdata",
594 + "stats",
595 + 130504,
596 + localhost->rrd_update_every,
597 + RRDSET_TYPE_LINE);
598 +
599 + rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
600 + rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
601 + rd_dirty = rrddim_add(st_pg_cache_pages, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
602 + rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
603 + rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
604 + rd_used_by_collectors =
605 + rrddim_add(st_pg_cache_pages, "used_by_collectors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
606 + } else
607 + rrdset_next(st_pg_cache_pages);
608 +
609 + rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]);
610 + rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
611 + rrddim_set_by_pointer(st_pg_cache_pages, rd_dirty, (collected_number)stats_array[0] + stats_array[4]);
612 + rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]);
613 + rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]);
614 + rrddim_set_by_pointer(st_pg_cache_pages, rd_used_by_collectors, (collected_number)stats_array[0]);
615 + rrdset_done(st_pg_cache_pages);
616 }
539 - else
540 - rrdset_next(st_pg_cache_hit_ratio);
541 -
542 - static unsigned long long old_hits = 0;
543 - static unsigned long long old_misses = 0;
544 - unsigned long long hits = stats_array[7];
545 - unsigned long long misses = stats_array[8];
546 - unsigned long long hits_delta;
547 - unsigned long long misses_delta;
548 - unsigned long long ratio;
549 -
550 - hits_delta = hits - old_hits;
551 - misses_delta = misses - old_misses;
552 - old_hits = hits;
553 - old_misses = misses;
554 -
555 - if (hits_delta + misses_delta) {
556 - ratio = (hits_delta * 100 * 1000) / (hits_delta + misses_delta);
557 - } else {
558 - ratio = 0;
617 +
618 + // ----------------------------------------------------------------
619 +
620 + {
621 + static RRDSET *st_long_term_pages = NULL;
622 + static RRDDIM *rd_total = NULL;
623 + static RRDDIM *rd_insertions = NULL;
624 + static RRDDIM *rd_deletions = NULL;
625 + static RRDDIM *rd_flushing_pressure_deletions = NULL;
626 +
627 + if (unlikely(!st_long_term_pages)) {
628 + st_long_term_pages = rrdset_create_localhost(
629 + "netdata",
630 + "dbengine_long_term_page_stats",
631 + NULL,
632 + "dbengine",
633 + NULL,
634 + "Netdata dbengine long-term page statistics",
635 + "pages",
636 + "netdata",
637 + "stats",
638 + 130505,
639 + localhost->rrd_update_every,
640 + RRDSET_TYPE_LINE);
641 +
642 + rd_total = rrddim_add(st_long_term_pages, "total", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
643 + rd_insertions = rrddim_add(st_long_term_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
644 + rd_deletions = rrddim_add(st_long_term_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
645 + rd_flushing_pressure_deletions = rrddim_add(
646 + st_long_term_pages, "flushing_pressure_deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
647 + } else
648 + rrdset_next(st_long_term_pages);
649 +
650 + rrddim_set_by_pointer(st_long_term_pages, rd_total, (collected_number)stats_array[2]);
651 + rrddim_set_by_pointer(st_long_term_pages, rd_insertions, (collected_number)stats_array[5]);
652 + rrddim_set_by_pointer(st_long_term_pages, rd_deletions, (collected_number)stats_array[6]);
653 + rrddim_set_by_pointer(
654 + st_long_term_pages, rd_flushing_pressure_deletions, (collected_number)stats_array[36]);
655 + rrdset_done(st_long_term_pages);
656 }
560 - rrddim_set_by_pointer(st_pg_cache_hit_ratio, rd_hit_ratio, ratio);
657
562 - rrdset_done(st_pg_cache_hit_ratio);
563 - }
658 + // ----------------------------------------------------------------
659 +
660 + {
661 + static RRDSET *st_io_stats = NULL;
662 + static RRDDIM *rd_reads = NULL;
663 + static RRDDIM *rd_writes = NULL;
664 +
665 + if (unlikely(!st_io_stats)) {
666 + st_io_stats = rrdset_create_localhost(
667 + "netdata",
668 + "dbengine_io_throughput",
669 + NULL,
670 + "dbengine",
671 + NULL,
672 + "Netdata DB engine I/O throughput",
673 + "MiB/s",
674 + "netdata",
675 + "stats",
676 + 130506,
677 + localhost->rrd_update_every,
678 + RRDSET_TYPE_LINE);
679 +
680 + rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
681 + rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
682 + } else
683 + rrdset_next(st_io_stats);
684 +
685 + rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[17]);
686 + rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[15]);
687 + rrdset_done(st_io_stats);
688 + }
689
565 - // ----------------------------------------------------------------
566 -
567 - {
568 - static RRDSET *st_pg_cache_pages = NULL;
569 - static RRDDIM *rd_descriptors = NULL;
570 - static RRDDIM *rd_populated = NULL;
571 - static RRDDIM *rd_dirty = NULL;
572 - static RRDDIM *rd_backfills = NULL;
573 - static RRDDIM *rd_evictions = NULL;
574 - static RRDDIM *rd_used_by_collectors = NULL;
575 -
576 - if (unlikely(!st_pg_cache_pages)) {
577 - st_pg_cache_pages = rrdset_create_localhost(
578 - "netdata"
579 - , "page_cache_stats"
580 - , NULL
581 - , "dbengine"
582 - , NULL
583 - , "Netdata dbengine page cache statistics"
584 - , "pages"
585 - , "netdata"
586 - , "stats"
587 - , 130504
588 - , localhost->rrd_update_every
589 - , RRDSET_TYPE_LINE
590 - );
591 -
592 - rd_descriptors = rrddim_add(st_pg_cache_pages, "descriptors", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
593 - rd_populated = rrddim_add(st_pg_cache_pages, "populated", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
594 - rd_dirty = rrddim_add(st_pg_cache_pages, "dirty", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
595 - rd_backfills = rrddim_add(st_pg_cache_pages, "backfills", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
596 - rd_evictions = rrddim_add(st_pg_cache_pages, "evictions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
597 - rd_used_by_collectors = rrddim_add(st_pg_cache_pages, "used_by_collectors", NULL, 1, 1,
598 - RRD_ALGORITHM_ABSOLUTE);
690 + // ----------------------------------------------------------------
691 +
692 + {
693 + static RRDSET *st_io_stats = NULL;
694 + static RRDDIM *rd_reads = NULL;
695 + static RRDDIM *rd_writes = NULL;
696 +
697 + if (unlikely(!st_io_stats)) {
698 + st_io_stats = rrdset_create_localhost(
699 + "netdata",
700 + "dbengine_io_operations",
701 + NULL,
702 + "dbengine",
703 + NULL,
704 + "Netdata DB engine I/O operations",
705 + "operations/s",
706 + "netdata",
707 + "stats",
708 + 130507,
709 + localhost->rrd_update_every,
710 + RRDSET_TYPE_LINE);
711 +
712 + rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
713 + rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
714 + } else
715 + rrdset_next(st_io_stats);
716 +
717 + rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[18]);
718 + rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]);
719 + rrdset_done(st_io_stats);
720 }
600 - else
601 - rrdset_next(st_pg_cache_pages);
602 -
603 - rrddim_set_by_pointer(st_pg_cache_pages, rd_descriptors, (collected_number)stats_array[27]);
604 - rrddim_set_by_pointer(st_pg_cache_pages, rd_populated, (collected_number)stats_array[3]);
605 - rrddim_set_by_pointer(st_pg_cache_pages, rd_dirty, (collected_number)stats_array[0] + stats_array[4]);
606 - rrddim_set_by_pointer(st_pg_cache_pages, rd_backfills, (collected_number)stats_array[9]);
607 - rrddim_set_by_pointer(st_pg_cache_pages, rd_evictions, (collected_number)stats_array[10]);
608 - rrddim_set_by_pointer(st_pg_cache_pages, rd_used_by_collectors, (collected_number)stats_array[0]);
609 - rrdset_done(st_pg_cache_pages);
610 - }
721
612 - // ----------------------------------------------------------------
722 + // ----------------------------------------------------------------
723 +
724 + {
725 + static RRDSET *st_errors = NULL;
726 + static RRDDIM *rd_fs_errors = NULL;
727 + static RRDDIM *rd_io_errors = NULL;
728 + static RRDDIM *pg_cache_over_half_dirty_events = NULL;
729 +
730 + if (unlikely(!st_errors)) {
731 + st_errors = rrdset_create_localhost(
732 + "netdata",
733 + "dbengine_global_errors",
734 + NULL,
735 + "dbengine",
736 + NULL,
737 + "Netdata DB engine errors",
738 + "errors/s",
739 + "netdata",
740 + "stats",
741 + 130508,
742 + localhost->rrd_update_every,
743 + RRDSET_TYPE_LINE);
744 +
745 + rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
746 + rd_fs_errors = rrddim_add(st_errors, "fs_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
747 + pg_cache_over_half_dirty_events =
748 + rrddim_add(st_errors, "pg_cache_over_half_dirty_events", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
749 + } else
750 + rrdset_next(st_errors);
751 +
752 + rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
753 + rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
754 + rrddim_set_by_pointer(st_errors, pg_cache_over_half_dirty_events, (collected_number)stats_array[34]);
755 + rrdset_done(st_errors);
756 + }
757
614 - {
615 - static RRDSET *st_long_term_pages = NULL;
616 - static RRDDIM *rd_total = NULL;
617 - static RRDDIM *rd_insertions = NULL;
618 - static RRDDIM *rd_deletions = NULL;
619 - static RRDDIM *rd_flushing_pressure_deletions = NULL;
758 + // ----------------------------------------------------------------
759 +
760 + {
761 + static RRDSET *st_fd = NULL;
762 + static RRDDIM *rd_fd_current = NULL;
763 + static RRDDIM *rd_fd_max = NULL;
764 +
765 + if (unlikely(!st_fd)) {
766 + st_fd = rrdset_create_localhost(
767 + "netdata",
768 + "dbengine_global_file_descriptors",
769 + NULL,
770 + "dbengine",
771 + NULL,
772 + "Netdata DB engine File Descriptors",
773 + "descriptors",
774 + "netdata",
775 + "stats",
776 + 130509,
777 + localhost->rrd_update_every,
778 + RRDSET_TYPE_LINE);
779 +
780 + rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
781 + rd_fd_max = rrddim_add(st_fd, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
782 + } else
783 + rrdset_next(st_fd);
784 +
785 + rrddim_set_by_pointer(st_fd, rd_fd_current, (collected_number)stats_array[32]);
786 + /* Careful here, modify this accordingly if the File-Descriptor budget ever changes */
787 + rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4);
788 + rrdset_done(st_fd);
789 + }
790
621 - if (unlikely(!st_long_term_pages)) {
622 - st_long_term_pages = rrdset_create_localhost(
623 - "netdata"
624 - , "dbengine_long_term_page_stats"
625 - , NULL
626 - , "dbengine"
627 - , NULL
628 - , "Netdata dbengine long-term page statistics"
629 - , "pages"
630 - , "netdata"
631 - , "stats"
632 - , 130505
633 - , localhost->rrd_update_every
634 - , RRDSET_TYPE_LINE
635 - );
636 -
637 - rd_total = rrddim_add(st_long_term_pages, "total", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
638 - rd_insertions = rrddim_add(st_long_term_pages, "insertions", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
639 - rd_deletions = rrddim_add(st_long_term_pages, "deletions", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
640 - rd_flushing_pressure_deletions = rrddim_add(st_long_term_pages, "flushing_pressure_deletions", NULL, -1,
641 - 1, RRD_ALGORITHM_INCREMENTAL);
791 + // ----------------------------------------------------------------
792 +
793 + {
794 + static RRDSET *st_ram_usage = NULL;
795 + static RRDDIM *rd_cached = NULL;
796 + static RRDDIM *rd_pinned = NULL;
797 + static RRDDIM *rd_metadata = NULL;
798 +
799 + collected_number cached_pages, pinned_pages, API_producers, populated_pages, metadata, pages_on_disk,
800 + page_cache_descriptors;
801 +
802 + if (unlikely(!st_ram_usage)) {
803 + st_ram_usage = rrdset_create_localhost(
804 + "netdata",
805 + "dbengine_ram",
806 + NULL,
807 + "dbengine",
808 + NULL,
809 + "Netdata DB engine RAM usage",
810 + "MiB",
811 + "netdata",
812 + "stats",
813 + 130510,
814 + localhost->rrd_update_every,
815 + RRDSET_TYPE_STACKED);
816 +
817 + rd_cached = rrddim_add(st_ram_usage, "cache", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
818 + rd_pinned = rrddim_add(st_ram_usage, "collectors", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
819 + rd_metadata = rrddim_add(st_ram_usage, "metadata", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE);
820 + } else
821 + rrdset_next(st_ram_usage);
822 +
823 + API_producers = (collected_number)stats_array[0];
824 + pages_on_disk = (collected_number)stats_array[2];
825 + populated_pages = (collected_number)stats_array[3];
826 + page_cache_descriptors = (collected_number)stats_array[27];
827 +
828 + if (API_producers * 2 > populated_pages) {
829 + pinned_pages = API_producers;
830 + } else {
831 + pinned_pages = API_producers * 2;
832 + }
833 + cached_pages = populated_pages - pinned_pages;
834 +
835 + metadata = page_cache_descriptors * sizeof(struct page_cache_descr);
836 + metadata += pages_on_disk * sizeof(struct rrdeng_page_descr);
837 + /* This is an empirical estimation for Judy array indexing and extent structures */
838 + metadata += pages_on_disk * 58;
839 +
840 + rrddim_set_by_pointer(st_ram_usage, rd_cached, cached_pages);
841 + rrddim_set_by_pointer(st_ram_usage, rd_pinned, pinned_pages);
842 + rrddim_set_by_pointer(st_ram_usage, rd_metadata, metadata);
843 + rrdset_done(st_ram_usage);
844 }
643 - else
644 - rrdset_next(st_long_term_pages);
645 -
646 - rrddim_set_by_pointer(st_long_term_pages, rd_total, (collected_number)stats_array[2]);
647 - rrddim_set_by_pointer(st_long_term_pages, rd_insertions, (collected_number)stats_array[5]);
648 - rrddim_set_by_pointer(st_long_term_pages, rd_deletions, (collected_number)stats_array[6]);
649 - rrddim_set_by_pointer(st_long_term_pages, rd_flushing_pressure_deletions,
650 - (collected_number)stats_array[36]);
651 - rrdset_done(st_long_term_pages);
845 }
846 + }
847 +#endif
848 +}
849 +
850 +static void update_heartbeat_charts() {
851 + RRDSET *st = rrdset_create_localhost(
852 + "netdata"
853 + , "heartbeat"
854 + , NULL
855 + , "heartbeat"
856 + , NULL
857 + , "System clock jitter"
858 + , "microseconds"
859 + , "netdata"
860 + , "stats"
861 + , 900000
862 + , localhost->rrd_update_every
863 + , RRDSET_TYPE_AREA
864 + );
865 +
866 + RRDDIM *rd_min = rrddim_add(st, "min", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
867 + RRDDIM *rd_max = rrddim_add(st, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
868 + RRDDIM *rd_avg = rrddim_add(st, "average", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
869 +
870 + rrdset_next(st);
871 +
872 + usec_t min, max, average;
873 + size_t count;
874 +
875 + heartbeat_statistics(&min, &max, &average, &count);
876 +
877 + rrddim_set_by_pointer(st, rd_min, (collected_number)min);
878 + rrddim_set_by_pointer(st, rd_max, (collected_number)max);
879 + rrddim_set_by_pointer(st, rd_avg, (collected_number)average);
880 +
881 + rrdset_done(st);
882 +}
883 +
884 +// ---------------------------------------------------------------------------------------------------------------------
885 +// worker utilization
886 +
887 +struct worker_job_type {
888 + char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
889 + size_t jobs_started;
890 + usec_t busy_time;
891 +
892 + RRDDIM *rd_jobs_started;
893 + RRDDIM *rd_busy_time;
894 +};
895 +
896 +struct worker_thread {
897 + pid_t pid;
898 + int enabled;
899 +
900 + int cpu_enabled;
901 +
902 + kernel_uint_t utime;
903 + kernel_uint_t stime;
904 +
905 + kernel_uint_t utime_old;
906 + kernel_uint_t stime_old;
907 +
908 + usec_t collected_time;
909 + usec_t collected_time_old;
910 +
911 + size_t jobs_started;
912 + usec_t busy_time;
913 +
914 + struct worker_thread *next;
915 +};
916 +
917 +struct worker_utilization {
918 + const char *name;
919 + const char *family;
920 + size_t priority;
921 + uint32_t flags;
922 +
923 + char *name_lowercase;
924 +
925 + struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
926 +
927 + size_t workers_registered;
928 + size_t workers_busy;
929 + usec_t workers_total_busy_time;
930 + usec_t workers_total_duration;
931 + size_t workers_total_jobs_started;
932 + double workers_min_busy_time;
933 + double workers_max_busy_time;
934
654 - // ----------------------------------------------------------------
935 + struct worker_thread *threads;
936
656 - {
657 - static RRDSET *st_io_stats = NULL;
658 - static RRDDIM *rd_reads = NULL;
659 - static RRDDIM *rd_writes = NULL;
937 + RRDSET *st_workers_time;
938 + RRDDIM *rd_workers_time_avg;
939 + RRDDIM *rd_workers_time_min;
940 + RRDDIM *rd_workers_time_max;
941
661 - if (unlikely(!st_io_stats)) {
662 - st_io_stats = rrdset_create_localhost(
942 + size_t workers_cpu_enabled;
943 + RRDSET *st_workers_cpu;
944 + RRDDIM *rd_workers_cpu_avg;
945 + RRDDIM *rd_workers_cpu_min;
946 + RRDDIM *rd_workers_cpu_max;
947 +
948 + RRDSET *st_workers_threads;
949 + RRDDIM *rd_workers_threads_free;
950 + RRDDIM *rd_workers_threads_busy;
951 +
952 + RRDSET *st_workers_jobs_per_job_type;
953 + RRDSET *st_workers_busy_per_job_type;
954 +};
955 +
956 +static void workers_utilization_update_chart(struct worker_utilization *wu) {
957 + if(!wu->workers_registered) return;
958 +
959 + //fprintf(stderr, "%-12s WORKER UTILIZATION: %-3.2f%%, %zu jobs done, %zu running, on %zu workers, min %-3.02f%%, max %-3.02f%%.\n",
960 + // wu->name,
961 + // (double)wu->workers_total_busy_time * 100.0 / (double)wu->workers_total_duration,
962 + // wu->workers_total_jobs_started, wu->workers_busy, wu->workers_registered,
963 + // wu->workers_min_busy_time, wu->workers_max_busy_time);
964 +
965 + // ----------------------------------------------------------------------
966 +
967 + if(unlikely(!wu->st_workers_time)) {
968 + char name[RRD_ID_LENGTH_MAX + 1];
969 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_time_%s", wu->name_lowercase);
970 +
971 + char context[RRD_ID_LENGTH_MAX + 1];
972 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.time", wu->name_lowercase);
973 +
974 + wu->st_workers_time = rrdset_create_localhost(
975 + "netdata"
976 + , name
977 + , NULL
978 + , wu->family
979 + , context
980 + , "Netdata Workers Busy Time (100% = all workers busy)"
981 + , "%"
982 + , "netdata"
983 + , "stats"
984 + , wu->priority
985 + , localhost->rrd_update_every
986 + , RRDSET_TYPE_AREA
987 + );
988 + }
989 +
990 + // we add the min and max dimensions only when we have multiple workers
991 +
992 + if(unlikely(!wu->rd_workers_time_min && wu->workers_registered > 1))
993 + wu->rd_workers_time_min = rrddim_add(wu->st_workers_time, "min", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
994 +
995 + if(unlikely(!wu->rd_workers_time_max && wu->workers_registered > 1))
996 + wu->rd_workers_time_max = rrddim_add(wu->st_workers_time, "max", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
997 +
998 + if(unlikely(!wu->rd_workers_time_avg))
999 + wu->rd_workers_time_avg = rrddim_add(wu->st_workers_time, "average", NULL, 1, 10000, RRD_ALGORITHM_ABSOLUTE);
1000 +
1001 + rrdset_next(wu->st_workers_time);
1002 +
1003 + if(wu->rd_workers_time_min)
1004 + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * 10000.0));
1005 +
1006 + if(wu->rd_workers_time_max)
1007 + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * 10000.0));
1008 +
1009 + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * 10000.0 / (double)wu->workers_total_duration));
1010 + rrdset_done(wu->st_workers_time);
1011 +
1012 + // ----------------------------------------------------------------------
1013 +
1014 +#ifdef __linux__
1015 + if(wu->workers_cpu_enabled || wu->st_workers_cpu) {
1016 + if(unlikely(!wu->st_workers_cpu)) {
1017 + char name[RRD_ID_LENGTH_MAX + 1];
1018 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_cpu_%s", wu->name_lowercase);
1019 +
1020 + char context[RRD_ID_LENGTH_MAX + 1];
1021 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.cpu", wu->name_lowercase);
1022 +
1023 + wu->st_workers_cpu = rrdset_create_localhost(
1024 "netdata"
664 - , "dbengine_io_throughput"
665 - , NULL
666 - , "dbengine"
1025 + , name
1026 , NULL
668 - , "Netdata DB engine I/O throughput"
669 - , "MiB/s"
1027 + , wu->family
1028 + , context
1029 + , "Netdata Workers CPU Utilization (100% = all workers busy)"
1030 + , "%"
1031 , "netdata"
1032 , "stats"
672 - , 130506
1033 + , wu->priority + 1
1034 , localhost->rrd_update_every
674 - , RRDSET_TYPE_LINE
675 - );
1035 + , RRDSET_TYPE_AREA
1036 + );
1037 + }
1038
677 - rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
678 - rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1024 * 1024, RRD_ALGORITHM_INCREMENTAL);
679 - }
680 - else
681 - rrdset_next(st_io_stats);
1039 + if (unlikely(!wu->rd_workers_cpu_min && wu->workers_registered > 1))
1040 + wu->rd_workers_cpu_min = rrddim_add(wu->st_workers_cpu, "min", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
1041
683 - rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[17]);
684 - rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[15]);
685 - rrdset_done(st_io_stats);
686 - }
1042 + if (unlikely(!wu->rd_workers_cpu_max && wu->workers_registered > 1))
1043 + wu->rd_workers_cpu_max = rrddim_add(wu->st_workers_cpu, "max", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
1044
688 - // ----------------------------------------------------------------
689 -
690 - {
691 - static RRDSET *st_io_stats = NULL;
692 - static RRDDIM *rd_reads = NULL;
693 - static RRDDIM *rd_writes = NULL;
694 -
695 - if (unlikely(!st_io_stats)) {
696 - st_io_stats = rrdset_create_localhost(
697 - "netdata"
698 - , "dbengine_io_operations"
699 - , NULL
700 - , "dbengine"
701 - , NULL
702 - , "Netdata DB engine I/O operations"
703 - , "operations/s"
704 - , "netdata"
705 - , "stats"
706 - , 130507
707 - , localhost->rrd_update_every
708 - , RRDSET_TYPE_LINE
709 - );
710 -
711 - rd_reads = rrddim_add(st_io_stats, "reads", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
712 - rd_writes = rrddim_add(st_io_stats, "writes", NULL, -1, 1, RRD_ALGORITHM_INCREMENTAL);
713 - }
714 - else
715 - rrdset_next(st_io_stats);
1045 + if(unlikely(!wu->rd_workers_cpu_avg))
1046 + wu->rd_workers_cpu_avg = rrddim_add(wu->st_workers_cpu, "average", NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
1047
717 - rrddim_set_by_pointer(st_io_stats, rd_reads, (collected_number)stats_array[18]);
718 - rrddim_set_by_pointer(st_io_stats, rd_writes, (collected_number)stats_array[16]);
719 - rrdset_done(st_io_stats);
720 - }
1048 + rrdset_next(wu->st_workers_cpu);
1049
722 - // ----------------------------------------------------------------
723 -
724 - {
725 - static RRDSET *st_errors = NULL;
726 - static RRDDIM *rd_fs_errors = NULL;
727 - static RRDDIM *rd_io_errors = NULL;
728 - static RRDDIM *pg_cache_over_half_dirty_events = NULL;
729 -
730 - if (unlikely(!st_errors)) {
731 - st_errors = rrdset_create_localhost(
732 - "netdata"
733 - , "dbengine_global_errors"
734 - , NULL
735 - , "dbengine"
736 - , NULL
737 - , "Netdata DB engine errors"
738 - , "errors/s"
739 - , "netdata"
740 - , "stats"
741 - , 130508
742 - , localhost->rrd_update_every
743 - , RRDSET_TYPE_LINE
744 - );
745 -
746 - rd_io_errors = rrddim_add(st_errors, "io_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
747 - rd_fs_errors = rrddim_add(st_errors, "fs_errors", NULL, 1, 1, RRD_ALGORITHM_INCREMENTAL);
748 - pg_cache_over_half_dirty_events = rrddim_add(st_errors, "pg_cache_over_half_dirty_events", NULL, 1, 1,
749 - RRD_ALGORITHM_INCREMENTAL);
750 - }
751 - else
752 - rrdset_next(st_errors);
1050 + size_t count = 0;
1051 + calculated_number min = 1000.0, max = 0.0, total = 0.0;
1052 + struct worker_thread *wt;
1053 + for(wt = wu->threads; wt ; wt = wt->next) {
1054 + if(!wt->cpu_enabled) continue;
1055 + count++;
1056
754 - rrddim_set_by_pointer(st_errors, rd_io_errors, (collected_number)stats_array[30]);
755 - rrddim_set_by_pointer(st_errors, rd_fs_errors, (collected_number)stats_array[31]);
756 - rrddim_set_by_pointer(st_errors, pg_cache_over_half_dirty_events, (collected_number)stats_array[34]);
757 - rrdset_done(st_errors);
1057 + usec_t delta = wt->collected_time - wt->collected_time_old;
1058 + calculated_number utime = (calculated_number)(wt->utime - wt->utime_old) / (calculated_number)system_hz * 100.0 * (calculated_number)USEC_PER_SEC / (calculated_number)delta;
1059 + calculated_number stime = (calculated_number)(wt->stime - wt->stime_old) / (calculated_number)system_hz * 100.0 * (calculated_number)USEC_PER_SEC / (calculated_number)delta;
1060 + calculated_number cpu_util = utime + stime;
1061 +
1062 + total += cpu_util;
1063 + if(cpu_util < min) min = cpu_util;
1064 + if(cpu_util > max) max = cpu_util;
1065 }
1066 + if(unlikely(min == 1000.0)) min = 0.0;
1067 +
1068 + if(wu->rd_workers_cpu_min)
1069 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(min * 10000ULL));
1070 +
1071 + if(wu->rd_workers_cpu_max)
1072 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(max * 10000ULL));
1073
760 - // ----------------------------------------------------------------
761 -
762 - {
763 - static RRDSET *st_fd = NULL;
764 - static RRDDIM *rd_fd_current = NULL;
765 - static RRDDIM *rd_fd_max = NULL;
766 -
767 - if (unlikely(!st_fd)) {
768 - st_fd = rrdset_create_localhost(
769 - "netdata"
770 - , "dbengine_global_file_descriptors"
771 - , NULL
772 - , "dbengine"
773 - , NULL
774 - , "Netdata DB engine File Descriptors"
775 - , "descriptors"
776 - , "netdata"
777 - , "stats"
778 - , 130509
779 - , localhost->rrd_update_every
780 - , RRDSET_TYPE_LINE
781 - );
782 -
783 - rd_fd_current = rrddim_add(st_fd, "current", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
784 - rd_fd_max = rrddim_add(st_fd, "max", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1074 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( total * 10000ULL / (calculated_number)count ));
1075 + rrdset_done(wu->st_workers_cpu);
1076 + }
1077 +#endif
1078 +
1079 + // ----------------------------------------------------------------------
1080 +
1081 + if(unlikely(!wu->st_workers_jobs_per_job_type)) {
1082 + char name[RRD_ID_LENGTH_MAX + 1];
1083 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_jobs_by_type_%s", wu->name_lowercase);
1084 +
1085 + char context[RRD_ID_LENGTH_MAX + 1];
1086 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.jobs_started_by_type", wu->name_lowercase);
1087 +
1088 + wu->st_workers_jobs_per_job_type = rrdset_create_localhost(
1089 + "netdata"
1090 + , name
1091 + , NULL
1092 + , wu->family
1093 + , context
1094 + , "Netdata Workers Jobs Started by Type"
1095 + , "jobs"
1096 + , "netdata"
1097 + , "stats"
1098 + , wu->priority + 2
1099 + , localhost->rrd_update_every
1100 + , RRDSET_TYPE_STACKED
1101 + );
1102 + }
1103 +
1104 + rrdset_next(wu->st_workers_jobs_per_job_type);
1105 +
1106 + {
1107 + size_t i;
1108 + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
1109 + if (wu->per_job_type[i].name[0]) {
1110 +
1111 + if(unlikely(!wu->per_job_type[i].rd_jobs_started))
1112 + wu->per_job_type[i].rd_jobs_started = rrddim_add(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].name, NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1113 +
1114 + rrddim_set_by_pointer(wu->st_workers_jobs_per_job_type, wu->per_job_type[i].rd_jobs_started, (collected_number)(wu->per_job_type[i].jobs_started));
1115 }
786 - else
787 - rrdset_next(st_fd);
1116 + }
1117 + }
1118
789 - rrddim_set_by_pointer(st_fd, rd_fd_current, (collected_number)stats_array[32]);
790 - /* Careful here, modify this accordingly if the File-Descriptor budget ever changes */
791 - rrddim_set_by_pointer(st_fd, rd_fd_max, (collected_number)rlimit_nofile.rlim_cur / 4);
792 - rrdset_done(st_fd);
1119 + rrdset_done(wu->st_workers_jobs_per_job_type);
1120 +
1121 + // ----------------------------------------------------------------------
1122 +
1123 + if(unlikely(!wu->st_workers_busy_per_job_type)) {
1124 + char name[RRD_ID_LENGTH_MAX + 1];
1125 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_busy_time_by_type_%s", wu->name_lowercase);
1126 +
1127 + char context[RRD_ID_LENGTH_MAX + 1];
1128 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.time_by_type", wu->name_lowercase);
1129 +
1130 + wu->st_workers_busy_per_job_type = rrdset_create_localhost(
1131 + "netdata"
1132 + , name
1133 + , NULL
1134 + , wu->family
1135 + , context
1136 + , "Netdata Workers Busy Time by Type"
1137 + , "ms"
1138 + , "netdata"
1139 + , "stats"
1140 + , wu->priority + 3
1141 + , localhost->rrd_update_every
1142 + , RRDSET_TYPE_STACKED
1143 + );
1144 + }
1145 +
1146 + rrdset_next(wu->st_workers_busy_per_job_type);
1147 +
1148 + {
1149 + size_t i;
1150 + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
1151 + if (wu->per_job_type[i].name[0]) {
1152 +
1153 + if(unlikely(!wu->per_job_type[i].rd_busy_time))
1154 + wu->per_job_type[i].rd_busy_time = rrddim_add(wu->st_workers_busy_per_job_type, wu->per_job_type[i].name, NULL, 1, USEC_PER_MS, RRD_ALGORITHM_ABSOLUTE);
1155 +
1156 + rrddim_set_by_pointer(wu->st_workers_busy_per_job_type, wu->per_job_type[i].rd_busy_time, (collected_number)(wu->per_job_type[i].busy_time));
1157 + }
1158 }
1159 + }
1160
795 - // ----------------------------------------------------------------
1161 + rrdset_done(wu->st_workers_busy_per_job_type);
1162
797 - {
798 - static RRDSET *st_ram_usage = NULL;
799 - static RRDDIM *rd_cached = NULL;
800 - static RRDDIM *rd_pinned = NULL;
801 - static RRDDIM *rd_metadata = NULL;
1163 + // ----------------------------------------------------------------------
1164
803 - collected_number cached_pages, pinned_pages, API_producers, populated_pages, metadata, pages_on_disk,
804 - page_cache_descriptors;
1165 + if(wu->st_workers_threads || wu->workers_registered > 1) {
1166 + if(unlikely(!wu->st_workers_threads)) {
1167 + char name[RRD_ID_LENGTH_MAX + 1];
1168 + snprintfz(name, RRD_ID_LENGTH_MAX, "workers_threads_%s", wu->name_lowercase);
1169
806 - if (unlikely(!st_ram_usage)) {
807 - st_ram_usage = rrdset_create_localhost(
1170 + char context[RRD_ID_LENGTH_MAX + 1];
1171 + snprintf(context, RRD_ID_LENGTH_MAX, "netdata.workers.%s.threads", wu->name_lowercase);
1172 +
1173 + wu->st_workers_threads = rrdset_create_localhost(
1174 "netdata"
809 - , "dbengine_ram"
810 - , NULL
811 - , "dbengine"
1175 + , name
1176 , NULL
813 - , "Netdata DB engine RAM usage"
814 - , "MiB"
1177 + , wu->family
1178 + , context
1179 + , "Netdata Workers Threads"
1180 + , "threads"
1181 , "netdata"
1182 , "stats"
817 - , 130510
1183 + , wu->priority + 4
1184 , localhost->rrd_update_every
1185 , RRDSET_TYPE_STACKED
820 - );
1186 + );
1187
822 - rd_cached = rrddim_add(st_ram_usage, "cache", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
823 - rd_pinned = rrddim_add(st_ram_usage, "collectors", NULL, 1, 256, RRD_ALGORITHM_ABSOLUTE);
824 - rd_metadata = rrddim_add(st_ram_usage, "metadata", NULL, 1, 1048576, RRD_ALGORITHM_ABSOLUTE);
825 - }
826 - else
827 - rrdset_next(st_ram_usage);
828 -
829 - API_producers = (collected_number)stats_array[0];
830 - pages_on_disk = (collected_number)stats_array[2];
831 - populated_pages = (collected_number)stats_array[3];
832 - page_cache_descriptors = (collected_number)stats_array[27];
833 -
834 - if (API_producers * 2 > populated_pages) {
835 - pinned_pages = API_producers;
836 - } else{
837 - pinned_pages = API_producers * 2;
838 - }
839 - cached_pages = populated_pages - pinned_pages;
1188 + wu->rd_workers_threads_free = rrddim_add(wu->st_workers_threads, "free", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1189 + wu->rd_workers_threads_busy = rrddim_add(wu->st_workers_threads, "busy", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
1190 + }
1191 + else
1192 + rrdset_next(wu->st_workers_threads);
1193
841 - metadata = page_cache_descriptors * sizeof(struct page_cache_descr);
842 - metadata += pages_on_disk * sizeof(struct rrdeng_page_descr);
843 - /* This is an empirical estimation for Judy array indexing and extent structures */
844 - metadata += pages_on_disk * 58;
1194 + rrddim_set_by_pointer(wu->st_workers_threads, wu->rd_workers_threads_free, (collected_number)(wu->workers_registered - wu->workers_busy));
1195 + rrddim_set_by_pointer(wu->st_workers_threads, wu->rd_workers_threads_busy, (collected_number)(wu->workers_busy));
1196 + rrdset_done(wu->st_workers_threads);
1197 + }
1198 +}
1199
846 - rrddim_set_by_pointer(st_ram_usage, rd_cached, cached_pages);
847 - rrddim_set_by_pointer(st_ram_usage, rd_pinned, pinned_pages);
848 - rrddim_set_by_pointer(st_ram_usage, rd_metadata, metadata);
849 - rrdset_done(st_ram_usage);
1200 +static void workers_utilization_reset_statistics(struct worker_utilization *wu) {
1201 + wu->workers_registered = 0;
1202 + wu->workers_busy = 0;
1203 + wu->workers_total_busy_time = 0;
1204 + wu->workers_total_duration = 0;
1205 + wu->workers_total_jobs_started = 0;
1206 + wu->workers_min_busy_time = 100.0;
1207 + wu->workers_max_busy_time = 0;
1208 + wu->workers_cpu_enabled = 0;
1209 +
1210 + size_t i;
1211 + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
1212 + if(unlikely(!wu->name_lowercase)) {
1213 + wu->name_lowercase = strdupz(wu->name);
1214 + char *s = wu->name_lowercase;
1215 + for( ; *s ; s++) *s = tolower(*s);
1216 }
1217 +
1218 + wu->per_job_type[i].jobs_started = 0;
1219 + wu->per_job_type[i].busy_time = 0;
1220 + }
1221 +
1222 + struct worker_thread *wt;
1223 + for(wt = wu->threads; wt ; wt = wt->next) {
1224 + wt->enabled = 0;
1225 + wt->cpu_enabled = 0;
1226 }
1227 +}
1228 +
1229 +static int read_thread_cpu_time_from_proc_stat(pid_t pid __maybe_unused, kernel_uint_t *utime __maybe_unused, kernel_uint_t *stime __maybe_unused) {
1230 +#ifdef __linux__
1231 + char filename[200 + 1];
1232 + snprintfz(filename, 200, "/proc/self/task/%d/stat", pid);
1233 +
1234 + procfile *ff = procfile_open(filename, " ", PROCFILE_FLAG_NO_ERROR_ON_FILE_IO);
1235 + if(!ff) return -1;
1236 +
1237 + ff = procfile_readall(ff);
1238 + if(!ff) return -1;
1239 +
1240 + *utime = str2kernel_uint_t(procfile_lineword(ff, 0, 13));
1241 + *stime = str2kernel_uint_t(procfile_lineword(ff, 0, 14));
1242 +
1243 + procfile_close(ff);
1244 + return 0;
1245 +#else
1246 + // TODO: add here cpu time detection per thread, for FreeBSD and MacOS
1247 + *utime = 0;
1248 + *stime = 0;
1249 + return 1;
1250 #endif
1251 +}
1252 +
1253 +static void workers_threads_cleanup(struct worker_utilization *wu) {
1254 + struct worker_thread *t;
1255 +
1256 + // free threads at the beginning of the linked list
1257 + while(wu->threads && !wu->threads->enabled) {
1258 + t = wu->threads;
1259 + wu->threads = t->next;
1260 + t->next = NULL;
1261 + freez(t);
1262 + }
1263 +
1264 + // free threads in the middle of the linked list
1265 + for(t = wu->threads; t && t->next ; t = t->next) {
1266 + if(t->next->enabled) continue;
1267 +
1268 + struct worker_thread *to_remove = t->next;
1269 + t->next = to_remove->next;
1270 + to_remove->next = NULL;
1271 + freez(to_remove);
1272 + }
1273 +}
1274 +
1275 +static struct worker_thread *worker_thread_find(struct worker_utilization *wu, pid_t pid) {
1276 + struct worker_thread *wt;
1277 + for(wt = wu->threads; wt && wt->pid != pid ; wt = wt->next) ;
1278 + return wt;
1279 +}
1280 +
1281 +static struct worker_thread *worker_thread_create(struct worker_utilization *wu, pid_t pid) {
1282 + struct worker_thread *wt;
1283 +
1284 + wt = (struct worker_thread *)callocz(1, sizeof(struct worker_thread));
1285 + wt->pid = pid;
1286 +
1287 + // link it
1288 + wt->next = wu->threads;
1289 + wu->threads = wt;
1290
1291 + return wt;
1292 }
1293
1294 +static struct worker_thread *worker_thread_find_or_create(struct worker_utilization *wu, pid_t pid) {
1295 + struct worker_thread *wt;
1296 + wt = worker_thread_find(wu, pid);
1297 + if(!wt) wt = worker_thread_create(wu, pid);
1298 +
1299 + return wt;
1300 +}
1301 +
1302 +static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unused, const char *thread_tag __maybe_unused, size_t utilization_usec __maybe_unused, size_t duration_usec __maybe_unused, size_t jobs_started __maybe_unused, size_t is_running __maybe_unused, const char **job_types_names __maybe_unused, size_t *job_types_jobs_started __maybe_unused, usec_t *job_types_busy_time __maybe_unused) {
1303 + struct worker_utilization *wu = (struct worker_utilization *)ptr;
1304 +
1305 + // find the worker_thread in the list
1306 + struct worker_thread *wt = worker_thread_find_or_create(wu, pid);
1307 +
1308 + wt->enabled = 1;
1309 + wt->busy_time = utilization_usec;
1310 + wt->jobs_started = jobs_started;
1311 +
1312 + wt->utime_old = wt->utime;
1313 + wt->stime_old = wt->stime;
1314 + wt->collected_time_old = wt->collected_time;
1315 +
1316 + wu->workers_total_busy_time += utilization_usec;
1317 + wu->workers_total_duration += duration_usec;
1318 + wu->workers_total_jobs_started += jobs_started;
1319 + wu->workers_busy += is_running;
1320 + wu->workers_registered++;
1321 +
1322 + double util = (double)utilization_usec * 100.0 / (double)duration_usec;
1323 + if(util > wu->workers_max_busy_time)
1324 + wu->workers_max_busy_time = util;
1325 +
1326 + if(util < wu->workers_min_busy_time)
1327 + wu->workers_min_busy_time = util;
1328 +
1329 + // accumulate per job type statistics
1330 + size_t i;
1331 + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
1332 + wu->per_job_type[i].jobs_started += job_types_jobs_started[i];
1333 + wu->per_job_type[i].busy_time += job_types_busy_time[i];
1334 +
1335 + // new job type found
1336 + if(unlikely(!wu->per_job_type[i].name[0] && job_types_names[i]))
1337 + strncpyz(wu->per_job_type[i].name, job_types_names[i], WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH);
1338 + }
1339 +
1340 + // find its CPU utilization
1341 + if((!read_thread_cpu_time_from_proc_stat(pid, &wt->utime, &wt->stime))) {
1342 + wt->cpu_enabled = 1;
1343 + wt->collected_time = now_realtime_usec();
1344 + }
1345 + wu->workers_cpu_enabled += wt->cpu_enabled;
1346 +}
1347 +
1348 +static struct worker_utilization all_workers_utilization[] = {
1349 + { .name = "STATS", .family = "workers global statistics", .priority = 1000000 },
1350 + { .name = "HEALTH", .family = "workers health alarms", .priority = 1000000 },
1351 + { .name = "MLTRAIN", .family = "workers ML training", .priority = 1000000 },
1352 + { .name = "MLDETECT", .family = "workers ML detection", .priority = 1000000 },
1353 + { .name = "STREAMRCV", .family = "workers streaming receive", .priority = 1000000 },
1354 + { .name = "STREAMSND", .family = "workers streaming send", .priority = 1000000 },
1355 + { .name = "DBENGINE", .family = "workers dbengine instances", .priority = 1000000 },
1356 + { .name = "WEB", .family = "workers web server", .priority = 1000000 },
1357 + { .name = "ACLKQUERY", .family = "workers aclk query", .priority = 1000000 },
1358 + { .name = "ACLKSYNC", .family = "workers aclk host sync", .priority = 1000000 },
1359 + { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 },
1360 + { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 },
1361 + { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 },
1362 + { .name = "PROC", .family = "workers plugin proc", .priority = 1000000 },
1363 + { .name = "FREEBSD", .family = "workers plugin freebsd", .priority = 1000000 },
1364 + { .name = "MACOS", .family = "workers plugin macos", .priority = 1000000 },
1365 + { .name = "CGROUPS", .family = "workers plugin cgroups", .priority = 1000000 },
1366 + { .name = "CGROUPSDISC", .family = "workers plugin cgroups find", .priority = 1000000 },
1367 + { .name = "DISKSPACE", .family = "workers plugin diskspace", .priority = 1000000 },
1368 + { .name = "TC", .family = "workers plugin tc", .priority = 1000000 },
1369 + { .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 },
1370 + { .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 },
1371 +
1372 + // has to be terminated with a NULL
1373 + { .name = NULL, .family = NULL }
1374 +};
1375 +
1376 +static void worker_utilization_charts(void) {
1377 + static size_t iterations = 0;
1378 + iterations++;
1379 +
1380 + int i;
1381 + for(i = 0; all_workers_utilization[i].name ;i++) {
1382 + workers_utilization_reset_statistics(&all_workers_utilization[i]);
1383 + workers_foreach(all_workers_utilization[i].name, worker_utilization_charts_callback, &all_workers_utilization[i]);
1384 +
1385 + // skip the first iteration, so that we don't accumulate startup utilization to our charts
1386 + if(likely(iterations > 1))
1387 + workers_utilization_update_chart(&all_workers_utilization[i]);
1388 +
1389 + workers_threads_cleanup(&all_workers_utilization[i]);
1390 + }
1391 +}
1392 +
1393 +static void worker_utilization_finish(void) {
1394 + int i;
1395 + for(i = 0; all_workers_utilization[i].name ;i++) {
1396 + struct worker_utilization *wu = &all_workers_utilization[i];
1397 +
1398 + if(wu->name_lowercase) {
1399 + freez(wu->name_lowercase);
1400 + wu->name_lowercase = NULL;
1401 + }
1402 +
1403 + // mark all threads as not enabled
1404 + struct worker_thread *t;
1405 + for(t = wu->threads; t ; t = t->next) t->enabled = 0;
1406 +
1407 + // let the cleanup job free them
1408 + workers_threads_cleanup(wu);
1409 + }
1410 +}
1411 +
1412 +// ---------------------------------------------------------------------------------------------------------------------
1413 +
1414 static void global_statistics_cleanup(void *ptr)
1415 {
1416 + worker_unregister();
1417 +
1418 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
1419 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
1420
1421 info("cleaning up...");
1422
1423 + worker_utilization_finish();
1424 +
1425 static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
1426 }
1427
1428 void *global_statistics_main(void *ptr)
1429 {
1430 + worker_register("STATS");
1431 + worker_register_job_name(WORKER_JOB_GLOBAL, "global");
1432 + worker_register_job_name(WORKER_JOB_REGISTRY, "registry");
1433 + worker_register_job_name(WORKER_JOB_WORKERS, "workers");
1434 + worker_register_job_name(WORKER_JOB_DBENGINE, "dbengine");
1435 +
1436 netdata_thread_cleanup_push(global_statistics_cleanup, ptr);
1437
1438 int update_every =
@@ -876,10 +1444,23 @@ void *global_statistics_main(void *ptr)
1444 heartbeat_t hb;
1445 heartbeat_init(&hb);
1446 while (!netdata_exit) {
1447 + worker_is_idle();
1448 heartbeat_next(&hb, step);
1449
1450 + worker_is_busy(WORKER_JOB_WORKERS);
1451 + worker_utilization_charts();
1452 +
1453 + worker_is_busy(WORKER_JOB_GLOBAL);
1454 global_statistics_charts();
1455 +
1456 + worker_is_busy(WORKER_JOB_REGISTRY);
1457 registry_statistics();
1458 +
1459 + worker_is_busy(WORKER_JOB_DBENGINE);
1460 + dbengine_statistics_charts();
1461 +
1462 + worker_is_busy(WORKER_JOB_HEARTBEAT);
1463 + update_heartbeat_charts();
1464 }
1465
1466 netdata_thread_cleanup_pop(1);
database/engine/rrdengine.c
+32 -1
@@ -11,6 +11,10 @@ rrdeng_stats_t global_flushing_pressure_page_deletions = 0;
11
12 static unsigned pages_per_extent = MAX_PAGES_PER_EXTENT;
13
14 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < (RRDENG_MAX_OPCODE + 2)
15 +#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least (RRDENG_MAX_OPCODE + 2)
16 +#endif
17 +
18 void *dbengine_page_alloc() {
19 void *page = netdata_mmap(NULL, RRDENG_BLOCK_SIZE, MAP_PRIVATE, enable_ksm);
20 if(!page) fatal("Cannot allocate dbengine page cache page, with mmap()");
@@ -23,6 +27,8 @@ void dbengine_page_free(void *page) {
27
28 static void sanity_check(void)
29 {
30 + BUILD_BUG_ON(WORKER_UTILIZATION_MAX_JOB_TYPES < (RRDENG_MAX_OPCODE + 2));
31 +
32 /* Magic numbers must fit in the super-blocks */
33 BUILD_BUG_ON(strlen(RRDENG_DF_MAGIC) > RRDENG_MAGIC_SZ);
34 BUILD_BUG_ON(strlen(RRDENG_JF_MAGIC) > RRDENG_MAGIC_SZ);
@@ -1085,13 +1091,17 @@ void async_cb(uv_async_t *handle)
1091
1092 void timer_cb(uv_timer_t* handle)
1093 {
1094 + worker_is_busy(RRDENG_MAX_OPCODE + 1);
1095 +
1096 struct rrdengine_worker_config* wc = handle->data;
1097 struct rrdengine_instance *ctx = wc->ctx;
1098
1099 uv_stop(handle->loop);
1100 uv_update_time(handle->loop);
1093 - if (unlikely(!ctx->metalog_ctx->initialized))
1101 + if (unlikely(!ctx->metalog_ctx->initialized)) {
1102 + worker_is_idle();
1103 return; /* Wait for the metadata log to initialize */
1104 + }
1105 rrdeng_test_quota(wc);
1106 debug(D_RRDENGINE, "%s: timeout reached.", __func__);
1107 if (likely(!wc->now_deleting_files && !wc->now_invalidating_dirty_pages)) {
@@ -1133,12 +1143,26 @@ void timer_cb(uv_timer_t* handle)
1143 debug(D_RRDENGINE, "%s", get_rrdeng_statistics(wc->ctx, buf, sizeof(buf)));
1144 }
1145 #endif
1146 +
1147 + worker_is_idle();
1148 }
1149
1150 #define MAX_CMD_BATCH_SIZE (256)
1151
1152 void rrdeng_worker(void* arg)
1153 {
1154 + worker_register("DBENGINE");
1155 + worker_register_job_name(RRDENG_NOOP, "noop");
1156 + worker_register_job_name(RRDENG_READ_PAGE, "page read");
1157 + worker_register_job_name(RRDENG_READ_EXTENT, "extent read");
1158 + worker_register_job_name(RRDENG_COMMIT_PAGE, "commit");
1159 + worker_register_job_name(RRDENG_FLUSH_PAGES, "flush");
1160 + worker_register_job_name(RRDENG_SHUTDOWN, "shutdown");
1161 + worker_register_job_name(RRDENG_INVALIDATE_OLDEST_MEMORY_PAGE, "page lru");
1162 + worker_register_job_name(RRDENG_QUIESCE, "quiesce");
1163 + worker_register_job_name(RRDENG_MAX_OPCODE, "cleanup");
1164 + worker_register_job_name(RRDENG_MAX_OPCODE + 1, "timer");
1165 +
1166 struct rrdengine_worker_config* wc = arg;
1167 struct rrdengine_instance *ctx = wc->ctx;
1168 uv_loop_t* loop;
@@ -1188,7 +1212,9 @@ void rrdeng_worker(void* arg)
1212 shutdown = 0;
1213 int set_name = 0;
1214 while (likely(shutdown == 0 || rrdeng_threads_alive(wc))) {
1215 + worker_is_idle();
1216 uv_run(loop, UV_RUN_DEFAULT);
1217 + worker_is_busy(RRDENG_MAX_OPCODE);
1218 rrdeng_cleanup_finished_threads(wc);
1219
1220 /* wait for commands */
@@ -1205,6 +1231,9 @@ void rrdeng_worker(void* arg)
1231 opcode = cmd.opcode;
1232 ++cmd_batch_size;
1233
1234 + if(likely(opcode != RRDENG_NOOP))
1235 + worker_is_busy(opcode);
1236 +
1237 switch (opcode) {
1238 case RRDENG_NOOP:
1239 /* the command queue was empty, do nothing */
@@ -1281,6 +1310,7 @@ void rrdeng_worker(void* arg)
1310 fatal_assert(0 == uv_loop_close(loop));
1311 freez(loop);
1312
1313 + worker_unregister();
1314 return;
1315
1316 error_after_timer_init:
@@ -1293,6 +1323,7 @@ error_after_loop_init:
1323 wc->error = UV_EAGAIN;
1324 /* wake up initialization thread */
1325 completion_mark_complete(&ctx->rrdengine_completion);
1326 + worker_unregister();
1327 }
1328
1329 /* C entry point for development purposes
database/sqlite/sqlite_aclk.c
+37
@@ -10,6 +10,11 @@
10 #include "../../aclk/aclk.h"
11 #endif
12
13 +void sanity_check(void) {
14 + // make sure the compiler will stop on misconfigurations
15 + BUILD_BUG_ON(WORKER_UTILIZATION_MAX_JOB_TYPES < ACLK_MAX_ENUMERATIONS_DEFINED);
16 +}
17 +
18 const char *aclk_sync_config[] = {
19 "CREATE TABLE IF NOT EXISTS dimension_delete (dimension_id blob, dimension_name text, chart_type_id text, "
20 "dim_id blob, chart_id blob, host_id blob, date_created);",
@@ -352,6 +357,29 @@ static void timer_cb(uv_timer_t* handle)
357
358 void aclk_database_worker(void *arg)
359 {
360 + worker_register("ACLKSYNC");
361 + worker_register_job_name(ACLK_DATABASE_NOOP, "noop");
362 +#ifdef ENABLE_NEW_CLOUD_PROTOCOL
363 + worker_register_job_name(ACLK_DATABASE_ADD_CHART, "chart add");
364 + worker_register_job_name(ACLK_DATABASE_ADD_DIMENSION, "dimension add");
365 + worker_register_job_name(ACLK_DATABASE_PUSH_CHART, "chart push");
366 + worker_register_job_name(ACLK_DATABASE_PUSH_CHART_CONFIG, "chart conf push");
367 + worker_register_job_name(ACLK_DATABASE_RESET_CHART, "chart reset");
368 + worker_register_job_name(ACLK_DATABASE_CHART_ACK, "chart ack");
369 + worker_register_job_name(ACLK_DATABASE_UPD_RETENTION, "retention check");
370 + worker_register_job_name(ACLK_DATABASE_DIM_DELETION, "dimension delete");
371 + worker_register_job_name(ACLK_DATABASE_ORPHAN_HOST, "node orphan");
372 +#endif
373 + worker_register_job_name(ACLK_DATABASE_ALARM_HEALTH_LOG, "alert log");
374 + worker_register_job_name(ACLK_DATABASE_CLEANUP, "cleanup");
375 + worker_register_job_name(ACLK_DATABASE_DELETE_HOST, "node delete");
376 + worker_register_job_name(ACLK_DATABASE_NODE_INFO, "node info");
377 + worker_register_job_name(ACLK_DATABASE_PUSH_ALERT, "alert push");
378 + worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_CONFIG, "alert conf push");
379 + worker_register_job_name(ACLK_DATABASE_PUSH_ALERT_SNAPSHOT, "alert snapshot");
380 + worker_register_job_name(ACLK_DATABASE_QUEUE_REMOVED_ALERTS, "alerts check");
381 + worker_register_job_name(ACLK_DATABASE_TIMER, "timer");
382 +
383 struct aclk_database_worker_config *wc = arg;
384 uv_loop_t *loop;
385 int ret;
@@ -413,6 +441,7 @@ void aclk_database_worker(void *arg)
441
442 debug(D_ACLK_SYNC,"Node %s reports pending message count = %u", wc->node_id, wc->chart_payload_count);
443 while (likely(!netdata_exit)) {
444 + worker_is_idle();
445 uv_run(loop, UV_RUN_DEFAULT);
446
447 /* wait for commands */
@@ -427,6 +456,10 @@ void aclk_database_worker(void *arg)
456
457 opcode = cmd.opcode;
458 ++cmd_batch_size;
459 +
460 + if(likely(opcode != ACLK_DATABASE_NOOP))
461 + worker_is_busy(opcode);
462 +
463 switch (opcode) {
464 case ACLK_DATABASE_NOOP:
465 /* the command queue was empty, do nothing */
@@ -439,6 +472,7 @@ void aclk_database_worker(void *arg)
472 if (wc->host == localhost)
473 sql_check_aclk_table_list(wc);
474 break;
475 +
476 case ACLK_DATABASE_DELETE_HOST:
477 debug(D_ACLK_SYNC,"Cleaning ACLK tables for %s", (char *) cmd.data);
478 sql_delete_aclk_table_list(wc, cmd);
@@ -577,6 +611,8 @@ void aclk_database_worker(void *arg)
611 wc->host->dbsync_worker = NULL;
612 freez(wc);
613 rrd_unlock();
614 +
615 + worker_unregister();
616 return;
617
618 error_after_timer_init:
@@ -585,6 +621,7 @@ error_after_async_init:
621 fatal_assert(0 == uv_loop_close(loop));
622 error_after_loop_init:
623 freez(loop);
624 + worker_unregister();
625 }
626
627 // -------------------------------------------------------------
database/sqlite/sqlite_aclk.h
+5 -1
@@ -133,7 +133,11 @@ enum aclk_database_opcode {
133 ACLK_DATABASE_PUSH_ALERT_CONFIG,
134 ACLK_DATABASE_PUSH_ALERT_SNAPSHOT,
135 ACLK_DATABASE_QUEUE_REMOVED_ALERTS,
136 - ACLK_DATABASE_TIMER
136 + ACLK_DATABASE_TIMER,
137 +
138 + // leave this last
139 + // we need it to check for worker utilization
140 + ACLK_MAX_ENUMERATIONS_DEFINED
141 };
142
143 struct aclk_chart_payload_t {
health/health.c
+41
@@ -573,6 +573,8 @@ static inline int check_if_resumed_from_suspension(void) {
573 }
574
575 static void health_main_cleanup(void *ptr) {
576 + worker_unregister();
577 +
578 struct netdata_static_thread *static_thread = (struct netdata_static_thread *)ptr;
579 static_thread->enabled = NETDATA_MAIN_THREAD_EXITING;
580
@@ -695,7 +697,31 @@ static void init_pending_foreach_alarms(RRDHOST *host) {
697 *
698 * @return It always returns NULL
699 */
700 +
701 +#define WORKER_HEALTH_JOB_RRD_LOCK 0
702 +#define WORKER_HEALTH_JOB_HOST_LOCK 1
703 +#define WORKER_HEALTH_JOB_DB_QUERY 2
704 +#define WORKER_HEALTH_JOB_CALC_EVAL 3
705 +#define WORKER_HEALTH_JOB_WARNING_EVAL 4
706 +#define WORKER_HEALTH_JOB_CRITICAL_EVAL 5
707 +#define WORKER_HEALTH_JOB_ALARM_LOG_ENTRY 6
708 +#define WORKER_HEALTH_JOB_ALARM_LOG_PROCESS 7
709 +
710 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 8
711 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 8
712 +#endif
713 +
714 void *health_main(void *ptr) {
715 + worker_register("HEALTH");
716 + worker_register_job_name(WORKER_HEALTH_JOB_RRD_LOCK, "rrd lock");
717 + worker_register_job_name(WORKER_HEALTH_JOB_HOST_LOCK, "host lock");
718 + worker_register_job_name(WORKER_HEALTH_JOB_DB_QUERY, "db lookup");
719 + worker_register_job_name(WORKER_HEALTH_JOB_CALC_EVAL, "calc eval");
720 + worker_register_job_name(WORKER_HEALTH_JOB_WARNING_EVAL, "warning eval");
721 + worker_register_job_name(WORKER_HEALTH_JOB_CRITICAL_EVAL, "critical eval");
722 + worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY, "alarm log entry");
723 + worker_register_job_name(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS, "alarm log process");
724 +
725 netdata_thread_cleanup_push(health_main_cleanup, ptr);
726
727 int min_run_every = (int)config_get_number(CONFIG_SECTION_HEALTH, "run at least every seconds", 10);
@@ -743,6 +769,7 @@ void *health_main(void *ptr) {
769 marked_aclk_reload_loop = loop;
770 #endif
771
772 + worker_is_busy(WORKER_HEALTH_JOB_RRD_LOCK);
773 rrd_rdlock();
774
775 RRDHOST *host;
@@ -772,6 +799,7 @@ void *health_main(void *ptr) {
799
800 init_pending_foreach_alarms(host);
801
802 + worker_is_busy(WORKER_HEALTH_JOB_HOST_LOCK);
803 rrdhost_rdlock(host);
804
805 // the first loop is to lookup values from the db
@@ -786,6 +814,7 @@ void *health_main(void *ptr) {
814 rrdset_flag_check(rc->rrdset, RRDSET_FLAG_OBSOLETE) &&
815 now > (rc->rrdset->last_collected_time.tv_sec + 60))) {
816 if (!rrdcalc_isrepeating(rc)) {
817 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
818 time_t now = now_realtime_sec();
819 ALARM_ENTRY *ae = health_create_alarm_entry(
820 host, rc->id, rc->next_event_id++, rc->config_hash_id, now, rc->name, rc->rrdset->id,
@@ -820,6 +849,8 @@ void *health_main(void *ptr) {
849 // if there is database lookup, do it
850
851 if (unlikely(RRDCALC_HAS_DB_LOOKUP(rc))) {
852 + worker_is_busy(WORKER_HEALTH_JOB_DB_QUERY);
853 +
854 /* time_t old_db_timestamp = rc->db_before; */
855 int value_is_null = 0;
856
@@ -876,6 +907,8 @@ void *health_main(void *ptr) {
907 // if there is calculation expression, run it
908
909 if (unlikely(rc->calculation)) {
910 + worker_is_busy(WORKER_HEALTH_JOB_CALC_EVAL);
911 +
912 if (unlikely(!expression_evaluate(rc->calculation))) {
913 // calculation failed
914 rc->value = NAN;
@@ -924,6 +957,8 @@ void *health_main(void *ptr) {
957 // check the warning expression
958
959 if (likely(rc->warning)) {
960 + worker_is_busy(WORKER_HEALTH_JOB_WARNING_EVAL);
961 +
962 if (unlikely(!expression_evaluate(rc->warning))) {
963 // calculation failed
964 rc->rrdcalc_flags |= RRDCALC_FLAG_WARN_ERROR;
@@ -948,6 +983,8 @@ void *health_main(void *ptr) {
983 // check the critical expression
984
985 if (likely(rc->critical)) {
986 + worker_is_busy(WORKER_HEALTH_JOB_CRITICAL_EVAL);
987 +
988 if (unlikely(!expression_evaluate(rc->critical))) {
989 // calculation failed
990 rc->rrdcalc_flags |= RRDCALC_FLAG_CRIT_ERROR;
@@ -1005,6 +1042,7 @@ void *health_main(void *ptr) {
1042 // check if the new status and the old differ
1043
1044 if (status != rc->status) {
1045 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1046 int delay = 0;
1047
1048 // apply trigger hysteresis
@@ -1086,6 +1124,7 @@ void *health_main(void *ptr) {
1124 }
1125
1126 if(unlikely(repeat_every > 0 && (rc->last_repeat + repeat_every) <= now)) {
1127 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_ENTRY);
1128 rc->last_repeat = now;
1129 if (likely(rc->times_repeat < UINT32_MAX)) rc->times_repeat++;
1130 ALARM_ENTRY *ae = health_create_alarm_entry(
@@ -1118,6 +1157,7 @@ void *health_main(void *ptr) {
1157
1158 // execute notifications
1159 // and cleanup
1160 + worker_is_busy(WORKER_HEALTH_JOB_ALARM_LOG_PROCESS);
1161 health_alarm_log_process(host);
1162
1163 if (unlikely(netdata_exit)) {
@@ -1156,6 +1196,7 @@ void *health_main(void *ptr) {
1196
1197 now = now_realtime_sec();
1198 if(now < next_run) {
1199 + worker_is_idle();
1200 debug(D_HEALTH, "Health monitoring iteration no %u done. Next iteration in %d secs", loop, (int) (next_run - now));
1201 sleep_usec(USEC_PER_SEC * (usec_t) (next_run - now));
1202 now = now_realtime_sec();
libnetdata/Makefile.am
+1
@@ -26,6 +26,7 @@ SUBDIRS = \
26 storage_number \
27 threads \
28 url \
29 + worker_utilization \
30 tests \
31 $(NULL)
32
libnetdata/clocks/clocks.c
+174 -71
@@ -7,6 +7,9 @@
7 static clockid_t clock_boottime_to_use = CLOCK_MONOTONIC;
8 static clockid_t clock_monotonic_to_use = CLOCK_MONOTONIC;
9
10 +usec_t clock_monotonic_resolution = 1000;
11 +usec_t clock_realtime_resolution = 1000;
12 +
13 #ifndef HAVE_CLOCK_GETTIME
14 inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
15 struct timeval tv;
@@ -20,15 +23,19 @@ inline int clock_gettime(clockid_t clk_id, struct timespec *ts) {
23 }
24 #endif
25
23 -// When running a binary with CLOCK_MONOTONIC_COARSE defined on a system with a linux kernel older than Linux 2.6.32 the
24 -// clock_gettime(2) system call fails with EINVAL. In that case it must fall-back to CLOCK_MONOTONIC.
26 +// Similar to CLOCK_MONOTONIC, but provides access to a raw hardware-based time that is not subject to NTP adjustments
27 +// or the incremental adjustments performed by adjtime(3). This clock does not count time that the system is suspended
28
26 -static void test_clock_monotonic_coarse(void) {
29 +static void test_clock_monotonic_raw(void) {
30 +#ifdef CLOCK_MONOTONIC_RAW
31 struct timespec ts;
28 - if(clock_gettime(CLOCK_MONOTONIC_COARSE, &ts) == -1 && errno == EINVAL)
32 + if(clock_gettime(CLOCK_MONOTONIC_RAW, &ts) == -1 && errno == EINVAL)
33 clock_monotonic_to_use = CLOCK_MONOTONIC;
34 else
31 - clock_monotonic_to_use = CLOCK_MONOTONIC_COARSE;
35 + clock_monotonic_to_use = CLOCK_MONOTONIC_RAW;
36 +#else
37 + clock_monotonic_to_use = CLOCK_MONOTONIC;
38 +#endif
39 }
40
41 // When running a binary with CLOCK_BOOTTIME defined on a system with a linux kernel older than Linux 2.6.39 the
@@ -42,14 +49,31 @@ static void test_clock_boottime(void) {
49 clock_boottime_to_use = CLOCK_BOOTTIME;
50 }
51
52 +static usec_t get_clock_resolution(clockid_t clock) {
53 + struct timespec ts;
54 + clock_getres(clock, &ts);
55 + return ts.tv_sec * USEC_PER_SEC + ts.tv_nsec * NSEC_PER_USEC;
56 +}
57 +
58 // perform any initializations required for clocks
59
60 void clocks_init(void) {
48 - // monotonic coarse has to be tested before boottime
49 - test_clock_monotonic_coarse();
61 + // monotonic raw has to be tested before boottime
62 + test_clock_monotonic_raw();
63
64 // boottime has to be tested after monotonic coarse
65 test_clock_boottime();
66 +
67 + clock_monotonic_resolution = get_clock_resolution(clock_monotonic_to_use);
68 + clock_realtime_resolution = get_clock_resolution(CLOCK_REALTIME);
69 +
70 + // if for any reason these are zero, netdata will crash
71 + // since we use them as modulo to calculations
72 + if(!clock_realtime_resolution)
73 + clock_realtime_resolution = 1000;
74 +
75 + if(!clock_monotonic_resolution)
76 + clock_monotonic_resolution = 1000;
77 }
78
79 inline time_t now_sec(clockid_t clk_id) {
@@ -155,8 +179,110 @@ inline usec_t dt_usec(struct timeval *now, struct timeval *old) {
179 return (ts1 > ts2) ? (ts1 - ts2) : (ts2 - ts1);
180 }
181
182 +void sleep_to_absolute_time(usec_t usec) {
183 + static int einval_printed = 0, enotsup_printed = 0, eunknown_printed = 0;
184 + clockid_t clock = CLOCK_REALTIME;
185 +
186 + struct timespec req = {
187 + .tv_sec = (time_t)(usec / USEC_PER_SEC),
188 + .tv_nsec = (suseconds_t)((usec % USEC_PER_SEC) * NSEC_PER_USEC)
189 + };
190 +
191 + int ret = 0;
192 + while( (ret = clock_nanosleep(clock, TIMER_ABSTIME, &req, NULL)) != 0 ) {
193 + if(ret == EINTR) continue;
194 + else {
195 + if (ret == EINVAL) {
196 + if (!einval_printed) {
197 + einval_printed++;
198 + error(
199 + "Invalid time given to clock_nanosleep(): clockid = %d, tv_sec = %ld, tv_nsec = %ld",
200 + clock,
201 + req.tv_sec,
202 + req.tv_nsec);
203 + }
204 + } else if (ret == ENOTSUP) {
205 + if (!enotsup_printed) {
206 + enotsup_printed++;
207 + error(
208 + "Invalid clock id given to clock_nanosleep(): clockid = %d, tv_sec = %ld, tv_nsec = %ld",
209 + clock,
210 + req.tv_sec,
211 + req.tv_nsec);
212 + }
213 + } else {
214 + if (!eunknown_printed) {
215 + eunknown_printed++;
216 + error(
217 + "Unknown return value %d from clock_nanosleep(): clockid = %d, tv_sec = %ld, tv_nsec = %ld",
218 + ret,
219 + clock,
220 + req.tv_sec,
221 + req.tv_nsec);
222 + }
223 + }
224 + sleep_usec(usec);
225 + }
226 + }
227 +};
228 +
229 +#define HEARTBEAT_ALIGNMENT_STATISTICS_SIZE 10
230 +netdata_mutex_t heartbeat_alignment_mutex = NETDATA_MUTEX_INITIALIZER;
231 +static size_t heartbeat_alignment_id = 0;
232 +
233 +struct heartbeat_thread_statistics {
234 + size_t sequence;
235 + usec_t dt;
236 +};
237 +static struct heartbeat_thread_statistics heartbeat_alignment_values[HEARTBEAT_ALIGNMENT_STATISTICS_SIZE] = { 0 };
238 +
239 +void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr, size_t *count_ptr) {
240 + struct heartbeat_thread_statistics current[HEARTBEAT_ALIGNMENT_STATISTICS_SIZE];
241 + static struct heartbeat_thread_statistics old[HEARTBEAT_ALIGNMENT_STATISTICS_SIZE] = { 0 };
242 +
243 + memcpy(current, heartbeat_alignment_values, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE);
244 +
245 + usec_t min = 0, max = 0, total = 0, average = 0;
246 + size_t i, count = 0;
247 + for(i = 0; i < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE ;i++) {
248 + if(current[i].sequence == old[i].sequence) continue;
249 + usec_t value = current[i].dt - old[i].dt;
250 +
251 + if(!count) {
252 + min = max = total = value;
253 + count = 1;
254 + }
255 + else {
256 + total += value;
257 + if(value < min) min = value;
258 + if(value > max) max = value;
259 + count++;
260 + }
261 + }
262 + average = total / count;
263 +
264 + if(min_ptr) *min_ptr = min;
265 + if(max_ptr) *max_ptr = max;
266 + if(average_ptr) *average_ptr = average;
267 + if(count_ptr) *count_ptr = count;
268 +
269 + memcpy(old, current, sizeof(struct heartbeat_thread_statistics) * HEARTBEAT_ALIGNMENT_STATISTICS_SIZE);
270 +}
271 +
272 inline void heartbeat_init(heartbeat_t *hb) {
159 - hb->monotonic = hb->realtime = 0ULL;
273 + hb->realtime = 0ULL;
274 + hb->randomness = 250 * USEC_PER_MS + ((now_realtime_usec() * clock_realtime_resolution) % (250 * USEC_PER_MS));
275 + hb->randomness -= (hb->randomness % clock_realtime_resolution);
276 +
277 + netdata_mutex_lock(&heartbeat_alignment_mutex);
278 + hb->statistics_id = heartbeat_alignment_id;
279 + heartbeat_alignment_id++;
280 + netdata_mutex_unlock(&heartbeat_alignment_mutex);
281 +
282 + if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) {
283 + heartbeat_alignment_values[hb->statistics_id].dt = 0;
284 + heartbeat_alignment_values[hb->statistics_id].sequence = 0;
285 + }
286 }
287
288 // waits for the next heartbeat
@@ -164,96 +290,73 @@ inline void heartbeat_init(heartbeat_t *hb) {
290 // it returns the dt using the realtime clock
291
292 usec_t heartbeat_next(heartbeat_t *hb, usec_t tick) {
167 - heartbeat_t now;
168 - now.monotonic = now_monotonic_usec();
169 - now.realtime = now_realtime_usec();
170 -
171 - usec_t next_monotonic = now.monotonic - (now.monotonic % tick) + tick;
172 -
173 - while(now.monotonic < next_monotonic) {
174 - sleep_usec(next_monotonic - now.monotonic);
175 - now.monotonic = now_monotonic_usec();
176 - now.realtime = now_realtime_usec();
293 + if(unlikely(hb->randomness > tick / 2)) {
294 + // TODO: The heartbeat tick should be specified at the heartbeat_init() function
295 + usec_t tmp = (now_realtime_usec() * clock_realtime_resolution) % (tick / 2);
296 + info("heartbeat randomness of %llu is too big for a tick of %llu - setting it to %llu", hb->randomness, tick, tmp);
297 + hb->randomness = tmp;
298 }
299
179 - if(likely(hb->realtime != 0ULL)) {
180 - usec_t dt_monotonic = now.monotonic - hb->monotonic;
181 - usec_t dt_realtime = now.realtime - hb->realtime;
300 + usec_t dt;
301 + usec_t now = now_realtime_usec();
302 + usec_t next = now - (now % tick) + tick + hb->randomness;
303
183 - hb->monotonic = now.monotonic;
184 - hb->realtime = now.realtime;
304 + // align the next time we want to the clock resolution
305 + if(next % clock_realtime_resolution)
306 + next = next - (next % clock_realtime_resolution) + clock_realtime_resolution;
307
186 - if(unlikely(dt_monotonic >= tick + tick / 2)) {
187 - errno = 0;
188 - error("heartbeat missed %llu monotonic microseconds", dt_monotonic - tick);
189 - }
308 + // sleep_usec() has a loop to guarantee we will sleep for at least the requested time.
309 + // According the specs, when we sleep for a relative time, clock adjustments should not affect the duration
310 + // we sleep.
311 + sleep_usec(next - now);
312 + now = now_realtime_usec();
313 + dt = now - hb->realtime;
314
191 - return dt_realtime;
315 + if(hb->statistics_id < HEARTBEAT_ALIGNMENT_STATISTICS_SIZE) {
316 + heartbeat_alignment_values[hb->statistics_id].dt += now - next;
317 + heartbeat_alignment_values[hb->statistics_id].sequence++;
318 }
193 - else {
194 - hb->monotonic = now.monotonic;
195 - hb->realtime = now.realtime;
196 - return 0ULL;
319 +
320 + if(unlikely(now < next)) {
321 + errno = 0;
322 + error("heartbeat clock: woke up %llu microseconds earlier than expected (can be due to the CLOCK_REALTIME set to the past).", next - now);
323 + }
324 + else if(unlikely(now - next > tick / 2)) {
325 + errno = 0;
326 + error("heartbeat clock: woke up %llu microseconds later than expected (can be due to system load or the CLOCK_REALTIME set to the future).", now - next);
327 }
198 -}
328
200 -// returned the elapsed time, since the last heartbeat
201 -// using the monotonic clock
329 + if(unlikely(!hb->realtime)) {
330 + // the first time return zero
331 + dt = 0;
332 + }
333
203 -inline usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb) {
204 - if(!hb || !hb->monotonic) return 0ULL;
205 - return now_monotonic_usec() - hb->monotonic;
334 + hb->realtime = now;
335 + return dt;
336 }
337
208 -int sleep_usec(usec_t usec) {
209 -
210 -#ifndef NETDATA_WITH_USLEEP
338 +void sleep_usec(usec_t usec) {
339 // we expect microseconds (1.000.000 per second)
340 // but timespec is nanoseconds (1.000.000.000 per second)
341 struct timespec rem, req = {
214 - .tv_sec = (time_t) (usec / 1000000),
215 - .tv_nsec = (suseconds_t) ((usec % 1000000) * 1000)
342 + .tv_sec = (time_t) (usec / USEC_PER_SEC),
343 + .tv_nsec = (suseconds_t) ((usec % USEC_PER_SEC) * NSEC_PER_USEC)
344 };
345
218 - while (nanosleep(&req, &rem) == -1) {
346 + while ((errno = clock_nanosleep(CLOCK_REALTIME, 0, &req, &rem)) != 0) {
347 if (likely(errno == EINTR)) {
220 - debug(D_SYSTEM, "nanosleep() interrupted (while sleeping for %llu microseconds).", usec);
348 req.tv_sec = rem.tv_sec;
349 req.tv_nsec = rem.tv_nsec;
350 } else {
224 - error("Cannot nanosleep() for %llu microseconds.", usec);
351 + error("Cannot clock_nanosleep(CLOCK_REALTIME) for %llu microseconds.", usec);
352 break;
353 }
354 }
228 -
229 - return 0;
230 -#else
231 - int ret = usleep(usec);
232 - if(unlikely(ret == -1 && errno == EINVAL)) {
233 - // on certain systems, usec has to be up to 999999
234 - if(usec > 999999) {
235 - int counter = usec / 999999;
236 - while(counter--)
237 - usleep(999999);
238 -
239 - usleep(usec % 999999);
240 - }
241 - else {
242 - error("Cannot usleep() for %llu microseconds.", usec);
243 - return ret;
244 - }
245 - }
246 -
247 - if(ret != 0)
248 - error("usleep() failed for %llu microseconds.", usec);
249 -
250 - return ret;
251 -#endif
355 }
356
357 static inline collected_number uptime_from_boottime(void) {
358 #ifdef CLOCK_BOOTTIME_IS_AVAILABLE
256 - return now_boottime_usec() / 1000;
359 + return (collected_number)(now_boottime_usec() / USEC_PER_MS);
360 #else
361 error("uptime cannot be read from CLOCK_BOOTTIME on this system.");
362 return 0;
libnetdata/clocks/clocks.h
+11 -14
@@ -22,8 +22,9 @@ typedef unsigned long long usec_t;
22 typedef long long susec_t;
23
24 typedef struct heartbeat {
25 - usec_t monotonic;
25 usec_t realtime;
26 + usec_t randomness;
27 + size_t statistics_id;
28 } heartbeat_t;
29
30 /* Linux value is as good as any other */
@@ -36,20 +37,14 @@ typedef struct heartbeat {
37 #define CLOCK_MONOTONIC CLOCK_REALTIME
38 #endif
39
39 -/* Prefer CLOCK_MONOTONIC_COARSE where available to reduce overhead. It has the same semantics as CLOCK_MONOTONIC */
40 -#ifndef CLOCK_MONOTONIC_COARSE
41 -/* fallback to CLOCK_MONOTONIC if not available */
42 -#define CLOCK_MONOTONIC_COARSE CLOCK_MONOTONIC
43 -#endif
44 -
40 #ifndef CLOCK_BOOTTIME
41
42 #ifdef CLOCK_UPTIME
43 /* CLOCK_BOOTTIME falls back to CLOCK_UPTIME on FreeBSD */
44 #define CLOCK_BOOTTIME CLOCK_UPTIME
45 #else // CLOCK_UPTIME
51 -/* CLOCK_BOOTTIME falls back to CLOCK_MONOTONIC */
52 -#define CLOCK_BOOTTIME CLOCK_MONOTONIC_COARSE
46 +/* CLOCK_BOOTTIME falls back to CLOCK_REALTIME */
47 +#define CLOCK_BOOTTIME CLOCK_REALTIME
48 #endif // CLOCK_UPTIME
49
50 #else // CLOCK_BOOTTIME
@@ -115,8 +110,6 @@ extern int clock_gettime(clockid_t clk_id, struct timespec *ts);
110 * All now_*_sec() functions return the time in seconds from the appropriate clock, or 0 on error.
111 * All now_*_usec() functions return the time in microseconds from the appropriate clock, or 0 on error.
112 *
118 - * Most functions will attempt to use CLOCK_MONOTONIC_COARSE if available to reduce contention overhead and improve
119 - * performance scaling. If high precision is required please use one of the available now_*_high_precision_* functions.
113 */
114 extern int now_realtime_timeval(struct timeval *tv);
115 extern time_t now_realtime_sec(void);
@@ -146,10 +139,9 @@ extern void heartbeat_init(heartbeat_t *hb);
139 */
140 extern usec_t heartbeat_next(heartbeat_t *hb, usec_t tick);
141
149 -/* Returns elapsed time in microseconds since last heartbeat */
150 -extern usec_t heartbeat_monotonic_dt_to_now_usec(heartbeat_t *hb);
142 +extern void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr, size_t *count_ptr);
143
152 -extern int sleep_usec(usec_t usec);
144 +extern void sleep_usec(usec_t usec);
145
146 extern void clocks_init(void);
147
@@ -160,4 +152,9 @@ extern int now_timeval(clockid_t clk_id, struct timeval *tv);
152
153 extern collected_number uptime_msec(char *filename);
154
155 +extern usec_t clock_monotonic_resolution;
156 +extern usec_t clock_realtime_resolution;
157 +
158 +extern void sleep_to_absolute_time(usec_t usec);
159 +
160 #endif /* NETDATA_CLOCKS_H */
libnetdata/libnetdata.h
+1
@@ -346,6 +346,7 @@ extern char *netdata_configured_host_prefix;
346 #include "health/health.h"
347 #include "string/utf8.h"
348 #include "onewayalloc/onewayalloc.h"
349 +#include "worker_utilization/worker_utilization.h"
350
351 // BEWARE: Outside of the C code this also exists in alarm-notify.sh
352 #define DEFAULT_CLOUD_BASE_URL "https://app.netdata.cloud"
libnetdata/worker_utilization/Makefile.am new
+8
@@ -0,0 +1,8 @@
1 +# SPDX-License-Identifier: GPL-3.0-or-later
2 +
3 +AUTOMAKE_OPTIONS = subdir-objects
4 +MAINTAINERCLEANFILES = $(srcdir)/Makefile.in
5 +
6 +dist_noinst_DATA = \
7 + README.md \
8 + $(NULL)
libnetdata/worker_utilization/README.md new
+58
@@ -0,0 +1,58 @@
1 +<!--
2 +title: "Worker Utilization"
3 +custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/onewayallocator/README.md
4 +-->
5 +
6 +# Worker Utilization
7 +
8 +This library is to be used when there are 1 or more worker threads accepting requests of some kind and servicing them.
9 +The goal is to provide a very simple way to monitor worker threads utilization, as a percentage of the time they are busy and the amount of requests served.
10 +
11 +## How to use
12 +
13 +When a working thread starts, call:
14 +
15 +```c
16 +void worker_register(const char *name);
17 +```
18 +
19 +This will create the necessary structures for the library to work.
20 +No need to keep a pointer to them. They are allocated as `__thread` variables.
21 +
22 +When the thread stops, call:
23 +
24 +```c
25 +void worker_unregister(void)
26 +```
27 +
28 +Again, no parameters, or return values.
29 +
30 +When you are about to do some work in the working thread, call:
31 +
32 +```c
33 +void worker_is_busy(void)
34 +```
35 +
36 +When you finish doing the job, call:
37 +
38 +```c
39 +void worker_is_idle(void)
40 +```
41 +
42 +Calls to `worker_is_busy()` can be made one after another (without calling
43 +`worker_is_idle()` between them) to switch jobs without losing any time between
44 +them and eliminating one of the 2 clock calls involved.
45 +
46 +## Implementation details
47 +
48 +Totally lockless, extremely fast, it should not introduce any kind of problems to the workers.
49 +Every time `worker_is_busy()` or `worker_is_idle()` are called, a call to `now_realtime_usec()`
50 +is done and a couple of variables are updated. That's it!
51 +
52 +The worker does not need to update the variables regularly. Based on the last status of the worker,
53 +the statistics collector of netdata will calculate if the thread is busy or idle all the time or
54 +part of the time. Works well for both thousands of jobs per second and unlimited working time
55 +(being totally busy with a single request for ages).
56 +
57 +The statistics collector is called by the global statistics thread of netdata. So, even if the workers
58 +are extremely busy with their jobs, netdata will be able to know how busy they are.
libnetdata/worker_utilization/worker_utilization.c new
+201
@@ -0,0 +1,201 @@
1 +#include "worker_utilization.h"
2 +
3 +#define WORKER_IDLE 'I'
4 +#define WORKER_BUSY 'B'
5 +
6 +struct worker_job_type {
7 + char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
8 + size_t worker_jobs_started;
9 + usec_t worker_busy_time;
10 +
11 + size_t statistics_jobs_started;
12 + usec_t statistics_busy_time;
13 +};
14 +
15 +struct worker {
16 + pid_t pid;
17 + const char *tag;
18 + const char *workname;
19 + uint32_t workname_hash;
20 +
21 + // only one variable is set by our statistics callers
22 + usec_t statistics_last_checkpoint;
23 + size_t statistics_last_jobs_started;
24 + usec_t statistics_last_busy_time;
25 +
26 + // the worker controlled variables
27 + size_t job_id;
28 + volatile size_t jobs_started;
29 + volatile usec_t busy_time;
30 + volatile usec_t last_action_timestamp;
31 + volatile char last_action;
32 +
33 + struct worker_job_type per_job_type[WORKER_UTILIZATION_MAX_JOB_TYPES];
34 +
35 + struct worker *next;
36 +};
37 +
38 +static netdata_mutex_t base_lock = NETDATA_MUTEX_INITIALIZER;
39 +static struct worker *base = NULL;
40 +static __thread struct worker *worker = NULL;
41 +
42 +void worker_register(const char *workname) {
43 + if(unlikely(worker)) return;
44 +
45 + worker = callocz(1, sizeof(struct worker));
46 + worker->pid = gettid();
47 + worker->tag = strdupz(netdata_thread_tag());
48 + worker->workname = strdupz(workname);
49 + worker->workname_hash = simple_hash(worker->workname);
50 +
51 + usec_t now = now_realtime_usec();
52 + worker->statistics_last_checkpoint = now;
53 + worker->last_action_timestamp = now;
54 + worker->last_action = WORKER_IDLE;
55 +
56 + netdata_mutex_lock(&base_lock);
57 + worker->next = base;
58 + base = worker;
59 + netdata_mutex_unlock(&base_lock);
60 +}
61 +
62 +void worker_register_job_name(size_t job_id, const char *name) {
63 + if(unlikely(!worker)) return;
64 +
65 + if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES)) {
66 + error("WORKER_UTILIZATION: job_id %zu is too big. Max is %zu", job_id, (size_t)(WORKER_UTILIZATION_MAX_JOB_TYPES - 1));
67 + return;
68 + }
69 +
70 + strncpy(worker->per_job_type[job_id].name, name, WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH);
71 +}
72 +
73 +void worker_unregister(void) {
74 + if(unlikely(!worker)) return;
75 +
76 + netdata_mutex_lock(&base_lock);
77 + if(base == worker)
78 + base = worker->next;
79 + else {
80 + struct worker *p;
81 + for(p = base; p && p->next && p->next != worker ;p = p->next);
82 + if(p && p->next == worker)
83 + p->next = worker->next;
84 + }
85 + netdata_mutex_unlock(&base_lock);
86 +
87 + freez((void *)worker->tag);
88 + freez((void *)worker->workname);
89 + freez(worker);
90 +
91 + worker = NULL;
92 +}
93 +
94 +static inline void worker_is_idle_with_time(usec_t now) {
95 + usec_t delta = now - worker->last_action_timestamp;
96 + worker->busy_time += delta;
97 + worker->per_job_type[worker->job_id].worker_busy_time += delta;
98 +
99 + // the worker was busy
100 + // set it to idle before we set the timestamp
101 +
102 + worker->last_action = WORKER_IDLE;
103 + if(likely(worker->last_action_timestamp < now))
104 + worker->last_action_timestamp = now;
105 +}
106 +
107 +void worker_is_idle(void) {
108 + if(unlikely(!worker)) return;
109 + if(unlikely(worker->last_action != WORKER_BUSY)) return;
110 +
111 + worker_is_idle_with_time(now_realtime_usec());
112 +}
113 +
114 +void worker_is_busy(size_t job_id) {
115 + if(unlikely(!worker)) return;
116 + if(unlikely(job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))
117 + job_id = 0;
118 +
119 + usec_t now = now_realtime_usec();
120 +
121 + if(worker->last_action == WORKER_BUSY)
122 + worker_is_idle_with_time(now);
123 +
124 + // the worker was idle
125 + // set the timestamp and then set it to busy
126 +
127 + worker->job_id = job_id;
128 + worker->per_job_type[job_id].worker_jobs_started++;
129 + worker->jobs_started++;
130 + worker->last_action_timestamp = now;
131 + worker->last_action = WORKER_BUSY;
132 +}
133 +
134 +
135 +// statistics interface
136 +
137 +void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data) {
138 + netdata_mutex_lock(&base_lock);
139 + uint32_t hash = simple_hash(workname);
140 + usec_t busy_time, delta;
141 + size_t i, jobs_started, jobs_running;
142 +
143 + struct worker *p;
144 + for(p = base; p ; p = p->next) {
145 + if(hash != p->workname_hash || strcmp(workname, p->workname)) continue;
146 +
147 + usec_t now = now_realtime_usec();
148 +
149 + // find per job type statistics
150 + const char *per_job_type_name[WORKER_UTILIZATION_MAX_JOB_TYPES];
151 + size_t per_job_type_jobs_started[WORKER_UTILIZATION_MAX_JOB_TYPES];
152 + usec_t per_job_type_busy_time[WORKER_UTILIZATION_MAX_JOB_TYPES];
153 + for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
154 + per_job_type_name[i] = p->per_job_type[i].name;
155 +
156 + size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started;
157 + per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_jobs_started;
158 + p->per_job_type[i].statistics_jobs_started = tmp_jobs_started;
159 +
160 + usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time;
161 + per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_busy_time;
162 + p->per_job_type[i].statistics_busy_time = tmp_busy_time;
163 + }
164 +
165 + // get a copy of the worker variables
166 + usec_t worker_busy_time = p->busy_time;
167 + size_t worker_jobs_started = p->jobs_started;
168 + char worker_last_action = p->last_action;
169 + usec_t worker_last_action_timestamp = p->last_action_timestamp;
170 +
171 + // this is the only variable both the worker thread and the statistics thread are writing
172 + // we set this only when the worker is busy, so that worker will not
173 + // accumulate all the busy time, but only the time after the point we collected statistics
174 + if(worker_last_action == WORKER_BUSY && p->last_action_timestamp == worker_last_action_timestamp && p->last_action == WORKER_BUSY)
175 + p->last_action_timestamp = now;
176 +
177 + // calculate delta busy time
178 + busy_time = worker_busy_time - p->statistics_last_busy_time;
179 + p->statistics_last_busy_time = worker_busy_time;
180 +
181 + // calculate delta jobs done
182 + jobs_started = worker_jobs_started - p->statistics_last_jobs_started;
183 + p->statistics_last_jobs_started = worker_jobs_started;
184 +
185 + jobs_running = 0;
186 + if(worker_last_action == WORKER_BUSY) {
187 + // the worker is still busy with something
188 + // let's add that busy time to the reported one
189 + busy_time += now - worker_last_action_timestamp;
190 + jobs_running = 1;
191 + }
192 +
193 + delta = now - p->statistics_last_checkpoint;
194 +
195 + p->statistics_last_checkpoint = now;
196 +
197 + callback(data, p->pid, p->tag, busy_time, delta, jobs_started, jobs_running, per_job_type_name, per_job_type_jobs_started, per_job_type_busy_time);
198 + }
199 +
200 + netdata_mutex_unlock(&base_lock);
201 +}
libnetdata/worker_utilization/worker_utilization.h new
+22
@@ -0,0 +1,22 @@
1 +#ifndef WORKER_UTILIZATION_H
2 +#define WORKER_UTILIZATION_H 1
3 +
4 +#include "../libnetdata.h"
5 +
6 +// workers interfaces
7 +
8 +#define WORKER_UTILIZATION_MAX_JOB_TYPES 50
9 +#define WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH 25
10 +
11 +extern void worker_register(const char *workname);
12 +extern void worker_register_job_name(size_t job_id, const char *name);
13 +extern void worker_unregister(void);
14 +
15 +extern void worker_is_idle(void);
16 +extern void worker_is_busy(size_t job_id);
17 +
18 +// statistics interface
19 +
20 +extern void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pid, const char *thread_tag, size_t utilization_usec, size_t duration_usec, size_t jobs_started, size_t is_running, const char **job_types_names, size_t *job_types_jobs_started, usec_t *job_types_busy_time), void *data);
21 +
22 +#endif // WORKER_UTILIZATION_H
ml/Host.cc
+32
@@ -358,6 +358,10 @@ void TrainableHost::trainDimension(Dimension *D, const TimePoint &NowTP) {
358 void TrainableHost::train() {
359 Duration<double> MaxSleepFor = Seconds{10 * updateEvery()};
360
361 + worker_register("MLTRAIN");
362 + worker_register_job_name(0, "dimensions");
363 +
364 + worker_is_busy(0);
365 while (!netdata_exit) {
366 netdata_thread_testcancel();
367 netdata_thread_disable_cancelability();
@@ -378,11 +382,23 @@ void TrainableHost::train() {
382 if (RealDuration >= AllottedDuration)
383 continue;
384
385 + worker_is_idle();
386 SleepFor = std::min(AllottedDuration - RealDuration, MaxSleepFor);
387 std::this_thread::sleep_for(SleepFor);
388 + worker_is_busy(0);
389 }
390 }
391
392 +#define WORKER_JOB_DETECT_DIMENSION 0
393 +#define WORKER_JOB_UPDATE_DETECTION_CHART 1
394 +#define WORKER_JOB_UPDATE_ANOMALY_RATES 2
395 +#define WORKER_JOB_UPDATE_CHARTS 3
396 +#define WORKER_JOB_SAVE_ANOMALY_EVENT 4
397 +
398 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 5
399 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 5
400 +#endif
401 +
402 void DetectableHost::detectOnce() {
403 auto P = BRW.insert(WindowAnomalyRate >= Cfg.HostAnomalyRateThreshold);
404 BitRateWindow::Edge Edge = P.first;
@@ -408,6 +424,8 @@ void DetectableHost::detectOnce() {
424 DimsOverThreshold.reserve(DimensionsMap.size());
425
426 for (auto &DP : DimensionsMap) {
427 + worker_is_busy(WORKER_JOB_DETECT_DIMENSION);
428 +
429 Dimension *D = DP.second;
430
431 auto P = D->detect(WindowLength, ResetBitCounter);
@@ -434,6 +452,7 @@ void DetectableHost::detectOnce() {
452 }
453
454 if (CollectAnomalyRates) {
455 + worker_is_busy(WORKER_JOB_UPDATE_ANOMALY_RATES);
456 AnomalyRateTimer = 0;
457 rrdset_done(AnomalyRateRS);
458 }
@@ -442,6 +461,7 @@ void DetectableHost::detectOnce() {
461 this->NumNormalDimensions = NumNormalDimensions;
462 this->NumTrainedDimensions = NumTrainedDimensions;
463
464 + worker_is_busy(WORKER_JOB_UPDATE_CHARTS);
465 updateDimensionsChart(getRH(), NumTrainedDimensions, NumNormalDimensions, NumAnomalousDimensions);
466 updateRateChart(getRH(), WindowAnomalyRate * 10000.0);
467 updateWindowLengthChart(getRH(), WindowLength);
@@ -454,6 +474,8 @@ void DetectableHost::detectOnce() {
474 if (!NewAnomalyEvent || (DimsOverThreshold.size() == 0))
475 return;
476
477 + worker_is_busy(WORKER_JOB_SAVE_ANOMALY_EVENT);
478 +
479 std::sort(DimsOverThreshold.begin(), DimsOverThreshold.end());
480 std::reverse(DimsOverThreshold.begin(), DimsOverThreshold.end());
481
@@ -476,6 +498,13 @@ void DetectableHost::detectOnce() {
498 }
499
500 void DetectableHost::detect() {
501 + worker_register("MLDETECT");
502 + worker_register_job_name(WORKER_JOB_DETECT_DIMENSION, "dimensions");
503 + worker_register_job_name(WORKER_JOB_UPDATE_DETECTION_CHART, "detection chart");
504 + worker_register_job_name(WORKER_JOB_UPDATE_ANOMALY_RATES, "anomaly rates");
505 + worker_register_job_name(WORKER_JOB_UPDATE_CHARTS, "charts");
506 + worker_register_job_name(WORKER_JOB_SAVE_ANOMALY_EVENT, "anomaly event");
507 +
508 std::this_thread::sleep_for(Seconds{10});
509
510 heartbeat_t HB;
@@ -483,10 +512,13 @@ void DetectableHost::detect() {
512
513 while (!netdata_exit) {
514 netdata_thread_testcancel();
515 + worker_is_idle();
516 heartbeat_next(&HB, updateEvery() * USEC_PER_SEC);
517
518 netdata_thread_disable_cancelability();
519 detectOnce();
520 +
521 + worker_is_busy(WORKER_JOB_UPDATE_DETECTION_CHART);
522 updateDetectionChart(getRH());
523 netdata_thread_enable_cancelability();
524 }
parser/parser.c
+7
@@ -133,10 +133,13 @@ int parser_add_keyword(PARSER *parser, char *keyword, keyword_function func)
133
134 tmp_keyword = callocz(1, sizeof(*tmp_keyword));
135
136 + tmp_keyword->worker_job_id = parser->worker_job_ids++;
137 tmp_keyword->keyword = strdupz(keyword);
138 tmp_keyword->keyword_hash = keyword_hash;
139 tmp_keyword->func[tmp_keyword->func_no++] = (void *) func;
140
141 + worker_register_job_name(tmp_keyword->worker_job_id, tmp_keyword->keyword);
142 +
143 tmp_keyword->next = parser->keyword;
144 parser->keyword = tmp_keyword;
145 return tmp_keyword->func_no;
@@ -265,10 +268,12 @@ inline int parser_action(PARSER *parser, char *input)
268
269 uint32_t command_hash = simple_hash(command);
270
271 + size_t worker_job_id;
272 while(tmp_keyword) {
273 if (command_hash == tmp_keyword->keyword_hash &&
274 (!strcmp(command, tmp_keyword->keyword))) {
275 action_function_list = &tmp_keyword->func[0];
276 + worker_job_id = tmp_keyword->worker_job_id;
277 break;
278 }
279 tmp_keyword = tmp_keyword->next;
@@ -284,12 +289,14 @@ inline int parser_action(PARSER *parser, char *input)
289 #endif
290 }
291 else {
292 + worker_is_busy(worker_job_id);
293 while ((action_function = *action_function_list) != NULL) {
294 rc = action_function(words, parser->user, parser->plugins_action);
295 if (unlikely(rc == PARSER_RC_ERROR || rc == PARSER_RC_STOP))
296 break;
297 action_function_list++;
298 }
299 + worker_is_idle();
300 }
301
302 if (likely(input == parser->buffer))
parser/parser.h
+2
@@ -54,6 +54,7 @@ typedef enum parser_input_type {
54 typedef PARSER_RC (*keyword_function)(char **, void *, PLUGINSD_ACTION *plugins_action);
55
56 typedef struct parser_keyword {
57 + size_t worker_job_id;
58 char *keyword;
59 uint32_t keyword_hash;
60 int func_no;
@@ -67,6 +68,7 @@ typedef struct parser_data {
68 } PARSER_DATA;
69
70 typedef struct parser {
71 + size_t worker_job_ids;
72 uint8_t version; // Parser version
73 RRDHOST *host;
74 void *input; // Input source e.g. stream
streaming/receiver.c
+4
@@ -30,6 +30,8 @@ void destroy_receiver_state(struct receiver_state *rpt) {
30 }
31
32 static void rrdpush_receiver_thread_cleanup(void *ptr) {
33 + worker_unregister();
34 +
35 static __thread int executed = 0;
36 if(!executed) {
37 executed = 1;
@@ -716,7 +718,9 @@ void *rrdpush_receiver_thread(void *ptr) {
718 struct receiver_state *rpt = (struct receiver_state *)ptr;
719 info("STREAM %s [%s]:%s: receive thread created (task id %d)", rpt->hostname, rpt->client_ip, rpt->client_port, gettid());
720
721 + worker_register("STREAMRCV");
722 rrdpush_receive(rpt);
723 + worker_unregister();
724
725 netdata_thread_cleanup_pop(1);
726 return NULL;
streaming/sender.c
+82 -14
@@ -2,6 +2,26 @@
2
3 #include "rrdpush.h"
4
5 +#define WORKER_SENDER_JOB_CONNECT 0
6 +#define WORKER_SENDER_JOB_PIPE_READ 1
7 +#define WORKER_SENDER_JOB_SOCKET_RECEIVE 2
8 +#define WORKER_SENDER_JOB_EXECUTE 3
9 +#define WORKER_SENDER_JOB_SOCKET_SEND 4
10 +#define WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE 5
11 +#define WORKER_SENDER_JOB_DISCONNECT_OVERFLOW 6
12 +#define WORKER_SENDER_JOB_DISCONNECT_TIMEOUT 7
13 +#define WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR 8
14 +#define WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR 9
15 +#define WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR 10
16 +#define WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED 11
17 +#define WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR 12
18 +#define WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR 13
19 +#define WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION 14
20 +
21 +#if WORKER_UTILIZATION_MAX_JOB_TYPES < 15
22 +#error WORKER_UTILIZATION_MAX_JOB_TYPES has to be at least 15
23 +#endif
24 +
25 extern struct config stream_config;
26 extern int netdata_use_ssl_on_stream;
27 extern char *netdata_ssl_ca_path;
@@ -21,8 +41,8 @@ static inline void rrdpush_sender_thread_close_socket(RRDHOST *host);
41 * Inform the user through the error log file and
42 * deactivate compression by downgrading the stream protocol.
43 */
24 -static inline void deactivate_compression(struct sender_state *s)
25 -{
44 +static inline void deactivate_compression(struct sender_state *s) {
45 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION);
46 error("STREAM_COMPRESSION: Deactivating compression to avoid stream corruption");
47 default_compression_enabled = 0;
48 s->rrdpush_compression = 0;
@@ -389,6 +409,7 @@ if(!s->rrdpush_compression)
409 err = SSL_get_error(host->ssl.conn, err);
410 error("SSL cannot connect with the server: %s ",ERR_error_string((long)SSL_get_error(host->ssl.conn,err),NULL));
411 if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
412 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
413 rrdpush_sender_thread_close_socket(host);
414 return 0;
415 }else {
@@ -399,6 +420,7 @@ if(!s->rrdpush_compression)
420 if (netdata_use_ssl_on_stream == NETDATA_SSL_FORCE) {
421 if (netdata_validate_server == NETDATA_SSL_VALID_CERTIFICATE) {
422 if ( security_test_certificate(host->ssl.conn)) {
423 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
424 error("Closing the stream connection, because the server SSL certificate is not valid.");
425 rrdpush_sender_thread_close_socket(host);
426 return 0;
@@ -411,6 +433,7 @@ if(!s->rrdpush_compression)
433 #else
434 if(send_timeout(host->rrdpush_sender_socket, http, strlen(http), 0, timeout) == -1) {
435 #endif
436 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
437 error("STREAM %s [send to %s]: failed to send HTTP header to remote netdata.", host->hostname, s->connected_to);
438 rrdpush_sender_thread_close_socket(host);
439 return 0;
@@ -426,6 +449,7 @@ if(!s->rrdpush_compression)
449 received = recv_timeout(host->rrdpush_sender_socket, http, HTTP_HEADER_SIZE, 0, timeout);
450 if(received == -1) {
451 #endif
452 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
453 error("STREAM %s [send to %s]: remote netdata does not respond.", host->hostname, s->connected_to);
454 rrdpush_sender_thread_close_socket(host);
455 return 0;
@@ -435,6 +459,7 @@ if(!s->rrdpush_compression)
459 debug(D_STREAM, "Response to sender from far end: %s", http);
460 int32_t version = (int32_t)parse_stream_version(host, http);
461 if(version == -1) {
462 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE);
463 error("STREAM %s [send to %s]: server is not replying properly (is it a netdata?).", host->hostname, s->connected_to);
464 rrdpush_sender_thread_close_socket(host);
465 return 0;
@@ -541,9 +566,9 @@ void attempt_to_send(struct sender_state *s) {
566 s->last_sent_t = now_monotonic_sec();
567 }
568 else if (ret == -1 && (errno == EAGAIN || errno == EINTR || errno == EWOULDBLOCK))
544 - debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", s->host->hostname,
545 - s->connected_to);
569 + debug(D_STREAM, "STREAM %s [send to %s]: unavailable after polling POLLOUT", s->host->hostname, s->connected_to);
570 else if (ret == -1) {
571 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR);
572 debug(D_STREAM, "STREAM: Send failed - closing socket...");
573 error("STREAM %s [send to %s]: failed to send metrics - closing connection - we have sent %zu bytes on this connection.", s->host->hostname, s->connected_to, s->sent_bytes_on_this_connection);
574 rrdpush_sender_thread_close_socket(s->host);
@@ -570,6 +595,8 @@ int ret;
595 int sslerrno = SSL_get_error(s->host->ssl.conn, desired);
596 if (sslerrno == SSL_ERROR_WANT_READ || sslerrno == SSL_ERROR_WANT_WRITE)
597 return;
598 +
599 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR);
600 u_long err;
601 char buf[256];
602 while ((err = ERR_get_error()) != 0) {
@@ -581,20 +608,25 @@ int ret;
608 return;
609 }
610 #endif
584 - ret = recv(s->host->rrdpush_sender_socket, s->read_buffer + s->read_len, sizeof(s->read_buffer) - s->read_len - 1,
585 - MSG_DONTWAIT);
611 + ret = recv(s->host->rrdpush_sender_socket, s->read_buffer + s->read_len, sizeof(s->read_buffer) - s->read_len - 1,MSG_DONTWAIT);
612 if (ret>0) {
613 s->read_len += ret;
614 return;
615 }
616 +
617 debug(D_STREAM, "Socket was POLLIN, but req %zu bytes gave %d", sizeof(s->read_buffer) - s->read_len - 1, ret);
618 +
619 if (ret<0 && (errno == EAGAIN || errno == EWOULDBLOCK || errno == EINTR))
620 return;
593 - if (ret==0)
621 +
622 + if (ret==0) {
623 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED);
624 error("STREAM %s [send to %s]: connection closed by far end. Restarting connection", s->host->hostname, s->connected_to);
595 - else
596 - error("STREAM %s [send to %s]: error during read (%d). Restarting connection", s->host->hostname, s->connected_to,
597 - ret);
625 + }
626 + else {
627 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR);
628 + error("STREAM %s [send to %s]: error during receive (%d). Restarting connection", s->host->hostname, s->connected_to, ret);
629 + }
630 rrdpush_sender_thread_close_socket(s->host);
631 }
632
@@ -615,6 +647,8 @@ void execute_commands(struct sender_state *s) {
647
648
649 static void rrdpush_sender_thread_cleanup_callback(void *ptr) {
650 + worker_unregister();
651 +
652 RRDHOST *host = (RRDHOST *)ptr;
653
654 netdata_mutex_lock(&host->sender->mutex);
@@ -707,6 +741,25 @@ void *rrdpush_sender_thread(void *ptr) {
741 fds[Collector].fd = s->host->rrdpush_sender_pipe[PIPE_READ];
742 fds[Collector].events = POLLIN;
743
744 + worker_register("STREAMSND");
745 + worker_register_job_name(WORKER_SENDER_JOB_CONNECT, "connect");
746 + worker_register_job_name(WORKER_SENDER_JOB_PIPE_READ, "pipe read");
747 + worker_register_job_name(WORKER_SENDER_JOB_SOCKET_RECEIVE, "receive");
748 + worker_register_job_name(WORKER_SENDER_JOB_EXECUTE, "execute");
749 + worker_register_job_name(WORKER_SENDER_JOB_SOCKET_SEND, "send");
750 +
751 + // disconnection reasons
752 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT, "disconnect timeout");
753 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR, "disconnect poll error");
754 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR, "disconnect socket error");
755 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW, "disconnect overflow");
756 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SSL_ERROR, "disconnect ssl error");
757 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_PARENT_CLOSED, "disconnect parent closed");
758 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_RECEIVE_ERROR, "disconnect receive error");
759 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_SEND_ERROR, "disconnect send error");
760 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_NO_COMPRESSION, "disconnect no compression");
761 + worker_register_job_name(WORKER_SENDER_JOB_DISCONNECT_BAD_HANDSHAKE, "disconnect bad handshake");
762 +
763 netdata_thread_cleanup_push(rrdpush_sender_thread_cleanup_callback, s->host);
764 for(; s->host->rrdpush_send_enabled && !netdata_exit ;) {
765 // check for outstanding cancellation requests
@@ -714,6 +767,7 @@ void *rrdpush_sender_thread(void *ptr) {
767
768 // The connection attempt blocks (after which we use the socket in nonblocking)
769 if(unlikely(s->host->rrdpush_sender_socket == -1)) {
770 + worker_is_busy(WORKER_SENDER_JOB_CONNECT);
771 s->overflow = 0;
772 s->read_len = 0;
773 s->buffer->read = 0;
@@ -731,11 +785,14 @@ void *rrdpush_sender_thread(void *ptr) {
785
786 // If the TCP window never opened then something is wrong, restart connection
787 if(unlikely(now_monotonic_sec() - s->last_sent_t > s->timeout)) {
788 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_TIMEOUT);
789 error("STREAM %s [send to %s]: could not send metrics for %d seconds - closing connection - we have sent %zu bytes on this connection via %zu send attempts.", s->host->hostname, s->connected_to, s->timeout, s->sent_bytes_on_this_connection, s->send_attempts);
790 rrdpush_sender_thread_close_socket(s->host);
791 continue;
792 }
793
794 + worker_is_idle();
795 +
796 // Wait until buffer opens in the socket or a rrdset_done_push wakes us
797 fds[Collector].revents = 0;
798 fds[Socket].revents = 0;
@@ -757,16 +814,18 @@ void *rrdpush_sender_thread(void *ptr) {
814 int retval = poll(fds, 2, 1000);
815 debug(D_STREAM, "STREAM: poll() finished collector=%d socket=%d (current chunk %zu bytes)...",
816 fds[Collector].revents, fds[Socket].revents, outstanding);
817 +
818 if(unlikely(netdata_exit)) break;
819
820 // Spurious wake-ups without error - loop again
763 - if (retval == 0 || ((retval == -1) && (errno == EAGAIN || errno == EINTR)))
764 - {
821 + if (retval == 0 || ((retval == -1) && (errno == EAGAIN || errno == EINTR))) {
822 debug(D_STREAM, "Spurious wakeup");
823 continue;
824 }
825 +
826 // Only errors from poll() are internal, but try restarting the connection
827 if(unlikely(retval == -1)) {
828 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_POLL_ERROR);
829 error("STREAM %s [send to %s]: failed to poll(). Closing socket.", s->host->hostname, s->connected_to);
830 rrdpush_sender_thread_close_socket(s->host);
831 continue;
@@ -774,6 +833,7 @@ void *rrdpush_sender_thread(void *ptr) {
833
834 // If the collector woke us up then empty the pipe to remove the signal
835 if (fds[Collector].revents & POLLIN || fds[Collector].revents & POLLPRI) {
836 + worker_is_busy(WORKER_SENDER_JOB_PIPE_READ);
837 debug(D_STREAM, "STREAM: Data added to send buffer (current buffer chunk %zu bytes)...", outstanding);
838
839 char buffer[1000 + 1];
@@ -782,13 +842,19 @@ void *rrdpush_sender_thread(void *ptr) {
842 }
843
844 // Read as much as possible to fill the buffer, split into full lines for execution.
785 - if (fds[Socket].revents & POLLIN)
845 + if (fds[Socket].revents & POLLIN) {
846 + worker_is_busy(WORKER_SENDER_JOB_SOCKET_RECEIVE);
847 attempt_read(s);
848 + }
849 +
850 + worker_is_busy(WORKER_SENDER_JOB_EXECUTE);
851 execute_commands(s);
852
853 // If we have data and have seen the TCP window open then try to close it by a transmission.
790 - if (outstanding && fds[Socket].revents & POLLOUT)
854 + if (outstanding && fds[Socket].revents & POLLOUT) {
855 + worker_is_busy(WORKER_SENDER_JOB_SOCKET_SEND);
856 attempt_to_send(s);
857 + }
858
859 // TODO-GAPS - why do we only check this on the socket, not the pipe?
860 if (outstanding) {
@@ -800,6 +866,7 @@ void *rrdpush_sender_thread(void *ptr) {
866 else if (unlikely(fds[Socket].revents & POLLNVAL))
867 error = "connection is invalid (POLLNVAL)";
868 if(unlikely(error)) {
869 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_SOCKER_ERROR);
870 error("STREAM %s [send to %s]: restart stream because %s - %zu bytes transmitted.", s->host->hostname,
871 s->connected_to, error, s->sent_bytes_on_this_connection);
872 rrdpush_sender_thread_close_socket(s->host);
@@ -808,6 +875,7 @@ void *rrdpush_sender_thread(void *ptr) {
875
876 // protection from overflow
877 if (s->overflow) {
878 + worker_is_busy(WORKER_SENDER_JOB_DISCONNECT_OVERFLOW);
879 errno = 0;
880 error("STREAM %s [send to %s]: buffer full (%zu-bytes) after %zu bytes. Restarting connection",
881 s->host->hostname, s->connected_to, s->buffer->size, s->sent_bytes_on_this_connection);
web/server/static/static-threaded.c
+89 -56
@@ -7,6 +7,20 @@ int web_client_timeout = DEFAULT_DISCONNECT_IDLE_WEB_CLIENTS_AFTER_SECONDS;
7 int web_client_first_request_timeout = DEFAULT_TIMEOUT_TO_RECEIVE_FIRST_WEB_REQUEST;
8 long web_client_streaming_rate_t = 0L;
9
10 +#define WORKER_JOB_ADD_CONNECTION 0
11 +#define WORKER_JOB_DEL_COLLECTION 1
12 +#define WORKER_JOB_ADD_FILE 2
13 +#define WORKER_JOB_DEL_FILE 3
14 +#define WORKER_JOB_READ_FILE 4
15 +#define WORKER_JOB_WRITE_FILE 5
16 +#define WORKER_JOB_RCV_DATA 6
17 +#define WORKER_JOB_SND_DATA 7
18 +#define WORKER_JOB_PROCESS 8
19 +
20 +#if (WORKER_UTILIZATION_MAX_JOB_TYPES < 9)
21 +#error Please increase WORKER_UTILIZATION_MAX_JOB_TYPES to at least 8
22 +#endif
23 +
24 /*
25 * --------------------------------------------------------------------------------------------------------------------
26 * Build web_client state from the pollinfo that describes an accepted connection.
@@ -71,11 +85,15 @@ static inline int web_server_check_client_status(struct web_client *w) {
85 static void *web_server_file_add_callback(POLLINFO *pi, short int *events, void *data) {
86 struct web_client *w = (struct web_client *)data;
87
88 + worker_is_busy(WORKER_JOB_ADD_FILE);
89 +
90 worker_private->files_read++;
91
92 debug(D_WEB_CLIENT, "%llu: ADDED FILE READ ON FD %d", w->id, pi->fd);
93 *events = POLLIN;
94 pi->data = w;
95 +
96 + worker_is_idle();
97 return w;
98 }
99
@@ -83,27 +101,36 @@ static void web_server_file_del_callback(POLLINFO *pi) {
101 struct web_client *w = (struct web_client *)pi->data;
102 debug(D_WEB_CLIENT, "%llu: RELEASE FILE READ ON FD %d", w->id, pi->fd);
103
104 + worker_is_busy(WORKER_JOB_DEL_FILE);
105 +
106 w->pollinfo_filecopy_slot = 0;
107
108 if(unlikely(!w->pollinfo_slot)) {
109 debug(D_WEB_CLIENT, "%llu: CROSS WEB CLIENT CLEANUP (iFD %d, oFD %d)", w->id, pi->fd, w->ofd);
110 web_client_release(w);
111 }
112 +
113 + worker_is_idle();
114 }
115
116 static int web_server_file_read_callback(POLLINFO *pi, short int *events) {
117 + int retval = -1;
118 struct web_client *w = (struct web_client *)pi->data;
119
120 + worker_is_busy(WORKER_JOB_READ_FILE);
121 +
122 // if there is no POLLINFO linked to this, it means the client disconnected
123 // stop the file reading too
124 if(unlikely(!w->pollinfo_slot)) {
125 debug(D_WEB_CLIENT, "%llu: PREVENTED ATTEMPT TO READ FILE ON FD %d, ON CLOSED WEB CLIENT", w->id, pi->fd);
101 - return -1;
126 + retval = -1;
127 + goto cleanup;
128 }
129
130 if(unlikely(w->mode != WEB_CLIENT_MODE_FILECOPY || w->ifd == w->ofd)) {
131 debug(D_WEB_CLIENT, "%llu: PREVENTED ATTEMPT TO READ FILE ON FD %d, ON NON-FILECOPY WEB CLIENT", w->id, pi->fd);
106 - return -1;
132 + retval = -1;
133 + goto cleanup;
134 }
135
136 debug(D_WEB_CLIENT, "%llu: READING FILE ON FD %d", w->id, pi->fd);
@@ -121,18 +148,25 @@ static int web_server_file_read_callback(POLLINFO *pi, short int *events) {
148
149 if(unlikely(ret <= 0 || w->ifd == w->ofd)) {
150 debug(D_WEB_CLIENT, "%llu: DONE READING FILE ON FD %d", w->id, pi->fd);
124 - return -1;
151 + retval = -1;
152 + goto cleanup;
153 }
154
155 *events = POLLIN;
128 - return 0;
156 + retval = 0;
157 +
158 +cleanup:
159 + worker_is_idle();
160 + return retval;
161 }
162
163 static int web_server_file_write_callback(POLLINFO *pi, short int *events) {
164 (void)pi;
165 (void)events;
166
167 + worker_is_busy(WORKER_JOB_WRITE_FILE);
168 error("Writing to web files is not supported!");
169 + worker_is_idle();
170
171 return -1;
172 }
@@ -143,6 +177,7 @@ static int web_server_file_write_callback(POLLINFO *pi, short int *events) {
177 static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data) {
178 (void)data; // Suppress warning on unused argument
179
180 + worker_is_busy(WORKER_JOB_ADD_CONNECTION);
181 worker_private->connected++;
182
183 size_t concurrent = worker_private->connected - worker_private->disconnected;
@@ -177,7 +212,7 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data
212 //this means that the mensage was not completely read, so
213 //I cannot identify it yet.
214 sock_setnonblock(w->ifd);
180 - return w;
215 + goto cleanup;
216 }
217
218 //The next two ifs are not together because I am reusing SSL structure
@@ -191,7 +226,7 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data
226 if (test[0] < 0x18){
227 WEB_CLIENT_IS_DEAD(w);
228 sock_setnonblock(w->ifd);
194 - return w;
229 + goto cleanup;
230 }
231 }
232 }
@@ -217,11 +252,16 @@ static void *web_server_add_callback(POLLINFO *pi, short int *events, void *data
252 #endif
253
254 debug(D_WEB_CLIENT, "%llu: ADDED CLIENT FD %d", w->id, pi->fd);
255 +
256 +cleanup:
257 + worker_is_idle();
258 return w;
259 }
260
261 // TCP client disconnected
262 static void web_server_del_callback(POLLINFO *pi) {
263 + worker_is_busy(WORKER_JOB_DEL_COLLECTION);
264 +
265 worker_private->disconnected++;
266
267 struct web_client *w = (struct web_client *)pi->data;
@@ -240,18 +280,27 @@ static void web_server_del_callback(POLLINFO *pi) {
280 debug(D_WEB_CLIENT, "%llu: CLOSING CLIENT FD %d", w->id, pi->fd);
281 web_client_release(w);
282 }
283 +
284 + worker_is_idle();
285 }
286
287 static int web_server_rcv_callback(POLLINFO *pi, short int *events) {
288 + int ret = -1;
289 + worker_is_busy(WORKER_JOB_RCV_DATA);
290 +
291 worker_private->receptions++;
292
293 struct web_client *w = (struct web_client *)pi->data;
294 int fd = pi->fd;
295
251 - if(unlikely(web_client_receive(w) < 0))
252 - return -1;
296 + if(unlikely(web_client_receive(w) < 0)) {
297 + ret = -1;
298 + goto cleanup;
299 + }
300
301 debug(D_WEB_CLIENT, "%llu: processing received data on fd %d.", w->id, fd);
302 + worker_is_idle();
303 + worker_is_busy(WORKER_JOB_PROCESS);
304 web_client_process_request(w);
305
306 if(unlikely(w->mode == WEB_CLIENT_MODE_FILECOPY)) {
@@ -282,7 +331,8 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) {
331 w->pollinfo_filecopy_slot = fpi->slot;
332 else {
333 error("Failed to add filecopy fd. Closing client.");
285 - return -1;
334 + ret = -1;
335 + goto cleanup;
336 }
337 }
338 }
@@ -295,10 +345,17 @@ static int web_server_rcv_callback(POLLINFO *pi, short int *events) {
345 if(unlikely(w->ofd == fd && web_client_has_wait_send(w)))
346 *events |= POLLOUT;
347
298 - return web_server_check_client_status(w);
348 + ret = web_server_check_client_status(w);
349 +
350 +cleanup:
351 + worker_is_idle();
352 + return ret;
353 }
354
355 static int web_server_snd_callback(POLLINFO *pi, short int *events) {
356 + int retval = -1;
357 + worker_is_busy(WORKER_JOB_SND_DATA);
358 +
359 worker_private->sends++;
360
361 struct web_client *w = (struct web_client *)pi->data;
@@ -306,8 +363,12 @@ static int web_server_snd_callback(POLLINFO *pi, short int *events) {
363
364 debug(D_WEB_CLIENT, "%llu: sending data on fd %d.", w->id, fd);
365
309 - if(unlikely(web_client_send(w) < 0))
310 - return -1;
366 + int ret = web_client_send(w);
367 +
368 + if(unlikely(ret < 0)) {
369 + retval = -1;
370 + goto cleanup;
371 + }
372
373 if(unlikely(w->ifd == fd && web_client_has_wait_receive(w)))
374 *events |= POLLIN;
@@ -315,50 +376,11 @@ static int web_server_snd_callback(POLLINFO *pi, short int *events) {
376 if(unlikely(w->ofd == fd && web_client_has_wait_send(w)))
377 *events |= POLLOUT;
378
318 - return web_server_check_client_status(w);
319 -}
379 + retval = web_server_check_client_status(w);
380
321 -static void web_server_tmr_callback(void *timer_data) {
322 - worker_private = (struct web_server_static_threaded_worker *)timer_data;
323 -
324 - static __thread RRDSET *st = NULL;
325 - static __thread RRDDIM *rd_user = NULL, *rd_system = NULL;
326 -
327 - if(unlikely(netdata_exit)) return;
328 -
329 - if(unlikely(!st)) {
330 - char id[100 + 1];
331 - char title[100 + 1];
332 -
333 - snprintfz(id, 100, "web_thread%d_cpu", worker_private->id + 1);
334 - snprintfz(title, 100, "Netdata web server thread CPU usage");
335 -
336 - st = rrdset_create_localhost(
337 - "netdata"
338 - , id
339 - , NULL
340 - , "web"
341 - , "netdata.web_cpu"
342 - , title
343 - , "milliseconds/s"
344 - , "web"
345 - , "stats"
346 - , 132000 + worker_private->id
347 - , default_rrd_update_every
348 - , RRDSET_TYPE_STACKED
349 - );
350 -
351 - rd_user = rrddim_add(st, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
352 - rd_system = rrddim_add(st, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
353 - }
354 - else
355 - rrdset_next(st);
356 -
357 - struct rusage rusage;
358 - getrusage(RUSAGE_THREAD, &rusage);
359 - rrddim_set_by_pointer(st, rd_user, rusage.ru_utime.tv_sec * 1000000ULL + rusage.ru_utime.tv_usec);
360 - rrddim_set_by_pointer(st, rd_system, rusage.ru_stime.tv_sec * 1000000ULL + rusage.ru_stime.tv_usec);
361 - rrdset_done(st);
381 +cleanup:
382 + worker_is_idle();
383 + return retval;
384 }
385
386 // ----------------------------------------------------------------------------
@@ -379,11 +401,22 @@ static void socket_listen_main_static_threaded_worker_cleanup(void *ptr) {
401 );
402
403 worker_private->running = 0;
404 + worker_unregister();
405 }
406
407 void *socket_listen_main_static_threaded_worker(void *ptr) {
408 worker_private = (struct web_server_static_threaded_worker *)ptr;
409 worker_private->running = 1;
410 + worker_register("WEB");
411 + worker_register_job_name(WORKER_JOB_ADD_CONNECTION, "connect");
412 + worker_register_job_name(WORKER_JOB_DEL_COLLECTION, "disconnect");
413 + worker_register_job_name(WORKER_JOB_ADD_FILE, "file start");
414 + worker_register_job_name(WORKER_JOB_DEL_FILE, "file end");
415 + worker_register_job_name(WORKER_JOB_READ_FILE, "file read");
416 + worker_register_job_name(WORKER_JOB_WRITE_FILE, "file write");
417 + worker_register_job_name(WORKER_JOB_RCV_DATA, "receive");
418 + worker_register_job_name(WORKER_JOB_SND_DATA, "send");
419 + worker_register_job_name(WORKER_JOB_PROCESS, "process");
420
421 netdata_thread_cleanup_push(socket_listen_main_static_threaded_worker_cleanup, ptr);
422
@@ -392,7 +425,7 @@ void *socket_listen_main_static_threaded_worker(void *ptr) {
425 , web_server_del_callback
426 , web_server_rcv_callback
427 , web_server_snd_callback
395 - , web_server_tmr_callback
428 + , NULL
429 , web_allow_connections_from
430 , web_allow_connections_dns
431 , NULL