@cryptotaxi247 / netdata-1 / commits / 33e20ac49

workers fixes and improvements (#12863)

Costa Tsaousis committed May 10, 2022 at 14:53 UTC 33e20ac4909596dae0ae718ba527e19d4a30ed70
4 files changed +210 -107
daemon/global_statistics.c
+138 -74
@@ -196,7 +196,7 @@ static void global_statistics_charts(void) {
196 "netdata"
197 , "clients"
198 , NULL
199 - , "netdata"
199 + , "api"
200 , NULL
201 , "Netdata Web Clients"
202 , "connected clients"
@@ -227,7 +227,7 @@ static void global_statistics_charts(void) {
227 "netdata"
228 , "requests"
229 , NULL
230 - , "netdata"
230 + , "api"
231 , NULL
232 , "Netdata Web Requests"
233 , "requests/s"
@@ -259,13 +259,13 @@ static void global_statistics_charts(void) {
259 "netdata"
260 , "net"
261 , NULL
262 - , "netdata"
262 + , "api"
263 , NULL
264 , "Netdata Network Traffic"
265 , "kilobits/s"
266 , "netdata"
267 , "stats"
268 - , 130000
268 + , 130400
269 , localhost->rrd_update_every
270 , RRDSET_TYPE_AREA
271 );
@@ -293,13 +293,13 @@ static void global_statistics_charts(void) {
293 "netdata"
294 , "response_time"
295 , NULL
296 - , "netdata"
296 + , "api"
297 , NULL
298 , "Netdata API Response Time"
299 , "milliseconds/request"
300 , "netdata"
301 , "stats"
302 - , 130400
302 + , 130500
303 , localhost->rrd_update_every
304 , RRDSET_TYPE_LINE
305 );
@@ -342,13 +342,13 @@ static void global_statistics_charts(void) {
342 "netdata"
343 , "compression_ratio"
344 , NULL
345 - , "netdata"
345 + , "api"
346 , NULL
347 , "Netdata API Responses Compression Savings Ratio"
348 , "percentage"
349 , "netdata"
350 , "stats"
351 - , 130500
351 + , 130600
352 , localhost->rrd_update_every
353 , RRDSET_TYPE_LINE
354 );
@@ -395,7 +395,7 @@ static void global_statistics_charts(void) {
395 , "queries/s"
396 , "netdata"
397 , "stats"
398 - , 130500
398 + , 131000
399 , localhost->rrd_update_every
400 , RRDSET_TYPE_LINE
401 );
@@ -428,7 +428,7 @@ static void global_statistics_charts(void) {
428 , "points/s"
429 , "netdata"
430 , "stats"
431 - , 130501
431 + , 131001
432 , localhost->rrd_update_every
433 , RRDSET_TYPE_AREA
434 );
@@ -499,7 +499,7 @@ static void dbengine_statistics_charts(void) {
499 "percentage",
500 "netdata",
501 "stats",
502 - 130502,
502 + 132000,
503 localhost->rrd_update_every,
504 RRDSET_TYPE_LINE);
505
@@ -539,7 +539,7 @@ static void dbengine_statistics_charts(void) {
539 "percentage",
540 "netdata",
541 "stats",
542 - 130503,
542 + 132003,
543 localhost->rrd_update_every,
544 RRDSET_TYPE_LINE);
545
@@ -592,7 +592,7 @@ static void dbengine_statistics_charts(void) {
592 "pages",
593 "netdata",
594 "stats",
595 - 130504,
595 + 132004,
596 localhost->rrd_update_every,
597 RRDSET_TYPE_LINE);
598
@@ -635,7 +635,7 @@ static void dbengine_statistics_charts(void) {
635 "pages",
636 "netdata",
637 "stats",
638 - 130505,
638 + 132005,
639 localhost->rrd_update_every,
640 RRDSET_TYPE_LINE);
641
@@ -673,7 +673,7 @@ static void dbengine_statistics_charts(void) {
673 "MiB/s",
674 "netdata",
675 "stats",
676 - 130506,
676 + 132006,
677 localhost->rrd_update_every,
678 RRDSET_TYPE_LINE);
679
@@ -705,7 +705,7 @@ static void dbengine_statistics_charts(void) {
705 "operations/s",
706 "netdata",
707 "stats",
708 - 130507,
708 + 132007,
709 localhost->rrd_update_every,
710 RRDSET_TYPE_LINE);
711
@@ -738,7 +738,7 @@ static void dbengine_statistics_charts(void) {
738 "errors/s",
739 "netdata",
740 "stats",
741 - 130508,
741 + 132008,
742 localhost->rrd_update_every,
743 RRDSET_TYPE_LINE);
744
@@ -773,7 +773,7 @@ static void dbengine_statistics_charts(void) {
773 "descriptors",
774 "netdata",
775 "stats",
776 - 130509,
776 + 132009,
777 localhost->rrd_update_every,
778 RRDSET_TYPE_LINE);
779
@@ -810,7 +810,7 @@ static void dbengine_statistics_charts(void) {
810 "MiB",
811 "netdata",
812 "stats",
813 - 130510,
813 + 132010,
814 localhost->rrd_update_every,
815 RRDSET_TYPE_STACKED);
816
@@ -884,6 +884,8 @@ static void update_heartbeat_charts() {
884 // ---------------------------------------------------------------------------------------------------------------------
885 // worker utilization
886
887 +#define WORKERS_MIN_PERCENT_DEFAULT 10000.0
888 +
889 struct worker_job_type {
890 char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
891 size_t jobs_started;
@@ -898,6 +900,7 @@ struct worker_thread {
900 int enabled;
901
902 int cpu_enabled;
903 + double cpu;
904
905 kernel_uint_t utime;
906 kernel_uint_t stime;
@@ -932,6 +935,11 @@ struct worker_utilization {
935 double workers_min_busy_time;
936 double workers_max_busy_time;
937
938 + size_t workers_cpu_registered;
939 + double workers_cpu_min;
940 + double workers_cpu_max;
941 + double workers_cpu_total;
942 +
943 struct worker_thread *threads;
944
945 RRDSET *st_workers_time;
@@ -939,7 +947,6 @@ struct worker_utilization {
947 RRDDIM *rd_workers_time_min;
948 RRDDIM *rd_workers_time_max;
949
942 - size_t workers_cpu_enabled;
950 RRDSET *st_workers_cpu;
951 RRDDIM *rd_workers_cpu_avg;
952 RRDDIM *rd_workers_cpu_min;
@@ -951,8 +958,78 @@ struct worker_utilization {
958
959 RRDSET *st_workers_jobs_per_job_type;
960 RRDSET *st_workers_busy_per_job_type;
961 +
962 + RRDDIM *rd_total_cpu_utilizaton;
963 +};
964 +
965 +static struct worker_utilization all_workers_utilization[] = {
966 + { .name = "STATS", .family = "workers global statistics", .priority = 1000000 },
967 + { .name = "HEALTH", .family = "workers health alarms", .priority = 1000000 },
968 + { .name = "MLTRAIN", .family = "workers ML training", .priority = 1000000 },
969 + { .name = "MLDETECT", .family = "workers ML detection", .priority = 1000000 },
970 + { .name = "STREAMRCV", .family = "workers streaming receive", .priority = 1000000 },
971 + { .name = "STREAMSND", .family = "workers streaming send", .priority = 1000000 },
972 + { .name = "DBENGINE", .family = "workers dbengine instances", .priority = 1000000 },
973 + { .name = "WEB", .family = "workers web server", .priority = 1000000 },
974 + { .name = "ACLKQUERY", .family = "workers aclk query", .priority = 1000000 },
975 + { .name = "ACLKSYNC", .family = "workers aclk host sync", .priority = 1000000 },
976 + { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 },
977 + { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 },
978 + { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 },
979 + { .name = "PROC", .family = "workers plugin proc", .priority = 1000000 },
980 + { .name = "FREEBSD", .family = "workers plugin freebsd", .priority = 1000000 },
981 + { .name = "MACOS", .family = "workers plugin macos", .priority = 1000000 },
982 + { .name = "CGROUPS", .family = "workers plugin cgroups", .priority = 1000000 },
983 + { .name = "CGROUPSDISC", .family = "workers plugin cgroups find", .priority = 1000000 },
984 + { .name = "DISKSPACE", .family = "workers plugin diskspace", .priority = 1000000 },
985 + { .name = "TC", .family = "workers plugin tc", .priority = 1000000 },
986 + { .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 },
987 + { .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 },
988 +
989 + // has to be terminated with a NULL
990 + { .name = NULL, .family = NULL }
991 };
992
993 +static void workers_total_cpu_utilization_chart(void) {
994 + size_t i, cpu_enabled = 0;
995 + for(i = 0; all_workers_utilization[i].name ;i++)
996 + if(all_workers_utilization[i].workers_cpu_registered) cpu_enabled++;
997 +
998 + if(!cpu_enabled) return;
999 +
1000 + static RRDSET *st = NULL;
1001 +
1002 + if(!st) {
1003 + st = rrdset_create_localhost(
1004 + "netdata",
1005 + "workers_cpu",
1006 + NULL,
1007 + "workers",
1008 + "netdata.workers.cpu_total",
1009 + "Netdata Workers CPU Utilization (100% = 1 core)",
1010 + "%",
1011 + "netdata",
1012 + "stats",
1013 + 999000,
1014 + localhost->rrd_update_every,
1015 + RRDSET_TYPE_STACKED);
1016 + }
1017 +
1018 + rrdset_next(st);
1019 +
1020 + for(i = 0; all_workers_utilization[i].name ;i++) {
1021 + struct worker_utilization *wu = &all_workers_utilization[i];
1022 + if(!wu->workers_cpu_registered) continue;
1023 +
1024 + if(!wu->rd_total_cpu_utilizaton)
1025 + wu->rd_total_cpu_utilizaton = rrddim_add(st, wu->name_lowercase, NULL, 1, 10000ULL, RRD_ALGORITHM_ABSOLUTE);
1026 +
1027 + rrddim_set_by_pointer(st, wu->rd_total_cpu_utilizaton, (collected_number)((double)wu->workers_cpu_total * 10000.0));
1028 + }
1029 +
1030 + rrdset_done(st);
1031 +}
1032 +
1033 static void workers_utilization_update_chart(struct worker_utilization *wu) {
1034 if(!wu->workers_registered) return;
1035
@@ -1000,19 +1077,25 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
1077
1078 rrdset_next(wu->st_workers_time);
1079
1080 + if(unlikely(wu->workers_min_busy_time == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_min_busy_time = 0.0;
1081 +
1082 if(wu->rd_workers_time_min)
1083 rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_min, (collected_number)((double)wu->workers_min_busy_time * 10000.0));
1084
1085 if(wu->rd_workers_time_max)
1086 rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_max, (collected_number)((double)wu->workers_max_busy_time * 10000.0));
1087
1009 - rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * 10000.0 / (double)wu->workers_total_duration));
1088 + if(wu->workers_total_duration == 0)
1089 + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, 0);
1090 + else
1091 + rrddim_set_by_pointer(wu->st_workers_time, wu->rd_workers_time_avg, (collected_number)((double)wu->workers_total_busy_time * 100.0 * 10000.0 / (double)wu->workers_total_duration));
1092 +
1093 rrdset_done(wu->st_workers_time);
1094
1095 // ----------------------------------------------------------------------
1096
1097 #ifdef __linux__
1015 - if(wu->workers_cpu_enabled || wu->st_workers_cpu) {
1098 + if(wu->workers_cpu_registered || wu->st_workers_cpu) {
1099 if(unlikely(!wu->st_workers_cpu)) {
1100 char name[RRD_ID_LENGTH_MAX + 1];
1101 snprintfz(name, RRD_ID_LENGTH_MAX, "workers_cpu_%s", wu->name_lowercase);
@@ -1047,31 +1130,19 @@ static void workers_utilization_update_chart(struct worker_utilization *wu) {
1130
1131 rrdset_next(wu->st_workers_cpu);
1132
1050 - size_t count = 0;
1051 - calculated_number min = 1000.0, max = 0.0, total = 0.0;
1052 - struct worker_thread *wt;
1053 - for(wt = wu->threads; wt ; wt = wt->next) {
1054 - if(!wt->cpu_enabled) continue;
1055 - count++;
1056 -
1057 - usec_t delta = wt->collected_time - wt->collected_time_old;
1058 - calculated_number utime = (calculated_number)(wt->utime - wt->utime_old) / (calculated_number)system_hz * 100.0 * (calculated_number)USEC_PER_SEC / (calculated_number)delta;
1059 - calculated_number stime = (calculated_number)(wt->stime - wt->stime_old) / (calculated_number)system_hz * 100.0 * (calculated_number)USEC_PER_SEC / (calculated_number)delta;
1060 - calculated_number cpu_util = utime + stime;
1061 -
1062 - total += cpu_util;
1063 - if(cpu_util < min) min = cpu_util;
1064 - if(cpu_util > max) max = cpu_util;
1065 - }
1066 - if(unlikely(min == 1000.0)) min = 0.0;
1133 + if(unlikely(wu->workers_cpu_min == WORKERS_MIN_PERCENT_DEFAULT)) wu->workers_cpu_min = 0.0;
1134
1135 if(wu->rd_workers_cpu_min)
1069 - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(min * 10000ULL));
1136 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_min, (collected_number)(wu->workers_cpu_min * 10000ULL));
1137
1138 if(wu->rd_workers_cpu_max)
1072 - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(max * 10000ULL));
1139 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_max, (collected_number)(wu->workers_cpu_max * 10000ULL));
1140 +
1141 + if(wu->workers_cpu_registered == 0)
1142 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, 0);
1143 + else
1144 + rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( wu->workers_cpu_total * 10000ULL / (calculated_number)wu->workers_cpu_registered ));
1145
1074 - rrddim_set_by_pointer(wu->st_workers_cpu, wu->rd_workers_cpu_avg, (collected_number)( total * 10000ULL / (calculated_number)count ));
1146 rrdset_done(wu->st_workers_cpu);
1147 }
1148 #endif
@@ -1203,9 +1274,13 @@ static void workers_utilization_reset_statistics(struct worker_utilization *wu)
1274 wu->workers_total_busy_time = 0;
1275 wu->workers_total_duration = 0;
1276 wu->workers_total_jobs_started = 0;
1206 - wu->workers_min_busy_time = 100.0;
1277 + wu->workers_min_busy_time = WORKERS_MIN_PERCENT_DEFAULT;
1278 wu->workers_max_busy_time = 0;
1208 - wu->workers_cpu_enabled = 0;
1279 +
1280 + wu->workers_cpu_registered = 0;
1281 + wu->workers_cpu_min = WORKERS_MIN_PERCENT_DEFAULT;
1282 + wu->workers_cpu_max = 0;
1283 + wu->workers_cpu_total = 0;
1284
1285 size_t i;
1286 for(i = 0; i < WORKER_UTILIZATION_MAX_JOB_TYPES ;i++) {
@@ -1339,39 +1414,21 @@ static void worker_utilization_charts_callback(void *ptr, pid_t pid __maybe_unus
1414
1415 // find its CPU utilization
1416 if((!read_thread_cpu_time_from_proc_stat(pid, &wt->utime, &wt->stime))) {
1342 - wt->cpu_enabled = 1;
1417 wt->collected_time = now_realtime_usec();
1344 - }
1345 - wu->workers_cpu_enabled += wt->cpu_enabled;
1346 -}
1418 + usec_t delta = wt->collected_time - wt->collected_time_old;
1419
1348 -static struct worker_utilization all_workers_utilization[] = {
1349 - { .name = "STATS", .family = "workers global statistics", .priority = 1000000 },
1350 - { .name = "HEALTH", .family = "workers health alarms", .priority = 1000000 },
1351 - { .name = "MLTRAIN", .family = "workers ML training", .priority = 1000000 },
1352 - { .name = "MLDETECT", .family = "workers ML detection", .priority = 1000000 },
1353 - { .name = "STREAMRCV", .family = "workers streaming receive", .priority = 1000000 },
1354 - { .name = "STREAMSND", .family = "workers streaming send", .priority = 1000000 },
1355 - { .name = "DBENGINE", .family = "workers dbengine instances", .priority = 1000000 },
1356 - { .name = "WEB", .family = "workers web server", .priority = 1000000 },
1357 - { .name = "ACLKQUERY", .family = "workers aclk query", .priority = 1000000 },
1358 - { .name = "ACLKSYNC", .family = "workers aclk host sync", .priority = 1000000 },
1359 - { .name = "PLUGINSD", .family = "workers plugins.d", .priority = 1000000 },
1360 - { .name = "STATSD", .family = "workers plugin statsd", .priority = 1000000 },
1361 - { .name = "STATSDFLUSH", .family = "workers plugin statsd flush", .priority = 1000000 },
1362 - { .name = "PROC", .family = "workers plugin proc", .priority = 1000000 },
1363 - { .name = "FREEBSD", .family = "workers plugin freebsd", .priority = 1000000 },
1364 - { .name = "MACOS", .family = "workers plugin macos", .priority = 1000000 },
1365 - { .name = "CGROUPS", .family = "workers plugin cgroups", .priority = 1000000 },
1366 - { .name = "CGROUPSDISC", .family = "workers plugin cgroups find", .priority = 1000000 },
1367 - { .name = "DISKSPACE", .family = "workers plugin diskspace", .priority = 1000000 },
1368 - { .name = "TC", .family = "workers plugin tc", .priority = 1000000 },
1369 - { .name = "TIMEX", .family = "workers plugin timex", .priority = 1000000 },
1370 - { .name = "IDLEJITTER", .family = "workers plugin idlejitter", .priority = 1000000 },
1420 + double utime = (double)(wt->utime - wt->utime_old) / (double)system_hz * 100.0 * (double)USEC_PER_SEC / (double)delta;
1421 + double stime = (double)(wt->stime - wt->stime_old) / (double)system_hz * 100.0 * (double)USEC_PER_SEC / (double)delta;
1422 + double cpu = utime + stime;
1423 + wt->cpu = cpu;
1424 + wt->cpu_enabled = 1;
1425
1372 - // has to be terminated with a NULL
1373 - { .name = NULL, .family = NULL }
1374 -};
1426 + wu->workers_cpu_total += cpu;
1427 + if(cpu < wu->workers_cpu_min) wu->workers_cpu_min = cpu;
1428 + if(cpu > wu->workers_cpu_max) wu->workers_cpu_max = cpu;
1429 + }
1430 + wu->workers_cpu_registered += wt->cpu_enabled;
1431 +}
1432
1433 static void worker_utilization_charts(void) {
1434 static size_t iterations = 0;
@@ -1388,6 +1445,8 @@ static void worker_utilization_charts(void) {
1445
1446 workers_threads_cleanup(&all_workers_utilization[i]);
1447 }
1448 +
1449 + workers_total_cpu_utilization_chart();
1450 }
1451
1452 static void worker_utilization_finish(void) {
@@ -1443,6 +1502,11 @@ void *global_statistics_main(void *ptr)
1502 usec_t step = update_every * USEC_PER_SEC;
1503 heartbeat_t hb;
1504 heartbeat_init(&hb);
1505 +
1506 + // keep the randomness at zero
1507 + // to make sure we are not close to any other thread
1508 + hb.randomness = 0;
1509 +
1510 while (!netdata_exit) {
1511 worker_is_idle();
1512 heartbeat_next(&hb, step);
libnetdata/clocks/clocks.c
+2 -1
@@ -259,7 +259,8 @@ void heartbeat_statistics(usec_t *min_ptr, usec_t *max_ptr, usec_t *average_ptr,
259 count++;
260 }
261 }
262 - average = total / count;
262 + if(count)
263 + average = total / count;
264
265 if(min_ptr) *min_ptr = min;
266 if(max_ptr) *max_ptr = max;
libnetdata/worker_utilization/README.md
+47 -15
@@ -5,8 +5,16 @@ custom_edit_url: https://github.com/netdata/netdata/edit/master/libnetdata/onewa
5
6 # Worker Utilization
7
8 -This library is to be used when there are 1 or more worker threads accepting requests of some kind and servicing them.
9 -The goal is to provide a very simple way to monitor worker threads utilization, as a percentage of the time they are busy and the amount of requests served.
8 +This library is to be used when there are 1 or more worker threads accepting requests
9 +of some kind and servicing them. The goal is to provide a very simple way to monitor
10 +worker threads utilization, as a percentage of the time they are busy and the amount
11 +of requests served.
12 +
13 +## Design goals
14 +
15 +1. Minimal, if any, impact on the performance of the workers
16 +2. Easy to be integrated into any kind of worker
17 +3. No state of any kind at the worker side
18
19 ## How to use
20
@@ -19,40 +27,64 @@ void worker_register(const char *name);
27 This will create the necessary structures for the library to work.
28 No need to keep a pointer to them. They are allocated as `__thread` variables.
29
30 +Then job types need to be defined. Job types are anything a worker does that can be
31 +counted and their execution time needs to be reported. The library is fast enough to
32 +be integrated even on workers that perform hundreds of thousands of actions per second.
33 +
34 +Job types are defined like this:
35 +
36 +```c
37 +void worker_register_job_type(size_t id, const char *name);
38 +```
39 +
40 +`id` is a number starting from zero. The library is compiled with a fixed size of 50
41 +ids (0 to 49). More can be allocated by setting `WORKER_UTILIZATION_MAX_JOB_TYPES` in
42 +`worker_utilization.h`. `name` can be any string up to 22 characters. This can be
43 +changed by setting `WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH` in `worker_utilization.h`.
44 +
45 +Each thread that calls `worker_register(name)` will allocate about 3kB for maintaining
46 +the information required.
47 +
48 When the thread stops, call:
49
50 ```c
25 -void worker_unregister(void)
51 +void worker_unregister(void);
52 ```
53
54 Again, no parameters, or return values.
55
56 +> IMPORTANT: cancellable threads need to add a call to `worker_unregister()` to the
57 +> `pop` function that cleans up the thread. Failure to do so, will result in about
58 +> 3kB of memory leak for every thread that is stopped.
59 +
60 When you are about to do some work in the working thread, call:
61
62 ```c
33 -void worker_is_busy(void)
63 +void worker_is_busy(size_t id);
64 ```
65
66 When you finish doing the job, call:
67
68 ```c
39 -void worker_is_idle(void)
69 +void worker_is_idle(void);
70 ```
71
42 -Calls to `worker_is_busy()` can be made one after another (without calling
72 +Calls to `worker_is_busy(id)` can be made one after another (without calling
73 `worker_is_idle()` between them) to switch jobs without losing any time between
74 them and eliminating one of the 2 clock calls involved.
75
76 ## Implementation details
77
48 -Totally lockless, extremely fast, it should not introduce any kind of problems to the workers.
49 -Every time `worker_is_busy()` or `worker_is_idle()` are called, a call to `now_realtime_usec()`
50 -is done and a couple of variables are updated. That's it!
78 +Totally lockless, extremely fast, it should not introduce any kind of problems to the
79 +workers. Every time `worker_is_busy(id)` or `worker_is_idle()` are called, a call to
80 +`now_realtime_usec()` is done and a couple of variables are updated. That's it!
81
52 -The worker does not need to update the variables regularly. Based on the last status of the worker,
53 -the statistics collector of netdata will calculate if the thread is busy or idle all the time or
54 -part of the time. Works well for both thousands of jobs per second and unlimited working time
55 -(being totally busy with a single request for ages).
82 +The worker does not need to update the variables regularly. Based on the last status
83 +of the worker, the statistics collector of netdata will calculate if the thread is
84 +busy or idle all the time or part of the time. Works well for both thousands of jobs
85 +per second and unlimited working time (being totally busy with a single request for
86 +ages).
87
57 -The statistics collector is called by the global statistics thread of netdata. So, even if the workers
58 -are extremely busy with their jobs, netdata will be able to know how busy they are.
88 +The statistics collector is called by the global statistics thread of netdata. So,
89 +even if the workers are extremely busy with their jobs, netdata will be able to know
90 +how busy they are.
libnetdata/worker_utilization/worker_utilization.c
+23 -17
@@ -5,11 +5,14 @@
5
6 struct worker_job_type {
7 char name[WORKER_UTILIZATION_MAX_JOB_NAME_LENGTH + 1];
8 - size_t worker_jobs_started;
9 - usec_t worker_busy_time;
8
11 - size_t statistics_jobs_started;
12 - usec_t statistics_busy_time;
9 + // statistics controlled variables
10 + size_t statistics_last_jobs_started;
11 + usec_t statistics_last_busy_time;
12 +
13 + // worker controlled variables
14 + volatile size_t worker_jobs_started;
15 + volatile usec_t worker_busy_time;
16 };
17
18 struct worker {
@@ -18,13 +21,13 @@ struct worker {
21 const char *workname;
22 uint32_t workname_hash;
23
21 - // only one variable is set by our statistics callers
22 - usec_t statistics_last_checkpoint;
24 + // statistics controlled variables
25 + volatile usec_t statistics_last_checkpoint;
26 size_t statistics_last_jobs_started;
27 usec_t statistics_last_busy_time;
28
29 // the worker controlled variables
27 - size_t job_id;
30 + volatile size_t job_id;
31 volatile size_t jobs_started;
32 volatile usec_t busy_time;
33 volatile usec_t last_action_timestamp;
@@ -154,22 +157,26 @@ void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pi
157 per_job_type_name[i] = p->per_job_type[i].name;
158
159 size_t tmp_jobs_started = p->per_job_type[i].worker_jobs_started;
157 - per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_jobs_started;
158 - p->per_job_type[i].statistics_jobs_started = tmp_jobs_started;
160 + per_job_type_jobs_started[i] = tmp_jobs_started - p->per_job_type[i].statistics_last_jobs_started;
161 + p->per_job_type[i].statistics_last_jobs_started = tmp_jobs_started;
162
163 usec_t tmp_busy_time = p->per_job_type[i].worker_busy_time;
161 - per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_busy_time;
162 - p->per_job_type[i].statistics_busy_time = tmp_busy_time;
164 + per_job_type_busy_time[i] = tmp_busy_time - p->per_job_type[i].statistics_last_busy_time;
165 + p->per_job_type[i].statistics_last_busy_time = tmp_busy_time;
166 }
167
168 // get a copy of the worker variables
169 + size_t worker_job_id = p->job_id;
170 usec_t worker_busy_time = p->busy_time;
171 size_t worker_jobs_started = p->jobs_started;
172 char worker_last_action = p->last_action;
173 usec_t worker_last_action_timestamp = p->last_action_timestamp;
174
175 + delta = now - p->statistics_last_checkpoint;
176 + p->statistics_last_checkpoint = now;
177 +
178 // this is the only variable both the worker thread and the statistics thread are writing
172 - // we set this only when the worker is busy, so that worker will not
179 + // we set this only when the worker is busy, so that the worker will not
180 // accumulate all the busy time, but only the time after the point we collected statistics
181 if(worker_last_action == WORKER_BUSY && p->last_action_timestamp == worker_last_action_timestamp && p->last_action == WORKER_BUSY)
182 p->last_action_timestamp = now;
@@ -186,14 +193,13 @@ void workers_foreach(const char *workname, void (*callback)(void *data, pid_t pi
193 if(worker_last_action == WORKER_BUSY) {
194 // the worker is still busy with something
195 // let's add that busy time to the reported one
189 - busy_time += now - worker_last_action_timestamp;
196 + usec_t dt = now - worker_last_action_timestamp;
197 + busy_time += dt;
198 + per_job_type_busy_time[worker_job_id] += dt;
199 + p->per_job_type[worker_job_id].statistics_last_busy_time += dt;
200 jobs_running = 1;
201 }
202
193 - delta = now - p->statistics_last_checkpoint;
194 -
195 - p->statistics_last_checkpoint = now;
196 -
203 callback(data, p->pid, p->tag, busy_time, delta, jobs_started, jobs_running, per_job_type_name, per_job_type_jobs_started, per_job_type_busy_time);
204 }
205