cpu stats per query thread (#10634)
* cpu stats per query thread * remove dim_name * limit calls to getrusage to MAX_GETRUSAGE_CALLS_PER_TICK per tick, per thread * proper parenthesis * use proper limits
Emmanuel Vasilakis committed
Feb 25, 2021 at 15:05 UTC
dbd1624837b9afb64111602431ece417d606b921
4 files changed
+56
aclk/legacy/aclk_query.c
+6
@@ -629,6 +629,12 @@ static int aclk_process_query(struct aclk_query_thread *t_info)
629
aclk_metrics_per_sample.queries_dispatched++;
630
aclk_queries_per_thread[t_info->idx]++;
631
ACLK_STATS_UNLOCK;
632
+
633
+ if (likely(getrusage_called_this_tick[t_info->idx] < MAX_GETRUSAGE_CALLS_PER_TICK)) {
634
+ getrusage(RUSAGE_THREAD, &rusage_per_thread[t_info->idx]);
635
+ getrusage_called_this_tick[t_info->idx]++;
636
+ }
637
+
638
}
639
640
aclk_query_free(this_query);
aclk/legacy/aclk_query.h
+3
@@ -8,8 +8,11 @@
8
9
#define ACLK_STABLE_TIMEOUT 3 // Minimum delay to mark AGENT as stable
10
11
+#define MAX_GETRUSAGE_CALLS_PER_TICK 5 // Maximum number of times getrusage can be called per tick, per thread.
12
+
13
extern pthread_cond_t query_cond_wait;
14
extern pthread_mutex_t query_lock_wait;
15
+extern uint8_t *getrusage_called_this_tick;
16
#define QUERY_THREAD_WAKEUP pthread_cond_signal(&query_cond_wait)
17
#define QUERY_THREAD_WAKEUP_ALL pthread_cond_broadcast(&query_cond_wait)
18
aclk/legacy/aclk_stats.c
+46
@@ -11,8 +11,17 @@ struct aclk_qt_data {
11
RRDDIM *dim;
12
} *aclk_qt_data = NULL;
13
14
+// ACLK per query thread cpu stats
15
+struct aclk_cpu_data {
16
+ RRDDIM *user;
17
+ RRDDIM *system;
18
+ RRDSET *st;
19
+} *aclk_cpu_data = NULL;
20
+
21
uint32_t *aclk_queries_per_thread = NULL;
22
uint32_t *aclk_queries_per_thread_sample = NULL;
23
+struct rusage *rusage_per_thread;
24
+uint8_t *getrusage_called_this_tick = NULL;
25
26
struct aclk_metrics aclk_metrics = {
27
.online = 0,
@@ -222,11 +231,42 @@ static void aclk_stats_mat_metric_process(struct aclk_metric_mat *metric, struct
231
rrdset_done(metric->st);
232
}
233
234
+static void aclk_stats_cpu_threads(void)
235
+{
236
+ char id[100 + 1];
237
+ char title[100 + 1];
238
+
239
+ for (int i = 0; i < query_thread_count; i++) {
240
+ if (unlikely(!aclk_cpu_data[i].st)) {
241
+
242
+ snprintfz(id, 100, "aclk_thread%d_cpu", i);
243
+ snprintfz(title, 100, "Cpu Usage For Thread No %d", i);
244
+
245
+ aclk_cpu_data[i].st = rrdset_create_localhost(
246
+ "netdata", id, NULL, "aclk", NULL, title, "milliseconds/s",
247
+ "netdata", "stats", 200008 + i, localhost->rrd_update_every, RRDSET_TYPE_STACKED);
248
+
249
+ aclk_cpu_data[i].user = rrddim_add(aclk_cpu_data[i].st, "user", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
250
+ aclk_cpu_data[i].system = rrddim_add(aclk_cpu_data[i].st, "system", NULL, 1, 1000, RRD_ALGORITHM_INCREMENTAL);
251
+
252
+ } else
253
+ rrdset_next(aclk_cpu_data[i].st);
254
+ }
255
+
256
+ for (int i = 0; i < query_thread_count; i++) {
257
+ rrddim_set_by_pointer(aclk_cpu_data[i].st, aclk_cpu_data[i].user, rusage_per_thread[i].ru_utime.tv_sec * 1000000ULL + rusage_per_thread[i].ru_utime.tv_usec);
258
+ rrddim_set_by_pointer(aclk_cpu_data[i].st, aclk_cpu_data[i].system, rusage_per_thread[i].ru_stime.tv_sec * 1000000ULL + rusage_per_thread[i].ru_stime.tv_usec);
259
+ rrdset_done(aclk_cpu_data[i].st);
260
+ }
261
+}
262
+
263
void aclk_stats_thread_cleanup()
264
{
265
freez(aclk_qt_data);
266
freez(aclk_queries_per_thread);
267
freez(aclk_queries_per_thread_sample);
268
+ freez(aclk_cpu_data);
269
+ freez(rusage_per_thread);
270
}
271
272
void *aclk_stats_main_thread(void *ptr)
@@ -235,8 +275,11 @@ void *aclk_stats_main_thread(void *ptr)
275
276
query_thread_count = args->query_thread_count;
277
aclk_qt_data = callocz(query_thread_count, sizeof(struct aclk_qt_data));
278
+ aclk_cpu_data = callocz(query_thread_count, sizeof(struct aclk_cpu_data));
279
aclk_queries_per_thread = callocz(query_thread_count, sizeof(uint32_t));
280
aclk_queries_per_thread_sample = callocz(query_thread_count, sizeof(uint32_t));
281
+ rusage_per_thread = callocz(query_thread_count, sizeof(struct rusage));
282
+ getrusage_called_this_tick = callocz(query_thread_count, sizeof(uint8_t));
283
284
heartbeat_t hb;
285
heartbeat_init(&hb);
@@ -264,6 +307,7 @@ void *aclk_stats_main_thread(void *ptr)
307
308
memcpy(aclk_queries_per_thread_sample, aclk_queries_per_thread, sizeof(uint32_t) * query_thread_count);
309
memset(aclk_queries_per_thread, 0, sizeof(uint32_t) * query_thread_count);
310
+ memset(getrusage_called_this_tick, 0, sizeof(uint8_t) * query_thread_count);
311
ACLK_STATS_UNLOCK;
312
313
aclk_stats_collect(&per_sample, &permanent);
@@ -275,6 +319,8 @@ void *aclk_stats_main_thread(void *ptr)
319
aclk_stats_cloud_req(&per_sample);
320
aclk_stats_query_threads(aclk_queries_per_thread_sample);
321
322
+ aclk_stats_cpu_threads();
323
+
324
#ifdef NETDATA_INTERNAL_CHECKS
325
aclk_stats_mat_metric_process(&aclk_mat_metrics.latency, &per_sample.latency);
326
#endif
aclk/legacy/aclk_stats.h
+1
@@ -83,6 +83,7 @@ extern struct aclk_metrics_per_sample {
83
} aclk_metrics_per_sample;
84
85
extern uint32_t *aclk_queries_per_thread;
86
+extern struct rusage *rusage_per_thread;
87
88
void *aclk_stats_main_thread(void *ptr);
89
void aclk_stats_thread_cleanup();