eBPF Soft IRQ latency (#11445)
Uman Shahzad committed
Aug 24, 2021 at 20:24 UTC
b2f9d9281bd1878bb56cb9fc4c2da83218e196b4
15 files changed
+369
-9
CMakeLists.txt
+2
@@ -504,6 +504,8 @@ set(EBPF_PROCESS_PLUGIN_FILES
504
collectors/ebpf.plugin/ebpf_process.h
505
collectors/ebpf.plugin/ebpf_socket.c
506
collectors/ebpf.plugin/ebpf_socket.h
507
+ collectors/ebpf.plugin/ebpf_softirq.c
508
+ collectors/ebpf.plugin/ebpf_softirq.h
509
collectors/ebpf.plugin/ebpf_sync.c
510
collectors/ebpf.plugin/ebpf_sync.h
511
collectors/ebpf.plugin/ebpf_swap.c
Makefile.am
+2
@@ -308,6 +308,8 @@ EBPF_PLUGIN_FILES = \
308
collectors/ebpf.plugin/ebpf_process.h \
309
collectors/ebpf.plugin/ebpf_socket.c \
310
collectors/ebpf.plugin/ebpf_socket.h \
311
+ collectors/ebpf.plugin/ebpf_softirq.c \
312
+ collectors/ebpf.plugin/ebpf_softirq.h \
313
collectors/ebpf.plugin/ebpf_sync.c \
314
collectors/ebpf.plugin/ebpf_sync.h \
315
collectors/ebpf.plugin/ebpf_swap.c \
collectors/ebpf.plugin/Makefile.am
+1
@@ -41,6 +41,7 @@ dist_ebpfconfig_DATA = \
41
ebpf.d/mount.conf \
42
ebpf.d/network.conf \
43
ebpf.d/process.conf \
44
+ ebpf.d/softirq.conf \
45
ebpf.d/sync.conf \
46
ebpf.d/swap.conf \
47
ebpf.d/vfs.conf \
collectors/ebpf.plugin/README.md
+3
@@ -320,6 +320,8 @@ The eBPF collector enables and runs the following eBPF programs by default:
320
When in `return` mode, it also creates charts showing errors when these operations are executed.
321
- `hardirq`: This eBPF program creates charts that show information about
322
time spent servicing individual hardware interrupt requests (hard IRQs).
323
+- `softirq`: This eBPF program creates charts that show information about
324
+ time spent servicing individual software interrupt requests (soft IRQs).
325
326
You can also enable the following eBPF programs:
327
- `cachestat`: Netdata's eBPF data collector creates charts about the memory page cache. When the integration with
@@ -355,6 +357,7 @@ The following configuration files are available:
357
- `process.conf`: Configuration for the `process` thread.
358
- `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and
359
also lets you specify which network the eBPF collector monitors.
360
+- `softirq.conf`: Configuration for the `softirq` thread.
361
- `sync.conf`: Configuration for the `sync` thread.
362
- `vfs.conf`: Configuration for the `vfs` thread.
363
collectors/ebpf.plugin/ebpf.c
+26
-1
@@ -135,6 +135,11 @@ ebpf_module_t ebpf_modules[] = {
135
.optional = 0, .apps_routine = NULL, .maps = NULL,
136
.pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &hardirq_config,
137
.config_file = NETDATA_HARDIRQ_CONFIG_FILE},
138
+ { .thread_name = "softirq", .config_name = "softirq", .enabled = 0, .start_routine = ebpf_softirq_thread,
139
+ .update_time = 1, .global_charts = 1, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
140
+ .optional = 0, .apps_routine = NULL, .maps = NULL,
141
+ .pid_map_size = ND_EBPF_DEFAULT_PID_SIZE, .names = NULL, .cfg = &softirq_config,
142
+ .config_file = NETDATA_SOFTIRQ_CONFIG_FILE},
143
{ .thread_name = NULL, .enabled = 0, .start_routine = NULL, .update_time = 1,
144
.global_charts = 0, .apps_charts = CONFIG_BOOLEAN_NO, .mode = MODE_ENTRY,
145
.optional = 0, .apps_routine = NULL, .maps = NULL, .pid_map_size = 0, .names = NULL,
@@ -729,6 +734,8 @@ void ebpf_print_help()
734
"\n"
735
" --return or -r Run the collector in return mode.\n"
736
"\n",
737
+ " --softirq or -t Enable chart related to soft IRQ latency.\n"
738
+ "\n"
739
" --sync or -s Enable chart related to sync run time.\n"
740
"\n"
741
" --swap or -w Enable chart related to swap run time.\n"
@@ -1154,6 +1161,13 @@ static void read_collector_values(int *disable_apps)
1161
started++;
1162
}
1163
1164
+ enabled = appconfig_get_boolean(&collector_config, EBPF_PROGRAMS_SECTION, "softirq",
1165
+ CONFIG_BOOLEAN_YES);
1166
+ if (enabled) {
1167
+ ebpf_enable_chart(EBPF_MODULE_SOFTIRQ_IDX, *disable_apps);
1168
+ started++;
1169
+ }
1170
+
1171
if (!started){
1172
ebpf_enable_all_charts(*disable_apps);
1173
// Read network viewer section
@@ -1246,6 +1260,7 @@ static void parse_args(int argc, char **argv)
1260
{"net", no_argument, 0, 'n' },
1261
{"process", no_argument, 0, 'p' },
1262
{"return", no_argument, 0, 'r' },
1263
+ {"softirq", no_argument, 0, 't' },
1264
{"sync", no_argument, 0, 's' },
1265
{"swap", no_argument, 0, 'w' },
1266
{"vfs", no_argument, 0, 'f' },
@@ -1263,7 +1278,7 @@ static void parse_args(int argc, char **argv)
1278
}
1279
1280
while (1) {
1266
- int c = getopt_long(argc, argv, "hvgacdkieqmnprswf", long_options, &option_index);
1281
+ int c = getopt_long(argc, argv, "hvgacdkieqmnprtswf", long_options, &option_index);
1282
if (c == -1)
1283
break;
1284
@@ -1371,6 +1386,14 @@ static void parse_args(int argc, char **argv)
1386
ebpf_set_thread_mode(MODE_RETURN);
1387
#ifdef NETDATA_INTERNAL_CHECKS
1388
info("EBPF running in \"return\" mode, because it was started with the option \"--return\" or \"-r\".");
1389
+#endif
1390
+ break;
1391
+ }
1392
+ case 't': {
1393
+ enabled = 1;
1394
+ ebpf_enable_chart(EBPF_MODULE_SOFTIRQ_IDX, disable_apps);
1395
+#ifdef NETDATA_INTERNAL_CHECKS
1396
+ info("EBPF enabling \"softirq\" chart, because it was started with the option \"--softirq\" or \"-t\".");
1397
#endif
1398
break;
1399
}
@@ -1680,6 +1703,8 @@ int main(int argc, char **argv)
1703
NULL, NULL, ebpf_modules[EBPF_MODULE_FD_IDX].start_routine},
1704
{"EBPF HARDIRQ" , NULL, NULL, 1,
1705
NULL, NULL, ebpf_modules[EBPF_MODULE_HARDIRQ_IDX].start_routine},
1706
+ {"EBPF SOFTIRQ" , NULL, NULL, 1,
1707
+ NULL, NULL, ebpf_modules[EBPF_MODULE_SOFTIRQ_IDX].start_routine},
1708
{NULL , NULL, NULL, 0,
1709
NULL, NULL, NULL}
1710
};
collectors/ebpf.plugin/ebpf.d.conf
+2
@@ -35,6 +35,7 @@
35
# `process` : This eBPF program creates charts that show information about process life.
36
# `socket` : This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
37
# bandwidth consumed by each.
38
+# `softirq` : Monitor latency of serving software interrupt requests (soft IRQs).
39
# `sync` : Montitor calls for syscall sync(2).
40
# `swap` : Monitor calls for internal swap functions.
41
# `vfs` : This eBPF program creates charts that show information about process VFS IO, VFS file manipulation and
@@ -49,6 +50,7 @@
50
mount = yes
51
process = yes
52
socket = yes
53
+ softirq = yes
54
sync = yes
55
swap = no
56
vfs = yes
collectors/ebpf.plugin/ebpf.d/softirq.conf
new
+8
@@ -0,0 +1,8 @@
1
+# The `ebpf load mode` option accepts the following values :
2
+# `entry` : The eBPF collector only monitors calls for the functions, and does not show charts related to errors.
3
+# `return : In the `return` mode, the eBPF collector monitors the same kernel functions as `entry`, but also creates
4
+# new charts for the return of these functions, such as errors.
5
+#
6
+[global]
7
+ ebpf load mode = entry
8
+ update every = 1
collectors/ebpf.plugin/ebpf.h
+3
-1
@@ -85,7 +85,8 @@ enum ebpf_module_indexes {
85
EBPF_MODULE_DISK_IDX,
86
EBPF_MODULE_MOUNT_IDX,
87
EBPF_MODULE_FD_IDX,
88
- EBPF_MODULE_HARDIRQ_IDX
88
+ EBPF_MODULE_HARDIRQ_IDX,
89
+ EBPF_MODULE_SOFTIRQ_IDX
90
};
91
92
typedef struct ebpf_tracepoint {
@@ -223,6 +224,7 @@ extern uint32_t ebpf_enable_tracepoints(ebpf_tracepoint_t *tps);
224
#define EBPF_COMMON_DIMENSION_DIFFERENCE "difference"
225
#define EBPF_COMMON_DIMENSION_PACKETS "packets"
226
#define EBPF_COMMON_DIMENSION_FILES "files"
227
+#define EBPF_COMMON_DIMENSION_MILLISECONDS "milliseconds"
228
229
// Common variables
230
extern int debug_enabled;
collectors/ebpf.plugin/ebpf_apps.h
+2
-1
@@ -23,12 +23,13 @@
23
#include "ebpf_disk.h"
24
#include "ebpf_fd.h"
25
#include "ebpf_filesystem.h"
26
+#include "ebpf_hardirq.h"
27
#include "ebpf_cachestat.h"
28
#include "ebpf_mount.h"
29
+#include "ebpf_softirq.h"
30
#include "ebpf_sync.h"
31
#include "ebpf_swap.h"
32
#include "ebpf_vfs.h"
31
-#include "ebpf_hardirq.h"
33
34
#define MAX_COMPARE_NAME 100
35
#define MAX_NAME 100
collectors/ebpf.plugin/ebpf_hardirq.c
+1
-1
@@ -344,7 +344,7 @@ static void hardirq_create_charts()
344
NETDATA_EBPF_SYSTEM_GROUP,
345
"hardirq_latency",
346
"Hardware IRQ latency",
347
- "milliseconds",
347
+ EBPF_COMMON_DIMENSION_MILLISECONDS,
348
"interrupts",
349
NULL,
350
NETDATA_EBPF_CHART_TYPE_STACKED,
collectors/ebpf.plugin/ebpf_softirq.c
new
+275
@@ -0,0 +1,275 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#include "ebpf.h"
4
+#include "ebpf_softirq.h"
5
+
6
+struct config softirq_config = { .first_section = NULL,
7
+ .last_section = NULL,
8
+ .mutex = NETDATA_MUTEX_INITIALIZER,
9
+ .index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
10
+ .rwlock = AVL_LOCK_INITIALIZER } };
11
+
12
+#define SOFTIRQ_MAP_LATENCY 0
13
+static ebpf_local_maps_t softirq_maps[] = {
14
+ {
15
+ .name = "tbl_softirq",
16
+ .internal_input = NETDATA_SOFTIRQ_MAX_IRQS,
17
+ .user_input = 0,
18
+ .type = NETDATA_EBPF_MAP_STATIC,
19
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
20
+ },
21
+ /* end */
22
+ {
23
+ .name = NULL,
24
+ .internal_input = 0,
25
+ .user_input = 0,
26
+ .type = NETDATA_EBPF_MAP_CONTROLLER,
27
+ .map_fd = ND_EBPF_MAP_FD_NOT_INITIALIZED
28
+ }
29
+};
30
+
31
+static ebpf_data_t softirq_data;
32
+
33
+#define SOFTIRQ_TP_CLASS_IRQ "irq"
34
+static ebpf_tracepoint_t softirq_tracepoints[] = {
35
+ {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_entry"},
36
+ {.enabled = false, .class = SOFTIRQ_TP_CLASS_IRQ, .event = "softirq_exit"},
37
+ /* end */
38
+ {.enabled = false, .class = NULL, .event = NULL}
39
+};
40
+
41
+// these must be in the order defined by the kernel:
42
+// https://elixir.bootlin.com/linux/v5.12.19/source/include/trace/events/irq.h#L13
43
+static softirq_val_t softirq_vals[] = {
44
+ {.name = "HI", .latency = 0},
45
+ {.name = "TIMER", .latency = 0},
46
+ {.name = "NET_TX", .latency = 0},
47
+ {.name = "NET_RX", .latency = 0},
48
+ {.name = "BLOCK", .latency = 0},
49
+ {.name = "IRQ_POLL", .latency = 0},
50
+ {.name = "TASKLET", .latency = 0},
51
+ {.name = "SCHED", .latency = 0},
52
+ {.name = "HRTIMER", .latency = 0},
53
+ {.name = "RCU", .latency = 0},
54
+};
55
+
56
+// tmp store for soft IRQ values we get from a per-CPU eBPF map.
57
+static softirq_ebpf_val_t *softirq_ebpf_vals = NULL;
58
+
59
+static struct bpf_link **probe_links = NULL;
60
+static struct bpf_object *objects = NULL;
61
+
62
+static int read_thread_closed = 1;
63
+
64
+static struct netdata_static_thread softirq_threads = {"SOFTIRQ KERNEL",
65
+ NULL, NULL, 1, NULL,
66
+ NULL, NULL };
67
+
68
+/**
69
+ * Clean up the main thread.
70
+ *
71
+ * @param ptr thread data.
72
+ */
73
+static void softirq_cleanup(void *ptr)
74
+{
75
+ for (int i = 0; softirq_tracepoints[i].class != NULL; i++) {
76
+ ebpf_disable_tracepoint(&softirq_tracepoints[i]);
77
+ }
78
+
79
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
80
+ if (!em->enabled) {
81
+ return;
82
+ }
83
+
84
+ heartbeat_t hb;
85
+ heartbeat_init(&hb);
86
+ uint32_t tick = 1 * USEC_PER_MS;
87
+ while (!read_thread_closed) {
88
+ usec_t dt = heartbeat_next(&hb, tick);
89
+ UNUSED(dt);
90
+ }
91
+
92
+ freez(softirq_ebpf_vals);
93
+ freez(softirq_threads.thread);
94
+
95
+ if (probe_links) {
96
+ struct bpf_program *prog;
97
+ size_t i = 0 ;
98
+ bpf_object__for_each_program(prog, objects) {
99
+ bpf_link__destroy(probe_links[i]);
100
+ i++;
101
+ }
102
+ bpf_object__close(objects);
103
+ }
104
+}
105
+
106
+/*****************************************************************
107
+ * MAIN LOOP
108
+ *****************************************************************/
109
+
110
+static void softirq_read_latency_map()
111
+{
112
+ int fd = softirq_maps[SOFTIRQ_MAP_LATENCY].map_fd;
113
+ int i;
114
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
115
+ int test = bpf_map_lookup_elem(fd, &i, softirq_ebpf_vals);
116
+ if (unlikely(test < 0)) {
117
+ continue;
118
+ }
119
+
120
+ uint64_t total_latency = 0;
121
+ int cpu_i;
122
+ int end = ebpf_nprocs;
123
+ for (cpu_i = 0; cpu_i < end; cpu_i++) {
124
+ total_latency += softirq_ebpf_vals[cpu_i].latency/1000;
125
+ }
126
+
127
+ softirq_vals[i].latency = total_latency;
128
+ }
129
+}
130
+
131
+/**
132
+ * Read eBPF maps for soft IRQ.
133
+ */
134
+static void *softirq_reader(void *ptr)
135
+{
136
+ read_thread_closed = 0;
137
+
138
+ heartbeat_t hb;
139
+ heartbeat_init(&hb);
140
+
141
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
142
+
143
+ usec_t step = NETDATA_SOFTIRQ_SLEEP_MS * em->update_time;
144
+ while (!close_ebpf_plugin) {
145
+ usec_t dt = heartbeat_next(&hb, step);
146
+ UNUSED(dt);
147
+
148
+ softirq_read_latency_map();
149
+ }
150
+
151
+ read_thread_closed = 1;
152
+ return NULL;
153
+}
154
+
155
+static void softirq_create_charts()
156
+{
157
+ ebpf_create_chart(
158
+ NETDATA_EBPF_SYSTEM_GROUP,
159
+ "softirq_latency",
160
+ "Software IRQ latency",
161
+ EBPF_COMMON_DIMENSION_MILLISECONDS,
162
+ "softirqs",
163
+ NULL,
164
+ NETDATA_EBPF_CHART_TYPE_STACKED,
165
+ NETDATA_CHART_PRIO_SYSTEM_SOFTIRQS+1,
166
+ NULL, NULL, 0,
167
+ NETDATA_EBPF_MODULE_NAME_SOFTIRQ
168
+ );
169
+
170
+ fflush(stdout);
171
+}
172
+
173
+static void softirq_create_dims()
174
+{
175
+ uint32_t i;
176
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
177
+ ebpf_write_global_dimension(
178
+ softirq_vals[i].name, softirq_vals[i].name,
179
+ ebpf_algorithms[NETDATA_EBPF_INCREMENTAL_IDX]
180
+ );
181
+ }
182
+}
183
+
184
+static inline void softirq_write_dims()
185
+{
186
+ uint32_t i;
187
+ for (i = 0; i < NETDATA_SOFTIRQ_MAX_IRQS; i++) {
188
+ write_chart_dimension(softirq_vals[i].name, softirq_vals[i].latency);
189
+ }
190
+}
191
+
192
+/**
193
+* Main loop for this collector.
194
+*/
195
+static void softirq_collector(ebpf_module_t *em)
196
+{
197
+ softirq_ebpf_vals = callocz(ebpf_nprocs, sizeof(softirq_ebpf_val_t));
198
+
199
+ // create reader thread.
200
+ softirq_threads.thread = mallocz(sizeof(netdata_thread_t));
201
+ softirq_threads.start_routine = softirq_reader;
202
+ netdata_thread_create(
203
+ softirq_threads.thread,
204
+ softirq_threads.name,
205
+ NETDATA_THREAD_OPTION_JOINABLE,
206
+ softirq_reader,
207
+ em
208
+ );
209
+
210
+ // create chart and static dims.
211
+ pthread_mutex_lock(&lock);
212
+ softirq_create_charts();
213
+ softirq_create_dims();
214
+ pthread_mutex_unlock(&lock);
215
+
216
+ // loop and read from published data until ebpf plugin is closed.
217
+ while (!close_ebpf_plugin) {
218
+ pthread_mutex_lock(&collect_data_mutex);
219
+ pthread_cond_wait(&collect_data_cond_var, &collect_data_mutex);
220
+ pthread_mutex_lock(&lock);
221
+
222
+ // write dims now for all hitherto discovered IRQs.
223
+ write_begin_chart(NETDATA_EBPF_SYSTEM_GROUP, "softirq_latency");
224
+ softirq_write_dims();
225
+ write_end_chart();
226
+
227
+ pthread_mutex_unlock(&lock);
228
+ pthread_mutex_unlock(&collect_data_mutex);
229
+ }
230
+}
231
+
232
+/*****************************************************************
233
+ * EBPF SOFTIRQ THREAD
234
+ *****************************************************************/
235
+
236
+/**
237
+ * Soft IRQ latency thread.
238
+ *
239
+ * @param ptr a `ebpf_module_t *`.
240
+ * @return always NULL.
241
+ */
242
+void *ebpf_softirq_thread(void *ptr)
243
+{
244
+ netdata_thread_cleanup_push(softirq_cleanup, ptr);
245
+
246
+ ebpf_module_t *em = (ebpf_module_t *)ptr;
247
+ em->maps = softirq_maps;
248
+
249
+ fill_ebpf_data(&softirq_data);
250
+
251
+ if (!em->enabled) {
252
+ goto endsoftirq;
253
+ }
254
+
255
+ if (ebpf_update_kernel(&softirq_data)) {
256
+ goto endsoftirq;
257
+ }
258
+
259
+ if (ebpf_enable_tracepoints(softirq_tracepoints) == 0) {
260
+ em->enabled = CONFIG_BOOLEAN_NO;
261
+ goto endsoftirq;
262
+ }
263
+
264
+ probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, softirq_data.map_fd);
265
+ if (!probe_links) {
266
+ goto endsoftirq;
267
+ }
268
+
269
+ softirq_collector(em);
270
+
271
+endsoftirq:
272
+ netdata_thread_cleanup_pop(1);
273
+
274
+ return NULL;
275
+}
collectors/ebpf.plugin/ebpf_softirq.h
new
+35
@@ -0,0 +1,35 @@
1
+// SPDX-License-Identifier: GPL-3.0-or-later
2
+
3
+#ifndef NETDATA_EBPF_SOFTIRQ_H
4
+#define NETDATA_EBPF_SOFTIRQ_H 1
5
+
6
+/*****************************************************************
7
+ * copied from kernel-collectors repo, with modifications needed
8
+ * for inclusion here.
9
+ *****************************************************************/
10
+
11
+#define NETDATA_SOFTIRQ_MAX_IRQS 10
12
+
13
+typedef struct softirq_ebpf_val {
14
+ uint64_t latency;
15
+ uint64_t ts;
16
+} softirq_ebpf_val_t;
17
+
18
+/*****************************************************************
19
+ * below this is eBPF plugin-specific code.
20
+ *****************************************************************/
21
+
22
+#define NETDATA_EBPF_MODULE_NAME_SOFTIRQ "softirq"
23
+#define NETDATA_SOFTIRQ_SLEEP_MS 650000ULL
24
+#define NETDATA_SOFTIRQ_CONFIG_FILE "softirq.conf"
25
+
26
+typedef struct sofirq_val {
27
+ uint64_t latency;
28
+ char *name;
29
+} softirq_val_t;
30
+
31
+extern struct config softirq_config;
32
+extern void *ebpf_softirq_thread(void *ptr);
33
+extern void ebpf_softirq_create_apps_charts(struct ebpf_module *em, void *ptr);
34
+
35
+#endif /* NETDATA_EBPF_SOFTIRQ_H */
packaging/ebpf.checksums
+3
-3
@@ -1,4 +1,4 @@
1
-a56eece7b09b795a3a75e317baff0bc46b3207a0a5535c179dc4bce6a144de36 netdata-kernel-collector-glibc-v0.7.8.1.tar.xz
2
-5e98ed15ab8ba667065f17f9722f4cf86f2932e454015c2a5c169ccb950bbb33 netdata-kernel-collector-musl-v0.7.8.1.tar.xz
3
-d93dff80d0b0e41478f734f44edfdb1bf1cfbd57db4c9f4fc34d703f436a77a6 netdata-kernel-collector-static-v0.7.8.1.tar.xz
1
+cb7079f209d34b00a4d8e1e7a228bc37b736d539b36a0649dcfda9743c64ddff netdata-kernel-collector-glibc-v0.7.9.tar.xz
2
+7832a73d6d4eeae92f38fc47fbdf77fa6c3b7442e1b3fd6cfc2b1a1df06d6895 netdata-kernel-collector-musl-v0.7.9.tar.xz
3
+620c459819363f207c8f91aec817be222859e523d3972bc5080737475b93ad40 netdata-kernel-collector-static-v0.7.9.tar.xz
4
packaging/ebpf.version
+1
-1
@@ -1,2 +1,2 @@
1
-v0.7.8.1
1
+v0.7.9
2
web/gui/dashboard_info.js
+5
-1
@@ -953,13 +953,17 @@ netdataDashboard.context = {
953
},
954
955
'system.hardirq_latency': {
956
- info: 'Total time spent servicing hardware interrupts.'
956
+ info: 'Total time spent servicing hardware interrupts. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/hardirqs_example.txt" target="_blank">hardirqs</a> from BCC tools.'
957
},
958
959
'system.softirqs': {
960
info: 'CPU softirqs in detail. At the <a href="#menu_cpu">CPUs</a> section, softirqs are analyzed per CPU core.'
961
},
962
963
+ 'system.softirq_latency': {
964
+ info: 'Total time spent servicing software interrupts. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/softirqs_example.txt" target="_blank">softirqs</a> from BCC tools.'
965
+ },
966
+
967
'system.processes': {
968
info: 'System processes. <b>Running</b> are the processes in the CPU. <b>Blocked</b> are processes that are willing to enter the CPU, but they cannot, e.g. because they wait for disk activity.'
969
},