Ebpf extend sync (#10814)
Extend synchronization thread.
thiagoftsm committed
Mar 24, 2021 at 12:00 UTC
a714066bcc47b4a0608e7d0eb601754054d114d3
7 files changed
+282
-50
collectors/ebpf.plugin/README.md
+18
-2
@@ -198,7 +198,7 @@ The eBPF collector enables and runs the following eBPF programs by default:
198
When in `return` mode, it also creates charts showing errors when these operations are executed.
199
- `network viewer`: This eBPF program creates charts with information about `TCP` and `UDP` functions, including the
200
bandwidth consumed by each.
201
-- `sync`: Montitor calls for syscall sync(2).
201
+- `sync`: Montitor calls for syscalls sync(2), fsync(2), fdatasync(2), syncfs(2), msync(2), and sync_file_range(2).
202
203
## Thread configuration
204
@@ -218,6 +218,7 @@ The following configuration files are available:
218
- `process.conf`: Configuration for the `process` thread.
219
- `network.conf`: Configuration for the `network viewer` thread. This config file overwrites the global options and
220
also lets you specify which network the eBPF collector monitors.
221
+- `sync.conf`: Configuration for the `sync` thread.
222
223
### Network configuration
224
@@ -261,7 +262,7 @@ The dimensions for the traffic charts are created using the destination IPs of t
262
changed setting `resolve hostname ips = yes` and restarting Netdata, after this Netdata will create dimensions using
263
the `hostnames` every time that is possible to resolve IPs to their hostnames.
264
264
-### `[service name]`
265
+#### `[service name]`
266
267
Netdata uses the list of services in `/etc/services` to plot network connection charts. If this file does not contain the
268
name for a particular service you use in your infrastructure, you will need to add it to the `[service name]` section.
@@ -274,6 +275,21 @@ service in network connection charts, and thus see the name of the service inste
275
19999 = Netdata
276
```
277
278
+### Sync configuration
279
+
280
+The sync configuration has specific options to disable monitoring for syscalls, as default option all syscalls are
281
+monitored.
282
+
283
+```conf
284
+[syscalls]
285
+ sync = yes
286
+ msync = yes
287
+ fsync = yes
288
+ fdatasync = yes
289
+ syncfs = yes
290
+ sync_file_range = yes
291
+```
292
+
293
## Troubleshooting
294
295
If the eBPF collector does not work, you can troubleshoot it by running the `ebpf.plugin` command and investigating its
collectors/ebpf.plugin/ebpf.d/sync.conf
+9
@@ -12,3 +12,12 @@
12
ebpf load mode = entry
13
apps = yes
14
update every = 2
15
+
16
+# List of monitored syscalls
17
+[syscalls]
18
+ sync = yes
19
+ msync = yes
20
+ fsync = yes
21
+ fdatasync = yes
22
+ syncfs = yes
23
+ sync_file_range = yes
collectors/ebpf.plugin/ebpf_sync.c
+212
-43
@@ -5,17 +5,14 @@
5
6
static ebpf_data_t sync_data;
7
8
-static struct bpf_link **probe_links = NULL;
9
-static struct bpf_object *objects = NULL;
10
-
11
-static char *sync_counter_dimension_name[NETDATA_SYNC_END] = { "sync" };
12
-static netdata_syscall_stat_t sync_counter_aggregated_data;
13
-static netdata_publish_syscall_t sync_counter_publish_aggregated;
8
+static char *sync_counter_dimension_name[NETDATA_SYNC_IDX_END] = { "sync", "syncfs", "msync", "fsync", "fdatasync",
9
+ "sync_file_range" };
10
+static netdata_syscall_stat_t sync_counter_aggregated_data[NETDATA_SYNC_IDX_END];
11
+static netdata_publish_syscall_t sync_counter_publish_aggregated[NETDATA_SYNC_IDX_END];
12
13
static int read_thread_closed = 1;
14
17
-static int *map_fd = NULL;
18
-static netdata_idx_t sync_hash_values = 0;
15
+static netdata_idx_t sync_hash_values[NETDATA_SYNC_IDX_END];
16
17
struct netdata_static_thread sync_threads = {"SYNC KERNEL", NULL, NULL, 1,
18
NULL, NULL, NULL};
@@ -26,6 +23,60 @@ struct config sync_config = { .first_section = NULL,
23
.index = { .avl_tree = { .root = NULL, .compar = appconfig_section_compare },
24
.rwlock = AVL_LOCK_INITIALIZER } };
25
26
+ebpf_sync_syscalls_t local_syscalls[] = {
27
+ {.syscall = "sync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
28
+ {.syscall = "syncfs", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
29
+ {.syscall = "msync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
30
+ {.syscall = "fsync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
31
+ {.syscall = "fdatasync", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
32
+ {.syscall = "sync_file_range", .enabled = CONFIG_BOOLEAN_YES, .objects = NULL, .probe_links = NULL},
33
+ {.syscall = NULL, .enabled = CONFIG_BOOLEAN_NO, .objects = NULL, .probe_links = NULL}
34
+};
35
+
36
+/*****************************************************************
37
+ *
38
+ * INITIALIZE THREAD
39
+ *
40
+ *****************************************************************/
41
+
42
+/*
43
+ * Initialize Syscalls
44
+ *
45
+ * Load the eBPF programs to monitor syscalls
46
+ *
47
+ * @return 0 on success and -1 otherwise.
48
+ */
49
+static int ebpf_sync_initialize_syscall(ebpf_module_t *em)
50
+{
51
+ int i;
52
+ const char *saved_name = em->thread_name;
53
+ for (i = 0; local_syscalls[i].syscall; i++) {
54
+ ebpf_sync_syscalls_t *w = &local_syscalls[i];
55
+ if (!w->probe_links && w->enabled) {
56
+ fill_ebpf_data(&w->kernel_info);
57
+ if (ebpf_update_kernel(&w->kernel_info)) {
58
+ em->thread_name = saved_name;
59
+ error("Cannot update the kernel for eBPF module %s", w->syscall);
60
+ return -1;
61
+ }
62
+
63
+ em->thread_name = w->syscall;
64
+ w->probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &w->objects, w->kernel_info.map_fd);
65
+ if (!w->probe_links) {
66
+ em->thread_name = saved_name;
67
+ return -1;
68
+ }
69
+ }
70
+ }
71
+ em->thread_name = saved_name;
72
+
73
+ memset(sync_counter_aggregated_data, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_syscall_stat_t));
74
+ memset(sync_counter_publish_aggregated, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_publish_syscall_t));
75
+ memset(sync_hash_values, 0 , NETDATA_SYNC_IDX_END * sizeof(netdata_idx_t));
76
+
77
+ return 0;
78
+}
79
+
80
/*****************************************************************
81
*
82
* DATA THREAD
@@ -39,12 +90,16 @@ struct config sync_config = { .first_section = NULL,
90
*/
91
static void read_global_table()
92
{
42
- uint32_t idx = NETDATA_SYNC_CALL;
93
netdata_idx_t stored;
44
- int fd = map_fd[NETDATA_SYNC_GLOBLAL_TABLE];
45
-
46
- if (!bpf_map_lookup_elem(fd, &idx, &stored)) {
47
- sync_hash_values = stored;
94
+ uint32_t idx = NETDATA_SYNC_CALL;
95
+ int i;
96
+ for (i = 0; local_syscalls[i].syscall; i++) {
97
+ if (local_syscalls[i].enabled) {
98
+ int fd = local_syscalls[i].kernel_info.map_fd[NETDATA_SYNC_GLOBLAL_TABLE];
99
+ if (!bpf_map_lookup_elem(fd, &idx, &stored)) {
100
+ sync_hash_values[i] = stored;
101
+ }
102
+ }
103
}
104
}
105
@@ -78,14 +133,57 @@ void *ebpf_sync_read_hash(void *ptr)
133
}
134
135
/**
81
- * Send global
136
+ * Create Sync charts
137
+ *
138
+ * Create charts and dimensions according user input.
139
+ *
140
+ * @param id chart id
141
+ * @param idx the first index with data.
142
+ * @param end the last index with data.
143
+ */
144
+static void ebpf_send_sync_chart(char *id,
145
+ int idx,
146
+ int end)
147
+{
148
+ write_begin_chart(NETDATA_EBPF_MEMORY_GROUP, id);
149
+
150
+ netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx];
151
+
152
+ while (move && idx <= end) {
153
+ if (local_syscalls[idx].enabled)
154
+ write_chart_dimension(move->name, sync_hash_values[idx]);
155
+
156
+ move = move->next;
157
+ idx++;
158
+ }
159
+
160
+ write_end_chart();
161
+}
162
+
163
+/**
164
+ * Send data
165
*
166
* Send global charts to Netdata
167
*/
85
-static void sync_send_global()
168
+static void sync_send_data()
169
{
87
- ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_SYNC_CHART,
88
- sync_counter_publish_aggregated.dimension, sync_hash_values);
170
+ if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled) {
171
+ ebpf_send_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART, NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX);
172
+ }
173
+
174
+ if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled)
175
+ ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_MSYNC_CHART,
176
+ sync_counter_publish_aggregated[NETDATA_SYNC_MSYNC_IDX].dimension,
177
+ sync_hash_values[NETDATA_SYNC_MSYNC_IDX]);
178
+
179
+ if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled) {
180
+ ebpf_send_sync_chart(NETDATA_EBPF_SYNC_CHART, NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX);
181
+ }
182
+
183
+ if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled)
184
+ ebpf_one_dimension_write_charts(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_FILE_SEGMENT_CHART,
185
+ sync_counter_publish_aggregated[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].dimension,
186
+ sync_hash_values[NETDATA_SYNC_SYNC_FILE_RANGE_IDX]);
187
}
188
189
/**
@@ -96,8 +194,6 @@ static void sync_collector(ebpf_module_t *em)
194
sync_threads.thread = mallocz(sizeof(netdata_thread_t));
195
sync_threads.start_routine = ebpf_sync_read_hash;
196
99
- map_fd = sync_data.map_fd;
100
-
197
netdata_thread_create(sync_threads.thread, sync_threads.name, NETDATA_THREAD_OPTION_JOINABLE,
198
ebpf_sync_read_hash, em);
199
@@ -107,7 +203,7 @@ static void sync_collector(ebpf_module_t *em)
203
204
pthread_mutex_lock(&lock);
205
110
- sync_send_global();
206
+ sync_send_data();
207
208
pthread_mutex_unlock(&lock);
209
pthread_mutex_unlock(&collect_data_mutex);
@@ -121,6 +217,30 @@ static void sync_collector(ebpf_module_t *em)
217
*
218
*****************************************************************/
219
220
+/**
221
+ * Cleanup Objects
222
+ *
223
+ * Cleanup loaded objects when thread was initialized.
224
+ */
225
+void ebpf_sync_cleanup_objects()
226
+{
227
+ int i;
228
+ for (i = 0; local_syscalls[i].syscall; i++) {
229
+ ebpf_sync_syscalls_t *w = &local_syscalls[i];
230
+ if (w->probe_links) {
231
+ freez(w->kernel_info.map_fd);
232
+
233
+ struct bpf_program *prog;
234
+ size_t j = 0 ;
235
+ bpf_object__for_each_program(prog, w->objects) {
236
+ bpf_link__destroy(w->probe_links[j]);
237
+ j++;
238
+ }
239
+ bpf_object__close(w->objects);
240
+ }
241
+ }
242
+}
243
+
244
/**
245
* Clean up the main thread.
246
*
@@ -140,15 +260,8 @@ static void ebpf_sync_cleanup(void *ptr)
260
UNUSED(dt);
261
}
262
263
+ ebpf_sync_cleanup_objects();
264
freez(sync_threads.thread);
144
-
145
- struct bpf_program *prog;
146
- size_t i = 0 ;
147
- bpf_object__for_each_program(prog, objects) {
148
- bpf_link__destroy(probe_links[i]);
149
- i++;
150
- }
151
- bpf_object__close(objects);
265
}
266
267
/*****************************************************************
@@ -157,6 +270,37 @@ static void ebpf_sync_cleanup(void *ptr)
270
*
271
*****************************************************************/
272
273
+/**
274
+ * Create Sync charts
275
+ *
276
+ * Create charts and dimensions according user input.
277
+ *
278
+ * @param id chart id
279
+ * @param title chart title
280
+ * @param order order number of the specified chart
281
+ * @param idx the first index with data.
282
+ * @param end the last index with data.
283
+ */
284
+static void ebpf_create_sync_chart(char *id,
285
+ char *title,
286
+ int order,
287
+ int idx,
288
+ int end)
289
+{
290
+ ebpf_write_chart_cmd(NETDATA_EBPF_MEMORY_GROUP, id, title, EBPF_COMMON_DIMENSION_CALL,
291
+ NETDATA_EBPF_SYNC_SUBMENU, NETDATA_EBPF_CHART_TYPE_LINE, NULL, order);
292
+
293
+ netdata_publish_syscall_t *move = &sync_counter_publish_aggregated[idx];
294
+
295
+ while (move && idx <= end) {
296
+ if (local_syscalls[idx].enabled)
297
+ ebpf_write_global_dimension(move->name, move->dimension, move->algorithm);
298
+
299
+ move = move->next;
300
+ idx++;
301
+ }
302
+}
303
+
304
/**
305
* Create global charts
306
*
@@ -164,11 +308,39 @@ static void ebpf_sync_cleanup(void *ptr)
308
*/
309
static void ebpf_create_sync_charts()
310
{
167
- ebpf_create_chart(NETDATA_EBPF_MEMORY_GROUP, NETDATA_EBPF_SYNC_CHART,
168
- "Monitor calls for <a href=\"https://linux.die.net/man/2/sync\">sync(2)</a> syscall.",
169
- EBPF_COMMON_DIMENSION_CALL, NETDATA_EBPF_SYNC_SUBMENU, NULL,
170
- NETDATA_EBPF_CHART_TYPE_LINE, 21300,
171
- ebpf_create_global_dimension, &sync_counter_publish_aggregated, 1);
311
+ if (local_syscalls[NETDATA_SYNC_FSYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_FDATASYNC_IDX].enabled)
312
+ ebpf_create_sync_chart(NETDATA_EBPF_FILE_SYNC_CHART,
313
+ "Monitor calls for <code>fsync(2)</code> and <code>fdatasync(2)</code>.", 21300,
314
+ NETDATA_SYNC_FSYNC_IDX, NETDATA_SYNC_FDATASYNC_IDX);
315
+
316
+ if (local_syscalls[NETDATA_SYNC_MSYNC_IDX].enabled)
317
+ ebpf_create_sync_chart(NETDATA_EBPF_MSYNC_CHART,
318
+ "Monitor calls for <code>msync(2)</code>.", 21301,
319
+ NETDATA_SYNC_MSYNC_IDX, NETDATA_SYNC_MSYNC_IDX);
320
+
321
+ if (local_syscalls[NETDATA_SYNC_SYNC_IDX].enabled || local_syscalls[NETDATA_SYNC_SYNCFS_IDX].enabled)
322
+ ebpf_create_sync_chart(NETDATA_EBPF_SYNC_CHART,
323
+ "Monitor calls for <code>sync(2)</code> and <code>syncfs(2)</code>.", 21302,
324
+ NETDATA_SYNC_SYNC_IDX, NETDATA_SYNC_SYNCFS_IDX);
325
+
326
+ if (local_syscalls[NETDATA_SYNC_SYNC_FILE_RANGE_IDX].enabled)
327
+ ebpf_create_sync_chart(NETDATA_EBPF_FILE_SEGMENT_CHART,
328
+ "Monitor calls for <code>sync_file_range(2)</code>.", 21303,
329
+ NETDATA_SYNC_SYNC_FILE_RANGE_IDX, NETDATA_SYNC_SYNC_FILE_RANGE_IDX);
330
+}
331
+
332
+/**
333
+ * Parse Syscalls
334
+ *
335
+ * Parse syscall options available inside ebpf.d/sync.conf
336
+ */
337
+static void ebpf_sync_parse_syscalls()
338
+{
339
+ int i;
340
+ for (i = 0; local_syscalls[i].syscall; i++) {
341
+ local_syscalls[i].enabled = appconfig_get_boolean(&sync_config, NETDATA_SYNC_CONFIG_NAME,
342
+ local_syscalls[i].syscall, CONFIG_BOOLEAN_YES);
343
+ }
344
}
345
346
/**
@@ -188,25 +360,22 @@ void *ebpf_sync_thread(void *ptr)
360
fill_ebpf_data(&sync_data);
361
362
ebpf_update_module(em, &sync_config, NETDATA_SYNC_CONFIG_FILE);
363
+ ebpf_sync_parse_syscalls();
364
365
if (!em->enabled)
366
goto endsync;
367
195
- if (ebpf_update_kernel(&sync_data)) {
196
- pthread_mutex_unlock(&lock);
197
- goto endsync;
198
- }
199
-
200
- probe_links = ebpf_load_program(ebpf_plugin_dir, em, kernel_string, &objects, sync_data.map_fd);
201
- if (!probe_links) {
368
+ if (ebpf_sync_initialize_syscall(em)) {
369
pthread_mutex_unlock(&lock);
370
goto endsync;
371
}
372
206
- int algorithm = NETDATA_EBPF_INCREMENTAL_IDX;
207
- ebpf_global_labels(&sync_counter_aggregated_data, &sync_counter_publish_aggregated,
373
+ int algorithms[NETDATA_SYNC_IDX_END] = { NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,
374
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX,
375
+ NETDATA_EBPF_INCREMENTAL_IDX, NETDATA_EBPF_INCREMENTAL_IDX };
376
+ ebpf_global_labels(sync_counter_aggregated_data, sync_counter_publish_aggregated,
377
sync_counter_dimension_name, sync_counter_dimension_name,
209
- &algorithm, NETDATA_SYNC_END);
378
+ algorithms, NETDATA_SYNC_IDX_END);
379
380
pthread_mutex_lock(&lock);
381
ebpf_create_sync_charts();
collectors/ebpf.plugin/ebpf_sync.h
+26
@@ -5,12 +5,38 @@
5
6
// charts
7
#define NETDATA_EBPF_SYNC_CHART "sync"
8
+#define NETDATA_EBPF_MSYNC_CHART "memory_map"
9
+#define NETDATA_EBPF_FILE_SYNC_CHART "file_sync"
10
+#define NETDATA_EBPF_FILE_SEGMENT_CHART "file_segment"
11
#define NETDATA_EBPF_SYNC_SUBMENU "synchronization (eBPF)"
12
13
#define NETDATA_EBPF_SYNC_SLEEP_MS 800000ULL
14
15
// configuration file
16
#define NETDATA_SYNC_CONFIG_FILE "sync.conf"
17
+#define NETDATA_SYNC_CONFIG_NAME "syscalls"
18
+
19
+enum sync_syscalls_index {
20
+ NETDATA_SYNC_SYNC_IDX,
21
+ NETDATA_SYNC_SYNCFS_IDX,
22
+ NETDATA_SYNC_MSYNC_IDX,
23
+ NETDATA_SYNC_FSYNC_IDX,
24
+ NETDATA_SYNC_FDATASYNC_IDX,
25
+ NETDATA_SYNC_SYNC_FILE_RANGE_IDX,
26
+
27
+ NETDATA_SYNC_IDX_END
28
+};
29
+
30
+typedef struct ebpf_sync_syscalls {
31
+ char *syscall;
32
+ int enabled;
33
+ uint32_t flags;
34
+
35
+ struct bpf_object *objects;
36
+ struct bpf_link **probe_links;
37
+
38
+ ebpf_data_t kernel_info;
39
+} ebpf_sync_syscalls_t;
40
41
enum netdata_sync_charts {
42
NETDATA_SYNC_CALL,
packaging/ebpf.checksums
+3
-3
@@ -1,3 +1,3 @@
1
-fd93c6cda92c1eda6bedb384dd79095bb1ba7e87493153ec455efa9cb6365b48 netdata-kernel-collector-glibc-v0.6.0.tar.xz
2
-a19e776eb714f2ed08eb4e43bfd9798a06fad8f1b15421e18b22a36583f41a6f netdata-kernel-collector-musl-v0.6.0.tar.xz
3
-582b2c5dec077266c5f993f3a735cc46f09448cd6b24e6397daafde97616b113 netdata-kernel-collector-static-v0.6.0.tar.xz
1
+380e31fe143e7b53bcebaaf03a04d143ae82e13318b264461ebb5d3ac9026ae5 netdata-kernel-collector-glibc-v0.6.1.tar.xz
2
+5a196ab8a00d307a4f6a5c213178bd62e5720173f433afc6e77dfa911fb6ca56 netdata-kernel-collector-musl-v0.6.1.tar.xz
3
+683e6676c1eee0cd4a7da5be953e94052e780de1ca375146a488d62593220c46 netdata-kernel-collector-static-v0.6.1.tar.xz
packaging/ebpf.version
+1
-1
@@ -1 +1 @@
1
-v0.6.0
1
+v0.6.1
web/gui/dashboard_info.js
+13
-1
@@ -1035,7 +1035,19 @@ netdataDashboard.context = {
1035
},
1036
1037
'mem.sync': {
1038
- info: 'System calls for <code>sync()</code> which flushes file system buffers to storage devices. These calls can cause performance perturbations, and it can be useful to know if they are happening and how frequently. Based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/syncsnoop.py" target="_blank">syncsnoop</a> from BCC tools.'
1038
+ info: 'System calls for <a href="https://man7.org/linux/man-pages/man2/sync.2.html" target="_blank">sync() and syncfs()</a> which flush the file system buffers to storage devices. Performance perturbations might be caused by these calls. The <code>sync()</code> calls are based on the eBPF <a href="https://github.com/iovisor/bcc/blob/master/tools/syncsnoop.py" target="_blank">syncsnoop</a> from BCC tools.'
1039
+ },
1040
+
1041
+ 'mem.file_sync': {
1042
+ info: 'System calls for <a href="https://man7.org/linux/man-pages/man2/fsync.2.html" target="_blank">fsync() and fdatasync()</a> transfer all modified page caches for the files on disk devices. These calls block until the device reports that the transfer has been completed.'
1043
+ },
1044
+
1045
+ 'mem.memory_map': {
1046
+ info: 'System calls for <a href="https://man7.org/linux/man-pages/man2/msync.2.html" target="_blank">msync()</a> which flushes changes made to the in-core copy of a file that was mapped.'
1047
+ },
1048
+
1049
+ 'mem.file_segment': {
1050
+ info: 'System calls for <a href="https://man7.org/linux/man-pages/man2/sync_file_range.2.html" target="_blank">sync_file_range()</a> permits fine control when synchronizing the open file referred to by the file descriptor fd with disk. This system call is extremely dangerous and should not be used in portable programs.'
1051
},
1052
1053
// ------------------------------------------------------------------------