Use a separate thread for slow mountpoints in the diskspace plugin (#13067)
Vladimir Kobal committed
Jun 15, 2022 at 12:35 UTC
f1563558810a1c0832dbf6bac9790afa9a345e3a
1 file changed
+362
-113
collectors/diskspace.plugin/plugin_diskspace.c
+362
-113
@@ -3,11 +3,17 @@
3
#include "../proc.plugin/plugin_proc.h"
4
5
#define PLUGIN_DISKSPACE_NAME "diskspace.plugin"
6
+#define THREAD_DISKSPACE_SLOW_NAME "PLUGIN[diskspace slow]"
7
8
#define DEFAULT_EXCLUDED_PATHS "/proc/* /sys/* /var/run/user/* /run/user/* /snap/* /var/lib/docker/*"
9
#define DEFAULT_EXCLUDED_FILESYSTEMS "*gvfs *gluster* *s3fs *ipfs *davfs2 *httpfs *sshfs *gdfs *moosefs fusectl autofs"
10
#define CONFIG_SECTION_DISKSPACE "plugin:proc:diskspace"
11
12
+#define MAX_STAT_USEC 10000
13
+#define SLOW_UPDATE_EVERY 5
14
+
15
+static netdata_thread_t *diskspace_slow_thread = NULL;
16
+
17
static struct mountinfo *disk_mountinfo_root = NULL;
18
static int check_for_new_mountpoints_every = 15;
19
static int cleanup_mount_points = 1;
@@ -34,6 +40,7 @@ struct mount_point_metadata {
40
int do_inodes;
41
int shown_error;
42
int updated;
43
+ int slow;
44
45
size_t collected; // the number of times this has been collected
46
@@ -52,13 +59,15 @@ static DICTIONARY *dict_mountpoints = NULL;
59
60
#define rrdset_obsolete_and_pointer_null(st) do { if(st) { rrdset_is_obsolete(st); (st) = NULL; } } while(st)
61
55
-int mount_point_cleanup(const char *name, void *entry, void *data) {
62
+int mount_point_cleanup(const char *name, void *entry, int slow) {
63
(void)name;
57
- (void)data;
58
-
64
+
65
struct mount_point_metadata *mp = (struct mount_point_metadata *)entry;
66
if(!mp) return 0;
67
68
+ if (slow != mp->slow)
69
+ return 0;
70
+
71
if(likely(mp->updated)) {
72
mp->updated = 0;
73
return 0;
@@ -84,6 +93,12 @@ int mount_point_cleanup(const char *name, void *entry, void *data) {
93
return 0;
94
}
95
96
+int mount_point_cleanup_cb(const char *name, void *entry, void *data) {
97
+ UNUSED(data);
98
+
99
+ return mount_point_cleanup(name, (struct mount_point_metadata *)entry, 0);
100
+}
101
+
102
// for the full list of protected mount points look at
103
// https://github.com/systemd/systemd/blob/1eb3ef78b4df28a9e9f464714208f2682f957e36/src/core/namespace.c#L142-L149
104
// https://github.com/systemd/systemd/blob/1eb3ef78b4df28a9e9f464714208f2682f957e36/src/core/namespace.c#L180-L194
@@ -106,12 +121,209 @@ int mount_point_is_protected(char *mount_point)
121
return 0;
122
}
123
109
-static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
124
+// a copy of basic mountinfo fields
125
+struct basic_mountinfo {
126
+ char *persistent_id;
127
+ char *root;
128
+ char *mount_point;
129
+ char *filesystem;
130
+
131
+ struct basic_mountinfo *next;
132
+};
133
+
134
+static struct basic_mountinfo *slow_mountinfo_tmp_root = NULL;
135
+static netdata_mutex_t slow_mountinfo_mutex;
136
+
137
+static struct basic_mountinfo *basic_mountinfo_create_and_copy(struct mountinfo* mi)
138
+{
139
+ struct basic_mountinfo *bmi = callocz(1, sizeof(struct basic_mountinfo));
140
+
141
+ if (mi) {
142
+ bmi->persistent_id = strdupz(mi->persistent_id);
143
+ bmi->root = strdupz(mi->root);
144
+ bmi->mount_point = strdupz(mi->mount_point);
145
+ bmi->filesystem = strdupz(mi->filesystem);
146
+ }
147
+
148
+ return bmi;
149
+}
150
+
151
+static void add_basic_mountinfo(struct basic_mountinfo **root, struct mountinfo *mi)
152
+{
153
+ if (!root)
154
+ return;
155
+
156
+ struct basic_mountinfo *bmi = basic_mountinfo_create_and_copy(mi);
157
+
158
+ bmi->next = *root;
159
+ *root = bmi;
160
+};
161
+
162
+static void free_basic_mountinfo(struct basic_mountinfo *bmi)
163
+{
164
+ if (bmi) {
165
+ freez(bmi->persistent_id);
166
+ freez(bmi->root);
167
+ freez(bmi->mount_point);
168
+ freez(bmi->filesystem);
169
+
170
+ freez(bmi);
171
+ }
172
+};
173
+
174
+static void free_basic_mountinfo_list(struct basic_mountinfo *root)
175
+{
176
+ struct basic_mountinfo *bmi = root, *next;
177
+
178
+ while (bmi) {
179
+ next = bmi->next;
180
+ free_basic_mountinfo(bmi);
181
+ bmi = next;
182
+ }
183
+}
184
+
185
+static void calculate_values_and_show_charts(
186
+ struct basic_mountinfo *mi,
187
+ struct mount_point_metadata *m,
188
+ struct statvfs *buff_statvfs,
189
+ int update_every)
190
+{
191
const char *family = mi->mount_point;
192
const char *disk = mi->persistent_id;
193
194
+ // logic found at get_fs_usage() in coreutils
195
+ unsigned long bsize = (buff_statvfs->f_frsize) ? buff_statvfs->f_frsize : buff_statvfs->f_bsize;
196
+
197
+ fsblkcnt_t bavail = buff_statvfs->f_bavail;
198
+ fsblkcnt_t btotal = buff_statvfs->f_blocks;
199
+ fsblkcnt_t bavail_root = buff_statvfs->f_bfree;
200
+ fsblkcnt_t breserved_root = bavail_root - bavail;
201
+ fsblkcnt_t bused = likely(btotal >= bavail_root) ? btotal - bavail_root : bavail_root - btotal;
202
+
203
+#ifdef NETDATA_INTERNAL_CHECKS
204
+ if(unlikely(btotal != bavail + breserved_root + bused))
205
+ error("DISKSPACE: disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused);
206
+#endif
207
+
208
+ // --------------------------------------------------------------------------
209
+
210
+ fsfilcnt_t favail = buff_statvfs->f_favail;
211
+ fsfilcnt_t ftotal = buff_statvfs->f_files;
212
+ fsfilcnt_t favail_root = buff_statvfs->f_ffree;
213
+ fsfilcnt_t freserved_root = favail_root - favail;
214
+ fsfilcnt_t fused = ftotal - favail_root;
215
+
216
+ if(m->do_inodes == CONFIG_BOOLEAN_AUTO && favail == (fsfilcnt_t)-1) {
217
+ // this file system does not support inodes reporting
218
+ // eg. cephfs
219
+ m->do_inodes = CONFIG_BOOLEAN_NO;
220
+ }
221
+
222
+#ifdef NETDATA_INTERNAL_CHECKS
223
+ if(unlikely(btotal != bavail + breserved_root + bused))
224
+ error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused);
225
+#endif
226
+
227
+ // --------------------------------------------------------------------------
228
+
229
+ int rendered = 0;
230
+
231
+ if(m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO &&
232
+ (bavail || breserved_root || bused ||
233
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
234
+ if(unlikely(!m->st_space) || m->st_space->update_every != update_every) {
235
+ m->do_space = CONFIG_BOOLEAN_YES;
236
+ m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
237
+ if(unlikely(!m->st_space || m->st_space->update_every != update_every)) {
238
+ char title[4096 + 1];
239
+ snprintfz(title, 4096, "Disk Space Usage");
240
+ m->st_space = rrdset_create_localhost(
241
+ "disk_space"
242
+ , disk
243
+ , NULL
244
+ , family
245
+ , "disk.space"
246
+ , title
247
+ , "GiB"
248
+ , PLUGIN_DISKSPACE_NAME
249
+ , NULL
250
+ , NETDATA_CHART_PRIO_DISKSPACE_SPACE
251
+ , update_every
252
+ , RRDSET_TYPE_STACKED
253
+ );
254
+ }
255
+
256
+ m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
257
+ m->rd_space_used = rrddim_add(m->st_space, "used", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
258
+ m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
259
+ }
260
+ else
261
+ rrdset_next(m->st_space);
262
+
263
+ rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number)bavail);
264
+ rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number)bused);
265
+ rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number)breserved_root);
266
+ rrdset_done(m->st_space);
267
+
268
+ rendered++;
269
+ }
270
+
271
+ // --------------------------------------------------------------------------
272
+
273
+ if(m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO &&
274
+ (favail || freserved_root || fused ||
275
+ netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
276
+ if(unlikely(!m->st_inodes) || m->st_inodes->update_every != update_every) {
277
+ m->do_inodes = CONFIG_BOOLEAN_YES;
278
+ m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
279
+ if(unlikely(!m->st_inodes) || m->st_inodes->update_every != update_every) {
280
+ char title[4096 + 1];
281
+ snprintfz(title, 4096, "Disk Files (inodes) Usage");
282
+ m->st_inodes = rrdset_create_localhost(
283
+ "disk_inodes"
284
+ , disk
285
+ , NULL
286
+ , family
287
+ , "disk.inodes"
288
+ , title
289
+ , "inodes"
290
+ , PLUGIN_DISKSPACE_NAME
291
+ , NULL
292
+ , NETDATA_CHART_PRIO_DISKSPACE_INODES
293
+ , update_every
294
+ , RRDSET_TYPE_STACKED
295
+ );
296
+ }
297
+
298
+ m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
299
+ m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
300
+ m->rd_inodes_reserved = rrddim_add(m->st_inodes, "reserved_for_root", "reserved for root", 1, 1, RRD_ALGORITHM_ABSOLUTE);
301
+ }
302
+ else
303
+ rrdset_next(m->st_inodes);
304
+
305
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number)favail);
306
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number)fused);
307
+ rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_reserved, (collected_number)freserved_root);
308
+ rrdset_done(m->st_inodes);
309
+
310
+ rendered++;
311
+ }
312
+
313
+ // --------------------------------------------------------------------------
314
+
315
+ if(likely(rendered))
316
+ m->collected++;
317
+}
318
+
319
+static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
320
+ const char *disk = mi->persistent_id;
321
+
322
static SIMPLE_PATTERN *excluded_mountpoints = NULL;
323
static SIMPLE_PATTERN *excluded_filesystems = NULL;
324
+
325
+ usec_t slow_timeout = MAX_STAT_USEC * update_every;
326
+
327
int do_space, do_inodes;
328
329
if(unlikely(!dict_mountpoints)) {
@@ -139,6 +351,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
351
352
struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
353
if(unlikely(!m)) {
354
+ int slow = 0;
355
char var_name[4096 + 1];
356
snprintfz(var_name, 4096, "plugin:proc:diskspace:%s", mi->mount_point);
357
@@ -158,7 +371,9 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
371
// check if the mount point is a directory #2407
372
// but only when it is enabled by default #4491
373
if(def_space != CONFIG_BOOLEAN_NO || def_inodes != CONFIG_BOOLEAN_NO) {
374
+ usec_t start_time = now_monotonic_high_precision_usec();
375
struct stat bs;
376
+
377
if(stat(mi->mount_point, &bs) == -1) {
378
error("DISKSPACE: Cannot stat() mount point '%s' (disk '%s', filesystem '%s', root '%s')."
379
, mi->mount_point
@@ -181,6 +396,9 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
396
def_inodes = CONFIG_BOOLEAN_NO;
397
}
398
}
399
+
400
+ if ((now_monotonic_high_precision_usec() - start_time) > slow_timeout)
401
+ slow = 1;
402
}
403
404
do_space = config_get_boolean_ondemand(var_name, "space usage", def_space);
@@ -191,6 +409,7 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
409
.do_inodes = do_inodes,
410
.shown_error = 0,
411
.updated = 0,
412
+ .slow = 0,
413
414
.collected = 0,
415
@@ -206,6 +425,13 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
425
};
426
427
m = dictionary_set(dict_mountpoints, mi->mount_point, &mp, sizeof(struct mount_point_metadata));
428
+
429
+ m->slow = slow;
430
+ }
431
+
432
+ if (m->slow) {
433
+ add_basic_mountinfo(&slow_mountinfo_tmp_root, mi);
434
+ return;
435
}
436
437
m->updated = 1;
@@ -221,7 +447,9 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
447
m->do_inodes != CONFIG_BOOLEAN_YES))
448
return;
449
450
+ usec_t start_time = now_monotonic_high_precision_usec();
451
struct statvfs buff_statvfs;
452
+
453
if (statvfs(mi->mount_point, &buff_statvfs) < 0) {
454
if(!m->shown_error) {
455
error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')"
@@ -234,135 +462,133 @@ static inline void do_disk_space_stats(struct mountinfo *mi, int update_every) {
462
}
463
return;
464
}
237
- m->shown_error = 0;
465
239
- // logic found at get_fs_usage() in coreutils
240
- unsigned long bsize = (buff_statvfs.f_frsize) ? buff_statvfs.f_frsize : buff_statvfs.f_bsize;
466
+ if ((now_monotonic_high_precision_usec() - start_time) > slow_timeout)
467
+ m->slow = 1;
468
242
- fsblkcnt_t bavail = buff_statvfs.f_bavail;
243
- fsblkcnt_t btotal = buff_statvfs.f_blocks;
244
- fsblkcnt_t bavail_root = buff_statvfs.f_bfree;
245
- fsblkcnt_t breserved_root = bavail_root - bavail;
246
- fsblkcnt_t bused;
247
- if(likely(btotal >= bavail_root))
248
- bused = btotal - bavail_root;
249
- else
250
- bused = bavail_root - btotal;
469
+ m->shown_error = 0;
470
252
-#ifdef NETDATA_INTERNAL_CHECKS
253
- if(unlikely(btotal != bavail + breserved_root + bused))
254
- error("DISKSPACE: disk block statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)btotal, (unsigned long long)bavail, (unsigned long long)breserved_root, (unsigned long long)bused);
255
-#endif
471
+ struct basic_mountinfo bmi;
472
+ bmi.mount_point = mi->mount_point;
473
+ bmi.persistent_id = mi->persistent_id;
474
+ bmi.filesystem = mi->filesystem;
475
+ bmi.root = mi->root;
476
257
- // --------------------------------------------------------------------------
477
+ calculate_values_and_show_charts(&bmi, m, &buff_statvfs, update_every);
478
+}
479
259
- fsfilcnt_t favail = buff_statvfs.f_favail;
260
- fsfilcnt_t ftotal = buff_statvfs.f_files;
261
- fsfilcnt_t favail_root = buff_statvfs.f_ffree;
262
- fsfilcnt_t freserved_root = favail_root - favail;
263
- fsfilcnt_t fused = ftotal - favail_root;
480
+static inline void do_slow_disk_space_stats(struct basic_mountinfo *mi, int update_every) {
481
+ struct mount_point_metadata *m = dictionary_get(dict_mountpoints, mi->mount_point);
482
265
- if(m->do_inodes == CONFIG_BOOLEAN_AUTO && favail == (fsfilcnt_t)-1) {
266
- // this file system does not support inodes reporting
267
- // eg. cephfs
268
- m->do_inodes = CONFIG_BOOLEAN_NO;
483
+ m->updated = 1;
484
+
485
+ struct statvfs buff_statvfs;
486
+ if (statvfs(mi->mount_point, &buff_statvfs) < 0) {
487
+ if(!m->shown_error) {
488
+ error("DISKSPACE: failed to statvfs() mount point '%s' (disk '%s', filesystem '%s', root '%s')"
489
+ , mi->mount_point
490
+ , mi->persistent_id
491
+ , mi->filesystem?mi->filesystem:""
492
+ , mi->root?mi->root:""
493
+ );
494
+ m->shown_error = 1;
495
+ }
496
+ return;
497
}
498
+ m->shown_error = 0;
499
271
-#ifdef NETDATA_INTERNAL_CHECKS
272
- if(unlikely(btotal != bavail + breserved_root + bused))
273
- error("DISKSPACE: disk inode statistics for '%s' (disk '%s') do not sum up: total = %llu, available = %llu, reserved = %llu, used = %llu", mi->mount_point, disk, (unsigned long long)ftotal, (unsigned long long)favail, (unsigned long long)freserved_root, (unsigned long long)fused);
274
-#endif
500
+ calculate_values_and_show_charts(mi, m, &buff_statvfs, update_every);
501
+}
502
276
- // --------------------------------------------------------------------------
503
+static void diskspace_slow_worker_cleanup(void *ptr)
504
+{
505
+ UNUSED(ptr);
506
278
- int rendered = 0;
507
+ info("cleaning up...");
508
280
- if(m->do_space == CONFIG_BOOLEAN_YES || (m->do_space == CONFIG_BOOLEAN_AUTO &&
281
- (bavail || breserved_root || bused ||
282
- netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
283
- if(unlikely(!m->st_space)) {
284
- m->do_space = CONFIG_BOOLEAN_YES;
285
- m->st_space = rrdset_find_active_bytype_localhost("disk_space", disk);
286
- if(unlikely(!m->st_space)) {
287
- char title[4096 + 1];
288
- snprintfz(title, 4096, "Disk Space Usage");
289
- m->st_space = rrdset_create_localhost(
290
- "disk_space"
291
- , disk
292
- , NULL
293
- , family
294
- , "disk.space"
295
- , title
296
- , "GiB"
297
- , PLUGIN_DISKSPACE_NAME
298
- , NULL
299
- , NETDATA_CHART_PRIO_DISKSPACE_SPACE
300
- , update_every
301
- , RRDSET_TYPE_STACKED
302
- );
303
- }
509
+ worker_unregister();
510
+}
511
305
- m->rd_space_avail = rrddim_add(m->st_space, "avail", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
306
- m->rd_space_used = rrddim_add(m->st_space, "used", NULL, (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
307
- m->rd_space_reserved = rrddim_add(m->st_space, "reserved_for_root", "reserved for root", (collected_number)bsize, 1024 * 1024 * 1024, RRD_ALGORITHM_ABSOLUTE);
308
- }
309
- else
310
- rrdset_next(m->st_space);
512
+#define WORKER_JOB_SLOW_MOUNTPOINT 0
513
+#define WORKER_JOB_SLOW_CLEANUP 1
514
312
- rrddim_set_by_pointer(m->st_space, m->rd_space_avail, (collected_number)bavail);
313
- rrddim_set_by_pointer(m->st_space, m->rd_space_used, (collected_number)bused);
314
- rrddim_set_by_pointer(m->st_space, m->rd_space_reserved, (collected_number)breserved_root);
315
- rrdset_done(m->st_space);
515
+struct slow_worker_data {
516
+ netdata_thread_t *slow_thread;
517
+ int update_every;
518
+};
519
317
- rendered++;
318
- }
520
+void *diskspace_slow_worker(void *ptr)
521
+{
522
+ struct slow_worker_data *data = (struct slow_worker_data *)ptr;
523
+
524
+ worker_register("DISKSPACE_SLOW");
525
+ worker_register_job_name(WORKER_JOB_SLOW_MOUNTPOINT, "mountpoint");
526
+ worker_register_job_name(WORKER_JOB_SLOW_CLEANUP, "cleanup");
527
320
- // --------------------------------------------------------------------------
528
+ struct basic_mountinfo *slow_mountinfo_root = NULL;
529
322
- if(m->do_inodes == CONFIG_BOOLEAN_YES || (m->do_inodes == CONFIG_BOOLEAN_AUTO &&
323
- (favail || freserved_root || fused ||
324
- netdata_zero_metrics_enabled == CONFIG_BOOLEAN_YES))) {
325
- if(unlikely(!m->st_inodes)) {
326
- m->do_inodes = CONFIG_BOOLEAN_YES;
327
- m->st_inodes = rrdset_find_active_bytype_localhost("disk_inodes", disk);
328
- if(unlikely(!m->st_inodes)) {
329
- char title[4096 + 1];
330
- snprintfz(title, 4096, "Disk Files (inodes) Usage");
331
- m->st_inodes = rrdset_create_localhost(
332
- "disk_inodes"
333
- , disk
334
- , NULL
335
- , family
336
- , "disk.inodes"
337
- , title
338
- , "inodes"
339
- , PLUGIN_DISKSPACE_NAME
340
- , NULL
341
- , NETDATA_CHART_PRIO_DISKSPACE_INODES
342
- , update_every
343
- , RRDSET_TYPE_STACKED
344
- );
345
- }
530
+ int slow_update_every = data->update_every > SLOW_UPDATE_EVERY ? data->update_every : SLOW_UPDATE_EVERY;
531
347
- m->rd_inodes_avail = rrddim_add(m->st_inodes, "avail", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
348
- m->rd_inodes_used = rrddim_add(m->st_inodes, "used", NULL, 1, 1, RRD_ALGORITHM_ABSOLUTE);
349
- m->rd_inodes_reserved = rrddim_add(m->st_inodes, "reserved_for_root", "reserved for root", 1, 1, RRD_ALGORITHM_ABSOLUTE);
532
+ netdata_thread_cleanup_push(diskspace_slow_worker_cleanup, data->slow_thread);
533
+
534
+ usec_t step = slow_update_every * USEC_PER_SEC;
535
+ heartbeat_t hb;
536
+ heartbeat_init(&hb);
537
+
538
+ while(!netdata_exit) {
539
+ worker_is_idle();
540
+ heartbeat_next(&hb, step);
541
+
542
+ usec_t start_time = now_monotonic_high_precision_usec();
543
+
544
+ if (!dict_mountpoints)
545
+ continue;
546
+
547
+ if(unlikely(netdata_exit)) break;
548
+
549
+ // --------------------------------------------------------------------------
550
+ // disk space metrics
551
+
552
+ worker_is_busy(WORKER_JOB_SLOW_MOUNTPOINT);
553
+
554
+ netdata_mutex_lock(&slow_mountinfo_mutex);
555
+ free_basic_mountinfo_list(slow_mountinfo_root);
556
+ slow_mountinfo_root = slow_mountinfo_tmp_root;
557
+ slow_mountinfo_tmp_root = NULL;
558
+ netdata_mutex_unlock(&slow_mountinfo_mutex);
559
+
560
+ struct basic_mountinfo *bmi;
561
+ for(bmi = slow_mountinfo_root; bmi; bmi = bmi->next) {
562
+ do_slow_disk_space_stats(bmi, slow_update_every);
563
+
564
+ if(unlikely(netdata_exit)) break;
565
}
351
- else
352
- rrdset_next(m->st_inodes);
566
354
- rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_avail, (collected_number)favail);
355
- rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_used, (collected_number)fused);
356
- rrddim_set_by_pointer(m->st_inodes, m->rd_inodes_reserved, (collected_number)freserved_root);
357
- rrdset_done(m->st_inodes);
567
+ if(unlikely(netdata_exit)) break;
568
359
- rendered++;
569
+ worker_is_busy(WORKER_JOB_SLOW_CLEANUP);
570
+
571
+ for(bmi = slow_mountinfo_root; bmi; bmi = bmi->next) {
572
+ struct mount_point_metadata *m = dictionary_get(dict_mountpoints, bmi->mount_point);
573
+
574
+ if (m)
575
+ mount_point_cleanup(bmi->mount_point, m, 1);
576
+ }
577
+
578
+ usec_t dt = now_monotonic_high_precision_usec() - start_time;
579
+ if (dt > step) {
580
+ slow_update_every = (dt / USEC_PER_SEC) * 3 / 2;
581
+ if (slow_update_every % SLOW_UPDATE_EVERY)
582
+ slow_update_every += SLOW_UPDATE_EVERY - slow_update_every % SLOW_UPDATE_EVERY;
583
+ step = slow_update_every * USEC_PER_SEC;
584
+ }
585
}
586
362
- // --------------------------------------------------------------------------
587
+ netdata_thread_cleanup_pop(1);
588
364
- if(likely(rendered))
365
- m->collected++;
589
+ free_basic_mountinfo_list(slow_mountinfo_root);
590
+
591
+ return NULL;
592
}
593
594
static void diskspace_main_cleanup(void *ptr) {
@@ -373,6 +599,13 @@ static void diskspace_main_cleanup(void *ptr) {
599
600
info("cleaning up...");
601
602
+ if (diskspace_slow_thread) {
603
+ netdata_thread_join(*diskspace_slow_thread, NULL);
604
+ freez(diskspace_slow_thread);
605
+ }
606
+
607
+ free_basic_mountinfo_list(slow_mountinfo_tmp_root);
608
+
609
static_thread->enabled = NETDATA_MAIN_THREAD_EXITED;
610
}
611
@@ -402,6 +635,19 @@ void *diskspace_main(void *ptr) {
635
if(check_for_new_mountpoints_every < update_every)
636
check_for_new_mountpoints_every = update_every;
637
638
+ netdata_mutex_init(&slow_mountinfo_mutex);
639
+
640
+ diskspace_slow_thread = mallocz(sizeof(netdata_thread_t));
641
+
642
+ struct slow_worker_data slow_worker_data = {.slow_thread = diskspace_slow_thread, .update_every = update_every};
643
+
644
+ netdata_thread_create(
645
+ diskspace_slow_thread,
646
+ THREAD_DISKSPACE_SLOW_NAME,
647
+ NETDATA_THREAD_OPTION_JOINABLE,
648
+ diskspace_slow_worker,
649
+ &slow_worker_data);
650
+
651
usec_t step = update_every * USEC_PER_SEC;
652
heartbeat_t hb;
653
heartbeat_init(&hb);
@@ -411,7 +657,6 @@ void *diskspace_main(void *ptr) {
657
658
if(unlikely(netdata_exit)) break;
659
414
-
660
// --------------------------------------------------------------------------
661
// this is smart enough not to reload it every time
662
@@ -421,9 +666,12 @@ void *diskspace_main(void *ptr) {
666
// --------------------------------------------------------------------------
667
// disk space metrics
668
669
+ netdata_mutex_lock(&slow_mountinfo_mutex);
670
+ free_basic_mountinfo_list(slow_mountinfo_tmp_root);
671
+ slow_mountinfo_tmp_root = NULL;
672
+
673
struct mountinfo *mi;
674
for(mi = disk_mountinfo_root; mi; mi = mi->next) {
426
-
675
if(unlikely(mi->flags & (MOUNTINFO_IS_DUMMY | MOUNTINFO_IS_BIND)))
676
continue;
677
@@ -435,12 +683,13 @@ void *diskspace_main(void *ptr) {
683
do_disk_space_stats(mi, update_every);
684
if(unlikely(netdata_exit)) break;
685
}
686
+ netdata_mutex_unlock(&slow_mountinfo_mutex);
687
688
if(unlikely(netdata_exit)) break;
689
690
if(dict_mountpoints) {
691
worker_is_busy(WORKER_JOB_CLEANUP);
443
- dictionary_walkthrough_read(dict_mountpoints, mount_point_cleanup, NULL);
692
+ dictionary_walkthrough_read(dict_mountpoints, mount_point_cleanup_cb, NULL);
693
}
694
695
}