python.d: add job file lock registry (#9564)
Ilya Mashchenko committed
Jul 21, 2020 at 09:46 UTC
def03a151cd042748b2687729a922d2f93fbd570
4 files changed
+521
REDISTRIBUTED.md
+4
@@ -178,4 +178,8 @@ connectivity is not available.
178
Copyright 2014, 2015, 2016 Ori Livneh [ori@wikimedia.org](mailto:ori@wikimedia.org)
179
[Apache-2.0](http://www.apache.org/licenses/LICENSE-2.0)
180
181
+- [filelock](https://github.com/benediktschmitt/py-filelock)
182
+
183
+ Copyright 2015, Benedikt Schmitt [Unlicense License](https://unlicense.org/)
184
+
185
[](<>)
collectors/python.d.plugin/Makefile.am
+1
@@ -141,6 +141,7 @@ dist_third_party_DATA = \
141
python_modules/third_party/mcrcon.py \
142
python_modules/third_party/boinc_client.py \
143
python_modules/third_party/monotonic.py \
144
+ python_modules/third_party/filelock.py \
145
$(NULL)
146
147
pythonyaml2dir=$(pythonmodulesdir)/pyyaml2
collectors/python.d.plugin/python.d.plugin.in
+65
@@ -52,6 +52,7 @@ ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'
52
ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'
53
ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'
54
ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'
55
+ENV_NETDATA_LOCKS_DIR = 'NETDATA_LOCKS_DIR'
56
57
58
def add_pythond_packages():
@@ -66,6 +67,7 @@ add_pythond_packages()
67
from bases.collection import safe_print
68
from bases.loggers import PythonDLogger
69
from bases.loaders import load_config
70
+from third_party import filelock
71
72
try:
73
from collections import OrderedDict
@@ -90,6 +92,10 @@ def dirs():
92
ENV_NETDATA_PLUGINS_DIR,
93
os.path.dirname(__file__),
94
)
95
+ locks = os.getenv(
96
+ ENV_NETDATA_LOCKS_DIR,
97
+ # TODO: add '@locksdir_POST@
98
+ )
99
modules_user_config = os.path.join(plugin_user_config, 'python.d')
100
modules_stock_config = os.path.join(plugin_stock_config, 'python.d')
101
modules = os.path.abspath(pluginsd + '/../python.d')
@@ -103,6 +109,7 @@ def dirs():
109
'modules_stock_config',
110
'modules',
111
'var_lib',
112
+ 'locks',
113
]
114
)
115
return Dirs(
@@ -112,6 +119,7 @@ def dirs():
119
modules_stock_config,
120
modules,
121
var_lib,
122
+ locks,
123
)
124
125
@@ -304,6 +312,9 @@ class Job(threading.Thread):
312
def init(self):
313
self.job = self.service(configuration=copy.deepcopy(self.config))
314
315
+ def full_name(self):
316
+ return self.job.name
317
+
318
def check(self):
319
ok = self.job.check()
320
self.checks -= self.checks != self.inf and not ok
@@ -445,6 +456,41 @@ class PluginConfig(dict):
456
return self['default_run']
457
458
459
+class FileLockRegistry:
460
+ def __init__(self, path):
461
+ self.path = path
462
+ self.locks = dict()
463
+
464
+ def register(self, name):
465
+ if name in self.locks:
466
+ return
467
+ file = os.path.join(self.path, name)
468
+ lock = filelock.FileLock(file)
469
+ lock.acquire(timeout=0)
470
+ self.locks[name] = lock
471
+
472
+ def unregister(self, name):
473
+ if name not in self.locks:
474
+ return
475
+ lock = self.locks[name]
476
+ lock.release()
477
+ del self.locks[name]
478
+
479
+
480
+class DummyRegistry:
481
+ def register(self, name):
482
+ pass
483
+
484
+ def unregister(self, name):
485
+ pass
486
+
487
+
488
+def create_jobs_registry():
489
+ if not DIRS.locks:
490
+ return DummyRegistry()
491
+ return FileLockRegistry(DIRS.locks)
492
+
493
+
494
class Plugin:
495
config_name = 'python.d.conf'
496
jobs_status_dump_name = 'pythond-jobs-statuses.json'
@@ -454,6 +500,7 @@ class Plugin:
500
self.min_update_every = min_update_every
501
self.config = PluginConfig(PLUGIN_BASE_CONF)
502
self.log = PythonDLogger()
503
+ self.registry = create_jobs_registry()
504
self.started_jobs = collections.defaultdict(dict)
505
self.jobs = list()
506
self.saver = None
@@ -604,12 +651,30 @@ class Plugin:
651
continue
652
self.log.info('{0}[{1}] : check success'.format(job.module_name, job.real_name))
653
654
+ try:
655
+ self.registry.register(job.full_name())
656
+ except filelock.Timeout as error:
657
+ self.log.info('{0}[{1}] : already registered by another process, skipping the job ({2})'.format(
658
+ job.module_name, job.real_name, error))
659
+ job.status = JOB_STATUS_DROPPED
660
+ continue
661
+ except Exception as error:
662
+ self.log.warning('{0}[{1}] : registration failed: {2}, skipping the job'.format(
663
+ job.module_name, job.real_name, error))
664
+ job.status = JOB_STATUS_DROPPED
665
+ continue
666
+
667
try:
668
job.create()
669
except Exception as error:
670
self.log.warning("{0}[{1}] : unhandled exception on create : {2}, skipping the job".format(
671
job.module_name, job.real_name, repr(error)))
672
job.status = JOB_STATUS_DROPPED
673
+ try:
674
+ self.registry.unregister(job.full_name())
675
+ except Exception as error:
676
+ self.log.warning('{0}[{1}] : deregistration failed: {2}'.format(
677
+ job.module_name, job.real_name, error))
678
continue
679
680
self.started_jobs[job.module_name] = job.actual_name
collectors/python.d.plugin/python_modules/third_party/filelock.py
new
+451
@@ -0,0 +1,451 @@
1
+# This is free and unencumbered software released into the public domain.
2
+#
3
+# Anyone is free to copy, modify, publish, use, compile, sell, or
4
+# distribute this software, either in source code form or as a compiled
5
+# binary, for any purpose, commercial or non-commercial, and by any
6
+# means.
7
+#
8
+# In jurisdictions that recognize copyright laws, the author or authors
9
+# of this software dedicate any and all copyright interest in the
10
+# software to the public domain. We make this dedication for the benefit
11
+# of the public at large and to the detriment of our heirs and
12
+# successors. We intend this dedication to be an overt act of
13
+# relinquishment in perpetuity of all present and future rights to this
14
+# software under copyright law.
15
+#
16
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
17
+# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
18
+# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
19
+# IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
20
+# OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
21
+# ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
22
+# OTHER DEALINGS IN THE SOFTWARE.
23
+#
24
+# For more information, please refer to <http://unlicense.org>
25
+
26
+"""
27
+A platform independent file lock that supports the with-statement.
28
+"""
29
+
30
+
31
+# Modules
32
+# ------------------------------------------------
33
+import logging
34
+import os
35
+import threading
36
+import time
37
+try:
38
+ import warnings
39
+except ImportError:
40
+ warnings = None
41
+
42
+try:
43
+ import msvcrt
44
+except ImportError:
45
+ msvcrt = None
46
+
47
+try:
48
+ import fcntl
49
+except ImportError:
50
+ fcntl = None
51
+
52
+
53
+# Backward compatibility
54
+# ------------------------------------------------
55
+try:
56
+ TimeoutError
57
+except NameError:
58
+ TimeoutError = OSError
59
+
60
+
61
+# Data
62
+# ------------------------------------------------
63
+__all__ = [
64
+ "Timeout",
65
+ "BaseFileLock",
66
+ "WindowsFileLock",
67
+ "UnixFileLock",
68
+ "SoftFileLock",
69
+ "FileLock"
70
+]
71
+
72
+__version__ = "3.0.12"
73
+
74
+
75
+_logger = None
76
+def logger():
77
+ """Returns the logger instance used in this module."""
78
+ global _logger
79
+ _logger = _logger or logging.getLogger(__name__)
80
+ return _logger
81
+
82
+
83
+# Exceptions
84
+# ------------------------------------------------
85
+class Timeout(TimeoutError):
86
+ """
87
+ Raised when the lock could not be acquired in *timeout*
88
+ seconds.
89
+ """
90
+
91
+ def __init__(self, lock_file):
92
+ """
93
+ """
94
+ #: The path of the file lock.
95
+ self.lock_file = lock_file
96
+ return None
97
+
98
+ def __str__(self):
99
+ temp = "The file lock '{}' could not be acquired."\
100
+ .format(self.lock_file)
101
+ return temp
102
+
103
+
104
+# Classes
105
+# ------------------------------------------------
106
+
107
+# This is a helper class which is returned by :meth:`BaseFileLock.acquire`
108
+# and wraps the lock to make sure __enter__ is not called twice when entering
109
+# the with statement.
110
+# If we would simply return *self*, the lock would be acquired again
111
+# in the *__enter__* method of the BaseFileLock, but not released again
112
+# automatically.
113
+#
114
+# :seealso: issue #37 (memory leak)
115
+class _Acquire_ReturnProxy(object):
116
+
117
+ def __init__(self, lock):
118
+ self.lock = lock
119
+ return None
120
+
121
+ def __enter__(self):
122
+ return self.lock
123
+
124
+ def __exit__(self, exc_type, exc_value, traceback):
125
+ self.lock.release()
126
+ return None
127
+
128
+
129
+class BaseFileLock(object):
130
+ """
131
+ Implements the base class of a file lock.
132
+ """
133
+
134
+ def __init__(self, lock_file, timeout = -1):
135
+ """
136
+ """
137
+ # The path to the lock file.
138
+ self._lock_file = lock_file
139
+
140
+ # The file descriptor for the *_lock_file* as it is returned by the
141
+ # os.open() function.
142
+ # This file lock is only NOT None, if the object currently holds the
143
+ # lock.
144
+ self._lock_file_fd = None
145
+
146
+ # The default timeout value.
147
+ self.timeout = timeout
148
+
149
+ # We use this lock primarily for the lock counter.
150
+ self._thread_lock = threading.Lock()
151
+
152
+ # The lock counter is used for implementing the nested locking
153
+ # mechanism. Whenever the lock is acquired, the counter is increased and
154
+ # the lock is only released, when this value is 0 again.
155
+ self._lock_counter = 0
156
+ return None
157
+
158
+ @property
159
+ def lock_file(self):
160
+ """
161
+ The path to the lock file.
162
+ """
163
+ return self._lock_file
164
+
165
+ @property
166
+ def timeout(self):
167
+ """
168
+ You can set a default timeout for the filelock. It will be used as
169
+ fallback value in the acquire method, if no timeout value (*None*) is
170
+ given.
171
+
172
+ If you want to disable the timeout, set it to a negative value.
173
+
174
+ A timeout of 0 means, that there is exactly one attempt to acquire the
175
+ file lock.
176
+
177
+ .. versionadded:: 2.0.0
178
+ """
179
+ return self._timeout
180
+
181
+ @timeout.setter
182
+ def timeout(self, value):
183
+ """
184
+ """
185
+ self._timeout = float(value)
186
+ return None
187
+
188
+ # Platform dependent locking
189
+ # --------------------------------------------
190
+
191
+ def _acquire(self):
192
+ """
193
+ Platform dependent. If the file lock could be
194
+ acquired, self._lock_file_fd holds the file descriptor
195
+ of the lock file.
196
+ """
197
+ raise NotImplementedError()
198
+
199
+ def _release(self):
200
+ """
201
+ Releases the lock and sets self._lock_file_fd to None.
202
+ """
203
+ raise NotImplementedError()
204
+
205
+ # Platform independent methods
206
+ # --------------------------------------------
207
+
208
+ @property
209
+ def is_locked(self):
210
+ """
211
+ True, if the object holds the file lock.
212
+
213
+ .. versionchanged:: 2.0.0
214
+
215
+ This was previously a method and is now a property.
216
+ """
217
+ return self._lock_file_fd is not None
218
+
219
+ def acquire(self, timeout=None, poll_intervall=0.05):
220
+ """
221
+ Acquires the file lock or fails with a :exc:`Timeout` error.
222
+
223
+ .. code-block:: python
224
+
225
+ # You can use this method in the context manager (recommended)
226
+ with lock.acquire():
227
+ pass
228
+
229
+ # Or use an equivalent try-finally construct:
230
+ lock.acquire()
231
+ try:
232
+ pass
233
+ finally:
234
+ lock.release()
235
+
236
+ :arg float timeout:
237
+ The maximum time waited for the file lock.
238
+ If ``timeout < 0``, there is no timeout and this method will
239
+ block until the lock could be acquired.
240
+ If ``timeout`` is None, the default :attr:`~timeout` is used.
241
+
242
+ :arg float poll_intervall:
243
+ We check once in *poll_intervall* seconds if we can acquire the
244
+ file lock.
245
+
246
+ :raises Timeout:
247
+ if the lock could not be acquired in *timeout* seconds.
248
+
249
+ .. versionchanged:: 2.0.0
250
+
251
+ This method returns now a *proxy* object instead of *self*,
252
+ so that it can be used in a with statement without side effects.
253
+ """
254
+ # Use the default timeout, if no timeout is provided.
255
+ if timeout is None:
256
+ timeout = self.timeout
257
+
258
+ # Increment the number right at the beginning.
259
+ # We can still undo it, if something fails.
260
+ with self._thread_lock:
261
+ self._lock_counter += 1
262
+
263
+ lock_id = id(self)
264
+ lock_filename = self._lock_file
265
+ start_time = time.time()
266
+ try:
267
+ while True:
268
+ with self._thread_lock:
269
+ if not self.is_locked:
270
+ logger().debug('Attempting to acquire lock %s on %s', lock_id, lock_filename)
271
+ self._acquire()
272
+
273
+ if self.is_locked:
274
+ logger().info('Lock %s acquired on %s', lock_id, lock_filename)
275
+ break
276
+ elif timeout >= 0 and time.time() - start_time > timeout:
277
+ logger().debug('Timeout on acquiring lock %s on %s', lock_id, lock_filename)
278
+ raise Timeout(self._lock_file)
279
+ else:
280
+ logger().debug(
281
+ 'Lock %s not acquired on %s, waiting %s seconds ...',
282
+ lock_id, lock_filename, poll_intervall
283
+ )
284
+ time.sleep(poll_intervall)
285
+ except:
286
+ # Something did go wrong, so decrement the counter.
287
+ with self._thread_lock:
288
+ self._lock_counter = max(0, self._lock_counter - 1)
289
+
290
+ raise
291
+ return _Acquire_ReturnProxy(lock = self)
292
+
293
+ def release(self, force = False):
294
+ """
295
+ Releases the file lock.
296
+
297
+ Please note, that the lock is only completly released, if the lock
298
+ counter is 0.
299
+
300
+ Also note, that the lock file itself is not automatically deleted.
301
+
302
+ :arg bool force:
303
+ If true, the lock counter is ignored and the lock is released in
304
+ every case.
305
+ """
306
+ with self._thread_lock:
307
+
308
+ if self.is_locked:
309
+ self._lock_counter -= 1
310
+
311
+ if self._lock_counter == 0 or force:
312
+ lock_id = id(self)
313
+ lock_filename = self._lock_file
314
+
315
+ logger().debug('Attempting to release lock %s on %s', lock_id, lock_filename)
316
+ self._release()
317
+ self._lock_counter = 0
318
+ logger().info('Lock %s released on %s', lock_id, lock_filename)
319
+
320
+ return None
321
+
322
+ def __enter__(self):
323
+ self.acquire()
324
+ return self
325
+
326
+ def __exit__(self, exc_type, exc_value, traceback):
327
+ self.release()
328
+ return None
329
+
330
+ def __del__(self):
331
+ self.release(force = True)
332
+ return None
333
+
334
+
335
+# Windows locking mechanism
336
+# ~~~~~~~~~~~~~~~~~~~~~~~~~
337
+
338
+class WindowsFileLock(BaseFileLock):
339
+ """
340
+ Uses the :func:`msvcrt.locking` function to hard lock the lock file on
341
+ windows systems.
342
+ """
343
+
344
+ def _acquire(self):
345
+ open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
346
+
347
+ try:
348
+ fd = os.open(self._lock_file, open_mode)
349
+ except OSError:
350
+ pass
351
+ else:
352
+ try:
353
+ msvcrt.locking(fd, msvcrt.LK_NBLCK, 1)
354
+ except (IOError, OSError):
355
+ os.close(fd)
356
+ else:
357
+ self._lock_file_fd = fd
358
+ return None
359
+
360
+ def _release(self):
361
+ fd = self._lock_file_fd
362
+ self._lock_file_fd = None
363
+ msvcrt.locking(fd, msvcrt.LK_UNLCK, 1)
364
+ os.close(fd)
365
+
366
+ try:
367
+ os.remove(self._lock_file)
368
+ # Probably another instance of the application
369
+ # that acquired the file lock.
370
+ except OSError:
371
+ pass
372
+ return None
373
+
374
+# Unix locking mechanism
375
+# ~~~~~~~~~~~~~~~~~~~~~~
376
+
377
+class UnixFileLock(BaseFileLock):
378
+ """
379
+ Uses the :func:`fcntl.flock` to hard lock the lock file on unix systems.
380
+ """
381
+
382
+ def _acquire(self):
383
+ open_mode = os.O_RDWR | os.O_CREAT | os.O_TRUNC
384
+ fd = os.open(self._lock_file, open_mode)
385
+
386
+ try:
387
+ fcntl.flock(fd, fcntl.LOCK_EX | fcntl.LOCK_NB)
388
+ except (IOError, OSError):
389
+ os.close(fd)
390
+ else:
391
+ self._lock_file_fd = fd
392
+ return None
393
+
394
+ def _release(self):
395
+ # Do not remove the lockfile:
396
+ #
397
+ # https://github.com/benediktschmitt/py-filelock/issues/31
398
+ # https://stackoverflow.com/questions/17708885/flock-removing-locked-file-without-race-condition
399
+ fd = self._lock_file_fd
400
+ self._lock_file_fd = None
401
+ fcntl.flock(fd, fcntl.LOCK_UN)
402
+ os.close(fd)
403
+ return None
404
+
405
+# Soft lock
406
+# ~~~~~~~~~
407
+
408
+class SoftFileLock(BaseFileLock):
409
+ """
410
+ Simply watches the existence of the lock file.
411
+ """
412
+
413
+ def _acquire(self):
414
+ open_mode = os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_TRUNC
415
+ try:
416
+ fd = os.open(self._lock_file, open_mode)
417
+ except (IOError, OSError):
418
+ pass
419
+ else:
420
+ self._lock_file_fd = fd
421
+ return None
422
+
423
+ def _release(self):
424
+ os.close(self._lock_file_fd)
425
+ self._lock_file_fd = None
426
+
427
+ try:
428
+ os.remove(self._lock_file)
429
+ # The file is already deleted and that's what we want.
430
+ except OSError:
431
+ pass
432
+ return None
433
+
434
+
435
+# Platform filelock
436
+# ~~~~~~~~~~~~~~~~~
437
+
438
+#: Alias for the lock, which should be used for the current platform. On
439
+#: Windows, this is an alias for :class:`WindowsFileLock`, on Unix for
440
+#: :class:`UnixFileLock` and otherwise for :class:`SoftFileLock`.
441
+FileLock = None
442
+
443
+if msvcrt:
444
+ FileLock = WindowsFileLock
445
+elif fcntl:
446
+ FileLock = UnixFileLock
447
+else:
448
+ FileLock = SoftFileLock
449
+
450
+ if warnings is not None:
451
+ warnings.warn("only soft file lock is available")