Added lock dir (#9584)
* Add lock dir * Clean directory on startup * Update environment variable name * Fix file removal * Add error message * collectors/python.d: change lock file name * collectors/python.d: add `nolock` cmd option Co-authored-by: ilyam8 <ilya@netdata.cloud>
Vladimir Kobal committed
Jul 23, 2020 at 16:39 UTC
219895300847a6347a1f4b8d82ad04455a682b03
5 files changed
+54
-13
collectors/python.d.plugin/python.d.plugin.in
+19
-13
@@ -52,7 +52,7 @@ ENV_NETDATA_STOCK_CONFIG_DIR = 'NETDATA_STOCK_CONFIG_DIR'
52
ENV_NETDATA_PLUGINS_DIR = 'NETDATA_PLUGINS_DIR'
53
ENV_NETDATA_LIB_DIR = 'NETDATA_LIB_DIR'
54
ENV_NETDATA_UPDATE_EVERY = 'NETDATA_UPDATE_EVERY'
55
-ENV_NETDATA_LOCKS_DIR = 'NETDATA_LOCKS_DIR'
55
+ENV_NETDATA_LOCK_DIR = 'NETDATA_LOCK_DIR'
56
57
58
def add_pythond_packages():
@@ -93,8 +93,8 @@ def dirs():
93
os.path.dirname(__file__),
94
)
95
locks = os.getenv(
96
- ENV_NETDATA_LOCKS_DIR,
97
- # TODO: add '@locksdir_POST@
96
+ ENV_NETDATA_LOCK_DIR,
97
+ os.path.join('@varlibdir_POST@', 'lock')
98
)
99
modules_user_config = os.path.join(plugin_user_config, 'python.d')
100
modules_stock_config = os.path.join(plugin_stock_config, 'python.d')
@@ -464,7 +464,7 @@ class FileLockRegistry:
464
def register(self, name):
465
if name in self.locks:
466
return
467
- file = os.path.join(self.path, name)
467
+ file = os.path.join(self.path, '{0}.collector.lock'.format(name))
468
lock = filelock.FileLock(file)
469
lock.acquire(timeout=0)
470
self.locks[name] = lock
@@ -485,22 +485,16 @@ class DummyRegistry:
485
pass
486
487
488
-def create_jobs_registry():
489
- if not DIRS.locks:
490
- return DummyRegistry()
491
- return FileLockRegistry(DIRS.locks)
492
-
493
-
488
class Plugin:
489
config_name = 'python.d.conf'
490
jobs_status_dump_name = 'pythond-jobs-statuses.json'
491
498
- def __init__(self, modules_to_run, min_update_every):
492
+ def __init__(self, modules_to_run, min_update_every, registry):
493
self.modules_to_run = modules_to_run
494
self.min_update_every = min_update_every
495
self.config = PluginConfig(PLUGIN_BASE_CONF)
496
self.log = PythonDLogger()
503
- self.registry = create_jobs_registry()
497
+ self.registry = registry
498
self.started_jobs = collections.defaultdict(dict)
499
self.jobs = list()
500
self.saver = None
@@ -748,6 +742,7 @@ def parse_command_line():
742
743
debug = False
744
trace = False
745
+ nolock = False
746
update_every = 1
747
modules_to_run = list()
748
@@ -764,6 +759,9 @@ def parse_command_line():
759
if 'trace' in opts:
760
trace = True
761
opts.remove('trace')
762
+ if 'nolock' in opts:
763
+ nolock = True
764
+ opts.remove('nolock')
765
if opts:
766
modules_to_run = list(opts)
767
@@ -773,13 +771,15 @@ def parse_command_line():
771
'update_every',
772
'debug',
773
'trace',
774
+ 'nolock',
775
'modules_to_run',
776
])
777
return cmd(
778
update_every,
779
debug,
780
trace,
782
- modules_to_run
781
+ nolock,
782
+ modules_to_run,
783
)
784
785
@@ -827,9 +827,15 @@ def main():
827
log.info('probably you meant : \n{0}'.format(pprint.pformat(guessed, width=1)))
828
return
829
830
+ if DIRS.locks and not cmd.nolock:
831
+ registry = FileLockRegistry(DIRS.locks)
832
+ else:
833
+ registry = DummyRegistry()
834
+
835
p = Plugin(
836
cmd.modules_to_run or AVAILABLE_MODULES,
837
cmd.update_every,
838
+ registry,
839
)
840
841
try:
daemon/common.c
+1
@@ -10,6 +10,7 @@ char *netdata_configured_primary_plugins_dir = NULL;
10
char *netdata_configured_web_dir = WEB_DIR;
11
char *netdata_configured_cache_dir = CACHE_DIR;
12
char *netdata_configured_varlib_dir = VARLIB_DIR;
13
+char *netdata_configured_lock_dir = NULL;
14
char *netdata_configured_home_dir = CACHE_DIR;
15
char *netdata_configured_host_prefix = NULL;
16
char *netdata_configured_timezone = NULL;
daemon/common.h
+1
@@ -92,6 +92,7 @@ extern char *netdata_configured_primary_plugins_dir;
92
extern char *netdata_configured_web_dir;
93
extern char *netdata_configured_cache_dir;
94
extern char *netdata_configured_varlib_dir;
95
+extern char *netdata_configured_lock_dir;
96
extern char *netdata_configured_home_dir;
97
extern char *netdata_configured_host_prefix;
98
extern char *netdata_configured_timezone;
daemon/daemon.c
+22
@@ -58,6 +58,22 @@ void create_needed_dir(const char *dir, uid_t uid, gid_t gid)
58
error("Cannot create directory '%s'", dir);
59
}
60
61
+void clean_directory(char *dirname)
62
+{
63
+ DIR *dir = opendir(dirname);
64
+ if(!dir) return;
65
+
66
+ int dir_fd = dirfd(dir);
67
+ struct dirent *de = NULL;
68
+
69
+ while((de = readdir(dir)))
70
+ if(de->d_type == DT_REG)
71
+ if (unlinkat(dir_fd, de->d_name, 0))
72
+ error("Cannot delete %s/%s", dirname, de->d_name);
73
+
74
+ closedir(dir);
75
+}
76
+
77
int become_user(const char *username, int pid_fd) {
78
int am_i_root = (getuid() == 0)?1:0;
79
@@ -72,8 +88,11 @@ int become_user(const char *username, int pid_fd) {
88
89
create_needed_dir(netdata_configured_cache_dir, uid, gid);
90
create_needed_dir(netdata_configured_varlib_dir, uid, gid);
91
+ create_needed_dir(netdata_configured_lock_dir, uid, gid);
92
create_needed_dir(claimingdirectory, uid, gid);
93
94
+ clean_directory(netdata_configured_lock_dir);
95
+
96
if(pidfile[0]) {
97
if(chown(pidfile, uid, gid) == -1)
98
error("Cannot chown '%s' to %u:%u", pidfile, (unsigned int)uid, (unsigned int)gid);
@@ -469,7 +488,10 @@ int become_daemon(int dont_fork, const char *user)
488
else {
489
create_needed_dir(netdata_configured_cache_dir, getuid(), getgid());
490
create_needed_dir(netdata_configured_varlib_dir, getuid(), getgid());
491
+ create_needed_dir(netdata_configured_lock_dir, getuid(), getgid());
492
create_needed_dir(claimingdirectory, getuid(), getgid());
493
+
494
+ clean_directory(netdata_configured_lock_dir);
495
}
496
497
if(pidfd != -1)
daemon/main.c
+11
@@ -434,6 +434,14 @@ static void log_init(void) {
434
setenv("NETDATA_ERRORS_PER_PERIOD", config_get(CONFIG_SECTION_GLOBAL, "errors to trigger flood protection", ""), 1);
435
}
436
437
+char *initialize_lock_directory_path(char *prefix)
438
+{
439
+ char filename[FILENAME_MAX + 1];
440
+ snprintfz(filename, FILENAME_MAX, "%s/lock", prefix);
441
+
442
+ return config_get(CONFIG_SECTION_GLOBAL, "lock directory", filename);
443
+}
444
+
445
static void backwards_compatible_config() {
446
// move [global] options to the [web] section
447
config_move(CONFIG_SECTION_GLOBAL, "http port listen backlog",
@@ -529,6 +537,8 @@ static void get_netdata_configured_variables() {
537
netdata_configured_varlib_dir = config_get(CONFIG_SECTION_GLOBAL, "lib directory", netdata_configured_varlib_dir);
538
netdata_configured_home_dir = config_get(CONFIG_SECTION_GLOBAL, "home directory", netdata_configured_home_dir);
539
540
+ netdata_configured_lock_dir = initialize_lock_directory_path(netdata_configured_varlib_dir);
541
+
542
{
543
pluginsd_initialize_plugin_directories();
544
netdata_configured_primary_plugins_dir = plugin_directories[PLUGINSD_STOCK_PLUGINS_DIRECTORY_PATH];
@@ -692,6 +702,7 @@ void set_global_environment() {
702
setenv("NETDATA_WEB_DIR" , verify_required_directory(netdata_configured_web_dir), 1);
703
setenv("NETDATA_CACHE_DIR" , verify_required_directory(netdata_configured_cache_dir), 1);
704
setenv("NETDATA_LIB_DIR" , verify_required_directory(netdata_configured_varlib_dir), 1);
705
+ setenv("NETDATA_LOCK_DIR" , netdata_configured_lock_dir, 1);
706
setenv("NETDATA_LOG_DIR" , verify_required_directory(netdata_configured_log_dir), 1);
707
setenv("HOME" , verify_required_directory(netdata_configured_home_dir), 1);
708
setenv("NETDATA_HOST_PREFIX" , netdata_configured_host_prefix, 1);