| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | #include "plugins_d.h" |
| 4 | #include "pluginsd_parser.h" |
| 5 | |
| 6 | char *plugin_directories[PLUGINSD_MAX_DIRECTORIES] = { [0] = PLUGINS_DIR, }; |
| 7 | struct plugind *pluginsd_root = NULL; |
| 8 | |
| 9 | static inline void pluginsd_sleep(const int seconds) { |
| 10 | int timeout_ms = seconds * 1000; |
| 11 | int waited_ms = 0; |
| 12 | while(waited_ms < timeout_ms) { |
| 13 | if(!service_running(SERVICE_COLLECTORS)) break; |
| 14 | sleep_usec(ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS * USEC_PER_MS); |
| 15 | waited_ms += ND_CHECK_CANCELLABILITY_WHILE_WAITING_EVERY_MS; |
| 16 | } |
| 17 | } |
| 18 | |
| 19 | inline size_t pluginsd_initialize_plugin_directories() |
| 20 | { |
| 21 | char plugins_dirs[(FILENAME_MAX * 2) + 1]; |
| 22 | static char *plugins_dir_list = NULL; |
| 23 | |
| 24 | // Get the configuration entry |
| 25 | if (likely(!plugins_dir_list)) { |
| 26 | snprintfz(plugins_dirs, FILENAME_MAX * 2, "\"%s\" \"%s/custom-plugins.d\"", PLUGINS_DIR, CONFIG_DIR); |
| 27 | plugins_dir_list = strdupz(inicfg_get_quoted_path_list(&netdata_config, CONFIG_SECTION_DIRECTORIES, "plugins", plugins_dirs)); |
| 28 | } |
| 29 | |
| 30 | // Parse it and store it to plugin directories |
| 31 | return quoted_strings_splitter_config(plugins_dir_list, plugin_directories, PLUGINSD_MAX_DIRECTORIES); |
| 32 | } |
| 33 | |
| 34 | static inline void plugin_set_disabled(struct plugind *cd) { |
| 35 | spinlock_lock(&cd->unsafe.spinlock); |
| 36 | cd->unsafe.enabled = false; |
| 37 | spinlock_unlock(&cd->unsafe.spinlock); |
| 38 | } |
| 39 | |
| 40 | bool plugin_is_enabled(struct plugind *cd) { |
| 41 | spinlock_lock(&cd->unsafe.spinlock); |
| 42 | bool ret = cd->unsafe.enabled; |
| 43 | spinlock_unlock(&cd->unsafe.spinlock); |
| 44 | return ret; |
| 45 | } |
| 46 | |
| 47 | static inline void plugin_set_running(struct plugind *cd) { |
| 48 | spinlock_lock(&cd->unsafe.spinlock); |
| 49 | cd->unsafe.running = true; |
| 50 | spinlock_unlock(&cd->unsafe.spinlock); |
| 51 | } |
| 52 | |
| 53 | static inline bool plugin_is_running(struct plugind *cd) { |
| 54 | spinlock_lock(&cd->unsafe.spinlock); |
| 55 | bool ret = cd->unsafe.running; |
| 56 | spinlock_unlock(&cd->unsafe.spinlock); |
| 57 | return ret; |
| 58 | } |
| 59 | |
| 60 | #define SERIAL_FAILURES_THRESHOLD 10 |
| 61 | static void pluginsd_worker_thread_handle_success(struct plugind *cd) { |
| 62 | if (likely(cd->successful_collections)) { |
| 63 | pluginsd_sleep(cd->update_every); |
| 64 | return; |
| 65 | } |
| 66 | |
| 67 | if (likely(cd->serial_failures <= SERIAL_FAILURES_THRESHOLD)) { |
| 68 | netdata_log_info("PLUGINSD: 'host:%s', '%s' (pid %d) does not generate useful output but it reports success (exits with 0). %s.", |
| 69 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, |
| 70 | plugin_is_enabled(cd) ? "Waiting a bit before starting it again." : "Will not start it again - it is now disabled."); |
| 71 | |
| 72 | pluginsd_sleep(cd->update_every * 10); |
| 73 | return; |
| 74 | } |
| 75 | |
| 76 | if (cd->serial_failures > SERIAL_FAILURES_THRESHOLD) { |
| 77 | netdata_log_error("PLUGINSD: 'host:'%s', '%s' (pid %d) does not generate useful output, " |
| 78 | "although it reports success (exits with 0)." |
| 79 | "We have tried to collect something %zu times - unsuccessfully. Disabling it.", |
| 80 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, cd->serial_failures); |
| 81 | plugin_set_disabled(cd); |
| 82 | return; |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | static void pluginsd_worker_thread_handle_error(struct plugind *cd, int worker_ret_code) { |
| 87 | if (worker_ret_code == -1) { |
| 88 | netdata_log_info("PLUGINSD: 'host:%s', '%s' (pid %d) exited abnormally. Disabling it.", |
| 89 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid); |
| 90 | plugin_set_disabled(cd); |
| 91 | return; |
| 92 | } |
| 93 | |
| 94 | if (!cd->successful_collections) { |
| 95 | netdata_log_error("PLUGINSD: 'host:%s', '%s' (pid %d) exited with error code %d and haven't collected any data. Disabling it.", |
| 96 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, worker_ret_code); |
| 97 | plugin_set_disabled(cd); |
| 98 | return; |
| 99 | } |
| 100 | |
| 101 | if (cd->serial_failures <= SERIAL_FAILURES_THRESHOLD) { |
| 102 | netdata_log_error("PLUGINSD: 'host:%s', '%s' (pid %d) exited with error code %d, but has given useful output in the past (%zu times). %s", |
| 103 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, worker_ret_code, cd->successful_collections, |
| 104 | plugin_is_enabled(cd) ? "Waiting a bit before starting it again." : "Will not start it again - it is disabled."); |
| 105 | |
| 106 | pluginsd_sleep(cd->update_every * 10); |
| 107 | return; |
| 108 | } |
| 109 | |
| 110 | if (cd->serial_failures > SERIAL_FAILURES_THRESHOLD) { |
| 111 | netdata_log_error("PLUGINSD: 'host:%s', '%s' (pid %d) exited with error code %d, but has given useful output in the past (%zu times)." |
| 112 | "We tried to restart it %zu times, but it failed to generate data. Disabling it.", |
| 113 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, worker_ret_code, |
| 114 | cd->successful_collections, cd->serial_failures); |
| 115 | plugin_set_disabled(cd); |
| 116 | return; |
| 117 | } |
| 118 | } |
| 119 | |
| 120 | #undef SERIAL_FAILURES_THRESHOLD |
| 121 | |
| 122 | |
| 123 | static void pluginsd_worker_thread(void *arg) { |
| 124 | struct plugind *cd = (struct plugind *) arg; |
| 125 | |
| 126 | worker_register("PLUGINSD"); |
| 127 | |
| 128 | plugin_set_running(cd); |
| 129 | |
| 130 | size_t count = 0; |
| 131 | |
| 132 | while(service_running(SERVICE_COLLECTORS)) { |
| 133 | cd->unsafe.pi = spawn_popen_run(string2str(cd->cmd)); |
| 134 | if(!cd->unsafe.pi) { |
| 135 | netdata_log_error("PLUGINSD: 'host:%s', cannot popen(\"%s\", \"r\").", |
| 136 | rrdhost_hostname(cd->host), string2str(cd->cmd)); |
| 137 | break; |
| 138 | } |
| 139 | cd->unsafe.pid = spawn_popen_pid(cd->unsafe.pi); |
| 140 | |
| 141 | nd_log(NDLS_DAEMON, NDLP_DEBUG, |
| 142 | "PLUGINSD: 'host:%s' connected to '%s' running on pid %d", |
| 143 | rrdhost_hostname(cd->host), |
| 144 | string2str(cd->fullfilename), cd->unsafe.pid); |
| 145 | |
| 146 | const char *plugin = strrchr(string2str(cd->fullfilename), '/'); |
| 147 | if(plugin) |
| 148 | plugin++; |
| 149 | else |
| 150 | plugin = string2str(cd->fullfilename); |
| 151 | |
| 152 | char module[100]; |
| 153 | snprintfz(module, sizeof(module), "plugins.d[%s]", plugin); |
| 154 | ND_LOG_STACK lgs[] = { |
| 155 | ND_LOG_FIELD_TXT(NDF_MODULE, module), |
| 156 | ND_LOG_FIELD_TXT(NDF_NIDL_NODE, rrdhost_hostname(cd->host)), |
| 157 | ND_LOG_FIELD_TXT(NDF_SRC_TRANSPORT, "pluginsd"), |
| 158 | ND_LOG_FIELD_END(), |
| 159 | }; |
| 160 | ND_LOG_STACK_PUSH(lgs); |
| 161 | |
| 162 | count = pluginsd_process(cd->host, cd, |
| 163 | spawn_popen_read_fd(cd->unsafe.pi), |
| 164 | spawn_popen_write_fd(cd->unsafe.pi), |
| 165 | 0); |
| 166 | |
| 167 | nd_log(NDLS_COLLECTORS, NDLP_WARNING, |
| 168 | "PLUGINSD: 'host:%s', '%s' (pid %d) disconnected after %zu successful data collections.", |
| 169 | rrdhost_hostname(cd->host), string2str(cd->fullfilename), cd->unsafe.pid, count); |
| 170 | |
| 171 | int worker_ret_code = spawn_popen_kill(cd->unsafe.pi, 3 * MSEC_PER_SEC); |
| 172 | cd->unsafe.pi = NULL; |
| 173 | |
| 174 | if(likely(worker_ret_code == 0)) |
| 175 | pluginsd_worker_thread_handle_success(cd); |
| 176 | else |
| 177 | pluginsd_worker_thread_handle_error(cd, worker_ret_code); |
| 178 | |
| 179 | cd->unsafe.pid = 0; |
| 180 | |
| 181 | if(unlikely(!plugin_is_enabled(cd))) |
| 182 | break; |
| 183 | } |
| 184 | |
| 185 | spinlock_lock(&cd->unsafe.spinlock); |
| 186 | |
| 187 | cd->unsafe.running = false; |
| 188 | cd->unsafe.pid = 0; |
| 189 | |
| 190 | POPEN_INSTANCE *pi = cd->unsafe.pi; |
| 191 | cd->unsafe.pi = NULL; |
| 192 | |
| 193 | spinlock_unlock(&cd->unsafe.spinlock); |
| 194 | |
| 195 | if (pi) |
| 196 | spawn_popen_kill(pi, 3 * MSEC_PER_SEC); |
| 197 | |
| 198 | worker_unregister(); |
| 199 | } |
| 200 | |
| 201 | static void pluginsd_main_cleanup(void *pptr) { |
| 202 | struct netdata_static_thread *static_thread = CLEANUP_FUNCTION_GET_PTR(pptr); |
| 203 | if(!static_thread) return; |
| 204 | |
| 205 | static_thread->enabled = NETDATA_MAIN_THREAD_EXITING; |
| 206 | netdata_log_info("PLUGINSD: cleaning up..."); |
| 207 | |
| 208 | struct plugind *cd = pluginsd_root; |
| 209 | while(cd) { |
| 210 | struct plugind *next = cd->next; |
| 211 | |
| 212 | spinlock_lock(&cd->unsafe.spinlock); |
| 213 | if (cd->unsafe.enabled && cd->unsafe.running && cd->unsafe.thread != 0) { |
| 214 | netdata_log_info("PLUGINSD: 'host:%s', stopping plugin thread: %s", |
| 215 | rrdhost_hostname(cd->host), string2str(cd->id)); |
| 216 | |
| 217 | nd_thread_signal_cancel(cd->unsafe.thread); |
| 218 | } |
| 219 | |
| 220 | DOUBLE_LINKED_LIST_REMOVE_ITEM_UNSAFE(pluginsd_root, cd, prev, next); |
| 221 | spinlock_unlock(&cd->unsafe.spinlock); |
| 222 | |
| 223 | if(cd->unsafe.thread) { |
| 224 | nd_thread_signal_cancel(cd->unsafe.thread); |
| 225 | nd_thread_join(cd->unsafe.thread); |
| 226 | cd->unsafe.thread = NULL; |
| 227 | } |
| 228 | |
| 229 | string_freez(cd->fullfilename); |
| 230 | string_freez(cd->filename); |
| 231 | string_freez(cd->id); |
| 232 | string_freez(cd->cmd); |
| 233 | freez(cd); |
| 234 | |
| 235 | cd = next; |
| 236 | } |
| 237 | |
| 238 | netdata_log_info("PLUGINSD: cleanup completed."); |
| 239 | static_thread->enabled = NETDATA_MAIN_THREAD_EXITED; |
| 240 | |
| 241 | worker_unregister(); |
| 242 | } |
| 243 | |
| 244 | static bool is_plugin(char *dst, size_t dst_size, const char *filename) { |
| 245 | #if defined(OS_WINDOWS) |
| 246 | const char *suffixes[] = { |
| 247 | ".plugin.exe", |
| 248 | "_plugin.exe", |
| 249 | "-plugin.exe", |
| 250 | ".plugin", |
| 251 | NULL |
| 252 | }; |
| 253 | #else |
| 254 | const char *suffixes[] = { |
| 255 | ".plugin", |
| 256 | "_plugin", |
| 257 | "-plugin", |
| 258 | NULL |
| 259 | }; |
| 260 | #endif |
| 261 | |
| 262 | size_t filename_len = strlen(filename); |
| 263 | |
| 264 | for (int i = 0; suffixes[i] != NULL; i++) { |
| 265 | size_t suffix_len = strlen(suffixes[i]); |
| 266 | |
| 267 | if (filename_len > suffix_len && |
| 268 | strcmp(suffixes[i], &filename[filename_len - suffix_len]) == 0) { |
| 269 | snprintfz(dst, dst_size, "%.*s", (int)(filename_len - suffix_len), filename); |
| 270 | return true; |
| 271 | } |
| 272 | } |
| 273 | |
| 274 | return false; |
| 275 | } |
| 276 | |
| 277 | void *pluginsd_main(void *ptr) { |
| 278 | CLEANUP_FUNCTION_REGISTER(pluginsd_main_cleanup) cleanup_ptr = ptr; |
| 279 | |
| 280 | int automatic_run = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_PLUGINS, "enable running new plugins", 1); |
| 281 | int scan_frequency = (int)inicfg_get_duration_seconds(&netdata_config, CONFIG_SECTION_PLUGINS, "check for new plugins every", 60); |
| 282 | if (scan_frequency < 1) |
| 283 | scan_frequency = 1; |
| 284 | |
| 285 | // disable some plugins by default |
| 286 | inicfg_get_boolean(&netdata_config, CONFIG_SECTION_PLUGINS, "slabinfo", CONFIG_BOOLEAN_NO); |
| 287 | // it crashes (both threads) on Alpine after we made it multi-threaded |
| 288 | // works with "--device /dev/ipmi0", but this is not default |
| 289 | // see https://github.com/netdata/netdata/pull/15564 for details |
| 290 | if (getenv("NETDATA_LISTENER_PORT")) |
| 291 | inicfg_get_boolean(&netdata_config, CONFIG_SECTION_PLUGINS, "freeipmi", CONFIG_BOOLEAN_NO); |
| 292 | |
| 293 | // store the errno for each plugins directory |
| 294 | // so that we don't log broken directories on each loop |
| 295 | int directory_errors[PLUGINSD_MAX_DIRECTORIES] = { 0 }; |
| 296 | |
| 297 | while (service_running(SERVICE_COLLECTORS)) { |
| 298 | int idx; |
| 299 | const char *directory_name; |
| 300 | |
| 301 | for (idx = 0; idx < PLUGINSD_MAX_DIRECTORIES && (directory_name = plugin_directories[idx]); idx++) { |
| 302 | if (unlikely(!service_running(SERVICE_COLLECTORS))) |
| 303 | break; |
| 304 | |
| 305 | errno_clear(); |
| 306 | DIR *dir = opendir(directory_name); |
| 307 | if (unlikely(!dir)) { |
| 308 | if (directory_errors[idx] != errno) { |
| 309 | directory_errors[idx] = errno; |
| 310 | netdata_log_error("cannot open plugins directory '%s'", directory_name); |
| 311 | } |
| 312 | continue; |
| 313 | } |
| 314 | |
| 315 | struct dirent *file = NULL; |
| 316 | while (likely((file = readdir(dir)))) { |
| 317 | if (unlikely(!service_running(SERVICE_COLLECTORS))) |
| 318 | break; |
| 319 | |
| 320 | netdata_log_debug(D_PLUGINSD, "examining file '%s'", file->d_name); |
| 321 | |
| 322 | if (unlikely(strcmp(file->d_name, ".") == 0 || strcmp(file->d_name, "..") == 0)) |
| 323 | continue; |
| 324 | |
| 325 | char pluginname[CONFIG_MAX_NAME + 1]; |
| 326 | if(!is_plugin(pluginname, sizeof(pluginname), file->d_name)) { |
| 327 | netdata_log_debug(D_PLUGINSD, "file '%s' does not look like a plugin", file->d_name); |
| 328 | continue; |
| 329 | } |
| 330 | |
| 331 | int enabled = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_PLUGINS, pluginname, automatic_run); |
| 332 | if (unlikely(!enabled)) { |
| 333 | netdata_log_debug(D_PLUGINSD, "plugin '%s' is not enabled", file->d_name); |
| 334 | continue; |
| 335 | } |
| 336 | |
| 337 | // check if it runs already |
| 338 | struct plugind *cd; |
| 339 | for (cd = pluginsd_root; cd; cd = cd->next) { |
| 340 | if (unlikely(strcmp(string2str(cd->filename), file->d_name) == 0)) { |
| 341 | break; |
| 342 | } |
| 343 | } |
| 344 | |
| 345 | if(cd) { |
| 346 | if (likely(plugin_is_running(cd))) { |
| 347 | netdata_log_debug(D_PLUGINSD, "plugin '%s' is already running", string2str(cd->filename)); |
| 348 | continue; |
| 349 | } |
| 350 | else if(cd->unsafe.thread) { |
| 351 | netdata_log_debug(D_PLUGINSD, "plugin '%s' gave up", string2str(cd->filename)); |
| 352 | nd_thread_signal_cancel(cd->unsafe.thread); |
| 353 | nd_thread_join(cd->unsafe.thread); |
| 354 | cd->unsafe.thread = NULL; |
| 355 | } |
| 356 | } |
| 357 | |
| 358 | // it is not running |
| 359 | // allocate a new one, or use the obsolete one |
| 360 | if (unlikely(!cd)) { |
| 361 | cd = callocz(sizeof(struct plugind), 1); |
| 362 | |
| 363 | { |
| 364 | char buf[CONFIG_MAX_NAME]; |
| 365 | snprintfz(buf, sizeof(buf), "plugin:%s", pluginname); |
| 366 | string_freez(cd->id); |
| 367 | cd->id = string_strdupz(buf); |
| 368 | } |
| 369 | |
| 370 | { |
| 371 | char buf[FILENAME_MAX + 1]; |
| 372 | strncpyz(buf, file->d_name, sizeof(buf) - 1); |
| 373 | string_freez(cd->filename); |
| 374 | cd->filename = string_strdupz(buf); |
| 375 | |
| 376 | snprintfz(buf, sizeof(buf), "%s/%s", directory_name, string2str(cd->filename)); |
| 377 | string_freez(cd->fullfilename); |
| 378 | cd->fullfilename = string_strdupz(buf); |
| 379 | } |
| 380 | |
| 381 | cd->host = localhost; |
| 382 | cd->unsafe.enabled = enabled; |
| 383 | cd->unsafe.running = false; |
| 384 | |
| 385 | cd->update_every = (int)inicfg_get_duration_seconds(&netdata_config, string2str(cd->id), "update every", localhost->rrd_update_every); |
| 386 | cd->started_t = now_realtime_sec(); |
| 387 | |
| 388 | { |
| 389 | const char *def = ""; |
| 390 | char buf[PLUGINSD_CMD_MAX + 1]; |
| 391 | |
| 392 | snprintfz( |
| 393 | buf, sizeof(buf), "exec %s %d %s", string2str(cd->fullfilename), |
| 394 | cd->update_every, inicfg_get(&netdata_config, string2str(cd->id), "command options", def)); |
| 395 | |
| 396 | string_freez(cd->cmd); |
| 397 | cd->cmd = string_strdupz(buf); |
| 398 | } |
| 399 | |
| 400 | // link it |
| 401 | DOUBLE_LINKED_LIST_PREPEND_ITEM_UNSAFE(pluginsd_root, cd, prev, next); |
| 402 | |
| 403 | if (plugin_is_enabled(cd)) { |
| 404 | char tag[NETDATA_THREAD_TAG_MAX + 1]; |
| 405 | snprintfz(tag, NETDATA_THREAD_TAG_MAX, "PD[%s]", pluginname); |
| 406 | |
| 407 | // spawn a new thread for it |
| 408 | cd->unsafe.thread = nd_thread_create(tag, NETDATA_THREAD_OPTION_DEFAULT, |
| 409 | pluginsd_worker_thread, cd); |
| 410 | } |
| 411 | } |
| 412 | } |
| 413 | |
| 414 | closedir(dir); |
| 415 | } |
| 416 | |
| 417 | pluginsd_sleep(scan_frequency); |
| 418 | } |
| 419 | |
| 420 | service_exits(); |
| 421 | return NULL; |
| 422 | } |