@cryptotaxi247 / netdata-1 / commits / bf6753d86

added worker last job id to status file (#19992)

* added worker last job id to status file * added worker last job id to sentry * send sentry breadcrumb on all events * fix potential crash in pluginsd_acquire_dimension() * verify uuids loaded from sql are valid * add fatal conditions for missing journalfile, datafile, and datafile->ctx * make sure we check for datafile->ctx on all uses of it * add magic numbers to datafiles to find the race condition

Costa Tsaousis committed Mar 29, 2025 at 15:16 UTC bf6753d86b1bf15fd30af51327affc0554e6c624

12 files changed +128 -74

src/daemon/daemon-status-file.c

+19

@@ -110,6 +110,7 @@ static uint64_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *msg,

110 ND_PROFILE profile;

111 EXIT_REASON exit_reason;

112 RRD_DB_MODE db_mode;

113 + uint32_t worker_job_id;

114 uint8_t db_tiers;

115 bool kubernetes;

116 bool sentry_available;

@@ -143,6 +144,7 @@ static uint64_t daemon_status_file_hash(DAEMON_STATUS_FILE *ds, const char *msg,

144 to_hash.sentry_fatal = ds->fatal.sentry,

145 to_hash.host_id = ds->host_id,

146 to_hash.machine_id = ds->machine_id,

147 + to_hash.worker_job_id = ds->fatal.worker_job_id,

148

149 strncpyz(to_hash.version, ds->version, sizeof(to_hash.version) - 1);

150 strncpyz(to_hash.filename, ds->fatal.filename, sizeof(to_hash.filename) - 1);

@@ -304,6 +306,9 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {

306 print_uint64_hex(buf, ds->fatal.fault_address);

307 buffer_json_member_add_string(wb, "fault_address", buf);

308 }

309 +

310 + if(ds->v >= 23)

311 + buffer_json_member_add_uint64(wb, "worker_job_id", ds->fatal.worker_job_id);

312 }

313 buffer_json_object_close(wb);

314

@@ -354,6 +359,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

359 bool required_v20 = version >= 20 ? strict : false;

360 bool required_v21 = version >= 21 ? strict : false;

361 bool required_v22 = version >= 22 ? strict : false;

362 + bool required_v23 = version >= 23 ? strict : false;

363

364 // Parse timestamp

365 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v1);

@@ -498,6 +504,9 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *

504 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "fault_address", buf, error, required_v18);

505 ds->fatal.fault_address = str2ull_encoded(buf);

506 }

507 +

508 + if(version >= 23)

509 + JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "worker_job_id", SIGNAL_CODE_2id_h, ds->fatal.worker_job_id, error, required_v23);

510 });

511

512 // Parse the last posted object

@@ -1474,6 +1483,9 @@ void daemon_status_file_register_fatal(const char *filename, const char *functio

1483 if(stack_trace && *stack_trace && stack_trace_is_empty(&session_status))

1484 strncpyz(session_status.fatal.stack_trace, stack_trace, sizeof(session_status.fatal.stack_trace) - 1);

1485

1486 + if(!session_status.fatal.worker_job_id)

1487 + session_status.fatal.worker_job_id = workers_get_last_job_id();

1488 +

1489 if(line)

1490 session_status.fatal.line = line;

1491

@@ -1541,6 +1553,9 @@ bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE c

1553 if(!session_status.fatal.thread_id)

1554 session_status.fatal.thread_id = gettid_cached();

1555

1556 + if(!session_status.fatal.worker_job_id)

1557 + session_status.fatal.worker_job_id = workers_get_last_job_id();

1558 +

1559 copy_and_clean_thread_name_if_empty(&session_status, nd_thread_tag_async_safe());

1560

1561 dsf_release(session_status);

@@ -1749,3 +1764,7 @@ ND_UUID daemon_status_file_get_host_id(void) {

1764 else

1765 return last_session_status.host_id;

1766 }

1767 +

1768 +size_t daemon_status_file_get_fatal_worker_job_id(void) {

1769 + return session_status.fatal.worker_job_id;

1770 +}

src/daemon/daemon-status-file.h

+4 -2

@@ -8,7 +8,7 @@

8 #include "database/rrd-database-mode.h"

9 #include "claim/cloud-status.h"

11 -#define STATUS_FILE_VERSION 22

11 +#define STATUS_FILE_VERSION 23

13 typedef enum {

14 DAEMON_STATUS_NONE,

@@ -93,11 +93,12 @@ typedef struct daemon_status_file {

93 char function[128];

94 char errno_str[64];

95 char message[512];

96 - char stack_trace[2048];

96 + char stack_trace[4096];

97 char thread[ND_THREAD_TAG_MAX + 1];

98 pid_t thread_id;

99 SIGNAL_CODE signal_code;

100 uintptr_t fault_address;

101 + uint32_t worker_job_id;

102 bool sentry; // true when the error was also reported to sentry

103 } fatal;

104

@@ -155,5 +156,6 @@ DAEMON_STATUS daemon_status_file_get_status(void);

156 size_t daemon_status_file_get_restarts(void);

157 ssize_t daemon_status_file_get_reliability(void);

158 ND_UUID daemon_status_file_get_host_id(void);

159 +size_t daemon_status_file_get_fatal_worker_job_id(void);

160

161 #endif //NETDATA_DAEMON_STATUS_FILE_H

src/daemon/sentry-native/sentry-native.c

+30 -44

@@ -5,10 +5,13 @@

6 #include "sentry.h"

8 +static void nd_sentry_add_deadly_signal_as_breadcrumb(void);

9 +

10 static char sentry_path[FILENAME_MAX] = "";

9 -static char g_sentry_event_message[512] = {0};

11 -bool nd_sentry_crash_report_enabled = true;

12 +static bool sentry_initialized = false;

13 +static bool breadcrumb_added = false;

14 +static bool nd_sentry_crash_report_enabled = true;

16 const char *nd_sentry_path(void) {

17 return sentry_path;

@@ -86,14 +89,7 @@ static sentry_value_t nd_sentry_on_hook(sentry_value_t event) {

89 return sentry_value_new_null();

90 }

89 - nd_sentry_set_tag_status();

90 - nd_sentry_set_tag_uptime();

91 - nd_sentry_set_tag("thread", daemon_status_file_get_fatal_thread());

92 - nd_sentry_set_tag_uint64("thread_id", daemon_status_file_get_fatal_thread_id());

93 -

94 - // set the title of the event

95 - if(g_sentry_event_message[0])

96 - sentry_value_set_by_key(event, "message", sentry_value_new_string(g_sentry_event_message));

92 + nd_sentry_add_deadly_signal_as_breadcrumb();

94 return event;

95 }

@@ -116,7 +112,7 @@ static sentry_value_t nd_sentry_before_send(

112 // sentry initialization

113

114 void nd_sentry_init(void) {

119 - if (!analytics_check_enabled())

115 + if (!analytics_check_enabled() || sentry_initialized)

116 return;

117

118 // path where sentry should save stuff

@@ -186,10 +182,12 @@ void nd_sentry_init(void) {

182 nd_sentry_set_tag_uint64("restarts", daemon_status_file_get_restarts());

183 nd_sentry_set_tag_int64("reliability", daemon_status_file_get_reliability());

184 nd_sentry_set_tag("stack_traces", daemon_status_file_get_stack_trace_backend());

185 +

186 + sentry_initialized = true;

187 }

188

189 void nd_sentry_fini(void) {

192 - if (!analytics_check_enabled())

190 + if(!sentry_initialized)

191 return;

192

193 sentry_close();

@@ -231,17 +229,20 @@ static void nd_sentry_add_key_value_uint64(sentry_value_t data, const char *key,

229 sentry_value_set_by_key(data, key, sentry_value_new_string(buf));

230 }

231

234 -void nd_sentry_add_fatal_message_as_breadcrumb(void) {

235 - if (!analytics_check_enabled())

232 +void nd_sentry_add_breadcrumb(const char *category, const char *message) {

233 + if(!sentry_initialized || breadcrumb_added)

234 return;

235

236 + nd_sentry_set_tag_status();

237 + nd_sentry_set_tag_uptime();

238 +

239 + nd_sentry_set_tag("thread", daemon_status_file_get_fatal_thread());

240 + nd_sentry_set_tag_uint64("thread_id", daemon_status_file_get_fatal_thread_id());

241 + nd_sentry_set_tag_uint64("worker_job_id", daemon_status_file_get_fatal_worker_job_id());

242 +

243 const char *function = daemon_status_file_get_fatal_function();

244 if(!function || !*function)

240 - function = "unknown";

241 - else

242 - strncpyz(g_sentry_event_message, function, sizeof(g_sentry_event_message) - 1);

243 -

244 - nd_sentry_set_tag_uptime();

245 + function = category;

246

247 // Set the transaction name to the function where the error occurred

248 // this should be low cardinality

@@ -250,7 +251,7 @@ void nd_sentry_add_fatal_message_as_breadcrumb(void) {

251 // Set the fingerprint to the function where the error occurred

252 sentry_set_fingerprint("{{ default }}", function, NULL);

253

253 - sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", "fatal() event details");

254 + sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", message);

255

256 sentry_value_t data = sentry_value_new_object();

257 nd_sentry_add_key_value_charp(data, "message", daemon_status_file_get_fatal_message());

@@ -262,35 +263,20 @@ void nd_sentry_add_fatal_message_as_breadcrumb(void) {

263 nd_sentry_add_key_value_charp(data, "errno", daemon_status_file_get_fatal_errno());

264 nd_sentry_add_key_value_charp(data, "stack_trace", daemon_status_file_get_fatal_stack_trace());

265 nd_sentry_add_key_value_charp(data, "status", DAEMON_STATUS_2str(daemon_status_file_get_status()));

266 + nd_sentry_add_key_value_uint64(data, "worker_job_id", daemon_status_file_get_fatal_worker_job_id());

267

268 sentry_value_set_by_key(crumb, "data", data);

269 sentry_add_breadcrumb(crumb);

270 }

271

270 -void nd_sentry_add_shutdown_timeout_as_breadcrumb(void) {

271 - if (!analytics_check_enabled())

272 - return;

273 -

274 - const char *function = "shutdown_timeout";

275 - strncpyz(g_sentry_event_message, function, sizeof(g_sentry_event_message) - 1);

276 -

277 - nd_sentry_set_tag_uptime();

278 -

279 - // Set the transaction name to the function where the error occurred

280 - // this should be low cardinality

281 - sentry_set_transaction(function);

282 -

283 - // Set the fingerprint to the function where the error occurred

284 - sentry_set_fingerprint("{{ default }}", function, NULL);

285 -

286 - sentry_value_t crumb = sentry_value_new_breadcrumb("fatal", "shutdown_timeout() event details");

272 +void nd_sentry_add_fatal_message_as_breadcrumb(void) {

273 + nd_sentry_add_breadcrumb("fatal", "fatal message event details");

274 +}

275

288 - sentry_value_t data = sentry_value_new_object();

289 - nd_sentry_add_key_value_charp(data, "function", function);

290 - nd_sentry_add_key_value_charp(data, "thread", nd_thread_tag());

291 - nd_sentry_add_key_value_uint64(data, "thread_id", gettid_cached());

292 - nd_sentry_add_key_value_charp(data, "status", DAEMON_STATUS_2str(daemon_status_file_get_status()));

276 +static void nd_sentry_add_deadly_signal_as_breadcrumb(void) {

277 + nd_sentry_add_breadcrumb("deadly_signal", "deadly signal event details");

278 +}

279

294 - sentry_value_set_by_key(crumb, "data", data);

295 - sentry_add_breadcrumb(crumb);

280 +void nd_sentry_add_shutdown_timeout_as_breadcrumb(void) {

281 + nd_sentry_add_breadcrumb("shutdown_timeout", "shutdown timeout event details");

282 }

src/database/engine/datafile.c

+15 -11

@@ -28,6 +28,7 @@ static struct rrdengine_datafile *datafile_alloc_and_init(struct rrdengine_insta

28 datafile->fileno = fileno;

29 fatal_assert(0 == uv_rwlock_init(&datafile->extent_rwlock));

30 datafile->ctx = ctx;

31 + datafile->magic1 = datafile->magic2 = DATAFILE_MAGIC;

33 datafile->users.available = true;

@@ -94,8 +95,8 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdo

95 // count the number of pages referencing this in the open cache

96 spinlock_unlock(&df->users.spinlock);

97 usec_t time_to_scan_ut = now_monotonic_usec();

97 - size_t clean_pages_in_open_cache = pgc_count_clean_pages_having_data_ptr(open_cache, (Word_t)df->ctx, df);

98 - size_t hot_pages_in_open_cache = pgc_count_hot_pages_having_data_ptr(open_cache, (Word_t)df->ctx, df);

98 + size_t clean_pages_in_open_cache = pgc_count_clean_pages_having_data_ptr(open_cache, (Word_t)datafile_ctx(df), df);

99 + size_t hot_pages_in_open_cache = pgc_count_hot_pages_having_data_ptr(open_cache, (Word_t)datafile_ctx(df), df);

100 time_to_scan_ut = now_monotonic_usec() - time_to_scan_ut;

101 spinlock_lock(&df->users.spinlock);

102

@@ -115,7 +116,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdo

116 "%zu clean and %zu hot open cache pages "

117 "- will be deleted shortly "

118 "(scanned open cache in %"PRIu64" usecs)",

118 - df->fileno, df->ctx->config.tier,

119 + df->fileno, datafile_ctx(df)->config.tier,

120 df->users.lockers,

121 df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],

122 df->users.lockers_by_reason[DATAFILE_ACQUIRE_PAGE_DETAILS],

@@ -132,7 +133,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdo

133 "%zu clean and %zu hot open cache pages "

134 "- will be deleted now "

135 "(scanned open cache in %"PRIu64" usecs)",

135 - df->fileno, df->ctx->config.tier,

136 + df->fileno, datafile_ctx(df)->config.tier,

137 df->users.lockers,

138 df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],

139 df->users.lockers_by_reason[DATAFILE_ACQUIRE_PAGE_DETAILS],

@@ -146,7 +147,7 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdo

147 "has %u lockers (oc:%u, pd:%u), "

148 "%zu clean and %zu hot open cache pages "

149 "(scanned open cache in %"PRIu64" usecs)",

149 - df->fileno, df->ctx->config.tier,

150 + df->fileno, datafile_ctx(df)->config.tier,

151 df->users.lockers,

152 df->users.lockers_by_reason[DATAFILE_ACQUIRE_OPEN_CACHE],

153 df->users.lockers_by_reason[DATAFILE_ACQUIRE_PAGE_DETAILS],

@@ -163,12 +164,12 @@ bool datafile_acquire_for_deletion(struct rrdengine_datafile *df, bool is_shutdo

164 void generate_datafilepath(struct rrdengine_datafile *datafile, char *str, size_t maxlen)

165 {

166 (void) snprintfz(str, maxlen - 1, "%s/" DATAFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL DATAFILE_EXTENSION,

166 - datafile->ctx->config.dbfiles_path, datafile->tier, datafile->fileno);

167 + datafile_ctx(datafile)->config.dbfiles_path, datafile->tier, datafile->fileno);

168 }

169

170 int close_data_file(struct rrdengine_datafile *datafile)

171 {

171 - struct rrdengine_instance *ctx = datafile->ctx;

172 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

173 uv_fs_t req;

174 int ret;

175 char path[RRDENG_PATH_MAX];

@@ -187,7 +188,7 @@ int close_data_file(struct rrdengine_datafile *datafile)

188

189 int unlink_data_file(struct rrdengine_datafile *datafile)

190 {

190 - struct rrdengine_instance *ctx = datafile->ctx;

191 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

192 uv_fs_t req;

193 int ret;

194 char path[RRDENG_PATH_MAX];

@@ -208,7 +209,7 @@ int unlink_data_file(struct rrdengine_datafile *datafile)

209

210 int destroy_data_file_unsafe(struct rrdengine_datafile *datafile)

211 {

211 - struct rrdengine_instance *ctx = datafile->ctx;

212 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

213 uv_fs_t req;

214 int ret;

215 char path[RRDENG_PATH_MAX];

@@ -243,7 +244,7 @@ int destroy_data_file_unsafe(struct rrdengine_datafile *datafile)

244

245 int create_data_file(struct rrdengine_datafile *datafile)

246 {

246 - struct rrdengine_instance *ctx = datafile->ctx;

247 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

248 uv_fs_t req;

249 uv_file file;

250 int ret, fd;

@@ -321,7 +322,7 @@ static int check_data_file_superblock(uv_file file)

322

323 static int load_data_file(struct rrdengine_datafile *datafile)

324 {

324 - struct rrdengine_instance *ctx = datafile->ctx;

325 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

326 uv_fs_t req;

327 uv_file file;

328 int ret, fd, error;

@@ -601,6 +602,9 @@ void finalize_data_files(struct rrdengine_instance *ctx)

602 spinlock_unlock(&datafile->writers.spinlock);

603 uv_rwlock_wrunlock(&ctx->datafiles.rwlock);

604

605 + memset(journalfile, 0, sizeof(*journalfile));

606 + memset(datafile, 0, sizeof(*datafile));

607 +

608 freez(journalfile);

609 freez(datafile);

610

src/database/engine/datafile.h

+17

@@ -44,8 +44,12 @@ typedef struct {

44 struct extent_page_details_list *base;

45 } EPDL_EXTENT;

47 +#define DATAFILE_MAGIC 0xDA7AF11E

48 +

49 /* only one event loop is supported for now */

50 struct rrdengine_datafile {

51 + uint32_t magic1;

52 +

53 unsigned tier;

54 unsigned fileno;

55 uv_file file;

@@ -79,6 +83,8 @@ struct rrdengine_datafile {

83 RW_SPINLOCK spinlock;

84 Pvoid_t epdl_per_extent;

85 } extent_epdl;

86 +

87 + uint32_t magic2;

88 };

90 bool datafile_acquire(struct rrdengine_datafile *df, DATAFILE_ACQUIRE_REASONS reason);

@@ -97,4 +103,15 @@ int create_new_datafile_pair(struct rrdengine_instance *ctx, bool having_lock);

103 int init_data_files(struct rrdengine_instance *ctx);

104 void finalize_data_files(struct rrdengine_instance *ctx);

105

106 +NEVERNULL ALWAYS_INLINE

107 +static struct rrdengine_instance *datafile_ctx(struct rrdengine_datafile *datafile) {

108 + if(unlikely(!datafile->ctx))

109 + fatal("DBENGINE: datafile %u of tier %u has no ctx", datafile->fileno, datafile->tier);

110 +

111 + if(unlikely(datafile->magic1 != DATAFILE_MAGIC || datafile->magic2 != DATAFILE_MAGIC))

112 + fatal("DBENGINE: datafile %u of tier %u has invalid magic", datafile->fileno, datafile->tier);

113 +

114 + return datafile->ctx;

115 +}

116 +

117 #endif /* NETDATA_DATAFILE_H */

\ No newline at end of file

src/database/engine/journalfile.c

+26 -14

@@ -61,13 +61,13 @@ void journalfile_v1_extent_write(struct rrdengine_instance *ctx, struct rrdengin

61 void journalfile_v2_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen)

62 {

63 (void) snprintfz(str, maxlen, "%s/" WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION_V2,

64 - datafile->ctx->config.dbfiles_path, datafile->tier, datafile->fileno);

64 + datafile_ctx(datafile)->config.dbfiles_path, datafile->tier, datafile->fileno);

65 }

67 void journalfile_v1_generate_path(struct rrdengine_datafile *datafile, char *str, size_t maxlen)

68 {

69 (void) snprintfz(str, maxlen - 1, "%s/" WALFILE_PREFIX RRDENG_FILE_NUMBER_PRINT_TMPL WALFILE_EXTENSION,

70 - datafile->ctx->config.dbfiles_path, datafile->tier, datafile->fileno);

70 + datafile_ctx(datafile)->config.dbfiles_path, datafile->tier, datafile->fileno);

71 }

73 // ----------------------------------------------------------------------------

@@ -155,15 +155,20 @@ ALWAYS_INLINE struct rrdengine_datafile *njfv2idx_find_and_acquire_j2_header(NJF

155 }

156

157 static void njfv2idx_add(struct rrdengine_datafile *datafile) {

158 + if(unlikely(!datafile))

159 + fatal("DBENGINE: NJFV2IDX trying to index a journal file with no datafile");

160 +

161 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

162 +

163 internal_fatal(datafile->journalfile->v2.last_time_s <= 0, "DBENGINE: NJFV2IDX trying to index a journal file with invalid first_time_s");

164

160 - rw_spinlock_write_lock(&datafile->ctx->njfv2idx.spinlock);

165 + rw_spinlock_write_lock(&ctx->njfv2idx.spinlock);

166 datafile->journalfile->njfv2idx.indexed_as = datafile->journalfile->v2.last_time_s;

167

168 do {

169 internal_fatal(datafile->journalfile->njfv2idx.indexed_as <= 0, "DBENGINE: NJFV2IDX journalfile is already indexed");

170

166 - Pvoid_t *PValue = JudyLIns(&datafile->ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

171 + Pvoid_t *PValue = JudyLIns(&ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

172 if (!PValue || PValue == PJERR)

173 fatal("DBENGINE: NJFV2IDX corrupted judy array");

174

@@ -177,21 +182,22 @@ static void njfv2idx_add(struct rrdengine_datafile *datafile) {

182 }

183 } while(1);

184

180 - rw_spinlock_write_unlock(&datafile->ctx->njfv2idx.spinlock);

185 + rw_spinlock_write_unlock(&ctx->njfv2idx.spinlock);

186 }

187

188 static void njfv2idx_remove(struct rrdengine_datafile *datafile) {

189 internal_fatal(!datafile->journalfile->njfv2idx.indexed_as, "DBENGINE: NJFV2IDX journalfile to remove is not indexed");

190

186 - rw_spinlock_write_lock(&datafile->ctx->njfv2idx.spinlock);

191 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

192 + rw_spinlock_write_lock(&ctx->njfv2idx.spinlock);

193

188 - int rc = JudyLDel(&datafile->ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

194 + int rc = JudyLDel(&ctx->njfv2idx.JudyL, datafile->journalfile->njfv2idx.indexed_as, PJE0);

195 (void)rc;

196 internal_fatal(!rc, "DBENGINE: NJFV2IDX cannot remove entry");

197

198 datafile->journalfile->njfv2idx.indexed_as = 0;

199

194 - rw_spinlock_write_unlock(&datafile->ctx->njfv2idx.spinlock);

200 + rw_spinlock_write_unlock(&ctx->njfv2idx.spinlock);

201 }

202

203 // ----------------------------------------------------------------------------

@@ -214,7 +220,7 @@ static struct journal_v2_header *journalfile_v2_mounted_data_get(struct rrdengin

220 journalfile->v2.flags &= ~(JOURNALFILE_FLAG_IS_AVAILABLE | JOURNALFILE_FLAG_IS_MOUNTED);

221 spinlock_unlock(&journalfile->v2.spinlock);

222

217 - ctx_fs_error(journalfile->datafile->ctx);

223 + ctx_fs_error(datafile_ctx(journalfile->datafile));

224 }

225 else {

226 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.journal_v2_mapped, 1, __ATOMIC_RELAXED);

@@ -276,7 +282,7 @@ static bool journalfile_v2_mounted_data_unmount(struct rrdengine_journalfile *jo

282 journalfile_v2_generate_path(journalfile->datafile, path, sizeof(path));

283 netdata_log_error("DBENGINE: failed to unmap index file '%s'", path);

284 internal_fatal(true, "DBENGINE: failed to unmap file '%s'", path);

279 - ctx_fs_error(journalfile->datafile->ctx);

285 + ctx_fs_error(datafile_ctx(journalfile->datafile));

286 }

287 else {

288 __atomic_add_fetch(&rrdeng_cache_efficiency_stats.journal_v2_unmapped, 1, __ATOMIC_RELAXED);

@@ -406,6 +412,12 @@ size_t journalfile_v2_data_size_get(struct rrdengine_journalfile *journalfile) {

412 }

413

414 void journalfile_v2_data_set(struct rrdengine_journalfile *journalfile, int fd, void *journal_data, uint32_t journal_data_size) {

415 + if(unlikely(!journalfile))

416 + fatal("DBENGINE: JOURNALFILE: trying to set journal data without a journalfile");

417 +

418 + if(unlikely(!journalfile->datafile))

419 + fatal("DBENGINE: JOURNALFILE: trying to set journal data without a datafile");

420 +

421 spinlock_lock(&journalfile->mmap.spinlock);

422 spinlock_lock(&journalfile->v2.spinlock);

423

@@ -488,7 +500,7 @@ static int close_uv_file(struct rrdengine_datafile *datafile, uv_file file)

500 if (ret < 0) {

501 journalfile_v1_generate_path(datafile, path, sizeof(path));

502 netdata_log_error("DBENGINE: uv_fs_close(%s): %s", path, uv_strerror(ret));

491 - ctx_fs_error(datafile->ctx);

503 + ctx_fs_error(datafile_ctx(datafile));

504 }

505 uv_fs_req_cleanup(&req);

506 return ret;

@@ -507,7 +519,7 @@ int journalfile_close(struct rrdengine_journalfile *journalfile, struct rrdengin

519 int journalfile_unlink(struct rrdengine_journalfile *journalfile)

520 {

521 struct rrdengine_datafile *datafile = journalfile->datafile;

510 - struct rrdengine_instance *ctx = datafile->ctx;

522 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

523 uv_fs_t req;

524 int ret;

525 char path[RRDENG_PATH_MAX];

@@ -528,7 +540,7 @@ int journalfile_unlink(struct rrdengine_journalfile *journalfile)

540

541 int journalfile_destroy_unsafe(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile)

542 {

531 - struct rrdengine_instance *ctx = datafile->ctx;

543 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

544 uv_fs_t req;

545 int ret;

546 char path[RRDENG_PATH_MAX];

@@ -572,7 +584,7 @@ int journalfile_destroy_unsafe(struct rrdengine_journalfile *journalfile, struct

584

585 int journalfile_create(struct rrdengine_journalfile *journalfile, struct rrdengine_datafile *datafile)

586 {

575 - struct rrdengine_instance *ctx = datafile->ctx;

587 + struct rrdengine_instance *ctx = datafile_ctx(datafile);

588 uv_fs_t req;

589 uv_file file;

590 int ret, fd;

src/database/engine/rrdengine.c

@@ -1293,6 +1293,9 @@ void datafile_delete(struct rrdengine_instance *ctx, struct rrdengine_datafile *

1293 rw_spinlock_write_unlock(&datafile->extent_epdl.spinlock);

1294 }

1295

1296 + memset(journal_file, 0, sizeof(*journal_file));

1297 + memset(datafile, 0, sizeof(*datafile));

1298 +

1299 freez(journal_file);

1300 freez(datafile);

1301

src/database/engine/rrdengineapi.c

+2 -2

@@ -1337,7 +1337,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S

1337

1338 time_t update_every_s;

1339

1340 - size_t points = descr->page_length / CTX_POINT_SIZE_BYTES(datafile->ctx);

1340 + size_t points = descr->page_length / CTX_POINT_SIZE_BYTES(datafile_ctx(datafile));

1341

1342 time_t start_time_s = journal_start_time_s + descr->delta_start_s;

1343 time_t end_time_s = journal_start_time_s + descr->delta_end_s;

@@ -1345,7 +1345,7 @@ static void populate_v2_statistics(struct rrdengine_datafile *datafile, RRDENG_S

1345 if(likely(points > 1))

1346 update_every_s = (time_t) ((end_time_s - start_time_s) / (points - 1));

1347 else {

1348 - update_every_s = (time_t) (nd_profile.update_every * get_tier_grouping(datafile->ctx->config.tier));

1348 + update_every_s = (time_t) (nd_profile.update_every * get_tier_grouping(datafile_ctx(datafile)->config.tier));

1349 stats->single_point_pages++;

1350 }

1351

src/database/sqlite/sqlite_metadata.c

@@ -2088,6 +2088,8 @@ size_t populate_metrics_from_database(void *mrg, void (*populate_cb)(void *mrg,

2088 usec_t started_ut = now_monotonic_usec();

2089 while (sqlite3_step(res) == SQLITE_ROW) {

2090 nd_uuid_t *uuid = (nd_uuid_t *)sqlite3_column_blob(res, 0);

2091 + if (!uuid || sqlite3_column_bytes(res, 0) != sizeof(nd_uuid_t))

2092 + continue;

2093

2094 for (size_t tier = 0; tier < nd_profile.storage_tiers ; tier++) {

2095 if (unlikely(!multidb_ctx[tier]))

src/libnetdata/worker_utilization/worker_utilization.c

@@ -95,6 +95,11 @@ static struct workers_globals {

95 };

97 static __thread struct worker *worker = NULL; // the current thread worker

98 +static __thread size_t last_job_id = 0;

99 +

100 +size_t workers_get_last_job_id() {

101 + return last_job_id;

102 +}

103

104 static ALWAYS_INLINE usec_t worker_now_monotonic_usec(void) {

105 #ifdef NETDATA_WITHOUT_WORKERS_LATENCY

@@ -256,6 +261,8 @@ static void worker_is_busy_do(size_t job_id) {

261 }

262

263 ALWAYS_INLINE void worker_is_busy(size_t job_id) {

264 + last_job_id = job_id;

265 +

266 if(likely(!worker || job_id >= WORKER_UTILIZATION_MAX_JOB_TYPES))

267 return;

268

src/libnetdata/worker_utilization/worker_utilization.h

@@ -43,6 +43,8 @@ void worker_register_job_name(size_t job_id, const char *name);

43 void worker_register_job_custom_metric(size_t job_id, const char *name, const char *units, WORKER_METRIC_TYPE type);

44 void worker_unregister(void);

46 +size_t workers_get_last_job_id();

47 +

48 void worker_is_idle(void);

49 void worker_is_busy(size_t job_id);

50 void worker_set_metric(size_t job_id, NETDATA_DOUBLE value);

src/plugins.d/pluginsd_internals.h

+1 -1

@@ -219,7 +219,7 @@ static ALWAYS_INLINE RRDDIM *pluginsd_acquire_dimension(RRDHOST *host, RRDSET *s

219 if(likely(rd)) {

220 const char *id = prd->id;

221

222 - if(strcmp(id, dimension) == 0) {

222 + if(id && *id && strcmp(id, dimension) == 0) {

223 // we found it cached

224 return rd;

225 }