@cryptotaxi247 / netdata
Code Commits Issues Pulls Sessions CI/CD
master
master
c 1,738 lines 74.8 KB
Raw
1 // SPDX-License-Identifier: GPL-3.0-or-later
2
3 #include "common.h"
4 #include "status-file.h"
5 #include "buildinfo.h"
6
7 #include <curl/curl.h>
8 #include <openssl/evp.h>
9 #include <openssl/pem.h>
10 #include <openssl/err.h>
11 #include "status-file-dmi.h"
12 #include "status-file-product.h"
13
14 #ifdef ENABLE_SENTRY
15 #include "sentry-native/sentry-native.h"
16 #endif
17
18 #include "status-file-dedup.h"
19 #include "status-file-io.h"
20
21 #define STATUS_FILENAME "status-netdata.json"
22
23 ENUM_STR_MAP_DEFINE(DAEMON_STATUS) = {
24 { DAEMON_STATUS_NONE, "none"},
25 { DAEMON_STATUS_INITIALIZING, "initializing"},
26 { DAEMON_STATUS_RUNNING, "running"},
27 { DAEMON_STATUS_EXITING, "exiting"},
28 { DAEMON_STATUS_EXITED, "exited"},
29
30 // terminator
31 { 0, NULL },
32 };
33 ENUM_STR_DEFINE_FUNCTIONS(DAEMON_STATUS, DAEMON_STATUS_NONE, "none");
34
35 ENUM_STR_MAP_DEFINE(DAEMON_OS_TYPE) = {
36 {DAEMON_OS_TYPE_UNKNOWN, "unknown"},
37 {DAEMON_OS_TYPE_LINUX, "linux"},
38 {DAEMON_OS_TYPE_FREEBSD, "freebsd"},
39 {DAEMON_OS_TYPE_MACOS, "macos"},
40 {DAEMON_OS_TYPE_WINDOWS, "windows"},
41
42 // terminator
43 { 0, NULL },
44 };
45 ENUM_STR_DEFINE_FUNCTIONS(DAEMON_OS_TYPE, DAEMON_OS_TYPE_UNKNOWN, "unknown");
46
47 static DAEMON_STATUS_FILE last_session_status = {
48 .v = 0,
49 .fatal = {
50 .spinlock = SPINLOCK_INITIALIZER,
51 },
52 };
53
54 static DAEMON_STATUS_FILE session_status = {
55 .v = STATUS_FILE_VERSION,
56 .fatal = {
57 .spinlock = SPINLOCK_INITIALIZER,
58 },
59 };
60
61 static void daemon_status_file_out_of_memory(void);
62
63 static void copy_and_clean_thread_name_if_empty(DAEMON_STATUS_FILE *ds, const char *name) {
64 if(ds->fatal.thread[0] && strcmp(ds->fatal.thread, "NO_NAME") != 0)
65 return;
66
67 if(!name || !*name) name = "NO_NAME";
68
69 safecpy(ds->fatal.thread, name);
70
71 // remove the variable part from the thread by removing [XXX] from it
72 unsigned char *p = (unsigned char *)strchr(ds->fatal.thread, '[');
73 if(p && isdigit(p[1]) && (isdigit(p[2]) || p[2] == ']'))
74 *p = '\0';
75 }
76
77 static bool stack_trace_is_empty(DAEMON_STATUS_FILE *ds) {
78 return !ds->fatal.stack_trace[0] || strncmp(ds->fatal.stack_trace, STACK_TRACE_INFO_PREFIX, strlen(STACK_TRACE_INFO_PREFIX)) == 0;
79 }
80
81 static void set_stack_trace_message_if_empty(DAEMON_STATUS_FILE *ds, const char *msg) {
82 if(stack_trace_is_empty(ds))
83 safecpy(ds->fatal.stack_trace, msg);
84 }
85
86 static void fill_dmi_info(DAEMON_STATUS_FILE *ds) {
87 if(!ds) return;
88
89 dmi_info_init(&ds->hw);
90 os_dmi_info_get(&ds->hw);
91 product_name_vendor_type(ds);
92 }
93
94 // --------------------------------------------------------------------------------------------------------------------
95 // json generation
96 // Each section is a separate function to stay within GCC's -fvar-tracking-assignments size limit.
97 // IMPORTANT: ALL functions here are called from signal handlers — no locks, no allocations.
98
99 static void dsf_json_agent(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
100 buffer_json_member_add_object(wb, "agent");
101 {
102 buffer_json_member_add_uuid(wb, "id", ds->host_id.uuid.uuid);
103
104 if(ds->v >= 24 && ds->host_id.last_modified_ut)
105 buffer_json_member_add_string(wb, "since", ds->host_id.last_modified_ut_rfc3339);
106
107 buffer_json_member_add_uuid_compact(wb, "ephemeral_id", ds->invocation.uuid);
108 buffer_json_member_add_string(wb, "version", ds->version);
109
110 buffer_json_member_add_time_t(wb, "uptime", ds->uptime);
111
112 buffer_json_member_add_uuid(wb, "node_id", ds->node_id.uuid);
113 buffer_json_member_add_uuid(wb, "claim_id", ds->claim_id.uuid);
114 buffer_json_member_add_uint64(wb, "restarts", ds->restarts);
115
116 if(ds->v >= 24)
117 buffer_json_member_add_uint64(wb, "crashes", ds->crashes);
118
119 // Only include PID if we're at version 27 or later
120 if(ds->v >= 27)
121 buffer_json_member_add_uint64(wb, "pid", (uint64_t)ds->pid);
122
123 if(ds->v >= 22) {
124 buffer_json_member_add_uint64(wb, "posts", ds->posts);
125 buffer_json_member_add_string(wb, "aclk", CLOUD_STATUS_2str(ds->cloud_status));
126 }
127
128 ND_PROFILE_2json(wb, "profile", ds->profile);
129 buffer_json_member_add_string(wb, "status", DAEMON_STATUS_2str(ds->status));
130 EXIT_REASON_2json(wb, "exit_reason", ds->exit_reason);
131
132 buffer_json_member_add_string_or_empty(wb, "install_type", ds->install_type);
133
134 if(ds->v >= 14) {
135 buffer_json_member_add_string(wb, "db_mode", rrd_memory_mode_name(ds->db_mode));
136 buffer_json_member_add_uint64(wb, "db_tiers", ds->db_tiers);
137 buffer_json_member_add_boolean(wb, "kubernetes", ds->kubernetes);
138 }
139
140 if(ds->v >= 16)
141 buffer_json_member_add_boolean(wb, "sentry_available", ds->sentry_available);
142
143 if(ds->v >= 18) {
144 buffer_json_member_add_int64(wb, "reliability", ds->reliability);
145 buffer_json_member_add_string(wb, "stack_traces", ds->stack_traces);
146 }
147
148 buffer_json_member_add_object(wb, "timings");
149 {
150 buffer_json_member_add_time_t(wb, "init", ds->timings.init);
151 buffer_json_member_add_time_t(wb, "exit", ds->timings.exit);
152 }
153 buffer_json_object_close(wb);
154 }
155 buffer_json_object_close(wb);
156 }
157
158 static void dsf_json_metrics(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
159 // Add metrics stats as a top-level node
160 buffer_json_member_add_object(wb, "metrics");
161 {
162 buffer_json_member_add_object(wb, "nodes");
163 {
164 buffer_json_member_add_uint64(wb, "total", ds->metrics_metadata.nodes.total);
165 buffer_json_member_add_uint64(wb, "receiving", ds->metrics_metadata.nodes.receiving);
166 buffer_json_member_add_uint64(wb, "sending", ds->metrics_metadata.nodes.sending);
167 buffer_json_member_add_uint64(wb, "archived", ds->metrics_metadata.nodes.archived);
168 }
169 buffer_json_object_close(wb);
170
171 buffer_json_member_add_object(wb, "metrics");
172 {
173 buffer_json_member_add_uint64(wb, "collected", ds->metrics_metadata.metrics.collected);
174 buffer_json_member_add_uint64(wb, "available", ds->metrics_metadata.metrics.available);
175 }
176 buffer_json_object_close(wb);
177
178 buffer_json_member_add_object(wb, "instances");
179 {
180 buffer_json_member_add_uint64(wb, "collected", ds->metrics_metadata.instances.collected);
181 buffer_json_member_add_uint64(wb, "available", ds->metrics_metadata.instances.available);
182 }
183 buffer_json_object_close(wb);
184
185 buffer_json_member_add_object(wb, "contexts");
186 {
187 buffer_json_member_add_uint64(wb, "collected", ds->metrics_metadata.contexts.collected);
188 buffer_json_member_add_uint64(wb, "available", ds->metrics_metadata.contexts.available);
189 }
190 buffer_json_object_close(wb);
191 }
192 buffer_json_object_close(wb);
193 }
194
195 static void dsf_json_host(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
196 buffer_json_member_add_object(wb, "host");
197 {
198 buffer_json_member_add_uuid_compact(wb, "id", ds->machine_id.uuid);
199 buffer_json_member_add_string_or_empty(wb, "architecture", ds->architecture);
200 buffer_json_member_add_string_or_empty(wb, "virtualization", ds->virtualization);
201 buffer_json_member_add_string_or_empty(wb, "container", ds->container);
202 buffer_json_member_add_time_t(wb, "uptime", ds->boottime);
203
204 if(ds->v >= 20) {
205 buffer_json_member_add_string_or_empty(wb, "timezone", ds->timezone);
206 buffer_json_member_add_string_or_empty(wb, "cloud_provider", ds->cloud_provider_type);
207 buffer_json_member_add_string_or_empty(wb, "cloud_instance", ds->cloud_instance_type);
208 buffer_json_member_add_string_or_empty(wb, "cloud_region", ds->cloud_instance_region);
209 }
210
211 buffer_json_member_add_uint64(wb, "system_cpus", ds->system_cpus);
212
213 buffer_json_member_add_object(wb, "boot");
214 {
215 buffer_json_member_add_uuid_compact(wb, "id", ds->boot_id.uuid);
216 }
217 buffer_json_object_close(wb);
218
219 buffer_json_member_add_object(wb, "memory");
220 if(OS_SYSTEM_MEMORY_OK(ds->memory)) {
221 buffer_json_member_add_uint64(wb, "total", ds->memory.ram_total_bytes);
222 buffer_json_member_add_uint64(wb, "free", ds->memory.ram_available_bytes);
223
224 if(ds->v >= 21) {
225 buffer_json_member_add_uint64(wb, "netdata", ds->netdata_max_rss);
226 buffer_json_member_add_uint64(wb, "oom_protection", ds->oom_protection);
227 }
228 }
229 buffer_json_object_close(wb);
230
231 buffer_json_member_add_object(wb, "disk");
232 {
233 buffer_json_member_add_object(wb, "db");
234 if (OS_SYSTEM_DISK_SPACE_OK(ds->var_cache)) {
235 buffer_json_member_add_uint64(wb, "total", ds->var_cache.total_bytes);
236 buffer_json_member_add_uint64(wb, "free", ds->var_cache.free_bytes);
237 buffer_json_member_add_uint64(wb, "inodes_total", ds->var_cache.total_inodes);
238 buffer_json_member_add_uint64(wb, "inodes_free", ds->var_cache.free_inodes);
239 buffer_json_member_add_boolean(wb, "read_only", ds->var_cache.is_read_only);
240 }
241 buffer_json_object_close(wb);
242
243 buffer_json_member_add_object(wb, "netdata");
244 buffer_json_member_add_uint64(wb, "dbengine", ds->disk_footprint.dbengine);
245 buffer_json_member_add_uint64(wb, "sqlite", ds->disk_footprint.sqlite);
246 buffer_json_member_add_uint64(wb, "other", ds->disk_footprint.other);
247 buffer_json_member_add_string(wb, "last_updated", ds->disk_footprint.last_updated_ut_rfc3339);
248 buffer_json_object_close(wb);
249 }
250 buffer_json_object_close(wb);
251 }
252 buffer_json_object_close(wb);
253 }
254
255 static void dsf_json_os(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
256 buffer_json_member_add_object(wb, "os");
257 {
258 buffer_json_member_add_string(wb, "type", DAEMON_OS_TYPE_2str(ds->os_type));
259 buffer_json_member_add_string_or_empty(wb, "kernel", ds->kernel_version);
260 buffer_json_member_add_string_or_empty(wb, "name", ds->os_name);
261 buffer_json_member_add_string_or_empty(wb, "version", ds->os_version);
262 buffer_json_member_add_string_or_empty(wb, "family", ds->os_id);
263 buffer_json_member_add_string_or_empty(wb, "platform", ds->os_id_like);
264 }
265 buffer_json_object_close(wb);
266 }
267
268 static void dsf_json_hw(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
269 buffer_json_member_add_object(wb, "hw");
270 {
271 buffer_json_member_add_object(wb, "sys");
272 {
273 buffer_json_member_add_string(wb, "vendor", ds->hw.sys.vendor);
274 buffer_json_member_add_string(wb, "uuid", ds->hw.sys.uuid);
275 // buffer_json_member_add_string(wb, "serial", ds->hw.sys.serial);
276 // buffer_json_member_add_string(wb, "asset_tag", ds->hw.sys.asset_tag);
277 }
278 buffer_json_object_close(wb);
279
280 buffer_json_member_add_object(wb, "product");
281 {
282 buffer_json_member_add_string(wb, "name", ds->hw.product.name);
283 buffer_json_member_add_string(wb, "version", ds->hw.product.version);
284 buffer_json_member_add_string(wb, "sku", ds->hw.product.sku);
285 buffer_json_member_add_string(wb, "family", ds->hw.product.family);
286 }
287 buffer_json_object_close(wb);
288
289 buffer_json_member_add_object(wb, "board");
290 {
291 buffer_json_member_add_string(wb, "name", ds->hw.board.name);
292 buffer_json_member_add_string(wb, "version", ds->hw.board.version);
293 buffer_json_member_add_string(wb, "vendor", ds->hw.board.vendor);
294 // buffer_json_member_add_string(wb, "serial", ds->hw.board.serial);
295 // buffer_json_member_add_string(wb, "asset_tag", ds->hw.board.asset_tag);
296 }
297 buffer_json_object_close(wb);
298
299 buffer_json_member_add_object(wb, "chassis");
300 {
301 buffer_json_member_add_string(wb, "type", ds->hw.chassis.type);
302 buffer_json_member_add_string(wb, "vendor", ds->hw.chassis.vendor);
303 buffer_json_member_add_string(wb, "version", ds->hw.chassis.version);
304 // buffer_json_member_add_string(wb, "serial", ds->hw.chassis.serial);
305 // buffer_json_member_add_string(wb, "asset_tag", ds->hw.chassis.asset_tag);
306 }
307 buffer_json_object_close(wb);
308
309 buffer_json_member_add_object(wb, "bios");
310 {
311 buffer_json_member_add_string(wb, "date", ds->hw.bios.date);
312 buffer_json_member_add_string(wb, "release", ds->hw.bios.release);
313 buffer_json_member_add_string(wb, "version", ds->hw.bios.version);
314 buffer_json_member_add_string(wb, "vendor", ds->hw.bios.vendor);
315 }
316 buffer_json_object_close(wb);
317 }
318 buffer_json_object_close(wb);
319 }
320
321 static void dsf_json_product(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
322 buffer_json_member_add_object(wb, "product");
323 {
324 buffer_json_member_add_string(wb, "vendor", ds->product.vendor);
325 buffer_json_member_add_string(wb, "name", ds->product.name);
326 buffer_json_member_add_string(wb, "type", ds->product.type);
327 }
328 buffer_json_object_close(wb);
329 }
330
331 static void dsf_json_fatal(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
332 buffer_json_member_add_object(wb, "fatal");
333 {
334 buffer_json_member_add_uint64(wb, "line", ds->fatal.line);
335 buffer_json_member_add_string_or_empty(wb, "filename", ds->fatal.filename);
336 buffer_json_member_add_string_or_empty(wb, "function", ds->fatal.function);
337 buffer_json_member_add_string_or_empty(wb, "message", ds->fatal.message);
338 buffer_json_member_add_string_or_empty(wb, "errno", ds->fatal.errno_str);
339 buffer_json_member_add_string_or_empty(wb, "thread", ds->fatal.thread);
340 buffer_json_member_add_uint64(wb, "thread_id", ds->fatal.thread_id);
341 buffer_json_member_add_string_or_empty(wb, "stack_trace", ds->fatal.stack_trace);
342
343 if(ds->v >= 16) {
344 char signal_code[UINT64_MAX_LENGTH];
345 SIGNAL_CODE_2str_h(ds->fatal.signal_code, signal_code, sizeof(signal_code));
346 buffer_json_member_add_string_or_empty(wb, "signal_code", signal_code);
347 }
348
349 if(ds->v >= 17)
350 buffer_json_member_add_boolean(wb, "sentry", ds->fatal.sentry);
351
352 if(ds->v >= 18) {
353 char buf[UINT64_HEX_MAX_LENGTH];
354
355 if(ds->fatal.signal_code)
356 print_uint64_hex(buf, ds->fatal.fault_address);
357 else
358 buf[0] = '\0';
359
360 buffer_json_member_add_string(wb, "fault_address", buf);
361 }
362
363 if(ds->v >= 23)
364 buffer_json_member_add_uint64(wb, "worker_job_id", ds->fatal.worker_job_id);
365 }
366 buffer_json_object_close(wb);
367 }
368
369 static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
370 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS
371 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS
372
373 dsf_acquire(*ds);
374
375 buffer_json_member_add_string(wb, "@timestamp", ds->timestamp_ut_rfc3339);
376 buffer_json_member_add_uint64(wb, "version", STATUS_FILE_VERSION);
377
378 dsf_json_agent(wb, ds);
379 dsf_json_metrics(wb, ds);
380 dsf_json_host(wb, ds);
381 dsf_json_os(wb, ds);
382 dsf_json_hw(wb, ds);
383 dsf_json_product(wb, ds);
384 dsf_json_fatal(wb, ds);
385
386 dsf_release(*ds);
387 }
388
389 // --------------------------------------------------------------------------------------------------------------------
390 // json parsing
391
392 static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *error) {
393 char path[1024]; path[0] = '\0';
394
395 DAEMON_STATUS_FILE *ds = data;
396
397 // change management, version to know which fields to expect
398 uint64_t version = 0;
399 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "version", version, error, JSONC_REQUIRED);
400 ds->v = version;
401
402 unsigned strict = JSONC_OPTIONAL; // allow missing fields and values
403 unsigned required_v1 = version >= 1 ? strict : JSONC_OPTIONAL;
404 unsigned required_v3 = version >= 3 ? strict : JSONC_OPTIONAL;
405 unsigned required_v4 = version >= 4 ? strict : JSONC_OPTIONAL;
406 unsigned required_v5 = version >= 5 ? strict : JSONC_OPTIONAL;
407 unsigned required_v10 = version >= 10 ? strict : JSONC_OPTIONAL;
408 unsigned required_v14 = version >= 14 ? strict : JSONC_OPTIONAL;
409 unsigned required_v16 = version >= 16 ? strict : JSONC_OPTIONAL;
410 unsigned required_v17 = version >= 17 ? strict : JSONC_OPTIONAL;
411 unsigned required_v18 = version >= 18 ? strict : JSONC_OPTIONAL;
412 unsigned required_v20 = version >= 20 ? strict : JSONC_OPTIONAL;
413 unsigned required_v21 = version >= 21 ? strict : JSONC_OPTIONAL;
414 unsigned required_v22 = version >= 22 ? strict : JSONC_OPTIONAL;
415 unsigned required_v23 = version >= 23 ? strict : JSONC_OPTIONAL;
416 unsigned required_v24 = version >= 24 ? strict : JSONC_OPTIONAL;
417 unsigned required_v25 = version >= 25 ? strict : JSONC_OPTIONAL;
418 unsigned required_v26 = version >= 26 ? strict : JSONC_OPTIONAL;
419 unsigned required_v27 = version >= 27 ? strict : JSONC_OPTIONAL;
420
421 // Parse timestamp
422 JSONC_PARSE_TXT2RFC3339_USEC_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", ds->timestamp_ut, error, required_v1);
423
424 const char *profile_key = version >= 18 ? "profile" : "ND_profile";
425 const char *status_key = version >= 18 ? "status" : "ND_status";
426 const char *exit_reason_key = version >= 18 ? "exit_reason" : "ND_exit_reason";
427 const char *node_id_key = version >= 18 ? "node_id" : "ND_node_id";
428 const char *claim_id_key = version >= 18 ? "claim_id" : "ND_claim_id";
429 const char *install_type_key = version >= 18 ? "install_type" : "ND_install_type";
430 const char *timings_key = version >= 18 ? "timings" : "ND_timings";
431 const char *restarts_key = version >= 18 ? "restarts" : "ND_restarts";
432 const char *db_mode_key = version >= 18 ? "db_mode" : "ND_db_mode";
433 const char *db_tiers_key = version >= 18 ? "db_tiers" : "ND_db_tiers";
434 const char *kubernetes_key = version >= 18 ? "kubernetes" : "ND_kubernetes";
435 const char *sentry_available_key = version >= 18 ? "sentry_available" : "ND_sentry_available";
436
437 // Parse agent object
438 JSONC_PARSE_SUBOBJECT(jobj, path, "agent", error, required_v1, {
439 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "id", ds->host_id.uuid.uuid, error, required_v1);
440
441 if(version >= 24)
442 JSONC_PARSE_TXT2RFC3339_USEC_OR_ERROR_AND_RETURN(jobj, path, "since", ds->host_id.last_modified_ut, error, required_v24);
443
444 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "ephemeral_id", ds->invocation.uuid, error, required_v1);
445 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->version, error, required_v1);
446 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "uptime", ds->uptime, error, required_v1);
447
448 JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, profile_key, ND_PROFILE_2id_one, ds->profile, error, required_v1);
449 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, status_key, DAEMON_STATUS_2id, ds->status, error, required_v1);
450 JSONC_PARSE_ARRAY_OF_TXT2BITMAP_OR_ERROR_AND_RETURN(jobj, path, exit_reason_key, EXIT_REASON_2id_one, ds->exit_reason, error, required_v1);
451 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, node_id_key, ds->node_id.uuid, error, required_v1);
452 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, claim_id_key, ds->claim_id.uuid, error, required_v1);
453 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, install_type_key, ds->install_type, error, required_v3);
454
455 JSONC_PARSE_SUBOBJECT(jobj, path, timings_key, error, required_v1, {
456 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "init", ds->timings.init, error, required_v1);
457 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "exit", ds->timings.exit, error, required_v1);
458 });
459
460 if(version >= 4)
461 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, restarts_key, ds->restarts, error, required_v4);
462
463 if(version >= 24)
464 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "crashes", ds->crashes, error, required_v24);
465
466 // Only try to parse PID if we're at version 27 or later
467 if(version >= 27)
468 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "pid", ds->pid, error, JSONC_OPTIONAL);
469
470 if(version >= 22) {
471 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "posts", ds->posts, error, required_v22);
472 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "aclk", CLOUD_STATUS_2id, ds->cloud_status, error, required_v22);
473 }
474
475 if(version >= 14) {
476 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, db_mode_key, rrd_memory_mode_id, ds->db_mode, error, required_v14);
477 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, db_tiers_key, ds->db_tiers, error, required_v14);
478 JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, kubernetes_key, ds->kubernetes, error, required_v14);
479 }
480 else {
481 ds->db_mode = default_rrd_memory_mode;
482 ds->db_tiers = nd_profile.storage_tiers;
483 ds->kubernetes = false;
484 }
485
486 if(version >= 17)
487 JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, sentry_available_key, ds->sentry_available, error, required_v17);
488 else if(version == 16)
489 JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "ND_sentry", ds->sentry_available, error, required_v16);
490 if(version >= 18) {
491 JSONC_PARSE_INT64_OR_ERROR_AND_RETURN(jobj, path, "reliability", ds->reliability, error, required_v18);
492 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "stack_traces", ds->stack_traces, error, required_v18);
493 }
494 });
495
496 // Parse host object
497 JSONC_PARSE_SUBOBJECT(jobj, path, "host", error, required_v1, {
498 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "id", ds->machine_id.uuid, error, required_v10);
499 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "architecture", ds->architecture, error, required_v1);
500 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "virtualization", ds->virtualization, error, required_v1);
501 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "container", ds->container, error, required_v1);
502 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "uptime", ds->boottime, error, required_v1);
503 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "system_cpus", ds->system_cpus, error, required_v27);
504
505 JSONC_PARSE_SUBOBJECT(jobj, path, "boot", error, required_v1, {
506 JSONC_PARSE_TXT2UUID_OR_ERROR_AND_RETURN(jobj, path, "id", ds->boot_id.uuid, error, required_v1);
507 });
508
509 JSONC_PARSE_SUBOBJECT(jobj, path, "memory", error, required_v1, {
510 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "total", ds->memory.ram_total_bytes, error, JSONC_OPTIONAL);
511 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "free", ds->memory.ram_available_bytes, error, JSONC_OPTIONAL);
512 if(!OS_SYSTEM_MEMORY_OK(ds->memory))
513 ds->memory = OS_SYSTEM_MEMORY_EMPTY;
514
515 if(version >= 21) {
516 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "netdata", ds->netdata_max_rss, error, required_v21);
517 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "oom_protection", ds->oom_protection, error, required_v21);
518 }
519 });
520
521 JSONC_PARSE_SUBOBJECT(jobj, path, "disk", error, required_v1, {
522 JSONC_PARSE_SUBOBJECT(jobj, path, "db", error, required_v1, {
523 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "total", ds->var_cache.total_bytes, error, JSONC_OPTIONAL);
524 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "free", ds->var_cache.free_bytes, error, JSONC_OPTIONAL);
525 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "inodes_total", ds->var_cache.total_inodes, error, JSONC_OPTIONAL);
526 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "inodes_free", ds->var_cache.free_inodes, error, JSONC_OPTIONAL);
527 JSONC_PARSE_BOOL_OR_ERROR_AND_RETURN(jobj, path, "read_only", ds->var_cache.is_read_only, error, JSONC_OPTIONAL);
528 if(!OS_SYSTEM_DISK_SPACE_OK(ds->var_cache))
529 ds->var_cache = OS_SYSTEM_DISK_SPACE_EMPTY;
530 });
531
532 JSONC_PARSE_SUBOBJECT(jobj, path, "netdata", error, required_v27, {
533 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "dbengine", ds->disk_footprint.dbengine, error, required_v27);
534 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "sqlite", ds->disk_footprint.sqlite, error, required_v27);
535 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "other", ds->disk_footprint.other, error, required_v27);
536 JSONC_PARSE_TXT2RFC3339_USEC_OR_ERROR_AND_RETURN(jobj, path, "last_updated", ds->disk_footprint.last_updated_ut, error, required_v27);
537 // Don't reset if not OK since this is a new field
538 });
539 });
540
541 if(version >= 20) {
542 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "timezone", ds->timezone, error, required_v20);
543 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_provider", ds->cloud_provider_type, error, required_v20);
544 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_instance", ds->cloud_instance_type, error, required_v20);
545 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "cloud_region", ds->cloud_instance_region, error, required_v20);
546 }
547 });
548
549 // Parse metrics metadata
550 JSONC_PARSE_SUBOBJECT(jobj, path, "metrics", error, required_v27, {
551 JSONC_PARSE_SUBOBJECT(jobj, path, "nodes", error, required_v27, {
552 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "total", ds->metrics_metadata.nodes.total, error, required_v27);
553 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "receiving", ds->metrics_metadata.nodes.receiving, error, required_v27);
554 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "sending", ds->metrics_metadata.nodes.sending, error, required_v27);
555 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "archived", ds->metrics_metadata.nodes.archived, error, required_v27);
556 });
557
558 JSONC_PARSE_SUBOBJECT(jobj, path, "metrics", error, required_v27, {
559 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "collected", ds->metrics_metadata.metrics.collected, error, required_v27);
560 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "available", ds->metrics_metadata.metrics.available, error, required_v27);
561 });
562
563 JSONC_PARSE_SUBOBJECT(jobj, path, "instances", error, required_v27, {
564 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "collected", ds->metrics_metadata.instances.collected, error, required_v27);
565 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "available", ds->metrics_metadata.instances.available, error, required_v27);
566 });
567
568 JSONC_PARSE_SUBOBJECT(jobj, path, "contexts", error, required_v27, {
569 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "collected", ds->metrics_metadata.contexts.collected, error, required_v27);
570 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "available", ds->metrics_metadata.contexts.available, error, required_v27);
571 });
572 });
573
574 // Parse os object
575 JSONC_PARSE_SUBOBJECT(jobj, path, "os", error, required_v1, {
576 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "type", DAEMON_OS_TYPE_2id, ds->os_type, error, required_v1);
577 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "kernel", ds->kernel_version, error, required_v1);
578 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "name", ds->os_name, error, required_v1);
579 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->os_version, error, required_v1);
580 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "family", ds->os_id, error, required_v1);
581 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "platform", ds->os_id_like, error, required_v1);
582 });
583
584 // Parse the hw object
585 JSONC_PARSE_SUBOBJECT(jobj, path, "hw", error, required_v25, {
586 JSONC_PARSE_SUBOBJECT(jobj, path, "sys", error, required_v25, {
587 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "vendor", ds->hw.sys.vendor, error, required_v25);
588 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "uuid", ds->hw.sys.uuid, error, required_v27);
589 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "serial", ds->hw.sys.serial, error, required_v26);
590 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "asset_tag", ds->hw.sys.asset_tag, error, required_v27);
591 });
592
593 JSONC_PARSE_SUBOBJECT(jobj, path, "product", error, required_v25, {
594 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "name", ds->hw.product.name, error, required_v25);
595 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->hw.product.version, error, required_v25);
596 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "sku", ds->hw.product.sku, error, required_v25);
597 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "family", ds->hw.product.family, error, required_v25);
598 });
599
600 JSONC_PARSE_SUBOBJECT(jobj, path, "board", error, required_v25, {
601 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "name", ds->hw.board.name, error, required_v25);
602 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->hw.board.version, error, required_v25);
603 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "vendor", ds->hw.board.vendor, error, required_v25);
604 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "serial", ds->hw.board.serial, error, required_v26);
605 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "asset_tag", ds->hw.board.asset_tag, error, required_v27);
606 });
607
608 JSONC_PARSE_SUBOBJECT(jobj, path, "chassis", error, required_v25, {
609 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "type", ds->hw.chassis.type, error, required_v25);
610 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "vendor", ds->hw.chassis.vendor, error, required_v25);
611 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->hw.chassis.version, error, required_v25);
612 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "serial", ds->hw.chassis.serial, error, required_v26);
613 // JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "asset_tag", ds->hw.chassis.asset_tag, error, required_v27);
614 });
615
616 JSONC_PARSE_SUBOBJECT(jobj, path, "bios", error, required_v25, {
617 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "date", ds->hw.bios.date, error, required_v25);
618 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "release", ds->hw.bios.release, error, required_v25);
619 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "version", ds->hw.bios.version, error, required_v25);
620 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "vendor", ds->hw.bios.vendor, error, required_v25);
621 });
622 });
623
624 // Parse product object
625 JSONC_PARSE_SUBOBJECT(jobj, path, "product", error, required_v26, {
626 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "vendor", ds->product.vendor, error, required_v26);
627 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "name", ds->product.name, error, required_v26);
628 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "type", ds->product.type, error, required_v26);
629 });
630
631 // Parse fatal object
632 JSONC_PARSE_SUBOBJECT(jobj, path, "fatal", error, required_v1, {
633 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "filename", ds->fatal.filename, error, required_v1);
634 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "function", ds->fatal.function, error, required_v1);
635 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "message", ds->fatal.message, error, required_v1);
636 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "stack_trace", ds->fatal.stack_trace, error, required_v1);
637 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "line", ds->fatal.line, error, required_v1);
638 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "errno", ds->fatal.errno_str, error, required_v3);
639 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "thread", ds->fatal.thread, error, required_v5);
640
641 if(version >= 16)
642 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "signal_code", SIGNAL_CODE_2id_h, ds->fatal.signal_code, error, required_v16);
643
644 if(version >= 17)
645 JSONC_PARSE_TXT2ENUM_OR_ERROR_AND_RETURN(jobj, path, "sentry", SIGNAL_CODE_2id_h, ds->fatal.sentry, error, required_v17);
646
647 if(version >= 18) {
648 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "thread_id", ds->fatal.thread_id, error, required_v18);
649
650 char buf[UINT64_HEX_MAX_LENGTH];
651 JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "fault_address", buf, error, required_v18);
652 if(buf[0])
653 ds->fatal.fault_address = str2ull_encoded(buf);
654 else
655 ds->fatal.fault_address = 0;
656 }
657
658 if(version >= 23)
659 JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "worker_job_id", ds->fatal.worker_job_id, error, required_v23);
660 });
661
662 return true;
663 }
664
665 // --------------------------------------------------------------------------------------------------------------------
666 // get the current status
667
668 static void daemon_status_file_migrate_once(void) {
669 FUNCTION_RUN_ONCE();
670
671 dsf_acquire(last_session_status);
672 dsf_acquire(session_status);
673
674 safecpy(session_status.version, NETDATA_VERSION);
675 session_status.machine_id = os_machine_id();
676
677 {
678 char *install_type = NULL, *prebuilt_arch = NULL, *prebuilt_dist = NULL;
679 get_install_type_internal(&install_type, &prebuilt_arch, &prebuilt_dist);
680
681 if(install_type)
682 safecpy(session_status.install_type, install_type);
683
684 freez(prebuilt_arch);
685 freez(prebuilt_dist);
686 freez(install_type);
687 }
688
689 #if defined(ENABLE_SENTRY)
690 session_status.sentry_available = true;
691 #else
692 session_status.sentry_available = false;
693 #endif
694
695 session_status.boot_id = os_boot_id();
696 if(!UUIDeq(session_status.boot_id, last_session_status.boot_id) && os_boot_ids_match(session_status.boot_id, last_session_status.boot_id)) {
697 // there is a slight difference in boot_id, but it is still the same boot
698 // copy the last boot_id
699 session_status.boot_id = last_session_status.boot_id;
700 }
701
702 session_status.claim_id = last_session_status.claim_id;
703 session_status.node_id = last_session_status.node_id;
704 session_status.host_id = *machine_guid_get();
705
706 safecpy(session_status.architecture, last_session_status.architecture);
707 safecpy(session_status.virtualization, last_session_status.virtualization);
708 safecpy(session_status.container, last_session_status.container);
709 safecpy(session_status.kernel_version, last_session_status.kernel_version);
710 safecpy(session_status.os_name, last_session_status.os_name);
711 safecpy(session_status.os_version, last_session_status.os_version);
712 safecpy(session_status.os_id, last_session_status.os_id);
713 safecpy(session_status.os_id_like, last_session_status.os_id_like);
714 safecpy(session_status.timezone, last_session_status.timezone);
715 safecpy(session_status.cloud_provider_type, last_session_status.cloud_provider_type);
716 safecpy(session_status.cloud_instance_type, last_session_status.cloud_instance_type);
717 safecpy(session_status.cloud_instance_region, last_session_status.cloud_instance_region);
718
719 session_status.posts = last_session_status.posts;
720 session_status.restarts = last_session_status.restarts + 1;
721 session_status.crashes = last_session_status.crashes;
722 session_status.reliability = last_session_status.reliability;
723
724 if(daemon_status_file_has_last_crashed(&last_session_status)) {
725 session_status.crashes++;
726 if(session_status.reliability > 0) session_status.reliability = 0;
727 session_status.reliability--;
728 }
729 else {
730 if(session_status.reliability < 0) session_status.reliability = 0;
731 session_status.reliability++;
732 }
733
734 #ifdef HAVE_LIBBACKTRACE
735 safecpy(session_status.stack_traces, stacktrace_backend());
736 #endif
737
738 fill_dmi_info(&session_status);
739
740 dsf_release(last_session_status);
741 dsf_release(session_status);
742 }
743
744 static void daemon_status_file_refresh(DAEMON_STATUS status) {
745 usec_t now_ut = now_realtime_usec();
746
747 dsf_acquire(session_status);
748
749 #if defined(OS_LINUX)
750 session_status.os_type = DAEMON_OS_TYPE_LINUX;
751 #elif defined(OS_FREEBSD)
752 session_status.os_type = DAEMON_OS_TYPE_FREEBSD;
753 #elif defined(OS_MACOS)
754 session_status.os_type = DAEMON_OS_TYPE_MACOS;
755 #elif defined(OS_WINDOWS)
756 session_status.os_type = DAEMON_OS_TYPE_WINDOWS;
757 #endif
758
759 if(!session_status.timings.init_started_ut)
760 session_status.timings.init_started_ut = now_ut;
761
762 if(status == DAEMON_STATUS_EXITING && !session_status.timings.exit_started_ut)
763 session_status.timings.exit_started_ut = now_ut;
764
765 if(session_status.status == DAEMON_STATUS_INITIALIZING)
766 session_status.timings.init = (time_t)((now_ut - session_status.timings.init_started_ut + USEC_PER_SEC/2) / USEC_PER_SEC);
767
768 if(session_status.status == DAEMON_STATUS_EXITING)
769 session_status.timings.exit = (time_t)((now_ut - session_status.timings.exit_started_ut + USEC_PER_SEC/2) / USEC_PER_SEC);
770
771 session_status.host_id = *machine_guid_get();
772 session_status.boottime = now_boottime_sec();
773 session_status.uptime = now_realtime_sec() - netdata_start_time;
774 session_status.timestamp_ut = now_ut;
775 rfc3339_datetime_ut(session_status.timestamp_ut_rfc3339, sizeof(session_status.timestamp_ut_rfc3339),
776 session_status.timestamp_ut, 2, true);
777 session_status.invocation = nd_log_get_invocation_id();
778 session_status.db_mode = default_rrd_memory_mode;
779 session_status.db_tiers = nd_profile.storage_tiers;
780 session_status.pid = getpid();
781
782 // we keep the highest cloud status, to know how the agent gets connected to netdata.cloud
783 CLOUD_STATUS cs = cloud_status();
784 if(!session_status.cloud_status || // it is ok to overwrite this
785 session_status.cloud_status == CLOUD_STATUS_AVAILABLE || // it is ok to overwrite this
786 session_status.cloud_status == CLOUD_STATUS_OFFLINE || // it is ok to overwrite this
787 cs == CLOUD_STATUS_BANNED || // this is a final state
788 cs == CLOUD_STATUS_ONLINE || // this is a final state
789 cs == CLOUD_STATUS_INDIRECT) // this is a final state
790 session_status.cloud_status = cs;
791
792
793 #ifdef ENABLE_DBENGINE
794 session_status.oom_protection = dbengine_out_of_memory_protection;
795 #else
796 session_status.oom_protection = 0;
797 #endif
798
799 OS_PROCESS_MEMORY proc_mem = os_process_memory(0);
800 if(OS_PROCESS_MEMORY_OK(proc_mem))
801 session_status.netdata_max_rss = proc_mem.max_rss;
802
803 session_status.claim_id = claim_id_get_uuid();
804
805 if(localhost) {
806 if(!UUIDiszero(localhost->host_id))
807 session_status.host_id.uuid = localhost->host_id;
808
809 if(!UUIDiszero(localhost->node_id))
810 session_status.node_id = localhost->node_id;
811 }
812
813 if(get_daemon_status_fields_from_system_info(&session_status))
814 product_name_vendor_type(&session_status);
815
816 {
817 SYSTEM_TZ tz = system_tz_get();
818 safecpy(session_status.timezone, tz.timezone);
819 system_tz_free(&tz);
820 }
821
822 session_status.exit_reason = exit_initiated_get();
823 session_status.profile = nd_profile_detect_and_configure(false);
824
825 if(status != DAEMON_STATUS_NONE)
826 session_status.status = status;
827
828 session_status.memory = os_system_memory(true);
829 session_status.var_cache = os_disk_space(netdata_configured_cache_dir);
830 session_status.system_cpus = os_get_system_cpus();
831
832 // Collect metrics metadata statistics
833 session_status.metrics_metadata = rrdstats_metadata_collect();
834
835 // Update disk footprint at most once every 10 minutes (600 seconds)
836 if ((now_ut - session_status.disk_footprint.last_updated_ut) >= 600 * USEC_PER_SEC ||
837 session_status.disk_footprint.last_updated_ut == 0) {
838 // Calculate disk footprint by categories
839 const char *dirs_to_measure[] = {
840 netdata_configured_varlib_dir,
841 netdata_configured_cache_dir
842 };
843
844 // Create patterns for different file types
845 SIMPLE_PATTERN *dbengine_pattern = simple_pattern_create("*dbengine*/*.ndf *dbengine*/*.njf*", " ", SIMPLE_PATTERN_EXACT, false);
846 SIMPLE_PATTERN *sqlite_pattern = simple_pattern_create("*.db *.wal *.shm", " ", SIMPLE_PATTERN_EXACT, false);
847
848 // Get total size first
849 DIR_SIZE total_size = dir_size_multiple(dirs_to_measure, 2, NULL, 0);
850
851 // Get DBEngine files size
852 DIR_SIZE dbengine_size = dir_size_multiple(dirs_to_measure, 2, dbengine_pattern, 0);
853 session_status.disk_footprint.dbengine = dbengine_size.bytes;
854
855 // Get SQLite files size
856 DIR_SIZE sqlite_size = dir_size_multiple(dirs_to_measure, 2, sqlite_pattern, 0);
857 session_status.disk_footprint.sqlite = sqlite_size.bytes;
858
859 // Calculate other files (total - dbengine - sqlite)
860 session_status.disk_footprint.other = total_size.bytes - dbengine_size.bytes - sqlite_size.bytes;
861
862 // Update last updated timestamp
863 session_status.disk_footprint.last_updated_ut = now_ut;
864 rfc3339_datetime_ut(session_status.disk_footprint.last_updated_ut_rfc3339,
865 sizeof(session_status.disk_footprint.last_updated_ut_rfc3339),
866 session_status.disk_footprint.last_updated_ut, 2, true);
867
868 // Clean up patterns
869 simple_pattern_free(dbengine_pattern);
870 simple_pattern_free(sqlite_pattern);
871 }
872
873 dsf_release(session_status);
874 }
875
876 // --------------------------------------------------------------------------------------------------------------------
877 // load a saved status
878
879 static bool status_file_load_and_parse(const char *filename, void *data) {
880 DAEMON_STATUS_FILE *status = data;
881
882 CLEAN_BUFFER *wb = buffer_create(0, NULL);
883 CLEAN_BUFFER *error = buffer_create(0, NULL);
884
885 if(!read_txt_file_to_buffer(filename, wb, 65536))
886 return false;
887
888 // Parse the JSON
889 return json_parse_payload_or_error(wb, error, daemon_status_file_from_json, status) == HTTP_RESP_OK;
890 }
891
892 // --------------------------------------------------------------------------------------------------------------------
893 // save the current status
894
895 static BUFFER *static_save_buffer = NULL;
896 static void static_save_buffer_init(void) {
897 if (!static_save_buffer)
898 static_save_buffer = buffer_create(16384, NULL);
899
900 buffer_flush(static_save_buffer);
901 }
902
903 static bool daemon_status_file_saved = false;
904 static void daemon_status_file_save(BUFFER *wb, DAEMON_STATUS_FILE *ds, bool log) {
905 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS
906 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS
907
908 // wb should have enough space to hold the JSON content, to avoid any allocations
909
910 buffer_flush(wb);
911 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
912 daemon_status_file_to_json(wb, ds);
913 buffer_json_finalize(wb);
914
915 if(status_file_io_save(STATUS_FILENAME, buffer_tostring(wb), buffer_strlen(wb), log))
916 daemon_status_file_saved = true;
917 }
918
919 // --------------------------------------------------------------------------------------------------------------------
920 // POST the last status to agent-events
921
922 struct post_status_file_thread_data {
923 const char *cause;
924 const char *msg;
925 ND_LOG_FIELD_PRIORITY priority;
926 DAEMON_STATUS_FILE *status;
927 };
928
929 static const char *agent_health(DAEMON_STATUS_FILE *ds) {
930 if(daemon_status_file_has_last_crashed(ds)) {
931 // it crashed
932
933 if(ds->restarts == 1)
934 return "crash-first";
935 else if(ds->reliability <= -2)
936 return "crash-loop";
937 else if(ds->reliability < 0)
938 return "crash-repeated";
939 else
940 return "crash-entered";
941 }
942
943 // it didn't crash
944 if(ds->restarts == 1)
945 return "healthy-first";
946 else if(ds->reliability >= 2)
947 return "healthy-loop";
948 else if(ds->reliability > 0)
949 return "healthy-repeated";
950 else
951 return "healthy-recovered";
952 }
953
954 // Callback to discard curl response data.
955 // Without this, curl writes to stdout which can crash on Windows when running as a service.
956 static size_t post_status_file_discard_response(void *ptr, size_t size, size_t nmemb, void *userdata) {
957 (void)ptr;
958 (void)userdata;
959 return size * nmemb;
960 }
961
962 static void post_status_file(struct post_status_file_thread_data *d) {
963 daemon_status_file_startup_step("startup(crash reports json)");
964
965 CLEAN_BUFFER *wb = buffer_create(0, NULL);
966 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
967 buffer_json_member_add_string(wb, "exit_cause", d->cause);
968 buffer_json_member_add_string(wb, "message", d->msg);
969 buffer_json_member_add_uint64(wb, "priority", d->priority);
970 buffer_json_member_add_uint64(wb, "version_saved", d->status->v);
971 buffer_json_member_add_string(wb, "agent_version_now", NETDATA_VERSION);
972 buffer_json_member_add_uint64(wb, "agent_pid_now", getpid());
973 buffer_json_member_add_boolean(wb, "host_memory_critical",
974 OS_SYSTEM_MEMORY_OK(d->status->memory) && d->status->memory.ram_available_bytes <= d->status->oom_protection);
975 buffer_json_member_add_uint64(wb, "host_memory_free_percent", (uint64_t)round(os_system_memory_available_percent(d->status->memory)));
976 buffer_json_member_add_string(wb, "agent_health", agent_health(d->status));
977 daemon_status_file_to_json(wb, d->status);
978 buffer_json_finalize(wb);
979
980 const char *json_data = buffer_tostring(wb);
981
982 CURL *curl = curl_easy_init();
983 if(!curl)
984 return;
985
986 daemon_status_file_startup_step("startup(crash reports curl)");
987
988 curl_easy_setopt(curl, CURLOPT_URL, "https://agent-events.netdata.cloud/agent-events");
989 curl_easy_setopt(curl, CURLOPT_POST, 1L);
990 curl_easy_setopt(curl, CURLOPT_POSTFIELDS, json_data);
991
992 // Discard response data - without this curl writes to stdout which can crash
993 // on Windows when running as a service or without a console
994 curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, post_status_file_discard_response);
995 curl_easy_setopt(curl, CURLOPT_WRITEDATA, NULL);
996
997 // Prevent signal-based timeouts which can cause issues on Windows/MSYS2
998 curl_easy_setopt(curl, CURLOPT_NOSIGNAL, 1L);
999
1000 struct curl_slist *headers = NULL;
1001 headers = curl_slist_append(headers, "Content-Type: application/json");
1002 curl_easy_setopt(curl, CURLOPT_HTTPHEADER, headers);
1003 curl_easy_setopt(curl, CURLOPT_TIMEOUT, 10L);
1004
1005 CURLcode rc = curl_easy_perform(curl);
1006 if(rc == CURLE_OK) {
1007 daemon_status_file_startup_step("startup(crash reports dedup)");
1008 session_status.posts++;
1009 nd_log(NDLS_DAEMON, NDLP_INFO, "Posted last status to agent-events successfully.");
1010 uint64_t hash = daemon_status_file_hash(d->status, d->msg, d->cause);
1011 dedup_keep_hash(&session_status, hash, false);
1012 daemon_status_file_save(wb, &session_status, true);
1013 }
1014 else
1015 nd_log(NDLS_DAEMON, NDLP_INFO, "Failed to post last status to agent-events.");
1016
1017 daemon_status_file_startup_step("startup(crash reports cleanup)");
1018
1019 curl_easy_cleanup(curl);
1020 curl_slist_free_all(headers);
1021 }
1022
1023 // --------------------------------------------------------------------------------------------------------------------
1024 // check last status on startup and post-crash report
1025
1026 struct log_priority {
1027 ND_LOG_FIELD_PRIORITY user;
1028 ND_LOG_FIELD_PRIORITY post;
1029 };
1030
1031 struct log_priority PRI_ALL_NORMAL = { NDLP_NOTICE, NDLP_DEBUG };
1032 struct log_priority PRI_USER_SHOULD_FIX = { NDLP_WARNING, NDLP_INFO };
1033 struct log_priority PRI_FATAL = { NDLP_ERR, NDLP_ERR };
1034 struct log_priority PRI_DEADLY_SIGNAL = { NDLP_CRIT, NDLP_CRIT };
1035 struct log_priority PRI_KILLED_HARD = { NDLP_ERR, NDLP_WARNING };
1036
1037 enum crash_report_t {
1038 DSF_REPORT_DISABLED = 0,
1039 DSF_REPORT_ALL,
1040 DSF_REPORT_CRASHES,
1041 };
1042
1043 static enum crash_report_t check_crash_reports_config(void) {
1044 bool default_enabled = analytics_check_enabled() ||
1045 !UUIDiszero(session_status.node_id) || !UUIDiszero(last_session_status.node_id) ||
1046 !UUIDiszero(session_status.claim_id) || !UUIDiszero(last_session_status.claim_id);
1047
1048 const char *t = inicfg_get(&netdata_config, CONFIG_SECTION_GLOBAL, "crash reports", default_enabled ? "all" : "off");
1049
1050 enum crash_report_t rc;
1051 if(!t || !*t)
1052 rc = default_enabled ? DSF_REPORT_ALL : DSF_REPORT_DISABLED;
1053 else if(strcmp(t, "all") == 0)
1054 rc = DSF_REPORT_ALL;
1055 else if(strcmp(t, "crashes") == 0)
1056 rc = DSF_REPORT_CRASHES;
1057 else
1058 rc = DSF_REPORT_DISABLED;
1059
1060 return rc;
1061 }
1062
1063 void daemon_status_file_init(void) {
1064 static_save_buffer_init();
1065 mallocz_register_out_of_memory_cb(daemon_status_file_out_of_memory);
1066
1067 status_file_io_load(STATUS_FILENAME, status_file_load_and_parse, &last_session_status);
1068
1069 // fill missing information on older versions of the status file
1070
1071 if(last_session_status.v <= 26)
1072 fill_dmi_info(&last_session_status);
1073
1074 if(last_session_status.v <= 27)
1075 last_session_status.system_cpus = os_get_system_cpus();
1076
1077 // Regenerate RFC3339 strings from loaded timestamps for async-signal-safe compatibility
1078 // (these fields don't exist in saved JSON, they're runtime-only for signal handlers)
1079 if(last_session_status.timestamp_ut)
1080 rfc3339_datetime_ut(last_session_status.timestamp_ut_rfc3339,
1081 sizeof(last_session_status.timestamp_ut_rfc3339),
1082 last_session_status.timestamp_ut, 2, true);
1083
1084 if(last_session_status.host_id.last_modified_ut)
1085 rfc3339_datetime_ut(last_session_status.host_id.last_modified_ut_rfc3339,
1086 sizeof(last_session_status.host_id.last_modified_ut_rfc3339),
1087 last_session_status.host_id.last_modified_ut, 2, true);
1088
1089 if(last_session_status.disk_footprint.last_updated_ut)
1090 rfc3339_datetime_ut(last_session_status.disk_footprint.last_updated_ut_rfc3339,
1091 sizeof(last_session_status.disk_footprint.last_updated_ut_rfc3339),
1092 last_session_status.disk_footprint.last_updated_ut, 2, true);
1093
1094 daemon_status_file_migrate_once();
1095 }
1096
1097 void daemon_status_file_check_crash(void) {
1098 struct log_priority pri = PRI_ALL_NORMAL;
1099
1100 bool new_version = strcmp(last_session_status.version, session_status.version) != 0;
1101 bool this_is_a_crash = false;
1102 bool no_previous_status = false;
1103 bool dump_json = true;
1104 const char *msg = "", *cause = "";
1105 switch(last_session_status.status) {
1106 default:
1107 case DAEMON_STATUS_NONE:
1108 // probably a previous version of netdata was running
1109 cause = "no last status";
1110 msg = "No status found for the previous Netdata session (new Netdata, or older version)";
1111 no_previous_status = true;
1112 break;
1113
1114 case DAEMON_STATUS_EXITED:
1115 if(last_session_status.exit_reason == EXIT_REASON_NONE) {
1116 cause = "exit no reason";
1117 msg = "Netdata was last stopped gracefully, without setting a reason";
1118 if(!last_session_status.timestamp_ut)
1119 dump_json = false;
1120 }
1121 else if(is_deadly_signal(last_session_status.exit_reason)) {
1122 cause = "deadly signal and exit";
1123 msg = "Netdata was last stopped gracefully after receiving a deadly signal";
1124 pri = PRI_DEADLY_SIGNAL;
1125 this_is_a_crash = true;
1126 }
1127 else if(last_session_status.exit_reason != EXIT_REASON_NONE &&
1128 !is_exit_reason_normal(last_session_status.exit_reason)) {
1129 cause = "fatal and exit";
1130 msg = "Netdata was last stopped gracefully after it encountered a fatal error";
1131 pri = PRI_FATAL;
1132 this_is_a_crash = true;
1133 }
1134 else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {
1135 cause = "exit on system shutdown";
1136 msg = "Netdata has gracefully stopped due to system shutdown";
1137 }
1138 else if(last_session_status.exit_reason & EXIT_REASON_UPDATE) {
1139 cause = "exit to update";
1140 msg = "Netdata has gracefully restarted to update to a new version";
1141 }
1142 else if(new_version) {
1143 cause = "exit and updated";
1144 msg = "Netdata has gracefully restarted and updated to a new version";
1145 last_session_status.exit_reason |= EXIT_REASON_UPDATE;
1146 }
1147 else {
1148 cause = "exit instructed";
1149 msg = "Netdata was last stopped gracefully";
1150 }
1151 break;
1152
1153 case DAEMON_STATUS_INITIALIZING:
1154 if (last_session_status.exit_reason == EXIT_REASON_NONE &&
1155 !UUIDiszero(session_status.boot_id) &&
1156 !UUIDiszero(last_session_status.boot_id) &&
1157 !os_boot_ids_match(session_status.boot_id, last_session_status.boot_id)) {
1158 cause = "abnormal power off";
1159 msg = "The system was abnormally powered off while Netdata was starting";
1160 pri = PRI_USER_SHOULD_FIX;
1161 }
1162 else if(is_deadly_signal(last_session_status.exit_reason)) {
1163 cause = "deadly signal on start";
1164 msg = "Netdata was last crashed while starting after receiving a deadly signal";
1165 pri = PRI_DEADLY_SIGNAL;
1166 this_is_a_crash = true;
1167 }
1168 else if (last_session_status.exit_reason & EXIT_REASON_OUT_OF_MEMORY) {
1169 cause = "out of memory";
1170 msg = "Netdata was last crashed while starting, because it couldn't allocate memory";
1171 pri = PRI_USER_SHOULD_FIX;
1172 }
1173 else if (last_session_status.exit_reason & EXIT_REASON_ALREADY_RUNNING) {
1174 cause = "already running";
1175 msg = "Netdata couldn't start, because it was already running";
1176 pri = PRI_USER_SHOULD_FIX;
1177 }
1178 else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
1179 last_session_status.var_cache.is_read_only) {
1180 cause = "disk read-only";
1181 msg = "Netdata couldn't start because the disk is readonly";
1182 pri = PRI_USER_SHOULD_FIX;
1183 }
1184 else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
1185 last_session_status.var_cache.free_bytes == 0) {
1186 cause = "disk full";
1187 msg = "Netdata couldn't start because the disk is full";
1188 pri = PRI_USER_SHOULD_FIX;
1189 }
1190 else if (OS_SYSTEM_DISK_SPACE_OK(last_session_status.var_cache) &&
1191 last_session_status.var_cache.free_bytes < 10 * 1024 * 1024) {
1192 cause = "disk almost full";
1193 msg = "Netdata couldn't start while the disk is almost full";
1194 pri = PRI_USER_SHOULD_FIX;
1195 }
1196 else if (last_session_status.exit_reason != EXIT_REASON_NONE &&
1197 !is_exit_reason_normal(last_session_status.exit_reason)) {
1198 cause = "fatal on start";
1199 msg = "Netdata was last crashed while starting, because of a fatal error";
1200 pri = PRI_FATAL;
1201 }
1202 else {
1203 cause = "killed hard on start";
1204 msg = "Netdata was last killed/crashed while starting";
1205 pri = PRI_KILLED_HARD;
1206 }
1207 this_is_a_crash = true;
1208 break;
1209
1210 case DAEMON_STATUS_EXITING:
1211 if(is_deadly_signal(last_session_status.exit_reason)) {
1212 cause = "deadly signal on exit";
1213 msg = "Netdata was last crashed while exiting after receiving a deadly signal";
1214 pri = PRI_DEADLY_SIGNAL;
1215 }
1216 else if(last_session_status.exit_reason & EXIT_REASON_SHUTDOWN_TIMEOUT) {
1217 cause = "exit timeout";
1218 msg = "Netdata was last killed because it couldn't shutdown on time";
1219 pri = PRI_FATAL;
1220 }
1221 else if(last_session_status.exit_reason != EXIT_REASON_NONE &&
1222 !is_exit_reason_normal(last_session_status.exit_reason)) {
1223 cause = "fatal on exit";
1224 msg = "Netdata was last killed/crashed while exiting after encountering an error";
1225 pri = PRI_FATAL;
1226 }
1227 else if(last_session_status.exit_reason & EXIT_REASON_SYSTEM_SHUTDOWN) {
1228 cause = "killed hard on shutdown";
1229 msg = "Netdata was last killed/crashed while exiting due to system shutdown";
1230 pri = PRI_KILLED_HARD;
1231 }
1232 else if(new_version || (last_session_status.exit_reason & EXIT_REASON_UPDATE)) {
1233 cause = "killed hard on update";
1234 msg = "Netdata was last killed/crashed while exiting to update to a new version";
1235 pri = PRI_KILLED_HARD;
1236 }
1237 else {
1238 cause = "killed hard on exit";
1239 msg = "Netdata was last killed/crashed while it was instructed to exit";
1240 pri = PRI_KILLED_HARD;
1241 }
1242 this_is_a_crash = true;
1243 break;
1244
1245 case DAEMON_STATUS_RUNNING: {
1246 if (last_session_status.exit_reason == EXIT_REASON_NONE &&
1247 !UUIDiszero(session_status.boot_id) &&
1248 !UUIDiszero(last_session_status.boot_id) &&
1249 !os_boot_ids_match(session_status.boot_id, last_session_status.boot_id)) {
1250 cause = "abnormal power off";
1251 msg = "The system was abnormally powered off while Netdata was running";
1252 pri = PRI_USER_SHOULD_FIX;
1253 }
1254 else if (last_session_status.exit_reason & EXIT_REASON_OUT_OF_MEMORY) {
1255 cause = "out of memory";
1256 msg = "Netdata was last crashed because it couldn't allocate memory";
1257 pri = PRI_USER_SHOULD_FIX;
1258 }
1259 else if(is_deadly_signal(last_session_status.exit_reason)) {
1260 cause = "deadly signal";
1261 msg = "Netdata was last crashed after receiving a deadly signal";
1262 pri = PRI_DEADLY_SIGNAL;
1263 this_is_a_crash = true;
1264 }
1265 else if (last_session_status.exit_reason != EXIT_REASON_NONE &&
1266 !is_exit_reason_normal(last_session_status.exit_reason)) {
1267 cause = "killed fatal";
1268 msg = "Netdata was last crashed due to a fatal error";
1269 pri = PRI_FATAL;
1270 }
1271 else if (OS_SYSTEM_MEMORY_OK(last_session_status.memory) &&
1272 last_session_status.memory.ram_available_bytes <= last_session_status.oom_protection) {
1273 cause = "killed hard low ram";
1274 msg = "Netdata was last killed/crashed while available memory was critically low";
1275 pri = PRI_KILLED_HARD;
1276 this_is_a_crash = true;
1277 }
1278 else {
1279 cause = "killed hard";
1280 msg = "Netdata was last killed/crashed while operating normally";
1281 pri = PRI_KILLED_HARD;
1282 this_is_a_crash = true;
1283 }
1284 break;
1285 }
1286 }
1287
1288 CLEAN_BUFFER *wb = buffer_create(0, NULL);
1289 buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_DEFAULT);
1290 if(dump_json)
1291 daemon_status_file_to_json(wb, &last_session_status);
1292 buffer_json_finalize(wb);
1293
1294 ND_LOG_STACK lgs[] = {
1295 ND_LOG_FIELD_UUID(NDF_MESSAGE_ID, &netdata_startup_msgid),
1296 ND_LOG_FIELD_END(),
1297 };
1298 ND_LOG_STACK_PUSH(lgs);
1299
1300 nd_log(NDLS_DAEMON, pri.user,
1301 "Netdata Agent version '%s' is starting...\n"
1302 "Last exit status: %s (%s):\n\n%s",
1303 NETDATA_VERSION, msg, cause, buffer_tostring(wb));
1304
1305 daemon_status_file_startup_step("startup(crash reports check)");
1306
1307 enum crash_report_t r = check_crash_reports_config();
1308 if( // must be first for netdata.conf option to be used
1309 (r == DSF_REPORT_ALL || (this_is_a_crash && r == DSF_REPORT_CRASHES)) &&
1310
1311 // we have a previous status, or we managed to save the current one
1312 (!no_previous_status || daemon_status_file_saved) &&
1313
1314 // we have more than 2 restarts, or this is not a CI run
1315 (last_session_status.restarts > 1 || !nd_is_running_under_ci()) &&
1316
1317 // we have not reported this
1318 !dedup_already_posted(&session_status, daemon_status_file_hash(&last_session_status, msg, cause), false)
1319
1320 ) {
1321 daemon_status_file_startup_step("startup(crash reports prep)");
1322
1323 netdata_conf_ssl();
1324
1325 if(no_previous_status) {
1326 last_session_status = session_status;
1327 last_session_status.status = DAEMON_STATUS_NONE;
1328 last_session_status.exit_reason = 0;
1329 memset(&last_session_status.fatal, 0, sizeof(last_session_status.fatal));
1330 }
1331
1332 struct post_status_file_thread_data d = {
1333 .cause = cause,
1334 .msg = msg,
1335 .status = &last_session_status,
1336 .priority = pri.post,
1337 };
1338
1339 post_status_file(&d);
1340
1341 // MacOS crashes when starting under launchctl, when we create a thread to post the status file,
1342 // so we post the status file synchronously, with a timeout of 10 seconds.
1343 }
1344 }
1345
1346 NEVER_INLINE
1347 static void daemon_status_file_save_twice_if_we_can_get_stack_trace(BUFFER *wb, DAEMON_STATUS_FILE *ds, bool force) {
1348 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS
1349 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS
1350
1351 #ifdef HAVE_LIBBACKTRACE
1352 if(stacktrace_available())
1353 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "will now attempt to get stack trace - if you see this message, we couldn't get it.");
1354 else
1355 #endif
1356 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "no stack trace backend available");
1357
1358 // save it without a stack trace to be sure we will have the event
1359 daemon_status_file_save(wb, ds, false);
1360
1361 if(!stack_trace_is_empty(ds) && !force)
1362 return;
1363
1364 buffer_flush(wb);
1365
1366 #ifdef HAVE_LIBBACKTRACE
1367 stacktrace_capture(wb);
1368
1369 // Store the first netdata function from the stack trace if available
1370 const char *first_nd_fn = stacktrace_root_cause_function();
1371 if (first_nd_fn && *first_nd_fn &&
1372 (!ds->fatal.function[0] || strncmp(ds->fatal.function, "thread:", 7) == 0))
1373 safecpy(ds->fatal.function, first_nd_fn);
1374 #endif
1375
1376 if(buffer_strlen(wb) > 0) {
1377 safecpy(ds->fatal.stack_trace, buffer_tostring(wb));
1378
1379 daemon_status_file_save(wb, ds, false);
1380 }
1381
1382 errno_clear();
1383 }
1384
1385 // --------------------------------------------------------------------------------------------------------------------
1386 // ng_log() hook for receiving fatal message information
1387
1388 NEVER_INLINE
1389 void daemon_status_file_register_fatal(const char *filename, const char *function, const char *message, const char *errno_str, const char *stack_trace, long line) {
1390 FUNCTION_RUN_ONCE();
1391
1392 dsf_acquire(session_status);
1393 spinlock_lock(&session_status.fatal.spinlock);
1394
1395 exit_initiated_add(EXIT_REASON_FATAL);
1396 session_status.exit_reason |= EXIT_REASON_FATAL;
1397
1398 if(!session_status.fatal.thread_id)
1399 session_status.fatal.thread_id = gettid_cached();
1400
1401 copy_and_clean_thread_name_if_empty(&session_status, nd_thread_tag());
1402
1403 if(filename && *filename)
1404 safecpy(session_status.fatal.filename, filename);
1405
1406 if(function && *function)
1407 safecpy(session_status.fatal.function, function);
1408
1409 if(message && *message)
1410 safecpy(session_status.fatal.message, message);
1411
1412 if(errno_str && *errno_str)
1413 safecpy(session_status.fatal.errno_str, errno_str);
1414
1415 if(stack_trace && *stack_trace && stack_trace_is_empty(&session_status))
1416 safecpy(session_status.fatal.stack_trace, stack_trace);
1417
1418 if(!session_status.fatal.worker_job_id)
1419 session_status.fatal.worker_job_id = workers_get_last_job_id();
1420
1421 if(line)
1422 session_status.fatal.line = line;
1423
1424 spinlock_unlock(&session_status.fatal.spinlock);
1425 dsf_release(session_status);
1426
1427 CLEAN_BUFFER *wb = buffer_create(0, NULL);
1428 daemon_status_file_save_twice_if_we_can_get_stack_trace(wb, &session_status, false);
1429
1430 freez((void *)filename);
1431 freez((void *)function);
1432 freez((void *)message);
1433 freez((void *)errno_str);
1434 freez((void *)stack_trace);
1435
1436 #ifdef ENABLE_SENTRY
1437 nd_sentry_add_fatal_message_as_breadcrumb();
1438 #endif
1439 }
1440
1441 // --------------------------------------------------------------------------------------------------------------------
1442
1443 void daemon_status_file_update_status(DAEMON_STATUS status) {
1444 int saved_errno = errno;
1445 CLEAN_BUFFER *wb = buffer_create(0, NULL);
1446 daemon_status_file_refresh(status);
1447 daemon_status_file_save(wb, &session_status, true);
1448 errno = saved_errno;
1449 }
1450
1451 static void daemon_status_file_out_of_memory(void) {
1452 FUNCTION_RUN_ONCE();
1453
1454 // DO NOT ALLOCATE IN THIS FUNCTION - WE DON'T HAVE ANY MEMORY AVAILABLE!
1455
1456 // the buffer should already be allocated, so this should normally do nothing
1457 static_save_buffer_init();
1458
1459 dsf_acquire(session_status);
1460 exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);
1461 session_status.exit_reason |= EXIT_REASON_OUT_OF_MEMORY;
1462 dsf_release(session_status);
1463
1464 daemon_status_file_save_twice_if_we_can_get_stack_trace(static_save_buffer, &session_status, true);
1465 }
1466
1467 NEVER_INLINE
1468 bool daemon_status_file_deadly_signal_received(EXIT_REASON reason, SIGNAL_CODE code, void *fault_address, bool chained_handler) {
1469 FUNCTION_RUN_ONCE_RET(true);
1470
1471 // IMPORTANT: NO LOCKS OR ALLOCATIONS HERE, THIS FUNCTION IS CALLED FROM SIGNAL HANDLERS
1472 // THIS FUNCTION MUST USE ONLY ASYNC-SIGNAL-SAFE OPERATIONS
1473
1474 dsf_acquire(session_status);
1475
1476 exit_initiated_add(reason);
1477 session_status.exit_reason |= reason;
1478 session_status.fatal.sentry = chained_handler;
1479
1480 if(code)
1481 session_status.fatal.signal_code = code;
1482
1483 if(fault_address)
1484 session_status.fatal.fault_address = (uintptr_t)fault_address;
1485
1486 if(!session_status.fatal.thread_id)
1487 session_status.fatal.thread_id = gettid_cached();
1488
1489 if(!session_status.fatal.worker_job_id)
1490 session_status.fatal.worker_job_id = workers_get_last_job_id();
1491
1492 copy_and_clean_thread_name_if_empty(&session_status, nd_thread_tag_async_safe());
1493
1494 if(!session_status.fatal.function[0] ||
1495 strncmp(session_status.fatal.function, "startup(", 8) == 0 ||
1496 strncmp(session_status.fatal.function, "shutdown(", 9) == 0) {
1497 size_t len = 0;
1498 len = strcatz(session_status.fatal.function, len, "thread:", sizeof(session_status.fatal.function));
1499 len = strcatz(session_status.fatal.function, len, session_status.fatal.thread, sizeof(session_status.fatal.function));
1500 if(session_status.fatal.worker_job_id <= WORKER_UTILIZATION_MAX_JOB_TYPES) {
1501 len = strcatz(session_status.fatal.function, len, ":", sizeof(session_status.fatal.function));
1502 len += print_uint64(&session_status.fatal.function[len], session_status.fatal.worker_job_id);
1503 }
1504 }
1505
1506 dsf_release(session_status);
1507
1508 // the buffer should already be allocated, so this should normally do nothing
1509 static_save_buffer_init();
1510
1511 // deduplicate the crash for sentry
1512 bool duplicate = false;
1513 if(chained_handler) {
1514 uint64_t hash = daemon_status_file_hash(&session_status, NULL, NULL);
1515 duplicate = dedup_already_posted(&session_status, hash, true);
1516 if (!duplicate) {
1517 // save this hash, so that we won't post it again to sentry
1518 dedup_keep_hash(&session_status, hash, true);
1519 }
1520 }
1521
1522 #ifdef HAVE_LIBBACKTRACE
1523 #if defined(OS_WINDOWS)
1524 // THE FOLLOWING CODE IS NOT ASYNC-SIGNAL-SAFE on MSYS2 due to internal locking in the runtime.
1525 // This can cause a deadlock when a signal is received while the lock is held.
1526 // The code is commented out to prevent the deadlock, at the cost of not saving the status file on a crash.
1527 #else
1528 bool safe_to_get_stack_trace = reason != EXIT_REASON_SIGABRT || stacktrace_capture_is_async_signal_safe();
1529 bool get_stack_trace = stacktrace_available() && safe_to_get_stack_trace && stack_trace_is_empty(&session_status);
1530
1531 // save it
1532 if(get_stack_trace)
1533 daemon_status_file_save_twice_if_we_can_get_stack_trace(static_save_buffer, &session_status, true);
1534 else {
1535 if (!stacktrace_available())
1536 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "no stack trace backend available");
1537 else
1538 set_stack_trace_message_if_empty(&session_status, STACK_TRACE_INFO_PREFIX "not safe to get a stack trace for this signal using this backend");
1539
1540 daemon_status_file_save(static_save_buffer, &session_status, false);
1541 }
1542 #endif // defined(OS_WINDOWS)
1543 #endif // HAVE_LIBBACKTRACE
1544
1545 return duplicate;
1546 }
1547
1548 // --------------------------------------------------------------------------------------------------------------------
1549 // shutdown related functions
1550
1551 static SPINLOCK shutdown_timeout_spinlock = SPINLOCK_INITIALIZER;
1552
1553 NEVER_INLINE
1554 void daemon_status_file_shutdown_timeout(BUFFER *trace) {
1555 FUNCTION_RUN_ONCE();
1556
1557 spinlock_lock(&shutdown_timeout_spinlock);
1558
1559 dsf_acquire(session_status);
1560 exit_initiated_add(EXIT_REASON_SHUTDOWN_TIMEOUT);
1561 session_status.exit_reason |= EXIT_REASON_SHUTDOWN_TIMEOUT;
1562 if(trace && buffer_strlen(trace) && stack_trace_is_empty(&session_status))
1563 safecpy(session_status.fatal.stack_trace, buffer_tostring(trace));
1564 dsf_release(session_status);
1565
1566 safecpy(session_status.fatal.function, "shutdown_timeout");
1567
1568 static_save_buffer_init();
1569 daemon_status_file_save(static_save_buffer, &session_status, false);
1570
1571 // keep the spinlock locked, to prevent further steps updating the status
1572 }
1573
1574 void daemon_status_file_shutdown_step(const char *step, const char *step_timings) {
1575 if(session_status.fatal.filename[0] || !spinlock_trylock(&shutdown_timeout_spinlock))
1576 // we have a fatal logged
1577 return;
1578
1579 if(step != NULL)
1580 snprintfz(session_status.fatal.function, sizeof(session_status.fatal.function), "shutdown(%s)", step);
1581 else
1582 session_status.fatal.function[0] = '\0';
1583
1584 if(step_timings && *step_timings && stack_trace_is_empty(&session_status))
1585 safecpy(session_status.fatal.stack_trace, step_timings);
1586
1587 daemon_status_file_update_status(DAEMON_STATUS_EXITING);
1588
1589 spinlock_unlock(&shutdown_timeout_spinlock);
1590 }
1591
1592 // --------------------------------------------------------------------------------------------------------------------
1593
1594 bool daemon_status_file_has_last_crashed(DAEMON_STATUS_FILE *ds) {
1595 if(!ds) ds = &last_session_status;
1596
1597 return (ds->status != DAEMON_STATUS_NONE && ds->status != DAEMON_STATUS_EXITED) ||
1598 !is_exit_reason_normal(ds->exit_reason);
1599 }
1600
1601 bool daemon_status_file_was_incomplete_shutdown(void) {
1602 return last_session_status.status == DAEMON_STATUS_EXITING;
1603 }
1604
1605 // --------------------------------------------------------------------------------------------------------------------
1606 // startup and shutdown steps
1607
1608 void daemon_status_file_startup_step(const char *step) {
1609 if(session_status.fatal.filename[0])
1610 // we have a fatal logged
1611 return;
1612
1613 if(step != NULL)
1614 safecpy(session_status.fatal.function, step);
1615 else
1616 session_status.fatal.function[0] = '\0';
1617
1618 daemon_status_file_update_status(DAEMON_STATUS_INITIALIZING);
1619 }
1620
1621 // --------------------------------------------------------------------------------------------------------------------
1622 // public API to get values
1623
1624 const char *daemon_status_file_get_install_type(void) {
1625 return session_status.install_type;
1626 }
1627
1628 const char *daemon_status_file_get_architecture(void) {
1629 return session_status.architecture;
1630 }
1631
1632 const char *daemon_status_file_get_virtualization(void) {
1633 return session_status.virtualization;
1634 }
1635
1636 const char *daemon_status_file_get_container(void) {
1637 return session_status.container;
1638 }
1639
1640 const char *daemon_status_file_get_os_name(void) {
1641 return session_status.os_name;
1642 }
1643
1644 const char *daemon_status_file_get_os_version(void) {
1645 return session_status.os_version;
1646 }
1647
1648 const char *daemon_status_file_get_os_id(void) {
1649 return session_status.os_id;
1650 }
1651
1652 const char *daemon_status_file_get_os_id_like(void) {
1653 return session_status.os_id_like;
1654 }
1655
1656 const char *daemon_status_file_get_cloud_provider_type(void) {
1657 return session_status.cloud_provider_type;
1658 }
1659
1660 const char *daemon_status_file_get_cloud_instance_type(void) {
1661 return session_status.cloud_instance_type;
1662 }
1663
1664 const char *daemon_status_file_get_cloud_instance_region(void) {
1665 return session_status.cloud_instance_region;
1666 }
1667
1668 const char *daemon_status_file_get_timezone(void) {
1669 return session_status.timezone;
1670 }
1671
1672 const char *daemon_status_file_get_fatal_filename(void) {
1673 return session_status.fatal.filename;
1674 }
1675
1676 const char *daemon_status_file_get_fatal_function(void) {
1677 return session_status.fatal.function;
1678 }
1679
1680 const char *daemon_status_file_get_fatal_message(void) {
1681 return session_status.fatal.message;
1682 }
1683
1684 const char *daemon_status_file_get_fatal_errno(void) {
1685 return session_status.fatal.errno_str;
1686 }
1687
1688 const char *daemon_status_file_get_fatal_stack_trace(void) {
1689 return session_status.fatal.stack_trace;
1690 }
1691
1692 const char *daemon_status_file_get_stack_trace_backend(void) {
1693 return session_status.stack_traces;
1694 }
1695
1696 const char *daemon_status_file_get_fatal_thread(void) {
1697 return session_status.fatal.thread;
1698 }
1699
1700 const char *daemon_status_file_get_sys_vendor(void) {
1701 return session_status.product.vendor;
1702 }
1703
1704 const char *daemon_status_file_get_product_name(void) {
1705 return session_status.product.name;
1706 }
1707
1708 const char *daemon_status_file_get_product_type(void) {
1709 return session_status.product.type;
1710 }
1711
1712 pid_t daemon_status_file_get_fatal_thread_id(void) {
1713 return session_status.fatal.thread_id;
1714 }
1715
1716 long daemon_status_file_get_fatal_line(void) {
1717 return session_status.fatal.line;
1718 }
1719
1720 DAEMON_STATUS daemon_status_file_get_status(void) {
1721 return session_status.status;
1722 }
1723
1724 size_t daemon_status_file_get_restarts(void) {
1725 return session_status.restarts;
1726 }
1727
1728 ssize_t daemon_status_file_get_reliability(void) {
1729 return session_status.reliability;
1730 }
1731
1732 ND_MACHINE_GUID daemon_status_file_get_host_id(void) {
1733 return last_session_status.host_id;
1734 }
1735
1736 size_t daemon_status_file_get_fatal_worker_job_id(void) {
1737 return session_status.fatal.worker_job_id;
1738 }