status file 21b (#19937)
add netdata max rss and oom protection to status file; added killed hard low mem exit cause
Costa Tsaousis committed
Mar 22, 2025 at 00:23 UTC
52a60fba39ec7413e78ef63565e415e90663468a
4 files changed
+41
-9
src/daemon/daemon-status-file.c
+23
@@ -228,6 +228,11 @@ static void daemon_status_file_to_json(BUFFER *wb, DAEMON_STATUS_FILE *ds) {
228
if(OS_SYSTEM_MEMORY_OK(ds->memory)) {
229
buffer_json_member_add_uint64(wb, "total", ds->memory.ram_total_bytes);
230
buffer_json_member_add_uint64(wb, "free", ds->memory.ram_available_bytes);
231
+
232
+ if(ds->v >= 21) {
233
+ buffer_json_member_add_uint64(wb, "netdata", ds->netdata_max_rss);
234
+ buffer_json_member_add_uint64(wb, "oom_protection", ds->oom_protection);
235
+ }
236
}
237
buffer_json_object_close(wb);
238
@@ -331,6 +336,7 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *
336
bool required_v17 = version >= 17 ? strict : false;
337
bool required_v18 = version >= 18 ? strict : false;
338
bool required_v20 = version >= 20 ? strict : false;
339
+ bool required_v21 = version >= 21 ? strict : false;
340
341
// Parse timestamp
342
JSONC_PARSE_TXT2CHAR_OR_ERROR_AND_RETURN(jobj, path, "@timestamp", datetime, error, required_v1);
@@ -414,6 +420,11 @@ static bool daemon_status_file_from_json(json_object *jobj, void *data, BUFFER *
420
JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "free", ds->memory.ram_available_bytes, error, false);
421
if(!OS_SYSTEM_MEMORY_OK(ds->memory))
422
ds->memory = OS_SYSTEM_MEMORY_EMPTY;
423
+
424
+ if(version >= 21) {
425
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "netdata", ds->netdata_max_rss, error, required_v21);
426
+ JSONC_PARSE_UINT64_OR_ERROR_AND_RETURN(jobj, path, "oom_protection", ds->oom_protection, error, required_v21);
427
+ }
428
});
429
430
JSONC_PARSE_SUBOBJECT(jobj, path, "disk", error, required_v1, {
@@ -617,6 +628,9 @@ static void daemon_status_file_refresh(DAEMON_STATUS status) {
628
session_status.db_mode = default_rrd_memory_mode;
629
session_status.db_tiers = nd_profile.storage_tiers;
630
631
+ session_status.oom_protection = dbengine_out_of_memory_protection;
632
+ session_status.netdata_max_rss = process_max_rss();
633
+
634
session_status.claim_id = claim_id_get_uuid();
635
636
if(localhost) {
@@ -984,6 +998,8 @@ void post_status_file(struct post_status_file_thread_data *d) {
998
buffer_json_member_add_uint64(wb, "priority", d->priority);
999
buffer_json_member_add_uint64(wb, "version_saved", d->status->v);
1000
buffer_json_member_add_string(wb, "agent_version_now", NETDATA_VERSION);
1001
+ buffer_json_member_add_boolean(wb, "host_memory_critical",
1002
+ OS_SYSTEM_MEMORY_OK(d->status->memory) && d->status->memory.ram_available_bytes <= d->status->oom_protection);
1003
daemon_status_file_to_json(wb, d->status);
1004
buffer_json_finalize(wb);
1005
@@ -1260,6 +1276,13 @@ void daemon_status_file_check_crash(void) {
1276
msg = "Netdata was last crashed due to a fatal error";
1277
pri = PRI_FATAL;
1278
}
1279
+ else if (OS_SYSTEM_MEMORY_OK(last_session_status.memory) &&
1280
+ last_session_status.memory.ram_available_bytes <= last_session_status.oom_protection) {
1281
+ cause = "killed hard low ram";
1282
+ msg = "Netdata was last killed/crashed while available memory was critically low";
1283
+ pri = PRI_KILLED_HARD;
1284
+ this_is_a_crash = true;
1285
+ }
1286
else {
1287
cause = "killed hard";
1288
msg = "Netdata was last killed/crashed while operating normally";
src/daemon/daemon-status-file.h
+2
@@ -61,6 +61,8 @@ typedef struct daemon_status_file {
61
time_t exit;
62
} timings;
63
64
+ uint64_t oom_protection;
65
+ uint64_t netdata_max_rss;
66
OS_SYSTEM_MEMORY memory;
67
OS_SYSTEM_DISK_SPACE var_cache;
68
src/libnetdata/memory/nd-mallocz.c
+15
-9
@@ -7,13 +7,7 @@ void mallocz_register_out_of_memory_cb(out_of_memory_cb cb) {
7
out_of_memory_callback = cb;
8
}
9
10
-ALWAYS_INLINE NORETURN
11
-void out_of_memory(const char *call, size_t size, const char *details) {
12
- exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);
13
-
14
- if(out_of_memory_callback)
15
- out_of_memory_callback();
16
-
10
+uint64_t process_max_rss(void) {
11
#if defined(OS_LINUX) || defined(OS_WINDOWS)
12
int rss_multiplier = 1024;
13
#else
@@ -22,14 +16,26 @@ void out_of_memory(const char *call, size_t size, const char *details) {
16
17
struct rusage usage = { 0 };
18
if(getrusage(RUSAGE_SELF, &usage) != 0)
25
- usage.ru_maxrss = 0;
19
+ return 0;
20
+
21
+ return usage.ru_maxrss * rss_multiplier;
22
+}
23
+
24
+ALWAYS_INLINE NORETURN
25
+void out_of_memory(const char *call, size_t size, const char *details) {
26
+ exit_initiated_add(EXIT_REASON_OUT_OF_MEMORY);
27
+
28
+ if(out_of_memory_callback)
29
+ out_of_memory_callback();
30
+
31
+ uint64_t max_rss = process_max_rss();
32
33
char mem_available[64];
34
char rss_used[64];
35
36
OS_SYSTEM_MEMORY sm = os_last_reported_system_memory();
37
size_snprintf(mem_available, sizeof(mem_available), sm.ram_available_bytes, "B", false);
32
- size_snprintf(rss_used, sizeof(rss_used), usage.ru_maxrss * rss_multiplier, "B", false);
38
+ size_snprintf(rss_used, sizeof(rss_used), max_rss, "B", false);
39
40
fatal("Out of memory on %s(%zu bytes)!\n"
41
"System memory available: %s, while our max RSS usage is: %s\n"
src/libnetdata/memory/nd-mallocz.h
+1
@@ -69,5 +69,6 @@ void mallocz_register_out_of_memory_cb(out_of_memory_cb cb);
69
70
NORETURN
71
void out_of_memory(const char *call, size_t size, const char *details);
72
+uint64_t process_max_rss(void);
73
74
#endif //NETDATA_ND_MALLOCZ_H