Strip intermediate objects from flattened JSON (#22085)
Strip intermediate objects from flattened JSON to prevent data duplication flatten_serde_json::flatten() preserves intermediate object nodes alongside leaf values, causing space waste. For example, flattening nested structures produced both the flattened leaves AND the parent objects as serialized JSON blobs, with leaf values duplicated into arrays. Introduce flatten_and_strip() that filters out intermediate objects after flattening. Split json_map_from_key_value_list (raw builder) from json_from_key_value_list (flatten+strip) to avoid double-flattening nested KvlistValues.
vkalintiris committed
Mar 30, 2026 at 14:42 UTC
912aa08a3689432c289a57d303942e68220a4b71
2 files changed
+19
-6
src/crates/netdata-otel/flatten_otel/src/lib.rs
+15
-2
@@ -13,6 +13,10 @@ pub use logs::{json_from_export_logs_service_request, json_from_log_record};
13
pub use metrics::flatten_metrics_request;
14
15
pub fn json_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValue> {
16
+ flatten_and_strip(&json_map_from_key_value_list(kvl))
17
+}
18
+
19
+fn json_map_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValue> {
20
let mut map = JsonMap::new();
21
22
for kv in kvl {
@@ -23,7 +27,14 @@ pub fn json_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValu
27
}
28
}
29
26
- flatten_serde_json::flatten(&map)
30
+ map
31
+}
32
+
33
+pub(crate) fn flatten_and_strip(map: &JsonMap<String, JsonValue>) -> JsonMap<String, JsonValue> {
34
+ flatten_serde_json::flatten(map)
35
+ .into_iter()
36
+ .filter(|(_k, v)| !v.is_object())
37
+ .collect()
38
}
39
40
fn json_from_any_value(any_value: &AnyValue) -> JsonValue {
@@ -42,7 +53,9 @@ fn json_from_any_value(any_value: &AnyValue) -> JsonValue {
53
let values: Vec<JsonValue> = array.values.iter().map(json_from_any_value).collect();
54
JsonValue::Array(values)
55
}
45
- Some(Value::KvlistValue(kvl)) => JsonValue::Object(json_from_key_value_list(&kvl.values)),
56
+ Some(Value::KvlistValue(kvl)) => {
57
+ JsonValue::Object(json_map_from_key_value_list(&kvl.values))
58
+ }
59
Some(Value::BytesValue(bytes)) => JsonValue::String(BASE64.encode(bytes)),
60
None => JsonValue::Null,
61
}
src/crates/netdata-otel/flatten_otel/src/logs.rs
+4
-4
@@ -5,8 +5,8 @@ use opentelemetry_proto::tonic::{
5
};
6
7
use crate::{
8
- json_from_any_value, json_from_instrumentation_scope, json_from_key_value_list,
9
- json_from_resource,
8
+ flatten_and_strip, json_from_any_value, json_from_instrumentation_scope,
9
+ json_from_key_value_list, json_from_resource,
10
};
11
12
pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &LogRecord) {
@@ -43,7 +43,7 @@ pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &Lo
43
let mut temp_map = JsonMap::new();
44
temp_map.insert("body".to_string(), parsed);
45
46
- let flattened_body = flatten_serde_json::flatten(&temp_map);
46
+ let flattened_body = flatten_and_strip(&temp_map);
47
for (key, value) in flattened_body {
48
jm.insert(format!("log.{}", key), value);
49
}
@@ -66,7 +66,7 @@ pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &Lo
66
let mut temp_map = JsonMap::new();
67
temp_map.insert("body".to_string(), body_json);
68
69
- let flattened_body = flatten_serde_json::flatten(&temp_map);
69
+ let flattened_body = flatten_and_strip(&temp_map);
70
for (key, value) in flattened_body {
71
jm.insert(format!("log.{}", key), value);
72
}