@cryptotaxi247 / netdata / commits / d41571962

Add OpenTelemetry plugin implementation. (#20765)

* Add OpenTelemetry plugin implementation. Collect OTEL metrics via Netdata's external plugin protocol, and logs via Netdata's implementation of systemd's journal file format. * User workspace deps for flog-otel * Remove cargo config * Add newline * Make github happy with yml files * Address two warnings * Adress review comments. * Explicitly list crates we want corrosion to import. Currently, this is just journal_reader_ffi and otel-plugin. The otel-plugin depends on the journal_reader crate and gets built by default on most Linux platforms. In the future, we might want to add a more fine-grained declaration of the crates we want. * Fix runtime checks for static builds. * Listen to localhost:4317 by default * Add keep-alive command to protocol * set MSRV to 1.85 * Fix lifetime warnings --------- Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud>

vkalintiris committed Aug 11, 2025 at 20:54 UTC d415719622419372f4e2c58ceed3358dbb39d353
67 files changed +6920 -1717
CMakeLists.txt
+28 -39
@@ -175,7 +175,7 @@ mark_as_advanced(ENABLE_DASHBOARD)
175
176 # Data collection plugins
177 option(ENABLE_PLUGIN_GO "Enable metric collectors written in Go" ${DEFAULT_FEATURE_STATE})
178 -option(ENABLE_PLUGIN_OTEL "Enable metrics collection via OpenTelemetry collector" False)
178 +option(ENABLE_PLUGIN_OTEL "Enable collection of OpenTelemetry metrics and logs" ${DEFAULT_FEATURE_STATE})
179 option(ENABLE_PLUGIN_PYTHON "Enable metric collectors written in Python" ${DEFAULT_FEATURE_STATE})
180
181 cmake_dependent_option(ENABLE_PLUGIN_APPS "Enable per-process resource usage monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX OR OS_FREEBSD OR OS_MACOS OR OS_WINDOWS" False)
@@ -227,13 +227,18 @@ cmake_dependent_option(FORCE_LEGACY_LIBBPF "Force usage of libbpf 0.0.9 instead
227 mark_as_advanced(FORCE_LEGACY_LIBBPF)
228
229 cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable netdata's journal file reader implementation" False "ENABLE_PLUGIN_SYSTEMD_JOURNAL" False)
230 -cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS "Enable netdata's journal file reader implementation benchmarks" False "ENABLE_NETDATA_JOURNAL_FILE_READER" False)
231 -mark_as_advanced(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS)
230
233 -if(ENABLE_PLUGIN_SYSTEMD_JOURNAL AND NOT ENABLE_NETDATA_JOURNAL_FILE_READER)
234 - set(ENABLE_LIBSYSTEMD_JOURNAL_FILE_READER True)
235 -else()
236 - set(ENABLE_LIBSYSTEMD_JOURNAL_FILE_READER False)
231 +# Setup Rust/Corrosion for plugins that need it
232 +if(ENABLE_NETDATA_JOURNAL_FILE_READER OR ENABLE_PLUGIN_OTEL)
233 + include(FetchContent)
234 + FetchContent_Declare(
235 + Corrosion
236 + GIT_REPOSITORY https://github.com/netdata/corrosion.git
237 + GIT_TAG f3b91559efca32c6b54837866ef35ba98ff5b2ca # stable/v0.5
238 + )
239 + FetchContent_MakeAvailable(Corrosion)
240 + corrosion_import_crate(MANIFEST_PATH src/crates/jf/Cargo.toml
241 + CRATES journal_reader_ffi otel-plugin)
242 endif()
243
244 option(ENABLE_MIMALLOC "Enable mimalloc allocator" OFF)
@@ -322,7 +327,7 @@ if(ENABLE_JEMALLOC)
327 endif()
328 endif()
329
325 -if(ENABLE_PLUGIN_GO OR ENABLE_PLUGIN_OTEL)
330 +if(ENABLE_PLUGIN_GO)
331 include(NetdataGoTools)
332
333 find_min_go_version("${CMAKE_SOURCE_DIR}/src/go")
@@ -2740,33 +2745,8 @@ if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2745 add_executable(systemd-journal.plugin ${SYSTEMD_JOURNAL_PLUGIN_FILES})
2746
2747 if(ENABLE_NETDATA_JOURNAL_FILE_READER)
2743 - include(FetchContent)
2744 - FetchContent_Declare(
2745 - Corrosion
2746 - GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
2747 - GIT_TAG v0.5
2748 - )
2749 - FetchContent_MakeAvailable(Corrosion)
2750 -
2751 - corrosion_import_crate(MANIFEST_PATH src/crates/jf/Cargo.toml)
2748 target_compile_definitions(systemd-journal.plugin PRIVATE HAVE_RUST_PROVIDER)
2749 target_link_libraries(systemd-journal.plugin journal_reader_ffi)
2754 -
2755 - if(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS)
2756 - set(JF_BENCHMARK_FILES src/crates/jf/benchmark/main.c
2757 - src/collectors/systemd-journal.plugin/provider/netdata_provider.c
2758 - src/collectors/systemd-journal.plugin/provider/netdata_provider.h
2759 - )
2760 -
2761 - add_executable(jfr.bench ${JF_BENCHMARK_FILES})
2762 - target_compile_definitions(jfr.bench PRIVATE HAVE_RUST_PROVIDER)
2763 - target_include_directories(jfr.bench PRIVATE "${CMAKE_SOURCE_DIR}/src")
2764 - target_link_libraries(jfr.bench journal_reader_ffi)
2765 -
2766 - add_executable(jfc.bench ${JF_BENCHMARK_FILES})
2767 - target_include_directories(jfc.bench PRIVATE "${CMAKE_SOURCE_DIR}/src")
2768 - target_link_libraries(jfc.bench "${SYSTEMD_LDFLAGS}")
2769 - endif()
2750 endif()
2751
2752 target_link_libraries(systemd-journal.plugin libnetdata)
@@ -3174,15 +3154,23 @@ if(ENABLE_PLUGIN_GO)
3154 endif()
3155
3156 #
3177 -# Handle OTel Collector plugin
3157 +# otel.plugin
3158 #
3179 -
3159 if(ENABLE_PLUGIN_OTEL)
3181 - include(${CMAKE_SOURCE_DIR}/src/go/otel-collector/CMakeLists.txt)
3160 + corrosion_install(TARGETS otel-plugin
3161 + PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE
3162 + RUNTIME DESTINATION usr/libexec/netdata/plugins.d
3163 + COMPONENT plugin-otel)
3164
3183 - install(PROGRAMS ${CMAKE_BINARY_DIR}/otel-collector/otelcol.plugin
3184 - COMPONENT plugin-otelcol
3185 - DESTINATION usr/libexec/netdata/plugins.d)
3165 + install(FILES src/crates/jf/otel-plugin/configs/otel.yml
3166 + COMPONENT plugin-otel
3167 + DESTINATION usr/lib/netdata/conf.d)
3168 +
3169 + install(FILES src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yml
3170 + COMPONENT plugin-otel
3171 + DESTINATION usr/lib/netdata/conf.d/otel.d/v1/metrics)
3172 +
3173 + install(DIRECTORY COMPONENT plugin-otel DESTINATION etc/netdata/otel.d/v1/metrics)
3174 endif()
3175
3176 #
@@ -3608,6 +3596,7 @@ if(CMAKE_BUILD_TYPE STREQUAL "Debug")
3596 ${CMAKE_BINARY_DIR}/tests/ebpf/ebpf_thread_function.sh
3597 DESTINATION usr/libexec/netdata/plugins.d)
3598 endif()
3599 +
3600 #
3601 # charts.d plugin
3602 #
netdata-installer.sh
+22 -2
@@ -216,6 +216,8 @@ USAGE: ${PROGRAM} [options]
216 --enable-plugin-systemd-journal Enable the systemd journal plugin. Default: enable it when libsystemd is available.
217 --disable-plugin-systemd-journal Explicitly disable the systemd journal plugin.
218 --internal-systemd-journal Enable the internal journal file reader instead of using libsystemd
219 + --enable-plugin-otel Enable the Netdata OpenTelemetry plugin. Default: disabled
220 + --disable-plugin-otel Explicitly disable the Netdata OpenTelemetry plugin.
221 --enable-exporting-kinesis Enable AWS Kinesis exporting connector. Default: enable it when libaws_cpp_sdk_kinesis
222 and its dependencies are available.
223 --disable-exporting-kinesis Explicitly disable AWS Kinesis exporting connector.
@@ -257,6 +259,7 @@ ENABLE_DBENGINE=1
259 ENABLE_GO=1
260 ENABLE_PYTHON=1
261 ENABLE_CHARTS=1
262 +ENABLE_OTEL=0
263 FORCE_LEGACY_CXX=0
264 NETDATA_CMAKE_OPTIONS="${NETDATA_CMAKE_OPTIONS-}"
265 REMOVE_BUILD=1
@@ -296,6 +299,8 @@ while [ -n "${1}" ]; do
299 "--enable-plugin-systemd-journal") ENABLE_SYSTEMD_JOURNAL=1 ;;
300 "--disable-plugin-systemd-journal") ENABLE_SYSTEMD_JOURNAL=0 ;;
301 "--internal-systemd-journal") USE_RUST_JOURNAL_FILE=1 ;;
302 + "--enable-plugin-otel") ENABLE_OTEL=1 ;;
303 + "--disable-plugin-otel") ENABLE_OTEL=0 ;;
304 "--enable-exporting-kinesis" | "--enable-backend-kinesis")
305 # TODO: Needs CMake Support
306 ;;
@@ -796,8 +801,8 @@ if [ "$(id -u)" -eq 0 ]; then
801 run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type d -exec chmod 0755 {} \;
802 run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -exec chmod 0644 {} \;
803 # shellcheck disable=SC2086
799 - run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -a -name \*.plugin -exec chown :${NETDATA_GROUP} {} \;
800 - run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -a -name \*.plugin -exec chmod 0750 {} \;
804 + run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -a -name \*plugin -exec chown :${NETDATA_GROUP} {} \;
805 + run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -a -name \*plugin -exec chmod 0750 {} \;
806 run find "${NETDATA_PREFIX}/usr/libexec/netdata" -type f -a -name \*.sh -exec chmod 0755 {} \;
807
808 if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/apps.plugin" ]; then
@@ -945,6 +950,21 @@ if [ "$(id -u)" -eq 0 ]; then
950 fi
951 fi
952
953 + if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/otel-plugin" ]; then
954 + run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/otel-plugin"
955 + capabilities=0
956 + if ! iscontainer && command -v setcap 1>/dev/null 2>&1; then
957 + run chmod 0750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/otel-plugin"
958 + if run setcap cap_net_bind_service=eip "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/otel-plugin"; then
959 + capabilities=1
960 + fi
961 + fi
962 +
963 + if [ $capabilities -eq 0 ]; then
964 + run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/otel-plugin"
965 + fi
966 + fi
967 +
968 else
969 # non-privileged user installation
970 run chown "${NETDATA_USER}:${NETDATA_GROUP}" "${NETDATA_LOG_DIR}"
netdata.spec.in
+38
@@ -62,6 +62,15 @@ AutoReqProv: yes
62 # Redefine centos_ver to standardize on a single macro
63 %{?rhel:%global centos_ver %rhel}
64
65 +# Only try to build OTEL plugin if we have rust
66 +%global _nd_rust_cmd %(command -v rustc)
67 +
68 +%if "%{_nd_rustc_cmd}" == ""
69 +%global _have_rust 0
70 +%else
71 +%global _have_rust 1
72 +%endif
73 +
74 # Disable FreeIPMI on Amazon Linux 2023 and newer
75 %if 0%{?amzn} >= 2023
76 %global _have_freeipmi 0
@@ -414,6 +423,11 @@ advanced correlations and fast root cause analysis, native horizontal scalabilit
423 %else
424 -DENABLE_PLUGIN_SYSTEMD_UNITS=Off \
425 %endif
426 + %if 0%{_have_rust}
427 + -DENABLE_PLUGIN_OTEL=On \
428 + %else
429 + -DENABLE_PLUGIN_OTEL=Off \
430 + %endif
431 -DENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE=On \
432 -DENABLE_BUNDLED_JSONC=Off \
433 -DENABLE_BUNDLED_YAML=Off \
@@ -1045,7 +1059,31 @@ fi
1059 %defattr(0644,root,root,0755)
1060 %{_datadir}/%{name}/web
1061
1062 +%if %{_have_rust}
1063 +%package plugin-otel
1064 +Summary: The Open Telemetry plugin for the Netdata Agent
1065 +Group: Applications/System
1066 +Requires: %{name} >= %{version}
1067 +Conflicts: %{name} < %{version}
1068 +
1069 +%description plugin-otel
1070 + This plugin allows the Netdata Agent to collect metrics and logs via the
1071 + OpenTelemetry gRPC protocol, providing integration with modern observability
1072 + stacks.
1073 +
1074 +%pre plugin-otel
1075 +if ! getent group %{name} > /dev/null; then
1076 + groupadd --system %{name}
1077 +fi
1078 +
1079 +%files plugin-otel
1080 +%defattr(0750,root,netdata,0750)
1081 +%caps(cap_net_bind_service=eip) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/otel-plugin
1082 +%endif
1083 +
1084 %changelog
1085 +* Tue Jul 29 2025 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-34
1086 +- Add OpenTelemetry plugin
1087 * Tue Jul 15 2025 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-33
1088 - Fix file handling for systemd-journal and systemd-units plugins
1089 * Tue May 27 2025 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-32
packaging/build-package.sh
+1
@@ -39,6 +39,7 @@ add_cmake_option ENABLE_PLUGIN_PYTHON On
39 add_cmake_option ENABLE_PLUGIN_CHARTS On
40 add_cmake_option ENABLE_PLUGIN_LOCAL_LISTENERS On
41 add_cmake_option ENABLE_PLUGIN_NFACCT On
42 +add_cmake_option ENABLE_PLUGIN_OTEL On
43 add_cmake_option ENABLE_PLUGIN_PERF On
44 add_cmake_option ENABLE_PLUGIN_SLABINFO On
45 add_cmake_option ENABLE_PLUGIN_SYSTEMD_JOURNAL On
packaging/cmake/Modules/Packaging.cmake
+25
@@ -329,6 +329,28 @@ set(CPACK_DEBIAN_PLUGIN-NETWORK-VIEWER_PACKAGE_CONTROL_EXTRA
329
330 set(CPACK_DEBIAN_PLUGIN-NETWORK-VIEWER_DEBUGINFO_PACKAGE On)
331
332 +#
333 +# otel.plugin
334 +#
335 +
336 +set(CPACK_COMPONENT_PLUGIN-OTEL_DEPENDS "netdata")
337 +set(CPACK_COMPONENT_PLUGIN-OTEL_DESCRIPTION
338 + "The OpenTelemetry collection plugin for the Netdata Agent
339 + This plugin allows the Netdata Agent to collect metrics and logs via
340 + OpenTelemetry gRPC protocol, providing integration with modern observability
341 + stacks.")
342 +
343 +set(CPACK_DEBIAN_PLUGIN-OTEL_PACKAGE_NAME "netdata-plugin-otel")
344 +set(CPACK_DEBIAN_PLUGIN-OTEL_PACKAGE_SECTION "net")
345 +set(CPACK_DEBIAN_PLUGIN-OTEL_PACKAGE_CONFLICTS "netdata (<< 1.40)")
346 +set(CPACK_DEBIAN_PLUGIN-OTEL_PACKAGE_PREDEPENDS "adduser")
347 +
348 +set(CPACK_DEBIAN_PLUGIN-OTEL_PACKAGE_CONTROL_EXTRA
349 + "${PKG_FILES_PATH}/deb/plugin-otel/preinst;"
350 + "${PKG_FILES_PATH}/deb/plugin-otel/postinst")
351 +
352 +set(CPACK_DEBIAN_PLUGIN-OTEL_DEBUGINFO_PACKAGE Off)
353 +
354 #
355 # nfacct.plugin
356 #
@@ -530,5 +552,8 @@ endif()
552 if(ENABLE_PLUGIN_XENSTAT)
553 list(APPEND CPACK_COMPONENTS_ALL "plugin-xenstat")
554 endif()
555 +if(ENABLE_PLUGIN_OTEL)
556 + list(APPEND CPACK_COMPONENTS_ALL "plugin-otel")
557 +endif()
558
559 include(CPack)
packaging/cmake/pkg-files/deb/plugin-otel/postinst new
+15
@@ -0,0 +1,15 @@
1 +#!/bin/sh
2 +
3 +set -e
4 +
5 +case "$1" in
6 + configure|reconfigure)
7 + chown root:netdata /usr/libexec/netdata/plugins.d/otel-plugin
8 + chmod 0750 /usr/libexec/netdata/plugins.d/otel-plugin
9 + if ! setcap "cap_net_bind_service=eip" /usr/libexec/netdata/plugins.d/otel-plugin; then
10 + chmod -f 4750 /usr/libexec/netdata/plugins.d/otel-plugin
11 + fi
12 + ;;
13 +esac
14 +
15 +exit 0
packaging/cmake/pkg-files/deb/plugin-otel/preinst new
+11
@@ -0,0 +1,11 @@
1 +#!/bin/sh
2 +
3 +set -e
4 +
5 +case "$1" in
6 + install)
7 + if ! getent group netdata > /dev/null; then
8 + addgroup --quiet --system netdata
9 + fi
10 + ;;
11 +esac
packaging/docker/Dockerfile
+2 -1
@@ -28,7 +28,7 @@ RUN chmod +x netdata-installer.sh && \
28 cp -rp /deps/* /usr/local/ && \
29 /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='$(uname -m)'" > ./system/.install-type && \
30 CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it --use-system-protobuf \
31 - ${EXTRA_INSTALL_OPTS} --disable-ebpf --install-no-prefix / "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)"
31 + ${EXTRA_INSTALL_OPTS} --disable-ebpf --enable-plugin-otel --install-no-prefix / "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)"
32
33 # files to one directory
34 RUN mkdir -p /app/usr/sbin/ \
@@ -126,6 +126,7 @@ RUN addgroup --gid ${NETDATA_GID} --system "${DOCKER_GRP}" && \
126 ndsudo \
127 slabinfo.plugin \
128 network-viewer.plugin \
129 + otel-plugin \
130 systemd-journal.plugin; do \
131 [ -f "/usr/libexec/netdata/plugins.d/$name" ] && chmod 4755 "/usr/libexec/netdata/plugins.d/$name"; \
132 done && \
packaging/installer/functions.sh
+1
@@ -377,6 +377,7 @@ prepare_cmake_options() {
377 enable_feature DBENGINE "${ENABLE_DBENGINE:-1}"
378 enable_feature ML "${NETDATA_ENABLE_ML:-1}"
379 enable_feature PLUGIN_APPS "${ENABLE_APPS:-1}"
380 + enable_feature PLUGIN_OTEL "${ENABLE_OTEL:-0}"
381
382 check_for_feature EXPORTER_PROMETHEUS_REMOTE_WRITE "${EXPORTER_PROMETHEUS}" snappy
383 check_for_feature EXPORTER_MONGODB "${EXPORTER_MONGODB}" libmongoc-1.0
packaging/makeself/jobs/70-netdata-git.install.sh
+2 -3
@@ -29,11 +29,11 @@ export NETDATA_BUILD_DIR
29 case "${BUILDARCH}" in
30 armv6l)
31 export NETDATA_CMAKE_OPTIONS="-DENABLE_LIBBACKTRACE=On"
32 - export INSTALLER_ARGS="--disable-plugin-systemd-journal"
32 + export INSTALLER_ARGS="--disable-plugin-systemd-journal --disable-plugin-otel"
33 ;;
34 *)
35 export NETDATA_CMAKE_OPTIONS="-DENABLE_LIBBACKTRACE=On"
36 - export INSTALLER_ARGS="--enable-plugin-systemd-journal --internal-systemd-journal"
36 + export INSTALLER_ARGS="--enable-plugin-systemd-journal --internal-systemd-journal --enable-plugin-otel"
37 ;;
38 esac
39
@@ -44,7 +44,6 @@ run ./netdata-installer.sh \
44 --dont-wait \
45 --dont-start-it \
46 --disable-exporting-mongodb \
47 - --enable-plugin-systemd-journal \
47 --dont-scrub-cflags-even-though-it-may-break-things \
48 --one-time-build \
49 --enable-lto \
packaging/makeself/jobs/81-netdata-runtime-check.sh
+5 -3
@@ -16,9 +16,11 @@ trap dump_log EXIT
16 export NETDATA_LIBEXEC_PREFIX="${NETDATA_INSTALL_PATH}/usr/libexec/netdata"
17 export NETDATA_SKIP_LIBEXEC_PARTS="freeipmi|xenstat|cups"
18
19 -if [ "$(uname -m)" != "x86_64" ]; then
20 - export NETDATA_SKIP_LIBEXEC_PARTS="${NETDATA_SKIP_LIBEXEC_PARTS}|ebpf"
21 -fi
19 +case "${BUILDARCH}" in
20 + x86_64) ;;
21 + armv6l) NETDATA_SKIP_LIBEXEC_PARTS="${NETDATA_SKIP_LIBEXEC_PARTS}|ebpf|otel|systemd-journal" ;;
22 + *) NETDATA_SKIP_LIBEXEC_PARTS="${NETDATA_SKIP_LIBEXEC_PARTS}|ebpf" ;;
23 +esac
24
25 "${NETDATA_INSTALL_PATH}/bin/netdata" -D > ./netdata.log 2>&1 &
26
packaging/runtime-check.sh
+1
@@ -63,6 +63,7 @@ plugins.d/local-listeners
63 plugins.d/ndsudo
64 plugins.d/network-viewer.plugin
65 plugins.d/nfacct.plugin
66 +plugins.d/otel-plugin
67 plugins.d/perf.plugin
68 plugins.d/python.d.plugin
69 plugins.d/slabinfo.plugin
packaging/windows/compile-on-windows.sh
+3
@@ -42,8 +42,11 @@ CFLAGS="${BUILD_CFLAGS}" /usr/bin/cmake \
42 -DENABLE_ML=On \
43 -DENABLE_PLUGIN_GO=On \
44 -DENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE=Off \
45 + -DENABLE_PLUGIN_OTEL=Off \
46 + -DENABLE_PLUGIN_SYSTEMD_JOURNAL=Off \
47 -DENABLE_BUNDLED_JSONC=On \
48 -DENABLE_BUNDLED_PROTOBUF=Off \
49 + -DRust_COMPILER=/ucrt64/bin/rustc \
50 ${EXTRA_CMAKE_OPTIONS:-}
51 ${GITHUB_ACTIONS+echo "::endgroup::"}
52
packaging/windows/msys2-dependencies.sh
+2
@@ -30,7 +30,9 @@ pacman -S --noconfirm --needed \
30 msys/libcurl-devel \
31 openssl-devel \
32 protobuf-devel \
33 + mingw-w64-x86_64-rust \
34 mingw-w64-x86_64-toolchain \
35 + mingw-w64-ucrt-x86_64-rust \
36 mingw-w64-ucrt-x86_64-toolchain \
37 mingw64/mingw-w64-x86_64-brotli \
38 mingw64/mingw-w64-x86_64-go \
src/collectors/systemd-journal.plugin/systemd-journal-files.c
+7
@@ -841,6 +841,13 @@ void nd_journal_init_files_and_directories(void)
841 journal_directories[d++].path = string_strdupz(path);
842 }
843
844 + const char *netdata_configured_log_dir = getenv("NETDATA_LOG_DIR");
845 + if (netdata_configured_log_dir != NULL) {
846 + char path[PATH_MAX];
847 + snprintfz(path, sizeof(path), "%s/otel", netdata_configured_host_prefix);
848 + journal_directories[d++].path = string_strdupz(path);
849 + };
850 +
851 // terminate the list
852 journal_directories[d].path = NULL;
853
src/crates/jf/Cargo.toml
+33 -6
@@ -2,20 +2,20 @@
2 resolver = "2"
3 members = [
4 "error",
5 - # "main",
5 "journal_file",
7 - # "journal_writer",
8 - "journal_reader",
6 "journal_reader_ffi",
10 - # "journal_logger",
11 - # "journal_forwarder",
7 + "journal_log",
8 "window_manager",
9 "sigbus",
10 + "otel-plugin",
11 + "flatten_otel",
12 + "flog-otel",
13 ]
14
15 [workspace.package]
16 version = "0.1.0"
17 edition = "2021"
18 +rust-version = "1.85"
19
20 [workspace.dependencies]
21 memmap2 = "0.9"
@@ -26,11 +26,38 @@ siphasher = "1.0"
26 twox-hash = { version = "2.1", default-features = false, features = ["std"] }
27 static_assertions = "1.1"
28 duct = "0.13"
29 +serde = { version = "1.0", features = ["derive"] }
30 serde_json = "1.0"
31 walkdir = "2"
32 ruzstd = "0.8"
32 -# systemd = { path = "/home/vk/repos/crates/rust-systemd"}
33 libc = "0.2"
34 +hex = "0.4"
35 +tempfile = "3"
36 +indexmap = "2.9"
37 +uuid = { version = "1.0", features = ["v4", "rng"] }
38 +chrono = { version = "0.4", features = ["serde"] }
39 +tracing = "0.1"
40 +tracing-subscriber = "0.3"
41 +
42 +# otel-plugin
43 +prost = "0.13"
44 +regex = "1.10"
45 +tokio = { version = "1.45", features = ["rt-multi-thread", "process", "io-util"] }
46 +tokio-stream = "0.1"
47 +tonic = "0.13"
48 +tonic-build = "0.13"
49 +prost-build = "0.13"
50 +hyper-util = "0.1"
51 +tower = "0.5"
52 +opentelemetry-proto = "0.30"
53 +atty = "0.2"
54 +anyhow = "1.0"
55 +humantime-serde = "1.1"
56 +humantime = "2.2"
57 +bytesize = "1.0"
58 +bytesize-serde = "0.2"
59 +
60 +clap = { version = "4.5", features = ["derive", "env"] }
61
62 [profile.release]
63 lto = true
src/crates/jf/benchmark/main.c deleted
-275
@@ -1,275 +0,0 @@
1 -#include <stdio.h>
2 -#include <stdlib.h>
3 -#include <string.h>
4 -
5 -#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
6 -
7 -void format_entry(NsdJournal *j, size_t entry_id) {
8 - size_t data_count = 0;
9 - char buf[4096];
10 -
11 - const void *data;
12 - size_t length;
13 - NSD_JOURNAL_FOREACH_DATA(j, data, length) {
14 - if (length > 4095) {
15 - fprintf(stderr, "length is more than 4KiB\n");
16 - nsd_journal_close(j);
17 - exit(EXIT_FAILURE);
18 - }
19 -
20 - memcpy(buf, data, length);
21 - buf[length] = '\0';
22 -
23 - fprintf(stdout, "E[%zu] D[%zu] %s\n", entry_id, data_count, buf);
24 - data_count += 1;
25 - }
26 -}
27 -
28 -static int process_unfiltered(const char *path)
29 -{
30 - NsdJournal *j;
31 - size_t total_bytes = 0;
32 -
33 - const char *paths[2] = {
34 - path,
35 - NULL,
36 - };
37 -
38 - // Open the specific journal file
39 - int r = nsd_journal_open_files(&j, &paths[0], 0);
40 - if (r < 0) {
41 - fprintf(stderr, "Failed to open journal file %s: %s\n",
42 - path, strerror(-r));
43 - return 1;
44 - }
45 -
46 - printf("Successfully opened journal file: %s\n", path);
47 -
48 - {
49 - const char *field = NULL;
50 - nsd_journal_restart_fields(j);
51 -
52 - // Enumerate all field names in the journal
53 - while (nsd_journal_enumerate_fields(j, &field) > 0) {
54 - printf("Field name: %s\n", field);
55 - }
56 - }
57 -
58 - // Move to the first entry
59 - r = nsd_journal_seek_head(j);
60 - if (r < 0) {
61 - fprintf(stderr, "Failed to seek to head: %s\n", strerror(-r));
62 - nsd_journal_close(j);
63 - return 1;
64 - }
65 -
66 - // Iterate through all entries
67 - size_t entry_count = 0;
68 - while ((r = nsd_journal_next(j)) > 0) {
69 - entry_count++;
70 -
71 - // Get all data fields
72 - const void *data;
73 - size_t length;
74 - NSD_JOURNAL_FOREACH_DATA(j, data, length) {
75 - // Count the bytes
76 - total_bytes += length;
77 - }
78 - }
79 -
80 - if (r < 0) {
81 - fprintf(stderr, "Failed to iterate journal: %s\n", strerror(-r));
82 - nsd_journal_close(j);
83 - return 1;
84 - }
85 -
86 - if (total_bytes == 10) {
87 - abort();
88 - }
89 -
90 - printf("Total entries processed: %zu\n", entry_count);
91 -
92 - // Close the journal
93 - nsd_journal_close(j);
94 - return 0;
95 -}
96 -
97 -static int process_filtered(const char *path)
98 -{
99 - NsdJournal *j;
100 - size_t total_bytes = 0;
101 -
102 - const char *paths[2] = {
103 - path,
104 - NULL,
105 - };
106 -
107 - // Open the specific journal file
108 - int r = nsd_journal_open_files(&j, &paths[0], 0);
109 - if (r < 0) {
110 - fprintf(stderr, "Failed to open journal file %s: %s\n",
111 - path, strerror(-r));
112 - return 1;
113 - }
114 -
115 - printf("Successfully opened journal file: %s\n", path);
116 -
117 - // Apply filters
118 - // Platform filters (OR condition)
119 - {
120 - r = nsd_journal_add_match(j, "AE_OS_PLATFORM=debian", strlen("AE_OS_PLATFORM=debian"));
121 - if (r < 0) {
122 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
123 - nsd_journal_close(j);
124 - return 1;
125 - }
126 -
127 - r = nsd_journal_add_match(j, "AE_OS_PLATFORM=fedora", strlen("AE_OS_PLATFORM=fedora"));
128 - if (r < 0) {
129 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
130 - nsd_journal_close(j);
131 - return 1;
132 - }
133 - }
134 -
135 - r = nsd_journal_add_conjunction(j); // AND
136 - if (r < 0) {
137 - fprintf(stderr, "Failed to add conjunction: %s\n", strerror(-r));
138 - nsd_journal_close(j);
139 - return 1;
140 - }
141 -
142 - {
143 - // Version filters (OR condition)
144 - r = nsd_journal_add_match(j, "AE_VERSION=17", strlen("AE_VERSION=17"));
145 - if (r < 0) {
146 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
147 - nsd_journal_close(j);
148 - return 1;
149 - }
150 -
151 - r = nsd_journal_add_match(j, "AE_VERSION=22", strlen("AE_VERSION=22"));
152 - if (r < 0) {
153 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
154 - nsd_journal_close(j);
155 - return 1;
156 - }
157 - }
158 -
159 - r = nsd_journal_add_conjunction(j); // AND
160 - if (r < 0) {
161 - fprintf(stderr, "Failed to add conjunction: %s\n", strerror(-r));
162 - nsd_journal_close(j);
163 - return 1;
164 - }
165 -
166 - // Priority filters (OR condition)
167 - {
168 - r = nsd_journal_add_match(j, "PRIORITY=7", strlen("PRIORITY=7"));
169 - if (r < 0) {
170 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
171 - nsd_journal_close(j);
172 - return 1;
173 - }
174 -
175 - r = nsd_journal_add_match(j, "PRIORITY=6", strlen("PRIORITY=6"));
176 - if (r < 0) {
177 - fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
178 - nsd_journal_close(j);
179 - return 1;
180 - }
181 - }
182 -
183 - r = nsd_journal_seek_tail(j);
184 - if (r < 0) {
185 - fprintf(stderr, "Failed to seek to head: %s\n", strerror(-r));
186 - nsd_journal_close(j);
187 - return 1;
188 - }
189 -
190 - size_t entry_count = 0;
191 - while ((r = nsd_journal_previous(j)) > 0) {
192 - format_entry(j, entry_count);
193 -
194 - entry_count++;
195 - }
196 -
197 - if (r < 0) {
198 - fprintf(stderr, "Failed to iterate journal: %s\n", strerror(-r));
199 - nsd_journal_close(j);
200 - return 1;
201 - }
202 -
203 - if (total_bytes == 10) {
204 - abort();
205 - }
206 -
207 - printf("Total entries processed: %zu\n\n", entry_count);
208 -
209 - // Close the journal
210 - nsd_journal_close(j);
211 - return 0;
212 -}
213 -
214 -long get_file_size(const char *filename) {
215 - FILE *file = fopen(filename, "rb");
216 -
217 - if (file == NULL) {
218 - abort();
219 - }
220 -
221 - // Seek to the end of the file
222 - fseek(file, 0, SEEK_END);
223 -
224 - // Get the current position (which is the size)
225 - long size = ftell(file);
226 -
227 - // Close the file
228 - fclose(file);
229 -
230 - return size;
231 -}
232 -
233 -
234 -int main(int argc, char *argv[]) {
235 - (void) argc;
236 - (void) argv;
237 -
238 - if (argc != 2) {
239 - fprintf(stderr, "usage: <binary> filtered|unfiltered\n");
240 - return 1;
241 - }
242 -
243 - printf("Processing entries for files...\n");
244 - const char *paths[] = {
245 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002b725a-0006314cd7a5cefd.journal",
246 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002c7398-00063157ce5e4da0.journal",
247 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002d4dd1-000631616affdc1c.journal",
248 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002e52a2-0006316e49ef1636.journal",
249 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002f0f22-00063175e2452287.journal",
250 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002ffa15-0006318392e11a33.journal",
251 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000030a308-00063189ec4c06b5.journal",
252 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000031a287-0006319ba73abb17.journal",
253 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000032b6a5-000631a6ddadfd47.journal",
254 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000033a684-000631b2794364a9.journal",
255 - "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000034afc5-000631c4c524ff14.journal",
256 - NULL,
257 - };
258 -
259 - size_t total_size = 0;
260 -
261 - for (size_t idx = 0; idx != 10; idx++) {
262 - total_size += get_file_size(paths[idx]);
263 -
264 - if (strcmp(argv[1], "filtered") == 0) {
265 - process_filtered(paths[idx]);
266 - } else if (strcmp(argv[1], "unfiltered") == 0) {
267 - process_unfiltered(paths[idx]);
268 - } else {
269 - fprintf(stderr, "Unknown argument: >>>%s<<<\n", argv[1]);
270 - return 1;
271 - }
272 - }
273 -
274 - fprintf(stdout, "Size of all logs: %zu MiB\n", total_size / (1024 * 1024));
275 -}
src/crates/jf/error/Cargo.toml
+1
@@ -2,6 +2,7 @@
2 name = "error"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 static_assertions = { workspace = true }
src/crates/jf/error/src/lib.rs
+20
@@ -77,6 +77,21 @@ pub enum JournalError {
77
78 #[error("ffi error")]
79 InvalidFfiOp,
80 +
81 + #[error("uuid encoding/decoding")]
82 + UuidSerde,
83 +
84 + #[error("invalid filename")]
85 + InvalidFilename,
86 +
87 + #[error("system time error")]
88 + SystemTimeError,
89 +
90 + #[error("directory not found")]
91 + DirectoryNotFound,
92 +
93 + #[error("not a directory")]
94 + NotADirectory,
95 }
96
97 const_assert!(std::mem::size_of::<JournalError>() <= 16);
@@ -108,6 +123,11 @@ impl JournalError {
123 JournalError::SigbusHandlerError => -22,
124 JournalError::UnknownCompressionMethod => -23,
125 JournalError::InvalidFfiOp => -24,
126 + JournalError::UuidSerde => -25,
127 + JournalError::InvalidFilename => -26,
128 + JournalError::SystemTimeError => -27,
129 + JournalError::DirectoryNotFound => -28,
130 + JournalError::NotADirectory => -29,
131 }
132 }
133 }
src/crates/jf/flatten_otel/Cargo.toml new
+10
@@ -0,0 +1,10 @@
1 +[package]
2 +name = "flatten_otel"
3 +version.workspace = true
4 +edition.workspace = true
5 +rust-version.workspace = true
6 +
7 +[dependencies]
8 +opentelemetry-proto = { workspace = true, features = ["logs", "metrics", "with-serde"] }
9 +flatten-serde-json = { git = "https://github.com/meilisearch/meilisearch" }
10 +serde_json = { workspace = true }
src/crates/jf/flatten_otel/src/lib.rs new
+86
@@ -0,0 +1,86 @@
1 +use serde_json::{Map as JsonMap, Value as JsonValue};
2 +
3 +use opentelemetry_proto::tonic::{
4 + common::v1::{AnyValue, InstrumentationScope, KeyValue},
5 + resource::v1::Resource,
6 +};
7 +
8 +mod logs;
9 +mod metrics;
10 +
11 +pub use logs::{json_from_export_logs_service_request, json_from_log_record};
12 +pub use metrics::flatten_metrics_request;
13 +
14 +fn json_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValue> {
15 + let mut map = JsonMap::new();
16 +
17 + for kv in kvl {
18 + if let Some(any_value) = &kv.value {
19 + map.insert(kv.key.clone(), json_from_any_value(any_value));
20 + } else {
21 + map.insert(kv.key.clone(), JsonValue::Null);
22 + }
23 + }
24 +
25 + flatten_serde_json::flatten(&map)
26 +}
27 +
28 +fn json_from_any_value(any_value: &AnyValue) -> JsonValue {
29 + use opentelemetry_proto::tonic::common::v1::any_value::Value;
30 +
31 + match &any_value.value {
32 + Some(Value::StringValue(s)) => JsonValue::String(s.clone()),
33 + Some(Value::BoolValue(b)) => JsonValue::Bool(*b),
34 + Some(Value::IntValue(i)) => JsonValue::Number(
35 + serde_json::Number::from_f64(*i as f64).unwrap_or_else(|| serde_json::Number::from(0)),
36 + ),
37 + Some(Value::DoubleValue(d)) => JsonValue::Number(
38 + serde_json::Number::from_f64(*d).unwrap_or_else(|| serde_json::Number::from(0)),
39 + ),
40 + Some(Value::ArrayValue(array)) => {
41 + let values: Vec<JsonValue> = array.values.iter().map(json_from_any_value).collect();
42 + JsonValue::Array(values)
43 + }
44 + Some(Value::KvlistValue(kvl)) => JsonValue::Object(json_from_key_value_list(&kvl.values)),
45 + Some(Value::BytesValue(_bytes)) => {
46 + todo!("Add support for byte values");
47 + }
48 + None => JsonValue::Null,
49 + }
50 +}
51 +
52 +fn json_from_resource(jm: &mut JsonMap<String, JsonValue>, resource: &Resource) {
53 + if !resource.attributes.is_empty() {
54 + let resource_attrs = json_from_key_value_list(&resource.attributes);
55 + for (key, value) in resource_attrs {
56 + jm.insert(format!("resource.attributes.{}", key), value);
57 + }
58 + }
59 +}
60 +
61 +fn json_from_instrumentation_scope(
62 + jm: &mut JsonMap<String, JsonValue>,
63 + scope: &InstrumentationScope,
64 +) {
65 + if !scope.name.is_empty() {
66 + jm.insert(
67 + "scope.name".to_string(),
68 + JsonValue::String(scope.name.clone()),
69 + );
70 + }
71 +
72 + if !scope.version.is_empty() {
73 + jm.insert(
74 + "scope.version".to_string(),
75 + JsonValue::String(scope.version.clone()),
76 + );
77 + }
78 +
79 + if !scope.attributes.is_empty() {
80 + let scope_attrs = json_from_key_value_list(&scope.attributes);
81 +
82 + for (key, value) in scope_attrs {
83 + jm.insert(format!("scope.attributes.{}", key), value);
84 + }
85 + }
86 +}
src/crates/jf/flatten_otel/src/logs.rs new
+126
@@ -0,0 +1,126 @@
1 +use serde_json::{Map as JsonMap, Value as JsonValue};
2 +
3 +use opentelemetry_proto::tonic::{
4 + collector::logs::v1::ExportLogsServiceRequest, logs::v1::LogRecord,
5 +};
6 +
7 +use crate::{
8 + json_from_any_value, json_from_instrumentation_scope, json_from_key_value_list,
9 + json_from_resource,
10 +};
11 +
12 +pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &LogRecord) {
13 + // Add log record fields with "log." prefix
14 + jm.insert(
15 + "log.time_unix_nano".to_string(),
16 + JsonValue::Number(serde_json::Number::from(log_record.time_unix_nano)),
17 + );
18 + jm.insert(
19 + "log.observed_time_unix_nano".to_string(),
20 + JsonValue::Number(serde_json::Number::from(log_record.observed_time_unix_nano)),
21 + );
22 + jm.insert(
23 + "log.severity_number".to_string(),
24 + JsonValue::Number(serde_json::Number::from(log_record.severity_number)),
25 + );
26 + jm.insert(
27 + "log.severity_text".to_string(),
28 + JsonValue::String(log_record.severity_text.clone()),
29 + );
30 +
31 + // Add body if present
32 + if let Some(body) = &log_record.body {
33 + let mut temp_map = JsonMap::new();
34 + temp_map.insert("body".to_string(), json_from_any_value(body));
35 +
36 + let flattened_body = flatten_serde_json::flatten(&temp_map);
37 + for (key, value) in flattened_body {
38 + if key != "body" {
39 + jm.insert(format!("log.{}", key), value);
40 + }
41 + }
42 + }
43 +
44 + // Add event name
45 + jm.insert(
46 + "log.event_name".to_string(),
47 + JsonValue::String(log_record.event_name.clone()),
48 + );
49 +
50 + // Add log record attributes
51 + let log_attrs = json_from_key_value_list(&log_record.attributes);
52 + for (key, value) in log_attrs {
53 + jm.insert(format!("log.attributes.{}", key), value);
54 + }
55 +
56 + jm.insert(
57 + "log.dropped_attributes_count".to_string(),
58 + JsonValue::Number(serde_json::Number::from(
59 + log_record.dropped_attributes_count,
60 + )),
61 + );
62 + jm.insert(
63 + "log.flags".to_string(),
64 + JsonValue::Number(serde_json::Number::from(log_record.flags)),
65 + );
66 +
67 + // Add trace_id and span_id as hex strings
68 + // if !log_record.trace_id.is_empty() {
69 + // jm.insert(
70 + // "log.trace_id".to_string(),
71 + // JsonValue::String(hex::encode(&log_record.trace_id)),
72 + // );
73 + // }
74 + // if !log_record.span_id.is_empty() {
75 + // jm.insert(
76 + // "log.span_id".to_string(),
77 + // JsonValue::String(hex::encode(&log_record.span_id)),
78 + // );
79 + // }
80 +}
81 +
82 +// TODO: this does not belong here, it should be part of the service.
83 +pub fn json_from_export_logs_service_request(request: &ExportLogsServiceRequest) -> JsonValue {
84 + let mut items = Vec::new();
85 +
86 + for resource_logs in &request.resource_logs {
87 + for scope_logs in &resource_logs.scope_logs {
88 + for log_record in &scope_logs.log_records {
89 + let mut jm = JsonMap::new();
90 +
91 + // Add resource information
92 + if let Some(resource) = resource_logs.resource.as_ref() {
93 + json_from_resource(&mut jm, resource);
94 + }
95 +
96 + // Add resource schema URL
97 + if !resource_logs.schema_url.is_empty() {
98 + jm.insert(
99 + "resource.schema_url".to_string(),
100 + JsonValue::String(resource_logs.schema_url.clone()),
101 + );
102 + }
103 +
104 + // Add scope information
105 + if let Some(scope) = scope_logs.scope.as_ref() {
106 + json_from_instrumentation_scope(&mut jm, scope);
107 + }
108 +
109 + // Add scope schema URL
110 + if !scope_logs.schema_url.is_empty() {
111 + jm.insert(
112 + "scope.schema_url".to_string(),
113 + JsonValue::String(scope_logs.schema_url.clone()),
114 + );
115 + }
116 +
117 + // Add log record information
118 + json_from_log_record(&mut jm, log_record);
119 +
120 + items.push(JsonValue::Object(jm));
121 + }
122 + }
123 + }
124 +
125 + JsonValue::Array(items)
126 +}
src/crates/jf/flatten_otel/src/metrics.rs new
+284
@@ -0,0 +1,284 @@
1 +use serde_json::{json, Map as JsonMap, Value as JsonValue};
2 +
3 +use opentelemetry_proto::tonic::{
4 + collector::metrics::v1::ExportMetricsServiceRequest,
5 + metrics::v1::{
6 + metric::Data, AggregationTemporality, Gauge, Histogram, HistogramDataPoint, Metric,
7 + NumberDataPoint, ResourceMetrics, ScopeMetrics, Sum,
8 + },
9 +};
10 +
11 +use crate::{json_from_instrumentation_scope, json_from_key_value_list, json_from_resource};
12 +
13 +pub fn flatten_metrics_request(
14 + req: &ExportMetricsServiceRequest,
15 +) -> Vec<JsonMap<String, JsonValue>> {
16 + req.resource_metrics
17 + .iter()
18 + .flat_map(flatten_resource_metrics)
19 + .collect()
20 +}
21 +
22 +fn flatten_resource_metrics(resource_metrics: &ResourceMetrics) -> Vec<JsonMap<String, JsonValue>> {
23 + resource_metrics
24 + .scope_metrics
25 + .iter()
26 + .flat_map(|scope_metrics| {
27 + let mut flattened_metrics = flatten_scope_metrics(scope_metrics);
28 +
29 + if let Some(resource) = &resource_metrics.resource {
30 + flattened_metrics
31 + .iter_mut()
32 + .for_each(|jm| json_from_resource(jm, resource));
33 + }
34 +
35 + flattened_metrics
36 + })
37 + .collect()
38 +}
39 +
40 +fn flatten_scope_metrics(scope_metrics: &ScopeMetrics) -> Vec<JsonMap<String, JsonValue>> {
41 + scope_metrics
42 + .metrics
43 + .iter()
44 + .flat_map(|metric| {
45 + let mut flattened_metrics = flatten_metric(metric);
46 +
47 + if let Some(scope) = &scope_metrics.scope {
48 + flattened_metrics
49 + .iter_mut()
50 + .for_each(|jm| json_from_instrumentation_scope(jm, scope));
51 + }
52 +
53 + flattened_metrics
54 + })
55 + .collect()
56 +}
57 +
58 +fn flatten_metric(metric: &Metric) -> Vec<JsonMap<String, JsonValue>> {
59 + let Some(data) = metric.data.as_ref() else {
60 + return Vec::new();
61 + };
62 +
63 + let mut flattened_metrics = match data {
64 + Data::Gauge(gauge) => flatten_gauge(gauge),
65 + Data::Sum(sum) => flatten_sum(sum),
66 + Data::Histogram(histogram) => flatten_histogram(histogram),
67 + Data::ExponentialHistogram(_) | Data::Summary(_) => {
68 + eprintln!(
69 + "Summary and exponential histogram metrics are not supported yet ('{}')",
70 + metric.name
71 + );
72 + return Vec::new();
73 + }
74 + };
75 +
76 + for jm in flattened_metrics.iter_mut() {
77 + // Add metric metadata
78 + jm.insert(
79 + "metric.name".to_string(),
80 + JsonValue::String(metric.name.clone()),
81 + );
82 + jm.insert(
83 + "metric.description".to_string(),
84 + JsonValue::String(metric.description.clone()),
85 + );
86 + jm.insert(
87 + "metric.unit".to_string(),
88 + JsonValue::String(metric.unit.clone()),
89 + );
90 +
91 + for (key, value) in json_from_key_value_list(&metric.metadata) {
92 + jm.insert(format!("metric.metadata.{}", key), value);
93 + }
94 + }
95 +
96 + flattened_metrics
97 +}
98 +
99 +fn flatten_gauge(gauge: &Gauge) -> Vec<JsonMap<String, JsonValue>> {
100 + let mut flattened_metrics = Vec::new();
101 +
102 + for data_point in &gauge.data_points {
103 + let mut jm = flatten_number_data_point(data_point);
104 +
105 + if jm.is_empty() {
106 + continue;
107 + }
108 +
109 + jm.insert(
110 + "metric.type".to_string(),
111 + JsonValue::String("gauge".to_string()),
112 + );
113 +
114 + flattened_metrics.push(jm);
115 + }
116 +
117 + flattened_metrics
118 +}
119 +
120 +fn flatten_sum(sum: &Sum) -> Vec<JsonMap<String, JsonValue>> {
121 + let mut flattened_metrics = Vec::new();
122 +
123 + let aggregation_temporality = match sum.aggregation_temporality {
124 + x if x == AggregationTemporality::Unspecified as i32 => "unspecified",
125 + x if x == AggregationTemporality::Delta as i32 => "delta",
126 + x if x == AggregationTemporality::Cumulative as i32 => "cumulative",
127 + _ => "unknown",
128 + };
129 +
130 + for data_point in &sum.data_points {
131 + let mut jm = flatten_number_data_point(data_point);
132 +
133 + if jm.is_empty() {
134 + continue;
135 + }
136 +
137 + jm.insert(
138 + "metric.type".to_string(),
139 + JsonValue::String("sum".to_string()),
140 + );
141 + jm.insert(
142 + "metric.aggregation_temporality".to_string(),
143 + JsonValue::String(aggregation_temporality.to_string()),
144 + );
145 + jm.insert(
146 + "metric.is_monotonic".to_string(),
147 + JsonValue::Bool(sum.is_monotonic),
148 + );
149 +
150 + flattened_metrics.push(jm);
151 + }
152 +
153 + flattened_metrics
154 +}
155 +
156 +fn flatten_histogram(histogram: &Histogram) -> Vec<JsonMap<String, JsonValue>> {
157 + let mut flattened_metrics = Vec::new();
158 +
159 + let aggregation_temporality = match histogram.aggregation_temporality {
160 + x if x == AggregationTemporality::Unspecified as i32 => "unspecified",
161 + x if x == AggregationTemporality::Delta as i32 => "delta",
162 + x if x == AggregationTemporality::Cumulative as i32 => "cumulative",
163 + _ => "unknown",
164 + };
165 +
166 + for data_point in &histogram.data_points {
167 + let mut bucket_maps = flatten_histogram_data_point(data_point);
168 +
169 + for jm in bucket_maps.iter_mut() {
170 + jm.insert(
171 + "metric.type".to_string(),
172 + JsonValue::String("histogram".to_string()),
173 + );
174 + jm.insert(
175 + "metric.aggregation_temporality".to_string(),
176 + JsonValue::String(aggregation_temporality.to_string()),
177 + );
178 + }
179 +
180 + flattened_metrics.extend(bucket_maps);
181 + }
182 +
183 + flattened_metrics
184 +}
185 +
186 +fn flatten_number_data_point(ndp: &NumberDataPoint) -> JsonMap<String, JsonValue> {
187 + let mut jm = JsonMap::new();
188 +
189 + let Some(value) = &ndp.value else {
190 + return jm;
191 + };
192 +
193 + match value {
194 + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsDouble(d) => {
195 + jm.insert("metric.value".to_string(), json!(d));
196 + }
197 + opentelemetry_proto::tonic::metrics::v1::number_data_point::Value::AsInt(i) => {
198 + jm.insert("metric.value".to_string(), JsonValue::Number((*i).into()));
199 + }
200 + };
201 +
202 + jm.insert(
203 + "metric.start_time_unix_nano".to_string(),
204 + JsonValue::Number(ndp.start_time_unix_nano.into()),
205 + );
206 + jm.insert(
207 + "metric.time_unix_nano".to_string(),
208 + JsonValue::Number(ndp.time_unix_nano.into()),
209 + );
210 +
211 + for (key, value) in json_from_key_value_list(&ndp.attributes) {
212 + jm.insert(format!("metric.attributes.{}", key), value);
213 + }
214 +
215 + if !ndp.exemplars.is_empty() {
216 + // todo!...
217 + }
218 +
219 + if ndp.flags != 0 {
220 + jm.insert(
221 + "metric.flags".to_string(),
222 + JsonValue::Number(ndp.flags.into()),
223 + );
224 + }
225 +
226 + jm
227 +}
228 +
229 +fn flatten_histogram_data_point(hdp: &HistogramDataPoint) -> Vec<JsonMap<String, JsonValue>> {
230 + let mut results = Vec::new();
231 +
232 + if hdp.bucket_counts.is_empty() || hdp.explicit_bounds.is_empty() {
233 + return results;
234 + }
235 +
236 + // Create base map with common fields
237 + let mut base_map = JsonMap::new();
238 + base_map.insert(
239 + "metric.start_time_unix_nano".to_string(),
240 + JsonValue::Number(hdp.start_time_unix_nano.into()),
241 + );
242 + base_map.insert(
243 + "metric.time_unix_nano".to_string(),
244 + JsonValue::Number(hdp.time_unix_nano.into()),
245 + );
246 +
247 + // Add attributes
248 + for (key, value) in json_from_key_value_list(&hdp.attributes) {
249 + base_map.insert(format!("metric.attributes.{}", key), value);
250 + }
251 +
252 + // Handle regular buckets
253 + for (&bound, &count) in hdp.explicit_bounds.iter().zip(hdp.bucket_counts.iter()) {
254 + let mut bucket_map = base_map.clone();
255 +
256 + // Set dimension name to bucket identifier
257 + let bucket_name = format!("{}", bound);
258 + bucket_map.insert(
259 + "metric.attributes._nd_dimension".to_string(),
260 + JsonValue::String("bucket".to_string()),
261 + );
262 + bucket_map.insert("bucket".to_string(), JsonValue::String(bucket_name.clone()));
263 + bucket_map.insert("metric.value".to_string(), JsonValue::from(count));
264 +
265 + results.push(bucket_map);
266 + }
267 +
268 + // Handle +Inf bucket if it exists
269 + if hdp.bucket_counts.len() > hdp.explicit_bounds.len() {
270 + let mut inf_map = base_map.clone();
271 + let inf_count = hdp.bucket_counts[hdp.bucket_counts.len() - 1];
272 +
273 + inf_map.insert(
274 + "metric.attributes._nd_dimension".to_string(),
275 + JsonValue::String("bucket".to_string()),
276 + );
277 + inf_map.insert("bucket".to_string(), JsonValue::String("+Inf".to_string()));
278 + inf_map.insert("metric.value".to_string(), JsonValue::from(inf_count));
279 +
280 + results.push(inf_map);
281 + }
282 +
283 + results
284 +}
src/crates/jf/flog-otel/Cargo.toml new
+22
@@ -0,0 +1,22 @@
1 +[package]
2 +name = "flog-otel"
3 +version.workspace = true
4 +edition.workspace = true
5 +rust-version.workspace = true
6 +
7 +[dependencies]
8 +tokio.workspace = true
9 +serde.workspace = true
10 +serde_json.workspace = true
11 +chrono.workspace = true
12 +clap.workspace = true
13 +uuid.workspace = true
14 +anyhow.workspace = true
15 +tracing.workspace = true
16 +tracing-subscriber.workspace = true
17 +tonic.workspace = true
18 +prost.workspace = true
19 +opentelemetry-proto.workspace = true
20 +
21 +[build-dependencies]
22 +tonic-build.workspace = true
src/crates/jf/flog-otel/src/main.rs new
+312
@@ -0,0 +1,312 @@
1 +use anyhow::Result;
2 +use chrono::{DateTime, Utc};
3 +use clap::Parser;
4 +use opentelemetry_proto::tonic::collector::logs::v1::{
5 + logs_service_client::LogsServiceClient, ExportLogsServiceRequest,
6 +};
7 +use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, KeyValue};
8 +use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs};
9 +use opentelemetry_proto::tonic::resource::v1::Resource;
10 +use serde::Deserialize;
11 +use std::process::Stdio;
12 +use std::sync::Arc;
13 +use tokio::io::{AsyncBufReadExt, BufReader};
14 +use tokio::sync::Mutex;
15 +use tokio::time::{Duration, Instant};
16 +use tracing::{error, info, warn};
17 +use tracing_subscriber;
18 +use uuid::Uuid;
19 +
20 +#[derive(Parser, Debug)]
21 +#[command(author, version, about, long_about = None)]
22 +struct Args {
23 + #[arg(short, long, default_value = "1048576")]
24 + rate_limit_bytes: u64,
25 +
26 + #[arg(short, long, default_value = "http://127.0.0.1:4317")]
27 + otel_endpoint: String,
28 +
29 + #[arg(short, long, default_value = "1000")]
30 + log_count: u32,
31 +
32 + #[arg(long, default_value = "false")]
33 + loop_forever: bool,
34 +}
35 +
36 +#[derive(Deserialize, Debug)]
37 +struct FlogEntry {
38 + host: String,
39 + #[serde(rename = "user-identifier")]
40 + user_identifier: String,
41 + datetime: String,
42 + method: String,
43 + request: String,
44 + protocol: String,
45 + status: u16,
46 + bytes: u64,
47 + referer: String,
48 +}
49 +
50 +struct RateLimiter {
51 + bytes_sent: u64,
52 + last_reset: Instant,
53 + limit: u64,
54 +}
55 +
56 +impl RateLimiter {
57 + fn new(limit: u64) -> Self {
58 + Self {
59 + bytes_sent: 0,
60 + last_reset: Instant::now(),
61 + limit,
62 + }
63 + }
64 +
65 + async fn can_send(&mut self, size: u64) -> bool {
66 + if self.last_reset.elapsed() >= Duration::from_secs(1) {
67 + self.bytes_sent = 0;
68 + self.last_reset = Instant::now();
69 + }
70 +
71 + if self.bytes_sent + size <= self.limit {
72 + self.bytes_sent += size;
73 + true
74 + } else {
75 + false
76 + }
77 + }
78 +}
79 +
80 +fn parse_flog_datetime(datetime_str: &str) -> Result<DateTime<Utc>> {
81 + let dt = chrono::DateTime::parse_from_str(datetime_str, "%d/%b/%Y:%H:%M:%S %z")?;
82 + Ok(dt.with_timezone(&Utc))
83 +}
84 +
85 +fn flog_to_otel(entry: FlogEntry) -> Result<LogRecord> {
86 + let dt = parse_flog_datetime(&entry.datetime)?;
87 + let time_unix_nano = dt.timestamp_nanos_opt().unwrap_or(0) as u64;
88 +
89 + let severity_number = match entry.status {
90 + 200..=299 => 9, // INFO
91 + 300..=399 => 5, // DEBUG
92 + 400..=499 => 13, // WARN
93 + 500..=599 => 17, // ERROR
94 + _ => 9, // INFO
95 + };
96 +
97 + let log_message = format!(
98 + "{} {} {} {} {} {}",
99 + entry.host, entry.method, entry.request, entry.protocol, entry.status, entry.bytes
100 + );
101 +
102 + let attributes = vec![
103 + KeyValue {
104 + key: "host".to_string(),
105 + value: Some(AnyValue {
106 + value: Some(any_value::Value::StringValue(entry.host)),
107 + }),
108 + },
109 + KeyValue {
110 + key: "method".to_string(),
111 + value: Some(AnyValue {
112 + value: Some(any_value::Value::StringValue(entry.method)),
113 + }),
114 + },
115 + KeyValue {
116 + key: "request".to_string(),
117 + value: Some(AnyValue {
118 + value: Some(any_value::Value::StringValue(entry.request)),
119 + }),
120 + },
121 + KeyValue {
122 + key: "protocol".to_string(),
123 + value: Some(AnyValue {
124 + value: Some(any_value::Value::StringValue(entry.protocol)),
125 + }),
126 + },
127 + KeyValue {
128 + key: "status".to_string(),
129 + value: Some(AnyValue {
130 + value: Some(any_value::Value::IntValue(entry.status as i64)),
131 + }),
132 + },
133 + KeyValue {
134 + key: "response_bytes".to_string(),
135 + value: Some(AnyValue {
136 + value: Some(any_value::Value::IntValue(entry.bytes as i64)),
137 + }),
138 + },
139 + KeyValue {
140 + key: "referer".to_string(),
141 + value: Some(AnyValue {
142 + value: Some(any_value::Value::StringValue(entry.referer)),
143 + }),
144 + },
145 + KeyValue {
146 + key: "user_identifier".to_string(),
147 + value: Some(AnyValue {
148 + value: Some(any_value::Value::StringValue(entry.user_identifier)),
149 + }),
150 + },
151 + ];
152 +
153 + let trace_id_bytes = Uuid::new_v4().as_u128().to_be_bytes().to_vec();
154 + let span_id_bytes = (Uuid::new_v4().as_u128() as u64).to_be_bytes().to_vec();
155 +
156 + Ok(LogRecord {
157 + time_unix_nano,
158 + severity_number: severity_number as i32,
159 + severity_text: String::new(),
160 + body: Some(AnyValue {
161 + value: Some(any_value::Value::StringValue(log_message)),
162 + }),
163 + attributes,
164 + dropped_attributes_count: 0,
165 + flags: 0,
166 + trace_id: trace_id_bytes,
167 + span_id: span_id_bytes,
168 + ..Default::default()
169 + })
170 +}
171 +
172 +async fn send_otel_logs(logs: Vec<LogRecord>, endpoint: &str) -> Result<()> {
173 + let mut client = LogsServiceClient::connect(endpoint.to_string()).await?;
174 +
175 + let resource_logs = vec![ResourceLogs {
176 + resource: Some(Resource {
177 + attributes: vec![
178 + KeyValue {
179 + key: "service.name".to_string(),
180 + value: Some(AnyValue {
181 + value: Some(any_value::Value::StringValue(
182 + "flog-otel-wrapper".to_string(),
183 + )),
184 + }),
185 + },
186 + KeyValue {
187 + key: "service.version".to_string(),
188 + value: Some(AnyValue {
189 + value: Some(any_value::Value::StringValue("0.1.0".to_string())),
190 + }),
191 + },
192 + ],
193 + dropped_attributes_count: 0,
194 + entity_refs: vec![],
195 + }),
196 + scope_logs: vec![ScopeLogs {
197 + scope: Some(
198 + opentelemetry_proto::tonic::common::v1::InstrumentationScope {
199 + name: "flog-otel-wrapper".to_string(),
200 + version: "0.1.0".to_string(),
201 + attributes: vec![],
202 + dropped_attributes_count: 0,
203 + },
204 + ),
205 + log_records: logs,
206 + schema_url: String::new(),
207 + }],
208 + schema_url: String::new(),
209 + }];
210 +
211 + let request = tonic::Request::new(ExportLogsServiceRequest { resource_logs });
212 +
213 + let response = client.export(request).await?;
214 +
215 + if response.get_ref().partial_success.is_some() {
216 + let partial_success = response.get_ref().partial_success.as_ref().unwrap();
217 + if partial_success.rejected_log_records > 0 {
218 + warn!("Some logs were rejected: {}", partial_success.error_message);
219 + }
220 + }
221 +
222 + info!("Successfully sent logs to OTEL collector via gRPC");
223 + Ok(())
224 +}
225 +
226 +#[tokio::main]
227 +async fn main() -> Result<()> {
228 + tracing_subscriber::fmt::init();
229 +
230 + let args = Args::parse();
231 + let rate_limiter = Arc::new(Mutex::new(RateLimiter::new(args.rate_limit_bytes)));
232 +
233 + info!(
234 + "Starting flog-otel-wrapper with rate limit: {} bytes/sec",
235 + args.rate_limit_bytes
236 + );
237 + info!("OTEL endpoint: {}", args.otel_endpoint);
238 +
239 + loop {
240 + let log_count_str = args.log_count.to_string();
241 + let mut flog_args = vec!["-f", "json", "-t", "stdout"];
242 +
243 + if args.loop_forever {
244 + flog_args.extend(&["-l"]);
245 + } else {
246 + flog_args.extend(&["-n", &log_count_str]);
247 + }
248 +
249 + let mut child = tokio::process::Command::new("flog")
250 + .args(&flog_args)
251 + .stdout(Stdio::piped())
252 + .spawn()?;
253 +
254 + let stdout = child.stdout.take().unwrap();
255 + let reader = BufReader::new(stdout);
256 + let mut lines = reader.lines();
257 +
258 + let mut log_batch = Vec::new();
259 + const BATCH_SIZE: usize = 100;
260 +
261 + while let Some(line) = lines.next_line().await? {
262 + match serde_json::from_str::<FlogEntry>(&line) {
263 + Ok(entry) => match flog_to_otel(entry) {
264 + Ok(otel_log) => {
265 + log_batch.push(otel_log);
266 +
267 + if log_batch.len() >= BATCH_SIZE {
268 + let batch_size = (log_batch.len() * 1024) as u64;
269 +
270 + let mut limiter = rate_limiter.lock().await;
271 + if limiter.can_send(batch_size).await {
272 + drop(limiter);
273 +
274 + if let Err(e) =
275 + send_otel_logs(log_batch.clone(), &args.otel_endpoint).await
276 + {
277 + error!("Failed to send OTEL logs: {}", e);
278 + }
279 + log_batch.clear();
280 + } else {
281 + drop(limiter);
282 + info!("Rate limit reached, waiting 1 second...");
283 + tokio::time::sleep(Duration::from_secs(1)).await;
284 + }
285 + }
286 + }
287 + Err(e) => error!("Failed to convert to OTEL format: {}", e),
288 + },
289 + Err(e) => error!("Failed to parse JSON: {}", e),
290 + }
291 + }
292 +
293 + if !log_batch.is_empty() {
294 + let batch_size = (log_batch.len() * 1024) as u64;
295 + let mut limiter = rate_limiter.lock().await;
296 + if limiter.can_send(batch_size).await {
297 + drop(limiter);
298 + if let Err(e) = send_otel_logs(log_batch, &args.otel_endpoint).await {
299 + error!("Failed to send final OTEL logs: {}", e);
300 + }
301 + }
302 + }
303 +
304 + child.wait().await?;
305 +
306 + if !args.loop_forever {
307 + break;
308 + }
309 + }
310 +
311 + Ok(())
312 +}
src/crates/jf/journal_file/Cargo.toml
+6
@@ -2,6 +2,7 @@
2 name = "journal_file"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 error = { path = "../error" }
@@ -11,3 +12,8 @@ ruzstd = { workspace = true }
12 siphasher = { workspace = true }
13 twox-hash = { workspace = true }
14 zerocopy = { workspace = true }
15 +hex = { workspace = true }
16 +rand = { workspace = true }
17 +
18 +[dev-dependencies]
19 +tempfile = { workspace = true }
src/crates/jf/journal_file/src/cursor.rs new
+237
@@ -0,0 +1,237 @@
1 +use crate::{file::JournalFile, filter::FilterExpr, offset_array, offset_array::Direction};
2 +use error::{JournalError, Result};
3 +use std::num::NonZeroU64;
4 +use window_manager::MemoryMap;
5 +
6 +#[derive(Debug, Copy, Clone, PartialEq, Eq)]
7 +pub enum Location {
8 + Head,
9 + Tail,
10 + Realtime(u64),
11 + Monotonic(u64, [u8; 16]),
12 + Seqnum(u64, Option<[u8; 16]>),
13 + XorHash(u64),
14 + ResolvedEntry(NonZeroU64),
15 +}
16 +
17 +impl Default for Location {
18 + fn default() -> Self {
19 + Self::Head
20 + }
21 +}
22 +
23 +#[derive(Debug)]
24 +pub struct JournalCursor {
25 + pub location: Location,
26 + pub filter_expr: Option<FilterExpr>,
27 + pub array_cursor: Option<offset_array::Cursor>,
28 +}
29 +
30 +impl JournalCursor {
31 + #[allow(clippy::new_without_default)]
32 + pub fn new() -> Self {
33 + Self {
34 + location: Location::Head,
35 + filter_expr: None,
36 + array_cursor: None,
37 + }
38 + }
39 +
40 + pub fn set_location(&mut self, location: Location) {
41 + self.location = location;
42 + self.array_cursor = None;
43 + }
44 +
45 + pub fn set_filter(&mut self, filter_expr: FilterExpr) {
46 + self.filter_expr = Some(filter_expr);
47 + // FIXME: should we set cursor to None?
48 + }
49 +
50 + pub fn clear_filter(&mut self) {
51 + self.filter_expr = None;
52 + self.array_cursor = None;
53 + self.set_location(Location::Head);
54 + }
55 +
56 + pub fn step<M: MemoryMap>(
57 + &mut self,
58 + journal_file: &JournalFile<M>,
59 + direction: Direction,
60 + ) -> Result<bool> {
61 + let new_location = if self.filter_expr.is_some() {
62 + self.resolve_filter_location(journal_file, direction)?
63 + } else {
64 + self.resolve_array_cursor(journal_file, direction)?
65 + };
66 +
67 + if let Some(location) = new_location {
68 + self.location = location;
69 + Ok(true)
70 + } else {
71 + Ok(false)
72 + }
73 + }
74 +
75 + pub fn position(&self) -> Result<NonZeroU64> {
76 + match self.location {
77 + Location::ResolvedEntry(entry_offset) => Ok(entry_offset),
78 + _ => Err(JournalError::UnsetCursor),
79 + }
80 + }
81 +
82 + fn resolve_array_cursor<M: MemoryMap>(
83 + &mut self,
84 + journal_file: &JournalFile<M>,
85 + direction: Direction,
86 + ) -> Result<Option<Location>> {
87 + let new_location = match (self.location, direction) {
88 + (Location::Head, Direction::Forward) => {
89 + let entry_list = journal_file
90 + .entry_list()
91 + .ok_or(JournalError::InvalidOffsetArrayOffset)?;
92 +
93 + let cursor = entry_list.cursor_head();
94 + if let Some(offset) = cursor.value(journal_file)? {
95 + self.array_cursor = Some(cursor);
96 + Some(Location::ResolvedEntry(offset))
97 + } else {
98 + None
99 + }
100 + }
101 + (Location::Head, Direction::Backward) => None,
102 + (Location::Tail, Direction::Forward) => None,
103 + (Location::Tail, Direction::Backward) => {
104 + let entry_list = journal_file
105 + .entry_list()
106 + .ok_or(JournalError::InvalidOffsetArrayOffset)?;
107 +
108 + let cursor = entry_list.cursor_tail(journal_file)?;
109 + if let Some(offset) = cursor.value(journal_file)? {
110 + self.array_cursor = Some(cursor);
111 + Some(Location::ResolvedEntry(offset))
112 + } else {
113 + None
114 + }
115 + }
116 + (Location::Realtime(realtime), _) => {
117 + let entry_list = journal_file
118 + .entry_list()
119 + .ok_or(JournalError::InvalidOffsetArrayOffset)?;
120 +
121 + let predicate = |entry_offset| {
122 + let entry_object = journal_file.entry_ref(entry_offset)?;
123 + Ok(entry_object.header.realtime < realtime)
124 + };
125 +
126 + let cursor = entry_list
127 + .directed_partition_point(journal_file, predicate, Direction::Forward)?
128 + .map(Ok)
129 + .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
130 +
131 + if let Some(offset) = cursor.value(journal_file)? {
132 + self.array_cursor = Some(cursor);
133 + Some(Location::ResolvedEntry(offset))
134 + } else {
135 + None
136 + }
137 + }
138 + (Location::ResolvedEntry(_), Direction::Forward) => {
139 + let Some(cursor) = self.array_cursor.unwrap().next(journal_file)? else {
140 + return Ok(None);
141 + };
142 +
143 + if let Some(offset) = cursor.value(journal_file)? {
144 + self.array_cursor = Some(cursor);
145 + Some(Location::ResolvedEntry(offset))
146 + } else {
147 + None
148 + }
149 + }
150 + (Location::ResolvedEntry(_), Direction::Backward) => {
151 + let Some(cursor) = self.array_cursor.unwrap().previous(journal_file)? else {
152 + return Ok(None);
153 + };
154 +
155 + if let Some(offset) = cursor.value(journal_file)? {
156 + self.array_cursor = Some(cursor);
157 + Some(Location::ResolvedEntry(offset))
158 + } else {
159 + None
160 + }
161 + }
162 + _ => {
163 + unimplemented!()
164 + }
165 + };
166 +
167 + Ok(new_location)
168 + }
169 +
170 + fn resolve_filter_location<M: MemoryMap>(
171 + &mut self,
172 + journal_file: &JournalFile<M>,
173 + direction: Direction,
174 + ) -> Result<Option<Location>> {
175 + let filter_expr = self.filter_expr.as_mut().unwrap();
176 +
177 + let resolved_location = match (self.location, direction) {
178 + (Location::Head, Direction::Forward) => filter_expr
179 + .head()
180 + .next(journal_file, NonZeroU64::MIN)?
181 + .map(Location::ResolvedEntry),
182 + (Location::Head, Direction::Backward) => None,
183 + (Location::Tail, Direction::Forward) => None,
184 + (Location::Tail, Direction::Backward) => filter_expr
185 + .tail(journal_file)?
186 + .previous(journal_file, NonZeroU64::MAX)?
187 + .map(Location::ResolvedEntry),
188 + (Location::Realtime(realtime), direction) => {
189 + let entry_list = journal_file
190 + .entry_list()
191 + .ok_or(JournalError::InvalidOffsetArrayOffset)?;
192 +
193 + let predicate = |entry_offset| {
194 + let entry_object = journal_file.entry_ref(entry_offset)?;
195 + Ok(entry_object.header.realtime < realtime)
196 + };
197 +
198 + let cursor = entry_list
199 + .directed_partition_point(journal_file, predicate, Direction::Forward)?
200 + .map(Ok)
201 + .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
202 +
203 + if let Some(entry_offset) = cursor.value(journal_file)? {
204 + match direction {
205 + Direction::Forward => filter_expr
206 + .head()
207 + .next(journal_file, entry_offset)?
208 + .map(Location::ResolvedEntry),
209 + Direction::Backward => filter_expr
210 + .tail(journal_file)?
211 + .previous(journal_file, entry_offset)?
212 + .map(Location::ResolvedEntry),
213 + }
214 + } else {
215 + None
216 + }
217 + }
218 + (Location::ResolvedEntry(location_offset), Direction::Forward) => filter_expr
219 + .next(journal_file, location_offset.saturating_add(1))?
220 + .map(Location::ResolvedEntry),
221 + (Location::ResolvedEntry(location_offset), Direction::Backward) => {
222 + if let Some(needle_offset) = NonZeroU64::new(location_offset.get() - 1) {
223 + filter_expr
224 + .previous(journal_file, needle_offset)?
225 + .map(Location::ResolvedEntry)
226 + } else {
227 + None
228 + }
229 + }
230 + _ => {
231 + unimplemented!();
232 + }
233 + };
234 +
235 + Ok(resolved_location)
236 + }
237 +}
src/crates/jf/journal_file/src/file.rs new
+1130
@@ -0,0 +1,1130 @@
1 +#![allow(unused_imports, clippy::field_reassign_with_default)]
2 +
3 +use crate::hash;
4 +use crate::object::*;
5 +use crate::offset_array;
6 +use error::{JournalError, Result};
7 +use std::cell::{RefCell, UnsafeCell};
8 +use std::fs::{File, OpenOptions};
9 +use std::marker::PhantomData;
10 +use std::num::NonZero;
11 +use std::num::NonZeroI128;
12 +use std::num::NonZeroU64;
13 +use std::path::Path;
14 +use window_manager::{MemoryMap, MemoryMapMut, WindowManager};
15 +use zerocopy::{ByteSlice, FromBytes, SplitByteSlice, SplitByteSliceMut};
16 +
17 +#[cfg(debug_assertions)]
18 +use std::backtrace::Backtrace;
19 +
20 +use crate::value_guard::ValueGuard;
21 +
22 +#[cfg(target_os = "linux")]
23 +pub fn load_machine_id() -> Result<[u8; 16]> {
24 + let content = std::fs::read_to_string("/etc/machine-id")?;
25 + let decoded = hex::decode(content.trim()).map_err(|_| JournalError::UuidSerde)?;
26 + let bytes: [u8; 16] = decoded.try_into().map_err(|_| JournalError::UuidSerde)?;
27 + Ok(bytes)
28 +}
29 +
30 +#[cfg(target_os = "macos")]
31 +pub fn load_machine_id() -> Result<[u8; 16]> {
32 + use std::process::Command;
33 +
34 + let output = Command::new("system_profiler")
35 + .arg("SPHardwareDataType")
36 + .output()
37 + .map_err(|_| JournalError::UuidSerde)?;
38 +
39 + if output.status.success() {
40 + let output_str = String::from_utf8_lossy(&output.stdout);
41 + for line in output_str.lines() {
42 + if line.contains("Hardware UUID:") {
43 + if let Some(uuid_str) = line.split("Hardware UUID:").nth(1) {
44 + let uuid_str = uuid_str.trim();
45 + let hex_str: String = uuid_str.chars().filter(|c| *c != '-').collect();
46 +
47 + if hex_str.len() == 32 {
48 + let mut bytes = [0u8; 16];
49 + for i in 0..16 {
50 + let hex_pair = &hex_str[i * 2..i * 2 + 2];
51 + bytes[i] = u8::from_str_radix(hex_pair, 16)
52 + .map_err(|_| JournalError::UuidSerde)?;
53 + }
54 + return Ok(bytes);
55 + }
56 + }
57 + }
58 + }
59 + }
60 +
61 + Err(JournalError::UuidSerde)
62 +}
63 +
64 +#[cfg(not(any(target_os = "linux", target_os = "macos")))]
65 +pub fn load_machine_id() -> Result<[u8; 16]> {
66 + Err(JournalError::UuidSerde)
67 +}
68 +
69 +#[cfg(target_os = "linux")]
70 +pub fn load_boot_id() -> Result<[u8; 16]> {
71 + let content = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?;
72 +
73 + let uuid_str = content.trim();
74 + let hex_str: String = uuid_str.chars().filter(|c| *c != '-').collect();
75 +
76 + if hex_str.len() != 32 {
77 + return Err(JournalError::UuidSerde);
78 + }
79 +
80 + let mut bytes = [0u8; 16];
81 + for i in 0..16 {
82 + let hex_pair = &hex_str[i * 2..i * 2 + 2];
83 + bytes[i] = u8::from_str_radix(hex_pair, 16).map_err(|_| JournalError::UuidSerde)?;
84 + }
85 +
86 + Ok(bytes)
87 +}
88 +
89 +#[cfg(target_os = "macos")]
90 +pub fn load_boot_id() -> Result<[u8; 16]> {
91 + use std::process::Command;
92 +
93 + let output = Command::new("sysctl")
94 + .arg("-n")
95 + .arg("kern.boottime")
96 + .output()
97 + .map_err(|_| JournalError::UuidSerde)?;
98 +
99 + if output.status.success() {
100 + let output_str = String::from_utf8_lossy(&output.stdout);
101 + // Parse "{ sec = 1753988677, usec = 131097 } Thu Jul 31 22:04:37 2025"
102 + // Extract sec and usec values
103 + if let (Some(sec_start), Some(usec_start)) =
104 + (output_str.find("sec = "), output_str.find("usec = "))
105 + {
106 + let sec_str = &output_str[sec_start + 6..];
107 + let sec_end = sec_str.find(',').unwrap_or(sec_str.len());
108 + let sec_str = &sec_str[..sec_end].trim();
109 +
110 + let usec_str = &output_str[usec_start + 7..];
111 + let usec_end = usec_str.find(' ').unwrap_or(usec_str.len());
112 + let usec_str = &usec_str[..usec_end].trim();
113 +
114 + if let (Ok(sec), Ok(usec)) = (sec_str.parse::<u64>(), usec_str.parse::<u64>()) {
115 + // Create a deterministic UUID from boot time
116 + // Use sec in first 8 bytes, usec in next 4 bytes, pad remaining with zeros
117 + let mut bytes = [0u8; 16];
118 + bytes[0..8].copy_from_slice(&sec.to_be_bytes());
119 + bytes[8..12].copy_from_slice(&(usec as u32).to_be_bytes());
120 + // bytes[12..16] remain zero-filled for consistency
121 + return Ok(bytes);
122 + }
123 + }
124 + }
125 +
126 + Err(JournalError::UuidSerde)
127 +}
128 +
129 +#[cfg(not(any(target_os = "linux", target_os = "macos")))]
130 +pub fn load_boot_id() -> Result<[u8; 16]> {
131 + Err(JournalError::UuidSerde)
132 +}
133 +
134 +// Size to pad objects to (8 bytes)
135 +const OBJECT_ALIGNMENT: u64 = 8;
136 +
137 +pub trait BucketVisitor<'a> {
138 + type Object: JournalObject<&'a [u8]> + HashableObject;
139 + type Output;
140 +
141 + /// Called for each object in the bucket. Return Some(output) to stop iteration,
142 + /// or None to continue to the next object.
143 + fn visit(&mut self, object: &ValueGuard<'a, Self::Object>) -> Result<Option<Self::Output>>;
144 +}
145 +
146 +struct PayloadMatcher<'data, T> {
147 + payload: &'data [u8],
148 + hash: u64,
149 + _phantom: PhantomData<T>,
150 +}
151 +
152 +impl<'data, B: ByteSlice> PayloadMatcher<'data, DataObject<B>> {
153 + fn data_matcher(payload: &'data [u8], hash: u64) -> Self {
154 + Self {
155 + payload,
156 + hash,
157 + _phantom: PhantomData::<DataObject<B>>,
158 + }
159 + }
160 +}
161 +
162 +impl<'data, B: ByteSlice> PayloadMatcher<'data, FieldObject<B>> {
163 + fn field_matcher(payload: &'data [u8], hash: u64) -> Self {
164 + Self {
165 + payload,
166 + hash,
167 + _phantom: PhantomData::<FieldObject<B>>,
168 + }
169 + }
170 +}
171 +
172 +impl<'a, T> BucketVisitor<'a> for PayloadMatcher<'_, T>
173 +where
174 + T: JournalObject<&'a [u8]> + HashableObject,
175 +{
176 + type Object = T;
177 + type Output = NonZeroU64;
178 +
179 + fn visit(&mut self, object: &ValueGuard<'a, Self::Object>) -> Result<Option<Self::Output>> {
180 + if object.hash() == self.hash && object.get_payload() == self.payload {
181 + Ok(Some(object.offset()))
182 + } else {
183 + Ok(None)
184 + }
185 + }
186 +}
187 +
188 +#[derive(Debug, Clone)]
189 +pub struct JournalFileOptions {
190 + machine_id: [u8; 16],
191 + boot_id: [u8; 16],
192 + seqnum_id: [u8; 16],
193 + file_id: [u8; 16],
194 + window_size: u64,
195 + data_hash_table_buckets: usize,
196 + field_hash_table_buckets: usize,
197 + enable_keyed_hash: bool,
198 +}
199 +
200 +impl JournalFileOptions {
201 + pub fn new(
202 + machine_id: [u8; 16],
203 + boot_id: [u8; 16],
204 + seqnum_id: [u8; 16],
205 + file_id: [u8; 16],
206 + ) -> Self {
207 + Self {
208 + machine_id,
209 + boot_id,
210 + seqnum_id,
211 + file_id,
212 + window_size: 64 * 1024,
213 + data_hash_table_buckets: 4096,
214 + field_hash_table_buckets: 512,
215 + enable_keyed_hash: true,
216 + }
217 + }
218 +
219 + pub fn with_window_size(mut self, size: u64) -> Self {
220 + assert_eq!(size % OBJECT_ALIGNMENT, 0);
221 + assert_eq!(size % 4096, 0, "Window size must be page-aligned");
222 + self.window_size = size;
223 + self
224 + }
225 +
226 + pub fn with_data_hash_table_buckets(mut self, buckets: usize) -> Self {
227 + assert!(
228 + buckets.is_power_of_two(),
229 + "Hash table buckets should be a power of two"
230 + );
231 + self.data_hash_table_buckets = buckets;
232 + self
233 + }
234 +
235 + pub fn with_field_hash_table_buckets(mut self, buckets: usize) -> Self {
236 + assert!(
237 + buckets.is_power_of_two(),
238 + "Hash table buckets should be a power of two"
239 + );
240 + self.field_hash_table_buckets = buckets;
241 + self
242 + }
243 +
244 + pub fn with_keyed_hash(mut self, enabled: bool) -> Self {
245 + self.enable_keyed_hash = enabled;
246 + self
247 + }
248 +
249 + pub fn create<M: MemoryMapMut>(self, path: impl AsRef<Path>) -> Result<JournalFile<M>> {
250 + JournalFile::create(path, self)
251 + }
252 +}
253 +
254 +/// Hash table bucket utilization statistics
255 +#[derive(Debug, Clone, Copy)]
256 +pub struct BucketUtilization {
257 + pub data_occupied: usize,
258 + pub data_total: usize,
259 + pub field_occupied: usize,
260 + pub field_total: usize,
261 +}
262 +
263 +impl BucketUtilization {
264 + pub fn data_utilization(&self) -> f64 {
265 + if self.data_total == 0 {
266 + 0.0
267 + } else {
268 + self.data_occupied as f64 / self.data_total as f64
269 + }
270 + }
271 +
272 + pub fn field_utilization(&self) -> f64 {
273 + if self.field_total == 0 {
274 + 0.0
275 + } else {
276 + self.field_occupied as f64 / self.field_total as f64
277 + }
278 + }
279 +}
280 +
281 +///
282 +/// A reader for systemd journal files that efficiently maps small regions of the file into memory.
283 +///
284 +/// # Memory Management
285 +///
286 +/// This implementation uses a window-based memory mapping strategy similar to systemd's original
287 +/// implementation. Instead of mapping the entire file, it maintains a small set of memory-mapped
288 +/// windows and reuses them as needed.
289 +///
290 +/// # Concurrency and Safety
291 +///
292 +/// `JournalFile` uses interior mutability to provide a safe API with the following characteristics:
293 +///
294 +/// - The window manager is wrapped in an `UnsafeCell` to allow mutation through a shared reference.
295 +/// - A single `RefCell<bool>` guards access to ensure only one object can be active at a time.
296 +/// - Methods like `data_object()` return a `ValueGuard<T>` that automatically releases the lock
297 +/// when dropped.
298 +///
299 +/// This design ensures that memory safety is maintained even though references to memory-mapped
300 +/// regions could be invalidated when new objects are created.
301 +pub struct JournalFile<M: MemoryMap> {
302 + // Persistent memory maps for journal header and data/field hash tables
303 + header_map: M,
304 + data_hash_table_map: Option<M>,
305 + field_hash_table_map: Option<M>,
306 +
307 + // Window manager for other objects
308 + window_manager: UnsafeCell<WindowManager<M>>,
309 +
310 + // Flag to track if any object is in use
311 + object_in_use: RefCell<bool>,
312 +
313 + #[cfg(debug_assertions)]
314 + prev_backtrace: RefCell<Backtrace>,
315 + #[cfg(debug_assertions)]
316 + backtrace: RefCell<Backtrace>,
317 +}
318 +
319 +fn map_hash_table<M: MemoryMap>(
320 + file: &File,
321 + offset: Option<NonZeroU64>,
322 + size: Option<NonZeroU64>,
323 +) -> Result<Option<M>> {
324 + let (Some(offset), Some(size)) = (offset, size) else {
325 + return Ok(None);
326 + };
327 +
328 + if offset.get() <= std::mem::size_of::<JournalHeader>() as u64 {
329 + return Err(JournalError::InvalidObjectLocation);
330 + }
331 + if size.get() <= std::mem::size_of::<ObjectHeader>() as u64 {
332 + return Err(JournalError::InvalidObjectLocation);
333 + }
334 +
335 + let offset = offset.get() - std::mem::size_of::<ObjectHeader>() as u64;
336 + let size = std::mem::size_of::<ObjectHeader>() as u64 + size.get();
337 + M::create(file, offset, size).map(Some)
338 +}
339 +
340 +impl<M: MemoryMap> JournalFile<M> {
341 + pub fn visit_bucket<'a, H, V>(
342 + &'a self,
343 + hash_table: Option<H>,
344 + hash: u64,
345 + mut visitor: V,
346 + ) -> Result<Option<V::Output>>
347 + where
348 + H: HashTable<Object = V::Object>,
349 + V: BucketVisitor<'a>,
350 + {
351 + let hash_table = hash_table.ok_or(JournalError::MissingHashTable)?;
352 + let bucket = hash_table.hash_item_ref(hash);
353 + let mut object_offset = bucket.head_hash_offset;
354 +
355 + while let Some(offset) = object_offset {
356 + let object_guard = self.journal_object_ref::<V::Object>(offset)?;
357 +
358 + if let Some(output) = visitor.visit(&object_guard)? {
359 + return Ok(Some(output));
360 + }
361 +
362 + object_offset = object_guard.next_hash_offset();
363 + }
364 +
365 + Ok(None)
366 + }
367 +
368 + pub fn open(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
369 + debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
370 +
371 + // Open file and check its size
372 + let file = OpenOptions::new().read(true).write(false).open(&path)?;
373 +
374 + // Create a memory map for the header
375 + let header_size = std::mem::size_of::<JournalHeader>() as u64;
376 + let header_map = M::create(&file, 0, header_size)?;
377 + let header = JournalHeader::ref_from_prefix(&header_map).unwrap().0;
378 + if header.signature != *b"LPKSHHRH" {
379 + return Err(JournalError::InvalidMagicNumber);
380 + }
381 +
382 + // Initialize the hash table maps if they exist
383 + let data_hash_table_map = map_hash_table(
384 + &file,
385 + header.data_hash_table_offset,
386 + header.data_hash_table_size,
387 + )?;
388 + let field_hash_table_map = map_hash_table(
389 + &file,
390 + header.field_hash_table_offset,
391 + header.field_hash_table_size,
392 + )?;
393 +
394 + // Create window manager for the rest of the objects
395 + let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
396 +
397 + Ok(JournalFile {
398 + header_map,
399 + data_hash_table_map,
400 + field_hash_table_map,
401 + window_manager,
402 + object_in_use: RefCell::new(false),
403 +
404 + #[cfg(debug_assertions)]
405 + prev_backtrace: RefCell::new(Backtrace::capture()),
406 + #[cfg(debug_assertions)]
407 + backtrace: RefCell::new(Backtrace::capture()),
408 + })
409 + }
410 +
411 + pub fn hash(&self, data: &[u8]) -> u64 {
412 + let is_keyed_hash = self
413 + .journal_header_ref()
414 + .has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash);
415 +
416 + hash::journal_hash_data(
417 + data,
418 + is_keyed_hash,
419 + if is_keyed_hash {
420 + Some(&self.journal_header_ref().file_id)
421 + } else {
422 + None
423 + },
424 + )
425 + }
426 +
427 + pub fn entry_list(&self) -> Option<offset_array::List> {
428 + let head_offset = self.journal_header_ref().entry_array_offset?;
429 + let total_items =
430 + std::num::NonZeroUsize::new(self.journal_header_ref().n_entries as usize)?;
431 + Some(offset_array::List::new(head_offset, total_items))
432 + }
433 +
434 + pub fn journal_header_ref(&self) -> &JournalHeader {
435 + JournalHeader::ref_from_prefix(&self.header_map).unwrap().0
436 + }
437 +
438 + pub fn data_hash_table_map(&self) -> Option<&M> {
439 + self.data_hash_table_map.as_ref()
440 + }
441 + pub fn field_hash_table_map(&self) -> Option<&M> {
442 + self.field_hash_table_map.as_ref()
443 + }
444 +
445 + pub fn data_hash_table_ref(&self) -> Option<DataHashTable<&[u8]>> {
446 + self.data_hash_table_map
447 + .as_ref()
448 + .and_then(|m| DataHashTable::<&[u8]>::from_data(m, false))
449 + }
450 +
451 + pub fn field_hash_table_ref(&self) -> Option<FieldHashTable<&[u8]>> {
452 + self.field_hash_table_map
453 + .as_ref()
454 + .and_then(|m| FieldHashTable::<&[u8]>::from_data(m, false))
455 + }
456 +
457 + pub fn object_header_ref(&self, position: NonZeroU64) -> Result<&ObjectHeader> {
458 + let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
459 + let window_manager = unsafe { &mut *self.window_manager.get() };
460 + let header_slice = window_manager.get_slice(position.get(), size_needed)?;
461 + Ok(ObjectHeader::ref_from_bytes(header_slice).unwrap())
462 + }
463 +
464 + fn object_data_ref(&self, offset: NonZeroU64, size_needed: u64) -> Result<&[u8]> {
465 + let window_manager = unsafe { &mut *self.window_manager.get() };
466 + let object_slice = window_manager.get_slice(offset.get(), size_needed)?;
467 + Ok(object_slice)
468 + }
469 +
470 + fn journal_object_ref<'a, T>(&'a self, offset: NonZeroU64) -> Result<ValueGuard<'a, T>>
471 + where
472 + T: JournalObject<&'a [u8]>,
473 + {
474 + // Check if any object is already in use
475 + let mut is_in_use = self.object_in_use.borrow_mut();
476 + if *is_in_use {
477 + #[cfg(debug_assertions)]
478 + {
479 + eprintln!(
480 + "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
481 + self.backtrace.borrow().to_string(),
482 + self.prev_backtrace.borrow().to_string()
483 + );
484 + }
485 + return Err(JournalError::ValueGuardInUse);
486 + }
487 +
488 + #[cfg(debug_assertions)]
489 + {
490 + self.backtrace.swap(&self.prev_backtrace);
491 + let _ = self.backtrace.replace(Backtrace::force_capture());
492 + }
493 +
494 + let is_compact = self
495 + .journal_header_ref()
496 + .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
497 +
498 + let size_needed = {
499 + let header = self.object_header_ref(offset)?;
500 + header.size
501 + };
502 +
503 + let data = self.object_data_ref(offset, size_needed)?;
504 + let Some(value) = T::from_data(data, is_compact) else {
505 + return Err(JournalError::ZerocopyFailure);
506 + };
507 +
508 + // Mark as in use
509 + *is_in_use = true;
510 +
511 + Ok(ValueGuard::new(offset, value, &self.object_in_use))
512 + }
513 +
514 + pub fn offset_array_ref(
515 + &self,
516 + offset: NonZeroU64,
517 + ) -> Result<ValueGuard<'_, OffsetArrayObject<&[u8]>>> {
518 + self.journal_object_ref(offset)
519 + }
520 +
521 + pub fn field_ref(&self, offset: NonZeroU64) -> Result<ValueGuard<'_, FieldObject<&[u8]>>> {
522 + self.journal_object_ref(offset)
523 + }
524 +
525 + pub fn entry_ref(&self, offset: NonZeroU64) -> Result<ValueGuard<'_, EntryObject<&[u8]>>> {
526 + self.journal_object_ref(offset)
527 + }
528 +
529 + pub fn data_ref(&self, offset: NonZeroU64) -> Result<ValueGuard<'_, DataObject<&[u8]>>> {
530 + self.journal_object_ref(offset)
531 + }
532 +
533 + pub fn tag_ref(&self, offset: NonZeroU64) -> Result<ValueGuard<'_, TagObject<&[u8]>>> {
534 + self.journal_object_ref(offset)
535 + }
536 +
537 + pub fn find_data_offset(&self, hash: u64, payload: &[u8]) -> Result<Option<NonZeroU64>> {
538 + let visitor = PayloadMatcher::data_matcher(payload, hash);
539 + self.visit_bucket(self.data_hash_table_ref(), hash, visitor)
540 + }
541 +
542 + pub fn find_field_offset(&self, hash: u64, payload: &[u8]) -> Result<Option<NonZeroU64>> {
543 + let visitor = PayloadMatcher::field_matcher(payload, hash);
544 + self.visit_bucket(self.field_hash_table_ref(), hash, visitor)
545 + }
546 +
547 + /// Run a directed partition point query on a data object's entry array
548 + ///
549 + /// This finds the first/last entry (depending on direction) that satisfies the given predicate
550 + /// in the entry array chain of the data object.
551 + pub fn data_object_directed_partition_point<F>(
552 + &self,
553 + data_offset: NonZeroU64,
554 + predicate: F,
555 + direction: offset_array::Direction,
556 + ) -> Result<Option<NonZeroU64>>
557 + where
558 + F: Fn(NonZeroU64) -> Result<bool>,
559 + {
560 + let Some(cursor) = self.data_ref(data_offset)?.inlined_cursor() else {
561 + return Ok(None);
562 + };
563 +
564 + let Some(best_match) = cursor.directed_partition_point(self, predicate, direction)? else {
565 + return Ok(None);
566 + };
567 +
568 + best_match.value(self)
569 + }
570 +
571 + /// Creates an iterator over all field objects in the field hash table
572 + pub fn fields(&self) -> FieldIterator<'_, M> {
573 + // Get the field hash table
574 + let field_hash_table = self.field_hash_table_ref();
575 +
576 + // Initialize with the first bucket
577 + let mut iterator = FieldIterator {
578 + journal: self,
579 + field_hash_table,
580 + current_bucket_index: 0,
581 + next_field_offset: None,
582 + };
583 +
584 + // Find the first non-empty bucket
585 + iterator.advance_to_next_nonempty_bucket();
586 +
587 + iterator
588 + }
589 +
590 + /// Creates an iterator over all DATA objects for the specified field
591 + pub fn field_data_objects<'a>(
592 + &'a self,
593 + field_name: &'a [u8],
594 + ) -> Result<FieldDataIterator<'a, M>> {
595 + // Find the field offset by name
596 + let field_hash = self.hash(field_name);
597 + let Some(field_offset) = self.find_field_offset(field_hash, field_name)? else {
598 + return Ok(FieldDataIterator {
599 + journal: self,
600 + current_data_offset: None,
601 + });
602 + };
603 +
604 + // Get the field object to access its head_data_offset
605 + let field_guard = self.field_ref(field_offset)?;
606 + let head_data_offset = field_guard.header.head_data_offset;
607 +
608 + // Create the iterator
609 + Ok(FieldDataIterator {
610 + journal: self,
611 + current_data_offset: head_data_offset,
612 + })
613 + }
614 +
615 + /// Creates an iterator over all DATA objects for a specific entry
616 + pub fn entry_data_objects(&self, entry_offset: NonZeroU64) -> Result<EntryDataIterator<'_, M>> {
617 + // Get the entry object to determine how many data items it has
618 + let entry_guard = self.entry_ref(entry_offset)?;
619 +
620 + // Get the total number of items
621 + let total_items = match &entry_guard.items {
622 + EntryItemsType::Regular(items) => items.len(),
623 + EntryItemsType::Compact(items) => items.len(),
624 + };
625 +
626 + // Create the iterator
627 + Ok(EntryDataIterator {
628 + journal: self,
629 + entry_offset: Some(entry_offset),
630 + current_index: 0,
631 + total_items,
632 + })
633 + }
634 +
635 + /// Get hash table bucket utilization statistics
636 + pub fn bucket_utilization(&self) -> Option<BucketUtilization> {
637 + let data_hash_table = self.data_hash_table_ref()?;
638 + let data_total = data_hash_table.items.len();
639 + let data_occupied = data_hash_table
640 + .items
641 + .iter()
642 + .filter(|item| item.head_hash_offset.is_some())
643 + .count();
644 +
645 + let field_hash_table = self.field_hash_table_ref()?;
646 + let field_total = field_hash_table.items.len();
647 + let field_occupied = field_hash_table
648 + .items
649 + .iter()
650 + .filter(|item| item.head_hash_offset.is_some())
651 + .count();
652 +
653 + Some(BucketUtilization {
654 + data_occupied,
655 + data_total,
656 + field_occupied,
657 + field_total,
658 + })
659 + }
660 +}
661 +
662 +impl<M: MemoryMapMut> JournalFile<M> {
663 + pub fn create(path: impl AsRef<Path>, options: JournalFileOptions) -> Result<Self> {
664 + let file = OpenOptions::new()
665 + .create(true)
666 + .truncate(true)
667 + .read(true)
668 + .write(true)
669 + .open(&path)?;
670 +
671 + // Calculate hash table sizes
672 + let data_hash_table_size =
673 + options.data_hash_table_buckets * std::mem::size_of::<HashItem>();
674 + let field_hash_table_size =
675 + options.field_hash_table_buckets * std::mem::size_of::<HashItem>();
676 +
677 + // Calculate hash table offsets
678 + let data_hash_table_offset = std::mem::size_of::<JournalHeader>() as u64
679 + + std::mem::size_of::<ObjectHeader>() as u64;
680 + let field_hash_table_offset = data_hash_table_offset
681 + + data_hash_table_size as u64
682 + + std::mem::size_of::<ObjectHeader>() as u64;
683 +
684 + // Create header with options configuration
685 + let mut header = JournalHeader::default();
686 + header.signature = *b"LPKSHHRH";
687 +
688 + // Set flags based on options configuration
689 + if options.enable_keyed_hash {
690 + header.incompatible_flags |= HeaderIncompatibleFlags::KeyedHash as u32;
691 + }
692 +
693 + // Set hash table configuration
694 + header.data_hash_table_offset = NonZeroU64::new(data_hash_table_offset);
695 + header.data_hash_table_size = NonZeroU64::new(data_hash_table_size as u64);
696 + header.field_hash_table_offset = NonZeroU64::new(field_hash_table_offset);
697 + header.field_hash_table_size = NonZeroU64::new(field_hash_table_size as u64);
698 +
699 + // Set other header fields
700 + header.tail_object_offset =
701 + NonZeroU64::new(data_hash_table_offset + data_hash_table_size as u64);
702 + header.header_size = std::mem::size_of::<JournalHeader>() as u64;
703 + header.n_objects = 2;
704 + header.arena_size =
705 + field_hash_table_offset + field_hash_table_size as u64 - header.header_size;
706 +
707 + // Set IDs from options
708 + header.machine_id = options.machine_id;
709 + header.tail_entry_boot_id = options.boot_id;
710 + header.file_id = options.file_id;
711 + header.seqnum_id = options.seqnum_id;
712 +
713 + // Create memory maps for hash tables
714 + let data_hash_table_map = map_hash_table(
715 + &file,
716 + header.data_hash_table_offset,
717 + header.data_hash_table_size,
718 + )?;
719 + let field_hash_table_map = map_hash_table(
720 + &file,
721 + header.field_hash_table_offset,
722 + header.field_hash_table_size,
723 + )?;
724 +
725 + // Create header memory map and write header
726 + let header_size = std::mem::size_of::<JournalHeader>() as u64;
727 + let mut header_map = M::create(&file, 0, header_size)?;
728 + {
729 + let header_mut = JournalHeader::mut_from_prefix(&mut header_map).unwrap().0;
730 + *header_mut = header;
731 + }
732 +
733 + // Create window manager for the rest of the objects
734 + let window_manager = UnsafeCell::new(WindowManager::new(file, options.window_size, 32)?);
735 +
736 + let jf = JournalFile {
737 + header_map,
738 + data_hash_table_map,
739 + field_hash_table_map,
740 + window_manager,
741 + object_in_use: RefCell::new(false),
742 +
743 + #[cfg(debug_assertions)]
744 + prev_backtrace: RefCell::new(Backtrace::capture()),
745 + #[cfg(debug_assertions)]
746 + backtrace: RefCell::new(Backtrace::capture()),
747 + };
748 +
749 + // write data hash table object header info
750 + {
751 + let offset = NonZeroU64::new(
752 + header.data_hash_table_offset.unwrap().get()
753 + - std::mem::size_of::<ObjectHeader>() as u64,
754 + )
755 + .unwrap();
756 + let size = header.data_hash_table_size.unwrap().get()
757 + + std::mem::size_of::<ObjectHeader>() as u64;
758 +
759 + let object_header = jf.object_header_mut(offset)?;
760 + object_header.type_ = ObjectType::DataHashTable as u8;
761 + object_header.size = size
762 + }
763 +
764 + // write field hash table object header info
765 + {
766 + let offset = NonZeroU64::new(
767 + header.field_hash_table_offset.unwrap().get()
768 + - std::mem::size_of::<ObjectHeader>() as u64,
769 + )
770 + .unwrap();
771 + let size = header.field_hash_table_size.unwrap().get()
772 + + std::mem::size_of::<ObjectHeader>() as u64;
773 +
774 + let object_header = jf.object_header_mut(offset)?;
775 + object_header.type_ = ObjectType::FieldHashTable as u8;
776 + object_header.size = size
777 + }
778 +
779 + Ok(jf)
780 + }
781 +
782 + pub fn journal_header_mut(&mut self) -> &mut JournalHeader {
783 + JournalHeader::mut_from_prefix(&mut self.header_map)
784 + .unwrap()
785 + .0
786 + }
787 +
788 + pub fn data_hash_table_mut(&mut self) -> Option<DataHashTable<&mut [u8]>> {
789 + self.data_hash_table_map
790 + .as_mut()
791 + .and_then(|m| DataHashTable::<&mut [u8]>::from_data_mut(m, false))
792 + }
793 +
794 + pub fn field_hash_table_mut(&mut self) -> Option<FieldHashTable<&mut [u8]>> {
795 + self.field_hash_table_map
796 + .as_mut()
797 + .and_then(|m| FieldHashTable::<&mut [u8]>::from_data_mut(m, false))
798 + }
799 +
800 + fn object_header_mut(&self, offset: NonZeroU64) -> Result<&mut ObjectHeader> {
801 + let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
802 + let window_manager = unsafe { &mut *self.window_manager.get() };
803 + let header_slice = window_manager.get_slice_mut(offset.get(), size_needed)?;
804 + Ok(ObjectHeader::mut_from_bytes(header_slice).unwrap())
805 + }
806 +
807 + fn object_data_mut(&self, offset: NonZeroU64, size_needed: u64) -> Result<&mut [u8]> {
808 + let window_manager = unsafe { &mut *self.window_manager.get() };
809 + let object_slice = window_manager.get_slice_mut(offset.get(), size_needed)?;
810 + Ok(object_slice)
811 + }
812 +
813 + fn journal_object_mut<'a, T>(
814 + &'a self,
815 + type_: ObjectType,
816 + offset: NonZeroU64,
817 + size: Option<u64>,
818 + ) -> Result<ValueGuard<'a, T>>
819 + where
820 + T: JournalObjectMut<&'a mut [u8]>,
821 + {
822 + // Check if any object is already in use
823 + let mut is_in_use = self.object_in_use.borrow_mut();
824 + if *is_in_use {
825 + #[cfg(debug_assertions)]
826 + {
827 + eprintln!(
828 + "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
829 + self.backtrace.borrow().to_string(),
830 + self.prev_backtrace.borrow().to_string()
831 + );
832 + }
833 + return Err(JournalError::ValueGuardInUse);
834 + }
835 +
836 + #[cfg(debug_assertions)]
837 + {
838 + self.backtrace.swap(&self.prev_backtrace);
839 + let _ = self.backtrace.replace(Backtrace::force_capture());
840 + }
841 +
842 + let is_compact = self
843 + .journal_header_ref()
844 + .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
845 +
846 + let size_needed = match size {
847 + Some(size) => {
848 + let header = self.object_header_mut(offset)?;
849 + header.type_ = type_ as u8;
850 + header.size = size;
851 + size
852 + }
853 + None => {
854 + let header = self.object_header_ref(offset)?;
855 + if header.type_ != type_ as u8 {
856 + return Err(JournalError::InvalidObjectType);
857 + }
858 + header.size
859 + }
860 + };
861 +
862 + let data = self.object_data_mut(offset, size_needed)?;
863 + let value = T::from_data_mut(data, is_compact).ok_or(JournalError::ZerocopyFailure)?;
864 +
865 + // Mark as in use
866 + *is_in_use = true;
867 + Ok(ValueGuard::new(offset, value, &self.object_in_use))
868 + }
869 +
870 + pub fn offset_array_mut(
871 + &self,
872 + offset: NonZeroU64,
873 + capacity: Option<NonZeroU64>,
874 + ) -> Result<ValueGuard<'_, OffsetArrayObject<&mut [u8]>>> {
875 + let size = capacity.map(|c| {
876 + let mut size = std::mem::size_of::<OffsetArrayObjectHeader>() as u64;
877 +
878 + let is_compact = self
879 + .journal_header_ref()
880 + .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
881 + if is_compact {
882 + size += c.get() * std::mem::size_of::<u32>() as u64;
883 + } else {
884 + size += c.get() * std::mem::size_of::<u64>() as u64;
885 + }
886 +
887 + size
888 + });
889 +
890 + let offset_array = self.journal_object_mut(ObjectType::EntryArray, offset, size);
891 + offset_array
892 + }
893 +
894 + pub fn field_mut(
895 + &self,
896 + offset: NonZeroU64,
897 + size: Option<u64>,
898 + ) -> Result<ValueGuard<'_, FieldObject<&mut [u8]>>> {
899 + let size = size.map(|n| std::mem::size_of::<FieldObjectHeader>() as u64 + n);
900 + self.journal_object_mut(ObjectType::Field, offset, size)
901 + }
902 +
903 + pub fn entry_mut(
904 + &self,
905 + offset: NonZeroU64,
906 + size: Option<u64>,
907 + ) -> Result<ValueGuard<'_, EntryObject<&mut [u8]>>> {
908 + let size = size.map(|n| std::mem::size_of::<DataObjectHeader>() as u64 + n);
909 + self.journal_object_mut(ObjectType::Entry, offset, size)
910 + }
911 +
912 + pub fn data_mut(
913 + &self,
914 + offset: NonZeroU64,
915 + size: Option<u64>,
916 + ) -> Result<ValueGuard<'_, DataObject<&mut [u8]>>> {
917 + let size = size.map(|n| std::mem::size_of::<DataObjectHeader>() as u64 + n);
918 + self.journal_object_mut(ObjectType::Data, offset, size)
919 + }
920 +
921 + pub fn tag_mut(
922 + &self,
923 + offset: NonZeroU64,
924 + new: bool,
925 + ) -> Result<ValueGuard<'_, TagObject<&mut [u8]>>> {
926 + let size = if new {
927 + Some(std::mem::size_of::<TagObjectHeader>() as u64)
928 + } else {
929 + None
930 + };
931 + self.journal_object_mut(ObjectType::Tag, offset, size)
932 + }
933 +}
934 +
935 +macro_rules! impl_hash_table_set_tail_offset {
936 + (
937 + $method_name:ident,
938 + $hash_table_ref:ident,
939 + $hash_table_mut:ident,
940 + $object_mut:ident
941 + ) => {
942 + pub fn $method_name(&mut self, hash: u64, object_offset: NonZeroU64) -> Result<()> {
943 + let hash_item = {
944 + let Some(ht) = self.$hash_table_ref() else {
945 + return Err(JournalError::MissingHashTable);
946 + };
947 + *ht.hash_item_ref(hash)
948 + };
949 +
950 + if let Some(tail_hash_offset) = hash_item.tail_hash_offset {
951 + let mut tail_object = self.$object_mut(tail_hash_offset, None)?;
952 + tail_object.set_next_hash_offset(object_offset);
953 + }
954 +
955 + let Some(mut ht) = self.$hash_table_mut() else {
956 + return Err(JournalError::MissingHashTable);
957 + };
958 +
959 + let hash_item = ht.hash_item_mut(hash);
960 + if hash_item.head_hash_offset.is_none() {
961 + hash_item.head_hash_offset = Some(object_offset);
962 + }
963 + hash_item.tail_hash_offset = Some(object_offset);
964 +
965 + Ok(())
966 + }
967 + };
968 +}
969 +
970 +impl<M: MemoryMapMut> JournalFile<M> {
971 + impl_hash_table_set_tail_offset!(
972 + data_hash_table_set_tail_offset,
973 + data_hash_table_ref,
974 + data_hash_table_mut,
975 + data_mut
976 + );
977 +
978 + impl_hash_table_set_tail_offset!(
979 + field_hash_table_set_tail_offset,
980 + field_hash_table_ref,
981 + field_hash_table_mut,
982 + field_mut
983 + );
984 +}
985 +
986 +/// Iterator that walks through all field objects in the field hash table
987 +pub struct FieldIterator<'a, M: MemoryMap> {
988 + journal: &'a JournalFile<M>,
989 + field_hash_table: Option<FieldHashTable<&'a [u8]>>,
990 + current_bucket_index: usize,
991 + next_field_offset: Option<NonZeroU64>,
992 +}
993 +
994 +impl<M: MemoryMap> FieldIterator<'_, M> {
995 + /// Advances to the next non-empty bucket
996 + fn advance_to_next_nonempty_bucket(&mut self) {
997 + // If we don't have a hash table, there's nothing to iterate
998 + let Some(hash_table) = &self.field_hash_table else {
999 + return;
1000 + };
1001 +
1002 + let items = &hash_table.items;
1003 +
1004 + // Find the next non-empty bucket
1005 + while self.current_bucket_index < items.len() {
1006 + let bucket = items[self.current_bucket_index];
1007 + if bucket.head_hash_offset.is_some() {
1008 + self.next_field_offset = bucket.head_hash_offset;
1009 + return;
1010 + }
1011 + self.current_bucket_index += 1;
1012 + }
1013 +
1014 + // No more non-empty buckets
1015 + self.next_field_offset = None;
1016 + }
1017 +}
1018 +
1019 +impl<'a, M: MemoryMap> Iterator for FieldIterator<'a, M> {
1020 + type Item = Result<ValueGuard<'a, FieldObject<&'a [u8]>>>;
1021 +
1022 + fn next(&mut self) -> Option<Self::Item> {
1023 + let offset = self.next_field_offset?;
1024 +
1025 + match self.journal.field_ref(offset) {
1026 + Ok(field_guard) => {
1027 + // Get the next field offset before we return the guard
1028 + self.next_field_offset = field_guard.header.next_hash_offset;
1029 +
1030 + // If we've reached the end of the chain, move to the next bucket
1031 + if self.next_field_offset.is_none() {
1032 + self.current_bucket_index += 1;
1033 + self.advance_to_next_nonempty_bucket();
1034 + }
1035 +
1036 + Some(Ok(field_guard))
1037 + }
1038 + Err(e) => {
1039 + self.next_field_offset = None;
1040 + Some(Err(e))
1041 + }
1042 + }
1043 + }
1044 +}
1045 +
1046 +/// Iterator that walks through all DATA objects for a specific field
1047 +pub struct FieldDataIterator<'a, M: MemoryMap> {
1048 + journal: &'a JournalFile<M>,
1049 + current_data_offset: Option<NonZeroU64>,
1050 +}
1051 +
1052 +impl<'a, M: MemoryMap> Iterator for FieldDataIterator<'a, M> {
1053 + type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
1054 +
1055 + fn next(&mut self) -> Option<Self::Item> {
1056 + let data_offset = self.current_data_offset?;
1057 +
1058 + match self.journal.data_ref(data_offset) {
1059 + Ok(data_guard) => {
1060 + // Get the next data offset before we return the guard
1061 + self.current_data_offset = data_guard.header.next_field_offset;
1062 + Some(Ok(data_guard))
1063 + }
1064 + Err(e) => {
1065 + self.current_data_offset = None;
1066 + Some(Err(e))
1067 + }
1068 + }
1069 + }
1070 +}
1071 +
1072 +/// Iterator that walks through all DATA objects for a specific entry
1073 +pub struct EntryDataIterator<'a, M: MemoryMap> {
1074 + journal: &'a JournalFile<M>,
1075 + entry_offset: Option<NonZeroU64>,
1076 + current_index: usize,
1077 + total_items: usize,
1078 +}
1079 +
1080 +impl<'a, M: MemoryMap> Iterator for EntryDataIterator<'a, M> {
1081 + type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
1082 +
1083 + fn next(&mut self) -> Option<Self::Item> {
1084 + let entry_offset = self.entry_offset?;
1085 +
1086 + // If we've reached the end of the data indices, return None
1087 + if self.current_index >= self.total_items {
1088 + return None;
1089 + }
1090 +
1091 + // Get the entry object to access the data offset
1092 + match self.journal.entry_ref(entry_offset) {
1093 + Ok(entry_guard) => {
1094 + let idx = self.current_index;
1095 + self.current_index += 1;
1096 +
1097 + let data_offset = match &entry_guard.items {
1098 + EntryItemsType::Regular(items) => {
1099 + if idx >= items.len() {
1100 + return None;
1101 + }
1102 + items[idx].object_offset
1103 + }
1104 + EntryItemsType::Compact(items) => {
1105 + if idx >= items.len() {
1106 + return None;
1107 + }
1108 + items[idx].object_offset as u64
1109 + }
1110 + };
1111 +
1112 + let data_offset = NonZeroU64::new(data_offset)?;
1113 +
1114 + // Drop the entry guard before obtaining the data object
1115 + drop(entry_guard);
1116 +
1117 + // Try to get the data object
1118 + match self.journal.data_ref(data_offset) {
1119 + Ok(data_guard) => Some(Ok(data_guard)),
1120 + Err(e) => Some(Err(e)),
1121 + }
1122 + }
1123 + Err(e) => {
1124 + // If we can't read the entry, return the error and stop iteration
1125 + self.current_index = self.total_items;
1126 + Some(Err(e))
1127 + }
1128 + }
1129 + }
1130 +}
src/crates/jf/journal_file/src/filter.rs new
+438
@@ -0,0 +1,438 @@
1 +use crate::{
2 + file::JournalFile,
3 + offset_array::{Direction, InlinedCursor},
4 +};
5 +use error::{JournalError, Result};
6 +use std::num::NonZeroU64;
7 +use window_manager::MemoryMap;
8 +
9 +#[derive(Clone, Debug)]
10 +pub enum FilterExpr {
11 + Match(u64, Option<InlinedCursor>),
12 + Conjunction(Vec<FilterExpr>),
13 + Disjunction(Vec<FilterExpr>),
14 +}
15 +
16 +impl FilterExpr {
17 + pub fn lookup<M: MemoryMap>(
18 + &self,
19 + journal_file: &JournalFile<M>,
20 + needle_offset: u64,
21 + direction: Direction,
22 + ) -> Result<Option<u64>> {
23 + let Some(needle_offset) = NonZeroU64::new(needle_offset) else {
24 + return Err(JournalError::InvalidOffset);
25 + };
26 +
27 + let predicate =
28 + move |entry_offset: NonZeroU64| -> Result<bool> { Ok(entry_offset < needle_offset) };
29 +
30 + match self {
31 + FilterExpr::Match(data_offset, _) => {
32 + let Some(data_offset) = NonZeroU64::new(*data_offset) else {
33 + return Err(JournalError::InvalidOffset);
34 + };
35 + let entry_offset = journal_file.data_object_directed_partition_point(
36 + data_offset,
37 + predicate,
38 + direction,
39 + )?;
40 + Ok(entry_offset.map(|x| x.get()))
41 + }
42 + FilterExpr::Conjunction(filter_exprs) => {
43 + let mut current_offset = needle_offset;
44 +
45 + loop {
46 + let previous_offset = current_offset;
47 +
48 + for filter_expr in filter_exprs {
49 + if direction == Direction::Backward {
50 + current_offset = current_offset.saturating_add(1);
51 + }
52 +
53 + match filter_expr.lookup(journal_file, current_offset.get(), direction)? {
54 + Some(new_offset) => {
55 + if new_offset == 0 {
56 + panic!("Wtf");
57 + }
58 + current_offset = NonZeroU64::new(new_offset).unwrap();
59 + }
60 + None => return Ok(None),
61 + }
62 + }
63 +
64 + if current_offset == previous_offset {
65 + return Ok(Some(current_offset.get()));
66 + }
67 + }
68 + }
69 + FilterExpr::Disjunction(filter_exprs) => {
70 + let cmp = match direction {
71 + Direction::Forward => std::cmp::min,
72 + Direction::Backward => std::cmp::max,
73 + };
74 +
75 + filter_exprs.iter().try_fold(None, |acc, expr| {
76 + let result = expr.lookup(journal_file, needle_offset.get(), direction)?;
77 +
78 + Ok(match (acc, result) {
79 + (None, Some(offset)) => Some(offset),
80 + (Some(best), Some(offset)) => Some(cmp(best, offset)),
81 + (acc, None) => acc,
82 + })
83 + })
84 + }
85 + }
86 + }
87 +
88 + pub fn head(&mut self) -> &mut Self {
89 + match self {
90 + FilterExpr::Match(_, None) => (),
91 + FilterExpr::Match(_, Some(ic)) => {
92 + *ic = ic.head();
93 + }
94 + FilterExpr::Conjunction(filter_exprs) => {
95 + for filter_expr in filter_exprs.iter_mut() {
96 + filter_expr.head();
97 + }
98 + }
99 + FilterExpr::Disjunction(filter_exprs) => {
100 + for filter_expr in filter_exprs.iter_mut() {
101 + filter_expr.head();
102 + }
103 + }
104 + }
105 +
106 + self
107 + }
108 +
109 + pub fn tail<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<&mut Self> {
110 + match self {
111 + FilterExpr::Match(_, None) => (),
112 + FilterExpr::Match(_, Some(ic)) => {
113 + *ic = ic.tail(journal_file)?;
114 + }
115 + FilterExpr::Conjunction(filter_exprs) => {
116 + for filter_expr in filter_exprs.iter_mut() {
117 + filter_expr.tail(journal_file)?;
118 + }
119 + }
120 + FilterExpr::Disjunction(filter_exprs) => {
121 + for filter_expr in filter_exprs.iter_mut() {
122 + filter_expr.tail(journal_file)?;
123 + }
124 + }
125 + }
126 +
127 + Ok(self)
128 + }
129 +
130 + // Returns the offset of the next matching entry, if any, with an offset
131 + // greater or equal to the needle offset.
132 + pub fn next<M: MemoryMap>(
133 + &mut self,
134 + journal_file: &JournalFile<M>,
135 + needle_offset: NonZeroU64,
136 + ) -> Result<Option<NonZeroU64>> {
137 + match self {
138 + FilterExpr::Match(_, None) => Ok(None),
139 + FilterExpr::Match(_, Some(ic)) => ic.next_until(journal_file, needle_offset),
140 + FilterExpr::Conjunction(filter_exprs) => {
141 + let mut needle_offset = needle_offset;
142 +
143 + loop {
144 + let previous_offset = needle_offset;
145 +
146 + for fe in filter_exprs.iter_mut() {
147 + if let Some(new_offset) = fe.next(journal_file, needle_offset)? {
148 + needle_offset = new_offset;
149 + } else {
150 + return Ok(None);
151 + }
152 + }
153 +
154 + if needle_offset == previous_offset {
155 + return Ok(Some(needle_offset));
156 + }
157 + }
158 + }
159 + FilterExpr::Disjunction(filter_exprs) => {
160 + let mut best_offset: Option<NonZeroU64> = None;
161 +
162 + for fe in filter_exprs.iter_mut() {
163 + if let Some(fe_offset) = fe.next(journal_file, needle_offset)? {
164 + best_offset = match best_offset {
165 + Some(offset) => Some(fe_offset.min(offset)),
166 + None => Some(fe_offset),
167 + };
168 + }
169 + }
170 +
171 + Ok(best_offset)
172 + }
173 + }
174 + }
175 +
176 + // Returns the offset of the previous matching entry, if any, with an offset
177 + // less or equal to the needle offset.
178 + pub fn previous<M: MemoryMap>(
179 + &mut self,
180 + journal_file: &JournalFile<M>,
181 + needle_offset: NonZeroU64,
182 + ) -> Result<Option<NonZeroU64>> {
183 + match self {
184 + FilterExpr::Match(_, None) => Ok(None),
185 + FilterExpr::Match(_, Some(ic)) => ic.previous_until(journal_file, needle_offset),
186 + FilterExpr::Conjunction(filter_exprs) => {
187 + let mut needle_offset = needle_offset;
188 +
189 + loop {
190 + let previous_offset = needle_offset;
191 +
192 + for fe in filter_exprs.iter_mut().rev() {
193 + if let Some(new_offset) = fe.previous(journal_file, needle_offset)? {
194 + needle_offset = new_offset;
195 + } else {
196 + return Ok(None);
197 + }
198 + }
199 +
200 + if needle_offset == previous_offset {
201 + return Ok(Some(needle_offset));
202 + }
203 + }
204 + }
205 + FilterExpr::Disjunction(filter_exprs) => {
206 + let mut best_offset: Option<NonZeroU64> = None;
207 +
208 + for fe in filter_exprs.iter_mut() {
209 + if let Some(fe_offset) = fe.previous(journal_file, needle_offset)? {
210 + best_offset = match best_offset {
211 + Some(offset) => Some(fe_offset.max(offset)),
212 + None => Some(fe_offset),
213 + };
214 + }
215 + }
216 +
217 + Ok(best_offset)
218 + }
219 + }
220 + }
221 +
222 + pub fn dump<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<String> {
223 + let mut output = String::new();
224 + self.dump_internal(journal_file, 0, &mut output)?;
225 + Ok(output)
226 + }
227 +
228 + /// Helper function for format_data_objects that handles nested expressions and indentation
229 + fn dump_internal<M: MemoryMap>(
230 + &self,
231 + journal_file: &JournalFile<M>,
232 + indent_level: usize,
233 + output: &mut String,
234 + ) -> Result<()> {
235 + let indent = " ".repeat(indent_level);
236 +
237 + match self {
238 + FilterExpr::Match(data_offset, inlined_cursor) => {
239 + // Load the data object
240 + let Some(data_offset) = NonZeroU64::new(*data_offset) else {
241 + return Err(JournalError::InvalidOffset);
242 + };
243 + let data_object = journal_file.data_ref(data_offset)?;
244 +
245 + // Get the payload as a string if possible
246 + let payload_bytes = data_object.payload_bytes();
247 + let payload_str = String::from_utf8_lossy(payload_bytes);
248 +
249 + // Format the data offset and payload
250 + output.push_str(&format!(
251 + "{}Match[offset=0x{:x}]: {}\n",
252 + indent, data_offset, payload_str
253 + ));
254 +
255 + // Format cursor information if available
256 + if let Some(ic) = inlined_cursor {
257 + output.push_str(&format!("{} Cursor: {:?}\n", indent, ic));
258 + }
259 + }
260 + FilterExpr::Conjunction(filter_exprs) => {
261 + output.push_str(&format!("{}Conjunction (AND) {{\n", indent));
262 + for expr in filter_exprs {
263 + expr.dump_internal(journal_file, indent_level + 1, output)?;
264 + }
265 + output.push_str(&format!("{}}}\n", indent));
266 + }
267 + FilterExpr::Disjunction(filter_exprs) => {
268 + output.push_str(&format!("{}Disjunction (OR) {{\n", indent));
269 + for expr in filter_exprs {
270 + expr.dump_internal(journal_file, indent_level + 1, output)?;
271 + }
272 + output.push_str(&format!("{}}}\n", indent));
273 + }
274 + }
275 +
276 + Ok(())
277 + }
278 +}
279 +
280 +#[derive(Debug, Clone, Copy, PartialEq, Eq)]
281 +pub enum LogicalOp {
282 + Conjunction,
283 + Disjunction,
284 +}
285 +
286 +#[derive(Debug)]
287 +pub struct JournalFilter {
288 + filter_expr: Option<FilterExpr>,
289 + current_matches: Vec<Vec<u8>>,
290 + current_op: LogicalOp,
291 +}
292 +
293 +impl Default for JournalFilter {
294 + fn default() -> Self {
295 + Self {
296 + filter_expr: None,
297 + current_matches: Vec::new(),
298 + current_op: LogicalOp::Conjunction,
299 + }
300 + }
301 +}
302 +
303 +impl JournalFilter {
304 + fn extract_key(kv_pair: &[u8]) -> Option<&[u8]> {
305 + if let Some(equal_pos) = kv_pair.iter().position(|&b| b == b'=') {
306 + Some(&kv_pair[..equal_pos])
307 + } else {
308 + None
309 + }
310 + }
311 +
312 + fn convert_current_matches<M: MemoryMap>(
313 + &mut self,
314 + journal_file: &JournalFile<M>,
315 + ) -> Result<Option<FilterExpr>> {
316 + if self.current_matches.is_empty() {
317 + return Ok(None);
318 + }
319 +
320 + let mut elements = Vec::new();
321 + let mut i = 0;
322 +
323 + while i < self.current_matches.len() {
324 + let current_key = Self::extract_key(&self.current_matches[i]).unwrap_or(&[]);
325 + let start = i;
326 +
327 + // Find all matches with the same key
328 + while i < self.current_matches.len()
329 + && Self::extract_key(&self.current_matches[i]).unwrap_or(&[]) == current_key
330 + {
331 + i += 1;
332 + }
333 +
334 + // If we have multiple values for this key, create a disjunction
335 + if i - start > 1 {
336 + let mut matches = Vec::with_capacity(i - start);
337 + for idx in start..i {
338 + let data = self.current_matches[idx].as_slice();
339 + let hash = journal_file.hash(data);
340 + let Some(offset) = journal_file.find_data_offset(hash, data)? else {
341 + // TODO: should we really return an error?
342 + return Err(JournalError::InvalidOffset);
343 + };
344 +
345 + let ic = journal_file.data_ref(offset)?.inlined_cursor();
346 + matches.push(FilterExpr::Match(offset.get(), ic));
347 + }
348 + elements.push(FilterExpr::Disjunction(matches));
349 + } else {
350 + let data = self.current_matches[start].as_slice();
351 + let hash = journal_file.hash(data);
352 + let Some(offset) = journal_file.find_data_offset(hash, data)? else {
353 + // TODO: should we really return an error?
354 + return Err(JournalError::InvalidOffset);
355 + };
356 +
357 + let ic = journal_file.data_ref(offset)?.inlined_cursor();
358 + elements.push(FilterExpr::Match(offset.get(), ic));
359 + }
360 + }
361 +
362 + self.current_matches.clear();
363 +
364 + match elements.len() {
365 + 0 => panic!("Could not create filter elements from current matches"),
366 + 1 => Ok(Some(elements.remove(0))),
367 + _ => Ok(Some(FilterExpr::Conjunction(elements))),
368 + }
369 + }
370 +
371 + pub fn add_match(&mut self, kv_pair: &[u8]) {
372 + if kv_pair.contains(&b'=') {
373 + let new_item = kv_pair.to_vec();
374 + let new_key = Self::extract_key(&new_item).unwrap_or(&[]);
375 +
376 + // Find the insertion position using binary search
377 + let pos = self
378 + .current_matches
379 + .binary_search_by(|item| {
380 + let key = Self::extract_key(item).unwrap_or(&[]);
381 + key.cmp(new_key)
382 + })
383 + .unwrap_or_else(|e| e);
384 +
385 + // Insert at the found position
386 + self.current_matches.insert(pos, new_item);
387 + }
388 + }
389 +
390 + pub fn set_operation<M: MemoryMap>(
391 + &mut self,
392 + journal_file: &JournalFile<M>,
393 + op: LogicalOp,
394 + ) -> Result<()> {
395 + let new_expr = self.convert_current_matches(journal_file)?;
396 + if new_expr.is_none() {
397 + self.current_op = op;
398 + return Ok(());
399 + }
400 +
401 + if self.filter_expr.is_none() {
402 + self.filter_expr = new_expr;
403 + self.current_op = op;
404 + return Ok(());
405 + }
406 +
407 + let new_expr = new_expr.unwrap();
408 + let current_expr = self.filter_expr.take().unwrap();
409 +
410 + self.filter_expr = Some(match (current_expr, self.current_op) {
411 + (FilterExpr::Disjunction(mut exprs), LogicalOp::Disjunction) => {
412 + exprs.push(new_expr);
413 + FilterExpr::Disjunction(exprs)
414 + }
415 + (FilterExpr::Conjunction(mut exprs), LogicalOp::Conjunction) => {
416 + exprs.push(new_expr);
417 + FilterExpr::Conjunction(exprs)
418 + }
419 + (current_expr, LogicalOp::Disjunction) => {
420 + FilterExpr::Disjunction(vec![current_expr, new_expr])
421 + }
422 + (current_expr, LogicalOp::Conjunction) => {
423 + FilterExpr::Conjunction(vec![current_expr, new_expr])
424 + }
425 + });
426 +
427 + self.current_op = op;
428 + Ok(())
429 + }
430 +
431 + pub fn build<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<FilterExpr> {
432 + self.set_operation(journal_file, self.current_op)?;
433 +
434 + self.current_matches.clear();
435 + self.current_op = LogicalOp::Conjunction;
436 + self.filter_expr.take().ok_or(JournalError::MalformedFilter)
437 + }
438 +}
src/crates/jf/journal_file/src/lib.rs
+29 -6
@@ -1,12 +1,35 @@
1 +// Modules - keep some public for advanced usage
2 +pub mod cursor;
3 +pub mod file;
4 +pub mod filter;
5 mod hash;
6 mod object;
3 -mod journal_file;
7 pub mod offset_array;
8 +pub mod reader;
9 mod value_guard;
10 +pub mod writer;
11
7 -pub use crate::hash::*;
8 -pub use error::Result;
12 +// Core functionality
13 +pub use file::{load_boot_id, BucketUtilization, JournalFile, JournalFileOptions};
14 +pub use reader::JournalReader;
15 +pub use writer::JournalWriter;
16 +
17 +// Essential types for working with readers
18 +pub use cursor::Location;
19 +pub use offset_array::Direction;
20 +
21 +// Advanced filtering (for users who need it)
22 +pub use cursor::JournalCursor;
23 +pub use filter::{FilterExpr, JournalFilter, LogicalOp};
24 +
25 +// For FFI compatibility and advanced object manipulation
26 +pub use object::HashableObject;
27 +
28 +// Re-export commonly needed external types
29 pub use memmap2::{Mmap, MmapMut};
10 -pub use object::*;
11 -pub use journal_file::{EntryDataIterator, FieldDataIterator, FieldIterator, JournalFile};
12 -pub use value_guard::ValueGuard;
30 +
31 +// Internal utilities that might be needed
32 +pub use crate::hash::journal_hash_data;
33 +
34 +// Internal re-exports needed by the crate itself (not part of public API)
35 +pub(crate) use object::*;
src/crates/jf/journal_file/src/object.rs
+285 -176
@@ -1,10 +1,9 @@
1 use crate::offset_array::{Cursor, InlinedCursor, List};
2 use error::{JournalError, Result};
3 -use std::fs::File;
4 -use std::num::{NonZeroU64, NonZeroUsize};
5 -use window_manager::MemoryMap;
3 +use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
4 use zerocopy::{
7 - ByteSlice, FromBytes, Immutable, IntoBytes, KnownLayout, Ref, SplitByteSlice, SplitByteSliceMut,
5 + ByteSlice, ByteSliceMut, FromBytes, Immutable, IntoBytes, KnownLayout, Ref, SplitByteSlice,
6 + SplitByteSliceMut,
7 };
8
9 pub trait HashableObject {
@@ -15,12 +14,145 @@ pub trait HashableObject {
14 fn get_payload(&self) -> &[u8];
15
16 /// Get the offset to the next object in the hash chain
18 - fn next_hash_offset(&self) -> u64;
17 + fn next_hash_offset(&self) -> Option<NonZeroU64>;
18
19 /// Get the object type
20 fn object_type() -> ObjectType;
21 }
22
23 +pub trait HashableObjectMut: HashableObject {
24 + /// Set the offset to the next object in the hash chain
25 + fn set_next_hash_offset(&mut self, offset: NonZeroU64);
26 +
27 + /// Set the payload of the object
28 + fn set_payload(&mut self, data: &[u8]);
29 +}
30 +
31 +/// Trait for hash table operations
32 +pub trait HashTable {
33 + /// The type of objects stored in this hash table
34 + type Object: HashableObject;
35 +
36 + /// Get the hash item for a given hash value
37 + fn hash_item_ref(&self, hash: u64) -> &HashItem;
38 +
39 + /// Get the length of the hash table (number of buckets)
40 + fn len(&self) -> usize;
41 +
42 + /// Make clippy happy
43 + fn is_empty(&self) -> bool {
44 + todo!()
45 + }
46 +}
47 +
48 +/// Trait for mutable hash table operations
49 +pub trait HashTableMut: HashTable {
50 + /// Get a mutable reference to the hash item for a given hash value
51 + fn hash_item_mut(&mut self, hash: u64) -> &mut HashItem;
52 +}
53 +
54 +pub struct DataHashTable<B: ByteSlice> {
55 + pub header: Ref<B, ObjectHeader>,
56 + pub items: Ref<B, [HashItem]>,
57 +}
58 +
59 +pub struct FieldHashTable<B: ByteSlice> {
60 + pub header: Ref<B, ObjectHeader>,
61 + pub items: Ref<B, [HashItem]>,
62 +}
63 +
64 +// Implement HashTable for DataHashTable
65 +impl<B: ByteSlice> HashTable for DataHashTable<B> {
66 + type Object = DataObject<B>;
67 +
68 + fn hash_item_ref(&self, hash: u64) -> &HashItem {
69 + let bucket_index = hash as usize % self.items.len();
70 + &self.items[bucket_index]
71 + }
72 +
73 + fn len(&self) -> usize {
74 + self.items.len()
75 + }
76 +}
77 +
78 +// Implement HashTable for FieldHashTable
79 +impl<B: ByteSlice> HashTable for FieldHashTable<B> {
80 + type Object = FieldObject<B>;
81 +
82 + fn hash_item_ref(&self, hash: u64) -> &HashItem {
83 + let bucket_index = hash as usize % self.items.len();
84 + &self.items[bucket_index]
85 + }
86 +
87 + fn len(&self) -> usize {
88 + self.items.len()
89 + }
90 +}
91 +
92 +// Implement HashTableMut for DataHashTable
93 +impl<B: ByteSliceMut> HashTableMut for DataHashTable<B> {
94 + fn hash_item_mut(&mut self, hash: u64) -> &mut HashItem {
95 + let bucket_index = hash as usize % self.items.len();
96 + &mut self.items[bucket_index]
97 + }
98 +}
99 +
100 +// Implement HashTableMut for FieldHashTable
101 +impl<B: ByteSliceMut> HashTableMut for FieldHashTable<B> {
102 + fn hash_item_mut(&mut self, hash: u64) -> &mut HashItem {
103 + let bucket_index = hash as usize % self.items.len();
104 + &mut self.items[bucket_index]
105 + }
106 +}
107 +
108 +// Implement JournalObject for DataHashTable
109 +impl<B: SplitByteSlice> JournalObject<B> for DataHashTable<B> {
110 + fn from_data(data: B, _is_compact: bool) -> Option<Self> {
111 + let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
112 +
113 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
114 + let items = zerocopy::Ref::from_bytes(items_data).ok()?;
115 +
116 + Some(DataHashTable { header, items })
117 + }
118 +}
119 +
120 +// Implement JournalObjectMut for DataHashTable
121 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for DataHashTable<B> {
122 + fn from_data_mut(data: B, _is_compact: bool) -> Option<Self> {
123 + let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
124 +
125 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
126 + let items = zerocopy::Ref::from_bytes(items_data).ok()?;
127 +
128 + Some(DataHashTable { header, items })
129 + }
130 +}
131 +
132 +// Implement JournalObject for FieldHashTable
133 +impl<B: SplitByteSlice> JournalObject<B> for FieldHashTable<B> {
134 + fn from_data(data: B, _is_compact: bool) -> Option<Self> {
135 + let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
136 +
137 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
138 + let items = zerocopy::Ref::from_bytes(items_data).ok()?;
139 +
140 + Some(FieldHashTable { header, items })
141 + }
142 +}
143 +
144 +// Implement JournalObjectMut for FieldHashTable
145 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for FieldHashTable<B> {
146 + fn from_data_mut(data: B, _is_compact: bool) -> Option<Self> {
147 + let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
148 +
149 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
150 + let items = zerocopy::Ref::from_bytes(items_data).ok()?;
151 +
152 + Some(FieldHashTable { header, items })
153 + }
154 +}
155 +
156 impl<B: ByteSlice> HashableObject for FieldObject<B> {
157 fn hash(&self) -> u64 {
158 self.header.hash
@@ -30,7 +162,7 @@ impl<B: ByteSlice> HashableObject for FieldObject<B> {
162 &self.payload
163 }
164
33 - fn next_hash_offset(&self) -> u64 {
165 + fn next_hash_offset(&self) -> Option<NonZeroU64> {
166 self.header.next_hash_offset
167 }
168
@@ -39,7 +171,17 @@ impl<B: ByteSlice> HashableObject for FieldObject<B> {
171 }
172 }
173
42 -impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> HashableObject for DataObject<B> {
174 +impl HashableObjectMut for FieldObject<&mut [u8]> {
175 + fn set_next_hash_offset(&mut self, next_hash_offset: NonZeroU64) {
176 + self.header.next_hash_offset = Some(next_hash_offset);
177 + }
178 +
179 + fn set_payload(&mut self, data: &[u8]) {
180 + self.payload.copy_from_slice(data);
181 + }
182 +}
183 +
184 +impl<B: ByteSlice> HashableObject for DataObject<B> {
185 fn hash(&self) -> u64 {
186 self.header.hash
187 }
@@ -48,7 +190,7 @@ impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> HashableObject for DataObj
190 self.payload_bytes()
191 }
192
51 - fn next_hash_offset(&self) -> u64 {
193 + fn next_hash_offset(&self) -> Option<NonZeroU64> {
194 self.header.next_hash_offset
195 }
196
@@ -57,15 +199,32 @@ impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> HashableObject for DataObj
199 }
200 }
201
202 +impl HashableObjectMut for DataObject<&mut [u8]> {
203 + fn set_next_hash_offset(&mut self, next_hash_offset: NonZeroU64) {
204 + self.header.next_hash_offset = Some(next_hash_offset);
205 + }
206 +
207 + fn set_payload(&mut self, data: &[u8]) {
208 + match &mut self.payload {
209 + DataPayloadType::Regular(payload) => {
210 + payload.copy_from_slice(data);
211 + }
212 + DataPayloadType::Compact { payload, .. } => {
213 + payload.copy_from_slice(data);
214 + }
215 + };
216 + }
217 +}
218 +
219 /// Trait to standardize creation of journal objects from byte slices
220 pub trait JournalObject<B: SplitByteSlice>: Sized {
221 /// Create a new journal object from a byte slice
222 fn from_data(data: B, is_compact: bool) -> Option<Self>;
223 }
224
66 -pub trait JournalObjectMut<B: SplitByteSliceMut>: Sized {
225 +pub trait JournalObjectMut<B: SplitByteSliceMut>: JournalObject<B> {
226 /// Create a new journal object from a byte slice
68 - fn from_data_mut(data: B, is_compact: bool) -> Self;
227 + fn from_data_mut(data: B, is_compact: bool) -> Option<Self>;
228 }
229
230 pub enum HeaderIncompatibleFlags {
@@ -104,30 +263,30 @@ impl TryFrom<u8> for JournalState {
263 #[derive(Default, Debug, Clone, Copy, FromBytes, IntoBytes, Immutable, KnownLayout)]
264 #[repr(C)]
265 pub struct JournalHeader {
107 - pub signature: [u8; 8], // "LPKSHHRH"
108 - pub compatible_flags: u32, // Compatible extension flags
109 - pub incompatible_flags: u32, // Incompatible extension flags
110 - pub state: u8, // File state (offline=0, online=1, archived=2)
111 - pub reserved: [u8; 7], // Reserved space
112 - pub file_id: [u8; 16], // Unique ID for this file
113 - pub machine_id: [u8; 16], // Machine ID this belongs to
114 - pub tail_entry_boot_id: [u8; 16], // Boot ID of the last entry
115 - pub seqnum_id: [u8; 16], // Sequence number ID
116 - pub header_size: u64, // Size of the header
117 - pub arena_size: u64, // Size of the data arena
118 - pub data_hash_table_offset: u64, // Offset of the data hash table
119 - pub data_hash_table_size: u64, // Size of the data hash table
120 - pub field_hash_table_offset: u64, // Offset of the field hash table
121 - pub field_hash_table_size: u64, // Size of the field hash table
122 - pub tail_object_offset: u64, // Offset of the last object
123 - pub n_objects: u64, // Number of objects
124 - pub n_entries: u64, // Number of entries
125 - pub tail_entry_seqnum: u64, // Sequence number of the last entry
126 - pub head_entry_seqnum: u64, // Sequence number of the first entry
127 - pub entry_array_offset: u64, // Offset of the entry array
128 - pub head_entry_realtime: u64, // Realtime timestamp of the first entry
129 - pub tail_entry_realtime: u64, // Realtime timestamp of the last entry
130 - pub tail_entry_monotonic: u64, // Monotonic timestamp of the last entry
266 + pub signature: [u8; 8], // "LPKSHHRH"
267 + pub compatible_flags: u32, // Compatible extension flags
268 + pub incompatible_flags: u32, // Incompatible extension flags
269 + pub state: u8, // File state (offline=0, online=1, archived=2)
270 + pub reserved: [u8; 7], // Reserved space
271 + pub file_id: [u8; 16], // Unique ID for this file
272 + pub machine_id: [u8; 16], // Machine ID this belongs to
273 + pub tail_entry_boot_id: [u8; 16], // Boot ID of the last entry
274 + pub seqnum_id: [u8; 16], // Sequence number ID
275 + pub header_size: u64, // Size of the header
276 + pub arena_size: u64, // Size of the data arena
277 + pub data_hash_table_offset: Option<NonZeroU64>, // Offset of the data hash table
278 + pub data_hash_table_size: Option<NonZeroU64>, // Size of the data hash table
279 + pub field_hash_table_offset: Option<NonZeroU64>, // Offset of the field hash table
280 + pub field_hash_table_size: Option<NonZeroU64>, // Size of the field hash table
281 + pub tail_object_offset: Option<NonZeroU64>, // Offset of the last object
282 + pub n_objects: u64, // Number of objects
283 + pub n_entries: u64, // Number of entries
284 + pub tail_entry_seqnum: u64, // Sequence number of the last entry
285 + pub head_entry_seqnum: u64, // Sequence number of the first entry
286 + pub entry_array_offset: Option<NonZeroU64>, // Offset of the entry array
287 + pub head_entry_realtime: u64, // Realtime timestamp of the first entry
288 + pub tail_entry_realtime: u64, // Realtime timestamp of the last entry
289 + pub tail_entry_monotonic: u64, // Monotonic timestamp of the last entry
290 }
291
292 /*
@@ -157,38 +316,6 @@ impl JournalHeader {
316 pub fn has_compatible_flag(&self, flag: HeaderCompatibleFlags) -> bool {
317 (self.compatible_flags & flag as u32) != 0
318 }
160 -
161 - fn map_hash_table<M: MemoryMap>(
162 - &self,
163 - file: &File,
164 - offset: u64,
165 - size: u64,
166 - ) -> Result<Option<M>> {
167 - if offset == 0 || size == 0 {
168 - return Ok(None);
169 - }
170 -
171 - let object_header_size = std::mem::size_of::<ObjectHeader>() as u64;
172 - if offset <= object_header_size || size < object_header_size {
173 - return Err(JournalError::InvalidObjectLocation);
174 - }
175 -
176 - let position = offset - object_header_size;
177 - let map_size = object_header_size + size;
178 - M::create(file, position, map_size).map(Some)
179 - }
180 -
181 - pub fn map_data_hash_table<M: MemoryMap>(&self, file: &File) -> Result<Option<M>> {
182 - self.map_hash_table(file, self.data_hash_table_offset, self.data_hash_table_size)
183 - }
184 -
185 - pub fn map_field_hash_table<M: MemoryMap>(&self, file: &File) -> Result<Option<M>> {
186 - self.map_hash_table(
187 - file,
188 - self.field_hash_table_offset,
189 - self.field_hash_table_size,
190 - )
191 - }
319 }
320
321 pub enum ObjectFlags {
@@ -264,49 +391,22 @@ impl ObjectHeader {
391 pub struct FieldObjectHeader {
392 pub object_header: ObjectHeader,
393 pub hash: u64,
267 - pub next_hash_offset: u64,
268 - pub head_data_offset: u64,
394 + pub next_hash_offset: Option<NonZeroU64>,
395 + pub head_data_offset: Option<NonZeroU64>,
396 }
397
398 #[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
399 #[repr(C)]
400 pub struct OffsetArrayObjectHeader {
401 pub object_header: ObjectHeader,
275 - pub next_offset_array: u64,
402 + pub next_offset_array: Option<NonZeroU64>,
403 }
404
405 #[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
406 #[repr(C)]
407 pub struct HashItem {
281 - pub head_hash_offset: u64,
282 - pub tail_hash_offset: u64,
283 -}
284 -
285 -pub struct HashTableObject<B: ByteSlice> {
286 - pub header: Ref<B, ObjectHeader>,
287 - pub items: Ref<B, [HashItem]>,
288 -}
289 -
290 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for HashTableObject<B> {
291 - fn from_data(data: B, _is_compact: bool) -> Option<Self> {
292 - let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
293 -
294 - let header = zerocopy::Ref::from_bytes(header_data).unwrap();
295 - let items = zerocopy::Ref::from_bytes(items_data).unwrap();
296 -
297 - Some(HashTableObject { header, items })
298 - }
299 -}
300 -
301 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for HashTableObject<B> {
302 - fn from_data_mut(data: B, _is_compact: bool) -> Self {
303 - let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).unwrap();
304 -
305 - let header = zerocopy::Ref::from_bytes(header_data).unwrap();
306 - let items = zerocopy::Ref::from_bytes(items_data).unwrap();
307 -
308 - HashTableObject { header, items }
309 - }
408 + pub head_hash_offset: Option<NonZeroU64>,
409 + pub tail_hash_offset: Option<NonZeroU64>,
410 }
411
412 #[derive(Debug)]
@@ -315,31 +415,39 @@ pub struct FieldObject<B: ByteSlice> {
415 pub payload: B,
416 }
417
318 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for FieldObject<B> {
418 +impl<B: SplitByteSlice> JournalObject<B> for FieldObject<B> {
419 fn from_data(data: B, _is_compact: bool) -> Option<Self> {
420 let (header, payload) = zerocopy::Ref::from_prefix(data).ok()?;
421 Some(FieldObject { header, payload })
422 }
423 }
424
325 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for FieldObject<B> {
326 - fn from_data_mut(data: B, _is_compact: bool) -> Self {
327 - let (header, payload) = zerocopy::Ref::from_prefix(data).unwrap();
328 -
329 - FieldObject { header, payload }
425 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for FieldObject<B> {
426 + fn from_data_mut(data: B, _is_compact: bool) -> Option<Self> {
427 + let (header, payload) = zerocopy::Ref::from_prefix(data).ok()?;
428 + Some(FieldObject { header, payload })
429 }
430 }
431
432 pub enum OffsetsType<B: ByteSlice> {
334 - Regular(Ref<B, [u64]>),
335 - Compact(Ref<B, [u32]>),
433 + Regular(Ref<B, [Option<NonZeroU64>]>),
434 + Compact(Ref<B, [Option<NonZeroU32>]>),
435 }
436
437 impl<B: ByteSlice> OffsetsType<B> {
339 - pub fn get(&self, index: usize) -> u64 {
438 + pub fn get(&self, index: usize) -> Option<NonZeroU64> {
439 match self {
440 OffsetsType::Regular(offsets) => offsets[index],
342 - OffsetsType::Compact(offsets) => offsets[index] as u64,
441 + OffsetsType::Compact(offsets) => offsets[index].map(NonZeroU64::from),
442 + }
443 + }
444 +}
445 +
446 +impl<B: ByteSliceMut> OffsetsType<B> {
447 + pub fn set(&mut self, index: usize, value: NonZeroU64) {
448 + match self {
449 + OffsetsType::Regular(offsets) => offsets[index] = Some(value),
450 + OffsetsType::Compact(offsets) => offsets[index] = NonZeroU32::new(value.get() as u32),
451 }
452 }
453 }
@@ -374,35 +482,22 @@ impl<B: ByteSlice> OffsetArrayObject<B> {
482 self.len(remaining_items) == 0
483 }
484
377 - pub fn get(&self, index: usize, remaining_items: usize) -> Result<u64> {
485 + pub fn get(&self, index: usize, remaining_items: usize) -> Result<Option<NonZeroU64>> {
486 if self.is_empty(remaining_items) {
487 return Err(JournalError::EmptyOffsetArrayNode);
488 }
489
382 - let offset = match &self.items {
383 - OffsetsType::Regular(items) => items[index],
384 - OffsetsType::Compact(items) => items[index] as u64,
385 - };
386 -
387 - if offset == 0 {
388 - Err(JournalError::InvalidOffsetArrayOffset)
389 - } else {
390 - Ok(offset)
391 - }
490 + Ok(self.items.get(index))
491 }
492 }
493
395 -impl<B: SplitByteSliceMut + std::fmt::Debug> OffsetArrayObject<B> {
494 +impl<B: ByteSliceMut> OffsetArrayObject<B> {
495 pub fn set(&mut self, index: usize, offset: NonZeroU64) -> Result<()> {
496 if index >= self.capacity() {
497 return Err(JournalError::OutOfBoundsIndex);
498 }
499
401 - match &mut self.items {
402 - OffsetsType::Regular(items) => items[index] = offset.get(),
403 - OffsetsType::Compact(items) => items[index] = offset.get() as u32,
404 - };
405 -
500 + self.items.set(index, offset);
501 Ok(())
502 }
503 }
@@ -415,7 +510,7 @@ impl<B: ByteSlice> std::fmt::Debug for OffsetArrayObject<B> {
510 }
511 }
512
418 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for OffsetArrayObject<B> {
513 +impl<B: SplitByteSlice> JournalObject<B> for OffsetArrayObject<B> {
514 fn from_data(data: B, is_compact: bool) -> Option<Self> {
515 let (header_data, items_data) = data
516 .split_at(std::mem::size_of::<OffsetArrayObjectHeader>())
@@ -438,26 +533,26 @@ impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for OffsetArrayObject
533 }
534 }
535
441 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for OffsetArrayObject<B> {
442 - fn from_data_mut(data: B, is_compact: bool) -> Self {
536 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for OffsetArrayObject<B> {
537 + fn from_data_mut(data: B, is_compact: bool) -> Option<Self> {
538 let (header_data, items_data) = data
539 .split_at(std::mem::size_of::<OffsetArrayObjectHeader>())
445 - .unwrap();
540 + .ok()?;
541
447 - let header = zerocopy::Ref::from_bytes(header_data).unwrap();
542 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
543
544 let items_type = if is_compact {
450 - let compact_items = zerocopy::Ref::from_bytes(items_data).unwrap();
545 + let compact_items = zerocopy::Ref::from_bytes(items_data).ok()?;
546 OffsetsType::Compact(compact_items)
547 } else {
453 - let regular_items = zerocopy::Ref::from_bytes(items_data).unwrap();
548 + let regular_items = zerocopy::Ref::from_bytes(items_data).ok()?;
549 OffsetsType::Regular(regular_items)
550 };
551
457 - OffsetArrayObject {
552 + Some(OffsetArrayObject {
553 header,
554 items: items_type,
460 - }
555 + })
556 }
557 }
558
@@ -492,6 +587,22 @@ pub enum EntryItemsType<B: ByteSlice> {
587 Compact(Ref<B, [CompactEntryItem]>),
588 }
589
590 +impl<B: ByteSliceMut> EntryItemsType<B> {
591 + pub fn set(&mut self, index: usize, object_offset: NonZeroU64, hash: Option<u64>) {
592 + match self {
593 + EntryItemsType::Regular(entry_items) => {
594 + entry_items[index].object_offset = object_offset.get();
595 + entry_items[index].hash = hash.unwrap();
596 + }
597 + EntryItemsType::Compact(entry_items) => {
598 + debug_assert!(hash.is_none());
599 + assert!(object_offset.get() < u32::MAX as u64);
600 + entry_items[index].object_offset = object_offset.get() as u32;
601 + }
602 + }
603 + }
604 +}
605 +
606 impl<B: ByteSlice> EntryItemsType<B> {
607 pub fn get(&self, index: usize) -> u64 {
608 match self {
@@ -538,7 +649,7 @@ impl<B: ByteSlice> std::fmt::Debug for EntryObject<B> {
649 }
650 }
651
541 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for EntryObject<B> {
652 +impl<B: SplitByteSlice> JournalObject<B> for EntryObject<B> {
653 fn from_data(data: B, is_compact: bool) -> Option<Self> {
654 let (header_data, items_data) = data
655 .split_at(std::mem::size_of::<EntryObjectHeader>())
@@ -561,26 +672,26 @@ impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for EntryObject<B> {
672 }
673 }
674
564 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for EntryObject<B> {
565 - fn from_data_mut(data: B, is_compact: bool) -> Self {
675 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for EntryObject<B> {
676 + fn from_data_mut(data: B, is_compact: bool) -> Option<Self> {
677 let (header_data, items_data) = data
678 .split_at(std::mem::size_of::<EntryObjectHeader>())
568 - .unwrap();
679 + .ok()?;
680
570 - let header = zerocopy::Ref::from_bytes(header_data).unwrap();
681 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
682
683 let items_type = if is_compact {
573 - let compact_items = zerocopy::Ref::from_bytes(items_data).unwrap();
684 + let compact_items = zerocopy::Ref::from_bytes(items_data).ok()?;
685 EntryItemsType::Compact(compact_items)
686 } else {
576 - let regular_items = zerocopy::Ref::from_bytes(items_data).unwrap();
687 + let regular_items = zerocopy::Ref::from_bytes(items_data).ok()?;
688 EntryItemsType::Regular(regular_items)
689 };
690
580 - EntryObject {
691 + Some(EntryObject {
692 header,
693 items: items_type,
583 - }
694 + })
695 }
696 }
697
@@ -589,11 +700,11 @@ impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for EntryObject
700 pub struct DataObjectHeader {
701 pub object_header: ObjectHeader,
702 pub hash: u64,
592 - pub next_hash_offset: u64,
593 - pub next_field_offset: u64,
594 - pub entry_offset: u64,
595 - pub entry_array_offset: u64,
596 - pub n_entries: u64,
703 + pub next_hash_offset: Option<NonZeroU64>,
704 + pub next_field_offset: Option<NonZeroU64>,
705 + pub entry_offset: Option<NonZeroU64>,
706 + pub entry_array_offset: Option<NonZeroU64>,
707 + pub n_entries: Option<NonZeroU64>,
708 }
709
710 impl DataObjectHeader {
@@ -614,21 +725,19 @@ impl DataObjectHeader {
725 }
726
727 pub fn inlined_cursor(&self) -> Option<InlinedCursor> {
617 - if self.n_entries == 0 {
618 - return None;
619 - }
620 -
621 - let inlined_offset = NonZeroU64::new(self.entry_offset)?;
622 - let cursor = self.entry_array_offset_list().map(Cursor::at_head);
728 + let inlined_offset = self.entry_offset?;
729 + let cursor = match self.n_entries?.get() {
730 + 1 => None,
731 + n => {
732 + let total_items = unsafe { NonZeroUsize::new_unchecked(n as usize - 1) };
733 + Some(Cursor::at_head(List::new(
734 + self.entry_array_offset?,
735 + total_items,
736 + )))
737 + }
738 + };
739 Some(InlinedCursor::new(inlined_offset, cursor))
740 }
625 -
626 - pub fn entry_array_offset_list(&self) -> Option<List> {
627 - let total_items = NonZeroUsize::new(self.n_entries.saturating_sub(1) as usize)?;
628 - let head_offset = NonZeroU64::new(self.entry_array_offset)?;
629 -
630 - Some(List::new(head_offset, total_items))
631 - }
741 }
742
743 #[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable, PartialEq, Eq)]
@@ -679,7 +788,7 @@ impl<B: ByteSlice> std::fmt::Debug for DataObject<B> {
788 }
789 }
790
682 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for DataObject<B> {
791 +impl<B: SplitByteSlice> JournalObject<B> for DataObject<B> {
792 fn from_data(data: B, is_compact: bool) -> Option<Self> {
793 let (header_data, remaining_data) = data
794 .split_at(std::mem::size_of::<DataObjectHeader>())
@@ -706,20 +815,20 @@ impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for DataObject<B> {
815 }
816 }
817
709 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for DataObject<B> {
710 - fn from_data_mut(data: B, is_compact: bool) -> Self {
818 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for DataObject<B> {
819 + fn from_data_mut(data: B, is_compact: bool) -> Option<Self> {
820 let (header_data, remaining_data) = data
821 .split_at(std::mem::size_of::<DataObjectHeader>())
713 - .unwrap();
822 + .ok()?;
823
715 - let header = zerocopy::Ref::from_bytes(header_data).unwrap();
824 + let header = zerocopy::Ref::from_bytes(header_data).ok()?;
825
826 let payload = if is_compact {
827 let (fields_data, payload_data) = remaining_data
828 .split_at(std::mem::size_of::<CompactDataFields>())
720 - .unwrap();
829 + .ok()?;
830
722 - let compact_fields = zerocopy::Ref::from_bytes(fields_data).unwrap();
831 + let compact_fields = zerocopy::Ref::from_bytes(fields_data).ok()?;
832
833 DataPayloadType::Compact {
834 compact_fields,
@@ -729,11 +838,11 @@ impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for DataObject<
838 DataPayloadType::Regular(remaining_data)
839 };
840
732 - DataObject { header, payload }
841 + Some(DataObject { header, payload })
842 }
843 }
844
736 -impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> DataObject<B> {
845 +impl<B: ByteSlice> DataObject<B> {
846 pub fn payload_bytes(&self) -> &[u8] {
847 match &self.payload {
848 DataPayloadType::Regular(payload) => payload,
@@ -806,21 +915,21 @@ impl<B: ByteSlice> std::fmt::Debug for TagObject<B> {
915 }
916 }
917
809 -impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for TagObject<B> {
918 +impl<B: SplitByteSlice> JournalObject<B> for TagObject<B> {
919 fn from_data(data: B, _is_compact: bool) -> Option<Self> {
920 let header = zerocopy::Ref::from_bytes(data).ok()?;
921 Some(TagObject { header })
922 }
923 }
924
816 -impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for TagObject<B> {
817 - fn from_data_mut(data: B, _is_compact: bool) -> Self {
818 - let header = zerocopy::Ref::from_bytes(data).unwrap();
819 - TagObject { header }
925 +impl<B: SplitByteSliceMut> JournalObjectMut<B> for TagObject<B> {
926 + fn from_data_mut(data: B, _is_compact: bool) -> Option<Self> {
927 + let header = zerocopy::Ref::from_bytes(data).ok()?;
928 + Some(TagObject { header })
929 }
930 }
931
823 -impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> TagObject<B> {
932 +impl<B: ByteSlice> TagObject<B> {
933 // Helper function to format tag as hex string
934 pub fn tag_as_hex(&self) -> String {
935 self.header
src/crates/jf/journal_file/src/offset_array.rs
+43 -25
@@ -1,4 +1,4 @@
1 -use crate::journal_file::JournalFile;
1 +use crate::file::JournalFile;
2 use error::{JournalError, Result};
3 use std::num::{NonZeroU64, NonZeroUsize};
4 use window_manager::MemoryMap;
@@ -25,21 +25,21 @@ impl Node {
25 offset: NonZeroU64,
26 remaining_items: NonZeroUsize,
27 ) -> Result<Self> {
28 - let array = journal_file.offset_array_ref(offset.get())?;
28 + let array = journal_file.offset_array_ref(offset)?;
29 let capacity =
30 NonZeroUsize::new(array.capacity()).ok_or(JournalError::EmptyOffsetArrayNode)?;
31
32 Ok(Self {
33 offset,
34 - next_offset: NonZeroU64::new(array.header.next_offset_array),
34 + next_offset: array.header.next_offset_array,
35 capacity,
36 remaining_items,
37 })
38 }
39
40 /// Get the offset of this array in the file
41 - pub fn offset(&self) -> u64 {
42 - self.offset.get()
41 + pub fn offset(&self) -> NonZeroU64 {
42 + self.offset
43 }
44
45 /// Get the maximum number of items this array can hold
@@ -74,12 +74,16 @@ impl Node {
74 }
75
76 /// Get an item at the specified index
77 - pub fn get<M: MemoryMap>(&self, journal_file: &JournalFile<M>, index: usize) -> Result<u64> {
77 + pub fn get<M: MemoryMap>(
78 + &self,
79 + journal_file: &JournalFile<M>,
80 + index: usize,
81 + ) -> Result<Option<NonZeroU64>> {
82 if index >= self.len().get() {
83 return Err(JournalError::InvalidOffsetArrayIndex);
84 }
85
82 - let array = journal_file.offset_array_ref(self.offset.get())?;
86 + let array = journal_file.offset_array_ref(self.offset)?;
87 array.get(index, self.remaining_items.get())
88 }
89
@@ -94,7 +98,7 @@ impl Node {
98 ) -> Result<usize>
99 where
100 M: MemoryMap,
97 - F: Fn(u64) -> Result<bool>,
101 + F: Fn(NonZeroU64) -> Result<bool>,
102 {
103 let mut left = left;
104 let mut right = right;
@@ -104,7 +108,9 @@ impl Node {
108
109 while left != right {
110 let mid = left.midpoint(right);
107 - let offset = self.get(journal_file, mid)?;
111 + let Some(offset) = self.get(journal_file, mid)? else {
112 + return Err(JournalError::InvalidOffset);
113 + };
114
115 if predicate(offset)? {
116 left = mid + 1;
@@ -127,7 +133,7 @@ impl Node {
133 ) -> Result<Option<usize>>
134 where
135 M: MemoryMap,
130 - F: Fn(u64) -> Result<bool>,
136 + F: Fn(NonZeroU64) -> Result<bool>,
137 {
138 let index = self.partition_point(journal_file, left, right, predicate)?;
139
@@ -229,7 +235,7 @@ impl List {
235 ) -> Result<Option<Cursor>>
236 where
237 M: MemoryMap,
232 - F: Fn(u64) -> Result<bool>,
238 + F: Fn(NonZeroU64) -> Result<bool>,
239 {
240 let mut last_cursor: Option<Cursor> = None;
241
@@ -361,7 +367,7 @@ impl Cursor {
367 Node::new(journal_file, self.array_offset, self.remaining_items)
368 }
369
364 - pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<u64> {
370 + pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<NonZeroU64>> {
371 self.node(journal_file)?.get(journal_file, self.array_index)
372 }
373
@@ -538,10 +544,10 @@ impl InlinedCursor {
544 unreachable!();
545 }
546
541 - pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<u64> {
547 + pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<NonZeroU64>> {
548 // Case 1: We're at the inlined entry
549 if self.at_inlined_offset {
544 - return Ok(self.inlined_offset.get());
550 + return Ok(Some(self.inlined_offset));
551 }
552
553 // Case 2: We're in the entry array
@@ -555,9 +561,12 @@ impl InlinedCursor {
561 pub fn next_until<M: MemoryMap>(
562 &mut self,
563 journal_file: &JournalFile<M>,
558 - offset: u64,
559 - ) -> Result<Option<u64>> {
560 - let current_offset = self.value(journal_file)?;
564 + offset: NonZeroU64,
565 + ) -> Result<Option<NonZeroU64>> {
566 + let Some(current_offset) = self.value(journal_file)? else {
567 + return Ok(None);
568 + };
569 +
570 if current_offset >= offset {
571 return Ok(Some(current_offset));
572 }
@@ -565,7 +574,10 @@ impl InlinedCursor {
574 while let Some(ic) = self.next(journal_file)? {
575 *self = ic;
576
568 - let current_offset = self.value(journal_file)?;
577 + let Some(current_offset) = self.value(journal_file)? else {
578 + break;
579 + };
580 +
581 if current_offset >= offset {
582 return Ok(Some(current_offset));
583 }
@@ -577,9 +589,12 @@ impl InlinedCursor {
589 pub fn previous_until<M: MemoryMap>(
590 &mut self,
591 journal_file: &JournalFile<M>,
580 - offset: u64,
581 - ) -> Result<Option<u64>> {
582 - let current_offset = self.value(journal_file)?;
592 + offset: NonZeroU64,
593 + ) -> Result<Option<NonZeroU64>> {
594 + let Some(current_offset) = self.value(journal_file)? else {
595 + return Ok(None);
596 + };
597 +
598 if current_offset <= offset {
599 return Ok(Some(current_offset));
600 }
@@ -587,7 +602,10 @@ impl InlinedCursor {
602 while let Some(ic) = self.previous(journal_file)? {
603 *self = ic;
604
590 - let current_offset = ic.value(journal_file)?;
605 + let Some(current_offset) = ic.value(journal_file)? else {
606 + break;
607 + };
608 +
609 if current_offset <= offset {
610 return Ok(Some(current_offset));
611 }
@@ -604,7 +622,7 @@ impl InlinedCursor {
622 ) -> Result<Option<Self>>
623 where
624 M: MemoryMap,
607 - F: Fn(u64) -> Result<bool>,
625 + F: Fn(NonZeroU64) -> Result<bool>,
626 {
627 // Variables to track our best match
628 let mut best_match: Option<Self> = None;
@@ -612,12 +630,12 @@ impl InlinedCursor {
630 // Handle the inlined entry based on direction
631 match direction {
632 Direction::Forward => {
615 - if !predicate(self.inlined_offset.get())? {
633 + if !predicate(self.inlined_offset)? {
634 return Ok(Some(self.head()));
635 }
636 }
637 Direction::Backward => {
620 - if predicate(self.inlined_offset.get())? {
638 + if predicate(self.inlined_offset)? {
639 // If predicate is true for inlined entry and we're going backward,
640 // this is potentially our best match
641 best_match = Some(self.head());
src/crates/jf/journal_file/src/reader.rs new
+194
@@ -0,0 +1,194 @@
1 +use crate::{
2 + cursor::{JournalCursor, Location},
3 + file::{EntryDataIterator, FieldDataIterator, FieldIterator, JournalFile},
4 + filter::{JournalFilter, LogicalOp},
5 + object::{DataObject, FieldObject},
6 + offset_array::Direction,
7 + value_guard::ValueGuard,
8 +};
9 +use error::Result;
10 +use std::num::NonZeroU64;
11 +use window_manager::MemoryMap;
12 +
13 +pub struct JournalReader<'a, M: MemoryMap> {
14 + cursor: JournalCursor,
15 +
16 + filter: Option<JournalFilter>,
17 + field_iterator: Option<FieldIterator<'a, M>>,
18 + field_data_iterator: Option<FieldDataIterator<'a, M>>,
19 + entry_data_iterator: Option<EntryDataIterator<'a, M>>,
20 +
21 + field_guard: Option<ValueGuard<'a, FieldObject<&'a [u8]>>>,
22 + data_guard: Option<ValueGuard<'a, DataObject<&'a [u8]>>>,
23 +}
24 +
25 +impl<M: MemoryMap> std::fmt::Debug for JournalReader<'_, M> {
26 + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
27 + f.debug_struct("JournalReader")
28 + // .field("cursor", &self.cursor)
29 + .field("field_guard", &self.field_guard)
30 + .field("data_guard", &self.data_guard)
31 + .finish()
32 + }
33 +}
34 +
35 +impl<M: MemoryMap> Default for JournalReader<'_, M> {
36 + fn default() -> Self {
37 + Self {
38 + cursor: JournalCursor::new(),
39 + filter: None,
40 + field_iterator: None,
41 + field_data_iterator: None,
42 + entry_data_iterator: None,
43 + field_guard: None,
44 + data_guard: None,
45 + }
46 + }
47 +}
48 +
49 +impl<'a, M: MemoryMap> JournalReader<'a, M> {
50 + pub fn dump(&self, journal_file: &'a JournalFile<M>) -> Result<String> {
51 + if let Some(filter_expr) = self.cursor.filter_expr.as_ref() {
52 + filter_expr.dump(journal_file)
53 + } else {
54 + Ok(String::from("no filter expr"))
55 + }
56 + }
57 +
58 + pub fn set_location(&mut self, location: Location) {
59 + self.cursor.set_location(location)
60 + }
61 +
62 + pub fn step(&mut self, journal_file: &'a JournalFile<M>, direction: Direction) -> Result<bool> {
63 + self.drop_guards();
64 +
65 + if let Some(filter) = self.filter.as_mut() {
66 + let filter_expr = filter.build(journal_file)?;
67 + self.cursor.set_filter(filter_expr);
68 + self.filter = None;
69 + }
70 +
71 + self.cursor.step(journal_file, direction)
72 + }
73 +
74 + pub fn add_match(&mut self, data: &[u8]) {
75 + self.filter.get_or_insert_default().add_match(data);
76 + }
77 +
78 + pub fn add_conjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
79 + self.filter
80 + .get_or_insert_default()
81 + .set_operation(journal_file, LogicalOp::Conjunction)
82 + }
83 +
84 + pub fn add_disjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
85 + self.filter
86 + .get_or_insert_default()
87 + .set_operation(journal_file, LogicalOp::Disjunction)
88 + }
89 +
90 + pub fn flush_matches(&mut self) {
91 + self.cursor.clear_filter();
92 + self.filter = None;
93 + }
94 +
95 + pub fn get_realtime_usec(&self, journal_file: &'a JournalFile<M>) -> Result<u64> {
96 + let entry_offset = self.cursor.position()?;
97 + let entry_object = journal_file.entry_ref(entry_offset)?;
98 + Ok(entry_object.header.realtime)
99 + }
100 +
101 + pub fn get_seqnum(&self, journal_file: &'a JournalFile<M>) -> Result<(u64, [u8; 16])> {
102 + let entry_offset = self.cursor.position()?;
103 + let entry_object = journal_file.entry_ref(entry_offset)?;
104 + Ok((
105 + entry_object.header.seqnum,
106 + journal_file.journal_header_ref().seqnum_id,
107 + ))
108 + }
109 +
110 + pub fn get_entry_offset(&self) -> Result<NonZeroU64> {
111 + self.cursor.position()
112 + }
113 +
114 + fn drop_guards(&mut self) {
115 + self.field_guard.take();
116 + self.data_guard.take();
117 + }
118 +
119 + pub fn fields_restart(&mut self) {
120 + self.drop_guards();
121 + self.field_iterator = None;
122 + }
123 +
124 + pub fn fields_enumerate(
125 + &mut self,
126 + journal_file: &'a JournalFile<M>,
127 + ) -> Result<Option<&ValueGuard<'_, FieldObject<&'a [u8]>>>> {
128 + self.drop_guards();
129 +
130 + if self.field_iterator.is_none() {
131 + self.field_iterator = Some(journal_file.fields());
132 + }
133 +
134 + if let Some(iter) = &mut self.field_iterator {
135 + self.field_guard = iter.next().transpose()?;
136 + Ok(self.field_guard.as_ref())
137 + } else {
138 + Ok(None)
139 + }
140 + }
141 +
142 + pub fn field_data_query_unique(
143 + &mut self,
144 + journal_file: &'a JournalFile<M>,
145 + field_name: &'a [u8],
146 + ) -> Result<()> {
147 + self.drop_guards();
148 +
149 + self.field_data_iterator = Some(journal_file.field_data_objects(field_name)?);
150 + Ok(())
151 + }
152 +
153 + pub fn field_data_restart(&mut self) {
154 + self.drop_guards();
155 + }
156 +
157 + pub fn field_data_enumerate(
158 + &mut self,
159 + _: &'a JournalFile<M>,
160 + ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
161 + self.drop_guards();
162 +
163 + if let Some(iter) = &mut self.field_data_iterator {
164 + self.data_guard = iter.next().transpose()?;
165 + Ok(self.data_guard.as_ref())
166 + } else {
167 + Ok(None)
168 + }
169 + }
170 +
171 + pub fn entry_data_restart(&mut self) {
172 + self.drop_guards();
173 + self.entry_data_iterator = None;
174 + }
175 +
176 + pub fn entry_data_enumerate(
177 + &mut self,
178 + journal_file: &'a JournalFile<M>,
179 + ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
180 + self.drop_guards();
181 +
182 + if self.entry_data_iterator.is_none() {
183 + let entry_offset = self.cursor.position()?;
184 + self.entry_data_iterator = Some(journal_file.entry_data_objects(entry_offset)?);
185 + }
186 +
187 + if let Some(iter) = &mut self.entry_data_iterator {
188 + self.data_guard = iter.next().transpose()?;
189 + Ok(self.data_guard.as_ref())
190 + } else {
191 + Ok(None)
192 + }
193 + }
194 +}
src/crates/jf/journal_file/src/value_guard.rs
+42 -2
@@ -28,13 +28,22 @@ use std::ops::{Deref, DerefMut};
28 /// their underlying memory might have been repurposed.
29 #[derive(Debug)]
30 pub struct ValueGuard<'a, T> {
31 + offset: NonZeroU64,
32 value: T,
33 in_use_flag: &'a RefCell<bool>,
34 }
35
36 impl<'a, T> ValueGuard<'a, T> {
36 - pub fn new(value: T, in_use_flag: &'a RefCell<bool>) -> Self {
37 - Self { value, in_use_flag }
37 + pub fn new(offset: NonZeroU64, value: T, in_use_flag: &'a RefCell<bool>) -> Self {
38 + Self {
39 + offset,
40 + value,
41 + in_use_flag,
42 + }
43 + }
44 +
45 + pub fn offset(&self) -> NonZeroU64 {
46 + self.offset
47 }
48 }
49
@@ -57,3 +66,34 @@ impl<T> Drop for ValueGuard<'_, T> {
66 *self.in_use_flag.borrow_mut() = false;
67 }
68 }
69 +
70 +use crate::{HashableObject, HashableObjectMut};
71 +use std::num::NonZeroU64;
72 +
73 +impl<T: HashableObject> HashableObject for ValueGuard<'_, T> {
74 + fn hash(&self) -> u64 {
75 + self.value.hash()
76 + }
77 +
78 + fn get_payload(&self) -> &[u8] {
79 + self.value.get_payload()
80 + }
81 +
82 + fn next_hash_offset(&self) -> Option<NonZeroU64> {
83 + self.value.next_hash_offset()
84 + }
85 +
86 + fn object_type() -> crate::ObjectType {
87 + T::object_type()
88 + }
89 +}
90 +
91 +impl<T: HashableObjectMut> HashableObjectMut for ValueGuard<'_, T> {
92 + fn set_next_hash_offset(&mut self, offset: NonZeroU64) {
93 + self.value.set_next_hash_offset(offset);
94 + }
95 +
96 + fn set_payload(&mut self, data: &[u8]) {
97 + self.value.set_payload(data);
98 + }
99 +}
src/crates/jf/journal_file/src/writer.rs new
+700
@@ -0,0 +1,700 @@
1 +#![allow(unused_imports, dead_code)]
2 +
3 +use crate::{
4 + journal_hash_data, CompactEntryItem, DataHashTable, DataObject, DataObjectHeader,
5 + DataPayloadType, EntryObject, EntryObjectHeader, FieldHashTable, FieldObject,
6 + FieldObjectHeader, HashItem, HashTable, HashTableMut, HashableObject, HashableObjectMut,
7 + HeaderIncompatibleFlags, JournalFile, JournalFileOptions, JournalHeader, JournalState,
8 + ObjectHeader, ObjectType, RegularEntryItem,
9 +};
10 +use error::{JournalError, Result};
11 +use memmap2::MmapMut;
12 +use rand::{seq::IndexedRandom, Rng};
13 +use std::num::{NonZeroU64, NonZeroUsize};
14 +use std::path::Path;
15 +use window_manager::MemoryMapMut;
16 +use zerocopy::{FromBytes, IntoBytes};
17 +
18 +const OBJECT_ALIGNMENT: u64 = 8;
19 +
20 +#[derive(Debug, Clone, Copy)]
21 +struct EntryItem {
22 + offset: NonZeroU64,
23 + hash: u64,
24 +}
25 +
26 +pub struct JournalWriter {
27 + tail_object_offset: NonZeroU64,
28 + append_offset: NonZeroU64,
29 + next_seqnum: u64,
30 + num_written_objects: u64,
31 + entry_items: Vec<EntryItem>,
32 + first_entry_monotonic: Option<u64>,
33 +}
34 +
35 +impl JournalWriter {
36 + /// Get current file size in bytes
37 + pub fn current_file_size(&self) -> u64 {
38 + self.append_offset.get()
39 + }
40 +
41 + /// Get the monotonic timestamp of the first entry written to this file
42 + pub fn first_entry_monotonic(&self) -> Option<u64> {
43 + self.first_entry_monotonic
44 + }
45 +
46 + pub fn new(journal_file: &mut JournalFile<MmapMut>) -> Result<Self> {
47 + let (append_offset, next_seqnum) = {
48 + let header = journal_file.journal_header_ref();
49 +
50 + let Some(tail_object_offset) = header.tail_object_offset else {
51 + return Err(JournalError::InvalidMagicNumber);
52 + };
53 +
54 + let tail_object = journal_file.object_header_ref(tail_object_offset)?;
55 +
56 + (
57 + tail_object_offset.saturating_add(tail_object.size),
58 + header.tail_entry_seqnum + 1,
59 + )
60 + };
61 +
62 + Ok(Self {
63 + tail_object_offset: journal_file
64 + .journal_header_ref()
65 + .tail_object_offset
66 + .unwrap(),
67 + append_offset,
68 + next_seqnum,
69 + num_written_objects: 0,
70 + entry_items: Vec::with_capacity(128),
71 + first_entry_monotonic: None,
72 + })
73 + }
74 +
75 + pub fn add_entry(
76 + &mut self,
77 + journal_file: &mut JournalFile<MmapMut>,
78 + items: &[&[u8]],
79 + realtime: u64,
80 + monotonic: u64,
81 + boot_id: [u8; 16],
82 + ) -> Result<()> {
83 + let header = journal_file.journal_header_ref();
84 + assert!(header.has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash));
85 +
86 + // Write the data/field objects while computing the entry's xor-hash
87 + // and storing each data object's offset/hash
88 + let mut xor_hash = 0;
89 + {
90 + self.entry_items.clear();
91 + for payload in items {
92 + let offset = self.add_data(journal_file, payload)?;
93 + let hash = {
94 + let data_guard = journal_file.data_ref(offset)?;
95 + data_guard.hash()
96 + };
97 +
98 + let entry_item = EntryItem { offset, hash };
99 + self.entry_items.push(entry_item);
100 +
101 + xor_hash ^= journal_hash_data(payload, true, None);
102 + }
103 +
104 + self.entry_items
105 + .sort_unstable_by(|a, b| a.offset.cmp(&b.offset));
106 + self.entry_items.dedup_by(|a, b| a.offset == b.offset);
107 + }
108 +
109 + // write the entry itself
110 + let entry_offset = self.append_offset;
111 + let entry_size = {
112 + let size = Some(self.entry_items.len() as u64 * 16);
113 + let mut entry_guard = journal_file.entry_mut(entry_offset, size)?;
114 +
115 + entry_guard.header.seqnum = self.next_seqnum;
116 + entry_guard.header.xor_hash = xor_hash;
117 + entry_guard.header.boot_id = boot_id;
118 + entry_guard.header.monotonic = monotonic;
119 + entry_guard.header.realtime = realtime;
120 +
121 + // set each entry item
122 + for (index, entry_item) in self.entry_items.iter().enumerate() {
123 + entry_guard
124 + .items
125 + .set(index, entry_item.offset, Some(entry_item.hash));
126 + }
127 +
128 + entry_guard.header.object_header.aligned_size()
129 + };
130 + self.object_added(entry_offset, entry_size);
131 +
132 + self.append_to_entry_array(journal_file, entry_offset)?;
133 + for entry_item_index in 0..self.entry_items.len() {
134 + self.link_data_to_entry(journal_file, entry_offset, entry_item_index)?;
135 + }
136 +
137 + self.entry_added(
138 + journal_file.journal_header_mut(),
139 + realtime,
140 + monotonic,
141 + boot_id,
142 + );
143 +
144 + Ok(())
145 + }
146 +
147 + fn object_added(&mut self, object_offset: NonZeroU64, object_size: u64) {
148 + self.tail_object_offset = object_offset;
149 + self.append_offset = object_offset.saturating_add(object_size);
150 + self.num_written_objects += 1;
151 + }
152 +
153 + fn entry_added(
154 + &mut self,
155 + header: &mut JournalHeader,
156 + realtime: u64,
157 + monotonic: u64,
158 + boot_id: [u8; 16],
159 + ) {
160 + header.n_entries += 1;
161 + header.n_objects += self.num_written_objects;
162 + header.tail_object_offset = Some(self.tail_object_offset);
163 + header.arena_size = self.append_offset.get() - header.header_size;
164 +
165 + if header.head_entry_seqnum == 0 {
166 + header.head_entry_seqnum = self.next_seqnum;
167 + }
168 + if header.head_entry_realtime == 0 {
169 + header.head_entry_realtime = realtime;
170 + }
171 + if self.first_entry_monotonic.is_none() {
172 + self.first_entry_monotonic = Some(monotonic);
173 + }
174 +
175 + header.tail_entry_seqnum = self.next_seqnum;
176 + header.tail_entry_realtime = realtime;
177 + header.tail_entry_monotonic = monotonic;
178 + header.tail_entry_boot_id = boot_id;
179 +
180 + self.next_seqnum += 1;
181 + self.num_written_objects = 0;
182 + }
183 +
184 + fn add_data(
185 + &mut self,
186 + journal_file: &mut JournalFile<MmapMut>,
187 + payload: &[u8],
188 + ) -> Result<NonZeroU64> {
189 + let hash = journal_file.hash(payload);
190 +
191 + match journal_file.find_data_offset(hash, payload)? {
192 + Some(data_offset) => Ok(data_offset),
193 + None => {
194 + // We will have to write the new data object at the current
195 + // tail offset
196 + let data_offset = self.append_offset;
197 + let data_size = {
198 + let mut data_guard =
199 + journal_file.data_mut(data_offset, Some(payload.len() as u64))?;
200 +
201 + data_guard.header.hash = hash;
202 + data_guard.set_payload(payload);
203 + data_guard.header.object_header.aligned_size()
204 + };
205 +
206 + self.object_added(data_offset, data_size);
207 +
208 + // Update hash table
209 + journal_file.data_hash_table_set_tail_offset(hash, data_offset)?;
210 +
211 + // Add the field object, if we have any
212 + if let Some(equals_pos) = payload.iter().position(|&b| b == b'=') {
213 + let field_offset = self.add_field(journal_file, &payload[..equals_pos])?;
214 +
215 + // Link data object to the linked-list
216 + {
217 + let head_data_offset = {
218 + let field_guard = journal_file.field_ref(field_offset)?;
219 + field_guard.header.head_data_offset
220 + };
221 +
222 + let mut data_guard = journal_file.data_mut(data_offset, None)?;
223 + data_guard.header.next_field_offset = head_data_offset;
224 + }
225 +
226 + // Link field to the head of the linked list
227 + {
228 + let mut field_guard = journal_file.field_mut(field_offset, None)?;
229 + field_guard.header.head_data_offset = Some(data_offset);
230 + };
231 + }
232 +
233 + Ok(data_offset)
234 + }
235 + }
236 + }
237 +
238 + fn add_field(
239 + &mut self,
240 + journal_file: &mut JournalFile<MmapMut>,
241 + payload: &[u8],
242 + ) -> Result<NonZeroU64> {
243 + let hash = journal_file.hash(payload);
244 +
245 + match journal_file.find_field_offset(hash, payload)? {
246 + Some(field_offset) => Ok(field_offset),
247 + None => {
248 + // We will have to write the new field object at the current
249 + // tail offset
250 + let field_offset = self.append_offset;
251 + let field_size = {
252 + let mut field_guard =
253 + journal_file.field_mut(field_offset, Some(payload.len() as u64))?;
254 +
255 + field_guard.header.hash = hash;
256 + field_guard.set_payload(payload);
257 + field_guard.header.object_header.aligned_size()
258 + };
259 + self.object_added(field_offset, field_size);
260 +
261 + // Update hash table
262 + journal_file.field_hash_table_set_tail_offset(hash, field_offset)?;
263 +
264 + // Return the offset where we wrote the newly added data object
265 + Ok(field_offset)
266 + }
267 + }
268 + }
269 +
270 + fn allocate_new_array(
271 + &mut self,
272 + journal_file: &JournalFile<MmapMut>,
273 + capacity: NonZeroU64,
274 + ) -> Result<NonZeroU64> {
275 + // let new_capacity = previous_capacity.saturating_mul(NonZeroU64::new(2).unwrap());
276 +
277 + let array_offset = self.append_offset;
278 + let array_size = {
279 + let array_guard = journal_file.offset_array_mut(array_offset, Some(capacity))?;
280 +
281 + array_guard.header.object_header.aligned_size()
282 + };
283 + self.object_added(array_offset, array_size);
284 +
285 + Ok(array_offset)
286 + }
287 +
288 + fn append_to_entry_array(
289 + &mut self,
290 + journal_file: &mut JournalFile<MmapMut>,
291 + entry_offset: NonZeroU64,
292 + ) -> Result<()> {
293 + let entry_array_offset = journal_file.journal_header_ref().entry_array_offset;
294 +
295 + if entry_array_offset.is_none() {
296 + journal_file.journal_header_mut().entry_array_offset = {
297 + let array_offset =
298 + self.allocate_new_array(journal_file, NonZeroU64::new(4096).unwrap())?;
299 + let mut array_guard = journal_file.offset_array_mut(array_offset, None)?;
300 + array_guard.set(0, entry_offset)?;
301 + Some(array_offset)
302 + };
303 + } else {
304 + let tail_node = {
305 + let entry_list = journal_file
306 + .entry_list()
307 + .ok_or(JournalError::EmptyOffsetArrayList)?;
308 + entry_list.tail(journal_file)?
309 + };
310 +
311 + if tail_node.len() < tail_node.capacity() {
312 + let mut array_guard = journal_file.offset_array_mut(tail_node.offset(), None)?;
313 + array_guard.set(tail_node.len().get(), entry_offset)?;
314 + } else {
315 + let new_array_offset = {
316 + let new_capacity = tail_node.capacity().get().saturating_mul(2) as u64;
317 + let new_array_offset = self
318 + .allocate_new_array(journal_file, NonZeroU64::new(new_capacity).unwrap())?;
319 + let mut array_guard = journal_file.offset_array_mut(new_array_offset, None)?;
320 + array_guard.set(0, entry_offset)?;
321 +
322 + new_array_offset
323 + };
324 +
325 + // Link the old tail to the new array
326 + {
327 + let mut array_guard =
328 + journal_file.offset_array_mut(tail_node.offset(), None)?;
329 + array_guard.header.next_offset_array = Some(new_array_offset);
330 + }
331 + }
332 + }
333 +
334 + Ok(())
335 + }
336 +
337 + fn append_to_data_entry_array(
338 + &mut self,
339 + journal_file: &mut JournalFile<MmapMut>,
340 + mut array_offset: NonZeroU64,
341 + entry_offset: NonZeroU64,
342 + current_count: u64,
343 + ) -> Result<()> {
344 + // Navigate to the tail of the array chain
345 + let mut current_index = 0u64;
346 + #[allow(unused_assignments)]
347 + let mut tail_offset = array_offset;
348 +
349 + loop {
350 + let array_guard = journal_file.offset_array_ref(array_offset)?;
351 + let capacity = array_guard.capacity() as u64;
352 +
353 + if current_index + capacity >= current_count {
354 + // This is the tail array
355 + tail_offset = array_offset;
356 + break;
357 + }
358 +
359 + current_index += capacity;
360 +
361 + let Some(next_offset) = array_guard.header.next_offset_array else {
362 + // This shouldn't happen if counts are correct
363 + return Err(JournalError::InvalidOffsetArrayOffset);
364 + };
365 +
366 + array_offset = next_offset;
367 + }
368 +
369 + // Try to add to the tail array
370 + let tail_capacity = {
371 + let tail_guard = journal_file.offset_array_ref(tail_offset)?;
372 + tail_guard.capacity() as u64
373 + };
374 +
375 + let entries_in_tail = current_count - current_index;
376 +
377 + if entries_in_tail < tail_capacity {
378 + // There's space in the tail array
379 + let mut tail_guard = journal_file.offset_array_mut(tail_offset, None)?;
380 + tail_guard.set(entries_in_tail as usize, entry_offset)?;
381 + } else {
382 + // Need to create a new array
383 + let new_capacity = NonZeroU64::new(tail_capacity * 2).unwrap(); // Double the size
384 + let new_array_offset = self.allocate_new_array(journal_file, new_capacity)?;
385 +
386 + // Link the old tail to the new array
387 + let mut tail_guard = journal_file.offset_array_mut(tail_offset, None)?;
388 + tail_guard.header.next_offset_array = Some(new_array_offset);
389 + drop(tail_guard);
390 +
391 + // Add entry to the new array
392 + let mut new_array_guard = journal_file.offset_array_mut(new_array_offset, None)?;
393 + new_array_guard.set(0, entry_offset)?;
394 + }
395 +
396 + Ok(())
397 + }
398 +
399 + fn link_data_to_entry(
400 + &mut self,
401 + journal_file: &mut JournalFile<MmapMut>,
402 + entry_offset: NonZeroU64,
403 + entry_item_index: usize,
404 + ) -> Result<()> {
405 + let data_offset = self.entry_items[entry_item_index].offset;
406 + let mut data_guard = journal_file.data_mut(data_offset, None)?;
407 +
408 + match data_guard.header.n_entries {
409 + None => {
410 + data_guard.header.entry_offset = Some(entry_offset);
411 + data_guard.header.n_entries = NonZeroU64::new(1);
412 + }
413 + Some(n_entries) => {
414 + match n_entries.get() {
415 + 0 => {
416 + unreachable!();
417 + }
418 + 1 => {
419 + drop(data_guard);
420 +
421 + // Create new entry array with initial capacity
422 + let array_capacity = NonZeroU64::new(64).unwrap();
423 + let array_offset = self.allocate_new_array(journal_file, array_capacity)?;
424 +
425 + // Load new array and set its first entry offset
426 + {
427 + let mut array_guard =
428 + journal_file.offset_array_mut(array_offset, None)?;
429 + array_guard.set(0, entry_offset)?;
430 + }
431 +
432 + // Update data object to point to the array
433 + let mut data_guard = journal_file.data_mut(data_offset, None)?;
434 + data_guard.header.entry_array_offset = Some(array_offset);
435 + data_guard.header.n_entries = NonZeroU64::new(2);
436 + }
437 + x => {
438 + // There's already an entry array, append to it
439 + let current_count = x - 1;
440 + let array_offset = data_guard.header.entry_array_offset.unwrap();
441 +
442 + // Drop the data guard to avoid borrow conflicts
443 + drop(data_guard);
444 +
445 + // Find the tail of the entry array chain and append
446 + self.append_to_data_entry_array(
447 + journal_file,
448 + array_offset,
449 + entry_offset,
450 + current_count,
451 + )?;
452 +
453 + // Update the count
454 + let mut data_guard = journal_file.data_mut(data_offset, None)?;
455 + data_guard.header.n_entries = NonZeroU64::new(x + 1);
456 + }
457 + }
458 + }
459 + }
460 +
461 + Ok(())
462 + }
463 +}
464 +
465 +#[cfg(test)]
466 +mod tests {
467 + use super::*;
468 + use crate::{load_boot_id, Direction, JournalFile, JournalReader, Location};
469 + use memmap2::Mmap;
470 + use std::collections::HashMap;
471 + use tempfile::NamedTempFile;
472 +
473 + fn generate_uuid() -> [u8; 16] {
474 + use rand::Rng;
475 + let mut rng = rand::rng();
476 + rng.random()
477 + }
478 +
479 + #[test]
480 + fn test_write_and_read_journal_entries() -> Result<()> {
481 + // Create test data - a hash map with key/values to add to the journal file
482 + let mut test_data = HashMap::new();
483 + test_data.insert(
484 + "MESSAGE",
485 + vec!["Hello, world!", "Another message", "Final message"],
486 + );
487 + test_data.insert("PRIORITY", vec!["6", "4", "3"]);
488 + test_data.insert(
489 + "_SYSTEMD_UNIT",
490 + vec!["test.service", "other.service", "test.service"],
491 + );
492 + test_data.insert("_PID", vec!["1234", "5678", "9999"]);
493 +
494 + // Create a temporary file for the journal
495 + let temp_file = NamedTempFile::new().map_err(JournalError::Io).unwrap();
496 + let journal_path = temp_file.path();
497 +
498 + // Step 1: Create and write to the journal
499 + let boot_id = load_boot_id().unwrap_or([1; 16]); // Use real boot_id or fallback
500 + let num_entries = test_data.values().next().unwrap().len();
501 +
502 + let options = JournalFileOptions::new(
503 + generate_uuid(),
504 + generate_uuid(),
505 + generate_uuid(),
506 + generate_uuid(),
507 + );
508 + let mut journal_file = JournalFile::create(journal_path, options)?;
509 + let iterations = 5000;
510 + for _ in 0..iterations {
511 + let mut writer = JournalWriter::new(&mut journal_file)?;
512 +
513 + // Write entries to the journal
514 + for i in 0..num_entries {
515 + let mut entry_data = Vec::new();
516 +
517 + // Build the entry data for this index
518 + for (key, values) in &test_data {
519 + let kv_pair = format!("{}={}", key, values[i]);
520 + entry_data.push(kv_pair.into_bytes());
521 + }
522 +
523 + // Convert to slice references for the writer
524 + let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
525 +
526 + // Write the entry with timestamps
527 + let realtime = 1000000 + (i as u64 * 1000); // Mock realtime in microseconds
528 + let monotonic = 500000 + (i as u64 * 1000); // Mock monotonic time
529 +
530 + writer.add_entry(&mut journal_file, &entry_refs, realtime, monotonic, boot_id)?;
531 + }
532 + }
533 +
534 + // Step 2: Read back and verify the journal contents
535 + {
536 + let journal_file = JournalFile::<Mmap>::open(journal_path, 8 * 1024)?;
537 + let mut reader = JournalReader::default();
538 +
539 + let hdr = journal_file.journal_header_ref();
540 + println!("Header: {:#?}", hdr);
541 +
542 + // Start from the head
543 + reader.set_location(Location::Head);
544 +
545 + let mut entries_read = 0;
546 + while reader.step(&journal_file, Direction::Forward)? {
547 + println!("Reading entry {}", entries_read);
548 +
549 + // Verify timestamps
550 + let realtime = reader.get_realtime_usec(&journal_file)?;
551 + let expected_realtime = 1000000 + ((entries_read % 3) * 1000);
552 + assert_eq!(
553 + realtime, expected_realtime,
554 + "Realtime mismatch for entry {}",
555 + entries_read
556 + );
557 +
558 + let (seqnum, _seqnum_id) = reader.get_seqnum(&journal_file)?;
559 + assert_eq!(
560 + seqnum,
561 + entries_read + 1,
562 + "Sequence number mismatch for entry {}",
563 + entries_read
564 + );
565 +
566 + // Read all data for this entry
567 + let mut entry_fields = HashMap::new();
568 + reader.entry_data_restart();
569 +
570 + while let Some(data_guard) = reader.entry_data_enumerate(&journal_file)? {
571 + let payload = data_guard.payload_bytes();
572 + let payload_str = String::from_utf8_lossy(payload);
573 +
574 + if let Some(eq_pos) = payload_str.find('=') {
575 + let key = &payload_str[..eq_pos];
576 + let value = &payload_str[eq_pos + 1..];
577 + entry_fields.insert(key.to_string(), value.to_string());
578 + }
579 + }
580 +
581 + // Verify the data matches what we wrote
582 + for (key, values) in &test_data {
583 + let expected_value = &values[entries_read as usize % 3];
584 + let actual_value = entry_fields.get(*key).unwrap_or_else(|| {
585 + panic!("Missing key '{}' in entry {}", key, entries_read)
586 + });
587 +
588 + assert_eq!(
589 + actual_value, expected_value,
590 + "Value mismatch for key '{}' in entry {}",
591 + key, entries_read
592 + );
593 + }
594 +
595 + println!("Read entry {}", entries_read);
596 + entries_read += 1;
597 + }
598 +
599 + assert_eq!(
600 + entries_read as usize,
601 + num_entries * iterations,
602 + "Number of entries read doesn't match written"
603 + );
604 + }
605 +
606 + // Step 3: Test filtering by specific fields
607 + {
608 + let journal_file = JournalFile::<Mmap>::open(journal_path, 64 * 1024)?;
609 + let mut reader = JournalReader::default();
610 +
611 + // Test filtering by _SYSTEMD_UNIT=test.service
612 + reader.add_match(b"_SYSTEMD_UNIT=test.service");
613 + reader.set_location(Location::Head);
614 +
615 + let mut filtered_entries = 0;
616 + while reader.step(&journal_file, Direction::Forward)? {
617 + // Verify this entry actually contains the filter match
618 + reader.entry_data_restart();
619 + let mut found_match = false;
620 +
621 + while let Some(data_guard) = reader.entry_data_enumerate(&journal_file)? {
622 + let payload = data_guard.payload_bytes();
623 + if payload == b"_SYSTEMD_UNIT=test.service" {
624 + found_match = true;
625 + break;
626 + }
627 + }
628 +
629 + assert!(
630 + found_match,
631 + "Filtered entry doesn't contain the expected field"
632 + );
633 + filtered_entries += 1;
634 + }
635 +
636 + // Should find 2 entries with _SYSTEMD_UNIT=test.service (entries 0 and 2)
637 + assert_eq!(filtered_entries, 2, "Expected 2 filtered entries");
638 + }
639 +
640 + println!("✅ All tests passed!");
641 + Ok(())
642 + }
643 +
644 + #[test]
645 + fn test_field_enumeration() -> Result<()> {
646 + // Create a simple journal with known fields
647 + let temp_file = NamedTempFile::new().map_err(JournalError::Io)?;
648 + let journal_path = temp_file.path();
649 +
650 + let test_fields = vec!["MESSAGE", "PRIORITY", "_SYSTEMD_UNIT"];
651 + let boot_id = [1; 16];
652 +
653 + // Write a single entry with multiple fields
654 + {
655 + let options = JournalFileOptions::new(
656 + generate_uuid(),
657 + generate_uuid(),
658 + generate_uuid(),
659 + generate_uuid(),
660 + );
661 +
662 + let mut journal_file = JournalFile::create(journal_path, options)?;
663 + let mut writer = JournalWriter::new(&mut journal_file)?;
664 +
665 + let entry_data = vec![
666 + b"MESSAGE=Test message".as_slice(),
667 + b"PRIORITY=6".as_slice(),
668 + b"_SYSTEMD_UNIT=test.service".as_slice(),
669 + ];
670 +
671 + writer.add_entry(&mut journal_file, &entry_data, 1000000, 500000, boot_id)?;
672 + }
673 +
674 + // Read back and enumerate fields
675 + {
676 + let journal_file = JournalFile::<Mmap>::open(journal_path, 8 * 1024)?;
677 + let mut reader = JournalReader::default();
678 +
679 + let mut found_fields = Vec::new();
680 + reader.fields_restart();
681 +
682 + while let Some(field_guard) = reader.fields_enumerate(&journal_file)? {
683 + let field_name = String::from_utf8_lossy(field_guard.payload);
684 + found_fields.push(field_name.to_string());
685 + }
686 +
687 + // Verify all expected fields were found
688 + for expected_field in &test_fields {
689 + assert!(
690 + found_fields.contains(&expected_field.to_string()),
691 + "Expected field '{}' not found. Found: {:?}",
692 + expected_field,
693 + found_fields
694 + );
695 + }
696 + }
697 +
698 + Ok(())
699 + }
700 +}
src/crates/jf/journal_forwarder/Cargo.toml deleted
-14
@@ -1,14 +0,0 @@
1 -[package]
2 -name = "journal_forwarder"
3 -version.workspace = true
4 -edition.workspace = true
5 -
6 -[dependencies]
7 -error = { path = "../error" }
8 -journal_file = { path = "../journal_file" }
9 -journal_logger = { path = "../journal_logger" }
10 -journal_reader = { path = "../journal_reader" }
11 -window_manager = { path = "../window_manager" }
12 -memmap2 = { workspace = true }
13 -rand = { workspace = true }
14 -walkdir = { workspace = true }
src/crates/jf/journal_forwarder/src/main.rs deleted
-214
@@ -1,214 +0,0 @@
1 -use error::Result;
2 -use journal_file::JournalFile;
3 -use journal_logger::JournalLogger;
4 -use journal_reader::JournalReader;
5 -use memmap2::Mmap;
6 -use rand::seq::{IndexedRandom, SliceRandom};
7 -use std::io::Read;
8 -use std::path::{Path, PathBuf};
9 -use std::time::Duration;
10 -use walkdir::WalkDir;
11 -
12 -fn process_one_cycle() -> Result<()> {
13 - // Find all journal files in the specified directory
14 - let journal_dir = "/var/log/journal";
15 - let journal_files = find_journal_files(journal_dir)?;
16 - if journal_files.is_empty() {
17 - eprintln!("No journal files found in {}", journal_dir);
18 - return Ok(());
19 - }
20 -
21 - // Pick a random journal file
22 - let mut rng = rand::rng();
23 - let random_file = journal_files.choose(&mut rng).unwrap();
24 -
25 - println!("Selected journal file: {}", random_file.display());
26 -
27 - // Open the selected journal file
28 - let journal_file = JournalFile::<Mmap>::open(random_file, 4096)?;
29 -
30 - // Choose random entries
31 - let entries = select_random_entries(&journal_file, 50)?;
32 - println!("Selected {} entries", entries.len());
33 -
34 - // Split entries into batches
35 - let batches = split_into_batches(entries, 5);
36 -
37 - // Forward each batch with a delay
38 - for (i, batch) in batches.iter().enumerate() {
39 - if i > 0 {
40 - std::thread::sleep(Duration::from_millis(200));
41 - }
42 -
43 - let mut logger = journal_logger::JournalLogger::new(
44 - "/home/vk/opt/sd/netdata/usr/sbin/log2journal",
45 - "/home/vk/opt/sd/netdata/usr/sbin/systemd-cat-native",
46 - );
47 -
48 - println!("Forwarding batch {} with {} entries", i + 1, batch.len());
49 - forward_entries(batch, &mut logger)?;
50 - }
51 -
52 - Ok(())
53 -}
54 -
55 -fn find_journal_files(journal_dir: &str) -> Result<Vec<PathBuf>> {
56 - let mut journal_files = Vec::new();
57 -
58 - for entry in WalkDir::new(journal_dir)
59 - .follow_links(true)
60 - .into_iter()
61 - .filter_map(|e| e.ok())
62 - {
63 - let path = entry.path();
64 - if path.is_file() && is_journal_file(path) {
65 - journal_files.push(path.to_path_buf());
66 - }
67 - }
68 -
69 - Ok(journal_files)
70 -}
71 -
72 -fn is_journal_file(path: &Path) -> bool {
73 - // Check if file begins with 8 bytes "LPKSHHRH"
74 - if let Ok(mut file) = std::fs::File::open(path) {
75 - let mut buffer = [0u8; 8];
76 - if let Ok(bytes_read) = file.read(&mut buffer) {
77 - if bytes_read == 8 && buffer == *b"LPKSHHRH" {
78 - return true;
79 - }
80 - }
81 - }
82 - false
83 -}
84 -
85 -fn select_random_entries(
86 - journal_file: &JournalFile<Mmap>,
87 - max_entries: usize,
88 -) -> Result<Vec<EntryData>> {
89 - let mut rng = rand::rng();
90 -
91 - // Get total number of entries in the journal
92 - let total_entries = journal_file.journal_header_ref().n_entries as usize;
93 - if total_entries == 0 {
94 - return Ok(Vec::new());
95 - }
96 -
97 - // Determine how many entries to select (min of max_entries and total_entries)
98 - let num_to_select = std::cmp::min(max_entries, total_entries);
99 -
100 - // Create a reader to navigate the journal
101 - let mut reader = JournalReader::default();
102 - let mut entries = Vec::new();
103 -
104 - // Generate random indices without repetition
105 - let mut indices: Vec<usize> = (0..total_entries).collect();
106 - indices.shuffle(&mut rng);
107 - indices.truncate(num_to_select);
108 -
109 - // Collect entries at the random indices
110 - for idx in indices {
111 - // Reset to head
112 - reader.set_location(journal_reader::Location::Head);
113 -
114 - // Skip forward to the random index
115 - for _ in 0..idx {
116 - if !reader.step(journal_file, journal_reader::Direction::Forward)? {
117 - break;
118 - }
119 - }
120 -
121 - // Get the entry at the current position
122 - if let Ok(entry_offset) = reader.get_entry_offset() {
123 - if let Ok(entry_data) = extract_entry_data(journal_file, entry_offset) {
124 - entries.push(entry_data);
125 - }
126 - }
127 - }
128 -
129 - Ok(entries)
130 -}
131 -
132 -// Structure to hold entry data for forwarding
133 -struct EntryData {
134 - fields: Vec<(String, String)>,
135 -}
136 -
137 -fn extract_entry_data(journal_file: &JournalFile<Mmap>, entry_offset: u64) -> Result<EntryData> {
138 - let mut fields = Vec::new();
139 -
140 - // Iterate through all data objects for this entry
141 - for data_result in journal_file.entry_data_objects(entry_offset)? {
142 - let data_object = data_result?;
143 - let payload = data_object.payload_bytes();
144 -
145 - // Find the first '=' character to split field and value
146 - if let Some(equals_pos) = payload.iter().position(|&b| b == b'=') {
147 - let field = String::from_utf8_lossy(&payload[0..equals_pos]).to_string();
148 - let value = String::from_utf8_lossy(&payload[equals_pos + 1..]).to_string();
149 -
150 - // Skip certain internal fields that shouldn't be forwarded
151 - if field.starts_with("_") || field == "MESSAGE_ID" || field == "PRIORITY" {
152 - continue;
153 - }
154 -
155 - fields.push((field, value));
156 - }
157 - }
158 -
159 - Ok(EntryData { fields })
160 -}
161 -
162 -fn split_into_batches(entries: Vec<EntryData>, num_batches: usize) -> Vec<Vec<EntryData>> {
163 - let mut batches = Vec::new();
164 - let entries_per_batch = (entries.len() + num_batches - 1) / num_batches.max(1);
165 -
166 - let mut entries_iter = entries.into_iter();
167 -
168 - for _ in 0..num_batches {
169 - let mut batch = Vec::new();
170 - for _ in 0..entries_per_batch {
171 - if let Some(entry) = entries_iter.next() {
172 - batch.push(entry);
173 - } else {
174 - break;
175 - }
176 - }
177 -
178 - if !batch.is_empty() {
179 - batches.push(batch);
180 - }
181 - }
182 -
183 - batches
184 -}
185 -
186 -fn forward_entries(entries: &[EntryData], logger: &mut JournalLogger) -> Result<()> {
187 - // Add a special field to indicate this is a forwarded entry
188 - for entry in entries {
189 - // Add all fields from the original entry
190 - for (key, value) in &entry.fields {
191 - logger.add_field(key, value);
192 - }
193 -
194 - // Add our own marker field
195 - logger.add_field("JOURNAL_FORWARDER", "1");
196 -
197 - // Flush this entry to the journal
198 - logger.flush().map_err(error::JournalError::Io)?;
199 - }
200 -
201 - Ok(())
202 -}
203 -
204 -fn main() -> Result<()> {
205 - // Main loop
206 - loop {
207 - if let Err(e) = process_one_cycle() {
208 - eprintln!("Error during cycle: {:?}", e);
209 - }
210 -
211 - // Wait for the next cycle
212 - std::thread::sleep(Duration::from_secs(1));
213 - }
214 -}
src/crates/jf/journal_log/Cargo.toml renamed
+7 -5
@@ -1,12 +1,14 @@
1 [package]
2 -name = "journal_writer"
2 +name = "journal_log"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
7 -journal_file = { path = "../journal_file" }
8 error = { path = "../error" }
9 -window_manager = { path = "../window_manager" }
9 +journal_file = { path = "../journal_file" }
10 memmap2 = { workspace = true }
11 -zerocopy = { workspace = true }
12 -rand = { workspace = true }
11 +uuid = { version = "1.0", features = ["v4", "rng"] }
12 +
13 +[dev-dependencies]
14 +tempfile = { workspace = true }
src/crates/jf/journal_log/src/lib.rs new
+632
@@ -0,0 +1,632 @@
1 +use error::{JournalError, Result};
2 +use journal_file::{
3 + load_boot_id, BucketUtilization, JournalFile, JournalFileOptions, JournalWriter,
4 +};
5 +use memmap2::MmapMut;
6 +use std::cmp::Ordering;
7 +use std::ffi::OsStr;
8 +use std::path::{Path, PathBuf};
9 +use std::time::{Duration, SystemTime, UNIX_EPOCH};
10 +
11 +#[derive(Debug, Clone, PartialEq, Eq)]
12 +pub struct JournalFileInfo {
13 + pub path: PathBuf,
14 + pub timestamp: SystemTime,
15 + pub counter: u64,
16 + pub size: Option<u64>,
17 +}
18 +
19 +impl JournalFileInfo {
20 + pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
21 + let path = path.as_ref();
22 + let filename = path
23 + .file_name()
24 + .and_then(OsStr::to_str)
25 + .ok_or(JournalError::InvalidFilename)?;
26 +
27 + let (timestamp, counter) = Self::parse_filename(filename)?;
28 +
29 + Ok(Self {
30 + path: path.to_path_buf(),
31 + timestamp,
32 + counter,
33 + size: None,
34 + })
35 + }
36 +
37 + pub fn from_parts(
38 + timestamp: SystemTime,
39 + counter: u64,
40 + size: Option<u64>,
41 + ) -> Result<JournalFileInfo> {
42 + let duration = timestamp
43 + .duration_since(UNIX_EPOCH)
44 + .map_err(|_| JournalError::SystemTimeError)?;
45 + let micros = duration.as_secs() * 1_000_000 + duration.subsec_micros() as u64;
46 + let path = PathBuf::from(format!("journal-{}-{}.journal", micros, counter));
47 +
48 + Ok(Self {
49 + path,
50 + timestamp,
51 + counter,
52 + size,
53 + })
54 + }
55 +
56 + /// Parse timestamp and counter from filename
57 + /// Expected format: "journal-{timestamp_micros}-{counter}.journal"
58 + fn parse_filename(filename: &str) -> Result<(SystemTime, u64)> {
59 + let name = filename.strip_suffix(".journal").unwrap_or(filename);
60 +
61 + if let Some(stripped) = name.strip_prefix("journal-") {
62 + let parts: Vec<&str> = stripped.split('-').collect();
63 + if parts.len() == 2 {
64 + let timestamp_micros: u64 = parts[0]
65 + .parse()
66 + .map_err(|_| JournalError::InvalidFilename)?;
67 + let counter: u64 = parts[1]
68 + .parse()
69 + .map_err(|_| JournalError::InvalidFilename)?;
70 +
71 + let timestamp = UNIX_EPOCH + Duration::from_micros(timestamp_micros);
72 + return Ok((timestamp, counter));
73 + }
74 + }
75 +
76 + Err(JournalError::InvalidFilename)
77 + }
78 +
79 + /// Get file size, loading from filesystem if not cached
80 + pub fn get_size(&mut self) -> Result<u64> {
81 + if let Some(size) = self.size {
82 + Ok(size)
83 + } else {
84 + let metadata = std::fs::metadata(&self.path)?;
85 + let size = metadata.len();
86 + self.size = Some(size);
87 + Ok(size)
88 + }
89 + }
90 +}
91 +
92 +// Implement ordering based on counter (for detecting duplicates and ordering)
93 +impl Ord for JournalFileInfo {
94 + fn cmp(&self, other: &Self) -> Ordering {
95 + // Order by counter only - this enables duplicate detection
96 + self.counter.cmp(&other.counter)
97 + }
98 +}
99 +
100 +impl PartialOrd for JournalFileInfo {
101 + fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
102 + Some(self.cmp(other))
103 + }
104 +}
105 +
106 +/// Determines when an active journal file should be sealed
107 +#[derive(Debug, Copy, Clone, Default)]
108 +pub struct RotationPolicy {
109 + /// Maximum file size before rotating (in bytes)
110 + pub size_of_journal_file: Option<u64>,
111 + /// Maximum duration that entries in a single file can span
112 + pub duration_of_journal_file: Option<Duration>,
113 +}
114 +
115 +impl RotationPolicy {
116 + pub fn with_size_of_journal_file(mut self, size_of_journal_file: u64) -> Self {
117 + self.size_of_journal_file = Some(size_of_journal_file);
118 + self
119 + }
120 +
121 + pub fn with_duration_of_journal_file(mut self, duration_of_journal_file: Duration) -> Self {
122 + self.duration_of_journal_file = Some(duration_of_journal_file);
123 + self
124 + }
125 +}
126 +
127 +/// Retention policy - determines when files should be removed
128 +#[derive(Debug, Copy, Clone, Default)]
129 +pub struct RetentionPolicy {
130 + /// Maximum number of journal files to keep
131 + pub number_of_journal_files: Option<usize>,
132 + /// Maximum total size of all journal files (in bytes)
133 + pub size_of_journal_files: Option<u64>,
134 + /// Maximum age of files to keep
135 + pub duration_of_journal_files: Option<Duration>,
136 +}
137 +
138 +impl RetentionPolicy {
139 + pub fn with_number_of_journal_files(mut self, number_of_journal_files: usize) -> Self {
140 + self.number_of_journal_files = Some(number_of_journal_files);
141 + self
142 + }
143 +
144 + pub fn with_size_of_journal_files(mut self, size_of_journal_files: u64) -> Self {
145 + self.size_of_journal_files = Some(size_of_journal_files);
146 + self
147 + }
148 +
149 + pub fn with_duration_of_journal_files(mut self, duration_of_journal_files: Duration) -> Self {
150 + self.duration_of_journal_files = Some(duration_of_journal_files);
151 + self
152 + }
153 +}
154 +
155 +/// Configuration for journal directory management
156 +#[derive(Debug, Clone)]
157 +pub struct JournalDirectoryConfig {
158 + /// Directory path where journal files are stored
159 + pub directory: PathBuf,
160 + /// Policy for when to rotate active files
161 + pub rotation_policy: RotationPolicy,
162 + /// Policy for when to remove old files
163 + pub retention_policy: RetentionPolicy,
164 +}
165 +
166 +impl JournalDirectoryConfig {
167 + pub fn new(directory: impl Into<PathBuf>) -> Self {
168 + Self {
169 + directory: directory.into(),
170 + rotation_policy: RotationPolicy::default(),
171 + retention_policy: RetentionPolicy::default(),
172 + }
173 + }
174 +
175 + pub fn with_sealing_policy(mut self, policy: RotationPolicy) -> Self {
176 + self.rotation_policy = policy;
177 + self
178 + }
179 +
180 + pub fn with_retention_policy(mut self, policy: RetentionPolicy) -> Self {
181 + self.retention_policy = policy;
182 + self
183 + }
184 +}
185 +
186 +/// Manages a directory of journal files with automatic cleanup and sealing
187 +#[derive(Debug)]
188 +pub struct JournalDirectory {
189 + config: JournalDirectoryConfig,
190 + /// Files ordered by counter (oldest counter first)
191 + files: Vec<JournalFileInfo>,
192 + /// Next counter value for new files
193 + next_counter: u64,
194 + /// Cached total size of all files
195 + total_size: u64,
196 +}
197 +
198 +impl JournalDirectory {
199 + /// Scan the directory and load existing journal files
200 + pub fn with_config(config: JournalDirectoryConfig) -> Result<Self> {
201 + // Create directory if it does not already exist.
202 + if !config.directory.exists() {
203 + std::fs::create_dir_all(&config.directory)?;
204 + } else if !config.directory.is_dir() {
205 + return Err(JournalError::NotADirectory);
206 + }
207 +
208 + let mut journal_directory = Self {
209 + config,
210 + files: Vec::new(),
211 + next_counter: 0,
212 + total_size: 0,
213 + };
214 +
215 + // Read all .journal files from directory
216 + for entry in std::fs::read_dir(&journal_directory.config.directory)? {
217 + let entry = entry?;
218 + let file_path = entry.path();
219 +
220 + if file_path.extension() != Some(OsStr::new("journal")) {
221 + continue;
222 + }
223 +
224 + match JournalFileInfo::from_path(&file_path) {
225 + Ok(mut file_info) => {
226 + // Load the actual file size from filesystem
227 + let file_size = file_info.get_size().unwrap_or(0);
228 + journal_directory.total_size += file_size;
229 + journal_directory.next_counter =
230 + journal_directory.next_counter.max(file_info.counter + 1);
231 + journal_directory.files.push(file_info);
232 + }
233 + Err(_) => {
234 + // Skip files with invalid names
235 + continue;
236 + }
237 + }
238 + }
239 +
240 + // Sort files by counter to maintain order
241 + journal_directory.files.sort();
242 +
243 + Ok(journal_directory)
244 + }
245 +
246 + pub fn directory_path(&self) -> &Path {
247 + &self.config.directory
248 + }
249 +
250 + pub fn get_full_path(&self, file_info: &JournalFileInfo) -> PathBuf {
251 + if file_info.path.is_absolute() {
252 + file_info.path.clone()
253 + } else {
254 + self.config.directory.join(&file_info.path)
255 + }
256 + }
257 +
258 + // Get information about all the files in the journal directory
259 + pub fn files(&self) -> Vec<JournalFileInfo> {
260 + self.files.clone()
261 + }
262 +
263 + /// Add a new journal file to the directory representation
264 + pub fn new_file(&mut self, existing_file: Option<JournalFileInfo>) -> Result<JournalFileInfo> {
265 + let timestamp = SystemTime::now();
266 + let new_file = JournalFileInfo::from_parts(timestamp, self.next_counter, None)?;
267 +
268 + self.files.push(new_file.clone());
269 + self.next_counter += 1;
270 +
271 + if let Some(existing_file) = existing_file {
272 + self.total_size += existing_file.size.unwrap_or(0);
273 + }
274 +
275 + Ok(new_file)
276 + }
277 +
278 + /// Remove the oldest file (by counter) from both filesystem and tracking
279 + fn remove_oldest_file(&mut self) -> Result<()> {
280 + if let Some(oldest_file) = self.files.first() {
281 + let file_path = self.get_full_path(oldest_file);
282 + let file_size = oldest_file.size.unwrap_or(0);
283 +
284 + // Remove from filesystem
285 + if let Err(e) = std::fs::remove_file(&file_path) {
286 + // Log error but continue cleanup - file might already be deleted
287 + eprintln!(
288 + "Warning: Failed to remove journal file {:?}: {}",
289 + file_path, e
290 + );
291 + }
292 +
293 + // Remove from tracking and update total size
294 + self.files.remove(0);
295 + self.total_size = self.total_size.saturating_sub(file_size);
296 + }
297 +
298 + Ok(())
299 + }
300 +
301 + /// Remove files older than the specified cutoff time
302 + fn remove_files_older_than(&mut self, cutoff_time: SystemTime) -> Result<()> {
303 + // Find files older than cutoff time
304 + let mut files_to_remove = Vec::new();
305 + for (index, file) in self.files.iter().enumerate() {
306 + if file.timestamp <= cutoff_time {
307 + files_to_remove.push(index);
308 + }
309 + }
310 +
311 + // Remove files in reverse order to maintain indices
312 + for &index in files_to_remove.iter().rev() {
313 + let file = &self.files[index];
314 + let file_path = self.get_full_path(file);
315 + let file_size = file.size.unwrap_or(0);
316 +
317 + // Remove from filesystem
318 + if let Err(e) = std::fs::remove_file(&file_path) {
319 + // Log error but continue cleanup
320 + eprintln!(
321 + "Warning: Failed to remove journal file {:?}: {}",
322 + file_path, e
323 + );
324 + }
325 +
326 + // Remove from tracking and update total size
327 + self.files.remove(index);
328 + self.total_size = self.total_size.saturating_sub(file_size);
329 + }
330 +
331 + Ok(())
332 + }
333 +
334 + /// Enforce the retention policy by removing old files
335 + pub fn enforce_retention_policy(&mut self) -> Result<()> {
336 + let policy = self.config.retention_policy;
337 +
338 + // 1. Remove by file count limit
339 + if let Some(max_files) = policy.number_of_journal_files {
340 + while self.files.len() > max_files {
341 + self.remove_oldest_file()?;
342 + }
343 + }
344 +
345 + // 2. Remove by total size limit
346 + if let Some(max_total_size) = policy.size_of_journal_files {
347 + while self.total_size > max_total_size && !self.files.is_empty() {
348 + self.remove_oldest_file()?;
349 + }
350 + }
351 +
352 + // 3. Remove by entry age limit
353 + if let Some(max_entry_age) = policy.duration_of_journal_files {
354 + let cutoff_time = SystemTime::now()
355 + .checked_sub(max_entry_age)
356 + .unwrap_or(SystemTime::UNIX_EPOCH);
357 + self.remove_files_older_than(cutoff_time)?;
358 + }
359 +
360 + Ok(())
361 + }
362 +}
363 +
364 +fn generate_uuid() -> [u8; 16] {
365 + uuid::Uuid::new_v4().into_bytes()
366 +}
367 +
368 +/// Configuration for JournalLog
369 +#[derive(Debug, Clone)]
370 +pub struct JournalLogConfig {
371 + /// Directory where journal files are stored
372 + pub journal_dir: PathBuf,
373 + /// Policy for when to rotate active files
374 + pub rotation_policy: RotationPolicy,
375 + /// Policy for when to remove old files
376 + pub retention_policy: RetentionPolicy,
377 +}
378 +
379 +impl JournalLogConfig {
380 + pub fn new(journal_dir: impl Into<PathBuf>) -> Self {
381 + Self {
382 + journal_dir: journal_dir.into(),
383 + rotation_policy: RotationPolicy::default()
384 + .with_size_of_journal_file(100 * 1024 * 1024) // 100MB
385 + .with_duration_of_journal_file(Duration::from_secs(2 * 3600)), // 2 hours
386 + retention_policy: RetentionPolicy::default()
387 + .with_number_of_journal_files(10) // 10 files
388 + .with_size_of_journal_files(1024 * 1024 * 1024) // 1GB
389 + .with_duration_of_journal_files(Duration::from_secs(7 * 24 * 3600)), // 7 days
390 + }
391 + }
392 +
393 + pub fn with_rotation_policy(mut self, policy: RotationPolicy) -> Self {
394 + self.rotation_policy = policy;
395 + self
396 + }
397 +
398 + pub fn with_retention_policy(mut self, policy: RetentionPolicy) -> Self {
399 + self.retention_policy = policy;
400 + self
401 + }
402 +}
403 +
404 +pub struct JournalLog {
405 + directory: JournalDirectory,
406 + current_file: Option<JournalFile<MmapMut>>,
407 + current_writer: Option<JournalWriter>,
408 + current_file_info: Option<JournalFileInfo>,
409 + machine_id: [u8; 16],
410 + boot_id: [u8; 16],
411 + seqnum_id: [u8; 16],
412 + previous_bucket_utilization: Option<BucketUtilization>,
413 +}
414 +
415 +/// Calculate optimal bucket sizes based on previous file utilization or rotation policy
416 +fn calculate_bucket_sizes(
417 + previous_utilization: Option<&BucketUtilization>,
418 + rotation_policy: &RotationPolicy,
419 +) -> (usize, usize) {
420 + if let Some(utilization) = previous_utilization {
421 + let data_utilization = utilization.data_utilization();
422 + let field_utilization = utilization.field_utilization();
423 +
424 + let data_buckets = if data_utilization > 0.75 {
425 + (utilization.data_total * 2).next_power_of_two()
426 + } else if data_utilization < 0.25 && utilization.data_total > 4096 {
427 + (utilization.data_total / 2).next_power_of_two()
428 + } else {
429 + utilization.data_total
430 + };
431 +
432 + let field_buckets = if field_utilization > 0.75 {
433 + (utilization.field_total * 2).next_power_of_two()
434 + } else if field_utilization < 0.25 && utilization.field_total > 512 {
435 + (utilization.field_total / 2).next_power_of_two()
436 + } else {
437 + utilization.field_total
438 + };
439 +
440 + (data_buckets, field_buckets)
441 + } else {
442 + // Initial sizing based on rotation policy max file size
443 + let max_file_size = rotation_policy
444 + .size_of_journal_file
445 + .unwrap_or(8 * 1024 * 1024);
446 +
447 + // 16 MiB -> 4096 data buckets
448 + let data_buckets = (max_file_size / 4096).max(1024).next_power_of_two() as usize;
449 + let field_buckets = 128; // Assume ~8:1 data:field ratio
450 +
451 + (data_buckets, field_buckets)
452 + }
453 +}
454 +
455 +impl JournalLog {
456 + pub fn new(config: JournalLogConfig) -> Result<Self> {
457 + let journal_config = JournalDirectoryConfig::new(&config.journal_dir)
458 + .with_sealing_policy(config.rotation_policy)
459 + .with_retention_policy(config.retention_policy);
460 +
461 + let mut directory = JournalDirectory::with_config(journal_config)?;
462 +
463 + // Enforce retention policy on startup to clean up any old files
464 + directory.enforce_retention_policy()?;
465 +
466 + let machine_id = journal_file::file::load_machine_id()?;
467 + let boot_id = load_boot_id()?;
468 + // TODO: Use NETDATA_INVOCATION_ID
469 + let seqnum_id = generate_uuid();
470 +
471 + Ok(JournalLog {
472 + directory,
473 + current_file: None,
474 + current_writer: None,
475 + current_file_info: None,
476 + machine_id,
477 + boot_id,
478 + seqnum_id,
479 + previous_bucket_utilization: None,
480 + })
481 + }
482 +
483 + fn ensure_active_journal(&mut self) -> Result<()> {
484 + // Check if rotation is needed before writing
485 + if let Some(writer) = &self.current_writer {
486 + if self.should_rotate(writer) {
487 + self.rotate_current_file()?;
488 + }
489 + }
490 +
491 + if self.current_file.is_none() {
492 + // Create a new journal file
493 + let file_info = self.directory.new_file(None)?;
494 +
495 + // Get the full path for the journal file
496 + let file_path = self.directory.get_full_path(&file_info);
497 +
498 + // Calculate optimal bucket sizes based on previous file utilization
499 + let (data_buckets, field_buckets) = calculate_bucket_sizes(
500 + self.previous_bucket_utilization.as_ref(),
501 + &self.directory.config.rotation_policy,
502 + );
503 +
504 + let options = JournalFileOptions::new(
505 + self.machine_id,
506 + self.boot_id,
507 + self.seqnum_id,
508 + generate_uuid(),
509 + )
510 + .with_window_size(8 * 1024 * 1024)
511 + .with_data_hash_table_buckets(data_buckets)
512 + .with_field_hash_table_buckets(field_buckets)
513 + .with_keyed_hash(true);
514 +
515 + let mut journal_file = JournalFile::create(&file_path, options)?;
516 + let writer = JournalWriter::new(&mut journal_file)?;
517 +
518 + self.current_file = Some(journal_file);
519 + self.current_writer = Some(writer);
520 + self.current_file_info = Some(file_info);
521 +
522 + // Enforce retention policy after creating new file to account for the new file count
523 + self.directory.enforce_retention_policy()?;
524 + }
525 +
526 + Ok(())
527 + }
528 +
529 + /// Checks if we have to rotate. Prioritizes file size over file creation
530 + /// time.
531 + fn should_rotate(&self, writer: &JournalWriter) -> bool {
532 + let policy = self.directory.config.rotation_policy;
533 +
534 + // Check if the file size went over the limit
535 + if let Some(max_size) = policy.size_of_journal_file {
536 + if writer.current_file_size() >= max_size {
537 + return true;
538 + }
539 + }
540 +
541 + // Check if the time span between first and last entries exceeds the limit
542 + let Some(file) = &self.current_file else {
543 + return false;
544 + };
545 + let Some(max_entry_span) = policy.duration_of_journal_file else {
546 + return false;
547 + };
548 + let Some(first_monotonic) = writer.first_entry_monotonic() else {
549 + return false;
550 + };
551 +
552 + let header = file.journal_header_ref();
553 + let last_monotonic = header.tail_entry_monotonic;
554 +
555 + // Convert monotonic timestamps (microseconds) to duration
556 + let entry_span = if last_monotonic >= first_monotonic {
557 + Duration::from_micros(last_monotonic - first_monotonic)
558 + } else {
559 + return false;
560 + };
561 +
562 + if entry_span >= max_entry_span {
563 + return true;
564 + }
565 +
566 + false
567 + }
568 +
569 + fn rotate_current_file(&mut self) -> Result<()> {
570 + // Capture bucket utilization before closing the file
571 + if let Some(file) = &self.current_file {
572 + self.previous_bucket_utilization = file.bucket_utilization();
573 + }
574 +
575 + // Update the current file's size in our tracking before closing
576 + if let (Some(file_info), Some(writer)) = (&mut self.current_file_info, &self.current_writer)
577 + {
578 + let current_size = writer.current_file_size();
579 + file_info.size = Some(current_size);
580 +
581 + // Update the size in the directory's file list
582 + if let Some(tracked_file) = self
583 + .directory
584 + .files
585 + .iter_mut()
586 + .find(|f| f.counter == file_info.counter)
587 + {
588 + let old_size = tracked_file.size.unwrap_or(0);
589 + tracked_file.size = Some(current_size);
590 +
591 + // Update total size tracking
592 + self.directory.total_size = self
593 + .directory
594 + .total_size
595 + .saturating_sub(old_size)
596 + .saturating_add(current_size);
597 + }
598 + }
599 +
600 + // Close current file
601 + self.current_file = None;
602 + self.current_writer = None;
603 + self.current_file_info = None;
604 +
605 + // Next call to ensure_active_journal() will create new file
606 + Ok(())
607 + }
608 +
609 + pub fn write_entry(&mut self, items: &[&[u8]]) -> Result<()> {
610 + if items.is_empty() {
611 + return Ok(());
612 + }
613 +
614 + self.ensure_active_journal()?;
615 +
616 + let journal_file = self.current_file.as_mut().unwrap();
617 + let writer = self.current_writer.as_mut().unwrap();
618 +
619 + let now = SystemTime::now();
620 + let realtime = now
621 + .duration_since(UNIX_EPOCH)
622 + .unwrap_or_default()
623 + .as_micros() as u64;
624 +
625 + // Use realtime for monotonic as well for simplicity
626 + let monotonic = realtime;
627 +
628 + writer.add_entry(journal_file, items, realtime, monotonic, self.boot_id)?;
629 +
630 + Ok(())
631 + }
632 +}
src/crates/jf/journal_logger/Cargo.toml deleted
-8
@@ -1,8 +0,0 @@
1 -[package]
2 -name = "journal_logger"
3 -version.workspace = true
4 -edition.workspace = true
5 -
6 -[dependencies]
7 -duct = { workspace = true }
8 -serde_json = { workspace = true }
src/crates/jf/journal_logger/src/lib.rs deleted
-69
@@ -1,69 +0,0 @@
1 -use duct::cmd;
2 -use std::collections::HashMap;
3 -use std::io::{self, Error, ErrorKind};
4 -use std::path::Path;
5 -
6 -/// A builder for creating systemd journal entries with custom fields
7 -pub struct JournalLogger {
8 - fields: HashMap<String, String>,
9 - log2journal_path: String,
10 - systemd_cat_path: String,
11 -}
12 -
13 -impl JournalLogger {
14 - /// Create a new JournalLogger with paths to the required executables
15 - pub fn new(log2journal_path: &str, systemd_cat_path: &str) -> Self {
16 - JournalLogger {
17 - fields: HashMap::new(),
18 - log2journal_path: log2journal_path.to_string(),
19 - systemd_cat_path: systemd_cat_path.to_string(),
20 - }
21 - }
22 -
23 - /// Add a field to the journal entry
24 - pub fn add_field(&mut self, key: &str, value: &str) -> &mut Self {
25 - self.fields.insert(key.to_string(), value.to_string());
26 - self
27 - }
28 -
29 - /// Flush the current fields to the journal and clear the fields
30 - pub fn flush(&mut self) -> io::Result<()> {
31 - // Verify that the required executables exist
32 - if !Path::new(&self.log2journal_path).exists() {
33 - return Err(Error::new(
34 - ErrorKind::NotFound,
35 - format!(
36 - "log2journal executable not found at {}",
37 - self.log2journal_path
38 - ),
39 - ));
40 - }
41 -
42 - if !Path::new(&self.systemd_cat_path).exists() {
43 - return Err(Error::new(
44 - ErrorKind::NotFound,
45 - format!(
46 - "systemd-cat executable not found at {}",
47 - self.systemd_cat_path
48 - ),
49 - ));
50 - }
51 -
52 - // Create the JSON string
53 - let json_data = serde_json::to_string(&self.fields)
54 - .map_err(|e| Error::new(ErrorKind::InvalidData, e))?;
55 -
56 - // Create the pipeline
57 - let pipeline = cmd!("echo", json_data)
58 - .pipe(cmd!(&self.log2journal_path, "json"))
59 - .pipe(cmd!(&self.systemd_cat_path));
60 -
61 - // Execute the pipeline
62 - pipeline.run()?;
63 -
64 - // Clear the fields for the next entry
65 - self.fields.clear();
66 -
67 - Ok(())
68 - }
69 -}
src/crates/jf/journal_reader/Cargo.toml
+1
@@ -2,6 +2,7 @@
2 name = "journal_reader"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 error = { path = "../error" }
src/crates/jf/journal_reader_ffi/Cargo.toml
+2 -1
@@ -2,10 +2,11 @@
2 name = "journal_reader_ffi"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 error = { path = "../error" }
8 -journal_reader = { path = "../journal_reader" }
9 +# journal_reader integrated into journal_file
10 journal_file = { path = "../journal_file" }
11 memmap2 = { workspace = true }
12 serde_json = { workspace = true }
src/crates/jf/journal_reader_ffi/src/lib.rs
+1 -2
@@ -1,5 +1,4 @@
1 -use journal_file::{HashableObject, JournalFile};
2 -use journal_reader::{Direction, JournalReader, Location};
1 +use journal_file::{Direction, HashableObject, JournalFile, JournalReader, Location};
2 use memmap2::Mmap;
3 use std::ffi::{c_char, c_int, c_void, CStr};
4
src/crates/jf/journal_writer/src/lib.rs deleted
-88
@@ -1,88 +0,0 @@
1 -// // #![allow(unused_imports, dead_code)]
2 -
3 -// use error::{JournalError, Result};
4 -// use journal_file::{
5 -// journal_hash_data, CompactEntryItem, DataObject, DataObjectHeader, DataPayloadType,
6 -// EntryObject, EntryObjectHeader, FieldObject, FieldObjectHeader, HashItem, HashTableObject,
7 -// HeaderIncompatibleFlags, JournalFile, JournalHeader, JournalState, ObjectHeader, ObjectType,
8 -// RegularEntryItem,
9 -// };
10 -// use memmap2::MmapMut;
11 -// use rand::{seq::IndexedRandom, Rng};
12 -// use std::num::NonZeroU64;
13 -// use std::path::Path;
14 -// use window_manager::MemoryMapMut;
15 -// use zerocopy::{FromBytes, IntoBytes};
16 -
17 -// const OBJECT_ALIGNMENT: u64 = 8;
18 -
19 -// #[derive(Default)]
20 -// pub struct JournalWriter {
21 -// tail_offset: u64,
22 -// offsets_buffer: Vec<u64>,
23 -// hash_buffer: Vec<u64>,
24 -// }
25 -
26 -// impl JournalWriter {
27 -// pub fn new(journal_file: &mut JournalFile<MmapMut>) -> Result<Self> {
28 -// Ok(Self {
29 -// tail_offset: 0,
30 -// offsets_buffer: Vec::with_capacity(128),
31 -// hash_buffer: Vec::with_capacity(128),
32 -// })
33 -// }
34 -
35 -// pub fn add_entry(
36 -// &mut self,
37 -// journal_file: &mut JournalFile<MmapMut>,
38 -// items: &[&[u8]],
39 -// realtime: u64,
40 -// monotonic: u64,
41 -// boot_id: [u8; 16],
42 -// ) -> Result<u64> {
43 -// let header = journal_file.journal_header_ref();
44 -
45 -// let is_keyed_hash = header.has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash);
46 -// let is_compact = header.has_incompatible_flag(HeaderIncompatibleFlags::Compact);
47 -// let file_id = header.file_id;
48 -
49 -// let mut current_offset = header.tail_object_offset;
50 -// if (current_offset == 0) || (current_offset % 8 != 0) {
51 -// return Err(JournalError::InvalidOffset);
52 -// }
53 -
54 -// let mut arena_size = header.arena_size;
55 -// if (current_offset == 0) || (current_offset % 8 != 0) {
56 -// return Err(JournalError::InvalidOffset);
57 -// }
58 -
59 -// // self.hash_buffer.clear();
60 -// // self.hash_buffer.extend(
61 -// // items
62 -// // .iter()
63 -// // .map(|item| journal_hash_data(item, is_keyed_hash, None)),
64 -// // );
65 -
66 -// // for payload in items.iter() {
67 -// // let hash = journal_file.hash(payload);
68 -// // match journal_file.find_data_offset_by_payload(payload, hash) {
69 -// // Ok(data_offset) => {
70 -// // self.offsets_buffer.push(data_offset);
71 -// // }
72 -// // Err(JournalError::MissingObjectFromHashTable) => {
73 -// // let size = payload.len() as u64;
74 -// // let data_object = journal_file.data_mut(current_offset, Some(size))?;
75 -
76 -// // current_offset += data_object.header.object_header.aligned_size();
77 -
78 -// // data_object.
79 -// // }
80 -// // Err(e) => {
81 -// // return Err(e);
82 -// // }
83 -// // };
84 -// // }
85 -
86 -// Ok(0)
87 -// }
88 -// }
src/crates/jf/main/Cargo.toml deleted
-15
@@ -1,15 +0,0 @@
1 -[package]
2 -name = "main"
3 -version.workspace = true
4 -edition.workspace = true
5 -
6 -[dependencies]
7 -error = { path = "../error" }
8 -journal_file = { path = "../journal_file" }
9 -journal_logger = { path = "../journal_logger" }
10 -journal_reader = { path = "../journal_reader" }
11 -sigbus = { path = "../sigbus" }
12 -window_manager = { path = "../window_manager" }
13 -rand = { workspace = true }
14 -systemd = { workspace = true }
15 -zerocopy = { workspace = true }
src/crates/jf/main/src/main.rs deleted
-669
@@ -1,669 +0,0 @@
1 -#![allow(dead_code)]
2 -
3 -use error::Result;
4 -use journal_file::*;
5 -use journal_reader::{Direction, JournalReader, Location};
6 -use std::collections::HashMap;
7 -use window_manager::MemoryMap;
8 -
9 -pub struct EntryData {
10 - pub offset: u64,
11 - pub realtime: u64,
12 - pub monotonic: u64,
13 - pub boot_id: String,
14 - pub seqnum: u64,
15 - pub fields: Vec<(String, String)>,
16 -}
17 -
18 -impl std::fmt::Debug for EntryData {
19 - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20 - // Start with a custom struct name
21 - write!(f, "@{:#x?} {{ fields: [", self.offset)?;
22 -
23 - // Iterate through fields and format each one
24 - for (i, (key, value)) in self.fields.iter().enumerate() {
25 - if i > 0 {
26 - write!(f, ", ")?;
27 - }
28 - write!(f, "({:?}, {:?})", key, value)?;
29 - }
30 -
31 - // Close the formatting
32 - write!(f, "] }}")
33 - }
34 -}
35 -
36 -impl EntryData {
37 - /// Extract all data from an entry into an owned structure
38 - pub fn from_offset<M: MemoryMap>(
39 - journal_file: &JournalFile<M>,
40 - entry_offset: u64,
41 - ) -> Result<EntryData> {
42 - // Get the entry object
43 - let entry_object = journal_file.entry_ref(entry_offset)?;
44 -
45 - // Extract basic information from the entry header
46 - let realtime = entry_object.header.realtime;
47 - let monotonic = entry_object.header.monotonic;
48 - let boot_id = format_uuid_bytes(&entry_object.header.boot_id);
49 - let seqnum = entry_object.header.seqnum;
50 -
51 - drop(entry_object);
52 -
53 - // Create a vector to hold all fields
54 - let mut fields = Vec::new();
55 -
56 - // Iterate through all data objects for this entry
57 - for data_result in journal_file.entry_data_objects(entry_offset)? {
58 - let data_object = data_result?;
59 - let payload = data_object.payload_bytes();
60 -
61 - // Find the first '=' character to split field and value
62 - if let Some(equals_pos) = payload.iter().position(|&b| b == b'=') {
63 - let field = String::from_utf8_lossy(&payload[0..equals_pos]).to_string();
64 - let value = String::from_utf8_lossy(&payload[equals_pos + 1..]).to_string();
65 -
66 - if field.starts_with("_") {
67 - continue;
68 - }
69 -
70 - fields.push((field, value));
71 - }
72 - }
73 -
74 - // Create and return the EntryData struct
75 - Ok(EntryData {
76 - offset: entry_offset,
77 - realtime,
78 - monotonic,
79 - boot_id,
80 - seqnum,
81 - fields,
82 - })
83 - }
84 -
85 - pub fn get_field(&self, name: &str) -> Option<&str> {
86 - self.fields
87 - .iter()
88 - .find(|(k, _)| k == name)
89 - .map(|(_, v)| v.as_str())
90 - }
91 -}
92 -
93 -fn format_uuid_bytes(bytes: &[u8; 16]) -> String {
94 - format!(
95 - "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
96 - bytes[0], bytes[1], bytes[2], bytes[3],
97 - bytes[4], bytes[5],
98 - bytes[6], bytes[7],
99 - bytes[8], bytes[9],
100 - bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]
101 - )
102 -}
103 -
104 -use systemd::journal;
105 -
106 -struct JournalWrapper<'a> {
107 - j: journal::Journal,
108 -
109 - jr: JournalReader<'a, Mmap>,
110 -}
111 -
112 -impl<'a> JournalWrapper<'a> {
113 - pub fn open(path: &str) -> Result<Self> {
114 - let opts = journal::OpenFilesOptions::default();
115 - let j = opts.open_files([path])?;
116 - let jr = JournalReader::default();
117 -
118 - Ok(Self { j, jr })
119 - }
120 -
121 - pub fn match_add(&mut self, data: &str) {
122 - self.j.match_add(data).unwrap();
123 - self.jr.add_match(data.as_bytes());
124 - }
125 -
126 - pub fn match_and(&mut self, journal_file: &'a JournalFile<Mmap>) {
127 - self.j.match_and().unwrap();
128 - self.jr.add_conjunction(journal_file).unwrap();
129 - }
130 -
131 - pub fn match_or(&mut self, journal_file: &'a JournalFile<Mmap>) {
132 - self.j.match_or().unwrap();
133 - self.jr.add_disjunction(journal_file).unwrap();
134 - }
135 -
136 - pub fn match_flush(&mut self) {
137 - self.j.match_flush().unwrap();
138 - self.jr.flush_matches();
139 - }
140 -
141 - pub fn seek_head(&mut self) {
142 - self.j.seek_head().unwrap();
143 - self.jr.set_location(Location::Head);
144 - }
145 -
146 - pub fn seek_tail(&mut self) {
147 - self.j.seek_tail().unwrap();
148 - self.jr.set_location(Location::Tail);
149 - }
150 -
151 - pub fn seek_realtime(&mut self, usec: u64) {
152 - self.j.seek_realtime_usec(usec).unwrap();
153 - self.jr.set_location(Location::Realtime(usec));
154 - }
155 -
156 - pub fn next(&mut self, journal_file: &'a JournalFile<Mmap>) -> bool {
157 - let r1 = self.j.next().unwrap();
158 - let r2 = self.jr.step(journal_file, Direction::Forward).unwrap();
159 -
160 - if r1 > 0 {
161 - if r2 {
162 - return r2;
163 - } else {
164 - panic!("r1: {:?}, r2: {:?}", r1, r2);
165 - }
166 - } else if r1 == 0 {
167 - if !r2 {
168 - return r2;
169 - } else {
170 - panic!("r1: {:?}, r2: {:?}", r1, r2);
171 - }
172 - } else {
173 - println!("WTF?");
174 - }
175 -
176 - r2
177 - }
178 -
179 - pub fn previous(&mut self, journal_file: &'a JournalFile<Mmap>) -> bool {
180 - let r1 = self.j.previous().unwrap();
181 - let r2 = self.jr.step(journal_file, Direction::Backward).unwrap();
182 -
183 - if r1 > 0 {
184 - if r2 {
185 - return r2;
186 - } else {
187 - panic!("r1: {:?}, r2: {:?}", r1, r2);
188 - }
189 - } else if r1 == 0 {
190 - if !r2 {
191 - return r2;
192 - } else {
193 - panic!("r1: {:?}, r2: {:?}", r1, r2);
194 - }
195 - } else {
196 - println!("WTF?");
197 - }
198 -
199 - r2
200 - }
201 -
202 - pub fn get_realtime_usec(&mut self, journal_file: &'a JournalFile<Mmap>) -> u64 {
203 - let usec1 = self.j.timestamp().unwrap();
204 - let usec2 = self.jr.get_realtime_usec(journal_file).unwrap();
205 -
206 - assert_eq!(usec1, usec2);
207 - usec1
208 - }
209 -}
210 -
211 -fn get_terms(path: &str) -> HashMap<String, Vec<String>> {
212 - let window_size = 8 * 1024 * 1024;
213 - let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
214 -
215 - let mut terms = HashMap::new();
216 - let mut fields = Vec::new();
217 - for field in journal_file.fields() {
218 - let field = field.unwrap();
219 - let field = String::from(String::from_utf8_lossy(field.get_payload()).clone());
220 - fields.push(field.clone());
221 - terms.insert(field, Vec::new());
222 - }
223 -
224 - for field in fields {
225 - for data in journal_file.field_data_objects(field.as_bytes()).unwrap() {
226 - let data = data.unwrap();
227 - if data.is_compressed() {
228 - continue;
229 - }
230 -
231 - let data_payload = String::from(String::from_utf8_lossy(data.get_payload()).clone());
232 -
233 - if data_payload.len() > 200 {
234 - continue;
235 - }
236 -
237 - terms.get_mut(&field).unwrap().push(data_payload);
238 - }
239 - }
240 -
241 - terms.retain(|_, value| !value.is_empty());
242 - terms
243 -}
244 -
245 -#[derive(Debug)]
246 -enum SeekType {
247 - Head,
248 - Tail,
249 - Realtime(u64),
250 -}
251 -
252 -fn get_timings(path: &str) -> Vec<u64> {
253 - let window_size = 8 * 1024 * 1024;
254 - let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
255 - let mut jw: JournalWrapper<'_> = JournalWrapper::open(path).unwrap();
256 -
257 - let mut v = Vec::new();
258 -
259 - jw.seek_head();
260 - loop {
261 - if !jw.next(&journal_file) {
262 - break;
263 - }
264 -
265 - let usec = jw.get_realtime_usec(&journal_file);
266 - v.push(usec);
267 - }
268 - assert!(v.is_sorted());
269 -
270 - v
271 -}
272 -
273 -use rand::{prelude::*, Rng};
274 -
275 -#[derive(Debug, Copy, Clone)]
276 -enum SeekOperation {
277 - Head,
278 - Tail,
279 - Realtime(u64),
280 -}
281 -
282 -fn select_seek_operation(rng: &mut ThreadRng, timings: &[u64]) -> SeekOperation {
283 - let duplicate_timestamps = [
284 - 1747729025279631,
285 - 1747729025280143,
286 - 1747729025280247,
287 - 1747729025358451,
288 - 1747729025387355,
289 - 1747729025387415,
290 - ];
291 -
292 - match rng.random_range(0..3) {
293 - 0 => SeekOperation::Head,
294 - 1 => SeekOperation::Tail,
295 - 2 => {
296 - let rt_idx = rng.random_range(0..timings.len());
297 -
298 - let usec = timings[rt_idx];
299 - if duplicate_timestamps.contains(&usec) {
300 - return SeekOperation::Head;
301 - }
302 -
303 - SeekOperation::Realtime(timings[rt_idx])
304 - }
305 - _ => unreachable!(),
306 - }
307 -}
308 -
309 -#[derive(Debug)]
310 -enum MatchOr {
311 - None,
312 - One(String),
313 - Two(String, String),
314 -}
315 -
316 -fn select_match_or(rng: &mut ThreadRng, terms: &HashMap<String, Vec<String>>) -> MatchOr {
317 - match rng.random_range(0..3) {
318 - 0 => MatchOr::None,
319 - 1 => {
320 - let key_index = rng.random_range(0..terms.len());
321 - let key = terms.keys().nth(key_index).unwrap();
322 -
323 - let value = terms.get(key).unwrap();
324 - let value_index = rng.random_range(0..value.len());
325 -
326 - MatchOr::One(value[value_index].clone())
327 - }
328 - 2 => {
329 - let first_term = {
330 - let key_index = rng.random_range(0..terms.len());
331 - let key = terms.keys().nth(key_index).unwrap();
332 -
333 - let value = terms.get(key).unwrap();
334 - let value_index = rng.random_range(0..value.len());
335 -
336 - value[value_index].clone()
337 - };
338 -
339 - let second_term = {
340 - let key_index = rng.random_range(0..terms.len());
341 - let key = terms.keys().nth(key_index).unwrap();
342 -
343 - let value = terms.get(key).unwrap();
344 - let value_index = rng.random_range(0..value.len());
345 -
346 - value[value_index].clone()
347 - };
348 -
349 - MatchOr::Two(first_term, second_term)
350 - }
351 - _ => {
352 - unreachable!()
353 - }
354 - }
355 -}
356 -
357 -#[derive(Debug, Clone)]
358 -enum MatchExpr {
359 - None,
360 - OrOne(String),
361 - OrTwo(String, String),
362 - And1(String, String),
363 - And2(String, (String, String)),
364 - And3((String, String), String),
365 - And4((String, String), (String, String)),
366 -}
367 -
368 -fn select_match_expression(rng: &mut ThreadRng, terms: &HashMap<String, Vec<String>>) -> MatchExpr {
369 - let mor1 = select_match_or(rng, terms);
370 - let mor2 = select_match_or(rng, terms);
371 -
372 - let expr = match (mor1, mor2) {
373 - (MatchOr::None, MatchOr::None) => MatchExpr::None,
374 -
375 - (MatchOr::None, MatchOr::One(d1)) => MatchExpr::OrOne(d1),
376 - (MatchOr::One(d1), MatchOr::None) => MatchExpr::OrOne(d1),
377 -
378 - (MatchOr::None, MatchOr::Two(d1, d2)) => MatchExpr::OrTwo(d1, d2),
379 - (MatchOr::Two(d1, d2), MatchOr::None) => MatchExpr::OrTwo(d1, d2),
380 -
381 - (MatchOr::One(d1), MatchOr::One(d2)) => MatchExpr::And1(d1, d2),
382 -
383 - (MatchOr::One(d1), MatchOr::Two(d2, d3)) => MatchExpr::And2(d1, (d2, d3)),
384 - (MatchOr::Two(d1, d2), MatchOr::One(d3)) => MatchExpr::And3((d1, d2), d3),
385 -
386 - (MatchOr::Two(d1, d2), MatchOr::Two(d3, d4)) => MatchExpr::And4((d1, d2), (d3, d4)),
387 - };
388 -
389 - match expr.clone() {
390 - MatchExpr::None | MatchExpr::OrOne(_) => expr,
391 - MatchExpr::OrTwo(d1, d2) => {
392 - if d1 == d2 {
393 - MatchExpr::None
394 - } else {
395 - expr
396 - }
397 - }
398 - MatchExpr::And1(d1, d2) => {
399 - if d1 == d2 {
400 - MatchExpr::None
401 - } else {
402 - expr
403 - }
404 - }
405 - MatchExpr::And2(d1, (d2, d3)) => {
406 - if d1 == d2 || d1 == d3 || d2 == d3 {
407 - MatchExpr::None
408 - } else {
409 - expr
410 - }
411 - }
412 - MatchExpr::And3((d1, d2), d3) => {
413 - if d1 == d2 || d1 == d3 || d2 == d3 {
414 - MatchExpr::None
415 - } else {
416 - expr
417 - }
418 - }
419 - MatchExpr::And4((d1, d2), (d3, d4)) => {
420 - if d1 == d2 || d1 == d3 || d1 == d4 || d2 == d3 || d2 == d4 || d3 == d4 {
421 - MatchExpr::None
422 - } else {
423 - expr
424 - }
425 - }
426 - }
427 -}
428 -
429 -#[derive(Debug, Clone, Copy)]
430 -enum IterationOperation {
431 - Next,
432 - Previous,
433 -}
434 -
435 -fn select_iteration_operation(rng: &mut ThreadRng) -> IterationOperation {
436 - match rng.random_range(0..2) {
437 - 0 => IterationOperation::Next,
438 - 1 => IterationOperation::Previous,
439 - _ => unreachable!(),
440 - }
441 -}
442 -
443 -fn apply_seek_operation(seek_operation: SeekOperation, jw: &mut JournalWrapper) {
444 - match seek_operation {
445 - SeekOperation::Head => jw.seek_head(),
446 - SeekOperation::Tail => jw.seek_tail(),
447 - SeekOperation::Realtime(usec) => jw.seek_realtime(usec),
448 - }
449 -}
450 -
451 -fn apply_iteration_operation<'a>(
452 - iteration_operation: IterationOperation,
453 - jw: &mut JournalWrapper<'a>,
454 - journal_file: &'a JournalFile<Mmap>,
455 -) -> bool {
456 - match iteration_operation {
457 - IterationOperation::Next => jw.next(journal_file),
458 - IterationOperation::Previous => jw.previous(journal_file),
459 - }
460 -}
461 -
462 -fn apply_match_expression<'a>(
463 - match_expr: MatchExpr,
464 - jw: &mut JournalWrapper<'a>,
465 - journal_file: &'a JournalFile<Mmap>,
466 -) -> bool {
467 - jw.match_flush();
468 -
469 - match match_expr.clone() {
470 - MatchExpr::None => {
471 - return true;
472 - }
473 - MatchExpr::OrOne(d) => {
474 - jw.match_add(&d);
475 - return true;
476 - }
477 - MatchExpr::OrTwo(d1, d2) => {
478 - jw.match_add(&d1);
479 - jw.match_add(&d2);
480 - return true;
481 - }
482 - MatchExpr::And1(d1, d2) => {
483 - jw.match_add(&d1);
484 - jw.match_and(journal_file);
485 - jw.match_add(&d2);
486 - return true;
487 - }
488 - MatchExpr::And2(d1, (d2, d3)) => {
489 - jw.match_add(&d1);
490 - jw.match_and(journal_file);
491 - jw.match_add(&d2);
492 - jw.match_add(&d3);
493 - return true;
494 - }
495 - MatchExpr::And3((d1, d2), d3) => {
496 - jw.match_add(&d1);
497 - jw.match_add(&d2);
498 - jw.match_and(journal_file);
499 - jw.match_add(&d3);
500 - return true;
501 - }
502 - MatchExpr::And4((d1, d2), (d3, d4)) => {
503 - jw.match_add(&d1);
504 - jw.match_add(&d2);
505 - jw.match_or(journal_file);
506 -
507 - jw.match_add(&d3);
508 - jw.match_add(&d4);
509 - jw.match_or(journal_file);
510 -
511 - jw.match_and(journal_file);
512 -
513 - return true;
514 - }
515 - };
516 -}
517 -
518 -fn filtered_test() {
519 - let path = "/tmp/foo.journal";
520 - let window_size = 8 * 1024 * 1024;
521 - let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
522 - println!(
523 - "num entries: {:?}",
524 - journal_file.journal_header_ref().n_entries
525 - );
526 - let mut jw = JournalWrapper::open(path).unwrap();
527 -
528 - let terms = get_terms(path);
529 - let timings = get_timings(path);
530 -
531 - let mut rng = rand::rng();
532 -
533 - let mut counter = 0;
534 - loop {
535 - let match_expr = select_match_expression(&mut rng, &terms);
536 - let applied = apply_match_expression(match_expr.clone(), &mut jw, &journal_file);
537 - if !applied {
538 - continue;
539 - }
540 -
541 - println!("[{}] match_expr: {:?}", counter, match_expr);
542 -
543 - let seek_operation = select_seek_operation(&mut rng, &timings);
544 - println!("[{}] seek: {:?}", counter, seek_operation);
545 - apply_seek_operation(seek_operation, &mut jw);
546 -
547 - let mut num_matches = 0;
548 - let iteration_operation = select_iteration_operation(&mut rng);
549 - println!("[{}] iteration: {:?}", counter, iteration_operation);
550 -
551 - for _ in 0..rng.random_range(0..2 * timings.len()) {
552 - let found = apply_iteration_operation(iteration_operation, &mut jw, &journal_file);
553 - if found {
554 - jw.get_realtime_usec(&journal_file);
555 - num_matches += 1;
556 - }
557 - }
558 -
559 - println!("[{}] num matches: {:?}\n", counter, num_matches);
560 - counter += 1;
561 - }
562 -}
563 -
564 -fn test_case() {
565 - let path = "/tmp/foo.journal";
566 -
567 - let window_size = 8 * 1024 * 1024;
568 - let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
569 - let mut jw = JournalWrapper::open(path).unwrap();
570 -
571 - let timings = get_timings(path);
572 - println!("timings: {:#?}", timings);
573 -
574 - jw.seek_realtime(u64::MAX);
575 - if jw.previous(&journal_file) {
576 - let value = jw.get_realtime_usec(&journal_file);
577 - println!("first value: {:?}", value);
578 - }
579 - // return;
580 -
581 - // jw.next(&journal_file);
582 - // let value = jw.get_realtime_usec(&journal_file);
583 - // println!("second value: {:?}", value);
584 -}
585 -
586 -fn main() {
587 - // {
588 - // let mut jf = JournalFile::<MmapMut>::create("/tmp/muh.journal", 4096).unwrap();
589 -
590 - // let dht = jf.data_hash_table_mut().unwrap();
591 - // let mut items = dht.items;
592 - // println!("dht items: {:?}", items.len());
593 - // items[0].head_hash_offset = 0xdeadbeef;
594 - // items[0].tail_hash_offset = 0xbeefdead;
595 -
596 - // let fht = jf.field_hash_table_mut().unwrap();
597 - // let mut items = fht.items;
598 - // println!("fht items: {:?}", items.len());
599 - // items[0].head_hash_offset = 0xaaaabbbb;
600 - // items[0].tail_hash_offset = 0xccccdddd;
601 -
602 - // let mut offset_array = jf.offset_array_mut(1024 * 1024, Some(8)).unwrap();
603 -
604 - // for i in 0..4 {
605 - // let offset = std::num::NonZeroU64::new(0xdead0000 + i).unwrap();
606 - // offset_array.set(i as usize, offset).unwrap();
607 - // }
608 - // }
609 -
610 - // let jf = JournalFile::<Mmap>::open("/tmp/muh.journal", 4096).unwrap();
611 -
612 - // let offset_array = jf.offset_array_ref(1024 * 1024).unwrap();
613 -
614 - // println!(
615 - // "tail object offset: 0x{:x?}",
616 - // jf.journal_header_ref().tail_object_offset
617 - // );
618 - // println!(
619 - // "fht offset: 0x{:x?}",
620 - // jf.journal_header_ref().field_hash_table_offset
621 - // );
622 - // println!(
623 - // "hash table header size: 0x{:x?}",
624 - // std::mem::size_of::<ObjectHeader>()
625 - // );
626 -
627 - // println!("offset_array: {:?}", offset_array);
628 -
629 - // for i in 0..4 {
630 - // let offset = offset_array.get(i, 8).unwrap();
631 - // println!("offset[{}]: 0x{:x?}", i, offset);
632 - // }
633 -
634 - filtered_test();
635 - // test_case()
636 -
637 - // altime();
638 -
639 - // let args: Vec<String> = std::env::args().collect();
640 - // if args.len() != 2 {
641 - // eprintln!("Usage: {} <journal_file_path>", args[0]);
642 - // std::process::exit(1);
643 - // }
644 -
645 - // if false {
646 - // create_logs();
647 - // return;
648 - // }
649 -
650 - // const WINDOW_SIZE: u64 = 4096;
651 - // match JournalFile::<Mmap>::open(&args[1], WINDOW_SIZE) {
652 - // Ok(journal_file) => {
653 - // if true {
654 - // if let Err(e) = test_cursor(&journal_file) {
655 - // panic!("Cursor tests failed: {:?}", e);
656 - // }
657 - // }
658 -
659 - // if true {
660 - // if let Err(e) = test_filter_expr(&journal_file) {
661 - // panic!("Filter expression tests failed: {:?}", e);
662 - // }
663 -
664 - // println!("Overall stat: {:?}", journal_file.stats());
665 - // }
666 - // }
667 - // Err(e) => panic!("Failed to open journal file: {:?}", e),
668 - // }
669 -}
src/crates/jf/otel-plugin/Cargo.toml new
+31
@@ -0,0 +1,31 @@
1 +[package]
2 +name = "otel-plugin"
3 +version.workspace = true
4 +edition.workspace = true
5 +rust-version.workspace = true
6 +
7 +[dependencies]
8 +tokio = { workspace = true }
9 +prost = { workspace = true }
10 +tonic = { workspace = true, features = ["gzip", "tls-ring"] }
11 +opentelemetry-proto = { workspace = true, features = ["metrics", "logs"] }
12 +
13 +flatten_otel = { path = "../flatten_otel" }
14 +journal_file = { path = "../journal_file" }
15 +journal_log = { path = "../journal_log" }
16 +memmap2 = { workspace = true }
17 +serde_json = { workspace = true, features = ["preserve_order"] }
18 +uuid = { version = "1.0", features = ["v4", "rng"] }
19 +flatten-serde-json = { git = "https://github.com/meilisearch/meilisearch", branch = "main", package = "flatten-serde-json" }
20 +base64 = "0.21"
21 +regex = { workspace = true }
22 +serde_yaml = "0.9.34"
23 +serde_regex = "1.1.0"
24 +serde = { version = "1.0.219", features = ["derive"] }
25 +atty = { workspace = true }
26 +clap = { workspace = true }
27 +anyhow = { workspace = true }
28 +humantime-serde = { workspace = true }
29 +humantime = { workspace = true }
30 +bytesize = { workspace = true }
31 +bytesize-serde = { workspace = true }
src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yml new
+172
@@ -0,0 +1,172 @@
1 +configs:
2 + - select:
3 + instrumentation_scope_name: .*hostmetricsreceiver.*networkscraper$
4 + metric_name: system\.network\.connections
5 + extract:
6 + chart_instance_pattern: metric.attributes.protocol
7 + dimension_name: metric.attributes.state
8 + - select:
9 + instrumentation_scope_name: .*hostmetricsreceiver.*networkscraper$
10 + metric_name: system\.network\.dropped
11 + extract:
12 + chart_instance_pattern: metric.attributes.device
13 + dimension_name: metric.attributes.direction
14 + - select:
15 + instrumentation_scope_name: .*hostmetricsreceiver.*networkscraper$
16 + metric_name: system\.network\.errors
17 + extract:
18 + chart_instance_pattern: metric.attributes.device
19 + dimension_name: metric.attributes.direction
20 + - select:
21 + instrumentation_scope_name: .*hostmetricsreceiver.*networkscraper$
22 + metric_name: system\.network\.io
23 + extract:
24 + chart_instance_pattern: metric.attributes.device
25 + dimension_name: metric.attributes.direction
26 + - select:
27 + instrumentation_scope_name: .*hostmetricsreceiver.*networkscraper$
28 + metric_name: system\.network\.packets
29 + extract:
30 + chart_instance_pattern: metric.attributes.device
31 + dimension_name: metric.attributes.direction
32 + - select:
33 + instrumentation_scope_name: .*hostmetricsreceiver.*cpuscraper$
34 + metric_name: system\.cpu\.time
35 + extract:
36 + chart_instance_pattern: metric.attributes.cpu
37 + dimension_name: metric.attributes.state
38 + - select:
39 + instrumentation_scope_name: .*hostmetricsreceiver.*cpuscraper$
40 + metric_name: system\.cpu\.frequency
41 + extract:
42 + chart_instance_pattern: metric.attributes.cpu
43 + - select:
44 + instrumentation_scope_name: .*hostmetricsreceiver.*cpuscraper$
45 + metric_name: system\.cpu\.utilization
46 + extract:
47 + chart_instance_pattern: metric.attributes.cpu
48 + dimension_name: metric.attributes.state
49 + - select:
50 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
51 + metric_name: system\.disk\.io$
52 + extract:
53 + chart_instance_pattern: metric.attributes.device
54 + dimension_name: metric.attributes.direction
55 + - select:
56 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
57 + metric_name: system\.disk\.io_time
58 + extract:
59 + chart_instance_pattern: metric.attributes.device
60 + - select:
61 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
62 + metric_name: system\.disk\.merged
63 + extract:
64 + chart_instance_pattern: metric.attributes.device
65 + dimension_name: metric.attributes.direction
66 + - select:
67 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
68 + metric_name: system\.disk\.operation_time
69 + extract:
70 + chart_instance_pattern: metric.attributes.device
71 + dimension_name: metric.attributes.direction
72 + - select:
73 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
74 + metric_name: system\.disk\.operations
75 + extract:
76 + chart_instance_pattern: metric.attributes.device
77 + dimension_name: metric.attributes.direction
78 + - select:
79 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
80 + metric_name: system\.disk\.pending_operations
81 + extract:
82 + chart_instance_pattern: metric.attributes.device
83 + - select:
84 + instrumentation_scope_name: .*hostmetricsreceiver.*diskscraper$
85 + metric_name: system\.disk\.weighted_io
86 + extract:
87 + chart_instance_pattern: metric.attributes.device
88 + - select:
89 + instrumentation_scope_name: .*hostmetricsreceiver.*filesystemscraper$
90 + metric_name: system\.filesystem\.inodes\.usage
91 + extract:
92 + chart_instance_pattern: metric.attributes.mountpoint
93 + dimension_name: metric.attributes.state
94 + - select:
95 + instrumentation_scope_name: .*hostmetricsreceiver.*filesystemscraper$
96 + metric_name: system\.filesystem\.usage
97 + extract:
98 + chart_instance_pattern: metric.attributes.mountpoint
99 + dimension_name: metric.attributes.state
100 + - select:
101 + instrumentation_scope_name: .*hostmetricsreceiver.*filesystemscraper$
102 + metric_name: system\.filesystem\.utilization
103 + extract:
104 + chart_instance_pattern: metric.attributes.mountpoint
105 + - select:
106 + instrumentation_scope_name: .*hostmetricsreceiver.*memoryscraper$
107 + metric_name: system\.memory\.utilization
108 + extract:
109 + dimension_name: metric.attributes.state
110 + - select:
111 + instrumentation_scope_name: .*hostmetricsreceiver.*pagingscraper$
112 + metric_name: system\.paging\.faults
113 + extract:
114 + dimension_name: metric.attributes.type
115 + - select:
116 + instrumentation_scope_name: .*hostmetricsreceiver.*pagingscraper$
117 + metric_name: system\.paging\.operations
118 + extract:
119 + chart_instance_pattern: metric.attributes.type
120 + dimension_name: metric.attributes.direction
121 + - select:
122 + instrumentation_scope_name: .*hostmetricsreceiver.*pagingscraper$
123 + metric_name: system\.paging\.usage
124 + extract:
125 + chart_instance_pattern: metric.attributes.device
126 + dimension_name: metric.attributes.state
127 + - select:
128 + instrumentation_scope_name: .*hostmetricsreceiver.*pagingscraper$
129 + metric_name: system\.paging\.utilization
130 + extract:
131 + chart_instance_pattern: metric.attributes.device
132 + dimension_name: metric.attributes.state
133 + - select:
134 + instrumentation_scope_name: .*hostmetricsreceiver.*processesscraper$
135 + metric_name: system\.processes\.count
136 + extract:
137 + dimension_name: metric.attributes.status
138 + - select:
139 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
140 + metric_name: process\.cpu\.time
141 + extract:
142 + dimension_name: metric.attributes.state
143 + - select:
144 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
145 + metric_name: process\.disk\.io
146 + extract:
147 + dimension_name: metric.attributes.direction
148 + - select:
149 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
150 + metric_name: process\.context_switches
151 + extract:
152 + dimension_name: metric.attributes.type
153 + - select:
154 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
155 + metric_name: process\.cpu\.utilization
156 + extract:
157 + dimension_name: metric.attributes.state
158 + - select:
159 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
160 + metric_name: process\.disk\.operations
161 + extract:
162 + dimension_name: metric.attributes.direction
163 + - select:
164 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
165 + metric_name: process\.paging\.faults
166 + extract:
167 + dimension_name: metric.attributes.type
168 + - select:
169 + instrumentation_scope_name: .*hostmetricsreceiver.*processscraper$
170 + metric_name: process\.paging\.faults
171 + extract:
172 + dimension_name: metric.attributes.type
src/crates/jf/otel-plugin/configs/otel.yml new
+49
@@ -0,0 +1,49 @@
1 +# OpenTelemetry Plugin Configuration
2 +# This file configures the OpenTelemetry metrics and logs ingestion for Netdata
3 +
4 +endpoint:
5 + # gRPC endpoint to listen on for OpenTelemetry data
6 + path: "127.0.0.1:4317"
7 +
8 + # Path to TLS certificate file (enables TLS when provided)
9 + tls_cert_path: null
10 +
11 + # Path to TLS private key file (required when TLS certificate is provided)
12 + tls_key_path: null
13 +
14 + # Path to TLS CA certificate file for client authentication (optional)
15 + tls_ca_cert_path: null
16 +
17 +metrics:
18 + # Print flattened metrics to stdout for debugging, instead of ingesting them
19 + print_flattened: false
20 +
21 + # Number of samples to buffer for collection interval detection
22 + buffer_samples: 10
23 +
24 + # Maximum number of new charts to create per collection interval
25 + throttle_charts: 100
26 +
27 + # Directory with configuration files for mapping OTEL metrics to Netdata charts
28 + # (relative paths are resolved based on Netdata's user configuration directory)
29 + chart_configs_dir: otel.d/v1/metrics
30 +
31 +logs:
32 + # Directory to store journal files for logs.
33 + # (relative paths are resolved based on Netdata's log directory)
34 + journal_dir: otel/v1
35 +
36 + # Maximum file size for individual journal files (e.g., "100MB", "1.5GB").
37 + size_of_journal_file: "100MB"
38 +
39 + # Maximum number of journal files to keep.
40 + number_of_journal_files: 10
41 +
42 + # Maximum total size for all journal files combined (e.g., "1GB", "500MB")
43 + size_of_journal_files: "1GB"
44 +
45 + # Maximum age for journal entries (e.g., "7 days", "1 week", "168h").
46 + duration_of_journal_files: "7 days"
47 +
48 + # Maximum time span that entries in a single journal file can cover (e.g., "2 hours", "1h", "30m").
49 + duration_of_journal_file: "2 hours"
src/crates/jf/otel-plugin/src/chart_config.rs new
+171
@@ -0,0 +1,171 @@
1 +use anyhow::{Context, Result};
2 +use regex::Regex;
3 +use serde::{Deserialize, Serialize};
4 +use serde_json::{Map as JsonMap, Value as JsonValue};
5 +use std::fs;
6 +use std::path::Path;
7 +
8 +#[derive(Debug, Clone, Serialize, Deserialize)]
9 +pub struct SelectCriteria {
10 + #[serde(with = "serde_regex", skip_serializing_if = "Option::is_none", default)]
11 + pub instrumentation_scope_name: Option<Regex>,
12 +
13 + #[serde(with = "serde_regex", skip_serializing_if = "Option::is_none", default)]
14 + pub instrumentation_scope_version: Option<Regex>,
15 +
16 + #[serde(with = "serde_regex")]
17 + pub metric_name: Regex,
18 +}
19 +
20 +#[derive(Debug, Clone, Serialize, Deserialize)]
21 +pub struct ExtractPattern {
22 + #[serde(skip_serializing_if = "Option::is_none")]
23 + pub chart_instance_pattern: Option<String>,
24 +
25 + #[serde(skip_serializing_if = "Option::is_none")]
26 + pub dimension_name: Option<String>,
27 +}
28 +
29 +#[derive(Debug, Clone, Serialize, Deserialize)]
30 +pub struct ChartConfig {
31 + pub select: SelectCriteria,
32 + pub extract: ExtractPattern,
33 +}
34 +
35 +impl ChartConfig {
36 + pub fn matches(&self, json_map: &JsonMap<String, JsonValue>) -> bool {
37 + if let Some(scope_regex) = &self.select.instrumentation_scope_name {
38 + if let Some(JsonValue::String(scope_name)) = json_map.get("scope.name") {
39 + if !scope_regex.is_match(scope_name) {
40 + return false;
41 + }
42 + } else {
43 + return false;
44 + }
45 + }
46 +
47 + if let Some(version_regex) = &self.select.instrumentation_scope_version {
48 + if let Some(JsonValue::String(scope_version)) = json_map.get("scope.version") {
49 + if !version_regex.is_match(scope_version) {
50 + return false;
51 + }
52 + } else {
53 + return false;
54 + }
55 + }
56 +
57 + if let Some(JsonValue::String(metric_name)) = json_map.get("metric.name") {
58 + self.select.metric_name.is_match(metric_name)
59 + } else {
60 + false
61 + }
62 + }
63 +}
64 +
65 +#[derive(Debug, Clone, Default, Serialize, Deserialize)]
66 +pub struct ChartConfigs {
67 + configs: Vec<ChartConfig>,
68 +}
69 +
70 +#[derive(Debug, Default, Clone)]
71 +pub struct ChartConfigManager {
72 + stock: ChartConfigs,
73 + user: ChartConfigs,
74 +}
75 +
76 +impl ChartConfigManager {
77 + pub fn with_default_configs() -> Self {
78 + let mut manager = Self::default();
79 + manager.load_stock_config();
80 + manager
81 + }
82 +
83 + pub fn find_matching_config(
84 + &self,
85 + json_map: &JsonMap<String, JsonValue>,
86 + ) -> Option<&ChartConfig> {
87 + // Chaining-order is important. We want to priority user configurations
88 + // and fall back to stock configurations if they are missing.
89 + self.user
90 + .configs
91 + .iter()
92 + .chain(self.stock.configs.iter())
93 + .find(|config| config.matches(json_map))
94 + }
95 +
96 + fn load_stock_config(&mut self) {
97 + const DEFAULT_CONFIGS_YAML: &str =
98 + include_str!("../configs/otel.d/v1/metrics/hostmetrics-receiver.yml");
99 +
100 + match serde_yaml::from_str::<ChartConfigs>(DEFAULT_CONFIGS_YAML) {
101 + Ok(configs) => {
102 + self.stock = configs;
103 + }
104 + Err(e) => {
105 + eprintln!("Failed to parse default configs YAML: {}", e);
106 + }
107 + }
108 + }
109 +
110 + pub fn load_user_configs<P: AsRef<Path>>(&mut self, config_dir: P) -> Result<()> {
111 + // check dir
112 + let config_path = config_dir.as_ref();
113 + if !config_path.exists() {
114 + return Err(anyhow::anyhow!(
115 + "Configuration directory does not exist: {}",
116 + config_path.display()
117 + ));
118 + }
119 + if !config_path.is_dir() {
120 + return Err(anyhow::anyhow!(
121 + "Configuration path is not a directory: {}",
122 + config_path.display()
123 + ));
124 + }
125 +
126 + // collect the yaml files
127 + let mut config_files: Vec<_> = std::fs::read_dir(config_path)
128 + .with_context(|| {
129 + format!(
130 + "Failed to read chart config directory: {}",
131 + config_path.display()
132 + )
133 + })?
134 + .filter_map(|entry| {
135 + let entry = entry.ok()?;
136 + let path = entry.path();
137 + if path.is_file()
138 + && matches!(
139 + path.extension().and_then(|s| s.to_str()),
140 + Some("yml" | "yaml")
141 + )
142 + {
143 + Some(path)
144 + } else {
145 + None
146 + }
147 + })
148 + .collect();
149 + config_files.sort();
150 +
151 + // deserialize them
152 + self.user = ChartConfigs::default();
153 + for path in config_files {
154 + match fs::read_to_string(&path) {
155 + Ok(contents) => match serde_yaml::from_str::<ChartConfigs>(&contents) {
156 + Ok(chart_configs) => {
157 + self.user.configs.extend(chart_configs.configs);
158 + }
159 + Err(e) => {
160 + eprintln!("Failed to parse YAML file {}: {}", path.display(), e);
161 + }
162 + },
163 + Err(e) => {
164 + eprintln!("Failed to read file {}: {}", path.display(), e);
165 + }
166 + }
167 + }
168 +
169 + Ok(())
170 + }
171 +}
src/crates/jf/otel-plugin/src/flattened_point.rs new
+171
@@ -0,0 +1,171 @@
1 +use serde_json::{Map as JsonMap, Value as JsonValue};
2 +use std::hash::{Hash, Hasher};
3 +
4 +use crate::regex_cache::RegexCache;
5 +
6 +#[derive(Default, Debug)]
7 +pub struct FlattenedPoint {
8 + pub attributes: JsonMap<String, JsonValue>,
9 +
10 + pub nd_instance_name: String,
11 + pub nd_dimension_name: String,
12 +
13 + pub metric_name: String,
14 + pub metric_description: String,
15 + pub metric_unit: String,
16 + pub metric_type: String,
17 +
18 + pub metric_time_unix_nano: u64,
19 + pub metric_value: f64,
20 +
21 + pub metric_is_monotonic: Option<bool>,
22 +}
23 +
24 +use crate::chart_config::ChartConfig;
25 +
26 +impl FlattenedPoint {
27 + pub fn new(
28 + mut json_map: JsonMap<String, JsonValue>,
29 + chart_config: Option<&ChartConfig>,
30 + regex_cache: &RegexCache,
31 + ) -> Option<Self> {
32 + let Some(JsonValue::String(metric_name)) = json_map.remove("metric.name") else {
33 + debug_assert!(false, "metric.name missing from json map");
34 + return None;
35 + };
36 +
37 + let Some(JsonValue::String(metric_description)) = json_map.remove("metric.description")
38 + else {
39 + debug_assert!(false, "metric.description missing from json map");
40 + return None;
41 + };
42 +
43 + let Some(JsonValue::String(metric_unit)) = json_map.remove("metric.unit") else {
44 + debug_assert!(false, "metric.unit missing from json map");
45 + return None;
46 + };
47 +
48 + let Some(JsonValue::String(metric_type)) = json_map.remove("metric.type") else {
49 + debug_assert!(false, "metric.type missing from json map");
50 + return None;
51 + };
52 +
53 + // Ignore start_time_unix for the time being.
54 + json_map.remove("metric.start_time_unix_nano");
55 +
56 + let Some(metric_time_unix_nano) = json_map
57 + .remove("metric.time_unix_nano")
58 + .and_then(|v| v.as_u64())
59 + else {
60 + debug_assert!(false, "metric.time_unix_nano missing from json map");
61 + return None;
62 + };
63 +
64 + let Some(metric_value) = json_map.remove("metric.value").and_then(|v| v.as_f64()) else {
65 + debug_assert!(false, "metric.value missing from json map");
66 + return None;
67 + };
68 +
69 + let metric_is_monotonic = json_map
70 + .remove("metric.is_monotonic")
71 + .and_then(|v| v.as_bool());
72 +
73 + if let Some(config) = chart_config {
74 + if let Some(chart_instance_pattern) = &config.extract.chart_instance_pattern {
75 + if !json_map.contains_key("metric.attributes._nd_chart_instance") {
76 + json_map.insert(
77 + "metric.attributes._nd_chart_instance".to_string(),
78 + JsonValue::String(chart_instance_pattern.clone()),
79 + );
80 + }
81 + }
82 +
83 + if let Some(dimension_name) = &config.extract.dimension_name {
84 + if !json_map.contains_key("metric.attributes._nd_dimension") {
85 + json_map.insert(
86 + "metric.attributes._nd_dimension".to_string(),
87 + JsonValue::String(dimension_name.clone()),
88 + );
89 + }
90 + }
91 + }
92 +
93 + let nd_dimension_name = {
94 + let nd_dimension_key = json_map
95 + .remove("metric.attributes._nd_dimension")
96 + .and_then(|v| v.as_str().map(String::from));
97 +
98 + if let Some(key) = nd_dimension_key {
99 + match json_map.remove(&key) {
100 + Some(JsonValue::String(s)) => s.clone(),
101 + Some(JsonValue::Number(n)) => n.to_string(),
102 + Some(JsonValue::Bool(b)) => b.to_string(),
103 + Some(value) => {
104 + eprintln!("Only strings/number/bool values can be used for dimension name >>>{:#?}<<<", value);
105 + return None;
106 + }
107 + _ => {
108 + eprintln!(
109 + "Dimension key >>>{:?}<<< not found in flattened representation.",
110 + key
111 + );
112 + return None;
113 + }
114 + }
115 + } else {
116 + String::from("value")
117 + }
118 + };
119 +
120 + let nd_instance_name = {
121 + let nd_chart_instance = json_map
122 + .remove("metric.attributes._nd_chart_instance")
123 + .and_then(|v| v.as_str().map(String::from))
124 + .and_then(|s| regex_cache.get(&s).ok());
125 +
126 + let mut matched_values = vec![metric_name.clone()];
127 + if let Some(pattern) = nd_chart_instance {
128 + for (key, value) in &json_map {
129 + if pattern.is_match(key) {
130 + let value_str = match value {
131 + JsonValue::String(s) => s.clone(),
132 + JsonValue::Number(n) => n.to_string(),
133 + JsonValue::Bool(b) => b.to_string(),
134 + JsonValue::Null => "null".to_string(),
135 + _ => serde_json::to_string(value).unwrap_or_default(),
136 + };
137 + matched_values.push(value_str);
138 + }
139 + }
140 + }
141 +
142 + let name = matched_values.join(".");
143 +
144 + let hash = {
145 + use std::hash::DefaultHasher;
146 +
147 + let mut state = DefaultHasher::new();
148 + name.hash(&mut state);
149 + json_map.hash(&mut state);
150 + metric_unit.hash(&mut state);
151 + metric_type.hash(&mut state);
152 + state.finish()
153 + };
154 +
155 + format!("{name}.{hash:016x}")
156 + };
157 +
158 + Some(Self {
159 + attributes: json_map,
160 + nd_instance_name,
161 + nd_dimension_name,
162 + metric_name,
163 + metric_description: metric_description.replace('\'', "\""),
164 + metric_unit,
165 + metric_type,
166 + metric_time_unix_nano,
167 + metric_value,
168 + metric_is_monotonic,
169 + })
170 + }
171 +}
src/crates/jf/otel-plugin/src/logs_service.rs new
+99
@@ -0,0 +1,99 @@
1 +use anyhow::{Context, Result};
2 +use flatten_otel::json_from_export_logs_service_request;
3 +use journal_log::{JournalLog, JournalLogConfig, RetentionPolicy, RotationPolicy};
4 +use opentelemetry_proto::tonic::collector::logs::v1::{
5 + logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse,
6 +};
7 +use serde_json::Value;
8 +use std::sync::{Arc, Mutex};
9 +use tonic::{Request, Response, Status};
10 +
11 +use crate::plugin_config::PluginConfig;
12 +
13 +pub struct NetdataLogsService {
14 + journal_log: Arc<Mutex<JournalLog>>,
15 +}
16 +
17 +impl NetdataLogsService {
18 + pub fn new(plugin_config: PluginConfig) -> Result<Self> {
19 + let logs_config = plugin_config.logs;
20 +
21 + let rotation_policy = RotationPolicy::default()
22 + .with_size_of_journal_file(logs_config.size_of_journal_file.as_u64())
23 + .with_duration_of_journal_file(logs_config.duration_of_journal_file);
24 +
25 + let retention_policy = RetentionPolicy::default()
26 + .with_number_of_journal_files(logs_config.number_of_journal_files)
27 + .with_size_of_journal_files(logs_config.size_of_journal_files.as_u64())
28 + .with_duration_of_journal_files(logs_config.duration_of_journal_files);
29 +
30 + let journal_config = JournalLogConfig::new(&logs_config.journal_dir)
31 + .with_rotation_policy(rotation_policy)
32 + .with_retention_policy(retention_policy);
33 +
34 + let journal_log = Arc::new(Mutex::new(JournalLog::new(journal_config).with_context(
35 + || {
36 + format!(
37 + "Failed to create journal log for directory: {}",
38 + logs_config.journal_dir
39 + )
40 + },
41 + )?));
42 + Ok(NetdataLogsService { journal_log })
43 + }
44 +
45 + fn json_to_entry_data(&self, json_value: &Value) -> Vec<Vec<u8>> {
46 + let mut entry_data = Vec::new();
47 +
48 + if let Value::Object(obj) = json_value {
49 + for (key, value) in obj {
50 + let value_str = match value {
51 + Value::String(s) => s.clone(),
52 + Value::Number(n) => n.to_string(),
53 + Value::Bool(b) => b.to_string(),
54 + Value::Null => "null".to_string(),
55 + _ => serde_json::to_string(value).unwrap_or_default(),
56 + };
57 +
58 + let kv_pair = format!("{}={}", key, value_str);
59 + entry_data.push(kv_pair.into_bytes());
60 + }
61 + }
62 +
63 + entry_data
64 + }
65 +}
66 +
67 +#[tonic::async_trait]
68 +impl LogsService for NetdataLogsService {
69 + async fn export(
70 + &self,
71 + request: Request<ExportLogsServiceRequest>,
72 + ) -> Result<Response<ExportLogsServiceResponse>, Status> {
73 + let req = request.into_inner();
74 +
75 + let json_array = json_from_export_logs_service_request(&req);
76 +
77 + if let Value::Array(entries) = json_array {
78 + for entry in entries {
79 + let entry_data = self.json_to_entry_data(&entry);
80 + if !entry_data.is_empty() {
81 + let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
82 + if let Err(e) = self.journal_log.lock().unwrap().write_entry(&entry_refs) {
83 + eprintln!("Failed to write log entry: {}", e);
84 + return Err(Status::internal(format!(
85 + "Failed to write log entry: {}",
86 + e
87 + )));
88 + }
89 + }
90 + }
91 + }
92 +
93 + let reply = ExportLogsServiceResponse {
94 + partial_success: None,
95 + };
96 +
97 + Ok(Response::new(reply))
98 + }
99 +}
src/crates/jf/otel-plugin/src/main.rs new
+104
@@ -0,0 +1,104 @@
1 +use anyhow::{Context, Result};
2 +use opentelemetry_proto::tonic::collector::{
3 + logs::v1::logs_service_server::LogsServiceServer,
4 + metrics::v1::metrics_service_server::MetricsServiceServer,
5 +};
6 +use tonic::transport::{Identity, Server, ServerTlsConfig};
7 +
8 +mod chart_config;
9 +mod flattened_point;
10 +mod netdata_chart;
11 +mod netdata_env;
12 +mod regex_cache;
13 +mod samples_table;
14 +
15 +mod plugin_config;
16 +use crate::plugin_config::PluginConfig;
17 +
18 +mod logs_service;
19 +use crate::logs_service::NetdataLogsService;
20 +
21 +mod metrics_service;
22 +use crate::metrics_service::NetdataMetricsService;
23 +
24 +async fn send_keepalive_periodically() {
25 + let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(60));
26 +
27 + loop {
28 + interval.tick().await;
29 + println!("PLUGIN_KEEPALIVE");
30 + }
31 +}
32 +
33 +#[tokio::main]
34 +async fn main() -> Result<()> {
35 + let config = PluginConfig::new().context("Failed to initialize plugin configuration")?;
36 +
37 + let addr =
38 + config.endpoint.path.parse().with_context(|| {
39 + format!("Failed to parse endpoint address: {}", config.endpoint.path)
40 + })?;
41 + let metrics_service =
42 + NetdataMetricsService::new(config.clone()).context("Failed to create metrics service")?;
43 + let logs_service =
44 + NetdataLogsService::new(config.clone()).context("Failed to create logs service")?;
45 +
46 + println!("TRUST_DURATIONS 1");
47 +
48 + let mut server_builder = Server::builder();
49 +
50 + // Configure TLS if provided
51 + if let (Some(cert_path), Some(key_path)) = (
52 + &config.endpoint.tls_cert_path,
53 + &config.endpoint.tls_key_path,
54 + ) {
55 + let cert = std::fs::read(cert_path)
56 + .with_context(|| format!("Failed to read TLS certificate from: {}", cert_path))?;
57 + let key = std::fs::read(key_path)
58 + .with_context(|| format!("Failed to read TLS private key from: {}", key_path))?;
59 + let identity = Identity::from_pem(cert, key);
60 +
61 + let mut tls_config_builder = ServerTlsConfig::new().identity(identity);
62 +
63 + // If CA certificate is provided, enable client authentication
64 + if let Some(ref ca_cert_path) = config.endpoint.tls_ca_cert_path {
65 + let ca_cert = std::fs::read(ca_cert_path)
66 + .with_context(|| format!("Failed to read CA certificate from: {}", ca_cert_path))?;
67 + tls_config_builder =
68 + tls_config_builder.client_ca_root(tonic::transport::Certificate::from_pem(ca_cert));
69 + }
70 +
71 + server_builder = server_builder
72 + .tls_config(tls_config_builder)
73 + .context("Failed to configure TLS")?;
74 + } else {
75 + eprintln!(
76 + "TLS disabled, using insecure connection on endpoint: {}",
77 + config.endpoint.path
78 + );
79 + }
80 +
81 + let server = server_builder
82 + .add_service(
83 + MetricsServiceServer::new(metrics_service)
84 + .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
85 + )
86 + .add_service(
87 + LogsServiceServer::new(logs_service)
88 + .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
89 + )
90 + .serve(addr);
91 +
92 + let keepalive = send_keepalive_periodically();
93 +
94 + tokio::select! {
95 + result = server => {
96 + result.with_context(|| format!("Failed to serve gRPC server on {}", addr))?;
97 + }
98 + _ = keepalive => {
99 + // This should never complete
100 + }
101 + }
102 +
103 + Ok(())
104 +}
src/crates/jf/otel-plugin/src/metrics_service.rs new
+140
@@ -0,0 +1,140 @@
1 +use anyhow::{Context, Result};
2 +use flatten_otel::flatten_metrics_request;
3 +use opentelemetry_proto::tonic::collector::metrics::v1::{
4 + metrics_service_server::MetricsService, ExportMetricsServiceRequest,
5 + ExportMetricsServiceResponse,
6 +};
7 +use std::collections::HashMap;
8 +use std::sync::Arc;
9 +use tokio::sync::RwLock;
10 +use tonic::{Request, Response, Status};
11 +
12 +use crate::chart_config::ChartConfigManager;
13 +use crate::flattened_point::FlattenedPoint;
14 +use crate::netdata_chart::NetdataChart;
15 +use crate::plugin_config::PluginConfig;
16 +use crate::regex_cache::RegexCache;
17 +
18 +#[derive(Default)]
19 +pub struct NetdataMetricsService {
20 + regex_cache: RegexCache,
21 + charts: Arc<RwLock<HashMap<String, NetdataChart>>>,
22 + config: Arc<PluginConfig>,
23 + chart_config_manager: ChartConfigManager,
24 + call_count: std::sync::atomic::AtomicU64,
25 +}
26 +
27 +impl NetdataMetricsService {
28 + pub fn new(config: PluginConfig) -> Result<Self> {
29 + let mut chart_config_manager = ChartConfigManager::with_default_configs();
30 +
31 + // Load user chart configs if directory is specified
32 + if let Some(chart_configs_dir) = &config.metrics.chart_configs_dir {
33 + chart_config_manager
34 + .load_user_configs(chart_configs_dir)
35 + .with_context(|| {
36 + format!(
37 + "Failed to load chart configs from directory: {}",
38 + chart_configs_dir
39 + )
40 + })?;
41 + }
42 +
43 + Ok(Self {
44 + regex_cache: RegexCache::default(),
45 + charts: Arc::default(),
46 + config: Arc::new(config),
47 + chart_config_manager,
48 + call_count: std::sync::atomic::AtomicU64::new(0),
49 + })
50 + }
51 +
52 + async fn cleanup_stale_charts(&self, max_age: std::time::Duration) {
53 + let now = std::time::SystemTime::now();
54 +
55 + let mut guard = self.charts.write().await;
56 + guard.retain(|_, chart| {
57 + let Some(chart_time) = chart.last_collection_time() else {
58 + return true;
59 + };
60 +
61 + now.duration_since(chart_time)
62 + .unwrap_or(std::time::Duration::ZERO)
63 + < max_age
64 + });
65 + }
66 +}
67 +
68 +#[tonic::async_trait]
69 +impl MetricsService for NetdataMetricsService {
70 + async fn export(
71 + &self,
72 + request: Request<ExportMetricsServiceRequest>,
73 + ) -> Result<Response<ExportMetricsServiceResponse>, Status> {
74 + let req = request.into_inner();
75 +
76 + let flattened_points = flatten_metrics_request(&req)
77 + .into_iter()
78 + .filter_map(|jm| {
79 + let cfg = self.chart_config_manager.find_matching_config(&jm);
80 + FlattenedPoint::new(jm, cfg, &self.regex_cache)
81 + })
82 + .collect::<Vec<_>>();
83 +
84 + if self.config.metrics.print_flattened {
85 + // Just print the flattened points
86 + for fp in &flattened_points {
87 + println!("{:#?}", fp);
88 + }
89 +
90 + return Ok(Response::new(ExportMetricsServiceResponse {
91 + partial_success: None,
92 + }));
93 + }
94 +
95 + // ingest
96 + {
97 + let mut newly_created_charts = 0;
98 +
99 + for fp in flattened_points.iter() {
100 + let mut guard = self.charts.write().await;
101 +
102 + if let Some(netdata_chart) = guard.get_mut(&fp.nd_instance_name) {
103 + netdata_chart.ingest(fp);
104 + } else if newly_created_charts < self.config.metrics.throttle_charts {
105 + let mut netdata_chart =
106 + NetdataChart::from_flattened_point(fp, self.config.metrics.buffer_samples);
107 + netdata_chart.ingest(fp);
108 + guard.insert(fp.nd_instance_name.clone(), netdata_chart);
109 +
110 + newly_created_charts += 1;
111 + }
112 + }
113 + }
114 +
115 + // process
116 + {
117 + let mut guard = self.charts.write().await;
118 +
119 + for netdata_chart in guard.values_mut() {
120 + netdata_chart.process();
121 + }
122 + }
123 +
124 + // cleanup stale charts
125 + {
126 + let prev_count = self
127 + .call_count
128 + .fetch_add(1, std::sync::atomic::Ordering::Relaxed);
129 +
130 + if prev_count % 60 == 0 {
131 + let one_hour = std::time::Duration::from_secs(3600);
132 + self.cleanup_stale_charts(one_hour).await;
133 + }
134 + }
135 +
136 + Ok(Response::new(ExportMetricsServiceResponse {
137 + partial_success: None,
138 + }))
139 + }
140 +}
src/crates/jf/otel-plugin/src/netdata_chart.rs new
+296
@@ -0,0 +1,296 @@
1 +use serde_json::{Map as JsonMap, Value as JsonValue};
2 +
3 +use crate::flattened_point::FlattenedPoint;
4 +use crate::samples_table::{CollectionInterval, SamplesTable};
5 +
6 +#[derive(Debug, Default, Clone)]
7 +enum ChartState {
8 + #[default]
9 + Uninitialized,
10 + InGap,
11 + Initialized,
12 + Empty,
13 +}
14 +
15 +#[derive(Debug)]
16 +pub struct NetdataChart {
17 + chart_id: String,
18 + metric_name: String,
19 + metric_description: String,
20 + metric_unit: String,
21 + metric_type: String,
22 + is_monotonic: Option<bool>,
23 + attributes: JsonMap<String, JsonValue>,
24 +
25 + samples_table: SamplesTable,
26 + last_samples_table_interval: Option<CollectionInterval>,
27 + last_collection_interval: Option<CollectionInterval>,
28 + chart_state: ChartState,
29 + samples_threshold: usize,
30 +
31 + multiplier: i32,
32 + divisor: i32,
33 +}
34 +
35 +impl NetdataChart {
36 + pub fn from_flattened_point(fp: &FlattenedPoint, samples_threshold: usize) -> Self {
37 + Self {
38 + chart_id: fp.nd_instance_name.clone(),
39 + metric_name: fp.metric_name.clone(),
40 + metric_description: fp.metric_description.clone(),
41 + metric_unit: fp.metric_unit.clone(),
42 + metric_type: fp.metric_type.clone(),
43 + attributes: fp.attributes.clone(),
44 + is_monotonic: fp.metric_is_monotonic,
45 +
46 + samples_table: SamplesTable::default(),
47 + last_samples_table_interval: None,
48 + last_collection_interval: None,
49 + chart_state: ChartState::Uninitialized,
50 +
51 + samples_threshold,
52 +
53 + multiplier: 1,
54 + divisor: 1,
55 + }
56 + }
57 +
58 + fn is_histogram(&self) -> bool {
59 + self.metric_type == "histogram"
60 + }
61 +
62 + pub fn ingest(&mut self, fp: &FlattenedPoint) {
63 + let dimension_name = &fp.nd_dimension_name;
64 + let value = fp.metric_value;
65 + let unix_time = fp.metric_time_unix_nano;
66 +
67 + let new_dimension = self.samples_table.insert(dimension_name, unix_time, value);
68 +
69 + if new_dimension {
70 + self.chart_state = ChartState::Uninitialized;
71 + self.last_samples_table_interval = None;
72 + self.last_collection_interval = None;
73 + }
74 + }
75 +
76 + fn initialize(&mut self) -> bool {
77 + // Clean up stale samples if we have a previous interval
78 + if let Some(ci) = &self.last_samples_table_interval {
79 + self.samples_table.drop_stale_samples(ci);
80 + }
81 +
82 + // Check if we have enough samples to determine frequency
83 + if self.samples_table.total_samples() < self.samples_threshold {
84 + return false;
85 + }
86 +
87 + // Store the old interval before calculating the new one
88 + let old_lci = self.last_collection_interval;
89 +
90 + // Set up collection intervals
91 + self.last_samples_table_interval =
92 + self.samples_table
93 + .collection_interval()
94 + .map(|ci| CollectionInterval {
95 + end_time: ci.end_time - ci.update_every.get(),
96 + update_every: ci.update_every,
97 + });
98 +
99 + self.last_collection_interval = self
100 + .last_samples_table_interval
101 + .and_then(|ci| ci.aligned_interval());
102 +
103 + (self.multiplier, self.divisor) = self.samples_table.scaling_factors();
104 +
105 + // Check if we need to emit a chart definition
106 + if let Some(new_lci) = &self.last_collection_interval {
107 + if let Some(old_lci) = old_lci {
108 + if old_lci.update_every != new_lci.update_every {
109 + // Update every changed, emit the chart definition again
110 + self.emit_chart_definition();
111 + }
112 + } else {
113 + // No previous collection interval, we need to emit the
114 + // chart definition first
115 + self.emit_chart_definition();
116 + }
117 + }
118 +
119 + true
120 + }
121 +
122 + pub fn process(&mut self) {
123 + loop {
124 + match &self.chart_state {
125 + ChartState::Uninitialized | ChartState::InGap => {
126 + if !self.initialize() {
127 + return;
128 + }
129 +
130 + self.chart_state = ChartState::Initialized;
131 + }
132 + ChartState::Initialized => {
133 + self.chart_state = self.process_next_interval();
134 + }
135 + ChartState::Empty => {
136 + self.chart_state = ChartState::Initialized;
137 + return;
138 + }
139 + }
140 + }
141 + }
142 +
143 + fn emit_chart_definition(&self) {
144 + let ci = self.last_collection_interval.unwrap();
145 + let ue = ci.update_every;
146 +
147 + let type_id = &self.chart_id;
148 + let name = "";
149 + let title = &self.metric_description;
150 + let units = &self.metric_unit;
151 + let family = &self.metric_name;
152 + let context = format!("otel.{}", &self.metric_name);
153 + let chart_type = if self.is_histogram() {
154 + "heatmap"
155 + } else {
156 + "line"
157 + };
158 + let priority = 1;
159 + let update_every = std::time::Duration::from_nanos(ue.get()).as_secs();
160 +
161 + println!(
162 + "CHART {type_id} '{name}' '{title}' '{units}' '{family}' '{context}' {chart_type} {priority} {update_every}"
163 + );
164 +
165 + for (key, value) in self.attributes.iter() {
166 + let value_str = match value {
167 + JsonValue::String(s) => s.clone(),
168 + JsonValue::Number(n) => n.to_string(),
169 + JsonValue::Bool(b) => b.to_string(),
170 + _ => continue,
171 + };
172 +
173 + println!("CLABEL '{key}' '{value_str}' 1");
174 + }
175 + println!("CLABEL_COMMIT");
176 +
177 + // Emit dimensions
178 + if self.is_histogram() {
179 + let mut dimension_names = self.samples_table.iter_dimensions().collect::<Vec<_>>();
180 +
181 + dimension_names.sort_by(|a, b| {
182 + let a_val = if *a == "+Inf" {
183 + f64::INFINITY
184 + } else {
185 + a.parse::<f64>().unwrap()
186 + };
187 + let b_val = if *b == "+Inf" {
188 + f64::INFINITY
189 + } else {
190 + b.parse::<f64>().unwrap()
191 + };
192 + a_val.partial_cmp(&b_val).unwrap()
193 + });
194 +
195 + for dimension_name in dimension_names {
196 + let algorithm = match self.is_monotonic {
197 + Some(true) => "incremental",
198 + _ => "absolute",
199 + };
200 + println!(
201 + "DIMENSION {} {} {} 1 {}",
202 + dimension_name, dimension_name, algorithm, self.divisor
203 + );
204 + }
205 + } else {
206 + for dimension_name in self.samples_table.iter_dimensions() {
207 + let algorithm = match self.is_monotonic {
208 + Some(true) => "incremental",
209 + _ => "absolute",
210 + };
211 + println!(
212 + "DIMENSION {} {} {} 1 {}",
213 + dimension_name, dimension_name, algorithm, self.divisor
214 + );
215 + }
216 + }
217 + }
218 +
219 + fn process_next_interval(&mut self) -> ChartState {
220 + let lsti = match &self.last_samples_table_interval {
221 + Some(interval) => interval,
222 + None => return ChartState::Empty,
223 + };
224 +
225 + let lci = match &self.last_collection_interval {
226 + Some(interval) => interval,
227 + None => return ChartState::Empty,
228 + };
229 +
230 + // Clean stale samples
231 + self.samples_table.drop_stale_samples(lsti);
232 + if self.samples_table.is_empty() {
233 + return ChartState::Empty;
234 + }
235 +
236 + // Check for gaps
237 + let have_gap = self
238 + .samples_table
239 + .iter_samples_buffers()
240 + .all(|sb| sb.first().is_none_or(|sp| lsti.is_in_gap(sp)));
241 +
242 + if have_gap {
243 + return ChartState::InGap;
244 + }
245 +
246 + // Collect samples to emit
247 + let mut samples_to_emit = Vec::new();
248 + for (dimension_name, sb) in &mut self.samples_table.iter_mut() {
249 + if let Some(sp) = sb.first() {
250 + if lsti.is_on_time(sp) {
251 + if let Some(sample) = sb.pop() {
252 + samples_to_emit.push((dimension_name.clone(), sample.value));
253 + }
254 + }
255 + }
256 + }
257 +
258 + // Emit data if we have samples
259 + if !samples_to_emit.is_empty() {
260 + self.emit_begin(lci.update_every.get());
261 + for (dimension_name, value) in samples_to_emit {
262 + self.emit_set(&dimension_name, value);
263 + }
264 + self.emit_end();
265 + }
266 +
267 + // Move to next interval
268 + self.last_samples_table_interval = Some(lsti.next_interval());
269 + self.last_collection_interval = Some(lci.next_interval());
270 +
271 + ChartState::Initialized
272 + }
273 +
274 + fn emit_begin(&self, update_every: u64) {
275 + let ue = std::time::Duration::from_nanos(update_every).as_micros() as u64;
276 + println!("BEGIN {} {}", self.chart_id, ue);
277 + }
278 +
279 + fn emit_set(&self, dimension_name: &str, value: f64) {
280 + println!("SET {} {}", dimension_name, value * self.divisor as f64);
281 + }
282 +
283 + fn emit_end(&self) {
284 + let collection_time = std::time::Duration::from_nanos(
285 + self.last_collection_interval.unwrap().collection_time(),
286 + )
287 + .as_secs();
288 + println!("END {collection_time}");
289 + }
290 +
291 + pub fn last_collection_time(&self) -> Option<std::time::SystemTime> {
292 + self.last_collection_interval.as_ref().map(|lci| {
293 + std::time::UNIX_EPOCH + std::time::Duration::from_nanos(lci.collection_time())
294 + })
295 + }
296 +}
src/crates/jf/otel-plugin/src/netdata_env.rs new
+204
@@ -0,0 +1,204 @@
1 +#![allow(dead_code)]
2 +
3 +use std::env;
4 +use std::path::PathBuf;
5 +
6 +#[derive(Debug, Clone, Default)]
7 +pub struct NetdataEnv {
8 + pub user_config_dir: Option<PathBuf>,
9 + pub stock_config_dir: Option<PathBuf>,
10 + pub plugins_dir: Option<PathBuf>,
11 + pub user_plugins_dirs: Option<Vec<PathBuf>>,
12 + pub web_dir: Option<PathBuf>,
13 + pub cache_dir: Option<PathBuf>,
14 + pub log_dir: Option<PathBuf>,
15 + pub host_prefix: Option<String>,
16 + pub debug_flags: Option<String>,
17 + pub update_every: Option<u64>,
18 + pub invocation_id: Option<String>,
19 + pub log_method: Option<LogMethod>,
20 + pub log_format: Option<LogFormat>,
21 + pub log_level: Option<LogLevel>,
22 + pub syslog_facility: Option<SyslogFacility>,
23 + pub errors_throttle_period: Option<u64>,
24 + pub errors_per_period: Option<u64>,
25 + pub systemd_journal_path: Option<PathBuf>,
26 +}
27 +
28 +#[derive(Debug, Clone)]
29 +pub enum LogMethod {
30 + Syslog,
31 + Journal,
32 + Stderr,
33 + None,
34 +}
35 +
36 +#[derive(Debug, Clone)]
37 +pub enum LogFormat {
38 + Journal,
39 + Logfmt,
40 + Json,
41 +}
42 +
43 +#[derive(Debug, Clone)]
44 +pub enum LogLevel {
45 + Emergency,
46 + Alert,
47 + Critical,
48 + Error,
49 + Warning,
50 + Notice,
51 + Info,
52 + Debug,
53 +}
54 +
55 +#[derive(Debug, Clone)]
56 +pub enum SyslogFacility {
57 + Auth,
58 + Authpriv,
59 + Cron,
60 + Daemon,
61 + Ftp,
62 + Kern,
63 + Lpr,
64 + Mail,
65 + News,
66 + Syslog,
67 + User,
68 + Uucp,
69 + Local0,
70 + Local1,
71 + Local2,
72 + Local3,
73 + Local4,
74 + Local5,
75 + Local6,
76 + Local7,
77 +}
78 +
79 +impl NetdataEnv {
80 + pub fn from_environment() -> Self {
81 + Self {
82 + user_config_dir: env::var("NETDATA_USER_CONFIG_DIR").ok().map(PathBuf::from),
83 + stock_config_dir: env::var("NETDATA_STOCK_CONFIG_DIR").ok().map(PathBuf::from),
84 + plugins_dir: env::var("NETDATA_PLUGINS_DIR").ok().map(PathBuf::from),
85 + user_plugins_dirs: env::var("NETDATA_USER_PLUGINS_DIRS")
86 + .ok()
87 + .map(|s| s.split(':').map(PathBuf::from).collect()),
88 + web_dir: env::var("NETDATA_WEB_DIR").ok().map(PathBuf::from),
89 + cache_dir: env::var("NETDATA_CACHE_DIR").ok().map(PathBuf::from),
90 + log_dir: env::var("NETDATA_LOG_DIR").ok().map(PathBuf::from),
91 + host_prefix: env::var("NETDATA_HOST_PREFIX").ok(),
92 + debug_flags: env::var("NETDATA_DEBUG_FLAGS").ok(),
93 + update_every: env::var("NETDATA_UPDATE_EVERY")
94 + .ok()
95 + .and_then(|s| s.parse().ok()),
96 + invocation_id: env::var("NETDATA_INVOCATION_ID").ok(),
97 + log_method: env::var("NETDATA_LOG_METHOD")
98 + .ok()
99 + .and_then(|s| s.parse().ok()),
100 + log_format: env::var("NETDATA_LOG_FORMAT")
101 + .ok()
102 + .and_then(|s| s.parse().ok()),
103 + log_level: env::var("NETDATA_LOG_LEVEL")
104 + .ok()
105 + .and_then(|s| s.parse().ok()),
106 + syslog_facility: env::var("NETDATA_SYSLOG_FACILITY")
107 + .ok()
108 + .and_then(|s| s.parse().ok()),
109 + errors_throttle_period: env::var("NETDATA_ERRORS_THROTTLE_PERIOD")
110 + .ok()
111 + .and_then(|s| s.parse().ok()),
112 + errors_per_period: env::var("NETDATA_ERRORS_PER_PERIOD")
113 + .ok()
114 + .and_then(|s| s.parse().ok()),
115 + systemd_journal_path: env::var("NETDATA_SYSTEMD_JOURNAL_PATH")
116 + .ok()
117 + .map(PathBuf::from),
118 + }
119 + }
120 +
121 + pub fn running_under_netdata(&self) -> bool {
122 + // we are overtly cautious, just one check would suffice
123 + self.user_config_dir.is_some()
124 + || self.stock_config_dir.is_some()
125 + || self.plugins_dir.is_some()
126 + || self.invocation_id.is_some()
127 + }
128 +}
129 +
130 +// Implement FromStr for the enums
131 +impl std::str::FromStr for LogMethod {
132 + type Err = String;
133 +
134 + fn from_str(s: &str) -> Result<Self, Self::Err> {
135 + match s.to_lowercase().as_str() {
136 + "syslog" => Ok(LogMethod::Syslog),
137 + "journal" => Ok(LogMethod::Journal),
138 + "stderr" => Ok(LogMethod::Stderr),
139 + "none" => Ok(LogMethod::None),
140 + _ => Err(format!("Invalid log method: {}", s)),
141 + }
142 + }
143 +}
144 +
145 +impl std::str::FromStr for LogFormat {
146 + type Err = String;
147 +
148 + fn from_str(s: &str) -> Result<Self, Self::Err> {
149 + match s.to_lowercase().as_str() {
150 + "journal" => Ok(LogFormat::Journal),
151 + "logfmt" => Ok(LogFormat::Logfmt),
152 + "json" => Ok(LogFormat::Json),
153 + _ => Err(format!("Invalid log format: {}", s)),
154 + }
155 + }
156 +}
157 +
158 +impl std::str::FromStr for LogLevel {
159 + type Err = String;
160 +
161 + fn from_str(s: &str) -> Result<Self, Self::Err> {
162 + match s.to_lowercase().as_str() {
163 + "emergency" => Ok(LogLevel::Emergency),
164 + "alert" => Ok(LogLevel::Alert),
165 + "critical" => Ok(LogLevel::Critical),
166 + "error" => Ok(LogLevel::Error),
167 + "warning" => Ok(LogLevel::Warning),
168 + "notice" => Ok(LogLevel::Notice),
169 + "info" => Ok(LogLevel::Info),
170 + "debug" => Ok(LogLevel::Debug),
171 + _ => Err(format!("Invalid log level: {}", s)),
172 + }
173 + }
174 +}
175 +
176 +impl std::str::FromStr for SyslogFacility {
177 + type Err = String;
178 +
179 + fn from_str(s: &str) -> Result<Self, Self::Err> {
180 + match s.to_lowercase().as_str() {
181 + "auth" => Ok(SyslogFacility::Auth),
182 + "authpriv" => Ok(SyslogFacility::Authpriv),
183 + "cron" => Ok(SyslogFacility::Cron),
184 + "daemon" => Ok(SyslogFacility::Daemon),
185 + "ftp" => Ok(SyslogFacility::Ftp),
186 + "kern" => Ok(SyslogFacility::Kern),
187 + "lpr" => Ok(SyslogFacility::Lpr),
188 + "mail" => Ok(SyslogFacility::Mail),
189 + "news" => Ok(SyslogFacility::News),
190 + "syslog" => Ok(SyslogFacility::Syslog),
191 + "user" => Ok(SyslogFacility::User),
192 + "uucp" => Ok(SyslogFacility::Uucp),
193 + "local0" => Ok(SyslogFacility::Local0),
194 + "local1" => Ok(SyslogFacility::Local1),
195 + "local2" => Ok(SyslogFacility::Local2),
196 + "local3" => Ok(SyslogFacility::Local3),
197 + "local4" => Ok(SyslogFacility::Local4),
198 + "local5" => Ok(SyslogFacility::Local5),
199 + "local6" => Ok(SyslogFacility::Local6),
200 + "local7" => Ok(SyslogFacility::Local7),
201 + _ => Err(format!("Invalid syslog facility: {}", s)),
202 + }
203 + }
204 +}
src/crates/jf/otel-plugin/src/plugin_config.rs new
+299
@@ -0,0 +1,299 @@
1 +use crate::netdata_env::NetdataEnv;
2 +
3 +use anyhow::{Context, Result};
4 +use bytesize::ByteSize;
5 +use clap::Parser;
6 +use serde::{Deserialize, Serialize};
7 +use std::fs;
8 +use std::path::Path;
9 +use std::time::Duration;
10 +
11 +#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
12 +#[serde(deny_unknown_fields)]
13 +pub struct EndpointConfig {
14 + /// gRPC endpoint to listen on
15 + #[arg(long = "otel-endpoint", default_value = "127.0.0.1:4317")]
16 + pub path: String,
17 +
18 + /// Path to TLS certificate file (enables TLS when provided)
19 + #[arg(long = "otel-tls-cert-path")]
20 + pub tls_cert_path: Option<String>,
21 +
22 + /// Path to TLS private key file (required when TLS certificate is provided)
23 + #[arg(long = "otel-tls-key-path")]
24 + pub tls_key_path: Option<String>,
25 +
26 + /// Path to TLS CA certificate file for client authentication (optional)
27 + #[arg(long = "otel-tls-ca-cert-path")]
28 + pub tls_ca_cert_path: Option<String>,
29 +}
30 +
31 +impl Default for EndpointConfig {
32 + fn default() -> Self {
33 + Self {
34 + path: String::from("127.0.0.1:4317"),
35 + tls_cert_path: None,
36 + tls_key_path: None,
37 + tls_ca_cert_path: None,
38 + }
39 + }
40 +}
41 +
42 +#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
43 +#[serde(deny_unknown_fields)]
44 +pub struct MetricsConfig {
45 + /// Print flattened metrics to stdout for debugging
46 + #[arg(long = "otel-metrics-print-flattened")]
47 + pub print_flattened: bool,
48 +
49 + /// Number of samples to buffer for collection interval detection
50 + #[arg(long = "otel-metrics-buffer-samples", default_value = "10")]
51 + pub buffer_samples: usize,
52 +
53 + /// Maximum number of new charts to create per collection interval
54 + #[arg(long = "otel-metrics-throttle-charts", default_value = "100")]
55 + pub throttle_charts: usize,
56 +
57 + /// Directory with configuration files for mapping OTEL metrics to Netdata charts
58 + #[arg(long = "otel-metrics-charts-configs-dir")]
59 + pub chart_configs_dir: Option<String>,
60 +}
61 +
62 +impl Default for MetricsConfig {
63 + fn default() -> Self {
64 + Self {
65 + print_flattened: false,
66 + buffer_samples: 10,
67 + throttle_charts: 100,
68 + chart_configs_dir: None,
69 + }
70 + }
71 +}
72 +
73 +/// Parse a duration string for clap (e.g., "7 days", "1 week", "168h")
74 +fn parse_duration(s: &str) -> Result<Duration, String> {
75 + humantime::parse_duration(s).map_err(|e| {
76 + format!(
77 + "Invalid duration format: '{}'. Use formats like '7 days', '1 week', '168h'. Error: {}",
78 + s, e
79 + )
80 + })
81 +}
82 +
83 +/// Parse a bytesize string for clap (e.g., "100MB", "1.5GB", "512MiB")
84 +fn parse_bytesize(s: &str) -> Result<ByteSize, String> {
85 + s.parse().map_err(|e| {
86 + format!(
87 + "Invalid size format: '{}'. Use formats like '100MB', '1.5GB', '512MiB'. Error: {}",
88 + s, e
89 + )
90 + })
91 +}
92 +
93 +#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
94 +#[serde(deny_unknown_fields)]
95 +pub struct LogsConfig {
96 + /// Directory to store journal files for logs
97 + #[arg(long = "otel-logs-journal-dir")]
98 + pub journal_dir: String,
99 +
100 + /// Maximum file size for journal files (accepts human-readable sizes like "100MB", "1.5GB")
101 + #[arg(
102 + long = "otel-logs-rotation-size-of-journal-file",
103 + default_value = "100MB",
104 + value_parser = parse_bytesize
105 + )]
106 + #[serde(with = "bytesize_serde")]
107 + pub size_of_journal_file: ByteSize,
108 +
109 + /// Maximum number of journal files to keep
110 + #[arg(
111 + long = "otel-logs-retention-number-of-journal-files",
112 + default_value = "10"
113 + )]
114 + pub number_of_journal_files: usize,
115 +
116 + /// Maximum total size for all journal files (accepts human-readable sizes like "1GB", "500MB")
117 + #[arg(
118 + long = "otel-logs-retention-size-of-journal-files",
119 + default_value = "1GB",
120 + value_parser = parse_bytesize
121 + )]
122 + #[serde(with = "bytesize_serde")]
123 + pub size_of_journal_files: ByteSize,
124 +
125 + /// Maximum age for journal entries (accepts human-readable durations like "7 days", "1 week", "168h")
126 + #[arg(
127 + long = "otel-logs-retention-duration-of-journal-files",
128 + default_value = "7 days",
129 + value_parser = parse_duration
130 + )]
131 + #[serde(with = "humantime_serde")]
132 + pub duration_of_journal_files: Duration,
133 +
134 + /// Maximum duration that entries in a single journal file can span (accepts human-readable durations like "2 hours", "1h", "30m")
135 + #[arg(
136 + long = "otel-logs-rotation-duration-of-journal-file",
137 + default_value = "2 hours",
138 + value_parser = parse_duration
139 + )]
140 + #[serde(with = "humantime_serde")]
141 + pub duration_of_journal_file: Duration,
142 +}
143 +
144 +impl Default for LogsConfig {
145 + fn default() -> Self {
146 + Self {
147 + journal_dir: String::from("/tmp/netdata-journals"),
148 + size_of_journal_file: ByteSize::mb(100),
149 + number_of_journal_files: 10,
150 + size_of_journal_files: ByteSize::gb(1),
151 + duration_of_journal_files: Duration::from_secs(7 * 24 * 60 * 60), // 7 days
152 + duration_of_journal_file: Duration::from_secs(2 * 60 * 60), // 2 hours
153 + }
154 + }
155 +}
156 +
157 +#[derive(Default, Debug, Parser, Clone, Serialize, Deserialize)]
158 +#[command(name = "otel-plugin")]
159 +#[command(about = "OpenTelemetry metrics and logs plugin.")]
160 +#[command(version = "0.1")]
161 +#[serde(deny_unknown_fields)]
162 +pub struct PluginConfig {
163 + // endpoint configuration (includes grpc endpoint and tls)
164 + #[command(flatten)]
165 + #[serde(rename = "endpoint")]
166 + pub endpoint: EndpointConfig,
167 +
168 + // metrics
169 + #[command(flatten)]
170 + #[serde(rename = "metrics")]
171 + pub metrics: MetricsConfig,
172 +
173 + // logs
174 + #[command(flatten)]
175 + #[serde(rename = "logs")]
176 + pub logs: LogsConfig,
177 +
178 + /// Collection interval (ignored)
179 + #[arg(hide = true, help = "Collection interval in seconds (ignored)")]
180 + #[serde(skip)]
181 + pub _update_frequency: Option<u32>,
182 +
183 + // netdata env variables
184 + #[arg(skip)]
185 + #[serde(skip)]
186 + pub _netdata_env: NetdataEnv,
187 +}
188 +
189 +impl PluginConfig {
190 + pub fn new() -> Result<Self> {
191 + let netdata_env = NetdataEnv::from_environment();
192 +
193 + let mut config = if netdata_env.running_under_netdata() {
194 + // Try user config first, fallback to stock config
195 + let user_config = netdata_env
196 + .user_config_dir
197 + .as_ref()
198 + .map(|path| path.join("otel.yml"))
199 + .and_then(|path| {
200 + Self::from_yaml_file(&path)
201 + .with_context(|| format!("Loading user config from {}", path.display()))
202 + .ok()
203 + });
204 +
205 + if let Some(config) = user_config {
206 + config
207 + } else if let Some(stock_path) = netdata_env
208 + .stock_config_dir
209 + .as_ref()
210 + .map(|p| p.join("otel.yml"))
211 + {
212 + Self::from_yaml_file(&stock_path).with_context(|| {
213 + format!("Loading stock config from {}", stock_path.display())
214 + })?
215 + } else {
216 + anyhow::bail!("No configuration directories available");
217 + }
218 + } else {
219 + // load from CLI args
220 + Self::parse()
221 + };
222 +
223 + // Resolve relative paths
224 + if let Some(charts_config_dir) = &mut config.metrics.chart_configs_dir {
225 + *charts_config_dir =
226 + resolve_relative_path(charts_config_dir, netdata_env.user_config_dir.as_deref());
227 + }
228 + config.logs.journal_dir =
229 + resolve_relative_path(&config.logs.journal_dir, netdata_env.log_dir.as_deref());
230 +
231 + // Validate configuration
232 + if config.metrics.buffer_samples == 0 {
233 + anyhow::bail!("buffer_samples must be greater than 0");
234 + }
235 +
236 + if config.metrics.throttle_charts == 0 {
237 + anyhow::bail!("throttle_charts must be greater than 0");
238 + }
239 +
240 + // Validate endpoint format (basic check)
241 + if !config.endpoint.path.contains(':') {
242 + anyhow::bail!(
243 + "endpoint must be in format host:port, got: {}",
244 + config.endpoint.path
245 + );
246 + }
247 +
248 + // Validate TLS configuration
249 + match (
250 + &config.endpoint.tls_cert_path,
251 + &config.endpoint.tls_key_path,
252 + ) {
253 + (Some(cert_path), Some(key_path)) => {
254 + if cert_path.is_empty() {
255 + anyhow::bail!("TLS certificate path cannot be empty when provided");
256 + }
257 + if key_path.is_empty() {
258 + anyhow::bail!("TLS private key path cannot be empty when provided");
259 + }
260 + }
261 + (Some(_), None) => {
262 + anyhow::bail!(
263 + "TLS private key path must be provided when TLS certificate is provided"
264 + );
265 + }
266 + (None, Some(_)) => {
267 + anyhow::bail!(
268 + "TLS certificate path must be provided when TLS private key is provided"
269 + );
270 + }
271 + (None, None) => {
272 + // TLS disabled, which is fine
273 + }
274 + }
275 +
276 + Ok(config)
277 + }
278 +
279 + pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self> {
280 + let path = path.as_ref();
281 + let contents = fs::read_to_string(path)
282 + .with_context(|| format!("Failed to read config file: {}", path.display()))?;
283 + let config: PluginConfig = serde_yaml::from_str(&contents)
284 + .with_context(|| format!("Failed to parse YAML config file: {}", path.display()))?;
285 + Ok(config)
286 + }
287 +}
288 +
289 +// Helper function to resolve relative paths
290 +fn resolve_relative_path(path: &str, base_dir: Option<&Path>) -> String {
291 + let path = Path::new(path);
292 + if path.is_absolute() {
293 + path.to_string_lossy().to_string()
294 + } else if let Some(base) = base_dir {
295 + base.join(path).to_string_lossy().to_string()
296 + } else {
297 + path.to_string_lossy().to_string()
298 + }
299 +}
src/crates/jf/otel-plugin/src/regex_cache.rs new
+22
@@ -0,0 +1,22 @@
1 +use regex::Regex;
2 +use std::collections::HashMap;
3 +use std::sync::{Arc, Mutex};
4 +
5 +#[derive(Default, Debug)]
6 +pub struct RegexCache {
7 + cache: Arc<Mutex<HashMap<String, Regex>>>,
8 +}
9 +
10 +impl RegexCache {
11 + pub fn get(&self, pattern: &str) -> Result<Regex, regex::Error> {
12 + let mut cache = self.cache.lock().unwrap();
13 +
14 + if let Some(regex) = cache.get(pattern) {
15 + return Ok(regex.clone());
16 + }
17 +
18 + let compiled_regex = Regex::new(pattern)?;
19 + cache.insert(pattern.to_string(), compiled_regex.clone());
20 + Ok(compiled_regex)
21 + }
22 +}
src/crates/jf/otel-plugin/src/samples_table.rs new
+221
@@ -0,0 +1,221 @@
1 +use std::collections::HashMap;
2 +use std::num::NonZeroU64;
3 +
4 +#[derive(Default, Clone, Copy, PartialEq, Debug)]
5 +pub struct SamplePoint {
6 + unix_time: u64,
7 + pub value: f64,
8 +}
9 +
10 +#[derive(Copy, Clone, Debug)]
11 +pub struct CollectionInterval {
12 + pub end_time: u64,
13 + pub update_every: NonZeroU64,
14 +}
15 +
16 +impl CollectionInterval {
17 + fn from_samples(sample_points: &[SamplePoint]) -> Option<Self> {
18 + if sample_points.len() < 2 {
19 + return None;
20 + }
21 +
22 + let collection_time = sample_points[0].unix_time;
23 + let mut update_every = u64::MAX;
24 +
25 + for w in sample_points.windows(2) {
26 + update_every = update_every.min(w[1].unix_time - w[0].unix_time);
27 + }
28 +
29 + NonZeroU64::new(update_every).map(|update_every| Self {
30 + end_time: collection_time,
31 + update_every,
32 + })
33 + }
34 +
35 + pub fn next_interval(&self) -> Self {
36 + Self {
37 + end_time: self.end_time + self.update_every.get(),
38 + update_every: self.update_every,
39 + }
40 + }
41 +
42 + pub fn collection_time(&self) -> u64 {
43 + self.end_time + self.update_every.get()
44 + }
45 +
46 + fn is_stale(&self, sp: &SamplePoint) -> bool {
47 + sp.unix_time < self.end_time
48 + }
49 +
50 + pub fn is_on_time(&self, sp: &SamplePoint) -> bool {
51 + let window = self.update_every.get() / 4;
52 + let window_start = self.end_time + self.update_every.get() - window;
53 + let window_end = self.end_time + self.update_every.get() + window;
54 +
55 + sp.unix_time >= window_start && sp.unix_time <= window_end
56 + }
57 +
58 + pub fn is_in_gap(&self, sp: &SamplePoint) -> bool {
59 + !self.is_stale(sp) && !self.is_on_time(sp)
60 + }
61 +
62 + pub fn aligned_interval(&self) -> Option<Self> {
63 + let dur = std::time::Duration::from_nanos(self.end_time);
64 + let end_time = dur.as_secs() + u64::from(dur.subsec_millis() >= 500);
65 +
66 + let dur = std::time::Duration::from_nanos(self.update_every.get());
67 + let update_every = dur.as_secs() + u64::from(dur.subsec_millis() >= 500);
68 +
69 + Self::from_secs(end_time, update_every)
70 + }
71 +
72 + fn from_secs(end_time: u64, update_every: u64) -> Option<Self> {
73 + let end_time = std::time::Duration::from_secs(end_time).as_nanos() as u64;
74 + let update_every = std::time::Duration::from_secs(update_every).as_nanos() as u64;
75 +
76 + NonZeroU64::new(update_every).map(|update_every| Self {
77 + end_time,
78 + update_every,
79 + })
80 + }
81 +}
82 +
83 +#[derive(Debug, Default, Clone)]
84 +pub struct SamplesBuffer(Vec<SamplePoint>);
85 +
86 +impl SamplesBuffer {
87 + pub fn push(&mut self, sp: SamplePoint) {
88 + match self.0.binary_search_by_key(&sp.unix_time, |p| p.unix_time) {
89 + Ok(idx) => self.0[idx] = sp,
90 + Err(idx) => self.0.insert(idx, sp),
91 + }
92 + }
93 +
94 + pub fn pop(&mut self) -> Option<SamplePoint> {
95 + if self.0.is_empty() {
96 + None
97 + } else {
98 + Some(self.0.remove(0))
99 + }
100 + }
101 +
102 + fn is_empty(&self) -> bool {
103 + self.0.is_empty()
104 + }
105 +
106 + pub fn first(&self) -> Option<&SamplePoint> {
107 + self.0.first()
108 + }
109 +
110 + pub fn len(&self) -> usize {
111 + self.0.len()
112 + }
113 +
114 + pub fn drop_stale_samples(&mut self, ci: &CollectionInterval) -> usize {
115 + let split_idx = self
116 + .0
117 + .iter()
118 + .position(|sp| !ci.is_stale(sp))
119 + .unwrap_or(self.0.len());
120 +
121 + self.0.drain(..split_idx);
122 +
123 + split_idx
124 + }
125 +
126 + pub fn collection_interval(&self) -> Option<CollectionInterval> {
127 + CollectionInterval::from_samples(&self.0)
128 + }
129 +}
130 +
131 +#[derive(Debug, Default)]
132 +pub struct SamplesTable {
133 + dimensions: HashMap<String, SamplesBuffer>,
134 +}
135 +
136 +impl SamplesTable {
137 + pub fn insert(&mut self, dimension: &str, unix_time: u64, value: f64) -> bool {
138 + let sp = SamplePoint { unix_time, value };
139 +
140 + let is_new_dimension = if let Some(sb) = self.dimensions.get_mut(dimension) {
141 + sb.push(sp);
142 + false
143 + } else {
144 + let mut sb = SamplesBuffer::default();
145 + sb.push(sp);
146 + self.dimensions.insert(dimension.to_string(), sb);
147 + true
148 + };
149 +
150 + is_new_dimension
151 + }
152 +
153 + pub fn is_empty(&self) -> bool {
154 + self.dimensions.values().all(|sb| sb.is_empty())
155 + }
156 +
157 + pub fn total_samples(&self) -> usize {
158 + self.dimensions
159 + .values()
160 + .map(|sb| sb.len())
161 + .max()
162 + .unwrap_or(0)
163 + }
164 +
165 + pub fn drop_stale_samples(&mut self, ci: &CollectionInterval) -> usize {
166 + let mut dropped_samples = 0;
167 +
168 + for sb in self.dimensions.values_mut() {
169 + dropped_samples += sb.drop_stale_samples(ci);
170 + }
171 +
172 + dropped_samples
173 + }
174 +
175 + pub fn collection_interval(&self) -> Option<CollectionInterval> {
176 + self.dimensions
177 + .values()
178 + .filter_map(|sb| sb.collection_interval())
179 + .min_by_key(|ci| ci.collection_time())
180 + }
181 +
182 + pub fn iter_mut(&mut self) -> impl Iterator<Item = (&String, &mut SamplesBuffer)> {
183 + self.dimensions.iter_mut()
184 + }
185 +
186 + pub fn iter_dimensions(&self) -> impl Iterator<Item = &String> {
187 + self.dimensions.keys()
188 + }
189 +
190 + pub fn iter_samples_buffers(&self) -> impl Iterator<Item = &SamplesBuffer> {
191 + self.dimensions.values()
192 + }
193 +
194 + // returns multiplier/divisor
195 + pub fn scaling_factors(&self) -> (i32, i32) {
196 + let mut has_nonzero = false;
197 +
198 + for buffer in self.dimensions.values() {
199 + for sample in &buffer.0 {
200 + let value = sample.value;
201 +
202 + // Check if value is outside the -100 to 100 range
203 + if !(-100.0..=100.0).contains(&value) {
204 + return (1, 1);
205 + }
206 +
207 + // Check for non-zero values
208 + if value != 0.0 {
209 + has_nonzero = true;
210 + }
211 + }
212 + }
213 +
214 + // Return 1/1000 scaling if all values are in range and at least one is non-zero
215 + if has_nonzero {
216 + (1, 1000)
217 + } else {
218 + (1, 1)
219 + }
220 + }
221 +}
src/crates/jf/sigbus/Cargo.toml
+1
@@ -2,6 +2,7 @@
2 name = "sigbus"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 error = { path = "../error" }
src/crates/jf/window_manager/Cargo.toml
+1
@@ -2,6 +2,7 @@
2 name = "window_manager"
3 version.workspace = true
4 edition.workspace = true
5 +rust-version.workspace = true
6
7 [dependencies]
8 error = { path = "../error" }
src/libnetdata/functions_evloop/functions_evloop.h
+3
@@ -82,6 +82,9 @@
82 #define PLUGINSD_KEYWORD_JSON_CMD_STREAM_PATH "STREAM_PATH"
83 #define PLUGINSD_KEYWORD_JSON_CMD_ML_MODEL "ML_MODEL"
84
85 +// trust BEGIN timestamps from the plugin
86 +#define PLUGINSD_KEYWORD_TRUST_DURATIONS "TRUST_DURATIONS"
87 +
88 typedef void (*functions_evloop_worker_execute_t)(const char *transaction, char *function, usec_t *stop_monotonic_ut,
89 bool *cancelled, BUFFER *payload, HTTP_ACCESS access,
90 const char *source, void *data);
src/plugins.d/gperf-config.txt
+21 -17
@@ -22,6 +22,8 @@
22 #define PLUGINSD_KEYWORD_ID_SET 11
23 #define PLUGINSD_KEYWORD_ID_VARIABLE 53
24 #define PLUGINSD_KEYWORD_ID_CONFIG 100
25 +#define PLUGINSD_KEYWORD_ID_TRUST_DURATIONS 101
26 +#define PLUGINSD_KEYWORD_ID_PLUGIN_KEEPALIVE 102
27
28 #define PLUGINSD_KEYWORD_ID_CLAIMED_ID 61
29 #define PLUGINSD_KEYWORD_ID_BEGIN2 2
@@ -87,32 +89,34 @@ OVERWRITE, PLUGINSD_KEYWORD_ID_OVERWRITE, PARSER_INIT_PL
89 SET, PLUGINSD_KEYWORD_ID_SET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 19
90 VARIABLE, PLUGINSD_KEYWORD_ID_VARIABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20
91 CONFIG, PLUGINSD_KEYWORD_ID_CONFIG, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 21
92 +TRUST_DURATIONS, PLUGINSD_KEYWORD_ID_TRUST_DURATIONS, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22
93 +PLUGIN_KEEPALIVE, PLUGINSD_KEYWORD_ID_PLUGIN_KEEPALIVE, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 23
94 #
95 # Streaming only keywords
96 #
93 -CLAIMED_ID, PLUGINSD_KEYWORD_ID_CLAIMED_ID, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22
94 -BEGIN2, PLUGINSD_KEYWORD_ID_BEGIN2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 23
95 -SET2, PLUGINSD_KEYWORD_ID_SET2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 24
96 -END2, PLUGINSD_KEYWORD_ID_END2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 25
97 +CLAIMED_ID, PLUGINSD_KEYWORD_ID_CLAIMED_ID, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 24
98 +BEGIN2, PLUGINSD_KEYWORD_ID_BEGIN2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 25
99 +SET2, PLUGINSD_KEYWORD_ID_SET2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 26
100 +END2, PLUGINSD_KEYWORD_ID_END2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 27
101 #
102 # Streaming Replication keywords
103 #
100 -CHART_DEFINITION_END, PLUGINSD_KEYWORD_ID_CHART_DEFINITION_END, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26
101 -RBEGIN, PLUGINSD_KEYWORD_ID_RBEGIN, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27
102 -RSET, PLUGINSD_KEYWORD_ID_RSET, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 28
103 -REND, PLUGINSD_KEYWORD_ID_REND, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 29
104 -RDSTATE, PLUGINSD_KEYWORD_ID_RDSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30
105 -RSSTATE, PLUGINSD_KEYWORD_ID_RSSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31
104 +CHART_DEFINITION_END, PLUGINSD_KEYWORD_ID_CHART_DEFINITION_END, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 28
105 +RBEGIN, PLUGINSD_KEYWORD_ID_RBEGIN, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 29
106 +RSET, PLUGINSD_KEYWORD_ID_RSET, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 30
107 +REND, PLUGINSD_KEYWORD_ID_REND, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31
108 +RDSTATE, PLUGINSD_KEYWORD_ID_RDSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 32
109 +RSSTATE, PLUGINSD_KEYWORD_ID_RSSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 33
110 #
111 # JSON
112 #
109 -JSON, PLUGINSD_KEYWORD_ID_JSON, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 32
113 +JSON, PLUGINSD_KEYWORD_ID_JSON, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 34
114 #
115 # obsolete - do nothing commands
116 #
113 -DYNCFG_ENABLE, PLUGINSD_KEYWORD_ID_DYNCFG_ENABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33
114 -DYNCFG_REGISTER_MODULE, PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_MODULE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34
115 -DYNCFG_REGISTER_JOB, PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
116 -DYNCFG_RESET, PLUGINSD_KEYWORD_ID_DYNCFG_RESET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36
117 -REPORT_JOB_STATUS, PLUGINSD_KEYWORD_ID_REPORT_JOB_STATUS, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37
118 -DELETE_JOB, PLUGINSD_KEYWORD_ID_DELETE_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 38
117 +DYNCFG_ENABLE, PLUGINSD_KEYWORD_ID_DYNCFG_ENABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35
118 +DYNCFG_REGISTER_MODULE, PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_MODULE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36
119 +DYNCFG_REGISTER_JOB, PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37
120 +DYNCFG_RESET, PLUGINSD_KEYWORD_ID_DYNCFG_RESET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 38
121 +REPORT_JOB_STATUS, PLUGINSD_KEYWORD_ID_REPORT_JOB_STATUS, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 39
122 +DELETE_JOB, PLUGINSD_KEYWORD_ID_DELETE_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 40
src/plugins.d/gperf-hashtable.h
+63 -59
@@ -54,6 +54,8 @@
54 #define PLUGINSD_KEYWORD_ID_SET 11
55 #define PLUGINSD_KEYWORD_ID_VARIABLE 53
56 #define PLUGINSD_KEYWORD_ID_CONFIG 100
57 +#define PLUGINSD_KEYWORD_ID_TRUST_DURATIONS 101
58 +#define PLUGINSD_KEYWORD_ID_PLUGIN_KEEPALIVE 102
59
60 #define PLUGINSD_KEYWORD_ID_CLAIMED_ID 61
61 #define PLUGINSD_KEYWORD_ID_BEGIN2 2
@@ -77,7 +79,7 @@
79 #define PLUGINSD_KEYWORD_ID_DELETE_JOB 906
80
81
80 -#define GPERF_PARSER_TOTAL_KEYWORDS 38
82 +#define GPERF_PARSER_TOTAL_KEYWORDS 40
83 #define GPERF_PARSER_MIN_WORD_LENGTH 3
84 #define GPERF_PARSER_MAX_WORD_LENGTH 22
85 #define GPERF_PARSER_MIN_HASH_VALUE 4
@@ -103,8 +105,8 @@ gperf_keyword_hash_function (register const char *str, register size_t len)
105 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
106 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
107 54, 54, 54, 54, 54, 31, 28, 2, 4, 0,
106 - 5, 54, 0, 25, 20, 54, 17, 54, 27, 0,
107 - 54, 54, 1, 16, 54, 15, 0, 54, 2, 0,
108 + 5, 54, 0, 25, 23, 54, 17, 54, 27, 0,
109 + 9, 54, 1, 16, 24, 15, 0, 54, 2, 0,
110 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
111 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
112 54, 54, 54, 54, 54, 54, 54, 54, 54, 54,
@@ -132,81 +134,85 @@ static const PARSER_KEYWORD gperf_keywords[] =
134 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
135 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
136 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
135 -#line 69 "gperf-config.txt"
137 +#line 71 "gperf-config.txt"
138 {"HOST", PLUGINSD_KEYWORD_ID_HOST, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 4},
137 -#line 103 "gperf-config.txt"
138 - {"REND", PLUGINSD_KEYWORD_ID_REND, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 29},
139 -#line 68 "gperf-config.txt"
139 +#line 107 "gperf-config.txt"
140 + {"REND", PLUGINSD_KEYWORD_ID_REND, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31},
141 +#line 70 "gperf-config.txt"
142 {"EXIT", PLUGINSD_KEYWORD_ID_EXIT, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 3},
141 -#line 77 "gperf-config.txt"
143 +#line 79 "gperf-config.txt"
144 {"CHART", PLUGINSD_KEYWORD_ID_CHART, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA|PARSER_REP_REPLICATION, WORKER_PARSER_FIRST_JOB + 9},
143 -#line 89 "gperf-config.txt"
145 +#line 91 "gperf-config.txt"
146 {"CONFIG", PLUGINSD_KEYWORD_ID_CONFIG, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 21},
145 -#line 86 "gperf-config.txt"
147 +#line 88 "gperf-config.txt"
148 {"OVERWRITE", PLUGINSD_KEYWORD_ID_OVERWRITE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 18},
147 -#line 72 "gperf-config.txt"
149 +#line 74 "gperf-config.txt"
150 {"HOST_LABEL", PLUGINSD_KEYWORD_ID_HOST_LABEL, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 7},
149 -#line 70 "gperf-config.txt"
151 +#line 72 "gperf-config.txt"
152 {"HOST_DEFINE", PLUGINSD_KEYWORD_ID_HOST_DEFINE, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 5},
151 -#line 104 "gperf-config.txt"
152 - {"RDSTATE", PLUGINSD_KEYWORD_ID_RDSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 30},
153 +#line 108 "gperf-config.txt"
154 + {"RDSTATE", PLUGINSD_KEYWORD_ID_RDSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 32},
155 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
154 -#line 118 "gperf-config.txt"
155 - {"DELETE_JOB", PLUGINSD_KEYWORD_ID_DELETE_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 38},
156 -#line 71 "gperf-config.txt"
156 +#line 122 "gperf-config.txt"
157 + {"DELETE_JOB", PLUGINSD_KEYWORD_ID_DELETE_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 40},
158 +#line 73 "gperf-config.txt"
159 {"HOST_DEFINE_END", PLUGINSD_KEYWORD_ID_HOST_DEFINE_END, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 6},
158 -#line 116 "gperf-config.txt"
159 - {"DYNCFG_RESET", PLUGINSD_KEYWORD_ID_DYNCFG_RESET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36},
160 -#line 113 "gperf-config.txt"
161 - {"DYNCFG_ENABLE", PLUGINSD_KEYWORD_ID_DYNCFG_ENABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 33},
160 +#line 120 "gperf-config.txt"
161 + {"DYNCFG_RESET", PLUGINSD_KEYWORD_ID_DYNCFG_RESET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 38},
162 #line 117 "gperf-config.txt"
163 - {"REPORT_JOB_STATUS", PLUGINSD_KEYWORD_ID_REPORT_JOB_STATUS, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37},
164 -#line 87 "gperf-config.txt"
163 + {"DYNCFG_ENABLE", PLUGINSD_KEYWORD_ID_DYNCFG_ENABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
164 +#line 121 "gperf-config.txt"
165 + {"REPORT_JOB_STATUS", PLUGINSD_KEYWORD_ID_REPORT_JOB_STATUS, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 39},
166 +#line 89 "gperf-config.txt"
167 {"SET", PLUGINSD_KEYWORD_ID_SET, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 19},
166 -#line 95 "gperf-config.txt"
167 - {"SET2", PLUGINSD_KEYWORD_ID_SET2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 24},
168 -#line 102 "gperf-config.txt"
169 - {"RSET", PLUGINSD_KEYWORD_ID_RSET, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 28},
170 -#line 100 "gperf-config.txt"
171 - {"CHART_DEFINITION_END", PLUGINSD_KEYWORD_ID_CHART_DEFINITION_END, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 26},
172 -#line 115 "gperf-config.txt"
173 - {"DYNCFG_REGISTER_JOB", PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 35},
174 -#line 105 "gperf-config.txt"
175 - {"RSSTATE", PLUGINSD_KEYWORD_ID_RSSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 31},
176 -#line 78 "gperf-config.txt"
168 +#line 99 "gperf-config.txt"
169 + {"SET2", PLUGINSD_KEYWORD_ID_SET2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 26},
170 +#line 106 "gperf-config.txt"
171 + {"RSET", PLUGINSD_KEYWORD_ID_RSET, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 30},
172 +#line 104 "gperf-config.txt"
173 + {"CHART_DEFINITION_END", PLUGINSD_KEYWORD_ID_CHART_DEFINITION_END, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 28},
174 +#line 119 "gperf-config.txt"
175 + {"DYNCFG_REGISTER_JOB", PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_JOB, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 37},
176 +#line 109 "gperf-config.txt"
177 + {"RSSTATE", PLUGINSD_KEYWORD_ID_RSSTATE, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 33},
178 +#line 80 "gperf-config.txt"
179 {"CLABEL", PLUGINSD_KEYWORD_ID_CLABEL, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 10},
178 -#line 114 "gperf-config.txt"
179 - {"DYNCFG_REGISTER_MODULE", PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_MODULE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 34},
180 -#line 66 "gperf-config.txt"
180 +#line 118 "gperf-config.txt"
181 + {"DYNCFG_REGISTER_MODULE", PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_MODULE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 36},
182 +#line 68 "gperf-config.txt"
183 {"FLUSH", PLUGINSD_KEYWORD_ID_FLUSH, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 1},
182 -#line 82 "gperf-config.txt"
184 +#line 84 "gperf-config.txt"
185 {"FUNCTION", PLUGINSD_KEYWORD_ID_FUNCTION, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 14},
184 -#line 93 "gperf-config.txt"
185 - {"CLAIMED_ID", PLUGINSD_KEYWORD_ID_CLAIMED_ID, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22},
186 -#line 81 "gperf-config.txt"
186 +#line 97 "gperf-config.txt"
187 + {"CLAIMED_ID", PLUGINSD_KEYWORD_ID_CLAIMED_ID, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 24},
188 +#line 83 "gperf-config.txt"
189 {"END", PLUGINSD_KEYWORD_ID_END, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 13},
188 -#line 96 "gperf-config.txt"
189 - {"END2", PLUGINSD_KEYWORD_ID_END2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 25},
190 -#line 79 "gperf-config.txt"
190 +#line 100 "gperf-config.txt"
191 + {"END2", PLUGINSD_KEYWORD_ID_END2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 27},
192 +#line 81 "gperf-config.txt"
193 {"CLABEL_COMMIT", PLUGINSD_KEYWORD_ID_CLABEL_COMMIT, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 11},
192 -#line 76 "gperf-config.txt"
194 +#line 78 "gperf-config.txt"
195 {"BEGIN", PLUGINSD_KEYWORD_ID_BEGIN, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 8},
194 -#line 94 "gperf-config.txt"
195 - {"BEGIN2", PLUGINSD_KEYWORD_ID_BEGIN2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 23},
196 -#line 101 "gperf-config.txt"
197 - {"RBEGIN", PLUGINSD_KEYWORD_ID_RBEGIN, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 27},
198 -#line 67 "gperf-config.txt"
196 +#line 98 "gperf-config.txt"
197 + {"BEGIN2", PLUGINSD_KEYWORD_ID_BEGIN2, PARSER_INIT_STREAMING|PARSER_REP_DATA, WORKER_PARSER_FIRST_JOB + 25},
198 +#line 105 "gperf-config.txt"
199 + {"RBEGIN", PLUGINSD_KEYWORD_ID_RBEGIN, PARSER_INIT_STREAMING|PARSER_REP_REPLICATION|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 29},
200 +#line 69 "gperf-config.txt"
201 {"DISABLE", PLUGINSD_KEYWORD_ID_DISABLE, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 2},
200 -#line 84 "gperf-config.txt"
202 +#line 86 "gperf-config.txt"
203 {"FUNCTION_PROGRESS", PLUGINSD_KEYWORD_ID_FUNCTION_PROGRESS, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 16},
202 -#line 80 "gperf-config.txt"
204 +#line 82 "gperf-config.txt"
205 {"DIMENSION", PLUGINSD_KEYWORD_ID_DIMENSION, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 12},
204 -#line 88 "gperf-config.txt"
206 +#line 90 "gperf-config.txt"
207 {"VARIABLE", PLUGINSD_KEYWORD_ID_VARIABLE, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 20},
206 -#line 109 "gperf-config.txt"
207 - {"JSON", PLUGINSD_KEYWORD_ID_JSON, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 32},
208 -#line 83 "gperf-config.txt"
208 +#line 92 "gperf-config.txt"
209 + {"TRUST_DURATIONS", PLUGINSD_KEYWORD_ID_TRUST_DURATIONS, PARSER_INIT_PLUGINSD|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 22},
210 +#line 85 "gperf-config.txt"
211 {"FUNCTION_RESULT_BEGIN", PLUGINSD_KEYWORD_ID_FUNCTION_RESULT_BEGIN, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING, WORKER_PARSER_FIRST_JOB + 15},
212 +#line 93 "gperf-config.txt"
213 + {"PLUGIN_KEEPALIVE", PLUGINSD_KEYWORD_ID_PLUGIN_KEEPALIVE, PARSER_INIT_PLUGINSD, WORKER_PARSER_FIRST_JOB + 23},
214 +#line 113 "gperf-config.txt"
215 + {"JSON", PLUGINSD_KEYWORD_ID_JSON, PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 34},
216 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
217 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
218 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
@@ -216,9 +222,7 @@ static const PARSER_KEYWORD gperf_keywords[] =
222 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
223 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
224 {(char*)0,0,PARSER_INIT_PLUGINSD,0},
219 - {(char*)0,0,PARSER_INIT_PLUGINSD,0},
220 - {(char*)0,0,PARSER_INIT_PLUGINSD,0},
221 -#line 85 "gperf-config.txt"
225 +#line 87 "gperf-config.txt"
226 {"LABEL", PLUGINSD_KEYWORD_ID_LABEL, PARSER_INIT_PLUGINSD|PARSER_INIT_STREAMING|PARSER_REP_METADATA, WORKER_PARSER_FIRST_JOB + 17}
227 };
228
src/plugins.d/plugins_d.c
+23 -18
@@ -242,28 +242,33 @@ static void pluginsd_main_cleanup(void *pptr) {
242 }
243
244 static bool is_plugin(char *dst, size_t dst_size, const char *filename) {
245 - size_t len = strlen(filename);
245 +#if defined(OS_WINDOWS)
246 + const char *suffixes[] = {
247 + ".plugin.exe",
248 + "_plugin.exe",
249 + "-plugin.exe",
250 + NULL
251 + };
252 +#else
253 + const char *suffixes[] = {
254 + ".plugin",
255 + "_plugin",
256 + "-plugin",
257 + NULL
258 + };
259 +#endif
260
247 - const char *suffix;
248 - size_t suffix_len;
261 + size_t filename_len = strlen(filename);
262
250 - suffix = ".plugin";
251 - suffix_len = strlen(suffix);
252 - if (len > suffix_len &&
253 - strcmp(suffix, &filename[len - suffix_len]) == 0) {
254 - snprintfz(dst, dst_size, "%.*s", (int)(len - suffix_len), filename);
255 - return true;
256 - }
263 + for (int i = 0; suffixes[i] != NULL; i++) {
264 + size_t suffix_len = strlen(suffixes[i]);
265
258 -#if defined(OS_WINDOWS)
259 - suffix = ".plugin.exe";
260 - suffix_len = strlen(suffix);
261 - if (len > suffix_len &&
262 - strcmp(suffix, &filename[len - suffix_len]) == 0) {
263 - snprintfz(dst, dst_size, "%.*s", (int)(len - suffix_len), filename);
264 - return true;
266 + if (filename_len > suffix_len &&
267 + strcmp(suffixes[i], &filename[filename_len - suffix_len]) == 0) {
268 + snprintfz(dst, dst_size, "%.*s", (int)(filename_len - suffix_len), filename);
269 + return true;
270 + }
271 }
266 -#endif
272
273 return false;
274 }
src/plugins.d/pluginsd_parser.c
+25
@@ -1068,11 +1068,32 @@ static ALWAYS_INLINE PARSER_RC pluginsd_end_v2(char **words __maybe_unused, size
1068 return PARSER_RC_OK;
1069 }
1070
1071 +static inline PARSER_RC pluginsd_trust_durations(char **words, size_t num_words, PARSER *parser) {
1072 + char *value = get_word(words, num_words, 1);
1073 +
1074 + if (!value || !*value)
1075 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_TRUST_DURATIONS, "missing parameter");
1076 +
1077 + int trusted = str2i(value);
1078 + if (trusted != 0 && trusted != 1)
1079 + return PLUGINSD_DISABLE_PLUGIN(parser, PLUGINSD_KEYWORD_TRUST_DURATIONS, "parameter must be 0 or 1");
1080 +
1081 + parser->user.trust_durations = trusted;
1082 +
1083 + netdata_log_debug(D_PLUGINSD, "PLUGINSD: trust durations set to %d", trusted);
1084 +
1085 + return PARSER_RC_OK;
1086 +}
1087 +
1088 static inline PARSER_RC pluginsd_exit(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
1089 netdata_log_info("PLUGINSD: plugin called EXIT.");
1090 return PARSER_RC_STOP;
1091 }
1092
1093 +static inline PARSER_RC pluginsd_plugin_keepalive(char **words __maybe_unused, size_t num_words __maybe_unused, PARSER *parser __maybe_unused) {
1094 + return PARSER_RC_OK;
1095 +}
1096 +
1097 static void pluginsd_json_stream_paths(PARSER *parser, void *action_data __maybe_unused) {
1098 stream_path_set_from_json(parser->user.host, buffer_tostring(parser->defer.response), false);
1099 buffer_free(parser->defer.response);
@@ -1322,6 +1343,10 @@ ALWAYS_INLINE PARSER_RC parser_execute(PARSER *parser, const PARSER_KEYWORD *key
1343 return pluginsd_exit(words, num_words, parser);
1344 case PLUGINSD_KEYWORD_ID_CONFIG:
1345 return pluginsd_config(words, num_words, parser);
1346 + case PLUGINSD_KEYWORD_ID_TRUST_DURATIONS:
1347 + return pluginsd_trust_durations(words, num_words, parser);
1348 + case PLUGINSD_KEYWORD_ID_PLUGIN_KEEPALIVE:
1349 + return pluginsd_plugin_keepalive(words, num_words, parser);
1350
1351 case PLUGINSD_KEYWORD_ID_DYNCFG_ENABLE:
1352 case PLUGINSD_KEYWORD_ID_DYNCFG_REGISTER_MODULE: