Split systemd-journal plugin and add Rust-based journal file reader (#20345)
* Split systemd-journal plugin and add Rust-based journal file reader This commit splits the systemd-journal plugin into two separate plugins, ie. one for reading journal logs and another for retrieving systemd units information. A new Rust-based journal file format reader is used on systems that do not have libsystemd available. * Disable systemd units and fix availability macros. * Restructure systemd journal file options to be a bit saner. - Make the options to select the journal file handling implementation depend on the journal plugin being enabled. - Make the benchmark option for the Rust journal file implementation depend on that implementation being enabled. Also, mark it as advanced so that it’s more obvious to users that this is a developer thing, not a ‘regular’ build thing. - Make the libsystemd journal file implementation autoselected based on whether the Rust implementation is enabled or not instead of having it be an option. This ensures that exactly one implementation is selected if we need one. * Enable systemd-units plugin by defualt and correctly autodetect it. * Fix RPM spec file. --------- Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud>
vkalintiris committed
May 28, 2025 at 15:34 UTC
adc4d66ea03e1737dc7361fecb26885aef896620
48 files changed
+6873
-114
CMakeLists.txt
+81
-6
@@ -195,6 +195,7 @@ cmake_dependent_option(ENABLE_PLUGIN_NFACCT "Enable Linux NFACCT metric collecti
195
cmake_dependent_option(ENABLE_PLUGIN_PERF "Enable Linux performance counter monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
196
cmake_dependent_option(ENABLE_PLUGIN_SLABINFO "Enable Linux kernel SLAB allocator monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
197
cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_JOURNAL "Enable systemd journal log collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
198
+cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_UNITS "Enable systemd units information collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
199
cmake_dependent_option(ENABLE_PLUGIN_XENSTAT "Enable Xen domain monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
200
201
# Metrics exporters
@@ -227,6 +228,16 @@ mark_as_advanced(ENABLE_LIBUNWIND)
228
cmake_dependent_option(FORCE_LEGACY_LIBBPF "Force usage of libbpf 0.0.9 instead of the latest version." False "ENABLE_PLUGIN_EBPF" False)
229
mark_as_advanced(FORCE_LEGACY_LIBBPF)
230
231
+cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable netdata's journal file reader implementation" False "ENABLE_PLUGIN_SYSTEMD_JOURNAL" False)
232
+cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS "Enable netdata's journal file reader implementation benchmarks" False "ENABLE_NETDATA_JOURNAL_FILE_READER" False)
233
+mark_as_advanced(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS)
234
+
235
+if(ENABLE_PLUGIN_SYSTEMD_JOURNAL AND NOT ENABLE_NETDATA_JOURNAL_FILE_READER)
236
+ set(ENABLE_LIBSYSTEMD_JOURNAL_FILE_READER True)
237
+else()
238
+ set(ENABLE_LIBSYSTEMD_JOURNAL_FILE_READER False)
239
+endif()
240
+
241
option(ENABLE_MIMALLOC "Enable mimalloc allocator" OFF)
242
243
if(ENABLE_MIMALLOC)
@@ -1758,13 +1769,16 @@ set(SYSTEMD_JOURNAL_PLUGIN_FILES
1769
src/collectors/systemd-journal.plugin/systemd-journal-fstat.c
1770
src/collectors/systemd-journal.plugin/systemd-journal-watcher.c
1771
src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
1772
+ src/collectors/systemd-journal.plugin/provider/netdata_provider.c
1773
+ src/collectors/systemd-journal.plugin/provider/netdata_provider.h
1774
+ src/collectors/systemd-journal.plugin/provider/rust_provider.h
1775
src/libnetdata/os/system-maps/system-services.h
1776
src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
1777
)
1778
1765
-if(ENABLE_SYSTEMD_DBUS)
1766
- list(APPEND SYSTEMD_JOURNAL_PLUGIN_FILES src/collectors/systemd-journal.plugin/systemd-units.c)
1767
-endif()
1779
+set(SYSTEMD_UNITS_PLUGIN_FILES
1780
+ src/collectors/systemd-units.plugin/plugin_systemd_units.c
1781
+)
1782
1783
set(STREAMING_PLUGIN_FILES
1784
src/streaming/stream.h
@@ -2799,12 +2813,47 @@ if(ENABLE_PLUGIN_CGROUP_NETWORK)
2813
DESTINATION usr/libexec/netdata/plugins.d)
2814
endif()
2815
2802
-if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2803
- if(NOT SYSTEMD_FOUND)
2804
- message(FATAL_ERROR "Systemd journal plugin requires systemd, but systemd was not found.")
2816
+# Enable rust implementation if we don't have systemd and we want the journal plugin
2817
+if(ENABLE_PLUGIN_SYSTEMD_JOURNAL AND NOT SYSTEMD_FOUND)
2818
+ if (NOT ENABLE_NETDATA_JOURNAL_FILE_READER)
2819
+ message(WARNING "Systemd journal package not found, will try netdata's journal reader which requires cargo.")
2820
+ set(ENABLE_NETDATA_JOURNAL_FILE_READER True)
2821
endif()
2822
+endif()
2823
2824
+if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2825
add_executable(systemd-journal.plugin ${SYSTEMD_JOURNAL_PLUGIN_FILES})
2826
+
2827
+ if(ENABLE_NETDATA_JOURNAL_FILE_READER)
2828
+ include(FetchContent)
2829
+ FetchContent_Declare(
2830
+ Corrosion
2831
+ GIT_REPOSITORY https://github.com/corrosion-rs/corrosion.git
2832
+ GIT_TAG v0.5
2833
+ )
2834
+ FetchContent_MakeAvailable(Corrosion)
2835
+
2836
+ corrosion_import_crate(MANIFEST_PATH src/crates/jf/Cargo.toml)
2837
+ target_compile_definitions(systemd-journal.plugin PRIVATE HAVE_RUST_PROVIDER)
2838
+ target_link_libraries(systemd-journal.plugin journal_reader_ffi)
2839
+
2840
+ if(ENABLE_NETDATA_JOURNAL_FILE_READER_BENCHMARKS)
2841
+ set(JF_BENCHMARK_FILES src/crates/jf/benchmark/main.c
2842
+ src/collectors/systemd-journal.plugin/provider/netdata_provider.c
2843
+ src/collectors/systemd-journal.plugin/provider/netdata_provider.h
2844
+ )
2845
+
2846
+ add_executable(jfr.bench ${JF_BENCHMARK_FILES})
2847
+ target_compile_definitions(jfr.bench PRIVATE HAVE_RUST_PROVIDER)
2848
+ target_include_directories(jfr.bench PRIVATE "${CMAKE_SOURCE_DIR}/src")
2849
+ target_link_libraries(jfr.bench journal_reader_ffi)
2850
+
2851
+ add_executable(jfc.bench ${JF_BENCHMARK_FILES})
2852
+ target_include_directories(jfc.bench PRIVATE "${CMAKE_SOURCE_DIR}/src")
2853
+ target_link_libraries(jfc.bench "${SYSTEMD_LDFLAGS}")
2854
+ endif()
2855
+ endif()
2856
+
2857
target_link_libraries(systemd-journal.plugin libnetdata)
2858
2859
install(TARGETS systemd-journal.plugin
@@ -2819,6 +2868,32 @@ if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2868
endif()
2869
endif()
2870
2871
+if(ENABLE_PLUGIN_SYSTEMD_UNITS)
2872
+ if(NOT SYSTEMD_FOUND)
2873
+ message(FATAL_ERROR "Systemd units plugin requires systemd, but systemd was not found.")
2874
+ endif()
2875
+
2876
+ if(SYSTEMD_VERSION LESS 221)
2877
+ message(FATAL_ERROR "Systemd units plugin requires systemd 221 or newer, but only systemd ${SYSTEMD_VERSION} was found.")
2878
+ endif()
2879
+
2880
+ include(FetchContent)
2881
+
2882
+ add_executable(systemd-units.plugin ${SYSTEMD_UNITS_PLUGIN_FILES})
2883
+ target_link_libraries(systemd-units.plugin libnetdata)
2884
+
2885
+ install(TARGETS systemd-units.plugin
2886
+ COMPONENT plugin-systemd-units
2887
+ DESTINATION usr/libexec/netdata/plugins.d)
2888
+
2889
+ if(BUILD_FOR_PACKAGING)
2890
+ install(FILES
2891
+ ${PKG_FILES_PATH}/copyright
2892
+ COMPONENT plugin-systemd-units
2893
+ DESTINATION usr/share/doc/netdata-plugin-systemd-units)
2894
+ endif()
2895
+endif()
2896
+
2897
if(OS_WINDOWS)
2898
add_executable(windows-events.plugin ${WINDOWS_EVENTS_PLUGIN_FILES})
2899
target_link_libraries(windows-events.plugin libnetdata wevtapi)
netdata.spec.in
+28
@@ -229,6 +229,7 @@ Suggests: %{name}-plugin-freeipmi = %{version}
229
Suggests: %{name}-plugin-cups = %{version}
230
%endif
231
Recommends: %{name}-plugin-systemd-journal = %{version}
232
+Recommends: %{name}-plugin-systmed-units = %{version}
233
Recommends: %{name}-plugin-network-viewer = %{version}
234
Recommends: %{name}-plugin-logs-management = %{version}
235
%else
@@ -409,6 +410,11 @@ advanced correlations and fast root cause analysis, native horizontal scalabilit
410
-DENABLE_PLUGIN_PERF=On \
411
-DENABLE_PLUGIN_SLABINFO=On \
412
-DENABLE_PLUGIN_SYSTEMD_JOURNAL=On \
413
+ %if 0%{?centos_ver} != 7 && 0%{?amazon_linux} != 2
414
+ -DENABLE_PLUGIN_SYSTEMD_UNITS=On \
415
+ %else
416
+ -DENABLE_PLUGIN_SYSTEMD_UNITS=Off \
417
+ %endif
418
-DENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE=On \
419
-DENABLE_BUNDLED_JSONC=Off \
420
-DENABLE_BUNDLED_YAML=Off \
@@ -953,6 +959,26 @@ fi
959
# CAP_DAC_READ_SEARCH required for data collection.
960
%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin
961
962
+%if 0%{?centos_ver} != 7 && 0%{?amazon_linux} != 2
963
+%package plugin-systemd-units
964
+Summary: The systemd units plugin for the Netdata Agent
965
+Group: Applications/System
966
+Requires: %{name} = %{version}
967
+Conflicts: %{name} < %{version}
968
+
969
+%description plugin-systemd-units
970
+ This plugin allows Netdata to collect metrics about systemd units.
971
+
972
+%pre plugin-systemd-units
973
+if ! getent group %{name} > /dev/null; then
974
+ groupadd --system %{name}
975
+fi
976
+
977
+%files plugin-systemd-journal
978
+%defattr(0750,root,netdata,0750)
979
+%{_libexecdir}/%{name}/plugins.d/systemd-units.plugin
980
+%endif
981
+
982
%if %{_have_xenstat}
983
%package plugin-xenstat
984
Summary: The xenstat plugin for the Netdata Agent
@@ -1020,6 +1046,8 @@ fi
1046
%{_datadir}/%{name}/web
1047
1048
%changelog
1049
+* Tue May 27 2025 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-32
1050
+- Correctly handle systmed-units plugin as it’s own package
1051
* Fri Mar 21 2025 Austin Hemmelgarn <austin@netdata.cloud> 0.0.0-31
1052
- Exclude build directory from install to make openSUSE happy
1053
* Tue Jan 28 2025 Konstantin Shalygin <k0ste@k0ste.ru> 0.0.0-30
packaging/build-package.sh
+1
@@ -43,6 +43,7 @@ add_cmake_option ENABLE_PLUGIN_NFACCT On
43
add_cmake_option ENABLE_PLUGIN_PERF On
44
add_cmake_option ENABLE_PLUGIN_SLABINFO On
45
add_cmake_option ENABLE_PLUGIN_SYSTEMD_JOURNAL On
46
+add_cmake_option ENABLE_PLUGIN_SYSTEMD_UNITS On
47
48
add_cmake_option ENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE On
49
add_cmake_option ENABLE_EXPORTER_MONGODB On
packaging/cmake/Modules/Packaging.cmake
+21
-2
@@ -415,7 +415,7 @@ set(CPACK_DEBIAN_PLUGIN-SLABINFO_PACKAGE_CONTROL_EXTRA
415
"${PKG_FILES_PATH}/deb/plugin-slabinfo/preinst;"
416
"${PKG_FILES_PATH}/deb/plugin-slabinfo/postinst")
417
418
-set(CPACK_DEBIAN_PLUGIN-SLABINFO_DEBUGINFO_PACKAGE On)
418
+set(CPACK_DEBIAN_PLUGIN-SLABINFO-DEBUGINFO_PACKAGE On)
419
420
#
421
# systemd-journal.plugin
@@ -435,7 +435,26 @@ set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_PACKAGE_CONTROL_EXTRA
435
"${PKG_FILES_PATH}/deb/plugin-systemd-journal/preinst;"
436
"${PKG_FILES_PATH}/deb/plugin-systemd-journal/postinst")
437
438
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD_JOURNAL_DEBUGINFO_PACKAGE On)
438
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_DEBUGINFO_PACKAGE On)
439
+
440
+#
441
+# systemd-units.plugin
442
+#
443
+
444
+set(CPACK_COMPONENT_PLUGIN-SYSTEMD-UNITS_DEPENDS "netdata")
445
+set(CPACK_COMPONENT_PLUGIN-SYSTEMD-UNITS_DESCRIPTION
446
+ "The systemd-units collector for the Netdata Agent
447
+ This plugin allows the Netdata Agent to collect metrics about systmed units.")
448
+
449
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD-UNITS_PACKAGE_NAME "netdata-plugin-systemd-units")
450
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD-UNITS_PACKAGE_SECTION "net")
451
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD-UNITS_PACKAGE_PREDEPENDS "adduser")
452
+
453
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD-UNITS_PACKAGE_CONTROL_EXTRA
454
+ "${PKG_FILES_PATH}/deb/plugin-systemd-units/preinst;"
455
+ "${PKG_FILES_PATH}/deb/plugin-systemd-units/postinst")
456
+
457
+set(CPACK_DEBIAN_PLUGIN-SYSTEMD_UNITS_DEBUGINFO_PACKAGE On)
458
459
#
460
# xenstat.plugin
packaging/cmake/pkg-files/deb/plugin-systemd-units/postinst
new
+11
@@ -0,0 +1,11 @@
1
+#!/bin/sh
2
+
3
+set -e
4
+
5
+case "$1" in
6
+ configure|reconfigure)
7
+ grep /usr/libexec/netdata /var/lib/dpkg/info/netdata-plugin-systemd-units.list | xargs -n 30 chown root:netdata
8
+ ;;
9
+esac
10
+
11
+exit 0
packaging/cmake/pkg-files/deb/plugin-systemd-units/preinst
new
+11
@@ -0,0 +1,11 @@
1
+#!/bin/sh
2
+
3
+set -e
4
+
5
+case "$1" in
6
+ install)
7
+ if ! getent group netdata > /dev/null; then
8
+ addgroup --quiet --system netdata
9
+ fi
10
+ ;;
11
+esac
packaging/installer/functions.sh
+6
@@ -343,6 +343,12 @@ prepare_cmake_options() {
343
344
enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}"
345
346
+ if check_for_module 'libsystemd >= 221'; then
347
+ enable_feature PLUGIN_SYSTEMD_UNITS 1
348
+ else
349
+ enable_feature PLUGIN_SYSTEMD_UNITS 0
350
+ fi
351
+
352
if command -v cups-config >/dev/null 2>&1 || check_for_module libcups || check_for_module cups; then
353
ENABLE_CUPS=1
354
else
src/collectors/systemd-journal.plugin/provider/netdata_provider.c
new
+182
@@ -0,0 +1,182 @@
1
+#include "netdata_provider.h"
2
+
3
+int32_t nsd_id128_from_string(const char *s, NsdId128 *ret)
4
+{
5
+#if defined(HAVE_RUST_PROVIDER)
6
+ return rsd_id128_from_string(s, (struct RsdId128 *) ret);
7
+#else
8
+ return sd_id128_from_string(s, (sd_id128_t *) ret);
9
+#endif
10
+}
11
+
12
+int32_t nsd_id128_equal(NsdId128 a, NsdId128 b)
13
+{
14
+#if defined(HAVE_RUST_PROVIDER)
15
+ return rsd_id128_equal(a, b);
16
+#else
17
+ return sd_id128_equal(a, b);
18
+#endif
19
+}
20
+
21
+int nsd_journal_open_files(NsdJournal **ret, const char *const *paths, int flags)
22
+{
23
+#if defined(HAVE_RUST_PROVIDER)
24
+ return rsd_journal_open_files(ret, paths, flags);
25
+#else
26
+ return sd_journal_open_files(ret, paths, flags);
27
+#endif
28
+}
29
+
30
+void nsd_journal_close(NsdJournal *j)
31
+{
32
+#if defined(HAVE_RUST_PROVIDER)
33
+ rsd_journal_close(j);
34
+#else
35
+ sd_journal_close(j);
36
+#endif
37
+}
38
+
39
+int nsd_journal_seek_head(NsdJournal *j)
40
+{
41
+#if defined(HAVE_RUST_PROVIDER)
42
+ return rsd_journal_seek_head(j);
43
+#else
44
+ return sd_journal_seek_head(j);
45
+#endif
46
+}
47
+
48
+int nsd_journal_seek_tail(NsdJournal *j)
49
+{
50
+#if defined(HAVE_RUST_PROVIDER)
51
+ return rsd_journal_seek_tail(j);
52
+#else
53
+ return sd_journal_seek_tail(j);
54
+#endif
55
+}
56
+
57
+int nsd_journal_seek_realtime_usec(NsdJournal *j, uint64_t usec)
58
+{
59
+#if defined(HAVE_RUST_PROVIDER)
60
+ return rsd_journal_seek_realtime_usec(j, usec);
61
+#else
62
+ return sd_journal_seek_realtime_usec(j, usec);
63
+#endif
64
+}
65
+
66
+int nsd_journal_next(NsdJournal *j)
67
+{
68
+#if defined(HAVE_RUST_PROVIDER)
69
+ return rsd_journal_next(j);
70
+#else
71
+ return sd_journal_next(j);
72
+#endif
73
+}
74
+
75
+int nsd_journal_previous(NsdJournal *j)
76
+{
77
+#if defined(HAVE_RUST_PROVIDER)
78
+ return rsd_journal_previous(j);
79
+#else
80
+ return sd_journal_previous(j);
81
+#endif
82
+}
83
+
84
+#if defined(HAVE_SD_JOURNAL_GET_SEQNUM)
85
+int nsd_journal_get_seqnum(NsdJournal *j, uint64_t *ret_seqnum, NsdId128 *ret_seqnum_id)
86
+{
87
+#if defined(HAVE_RUST_PROVIDER)
88
+ return rsd_journal_get_seqnum(j, ret_seqnum, ret_seqnum_id);
89
+#else
90
+ return sd_journal_get_seqnum(j, ret_seqnum, ret_seqnum_id);
91
+#endif
92
+}
93
+#endif /* HAVE_SD_JOURNAL_GET_SEQNUM */
94
+
95
+int nsd_journal_get_realtime_usec(NsdJournal *j, uint64_t *ret)
96
+{
97
+#if defined(HAVE_RUST_PROVIDER)
98
+ return rsd_journal_get_realtime_usec(j, ret);
99
+#else
100
+ return sd_journal_get_realtime_usec(j, ret);
101
+#endif
102
+}
103
+
104
+#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
105
+int nsd_journal_enumerate_fields(NsdJournal *j, const char **field)
106
+{
107
+#if defined(HAVE_RUST_PROVIDER)
108
+ return rsd_journal_enumerate_fields(j, field);
109
+#else
110
+ return sd_journal_enumerate_fields(j, field);
111
+#endif
112
+}
113
+#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
114
+
115
+#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
116
+void nsd_journal_restart_fields(NsdJournal *j)
117
+{
118
+#if defined(HAVE_RUST_PROVIDER)
119
+ rsd_journal_restart_fields(j);
120
+#else
121
+ sd_journal_restart_fields(j);
122
+#endif
123
+}
124
+#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
125
+
126
+#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
127
+int nsd_journal_query_unique(NsdJournal *j, const char *field)
128
+{
129
+#if defined(HAVE_RUST_PROVIDER)
130
+ return rsd_journal_query_unique(j, field);
131
+#else
132
+ return sd_journal_query_unique(j, field);
133
+#endif
134
+}
135
+#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
136
+
137
+#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
138
+void nsd_journal_restart_unique(NsdJournal *j)
139
+{
140
+#if defined(HAVE_RUST_PROVIDER)
141
+ rsd_journal_restart_unique(j);
142
+#else
143
+ sd_journal_restart_unique(j);
144
+#endif
145
+}
146
+#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
147
+
148
+int nsd_journal_add_match(NsdJournal *j, const void *data, uintptr_t size)
149
+{
150
+#if defined(HAVE_RUST_PROVIDER)
151
+ return rsd_journal_add_match(j, data, size);
152
+#else
153
+ return sd_journal_add_match(j, data, size);
154
+#endif
155
+}
156
+
157
+int nsd_journal_add_conjunction(NsdJournal *j)
158
+{
159
+#if defined(HAVE_RUST_PROVIDER)
160
+ return rsd_journal_add_conjunction(j);
161
+#else
162
+ return sd_journal_add_conjunction(j);
163
+#endif
164
+}
165
+
166
+int nsd_journal_add_disjunction(NsdJournal *j)
167
+{
168
+#if defined(HAVE_RUST_PROVIDER)
169
+ return rsd_journal_add_disjunction(j);
170
+#else
171
+ return sd_journal_add_disjunction(j);
172
+#endif
173
+}
174
+
175
+void nsd_journal_flush_matches(NsdJournal *j)
176
+{
177
+#if defined(HAVE_RUST_PROVIDER)
178
+ rsd_journal_flush_matches(j);
179
+#else
180
+ sd_journal_flush_matches(j);
181
+#endif
182
+}
src/collectors/systemd-journal.plugin/provider/netdata_provider.h
new
+78
@@ -0,0 +1,78 @@
1
+#ifndef ND_SD_JOURNAL_PROVIDER_NETDATA_H
2
+#define ND_SD_JOURNAL_PROVIDER_NETDATA_H
3
+
4
+#ifdef __cplusplus
5
+extern "C" {
6
+#endif /* __cplusplus */
7
+
8
+#include "config.h"
9
+
10
+#if defined(HAVE_RUST_PROVIDER)
11
+ #include "rust_provider.h"
12
+#else
13
+ #include <systemd/sd-journal.h>
14
+#endif
15
+
16
+#if defined(HAVE_RUST_PROVIDER)
17
+ typedef struct RsdJournal NsdJournal;
18
+
19
+ #define NSD_JOURNAL_FOREACH_DATA(j, data, l) RSD_JOURNAL_FOREACH_DATA(j, data, l)
20
+ #define NSD_JOURNAL_FOREACH_UNIQUE(j, data, l) RSD_JOURNAL_FOREACH_UNIQUE(j, data, l)
21
+ #define NSD_JOURNAL_FOREACH_FIELD(j, field) RSD_JOURNAL_FOREACH_FIELD(j, field)
22
+
23
+ typedef struct RsdId128 NsdId128;
24
+
25
+ #define NSD_ID128_NULL RSD_ID128_NULL
26
+ #define NSD_ID128_STRING_MAX RSD_ID128_STRING_MAX
27
+ #define NSD_ID128_UUID_STRING_MAX RSD_ID128_UUID_STRING_MAX
28
+#else
29
+ typedef struct sd_journal NsdJournal;
30
+ typedef sd_id128_t NsdId128;
31
+
32
+ #define NSD_ID128_NULL SD_ID128_NULL
33
+ #define NSD_ID128_STRING_MAX SD_ID128_STRING_MAX
34
+ #define NSD_ID128_UUID_STRING_MAX SD_ID128_UUID_STRING_MAX
35
+
36
+ #define NSD_JOURNAL_FOREACH_DATA(j, data, l) SD_JOURNAL_FOREACH_DATA(j, data, l)
37
+ #if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
38
+ #define NSD_JOURNAL_FOREACH_UNIQUE(j, data, l) SD_JOURNAL_FOREACH_UNIQUE(j, data, l)
39
+ #define NSD_JOURNAL_FOREACH_FIELD(j, field) SD_JOURNAL_FOREACH_FIELD(j, field)
40
+ #endif
41
+#endif
42
+
43
+int32_t nsd_id128_from_string(const char *s, NsdId128 *ret);
44
+int32_t nsd_id128_equal(NsdId128 a, NsdId128 b);
45
+
46
+int nsd_journal_open_files(NsdJournal **ret, const char *const *paths, int flags);
47
+void nsd_journal_close(NsdJournal *j);
48
+
49
+int nsd_journal_seek_head(NsdJournal *j);
50
+int nsd_journal_seek_tail(NsdJournal *j);
51
+int nsd_journal_seek_realtime_usec(NsdJournal *j, uint64_t usec);
52
+
53
+int nsd_journal_next(NsdJournal *j);
54
+int nsd_journal_previous(NsdJournal *j);
55
+
56
+#if defined(HAVE_SD_JOURNAL_GET_SEQNUM)
57
+int nsd_journal_get_seqnum(NsdJournal *j, uint64_t *ret_seqnum, NsdId128 *ret_seqnum_id);
58
+#endif
59
+int nsd_journal_get_realtime_usec(NsdJournal *j, uint64_t *ret);
60
+
61
+#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
62
+int nsd_journal_enumerate_fields(NsdJournal *j, const char **field);
63
+void nsd_journal_restart_fields(NsdJournal *j);
64
+
65
+int nsd_journal_query_unique(NsdJournal *j, const char *field);
66
+void nsd_journal_restart_unique(NsdJournal *j);
67
+#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
68
+
69
+int nsd_journal_add_match(NsdJournal *j, const void *data, uintptr_t size);
70
+int nsd_journal_add_conjunction(NsdJournal *j);
71
+int nsd_journal_add_disjunction(NsdJournal *j);
72
+void nsd_journal_flush_matches(NsdJournal *j);
73
+
74
+#ifdef __cplusplus
75
+}
76
+#endif /* __cplusplus */
77
+
78
+#endif /* ND_SD_JOURNAL_PROVIDER_NETDATA_H */
src/collectors/systemd-journal.plugin/provider/rust_provider.h
new
+27
@@ -0,0 +1,27 @@
1
+#ifndef ND_SD_JOURNAL_PROVIDER_RUST_H
2
+#define ND_SD_JOURNAL_PROVIDER_RUST_H
3
+
4
+#include "crates/jf/journal_reader_ffi/journal_reader_ffi.h"
5
+
6
+#define RSD_ID128_NULL ((const RsdId128){.bytes = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}})
7
+#define RSD_ID128_STRING_MAX 33U
8
+#define RSD_ID128_UUID_STRING_MAX 37U
9
+
10
+#define RSD_JOURNAL_FOREACH_DATA(j, data, l) \
11
+ for (rsd_journal_restart_data(j); rsd_journal_enumerate_available_data((j), &(data), &(l)) > 0;)
12
+
13
+#define RSD_JOURNAL_FOREACH_UNIQUE(j, data, l) \
14
+ for (rsd_journal_restart_unique(j); rsd_journal_enumerate_available_unique((j), &(data), &(l)) > 0;)
15
+
16
+#define RSD_JOURNAL_FOREACH_FIELD(j, field) \
17
+ for (rsd_journal_restart_fields(j); rsd_journal_enumerate_fields((j), &(field)) > 0;)
18
+
19
+#ifndef HAVE_SD_JOURNAL_RESTART_FIELDS
20
+#define HAVE_SD_JOURNAL_RESTART_FIELDS 1
21
+#endif
22
+
23
+#ifndef HAVE_SD_JOURNAL_GET_SEQNUM
24
+#define HAVE_SD_JOURNAL_GET_SEQNUM 1
25
+#endif
26
+
27
+#endif /* ND_SD_JOURNAL_PROVIDER_RUST_H */
src/collectors/systemd-journal.plugin/systemd-internals.h
+4
-20
@@ -5,9 +5,9 @@
5
6
#include "collectors/all.h"
7
#include "libnetdata/libnetdata.h"
8
+#include "provider/netdata_provider.h"
9
10
#include <linux/capability.h>
10
-#include <systemd/sd-journal.h>
11
#include <syslog.h>
12
13
#define ND_SD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
@@ -17,10 +17,6 @@
17
#define ND_SD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS 250
18
#define ND_SD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
19
20
-#define ND_SD_UNITS_FUNCTION_DESCRIPTION "View the status of systemd units"
21
-#define ND_SD_UNITS_FUNCTION_NAME "systemd-list-units"
22
-#define ND_SD_UNITS_DEFAULT_TIMEOUT 30
23
-
20
extern __thread size_t fstat_thread_calls;
21
extern __thread size_t fstat_thread_cached_responses;
22
void fstat_cache_enable_on_thread(void);
@@ -67,8 +63,8 @@ struct nd_journal_file {
63
64
uint64_t first_seqnum;
65
uint64_t last_seqnum;
70
- sd_id128_t first_writer_id;
71
- sd_id128_t last_writer_id;
66
+ NsdId128 first_writer_id;
67
+ NsdId128 last_writer_id;
68
69
uint64_t messages_in_file;
70
};
@@ -115,7 +111,7 @@ void nd_sd_journal_transform_gid(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATI
111
void nd_sd_journal_transform_cap_effective(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
112
void nd_sd_journal_transform_timestamp_usec(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
113
118
-usec_t nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file *njf, const char *boot_id);
114
+usec_t nd_journal_file_update_annotation_boot_id(NsdJournal *j, struct nd_journal_file *njf, const char *boot_id);
115
116
#define MAX_JOURNAL_DIRECTORIES 100
117
struct journal_directory {
@@ -141,18 +137,6 @@ void nd_sd_journal_transform_message_id(FACETS *facets, BUFFER *wb, FACETS_TRANS
137
void *nd_journal_watcher_main(void *arg);
138
void nd_journal_watcher_restart(void);
139
144
-#ifdef ENABLE_SYSTEMD_DBUS
145
-void function_systemd_units(
146
- const char *transaction,
147
- char *function,
148
- usec_t *stop_monotonic_ut,
149
- bool *cancelled,
150
- BUFFER *payload,
151
- HTTP_ACCESS access __maybe_unused,
152
- const char *source,
153
- void *data);
154
-#endif
155
-
140
static inline bool parse_journal_field(
141
const char *data,
142
size_t data_length,
src/collectors/systemd-journal.plugin/systemd-journal-files.c
+33
-23
@@ -1,5 +1,6 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
+#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
4
#include "systemd-internals.h"
5
6
#define ND_SD_JOURNAL_MAX_SOURCE_LEN 64
@@ -21,7 +22,7 @@ void buffer_json_journal_versions(BUFFER *wb)
22
buffer_json_object_close(wb);
23
}
24
24
-static bool journal_sd_id128_parse(const char *in, sd_id128_t *ret)
25
+static bool journal_sd_id128_parse(const char *in, NsdId128 *ret)
26
{
27
while (isspace(*in))
28
in++;
@@ -31,8 +32,8 @@ static bool journal_sd_id128_parse(const char *in, sd_id128_t *ret)
32
uuid[32] = '\0';
33
34
if (strlen(uuid) == 32) {
34
- sd_id128_t read;
35
- if (sd_id128_from_string(uuid, &read) == 0) {
35
+ NsdId128 read;
36
+ if (nsd_id128_from_string(uuid, &read) == 0) {
37
*ret = read;
38
return true;
39
}
@@ -42,7 +43,7 @@ static bool journal_sd_id128_parse(const char *in, sd_id128_t *ret)
43
}
44
45
usec_t
45
-nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file *njf __maybe_unused, const char *boot_id)
46
+nd_journal_file_update_annotation_boot_id(NsdJournal *j, struct nd_journal_file *njf __maybe_unused, const char *boot_id)
47
{
48
usec_t ut = UINT64_MAX;
49
int r;
@@ -50,9 +51,9 @@ nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file
51
char m[100];
52
size_t len = snprintfz(m, sizeof(m), "_BOOT_ID=%s", boot_id);
53
53
- sd_journal_flush_matches(j);
54
+ nsd_journal_flush_matches(j);
55
55
- r = sd_journal_add_match(j, m, len);
56
+ r = nsd_journal_add_match(j, m, len);
57
if (r < 0) {
58
errno = -r;
59
internal_error(
@@ -66,7 +67,7 @@ nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file
67
return UINT64_MAX;
68
}
69
69
- r = sd_journal_seek_head(j);
70
+ r = nsd_journal_seek_head(j);
71
if (r < 0) {
72
errno = -r;
73
internal_error(
@@ -79,7 +80,7 @@ nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file
80
return UINT64_MAX;
81
}
82
82
- r = sd_journal_next(j);
83
+ r = nsd_journal_next(j);
84
if (r < 0) {
85
errno = -r;
86
internal_error(
@@ -90,9 +91,18 @@ nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file
91
njf->filename,
92
r);
93
return UINT64_MAX;
94
+ } else if (r == 0) {
95
+ internal_error(
96
+ true,
97
+ "GVD: while looking for the first timestamp of boot_id '%s', "
98
+ "sd_journal_next() on file '%s' returned %d",
99
+ boot_id,
100
+ njf->filename,
101
+ r);
102
+ return UINT64_MAX;
103
}
104
95
- r = sd_journal_get_realtime_usec(j, &ut);
105
+ r = nsd_journal_get_realtime_usec(j, &ut);
106
if (r < 0 || !ut || ut == UINT64_MAX) {
107
errno = -r;
108
internal_error(
@@ -114,12 +124,12 @@ nd_journal_file_update_annotation_boot_id(sd_journal *j, struct nd_journal_file
124
}
125
126
static void
117
-nd_journal_file_get_boot_id_annotations(sd_journal *j __maybe_unused, struct nd_journal_file *njf __maybe_unused)
127
+nd_journal_file_get_boot_id_annotations(NsdJournal *j __maybe_unused, struct nd_journal_file *njf __maybe_unused)
128
{
129
#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
120
- sd_journal_flush_matches(j);
130
+ nsd_journal_flush_matches(j);
131
122
- int r = sd_journal_query_unique(j, "_BOOT_ID");
132
+ int r = nsd_journal_query_unique(j, "_BOOT_ID");
133
if (r < 0) {
134
errno = -r;
135
internal_error(
@@ -137,7 +147,7 @@ nd_journal_file_get_boot_id_annotations(sd_journal *j __maybe_unused, struct nd_
147
148
DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
149
140
- SD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
150
+ NSD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
151
{
152
const char *key, *value;
153
size_t key_length, value_length;
@@ -178,8 +188,8 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
188
[1] = NULL,
189
};
190
181
- sd_journal *j = NULL;
182
- if (sd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
191
+ NsdJournal *j = NULL;
192
+ if (nsd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
193
netdata_log_error("JOURNAL: cannot open file '%s' to update msg_ut", filename);
194
fstat_cache_disable_on_thread();
195
@@ -197,16 +207,16 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
207
208
usec_t first_ut = 0, last_ut = 0;
209
uint64_t first_seqnum = 0, last_seqnum = 0;
200
- sd_id128_t first_writer_id = SD_ID128_NULL, last_writer_id = SD_ID128_NULL;
210
+ NsdId128 first_writer_id = NSD_ID128_NULL, last_writer_id = NSD_ID128_NULL;
211
202
- if (sd_journal_seek_head(j) < 0 || sd_journal_next(j) < 0 || sd_journal_get_realtime_usec(j, &first_ut) < 0 ||
212
+ if (nsd_journal_seek_head(j) < 0 || nsd_journal_next(j) < 0 || nsd_journal_get_realtime_usec(j, &first_ut) < 0 ||
213
!first_ut) {
214
internal_error(true, "cannot find the timestamp of the first message in '%s'", filename);
215
first_ut = 0;
216
}
217
#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
218
else {
209
- if (sd_journal_get_seqnum(j, &first_seqnum, &first_writer_id) < 0 || !first_seqnum) {
219
+ if (nsd_journal_get_seqnum(j, &first_seqnum, &first_writer_id) < 0 || !first_seqnum) {
220
internal_error(true, "cannot find the first seqnums of the first message in '%s'", filename);
221
first_seqnum = 0;
222
memset(&first_writer_id, 0, sizeof(first_writer_id));
@@ -214,14 +224,14 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
224
}
225
#endif
226
217
- if (sd_journal_seek_tail(j) < 0 || sd_journal_previous(j) < 0 || sd_journal_get_realtime_usec(j, &last_ut) < 0 ||
227
+ if (nsd_journal_seek_tail(j) < 0 || nsd_journal_previous(j) < 0 || nsd_journal_get_realtime_usec(j, &last_ut) < 0 ||
228
!last_ut) {
229
internal_error(true, "cannot find the timestamp of the last message in '%s'", filename);
230
last_ut = njf->file_last_modified_ut;
231
}
232
#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
233
else {
224
- if (sd_journal_get_seqnum(j, &last_seqnum, &last_writer_id) < 0 || !last_seqnum) {
234
+ if (nsd_journal_get_seqnum(j, &last_seqnum, &last_writer_id) < 0 || !last_seqnum) {
235
internal_error(true, "cannot find the last seqnums of the first message in '%s'", filename);
236
last_seqnum = 0;
237
memset(&last_writer_id, 0, sizeof(last_writer_id));
@@ -249,7 +259,7 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
259
if (is_journal_file(filename, -1, &dot_journal) && dot_journal && dot_journal > dash_first_msg_ut) {
260
if (dash_seqnum - at - 1 == 32 && dash_first_msg_ut - dash_seqnum - 1 == 16 &&
261
dot_journal - dash_first_msg_ut - 1 == 16) {
252
- sd_id128_t writer;
262
+ NsdId128 writer;
263
if (journal_sd_id128_parse(at + 1, &writer)) {
264
char *endptr = NULL;
265
uint64_t seqnum = strtoul(dash_seqnum + 1, &endptr, 16);
@@ -282,7 +292,7 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
292
njf->msg_last_ut = njf->file_last_modified_ut;
293
294
if (last_seqnum > first_seqnum) {
285
- if (!sd_id128_equal(first_writer_id, last_writer_id)) {
295
+ if (!nsd_id128_equal(first_writer_id, last_writer_id)) {
296
njf->messages_in_file = 0;
297
nd_log(
298
NDLS_COLLECTORS,
@@ -295,7 +305,7 @@ void nd_journal_file_update_header(const char *filename, struct nd_journal_file
305
njf->messages_in_file = 0;
306
307
nd_journal_file_get_boot_id_annotations(j, njf);
298
- sd_journal_close(j);
308
+ nsd_journal_close(j);
309
fstat_cache_disable_on_thread();
310
311
njf->last_scan_header_vs_last_modified_ut = njf->file_last_modified_ut;
src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
+7
-7
@@ -255,7 +255,7 @@ static inline size_t sampling_running_file_query_estimate_remaining_lines_by_tim
255
}
256
257
static inline size_t sampling_running_file_query_estimate_remaining_lines(
258
- sd_journal *j __maybe_unused,
258
+ NsdJournal *j __maybe_unused,
259
LOGS_QUERY_STATUS *lqs,
260
struct nd_journal_file *jf,
261
FACETS_ANCHOR_DIRECTION direction,
@@ -267,10 +267,10 @@ static inline size_t sampling_running_file_query_estimate_remaining_lines(
267
size_t expected_matching_logs_by_seqnum = 0;
268
double proportion_by_seqnum = 0.0;
269
uint64_t current_msg_seqnum;
270
- sd_id128_t current_msg_writer;
271
- if (!lqs->c.query_file.first_msg_seqnum || sd_journal_get_seqnum(j, ¤t_msg_seqnum, ¤t_msg_writer) < 0) {
270
+ NsdId128 current_msg_writer;
271
+ if (!lqs->c.query_file.first_msg_seqnum || nsd_journal_get_seqnum(j, ¤t_msg_seqnum, ¤t_msg_writer) < 0) {
272
lqs->c.query_file.first_msg_seqnum = 0;
273
- lqs->c.query_file.first_msg_writer = SD_ID128_NULL;
273
+ lqs->c.query_file.first_msg_writer = NSD_ID128_NULL;
274
} else if (jf->messages_in_file) {
275
size_t scanned_lines = sampling_file_lines_scanned_so_far(lqs);
276
@@ -303,7 +303,7 @@ static inline size_t sampling_running_file_query_estimate_remaining_lines(
303
}
304
305
static inline void sampling_decide_file_sampling_every(
306
- sd_journal *j,
306
+ NsdJournal *j,
307
LOGS_QUERY_STATUS *lqs,
308
struct nd_journal_file *jf,
309
FACETS_ANCHOR_DIRECTION direction,
@@ -331,7 +331,7 @@ typedef enum {
331
} sampling_t;
332
333
static inline sampling_t is_row_in_sample(
334
- sd_journal *j,
334
+ NsdJournal *j,
335
LOGS_QUERY_STATUS *lqs,
336
struct nd_journal_file *jf,
337
usec_t msg_ut,
@@ -397,7 +397,7 @@ static inline sampling_t is_row_in_sample(
397
398
static inline void sampling_update_running_query_file_estimates(
399
FACETS *facets,
400
- sd_journal *j,
400
+ NsdJournal *j,
401
LOGS_QUERY_STATUS *lqs,
402
struct nd_journal_file *jf,
403
usec_t msg_ut,
src/collectors/systemd-journal.plugin/systemd-journal.c
+29
-28
@@ -6,6 +6,7 @@
6
*
7
*/
8
9
+#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
10
#include "systemd-internals.h"
11
12
#define ND_SD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
@@ -29,7 +30,7 @@ struct lqs_extension {
30
usec_t stop_ut;
31
usec_t first_msg_ut;
32
32
- sd_id128_t first_msg_writer;
33
+ NsdId128 first_msg_writer;
34
uint64_t first_msg_seqnum;
35
} query_file;
36
@@ -210,11 +211,11 @@ static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value)
211
return ND_SD_JF_NONE;
212
}
213
213
-static inline bool nd_sd_journal_seek_to(sd_journal *j, usec_t timestamp)
214
+static inline bool nd_sd_journal_seek_to(NsdJournal *j, usec_t timestamp)
215
{
215
- if (sd_journal_seek_realtime_usec(j, timestamp) < 0) {
216
+ if (nsd_journal_seek_realtime_usec(j, timestamp) < 0) {
217
netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, timestamp);
217
- if (sd_journal_seek_tail(j) < 0) {
218
+ if (nsd_journal_seek_tail(j) < 0) {
219
netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to journal's tail");
220
return false;
221
}
@@ -226,7 +227,7 @@ static inline bool nd_sd_journal_seek_to(sd_journal *j, usec_t timestamp)
227
#define JD_SOURCE_REALTIME_TIMESTAMP "_SOURCE_REALTIME_TIMESTAMP"
228
229
static inline size_t
229
-nd_sd_journal_process_row(sd_journal *j, FACETS *facets, struct nd_journal_file *njf, usec_t *msg_ut)
230
+nd_sd_journal_process_row(NsdJournal *j, FACETS *facets, struct nd_journal_file *njf, usec_t *msg_ut)
231
{
232
const void *data;
233
size_t length, bytes = 0;
@@ -234,7 +235,7 @@ nd_sd_journal_process_row(sd_journal *j, FACETS *facets, struct nd_journal_file
235
facets_add_key_value_length(
236
facets, JOURNAL_KEY_ND_JOURNAL_FILE, sizeof(JOURNAL_KEY_ND_JOURNAL_FILE) - 1, njf->filename, njf->filename_len);
237
237
- SD_JOURNAL_FOREACH_DATA(j, data, length)
238
+ NSD_JOURNAL_FOREACH_DATA(j, data, length)
239
{
240
const char *key, *value;
241
size_t key_length, value_length;
@@ -309,7 +310,7 @@ static inline ND_SD_JOURNAL_STATUS check_stop(const bool *cancelled, const usec_
310
}
311
312
ND_SD_JOURNAL_STATUS nd_sd_journal_query_backward(
312
- sd_journal *j,
313
+ NsdJournal *j,
314
BUFFER *wb __maybe_unused,
315
FACETS *facets,
316
struct nd_journal_file *njf,
@@ -339,9 +340,9 @@ ND_SD_JOURNAL_STATUS nd_sd_journal_query_backward(
340
ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK;
341
342
facets_rows_begin(facets);
342
- while (status == ND_SD_JOURNAL_OK && sd_journal_previous(j) > 0) {
343
+ while (status == ND_SD_JOURNAL_OK && nsd_journal_previous(j) > 0) {
344
usec_t msg_ut = 0;
344
- if (sd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
345
+ if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
346
errors_no_timestamp++;
347
continue;
348
}
@@ -360,10 +361,10 @@ ND_SD_JOURNAL_STATUS nd_sd_journal_query_backward(
361
fqs->c.query_file.first_msg_ut = msg_ut;
362
363
#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
363
- if (sd_journal_get_seqnum(j, &fqs->c.query_file.first_msg_seqnum, &fqs->c.query_file.first_msg_writer) <
364
+ if (nsd_journal_get_seqnum(j, &fqs->c.query_file.first_msg_seqnum, &fqs->c.query_file.first_msg_writer) <
365
0) {
366
fqs->c.query_file.first_msg_seqnum = 0;
366
- fqs->c.query_file.first_msg_writer = SD_ID128_NULL;
367
+ fqs->c.query_file.first_msg_writer = NSD_ID128_NULL;
368
}
369
#endif
370
}
@@ -425,7 +426,7 @@ ND_SD_JOURNAL_STATUS nd_sd_journal_query_backward(
426
}
427
428
ND_SD_JOURNAL_STATUS nd_sd_journal_query_forward(
428
- sd_journal *j,
429
+ NsdJournal *j,
430
BUFFER *wb __maybe_unused,
431
FACETS *facets,
432
struct nd_journal_file *njf,
@@ -455,9 +456,9 @@ ND_SD_JOURNAL_STATUS nd_sd_journal_query_forward(
456
ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK;
457
458
facets_rows_begin(facets);
458
- while (status == ND_SD_JOURNAL_OK && sd_journal_next(j) > 0) {
459
+ while (status == ND_SD_JOURNAL_OK && nsd_journal_next(j) > 0) {
460
usec_t msg_ut = 0;
460
- if (sd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
461
+ if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
462
errors_no_timestamp++;
463
continue;
464
}
@@ -532,7 +533,7 @@ ND_SD_JOURNAL_STATUS nd_sd_journal_query_forward(
533
return status;
534
}
535
535
-bool nd_sd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t last_modified)
536
+bool nd_sd_journal_check_if_modified_since(NsdJournal *j, usec_t seek_to, usec_t last_modified)
537
{
538
// return true, if data have been modified since the timestamp
539
@@ -543,9 +544,9 @@ bool nd_sd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t
544
return false;
545
546
usec_t first_msg_ut = 0;
546
- while (sd_journal_previous(j) > 0) {
547
+ while (nsd_journal_previous(j) > 0) {
548
usec_t msg_ut;
548
- if (sd_journal_get_realtime_usec(j, &msg_ut) < 0)
549
+ if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0)
550
continue;
551
552
first_msg_ut = msg_ut;
@@ -556,7 +557,7 @@ bool nd_sd_journal_check_if_modified_since(sd_journal *j, usec_t seek_to, usec_t
557
}
558
559
#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
559
-static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets, LOGS_QUERY_STATUS *lqs)
560
+static bool netdata_systemd_filtering_by_journal(NsdJournal *j, FACETS *facets, LOGS_QUERY_STATUS *lqs)
561
{
562
const char *field = NULL;
563
const void *data = NULL;
@@ -565,7 +566,7 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
566
size_t failures = 0;
567
size_t filters_added = 0;
568
568
- SD_JOURNAL_FOREACH_FIELD(j, field)
569
+ NSD_JOURNAL_FOREACH_FIELD(j, field)
570
{ // for each key
571
bool interesting;
572
@@ -575,11 +576,11 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
576
interesting = facets_key_name_is_facet(facets, field);
577
578
if (interesting) {
578
- if (sd_journal_query_unique(j, field) >= 0) {
579
+ if (nsd_journal_query_unique(j, field) >= 0) {
580
bool added_this_key = false;
581
size_t added_values = 0;
582
582
- SD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
583
+ NSD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
584
{ // for each value of the key
585
const char *key, *value;
586
size_t key_length, value_length;
@@ -593,16 +594,16 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
594
continue;
595
596
if (added_keys && !added_this_key) {
596
- if (sd_journal_add_conjunction(j) < 0) // key AND key AND key
597
+ if (nsd_journal_add_conjunction(j) < 0) // key AND key AND key
598
failures++;
599
600
added_this_key = true;
601
added_keys++;
602
} else if (added_values)
602
- if (sd_journal_add_disjunction(j) < 0) // value OR value OR value
603
+ if (nsd_journal_add_disjunction(j) < 0) // value OR value OR value
604
failures++;
605
605
- if (sd_journal_add_match(j, data, data_length) < 0)
606
+ if (nsd_journal_add_match(j, data, data_length) < 0)
607
failures++;
608
609
if (!added_keys) {
@@ -619,7 +620,7 @@ static bool netdata_systemd_filtering_by_journal(sd_journal *j, FACETS *facets,
620
621
if (failures) {
622
lqs_log_error(lqs, "failed to setup journal filter, will run the full query.");
622
- sd_journal_flush_matches(j);
623
+ nsd_journal_flush_matches(j);
624
return true;
625
}
626
@@ -634,7 +635,7 @@ static ND_SD_JOURNAL_STATUS nd_sd_journal_query_one_file(
635
struct nd_journal_file *njf,
636
LOGS_QUERY_STATUS *fqs)
637
{
637
- sd_journal *j = NULL;
638
+ NsdJournal *j = NULL;
639
errno_clear();
640
641
fstat_cache_enable_on_thread();
@@ -644,7 +645,7 @@ static ND_SD_JOURNAL_STATUS nd_sd_journal_query_one_file(
645
[1] = NULL,
646
};
647
647
- if (sd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
648
+ if (nsd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
649
netdata_log_error("JOURNAL: cannot open file '%s' for query", filename);
650
fstat_cache_disable_on_thread();
651
return ND_SD_JOURNAL_FAILED_TO_OPEN;
@@ -672,7 +673,7 @@ static ND_SD_JOURNAL_STATUS nd_sd_journal_query_one_file(
673
} else
674
status = ND_SD_JOURNAL_NO_FILE_MATCHED;
675
675
- sd_journal_close(j);
676
+ nsd_journal_close(j);
677
fstat_cache_disable_on_thread();
678
679
return status;
src/collectors/systemd-journal.plugin/systemd-main.c
-25
@@ -54,15 +54,6 @@ int main(int argc __maybe_unused, char **argv __maybe_unused)
54
function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
55
exit(1);
56
}
57
-#ifdef ENABLE_SYSTEMD_DBUS
58
- if (argc == 2 && strcmp(argv[1], "debug-units") == 0) {
59
- bool cancelled = false;
60
- usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
61
- function_systemd_units(
62
- "123", "systemd-units", &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
63
- exit(1);
64
- }
65
-#endif
57
58
// ------------------------------------------------------------------------
59
// watcher thread
@@ -78,11 +69,6 @@ int main(int argc __maybe_unused, char **argv __maybe_unused)
69
functions_evloop_add_function(
70
wg, ND_SD_JOURNAL_FUNCTION_NAME, function_systemd_journal, ND_SD_JOURNAL_DEFAULT_TIMEOUT, NULL);
71
81
-#ifdef ENABLE_SYSTEMD_DBUS
82
- functions_evloop_add_function(
83
- wg, ND_SD_UNITS_FUNCTION_NAME, function_systemd_units, ND_SD_UNITS_DEFAULT_TIMEOUT, NULL);
84
-#endif
85
-
72
nd_systemd_journal_dyncfg_init(wg);
73
74
// ------------------------------------------------------------------------
@@ -99,17 +85,6 @@ int main(int argc __maybe_unused, char **argv __maybe_unused)
85
(HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
86
RRDFUNCTIONS_PRIORITY_DEFAULT);
87
102
-#ifdef ENABLE_SYSTEMD_DBUS
103
- fprintf(
104
- stdout,
105
- PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"top\" " HTTP_ACCESS_FORMAT " %d\n",
106
- ND_SD_UNITS_FUNCTION_NAME,
107
- ND_SD_UNITS_DEFAULT_TIMEOUT,
108
- ND_SD_UNITS_FUNCTION_DESCRIPTION,
109
- (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
110
- RRDFUNCTIONS_PRIORITY_DEFAULT);
111
-#endif
112
-
88
fflush(stdout);
89
netdata_mutex_unlock(&stdout_mutex);
90
src/collectors/systemd-units.plugin/plugin_systemd_units.c
renamed
+80
-3
@@ -1,13 +1,23 @@
1
// SPDX-License-Identifier: GPL-3.0-or-later
2
3
-#include "systemd-internals.h"
3
+#include "collectors/all.h"
4
+#include "libnetdata/libnetdata.h"
5
+#include "libnetdata/required_dummies.h"
6
5
-#ifdef ENABLE_SYSTEMD_DBUS
7
+#include <linux/capability.h>
8
+#include <syslog.h>
9
#include <systemd/sd-bus.h>
10
11
+#define ND_SD_JOURNAL_WORKER_THREADS 2
12
+#define ND_SD_UNITS_FUNCTION_DESCRIPTION "View the status of systemd units"
13
+#define ND_SD_UNITS_FUNCTION_NAME "systemd-list-units"
14
+#define ND_SD_UNITS_DEFAULT_TIMEOUT 30
15
+
16
#define ND_SD_UNITS_MAX_PARAMS 10
17
#define ND_SD_UNITS_DBUS_TYPES "(ssssssouso)"
18
19
+netdata_mutex_t stdout_mutex = NETDATA_MUTEX_INITIALIZER;
20
+
21
// ----------------------------------------------------------------------------
22
// copied from systemd: string-table.h
23
@@ -2177,4 +2187,71 @@ void function_systemd_units(
2187
systemd_units_free_all(base);
2188
}
2189
2180
-#endif // ENABLE_SYSTEMD_DBUS
2190
+static bool plugin_should_exit = false;
2191
+
2192
+int main(int argc __maybe_unused, char **argv __maybe_unused)
2193
+{
2194
+ nd_thread_tag_set("sd-unit.plugin");
2195
+ nd_log_initialize_for_external_plugins("systemd-units.plugin");
2196
+ netdata_threads_init_for_external_plugins(0);
2197
+
2198
+ netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
2199
+ if (verify_netdata_host_prefix(true) == -1)
2200
+ exit(1);
2201
+
2202
+ // ------------------------------------------------------------------------
2203
+ // debug
2204
+
2205
+ if (argc == 2 && strcmp(argv[1], "debug-units") == 0) {
2206
+ bool cancelled = false;
2207
+ usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
2208
+ function_systemd_units(
2209
+ "123", "systemd-units", &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
2210
+ exit(1);
2211
+ }
2212
+
2213
+ // ------------------------------------------------------------------------
2214
+ // the event loop for functions
2215
+
2216
+ struct functions_evloop_globals *wg =
2217
+ functions_evloop_init(ND_SD_JOURNAL_WORKER_THREADS, "SDU", &stdout_mutex, &plugin_should_exit);
2218
+
2219
+ functions_evloop_add_function(
2220
+ wg, ND_SD_UNITS_FUNCTION_NAME, function_systemd_units, ND_SD_UNITS_DEFAULT_TIMEOUT, NULL);
2221
+
2222
+ // ------------------------------------------------------------------------
2223
+ // register functions to netdata
2224
+
2225
+ netdata_mutex_lock(&stdout_mutex);
2226
+
2227
+ fprintf(
2228
+ stdout,
2229
+ PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"top\" " HTTP_ACCESS_FORMAT " %d\n",
2230
+ ND_SD_UNITS_FUNCTION_NAME,
2231
+ ND_SD_UNITS_DEFAULT_TIMEOUT,
2232
+ ND_SD_UNITS_FUNCTION_DESCRIPTION,
2233
+ (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
2234
+ RRDFUNCTIONS_PRIORITY_DEFAULT);
2235
+
2236
+ fflush(stdout);
2237
+ netdata_mutex_unlock(&stdout_mutex);
2238
+
2239
+ // ------------------------------------------------------------------------
2240
+
2241
+ usec_t send_newline_ut = 0;
2242
+ const bool tty = isatty(fileno(stdout)) == 1;
2243
+
2244
+ heartbeat_t hb;
2245
+ heartbeat_init(&hb, USEC_PER_SEC);
2246
+ while (!plugin_should_exit) {
2247
+ usec_t dt_ut = heartbeat_next(&hb);
2248
+ send_newline_ut += dt_ut;
2249
+
2250
+ if (!tty && send_newline_ut > USEC_PER_SEC) {
2251
+ send_newline_and_flush(&stdout_mutex);
2252
+ send_newline_ut = 0;
2253
+ }
2254
+ }
2255
+
2256
+ exit(0);
2257
+}
src/crates/jf/.gitignore
new
+4
@@ -0,0 +1,4 @@
1
+target/
2
+.idea/
3
+Cargo.lock
4
+journal_reader_ffi/journal_reader_ffi.h
src/crates/jf/Cargo.toml
new
+37
@@ -0,0 +1,37 @@
1
+[workspace]
2
+resolver = "2"
3
+members = [
4
+ "error",
5
+ "main",
6
+ "journal_file",
7
+ "journal_writer",
8
+ "journal_reader",
9
+ "journal_reader_ffi",
10
+ "journal_logger",
11
+ "journal_forwarder",
12
+ "window_manager",
13
+ "sigbus",
14
+]
15
+
16
+[workspace.package]
17
+version = "0.1.0"
18
+edition = "2021"
19
+
20
+[workspace.dependencies]
21
+memmap2 = "0.9"
22
+zerocopy = { version = "0.9.0-alpha.0", features = ["derive"] }
23
+thiserror = "2"
24
+rand = "0.9"
25
+siphasher = "1.0"
26
+twox-hash = { version = "2.1", default-features = false, features = ["std"] }
27
+static_assertions = "1.1"
28
+duct = "0.13"
29
+serde_json = "1.0"
30
+walkdir = "2"
31
+ruzstd = "0.8"
32
+systemd = { path = "/home/vk/repos/crates/rust-systemd"}
33
+libc = "0.2"
34
+
35
+[profile.release]
36
+lto = true
37
+codegen-units = 1
src/crates/jf/LICENSE
new
+684
@@ -0,0 +1,684 @@
1
+ GNU GENERAL PUBLIC LICENSE
2
+ Version 3, 29 June 2007
3
+
4
+ Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
5
+ Everyone is permitted to copy and distribute verbatim copies
6
+ of this license document, but changing it is not allowed.
7
+
8
+ Preamble
9
+
10
+ The GNU General Public License is a free, copyleft license for
11
+software and other kinds of works.
12
+
13
+ The licenses for most software and other practical works are designed
14
+to take away your freedom to share and change the works. By contrast,
15
+the GNU General Public License is intended to guarantee your freedom to
16
+share and change all versions of a program--to make sure it remains free
17
+software for all its users. We, the Free Software Foundation, use the
18
+GNU General Public License for most of our software; it applies also to
19
+any other work released this way by its authors. You can apply it to
20
+your programs, too.
21
+
22
+ When we speak of free software, we are referring to freedom, not
23
+price. Our General Public Licenses are designed to make sure that you
24
+have the freedom to distribute copies of free software (and charge for
25
+them if you wish), that you receive source code or can get it if you
26
+want it, that you can change the software or use pieces of it in new
27
+free programs, and that you know you can do these things.
28
+
29
+ To protect your rights, we need to prevent others from denying you
30
+these rights or asking you to surrender the rights. Therefore, you have
31
+certain responsibilities if you distribute copies of the software, or if
32
+you modify it: responsibilities to respect the freedom of others.
33
+
34
+ For example, if you distribute copies of such a program, whether
35
+gratis or for a fee, you must pass on to the recipients the same
36
+freedoms that you received. You must make sure that they, too, receive
37
+or can get the source code. And you must show them these terms so they
38
+know their rights.
39
+
40
+ Developers that use the GNU GPL protect your rights with two steps:
41
+(1) assert copyright on the software, and (2) offer you this License
42
+giving you legal permission to copy, distribute and/or modify it.
43
+
44
+ For the developers' and authors' protection, the GPL clearly explains
45
+that there is no warranty for this free software. For both users' and
46
+authors' sake, the GPL requires that modified versions be marked as
47
+changed, so that their problems will not be attributed erroneously to
48
+authors of previous versions.
49
+
50
+ Some devices are designed to deny users access to install or run
51
+modified versions of the software inside them, although the manufacturer
52
+can do so. This is fundamentally incompatible with the aim of
53
+protecting users' freedom to change the software. The systematic
54
+pattern of such abuse occurs in the area of products for individuals to
55
+use, which is precisely where it is most unacceptable. Therefore, we
56
+have designed this version of the GPL to prohibit the practice for those
57
+products. If such problems arise substantially in other domains, we
58
+stand ready to extend this provision to those domains in future versions
59
+of the GPL, as needed to protect the freedom of users.
60
+
61
+ Finally, every program is threatened constantly by software patents.
62
+States should not allow patents to restrict development and use of
63
+software on general-purpose computers, but in those that do, we wish to
64
+avoid the special danger that patents applied to a free program could
65
+make it effectively proprietary. To prevent this, the GPL assures that
66
+patents cannot be used to render the program non-free.
67
+
68
+ The precise terms and conditions for copying, distribution and
69
+modification follow.
70
+
71
+ TERMS AND CONDITIONS
72
+
73
+ 0. Definitions.
74
+
75
+ "This License" refers to version 3 of the GNU General Public License.
76
+
77
+ "Copyright" also means copyright-like laws that apply to other kinds of
78
+works, such as semiconductor masks.
79
+
80
+ "The Program" refers to any copyrightable work licensed under this
81
+License. Each licensee is addressed as "you". "Licensees" and
82
+"recipients" may be individuals or organizations.
83
+
84
+ To "modify" a work means to copy from or adapt all or part of the work
85
+in a fashion requiring copyright permission, other than the making of an
86
+exact copy. The resulting work is called a "modified version" of the
87
+earlier work or a work "based on" the earlier work.
88
+
89
+ A "covered work" means either the unmodified Program or a work based
90
+on the Program.
91
+
92
+ To "propagate" a work means to do anything with it that, without
93
+permission, would make you directly or secondarily liable for
94
+infringement under applicable copyright law, except executing it on a
95
+computer or modifying a private copy. Propagation includes copying,
96
+distribution (with or without modification), making available to the
97
+public, and in some countries other activities as well.
98
+
99
+ To "convey" a work means any kind of propagation that enables other
100
+parties to make or receive copies. Mere interaction with a user through
101
+a computer network, with no transfer of a copy, is not conveying.
102
+
103
+ An interactive user interface displays "Appropriate Legal Notices"
104
+to the extent that it includes a convenient and prominently visible
105
+feature that (1) displays an appropriate copyright notice, and (2)
106
+tells the user that there is no warranty for the work (except to the
107
+extent that warranties are provided), that licensees may convey the
108
+work under this License, and how to view a copy of this License. If
109
+the interface presents a list of user commands or options, such as a
110
+menu, a prominent item in the list meets this criterion.
111
+
112
+ 1. Source Code.
113
+
114
+ The "source code" for a work means the preferred form of the work
115
+for making modifications to it. "Object code" means any non-source
116
+form of a work.
117
+
118
+ A "Standard Interface" means an interface that either is an official
119
+standard defined by a recognized standards body, or, in the case of
120
+interfaces specified for a particular programming language, one that
121
+is widely used among developers working in that language.
122
+
123
+ The "System Libraries" of an executable work include anything, other
124
+than the work as a whole, that (a) is included in the normal form of
125
+packaging a Major Component, but which is not part of that Major
126
+Component, and (b) serves only to enable use of the work with that
127
+Major Component, or to implement a Standard Interface for which an
128
+implementation is available to the public in source code form. A
129
+"Major Component", in this context, means a major essential component
130
+(kernel, window system, and so on) of the specific operating system
131
+(if any) on which the executable work runs, or a compiler used to
132
+produce the work, or an object code interpreter used to run it.
133
+
134
+ The "Corresponding Source" for a work in object code form means all
135
+the source code needed to generate, install, and (for an executable
136
+work) run the object code and to modify the work, including scripts to
137
+control those activities. However, it does not include the work's
138
+System Libraries, or general-purpose tools or generally available free
139
+programs which are used unmodified in performing those activities but
140
+which are not part of the work. For example, Corresponding Source
141
+includes interface definition files associated with source files for
142
+the work, and the source code for shared libraries and dynamically
143
+linked subprograms that the work is specifically designed to require,
144
+such as by intimate data communication or control flow between those
145
+subprograms and other parts of the work.
146
+
147
+ The Corresponding Source need not include anything that users
148
+can regenerate automatically from other parts of the Corresponding
149
+Source.
150
+
151
+ The Corresponding Source for a work in source code form is that
152
+same work.
153
+
154
+ 2. Basic Permissions.
155
+
156
+ All rights granted under this License are granted for the term of
157
+copyright on the Program, and are irrevocable provided the stated
158
+conditions are met. This License explicitly affirms your unlimited
159
+permission to run the unmodified Program. The output from running a
160
+covered work is covered by this License only if the output, given its
161
+content, constitutes a covered work. This License acknowledges your
162
+rights of fair use or other equivalent, as provided by copyright law.
163
+
164
+ You may make, run and propagate covered works that you do not
165
+convey, without conditions so long as your license otherwise remains
166
+in force. You may convey covered works to others for the sole purpose
167
+of having them make modifications exclusively for you, or provide you
168
+with facilities for running those works, provided that you comply with
169
+the terms of this License in conveying all material for which you do
170
+not control copyright. Those thus making or running the covered works
171
+for you must do so exclusively on your behalf, under your direction
172
+and control, on terms that prohibit them from making any copies of
173
+your copyrighted material outside their relationship with you.
174
+
175
+ Conveying under any other circumstances is permitted solely under
176
+the conditions stated below. Sublicensing is not allowed; section 10
177
+makes it unnecessary.
178
+
179
+ 3. Protecting Users' Legal Rights From Anti-Circumvention Law.
180
+
181
+ No covered work shall be deemed part of an effective technological
182
+measure under any applicable law fulfilling obligations under article
183
+11 of the WIPO copyright treaty adopted on 20 December 1996, or
184
+similar laws prohibiting or restricting circumvention of such
185
+measures.
186
+
187
+ When you convey a covered work, you waive any legal power to forbid
188
+circumvention of technological measures to the extent such circumvention
189
+is effected by exercising rights under this License with respect to
190
+the covered work, and you disclaim any intention to limit operation or
191
+modification of the work as a means of enforcing, against the work's
192
+users, your or third parties' legal rights to forbid circumvention of
193
+technological measures.
194
+
195
+ 4. Conveying Verbatim Copies.
196
+
197
+ You may convey verbatim copies of the Program's source code as you
198
+receive it, in any medium, provided that you conspicuously and
199
+appropriately publish on each copy an appropriate copyright notice;
200
+keep intact all notices stating that this License and any
201
+non-permissive terms added in accord with section 7 apply to the code;
202
+keep intact all notices of the absence of any warranty; and give all
203
+recipients a copy of this License along with the Program.
204
+
205
+ You may charge any price or no price for each copy that you convey,
206
+and you may offer support or warranty protection for a fee.
207
+
208
+ 5. Conveying Modified Source Versions.
209
+
210
+ You may convey a work based on the Program, or the modifications to
211
+produce it from the Program, in the form of source code under the
212
+terms of section 4, provided that you also meet all of these conditions:
213
+
214
+ a) The work must carry prominent notices stating that you modified
215
+ it, and giving a relevant date.
216
+
217
+ b) The work must carry prominent notices stating that it is
218
+ released under this License and any conditions added under section
219
+ 7. This requirement modifies the requirement in section 4 to
220
+ "keep intact all notices".
221
+
222
+ c) You must license the entire work, as a whole, under this
223
+ License to anyone who comes into possession of a copy. This
224
+ License will therefore apply, along with any applicable section 7
225
+ additional terms, to the whole of the work, and all its parts,
226
+ regardless of how they are packaged. This License gives no
227
+ permission to license the work in any other way, but it does not
228
+ invalidate such permission if you have separately received it.
229
+
230
+ d) If the work has interactive user interfaces, each must display
231
+ Appropriate Legal Notices; however, if the Program has interactive
232
+ interfaces that do not display Appropriate Legal Notices, your
233
+ work need not make them do so.
234
+
235
+ A compilation of a covered work with other separate and independent
236
+works, which are not by their nature extensions of the covered work,
237
+and which are not combined with it such as to form a larger program,
238
+in or on a volume of a storage or distribution medium, is called an
239
+"aggregate" if the compilation and its resulting copyright are not
240
+used to limit the access or legal rights of the compilation's users
241
+beyond what the individual works permit. Inclusion of a covered work
242
+in an aggregate does not cause this License to apply to the other
243
+parts of the aggregate.
244
+
245
+ 6. Conveying Non-Source Forms.
246
+
247
+ You may convey a covered work in object code form under the terms
248
+of sections 4 and 5, provided that you also convey the
249
+machine-readable Corresponding Source under the terms of this License,
250
+in one of these ways:
251
+
252
+ a) Convey the object code in, or embodied in, a physical product
253
+ (including a physical distribution medium), accompanied by the
254
+ Corresponding Source fixed on a durable physical medium
255
+ customarily used for software interchange.
256
+
257
+ b) Convey the object code in, or embodied in, a physical product
258
+ (including a physical distribution medium), accompanied by a
259
+ written offer, valid for at least three years and valid for as
260
+ long as you offer spare parts or customer support for that product
261
+ model, to give anyone who possesses the object code either (1) a
262
+ copy of the Corresponding Source for all the software in the
263
+ product that is covered by this License, on a durable physical
264
+ medium customarily used for software interchange, for a price no
265
+ more than your reasonable cost of physically performing this
266
+ conveying of source, or (2) access to copy the
267
+ Corresponding Source from a network server at no charge.
268
+
269
+ c) Convey individual copies of the object code with a copy of the
270
+ written offer to provide the Corresponding Source. This
271
+ alternative is allowed only occasionally and noncommercially, and
272
+ only if you received the object code with such an offer, in accord
273
+ with subsection 6b.
274
+
275
+ d) Convey the object code by offering access from a designated
276
+ place (gratis or for a charge), and offer equivalent access to the
277
+ Corresponding Source in the same way through the same place at no
278
+ further charge. You need not require recipients to copy the
279
+ Corresponding Source along with the object code. If the place to
280
+ copy the object code is a network server, the Corresponding Source
281
+ may be on a different server (operated by you or a third party)
282
+ that supports equivalent copying facilities, provided you maintain
283
+ clear directions next to the object code saying where to find the
284
+ Corresponding Source. Regardless of what server hosts the
285
+ Corresponding Source, you remain obligated to ensure that it is
286
+ available for as long as needed to satisfy these requirements.
287
+
288
+ e) Convey the object code using peer-to-peer transmission, provided
289
+ you inform other peers where the object code and Corresponding
290
+ Source of the work are being offered to the general public at no
291
+ charge under subsection 6d.
292
+
293
+ A separable portion of the object code, whose source code is excluded
294
+from the Corresponding Source as a System Library, need not be
295
+included in conveying the object code work.
296
+
297
+ A "User Product" is either (1) a "consumer product", which means any
298
+tangible personal property which is normally used for personal, family,
299
+or household purposes, or (2) anything designed or sold for incorporation
300
+into a dwelling. In determining whether a product is a consumer product,
301
+doubtful cases shall be resolved in favor of coverage. For a particular
302
+product received by a particular user, "normally used" refers to a
303
+typical or common use of that class of product, regardless of the status
304
+of the particular user or of the way in which the particular user
305
+actually uses, or expects or is expected to use, the product. A product
306
+is a consumer product regardless of whether the product has substantial
307
+commercial, industrial or non-consumer uses, unless such uses represent
308
+the only significant mode of use of the product.
309
+
310
+ "Installation Information" for a User Product means any methods,
311
+procedures, authorization keys, or other information required to install
312
+and execute modified versions of a covered work in that User Product from
313
+a modified version of its Corresponding Source. The information must
314
+suffice to ensure that the continued functioning of the modified object
315
+code is in no case prevented or interfered with solely because
316
+modification has been made.
317
+
318
+ If you convey an object code work under this section in, or with, or
319
+specifically for use in, a User Product, and the conveying occurs as
320
+part of a transaction in which the right of possession and use of the
321
+User Product is transferred to the recipient in perpetuity or for a
322
+fixed term (regardless of how the transaction is characterized), the
323
+Corresponding Source conveyed under this section must be accompanied
324
+by the Installation Information. But this requirement does not apply
325
+if neither you nor any third party retains the ability to install
326
+modified object code on the User Product (for example, the work has
327
+been installed in ROM).
328
+
329
+ The requirement to provide Installation Information does not include a
330
+requirement to continue to provide support service, warranty, or updates
331
+for a work that has been modified or installed by the recipient, or for
332
+the User Product in which it has been modified or installed. Access to a
333
+network may be denied when the modification itself materially and
334
+adversely affects the operation of the network or violates the rules and
335
+protocols for communication across the network.
336
+
337
+ Corresponding Source conveyed, and Installation Information provided,
338
+in accord with this section must be in a format that is publicly
339
+documented (and with an implementation available to the public in
340
+source code form), and must require no special password or key for
341
+unpacking, reading or copying.
342
+
343
+ 7. Additional Terms.
344
+
345
+ "Additional permissions" are terms that supplement the terms of this
346
+License by making exceptions from one or more of its conditions.
347
+Additional permissions that are applicable to the entire Program shall
348
+be treated as though they were included in this License, to the extent
349
+that they are valid under applicable law. If additional permissions
350
+apply only to part of the Program, that part may be used separately
351
+under those permissions, but the entire Program remains governed by
352
+this License without regard to the additional permissions.
353
+
354
+ When you convey a copy of a covered work, you may at your option
355
+remove any additional permissions from that copy, or from any part of
356
+it. (Additional permissions may be written to require their own
357
+removal in certain cases when you modify the work.) You may place
358
+additional permissions on material, added by you to a covered work,
359
+for which you have or can give appropriate copyright permission.
360
+
361
+ Notwithstanding any other provision of this License, for material you
362
+add to a covered work, you may (if authorized by the copyright holders of
363
+that material) supplement the terms of this License with terms:
364
+
365
+ a) Disclaiming warranty or limiting liability differently from the
366
+ terms of sections 15 and 16 of this License; or
367
+
368
+ b) Requiring preservation of specified reasonable legal notices or
369
+ author attributions in that material or in the Appropriate Legal
370
+ Notices displayed by works containing it; or
371
+
372
+ c) Prohibiting misrepresentation of the origin of that material, or
373
+ requiring that modified versions of such material be marked in
374
+ reasonable ways as different from the original version; or
375
+
376
+ d) Limiting the use for publicity purposes of names of licensors or
377
+ authors of the material; or
378
+
379
+ e) Declining to grant rights under trademark law for use of some
380
+ trade names, trademarks, or service marks; or
381
+
382
+ f) Requiring indemnification of licensors and authors of that
383
+ material by anyone who conveys the material (or modified versions of
384
+ it) with contractual assumptions of liability to the recipient, for
385
+ any liability that these contractual assumptions directly impose on
386
+ those licensors and authors.
387
+
388
+ All other non-permissive additional terms are considered "further
389
+restrictions" within the meaning of section 10. If the Program as you
390
+received it, or any part of it, contains a notice stating that it is
391
+governed by this License along with a term that is a further
392
+restriction, you may remove that term. If a license document contains
393
+a further restriction but permits relicensing or conveying under this
394
+License, you may add to a covered work material governed by the terms
395
+of that license document, provided that the further restriction does
396
+not survive such relicensing or conveying.
397
+
398
+ If you add terms to a covered work in accord with this section, you
399
+must place, in the relevant source files, a statement of the
400
+additional terms that apply to those files, or a notice indicating
401
+where to find the applicable terms.
402
+
403
+ Additional terms, permissive or non-permissive, may be stated in the
404
+form of a separately written license, or stated as exceptions;
405
+the above requirements apply either way.
406
+
407
+ 8. Termination.
408
+
409
+ You may not propagate or modify a covered work except as expressly
410
+provided under this License. Any attempt otherwise to propagate or
411
+modify it is void, and will automatically terminate your rights under
412
+this License (including any patent licenses granted under the third
413
+paragraph of section 11).
414
+
415
+ However, if you cease all violation of this License, then your
416
+license from a particular copyright holder is reinstated (a)
417
+provisionally, unless and until the copyright holder explicitly and
418
+finally terminates your license, and (b) permanently, if the copyright
419
+holder fails to notify you of the violation by some reasonable means
420
+prior to 60 days after the cessation.
421
+
422
+ Moreover, your license from a particular copyright holder is
423
+reinstated permanently if the copyright holder notifies you of the
424
+violation by some reasonable means, this is the first time you have
425
+received notice of violation of this License (for any work) from that
426
+copyright holder, and you cure the violation prior to 30 days after
427
+your receipt of the notice.
428
+
429
+ Termination of your rights under this section does not terminate the
430
+licenses of parties who have received copies or rights from you under
431
+this License. If your rights have been terminated and not permanently
432
+reinstated, you do not qualify to receive new licenses for the same
433
+material under section 10.
434
+
435
+ 9. Acceptance Not Required for Having Copies.
436
+
437
+ You are not required to accept this License in order to receive or
438
+run a copy of the Program. Ancillary propagation of a covered work
439
+occurring solely as a consequence of using peer-to-peer transmission
440
+to receive a copy likewise does not require acceptance. However,
441
+nothing other than this License grants you permission to propagate or
442
+modify any covered work. These actions infringe copyright if you do
443
+not accept this License. Therefore, by modifying or propagating a
444
+covered work, you indicate your acceptance of this License to do so.
445
+
446
+ 10. Automatic Licensing of Downstream Recipients.
447
+
448
+ Each time you convey a covered work, the recipient automatically
449
+receives a license from the original licensors, to run, modify and
450
+propagate that work, subject to this License. You are not responsible
451
+for enforcing compliance by third parties with this License.
452
+
453
+ An "entity transaction" is a transaction transferring control of an
454
+organization, or substantially all assets of one, or subdividing an
455
+organization, or merging organizations. If propagation of a covered
456
+work results from an entity transaction, each party to that
457
+transaction who receives a copy of the work also receives whatever
458
+licenses to the work the party's predecessor in interest had or could
459
+give under the previous paragraph, plus a right to possession of the
460
+Corresponding Source of the work from the predecessor in interest, if
461
+the predecessor has it or can get it with reasonable efforts.
462
+
463
+ You may not impose any further restrictions on the exercise of the
464
+rights granted or affirmed under this License. For example, you may
465
+not impose a license fee, royalty, or other charge for exercise of
466
+rights granted under this License, and you may not initiate litigation
467
+(including a cross-claim or counterclaim in a lawsuit) alleging that
468
+any patent claim is infringed by making, using, selling, offering for
469
+sale, or importing the Program or any portion of it.
470
+
471
+ 11. Patents.
472
+
473
+ A "contributor" is a copyright holder who authorizes use under this
474
+License of the Program or a work on which the Program is based. The
475
+work thus licensed is called the contributor's "contributor version".
476
+
477
+ A contributor's "essential patent claims" are all patent claims
478
+owned or controlled by the contributor, whether already acquired or
479
+hereafter acquired, that would be infringed by some manner, permitted
480
+by this License, of making, using, or selling its contributor version,
481
+but do not include claims that would be infringed only as a
482
+consequence of further modification of the contributor version. For
483
+purposes of this definition, "control" includes the right to grant
484
+patent sublicenses in a manner consistent with the requirements of
485
+this License.
486
+
487
+ Each contributor grants you a non-exclusive, worldwide, royalty-free
488
+patent license under the contributor's essential patent claims, to
489
+make, use, sell, offer for sale, import and otherwise run, modify and
490
+propagate the contents of its contributor version.
491
+
492
+ In the following three paragraphs, a "patent license" is any express
493
+agreement or commitment, however denominated, not to enforce a patent
494
+(such as an express permission to practice a patent or covenant not to
495
+sue for patent infringement). To "grant" such a patent license to a
496
+party means to make such an agreement or commitment not to enforce a
497
+patent against the party.
498
+
499
+ If you convey a covered work, knowingly relying on a patent license,
500
+and the Corresponding Source of the work is not available for anyone
501
+to copy, free of charge and under the terms of this License, through a
502
+publicly available network server or other readily accessible means,
503
+then you must either (1) cause the Corresponding Source to be so
504
+available, or (2) arrange to deprive yourself of the benefit of the
505
+patent license for this particular work, or (3) arrange, in a manner
506
+consistent with the requirements of this License, to extend the patent
507
+license to downstream recipients. "Knowingly relying" means you have
508
+actual knowledge that, but for the patent license, your conveying the
509
+covered work in a country, or your recipient's use of the covered work
510
+in a country, would infringe one or more identifiable patents in that
511
+country that you have reason to believe are valid.
512
+
513
+ If, pursuant to or in connection with a single transaction or
514
+arrangement, you convey, or propagate by procuring conveyance of, a
515
+covered work, and grant a patent license to some of the parties
516
+receiving the covered work authorizing them to use, propagate, modify
517
+or convey a specific copy of the covered work, then the patent license
518
+you grant is automatically extended to all recipients of the covered
519
+work and works based on it.
520
+
521
+ A patent license is "discriminatory" if it does not include within
522
+the scope of its coverage, prohibits the exercise of, or is
523
+conditioned on the non-exercise of one or more of the rights that are
524
+specifically granted under this License. You may not convey a covered
525
+work if you are a party to an arrangement with a third party that is
526
+in the business of distributing software, under which you make payment
527
+to the third party based on the extent of your activity of conveying
528
+the work, and under which the third party grants, to any of the
529
+parties who would receive the covered work from you, a discriminatory
530
+patent license (a) in connection with copies of the covered work
531
+conveyed by you (or copies made from those copies), or (b) primarily
532
+for and in connection with specific products or compilations that
533
+contain the covered work, unless you entered into that arrangement,
534
+or that patent license was granted, prior to 28 March 2007.
535
+
536
+ Nothing in this License shall be construed as excluding or limiting
537
+any implied license or other defenses to infringement that may
538
+otherwise be available to you under applicable patent law.
539
+
540
+ 12. No Surrender of Others' Freedom.
541
+
542
+ If conditions are imposed on you (whether by court order, agreement or
543
+otherwise) that contradict the conditions of this License, they do not
544
+excuse you from the conditions of this License. If you cannot convey a
545
+covered work so as to satisfy simultaneously your obligations under this
546
+License and any other pertinent obligations, then as a consequence you may
547
+not convey it at all. For example, if you agree to terms that obligate you
548
+to collect a royalty for further conveying from those to whom you convey
549
+the Program, the only way you could satisfy both those terms and this
550
+License would be to refrain entirely from conveying the Program.
551
+
552
+ 13. Use with the GNU Affero General Public License.
553
+
554
+ Notwithstanding any other provision of this License, you have
555
+permission to link or combine any covered work with a work licensed
556
+under version 3 of the GNU Affero General Public License into a single
557
+combined work, and to convey the resulting work. The terms of this
558
+License will continue to apply to the part which is the covered work,
559
+but the special requirements of the GNU Affero General Public License,
560
+section 13, concerning interaction through a network will apply to the
561
+combination as such.
562
+
563
+ 14. Revised Versions of this License.
564
+
565
+ The Free Software Foundation may publish revised and/or new versions of
566
+the GNU General Public License from time to time. Such new versions will
567
+be similar in spirit to the present version, but may differ in detail to
568
+address new problems or concerns.
569
+
570
+ Each version is given a distinguishing version number. If the
571
+Program specifies that a certain numbered version of the GNU General
572
+Public License "or any later version" applies to it, you have the
573
+option of following the terms and conditions either of that numbered
574
+version or of any later version published by the Free Software
575
+Foundation. If the Program does not specify a version number of the
576
+GNU General Public License, you may choose any version ever published
577
+by the Free Software Foundation.
578
+
579
+ If the Program specifies that a proxy can decide which future
580
+versions of the GNU General Public License can be used, that proxy's
581
+public statement of acceptance of a version permanently authorizes you
582
+to choose that version for the Program.
583
+
584
+ Later license versions may give you additional or different
585
+permissions. However, no additional obligations are imposed on any
586
+author or copyright holder as a result of your choosing to follow a
587
+later version.
588
+
589
+ 15. Disclaimer of Warranty.
590
+
591
+ THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY
592
+APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT
593
+HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY
594
+OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO,
595
+THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
596
+PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM
597
+IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF
598
+ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
599
+
600
+ 16. Limitation of Liability.
601
+
602
+ IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING
603
+WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS
604
+THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY
605
+GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE
606
+USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF
607
+DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD
608
+PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS),
609
+EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF
610
+SUCH DAMAGES.
611
+
612
+ 17. Interpretation of Sections 15 and 16.
613
+
614
+ If the disclaimer of warranty and limitation of liability provided
615
+above cannot be given local legal effect according to their terms,
616
+reviewing courts shall apply local law that most closely approximates
617
+an absolute waiver of all civil liability in connection with the
618
+Program, unless a warranty or assumption of liability accompanies a
619
+copy of the Program in return for a fee.
620
+
621
+ END OF TERMS AND CONDITIONS
622
+
623
+ How to Apply These Terms to Your New Programs
624
+
625
+ If you develop a new program, and you want it to be of the greatest
626
+possible use to the public, the best way to achieve this is to make it
627
+free software which everyone can redistribute and change under these terms.
628
+
629
+ To do so, attach the following notices to the program. It is safest
630
+to attach them to the start of each source file to most effectively
631
+state the exclusion of warranty; and each file should have at least
632
+the "copyright" line and a pointer to where the full notice is found.
633
+
634
+ {one line to give the program's name and a brief idea of what it does.}
635
+ Copyright (C) {year} {name of author}
636
+
637
+ This program is free software: you can redistribute it and/or modify
638
+ it under the terms of the GNU General Public License as published by
639
+ the Free Software Foundation, either version 3 of the License, or
640
+ (at your option) any later version.
641
+
642
+ This program is distributed in the hope that it will be useful,
643
+ but WITHOUT ANY WARRANTY; without even the implied warranty of
644
+ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
645
+ GNU General Public License for more details.
646
+
647
+ You should have received a copy of the GNU General Public License
648
+ along with this program. If not, see <http://www.gnu.org/licenses/>.
649
+
650
+Also add information on how to contact you by electronic and paper mail.
651
+
652
+ If the program does terminal interaction, make it output a short
653
+notice like this when it starts in an interactive mode:
654
+
655
+ {project} Copyright (C) {year} {fullname}
656
+ This program comes with ABSOLUTELY NO WARRANTY; for details type `show w'.
657
+ This is free software, and you are welcome to redistribute it
658
+ under certain conditions; type `show c' for details.
659
+
660
+The hypothetical commands `show w' and `show c' should show the appropriate
661
+parts of the General Public License. Of course, your program's commands
662
+might be different; for a GUI interface, you would use an "about box".
663
+
664
+ You should also get your employer (if you work as a programmer) or school,
665
+if any, to sign a "copyright disclaimer" for the program, if necessary.
666
+For more information on this, and how to apply and follow the GNU GPL, see
667
+<http://www.gnu.org/licenses/>.
668
+
669
+ The GNU General Public License does not permit incorporating your program
670
+into proprietary programs. If your program is a subroutine library, you
671
+may consider it more useful to permit linking proprietary applications with
672
+the library. If this is what you want to do, use the GNU Lesser General
673
+Public License instead of this License. But first, please read
674
+<http://www.gnu.org/philosophy/why-not-lgpl.html>.
675
+
676
+---------------------------------------------------------------------------
677
+
678
+Note:
679
+Individual files contain the following tag instead of the full license text.
680
+
681
+ SPDX-License-Identifier: GPL-3.0-or-later
682
+
683
+This enables machine processing of license information based on the SPDX
684
+License Identifiers that are here available: http://spdx.org/licenses/
src/crates/jf/benchmark/main.c
new
+275
@@ -0,0 +1,275 @@
1
+#include <stdio.h>
2
+#include <stdlib.h>
3
+#include <string.h>
4
+
5
+#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
6
+
7
+void format_entry(NsdJournal *j, size_t entry_id) {
8
+ size_t data_count = 0;
9
+ char buf[4096];
10
+
11
+ const void *data;
12
+ size_t length;
13
+ NSD_JOURNAL_FOREACH_DATA(j, data, length) {
14
+ if (length > 4095) {
15
+ fprintf(stderr, "length is more than 4KiB\n");
16
+ nsd_journal_close(j);
17
+ exit(EXIT_FAILURE);
18
+ }
19
+
20
+ memcpy(buf, data, length);
21
+ buf[length] = '\0';
22
+
23
+ fprintf(stdout, "E[%zu] D[%zu] %s\n", entry_id, data_count, buf);
24
+ data_count += 1;
25
+ }
26
+}
27
+
28
+static int process_unfiltered(const char *path)
29
+{
30
+ NsdJournal *j;
31
+ size_t total_bytes = 0;
32
+
33
+ const char *paths[2] = {
34
+ path,
35
+ NULL,
36
+ };
37
+
38
+ // Open the specific journal file
39
+ int r = nsd_journal_open_files(&j, &paths[0], 0);
40
+ if (r < 0) {
41
+ fprintf(stderr, "Failed to open journal file %s: %s\n",
42
+ path, strerror(-r));
43
+ return 1;
44
+ }
45
+
46
+ printf("Successfully opened journal file: %s\n", path);
47
+
48
+ {
49
+ const char *field = NULL;
50
+ nsd_journal_restart_fields(j);
51
+
52
+ // Enumerate all field names in the journal
53
+ while (nsd_journal_enumerate_fields(j, &field) > 0) {
54
+ printf("Field name: %s\n", field);
55
+ }
56
+ }
57
+
58
+ // Move to the first entry
59
+ r = nsd_journal_seek_head(j);
60
+ if (r < 0) {
61
+ fprintf(stderr, "Failed to seek to head: %s\n", strerror(-r));
62
+ nsd_journal_close(j);
63
+ return 1;
64
+ }
65
+
66
+ // Iterate through all entries
67
+ size_t entry_count = 0;
68
+ while ((r = nsd_journal_next(j)) > 0) {
69
+ entry_count++;
70
+
71
+ // Get all data fields
72
+ const void *data;
73
+ size_t length;
74
+ NSD_JOURNAL_FOREACH_DATA(j, data, length) {
75
+ // Count the bytes
76
+ total_bytes += length;
77
+ }
78
+ }
79
+
80
+ if (r < 0) {
81
+ fprintf(stderr, "Failed to iterate journal: %s\n", strerror(-r));
82
+ nsd_journal_close(j);
83
+ return 1;
84
+ }
85
+
86
+ if (total_bytes == 10) {
87
+ abort();
88
+ }
89
+
90
+ printf("Total entries processed: %zu\n", entry_count);
91
+
92
+ // Close the journal
93
+ nsd_journal_close(j);
94
+ return 0;
95
+}
96
+
97
+static int process_filtered(const char *path)
98
+{
99
+ NsdJournal *j;
100
+ size_t total_bytes = 0;
101
+
102
+ const char *paths[2] = {
103
+ path,
104
+ NULL,
105
+ };
106
+
107
+ // Open the specific journal file
108
+ int r = nsd_journal_open_files(&j, &paths[0], 0);
109
+ if (r < 0) {
110
+ fprintf(stderr, "Failed to open journal file %s: %s\n",
111
+ path, strerror(-r));
112
+ return 1;
113
+ }
114
+
115
+ printf("Successfully opened journal file: %s\n", path);
116
+
117
+ // Apply filters
118
+ // Platform filters (OR condition)
119
+ {
120
+ r = nsd_journal_add_match(j, "AE_OS_PLATFORM=debian", strlen("AE_OS_PLATFORM=debian"));
121
+ if (r < 0) {
122
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
123
+ nsd_journal_close(j);
124
+ return 1;
125
+ }
126
+
127
+ r = nsd_journal_add_match(j, "AE_OS_PLATFORM=fedora", strlen("AE_OS_PLATFORM=fedora"));
128
+ if (r < 0) {
129
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
130
+ nsd_journal_close(j);
131
+ return 1;
132
+ }
133
+ }
134
+
135
+ r = nsd_journal_add_conjunction(j); // AND
136
+ if (r < 0) {
137
+ fprintf(stderr, "Failed to add conjunction: %s\n", strerror(-r));
138
+ nsd_journal_close(j);
139
+ return 1;
140
+ }
141
+
142
+ {
143
+ // Version filters (OR condition)
144
+ r = nsd_journal_add_match(j, "AE_VERSION=17", strlen("AE_VERSION=17"));
145
+ if (r < 0) {
146
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
147
+ nsd_journal_close(j);
148
+ return 1;
149
+ }
150
+
151
+ r = nsd_journal_add_match(j, "AE_VERSION=22", strlen("AE_VERSION=22"));
152
+ if (r < 0) {
153
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
154
+ nsd_journal_close(j);
155
+ return 1;
156
+ }
157
+ }
158
+
159
+ r = nsd_journal_add_conjunction(j); // AND
160
+ if (r < 0) {
161
+ fprintf(stderr, "Failed to add conjunction: %s\n", strerror(-r));
162
+ nsd_journal_close(j);
163
+ return 1;
164
+ }
165
+
166
+ // Priority filters (OR condition)
167
+ {
168
+ r = nsd_journal_add_match(j, "PRIORITY=7", strlen("PRIORITY=7"));
169
+ if (r < 0) {
170
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
171
+ nsd_journal_close(j);
172
+ return 1;
173
+ }
174
+
175
+ r = nsd_journal_add_match(j, "PRIORITY=6", strlen("PRIORITY=6"));
176
+ if (r < 0) {
177
+ fprintf(stderr, "Failed to add match: %s\n", strerror(-r));
178
+ nsd_journal_close(j);
179
+ return 1;
180
+ }
181
+ }
182
+
183
+ r = nsd_journal_seek_tail(j);
184
+ if (r < 0) {
185
+ fprintf(stderr, "Failed to seek to head: %s\n", strerror(-r));
186
+ nsd_journal_close(j);
187
+ return 1;
188
+ }
189
+
190
+ size_t entry_count = 0;
191
+ while ((r = nsd_journal_previous(j)) > 0) {
192
+ format_entry(j, entry_count);
193
+
194
+ entry_count++;
195
+ }
196
+
197
+ if (r < 0) {
198
+ fprintf(stderr, "Failed to iterate journal: %s\n", strerror(-r));
199
+ nsd_journal_close(j);
200
+ return 1;
201
+ }
202
+
203
+ if (total_bytes == 10) {
204
+ abort();
205
+ }
206
+
207
+ printf("Total entries processed: %zu\n\n", entry_count);
208
+
209
+ // Close the journal
210
+ nsd_journal_close(j);
211
+ return 0;
212
+}
213
+
214
+long get_file_size(const char *filename) {
215
+ FILE *file = fopen(filename, "rb");
216
+
217
+ if (file == NULL) {
218
+ abort();
219
+ }
220
+
221
+ // Seek to the end of the file
222
+ fseek(file, 0, SEEK_END);
223
+
224
+ // Get the current position (which is the size)
225
+ long size = ftell(file);
226
+
227
+ // Close the file
228
+ fclose(file);
229
+
230
+ return size;
231
+}
232
+
233
+
234
+int main(int argc, char *argv[]) {
235
+ (void) argc;
236
+ (void) argv;
237
+
238
+ if (argc != 2) {
239
+ fprintf(stderr, "usage: <binary> filtered|unfiltered\n");
240
+ return 1;
241
+ }
242
+
243
+ printf("Processing entries for files...\n");
244
+ const char *paths[] = {
245
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002b725a-0006314cd7a5cefd.journal",
246
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002c7398-00063157ce5e4da0.journal",
247
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002d4dd1-000631616affdc1c.journal",
248
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002e52a2-0006316e49ef1636.journal",
249
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002f0f22-00063175e2452287.journal",
250
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-00000000002ffa15-0006318392e11a33.journal",
251
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000030a308-00063189ec4c06b5.journal",
252
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000031a287-0006319ba73abb17.journal",
253
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000032b6a5-000631a6ddadfd47.journal",
254
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000033a684-000631b2794364a9.journal",
255
+ "/var/log/journal/ec2ce35ddef16e80b43d6cd9f008dcba.agent-events/system@67fcfeba8339461c9a8dc77363c2c739-000000000034afc5-000631c4c524ff14.journal",
256
+ NULL,
257
+ };
258
+
259
+ size_t total_size = 0;
260
+
261
+ for (size_t idx = 0; idx != 10; idx++) {
262
+ total_size += get_file_size(paths[idx]);
263
+
264
+ if (strcmp(argv[1], "filtered") == 0) {
265
+ process_filtered(paths[idx]);
266
+ } else if (strcmp(argv[1], "unfiltered") == 0) {
267
+ process_unfiltered(paths[idx]);
268
+ } else {
269
+ fprintf(stderr, "Unknown argument: >>>%s<<<\n", argv[1]);
270
+ return 1;
271
+ }
272
+ }
273
+
274
+ fprintf(stdout, "Size of all logs: %zu MiB\n", total_size / (1024 * 1024));
275
+}
src/crates/jf/error/Cargo.toml
new
+9
@@ -0,0 +1,9 @@
1
+[package]
2
+name = "error"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+static_assertions = { workspace = true }
8
+thiserror = { workspace = true }
9
+zerocopy = { workspace = true }
src/crates/jf/error/src/lib.rs
new
+121
@@ -0,0 +1,121 @@
1
+#[macro_use]
2
+extern crate static_assertions;
3
+
4
+use std::io;
5
+use thiserror::Error;
6
+
7
+#[derive(Error, Debug)]
8
+pub enum JournalError {
9
+ #[error("invalid magic number")]
10
+ InvalidMagicNumber,
11
+
12
+ #[error("invalid journal file state")]
13
+ InvalidJournalFileState,
14
+
15
+ #[error("invalid object type")]
16
+ InvalidObjectType,
17
+
18
+ #[error("invalid object location")]
19
+ InvalidObjectLocation,
20
+
21
+ #[error("invalid zerocopy size")]
22
+ InvalidZeroCopySize,
23
+
24
+ #[error("previous object is still in use")]
25
+ ValueGuardInUse,
26
+
27
+ #[error("i/o error during object operation: {0}")]
28
+ Io(#[from] io::Error),
29
+
30
+ #[error("missing hash table")]
31
+ MissingHashTable,
32
+
33
+ #[error("missing object from hash table")]
34
+ MissingObjectFromHashTable,
35
+
36
+ #[error("invalid offset array offset")]
37
+ InvalidOffsetArrayOffset,
38
+
39
+ #[error("invalid offset array index")]
40
+ InvalidOffsetArrayIndex,
41
+
42
+ #[error("empty offset array list")]
43
+ EmptyOffsetArrayList,
44
+
45
+ #[error("empty offset array node")]
46
+ EmptyOffsetArrayNode,
47
+
48
+ #[error("empty inline cursor")]
49
+ EmptyInlineCursor,
50
+
51
+ #[error("unset cursor")]
52
+ UnsetCursor,
53
+
54
+ #[error("malformed filter")]
55
+ MalformedFilter,
56
+
57
+ #[error("Invalid field")]
58
+ InvalidField,
59
+
60
+ #[error("Decompressor error")]
61
+ DecompressorError,
62
+
63
+ #[error("out of bounds index")]
64
+ OutOfBoundsIndex,
65
+
66
+ #[error("invalid offset")]
67
+ InvalidOffset,
68
+
69
+ #[error("zerocopy failure")]
70
+ ZerocopyFailure,
71
+
72
+ #[error("sigbus handler error")]
73
+ SigbusHandlerError,
74
+
75
+ #[error("unknown compression method")]
76
+ UnknownCompressionMethod,
77
+
78
+ #[error("ffi error")]
79
+ InvalidFfiOp,
80
+}
81
+
82
+const_assert!(std::mem::size_of::<JournalError>() <= 16);
83
+
84
+impl JournalError {
85
+ pub fn to_error_code(&self) -> i32 {
86
+ match self {
87
+ JournalError::InvalidMagicNumber => -1,
88
+ JournalError::InvalidJournalFileState => -2,
89
+ JournalError::InvalidObjectType => -3,
90
+ JournalError::InvalidObjectLocation => -4,
91
+ JournalError::InvalidZeroCopySize => -5,
92
+ JournalError::ValueGuardInUse => -6,
93
+ JournalError::Io(_) => -7,
94
+ JournalError::MissingHashTable => -8,
95
+ JournalError::MissingObjectFromHashTable => -9,
96
+ JournalError::InvalidOffsetArrayOffset => -10,
97
+ JournalError::InvalidOffsetArrayIndex => -11,
98
+ JournalError::EmptyOffsetArrayList => -12,
99
+ JournalError::EmptyOffsetArrayNode => -13,
100
+ JournalError::EmptyInlineCursor => -14,
101
+ JournalError::UnsetCursor => -15,
102
+ JournalError::MalformedFilter => -16,
103
+ JournalError::InvalidField => -17,
104
+ JournalError::DecompressorError => -18,
105
+ JournalError::OutOfBoundsIndex => -19,
106
+ JournalError::InvalidOffset => -20,
107
+ JournalError::ZerocopyFailure => -21,
108
+ JournalError::SigbusHandlerError => -22,
109
+ JournalError::UnknownCompressionMethod => -23,
110
+ JournalError::InvalidFfiOp => -24,
111
+ }
112
+ }
113
+}
114
+
115
+impl<T: zerocopy::KnownLayout> From<zerocopy::SizeError<&[u8], T>> for JournalError {
116
+ fn from(_: zerocopy::SizeError<&[u8], T>) -> Self {
117
+ JournalError::InvalidZeroCopySize
118
+ }
119
+}
120
+
121
+pub type Result<T> = std::result::Result<T, JournalError>;
src/crates/jf/journal_file/Cargo.toml
new
+13
@@ -0,0 +1,13 @@
1
+[package]
2
+name = "journal_file"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+window_manager = { path = "../window_manager" }
9
+memmap2 = { workspace = true }
10
+ruzstd = { workspace = true }
11
+siphasher = { workspace = true }
12
+twox-hash = { workspace = true }
13
+zerocopy = { workspace = true }
src/crates/jf/journal_file/src/hash.rs
new
+31
@@ -0,0 +1,31 @@
1
+use siphasher::sip::SipHasher24;
2
+use std::hash::Hasher;
3
+
4
+fn jenkins_hash64(data: &[u8]) -> u64 {
5
+ // FIXME: user real jenkins hasher
6
+ let mut hasher = twox_hash::XxHash64::default();
7
+ hasher.write(data);
8
+ hasher.finish()
9
+}
10
+
11
+fn siphash24(data: &[u8], key: &[u8; 16]) -> u64 {
12
+ let k0 = u64::from_le_bytes(key[0..8].try_into().unwrap());
13
+ let k1 = u64::from_le_bytes(key[8..16].try_into().unwrap());
14
+
15
+ let mut hasher = SipHasher24::new_with_keys(k0, k1);
16
+ hasher.write(data);
17
+ hasher.finish()
18
+}
19
+
20
+pub fn journal_hash_data(data: &[u8], is_keyed_hash: bool, file_id: Option<&[u8; 16]>) -> u64 {
21
+ if is_keyed_hash {
22
+ if let Some(file_id) = file_id {
23
+ siphash24(data, file_id)
24
+ } else {
25
+ // FIXME: verify fallback behaviour
26
+ jenkins_hash64(data)
27
+ }
28
+ } else {
29
+ jenkins_hash64(data)
30
+ }
31
+}
src/crates/jf/journal_file/src/journal_file.rs
new
+816
@@ -0,0 +1,816 @@
1
+#![allow(clippy::field_reassign_with_default)]
2
+
3
+use crate::hash;
4
+use crate::object::*;
5
+use crate::offset_array;
6
+use error::{JournalError, Result};
7
+use std::cell::{RefCell, UnsafeCell};
8
+use std::fs::OpenOptions;
9
+use std::path::Path;
10
+use window_manager::{MemoryMap, MemoryMapMut, WindowManager};
11
+use zerocopy::FromBytes;
12
+
13
+#[cfg(debug_assertions)]
14
+use std::backtrace::Backtrace;
15
+
16
+use crate::value_guard::ValueGuard;
17
+
18
+// Size to pad objects to (8 bytes)
19
+const OBJECT_ALIGNMENT: u64 = 8;
20
+
21
+/// A reader for systemd journal files that efficiently maps small regions of the file into memory.
22
+///
23
+/// # Memory Management
24
+///
25
+/// This implementation uses a window-based memory mapping strategy similar to systemd's original
26
+/// implementation. Instead of mapping the entire file, it maintains a small set of memory-mapped
27
+/// windows and reuses them as needed.
28
+///
29
+/// # Concurrency and Safety
30
+///
31
+/// `JournalFile` uses interior mutability to provide a safe API with the following characteristics:
32
+///
33
+/// - The window manager is wrapped in an `UnsafeCell` to allow mutation through a shared reference.
34
+/// - A single `RefCell<bool>` guards access to ensure only one object can be active at a time.
35
+/// - Methods like `data_object()` return a `ValueGuard<T>` that automatically releases the lock
36
+/// when dropped.
37
+///
38
+/// This design ensures that memory safety is maintained even though references to memory-mapped
39
+/// regions could be invalidated when new objects are created.
40
+pub struct JournalFile<M: MemoryMap> {
41
+ // Persistent memory maps for journal header and data/field hash tables
42
+ header_map: M,
43
+ data_hash_table_map: Option<M>,
44
+ field_hash_table_map: Option<M>,
45
+
46
+ // Window manager for other objects
47
+ window_manager: UnsafeCell<WindowManager<M>>,
48
+
49
+ // Flag to track if any object is in use
50
+ object_in_use: RefCell<bool>,
51
+
52
+ #[cfg(debug_assertions)]
53
+ prev_backtrace: RefCell<Backtrace>,
54
+ #[cfg(debug_assertions)]
55
+ backtrace: RefCell<Backtrace>,
56
+}
57
+
58
+impl<M: MemoryMapMut> JournalFile<M> {
59
+ pub fn create(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
60
+ debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
61
+
62
+ let file = OpenOptions::new()
63
+ .create(true)
64
+ .truncate(true)
65
+ .read(true)
66
+ .write(true)
67
+ .open(&path)?;
68
+
69
+ let mut header = JournalHeader::default();
70
+ header.signature = *b"LPKSHHRH";
71
+
72
+ header.data_hash_table_offset = std::mem::size_of::<JournalHeader>() as u64
73
+ + std::mem::size_of::<ObjectHeader>() as u64;
74
+ header.data_hash_table_size = 4096 * std::mem::size_of::<HashItem>() as u64;
75
+
76
+ header.field_hash_table_offset = header.data_hash_table_offset
77
+ + header.data_hash_table_size
78
+ + std::mem::size_of::<ObjectHeader>() as u64;
79
+ header.field_hash_table_size = 512 * std::mem::size_of::<HashItem>() as u64;
80
+
81
+ debug_assert_eq!(header.data_hash_table_offset % OBJECT_ALIGNMENT, 0);
82
+ debug_assert_eq!(header.data_hash_table_size % OBJECT_ALIGNMENT, 0);
83
+ let data_hash_table_map = header.map_data_hash_table(&file)?;
84
+
85
+ debug_assert_eq!(header.field_hash_table_offset % OBJECT_ALIGNMENT, 0);
86
+ debug_assert_eq!(header.field_hash_table_size % OBJECT_ALIGNMENT, 0);
87
+ let field_hash_table_map = header.map_field_hash_table(&file)?;
88
+
89
+ header.tail_object_offset = header.data_hash_table_offset + header.data_hash_table_size;
90
+ header.n_objects = 2;
91
+
92
+ debug_assert_eq!(header.tail_object_offset % OBJECT_ALIGNMENT, 0);
93
+
94
+ let header_size = std::mem::size_of::<JournalHeader>() as u64;
95
+ let mut header_map = M::create(&file, 0, header_size)?;
96
+ {
97
+ let header_mut = JournalHeader::mut_from_prefix(&mut header_map).unwrap().0;
98
+ *header_mut = header;
99
+ }
100
+
101
+ // Create window manager for the rest of the objects
102
+ let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
103
+
104
+ Ok(JournalFile {
105
+ header_map,
106
+ data_hash_table_map,
107
+ field_hash_table_map,
108
+ window_manager,
109
+ object_in_use: RefCell::new(false),
110
+
111
+ #[cfg(debug_assertions)]
112
+ prev_backtrace: RefCell::new(Backtrace::capture()),
113
+ #[cfg(debug_assertions)]
114
+ backtrace: RefCell::new(Backtrace::capture()),
115
+ })
116
+ }
117
+
118
+ pub fn journal_header_mut(&mut self) -> &mut JournalHeader {
119
+ JournalHeader::mut_from_prefix(&mut self.header_map)
120
+ .unwrap()
121
+ .0
122
+ }
123
+
124
+ pub fn data_hash_table_mut(&mut self) -> Option<HashTableObject<&mut [u8]>> {
125
+ self.data_hash_table_map
126
+ .as_mut()
127
+ .map(|m| HashTableObject::<&mut [u8]>::from_data_mut(m, false))
128
+ }
129
+
130
+ pub fn field_hash_table_mut(&mut self) -> Option<HashTableObject<&mut [u8]>> {
131
+ self.field_hash_table_map
132
+ .as_mut()
133
+ .map(|m| HashTableObject::<&mut [u8]>::from_data_mut(m, false))
134
+ }
135
+
136
+ fn object_header_mut(&self, position: u64) -> Result<&mut ObjectHeader> {
137
+ let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
138
+ let window_manager = unsafe { &mut *self.window_manager.get() };
139
+ let header_slice = window_manager.get_slice_mut(position, size_needed)?;
140
+ Ok(ObjectHeader::mut_from_bytes(header_slice).unwrap())
141
+ }
142
+
143
+ fn object_data_mut(&self, position: u64, size_needed: u64) -> Result<&mut [u8]> {
144
+ let window_manager = unsafe { &mut *self.window_manager.get() };
145
+ let object_slice = window_manager.get_slice_mut(position, size_needed)?;
146
+ Ok(object_slice)
147
+ }
148
+
149
+ fn journal_object_mut<'a, T>(
150
+ &'a self,
151
+ type_: ObjectType,
152
+ position: u64,
153
+ size: Option<u64>,
154
+ ) -> Result<ValueGuard<'a, T>>
155
+ where
156
+ T: JournalObjectMut<&'a mut [u8]>,
157
+ {
158
+ // Check if any object is already in use
159
+ let mut is_in_use = self.object_in_use.borrow_mut();
160
+ if *is_in_use {
161
+ #[cfg(debug_assertions)]
162
+ {
163
+ eprintln!(
164
+ "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
165
+ self.backtrace.borrow().to_string(),
166
+ self.prev_backtrace.borrow().to_string()
167
+ );
168
+ }
169
+ return Err(JournalError::ValueGuardInUse);
170
+ }
171
+
172
+ #[cfg(debug_assertions)]
173
+ {
174
+ self.backtrace.swap(&self.prev_backtrace);
175
+ let _ = self.backtrace.replace(Backtrace::force_capture());
176
+ }
177
+
178
+ let is_compact = self
179
+ .journal_header_ref()
180
+ .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
181
+
182
+ let size_needed = match size {
183
+ Some(size) => {
184
+ let header = self.object_header_mut(position)?;
185
+ header.type_ = type_ as u8;
186
+ header.size = size;
187
+ size
188
+ }
189
+ None => {
190
+ let header = self.object_header_ref(position)?;
191
+ if header.type_ != type_ as u8 {
192
+ return Err(JournalError::InvalidObjectType);
193
+ }
194
+ header.size
195
+ }
196
+ };
197
+
198
+ let data = self.object_data_mut(position, size_needed)?;
199
+ let object = T::from_data_mut(data, is_compact);
200
+
201
+ // Mark as in use
202
+ *is_in_use = true;
203
+ Ok(ValueGuard::new(object, &self.object_in_use))
204
+ }
205
+
206
+ pub fn offset_array_mut(
207
+ &self,
208
+ position: u64,
209
+ capacity: Option<u64>,
210
+ ) -> Result<ValueGuard<OffsetArrayObject<&mut [u8]>>> {
211
+ let size = capacity.map(|c| {
212
+ let mut size = std::mem::size_of::<OffsetArrayObjectHeader>() as u64;
213
+
214
+ let is_compact = self
215
+ .journal_header_ref()
216
+ .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
217
+ if is_compact {
218
+ size += c * std::mem::size_of::<u32>() as u64;
219
+ } else {
220
+ size += c * std::mem::size_of::<u64>() as u64;
221
+ }
222
+
223
+ size
224
+ });
225
+
226
+ let offset_array = self.journal_object_mut(ObjectType::EntryArray, position, size);
227
+ offset_array
228
+ }
229
+
230
+ pub fn field_mut(
231
+ &self,
232
+ position: u64,
233
+ size: Option<u64>,
234
+ ) -> Result<ValueGuard<FieldObject<&mut [u8]>>> {
235
+ let size = size.map(|n| std::mem::size_of::<FieldObjectHeader>() as u64 + n);
236
+ self.journal_object_mut(ObjectType::Field, position, size)
237
+ }
238
+
239
+ pub fn entry_mut(&self, position: u64) -> Result<ValueGuard<EntryObject<&mut [u8]>>> {
240
+ self.journal_object_mut(ObjectType::Entry, position, None)
241
+ }
242
+
243
+ pub fn data_mut(
244
+ &self,
245
+ position: u64,
246
+ size: Option<u64>,
247
+ ) -> Result<ValueGuard<DataObject<&mut [u8]>>> {
248
+ let size = size.map(|n| std::mem::size_of::<DataObjectHeader>() as u64 + n);
249
+ self.journal_object_mut(ObjectType::Data, position, size)
250
+ }
251
+
252
+ pub fn tag_mut(&self, position: u64, new: bool) -> Result<ValueGuard<TagObject<&mut [u8]>>> {
253
+ let size = if new {
254
+ Some(std::mem::size_of::<TagObjectHeader>() as u64)
255
+ } else {
256
+ None
257
+ };
258
+ self.journal_object_mut(ObjectType::Tag, position, size)
259
+ }
260
+}
261
+
262
+impl<M: MemoryMap> JournalFile<M> {
263
+ pub fn open(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
264
+ debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
265
+
266
+ // Open file and check its size
267
+ let file = OpenOptions::new().read(true).write(false).open(&path)?;
268
+
269
+ // Create a memory map for the header
270
+ let header_size = std::mem::size_of::<JournalHeader>() as u64;
271
+ let header_map = M::create(&file, 0, header_size)?;
272
+ let header = JournalHeader::ref_from_prefix(&header_map).unwrap().0;
273
+ if header.signature != *b"LPKSHHRH" {
274
+ return Err(JournalError::InvalidMagicNumber);
275
+ }
276
+
277
+ // Initialize the hash table maps if they exist
278
+ let data_hash_table_map = header.map_data_hash_table(&file)?;
279
+ let field_hash_table_map = header.map_field_hash_table(&file)?;
280
+
281
+ // Create window manager for the rest of the objects
282
+ let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
283
+
284
+ Ok(JournalFile {
285
+ header_map,
286
+ data_hash_table_map,
287
+ field_hash_table_map,
288
+ window_manager,
289
+ object_in_use: RefCell::new(false),
290
+
291
+ #[cfg(debug_assertions)]
292
+ prev_backtrace: RefCell::new(Backtrace::capture()),
293
+ #[cfg(debug_assertions)]
294
+ backtrace: RefCell::new(Backtrace::capture()),
295
+ })
296
+ }
297
+
298
+ pub fn hash(&self, data: &[u8]) -> u64 {
299
+ let is_keyed_hash = self
300
+ .journal_header_ref()
301
+ .has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash);
302
+
303
+ hash::journal_hash_data(
304
+ data,
305
+ is_keyed_hash,
306
+ if is_keyed_hash {
307
+ Some(&self.journal_header_ref().file_id)
308
+ } else {
309
+ None
310
+ },
311
+ )
312
+ }
313
+
314
+ pub fn entry_list(&self) -> Option<offset_array::List> {
315
+ let head_offset = std::num::NonZeroU64::new(self.journal_header_ref().entry_array_offset)?;
316
+ let total_items =
317
+ std::num::NonZeroUsize::new(self.journal_header_ref().n_entries as usize)?;
318
+ Some(offset_array::List::new(head_offset, total_items))
319
+ }
320
+
321
+ pub fn journal_header_ref(&self) -> &JournalHeader {
322
+ JournalHeader::ref_from_prefix(&self.header_map).unwrap().0
323
+ }
324
+
325
+ pub fn data_hash_table_ref(&self) -> Option<HashTableObject<&[u8]>> {
326
+ self.data_hash_table_map
327
+ .as_ref()
328
+ .and_then(|m| HashTableObject::<&[u8]>::from_data(m, false))
329
+ }
330
+
331
+ pub fn field_hash_table_ref(&self) -> Option<HashTableObject<&[u8]>> {
332
+ self.field_hash_table_map
333
+ .as_ref()
334
+ .and_then(|m| HashTableObject::<&[u8]>::from_data(m, false))
335
+ }
336
+
337
+ fn object_header_ref(&self, position: u64) -> Result<&ObjectHeader> {
338
+ let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
339
+ let window_manager = unsafe { &mut *self.window_manager.get() };
340
+ let header_slice = window_manager.get_slice(position, size_needed)?;
341
+ Ok(ObjectHeader::ref_from_bytes(header_slice).unwrap())
342
+ }
343
+
344
+ fn object_data_ref(&self, position: u64, size_needed: u64) -> Result<&[u8]> {
345
+ let window_manager = unsafe { &mut *self.window_manager.get() };
346
+ let object_slice = window_manager.get_slice(position, size_needed)?;
347
+ Ok(object_slice)
348
+ }
349
+
350
+ fn journal_object_ref<'a, T>(&'a self, position: u64) -> Result<ValueGuard<'a, T>>
351
+ where
352
+ T: JournalObject<&'a [u8]>,
353
+ {
354
+ // Check if any object is already in use
355
+ let mut is_in_use = self.object_in_use.borrow_mut();
356
+ if *is_in_use {
357
+ #[cfg(debug_assertions)]
358
+ {
359
+ eprintln!(
360
+ "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
361
+ self.backtrace.borrow().to_string(),
362
+ self.prev_backtrace.borrow().to_string()
363
+ );
364
+ }
365
+ return Err(JournalError::ValueGuardInUse);
366
+ }
367
+
368
+ #[cfg(debug_assertions)]
369
+ {
370
+ self.backtrace.swap(&self.prev_backtrace);
371
+ let _ = self.backtrace.replace(Backtrace::force_capture());
372
+ }
373
+
374
+ let is_compact = self
375
+ .journal_header_ref()
376
+ .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
377
+
378
+ let size_needed = {
379
+ let header = self.object_header_ref(position)?;
380
+ header.size
381
+ };
382
+
383
+ let data = self.object_data_ref(position, size_needed)?;
384
+ let Some(object) = T::from_data(data, is_compact) else {
385
+ return Err(JournalError::ZerocopyFailure);
386
+ };
387
+
388
+ // Mark as in use
389
+ *is_in_use = true;
390
+
391
+ Ok(ValueGuard::new(object, &self.object_in_use))
392
+ }
393
+
394
+ pub fn offset_array_ref(&self, position: u64) -> Result<ValueGuard<OffsetArrayObject<&[u8]>>> {
395
+ self.journal_object_ref(position)
396
+ }
397
+
398
+ pub fn field_ref(&self, position: u64) -> Result<ValueGuard<FieldObject<&[u8]>>> {
399
+ self.journal_object_ref(position)
400
+ }
401
+
402
+ pub fn entry_ref(&self, position: u64) -> Result<ValueGuard<EntryObject<&[u8]>>> {
403
+ self.journal_object_ref(position)
404
+ }
405
+
406
+ pub fn data_ref(&self, position: u64) -> Result<ValueGuard<DataObject<&[u8]>>> {
407
+ self.journal_object_ref(position)
408
+ }
409
+
410
+ pub fn tag_ref(&self, position: u64) -> Result<ValueGuard<TagObject<&[u8]>>> {
411
+ self.journal_object_ref(position)
412
+ }
413
+
414
+ fn lookup_hash_table<'a, T, F>(
415
+ &'a self,
416
+ hash_table: Option<HashTableObject<&[u8]>>,
417
+ data: &[u8],
418
+ hash: u64,
419
+ fetch_fn: F,
420
+ ) -> Result<u64>
421
+ where
422
+ T: HashableObject,
423
+ F: Fn(u64) -> Result<ValueGuard<'a, T>>,
424
+ {
425
+ let hash_table = hash_table.ok_or(JournalError::MissingHashTable)?;
426
+
427
+ // Find the right bucket in the hash table
428
+ let hash_table_size = hash_table.items.len();
429
+ let bucket_idx = (hash % hash_table_size as u64) as usize;
430
+
431
+ // Get the head object offset from the bucket
432
+ let bucket = hash_table.items[bucket_idx];
433
+ let mut object_offset = bucket.head_hash_offset;
434
+
435
+ // Traverse the linked list of objects in this bucket
436
+ while object_offset != 0 {
437
+ match fetch_fn(object_offset) {
438
+ Ok(object_guard) => {
439
+ // Check if this is the object we're looking for
440
+ if object_guard.hash() == hash && object_guard.get_payload() == data {
441
+ return Ok(object_offset);
442
+ }
443
+
444
+ // Move to the next object in the chain
445
+ object_offset = object_guard.next_hash_offset();
446
+ }
447
+ Err(e) => {
448
+ return Err(e);
449
+ }
450
+ }
451
+ }
452
+
453
+ Err(JournalError::MissingObjectFromHashTable)
454
+ }
455
+
456
+ /// Finds a field object by name and returns its offset
457
+ pub fn find_field_offset_by_name(&self, field_name: &[u8], hash: u64) -> Result<u64> {
458
+ self.lookup_hash_table::<FieldObject<&[u8]>, _>(
459
+ self.field_hash_table_ref(),
460
+ field_name,
461
+ hash,
462
+ |offset| self.field_ref(offset),
463
+ )
464
+ }
465
+
466
+ /// Finds a data object by payload and returns its offset
467
+ pub fn find_data_offset_by_payload(&self, payload: &[u8], hash: u64) -> Result<u64> {
468
+ self.lookup_hash_table::<DataObject<&[u8]>, _>(
469
+ self.data_hash_table_ref(),
470
+ payload,
471
+ hash,
472
+ |offset| self.data_ref(offset),
473
+ )
474
+ }
475
+
476
+ /// Run a directed partition point query on a data object's entry array
477
+ ///
478
+ /// This finds the first/last entry (depending on direction) that satisfies the given predicate
479
+ /// in the entry array chain of the data object.
480
+ pub fn data_object_directed_partition_point<F>(
481
+ &self,
482
+ data_offset: u64,
483
+ predicate: F,
484
+ direction: offset_array::Direction,
485
+ ) -> Result<Option<u64>>
486
+ where
487
+ F: Fn(u64) -> Result<bool>,
488
+ {
489
+ let Some(cursor) = self.data_ref(data_offset)?.inlined_cursor() else {
490
+ return Ok(None);
491
+ };
492
+
493
+ let best_match = cursor.directed_partition_point(self, predicate, direction)?;
494
+
495
+ // Convert the result to an entry offset
496
+ best_match.map(|c| c.value(self)).transpose()
497
+ }
498
+
499
+ /// Creates an iterator over all offsets of an offset array list
500
+ pub fn array_offsets(&self, position: u64) -> Result<OffsetArrayListIterator<'_, M>> {
501
+ Ok(OffsetArrayListIterator {
502
+ journal: self,
503
+ offset: position,
504
+ capacity: if position == 0 {
505
+ 0
506
+ } else {
507
+ self.offset_array_ref(position)?.capacity()
508
+ },
509
+ index: 0,
510
+ })
511
+ }
512
+
513
+ /// Creates an iterator over all entry offsets in the journal
514
+ pub fn entry_offsets(&self) -> Result<OffsetArrayListIterator<'_, M>> {
515
+ self.array_offsets(self.journal_header_ref().entry_array_offset)
516
+ }
517
+
518
+ /// Creates an iterator over all field objects in the field hash table
519
+ pub fn fields(&self) -> FieldIterator<'_, M> {
520
+ // Get the field hash table
521
+ let field_hash_table = self.field_hash_table_ref();
522
+
523
+ // Initialize with the first bucket
524
+ let mut iterator = FieldIterator {
525
+ journal: self,
526
+ field_hash_table,
527
+ current_bucket_index: 0,
528
+ next_field_offset: 0,
529
+ };
530
+
531
+ // Find the first non-empty bucket
532
+ iterator.advance_to_next_nonempty_bucket();
533
+
534
+ iterator
535
+ }
536
+
537
+ /// Creates an iterator over all DATA objects for the specified field
538
+ pub fn field_data_objects<'a>(
539
+ &'a self,
540
+ field_name: &'a [u8],
541
+ ) -> Result<FieldDataIterator<'a, M>> {
542
+ // Find the field offset by name
543
+ let field_hash = self.hash(field_name);
544
+ let field_offset = self.find_field_offset_by_name(field_name, field_hash)?;
545
+
546
+ // Get the field object to access its head_data_offset
547
+ let field_guard = self.field_ref(field_offset)?;
548
+ let head_data_offset = field_guard.header.head_data_offset;
549
+
550
+ // Create the iterator
551
+ Ok(FieldDataIterator {
552
+ journal: self,
553
+ current_data_offset: head_data_offset,
554
+ })
555
+ }
556
+
557
+ /// Creates an iterator over all DATA objects for a specific entry
558
+ pub fn entry_data_objects(&self, entry_offset: u64) -> Result<EntryDataIterator<'_, M>> {
559
+ // Get the entry object to determine how many data items it has
560
+ let entry_guard = self.entry_ref(entry_offset)?;
561
+
562
+ // Get the total number of items
563
+ let total_items = match &entry_guard.items {
564
+ EntryItemsType::Regular(items) => items.len(),
565
+ EntryItemsType::Compact(items) => items.len(),
566
+ };
567
+
568
+ // Create the iterator
569
+ Ok(EntryDataIterator {
570
+ journal: self,
571
+ entry_offset,
572
+ current_index: 0,
573
+ total_items,
574
+ })
575
+ }
576
+}
577
+
578
+/*
579
+ * Offset array iteration
580
+*/
581
+
582
+/// Iterator that returns all offsets in an offset array list
583
+pub struct OffsetArrayListIterator<'a, M: MemoryMap> {
584
+ journal: &'a JournalFile<M>,
585
+ offset: u64,
586
+ capacity: usize,
587
+ index: usize,
588
+}
589
+
590
+impl<M: MemoryMap> Iterator for OffsetArrayListIterator<'_, M> {
591
+ type Item = Result<u64>;
592
+
593
+ fn next(&mut self) -> Option<Self::Item> {
594
+ // If we've reached the end of offsets in the offset array list, return None
595
+ if self.offset == 0 {
596
+ return None;
597
+ }
598
+
599
+ // Check if we need to move to the next offset array
600
+ if self.index >= self.capacity {
601
+ // Get the next offset array offset
602
+ let next_offset = match self.journal.offset_array_ref(self.offset) {
603
+ Ok(array_guard) => array_guard.header.next_offset_array,
604
+ Err(e) => return Some(Err(e)),
605
+ };
606
+
607
+ // If there's no next offset array, we're done
608
+ if next_offset == 0 {
609
+ self.offset = 0;
610
+ return None;
611
+ }
612
+
613
+ // Set up the next offset array
614
+ match self.journal.offset_array_ref(next_offset) {
615
+ Ok(array_guard) => {
616
+ self.offset = next_offset;
617
+ self.capacity = array_guard.capacity();
618
+ self.index = 0;
619
+ }
620
+ Err(e) => return Some(Err(e)),
621
+ }
622
+ }
623
+
624
+ // Get the current offset from the offset array
625
+ let offset = match self.journal.offset_array_ref(self.offset) {
626
+ Ok(array_guard) => match &array_guard.items {
627
+ OffsetsType::Regular(offsets) => {
628
+ if self.index < offsets.len() {
629
+ offsets[self.index]
630
+ } else {
631
+ 0
632
+ }
633
+ }
634
+ OffsetsType::Compact(offsets) => {
635
+ if self.index < offsets.len() {
636
+ offsets[self.index] as u64
637
+ } else {
638
+ 0
639
+ }
640
+ }
641
+ },
642
+ Err(e) => return Some(Err(e)),
643
+ };
644
+
645
+ // Increment index for next iteration
646
+ self.index += 1;
647
+
648
+ // If offset is zero, we've reached the end of all entries
649
+ if offset == 0 {
650
+ self.offset = 0;
651
+ return None;
652
+ }
653
+
654
+ Some(Ok(offset))
655
+ }
656
+}
657
+
658
+/// Iterator that walks through all field objects in the field hash table
659
+pub struct FieldIterator<'a, M: MemoryMap> {
660
+ journal: &'a JournalFile<M>,
661
+ field_hash_table: Option<HashTableObject<&'a [u8]>>,
662
+ current_bucket_index: usize,
663
+ next_field_offset: u64,
664
+}
665
+
666
+impl<M: MemoryMap> FieldIterator<'_, M> {
667
+ /// Advances to the next non-empty bucket
668
+ fn advance_to_next_nonempty_bucket(&mut self) {
669
+ // If we don't have a hash table, there's nothing to iterate
670
+ let Some(hash_table) = &self.field_hash_table else {
671
+ return;
672
+ };
673
+
674
+ let items = &hash_table.items;
675
+ let num_buckets = items.len();
676
+
677
+ // Find the next non-empty bucket
678
+ while self.current_bucket_index < num_buckets {
679
+ let bucket = items[self.current_bucket_index];
680
+ if bucket.head_hash_offset != 0 {
681
+ // Found a non-empty bucket
682
+ self.next_field_offset = bucket.head_hash_offset;
683
+ return;
684
+ }
685
+ self.current_bucket_index += 1;
686
+ }
687
+
688
+ // No more non-empty buckets
689
+ self.next_field_offset = 0;
690
+ }
691
+}
692
+
693
+impl<'a, M: MemoryMap> Iterator for FieldIterator<'a, M> {
694
+ type Item = Result<ValueGuard<'a, FieldObject<&'a [u8]>>>;
695
+
696
+ fn next(&mut self) -> Option<Self::Item> {
697
+ // If we've reached the end, return None
698
+ if self.next_field_offset == 0 {
699
+ return None;
700
+ }
701
+
702
+ // Save the current offset to return
703
+ let offset = self.next_field_offset;
704
+
705
+ // Try to get the field object
706
+ match self.journal.field_ref(offset) {
707
+ Ok(field_guard) => {
708
+ // Get the next field offset before we return the guard
709
+ self.next_field_offset = field_guard.header.next_hash_offset;
710
+
711
+ // If we've reached the end of the chain, move to the next bucket
712
+ if self.next_field_offset == 0 {
713
+ self.current_bucket_index += 1;
714
+ self.advance_to_next_nonempty_bucket();
715
+ }
716
+
717
+ Some(Ok(field_guard))
718
+ }
719
+ Err(e) => {
720
+ // If we can't read the field, return the error and stop iteration
721
+ self.next_field_offset = 0;
722
+ Some(Err(e))
723
+ }
724
+ }
725
+ }
726
+}
727
+
728
+/// Iterator that walks through all DATA objects for a specific field
729
+pub struct FieldDataIterator<'a, M: MemoryMap> {
730
+ journal: &'a JournalFile<M>,
731
+ current_data_offset: u64,
732
+}
733
+
734
+impl<'a, M: MemoryMap> Iterator for FieldDataIterator<'a, M> {
735
+ type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
736
+
737
+ fn next(&mut self) -> Option<Self::Item> {
738
+ // If we've reached the end, return None
739
+ if self.current_data_offset == 0 {
740
+ return None;
741
+ }
742
+
743
+ // Save the current offset to return
744
+ let offset = self.current_data_offset;
745
+
746
+ // Try to get the data object
747
+ match self.journal.data_ref(offset) {
748
+ Ok(data_guard) => {
749
+ // Get the next data offset before we return the guard
750
+ self.current_data_offset = data_guard.header.next_field_offset;
751
+ Some(Ok(data_guard))
752
+ }
753
+ Err(e) => {
754
+ // If we can't read the data object, return the error and stop iteration
755
+ self.current_data_offset = 0;
756
+ Some(Err(e))
757
+ }
758
+ }
759
+ }
760
+}
761
+
762
+/// Iterator that walks through all DATA objects for a specific entry
763
+pub struct EntryDataIterator<'a, M: MemoryMap> {
764
+ journal: &'a JournalFile<M>,
765
+ entry_offset: u64,
766
+ current_index: usize,
767
+ total_items: usize,
768
+}
769
+
770
+impl<'a, M: MemoryMap> Iterator for EntryDataIterator<'a, M> {
771
+ type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
772
+
773
+ fn next(&mut self) -> Option<Self::Item> {
774
+ // If we've reached the end of the data indices, return None
775
+ if self.current_index >= self.total_items {
776
+ return None;
777
+ }
778
+
779
+ // Get the entry object to access the data offset
780
+ match self.journal.entry_ref(self.entry_offset) {
781
+ Ok(entry_guard) => {
782
+ let idx = self.current_index;
783
+ self.current_index += 1;
784
+
785
+ let data_offset = match &entry_guard.items {
786
+ EntryItemsType::Regular(items) => {
787
+ if idx >= items.len() {
788
+ return None;
789
+ }
790
+ items[idx].object_offset
791
+ }
792
+ EntryItemsType::Compact(items) => {
793
+ if idx >= items.len() {
794
+ return None;
795
+ }
796
+ items[idx].object_offset as u64
797
+ }
798
+ };
799
+
800
+ // Drop the entry guard before obtaining the data object
801
+ drop(entry_guard);
802
+
803
+ // Try to get the data object
804
+ match self.journal.data_ref(data_offset) {
805
+ Ok(data_guard) => Some(Ok(data_guard)),
806
+ Err(e) => Some(Err(e)),
807
+ }
808
+ }
809
+ Err(e) => {
810
+ // If we can't read the entry, return the error and stop iteration
811
+ self.current_index = self.total_items;
812
+ Some(Err(e))
813
+ }
814
+ }
815
+ }
816
+}
src/crates/jf/journal_file/src/lib.rs
new
+12
@@ -0,0 +1,12 @@
1
+mod hash;
2
+mod object;
3
+mod journal_file;
4
+pub mod offset_array;
5
+mod value_guard;
6
+
7
+pub use crate::hash::*;
8
+pub use error::Result;
9
+pub use memmap2::{Mmap, MmapMut};
10
+pub use object::*;
11
+pub use journal_file::{EntryDataIterator, FieldDataIterator, FieldIterator, JournalFile};
12
+pub use value_guard::ValueGuard;
src/crates/jf/journal_file/src/object.rs
new
+832
@@ -0,0 +1,832 @@
1
+use crate::offset_array::{Cursor, InlinedCursor, List};
2
+use error::{JournalError, Result};
3
+use std::fs::File;
4
+use std::num::{NonZeroU64, NonZeroUsize};
5
+use window_manager::MemoryMap;
6
+use zerocopy::{
7
+ ByteSlice, FromBytes, Immutable, IntoBytes, KnownLayout, Ref, SplitByteSlice, SplitByteSliceMut,
8
+};
9
+
10
+pub trait HashableObject {
11
+ /// Get the hash value of this object
12
+ fn hash(&self) -> u64;
13
+
14
+ /// Get the payload data for matching
15
+ fn get_payload(&self) -> &[u8];
16
+
17
+ /// Get the offset to the next object in the hash chain
18
+ fn next_hash_offset(&self) -> u64;
19
+
20
+ /// Get the object type
21
+ fn object_type() -> ObjectType;
22
+}
23
+
24
+impl<B: ByteSlice> HashableObject for FieldObject<B> {
25
+ fn hash(&self) -> u64 {
26
+ self.header.hash
27
+ }
28
+
29
+ fn get_payload(&self) -> &[u8] {
30
+ &self.payload
31
+ }
32
+
33
+ fn next_hash_offset(&self) -> u64 {
34
+ self.header.next_hash_offset
35
+ }
36
+
37
+ fn object_type() -> ObjectType {
38
+ ObjectType::Field
39
+ }
40
+}
41
+
42
+impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> HashableObject for DataObject<B> {
43
+ fn hash(&self) -> u64 {
44
+ self.header.hash
45
+ }
46
+
47
+ fn get_payload(&self) -> &[u8] {
48
+ self.payload_bytes()
49
+ }
50
+
51
+ fn next_hash_offset(&self) -> u64 {
52
+ self.header.next_hash_offset
53
+ }
54
+
55
+ fn object_type() -> ObjectType {
56
+ ObjectType::Data
57
+ }
58
+}
59
+
60
+/// Trait to standardize creation of journal objects from byte slices
61
+pub trait JournalObject<B: SplitByteSlice>: Sized {
62
+ /// Create a new journal object from a byte slice
63
+ fn from_data(data: B, is_compact: bool) -> Option<Self>;
64
+}
65
+
66
+pub trait JournalObjectMut<B: SplitByteSliceMut>: Sized {
67
+ /// Create a new journal object from a byte slice
68
+ fn from_data_mut(data: B, is_compact: bool) -> Self;
69
+}
70
+
71
+pub enum HeaderIncompatibleFlags {
72
+ CompressedXz = 1 << 0,
73
+ CompressedLz4 = 1 << 1,
74
+ KeyedHash = 1 << 2,
75
+ CompressedZstd = 1 << 3,
76
+ Compact = 1 << 4,
77
+}
78
+
79
+pub enum HeaderCompatibleFlags {
80
+ Sealed = 1 << 0,
81
+ TailEntryBootId = 1 << 1,
82
+}
83
+
84
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
85
+pub enum JournalState {
86
+ Offline = 0,
87
+ Online = 1,
88
+ Archived = 2,
89
+}
90
+
91
+impl TryFrom<u8> for JournalState {
92
+ type Error = JournalError;
93
+
94
+ fn try_from(value: u8) -> Result<Self> {
95
+ match value {
96
+ 0 => Ok(JournalState::Offline),
97
+ 1 => Ok(JournalState::Online),
98
+ 2 => Ok(JournalState::Archived),
99
+ _ => Err(JournalError::InvalidJournalFileState),
100
+ }
101
+ }
102
+}
103
+
104
+#[derive(Default, Debug, Clone, Copy, FromBytes, IntoBytes, Immutable, KnownLayout)]
105
+#[repr(C)]
106
+pub struct JournalHeader {
107
+ pub signature: [u8; 8], // "LPKSHHRH"
108
+ pub compatible_flags: u32, // Compatible extension flags
109
+ pub incompatible_flags: u32, // Incompatible extension flags
110
+ pub state: u8, // File state (offline=0, online=1, archived=2)
111
+ pub reserved: [u8; 7], // Reserved space
112
+ pub file_id: [u8; 16], // Unique ID for this file
113
+ pub machine_id: [u8; 16], // Machine ID this belongs to
114
+ pub tail_entry_boot_id: [u8; 16], // Boot ID of the last entry
115
+ pub seqnum_id: [u8; 16], // Sequence number ID
116
+ pub header_size: u64, // Size of the header
117
+ pub arena_size: u64, // Size of the data arena
118
+ pub data_hash_table_offset: u64, // Offset of the data hash table
119
+ pub data_hash_table_size: u64, // Size of the data hash table
120
+ pub field_hash_table_offset: u64, // Offset of the field hash table
121
+ pub field_hash_table_size: u64, // Size of the field hash table
122
+ pub tail_object_offset: u64, // Offset of the last object
123
+ pub n_objects: u64, // Number of objects
124
+ pub n_entries: u64, // Number of entries
125
+ pub tail_entry_seqnum: u64, // Sequence number of the last entry
126
+ pub head_entry_seqnum: u64, // Sequence number of the first entry
127
+ pub entry_array_offset: u64, // Offset of the entry array
128
+ pub head_entry_realtime: u64, // Realtime timestamp of the first entry
129
+ pub tail_entry_realtime: u64, // Realtime timestamp of the last entry
130
+ pub tail_entry_monotonic: u64, // Monotonic timestamp of the last entry
131
+}
132
+
133
+/*
134
+ NOTE: For the time being, we do not need the following fields.
135
+
136
+ // Added in 187
137
+ pub n_data: u64, // Number of data objects
138
+ pub n_fields: u64, // Number of field objects
139
+ // Added in 189
140
+ pub n_tags: u64, // Number of tag objects
141
+ pub n_entry_arrays: u64, // Number of entry array objects
142
+ // Added in 246
143
+ pub data_hash_chain_depth: u64, // Deepest chain in data hash table
144
+ pub field_hash_chain_depth: u64, // Deepest chain in field hash table
145
+ // Added in 252
146
+ pub tail_entry_array_offset: u32, // Offset to the tail entry array
147
+ pub tail_entry_array_n_entries: u32, // Number of entries in the tail entry array
148
+ // Added in 254
149
+ pub tail_entry_offset: u64, // Offset to the tail entry
150
+*/
151
+
152
+impl JournalHeader {
153
+ pub fn has_incompatible_flag(&self, flag: HeaderIncompatibleFlags) -> bool {
154
+ (self.incompatible_flags & flag as u32) != 0
155
+ }
156
+
157
+ pub fn has_compatible_flag(&self, flag: HeaderCompatibleFlags) -> bool {
158
+ (self.compatible_flags & flag as u32) != 0
159
+ }
160
+
161
+ fn map_hash_table<M: MemoryMap>(
162
+ &self,
163
+ file: &File,
164
+ offset: u64,
165
+ size: u64,
166
+ ) -> Result<Option<M>> {
167
+ if offset == 0 || size == 0 {
168
+ return Ok(None);
169
+ }
170
+
171
+ let object_header_size = std::mem::size_of::<ObjectHeader>() as u64;
172
+ if offset <= object_header_size || size < object_header_size {
173
+ return Err(JournalError::InvalidObjectLocation);
174
+ }
175
+
176
+ let position = offset - object_header_size;
177
+ let map_size = object_header_size + size;
178
+ M::create(file, position, map_size).map(Some)
179
+ }
180
+
181
+ pub fn map_data_hash_table<M: MemoryMap>(&self, file: &File) -> Result<Option<M>> {
182
+ self.map_hash_table(file, self.data_hash_table_offset, self.data_hash_table_size)
183
+ }
184
+
185
+ pub fn map_field_hash_table<M: MemoryMap>(&self, file: &File) -> Result<Option<M>> {
186
+ self.map_hash_table(
187
+ file,
188
+ self.field_hash_table_offset,
189
+ self.field_hash_table_size,
190
+ )
191
+ }
192
+}
193
+
194
+pub enum ObjectFlags {
195
+ CompressedXz = 1 << 0,
196
+ CompressedLz4 = 1 << 1,
197
+ CompressedZstd = 1 << 2,
198
+}
199
+
200
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
201
+#[repr(u8)]
202
+pub enum ObjectType {
203
+ Unused = 0,
204
+ Data = 1,
205
+ Field = 2,
206
+ Entry = 3,
207
+ DataHashTable = 4,
208
+ FieldHashTable = 5,
209
+ EntryArray = 6,
210
+ Tag = 7,
211
+}
212
+
213
+impl TryFrom<u8> for ObjectType {
214
+ type Error = JournalError;
215
+
216
+ fn try_from(value: u8) -> Result<Self> {
217
+ match value {
218
+ 0 => Ok(ObjectType::Unused),
219
+ 1 => Ok(ObjectType::Data),
220
+ 2 => Ok(ObjectType::Field),
221
+ 3 => Ok(ObjectType::Entry),
222
+ 4 => Ok(ObjectType::DataHashTable),
223
+ 5 => Ok(ObjectType::FieldHashTable),
224
+ 6 => Ok(ObjectType::EntryArray),
225
+ 7 => Ok(ObjectType::Tag),
226
+ _ => Err(JournalError::InvalidObjectType),
227
+ }
228
+ }
229
+}
230
+
231
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
232
+#[repr(C)]
233
+pub struct ObjectHeader {
234
+ pub type_: u8,
235
+ pub flags: u8,
236
+ pub reserved: [u8; 6],
237
+ pub size: u64,
238
+}
239
+
240
+impl ObjectHeader {
241
+ pub fn xz_compressed(&self) -> bool {
242
+ (self.flags & ObjectFlags::CompressedXz as u8) != 0
243
+ }
244
+
245
+ pub fn lz4_compressed(&self) -> bool {
246
+ (self.flags & ObjectFlags::CompressedLz4 as u8) != 0
247
+ }
248
+
249
+ pub fn zstd_compressed(&self) -> bool {
250
+ (self.flags & ObjectFlags::CompressedZstd as u8) != 0
251
+ }
252
+
253
+ pub fn is_compressed(&self) -> bool {
254
+ self.zstd_compressed() | self.lz4_compressed() | self.xz_compressed()
255
+ }
256
+
257
+ pub fn aligned_size(&self) -> u64 {
258
+ (self.size + 7) & !7
259
+ }
260
+}
261
+
262
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
263
+#[repr(C)]
264
+pub struct FieldObjectHeader {
265
+ pub object_header: ObjectHeader,
266
+ pub hash: u64,
267
+ pub next_hash_offset: u64,
268
+ pub head_data_offset: u64,
269
+}
270
+
271
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
272
+#[repr(C)]
273
+pub struct OffsetArrayObjectHeader {
274
+ pub object_header: ObjectHeader,
275
+ pub next_offset_array: u64,
276
+}
277
+
278
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
279
+#[repr(C)]
280
+pub struct HashItem {
281
+ pub head_hash_offset: u64,
282
+ pub tail_hash_offset: u64,
283
+}
284
+
285
+pub struct HashTableObject<B: ByteSlice> {
286
+ pub header: Ref<B, ObjectHeader>,
287
+ pub items: Ref<B, [HashItem]>,
288
+}
289
+
290
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for HashTableObject<B> {
291
+ fn from_data(data: B, _is_compact: bool) -> Option<Self> {
292
+ let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).ok()?;
293
+
294
+ let header = zerocopy::Ref::from_bytes(header_data).unwrap();
295
+ let items = zerocopy::Ref::from_bytes(items_data).unwrap();
296
+
297
+ Some(HashTableObject { header, items })
298
+ }
299
+}
300
+
301
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for HashTableObject<B> {
302
+ fn from_data_mut(data: B, _is_compact: bool) -> Self {
303
+ let (header_data, items_data) = data.split_at(std::mem::size_of::<ObjectHeader>()).unwrap();
304
+
305
+ let header = zerocopy::Ref::from_bytes(header_data).unwrap();
306
+ let items = zerocopy::Ref::from_bytes(items_data).unwrap();
307
+
308
+ HashTableObject { header, items }
309
+ }
310
+}
311
+
312
+#[derive(Debug)]
313
+pub struct FieldObject<B: ByteSlice> {
314
+ pub header: Ref<B, FieldObjectHeader>,
315
+ pub payload: B,
316
+}
317
+
318
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for FieldObject<B> {
319
+ fn from_data(data: B, _is_compact: bool) -> Option<Self> {
320
+ let (header, payload) = zerocopy::Ref::from_prefix(data).ok()?;
321
+ Some(FieldObject { header, payload })
322
+ }
323
+}
324
+
325
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for FieldObject<B> {
326
+ fn from_data_mut(data: B, _is_compact: bool) -> Self {
327
+ let (header, payload) = zerocopy::Ref::from_prefix(data).unwrap();
328
+
329
+ FieldObject { header, payload }
330
+ }
331
+}
332
+
333
+pub enum OffsetsType<B: ByteSlice> {
334
+ Regular(Ref<B, [u64]>),
335
+ Compact(Ref<B, [u32]>),
336
+}
337
+
338
+impl<B: ByteSlice> OffsetsType<B> {
339
+ pub fn get(&self, index: usize) -> u64 {
340
+ match self {
341
+ OffsetsType::Regular(offsets) => offsets[index],
342
+ OffsetsType::Compact(offsets) => offsets[index] as u64,
343
+ }
344
+ }
345
+}
346
+
347
+impl<B: ByteSlice> std::fmt::Debug for OffsetsType<B> {
348
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
349
+ match self {
350
+ OffsetsType::Regular(items) => write!(f, "Regular({} items)", items.len()),
351
+ OffsetsType::Compact(items) => write!(f, "Compact({} items)", items.len()),
352
+ }
353
+ }
354
+}
355
+
356
+pub struct OffsetArrayObject<B: ByteSlice> {
357
+ pub header: Ref<B, OffsetArrayObjectHeader>,
358
+ pub items: OffsetsType<B>,
359
+}
360
+
361
+impl<B: ByteSlice> OffsetArrayObject<B> {
362
+ pub fn capacity(&self) -> usize {
363
+ match &self.items {
364
+ OffsetsType::Regular(offsets) => offsets.len(),
365
+ OffsetsType::Compact(offsets) => offsets.len(),
366
+ }
367
+ }
368
+
369
+ pub fn len(&self, remaining_items: usize) -> usize {
370
+ self.capacity().min(remaining_items)
371
+ }
372
+
373
+ pub fn is_empty(&self, remaining_items: usize) -> bool {
374
+ self.len(remaining_items) == 0
375
+ }
376
+
377
+ pub fn get(&self, index: usize, remaining_items: usize) -> Result<u64> {
378
+ if self.is_empty(remaining_items) {
379
+ return Err(JournalError::EmptyOffsetArrayNode);
380
+ }
381
+
382
+ let offset = match &self.items {
383
+ OffsetsType::Regular(items) => items[index],
384
+ OffsetsType::Compact(items) => items[index] as u64,
385
+ };
386
+
387
+ if offset == 0 {
388
+ Err(JournalError::InvalidOffsetArrayOffset)
389
+ } else {
390
+ Ok(offset)
391
+ }
392
+ }
393
+}
394
+
395
+impl<B: SplitByteSliceMut + std::fmt::Debug> OffsetArrayObject<B> {
396
+ pub fn set(&mut self, index: usize, offset: NonZeroU64) -> Result<()> {
397
+ if index >= self.capacity() {
398
+ return Err(JournalError::OutOfBoundsIndex);
399
+ }
400
+
401
+ match &mut self.items {
402
+ OffsetsType::Regular(items) => items[index] = offset.get(),
403
+ OffsetsType::Compact(items) => items[index] = offset.get() as u32,
404
+ };
405
+
406
+ Ok(())
407
+ }
408
+}
409
+
410
+impl<B: ByteSlice> std::fmt::Debug for OffsetArrayObject<B> {
411
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
412
+ f.debug_struct("JournalHeader")
413
+ .field("header", &self.header)
414
+ .finish()
415
+ }
416
+}
417
+
418
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for OffsetArrayObject<B> {
419
+ fn from_data(data: B, is_compact: bool) -> Option<Self> {
420
+ let (header_data, items_data) = data
421
+ .split_at(std::mem::size_of::<OffsetArrayObjectHeader>())
422
+ .ok()?;
423
+
424
+ let header = zerocopy::Ref::from_bytes(header_data).ok()?;
425
+
426
+ let items_type = if is_compact {
427
+ let compact_items = zerocopy::Ref::from_bytes(items_data).ok()?;
428
+ OffsetsType::Compact(compact_items)
429
+ } else {
430
+ let regular_items = zerocopy::Ref::from_bytes(items_data).ok()?;
431
+ OffsetsType::Regular(regular_items)
432
+ };
433
+
434
+ Some(OffsetArrayObject {
435
+ header,
436
+ items: items_type,
437
+ })
438
+ }
439
+}
440
+
441
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for OffsetArrayObject<B> {
442
+ fn from_data_mut(data: B, is_compact: bool) -> Self {
443
+ let (header_data, items_data) = data
444
+ .split_at(std::mem::size_of::<OffsetArrayObjectHeader>())
445
+ .unwrap();
446
+
447
+ let header = zerocopy::Ref::from_bytes(header_data).unwrap();
448
+
449
+ let items_type = if is_compact {
450
+ let compact_items = zerocopy::Ref::from_bytes(items_data).unwrap();
451
+ OffsetsType::Compact(compact_items)
452
+ } else {
453
+ let regular_items = zerocopy::Ref::from_bytes(items_data).unwrap();
454
+ OffsetsType::Regular(regular_items)
455
+ };
456
+
457
+ OffsetArrayObject {
458
+ header,
459
+ items: items_type,
460
+ }
461
+ }
462
+}
463
+
464
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
465
+#[repr(C)]
466
+pub struct EntryObjectHeader {
467
+ pub object_header: ObjectHeader,
468
+ pub seqnum: u64,
469
+ pub realtime: u64,
470
+ pub monotonic: u64,
471
+ pub boot_id: [u8; 16], // UUID/128-bit ID
472
+ pub xor_hash: u64,
473
+}
474
+
475
+// For regular (non-compact) format - an array of these follows the header
476
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
477
+#[repr(C)]
478
+pub struct RegularEntryItem {
479
+ pub object_offset: u64,
480
+ pub hash: u64,
481
+}
482
+
483
+// For compact format - an array of these follows the header
484
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
485
+#[repr(C)]
486
+pub struct CompactEntryItem {
487
+ pub object_offset: u32,
488
+}
489
+
490
+pub enum EntryItemsType<B: ByteSlice> {
491
+ Regular(Ref<B, [RegularEntryItem]>),
492
+ Compact(Ref<B, [CompactEntryItem]>),
493
+}
494
+
495
+impl<B: ByteSlice> EntryItemsType<B> {
496
+ pub fn get(&self, index: usize) -> u64 {
497
+ match self {
498
+ EntryItemsType::Regular(entry_items) => entry_items[index].object_offset,
499
+ EntryItemsType::Compact(entry_items) => entry_items[index].object_offset as u64,
500
+ }
501
+ }
502
+
503
+ pub fn len(&self) -> usize {
504
+ match self {
505
+ EntryItemsType::Regular(entry_items) => entry_items.len(),
506
+ EntryItemsType::Compact(entry_items) => entry_items.len(),
507
+ }
508
+ }
509
+
510
+ pub fn is_empty(&self) -> bool {
511
+ match self {
512
+ EntryItemsType::Regular(entry_items) => entry_items.is_empty(),
513
+ EntryItemsType::Compact(entry_items) => entry_items.is_empty(),
514
+ }
515
+ }
516
+}
517
+
518
+impl<B: ByteSlice> std::fmt::Debug for EntryItemsType<B> {
519
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
520
+ match self {
521
+ EntryItemsType::Regular(items) => write!(f, "Regular({} items)", items.len()),
522
+ EntryItemsType::Compact(items) => write!(f, "Compact({} items)", items.len()),
523
+ }
524
+ }
525
+}
526
+
527
+pub struct EntryObject<B: ByteSlice> {
528
+ pub header: Ref<B, EntryObjectHeader>,
529
+ pub items: EntryItemsType<B>,
530
+}
531
+
532
+impl<B: ByteSlice> std::fmt::Debug for EntryObject<B> {
533
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
534
+ f.debug_struct("EntryObject")
535
+ .field("header", &self.header)
536
+ .field("items", &self.items)
537
+ .finish()
538
+ }
539
+}
540
+
541
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for EntryObject<B> {
542
+ fn from_data(data: B, is_compact: bool) -> Option<Self> {
543
+ let (header_data, items_data) = data
544
+ .split_at(std::mem::size_of::<EntryObjectHeader>())
545
+ .ok()?;
546
+
547
+ let header = zerocopy::Ref::from_bytes(header_data).ok()?;
548
+
549
+ let items_type = if is_compact {
550
+ let compact_items = zerocopy::Ref::from_bytes(items_data).ok()?;
551
+ EntryItemsType::Compact(compact_items)
552
+ } else {
553
+ let regular_items = zerocopy::Ref::from_bytes(items_data).ok()?;
554
+ EntryItemsType::Regular(regular_items)
555
+ };
556
+
557
+ Some(EntryObject {
558
+ header,
559
+ items: items_type,
560
+ })
561
+ }
562
+}
563
+
564
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for EntryObject<B> {
565
+ fn from_data_mut(data: B, is_compact: bool) -> Self {
566
+ let (header_data, items_data) = data
567
+ .split_at(std::mem::size_of::<EntryObjectHeader>())
568
+ .unwrap();
569
+
570
+ let header = zerocopy::Ref::from_bytes(header_data).unwrap();
571
+
572
+ let items_type = if is_compact {
573
+ let compact_items = zerocopy::Ref::from_bytes(items_data).unwrap();
574
+ EntryItemsType::Compact(compact_items)
575
+ } else {
576
+ let regular_items = zerocopy::Ref::from_bytes(items_data).unwrap();
577
+ EntryItemsType::Regular(regular_items)
578
+ };
579
+
580
+ EntryObject {
581
+ header,
582
+ items: items_type,
583
+ }
584
+ }
585
+}
586
+
587
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
588
+#[repr(C)]
589
+pub struct DataObjectHeader {
590
+ pub object_header: ObjectHeader,
591
+ pub hash: u64,
592
+ pub next_hash_offset: u64,
593
+ pub next_field_offset: u64,
594
+ pub entry_offset: u64,
595
+ pub entry_array_offset: u64,
596
+ pub n_entries: u64,
597
+}
598
+
599
+impl DataObjectHeader {
600
+ pub fn xz_compressed(&self) -> bool {
601
+ self.object_header.xz_compressed()
602
+ }
603
+
604
+ pub fn lz4_compressed(&self) -> bool {
605
+ self.object_header.lz4_compressed()
606
+ }
607
+
608
+ pub fn zstd_compressed(&self) -> bool {
609
+ self.object_header.zstd_compressed()
610
+ }
611
+
612
+ pub fn is_compressed(&self) -> bool {
613
+ self.object_header.is_compressed()
614
+ }
615
+
616
+ pub fn inlined_cursor(&self) -> Option<InlinedCursor> {
617
+ if self.n_entries == 0 {
618
+ return None;
619
+ }
620
+
621
+ let inlined_offset = NonZeroU64::new(self.entry_offset)?;
622
+ let cursor = self.entry_array_offset_list().map(Cursor::at_head);
623
+ Some(InlinedCursor::new(inlined_offset, cursor))
624
+ }
625
+
626
+ pub fn entry_array_offset_list(&self) -> Option<List> {
627
+ let total_items = NonZeroUsize::new(self.n_entries.saturating_sub(1) as usize)?;
628
+ let head_offset = NonZeroU64::new(self.entry_array_offset)?;
629
+
630
+ Some(List::new(head_offset, total_items))
631
+ }
632
+}
633
+
634
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable, PartialEq, Eq)]
635
+#[repr(C)]
636
+pub struct CompactDataFields {
637
+ pub tail_entry_array_offset: u32,
638
+ pub tail_entry_array_n_entries: u32,
639
+}
640
+
641
+#[derive(PartialEq, Eq)]
642
+pub enum DataPayloadType<B: ByteSlice> {
643
+ Regular(B),
644
+ Compact {
645
+ compact_fields: Ref<B, CompactDataFields>,
646
+ payload: B,
647
+ },
648
+}
649
+
650
+impl<B: ByteSlice> std::fmt::Debug for DataPayloadType<B> {
651
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
652
+ match self {
653
+ DataPayloadType::Regular(payload) => write!(f, "Regular({} bytes)", payload.len()),
654
+ DataPayloadType::Compact {
655
+ compact_fields,
656
+ payload,
657
+ } => write!(
658
+ f,
659
+ "Compact(fields: {:?}, payload: {} bytes)",
660
+ compact_fields,
661
+ payload.len()
662
+ ),
663
+ }
664
+ }
665
+}
666
+
667
+// Complete Data Object structure
668
+pub struct DataObject<B: ByteSlice> {
669
+ pub header: Ref<B, DataObjectHeader>,
670
+ pub payload: DataPayloadType<B>,
671
+}
672
+
673
+impl<B: ByteSlice> std::fmt::Debug for DataObject<B> {
674
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
675
+ f.debug_struct("DataObject")
676
+ .field("header", &self.header)
677
+ .field("payload", &self.payload)
678
+ .finish()
679
+ }
680
+}
681
+
682
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for DataObject<B> {
683
+ fn from_data(data: B, is_compact: bool) -> Option<Self> {
684
+ let (header_data, remaining_data) = data
685
+ .split_at(std::mem::size_of::<DataObjectHeader>())
686
+ .ok()?;
687
+
688
+ let header = zerocopy::Ref::from_bytes(header_data).ok()?;
689
+
690
+ let payload = if is_compact {
691
+ let (fields_data, payload_data) = remaining_data
692
+ .split_at(std::mem::size_of::<CompactDataFields>())
693
+ .ok()?;
694
+
695
+ let compact_fields = zerocopy::Ref::from_bytes(fields_data).ok()?;
696
+
697
+ DataPayloadType::Compact {
698
+ compact_fields,
699
+ payload: payload_data,
700
+ }
701
+ } else {
702
+ DataPayloadType::Regular(remaining_data)
703
+ };
704
+
705
+ Some(DataObject { header, payload })
706
+ }
707
+}
708
+
709
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for DataObject<B> {
710
+ fn from_data_mut(data: B, is_compact: bool) -> Self {
711
+ let (header_data, remaining_data) = data
712
+ .split_at(std::mem::size_of::<DataObjectHeader>())
713
+ .unwrap();
714
+
715
+ let header = zerocopy::Ref::from_bytes(header_data).unwrap();
716
+
717
+ let payload = if is_compact {
718
+ let (fields_data, payload_data) = remaining_data
719
+ .split_at(std::mem::size_of::<CompactDataFields>())
720
+ .unwrap();
721
+
722
+ let compact_fields = zerocopy::Ref::from_bytes(fields_data).unwrap();
723
+
724
+ DataPayloadType::Compact {
725
+ compact_fields,
726
+ payload: payload_data,
727
+ }
728
+ } else {
729
+ DataPayloadType::Regular(remaining_data)
730
+ };
731
+
732
+ DataObject { header, payload }
733
+ }
734
+}
735
+
736
+impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> DataObject<B> {
737
+ pub fn payload_bytes(&self) -> &[u8] {
738
+ match &self.payload {
739
+ DataPayloadType::Regular(payload) => payload,
740
+ DataPayloadType::Compact { payload, .. } => payload,
741
+ }
742
+ }
743
+
744
+ pub fn inlined_cursor(&self) -> Option<InlinedCursor> {
745
+ self.header.inlined_cursor()
746
+ }
747
+
748
+ pub fn is_compressed(&self) -> bool {
749
+ self.header.is_compressed()
750
+ }
751
+
752
+ pub fn xz_compressed(&self) -> bool {
753
+ self.header.xz_compressed()
754
+ }
755
+
756
+ pub fn lz4_compressed(&self) -> bool {
757
+ self.header.lz4_compressed()
758
+ }
759
+
760
+ pub fn zstd_compressed(&self) -> bool {
761
+ self.header.zstd_compressed()
762
+ }
763
+
764
+ pub fn decompress(&self, buf: &mut Vec<u8>) -> Result<usize> {
765
+ debug_assert!(self.is_compressed());
766
+
767
+ if self.zstd_compressed() {
768
+ use ruzstd::decoding::StreamingDecoder;
769
+ use ruzstd::io::Read;
770
+
771
+ let payload = self.payload_bytes();
772
+ let mut decoder =
773
+ StreamingDecoder::new(payload).map_err(|_| JournalError::DecompressorError)?;
774
+
775
+ buf.clear();
776
+ decoder
777
+ .read_to_end(buf)
778
+ .map_err(|_| JournalError::DecompressorError)
779
+ } else {
780
+ Err(JournalError::UnknownCompressionMethod)
781
+ }
782
+ }
783
+}
784
+
785
+// SHA-256 HMAC is 32 bytes (256 bits)
786
+pub const TAG_LENGTH: usize = 256 / 8;
787
+
788
+#[derive(Debug, Copy, Clone, FromBytes, IntoBytes, KnownLayout, Immutable)]
789
+#[repr(C)]
790
+pub struct TagObjectHeader {
791
+ pub object_header: ObjectHeader,
792
+ pub seqnum: u64,
793
+ pub epoch: u64,
794
+ pub tag: [u8; TAG_LENGTH], // SHA-256 HMAC
795
+}
796
+
797
+pub struct TagObject<B: ByteSlice> {
798
+ pub header: Ref<B, TagObjectHeader>,
799
+}
800
+
801
+impl<B: ByteSlice> std::fmt::Debug for TagObject<B> {
802
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
803
+ f.debug_struct("TagObject")
804
+ .field("header", &self.header)
805
+ .finish()
806
+ }
807
+}
808
+
809
+impl<B: SplitByteSlice + std::fmt::Debug> JournalObject<B> for TagObject<B> {
810
+ fn from_data(data: B, _is_compact: bool) -> Option<Self> {
811
+ let header = zerocopy::Ref::from_bytes(data).ok()?;
812
+ Some(TagObject { header })
813
+ }
814
+}
815
+
816
+impl<B: SplitByteSliceMut + std::fmt::Debug> JournalObjectMut<B> for TagObject<B> {
817
+ fn from_data_mut(data: B, _is_compact: bool) -> Self {
818
+ let header = zerocopy::Ref::from_bytes(data).unwrap();
819
+ TagObject { header }
820
+ }
821
+}
822
+
823
+impl<B: ByteSlice + SplitByteSlice + std::fmt::Debug> TagObject<B> {
824
+ // Helper function to format tag as hex string
825
+ pub fn tag_as_hex(&self) -> String {
826
+ self.header
827
+ .tag
828
+ .iter()
829
+ .map(|b| format!("{:02x}", b))
830
+ .collect()
831
+ }
832
+}
src/crates/jf/journal_file/src/offset_array.rs
new
+668
@@ -0,0 +1,668 @@
1
+use crate::journal_file::JournalFile;
2
+use error::{JournalError, Result};
3
+use std::num::{NonZeroU64, NonZeroUsize};
4
+use window_manager::MemoryMap;
5
+
6
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7
+pub enum Direction {
8
+ Forward,
9
+ Backward,
10
+}
11
+
12
+/// A reference to a single array of offsets in the journal file
13
+pub struct Node {
14
+ offset: NonZeroU64,
15
+ next_offset: Option<NonZeroU64>,
16
+ capacity: NonZeroUsize,
17
+ // Number of items remaining in this array and subsequent arrays
18
+ remaining_items: NonZeroUsize,
19
+}
20
+
21
+impl Node {
22
+ /// Create a new offset array reference
23
+ fn new<M: MemoryMap>(
24
+ journal_file: &JournalFile<M>,
25
+ offset: NonZeroU64,
26
+ remaining_items: NonZeroUsize,
27
+ ) -> Result<Self> {
28
+ let array = journal_file.offset_array_ref(offset.get())?;
29
+ let capacity =
30
+ NonZeroUsize::new(array.capacity()).ok_or(JournalError::EmptyOffsetArrayNode)?;
31
+
32
+ Ok(Self {
33
+ offset,
34
+ next_offset: NonZeroU64::new(array.header.next_offset_array),
35
+ capacity,
36
+ remaining_items,
37
+ })
38
+ }
39
+
40
+ /// Get the offset of this array in the file
41
+ pub fn offset(&self) -> u64 {
42
+ self.offset.get()
43
+ }
44
+
45
+ /// Get the maximum number of items this array can hold
46
+ pub fn capacity(&self) -> NonZeroUsize {
47
+ self.capacity
48
+ }
49
+
50
+ /// Get the number of items available in this array
51
+ pub fn len(&self) -> NonZeroUsize {
52
+ self.capacity.min(self.remaining_items)
53
+ }
54
+
55
+ /// Check if this array has a next array in the chain
56
+ pub fn has_next(&self) -> bool {
57
+ self.next_offset.is_some() && self.remaining_items > self.len()
58
+ }
59
+
60
+ /// Get the next array in the chain, if any
61
+ pub fn next<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<Self>> {
62
+ if !self.has_next() {
63
+ return Ok(None);
64
+ }
65
+
66
+ let next_offset = self.next_offset.unwrap();
67
+ let remaining_items = {
68
+ let n = self.remaining_items.get().saturating_sub(self.len().get());
69
+ NonZeroUsize::new(n).ok_or(JournalError::EmptyOffsetArrayNode)?
70
+ };
71
+ let node = Self::new(journal_file, next_offset, remaining_items);
72
+
73
+ Some(node).transpose()
74
+ }
75
+
76
+ /// Get an item at the specified index
77
+ pub fn get<M: MemoryMap>(&self, journal_file: &JournalFile<M>, index: usize) -> Result<u64> {
78
+ if index >= self.len().get() {
79
+ return Err(JournalError::InvalidOffsetArrayIndex);
80
+ }
81
+
82
+ let array = journal_file.offset_array_ref(self.offset.get())?;
83
+ array.get(index, self.remaining_items.get())
84
+ }
85
+
86
+ /// Returns the first index where the predicate returns false, or array length if
87
+ /// the predicate is true for all elements
88
+ pub fn partition_point<M, F>(
89
+ &self,
90
+ journal_file: &JournalFile<M>,
91
+ left: usize,
92
+ right: usize,
93
+ predicate: F,
94
+ ) -> Result<usize>
95
+ where
96
+ M: MemoryMap,
97
+ F: Fn(u64) -> Result<bool>,
98
+ {
99
+ let mut left = left;
100
+ let mut right = right;
101
+
102
+ debug_assert!(left <= right);
103
+ debug_assert!(right <= self.len().get());
104
+
105
+ while left != right {
106
+ let mid = left.midpoint(right);
107
+ let offset = self.get(journal_file, mid)?;
108
+
109
+ if predicate(offset)? {
110
+ left = mid + 1;
111
+ } else {
112
+ right = mid;
113
+ }
114
+ }
115
+
116
+ Ok(left)
117
+ }
118
+
119
+ /// Find the forward or backward (depending on direction) position that matches the predicate.
120
+ pub fn directed_partition_point<M, F>(
121
+ &self,
122
+ journal_file: &JournalFile<M>,
123
+ left: usize,
124
+ right: usize,
125
+ predicate: F,
126
+ direction: Direction,
127
+ ) -> Result<Option<usize>>
128
+ where
129
+ M: MemoryMap,
130
+ F: Fn(u64) -> Result<bool>,
131
+ {
132
+ let index = self.partition_point(journal_file, left, right, predicate)?;
133
+
134
+ Ok(match direction {
135
+ Direction::Forward => {
136
+ if index < self.len().get() {
137
+ Some(index)
138
+ } else {
139
+ None
140
+ }
141
+ }
142
+ Direction::Backward => {
143
+ if index > 0 {
144
+ Some(index - 1)
145
+ } else {
146
+ None
147
+ }
148
+ }
149
+ })
150
+ }
151
+}
152
+
153
+impl std::fmt::Debug for Node {
154
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
155
+ let next_offset = self.next_offset.map(|x| x.get()).unwrap_or(0);
156
+
157
+ f.debug_struct("Node")
158
+ .field("offset", &format!("0x{:x}", self.offset))
159
+ .field("next_offset", &format!("0x{:x}", next_offset))
160
+ .field("capacity", &self.capacity)
161
+ .field("len", &self.len())
162
+ .field("remaining_items", &self.remaining_items)
163
+ .finish()
164
+ }
165
+}
166
+
167
+/// A linked list of offset arrays
168
+#[derive(Copy, Clone)]
169
+pub struct List {
170
+ head_offset: NonZeroU64,
171
+ total_items: NonZeroUsize,
172
+}
173
+
174
+impl std::fmt::Debug for List {
175
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
176
+ f.debug_struct("List")
177
+ .field("head_offset", &format!("0x{:x}", self.head_offset))
178
+ .field("total_items", &self.total_items)
179
+ .finish()
180
+ }
181
+}
182
+
183
+impl List {
184
+ /// Create a new list from head offset and total items
185
+ pub fn new(head_offset: NonZeroU64, total_items: NonZeroUsize) -> Self {
186
+ Self {
187
+ head_offset,
188
+ total_items,
189
+ }
190
+ }
191
+
192
+ /// Get the head array of this chain
193
+ pub fn head<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Node> {
194
+ Node::new(journal_file, self.head_offset, self.total_items)
195
+ }
196
+
197
+ /// Get the tail array of this list by traversing from head to tail
198
+ pub fn tail<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Node> {
199
+ let mut current = self.head(journal_file)?;
200
+
201
+ while let Some(next) = current.next(journal_file)? {
202
+ current = next;
203
+ }
204
+
205
+ Ok(current)
206
+ }
207
+
208
+ /// Get a cursor at the first position in the chain
209
+ pub fn cursor_head(self) -> Cursor {
210
+ Cursor::at_head(self)
211
+ }
212
+
213
+ /// Get a cursor at the last position in the chain
214
+ pub fn cursor_tail<M: MemoryMap>(self, journal_file: &JournalFile<M>) -> Result<Cursor> {
215
+ Cursor::at_tail(journal_file, self)
216
+ }
217
+
218
+ /// Finds the first/last array item position where the predicate function becomes false
219
+ /// in a chain of offset arrays.
220
+ ///
221
+ /// # Parameters
222
+ /// * `predicate` - Function that takes an array item value and returns true if the search should continue.
223
+ /// * `direction` - Direction of the search (Forward or Backward)
224
+ pub fn directed_partition_point<M, F>(
225
+ self,
226
+ journal_file: &JournalFile<M>,
227
+ predicate: F,
228
+ direction: Direction,
229
+ ) -> Result<Option<Cursor>>
230
+ where
231
+ M: MemoryMap,
232
+ F: Fn(u64) -> Result<bool>,
233
+ {
234
+ let mut last_cursor: Option<Cursor> = None;
235
+
236
+ let mut node = self.head(journal_file)?;
237
+
238
+ loop {
239
+ let left = 0;
240
+ let right = node.len().get();
241
+
242
+ if let Some(index) =
243
+ node.directed_partition_point(journal_file, left, right, &predicate, direction)?
244
+ {
245
+ let cursor = Cursor::at_position(
246
+ journal_file,
247
+ self,
248
+ node.offset,
249
+ index,
250
+ node.remaining_items,
251
+ )?;
252
+
253
+ match direction {
254
+ Direction::Forward => {
255
+ return Ok(Some(cursor));
256
+ }
257
+ Direction::Backward => {
258
+ // In backward direction, save this match and continue
259
+ // to ensure we'll find the last match
260
+ last_cursor = Some(cursor);
261
+
262
+ // If this match is at the end of the array and there's a next array,
263
+ // we should check the next array as well
264
+ if index == node.len().get() - 1 && node.has_next() {
265
+ // continue;
266
+ } else {
267
+ return Ok(last_cursor);
268
+ }
269
+ }
270
+ }
271
+ } else if direction == Direction::Backward {
272
+ // No match in this array for backward direction
273
+ return Ok(last_cursor);
274
+ }
275
+
276
+ if let Some(nd) = node.next(journal_file)? {
277
+ node = nd;
278
+ } else {
279
+ break;
280
+ }
281
+ }
282
+
283
+ // For backward direction, return the last match we found (if any)
284
+ if direction == Direction::Backward {
285
+ return Ok(last_cursor);
286
+ }
287
+
288
+ // No match found in any array
289
+ Ok(None)
290
+ }
291
+}
292
+
293
+/// A cursor pointing to a specific position within an offset array chain
294
+#[derive(Clone, Copy)]
295
+pub struct Cursor {
296
+ list: List,
297
+ array_offset: NonZeroU64,
298
+ array_index: usize,
299
+ remaining_items: NonZeroUsize,
300
+}
301
+
302
+impl Cursor {
303
+ pub fn head(&self) -> Self {
304
+ Self::at_head(self.list)
305
+ }
306
+
307
+ /// Create a cursor at the head of the chain
308
+ pub fn at_head(list: List) -> Self {
309
+ Self {
310
+ list,
311
+ array_offset: list.head_offset,
312
+ array_index: 0,
313
+ remaining_items: list.total_items,
314
+ }
315
+ }
316
+
317
+ /// Create a cursor at the tail of the chain
318
+ pub fn at_tail<M: MemoryMap>(journal_file: &JournalFile<M>, list: List) -> Result<Self> {
319
+ let mut current_array = list.head(journal_file)?;
320
+
321
+ while let Some(next_array) = current_array.next(journal_file)? {
322
+ current_array = next_array;
323
+ }
324
+
325
+ Ok(Self {
326
+ list,
327
+ array_offset: current_array.offset,
328
+ array_index: current_array.len().get() - 1,
329
+ remaining_items: current_array.len(),
330
+ })
331
+ }
332
+
333
+ /// Create a cursor at a specific position
334
+ pub fn at_position<M: MemoryMap>(
335
+ journal_file: &JournalFile<M>,
336
+ offset_array_list: List,
337
+ array_offset: NonZeroU64,
338
+ array_index: usize,
339
+ remaining_items: NonZeroUsize,
340
+ ) -> Result<Self> {
341
+ debug_assert!(offset_array_list.total_items >= remaining_items);
342
+
343
+ // Verify the array exists
344
+ let array = Node::new(journal_file, array_offset, remaining_items)?;
345
+
346
+ // Verify the index is valid
347
+ if array_index >= array.len().get() {
348
+ return Err(JournalError::InvalidOffsetArrayIndex);
349
+ }
350
+
351
+ Ok(Self {
352
+ list: offset_array_list,
353
+ array_offset,
354
+ array_index,
355
+ remaining_items,
356
+ })
357
+ }
358
+
359
+ /// Get the current array this cursor points to
360
+ pub fn node<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Node> {
361
+ Node::new(journal_file, self.array_offset, self.remaining_items)
362
+ }
363
+
364
+ pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<u64> {
365
+ self.node(journal_file)?.get(journal_file, self.array_index)
366
+ }
367
+
368
+ /// Move to the next position
369
+ pub fn next<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<Self>> {
370
+ let array_node = self.node(journal_file)?;
371
+
372
+ // FIXME: overtly defensive/expensive...
373
+ if self.array_index + 1 < array_node.len().get() {
374
+ // Next item is in the same array
375
+ return Ok(Some(Self {
376
+ list: self.list,
377
+ array_offset: self.array_offset,
378
+ array_index: self.array_index + 1,
379
+ remaining_items: self.remaining_items,
380
+ }));
381
+ }
382
+
383
+ if !array_node.has_next() {
384
+ return Ok(None);
385
+ }
386
+
387
+ let next_array = array_node.next(journal_file)?.unwrap();
388
+
389
+ match NonZeroUsize::new(
390
+ self.remaining_items
391
+ .get()
392
+ .saturating_sub(array_node.len().get()),
393
+ ) {
394
+ None => Ok(None),
395
+ Some(remaining_items) => Ok(Some(Self {
396
+ list: self.list,
397
+ array_offset: next_array.offset,
398
+ array_index: 0,
399
+ remaining_items,
400
+ })),
401
+ }
402
+ }
403
+
404
+ /// Move to the previous position
405
+ pub fn previous<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<Self>> {
406
+ if self.array_index > 0 {
407
+ // Previous item is in the same array
408
+ return Ok(Some(Self {
409
+ list: self.list,
410
+ array_offset: self.array_offset,
411
+ array_index: self.array_index - 1,
412
+ remaining_items: self.remaining_items,
413
+ }));
414
+ }
415
+
416
+ if self.array_offset == self.list.head_offset {
417
+ return Ok(None);
418
+ }
419
+
420
+ let mut node = self.list.head(journal_file)?;
421
+ while node.has_next() {
422
+ if node.next_offset == Some(self.array_offset) {
423
+ return Ok(Some(Self {
424
+ list: self.list,
425
+ array_offset: node.offset,
426
+ array_index: node.len().get() - 1,
427
+ remaining_items: node.remaining_items,
428
+ }));
429
+ }
430
+
431
+ node = node.next(journal_file)?.unwrap();
432
+ }
433
+
434
+ Err(JournalError::InvalidOffsetArrayOffset)
435
+ }
436
+}
437
+
438
+impl std::fmt::Debug for Cursor {
439
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
440
+ f.debug_struct("Cursor")
441
+ .field("array_offset", &format!("0x{:x}", self.array_offset))
442
+ .field("array_index", &self.array_index)
443
+ .field("remaining_items", &self.remaining_items)
444
+ .finish()
445
+ }
446
+}
447
+
448
+#[derive(Debug, Copy, Clone)]
449
+pub struct InlinedCursor {
450
+ inlined_offset: NonZeroU64,
451
+ cursor: Option<Cursor>,
452
+ at_inlined_offset: bool,
453
+}
454
+
455
+impl InlinedCursor {
456
+ pub fn new(inlined_offset: NonZeroU64, cursor: Option<Cursor>) -> Self {
457
+ Self {
458
+ inlined_offset,
459
+ cursor,
460
+ at_inlined_offset: true,
461
+ }
462
+ }
463
+
464
+ pub fn head(&self) -> Self {
465
+ Self {
466
+ inlined_offset: self.inlined_offset,
467
+ cursor: self.cursor.as_ref().map(|c| c.head()),
468
+ at_inlined_offset: true,
469
+ }
470
+ }
471
+
472
+ pub fn tail<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Self> {
473
+ // Start with a copy of the current cursor
474
+ let mut result = *self;
475
+
476
+ // If we have an entry array list cursor, move it to the tail
477
+ if let Some(cursor) = self.cursor {
478
+ result.cursor = Some(cursor.list.cursor_tail(journal_file)?);
479
+ result.at_inlined_offset = false;
480
+ }
481
+
482
+ Ok(result)
483
+ }
484
+
485
+ fn next<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<Self>> {
486
+ // Case 1: We're at the inlined entry, move to the first array entry
487
+ if self.at_inlined_offset {
488
+ if self.cursor.is_some() {
489
+ return Ok(Some(Self {
490
+ inlined_offset: self.inlined_offset,
491
+ cursor: self.cursor,
492
+ at_inlined_offset: false,
493
+ }));
494
+ } else {
495
+ return Ok(None);
496
+ }
497
+ }
498
+
499
+ // Case 2: We're already in the entry array
500
+ if let Some(current_cursor) = self.cursor.as_ref() {
501
+ let next_cursor = current_cursor.next(journal_file)?;
502
+
503
+ if next_cursor.is_some() {
504
+ return Ok(Some(Self {
505
+ inlined_offset: self.inlined_offset,
506
+ cursor: next_cursor,
507
+ at_inlined_offset: false,
508
+ }));
509
+ } else {
510
+ return Ok(None);
511
+ }
512
+ }
513
+
514
+ // No more entries
515
+ Ok(None)
516
+ }
517
+
518
+ fn previous<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<Option<Self>> {
519
+ if self.at_inlined_offset {
520
+ return Ok(None);
521
+ }
522
+
523
+ if let Some(current_cursor) = self.cursor {
524
+ // Try to move to the previous position in the array
525
+ if let Some(prev_cursor) = current_cursor.previous(journal_file)? {
526
+ // We can move back within the array
527
+ let mut ic = *self;
528
+ ic.cursor = Some(prev_cursor);
529
+ return Ok(Some(ic));
530
+ } else {
531
+ // We're at the first array position, move to the inlined entry
532
+ let mut ic = *self;
533
+ ic.at_inlined_offset = true;
534
+ return Ok(Some(ic));
535
+ }
536
+ }
537
+
538
+ unreachable!();
539
+ }
540
+
541
+ pub fn value<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<u64> {
542
+ // Case 1: We're at the inlined entry
543
+ if self.at_inlined_offset {
544
+ return Ok(self.inlined_offset.get());
545
+ }
546
+
547
+ // Case 2: We're in the entry array
548
+ if let Some(cursor) = self.cursor {
549
+ return cursor.value(journal_file);
550
+ }
551
+
552
+ unreachable!();
553
+ }
554
+
555
+ pub fn next_until<M: MemoryMap>(
556
+ &mut self,
557
+ journal_file: &JournalFile<M>,
558
+ offset: u64,
559
+ ) -> Result<Option<u64>> {
560
+ let current_offset = self.value(journal_file)?;
561
+ if current_offset >= offset {
562
+ return Ok(Some(current_offset));
563
+ }
564
+
565
+ while let Some(ic) = self.next(journal_file)? {
566
+ *self = ic;
567
+
568
+ let current_offset = self.value(journal_file)?;
569
+ if current_offset >= offset {
570
+ return Ok(Some(current_offset));
571
+ }
572
+ }
573
+
574
+ Ok(None)
575
+ }
576
+
577
+ pub fn previous_until<M: MemoryMap>(
578
+ &mut self,
579
+ journal_file: &JournalFile<M>,
580
+ offset: u64,
581
+ ) -> Result<Option<u64>> {
582
+ let current_offset = self.value(journal_file)?;
583
+ if current_offset <= offset {
584
+ return Ok(Some(current_offset));
585
+ }
586
+
587
+ while let Some(ic) = self.previous(journal_file)? {
588
+ *self = ic;
589
+
590
+ let current_offset = ic.value(journal_file)?;
591
+ if current_offset <= offset {
592
+ return Ok(Some(current_offset));
593
+ }
594
+ }
595
+
596
+ Ok(None)
597
+ }
598
+
599
+ pub fn directed_partition_point<M, F>(
600
+ &self,
601
+ journal_file: &JournalFile<M>,
602
+ predicate: F,
603
+ direction: Direction,
604
+ ) -> Result<Option<Self>>
605
+ where
606
+ M: MemoryMap,
607
+ F: Fn(u64) -> Result<bool>,
608
+ {
609
+ // Variables to track our best match
610
+ let mut best_match: Option<Self> = None;
611
+
612
+ // Handle the inlined entry based on direction
613
+ match direction {
614
+ Direction::Forward => {
615
+ if !predicate(self.inlined_offset.get())? {
616
+ return Ok(Some(self.head()));
617
+ }
618
+ }
619
+ Direction::Backward => {
620
+ if predicate(self.inlined_offset.get())? {
621
+ // If predicate is true for inlined entry and we're going backward,
622
+ // this is potentially our best match
623
+ best_match = Some(self.head());
624
+ }
625
+ }
626
+ }
627
+
628
+ // If we have an array cursor, check it too using binary search
629
+ if let Some(cursor) = self.cursor {
630
+ let ic = cursor
631
+ .list
632
+ .directed_partition_point(journal_file, predicate, direction)?;
633
+
634
+ if let Some(ic) = ic {
635
+ // Create a new InlinedCursor with this array cursor
636
+ let array_match = Self {
637
+ inlined_offset: self.inlined_offset,
638
+ cursor: Some(ic),
639
+ at_inlined_offset: false,
640
+ };
641
+
642
+ // Compare with our current best match
643
+ if best_match.is_none() {
644
+ best_match = Some(array_match);
645
+ } else {
646
+ // Choose the better match based on direction
647
+ let best_offset = best_match.as_ref().unwrap().value(journal_file)?;
648
+ let array_offset = array_match.value(journal_file)?;
649
+
650
+ match direction {
651
+ Direction::Forward => {
652
+ if array_offset < best_offset {
653
+ best_match = Some(array_match);
654
+ }
655
+ }
656
+ Direction::Backward => {
657
+ if array_offset > best_offset {
658
+ best_match = Some(array_match);
659
+ }
660
+ }
661
+ }
662
+ }
663
+ }
664
+ }
665
+
666
+ Ok(best_match)
667
+ }
668
+}
src/crates/jf/journal_file/src/value_guard.rs
new
+59
@@ -0,0 +1,59 @@
1
+use std::cell::RefCell;
2
+use std::ops::{Deref, DerefMut};
3
+
4
+/// A guard that ensures exclusive access to objects obtained from a shared memory window.
5
+///
6
+/// # Purpose
7
+///
8
+/// `ValueGuard` enforces that only one journal object can be accessed at a time.
9
+/// This is necessary because:
10
+///
11
+/// 1. The underlying window manager uses a limited number of memory-mapped windows
12
+/// that are shared between all objects.
13
+/// 2. Creating a new object could invalidate memory references held by previously
14
+/// created objects, even though the objects themselves are immutable.
15
+///
16
+/// # Usage
17
+///
18
+/// When a `ValueGuard<T>` is returned from methods like `data_object()`, it provides
19
+/// read-only access to the underlying object via the `Deref` trait. When the guard
20
+/// is dropped (goes out of scope), it automatically releases the lock, allowing new
21
+/// objects to be created.
22
+///
23
+/// # Safety and Interior Mutability
24
+///
25
+/// Although the returned objects are immutable, the window manager that provides
26
+/// the memory they reference uses interior mutability (via `UnsafeCell`) to reuse
27
+/// memory-mapped regions. This guard ensures that objects are not accessed after
28
+/// their underlying memory might have been repurposed.
29
+#[derive(Debug)]
30
+pub struct ValueGuard<'a, T> {
31
+ value: T,
32
+ in_use_flag: &'a RefCell<bool>,
33
+}
34
+
35
+impl<'a, T> ValueGuard<'a, T> {
36
+ pub fn new(value: T, in_use_flag: &'a RefCell<bool>) -> Self {
37
+ Self { value, in_use_flag }
38
+ }
39
+}
40
+
41
+impl<T> Deref for ValueGuard<'_, T> {
42
+ type Target = T;
43
+
44
+ fn deref(&self) -> &Self::Target {
45
+ &self.value
46
+ }
47
+}
48
+
49
+impl<T> DerefMut for ValueGuard<'_, T> {
50
+ fn deref_mut(&mut self) -> &mut Self::Target {
51
+ &mut self.value
52
+ }
53
+}
54
+
55
+impl<T> Drop for ValueGuard<'_, T> {
56
+ fn drop(&mut self) {
57
+ *self.in_use_flag.borrow_mut() = false;
58
+ }
59
+}
src/crates/jf/journal_forwarder/Cargo.toml
new
+14
@@ -0,0 +1,14 @@
1
+[package]
2
+name = "journal_forwarder"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+journal_file = { path = "../journal_file" }
9
+journal_logger = { path = "../journal_logger" }
10
+journal_reader = { path = "../journal_reader" }
11
+window_manager = { path = "../window_manager" }
12
+memmap2 = { workspace = true }
13
+rand = { workspace = true }
14
+walkdir = { workspace = true }
src/crates/jf/journal_forwarder/src/main.rs
new
+214
@@ -0,0 +1,214 @@
1
+use error::Result;
2
+use journal_file::JournalFile;
3
+use journal_logger::JournalLogger;
4
+use journal_reader::JournalReader;
5
+use memmap2::Mmap;
6
+use rand::seq::{IndexedRandom, SliceRandom};
7
+use std::io::Read;
8
+use std::path::{Path, PathBuf};
9
+use std::time::Duration;
10
+use walkdir::WalkDir;
11
+
12
+fn process_one_cycle() -> Result<()> {
13
+ // Find all journal files in the specified directory
14
+ let journal_dir = "/var/log/journal";
15
+ let journal_files = find_journal_files(journal_dir)?;
16
+ if journal_files.is_empty() {
17
+ eprintln!("No journal files found in {}", journal_dir);
18
+ return Ok(());
19
+ }
20
+
21
+ // Pick a random journal file
22
+ let mut rng = rand::rng();
23
+ let random_file = journal_files.choose(&mut rng).unwrap();
24
+
25
+ println!("Selected journal file: {}", random_file.display());
26
+
27
+ // Open the selected journal file
28
+ let journal_file = JournalFile::<Mmap>::open(random_file, 4096)?;
29
+
30
+ // Choose random entries
31
+ let entries = select_random_entries(&journal_file, 50)?;
32
+ println!("Selected {} entries", entries.len());
33
+
34
+ // Split entries into batches
35
+ let batches = split_into_batches(entries, 5);
36
+
37
+ // Forward each batch with a delay
38
+ for (i, batch) in batches.iter().enumerate() {
39
+ if i > 0 {
40
+ std::thread::sleep(Duration::from_millis(200));
41
+ }
42
+
43
+ let mut logger = journal_logger::JournalLogger::new(
44
+ "/home/vk/opt/sd/netdata/usr/sbin/log2journal",
45
+ "/home/vk/opt/sd/netdata/usr/sbin/systemd-cat-native",
46
+ );
47
+
48
+ println!("Forwarding batch {} with {} entries", i + 1, batch.len());
49
+ forward_entries(batch, &mut logger)?;
50
+ }
51
+
52
+ Ok(())
53
+}
54
+
55
+fn find_journal_files(journal_dir: &str) -> Result<Vec<PathBuf>> {
56
+ let mut journal_files = Vec::new();
57
+
58
+ for entry in WalkDir::new(journal_dir)
59
+ .follow_links(true)
60
+ .into_iter()
61
+ .filter_map(|e| e.ok())
62
+ {
63
+ let path = entry.path();
64
+ if path.is_file() && is_journal_file(path) {
65
+ journal_files.push(path.to_path_buf());
66
+ }
67
+ }
68
+
69
+ Ok(journal_files)
70
+}
71
+
72
+fn is_journal_file(path: &Path) -> bool {
73
+ // Check if file begins with 8 bytes "LPKSHHRH"
74
+ if let Ok(mut file) = std::fs::File::open(path) {
75
+ let mut buffer = [0u8; 8];
76
+ if let Ok(bytes_read) = file.read(&mut buffer) {
77
+ if bytes_read == 8 && buffer == *b"LPKSHHRH" {
78
+ return true;
79
+ }
80
+ }
81
+ }
82
+ false
83
+}
84
+
85
+fn select_random_entries(
86
+ journal_file: &JournalFile<Mmap>,
87
+ max_entries: usize,
88
+) -> Result<Vec<EntryData>> {
89
+ let mut rng = rand::rng();
90
+
91
+ // Get total number of entries in the journal
92
+ let total_entries = journal_file.journal_header_ref().n_entries as usize;
93
+ if total_entries == 0 {
94
+ return Ok(Vec::new());
95
+ }
96
+
97
+ // Determine how many entries to select (min of max_entries and total_entries)
98
+ let num_to_select = std::cmp::min(max_entries, total_entries);
99
+
100
+ // Create a reader to navigate the journal
101
+ let mut reader = JournalReader::default();
102
+ let mut entries = Vec::new();
103
+
104
+ // Generate random indices without repetition
105
+ let mut indices: Vec<usize> = (0..total_entries).collect();
106
+ indices.shuffle(&mut rng);
107
+ indices.truncate(num_to_select);
108
+
109
+ // Collect entries at the random indices
110
+ for idx in indices {
111
+ // Reset to head
112
+ reader.set_location(journal_reader::Location::Head);
113
+
114
+ // Skip forward to the random index
115
+ for _ in 0..idx {
116
+ if !reader.step(journal_file, journal_reader::Direction::Forward)? {
117
+ break;
118
+ }
119
+ }
120
+
121
+ // Get the entry at the current position
122
+ if let Ok(entry_offset) = reader.get_entry_offset() {
123
+ if let Ok(entry_data) = extract_entry_data(journal_file, entry_offset) {
124
+ entries.push(entry_data);
125
+ }
126
+ }
127
+ }
128
+
129
+ Ok(entries)
130
+}
131
+
132
+// Structure to hold entry data for forwarding
133
+struct EntryData {
134
+ fields: Vec<(String, String)>,
135
+}
136
+
137
+fn extract_entry_data(journal_file: &JournalFile<Mmap>, entry_offset: u64) -> Result<EntryData> {
138
+ let mut fields = Vec::new();
139
+
140
+ // Iterate through all data objects for this entry
141
+ for data_result in journal_file.entry_data_objects(entry_offset)? {
142
+ let data_object = data_result?;
143
+ let payload = data_object.payload_bytes();
144
+
145
+ // Find the first '=' character to split field and value
146
+ if let Some(equals_pos) = payload.iter().position(|&b| b == b'=') {
147
+ let field = String::from_utf8_lossy(&payload[0..equals_pos]).to_string();
148
+ let value = String::from_utf8_lossy(&payload[equals_pos + 1..]).to_string();
149
+
150
+ // Skip certain internal fields that shouldn't be forwarded
151
+ if field.starts_with("_") || field == "MESSAGE_ID" || field == "PRIORITY" {
152
+ continue;
153
+ }
154
+
155
+ fields.push((field, value));
156
+ }
157
+ }
158
+
159
+ Ok(EntryData { fields })
160
+}
161
+
162
+fn split_into_batches(entries: Vec<EntryData>, num_batches: usize) -> Vec<Vec<EntryData>> {
163
+ let mut batches = Vec::new();
164
+ let entries_per_batch = (entries.len() + num_batches - 1) / num_batches.max(1);
165
+
166
+ let mut entries_iter = entries.into_iter();
167
+
168
+ for _ in 0..num_batches {
169
+ let mut batch = Vec::new();
170
+ for _ in 0..entries_per_batch {
171
+ if let Some(entry) = entries_iter.next() {
172
+ batch.push(entry);
173
+ } else {
174
+ break;
175
+ }
176
+ }
177
+
178
+ if !batch.is_empty() {
179
+ batches.push(batch);
180
+ }
181
+ }
182
+
183
+ batches
184
+}
185
+
186
+fn forward_entries(entries: &[EntryData], logger: &mut JournalLogger) -> Result<()> {
187
+ // Add a special field to indicate this is a forwarded entry
188
+ for entry in entries {
189
+ // Add all fields from the original entry
190
+ for (key, value) in &entry.fields {
191
+ logger.add_field(key, value);
192
+ }
193
+
194
+ // Add our own marker field
195
+ logger.add_field("JOURNAL_FORWARDER", "1");
196
+
197
+ // Flush this entry to the journal
198
+ logger.flush().map_err(error::JournalError::Io)?;
199
+ }
200
+
201
+ Ok(())
202
+}
203
+
204
+fn main() -> Result<()> {
205
+ // Main loop
206
+ loop {
207
+ if let Err(e) = process_one_cycle() {
208
+ eprintln!("Error during cycle: {:?}", e);
209
+ }
210
+
211
+ // Wait for the next cycle
212
+ std::thread::sleep(Duration::from_secs(1));
213
+ }
214
+}
src/crates/jf/journal_logger/Cargo.toml
new
+8
@@ -0,0 +1,8 @@
1
+[package]
2
+name = "journal_logger"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+duct = { workspace = true }
8
+serde_json = { workspace = true }
src/crates/jf/journal_logger/src/lib.rs
new
+69
@@ -0,0 +1,69 @@
1
+use duct::cmd;
2
+use std::collections::HashMap;
3
+use std::io::{self, Error, ErrorKind};
4
+use std::path::Path;
5
+
6
+/// A builder for creating systemd journal entries with custom fields
7
+pub struct JournalLogger {
8
+ fields: HashMap<String, String>,
9
+ log2journal_path: String,
10
+ systemd_cat_path: String,
11
+}
12
+
13
+impl JournalLogger {
14
+ /// Create a new JournalLogger with paths to the required executables
15
+ pub fn new(log2journal_path: &str, systemd_cat_path: &str) -> Self {
16
+ JournalLogger {
17
+ fields: HashMap::new(),
18
+ log2journal_path: log2journal_path.to_string(),
19
+ systemd_cat_path: systemd_cat_path.to_string(),
20
+ }
21
+ }
22
+
23
+ /// Add a field to the journal entry
24
+ pub fn add_field(&mut self, key: &str, value: &str) -> &mut Self {
25
+ self.fields.insert(key.to_string(), value.to_string());
26
+ self
27
+ }
28
+
29
+ /// Flush the current fields to the journal and clear the fields
30
+ pub fn flush(&mut self) -> io::Result<()> {
31
+ // Verify that the required executables exist
32
+ if !Path::new(&self.log2journal_path).exists() {
33
+ return Err(Error::new(
34
+ ErrorKind::NotFound,
35
+ format!(
36
+ "log2journal executable not found at {}",
37
+ self.log2journal_path
38
+ ),
39
+ ));
40
+ }
41
+
42
+ if !Path::new(&self.systemd_cat_path).exists() {
43
+ return Err(Error::new(
44
+ ErrorKind::NotFound,
45
+ format!(
46
+ "systemd-cat executable not found at {}",
47
+ self.systemd_cat_path
48
+ ),
49
+ ));
50
+ }
51
+
52
+ // Create the JSON string
53
+ let json_data = serde_json::to_string(&self.fields)
54
+ .map_err(|e| Error::new(ErrorKind::InvalidData, e))?;
55
+
56
+ // Create the pipeline
57
+ let pipeline = cmd!("echo", json_data)
58
+ .pipe(cmd!(&self.log2journal_path, "json"))
59
+ .pipe(cmd!(&self.systemd_cat_path));
60
+
61
+ // Execute the pipeline
62
+ pipeline.run()?;
63
+
64
+ // Clear the fields for the next entry
65
+ self.fields.clear();
66
+
67
+ Ok(())
68
+ }
69
+}
src/crates/jf/journal_reader/Cargo.toml
new
+9
@@ -0,0 +1,9 @@
1
+[package]
2
+name = "journal_reader"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+journal_file = { path = "../journal_file" }
9
+window_manager = { path = "../window_manager" }
src/crates/jf/journal_reader/src/journal_cursor.rs
new
+214
@@ -0,0 +1,214 @@
1
+use crate::journal_filter::FilterExpr;
2
+use error::{JournalError, Result};
3
+use journal_file::{offset_array, offset_array::Direction, JournalFile};
4
+use window_manager::MemoryMap;
5
+
6
+#[derive(Debug, Copy, Clone, PartialEq, Eq)]
7
+pub enum Location {
8
+ Head,
9
+ Tail,
10
+ Realtime(u64),
11
+ Monotonic(u64, [u8; 16]),
12
+ Seqnum(u64, Option<[u8; 16]>),
13
+ XorHash(u64),
14
+ ResolvedEntry(u64),
15
+}
16
+
17
+impl Default for Location {
18
+ fn default() -> Self {
19
+ Self::Head
20
+ }
21
+}
22
+
23
+#[derive(Debug)]
24
+pub struct JournalCursor {
25
+ pub location: Location,
26
+ pub filter_expr: Option<FilterExpr>,
27
+ pub array_cursor: Option<offset_array::Cursor>,
28
+}
29
+
30
+impl JournalCursor {
31
+ #[allow(clippy::new_without_default)]
32
+ pub fn new() -> Self {
33
+ Self {
34
+ location: Location::Head,
35
+ filter_expr: None,
36
+ array_cursor: None,
37
+ }
38
+ }
39
+
40
+ pub fn set_location(&mut self, location: Location) {
41
+ self.location = location;
42
+ self.array_cursor = None;
43
+ }
44
+
45
+ pub fn set_filter(&mut self, filter_expr: FilterExpr) {
46
+ self.filter_expr = Some(filter_expr);
47
+ // FIXME: should we set cursor to None?
48
+ }
49
+
50
+ pub fn clear_filter(&mut self) {
51
+ self.filter_expr = None;
52
+ self.array_cursor = None;
53
+ self.set_location(Location::Head);
54
+ }
55
+
56
+ pub fn step<M: MemoryMap>(
57
+ &mut self,
58
+ journal_file: &JournalFile<M>,
59
+ direction: Direction,
60
+ ) -> Result<bool> {
61
+ let new_location = if self.filter_expr.is_some() {
62
+ self.resolve_filter_location(journal_file, direction)?
63
+ } else {
64
+ self.resolve_array_cursor(journal_file, direction)?
65
+ };
66
+
67
+ if let Some(location) = new_location {
68
+ self.location = location;
69
+ Ok(true)
70
+ } else {
71
+ Ok(false)
72
+ }
73
+ }
74
+
75
+ pub fn position(&self) -> Result<u64> {
76
+ match self.location {
77
+ Location::ResolvedEntry(entry_offset) => Ok(entry_offset),
78
+ _ => Err(JournalError::UnsetCursor),
79
+ }
80
+ }
81
+
82
+ fn resolve_array_cursor<M: MemoryMap>(
83
+ &mut self,
84
+ journal_file: &JournalFile<M>,
85
+ direction: Direction,
86
+ ) -> Result<Option<Location>> {
87
+ let new_location = match (self.location, direction) {
88
+ (Location::Head, Direction::Forward) => {
89
+ let entry_list = journal_file
90
+ .entry_list()
91
+ .ok_or(JournalError::InvalidOffsetArrayOffset)?;
92
+
93
+ let cursor = entry_list.cursor_head();
94
+ let offset = cursor.value(journal_file)?;
95
+ self.array_cursor = Some(cursor);
96
+ Some(Location::ResolvedEntry(offset))
97
+ }
98
+ (Location::Head, Direction::Backward) => None,
99
+ (Location::Tail, Direction::Forward) => None,
100
+ (Location::Tail, Direction::Backward) => {
101
+ let entry_list = journal_file
102
+ .entry_list()
103
+ .ok_or(JournalError::InvalidOffsetArrayOffset)?;
104
+
105
+ let cursor = entry_list.cursor_tail(journal_file)?;
106
+ let offset = cursor.value(journal_file)?;
107
+ self.array_cursor = Some(cursor);
108
+ Some(Location::ResolvedEntry(offset))
109
+ }
110
+ (Location::Realtime(realtime), _) => {
111
+ let entry_list = journal_file
112
+ .entry_list()
113
+ .ok_or(JournalError::InvalidOffsetArrayOffset)?;
114
+
115
+ let predicate = |entry_offset| {
116
+ let entry_object = journal_file.entry_ref(entry_offset)?;
117
+ Ok(entry_object.header.realtime < realtime)
118
+ };
119
+
120
+ let cursor = entry_list
121
+ .directed_partition_point(journal_file, predicate, Direction::Forward)?
122
+ .map(Ok)
123
+ .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
124
+
125
+ let offset = cursor.value(journal_file)?;
126
+ self.array_cursor = Some(cursor);
127
+ Some(Location::ResolvedEntry(offset))
128
+ }
129
+ (Location::ResolvedEntry(_), Direction::Forward) => {
130
+ let Some(cursor) = self.array_cursor.unwrap().next(journal_file)? else {
131
+ return Ok(None);
132
+ };
133
+
134
+ let offset = cursor.value(journal_file)?;
135
+ self.array_cursor = Some(cursor);
136
+ Some(Location::ResolvedEntry(offset))
137
+ }
138
+ (Location::ResolvedEntry(_), Direction::Backward) => {
139
+ let Some(cursor) = self.array_cursor.unwrap().previous(journal_file)? else {
140
+ return Ok(None);
141
+ };
142
+
143
+ let offset = cursor.value(journal_file)?;
144
+ self.array_cursor = Some(cursor);
145
+ Some(Location::ResolvedEntry(offset))
146
+ }
147
+ _ => {
148
+ unimplemented!()
149
+ }
150
+ };
151
+
152
+ Ok(new_location)
153
+ }
154
+
155
+ fn resolve_filter_location<M: MemoryMap>(
156
+ &mut self,
157
+ journal_file: &JournalFile<M>,
158
+ direction: Direction,
159
+ ) -> Result<Option<Location>> {
160
+ let filter_expr = self.filter_expr.as_mut().unwrap();
161
+
162
+ let resolved_location = match (self.location, direction) {
163
+ (Location::Head, Direction::Forward) => filter_expr
164
+ .head()
165
+ .next(journal_file, u64::MIN)?
166
+ .map(Location::ResolvedEntry),
167
+ (Location::Head, Direction::Backward) => None,
168
+ (Location::Tail, Direction::Forward) => None,
169
+ (Location::Tail, Direction::Backward) => filter_expr
170
+ .tail(journal_file)?
171
+ .previous(journal_file, u64::MAX)?
172
+ .map(Location::ResolvedEntry),
173
+ (Location::Realtime(realtime), direction) => {
174
+ let entry_list = journal_file
175
+ .entry_list()
176
+ .ok_or(JournalError::InvalidOffsetArrayOffset)?;
177
+
178
+ let predicate = |entry_offset| {
179
+ let entry_object = journal_file.entry_ref(entry_offset)?;
180
+ Ok(entry_object.header.realtime < realtime)
181
+ };
182
+
183
+ let cursor = entry_list
184
+ .directed_partition_point(journal_file, predicate, Direction::Forward)?
185
+ .map(Ok)
186
+ .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
187
+
188
+ let entry_offset = cursor.value(journal_file)?;
189
+
190
+ match direction {
191
+ Direction::Forward => filter_expr
192
+ .head()
193
+ .next(journal_file, entry_offset)?
194
+ .map(Location::ResolvedEntry),
195
+ Direction::Backward => filter_expr
196
+ .tail(journal_file)?
197
+ .previous(journal_file, entry_offset)?
198
+ .map(Location::ResolvedEntry),
199
+ }
200
+ }
201
+ (Location::ResolvedEntry(location_offset), Direction::Forward) => filter_expr
202
+ .next(journal_file, location_offset + 1)?
203
+ .map(Location::ResolvedEntry),
204
+ (Location::ResolvedEntry(location_offset), Direction::Backward) => filter_expr
205
+ .previous(journal_file, location_offset - 1)?
206
+ .map(Location::ResolvedEntry),
207
+ _ => {
208
+ unimplemented!();
209
+ }
210
+ };
211
+
212
+ Ok(resolved_location)
213
+ }
214
+}
src/crates/jf/journal_reader/src/journal_filter.rs
new
+416
@@ -0,0 +1,416 @@
1
+use error::{JournalError, Result};
2
+use journal_file::{
3
+ offset_array::{Direction, InlinedCursor},
4
+ JournalFile,
5
+};
6
+use window_manager::MemoryMap;
7
+
8
+#[derive(Clone, Debug)]
9
+pub enum FilterExpr {
10
+ Match(u64, Option<InlinedCursor>),
11
+ Conjunction(Vec<FilterExpr>),
12
+ Disjunction(Vec<FilterExpr>),
13
+}
14
+
15
+impl FilterExpr {
16
+ pub fn lookup<M: MemoryMap>(
17
+ &self,
18
+ journal_file: &JournalFile<M>,
19
+ needle_offset: u64,
20
+ direction: Direction,
21
+ ) -> Result<Option<u64>> {
22
+ let predicate =
23
+ move |entry_offset: u64| -> Result<bool> { Ok(entry_offset < needle_offset) };
24
+
25
+ match self {
26
+ FilterExpr::Match(data_offset, _) => {
27
+ let entry_offset = journal_file.data_object_directed_partition_point(
28
+ *data_offset,
29
+ predicate,
30
+ direction,
31
+ )?;
32
+ Ok(entry_offset)
33
+ }
34
+ FilterExpr::Conjunction(filter_exprs) => {
35
+ let mut current_offset = needle_offset;
36
+
37
+ loop {
38
+ let previous_offset = current_offset;
39
+
40
+ for filter_expr in filter_exprs {
41
+ if direction == Direction::Backward {
42
+ current_offset = current_offset.saturating_add(1);
43
+ }
44
+
45
+ match filter_expr.lookup(journal_file, current_offset, direction)? {
46
+ Some(new_offset) => current_offset = new_offset,
47
+ None => return Ok(None),
48
+ }
49
+ }
50
+
51
+ if current_offset == previous_offset {
52
+ return Ok(Some(current_offset));
53
+ }
54
+ }
55
+ }
56
+ FilterExpr::Disjunction(filter_exprs) => {
57
+ let cmp = match direction {
58
+ Direction::Forward => std::cmp::min,
59
+ Direction::Backward => std::cmp::max,
60
+ };
61
+
62
+ filter_exprs.iter().try_fold(None, |acc, expr| {
63
+ let result = expr.lookup(journal_file, needle_offset, direction)?;
64
+
65
+ Ok(match (acc, result) {
66
+ (None, Some(offset)) => Some(offset),
67
+ (Some(best), Some(offset)) => Some(cmp(best, offset)),
68
+ (acc, None) => acc,
69
+ })
70
+ })
71
+ }
72
+ }
73
+ }
74
+
75
+ pub fn head(&mut self) -> &mut Self {
76
+ match self {
77
+ FilterExpr::Match(_, None) => (),
78
+ FilterExpr::Match(_, Some(ic)) => {
79
+ *ic = ic.head();
80
+ }
81
+ FilterExpr::Conjunction(filter_exprs) => {
82
+ for filter_expr in filter_exprs.iter_mut() {
83
+ filter_expr.head();
84
+ }
85
+ }
86
+ FilterExpr::Disjunction(filter_exprs) => {
87
+ for filter_expr in filter_exprs.iter_mut() {
88
+ filter_expr.head();
89
+ }
90
+ }
91
+ }
92
+
93
+ self
94
+ }
95
+
96
+ pub fn tail<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<&mut Self> {
97
+ match self {
98
+ FilterExpr::Match(_, None) => (),
99
+ FilterExpr::Match(_, Some(ic)) => {
100
+ *ic = ic.tail(journal_file)?;
101
+ }
102
+ FilterExpr::Conjunction(filter_exprs) => {
103
+ for filter_expr in filter_exprs.iter_mut() {
104
+ filter_expr.tail(journal_file)?;
105
+ }
106
+ }
107
+ FilterExpr::Disjunction(filter_exprs) => {
108
+ for filter_expr in filter_exprs.iter_mut() {
109
+ filter_expr.tail(journal_file)?;
110
+ }
111
+ }
112
+ }
113
+
114
+ Ok(self)
115
+ }
116
+
117
+ // Returns the offset of the next matching entry, if any, with an offset
118
+ // greater or equal to the needle offset.
119
+ pub fn next<M: MemoryMap>(
120
+ &mut self,
121
+ journal_file: &JournalFile<M>,
122
+ needle_offset: u64,
123
+ ) -> Result<Option<u64>> {
124
+ match self {
125
+ FilterExpr::Match(_, None) => Ok(None),
126
+ FilterExpr::Match(_, Some(ic)) => ic.next_until(journal_file, needle_offset),
127
+ FilterExpr::Conjunction(filter_exprs) => {
128
+ let mut needle_offset = needle_offset;
129
+
130
+ loop {
131
+ let previous_offset = needle_offset;
132
+
133
+ for fe in filter_exprs.iter_mut() {
134
+ if let Some(new_offset) = fe.next(journal_file, needle_offset)? {
135
+ needle_offset = new_offset;
136
+ } else {
137
+ return Ok(None);
138
+ }
139
+ }
140
+
141
+ if needle_offset == previous_offset {
142
+ return Ok(Some(needle_offset));
143
+ }
144
+ }
145
+ }
146
+ FilterExpr::Disjunction(filter_exprs) => {
147
+ let mut best_offset: Option<u64> = None;
148
+
149
+ for fe in filter_exprs.iter_mut() {
150
+ if let Some(fe_offset) = fe.next(journal_file, needle_offset)? {
151
+ best_offset = match best_offset {
152
+ Some(offset) => Some(fe_offset.min(offset)),
153
+ None => Some(fe_offset),
154
+ };
155
+ }
156
+ }
157
+
158
+ Ok(best_offset)
159
+ }
160
+ }
161
+ }
162
+
163
+ // Returns the offset of the previous matching entry, if any, with an offset
164
+ // less or equal to the needle offset.
165
+ pub fn previous<M: MemoryMap>(
166
+ &mut self,
167
+ journal_file: &JournalFile<M>,
168
+ needle_offset: u64,
169
+ ) -> Result<Option<u64>> {
170
+ match self {
171
+ FilterExpr::Match(_, None) => Ok(None),
172
+ FilterExpr::Match(_, Some(ic)) => ic.previous_until(journal_file, needle_offset),
173
+ FilterExpr::Conjunction(filter_exprs) => {
174
+ let mut needle_offset = needle_offset;
175
+
176
+ loop {
177
+ let previous_offset = needle_offset;
178
+
179
+ for fe in filter_exprs.iter_mut().rev() {
180
+ if let Some(new_offset) = fe.previous(journal_file, needle_offset)? {
181
+ needle_offset = new_offset;
182
+ } else {
183
+ return Ok(None);
184
+ }
185
+ }
186
+
187
+ if needle_offset == previous_offset {
188
+ return Ok(Some(needle_offset));
189
+ }
190
+ }
191
+ }
192
+ FilterExpr::Disjunction(filter_exprs) => {
193
+ let mut best_offset: Option<u64> = None;
194
+
195
+ for fe in filter_exprs.iter_mut() {
196
+ if let Some(fe_offset) = fe.previous(journal_file, needle_offset)? {
197
+ best_offset = match best_offset {
198
+ Some(offset) => Some(fe_offset.max(offset)),
199
+ None => Some(fe_offset),
200
+ };
201
+ }
202
+ }
203
+
204
+ Ok(best_offset)
205
+ }
206
+ }
207
+ }
208
+
209
+ pub fn dump<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<String> {
210
+ let mut output = String::new();
211
+ self.dump_internal(journal_file, 0, &mut output)?;
212
+ Ok(output)
213
+ }
214
+
215
+ /// Helper function for format_data_objects that handles nested expressions and indentation
216
+ fn dump_internal<M: MemoryMap>(
217
+ &self,
218
+ journal_file: &JournalFile<M>,
219
+ indent_level: usize,
220
+ output: &mut String,
221
+ ) -> Result<()> {
222
+ let indent = " ".repeat(indent_level);
223
+
224
+ match self {
225
+ FilterExpr::Match(data_offset, inlined_cursor) => {
226
+ // Load the data object
227
+ let data_object = journal_file.data_ref(*data_offset)?;
228
+
229
+ // Get the payload as a string if possible
230
+ let payload_bytes = data_object.payload_bytes();
231
+ let payload_str = String::from_utf8_lossy(payload_bytes);
232
+
233
+ // Format the data offset and payload
234
+ output.push_str(&format!(
235
+ "{}Match[offset=0x{:x}]: {}\n",
236
+ indent, data_offset, payload_str
237
+ ));
238
+
239
+ // Format cursor information if available
240
+ if let Some(ic) = inlined_cursor {
241
+ output.push_str(&format!("{} Cursor: {:?}\n", indent, ic));
242
+ }
243
+ }
244
+ FilterExpr::Conjunction(filter_exprs) => {
245
+ output.push_str(&format!("{}Conjunction (AND) {{\n", indent));
246
+ for expr in filter_exprs {
247
+ expr.dump_internal(journal_file, indent_level + 1, output)?;
248
+ }
249
+ output.push_str(&format!("{}}}\n", indent));
250
+ }
251
+ FilterExpr::Disjunction(filter_exprs) => {
252
+ output.push_str(&format!("{}Disjunction (OR) {{\n", indent));
253
+ for expr in filter_exprs {
254
+ expr.dump_internal(journal_file, indent_level + 1, output)?;
255
+ }
256
+ output.push_str(&format!("{}}}\n", indent));
257
+ }
258
+ }
259
+
260
+ Ok(())
261
+ }
262
+}
263
+
264
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
265
+pub enum LogicalOp {
266
+ Conjunction,
267
+ Disjunction,
268
+}
269
+
270
+#[derive(Debug)]
271
+pub struct JournalFilter {
272
+ filter_expr: Option<FilterExpr>,
273
+ current_matches: Vec<Vec<u8>>,
274
+ current_op: LogicalOp,
275
+}
276
+
277
+impl Default for JournalFilter {
278
+ fn default() -> Self {
279
+ Self {
280
+ filter_expr: None,
281
+ current_matches: Vec::new(),
282
+ current_op: LogicalOp::Conjunction,
283
+ }
284
+ }
285
+}
286
+
287
+impl JournalFilter {
288
+ fn extract_key(kv_pair: &[u8]) -> Option<&[u8]> {
289
+ if let Some(equal_pos) = kv_pair.iter().position(|&b| b == b'=') {
290
+ Some(&kv_pair[..equal_pos])
291
+ } else {
292
+ None
293
+ }
294
+ }
295
+
296
+ fn convert_current_matches<M: MemoryMap>(
297
+ &mut self,
298
+ journal_file: &JournalFile<M>,
299
+ ) -> Result<Option<FilterExpr>> {
300
+ if self.current_matches.is_empty() {
301
+ return Ok(None);
302
+ }
303
+
304
+ let mut elements = Vec::new();
305
+ let mut i = 0;
306
+
307
+ while i < self.current_matches.len() {
308
+ let current_key = Self::extract_key(&self.current_matches[i]).unwrap_or(&[]);
309
+ let start = i;
310
+
311
+ // Find all matches with the same key
312
+ while i < self.current_matches.len()
313
+ && Self::extract_key(&self.current_matches[i]).unwrap_or(&[]) == current_key
314
+ {
315
+ i += 1;
316
+ }
317
+
318
+ // If we have multiple values for this key, create a disjunction
319
+ if i - start > 1 {
320
+ let mut matches = Vec::with_capacity(i - start);
321
+ for idx in start..i {
322
+ let data = self.current_matches[idx].as_slice();
323
+ let hash = journal_file.hash(data);
324
+ let offset = journal_file.find_data_offset_by_payload(data, hash)?;
325
+
326
+ let ic = journal_file.data_ref(offset)?.inlined_cursor();
327
+ matches.push(FilterExpr::Match(offset, ic));
328
+ }
329
+ elements.push(FilterExpr::Disjunction(matches));
330
+ } else {
331
+ let data = self.current_matches[start].as_slice();
332
+ let hash = journal_file.hash(data);
333
+ let offset = journal_file.find_data_offset_by_payload(data, hash)?;
334
+
335
+ let ic = journal_file.data_ref(offset)?.inlined_cursor();
336
+ elements.push(FilterExpr::Match(offset, ic));
337
+ }
338
+ }
339
+
340
+ self.current_matches.clear();
341
+
342
+ match elements.len() {
343
+ 0 => panic!("Could not create filter elements from current matches"),
344
+ 1 => Ok(Some(elements.remove(0))),
345
+ _ => Ok(Some(FilterExpr::Conjunction(elements))),
346
+ }
347
+ }
348
+
349
+ pub fn add_match(&mut self, kv_pair: &[u8]) {
350
+ if kv_pair.contains(&b'=') {
351
+ let new_item = kv_pair.to_vec();
352
+ let new_key = Self::extract_key(&new_item).unwrap_or(&[]);
353
+
354
+ // Find the insertion position using binary search
355
+ let pos = self
356
+ .current_matches
357
+ .binary_search_by(|item| {
358
+ let key = Self::extract_key(item).unwrap_or(&[]);
359
+ key.cmp(new_key)
360
+ })
361
+ .unwrap_or_else(|e| e);
362
+
363
+ // Insert at the found position
364
+ self.current_matches.insert(pos, new_item);
365
+ }
366
+ }
367
+
368
+ pub fn set_operation<M: MemoryMap>(
369
+ &mut self,
370
+ journal_file: &JournalFile<M>,
371
+ op: LogicalOp,
372
+ ) -> Result<()> {
373
+ let new_expr = self.convert_current_matches(journal_file)?;
374
+ if new_expr.is_none() {
375
+ self.current_op = op;
376
+ return Ok(());
377
+ }
378
+
379
+ if self.filter_expr.is_none() {
380
+ self.filter_expr = new_expr;
381
+ self.current_op = op;
382
+ return Ok(());
383
+ }
384
+
385
+ let new_expr = new_expr.unwrap();
386
+ let current_expr = self.filter_expr.take().unwrap();
387
+
388
+ self.filter_expr = Some(match (current_expr, self.current_op) {
389
+ (FilterExpr::Disjunction(mut exprs), LogicalOp::Disjunction) => {
390
+ exprs.push(new_expr);
391
+ FilterExpr::Disjunction(exprs)
392
+ }
393
+ (FilterExpr::Conjunction(mut exprs), LogicalOp::Conjunction) => {
394
+ exprs.push(new_expr);
395
+ FilterExpr::Conjunction(exprs)
396
+ }
397
+ (current_expr, LogicalOp::Disjunction) => {
398
+ FilterExpr::Disjunction(vec![current_expr, new_expr])
399
+ }
400
+ (current_expr, LogicalOp::Conjunction) => {
401
+ FilterExpr::Conjunction(vec![current_expr, new_expr])
402
+ }
403
+ });
404
+
405
+ self.current_op = op;
406
+ Ok(())
407
+ }
408
+
409
+ pub fn build<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<FilterExpr> {
410
+ self.set_operation(journal_file, self.current_op)?;
411
+
412
+ self.current_matches.clear();
413
+ self.current_op = LogicalOp::Conjunction;
414
+ self.filter_expr.take().ok_or(JournalError::MalformedFilter)
415
+ }
416
+}
src/crates/jf/journal_reader/src/lib.rs
new
+197
@@ -0,0 +1,197 @@
1
+use error::Result;
2
+use journal_file::{
3
+ offset_array, DataObject, EntryDataIterator, FieldDataIterator, FieldIterator, FieldObject,
4
+ JournalFile, ValueGuard,
5
+};
6
+use window_manager::MemoryMap;
7
+
8
+pub mod journal_filter;
9
+use journal_filter::{JournalFilter, LogicalOp};
10
+pub mod journal_cursor;
11
+use journal_cursor::JournalCursor;
12
+pub use journal_cursor::Location;
13
+
14
+pub use offset_array::Direction;
15
+
16
+pub struct JournalReader<'a, M: MemoryMap> {
17
+ cursor: JournalCursor,
18
+
19
+ filter: Option<JournalFilter>,
20
+ field_iterator: Option<FieldIterator<'a, M>>,
21
+ field_data_iterator: Option<FieldDataIterator<'a, M>>,
22
+ entry_data_iterator: Option<EntryDataIterator<'a, M>>,
23
+
24
+ field_guard: Option<ValueGuard<'a, FieldObject<&'a [u8]>>>,
25
+ data_guard: Option<ValueGuard<'a, DataObject<&'a [u8]>>>,
26
+}
27
+
28
+impl<M: MemoryMap> std::fmt::Debug for JournalReader<'_, M> {
29
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
30
+ f.debug_struct("JournalReader")
31
+ // .field("cursor", &self.cursor)
32
+ .field("field_guard", &self.field_guard)
33
+ .field("data_guard", &self.data_guard)
34
+ .finish()
35
+ }
36
+}
37
+
38
+impl<M: MemoryMap> Default for JournalReader<'_, M> {
39
+ fn default() -> Self {
40
+ Self {
41
+ cursor: JournalCursor::new(),
42
+ filter: None,
43
+ field_iterator: None,
44
+ field_data_iterator: None,
45
+ entry_data_iterator: None,
46
+ field_guard: None,
47
+ data_guard: None,
48
+ }
49
+ }
50
+}
51
+
52
+impl<'a, M: MemoryMap> JournalReader<'a, M> {
53
+ pub fn dump(&self, journal_file: &'a JournalFile<M>) -> Result<String> {
54
+ if let Some(filter_expr) = self.cursor.filter_expr.as_ref() {
55
+ filter_expr.dump(journal_file)
56
+ } else {
57
+ Ok(String::from("no filter expr"))
58
+ }
59
+ }
60
+
61
+ pub fn set_location(&mut self, location: Location) {
62
+ self.cursor.set_location(location)
63
+ }
64
+
65
+ pub fn step(&mut self, journal_file: &'a JournalFile<M>, direction: Direction) -> Result<bool> {
66
+ self.drop_guards();
67
+
68
+ if let Some(filter) = self.filter.as_mut() {
69
+ let filter_expr = filter.build(journal_file)?;
70
+ self.cursor.set_filter(filter_expr);
71
+ self.filter = None;
72
+ }
73
+
74
+ self.cursor.step(journal_file, direction)
75
+ }
76
+
77
+ pub fn add_match(&mut self, data: &[u8]) {
78
+ self.filter.get_or_insert_default().add_match(data);
79
+ }
80
+
81
+ pub fn add_conjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
82
+ self.filter
83
+ .get_or_insert_default()
84
+ .set_operation(journal_file, LogicalOp::Conjunction)
85
+ }
86
+
87
+ pub fn add_disjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
88
+ self.filter
89
+ .get_or_insert_default()
90
+ .set_operation(journal_file, LogicalOp::Disjunction)
91
+ }
92
+
93
+ pub fn flush_matches(&mut self) {
94
+ self.cursor.clear_filter();
95
+ self.filter = None;
96
+ }
97
+
98
+ pub fn get_realtime_usec(&self, journal_file: &'a JournalFile<M>) -> Result<u64> {
99
+ let entry_offset = self.cursor.position()?;
100
+ let entry_object = journal_file.entry_ref(entry_offset)?;
101
+ Ok(entry_object.header.realtime)
102
+ }
103
+
104
+ pub fn get_seqnum(&self, journal_file: &'a JournalFile<M>) -> Result<(u64, [u8; 16])> {
105
+ let entry_offset = self.cursor.position()?;
106
+ let entry_object = journal_file.entry_ref(entry_offset)?;
107
+ Ok((
108
+ entry_object.header.seqnum,
109
+ journal_file.journal_header_ref().seqnum_id,
110
+ ))
111
+ }
112
+
113
+ pub fn get_entry_offset(&self) -> Result<u64> {
114
+ self.cursor.position()
115
+ }
116
+
117
+ fn drop_guards(&mut self) {
118
+ self.field_guard.take();
119
+ self.data_guard.take();
120
+ }
121
+
122
+ pub fn fields_restart(&mut self) {
123
+ self.drop_guards();
124
+ self.field_iterator = None;
125
+ }
126
+
127
+ pub fn fields_enumerate(
128
+ &mut self,
129
+ journal_file: &'a JournalFile<M>,
130
+ ) -> Result<Option<&ValueGuard<FieldObject<&'a [u8]>>>> {
131
+ self.drop_guards();
132
+
133
+ if self.field_iterator.is_none() {
134
+ self.field_iterator = Some(journal_file.fields());
135
+ }
136
+
137
+ if let Some(iter) = &mut self.field_iterator {
138
+ self.field_guard = iter.next().transpose()?;
139
+ Ok(self.field_guard.as_ref())
140
+ } else {
141
+ Ok(None)
142
+ }
143
+ }
144
+
145
+ pub fn field_data_query_unique(
146
+ &mut self,
147
+ journal_file: &'a JournalFile<M>,
148
+ field_name: &'a [u8],
149
+ ) -> Result<()> {
150
+ self.drop_guards();
151
+
152
+ self.field_data_iterator = Some(journal_file.field_data_objects(field_name)?);
153
+ Ok(())
154
+ }
155
+
156
+ pub fn field_data_restart(&mut self) {
157
+ self.drop_guards();
158
+ }
159
+
160
+ pub fn field_data_enumerate(
161
+ &mut self,
162
+ _: &'a JournalFile<M>,
163
+ ) -> Result<Option<&ValueGuard<DataObject<&'a [u8]>>>> {
164
+ self.drop_guards();
165
+
166
+ if let Some(iter) = &mut self.field_data_iterator {
167
+ self.data_guard = iter.next().transpose()?;
168
+ Ok(self.data_guard.as_ref())
169
+ } else {
170
+ Ok(None)
171
+ }
172
+ }
173
+
174
+ pub fn entry_data_restart(&mut self) {
175
+ self.drop_guards();
176
+ self.entry_data_iterator = None;
177
+ }
178
+
179
+ pub fn entry_data_enumerate(
180
+ &mut self,
181
+ journal_file: &'a JournalFile<M>,
182
+ ) -> Result<Option<&ValueGuard<DataObject<&'a [u8]>>>> {
183
+ self.drop_guards();
184
+
185
+ if self.entry_data_iterator.is_none() {
186
+ let entry_offset = self.cursor.position()?;
187
+ self.entry_data_iterator = Some(journal_file.entry_data_objects(entry_offset)?);
188
+ }
189
+
190
+ if let Some(iter) = &mut self.entry_data_iterator {
191
+ self.data_guard = iter.next().transpose()?;
192
+ Ok(self.data_guard.as_ref())
193
+ } else {
194
+ Ok(None)
195
+ }
196
+ }
197
+}
src/crates/jf/journal_reader_ffi/Cargo.toml
new
+18
@@ -0,0 +1,18 @@
1
+[package]
2
+name = "journal_reader_ffi"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+journal_reader = { path = "../journal_reader" }
9
+journal_file = { path = "../journal_file" }
10
+memmap2 = { workspace = true }
11
+serde_json = { workspace = true }
12
+sigbus = { path = "../sigbus" }
13
+
14
+[build-dependencies]
15
+cbindgen = "0.28.0"
16
+
17
+[lib]
18
+crate-type = ["staticlib"]
src/crates/jf/journal_reader_ffi/build.rs
new
+21
@@ -0,0 +1,21 @@
1
+extern crate cbindgen;
2
+
3
+use std::env;
4
+
5
+fn main() {
6
+ let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
7
+
8
+ cbindgen::Builder::new()
9
+ .with_crate(crate_dir)
10
+ .with_language(cbindgen::Language::C)
11
+ .with_cpp_compat(true)
12
+ .with_include_guard("JOURNAL_READER_FFI_H")
13
+ .rename_item("SdJournal", "sd_journal")
14
+ .rename_item("SdId128", "sd_id128_t")
15
+ .generate()
16
+ .expect("Unable to generate bindings")
17
+ .write_to_file("journal_reader_ffi.h");
18
+
19
+ println!("cargo:rerun-if-changed=cbindgen.toml");
20
+ println!("cargo:rerun-if-changed=src/");
21
+}
src/crates/jf/journal_reader_ffi/src/lib.rs
new
+449
@@ -0,0 +1,449 @@
1
+use journal_file::{HashableObject, JournalFile};
2
+use journal_reader::{Direction, JournalReader, Location};
3
+use memmap2::Mmap;
4
+use std::ffi::{c_char, c_int, c_void, CStr};
5
+
6
+#[repr(C)]
7
+#[derive(Debug, Clone, Copy)]
8
+pub struct RsdId128 {
9
+ pub bytes: [u8; 16],
10
+}
11
+
12
+fn unhexchar(c: u8) -> Result<u8, i32> {
13
+ match c {
14
+ b'0'..=b'9' => Ok(c - b'0'),
15
+ b'a'..=b'f' => Ok(c - b'a' + 10),
16
+ b'A'..=b'F' => Ok(c - b'A' + 10),
17
+ _ => Err(-22), // -EINVAL
18
+ }
19
+}
20
+
21
+#[no_mangle]
22
+unsafe extern "C" fn rsd_id128_from_string(s: *const c_char, ret: *mut RsdId128) -> i32 {
23
+ debug_assert!(!s.is_null());
24
+ debug_assert!(!ret.is_null());
25
+
26
+ let c_str = match CStr::from_ptr(s).to_str() {
27
+ Ok(s) => s,
28
+ Err(_) => return -1,
29
+ };
30
+
31
+ let res = &mut *ret;
32
+ let mut n: usize = 0;
33
+ let mut i: usize = 0;
34
+ let mut is_guid = false;
35
+
36
+ let bytes = c_str.as_bytes();
37
+
38
+ while n < 16 {
39
+ if i >= bytes.len() {
40
+ return -1;
41
+ }
42
+
43
+ if bytes[i] == b'-' {
44
+ if i == 8 {
45
+ is_guid = true;
46
+ } else if i == 13 || i == 18 || i == 23 {
47
+ if !is_guid {
48
+ return -1;
49
+ }
50
+ } else {
51
+ return -1;
52
+ }
53
+
54
+ i += 1;
55
+ continue;
56
+ }
57
+
58
+ if i + 1 >= bytes.len() {
59
+ return -1;
60
+ }
61
+
62
+ let a = match unhexchar(bytes[i]) {
63
+ Ok(val) => val,
64
+ Err(e) => return e,
65
+ };
66
+ i += 1;
67
+
68
+ let b = match unhexchar(bytes[i]) {
69
+ Ok(val) => val,
70
+ Err(e) => return e,
71
+ };
72
+ i += 1;
73
+
74
+ res.bytes[n] = (a << 4) | b;
75
+ n += 1;
76
+ }
77
+
78
+ let expected_len = if is_guid { 36 } else { 32 };
79
+ if i != expected_len || i >= bytes.len() || bytes[i] != 0 {
80
+ return -1;
81
+ }
82
+
83
+ 0
84
+}
85
+
86
+#[no_mangle]
87
+pub extern "C" fn rsd_id128_equal(a: RsdId128, b: RsdId128) -> i32 {
88
+ (a.bytes == b.bytes) as i32
89
+}
90
+
91
+impl PartialEq for RsdId128 {
92
+ fn eq(&self, other: &Self) -> bool {
93
+ self.bytes == other.bytes
94
+ }
95
+}
96
+
97
+impl Eq for RsdId128 {}
98
+
99
+struct RsdJournal<'a> {
100
+ journal_file: Box<JournalFile<Mmap>>,
101
+ reader: JournalReader<'a, Mmap>,
102
+ field_buffer: Vec<u8>,
103
+ decompressed_payload: Vec<u8>,
104
+}
105
+
106
+#[no_mangle]
107
+unsafe extern "C" fn rsd_journal_open_files(
108
+ ret: *mut *mut RsdJournal,
109
+ paths: *const *const c_char,
110
+ _flags: c_int,
111
+) -> c_int {
112
+ debug_assert!(!ret.is_null());
113
+ debug_assert!(!paths.is_null());
114
+
115
+ if sigbus::install_handler().is_err() {
116
+ eprintln!("Failed to install sigbus handler");
117
+ }
118
+
119
+ // Get the first path
120
+ let path_ptr = *paths;
121
+ if path_ptr.is_null() {
122
+ return error::JournalError::InvalidFfiOp.to_error_code();
123
+ }
124
+
125
+ // Convert C string to Rust string
126
+ let path = match CStr::from_ptr(path_ptr).to_str() {
127
+ Ok(s) => s,
128
+ Err(_) => {
129
+ return error::JournalError::InvalidFfiOp.to_error_code();
130
+ }
131
+ };
132
+
133
+ // Create the ObjectFile
134
+ let window_size = 512 * 1024 * 1024;
135
+ let journal_file = match JournalFile::<Mmap>::open(path, window_size) {
136
+ Ok(f) => Box::new(f),
137
+ Err(e) => {
138
+ return e.to_error_code();
139
+ }
140
+ };
141
+
142
+ let journal = Box::new(RsdJournal {
143
+ reader: JournalReader::default(),
144
+ journal_file,
145
+ field_buffer: Vec::with_capacity(256),
146
+ decompressed_payload: Vec::new(),
147
+ });
148
+
149
+ // Pass ownership to the caller
150
+ *ret = Box::into_raw(journal);
151
+
152
+ 0
153
+}
154
+
155
+#[no_mangle]
156
+unsafe extern "C" fn rsd_journal_close(j: *mut RsdJournal) {
157
+ debug_assert!(!j.is_null());
158
+ let _ = Box::from_raw(j);
159
+}
160
+
161
+#[no_mangle]
162
+unsafe extern "C" fn rsd_journal_seek_head(j: *mut RsdJournal) -> c_int {
163
+ debug_assert!(!j.is_null());
164
+ let journal = &mut *j;
165
+ journal.reader.set_location(Location::Head);
166
+ 0
167
+}
168
+
169
+#[no_mangle]
170
+unsafe extern "C" fn rsd_journal_seek_tail(j: *mut RsdJournal) -> c_int {
171
+ debug_assert!(!j.is_null());
172
+ let journal = &mut *j;
173
+ journal.reader.set_location(Location::Tail);
174
+ 0
175
+}
176
+
177
+#[no_mangle]
178
+unsafe extern "C" fn rsd_journal_seek_realtime_usec(j: *mut RsdJournal, usec: u64) -> c_int {
179
+ debug_assert!(!j.is_null());
180
+ let journal = &mut *j;
181
+ journal.reader.set_location(Location::Realtime(usec));
182
+ 0
183
+}
184
+
185
+#[no_mangle]
186
+unsafe extern "C" fn rsd_journal_next(j: *mut RsdJournal) -> c_int {
187
+ debug_assert!(!j.is_null());
188
+ let journal = &mut *j;
189
+
190
+ match journal
191
+ .reader
192
+ .step(&journal.journal_file, Direction::Forward)
193
+ {
194
+ Ok(has_entry) => {
195
+ if has_entry {
196
+ 1
197
+ } else {
198
+ 0
199
+ }
200
+ }
201
+ Err(e) => e.to_error_code(),
202
+ }
203
+}
204
+
205
+#[no_mangle]
206
+unsafe extern "C" fn rsd_journal_previous(j: *mut RsdJournal) -> c_int {
207
+ debug_assert!(!j.is_null());
208
+ let journal = &mut *j;
209
+
210
+ match journal
211
+ .reader
212
+ .step(&journal.journal_file, Direction::Backward)
213
+ {
214
+ Ok(has_entry) => {
215
+ if has_entry {
216
+ 1
217
+ } else {
218
+ 0
219
+ }
220
+ }
221
+ Err(e) => e.to_error_code(),
222
+ }
223
+}
224
+
225
+#[no_mangle]
226
+unsafe extern "C" fn rsd_journal_get_seqnum(
227
+ j: *mut RsdJournal,
228
+ ret_seqnum: *mut u64,
229
+ ret_seqnum_id: *mut RsdId128,
230
+) -> c_int {
231
+ debug_assert!(!j.is_null());
232
+ debug_assert!(!ret_seqnum.is_null());
233
+ debug_assert!(!ret_seqnum_id.is_null());
234
+
235
+ let journal = &mut *j;
236
+ match journal.reader.get_seqnum(&journal.journal_file) {
237
+ Ok((seqnum, boot_id)) => {
238
+ *ret_seqnum = seqnum;
239
+
240
+ if !ret_seqnum_id.is_null() {
241
+ *ret_seqnum_id = RsdId128 { bytes: boot_id };
242
+ }
243
+
244
+ 0
245
+ }
246
+ Err(e) => e.to_error_code(),
247
+ }
248
+}
249
+
250
+#[no_mangle]
251
+unsafe extern "C" fn rsd_journal_get_realtime_usec(j: *mut RsdJournal, ret: *mut u64) -> c_int {
252
+ debug_assert!(!j.is_null());
253
+ debug_assert!(!ret.is_null());
254
+
255
+ let journal = &mut *j;
256
+
257
+ match journal.reader.get_realtime_usec(&journal.journal_file) {
258
+ Ok(realtime) => {
259
+ *ret = realtime;
260
+ 0
261
+ }
262
+ Err(e) => e.to_error_code(),
263
+ }
264
+}
265
+
266
+#[no_mangle]
267
+unsafe extern "C" fn rsd_journal_restart_data(j: *mut RsdJournal) {
268
+ debug_assert!(!j.is_null());
269
+
270
+ let journal = &mut *j;
271
+ journal.reader.entry_data_restart();
272
+}
273
+
274
+#[no_mangle]
275
+unsafe extern "C" fn rsd_journal_enumerate_available_data(
276
+ j: *mut RsdJournal,
277
+ data: *mut *const c_void,
278
+ l: *mut usize,
279
+) -> c_int {
280
+ debug_assert!(!j.is_null());
281
+ debug_assert!(!data.is_null());
282
+ debug_assert!(!l.is_null());
283
+
284
+ let journal = &mut *j;
285
+
286
+ match journal.reader.entry_data_enumerate(&journal.journal_file) {
287
+ Ok(Some(data_guard)) => {
288
+ if data_guard.is_compressed() {
289
+ return match data_guard.decompress(&mut journal.decompressed_payload) {
290
+ Ok(n) => {
291
+ *l = n;
292
+ *data = journal.decompressed_payload.as_ptr() as *const c_void;
293
+ 1
294
+ }
295
+ Err(error::JournalError::UnknownCompressionMethod) => {
296
+ eprintln!("unknown compression method");
297
+ -1
298
+ }
299
+ Err(_) => -1,
300
+ };
301
+ } else {
302
+ let payload = data_guard.payload_bytes();
303
+ *l = payload.len();
304
+ *data = payload.as_ptr() as *const c_void;
305
+ }
306
+ 1
307
+ }
308
+ Ok(None) => 0,
309
+ Err(e) => e.to_error_code(),
310
+ }
311
+}
312
+
313
+#[no_mangle]
314
+unsafe extern "C" fn rsd_journal_restart_fields(j: *mut RsdJournal) {
315
+ debug_assert!(!j.is_null());
316
+
317
+ let journal = &mut *j;
318
+ journal.reader.fields_restart();
319
+}
320
+
321
+#[no_mangle]
322
+unsafe extern "C" fn rsd_journal_enumerate_fields(
323
+ j: *mut RsdJournal,
324
+ field: *mut *const c_char,
325
+) -> c_int {
326
+ debug_assert!(!j.is_null());
327
+ debug_assert!(!field.is_null());
328
+
329
+ let journal = &mut *j;
330
+
331
+ match journal.reader.fields_enumerate(&journal.journal_file) {
332
+ Ok(Some(field_guard)) => {
333
+ let field_name = field_guard.get_payload();
334
+
335
+ journal.field_buffer.clear();
336
+ journal.field_buffer.extend_from_slice(field_name);
337
+ journal.field_buffer.push(0);
338
+ *field = journal.field_buffer.as_ptr() as *const c_char;
339
+
340
+ 1
341
+ }
342
+ Ok(None) => 0,
343
+ Err(e) => e.to_error_code(),
344
+ }
345
+}
346
+
347
+#[no_mangle]
348
+unsafe extern "C" fn rsd_journal_query_unique(j: *mut RsdJournal, field: *const c_char) -> c_int {
349
+ debug_assert!(!j.is_null());
350
+ debug_assert!(!field.is_null());
351
+
352
+ let journal = &mut *j;
353
+ let field_cstr = CStr::from_ptr(field);
354
+ let field_name = field_cstr.to_bytes();
355
+
356
+ match journal
357
+ .reader
358
+ .field_data_query_unique(&journal.journal_file, field_name)
359
+ {
360
+ Ok(_) => 0,
361
+ Err(e) => e.to_error_code(),
362
+ }
363
+}
364
+
365
+#[no_mangle]
366
+unsafe extern "C" fn rsd_journal_restart_unique(j: *mut RsdJournal) {
367
+ debug_assert!(!j.is_null());
368
+ let journal = &mut *j;
369
+ journal.reader.field_data_restart();
370
+}
371
+
372
+#[no_mangle]
373
+unsafe extern "C" fn rsd_journal_enumerate_available_unique(
374
+ j: *mut RsdJournal,
375
+ data: *mut *const c_void,
376
+ l: *mut usize,
377
+) -> c_int {
378
+ debug_assert!(!j.is_null());
379
+ debug_assert!(!data.is_null());
380
+ debug_assert!(!l.is_null());
381
+
382
+ let journal = &mut *j;
383
+
384
+ match journal.reader.field_data_enumerate(&journal.journal_file) {
385
+ Ok(Some(data_guard)) => {
386
+ let payload = data_guard.payload_bytes();
387
+ *data = payload.as_ptr() as *const c_void;
388
+ *l = payload.len();
389
+
390
+ 1
391
+ }
392
+ Ok(None) => 0,
393
+ Err(e) => e.to_error_code(),
394
+ }
395
+}
396
+
397
+#[no_mangle]
398
+unsafe extern "C" fn rsd_journal_add_match(
399
+ j: *mut RsdJournal,
400
+ data: *const c_void,
401
+ size: usize,
402
+) -> c_int {
403
+ debug_assert!(!j.is_null());
404
+ debug_assert!(!data.is_null());
405
+
406
+ let journal = &mut *j;
407
+
408
+ let data_slice = if size == 0 {
409
+ let mut len = 0;
410
+ let data_ptr = data as *const u8;
411
+ while *data_ptr.add(len) != 0 {
412
+ len += 1;
413
+ }
414
+ std::slice::from_raw_parts(data as *const u8, len)
415
+ } else {
416
+ std::slice::from_raw_parts(data as *const u8, size)
417
+ };
418
+
419
+ journal.reader.add_match(data_slice);
420
+ 0
421
+}
422
+
423
+#[no_mangle]
424
+unsafe extern "C" fn rsd_journal_add_conjunction(j: *mut RsdJournal) -> c_int {
425
+ debug_assert!(!j.is_null());
426
+ let journal = &mut *j;
427
+ match journal.reader.add_conjunction(&journal.journal_file) {
428
+ Ok(_) => 0,
429
+ Err(e) => e.to_error_code(),
430
+ }
431
+}
432
+
433
+#[no_mangle]
434
+unsafe extern "C" fn rsd_journal_add_disjunction(j: *mut RsdJournal) -> c_int {
435
+ debug_assert!(!j.is_null());
436
+
437
+ let journal = &mut *j;
438
+ match journal.reader.add_disjunction(&journal.journal_file) {
439
+ Ok(_) => 0,
440
+ Err(e) => e.to_error_code(),
441
+ }
442
+}
443
+
444
+#[no_mangle]
445
+unsafe extern "C" fn rsd_journal_flush_matches(j: *mut RsdJournal) {
446
+ debug_assert!(!j.is_null());
447
+ let journal = &mut *j;
448
+ journal.reader.flush_matches();
449
+}
src/crates/jf/journal_writer/Cargo.toml
new
+12
@@ -0,0 +1,12 @@
1
+[package]
2
+name = "journal_writer"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+journal_file = { path = "../journal_file" }
8
+error = { path = "../error" }
9
+window_manager = { path = "../window_manager" }
10
+memmap2 = { workspace = true }
11
+zerocopy = { workspace = true }
12
+rand = { workspace = true }
src/crates/jf/journal_writer/src/lib.rs
new
+88
@@ -0,0 +1,88 @@
1
+// // #![allow(unused_imports, dead_code)]
2
+
3
+// use error::{JournalError, Result};
4
+// use journal_file::{
5
+// journal_hash_data, CompactEntryItem, DataObject, DataObjectHeader, DataPayloadType,
6
+// EntryObject, EntryObjectHeader, FieldObject, FieldObjectHeader, HashItem, HashTableObject,
7
+// HeaderIncompatibleFlags, JournalFile, JournalHeader, JournalState, ObjectHeader, ObjectType,
8
+// RegularEntryItem,
9
+// };
10
+// use memmap2::MmapMut;
11
+// use rand::{seq::IndexedRandom, Rng};
12
+// use std::num::NonZeroU64;
13
+// use std::path::Path;
14
+// use window_manager::MemoryMapMut;
15
+// use zerocopy::{FromBytes, IntoBytes};
16
+
17
+// const OBJECT_ALIGNMENT: u64 = 8;
18
+
19
+// #[derive(Default)]
20
+// pub struct JournalWriter {
21
+// tail_offset: u64,
22
+// offsets_buffer: Vec<u64>,
23
+// hash_buffer: Vec<u64>,
24
+// }
25
+
26
+// impl JournalWriter {
27
+// pub fn new(journal_file: &mut JournalFile<MmapMut>) -> Result<Self> {
28
+// Ok(Self {
29
+// tail_offset: 0,
30
+// offsets_buffer: Vec::with_capacity(128),
31
+// hash_buffer: Vec::with_capacity(128),
32
+// })
33
+// }
34
+
35
+// pub fn add_entry(
36
+// &mut self,
37
+// journal_file: &mut JournalFile<MmapMut>,
38
+// items: &[&[u8]],
39
+// realtime: u64,
40
+// monotonic: u64,
41
+// boot_id: [u8; 16],
42
+// ) -> Result<u64> {
43
+// let header = journal_file.journal_header_ref();
44
+
45
+// let is_keyed_hash = header.has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash);
46
+// let is_compact = header.has_incompatible_flag(HeaderIncompatibleFlags::Compact);
47
+// let file_id = header.file_id;
48
+
49
+// let mut current_offset = header.tail_object_offset;
50
+// if (current_offset == 0) || (current_offset % 8 != 0) {
51
+// return Err(JournalError::InvalidOffset);
52
+// }
53
+
54
+// let mut arena_size = header.arena_size;
55
+// if (current_offset == 0) || (current_offset % 8 != 0) {
56
+// return Err(JournalError::InvalidOffset);
57
+// }
58
+
59
+// // self.hash_buffer.clear();
60
+// // self.hash_buffer.extend(
61
+// // items
62
+// // .iter()
63
+// // .map(|item| journal_hash_data(item, is_keyed_hash, None)),
64
+// // );
65
+
66
+// // for payload in items.iter() {
67
+// // let hash = journal_file.hash(payload);
68
+// // match journal_file.find_data_offset_by_payload(payload, hash) {
69
+// // Ok(data_offset) => {
70
+// // self.offsets_buffer.push(data_offset);
71
+// // }
72
+// // Err(JournalError::MissingObjectFromHashTable) => {
73
+// // let size = payload.len() as u64;
74
+// // let data_object = journal_file.data_mut(current_offset, Some(size))?;
75
+
76
+// // current_offset += data_object.header.object_header.aligned_size();
77
+
78
+// // data_object.
79
+// // }
80
+// // Err(e) => {
81
+// // return Err(e);
82
+// // }
83
+// // };
84
+// // }
85
+
86
+// Ok(0)
87
+// }
88
+// }
src/crates/jf/main/Cargo.toml
new
+15
@@ -0,0 +1,15 @@
1
+[package]
2
+name = "main"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+journal_file = { path = "../journal_file" }
9
+journal_logger = { path = "../journal_logger" }
10
+journal_reader = { path = "../journal_reader" }
11
+sigbus = { path = "../sigbus" }
12
+window_manager = { path = "../window_manager" }
13
+rand = { workspace = true }
14
+systemd = { workspace = true }
15
+zerocopy = { workspace = true }
src/crates/jf/main/src/main.rs
new
+669
@@ -0,0 +1,669 @@
1
+#![allow(dead_code)]
2
+
3
+use error::Result;
4
+use journal_file::*;
5
+use journal_reader::{Direction, JournalReader, Location};
6
+use std::collections::HashMap;
7
+use window_manager::MemoryMap;
8
+
9
+pub struct EntryData {
10
+ pub offset: u64,
11
+ pub realtime: u64,
12
+ pub monotonic: u64,
13
+ pub boot_id: String,
14
+ pub seqnum: u64,
15
+ pub fields: Vec<(String, String)>,
16
+}
17
+
18
+impl std::fmt::Debug for EntryData {
19
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
20
+ // Start with a custom struct name
21
+ write!(f, "@{:#x?} {{ fields: [", self.offset)?;
22
+
23
+ // Iterate through fields and format each one
24
+ for (i, (key, value)) in self.fields.iter().enumerate() {
25
+ if i > 0 {
26
+ write!(f, ", ")?;
27
+ }
28
+ write!(f, "({:?}, {:?})", key, value)?;
29
+ }
30
+
31
+ // Close the formatting
32
+ write!(f, "] }}")
33
+ }
34
+}
35
+
36
+impl EntryData {
37
+ /// Extract all data from an entry into an owned structure
38
+ pub fn from_offset<M: MemoryMap>(
39
+ journal_file: &JournalFile<M>,
40
+ entry_offset: u64,
41
+ ) -> Result<EntryData> {
42
+ // Get the entry object
43
+ let entry_object = journal_file.entry_ref(entry_offset)?;
44
+
45
+ // Extract basic information from the entry header
46
+ let realtime = entry_object.header.realtime;
47
+ let monotonic = entry_object.header.monotonic;
48
+ let boot_id = format_uuid_bytes(&entry_object.header.boot_id);
49
+ let seqnum = entry_object.header.seqnum;
50
+
51
+ drop(entry_object);
52
+
53
+ // Create a vector to hold all fields
54
+ let mut fields = Vec::new();
55
+
56
+ // Iterate through all data objects for this entry
57
+ for data_result in journal_file.entry_data_objects(entry_offset)? {
58
+ let data_object = data_result?;
59
+ let payload = data_object.payload_bytes();
60
+
61
+ // Find the first '=' character to split field and value
62
+ if let Some(equals_pos) = payload.iter().position(|&b| b == b'=') {
63
+ let field = String::from_utf8_lossy(&payload[0..equals_pos]).to_string();
64
+ let value = String::from_utf8_lossy(&payload[equals_pos + 1..]).to_string();
65
+
66
+ if field.starts_with("_") {
67
+ continue;
68
+ }
69
+
70
+ fields.push((field, value));
71
+ }
72
+ }
73
+
74
+ // Create and return the EntryData struct
75
+ Ok(EntryData {
76
+ offset: entry_offset,
77
+ realtime,
78
+ monotonic,
79
+ boot_id,
80
+ seqnum,
81
+ fields,
82
+ })
83
+ }
84
+
85
+ pub fn get_field(&self, name: &str) -> Option<&str> {
86
+ self.fields
87
+ .iter()
88
+ .find(|(k, _)| k == name)
89
+ .map(|(_, v)| v.as_str())
90
+ }
91
+}
92
+
93
+fn format_uuid_bytes(bytes: &[u8; 16]) -> String {
94
+ format!(
95
+ "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}",
96
+ bytes[0], bytes[1], bytes[2], bytes[3],
97
+ bytes[4], bytes[5],
98
+ bytes[6], bytes[7],
99
+ bytes[8], bytes[9],
100
+ bytes[10], bytes[11], bytes[12], bytes[13], bytes[14], bytes[15]
101
+ )
102
+}
103
+
104
+use systemd::journal;
105
+
106
+struct JournalWrapper<'a> {
107
+ j: journal::Journal,
108
+
109
+ jr: JournalReader<'a, Mmap>,
110
+}
111
+
112
+impl<'a> JournalWrapper<'a> {
113
+ pub fn open(path: &str) -> Result<Self> {
114
+ let opts = journal::OpenFilesOptions::default();
115
+ let j = opts.open_files([path])?;
116
+ let jr = JournalReader::default();
117
+
118
+ Ok(Self { j, jr })
119
+ }
120
+
121
+ pub fn match_add(&mut self, data: &str) {
122
+ self.j.match_add(data).unwrap();
123
+ self.jr.add_match(data.as_bytes());
124
+ }
125
+
126
+ pub fn match_and(&mut self, journal_file: &'a JournalFile<Mmap>) {
127
+ self.j.match_and().unwrap();
128
+ self.jr.add_conjunction(journal_file).unwrap();
129
+ }
130
+
131
+ pub fn match_or(&mut self, journal_file: &'a JournalFile<Mmap>) {
132
+ self.j.match_or().unwrap();
133
+ self.jr.add_disjunction(journal_file).unwrap();
134
+ }
135
+
136
+ pub fn match_flush(&mut self) {
137
+ self.j.match_flush().unwrap();
138
+ self.jr.flush_matches();
139
+ }
140
+
141
+ pub fn seek_head(&mut self) {
142
+ self.j.seek_head().unwrap();
143
+ self.jr.set_location(Location::Head);
144
+ }
145
+
146
+ pub fn seek_tail(&mut self) {
147
+ self.j.seek_tail().unwrap();
148
+ self.jr.set_location(Location::Tail);
149
+ }
150
+
151
+ pub fn seek_realtime(&mut self, usec: u64) {
152
+ self.j.seek_realtime_usec(usec).unwrap();
153
+ self.jr.set_location(Location::Realtime(usec));
154
+ }
155
+
156
+ pub fn next(&mut self, journal_file: &'a JournalFile<Mmap>) -> bool {
157
+ let r1 = self.j.next().unwrap();
158
+ let r2 = self.jr.step(journal_file, Direction::Forward).unwrap();
159
+
160
+ if r1 > 0 {
161
+ if r2 {
162
+ return r2;
163
+ } else {
164
+ panic!("r1: {:?}, r2: {:?}", r1, r2);
165
+ }
166
+ } else if r1 == 0 {
167
+ if !r2 {
168
+ return r2;
169
+ } else {
170
+ panic!("r1: {:?}, r2: {:?}", r1, r2);
171
+ }
172
+ } else {
173
+ println!("WTF?");
174
+ }
175
+
176
+ r2
177
+ }
178
+
179
+ pub fn previous(&mut self, journal_file: &'a JournalFile<Mmap>) -> bool {
180
+ let r1 = self.j.previous().unwrap();
181
+ let r2 = self.jr.step(journal_file, Direction::Backward).unwrap();
182
+
183
+ if r1 > 0 {
184
+ if r2 {
185
+ return r2;
186
+ } else {
187
+ panic!("r1: {:?}, r2: {:?}", r1, r2);
188
+ }
189
+ } else if r1 == 0 {
190
+ if !r2 {
191
+ return r2;
192
+ } else {
193
+ panic!("r1: {:?}, r2: {:?}", r1, r2);
194
+ }
195
+ } else {
196
+ println!("WTF?");
197
+ }
198
+
199
+ r2
200
+ }
201
+
202
+ pub fn get_realtime_usec(&mut self, journal_file: &'a JournalFile<Mmap>) -> u64 {
203
+ let usec1 = self.j.timestamp().unwrap();
204
+ let usec2 = self.jr.get_realtime_usec(journal_file).unwrap();
205
+
206
+ assert_eq!(usec1, usec2);
207
+ usec1
208
+ }
209
+}
210
+
211
+fn get_terms(path: &str) -> HashMap<String, Vec<String>> {
212
+ let window_size = 8 * 1024 * 1024;
213
+ let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
214
+
215
+ let mut terms = HashMap::new();
216
+ let mut fields = Vec::new();
217
+ for field in journal_file.fields() {
218
+ let field = field.unwrap();
219
+ let field = String::from(String::from_utf8_lossy(field.get_payload()).clone());
220
+ fields.push(field.clone());
221
+ terms.insert(field, Vec::new());
222
+ }
223
+
224
+ for field in fields {
225
+ for data in journal_file.field_data_objects(field.as_bytes()).unwrap() {
226
+ let data = data.unwrap();
227
+ if data.is_compressed() {
228
+ continue;
229
+ }
230
+
231
+ let data_payload = String::from(String::from_utf8_lossy(data.get_payload()).clone());
232
+
233
+ if data_payload.len() > 200 {
234
+ continue;
235
+ }
236
+
237
+ terms.get_mut(&field).unwrap().push(data_payload);
238
+ }
239
+ }
240
+
241
+ terms.retain(|_, value| !value.is_empty());
242
+ terms
243
+}
244
+
245
+#[derive(Debug)]
246
+enum SeekType {
247
+ Head,
248
+ Tail,
249
+ Realtime(u64),
250
+}
251
+
252
+fn get_timings(path: &str) -> Vec<u64> {
253
+ let window_size = 8 * 1024 * 1024;
254
+ let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
255
+ let mut jw: JournalWrapper<'_> = JournalWrapper::open(path).unwrap();
256
+
257
+ let mut v = Vec::new();
258
+
259
+ jw.seek_head();
260
+ loop {
261
+ if !jw.next(&journal_file) {
262
+ break;
263
+ }
264
+
265
+ let usec = jw.get_realtime_usec(&journal_file);
266
+ v.push(usec);
267
+ }
268
+ assert!(v.is_sorted());
269
+
270
+ v
271
+}
272
+
273
+use rand::{prelude::*, Rng};
274
+
275
+#[derive(Debug, Copy, Clone)]
276
+enum SeekOperation {
277
+ Head,
278
+ Tail,
279
+ Realtime(u64),
280
+}
281
+
282
+fn select_seek_operation(rng: &mut ThreadRng, timings: &[u64]) -> SeekOperation {
283
+ let duplicate_timestamps = [
284
+ 1747729025279631,
285
+ 1747729025280143,
286
+ 1747729025280247,
287
+ 1747729025358451,
288
+ 1747729025387355,
289
+ 1747729025387415,
290
+ ];
291
+
292
+ match rng.random_range(0..3) {
293
+ 0 => SeekOperation::Head,
294
+ 1 => SeekOperation::Tail,
295
+ 2 => {
296
+ let rt_idx = rng.random_range(0..timings.len());
297
+
298
+ let usec = timings[rt_idx];
299
+ if duplicate_timestamps.contains(&usec) {
300
+ return SeekOperation::Head;
301
+ }
302
+
303
+ SeekOperation::Realtime(timings[rt_idx])
304
+ }
305
+ _ => unreachable!(),
306
+ }
307
+}
308
+
309
+#[derive(Debug)]
310
+enum MatchOr {
311
+ None,
312
+ One(String),
313
+ Two(String, String),
314
+}
315
+
316
+fn select_match_or(rng: &mut ThreadRng, terms: &HashMap<String, Vec<String>>) -> MatchOr {
317
+ match rng.random_range(0..3) {
318
+ 0 => MatchOr::None,
319
+ 1 => {
320
+ let key_index = rng.random_range(0..terms.len());
321
+ let key = terms.keys().nth(key_index).unwrap();
322
+
323
+ let value = terms.get(key).unwrap();
324
+ let value_index = rng.random_range(0..value.len());
325
+
326
+ MatchOr::One(value[value_index].clone())
327
+ }
328
+ 2 => {
329
+ let first_term = {
330
+ let key_index = rng.random_range(0..terms.len());
331
+ let key = terms.keys().nth(key_index).unwrap();
332
+
333
+ let value = terms.get(key).unwrap();
334
+ let value_index = rng.random_range(0..value.len());
335
+
336
+ value[value_index].clone()
337
+ };
338
+
339
+ let second_term = {
340
+ let key_index = rng.random_range(0..terms.len());
341
+ let key = terms.keys().nth(key_index).unwrap();
342
+
343
+ let value = terms.get(key).unwrap();
344
+ let value_index = rng.random_range(0..value.len());
345
+
346
+ value[value_index].clone()
347
+ };
348
+
349
+ MatchOr::Two(first_term, second_term)
350
+ }
351
+ _ => {
352
+ unreachable!()
353
+ }
354
+ }
355
+}
356
+
357
+#[derive(Debug, Clone)]
358
+enum MatchExpr {
359
+ None,
360
+ OrOne(String),
361
+ OrTwo(String, String),
362
+ And1(String, String),
363
+ And2(String, (String, String)),
364
+ And3((String, String), String),
365
+ And4((String, String), (String, String)),
366
+}
367
+
368
+fn select_match_expression(rng: &mut ThreadRng, terms: &HashMap<String, Vec<String>>) -> MatchExpr {
369
+ let mor1 = select_match_or(rng, terms);
370
+ let mor2 = select_match_or(rng, terms);
371
+
372
+ let expr = match (mor1, mor2) {
373
+ (MatchOr::None, MatchOr::None) => MatchExpr::None,
374
+
375
+ (MatchOr::None, MatchOr::One(d1)) => MatchExpr::OrOne(d1),
376
+ (MatchOr::One(d1), MatchOr::None) => MatchExpr::OrOne(d1),
377
+
378
+ (MatchOr::None, MatchOr::Two(d1, d2)) => MatchExpr::OrTwo(d1, d2),
379
+ (MatchOr::Two(d1, d2), MatchOr::None) => MatchExpr::OrTwo(d1, d2),
380
+
381
+ (MatchOr::One(d1), MatchOr::One(d2)) => MatchExpr::And1(d1, d2),
382
+
383
+ (MatchOr::One(d1), MatchOr::Two(d2, d3)) => MatchExpr::And2(d1, (d2, d3)),
384
+ (MatchOr::Two(d1, d2), MatchOr::One(d3)) => MatchExpr::And3((d1, d2), d3),
385
+
386
+ (MatchOr::Two(d1, d2), MatchOr::Two(d3, d4)) => MatchExpr::And4((d1, d2), (d3, d4)),
387
+ };
388
+
389
+ match expr.clone() {
390
+ MatchExpr::None | MatchExpr::OrOne(_) => expr,
391
+ MatchExpr::OrTwo(d1, d2) => {
392
+ if d1 == d2 {
393
+ MatchExpr::None
394
+ } else {
395
+ expr
396
+ }
397
+ }
398
+ MatchExpr::And1(d1, d2) => {
399
+ if d1 == d2 {
400
+ MatchExpr::None
401
+ } else {
402
+ expr
403
+ }
404
+ }
405
+ MatchExpr::And2(d1, (d2, d3)) => {
406
+ if d1 == d2 || d1 == d3 || d2 == d3 {
407
+ MatchExpr::None
408
+ } else {
409
+ expr
410
+ }
411
+ }
412
+ MatchExpr::And3((d1, d2), d3) => {
413
+ if d1 == d2 || d1 == d3 || d2 == d3 {
414
+ MatchExpr::None
415
+ } else {
416
+ expr
417
+ }
418
+ }
419
+ MatchExpr::And4((d1, d2), (d3, d4)) => {
420
+ if d1 == d2 || d1 == d3 || d1 == d4 || d2 == d3 || d2 == d4 || d3 == d4 {
421
+ MatchExpr::None
422
+ } else {
423
+ expr
424
+ }
425
+ }
426
+ }
427
+}
428
+
429
+#[derive(Debug, Clone, Copy)]
430
+enum IterationOperation {
431
+ Next,
432
+ Previous,
433
+}
434
+
435
+fn select_iteration_operation(rng: &mut ThreadRng) -> IterationOperation {
436
+ match rng.random_range(0..2) {
437
+ 0 => IterationOperation::Next,
438
+ 1 => IterationOperation::Previous,
439
+ _ => unreachable!(),
440
+ }
441
+}
442
+
443
+fn apply_seek_operation(seek_operation: SeekOperation, jw: &mut JournalWrapper) {
444
+ match seek_operation {
445
+ SeekOperation::Head => jw.seek_head(),
446
+ SeekOperation::Tail => jw.seek_tail(),
447
+ SeekOperation::Realtime(usec) => jw.seek_realtime(usec),
448
+ }
449
+}
450
+
451
+fn apply_iteration_operation<'a>(
452
+ iteration_operation: IterationOperation,
453
+ jw: &mut JournalWrapper<'a>,
454
+ journal_file: &'a JournalFile<Mmap>,
455
+) -> bool {
456
+ match iteration_operation {
457
+ IterationOperation::Next => jw.next(journal_file),
458
+ IterationOperation::Previous => jw.previous(journal_file),
459
+ }
460
+}
461
+
462
+fn apply_match_expression<'a>(
463
+ match_expr: MatchExpr,
464
+ jw: &mut JournalWrapper<'a>,
465
+ journal_file: &'a JournalFile<Mmap>,
466
+) -> bool {
467
+ jw.match_flush();
468
+
469
+ match match_expr.clone() {
470
+ MatchExpr::None => {
471
+ return true;
472
+ }
473
+ MatchExpr::OrOne(d) => {
474
+ jw.match_add(&d);
475
+ return true;
476
+ }
477
+ MatchExpr::OrTwo(d1, d2) => {
478
+ jw.match_add(&d1);
479
+ jw.match_add(&d2);
480
+ return true;
481
+ }
482
+ MatchExpr::And1(d1, d2) => {
483
+ jw.match_add(&d1);
484
+ jw.match_and(journal_file);
485
+ jw.match_add(&d2);
486
+ return true;
487
+ }
488
+ MatchExpr::And2(d1, (d2, d3)) => {
489
+ jw.match_add(&d1);
490
+ jw.match_and(journal_file);
491
+ jw.match_add(&d2);
492
+ jw.match_add(&d3);
493
+ return true;
494
+ }
495
+ MatchExpr::And3((d1, d2), d3) => {
496
+ jw.match_add(&d1);
497
+ jw.match_add(&d2);
498
+ jw.match_and(journal_file);
499
+ jw.match_add(&d3);
500
+ return true;
501
+ }
502
+ MatchExpr::And4((d1, d2), (d3, d4)) => {
503
+ jw.match_add(&d1);
504
+ jw.match_add(&d2);
505
+ jw.match_or(journal_file);
506
+
507
+ jw.match_add(&d3);
508
+ jw.match_add(&d4);
509
+ jw.match_or(journal_file);
510
+
511
+ jw.match_and(journal_file);
512
+
513
+ return true;
514
+ }
515
+ };
516
+}
517
+
518
+fn filtered_test() {
519
+ let path = "/tmp/foo.journal";
520
+ let window_size = 8 * 1024 * 1024;
521
+ let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
522
+ println!(
523
+ "num entries: {:?}",
524
+ journal_file.journal_header_ref().n_entries
525
+ );
526
+ let mut jw = JournalWrapper::open(path).unwrap();
527
+
528
+ let terms = get_terms(path);
529
+ let timings = get_timings(path);
530
+
531
+ let mut rng = rand::rng();
532
+
533
+ let mut counter = 0;
534
+ loop {
535
+ let match_expr = select_match_expression(&mut rng, &terms);
536
+ let applied = apply_match_expression(match_expr.clone(), &mut jw, &journal_file);
537
+ if !applied {
538
+ continue;
539
+ }
540
+
541
+ println!("[{}] match_expr: {:?}", counter, match_expr);
542
+
543
+ let seek_operation = select_seek_operation(&mut rng, &timings);
544
+ println!("[{}] seek: {:?}", counter, seek_operation);
545
+ apply_seek_operation(seek_operation, &mut jw);
546
+
547
+ let mut num_matches = 0;
548
+ let iteration_operation = select_iteration_operation(&mut rng);
549
+ println!("[{}] iteration: {:?}", counter, iteration_operation);
550
+
551
+ for _ in 0..rng.random_range(0..2 * timings.len()) {
552
+ let found = apply_iteration_operation(iteration_operation, &mut jw, &journal_file);
553
+ if found {
554
+ jw.get_realtime_usec(&journal_file);
555
+ num_matches += 1;
556
+ }
557
+ }
558
+
559
+ println!("[{}] num matches: {:?}\n", counter, num_matches);
560
+ counter += 1;
561
+ }
562
+}
563
+
564
+fn test_case() {
565
+ let path = "/tmp/foo.journal";
566
+
567
+ let window_size = 8 * 1024 * 1024;
568
+ let journal_file = JournalFile::<Mmap>::open(path, window_size).unwrap();
569
+ let mut jw = JournalWrapper::open(path).unwrap();
570
+
571
+ let timings = get_timings(path);
572
+ println!("timings: {:#?}", timings);
573
+
574
+ jw.seek_realtime(u64::MAX);
575
+ if jw.previous(&journal_file) {
576
+ let value = jw.get_realtime_usec(&journal_file);
577
+ println!("first value: {:?}", value);
578
+ }
579
+ // return;
580
+
581
+ // jw.next(&journal_file);
582
+ // let value = jw.get_realtime_usec(&journal_file);
583
+ // println!("second value: {:?}", value);
584
+}
585
+
586
+fn main() {
587
+ // {
588
+ // let mut jf = JournalFile::<MmapMut>::create("/tmp/muh.journal", 4096).unwrap();
589
+
590
+ // let dht = jf.data_hash_table_mut().unwrap();
591
+ // let mut items = dht.items;
592
+ // println!("dht items: {:?}", items.len());
593
+ // items[0].head_hash_offset = 0xdeadbeef;
594
+ // items[0].tail_hash_offset = 0xbeefdead;
595
+
596
+ // let fht = jf.field_hash_table_mut().unwrap();
597
+ // let mut items = fht.items;
598
+ // println!("fht items: {:?}", items.len());
599
+ // items[0].head_hash_offset = 0xaaaabbbb;
600
+ // items[0].tail_hash_offset = 0xccccdddd;
601
+
602
+ // let mut offset_array = jf.offset_array_mut(1024 * 1024, Some(8)).unwrap();
603
+
604
+ // for i in 0..4 {
605
+ // let offset = std::num::NonZeroU64::new(0xdead0000 + i).unwrap();
606
+ // offset_array.set(i as usize, offset).unwrap();
607
+ // }
608
+ // }
609
+
610
+ // let jf = JournalFile::<Mmap>::open("/tmp/muh.journal", 4096).unwrap();
611
+
612
+ // let offset_array = jf.offset_array_ref(1024 * 1024).unwrap();
613
+
614
+ // println!(
615
+ // "tail object offset: 0x{:x?}",
616
+ // jf.journal_header_ref().tail_object_offset
617
+ // );
618
+ // println!(
619
+ // "fht offset: 0x{:x?}",
620
+ // jf.journal_header_ref().field_hash_table_offset
621
+ // );
622
+ // println!(
623
+ // "hash table header size: 0x{:x?}",
624
+ // std::mem::size_of::<ObjectHeader>()
625
+ // );
626
+
627
+ // println!("offset_array: {:?}", offset_array);
628
+
629
+ // for i in 0..4 {
630
+ // let offset = offset_array.get(i, 8).unwrap();
631
+ // println!("offset[{}]: 0x{:x?}", i, offset);
632
+ // }
633
+
634
+ filtered_test();
635
+ // test_case()
636
+
637
+ // altime();
638
+
639
+ // let args: Vec<String> = std::env::args().collect();
640
+ // if args.len() != 2 {
641
+ // eprintln!("Usage: {} <journal_file_path>", args[0]);
642
+ // std::process::exit(1);
643
+ // }
644
+
645
+ // if false {
646
+ // create_logs();
647
+ // return;
648
+ // }
649
+
650
+ // const WINDOW_SIZE: u64 = 4096;
651
+ // match JournalFile::<Mmap>::open(&args[1], WINDOW_SIZE) {
652
+ // Ok(journal_file) => {
653
+ // if true {
654
+ // if let Err(e) = test_cursor(&journal_file) {
655
+ // panic!("Cursor tests failed: {:?}", e);
656
+ // }
657
+ // }
658
+
659
+ // if true {
660
+ // if let Err(e) = test_filter_expr(&journal_file) {
661
+ // panic!("Filter expression tests failed: {:?}", e);
662
+ // }
663
+
664
+ // println!("Overall stat: {:?}", journal_file.stats());
665
+ // }
666
+ // }
667
+ // Err(e) => panic!("Failed to open journal file: {:?}", e),
668
+ // }
669
+}
src/crates/jf/sigbus/Cargo.toml
new
+8
@@ -0,0 +1,8 @@
1
+[package]
2
+name = "sigbus"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+libc = { workspace = true }
src/crates/jf/sigbus/src/lib.rs
new
+49
@@ -0,0 +1,49 @@
1
+use error::{JournalError, Result};
2
+use std::sync::atomic::{AtomicBool, Ordering};
3
+use std::sync::OnceLock;
4
+
5
+static SIGBUS_OCCURRED: AtomicBool = AtomicBool::new(false);
6
+static HANDLER_INSTALLED: OnceLock<i32> = OnceLock::new();
7
+
8
+extern "C" fn sigbus_handler(
9
+ _sig: libc::c_int,
10
+ info: *mut libc::siginfo_t,
11
+ _ucontext: *mut libc::c_void,
12
+) {
13
+ unsafe {
14
+ let si = &*info;
15
+ let fault_addr = si.si_addr();
16
+
17
+ let page_addr = (fault_addr as usize & !(4096 - 1)) as *mut libc::c_void;
18
+ libc::mmap(
19
+ page_addr,
20
+ 4096,
21
+ libc::PROT_READ,
22
+ libc::MAP_PRIVATE | libc::MAP_ANONYMOUS | libc::MAP_FIXED,
23
+ -1,
24
+ 0,
25
+ );
26
+
27
+ SIGBUS_OCCURRED.store(true, Ordering::Relaxed);
28
+ }
29
+}
30
+
31
+pub fn signalled() -> bool {
32
+ SIGBUS_OCCURRED.load(Ordering::Relaxed)
33
+}
34
+
35
+pub fn install_handler() -> Result<()> {
36
+ let rc = HANDLER_INSTALLED.get_or_init(|| unsafe {
37
+ let mut sa: libc::sigaction = std::mem::zeroed();
38
+
39
+ sa.sa_flags = libc::SA_SIGINFO;
40
+ sa.sa_sigaction = sigbus_handler as usize;
41
+
42
+ libc::sigaction(libc::SIGBUS, &sa, std::ptr::null_mut())
43
+ });
44
+
45
+ match rc {
46
+ -1 => Err(JournalError::SigbusHandlerError),
47
+ _ => Ok(()),
48
+ }
49
+}
src/crates/jf/window_manager/Cargo.toml
new
+8
@@ -0,0 +1,8 @@
1
+[package]
2
+name = "window_manager"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[dependencies]
7
+error = { path = "../error" }
8
+memmap2 = { workspace = true }
src/crates/jf/window_manager/src/lib.rs
new
+235
@@ -0,0 +1,235 @@
1
+use error::Result;
2
+use memmap2::{Mmap, MmapMut, MmapOptions};
3
+use std::fs::File;
4
+use std::ops::{Deref, DerefMut};
5
+
6
+const PAGE_SIZE: u64 = 4096;
7
+
8
+pub trait MemoryMap: Deref<Target = [u8]> {
9
+ fn create(file: &File, offset: u64, size: u64) -> Result<Self>
10
+ where
11
+ Self: Sized;
12
+}
13
+
14
+pub trait MemoryMapMut: MemoryMap + DerefMut {}
15
+
16
+impl MemoryMap for Mmap {
17
+ fn create(file: &File, offset: u64, size: u64) -> Result<Self> {
18
+ let mmap = unsafe {
19
+ MmapOptions::new()
20
+ .offset(offset)
21
+ .len(size as usize)
22
+ .map(file)?
23
+ };
24
+
25
+ Ok(mmap)
26
+ }
27
+}
28
+
29
+impl MemoryMap for MmapMut {
30
+ fn create(file: &File, offset: u64, size: u64) -> Result<Self> {
31
+ let required_size = offset + size;
32
+
33
+ if required_size > file.metadata()?.len() {
34
+ file.set_len(required_size)?;
35
+ }
36
+
37
+ let mmap = unsafe {
38
+ MmapOptions::new()
39
+ .offset(offset)
40
+ .len(size as usize)
41
+ .map_mut(file)?
42
+ };
43
+
44
+ Ok(mmap)
45
+ }
46
+}
47
+
48
+impl MemoryMapMut for MmapMut {}
49
+
50
+struct Window<M: MemoryMap> {
51
+ offset: u64,
52
+ size: u64,
53
+ mmap: M,
54
+}
55
+
56
+impl<M: MemoryMap> std::fmt::Debug for Window<M> {
57
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
58
+ f.debug_struct("Window")
59
+ .field("offset", &self.offset)
60
+ .field("size", &self.size)
61
+ .finish()
62
+ }
63
+}
64
+
65
+impl<M: MemoryMap> Window<M> {
66
+ fn end_offset(&self) -> u64 {
67
+ self.offset + self.size
68
+ }
69
+
70
+ fn contains(&self, position: u64) -> bool {
71
+ position >= self.offset && position < self.end_offset()
72
+ }
73
+
74
+ fn contains_range(&self, position: u64, size: u64) -> bool {
75
+ position >= self.offset && position + size <= self.end_offset()
76
+ }
77
+
78
+ fn get_slice(&self, position: u64, size: u64) -> &[u8] {
79
+ debug_assert!(self.contains_range(position, size));
80
+
81
+ let offset = (position - self.offset) as usize;
82
+ &self.mmap[offset..offset + size as usize]
83
+ }
84
+}
85
+
86
+impl<M: MemoryMapMut> Window<M> {
87
+ pub fn get_mut_slice(&mut self, position: u64, size: u64) -> &mut [u8] {
88
+ debug_assert!(self.contains_range(position, size));
89
+
90
+ let offset = (position - self.offset) as usize;
91
+ &mut self.mmap[offset..offset + size as usize]
92
+ }
93
+}
94
+
95
+pub struct WindowManager<M: MemoryMap> {
96
+ file: File,
97
+ _file_size: u64,
98
+ chunk_size: u64,
99
+ active_window_idx: Option<usize>,
100
+ max_windows: usize,
101
+ windows: Vec<Window<M>>,
102
+}
103
+
104
+impl<M: MemoryMap> WindowManager<M> {
105
+ pub fn new(file: File, chunk_size: u64, max_windows: usize) -> Result<Self> {
106
+ debug_assert!(chunk_size != 0 && (chunk_size % PAGE_SIZE) == 0);
107
+ debug_assert!(max_windows != 0);
108
+
109
+ let _file_size = file.metadata()?.len();
110
+
111
+ Ok(WindowManager {
112
+ file,
113
+ _file_size,
114
+ chunk_size,
115
+ max_windows,
116
+ windows: Vec::new(),
117
+ active_window_idx: None,
118
+ })
119
+ }
120
+
121
+ fn get_chunk_aligned_start(&self, position: u64) -> u64 {
122
+ (position / self.chunk_size) * self.chunk_size
123
+ }
124
+
125
+ fn get_chunk_aligned_end(&self, position: u64) -> u64 {
126
+ position.div_ceil(self.chunk_size) * self.chunk_size
127
+ }
128
+
129
+ fn create_window(&self, window_start: u64, chunk_count: u64) -> Result<Window<M>> {
130
+ debug_assert_ne!(chunk_count, 0);
131
+
132
+ let size = chunk_count * self.chunk_size;
133
+ let mmap = M::create(&self.file, window_start, size)?;
134
+ Ok(Window {
135
+ offset: window_start,
136
+ size,
137
+ mmap,
138
+ })
139
+ }
140
+
141
+ fn find_window_to_evict(&self) -> usize {
142
+ if self.active_window_idx == Some(0) && self.windows.len() > 1 {
143
+ 1
144
+ } else {
145
+ 0
146
+ }
147
+ }
148
+
149
+ fn lookup_window_by_range(&self, position: u64, size_needed: u64) -> Option<usize> {
150
+ if let Some(idx) = self.active_window_idx {
151
+ if self.windows[idx].contains_range(position, size_needed) {
152
+ return Some(idx);
153
+ }
154
+ }
155
+
156
+ for (idx, window) in self.windows.iter().enumerate() {
157
+ if window.contains_range(position, size_needed) {
158
+ return Some(idx);
159
+ }
160
+ }
161
+
162
+ None
163
+ }
164
+
165
+ fn lookup_window_by_position(&self, position: u64) -> Option<usize> {
166
+ if let Some(idx) = self.active_window_idx {
167
+ if self.windows[idx].contains(position) {
168
+ return Some(idx);
169
+ }
170
+ }
171
+
172
+ for (idx, window) in self.windows.iter().enumerate() {
173
+ if window.contains(position) {
174
+ return Some(idx);
175
+ }
176
+ }
177
+
178
+ None
179
+ }
180
+
181
+ fn get_window(&mut self, position: u64, size_needed: u64) -> Result<&mut Window<M>> {
182
+ if let Some(idx) = self.lookup_window_by_range(position, size_needed) {
183
+ // Use the existing window
184
+ Ok(&mut self.windows[idx])
185
+ } else if let Some(idx) = self.lookup_window_by_position(position) {
186
+ // Remap the window
187
+
188
+ let window = self.windows.remove(idx);
189
+
190
+ let window_start = window.offset;
191
+ let window_end = self.get_chunk_aligned_end(position + size_needed);
192
+ let num_chunks = (window_end - window_start) / self.chunk_size;
193
+
194
+ let new_window = self.create_window(window_start, num_chunks)?;
195
+
196
+ self.windows.push(new_window);
197
+ self.active_window_idx = Some(self.windows.len() - 1);
198
+ Ok(self.windows.last_mut().unwrap())
199
+ } else {
200
+ // Create a brand new window
201
+
202
+ if self.windows.len() >= self.max_windows {
203
+ self.windows.remove(self.find_window_to_evict());
204
+ }
205
+
206
+ // NOTE: the active window index might have been invalidated. In
207
+ // the scope that follows, we should not use code that relies on it.
208
+ {
209
+ // Calculate window start for this position
210
+ let window_start = self.get_chunk_aligned_start(position);
211
+ let window_end = self.get_chunk_aligned_end(position + size_needed);
212
+ let num_chunks = (window_end - window_start) / self.chunk_size;
213
+
214
+ let new_window = self.create_window(window_start, num_chunks)?;
215
+
216
+ self.windows.push(new_window);
217
+ }
218
+
219
+ self.active_window_idx = Some(self.windows.len() - 1);
220
+ Ok(self.windows.last_mut().unwrap())
221
+ }
222
+ }
223
+
224
+ pub fn get_slice(&mut self, position: u64, size: u64) -> Result<&[u8]> {
225
+ let window = self.get_window(position, size)?;
226
+ Ok(window.get_slice(position, size))
227
+ }
228
+}
229
+
230
+impl<M: MemoryMapMut> WindowManager<M> {
231
+ pub fn get_slice_mut(&mut self, position: u64, size: u64) -> Result<&mut [u8]> {
232
+ let window = self.get_window(position, size)?;
233
+ Ok(window.get_mut_slice(position, size))
234
+ }
235
+}