OTEL logs (#21356)
Ingest otel logs through the otel plugin and visualize them with journal-viewer plugin. --------- Co-authored-by: Austin S. Hemmelgarn <austin@netdata.cloud> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: netdatabot <bot@netdata.cloud> Co-authored-by: thiagoftsm <thiagoftsm@gmail.com> Co-authored-by: Ilya Mashchenko <ilya@netdata.cloud> Co-authored-by: Stelios Fragkakis <52996999+stelfrag@users.noreply.github.com> Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> Co-authored-by: cubic-dev-ai[bot] <191113872+cubic-dev-ai[bot]@users.noreply.github.com>
vkalintiris committed
Jan 14, 2026 at 16:54 UTC
d0905d9b99b9c6d8bd24847b020641f9b4455f55
234 files changed
+36047
-12048
.github/labeler.yml
+2
-1
@@ -195,7 +195,8 @@ collectors/systemd-journal:
195
- any:
196
- changed-files:
197
- any-glob-to-any-file:
198
- - src/collectors/systemd-journal.plugin/**
198
+ - src/crates/netdata-log-viewer/**
199
+ - docs/logs/**
200
201
collectors/tc:
202
- any:
.github/workflows/docker.yml
+4
@@ -248,6 +248,7 @@ jobs:
248
build-args: |
249
OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
250
EXTRA_INSTALL_OPTS=${{ needs.file-check.outputs.skip-go }}
251
+ BUILD_ARCH=${{ matrix.arch }}
252
BUILD_VERSION=test
253
BUILD_DATE=${{ github.event.repository.updated_at }}
254
- name: Test Image
@@ -369,6 +370,7 @@ jobs:
370
outputs: type=image,name=netdata/netdata,push-by-digest=true,name-canonical=true,push=true
371
build-args: |
372
OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
373
+ BUILD_ARCH=${{ matrix.arch }}
374
BUILD_VERSION=${{ github.event.inputs.version }}
375
BUILD_DATE=${{ github.event.repository.updated_at }}
376
- name: Export Digest
@@ -526,6 +528,7 @@ jobs:
528
cache-from: type=local,src=/tmp/build-cache
529
build-args: |
530
OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
531
+ BUILD_ARCH=${{ matrix.arch }}
532
BUILD_VERSION=${{ github.event.inputs.version }}
533
BUILD_DATE=${{ github.event.repository.updated_at }}
534
outputs: type=image,name=quay.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true
@@ -685,6 +688,7 @@ jobs:
688
cache-from: type=local,src=/tmp/build-cache
689
build-args: |
690
OFFICIAL_IMAGE=${{ env.OFFICIAL_IMAGE }}
691
+ BUILD_ARCH=${{ matrix.arch }}
692
BUILD_VERSION=${{ github.event.inputs.version }}
693
BUILD_DATE=${{ github.event.repository.updated_at }}
694
outputs: type=image,name=ghcr.io/netdata/netdata,push-by-digest=true,name-canonical=true,push=true
CMakeLists.txt
+51
-54
@@ -132,6 +132,7 @@ mark_as_advanced(ENABLE_DASHBOARD)
132
# Data collection plugins
133
option(ENABLE_PLUGIN_GO "Enable metric collectors written in Go" ${DEFAULT_FEATURE_STATE})
134
option(ENABLE_PLUGIN_OTEL "Enable collection of OpenTelemetry metrics and logs" ${DEFAULT_FEATURE_STATE})
135
+option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable journal viewer plugin" ${DEFAULT_FEATURE_STATE})
136
option(ENABLE_PLUGIN_PYTHON "Enable metric collectors written in Python" ${DEFAULT_FEATURE_STATE})
137
138
cmake_dependent_option(ENABLE_PLUGIN_APPS "Enable per-process resource usage monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX OR OS_FREEBSD OR OS_MACOS OR OS_WINDOWS" False)
@@ -150,7 +151,6 @@ cmake_dependent_option(ENABLE_PLUGIN_NETWORK_VIEWER "Enable network viewer funct
151
cmake_dependent_option(ENABLE_PLUGIN_NFACCT "Enable Linux NFACCT metric collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
152
cmake_dependent_option(ENABLE_PLUGIN_PERF "Enable Linux performance counter monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
153
cmake_dependent_option(ENABLE_PLUGIN_SLABINFO "Enable Linux kernel SLAB allocator monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
153
-cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_JOURNAL "Enable systemd journal log collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
154
cmake_dependent_option(ENABLE_PLUGIN_SYSTEMD_UNITS "Enable systemd units information collection" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
155
cmake_dependent_option(ENABLE_PLUGIN_XENSTAT "Enable Xen domain monitoring" ${DEFAULT_FEATURE_STATE} "OS_LINUX" False)
156
@@ -184,7 +184,6 @@ mark_as_advanced(ENABLE_LIBUNWIND)
184
cmake_dependent_option(FORCE_LEGACY_LIBBPF "Force usage of libbpf 0.0.9 instead of the latest version." False "ENABLE_PLUGIN_EBPF" False)
185
mark_as_advanced(FORCE_LEGACY_LIBBPF)
186
187
-cmake_dependent_option(ENABLE_NETDATA_JOURNAL_FILE_READER "Enable netdata's journal file reader implementation" False "ENABLE_PLUGIN_SYSTEMD_JOURNAL" False)
187
188
# Setup Rust/Corrosion for plugins that need it
189
if(ENABLE_NETDATA_JOURNAL_FILE_READER OR ENABLE_PLUGIN_OTEL)
@@ -195,8 +194,33 @@ if(ENABLE_NETDATA_JOURNAL_FILE_READER OR ENABLE_PLUGIN_OTEL)
194
GIT_TAG f3b91559efca32c6b54837866ef35ba98ff5b2ca # stable/v0.5
195
)
196
FetchContent_MakeAvailable(Corrosion)
198
- corrosion_import_crate(MANIFEST_PATH src/crates/jf/Cargo.toml
199
- CRATES journal_reader_ffi otel-plugin)
197
+
198
+ if(ENABLE_NETDATA_JOURNAL_FILE_READER AND ENABLE_PLUGIN_OTEL)
199
+ corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
200
+ CRATES journal-viewer-plugin otel-plugin)
201
+ elseif(ENABLE_NETDATA_JOURNAL_FILE_READER)
202
+ corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
203
+ CRATES journal-viewer-plugin)
204
+ elseif(ENABLE_PLUGIN_OTEL)
205
+ corrosion_import_crate(MANIFEST_PATH src/crates/Cargo.toml
206
+ CRATES otel-plugin)
207
+ endif()
208
+
209
+ if(ENABLE_NETDATA_JOURNAL_FILE_READER)
210
+ if(STATIC_BUILD)
211
+ corrosion_add_target_rustflags(journal-viewer-plugin --cfg=tracing_unstable "-C" "target-feature=+crt-static")
212
+ else()
213
+ corrosion_add_target_rustflags(journal-viewer-plugin --cfg=tracing_unstable)
214
+ endif()
215
+ endif()
216
+
217
+ if(ENABLE_PLUGIN_OTEL)
218
+ if(STATIC_BUILD)
219
+ corrosion_add_target_rustflags(otel-plugin --cfg=tracing_unstable "-C" "target-feature=+crt-static")
220
+ else()
221
+ corrosion_add_target_rustflags(otel-plugin --cfg=tracing_unstable)
222
+ endif()
223
+ endif()
224
endif()
225
226
option(ENABLE_MIMALLOC "Enable mimalloc allocator" OFF)
@@ -1674,23 +1698,6 @@ set(STATSD_PLUGIN_FILES
1698
src/collectors/statsd.plugin/statsd.c
1699
)
1700
1677
-set(SYSTEMD_JOURNAL_PLUGIN_FILES
1678
- src/collectors/systemd-journal.plugin/systemd-journal.c
1679
- src/collectors/systemd-journal.plugin/systemd-journal-fstat.c
1680
- src/collectors/systemd-journal.plugin/systemd-internals.h
1681
- src/collectors/systemd-journal.plugin/systemd-main.c
1682
- src/collectors/systemd-journal.plugin/systemd-journal.c
1683
- src/collectors/systemd-journal.plugin/systemd-journal-annotations.c
1684
- src/collectors/systemd-journal.plugin/systemd-journal-files.c
1685
- src/collectors/systemd-journal.plugin/systemd-journal-watcher.c
1686
- src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
1687
- src/collectors/systemd-journal.plugin/provider/netdata_provider.c
1688
- src/collectors/systemd-journal.plugin/provider/netdata_provider.h
1689
- src/collectors/systemd-journal.plugin/provider/rust_provider.h
1690
- src/libnetdata/os/system-maps/system-services.h
1691
- src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
1692
-)
1693
-
1701
set(SYSTEMD_UNITS_PLUGIN_FILES
1702
src/collectors/systemd-units.plugin/plugin_systemd_units.c
1703
)
@@ -2745,36 +2752,6 @@ if(ENABLE_PLUGIN_CGROUP_NETWORK)
2752
DESTINATION usr/libexec/netdata/plugins.d)
2753
endif()
2754
2748
-# Enable rust implementation if we don't have systemd and we want the journal plugin
2749
-if(ENABLE_PLUGIN_SYSTEMD_JOURNAL AND NOT SYSTEMD_FOUND)
2750
- if (NOT ENABLE_NETDATA_JOURNAL_FILE_READER)
2751
- message(WARNING "Systemd journal package not found, will try netdata's journal reader which requires cargo.")
2752
- set(ENABLE_NETDATA_JOURNAL_FILE_READER True)
2753
- endif()
2754
-endif()
2755
-
2756
-if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
2757
- add_executable(systemd-journal.plugin ${SYSTEMD_JOURNAL_PLUGIN_FILES})
2758
-
2759
- if(ENABLE_NETDATA_JOURNAL_FILE_READER)
2760
- target_compile_definitions(systemd-journal.plugin PRIVATE HAVE_RUST_PROVIDER)
2761
- target_link_libraries(systemd-journal.plugin journal_reader_ffi)
2762
- endif()
2763
-
2764
- target_link_libraries(systemd-journal.plugin libnetdata)
2765
-
2766
- install(TARGETS systemd-journal.plugin
2767
- COMPONENT plugin-systemd-journal
2768
- DESTINATION usr/libexec/netdata/plugins.d)
2769
-
2770
- if(BUILD_FOR_PACKAGING)
2771
- install(FILES
2772
- ${PKG_FILES_PATH}/copyright
2773
- COMPONENT plugin-systemd-journal
2774
- DESTINATION usr/share/doc/netdata-plugin-systemd-journal)
2775
- endif()
2776
-endif()
2777
-
2755
if(ENABLE_PLUGIN_SYSTEMD_UNITS)
2756
if(NOT SYSTEMD_FOUND)
2757
message(FATAL_ERROR "Systemd units plugin requires systemd, but systemd was not found.")
@@ -3242,17 +3219,28 @@ if(ENABLE_PLUGIN_OTEL)
3219
RUNTIME DESTINATION usr/libexec/netdata/plugins.d
3220
COMPONENT plugin-otel)
3221
3245
- install(FILES src/crates/jf/otel-plugin/configs/otel.yml
3222
+ install(FILES src/crates/netdata-otel/otel-plugin/configs/otel.yaml
3223
COMPONENT plugin-otel
3224
DESTINATION usr/lib/netdata/conf.d)
3225
3249
- install(FILES src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yml
3226
+ install(FILES src/crates/netdata-otel/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yaml
3227
COMPONENT plugin-otel
3228
DESTINATION usr/lib/netdata/conf.d/otel.d/v1/metrics)
3229
3230
install(DIRECTORY COMPONENT plugin-otel DESTINATION etc/netdata/otel.d/v1/metrics)
3231
endif()
3232
3233
+#
3234
+# journal-viewer-plugin
3235
+#
3236
+# Handle journal-viewer plugin
3237
+if(ENABLE_NETDATA_JOURNAL_FILE_READER)
3238
+ corrosion_install(TARGETS journal-viewer-plugin
3239
+ PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE
3240
+ RUNTIME DESTINATION usr/libexec/netdata/plugins.d
3241
+ COMPONENT plugin-journal-viewer)
3242
+endif()
3243
+
3244
#
3245
# Generate config file
3246
#
@@ -3327,6 +3315,16 @@ set(logdir_POST "${LOG_DIR}")
3315
set(netdata_user_POST "${NETDATA_USER}")
3316
set(netdata_group_POST "${NETDATA_USER}")
3317
3318
+if(ENABLE_NETDATA_JOURNAL_FILE_READER)
3319
+ configure_file(src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml.in
3320
+ src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml
3321
+ @ONLY)
3322
+
3323
+ install(FILES ${CMAKE_BINARY_DIR}/src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml
3324
+ COMPONENT plugin-journal-viewer
3325
+ DESTINATION usr/lib/netdata/conf.d)
3326
+endif()
3327
+
3328
if(NOT OS_WINDOWS)
3329
configure_file(src/claim/netdata-claim.sh.in src/claim/netdata-claim.sh @ONLY)
3330
install(PROGRAMS
@@ -3745,7 +3743,6 @@ endif()
3743
3744
# confs
3745
install(FILES
3748
- src/collectors/systemd-journal.plugin/schema.d/systemd-journal%3Amonitored-directories.json
3746
src/health/schema.d/health%3Aalert%3Aprototype.json
3747
COMPONENT netdata
3748
DESTINATION usr/lib/netdata/conf.d/schema.d)
docs/.map/map.csv
+4
-4
@@ -114,7 +114,7 @@ https://github.com/netdata/netdata/edit/master/src/collectors/README.md,Collecti
114
https://github.com/netdata/netdata/edit/master/src/collectors/REFERENCE.md,Collectors configuration,Published,Collecting Metrics,,
115
https://github.com/netdata/agent-service-discovery/edit/master/README.md,Service discovery,Published,Collecting Metrics,,
116
https://github.com/netdata/netdata/edit/master/src/collectors/statsd.plugin/README.md,StatsD,Published,Collecting Metrics,,
117
-https://github.com/netdata/netdata/edit/master/src/crates/jf/otel-plugin/README.md,OpenTelemetry Metrics,Published,Collecting Metrics/OpenTelemetry,,Ingesting storing and visualizing OpenTelemetry metrics
117
+https://github.com/netdata/netdata/edit/master/src/crates/netdata-otel/otel-plugin/README.md,OpenTelemetry Metrics,Published,Collecting Metrics/OpenTelemetry,,Ingesting storing and visualizing OpenTelemetry metrics
118
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/README.md,Metrics Centralization Points,Published,Netdata Parents/Metrics Centralization Points,,
119
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/configuration.md,Configuring Metrics Centralization Points,Published,Netdata Parents/Metrics Centralization Points,,
120
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/metrics-centralization-points/sizing-netdata-parents.md,Sizing Netdata Parents,Published,Netdata Parents/Metrics Centralization Points,,
@@ -140,15 +140,15 @@ https://github.com/netdata/netdata/edit/master/src/web/api/exporters/shell/READM
140
,,,,,
141
,,,,,
142
logs_integrations,,,,,
143
-https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/README.md,Systemd Journal Plugin Reference,Published,Logs/Systemd Journal Logs,,View and analyze logs available in systemd journal
144
-https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/forward_secure_sealing.md,Forward Secure Sealing (FSS) in Systemd-Journal,Published,Logs/Systemd Journal Logs,,
143
+https://github.com/netdata/netdata/edit/master/docs/logs/README.md,Journal Viewer Plugin,Published,Logs/Systemd Journal Logs,,View and analyze logs available in systemd journal
144
+https://github.com/netdata/netdata/edit/master/docs/logs/forward_secure_sealing.md,Forward Secure Sealing (FSS) in Systemd-Journal,Published,Logs/Systemd Journal Logs,,
145
https://github.com/netdata/netdata/edit/master/src/collectors/windows-events.plugin/README.md,Windows Events Plugin Reference,Published,Logs/Windows Event Logs,,
146
https://github.com/netdata/netdata/edit/master/src/collectors/log2journal/README.md,log2journal,Published,Logs/log2journal,,
147
https://github.com/netdata/netdata/edit/master/src/libnetdata/log/systemd-cat-native.md,systemd-cat-native,Published,Logs/systemd-cat-native,,
148
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/README.md,Logs Centralization Points with systemd-journald,Published,Logs/Logs Centralization Points with systemd-journald,,
149
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-with-encryption-using-self-signed-certificates.md,Passive journal centralization with encryption using self-signed certificates,Published,Logs/Logs Centralization Points with systemd-journald,,
150
https://github.com/netdata/netdata/edit/master/docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-without-encryption.md,Passive journal centralization without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
151
-https://github.com/netdata/netdata/edit/master/src/collectors/systemd-journal.plugin/active_journal_centralization_guide_no_encryption.md,Active journal source without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
151
+https://github.com/netdata/netdata/edit/master/docs/logs/active_journal_centralization_guide_no_encryption.md,Active journal source without encryption,Published,Logs/Logs Centralization Points with systemd-journald,,
152
,,,,,
153
,,,,,
154
https://github.com/netdata/netdata/edit/master/docs/top-monitoring-netdata-functions.md,Top Consumers,Published,Top Consumers,,Present the Netdata Functions what these are and why they should be used.
docs/category-overview-pages/working-with-logs.md
+1
-1
@@ -2,7 +2,7 @@
2
3
This section talks about the ways Netdata collects and visualizes logs.
4
5
-The [systemd journal plugin](/src/collectors/systemd-journal.plugin) is the core Netdata component for reading systemd journal logs.
5
+The [journal viewer plugin](/docs/logs/README.md) is the core Netdata component for reading systemd journal logs.
6
7
For structured logs, Netdata provides tools like [log2journal](/src/collectors/log2journal/README.md) and [systemd-cat-native](/src/libnetdata/log/systemd-cat-native.md) to convert them into compatible systemd journal entries.
8
docs/dashboards-and-charts/logs-tab.md
+8
-8
@@ -1,16 +1,16 @@
1
# Logs tab
2
3
-The Logs tab is using the [`systemd` journal plugin](/src/collectors/systemd-journal.plugin/README.md), to present a structured view into your infrastructure's `systemd` logs.
3
+The Logs tab is using the [`journal-viewer` plugin](/docs/logs/README.md), to present a structured view into your infrastructure's `systemd` logs.
4
5
We have a thorough section explaining how you can [work with logs](https://learn.netdata.cloud/docs/logs), detailing how the plugin works, and what other utilities are used under the hood to provide you with the visualizations and the log entries.
6
7
-The [`systemd` journal plugin](/src/collectors/systemd-journal.plugin/README.md) documentation has information about:
7
+The [`journal-viewer` plugin](/docs/logs/README.md) documentation has information about:
8
9
-- [Key features the plugin provides](/src/collectors/systemd-journal.plugin/README.md#key-features)
10
-- [Journal sources](/src/collectors/systemd-journal.plugin/README.md#journal-sources)
11
-- [Journal fields](/src/collectors/systemd-journal.plugin/README.md#journal-fields)
12
-- [Full-text search](/src/collectors/systemd-journal.plugin/README.md#full-text-search)
13
-- [Query performance](/src/collectors/systemd-journal.plugin/README.md#query-performance)
14
-- [Performance at scale](/src/collectors/systemd-journal.plugin/README.md#performance-at-scale)
9
+- [Key features the plugin provides](/docs/logs/README.md#key-features)
10
+- [Journal sources](/docs/logs/README.md#journal-sources)
11
+- [Journal fields](/docs/logs/README.md#journal-fields)
12
+- [Full-text search](/docs/logs/README.md#full-text-search)
13
+- [Query performance](/docs/logs/README.md#query-performance)
14
+- [Performance at scale](/docs/logs/README.md#performance-at-scale)
15
16
We recommend you to read through that document, to better understand how the plugin and the visualizations work.
docs/developer-and-contributor-corner/dyncfg.md
+4
-5
@@ -150,13 +150,12 @@ The health module manages alert definitions through DynCfg:
150
</details>
151
152
<details>
153
-<summary><strong>systemd-journal.plugin (External Plugin, C)</strong></summary><br/>
153
+<summary><strong>journal-viewer-plugin (External Plugin, Rust)</strong></summary><br/>
154
155
-External C plugin that manages journal directory configurations:
155
+External Rust plugin that provides systemd journal log viewing and analysis:
156
157
-- **File**: `src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c`
158
-- **Pattern**: SINGLE configuration type
159
-- **Use Case**: Managing journal directory paths
157
+- **Location**: `src/crates/netdata-log-viewer/journal-viewer-plugin/`
158
+- **Use Case**: Viewing and analyzing systemd journal logs
159
160
<br/>
161
</details>
docs/logs/README.md
renamed
+4
-4
@@ -1,8 +1,8 @@
1
-# `systemd` journal plugin
1
+# `journal-viewer` plugin
2
3
[KEY FEATURES](#key-features) | [PREREQUISITES](#prerequisites) | [JOURNAL SOURCES](#journal-sources) | [JOURNAL FIELDS](#journal-fields) | [VISUALIZATION](#visualization-capabilities) | [PLAY MODE](#play-mode) | [FULL TEXT SEARCH](#full-text-search) | [QUERY PERFORMANCE](#query-performance) | [PERFORMANCE AT SCALE](#performance-at-scale) | [BEST PRACTICES](#best-practices-for-better-performance) | [CONFIGURATION](#configuration-and-maintenance) | [FAQ](#faq) | [HOW TO TROUBLESHOOT COMMON ISSUES](#how-to-troubleshoot-common-issues) | [HOW TO VERIFY SETUP](#how-to-verify-setup)
4
5
-The `systemd` journal plugin provides an efficient way to view, explore, and analyze `systemd` journal logs directly from the Netdata dashboard. It combines powerful filtering, real-time updates, and visual analysis tools to help you troubleshoot system issues effectively.
5
+The `journal-viewer` plugin provides an efficient way to view, explore, and analyze `systemd` journal logs directly from the Netdata dashboard. It combines powerful filtering, real-time updates, and visual analysis tools to help you troubleshoot system issues effectively.
6
7
8
@@ -544,7 +544,7 @@ Sampling ensures responsiveness at scale, but selecting sources and filters rema
544
545
| Error Message | Meaning | Solution |
546
|--------------|---------|----------|
547
-| "Plugin not available" | The systemd-journal plugin isn't loaded | Check your Netdata installation type (must not be Alpine or static) |
547
+| "Plugin not available" | The journal-viewer plugin isn't loaded | Check your Netdata installation type (must not be Alpine or static) |
548
| "Unable to open journal" | Permission issues accessing journal files | Ensure Netdata has proper permissions for journal directories |
549
| "Timeout while querying" | Query is taking too long to complete | Reduce the query scope with filters or shorter timeframes |
550
| "No sources detected" | Cannot find valid journal files | Check journal file locations and setup |
@@ -558,7 +558,7 @@ Sampling ensures responsiveness at scale, but selecting sources and filters rema
558
sudo netdata -W plugins
559
```
560
561
-Check that the `systemd-journal` plugin is listed as active.
561
+Check that the `journal-viewer-plugin` is listed as active.
562
563
### How to confirm journal sources are detected
564
docs/logs/active_journal_centralization_guide_no_encryption.md
renamed
docs/logs/forward_secure_sealing.md
renamed
docs/logs/systemd-journal-self-signed-certs.sh
renamed
docs/observability-centralization-points/logs-centralization-points-with-systemd-journald/passive-journal-centralization-with-encryption-using-self-signed-certificates.md
+2
-2
@@ -20,7 +20,7 @@ This means that if both certificates are issued by the same certificate authorit
20
21
## Self-signed certificates
22
23
-To simplify the process of creating and managing self-signed certificates, we have created [this bash script](https://github.com/netdata/netdata/blob/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh).
23
+To simplify the process of creating and managing self-signed certificates, we have created [this bash script](https://github.com/netdata/netdata/blob/master/docs/logs/systemd-journal-self-signed-certs.sh).
24
25
This helps to also automate the distribution of the certificates to your servers (it generates a new bash script for each of your servers, which includes everything required, including the certificates).
26
@@ -34,7 +34,7 @@ On the server that will issue the certificates (usually the centralization serve
34
sudo apt-get install systemd-journal-remote openssl
35
36
# download the script and make it executable
37
-curl >systemd-journal-self-signed-certs.sh "https://raw.githubusercontent.com/netdata/netdata/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh"
37
+curl >systemd-journal-self-signed-certs.sh "https://raw.githubusercontent.com/netdata/netdata/master/docs/logs/systemd-journal-self-signed-certs.sh"
38
chmod 750 systemd-journal-self-signed-certs.sh
39
```
40
docs/security-and-privacy-design/netdata-kubernetes.md
+1
-1
@@ -26,7 +26,7 @@ This architecture ensures that the components responsible for broader network ex
26
Even within the `child` agent container, which might be granted elevated permissions by Kubernetes (e.g., host mounts, capabilities), Netdata enforces strict internal privilege separation:
27
28
- **Unprivileged Core Daemon:** The main `netdata` daemon process, which orchestrates data collection, communicates with the parent, and handles basic tasks, runs as a **non-root user** (typically the `netdata` user). Key internal plugins responsible for collecting core metrics, such as `proc.plugin` (for `/proc` based metrics) and `cgroups.plugin` (for container metrics via `/sys/fs/cgroup`), also run as threads within this unprivileged daemon process.
29
-- **Dedicated Privileged Helpers/Plugins:** Operations that require elevated privileges or specific capabilities to access protected host resources (e.g., detailed process information via `apps.plugin`, certain network operations via `network-viewer.plugin` or `cgroup-network`, logs querying via `systemd-journal.plugin`) are delegated to **dedicated external plugins or helper processes**.
29
+- **Dedicated Privileged Helpers/Plugins:** Operations that require elevated privileges or specific capabilities to access protected host resources (e.g., detailed process information via `apps.plugin`, certain network operations via `network-viewer.plugin` or `cgroup-network`, logs querying via `journal-viewer-plugin`) are delegated to **dedicated external plugins or helper processes**.
30
- **Isolation via `setuid`:** These external plugins/helpers run as separate, isolated processes spawned by the main daemon. Within the container, the specific helper executables requiring elevated rights are configured with the **`setuid` root** permission bit. This standard Linux mechanism allows only these designated programs to execute specific, pre-defined tasks with root privileges _within the container's environment_.
31
- **Contained Capabilities:** Consequently, even if the _container_ is granted capabilities like `SYS_ADMIN` or `SYS_PTRACE`, the unprivileged main `netdata` daemon and its non-`setuid` plugins **cannot** directly utilize these capabilities. Only the specific `setuid` root helpers designed for tasks needing those privileges can effectively leverage them.
32
- **Narrowly Scoped Tasks:** Each privileged helper is designed to perform a narrow, hard-coded task – typically reading specific system information and passing it back to the main daemon.
docs/top-monitoring-netdata-functions.md
+2
-2
@@ -17,8 +17,8 @@ Beyond their primary roles of collecting metrics, collectors can execute specifi
17
| Network-connections | Real-time monitoring of all network connections, showing established connections, ports, protocols, and connection states across TCP/UDP services. | `netstat`, `ss` | yes | [network-viewer](https://github.com/netdata/netdata/tree/master/src/collectors/network-viewer.plugin) |
18
| Network-interfaces | Network traffic, packet drop rates, interface states, MTU, speed, and duplex mode for all network interfaces. | `bmon`, `bwm-ng` | no | [proc](https://github.com/netdata/netdata/tree/master/src/collectors/proc.plugin#readme) |
19
| Processes | Real-time information about the system's resource usage, including CPU utilization, memory consumption, and disk IO for every running process. | `top`, `htop` | yes | [apps](/src/collectors/apps.plugin/README.md) |
20
-| Systemd-journal | Viewing, exploring and analyzing systemd journal logs. | `journalctl` | yes | [systemd-journal](https://github.com/netdata/netdata/tree/master/src/collectors/systemd-journal.plugin#readme) |
21
-| Systemd-list-units | Information about all systemd units, including their active state, description, whether or not they are enabled, and more. | `systemctl list-units` | yes | [systemd-journal](https://github.com/netdata/netdata/tree/master/src/collectors/systemd-journal.plugin#readme) |
20
+| Systemd-journal | Viewing, exploring and analyzing systemd journal logs. | `journalctl` | yes | [journal-viewer](/docs/logs/README.md) |
21
+| Systemd-list-units | Information about all systemd units, including their active state, description, whether or not they are enabled, and more. | `systemctl list-units` | yes | [journal-viewer](/docs/logs/README.md) |
22
| Systemd-services | System resource utilization for all running systemd services: CPU, memory, and disk IO. | `systemd-cgtop` | no | [cgroups](https://github.com/netdata/netdata/tree/master/src/collectors/cgroups.plugin#readme) |
23
| Netdata-api-calls | Real-time tracing of API calls made to the Netdata Agent. It provides information on query, source, status, elapsed time, and more. | | yes | |
24
| Netdata-streaming | Comprehensive overview of all Netdata children instances, offering detailed information about their status, replication completion time, and many more. | | yes | |
integrations/logs/metadata.yaml
+1
-1
@@ -3,7 +3,7 @@
3
- id: "logs-systemd-journal"
4
meta:
5
name: "Systemd Journal Logs"
6
- link: "https://github.com/netdata/netdata/blob/master/src/collectors/systemd-journal.plugin/README.md"
6
+ link: "https://github.com/netdata/netdata/blob/master/docs/logs/README.md"
7
categories:
8
- logs
9
icon_filename: "netdata.png"
netdata-installer.sh
+7
-9
@@ -213,11 +213,10 @@ USAGE: ${PROGRAM} [options]
213
--disable-plugin-nfacct Explicitly disable the nfacct plugin.
214
--enable-plugin-xenstat Enable the xenstat plugin. Default: enable it when libxenstat and libyajl are available.
215
--disable-plugin-xenstat Explicitly disable the xenstat plugin.
216
- --enable-plugin-systemd-journal Enable the systemd journal plugin. Default: enable it when libsystemd is available.
217
- --disable-plugin-systemd-journal Explicitly disable the systemd journal plugin.
218
- --internal-systemd-journal Enable the internal journal file reader instead of using libsystemd
216
--enable-plugin-otel Enable the Netdata OpenTelemetry plugin. Default: disabled
217
--disable-plugin-otel Explicitly disable the Netdata OpenTelemetry plugin.
218
+ --enable-plugin-systemd-journal Enable the systemd journal reader plugin. Default: enabled if Rust is available and the build isn’t known broken
219
+ --disable-plugin-systemd-journal Explicitly disable the systemd journal reader plugin.
220
--enable-plugin-ibm Enable the IBM ecosystem monitoring plugin. Default: disabled
221
--disable-plugin-ibm Explicitly disable the IBM ecosystem monitoring plugin.
222
--enable-exporting-kinesis Enable AWS Kinesis exporting connector. Default: enable it when libaws_cpp_sdk_kinesis
@@ -301,7 +300,6 @@ while [ -n "${1}" ]; do
300
"--disable-plugin-xenstat") ENABLE_XENSTAT=0 ;;
301
"--enable-plugin-systemd-journal") ENABLE_SYSTEMD_JOURNAL=1 ;;
302
"--disable-plugin-systemd-journal") ENABLE_SYSTEMD_JOURNAL=0 ;;
304
- "--internal-systemd-journal") USE_RUST_JOURNAL_FILE=1 ;;
303
"--enable-plugin-otel") ENABLE_OTEL=1 ;;
304
"--disable-plugin-otel") ENABLE_OTEL=0 ;;
305
"--enable-plugin-ibm") ENABLE_IBM=1 ;;
@@ -819,18 +817,18 @@ if [ "$(id -u)" -eq 0 ]; then
817
fi
818
fi
819
822
- if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin" ]; then
823
- run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
820
+ if [ -f "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/journal-viewer-plugin" ]; then
821
+ run chown "root:${NETDATA_GROUP}" "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/journal-viewer-plugin"
822
capabilities=0
823
if ! iscontainer && command -v setcap 1> /dev/null 2>&1; then
826
- run chmod 0750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
827
- if run setcap cap_dac_read_search+ep "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"; then
824
+ run chmod 0750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/journal-viewer-plugin"
825
+ if run setcap cap_dac_read_search+ep "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/journal-viewer-plugin"; then
826
capabilities=1
827
fi
828
fi
829
830
if [ $capabilities -eq 0 ]; then
833
- run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/systemd-journal.plugin"
831
+ run chmod 4750 "${NETDATA_PREFIX}/usr/libexec/netdata/plugins.d/journal-viewer-plugin"
832
fi
833
fi
834
netdata.spec.in
+15
-16
@@ -270,11 +270,11 @@ Suggests: %{name}-plugin-freeipmi = %{version}
270
%if %{_have_cups}
271
Suggests: %{name}-plugin-cups = %{version}
272
%endif
273
-Recommends: %{name}-plugin-systemd-journal = %{version}
273
+Recommends: %{name}-plugin-journal-viewer = %{version}
274
Recommends: %{name}-plugin-systmed-units = %{version}
275
Recommends: %{name}-plugin-network-viewer = %{version}
276
%else
277
-Requires: %{name}-plugin-systemd-journal = %{version}
277
+Requires: %{name}-plugin-journal-viewer = %{version}
278
Requires: %{name}-plugin-network-viewer = %{version}
279
%endif
280
@@ -327,10 +327,6 @@ BuildRequires: golang >= 1.25
327
%endif
328
# end - go.d.plugin plugin dependencies
329
330
-# systemd-journal dependencies
331
-BuildRequires: pkgconfig(libsystemd)
332
-# end - systemd-journal dependencies
333
-
330
# Prometheus remote write dependencies
331
%if 0%{?suse_version}
332
BuildRequires: snappy-devel
@@ -702,8 +698,9 @@ rm -rf "${RPM_BUILD_ROOT}"
698
# perf belongs to a different sub-package
699
%exclude %{_libexecdir}/%{name}/plugins.d/perf.plugin
700
705
-# systemd-journal belongs to a different sub-package
706
-%exclude %{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin
701
+# journal-viewer belongs to a different sub-package
702
+%exclude %{_libexecdir}/%{name}/plugins.d/journal-viewer-plugin
703
+%exclude %{_libdir}/%{name}/conf.d/journal-viewer.yaml
704
705
# xenstat belongs to a different sub-package
706
%exclude %{_libexecdir}/%{name}/plugins.d/xenstat.plugin
@@ -3121,23 +3118,25 @@ Requires(pre): %{name}-user >= %{version}
3118
# CAP_DAC_READ_SEARCH required for data collection.
3119
%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/debugfs.plugin
3120
3124
-%package plugin-systemd-journal
3125
-Summary: The systemd-journal plugin for the Netdata Agent
3121
+%package plugin-journal-viewer
3122
+Summary: The journal viewer plugin for the Netdata Agent
3123
Group: Applications/System
3124
Requires: %{name} = %{version}
3125
Conflicts: %{name} < %{version}
3126
+Conflicts: %{name}-plugin-systemd-journal
3127
+Obsoletes: %{name}-plugin-systemd-journal < %{version}
3128
%if ! %{_have_sysuser}
3129
Requires(pre): %{name}-user >= %{version}
3130
%endif
3131
3133
-%description plugin-systemd-journal
3134
- This plugin allows the Netdata Agent to present entries from the systemd
3135
- journal on Netdata Cloud or the local Agent Dashboard.
3132
+%description plugin-journal-viewer
3133
+ This plugin provides systemd journal log viewing and querying functionality
3134
+ with histogram analysis and faceted search capabilities for the Netdata Agent.
3135
3137
-%files plugin-systemd-journal
3136
+%files plugin-journal-viewer
3137
%defattr(0750,root,netdata,0750)
3139
-# CAP_DAC_READ_SEARCH required for data collection.
3140
-%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/systemd-journal.plugin
3138
+# CAP_DAC_READ_SEARCH required for reading journal files.
3139
+%caps(cap_dac_read_search=ep) %attr(0750,root,netdata) %{_libexecdir}/%{name}/plugins.d/journal-viewer-plugin
3140
3141
%if 0%{?centos_ver} != 7 && 0%{?amazon_linux} != 2
3142
%package plugin-systemd-units
packaging/cmake/Modules/Packaging.cmake
+19
-15
@@ -97,7 +97,7 @@ set(CPACK_DEBIAN_NETDATA_PACKAGE_PREDEPENDS "netdata-user, libcap2-bin")
97
set(CPACK_DEBIAN_NETDATA_PACKAGE_SUGGESTS
98
"netdata-plugin-cups, netdata-plugin-freeipmi, netdata-plugin-ibm")
99
set(CPACK_DEBIAN_NETDATA_PACKAGE_RECOMMENDS
100
- "netdata-plugin-systemd-journal, netdata-plugin-systemd-units, \
100
+ "netdata-plugin-journal-viewer, netdata-plugin-systemd-units, \
101
netdata-plugin-network-viewer")
102
set(CPACK_DEBIAN_NETDATA_PACKAGE_CONFLICTS
103
"netdata-core, netdata-plugins-bash, netdata-plugins-python, netdata-web")
@@ -518,23 +518,27 @@ set(CPACK_DEBIAN_PLUGIN-SLABINFO_PACKAGE_CONTROL_EXTRA
518
set(CPACK_DEBIAN_PLUGIN-SLABINFO-DEBUGINFO_PACKAGE On)
519
520
#
521
-# systemd-journal.plugin
521
+# journal-viewer.plugin
522
#
523
524
-set(CPACK_COMPONENT_PLUGIN-SYSTEMD-JOURNAL_DEPENDS "netdata")
525
-set(CPACK_COMPONENT_PLUGIN-SYSTEMD-JOURNAL_DESCRIPTION
526
- "The systemd-journal collector for the Netdata Agent
527
- This plugin allows the Netdata Agent to present logs from the systemd
528
- journal on Netdata Cloud or the local Agent dashboard.")
524
+set(CPACK_COMPONENT_PLUGIN-JOURNAL-VIEWER_DEPENDS "netdata")
525
+set(CPACK_COMPONENT_PLUGIN-JOURNAL-VIEWER_DESCRIPTION
526
+ "The journal viewer plugin for the Netdata Agent
527
+ This plugin provides systemd journal log viewing and querying functionality
528
+ with histogram analysis and faceted search capabilities.")
529
530
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_PACKAGE_NAME "netdata-plugin-systemd-journal")
531
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_PACKAGE_SECTION "net")
532
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_PACKAGE_PREDEPENDS "libcap2-bin, adduser")
530
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_NAME "netdata-plugin-journal-viewer")
531
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_SECTION "net")
532
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_PREDEPENDS "libcap2-bin, adduser")
533
534
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_PACKAGE_CONTROL_EXTRA
535
- "${PKG_FILES_PATH}/deb/plugin-systemd-journal/postinst")
534
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_CONTROL_EXTRA
535
+ "${PKG_FILES_PATH}/deb/plugin-journal-viewer/postinst")
536
537
-set(CPACK_DEBIAN_PLUGIN-SYSTEMD-JOURNAL_DEBUGINFO_PACKAGE On)
537
+# Replaces old systemd-journal plugin
538
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_CONFLICTS "netdata-plugin-systemd-journal")
539
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_PACKAGE_REPLACES "netdata-plugin-systemd-journal")
540
+
541
+set(CPACK_DEBIAN_PLUGIN-JOURNAL-VIEWER_DEBUGINFO_PACKAGE Off)
542
543
#
544
# systemd-units.plugin
@@ -626,8 +630,8 @@ endif()
630
if(ENABLE_PLUGIN_SLABINFO)
631
list(APPEND CPACK_COMPONENTS_ALL "plugin-slabinfo")
632
endif()
629
-if(ENABLE_PLUGIN_SYSTEMD_JOURNAL)
630
- list(APPEND CPACK_COMPONENTS_ALL "plugin-systemd-journal")
633
+if(ENABLE_PLUGIN_OTEL)
634
+ list(APPEND CPACK_COMPONENTS_ALL "plugin-journal-viewer")
635
endif()
636
if(ENABLE_PLUGIN_SYSTEMD_UNITS)
637
list(APPEND CPACK_COMPONENTS_ALL "plugin-systemd-units")
packaging/cmake/pkg-files/deb/plugin-journal-viewer/postinst
new
+15
@@ -0,0 +1,15 @@
1
+#!/bin/sh
2
+
3
+set -e
4
+
5
+case "$1" in
6
+ configure|reconfigure)
7
+ chown root:netdata /usr/libexec/netdata/plugins.d/journal-viewer-plugin
8
+ chmod 0750 /usr/libexec/netdata/plugins.d/journal-viewer-plugin
9
+ if ! setcap "cap_dac_read_search=eip" /usr/libexec/netdata/plugins.d/journal-viewer-plugin; then
10
+ chmod -f 4750 /usr/libexec/netdata/plugins.d/journal-viewer-plugin
11
+ fi
12
+ ;;
13
+esac
14
+
15
+exit 0
packaging/cmake/pkg-files/deb/plugin-systemd-journal/postinst
deleted
-15
@@ -1,15 +0,0 @@
1
-#!/bin/sh
2
-
3
-set -e
4
-
5
-case "$1" in
6
- configure|reconfigure)
7
- chown root:netdata /usr/libexec/netdata/plugins.d/systemd-journal.plugin
8
- chmod 0750 /usr/libexec/netdata/plugins.d/systemd-journal.plugin
9
- if ! setcap "cap_dac_read_search=eip" /usr/libexec/netdata/plugins.d/systemd-journal.plugin; then
10
- chmod -f 4750 /usr/libexec/netdata/plugins.d/systemd-journal.plugin
11
- fi
12
- ;;
13
-esac
14
-
15
-exit 0
packaging/docker/Dockerfile
+34
-21
@@ -19,22 +19,33 @@ ARG DEBUG_BUILD
19
20
ENV DEBUG_BUILD=$DEBUG_BUILD
21
22
+ARG BUILD_ARCH
23
+
24
+ENV BUILD_ARCH=$BUILD_ARCH
25
+
26
# Copy source
27
COPY . /opt/netdata.git
28
WORKDIR /opt/netdata.git
29
30
# Install from source
31
RUN chmod +x netdata-installer.sh && \
28
- cp -rp /deps/* /usr/local/ && \
29
- /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='$(uname -m)'" > ./system/.install-type && \
30
- CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it \
31
- --use-system-protobuf \
32
- --disable-ebpf \
33
- --enable-plugin-otel \
34
- --internal-systemd-journal \
35
- ${EXTRA_INSTALL_OPTS} \
36
- --install-no-prefix / \
37
- "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)"
32
+ cp -rp /deps/* /usr/local/ && \
33
+ BUILD_ARCH="${BUILD_ARCH:-"$(uname -m)"}" && \
34
+ /bin/echo -e "INSTALL_TYPE='oci'\nPREBUILT_ARCH='${BUILD_ARCH}'" > ./system/.install-type && \
35
+ if [ "${BUILD_ARCH}" = "armv7l" ]; then \
36
+ EXTRA_INSTALL_OPTS="${EXTRA_INSTALL_OPTS} --disable-plugin-systemd-journal" ; \
37
+ else \
38
+ EXTRA_INSTALL_OPTS="${EXTRA_INSTALL_OPTS} --enable-plugin-systemd-journal" ; \
39
+ fi && \
40
+ NETDATA_BUILD_DIR=/build \
41
+ CFLAGS="$(packaging/docker/gen-cflags.sh)" LDFLAGS="-Wl,--gc-sections" ./netdata-installer.sh --dont-wait --dont-start-it \
42
+ --use-system-protobuf \
43
+ --disable-ebpf \
44
+ --enable-plugin-otel \
45
+ ${EXTRA_INSTALL_OPTS} \
46
+ --install-no-prefix / \
47
+ "$([ "$RELEASE_CHANNEL" = stable ] && echo --stable-channel)" && \
48
+ rm -rf /build /root/.cargo
49
50
# files to one directory
51
RUN mkdir -p /app/usr/sbin/ \
@@ -109,20 +120,20 @@ COPY --from=builder /app /
120
# Create netdata user and apply the permissions as described in
121
# https://docs.netdata.cloud/docs/netdata-security/#netdata-directories, but own everything by root group due to https://github.com/netdata/netdata/pull/6543
122
# hadolint ignore=DL3013
112
-RUN addgroup --gid ${NETDATA_GID} --system "${DOCKER_GRP}" && \
113
- adduser --system --no-create-home --shell /usr/sbin/nologin --uid ${NETDATA_UID} --home /etc/netdata --group "${DOCKER_USR}" && \
114
- chown -R root:root \
123
+RUN addgroup --stdoutmsglevel=info --gid ${NETDATA_GID} --system "${DOCKER_GRP}" && \
124
+ adduser --stdoutmsglevel=info --system --no-create-home --shell /usr/sbin/nologin --uid ${NETDATA_UID} --home /etc/netdata --group "${DOCKER_USR}" && \
125
+ chown -vR root:root \
126
/etc/netdata \
127
/usr/share/netdata \
128
/usr/libexec/netdata && \
118
- chown -R netdata:root \
129
+ chown -vR netdata:root \
130
/usr/lib/netdata \
131
/var/cache/netdata \
132
/var/lib/netdata \
133
/var/log/netdata && \
123
- chown -R netdata:netdata /var/lib/netdata/cloud.d && \
124
- chmod 0700 /var/lib/netdata/cloud.d && \
125
- chmod 0755 /usr/libexec/netdata/plugins.d/*.plugin && \
134
+ chown -vR netdata:netdata /var/lib/netdata/cloud.d && \
135
+ chmod -v 0700 /var/lib/netdata/cloud.d && \
136
+ chmod -v 0755 /usr/libexec/netdata/plugins.d/*.plugin && \
137
for name in cgroup-network \
138
local-listeners \
139
apps.plugin \
@@ -134,12 +145,14 @@ RUN addgroup --gid ${NETDATA_GID} --system "${DOCKER_GRP}" && \
145
slabinfo.plugin \
146
network-viewer.plugin \
147
otel-plugin \
137
- systemd-journal.plugin; do \
138
- [ -f "/usr/libexec/netdata/plugins.d/$name" ] && chmod 4755 "/usr/libexec/netdata/plugins.d/$name"; \
148
+ journal-viewer-plugin; do \
149
+ if [ -f "/usr/libexec/netdata/plugins.d/$name" ] ; then \
150
+ chmod -v 4755 "/usr/libexec/netdata/plugins.d/$name"; \
151
+ fi \
152
done && \
153
# Group write permissions due to: https://github.com/netdata/netdata/pull/6543
141
- find /var/lib/netdata /var/cache/netdata -type d -exec chmod 0770 {} \; && \
142
- find /var/lib/netdata /var/cache/netdata -type f -exec chmod 0660 {} \; && \
154
+ find /var/lib/netdata /var/cache/netdata -type d -exec chmod -v 0770 {} \; && \
155
+ find /var/lib/netdata /var/cache/netdata -type f -exec chmod -v 0660 {} \; && \
156
cp -va /etc/netdata /etc/netdata.stock
157
158
ENTRYPOINT ["/usr/sbin/run.sh"]
packaging/docker/README.md
+3
-3
@@ -38,8 +38,8 @@ The Netdata container requires specific **privileges** and **mounts** to provide
38
| go.d.plugin | /var/log | Web servers logs tailing. See [weblog](https://github.com/netdata/go.d.plugin/tree/master/modules/weblog#readme) collector. |
39
| apps.plugin | /etc/passwd, /etc/group | Monitoring of host system resource usage by each user and user group. |
40
| proc.plugin | /proc | Host system monitoring (CPU, memory, network interfaces, disks, etc.). |
41
-| systemd-journal.plugin | /var/log | Viewing, exploring and analyzing systemd journal logs. |
42
-| systemd-journal.plugin | /run/dbus | Systemd-list-units function: information about all systemd units, including their active state, description, whether they are enabled, and more. |
41
+| journal-viewer-plugin | /var/log | Viewing, exploring and analyzing systemd journal logs. |
42
+| systemd-units.plugin | /run/dbus | Systemd-list-units function: information about all systemd units, including their active state, description, whether they are enabled, and more. |
43
| go.d.plugin | /run/dbus | [go.d/systemdunits](https://github.com/netdata/go.d.plugin/tree/master/modules/systemdunits#readme) |
44
45
</details>
@@ -475,7 +475,7 @@ The following components won't work:
475
- freeipmi.plugin
476
- perf.plugin
477
- slabinfo.plugin
478
-- systemd-journal.plugin
478
+- journal-viewer-plugin
479
480
This method creates a [volume](https://docs.docker.com/storage/volumes/) for Netdata's configuration files
481
_within the container_ at `/etc/netdata`.
packaging/installer/functions.sh
+3
-13
@@ -330,24 +330,14 @@ prepare_cmake_options() {
330
fi
331
332
if [ -z "${ENABLE_SYSTEMD_JOURNAL}" ]; then
333
- if check_for_module libsystemd; then
334
- if check_for_module libelogind; then
335
- ENABLE_SYSTEMD_JOURNAL=0
336
- else
337
- ENABLE_SYSTEMD_JOURNAL=1
338
- fi
333
+ if command -v rustc > /dev/null 2>&1 && [ "$(uname -m)" != 'armv7l' ] && [ "$(uname -m)" != 'armv6l' ]; then
334
+ ENABLE_SYSTEMD_JOURNAL=1
335
else
336
ENABLE_SYSTEMD_JOURNAL=0
337
fi
338
fi
339
344
- enable_feature PLUGIN_SYSTEMD_JOURNAL "${ENABLE_SYSTEMD_JOURNAL}"
345
-
346
- if [ "${ENABLE_SYSTEMD_JOURNAL}" -eq 1 ] && [ -n "${USE_RUST_JOURNAL_FILE}" ]; then
347
- enable_feature NETDATA_JOURNAL_FILE_READER 1
348
- else
349
- enable_feature NETDATA_JOURNAL_FILE_READER 0
350
- fi
340
+ enable_feature NETDATA_JOURNAL_FILE_READER "${ENABLE_SYSTEMD_JOURNAL}"
341
342
if check_for_module 'libsystemd >= 221'; then
343
enable_feature PLUGIN_SYSTEMD_UNITS 1
packaging/installer/netdata-updater.sh
-13
@@ -1262,19 +1262,6 @@ update_binpkg() {
1262
# shellcheck disable=SC2086
1263
env ${env} ${pm_cmd} ${upgrade_subcmd} ${pkg_install_opts} netdata >&3 2>&3 || fatal "Failed to update Netdata package." U000F
1264
1265
- if ${pkg_installed_check} systemd > /dev/null 2>&1; then
1266
- if [ "${NETDATA_NO_SYSTEMD_JOURNAL}" -eq 0 ]; then
1267
- if ! ${pkg_installed_check} netdata-plugin-systemd-journal > /dev/null 2>&1; then
1268
- env ${env} ${pm_cmd} ${install_subcmd} ${pkg_install_opts} netdata-plugin-systemd-journal >&3 2>&3
1269
-
1270
- if [ -n "${mark_auto_cmd}" ]; then
1271
- # shellcheck disable=SC2086
1272
- env ${env} ${mark_auto_cmd} netdata-plugin-systemd-journal >&3 2>&3
1273
- fi
1274
- fi
1275
- fi
1276
- fi
1277
-
1265
current_version="$(get_current_version)"
1266
latest_version="$(get_latest_version)"
1267
packaging/makeself/jobs/70-netdata-git.install.sh
+6
-7
@@ -22,18 +22,17 @@ export IS_NETDATA_STATIC_BINARY="yes"
22
NETDATA_BUILD_DIR="$(build_path netdata)"
23
export NETDATA_BUILD_DIR
24
25
-# Needed to make Rust play nice with our static builds
26
-# Once Cargo’s profile-rustflags feature is a bit more widespread, we should switch to using that to specify this.
27
-export RUSTFLAGS="-C target-feature=+crt-static"
25
+export NETDATA_CMAKE_OPTIONS="-DSTATIC_BUILD=On -DENABLE_LIBBACKTRACE=On"
26
27
case "${BUILDARCH}" in
28
armv6l)
31
- export NETDATA_CMAKE_OPTIONS="-DSTATIC_BUILD=On -DENABLE_LIBBACKTRACE=On"
32
- export INSTALLER_ARGS="--disable-plugin-systemd-journal --disable-plugin-otel"
29
+ export INSTALLER_ARGS="--disable-plugin-otel --disable-plugin-systemd-journal"
30
+ ;;
31
+ armv7l)
32
+ export INSTALLER_ARGS="--enable-plugin-otel --disable-plugin-systemd-journal"
33
;;
34
*)
35
- export NETDATA_CMAKE_OPTIONS="-DSTATIC_BUILD=On -DENABLE_LIBBACKTRACE=On"
36
- export INSTALLER_ARGS="--enable-plugin-systemd-journal --internal-systemd-journal --enable-plugin-otel"
35
+ export INSTALLER_ARGS="--enable-plugin-otel --enable-plugin-systemd-journal"
36
;;
37
esac
38
packaging/windows/compile-on-windows.sh
+1
-1
@@ -43,7 +43,7 @@ CFLAGS="${BUILD_CFLAGS}" /usr/bin/cmake \
43
-DENABLE_PLUGIN_GO=On \
44
-DENABLE_EXPORTER_PROMETHEUS_REMOTE_WRITE=Off \
45
-DENABLE_PLUGIN_OTEL=Off \
46
- -DENABLE_PLUGIN_SYSTEMD_JOURNAL=Off \
46
+ -DENABLE_NETDATA_JOURNAL_FILE_READER=Off \
47
-DENABLE_BUNDLED_JSONC=On \
48
-DENABLE_BUNDLED_PROTOBUF=Off \
49
-DRust_COMPILER=/ucrt64/bin/rustc \
src/collectors/README.md
-1
@@ -27,7 +27,6 @@ This section outlines the required privileges and how they are configured in dif
27
|------------------------|-------------------------------------------------|-----------------------------------------------------|
28
| apps.plugin | CAP_DAC_READ_SEARCH, CAP_SYS_PTRACE | setuid root |
29
| debugfs.plugin | CAP_DAC_READ_SEARCH | setuid root |
30
-| systemd-journal.plugin | CAP_DAC_READ_SEARCH | setuid root |
30
| perf.plugin | CAP_PERFMON | setuid root |
31
| slabinfo.plugin | CAP_DAC_READ_SEARCH | setuid root |
32
| go.d.plugin | CAP_DAC_READ_SEARCH, CAP_NET_ADMIN, CAP_NET_RAW | setuid root |
src/collectors/systemd-journal.plugin/passive_journal_centralization_guide_no_encryption.md
deleted
-150
@@ -1,150 +0,0 @@
1
-# Passive journal centralization without encryption
2
-
3
-This page will guide you through creating a passive journal centralization setup without the use of encryption.
4
-
5
-Once you centralize your infrastructure logs to a server, Netdata will automatically detects all the logs from all servers and organize them in sources.
6
-With the setup described in this document, journal files are identified by the IPs of the clients sending the logs. Netdata will automatically do
7
-reverse DNS lookups to find the names of the server and name the sources on the dashboard accordingly.
8
-
9
-A _passive_ journal server waits for clients to push their metrics to it, so in this setup we will:
10
-
11
-1. configure `systemd-journal-remote` on the server, to listen for incoming connections.
12
-2. configure `systemd-journal-upload` on the clients, to push their logs to the server.
13
-
14
-> ⚠️ **IMPORTANT**<br/>
15
-> These instructions will copy your logs to a central server, without any encryption or authorization.<br/>
16
-> DO NOT USE THIS ON NON-TRUSTED NETWORKS.
17
-
18
-## Server configuration
19
-
20
-On the centralization server install `systemd-journal-remote`:
21
-
22
-```bash
23
-# change this according to your distro
24
-sudo apt-get install systemd-journal-remote
25
-```
26
-
27
-Make sure the journal transfer protocol is `http`:
28
-
29
-```bash
30
-sudo cp /lib/systemd/system/systemd-journal-remote.service /etc/systemd/system/
31
-
32
-# edit it to make sure it says:
33
-# --listen-http=-3
34
-# not:
35
-# --listen-https=-3
36
-sudo nano /etc/systemd/system/systemd-journal-remote.service
37
-
38
-# reload systemd
39
-sudo systemctl daemon-reload
40
-```
41
-
42
-Optionally, if you want to change the port (the default is `19532`), edit `systemd-journal-remote.socket`
43
-
44
-```bash
45
-# edit the socket file
46
-sudo systemctl edit systemd-journal-remote.socket
47
-```
48
-
49
-and add the following lines into the instructed place, and choose your desired port; save and exit.
50
-
51
-```bash
52
-[Socket]
53
-ListenStream=<DESIRED_PORT>
54
-```
55
-
56
-Finally, enable it, so that it will start automatically upon receiving a connection:
57
-
58
-```bash
59
-# enable systemd-journal-remote
60
-sudo systemctl enable --now systemd-journal-remote.socket
61
-sudo systemctl enable systemd-journal-remote.service
62
-```
63
-
64
-`systemd-journal-remote` is now listening for incoming journals from remote hosts.
65
-
66
-## Client configuration
67
-
68
-On the clients, install `systemd-journal-remote` (it includes `systemd-journal-upload`):
69
-
70
-```bash
71
-# change this according to your distro
72
-sudo apt-get install systemd-journal-remote
73
-```
74
-
75
-Edit `/etc/systemd/journal-upload.conf` and set the IP address and the port of the server, like so:
76
-
77
-```text
78
-[Upload]
79
-URL=http://centralization.server.ip:19532
80
-```
81
-
82
-Edit `systemd-journal-upload`, and add `Restart=always` to make sure the client will keep trying to push logs, even if the server is temporarily not there, like this:
83
-
84
-```bash
85
-sudo systemctl edit systemd-journal-upload
86
-```
87
-
88
-At the top, add:
89
-
90
-```text
91
-[Service]
92
-Restart=always
93
-```
94
-
95
-Enable and start `systemd-journal-upload`, like this:
96
-
97
-```bash
98
-sudo systemctl enable systemd-journal-upload
99
-sudo systemctl start systemd-journal-upload
100
-```
101
-
102
-## Verify it works
103
-
104
-To verify the central server is receiving logs, run this on the central server:
105
-
106
-```bash
107
-sudo ls -l /var/log/journal/remote/
108
-```
109
-
110
-You should see new files from the client's IP.
111
-
112
-Also, `systemctl status systemd-journal-remote` should show something like this:
113
-
114
-```bash
115
-systemd-journal-remote.service - Journal Remote Sink Service
116
- Loaded: loaded (/etc/systemd/system/systemd-journal-remote.service; indirect; preset: disabled)
117
- Active: active (running) since Sun 2023-10-15 14:29:46 EEST; 2h 24min ago
118
-TriggeredBy: ● systemd-journal-remote.socket
119
- Docs: man:systemd-journal-remote(8)
120
- man:journal-remote.conf(5)
121
- Main PID: 2118153 (systemd-journal)
122
- Status: "Processing requests..."
123
- Tasks: 1 (limit: 154152)
124
- Memory: 2.2M
125
- CPU: 71ms
126
- CGroup: /system.slice/systemd-journal-remote.service
127
- └─2118153 /usr/lib/systemd/systemd-journal-remote --listen-http=-3 --output=/var/log/journal/remote/
128
-```
129
-
130
-Note the `status: "Processing requests..."` and the PID under `CGroup`.
131
-
132
-On the client `systemctl status systemd-journal-upload` should show something like this:
133
-
134
-```bash
135
-● systemd-journal-upload.service - Journal Remote Upload Service
136
- Loaded: loaded (/lib/systemd/system/systemd-journal-upload.service; enabled; vendor preset: disabled)
137
- Drop-In: /etc/systemd/system/systemd-journal-upload.service.d
138
- └─override.conf
139
- Active: active (running) since Sun 2023-10-15 10:39:04 UTC; 3h 17min ago
140
- Docs: man:systemd-journal-upload(8)
141
- Main PID: 4169 (systemd-journal)
142
- Status: "Processing input..."
143
- Tasks: 1 (limit: 13868)
144
- Memory: 3.5M
145
- CPU: 1.081s
146
- CGroup: /system.slice/systemd-journal-upload.service
147
- └─4169 /lib/systemd/systemd-journal-upload --save-state
148
-```
149
-
150
-Note the `Status: "Processing input..."` and the PID under `CGroup`.
src/collectors/systemd-journal.plugin/passive_journal_centralization_guide_self_signed_certs.md
deleted
-249
@@ -1,249 +0,0 @@
1
-# Passive journal centralization with encryption using self-signed certificates
2
-
3
-This page will guide you through creating a **passive** journal centralization setup using **self-signed certificates** for encryption and authorization.
4
-
5
-Once you centralize your infrastructure logs to a server, Netdata will automatically detect all the logs from all servers and organize them in sources. With the setup described in this document, on recent systemd versions, Netdata will automatically name all remote sources using the names of the clients, as they are described at their certificates (on older versions, the names will be IPs or reverse DNS lookups of the IPs).
6
-
7
-A **passive** journal server waits for clients to push their metrics to it, so in this setup we will:
8
-
9
-1. configure a certificates authority and issue self-signed certificates for your servers.
10
-2. configure `systemd-journal-remote` on the server, to listen for incoming connections.
11
-3. configure `systemd-journal-upload` on the clients, to push their logs to the server.
12
-
13
-Keep in mind that the authorization involved works like this:
14
-
15
-1. The server (`systemd-journal-remote`) validates that the client (`systemd-journal-upload`) uses a trusted certificate (a certificate issued by the same certificate authority as its own).
16
- So, **the server will accept logs from any client having a valid certificate**.
17
-2. The client (`systemd-journal-upload`) validates that the receiver (`systemd-journal-remote`) uses a trusted certificate (like the server does) and it also checks that the hostname or IP of the URL specified to its configuration, matches one of the names or IPs of the server it gets connected to. So, **the client does a validation that it connected to the right server**, using the URL hostname against the names and IPs of the server on its certificate.
18
-
19
-This means, that if both certificates are issued by the same certificate authority, only the client can potentially reject the server.
20
-
21
-## Self-signed certificates
22
-
23
-To simplify the process of creating and managing self-signed certificates, we have created [this bash script](https://github.com/netdata/netdata/blob/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh).
24
-
25
-This helps to also automate the distribution of the certificates to your servers (it generates a new bash script for each of your servers, which includes everything required, including the certificates).
26
-
27
-We suggest to keep this script and all the involved certificates at the journals centralization server, in the directory `/etc/ssl/systemd-journal`, so that you can make future changes as required. If you prefer to keep the certificate authority and all the certificates at a more secure location, just use the script on that location.
28
-
29
-On the server that will issue the certificates (usually the centralizaton server), do the following:
30
-
31
-```bash
32
-# install systemd-journal-remote to add the users and groups required and openssl for the certs
33
-# change this according to your distro
34
-sudo apt-get install systemd-journal-remote openssl
35
-
36
-# download the script and make it executable
37
-curl >systemd-journal-self-signed-certs.sh "https://raw.githubusercontent.com/netdata/netdata/master/src/collectors/systemd-journal.plugin/systemd-journal-self-signed-certs.sh"
38
-chmod 750 systemd-journal-self-signed-certs.sh
39
-```
40
-
41
-To create certificates for your servers, run this:
42
-
43
-```bash
44
-sudo ./systemd-journal-self-signed-certs.sh "server1" "DNS:hostname1" "IP:10.0.0.1"
45
-```
46
-
47
-Where:
48
-
49
-- `server1` is the canonical name of the server. On newer systemd version, this name will be used by `systemd-journal-remote` and Netdata when you view the logs on the dashboard.
50
-- `DNS:hostname1` is a DNS name that the server is reachable at. Add `"DNS:xyz"` multiple times to define multiple DNS names for the server.
51
-- `IP:10.0.0.1` is an IP that the server is reachable at. Add `"IP:xyz"` multiple times to define multiple IPs for the server.
52
-
53
-Repeat this process to create the certificates for all your servers. You can add servers as required, at any time in the future.
54
-
55
-Existing certificates are never re-generated. Typically certificates need to be revoked and new ones to be issued. But `systemd-journal-remote` tools do not support handling revocations. So, the only option you have to re-issue a certificate is to delete its files in `/etc/ssl/systemd-journal` and run the script again to create a new one.
56
-
57
-Once you run the script of each of your servers, in `/etc/ssl/systemd-journal` you will find shell scripts named `runme-on-XXX.sh`, where `XXX` are the canonical names of your servers.
58
-
59
-These `runme-on-XXX.sh` include everything to install the certificates, fix their file permissions to be accessible by `systemd-journal-remote` and `systemd-journal-upload`, and update `/etc/systemd/journal-remote.conf` and `/etc/systemd/journal-upload.conf`.
60
-
61
-You can copy and paste (or `scp`) these scripts on your server and each of your clients:
62
-
63
-```bash
64
-sudo scp /etc/ssl/systemd-journal/runme-on-XXX.sh XXX:/tmp/
65
-```
66
-
67
-For the rest of this guide, we assume that you have copied the right `runme-on-XXX.sh` at the `/tmp` of all the servers for which you issued certificates.
68
-
69
-### note about certificates file permissions
70
-
71
-It is worth noting that `systemd-journal` certificates need to be owned by `systemd-journal-remote:systemd-journal`.
72
-
73
-Both the user `systemd-journal-remote` and the group `systemd-journal` are automatically added by the `systemd-journal-remote` package. However, `systemd-journal-upload` (and `systemd-journal-gatewayd` - that is not used in this guide) use dynamic users. Thankfully they are added to the `systemd-journal` remote group.
74
-
75
-So, by having the certificates owned by `systemd-journal-remote:systemd-journal`, satisfies both `systemd-journal-remote` which is not in the `systemd-journal` group, and `systemd-journal-upload` (and `systemd-journal-gatewayd`) which use dynamic users.
76
-
77
-You don't need to do anything about it (the scripts take care of everything), but it is worth noting how this works.
78
-
79
-## Server configuration
80
-
81
-On the centralization server install `systemd-journal-remote`:
82
-
83
-```bash
84
-# change this according to your distro
85
-sudo apt-get install systemd-journal-remote
86
-```
87
-
88
-Make sure the journal transfer protocol is `https`:
89
-
90
-```bash
91
-sudo cp /lib/systemd/system/systemd-journal-remote.service /etc/systemd/system/
92
-
93
-# edit it to make sure it says:
94
-# --listen-https=-3
95
-# not:
96
-# --listen-http=-3
97
-sudo nano /etc/systemd/system/systemd-journal-remote.service
98
-
99
-# reload systemd
100
-sudo systemctl daemon-reload
101
-```
102
-
103
-Optionally, if you want to change the port (the default is `19532`), edit `systemd-journal-remote.socket`
104
-
105
-```bash
106
-# edit the socket file
107
-sudo systemctl edit systemd-journal-remote.socket
108
-```
109
-
110
-and add the following lines into the instructed place, and choose your desired port; save and exit.
111
-
112
-```bash
113
-[Socket]
114
-ListenStream=<DESIRED_PORT>
115
-```
116
-
117
-Next, run the `runme-on-XXX.sh` script on the server:
118
-
119
-```bash
120
-# if you run the certificate authority on the server:
121
-sudo /etc/ssl/systemd-journal/runme-on-XXX.sh
122
-
123
-# if you run the certificate authority elsewhere,
124
-# assuming you have coped the runme-on-XXX.sh script (as described above):
125
-sudo bash /tmp/runme-on-XXX.sh
126
-```
127
-
128
-This will install the certificates in `/etc/ssl/systemd-journal`, set the right file permissions, and update `/etc/systemd/journal-remote.conf` and `/etc/systemd/journal-upload.conf` to use the right certificate files.
129
-
130
-Finally, enable it, so that it will start automatically upon receiving a connection:
131
-
132
-```bash
133
-# enable systemd-journal-remote
134
-sudo systemctl enable --now systemd-journal-remote.socket
135
-sudo systemctl enable systemd-journal-remote.service
136
-```
137
-
138
-`systemd-journal-remote` is now listening for incoming journals from remote hosts.
139
-
140
-> When done, remember to `rm /tmp/runme-on-*.sh` to make sure your certificates are secure.
141
-
142
-## Client configuration
143
-
144
-On the clients, install `systemd-journal-remote` (it includes `systemd-journal-upload`):
145
-
146
-```bash
147
-# change this according to your distro
148
-sudo apt-get install systemd-journal-remote
149
-```
150
-
151
-Edit `/etc/systemd/journal-upload.conf` and set the IP address and the port of the server, like so:
152
-
153
-```text
154
-[Upload]
155
-URL=https://centralization.server.ip:19532
156
-```
157
-
158
-Make sure that `centralization.server.ip` is one of the `DNS:` or `IP:` parameters you defined when you created the centralization server certificates. If it is not, the client may reject to connect.
159
-
160
-Next, edit `systemd-journal-upload.service`, and add `Restart=always` to make sure the client will keep trying to push logs, even if the server is temporarily not there, like this:
161
-
162
-```bash
163
-sudo systemctl edit systemd-journal-upload.service
164
-```
165
-
166
-At the top, add:
167
-
168
-```text
169
-[Service]
170
-Restart=always
171
-```
172
-
173
-Enable `systemd-journal-upload.service`, like this:
174
-
175
-```bash
176
-sudo systemctl enable systemd-journal-upload.service
177
-```
178
-
179
-Assuming that you have in `/tmp` the relevant `runme-on-XXX.sh` script for this client, run:
180
-
181
-```bash
182
-sudo bash /tmp/runme-on-XXX.sh
183
-```
184
-
185
-This will install the certificates in `/etc/ssl/systemd-journal`, set the right file permissions, and update `/etc/systemd/journal-remote.conf` and `/etc/systemd/journal-upload.conf` to use the right certificate files.
186
-
187
-Finally, restart `systemd-journal-upload.service`:
188
-
189
-```bash
190
-sudo systemctl restart systemd-journal-upload.service
191
-```
192
-
193
-The client should now be pushing logs to the central server.
194
-
195
-> When done, remember to `rm /tmp/runme-on-*.sh` to make sure your certificates are secure.
196
-
197
-Here it is in action, in Netdata:
198
-
199
-
200
-
201
-## Verify it works
202
-
203
-To verify the central server is receiving logs, run this on the central server:
204
-
205
-```bash
206
-sudo ls -l /var/log/journal/remote/
207
-```
208
-
209
-Depending on the `systemd` version you use, you should see new files from the clients' canonical names (as defined at their certificates) or IPs.
210
-
211
-Also, `systemctl status systemd-journal-remote` should show something like this:
212
-
213
-```bash
214
-systemd-journal-remote.service - Journal Remote Sink Service
215
- Loaded: loaded (/etc/systemd/system/systemd-journal-remote.service; indirect; preset: disabled)
216
- Active: active (running) since Sun 2023-10-15 14:29:46 EEST; 2h 24min ago
217
-TriggeredBy: ● systemd-journal-remote.socket
218
- Docs: man:systemd-journal-remote(8)
219
- man:journal-remote.conf(5)
220
- Main PID: 2118153 (systemd-journal)
221
- Status: "Processing requests..."
222
- Tasks: 1 (limit: 154152)
223
- Memory: 2.2M
224
- CPU: 71ms
225
- CGroup: /system.slice/systemd-journal-remote.service
226
- └─2118153 /usr/lib/systemd/systemd-journal-remote --listen-https=-3 --output=/var/log/journal/remote/
227
-```
228
-
229
-Note the `status: "Processing requests..."` and the PID under `CGroup`.
230
-
231
-On the client `systemctl status systemd-journal-upload` should show something like this:
232
-
233
-```bash
234
-● systemd-journal-upload.service - Journal Remote Upload Service
235
- Loaded: loaded (/lib/systemd/system/systemd-journal-upload.service; enabled; vendor preset: disabled)
236
- Drop-In: /etc/systemd/system/systemd-journal-upload.service.d
237
- └─override.conf
238
- Active: active (running) since Sun 2023-10-15 10:39:04 UTC; 3h 17min ago
239
- Docs: man:systemd-journal-upload(8)
240
- Main PID: 4169 (systemd-journal)
241
- Status: "Processing input..."
242
- Tasks: 1 (limit: 13868)
243
- Memory: 3.5M
244
- CPU: 1.081s
245
- CGroup: /system.slice/systemd-journal-upload.service
246
- └─4169 /lib/systemd/systemd-journal-upload --save-state
247
-```
248
-
249
-Note the `Status: "Processing input..."` and the PID under `CGroup`.
src/collectors/systemd-journal.plugin/provider/netdata_provider.c
deleted
-182
@@ -1,182 +0,0 @@
1
-#include "netdata_provider.h"
2
-
3
-int32_t nsd_id128_from_string(const char *s, NsdId128 *ret)
4
-{
5
-#if defined(HAVE_RUST_PROVIDER)
6
- return rsd_id128_from_string(s, (struct RsdId128 *) ret);
7
-#else
8
- return sd_id128_from_string(s, (sd_id128_t *) ret);
9
-#endif
10
-}
11
-
12
-int32_t nsd_id128_equal(NsdId128 a, NsdId128 b)
13
-{
14
-#if defined(HAVE_RUST_PROVIDER)
15
- return rsd_id128_equal(a, b);
16
-#else
17
- return sd_id128_equal(a, b);
18
-#endif
19
-}
20
-
21
-int nsd_journal_open_files(NsdJournal **ret, const char *const *paths, int flags)
22
-{
23
-#if defined(HAVE_RUST_PROVIDER)
24
- return rsd_journal_open_files(ret, paths, flags);
25
-#else
26
- return sd_journal_open_files(ret, paths, flags);
27
-#endif
28
-}
29
-
30
-void nsd_journal_close(NsdJournal *j)
31
-{
32
-#if defined(HAVE_RUST_PROVIDER)
33
- rsd_journal_close(j);
34
-#else
35
- sd_journal_close(j);
36
-#endif
37
-}
38
-
39
-int nsd_journal_seek_head(NsdJournal *j)
40
-{
41
-#if defined(HAVE_RUST_PROVIDER)
42
- return rsd_journal_seek_head(j);
43
-#else
44
- return sd_journal_seek_head(j);
45
-#endif
46
-}
47
-
48
-int nsd_journal_seek_tail(NsdJournal *j)
49
-{
50
-#if defined(HAVE_RUST_PROVIDER)
51
- return rsd_journal_seek_tail(j);
52
-#else
53
- return sd_journal_seek_tail(j);
54
-#endif
55
-}
56
-
57
-int nsd_journal_seek_realtime_usec(NsdJournal *j, uint64_t usec)
58
-{
59
-#if defined(HAVE_RUST_PROVIDER)
60
- return rsd_journal_seek_realtime_usec(j, usec);
61
-#else
62
- return sd_journal_seek_realtime_usec(j, usec);
63
-#endif
64
-}
65
-
66
-int nsd_journal_next(NsdJournal *j)
67
-{
68
-#if defined(HAVE_RUST_PROVIDER)
69
- return rsd_journal_next(j);
70
-#else
71
- return sd_journal_next(j);
72
-#endif
73
-}
74
-
75
-int nsd_journal_previous(NsdJournal *j)
76
-{
77
-#if defined(HAVE_RUST_PROVIDER)
78
- return rsd_journal_previous(j);
79
-#else
80
- return sd_journal_previous(j);
81
-#endif
82
-}
83
-
84
-#if defined(HAVE_SD_JOURNAL_GET_SEQNUM)
85
-int nsd_journal_get_seqnum(NsdJournal *j, uint64_t *ret_seqnum, NsdId128 *ret_seqnum_id)
86
-{
87
-#if defined(HAVE_RUST_PROVIDER)
88
- return rsd_journal_get_seqnum(j, ret_seqnum, ret_seqnum_id);
89
-#else
90
- return sd_journal_get_seqnum(j, ret_seqnum, ret_seqnum_id);
91
-#endif
92
-}
93
-#endif /* HAVE_SD_JOURNAL_GET_SEQNUM */
94
-
95
-int nsd_journal_get_realtime_usec(NsdJournal *j, uint64_t *ret)
96
-{
97
-#if defined(HAVE_RUST_PROVIDER)
98
- return rsd_journal_get_realtime_usec(j, ret);
99
-#else
100
- return sd_journal_get_realtime_usec(j, ret);
101
-#endif
102
-}
103
-
104
-#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
105
-int nsd_journal_enumerate_fields(NsdJournal *j, const char **field)
106
-{
107
-#if defined(HAVE_RUST_PROVIDER)
108
- return rsd_journal_enumerate_fields(j, field);
109
-#else
110
- return sd_journal_enumerate_fields(j, field);
111
-#endif
112
-}
113
-#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
114
-
115
-#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
116
-void nsd_journal_restart_fields(NsdJournal *j)
117
-{
118
-#if defined(HAVE_RUST_PROVIDER)
119
- rsd_journal_restart_fields(j);
120
-#else
121
- sd_journal_restart_fields(j);
122
-#endif
123
-}
124
-#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
125
-
126
-#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
127
-int nsd_journal_query_unique(NsdJournal *j, const char *field)
128
-{
129
-#if defined(HAVE_RUST_PROVIDER)
130
- return rsd_journal_query_unique(j, field);
131
-#else
132
- return sd_journal_query_unique(j, field);
133
-#endif
134
-}
135
-#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
136
-
137
-#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
138
-void nsd_journal_restart_unique(NsdJournal *j)
139
-{
140
-#if defined(HAVE_RUST_PROVIDER)
141
- rsd_journal_restart_unique(j);
142
-#else
143
- sd_journal_restart_unique(j);
144
-#endif
145
-}
146
-#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
147
-
148
-int nsd_journal_add_match(NsdJournal *j, const void *data, uintptr_t size)
149
-{
150
-#if defined(HAVE_RUST_PROVIDER)
151
- return rsd_journal_add_match(j, data, size);
152
-#else
153
- return sd_journal_add_match(j, data, size);
154
-#endif
155
-}
156
-
157
-int nsd_journal_add_conjunction(NsdJournal *j)
158
-{
159
-#if defined(HAVE_RUST_PROVIDER)
160
- return rsd_journal_add_conjunction(j);
161
-#else
162
- return sd_journal_add_conjunction(j);
163
-#endif
164
-}
165
-
166
-int nsd_journal_add_disjunction(NsdJournal *j)
167
-{
168
-#if defined(HAVE_RUST_PROVIDER)
169
- return rsd_journal_add_disjunction(j);
170
-#else
171
- return sd_journal_add_disjunction(j);
172
-#endif
173
-}
174
-
175
-void nsd_journal_flush_matches(NsdJournal *j)
176
-{
177
-#if defined(HAVE_RUST_PROVIDER)
178
- rsd_journal_flush_matches(j);
179
-#else
180
- sd_journal_flush_matches(j);
181
-#endif
182
-}
src/collectors/systemd-journal.plugin/provider/netdata_provider.h
deleted
-78
@@ -1,78 +0,0 @@
1
-#ifndef ND_SD_JOURNAL_PROVIDER_NETDATA_H
2
-#define ND_SD_JOURNAL_PROVIDER_NETDATA_H
3
-
4
-#ifdef __cplusplus
5
-extern "C" {
6
-#endif /* __cplusplus */
7
-
8
-#include "config.h"
9
-
10
-#if defined(HAVE_RUST_PROVIDER)
11
- #include "rust_provider.h"
12
-#else
13
- #include <systemd/sd-journal.h>
14
-#endif
15
-
16
-#if defined(HAVE_RUST_PROVIDER)
17
- typedef struct RsdJournal NsdJournal;
18
-
19
- #define NSD_JOURNAL_FOREACH_DATA(j, data, l) RSD_JOURNAL_FOREACH_DATA(j, data, l)
20
- #define NSD_JOURNAL_FOREACH_UNIQUE(j, data, l) RSD_JOURNAL_FOREACH_UNIQUE(j, data, l)
21
- #define NSD_JOURNAL_FOREACH_FIELD(j, field) RSD_JOURNAL_FOREACH_FIELD(j, field)
22
-
23
- typedef struct RsdId128 NsdId128;
24
-
25
- #define NSD_ID128_NULL RSD_ID128_NULL
26
- #define NSD_ID128_STRING_MAX RSD_ID128_STRING_MAX
27
- #define NSD_ID128_UUID_STRING_MAX RSD_ID128_UUID_STRING_MAX
28
-#else
29
- typedef struct sd_journal NsdJournal;
30
- typedef sd_id128_t NsdId128;
31
-
32
- #define NSD_ID128_NULL SD_ID128_NULL
33
- #define NSD_ID128_STRING_MAX SD_ID128_STRING_MAX
34
- #define NSD_ID128_UUID_STRING_MAX SD_ID128_UUID_STRING_MAX
35
-
36
- #define NSD_JOURNAL_FOREACH_DATA(j, data, l) SD_JOURNAL_FOREACH_DATA(j, data, l)
37
- #if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
38
- #define NSD_JOURNAL_FOREACH_UNIQUE(j, data, l) SD_JOURNAL_FOREACH_UNIQUE(j, data, l)
39
- #define NSD_JOURNAL_FOREACH_FIELD(j, field) SD_JOURNAL_FOREACH_FIELD(j, field)
40
- #endif
41
-#endif
42
-
43
-int32_t nsd_id128_from_string(const char *s, NsdId128 *ret);
44
-int32_t nsd_id128_equal(NsdId128 a, NsdId128 b);
45
-
46
-int nsd_journal_open_files(NsdJournal **ret, const char *const *paths, int flags);
47
-void nsd_journal_close(NsdJournal *j);
48
-
49
-int nsd_journal_seek_head(NsdJournal *j);
50
-int nsd_journal_seek_tail(NsdJournal *j);
51
-int nsd_journal_seek_realtime_usec(NsdJournal *j, uint64_t usec);
52
-
53
-int nsd_journal_next(NsdJournal *j);
54
-int nsd_journal_previous(NsdJournal *j);
55
-
56
-#if defined(HAVE_SD_JOURNAL_GET_SEQNUM)
57
-int nsd_journal_get_seqnum(NsdJournal *j, uint64_t *ret_seqnum, NsdId128 *ret_seqnum_id);
58
-#endif
59
-int nsd_journal_get_realtime_usec(NsdJournal *j, uint64_t *ret);
60
-
61
-#if defined(HAVE_SD_JOURNAL_RESTART_FIELDS)
62
-int nsd_journal_enumerate_fields(NsdJournal *j, const char **field);
63
-void nsd_journal_restart_fields(NsdJournal *j);
64
-
65
-int nsd_journal_query_unique(NsdJournal *j, const char *field);
66
-void nsd_journal_restart_unique(NsdJournal *j);
67
-#endif /* HAVE_SD_JOURNAL_RESTART_FIELDS */
68
-
69
-int nsd_journal_add_match(NsdJournal *j, const void *data, uintptr_t size);
70
-int nsd_journal_add_conjunction(NsdJournal *j);
71
-int nsd_journal_add_disjunction(NsdJournal *j);
72
-void nsd_journal_flush_matches(NsdJournal *j);
73
-
74
-#ifdef __cplusplus
75
-}
76
-#endif /* __cplusplus */
77
-
78
-#endif /* ND_SD_JOURNAL_PROVIDER_NETDATA_H */
src/collectors/systemd-journal.plugin/provider/rust_provider.h
deleted
-27
@@ -1,27 +0,0 @@
1
-#ifndef ND_SD_JOURNAL_PROVIDER_RUST_H
2
-#define ND_SD_JOURNAL_PROVIDER_RUST_H
3
-
4
-#include "crates/jf/journal_reader_ffi/journal_reader_ffi.h"
5
-
6
-#define RSD_ID128_NULL ((const RsdId128){.bytes = {0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}})
7
-#define RSD_ID128_STRING_MAX 33U
8
-#define RSD_ID128_UUID_STRING_MAX 37U
9
-
10
-#define RSD_JOURNAL_FOREACH_DATA(j, data, l) \
11
- for (rsd_journal_restart_data(j); rsd_journal_enumerate_available_data((j), &(data), &(l)) > 0;)
12
-
13
-#define RSD_JOURNAL_FOREACH_UNIQUE(j, data, l) \
14
- for (rsd_journal_restart_unique(j); rsd_journal_enumerate_available_unique((j), &(data), &(l)) > 0;)
15
-
16
-#define RSD_JOURNAL_FOREACH_FIELD(j, field) \
17
- for (rsd_journal_restart_fields(j); rsd_journal_enumerate_fields((j), &(field)) > 0;)
18
-
19
-#ifndef HAVE_SD_JOURNAL_RESTART_FIELDS
20
-#define HAVE_SD_JOURNAL_RESTART_FIELDS 1
21
-#endif
22
-
23
-#ifndef HAVE_SD_JOURNAL_GET_SEQNUM
24
-#define HAVE_SD_JOURNAL_GET_SEQNUM 1
25
-#endif
26
-
27
-#endif /* ND_SD_JOURNAL_PROVIDER_RUST_H */
src/collectors/systemd-journal.plugin/schema.d/systemd-journal%3Amonitored-directories.json
deleted
-38
@@ -1,38 +0,0 @@
1
-{
2
- "jsonSchema": {
3
- "$schema": "http://json-schema.org/draft-07/schema#",
4
- "type": "object",
5
- "properties": {
6
- "journalDirectories": {
7
- "title": "systemd-journal directories",
8
- "description": "The list of directories `systemd-journald` and `systemd-journal-remote` store journal files. Netdata monitors these directories to automatically detect changes.",
9
- "type": "array",
10
- "items": {
11
- "title": "Absolute Path",
12
- "type": "string",
13
- "pattern": "^/.+$"
14
- },
15
- "maxItems": 100,
16
- "uniqueItems": true
17
- }
18
- },
19
- "required": [
20
- "journalDirectories"
21
- ]
22
- },
23
- "uiSchema": {
24
- "journalDirectories": {
25
- "ui:listFlavour": "list",
26
- "ui:options": {
27
- "addable": true,
28
- "orderable": false,
29
- "removable": true
30
- },
31
- "items": {
32
- "ui:placeholder": "Enter absolute directory path",
33
- "ui:widget": "text",
34
- "ui:emptyValue": ""
35
- }
36
- }
37
- }
38
-}
src/collectors/systemd-journal.plugin/systemd-internals.h
deleted
-174
@@ -1,174 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef NETDATA_COLLECTORS_SYSTEMD_INTERNALS_H
4
-#define NETDATA_COLLECTORS_SYSTEMD_INTERNALS_H
5
-
6
-#include "collectors/all.h"
7
-#include "libnetdata/libnetdata.h"
8
-#include "provider/netdata_provider.h"
9
-
10
-#include <linux/capability.h>
11
-#include <syslog.h>
12
-
13
-#define ND_SD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
14
-#define ND_SD_JOURNAL_FUNCTION_NAME "systemd-journal"
15
-#define ND_SD_JOURNAL_DEFAULT_TIMEOUT 60
16
-#define ND_SD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE 0.01
17
-#define ND_SD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS 250
18
-#define ND_SD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC (5 * 60 * USEC_PER_SEC)
19
-
20
-#define ND_SD_UNITS_FUNCTION_DESCRIPTION "Lists all systemd units (services, timers, mounts, etc.) with their current state and status."
21
-#define ND_SD_UNITS_FUNCTION_NAME "systemd-list-units"
22
-#define ND_SD_UNITS_DEFAULT_TIMEOUT 30
23
-
24
-extern __thread size_t fstat_thread_calls;
25
-extern __thread size_t fstat_thread_cached_responses;
26
-void fstat_cache_enable_on_thread(void);
27
-void fstat_cache_disable_on_thread(void);
28
-
29
-extern netdata_mutex_t stdout_mutex;
30
-
31
-typedef enum {
32
- ND_SD_JOURNAL_NO_FILE_MATCHED,
33
- ND_SD_JOURNAL_FAILED_TO_OPEN,
34
- ND_SD_JOURNAL_FAILED_TO_SEEK,
35
- ND_SD_JOURNAL_TIMED_OUT,
36
- ND_SD_JOURNAL_OK,
37
- ND_SD_JOURNAL_NOT_MODIFIED,
38
- ND_SD_JOURNAL_CANCELLED,
39
-} ND_SD_JOURNAL_STATUS;
40
-
41
-typedef enum {
42
- ND_SD_JF_NONE = 0,
43
- ND_SD_JF_ALL = (1 << 0),
44
- ND_SD_JF_LOCAL_ALL = (1 << 1),
45
- ND_SD_JF_REMOTE_ALL = (1 << 2),
46
- ND_SD_JF_LOCAL_SYSTEM = (1 << 3),
47
- ND_SD_JF_LOCAL_USER = (1 << 4),
48
- ND_SD_JF_LOCAL_NAMESPACE = (1 << 5),
49
- ND_SD_JF_LOCAL_OTHER = (1 << 6),
50
-} SD_JOURNAL_FILE_SOURCE_TYPE;
51
-
52
-struct nd_journal_file {
53
- const char *filename;
54
- size_t filename_len;
55
- STRING *source;
56
- SD_JOURNAL_FILE_SOURCE_TYPE source_type;
57
- usec_t file_last_modified_ut;
58
- usec_t msg_first_ut;
59
- usec_t msg_last_ut;
60
- size_t size;
61
- bool logged_failure;
62
- bool logged_journalctl_failure;
63
- usec_t max_journal_vs_realtime_delta_ut;
64
-
65
- usec_t last_scan_monotonic_ut;
66
- usec_t last_scan_header_vs_last_modified_ut;
67
-
68
- uint64_t first_seqnum;
69
- uint64_t last_seqnum;
70
- NsdId128 first_writer_id;
71
- NsdId128 last_writer_id;
72
-
73
- uint64_t messages_in_file;
74
-};
75
-
76
-#define ND_SD_JF_SOURCE_ALL_NAME "all"
77
-#define ND_SD_JF_SOURCE_LOCAL_NAME "all-local-logs"
78
-#define ND_SD_JF_SOURCE_LOCAL_SYSTEM_NAME "all-local-system-logs"
79
-#define ND_SD_JF_SOURCE_LOCAL_USERS_NAME "all-local-user-logs"
80
-#define ND_SD_JF_SOURCE_LOCAL_OTHER_NAME "all-uncategorized"
81
-#define ND_SD_JF_SOURCE_NAMESPACES_NAME "all-local-namespaces"
82
-#define ND_SD_JF_SOURCE_REMOTES_NAME "all-remote-systems"
83
-
84
-#define ND_SD_JOURNAL_OPEN_FLAGS (0)
85
-
86
-#define JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT (5 * USEC_PER_SEC) // assume a 5-seconds latency
87
-#define JOURNAL_VS_REALTIME_DELTA_MAX_UT (2 * 60 * USEC_PER_SEC) // up to 2-minutes latency
88
-
89
-extern DICTIONARY *nd_journal_files_registry;
90
-extern DICTIONARY *used_hashes_registry;
91
-extern DICTIONARY *boot_ids_to_first_ut;
92
-
93
-int nd_journal_file_dict_items_backward_compar(const void *a, const void *b);
94
-int nd_journal_file_dict_items_forward_compar(const void *a, const void *b);
95
-void buffer_json_journal_versions(BUFFER *wb);
96
-void available_journal_file_sources_to_json_array(BUFFER *wb);
97
-bool nd_journal_files_completed_once(void);
98
-void nd_journal_files_registry_update(void);
99
-void nd_journal_directory_scan_recursively(DICTIONARY *files, DICTIONARY *dirs, const char *dirname, int depth);
100
-
101
-FACET_ROW_SEVERITY syslog_priority_to_facet_severity(FACETS *facets, FACET_ROW *row, void *data);
102
-
103
-void nd_sd_journal_dynamic_row_id(
104
- FACETS *facets,
105
- BUFFER *json_array,
106
- FACET_ROW_KEY_VALUE *rkv,
107
- FACET_ROW *row,
108
- void *data);
109
-void nd_sd_journal_transform_priority(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
110
-void nd_sd_journal_transform_syslog_facility(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
111
-void nd_sd_journal_transform_errno(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
112
-void nd_sd_journal_transform_boot_id(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
113
-void nd_sd_journal_transform_uid(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
114
-void nd_sd_journal_transform_gid(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
115
-void nd_sd_journal_transform_cap_effective(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
116
-void nd_sd_journal_transform_timestamp_usec(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
117
-
118
-usec_t nd_journal_file_update_annotation_boot_id(NsdJournal *j, struct nd_journal_file *njf, const char *boot_id);
119
-
120
-#define MAX_JOURNAL_DIRECTORIES 100
121
-struct journal_directory {
122
- STRING *path;
123
-};
124
-extern struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES];
125
-
126
-void nd_journal_init_files_and_directories(void);
127
-void function_systemd_journal(
128
- const char *transaction,
129
- char *function,
130
- usec_t *stop_monotonic_ut,
131
- bool *cancelled,
132
- BUFFER *payload,
133
- HTTP_ACCESS access __maybe_unused,
134
- const char *source,
135
- void *data);
136
-void nd_journal_file_update_header(const char *filename, struct nd_journal_file *njf);
137
-
138
-void nd_sd_journal_annotations_init(void);
139
-void nd_sd_journal_transform_message_id(FACETS *facets, BUFFER *wb, FACETS_TRANSFORMATION_SCOPE scope, void *data);
140
-
141
-void nd_journal_watcher_main(void *arg);
142
-void nd_journal_watcher_restart(void);
143
-
144
-static inline bool parse_journal_field(
145
- const char *data,
146
- size_t data_length,
147
- const char **key,
148
- size_t *key_length,
149
- const char **value,
150
- size_t *value_length)
151
-{
152
- const char *k = data;
153
- const char *equal = strchr(k, '=');
154
- if (unlikely(!equal))
155
- return false;
156
-
157
- size_t kl = equal - k;
158
-
159
- const char *v = ++equal;
160
- size_t vl = data_length - kl - 1;
161
-
162
- *key = k;
163
- *key_length = kl;
164
- *value = v;
165
- *value_length = vl;
166
-
167
- return true;
168
-}
169
-
170
-void nd_systemd_journal_dyncfg_init(struct functions_evloop_globals *wg);
171
-
172
-bool is_journal_file(const char *filename, ssize_t len, const char **start_of_extension);
173
-
174
-#endif //NETDATA_COLLECTORS_SYSTEMD_INTERNALS_H
src/collectors/systemd-journal.plugin/systemd-journal-annotations.c
deleted
-737
@@ -1,737 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "systemd-internals.h"
4
-
5
-const char *errno_map[] = {
6
- [1] = "1 (EPERM)", // "Operation not permitted",
7
- [2] = "2 (ENOENT)", // "No such file or directory",
8
- [3] = "3 (ESRCH)", // "No such process",
9
- [4] = "4 (EINTR)", // "Interrupted system call",
10
- [5] = "5 (EIO)", // "Input/output error",
11
- [6] = "6 (ENXIO)", // "No such device or address",
12
- [7] = "7 (E2BIG)", // "Argument list too long",
13
- [8] = "8 (ENOEXEC)", // "Exec format error",
14
- [9] = "9 (EBADF)", // "Bad file descriptor",
15
- [10] = "10 (ECHILD)", // "No child processes",
16
- [11] = "11 (EAGAIN)", // "Resource temporarily unavailable",
17
- [12] = "12 (ENOMEM)", // "Cannot allocate memory",
18
- [13] = "13 (EACCES)", // "Permission denied",
19
- [14] = "14 (EFAULT)", // "Bad address",
20
- [15] = "15 (ENOTBLK)", // "Block device required",
21
- [16] = "16 (EBUSY)", // "Device or resource busy",
22
- [17] = "17 (EEXIST)", // "File exists",
23
- [18] = "18 (EXDEV)", // "Invalid cross-device link",
24
- [19] = "19 (ENODEV)", // "No such device",
25
- [20] = "20 (ENOTDIR)", // "Not a directory",
26
- [21] = "21 (EISDIR)", // "Is a directory",
27
- [22] = "22 (EINVAL)", // "Invalid argument",
28
- [23] = "23 (ENFILE)", // "Too many open files in system",
29
- [24] = "24 (EMFILE)", // "Too many open files",
30
- [25] = "25 (ENOTTY)", // "Inappropriate ioctl for device",
31
- [26] = "26 (ETXTBSY)", // "Text file busy",
32
- [27] = "27 (EFBIG)", // "File too large",
33
- [28] = "28 (ENOSPC)", // "No space left on device",
34
- [29] = "29 (ESPIPE)", // "Illegal seek",
35
- [30] = "30 (EROFS)", // "Read-only file system",
36
- [31] = "31 (EMLINK)", // "Too many links",
37
- [32] = "32 (EPIPE)", // "Broken pipe",
38
- [33] = "33 (EDOM)", // "Numerical argument out of domain",
39
- [34] = "34 (ERANGE)", // "Numerical result out of range",
40
- [35] = "35 (EDEADLK)", // "Resource deadlock avoided",
41
- [36] = "36 (ENAMETOOLONG)", // "File name too long",
42
- [37] = "37 (ENOLCK)", // "No locks available",
43
- [38] = "38 (ENOSYS)", // "Function not implemented",
44
- [39] = "39 (ENOTEMPTY)", // "Directory not empty",
45
- [40] = "40 (ELOOP)", // "Too many levels of symbolic links",
46
- [42] = "42 (ENOMSG)", // "No message of desired type",
47
- [43] = "43 (EIDRM)", // "Identifier removed",
48
- [44] = "44 (ECHRNG)", // "Channel number out of range",
49
- [45] = "45 (EL2NSYNC)", // "Level 2 not synchronized",
50
- [46] = "46 (EL3HLT)", // "Level 3 halted",
51
- [47] = "47 (EL3RST)", // "Level 3 reset",
52
- [48] = "48 (ELNRNG)", // "Link number out of range",
53
- [49] = "49 (EUNATCH)", // "Protocol driver not attached",
54
- [50] = "50 (ENOCSI)", // "No CSI structure available",
55
- [51] = "51 (EL2HLT)", // "Level 2 halted",
56
- [52] = "52 (EBADE)", // "Invalid exchange",
57
- [53] = "53 (EBADR)", // "Invalid request descriptor",
58
- [54] = "54 (EXFULL)", // "Exchange full",
59
- [55] = "55 (ENOANO)", // "No anode",
60
- [56] = "56 (EBADRQC)", // "Invalid request code",
61
- [57] = "57 (EBADSLT)", // "Invalid slot",
62
- [59] = "59 (EBFONT)", // "Bad font file format",
63
- [60] = "60 (ENOSTR)", // "Device not a stream",
64
- [61] = "61 (ENODATA)", // "No data available",
65
- [62] = "62 (ETIME)", // "Timer expired",
66
- [63] = "63 (ENOSR)", // "Out of streams resources",
67
- [64] = "64 (ENONET)", // "Machine is not on the network",
68
- [65] = "65 (ENOPKG)", // "Package not installed",
69
- [66] = "66 (EREMOTE)", // "Object is remote",
70
- [67] = "67 (ENOLINK)", // "Link has been severed",
71
- [68] = "68 (EADV)", // "Advertise error",
72
- [69] = "69 (ESRMNT)", // "Srmount error",
73
- [70] = "70 (ECOMM)", // "Communication error on send",
74
- [71] = "71 (EPROTO)", // "Protocol error",
75
- [72] = "72 (EMULTIHOP)", // "Multihop attempted",
76
- [73] = "73 (EDOTDOT)", // "RFS specific error",
77
- [74] = "74 (EBADMSG)", // "Bad message",
78
- [75] = "75 (EOVERFLOW)", // "Value too large for defined data type",
79
- [76] = "76 (ENOTUNIQ)", // "Name not unique on network",
80
- [77] = "77 (EBADFD)", // "File descriptor in bad state",
81
- [78] = "78 (EREMCHG)", // "Remote address changed",
82
- [79] = "79 (ELIBACC)", // "Can not access a needed shared library",
83
- [80] = "80 (ELIBBAD)", // "Accessing a corrupted shared library",
84
- [81] = "81 (ELIBSCN)", // ".lib section in a.out corrupted",
85
- [82] = "82 (ELIBMAX)", // "Attempting to link in too many shared libraries",
86
- [83] = "83 (ELIBEXEC)", // "Cannot exec a shared library directly",
87
- [84] = "84 (EILSEQ)", // "Invalid or incomplete multibyte or wide character",
88
- [85] = "85 (ERESTART)", // "Interrupted system call should be restarted",
89
- [86] = "86 (ESTRPIPE)", // "Streams pipe error",
90
- [87] = "87 (EUSERS)", // "Too many users",
91
- [88] = "88 (ENOTSOCK)", // "Socket operation on non-socket",
92
- [89] = "89 (EDESTADDRREQ)", // "Destination address required",
93
- [90] = "90 (EMSGSIZE)", // "Message too long",
94
- [91] = "91 (EPROTOTYPE)", // "Protocol wrong type for socket",
95
- [92] = "92 (ENOPROTOOPT)", // "Protocol not available",
96
- [93] = "93 (EPROTONOSUPPORT)", // "Protocol not supported",
97
- [94] = "94 (ESOCKTNOSUPPORT)", // "Socket type not supported",
98
- [95] = "95 (ENOTSUP)", // "Operation not supported",
99
- [96] = "96 (EPFNOSUPPORT)", // "Protocol family not supported",
100
- [97] = "97 (EAFNOSUPPORT)", // "Address family not supported by protocol",
101
- [98] = "98 (EADDRINUSE)", // "Address already in use",
102
- [99] = "99 (EADDRNOTAVAIL)", // "Cannot assign requested address",
103
- [100] = "100 (ENETDOWN)", // "Network is down",
104
- [101] = "101 (ENETUNREACH)", // "Network is unreachable",
105
- [102] = "102 (ENETRESET)", // "Network dropped connection on reset",
106
- [103] = "103 (ECONNABORTED)", // "Software caused connection abort",
107
- [104] = "104 (ECONNRESET)", // "Connection reset by peer",
108
- [105] = "105 (ENOBUFS)", // "No buffer space available",
109
- [106] = "106 (EISCONN)", // "Transport endpoint is already connected",
110
- [107] = "107 (ENOTCONN)", // "Transport endpoint is not connected",
111
- [108] = "108 (ESHUTDOWN)", // "Cannot send after transport endpoint shutdown",
112
- [109] = "109 (ETOOMANYREFS)", // "Too many references: cannot splice",
113
- [110] = "110 (ETIMEDOUT)", // "Connection timed out",
114
- [111] = "111 (ECONNREFUSED)", // "Connection refused",
115
- [112] = "112 (EHOSTDOWN)", // "Host is down",
116
- [113] = "113 (EHOSTUNREACH)", // "No route to host",
117
- [114] = "114 (EALREADY)", // "Operation already in progress",
118
- [115] = "115 (EINPROGRESS)", // "Operation now in progress",
119
- [116] = "116 (ESTALE)", // "Stale file handle",
120
- [117] = "117 (EUCLEAN)", // "Structure needs cleaning",
121
- [118] = "118 (ENOTNAM)", // "Not a XENIX named type file",
122
- [119] = "119 (ENAVAIL)", // "No XENIX semaphores available",
123
- [120] = "120 (EISNAM)", // "Is a named type file",
124
- [121] = "121 (EREMOTEIO)", // "Remote I/O error",
125
- [122] = "122 (EDQUOT)", // "Disk quota exceeded",
126
- [123] = "123 (ENOMEDIUM)", // "No medium found",
127
- [124] = "124 (EMEDIUMTYPE)", // "Wrong medium type",
128
- [125] = "125 (ECANCELED)", // "Operation canceled",
129
- [126] = "126 (ENOKEY)", // "Required key not available",
130
- [127] = "127 (EKEYEXPIRED)", // "Key has expired",
131
- [128] = "128 (EKEYREVOKED)", // "Key has been revoked",
132
- [129] = "129 (EKEYREJECTED)", // "Key was rejected by service",
133
- [130] = "130 (EOWNERDEAD)", // "Owner died",
134
- [131] = "131 (ENOTRECOVERABLE)", // "State not recoverable",
135
- [132] = "132 (ERFKILL)", // "Operation not possible due to RF-kill",
136
- [133] = "133 (EHWPOISON)", // "Memory page has hardware error",
137
-};
138
-
139
-const char *linux_capabilities[] = {
140
- [CAP_CHOWN] = "CHOWN",
141
- [CAP_DAC_OVERRIDE] = "DAC_OVERRIDE",
142
- [CAP_DAC_READ_SEARCH] = "DAC_READ_SEARCH",
143
- [CAP_FOWNER] = "FOWNER",
144
- [CAP_FSETID] = "FSETID",
145
- [CAP_KILL] = "KILL",
146
- [CAP_SETGID] = "SETGID",
147
- [CAP_SETUID] = "SETUID",
148
- [CAP_SETPCAP] = "SETPCAP",
149
- [CAP_LINUX_IMMUTABLE] = "LINUX_IMMUTABLE",
150
- [CAP_NET_BIND_SERVICE] = "NET_BIND_SERVICE",
151
- [CAP_NET_BROADCAST] = "NET_BROADCAST",
152
- [CAP_NET_ADMIN] = "NET_ADMIN",
153
- [CAP_NET_RAW] = "NET_RAW",
154
- [CAP_IPC_LOCK] = "IPC_LOCK",
155
- [CAP_IPC_OWNER] = "IPC_OWNER",
156
- [CAP_SYS_MODULE] = "SYS_MODULE",
157
- [CAP_SYS_RAWIO] = "SYS_RAWIO",
158
- [CAP_SYS_CHROOT] = "SYS_CHROOT",
159
- [CAP_SYS_PTRACE] = "SYS_PTRACE",
160
- [CAP_SYS_PACCT] = "SYS_PACCT",
161
- [CAP_SYS_ADMIN] = "SYS_ADMIN",
162
- [CAP_SYS_BOOT] = "SYS_BOOT",
163
- [CAP_SYS_NICE] = "SYS_NICE",
164
- [CAP_SYS_RESOURCE] = "SYS_RESOURCE",
165
- [CAP_SYS_TIME] = "SYS_TIME",
166
- [CAP_SYS_TTY_CONFIG] = "SYS_TTY_CONFIG",
167
- [CAP_MKNOD] = "MKNOD",
168
- [CAP_LEASE] = "LEASE",
169
- [CAP_AUDIT_WRITE] = "AUDIT_WRITE",
170
- [CAP_AUDIT_CONTROL] = "AUDIT_CONTROL",
171
- [CAP_SETFCAP] = "SETFCAP",
172
- [CAP_MAC_OVERRIDE] = "MAC_OVERRIDE",
173
- [CAP_MAC_ADMIN] = "MAC_ADMIN",
174
- [CAP_SYSLOG] = "SYSLOG",
175
- [CAP_WAKE_ALARM] = "WAKE_ALARM",
176
- [CAP_BLOCK_SUSPEND] = "BLOCK_SUSPEND",
177
- [37 /*CAP_AUDIT_READ*/] = "AUDIT_READ",
178
- [38 /*CAP_PERFMON*/] = "PERFMON",
179
- [39 /*CAP_BPF*/] = "BPF",
180
- [40 /* CAP_CHECKPOINT_RESTORE */] = "CHECKPOINT_RESTORE",
181
-};
182
-
183
-static const char *syslog_facility_to_name(int facility)
184
-{
185
- switch (facility) {
186
- case LOG_FAC(LOG_KERN):
187
- return "kern";
188
- case LOG_FAC(LOG_USER):
189
- return "user";
190
- case LOG_FAC(LOG_MAIL):
191
- return "mail";
192
- case LOG_FAC(LOG_DAEMON):
193
- return "daemon";
194
- case LOG_FAC(LOG_AUTH):
195
- return "auth";
196
- case LOG_FAC(LOG_SYSLOG):
197
- return "syslog";
198
- case LOG_FAC(LOG_LPR):
199
- return "lpr";
200
- case LOG_FAC(LOG_NEWS):
201
- return "news";
202
- case LOG_FAC(LOG_UUCP):
203
- return "uucp";
204
- case LOG_FAC(LOG_CRON):
205
- return "cron";
206
- case LOG_FAC(LOG_AUTHPRIV):
207
- return "authpriv";
208
- case LOG_FAC(LOG_FTP):
209
- return "ftp";
210
- case LOG_FAC(LOG_LOCAL0):
211
- return "local0";
212
- case LOG_FAC(LOG_LOCAL1):
213
- return "local1";
214
- case LOG_FAC(LOG_LOCAL2):
215
- return "local2";
216
- case LOG_FAC(LOG_LOCAL3):
217
- return "local3";
218
- case LOG_FAC(LOG_LOCAL4):
219
- return "local4";
220
- case LOG_FAC(LOG_LOCAL5):
221
- return "local5";
222
- case LOG_FAC(LOG_LOCAL6):
223
- return "local6";
224
- case LOG_FAC(LOG_LOCAL7):
225
- return "local7";
226
- default:
227
- return NULL;
228
- }
229
-}
230
-
231
-static const char *syslog_priority_to_name(int priority)
232
-{
233
- switch (priority) {
234
- case LOG_ALERT:
235
- return "alert";
236
- case LOG_CRIT:
237
- return "critical";
238
- case LOG_DEBUG:
239
- return "debug";
240
- case LOG_EMERG:
241
- return "panic";
242
- case LOG_ERR:
243
- return "error";
244
- case LOG_INFO:
245
- return "info";
246
- case LOG_NOTICE:
247
- return "notice";
248
- case LOG_WARNING:
249
- return "warning";
250
- default:
251
- return NULL;
252
- }
253
-}
254
-
255
-FACET_ROW_SEVERITY
256
-syslog_priority_to_facet_severity(FACETS *facets __maybe_unused, FACET_ROW *row, void *data __maybe_unused)
257
-{
258
- // same to
259
- // https://github.com/systemd/systemd/blob/aab9e4b2b86905a15944a1ac81e471b5b7075932/src/basic/terminal-util.c#L1501
260
- // function get_log_colors()
261
-
262
- FACET_ROW_KEY_VALUE *priority_rkv = dictionary_get(row->dict, "PRIORITY");
263
- if (!priority_rkv || priority_rkv->empty)
264
- return FACET_ROW_SEVERITY_NORMAL;
265
-
266
- int priority = str2i(buffer_tostring(priority_rkv->wb));
267
-
268
- if (priority <= LOG_ERR)
269
- return FACET_ROW_SEVERITY_CRITICAL;
270
-
271
- else if (priority <= LOG_WARNING)
272
- return FACET_ROW_SEVERITY_WARNING;
273
-
274
- else if (priority <= LOG_NOTICE)
275
- return FACET_ROW_SEVERITY_NOTICE;
276
-
277
- else if (priority >= LOG_DEBUG)
278
- return FACET_ROW_SEVERITY_DEBUG;
279
-
280
- return FACET_ROW_SEVERITY_NORMAL;
281
-}
282
-
283
-void nd_sd_journal_transform_syslog_facility(
284
- FACETS *facets __maybe_unused,
285
- BUFFER *wb,
286
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
287
- void *data __maybe_unused)
288
-{
289
- const char *v = buffer_tostring(wb);
290
- if (*v && isdigit(*v)) {
291
- int facility = str2i(buffer_tostring(wb));
292
- const char *name = syslog_facility_to_name(facility);
293
- if (name) {
294
- buffer_flush(wb);
295
- buffer_strcat(wb, name);
296
- }
297
- }
298
-}
299
-
300
-void nd_sd_journal_transform_priority(
301
- FACETS *facets __maybe_unused,
302
- BUFFER *wb,
303
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
304
- void *data __maybe_unused)
305
-{
306
- if (scope == FACETS_TRANSFORM_FACET_SORT)
307
- return;
308
-
309
- const char *v = buffer_tostring(wb);
310
- if (*v && isdigit(*v)) {
311
- int priority = str2i(buffer_tostring(wb));
312
- const char *name = syslog_priority_to_name(priority);
313
- if (name) {
314
- buffer_flush(wb);
315
- buffer_strcat(wb, name);
316
- }
317
- }
318
-}
319
-
320
-void nd_sd_journal_transform_errno(
321
- FACETS *facets __maybe_unused,
322
- BUFFER *wb,
323
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
324
- void *data __maybe_unused)
325
-{
326
- if (scope == FACETS_TRANSFORM_FACET_SORT)
327
- return;
328
-
329
- const char *v = buffer_tostring(wb);
330
- if (*v && isdigit(*v)) {
331
- unsigned err_no = str2u(buffer_tostring(wb));
332
- if (err_no > 0 && err_no < sizeof(errno_map) / sizeof(*errno_map)) {
333
- const char *name = errno_map[err_no];
334
- if (name) {
335
- buffer_flush(wb);
336
- buffer_strcat(wb, name);
337
- }
338
- }
339
- }
340
-}
341
-
342
-DICTIONARY *boot_ids_to_first_ut = NULL;
343
-
344
-void nd_sd_journal_transform_boot_id(
345
- FACETS *facets __maybe_unused,
346
- BUFFER *wb,
347
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
348
- void *data __maybe_unused)
349
-{
350
- const char *boot_id = buffer_tostring(wb);
351
- if (*boot_id && isxdigit(*boot_id)) {
352
- usec_t ut = UINT64_MAX;
353
- usec_t *p_ut = dictionary_get(boot_ids_to_first_ut, boot_id);
354
- if (!p_ut) {
355
-#ifndef HAVE_SD_JOURNAL_RESTART_FIELDS
356
- struct nd_journal_file *njf;
357
- dfe_start_read(nd_journal_files_registry, njf)
358
- {
359
- const char *files[2] = {
360
- [0] = njf_dfe.name,
361
- [1] = NULL,
362
- };
363
-
364
- sd_journal *j = NULL;
365
- int r = sd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS);
366
- if (r < 0 || !j) {
367
- internal_error(
368
- true,
369
- "JOURNAL: while looking for the first timestamp of boot_id '%s', "
370
- "sd_journal_open_files('%s') returned %d",
371
- boot_id,
372
- jf_dfe.name,
373
- r);
374
- continue;
375
- }
376
-
377
- ut = nd_journal_file_update_annotation_boot_id(j, njf, boot_id);
378
- sd_journal_close(j);
379
- }
380
- dfe_done(njf);
381
-#endif
382
- } else
383
- ut = *p_ut;
384
-
385
- if (ut && ut != UINT64_MAX) {
386
- char buffer[RFC3339_MAX_LENGTH];
387
- rfc3339_datetime_ut(buffer, sizeof(buffer), ut, 0, true);
388
-
389
- switch (scope) {
390
- default:
391
- case FACETS_TRANSFORM_DATA:
392
- case FACETS_TRANSFORM_VALUE:
393
- buffer_sprintf(wb, " (%s) ", buffer);
394
- break;
395
-
396
- case FACETS_TRANSFORM_FACET:
397
- case FACETS_TRANSFORM_FACET_SORT:
398
- case FACETS_TRANSFORM_HISTOGRAM:
399
- buffer_flush(wb);
400
- buffer_sprintf(wb, "%s", buffer);
401
- break;
402
- }
403
- }
404
- }
405
-}
406
-
407
-void nd_sd_journal_transform_uid(
408
- FACETS *facets __maybe_unused,
409
- BUFFER *wb,
410
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
411
- void *data __maybe_unused)
412
-{
413
- if (scope == FACETS_TRANSFORM_FACET_SORT)
414
- return;
415
-
416
- const char *v = buffer_tostring(wb);
417
- if (*v && isdigit(*v)) {
418
- uid_t uid = str2i(buffer_tostring(wb));
419
- CACHED_USERNAME cu = cached_username_get_by_uid(uid);
420
- buffer_contents_replace(wb, string2str(cu.username), string_strlen(cu.username));
421
- cached_username_release(cu);
422
- }
423
-}
424
-
425
-void nd_sd_journal_transform_gid(
426
- FACETS *facets __maybe_unused,
427
- BUFFER *wb,
428
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
429
- void *data __maybe_unused)
430
-{
431
- if (scope == FACETS_TRANSFORM_FACET_SORT)
432
- return;
433
-
434
- const char *v = buffer_tostring(wb);
435
- if (*v && isdigit(*v)) {
436
- gid_t gid = str2i(buffer_tostring(wb));
437
- CACHED_GROUPNAME cg = cached_groupname_get_by_gid(gid);
438
- buffer_contents_replace(wb, string2str(cg.groupname), string_strlen(cg.groupname));
439
- cached_groupname_release(cg);
440
- }
441
-}
442
-
443
-void nd_sd_journal_transform_cap_effective(
444
- FACETS *facets __maybe_unused,
445
- BUFFER *wb,
446
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
447
- void *data __maybe_unused)
448
-{
449
- if (scope == FACETS_TRANSFORM_FACET_SORT)
450
- return;
451
-
452
- const char *v = buffer_tostring(wb);
453
- if (*v && isdigit(*v)) {
454
- uint64_t cap = strtoul(buffer_tostring(wb), NULL, 16);
455
- if (cap) {
456
- buffer_fast_strcat(wb, " (", 2);
457
- for (size_t i = 0, added = 0; i < sizeof(linux_capabilities) / sizeof(linux_capabilities[0]); i++) {
458
- if (linux_capabilities[i] && (cap & (1ULL << i))) {
459
- if (added)
460
- buffer_fast_strcat(wb, " | ", 3);
461
-
462
- buffer_strcat(wb, linux_capabilities[i]);
463
- added++;
464
- }
465
- }
466
- buffer_fast_strcat(wb, ")", 1);
467
- }
468
- }
469
-}
470
-
471
-void nd_sd_journal_transform_timestamp_usec(
472
- FACETS *facets __maybe_unused,
473
- BUFFER *wb,
474
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
475
- void *data __maybe_unused)
476
-{
477
- if (scope == FACETS_TRANSFORM_FACET_SORT)
478
- return;
479
-
480
- const char *v = buffer_tostring(wb);
481
- if (*v && isdigit(*v)) {
482
- uint64_t ut = str2ull(buffer_tostring(wb), NULL);
483
- if (ut) {
484
- char buffer[RFC3339_MAX_LENGTH];
485
- rfc3339_datetime_ut(buffer, sizeof(buffer), ut, 6, true);
486
- buffer_sprintf(wb, " (%s)", buffer);
487
- }
488
- }
489
-}
490
-
491
-void nd_sd_journal_dynamic_row_id(
492
- FACETS *facets __maybe_unused,
493
- BUFFER *json_array,
494
- FACET_ROW_KEY_VALUE *rkv,
495
- FACET_ROW *row,
496
- void *data __maybe_unused)
497
-{
498
- FACET_ROW_KEY_VALUE *pid_rkv = dictionary_get(row->dict, "_PID");
499
- const char *pid = pid_rkv ? buffer_tostring(pid_rkv->wb) : FACET_VALUE_UNSET;
500
-
501
- const char *identifier = NULL;
502
- FACET_ROW_KEY_VALUE *container_name_rkv = dictionary_get(row->dict, "CONTAINER_NAME");
503
- if (container_name_rkv && !container_name_rkv->empty)
504
- identifier = buffer_tostring(container_name_rkv->wb);
505
-
506
- if (!identifier) {
507
- FACET_ROW_KEY_VALUE *syslog_identifier_rkv = dictionary_get(row->dict, "SYSLOG_IDENTIFIER");
508
- if (syslog_identifier_rkv && !syslog_identifier_rkv->empty)
509
- identifier = buffer_tostring(syslog_identifier_rkv->wb);
510
-
511
- if (!identifier) {
512
- FACET_ROW_KEY_VALUE *comm_rkv = dictionary_get(row->dict, "_COMM");
513
- if (comm_rkv && !comm_rkv->empty)
514
- identifier = buffer_tostring(comm_rkv->wb);
515
- }
516
- }
517
-
518
- buffer_flush(rkv->wb);
519
-
520
- if (!identifier || !*identifier)
521
- buffer_strcat(rkv->wb, FACET_VALUE_UNSET);
522
- else if (!pid || !*pid)
523
- buffer_sprintf(rkv->wb, "%s", identifier);
524
- else
525
- buffer_sprintf(rkv->wb, "%s[%s]", identifier, pid);
526
-
527
- buffer_json_add_array_item_string(json_array, buffer_tostring(rkv->wb));
528
-}
529
-
530
-struct message_id_info {
531
- const char *msg;
532
-};
533
-
534
-static DICTIONARY *known_journal_messages_ids = NULL;
535
-
536
-#define msgid_into_dict(uuid, message) \
537
- do { \
538
- i.msg = message; \
539
- dictionary_set(known_journal_messages_ids, uuid, &i, sizeof(i)); \
540
- } while (0)
541
-
542
-static void nd_sd_journal_message_ids_init(void)
543
-{
544
- known_journal_messages_ids = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
545
- struct message_id_info i = {0};
546
-
547
- // systemd
548
- // https://github.com/systemd/systemd/blob/main/catalog/systemd.catalog.in
549
- msgid_into_dict("f77379a8490b408bbe5f6940505a777b", "Journal started");
550
- msgid_into_dict("d93fb3c9c24d451a97cea615ce59c00b", "Journal stopped");
551
- msgid_into_dict("a596d6fe7bfa4994828e72309e95d61e", "Journal messages suppressed");
552
- msgid_into_dict("e9bf28e6e834481bb6f48f548ad13606", "Journal messages missed");
553
- msgid_into_dict("ec387f577b844b8fa948f33cad9a75e6", "Journal disk space usage");
554
- msgid_into_dict("fc2e22bc6ee647b6b90729ab34a250b1", "Coredump");
555
- msgid_into_dict("5aadd8e954dc4b1a8c954d63fd9e1137", "Coredump truncated");
556
- msgid_into_dict("1f4e0a44a88649939aaea34fc6da8c95", "Backtrace"); // not found in systemd catalog
557
- msgid_into_dict("8d45620c1a4348dbb17410da57c60c66", "User Session created");
558
- msgid_into_dict("3354939424b4456d9802ca8333ed424a", "User Session terminated");
559
- msgid_into_dict("fcbefc5da23d428093f97c82a9290f7b", "Seat started");
560
- msgid_into_dict("e7852bfe46784ed0accde04bc864c2d5", "Seat removed");
561
- msgid_into_dict("24d8d4452573402496068381a6312df2", "VM or container started");
562
- msgid_into_dict("58432bd3bace477cb514b56381b8a758", "VM or container stopped");
563
- msgid_into_dict("c7a787079b354eaaa9e77b371893cd27", "Time change");
564
- msgid_into_dict("45f82f4aef7a4bbf942ce861d1f20990", "Timezone change");
565
- msgid_into_dict("50876a9db00f4c40bde1a2ad381c3a1b", "System configuration issues");
566
- msgid_into_dict("b07a249cd024414a82dd00cd181378ff", "System start-up completed");
567
- msgid_into_dict("eed00a68ffd84e31882105fd973abdd1", "User start-up completed");
568
- msgid_into_dict("6bbd95ee977941e497c48be27c254128", "Sleep start");
569
- msgid_into_dict("8811e6df2a8e40f58a94cea26f8ebf14", "Sleep stop");
570
- msgid_into_dict("98268866d1d54a499c4e98921d93bc40", "System shutdown initiated");
571
- msgid_into_dict("c14aaf76ec284a5fa1f105f88dfb061c", "System factory reset initiated");
572
- msgid_into_dict("d9ec5e95e4b646aaaea2fd05214edbda", "Container init crashed");
573
- msgid_into_dict("3ed0163e868a4417ab8b9e210407a96c", "System reboot failed after crash");
574
- msgid_into_dict("645c735537634ae0a32b15a7c6cba7d4", "Init execution froze");
575
- msgid_into_dict("5addb3a06a734d3396b794bf98fb2d01", "Init crashed no coredump");
576
- msgid_into_dict("5c9e98de4ab94c6a9d04d0ad793bd903", "Init crashed no fork");
577
- msgid_into_dict("5e6f1f5e4db64a0eaee3368249d20b94", "Init crashed unknown signal");
578
- msgid_into_dict("83f84b35ee264f74a3896a9717af34cb", "Init crashed systemd signal");
579
- msgid_into_dict("3a73a98baf5b4b199929e3226c0be783", "Init crashed process signal");
580
- msgid_into_dict("2ed18d4f78ca47f0a9bc25271c26adb4", "Init crashed waitpid failed");
581
- msgid_into_dict("56b1cd96f24246c5b607666fda952356", "Init crashed coredump failed");
582
- msgid_into_dict("4ac7566d4d7548f4981f629a28f0f829", "Init crashed coredump");
583
- msgid_into_dict("38e8b1e039ad469291b18b44c553a5b7", "Crash shell failed to fork");
584
- msgid_into_dict("872729b47dbe473eb768ccecd477beda", "Crash shell failed to execute");
585
- msgid_into_dict("658a67adc1c940b3b3316e7e8628834a", "Selinux failed");
586
- msgid_into_dict("e6f456bd92004d9580160b2207555186", "Battery low warning");
587
- msgid_into_dict("267437d33fdd41099ad76221cc24a335", "Battery low powering off");
588
- msgid_into_dict("79e05b67bc4545d1922fe47107ee60c5", "Manager mainloop failed");
589
- msgid_into_dict("dbb136b10ef4457ba47a795d62f108c9", "Manager no xdgdir path");
590
- msgid_into_dict("ed158c2df8884fa584eead2d902c1032", "Init failed to drop capability bounding set of usermode");
591
- msgid_into_dict("42695b500df048298bee37159caa9f2e", "Init failed to drop capability bounding set");
592
- msgid_into_dict("bfc2430724ab44499735b4f94cca9295", "User manager can't disable new privileges");
593
- msgid_into_dict("59288af523be43a28d494e41e26e4510", "Manager failed to start default target");
594
- msgid_into_dict("689b4fcc97b4486ea5da92db69c9e314", "Manager failed to isolate default target");
595
- msgid_into_dict("5ed836f1766f4a8a9fc5da45aae23b29", "Manager failed to collect passed file descriptors");
596
- msgid_into_dict("6a40fbfbd2ba4b8db02fb40c9cd090d7", "Init failed to fix up environment variables");
597
- msgid_into_dict("0e54470984ac419689743d957a119e2e", "Manager failed to allocate");
598
- msgid_into_dict("d67fa9f847aa4b048a2ae33535331adb", "Manager failed to write Smack");
599
- msgid_into_dict("af55a6f75b544431b72649f36ff6d62c", "System shutdown critical error");
600
- msgid_into_dict("d18e0339efb24a068d9c1060221048c2", "Init failed to fork off valgrind");
601
- msgid_into_dict("7d4958e842da4a758f6c1cdc7b36dcc5", "Unit starting");
602
- msgid_into_dict("39f53479d3a045ac8e11786248231fbf", "Unit started");
603
- msgid_into_dict("be02cf6855d2428ba40df7e9d022f03d", "Unit failed");
604
- msgid_into_dict("de5b426a63be47a7b6ac3eaac82e2f6f", "Unit stopping");
605
- msgid_into_dict("9d1aaa27d60140bd96365438aad20286", "Unit stopped");
606
- msgid_into_dict("d34d037fff1847e6ae669a370e694725", "Unit reloading");
607
- msgid_into_dict("7b05ebc668384222baa8881179cfda54", "Unit reloaded");
608
- msgid_into_dict("5eb03494b6584870a536b337290809b3", "Unit restart scheduled");
609
- msgid_into_dict("ae8f7b866b0347b9af31fe1c80b127c0", "Unit resources");
610
- msgid_into_dict("7ad2d189f7e94e70a38c781354912448", "Unit success");
611
- msgid_into_dict("0e4284a0caca4bfc81c0bb6786972673", "Unit skipped");
612
- msgid_into_dict("d9b373ed55a64feb8242e02dbe79a49c", "Unit failure result");
613
- msgid_into_dict("641257651c1b4ec9a8624d7a40a9e1e7", "Process execution failed");
614
- msgid_into_dict("98e322203f7a4ed290d09fe03c09fe15", "Unit process exited");
615
- msgid_into_dict("0027229ca0644181a76c4e92458afa2e", "Syslog forward missed");
616
- msgid_into_dict("1dee0369c7fc4736b7099b38ecb46ee7", "Mount point is not empty");
617
- msgid_into_dict("d989611b15e44c9dbf31e3c81256e4ed", "Unit oomd kill"); // not found in systemd catalog
618
- msgid_into_dict("fe6faa94e7774663a0da52717891d8ef", "Unit out of memory");
619
- msgid_into_dict("b72ea4a2881545a0b50e200e55b9b06f", "Lid opened");
620
- msgid_into_dict("b72ea4a2881545a0b50e200e55b9b070", "Lid closed");
621
- msgid_into_dict("f5f416b862074b28927a48c3ba7d51ff", "System docked");
622
- msgid_into_dict("51e171bd585248568110144c517cca53", "System undocked");
623
- msgid_into_dict("b72ea4a2881545a0b50e200e55b9b071", "Power key");
624
- msgid_into_dict("3e0117101eb243c1b9a50db3494ab10b", "Power key long press");
625
- msgid_into_dict("9fa9d2c012134ec385451ffe316f97d0", "Reboot key");
626
- msgid_into_dict("f1c59a58c9d943668965c337caec5975", "Reboot key long press");
627
- msgid_into_dict("b72ea4a2881545a0b50e200e55b9b072", "Suspend key");
628
- msgid_into_dict("bfdaf6d312ab4007bc1fe40a15df78e8", "Suspend key long press");
629
- msgid_into_dict("b72ea4a2881545a0b50e200e55b9b073", "Hibernate key");
630
- msgid_into_dict("167836df6f7f428e98147227b2dc8945", "Hibernate key long press");
631
- msgid_into_dict("c772d24e9a884cbeb9ea12625c306c01", "Invalid configuration"); // not found in systemd catalog
632
- msgid_into_dict("1675d7f172174098b1108bf8c7dc8f5d", "DNSSEC validation failed");
633
- msgid_into_dict("4d4408cfd0d144859184d1e65d7c8a65", "DNSSEC trust anchor revoked");
634
- msgid_into_dict("36db2dfa5a9045e1bd4af5f93e1cf057", "DNSSEC turned off");
635
- msgid_into_dict("b61fdac612e94b9182285b998843061f", "Username unsafe");
636
- msgid_into_dict("1b3bb94037f04bbf81028e135a12d293", "Mount point path not suitable");
637
- msgid_into_dict("010190138f494e29a0ef6669749531aa", "Device path not suitable"); // not found in systemd catalog
638
- msgid_into_dict("b480325f9c394a7b802c231e51a2752c", "Nobody user unsuitable");
639
- msgid_into_dict("1c0454c1bd2241e0ac6fefb4bc631433", "Systemd udev settle deprecated");
640
- msgid_into_dict("7c8a41f37b764941a0e1780b1be2f037", "Time initial sync");
641
- msgid_into_dict("7db73c8af0d94eeb822ae04323fe6ab6", "Time initial bump");
642
- msgid_into_dict("9e7066279dc8403da79ce4b1a69064b2", "Shutdown scheduled");
643
- msgid_into_dict("249f6fb9e6e2428c96f3f0875681ffa3", "Shutdown canceled");
644
- msgid_into_dict("3f7d5ef3e54f4302b4f0b143bb270cab", "TPM PCR Extended");
645
- msgid_into_dict("f9b0be465ad540d0850ad32172d57c21", "Memory Trimmed");
646
- msgid_into_dict("a8fa8dacdb1d443e9503b8be367a6adb", "SysV Service Found");
647
- msgid_into_dict("187c62eb1e7f463bb530394f52cb090f", "Portable Service attached");
648
- msgid_into_dict("76c5c754d628490d8ecba4c9d042112b", "Portable Service detached");
649
- msgid_into_dict("9cf56b8baf9546cf9478783a8de42113", "systemd-networkd sysctl changed by foreign process");
650
- msgid_into_dict("ad7089f928ac4f7ea00c07457d47ba8a", "SRK into TPM authorization failure");
651
- msgid_into_dict("b2bcbaf5edf948e093ce50bbea0e81ec", "Secure Attention Key (SAK) was pressed");
652
-
653
- // dbus
654
- // https://github.com/bus1/dbus-broker/blob/main/src/catalog/catalog-ids.h
655
- msgid_into_dict("7fc63312330b479bb32e598d47cef1a8", "dbus activate no unit");
656
- msgid_into_dict("ee9799dab1e24d81b7bee7759a543e1b", "dbus activate masked unit");
657
- msgid_into_dict("a0fa58cafd6f4f0c8d003d16ccf9e797", "dbus broker exited");
658
- msgid_into_dict("c8c6cde1c488439aba371a664353d9d8", "dbus dirwatch");
659
- msgid_into_dict("8af3357071af4153af414daae07d38e7", "dbus dispatch stats");
660
- msgid_into_dict("199d4300277f495f84ba4028c984214c", "dbus no sopeergroup");
661
- msgid_into_dict("b209c0d9d1764ab38d13b8e00d1784d6", "dbus protocol violation");
662
- msgid_into_dict("6fa70fa776044fa28be7a21daf42a108", "dbus receive failed");
663
- msgid_into_dict("0ce0fa61d1a9433dabd67417f6b8e535", "dbus service failed open");
664
- msgid_into_dict("24dc708d9e6a4226a3efe2033bb744de", "dbus service invalid");
665
- msgid_into_dict("f15d2347662d483ea9bcd8aa1a691d28", "dbus sighup");
666
-
667
- // gnome
668
- // https://gitlab.gnome.org/GNOME/gnome-session/-/blob/main/gnome-session/gsm-manager.c
669
- msgid_into_dict("0ce153587afa4095832d233c17a88001", "Gnome SM startup succeeded");
670
- msgid_into_dict("10dd2dc188b54a5e98970f56499d1f73", "Gnome SM unrecoverable failure");
671
-
672
- // gnome-shell
673
- // https://gitlab.gnome.org/GNOME/gnome-shell/-/blob/main/js/ui/main.js#L56
674
- msgid_into_dict("f3ea493c22934e26811cd62abe8e203a", "Gnome shell started");
675
-
676
- // flathub
677
- // https://docs.flatpak.org/de/latest/flatpak-command-reference.html
678
- msgid_into_dict("c7b39b1e006b464599465e105b361485", "Flatpak cache");
679
-
680
- // ???
681
- msgid_into_dict("75ba3deb0af041a9a46272ff85d9e73e", "Flathub pulls");
682
- msgid_into_dict("f02bce89a54e4efab3a94a797d26204a", "Flathub pull errors");
683
-
684
- // ??
685
- msgid_into_dict("dd11929c788e48bdbb6276fb5f26b08a", "Boltd starting");
686
-
687
- // Netdata
688
- msgid_into_dict("1e6061a9fbd44501b3ccc368119f2b69", "Netdata startup");
689
- msgid_into_dict("ed4cdb8f1beb4ad3b57cb3cae2d162fa", "Netdata connection from child");
690
- msgid_into_dict("6e2e3839067648968b646045dbf28d66", "Netdata connection to parent");
691
- msgid_into_dict("9ce0cb58ab8b44df82c4bf1ad9ee22de", "Netdata alert transition");
692
- msgid_into_dict("6db0018e83e34320ae2a659d78019fb7", "Netdata alert notification");
693
- msgid_into_dict("23e93dfccbf64e11aac858b9410d8a82", "Netdata fatal message");
694
- msgid_into_dict("8ddaf5ba33a74078b609250db1e951f3", "Sensor state transition");
695
- msgid_into_dict("ec87a56120d5431bace51e2fb8bba243", "Netdata log flood protection");
696
- msgid_into_dict("acb33cb95778476baac702eb7e4e151d", "Netdata Cloud connection");
697
- msgid_into_dict("d1f59606dd4d41e3b217a0cfcae8e632", "Netdata extreme cardinality");
698
- msgid_into_dict("02f47d350af5449197bf7a95b605a468", "Netdata exit reason");
699
- msgid_into_dict("4fdf40816c124623a032b7fe73beacb8", "Netdata dynamic configuration");
700
-}
701
-
702
-void nd_sd_journal_transform_message_id(
703
- FACETS *facets __maybe_unused,
704
- BUFFER *wb,
705
- FACETS_TRANSFORMATION_SCOPE scope __maybe_unused,
706
- void *data __maybe_unused)
707
-{
708
- const char *message_id = buffer_tostring(wb);
709
- struct message_id_info *i = dictionary_get(known_journal_messages_ids, message_id);
710
-
711
- if (!i)
712
- return;
713
-
714
- switch (scope) {
715
- default:
716
- case FACETS_TRANSFORM_DATA:
717
- case FACETS_TRANSFORM_VALUE:
718
- buffer_sprintf(wb, " (%s)", i->msg);
719
- break;
720
-
721
- case FACETS_TRANSFORM_FACET:
722
- case FACETS_TRANSFORM_FACET_SORT:
723
- case FACETS_TRANSFORM_HISTOGRAM:
724
- buffer_flush(wb);
725
- buffer_strcat(wb, i->msg);
726
- break;
727
- }
728
-}
729
-
730
-void nd_sd_journal_annotations_init(void)
731
-{
732
- cached_usernames_init();
733
- cached_groupnames_init();
734
- update_cached_host_users();
735
- update_cached_host_groups();
736
- nd_sd_journal_message_ids_init();
737
-}
src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c
deleted
-177
@@ -1,177 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "systemd-internals.h"
4
-
5
-#define JOURNAL_DIRECTORIES_JSON_NODE "journalDirectories"
6
-
7
-static bool is_directory(const char *dir)
8
-{
9
- struct stat statbuf;
10
- if (stat(dir, &statbuf) != 0) {
11
- // Error in stat() means the path probably doesn't exist or can't be accessed.
12
- return false;
13
- }
14
- // S_ISDIR macro is true if the path is a directory.
15
- return S_ISDIR(statbuf.st_mode) ? true : false;
16
-}
17
-
18
-static const char *is_valid_dir(const char *dir)
19
-{
20
- if (strcmp(dir, "/") == 0)
21
- return "/ is not acceptable";
22
-
23
- if (!strstartswith(dir, "/"))
24
- return "only directories starting with / are accepted";
25
-
26
- if (strstr(dir, "/./"))
27
- return "directory contains /./";
28
-
29
- if (strstr(dir, "/../") || strendswith(dir, "/.."))
30
- return "directory contains /../";
31
-
32
- if (strstartswith(dir, "/dev/") || strcmp(dir, "/dev") == 0)
33
- return "directory contains /dev";
34
-
35
- if (strstartswith(dir, "/proc/") || strcmp(dir, "/proc") == 0)
36
- return "directory contains /proc";
37
-
38
- if (strstartswith(dir, "/sys/") || strcmp(dir, "/sys") == 0)
39
- return "directory contains /sys";
40
-
41
- if (strstartswith(dir, "/etc/") || strcmp(dir, "/etc") == 0)
42
- return "directory contains /etc";
43
-
44
- if (strstartswith(dir, "/lib/") || strcmp(dir, "/lib") == 0 || strstartswith(dir, "/lib32/") ||
45
- strcmp(dir, "/lib32") == 0 || strstartswith(dir, "/lib64/") || strcmp(dir, "/lib64") == 0)
46
- return "directory contains /lib";
47
-
48
- return NULL;
49
-}
50
-
51
-static int systemd_journal_directories_dyncfg_update(BUFFER *result, BUFFER *payload)
52
-{
53
- if (!payload || !buffer_strlen(payload))
54
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "empty payload received");
55
-
56
- CLEAN_JSON_OBJECT *jobj = json_tokener_parse(buffer_tostring(payload));
57
- if (!jobj)
58
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "cannot parse json payload");
59
-
60
- struct json_object *journalDirectories;
61
- json_object_object_get_ex(jobj, JOURNAL_DIRECTORIES_JSON_NODE, &journalDirectories);
62
-
63
- if (json_object_get_type(journalDirectories) != json_type_array)
64
- return dyncfg_default_response(
65
- result, HTTP_RESP_BAD_REQUEST, "member " JOURNAL_DIRECTORIES_JSON_NODE " is not an array");
66
-
67
- size_t n_directories = json_object_array_length(journalDirectories);
68
- if (n_directories > MAX_JOURNAL_DIRECTORIES)
69
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "too many directories configured");
70
-
71
- // validate the directories
72
- for (size_t i = 0; i < n_directories; i++) {
73
- struct json_object *dir = json_object_array_get_idx(journalDirectories, i);
74
- const char *s = json_object_get_string(dir);
75
- if (s && *s) {
76
- const char *msg = is_valid_dir(s);
77
- if (msg)
78
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, msg);
79
- }
80
- }
81
-
82
- size_t added = 0, not_found = 0;
83
- for (size_t i = 0; i < n_directories; i++) {
84
- struct json_object *dir = json_object_array_get_idx(journalDirectories, i);
85
- const char *s = json_object_get_string(dir);
86
- if (s && *s) {
87
- string_freez(journal_directories[added].path);
88
- journal_directories[added++].path = string_strdupz(s);
89
-
90
- if (!is_directory(s))
91
- not_found++;
92
- }
93
- }
94
-
95
- if (!added)
96
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "no directories in the payload");
97
- else {
98
- for (size_t i = added; i < MAX_JOURNAL_DIRECTORIES; i++) {
99
- string_freez(journal_directories[i].path);
100
- journal_directories[i].path = NULL;
101
- }
102
- }
103
-
104
- nd_journal_watcher_restart();
105
-
106
- return dyncfg_default_response(
107
- result, HTTP_RESP_OK, not_found ? "added, but some directories are not found in the filesystem" : "");
108
-}
109
-
110
-static int systemd_journal_directories_dyncfg_get(BUFFER *wb)
111
-{
112
- buffer_flush(wb);
113
- buffer_json_initialize(wb, "\"", "\"", 0, true, BUFFER_JSON_OPTIONS_MINIFY);
114
-
115
- buffer_json_member_add_array(wb, JOURNAL_DIRECTORIES_JSON_NODE);
116
- for (size_t i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
117
- if (!journal_directories[i].path)
118
- break;
119
-
120
- buffer_json_add_array_item_string(wb, string2str(journal_directories[i].path));
121
- }
122
- buffer_json_array_close(wb);
123
-
124
- buffer_json_finalize(wb);
125
- return HTTP_RESP_OK;
126
-}
127
-
128
-static int systemd_journal_directories_dyncfg_cb(
129
- const char *transaction,
130
- const char *id,
131
- DYNCFG_CMDS cmd,
132
- const char *add_name __maybe_unused,
133
- BUFFER *payload,
134
- usec_t *stop_monotonic_ut __maybe_unused,
135
- bool *cancelled __maybe_unused,
136
- BUFFER *result,
137
- HTTP_ACCESS access __maybe_unused,
138
- const char *source __maybe_unused,
139
- void *data __maybe_unused)
140
-{
141
- CLEAN_BUFFER *action = buffer_create(100, NULL);
142
- dyncfg_cmds2buffer(cmd, action);
143
-
144
- if (cmd == DYNCFG_CMD_GET)
145
- return systemd_journal_directories_dyncfg_get(result);
146
-
147
- if (cmd == DYNCFG_CMD_UPDATE)
148
- return systemd_journal_directories_dyncfg_update(result, payload);
149
-
150
- nd_log(
151
- NDLS_COLLECTORS,
152
- NDLP_ERR,
153
- "DYNCFG: unhandled transaction '%s', id '%s' cmd '%s', payload: %s",
154
- transaction,
155
- id,
156
- buffer_tostring(action),
157
- payload ? buffer_tostring(payload) : "");
158
-
159
- return dyncfg_default_response(result, HTTP_RESP_BAD_REQUEST, "the command is not handled by this plugin");
160
-}
161
-
162
-void nd_systemd_journal_dyncfg_init(struct functions_evloop_globals *wg)
163
-{
164
- functions_evloop_dyncfg_add(
165
- wg,
166
- "systemd-journal:monitored-directories",
167
- "/logs/systemd-journal",
168
- DYNCFG_STATUS_RUNNING,
169
- DYNCFG_TYPE_SINGLE,
170
- DYNCFG_SOURCE_TYPE_INTERNAL,
171
- "internal",
172
- DYNCFG_CMD_SCHEMA | DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE,
173
- HTTP_ACCESS_NONE,
174
- HTTP_ACCESS_NONE,
175
- systemd_journal_directories_dyncfg_cb,
176
- NULL);
177
-}
src/collectors/systemd-journal.plugin/systemd-journal-files.c
deleted
-872
@@ -1,872 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
4
-#include "systemd-internals.h"
5
-
6
-#define ND_SD_JOURNAL_MAX_SOURCE_LEN 64
7
-#define VAR_LOG_JOURNAL_MAX_DEPTH 10
8
-
9
-struct journal_directory journal_directories[MAX_JOURNAL_DIRECTORIES] = {0};
10
-DICTIONARY *nd_journal_files_registry = NULL;
11
-DICTIONARY *used_hashes_registry = NULL;
12
-
13
-static usec_t systemd_journal_session = 0;
14
-
15
-void buffer_json_journal_versions(BUFFER *wb)
16
-{
17
- buffer_json_member_add_object(wb, "versions");
18
- {
19
- buffer_json_member_add_uint64(
20
- wb, "sources", systemd_journal_session + dictionary_version(nd_journal_files_registry));
21
- }
22
- buffer_json_object_close(wb);
23
-}
24
-
25
-static bool journal_sd_id128_parse(const char *in, NsdId128 *ret)
26
-{
27
- while (isspace(*in))
28
- in++;
29
-
30
- char uuid[33];
31
- strncpyz(uuid, in, 32);
32
- uuid[32] = '\0';
33
-
34
- if (strlen(uuid) == 32) {
35
- NsdId128 read;
36
- if (nsd_id128_from_string(uuid, &read) == 0) {
37
- *ret = read;
38
- return true;
39
- }
40
- }
41
-
42
- return false;
43
-}
44
-
45
-usec_t
46
-nd_journal_file_update_annotation_boot_id(NsdJournal *j, struct nd_journal_file *njf __maybe_unused, const char *boot_id)
47
-{
48
- usec_t ut = UINT64_MAX;
49
- int r;
50
-
51
- char m[100];
52
- size_t len = snprintfz(m, sizeof(m), "_BOOT_ID=%s", boot_id);
53
-
54
- nsd_journal_flush_matches(j);
55
-
56
- r = nsd_journal_add_match(j, m, len);
57
- if (r < 0) {
58
- errno = -r;
59
- internal_error(
60
- true,
61
- "JOURNAL: while looking for the first timestamp of boot_id '%s', "
62
- "sd_journal_add_match('%s') on file '%s' returned %d",
63
- boot_id,
64
- m,
65
- njf->filename,
66
- r);
67
- return UINT64_MAX;
68
- }
69
-
70
- r = nsd_journal_seek_head(j);
71
- if (r < 0) {
72
- errno = -r;
73
- internal_error(
74
- true,
75
- "JOURNAL: while looking for the first timestamp of boot_id '%s', "
76
- "sd_journal_seek_head() on file '%s' returned %d",
77
- boot_id,
78
- njf->filename,
79
- r);
80
- return UINT64_MAX;
81
- }
82
-
83
- r = nsd_journal_next(j);
84
- if (r < 0) {
85
- errno = -r;
86
- internal_error(
87
- true,
88
- "JOURNAL: while looking for the first timestamp of boot_id '%s', "
89
- "sd_journal_next() on file '%s' returned %d",
90
- boot_id,
91
- njf->filename,
92
- r);
93
- return UINT64_MAX;
94
- } else if (r == 0) {
95
- internal_error(
96
- true,
97
- "GVD: while looking for the first timestamp of boot_id '%s', "
98
- "sd_journal_next() on file '%s' returned %d",
99
- boot_id,
100
- njf->filename,
101
- r);
102
- return UINT64_MAX;
103
- }
104
-
105
- r = nsd_journal_get_realtime_usec(j, &ut);
106
- if (r < 0 || !ut || ut == UINT64_MAX) {
107
- errno = -r;
108
- internal_error(
109
- r != -EADDRNOTAVAIL,
110
- "JOURNAL: while looking for the first timestamp of boot_id '%s', "
111
- "sd_journal_get_realtime_usec() on file '%s' returned %d",
112
- boot_id,
113
- njf->filename,
114
- r);
115
- return UINT64_MAX;
116
- }
117
-
118
- if (ut && ut != UINT64_MAX) {
119
- dictionary_set(boot_ids_to_first_ut, boot_id, &ut, sizeof(ut));
120
- return ut;
121
- }
122
-
123
- return UINT64_MAX;
124
-}
125
-
126
-static void
127
-nd_journal_file_get_boot_id_annotations(NsdJournal *j __maybe_unused, struct nd_journal_file *njf __maybe_unused)
128
-{
129
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
130
- nsd_journal_flush_matches(j);
131
-
132
- int r = nsd_journal_query_unique(j, "_BOOT_ID");
133
- if (r < 0) {
134
- errno = -r;
135
- internal_error(
136
- true,
137
- "JOURNAL: while querying for the unique _BOOT_ID values, "
138
- "sd_journal_query_unique() on file '%s' returned %d",
139
- njf->filename,
140
- r);
141
- errno = -r;
142
- return;
143
- }
144
-
145
- const void *data = NULL;
146
- size_t data_length;
147
-
148
- DICTIONARY *dict = dictionary_create(DICT_OPTION_SINGLE_THREADED);
149
-
150
- NSD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
151
- {
152
- const char *key, *value;
153
- size_t key_length, value_length;
154
-
155
- if (!parse_journal_field(data, data_length, &key, &key_length, &value, &value_length))
156
- continue;
157
-
158
- if (value_length != 32)
159
- continue;
160
-
161
- char buf[33];
162
- memcpy(buf, value, 32);
163
- buf[32] = '\0';
164
-
165
- dictionary_set(dict, buf, NULL, 0);
166
- }
167
-
168
- void *nothing;
169
- dfe_start_read(dict, nothing)
170
- {
171
- nd_journal_file_update_annotation_boot_id(j, njf, nothing_dfe.name);
172
- }
173
- dfe_done(nothing);
174
-
175
- dictionary_destroy(dict);
176
-#endif
177
-}
178
-
179
-void nd_journal_file_update_header(const char *filename, struct nd_journal_file *njf)
180
-{
181
- if (njf->last_scan_header_vs_last_modified_ut == njf->file_last_modified_ut)
182
- return;
183
-
184
- fstat_cache_enable_on_thread();
185
-
186
- const char *files[2] = {
187
- [0] = filename,
188
- [1] = NULL,
189
- };
190
-
191
- NsdJournal *j = NULL;
192
- if (nsd_journal_open_files(&j, files, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
193
- netdata_log_error("JOURNAL: cannot open file '%s' to update msg_ut", filename);
194
- fstat_cache_disable_on_thread();
195
-
196
- if (!njf->logged_failure) {
197
- netdata_log_error(
198
- "cannot open journal file '%s', using file timestamps to understand time-frame.", filename);
199
- njf->logged_failure = true;
200
- }
201
-
202
- njf->msg_first_ut = 0;
203
- njf->msg_last_ut = njf->file_last_modified_ut;
204
- njf->last_scan_header_vs_last_modified_ut = njf->file_last_modified_ut;
205
- return;
206
- }
207
-
208
- usec_t first_ut = 0, last_ut = 0;
209
- uint64_t first_seqnum = 0, last_seqnum = 0;
210
- NsdId128 first_writer_id = NSD_ID128_NULL, last_writer_id = NSD_ID128_NULL;
211
-
212
- if (nsd_journal_seek_head(j) < 0 || nsd_journal_next(j) < 0 || nsd_journal_get_realtime_usec(j, &first_ut) < 0 ||
213
- !first_ut) {
214
- internal_error(true, "cannot find the timestamp of the first message in '%s'", filename);
215
- first_ut = 0;
216
- }
217
-#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
218
- else {
219
- if (nsd_journal_get_seqnum(j, &first_seqnum, &first_writer_id) < 0 || !first_seqnum) {
220
- internal_error(true, "cannot find the first seqnums of the first message in '%s'", filename);
221
- first_seqnum = 0;
222
- memset(&first_writer_id, 0, sizeof(first_writer_id));
223
- }
224
- }
225
-#endif
226
-
227
- if (nsd_journal_seek_tail(j) < 0 || nsd_journal_previous(j) < 0 || nsd_journal_get_realtime_usec(j, &last_ut) < 0 ||
228
- !last_ut) {
229
- internal_error(true, "cannot find the timestamp of the last message in '%s'", filename);
230
- last_ut = njf->file_last_modified_ut;
231
- }
232
-#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
233
- else {
234
- if (nsd_journal_get_seqnum(j, &last_seqnum, &last_writer_id) < 0 || !last_seqnum) {
235
- internal_error(true, "cannot find the last seqnums of the first message in '%s'", filename);
236
- last_seqnum = 0;
237
- memset(&last_writer_id, 0, sizeof(last_writer_id));
238
- }
239
- }
240
-#endif
241
-
242
- if (first_ut > last_ut) {
243
- internal_error(true, "timestamps are flipped in file '%s'", filename);
244
- usec_t t = first_ut;
245
- first_ut = last_ut;
246
- last_ut = t;
247
- }
248
-
249
- if (!first_seqnum || !first_ut) {
250
- // extract these from the filename - if possible
251
-
252
- const char *at = strchr(filename, '@');
253
- if (at) {
254
- const char *dash_seqnum = strchr(at + 1, '-');
255
- if (dash_seqnum) {
256
- const char *dash_first_msg_ut = strchr(dash_seqnum + 1, '-');
257
- if (dash_first_msg_ut) {
258
- const char *dot_journal = NULL;
259
- if (is_journal_file(filename, -1, &dot_journal) && dot_journal && dot_journal > dash_first_msg_ut) {
260
- if (dash_seqnum - at - 1 == 32 && dash_first_msg_ut - dash_seqnum - 1 == 16 &&
261
- dot_journal - dash_first_msg_ut - 1 == 16) {
262
- NsdId128 writer;
263
- if (journal_sd_id128_parse(at + 1, &writer)) {
264
- char *endptr = NULL;
265
- uint64_t seqnum = strtoul(dash_seqnum + 1, &endptr, 16);
266
- if (endptr == dash_first_msg_ut) {
267
- uint64_t ts = strtoul(dash_first_msg_ut + 1, &endptr, 16);
268
- if (endptr == dot_journal) {
269
- first_seqnum = seqnum;
270
- first_writer_id = writer;
271
- first_ut = ts;
272
- }
273
- }
274
- }
275
- }
276
- }
277
- }
278
- }
279
- }
280
- }
281
-
282
- njf->first_seqnum = first_seqnum;
283
- njf->last_seqnum = last_seqnum;
284
-
285
- njf->first_writer_id = first_writer_id;
286
- njf->last_writer_id = last_writer_id;
287
-
288
- njf->msg_first_ut = first_ut;
289
- njf->msg_last_ut = last_ut;
290
-
291
- if (!njf->msg_last_ut)
292
- njf->msg_last_ut = njf->file_last_modified_ut;
293
-
294
- if (last_seqnum > first_seqnum) {
295
- if (!nsd_id128_equal(first_writer_id, last_writer_id)) {
296
- njf->messages_in_file = 0;
297
- nd_log(
298
- NDLS_COLLECTORS,
299
- NDLP_NOTICE,
300
- "The writers of the first and the last message in file '%s' differ.",
301
- filename);
302
- } else
303
- njf->messages_in_file = last_seqnum - first_seqnum + 1;
304
- } else
305
- njf->messages_in_file = 0;
306
-
307
- nd_journal_file_get_boot_id_annotations(j, njf);
308
- nsd_journal_close(j);
309
- fstat_cache_disable_on_thread();
310
-
311
- njf->last_scan_header_vs_last_modified_ut = njf->file_last_modified_ut;
312
-}
313
-
314
-static STRING *string_strdupz_source(const char *s, const char *e, size_t max_len, const char *prefix)
315
-{
316
- char buf[max_len];
317
- size_t len;
318
- char *dst = buf;
319
-
320
- if (prefix) {
321
- len = strlen(prefix);
322
- memcpy(buf, prefix, len);
323
- dst = &buf[len];
324
- max_len -= len;
325
- }
326
-
327
- len = e - s;
328
- if (len >= max_len)
329
- len = max_len - 1;
330
- memcpy(dst, s, len);
331
- dst[len] = '\0';
332
- buf[max_len - 1] = '\0';
333
-
334
- for (size_t i = 0; buf[i]; i++)
335
- if (!is_netdata_api_valid_character(buf[i]))
336
- buf[i] = '_';
337
-
338
- return string_strdupz(buf);
339
-}
340
-
341
-static void files_registry_insert_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused)
342
-{
343
- struct nd_journal_file *njf = value;
344
- njf->filename = dictionary_acquired_item_name(item);
345
- njf->filename_len = strlen(njf->filename);
346
- njf->source_type = ND_SD_JF_ALL;
347
-
348
- // based on the filename
349
- // decide the source to show to the user
350
- const char *s = strrchr(njf->filename, '/');
351
- if (s) {
352
- if (strstr(njf->filename, "/remote/")) {
353
- njf->source_type |= ND_SD_JF_REMOTE_ALL;
354
-
355
- if (strncmp(s, "/remote-", 8) == 0) {
356
- s = &s[8]; // skip "/remote-"
357
-
358
- char *e = strchr(s, '@');
359
- if (!e)
360
- is_journal_file(s, -1, (const char **)&e);
361
-
362
- if (e) {
363
- const char *d = s;
364
- for (; d < e && (isdigit(*d) || *d == '.' || *d == ':'); d++)
365
- ;
366
- if (d == e) {
367
- // a valid IP address
368
- char ip[e - s + 1];
369
- memcpy(ip, s, e - s);
370
- ip[e - s] = '\0';
371
- char buf[ND_SD_JOURNAL_MAX_SOURCE_LEN];
372
- if (ip_to_hostname(ip, buf, sizeof(buf)))
373
- njf->source =
374
- string_strdupz_source(buf, &buf[strlen(buf)], ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-");
375
- else {
376
- internal_error(true, "Cannot find the hostname for IP '%s'", ip);
377
- njf->source = string_strdupz_source(s, e, ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-");
378
- }
379
- } else
380
- njf->source = string_strdupz_source(s, e, ND_SD_JOURNAL_MAX_SOURCE_LEN, "remote-");
381
- }
382
- }
383
- } else {
384
- njf->source_type |= ND_SD_JF_LOCAL_ALL;
385
-
386
- const char *t = s - 1;
387
- while (t >= njf->filename && *t != '.' && *t != '/')
388
- t--;
389
-
390
- if (t >= njf->filename && *t == '.') {
391
- njf->source_type |= ND_SD_JF_LOCAL_NAMESPACE;
392
- njf->source = string_strdupz_source(t + 1, s, ND_SD_JOURNAL_MAX_SOURCE_LEN, "namespace-");
393
- } else if (strncmp(s, "/system", 7) == 0)
394
- njf->source_type |= ND_SD_JF_LOCAL_SYSTEM;
395
-
396
- else if (strncmp(s, "/user", 5) == 0)
397
- njf->source_type |= ND_SD_JF_LOCAL_USER;
398
-
399
- else
400
- njf->source_type |= ND_SD_JF_LOCAL_OTHER;
401
- }
402
- } else
403
- njf->source_type |= ND_SD_JF_LOCAL_ALL | ND_SD_JF_LOCAL_OTHER;
404
-
405
- njf->msg_last_ut = njf->file_last_modified_ut;
406
-
407
- nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "Journal file added to the journal files registry: '%s'", njf->filename);
408
-}
409
-
410
-static bool files_registry_conflict_cb(
411
- const DICTIONARY_ITEM *item __maybe_unused,
412
- void *old_value,
413
- void *new_value,
414
- void *data __maybe_unused)
415
-{
416
- struct nd_journal_file *njf_old = old_value;
417
- struct nd_journal_file *njf_new = new_value;
418
-
419
- if (njf_new->last_scan_monotonic_ut > njf_old->last_scan_monotonic_ut)
420
- njf_old->last_scan_monotonic_ut = njf_new->last_scan_monotonic_ut;
421
-
422
- if (njf_new->file_last_modified_ut > njf_old->file_last_modified_ut) {
423
- njf_old->file_last_modified_ut = njf_new->file_last_modified_ut;
424
- njf_old->size = njf_new->size;
425
-
426
- njf_old->msg_last_ut = njf_old->file_last_modified_ut;
427
- }
428
-
429
- return false;
430
-}
431
-
432
-struct nd_journal_file_source {
433
- usec_t first_ut;
434
- usec_t last_ut;
435
- size_t count;
436
- uint64_t size;
437
-};
438
-
439
-#define print_duration(dst, dst_len, pos, remaining, duration, one, many, printed) \
440
- do { \
441
- if ((remaining) > (duration)) { \
442
- uint64_t _count = (remaining) / (duration); \
443
- uint64_t _rem = (remaining) - (_count * (duration)); \
444
- (pos) += snprintfz( \
445
- &(dst)[pos], \
446
- (dst_len) - (pos), \
447
- "%s%s%" PRIu64 " %s", \
448
- (printed) ? ", " : "", \
449
- _rem ? "" : "and ", \
450
- _count, \
451
- _count > 1 ? (many) : (one)); \
452
- (remaining) = _rem; \
453
- (printed) = true; \
454
- } \
455
- } while (0)
456
-
457
-static int nd_journal_file_to_json_array_cb(const DICTIONARY_ITEM *item, void *entry, void *data)
458
-{
459
- struct nd_journal_file_source *nd_jfs = entry;
460
- BUFFER *wb = data;
461
-
462
- const char *name = dictionary_acquired_item_name(item);
463
-
464
- buffer_json_add_array_item_object(wb);
465
- {
466
- char size_for_humans[128];
467
- size_snprintf(size_for_humans, sizeof(size_for_humans), nd_jfs->size, "B", false);
468
-
469
- char duration_for_humans[128];
470
- duration_snprintf(
471
- duration_for_humans,
472
- sizeof(duration_for_humans),
473
- (time_t)((nd_jfs->last_ut - nd_jfs->first_ut) / USEC_PER_SEC),
474
- "s",
475
- true);
476
-
477
- char last_ut[RFC3339_MAX_LENGTH];
478
- rfc3339_datetime_ut(last_ut, sizeof(last_ut), nd_jfs->last_ut, 0, true);
479
-
480
- char info[1024];
481
- snprintfz(
482
- info,
483
- sizeof(info),
484
- "%zu files, total size %s, covering %s, last entry at %s",
485
- nd_jfs->count,
486
- size_for_humans,
487
- duration_for_humans,
488
- last_ut);
489
-
490
- buffer_json_member_add_string(wb, "id", name);
491
- buffer_json_member_add_string(wb, "name", name);
492
- buffer_json_member_add_string(wb, "pill", size_for_humans);
493
- buffer_json_member_add_string(wb, "info", info);
494
- }
495
- buffer_json_object_close(wb); // options object
496
-
497
- return 1;
498
-}
499
-
500
-static bool nd_journal_file_merge_sizes(
501
- const DICTIONARY_ITEM *item __maybe_unused,
502
- void *old_value,
503
- void *new_value,
504
- void *data __maybe_unused)
505
-{
506
- struct nd_journal_file_source *jfs = old_value, *njfs = new_value;
507
- jfs->count += njfs->count;
508
- jfs->size += njfs->size;
509
-
510
- if (njfs->first_ut && njfs->first_ut < jfs->first_ut)
511
- jfs->first_ut = njfs->first_ut;
512
-
513
- if (njfs->last_ut && njfs->last_ut > jfs->last_ut)
514
- jfs->last_ut = njfs->last_ut;
515
-
516
- return false;
517
-}
518
-
519
-void available_journal_file_sources_to_json_array(BUFFER *wb)
520
-{
521
- DICTIONARY *dict = dictionary_create(
522
- DICT_OPTION_SINGLE_THREADED | DICT_OPTION_NAME_LINK_DONT_CLONE | DICT_OPTION_DONT_OVERWRITE_VALUE);
523
- dictionary_register_conflict_callback(dict, nd_journal_file_merge_sizes, NULL);
524
-
525
- struct nd_journal_file_source njfs_tmp = {0};
526
-
527
- struct nd_journal_file *njf;
528
- dfe_start_read(nd_journal_files_registry, njf)
529
- {
530
- njfs_tmp.first_ut = njf->msg_first_ut;
531
- njfs_tmp.last_ut = njf->msg_last_ut;
532
- njfs_tmp.count = 1;
533
- njfs_tmp.size = njf->size;
534
-
535
- dictionary_set(dict, ND_SD_JF_SOURCE_ALL_NAME, &njfs_tmp, sizeof(njfs_tmp));
536
-
537
- if (njf->source_type & ND_SD_JF_LOCAL_ALL)
538
- dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_NAME, &njfs_tmp, sizeof(njfs_tmp));
539
- if (njf->source_type & ND_SD_JF_LOCAL_SYSTEM)
540
- dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_SYSTEM_NAME, &njfs_tmp, sizeof(njfs_tmp));
541
- if (njf->source_type & ND_SD_JF_LOCAL_USER)
542
- dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_USERS_NAME, &njfs_tmp, sizeof(njfs_tmp));
543
- if (njf->source_type & ND_SD_JF_LOCAL_OTHER)
544
- dictionary_set(dict, ND_SD_JF_SOURCE_LOCAL_OTHER_NAME, &njfs_tmp, sizeof(njfs_tmp));
545
- if (njf->source_type & ND_SD_JF_LOCAL_NAMESPACE)
546
- dictionary_set(dict, ND_SD_JF_SOURCE_NAMESPACES_NAME, &njfs_tmp, sizeof(njfs_tmp));
547
- if (njf->source_type & ND_SD_JF_REMOTE_ALL)
548
- dictionary_set(dict, ND_SD_JF_SOURCE_REMOTES_NAME, &njfs_tmp, sizeof(njfs_tmp));
549
- if (njf->source)
550
- dictionary_set(dict, string2str(njf->source), &njfs_tmp, sizeof(njfs_tmp));
551
- }
552
- dfe_done(jf);
553
-
554
- dictionary_sorted_walkthrough_read(dict, nd_journal_file_to_json_array_cb, wb);
555
-
556
- dictionary_destroy(dict);
557
-}
558
-
559
-static void files_registry_delete_cb(const DICTIONARY_ITEM *item, void *value, void *data __maybe_unused)
560
-{
561
- struct nd_journal_file *njf = value;
562
- const char *filename = dictionary_acquired_item_name(item);
563
- (void)filename;
564
-
565
- internal_error(true, "removed journal file '%s'", filename);
566
- string_freez(njf->source);
567
-}
568
-
569
-#define EXT_DOT_JOURNAL ".journal"
570
-#define EXT_DOT_JOURNAL_TILDA ".journal~"
571
-
572
-static struct {
573
- const char *ext;
574
- ssize_t len;
575
-} valid_journal_extension[] = {
576
- {.ext = EXT_DOT_JOURNAL, .len = sizeof(EXT_DOT_JOURNAL) - 1},
577
- {.ext = EXT_DOT_JOURNAL_TILDA, .len = sizeof(EXT_DOT_JOURNAL_TILDA) - 1},
578
-};
579
-
580
-bool is_journal_file(const char *filename, ssize_t len, const char **start_of_extension)
581
-{
582
- if (len < 0)
583
- len = (ssize_t)strlen(filename);
584
-
585
- for (size_t i = 0; i < _countof(valid_journal_extension); i++) {
586
- const char *ext = valid_journal_extension[i].ext;
587
- ssize_t elen = valid_journal_extension[i].len;
588
-
589
- if (len > elen && strcmp(filename + len - elen, ext) == 0) {
590
- if (start_of_extension)
591
- *start_of_extension = filename + len - elen;
592
- return true;
593
- }
594
- }
595
-
596
- if (start_of_extension)
597
- *start_of_extension = NULL;
598
-
599
- return false;
600
-}
601
-
602
-void nd_journal_directory_scan_recursively(DICTIONARY *files, DICTIONARY *dirs, const char *dirname, int depth)
603
-{
604
- if (depth > VAR_LOG_JOURNAL_MAX_DEPTH)
605
- return;
606
-
607
- DIR *dir;
608
- struct dirent *entry;
609
- char full_path[FILENAME_MAX];
610
-
611
- // Open the directory.
612
- if ((dir = opendir(dirname)) == NULL) {
613
- if (errno != ENOENT && errno != ENOTDIR)
614
- netdata_log_error("Cannot opendir() '%s'", dirname);
615
- return;
616
- }
617
-
618
- bool existing = false;
619
- bool *found = dictionary_set(dirs, dirname, &existing, sizeof(existing));
620
- if (*found)
621
- return;
622
- *found = true;
623
-
624
- // Read each entry in the directory.
625
- while ((entry = readdir(dir)) != NULL) {
626
- if (strcmp(entry->d_name, ".") == 0 || strcmp(entry->d_name, "..") == 0)
627
- continue;
628
-
629
- ssize_t len = snprintfz(full_path, sizeof(full_path), "%s/%s", dirname, entry->d_name);
630
-
631
- if (entry->d_type == DT_DIR) {
632
- nd_journal_directory_scan_recursively(files, dirs, full_path, depth++);
633
- } else if (entry->d_type == DT_REG && is_journal_file(full_path, len, NULL)) {
634
- if (files)
635
- dictionary_set(files, full_path, NULL, 0);
636
-
637
- send_newline_and_flush(&stdout_mutex);
638
- } else if (entry->d_type == DT_LNK) {
639
- struct stat info;
640
- if (stat(full_path, &info) == -1)
641
- continue;
642
-
643
- if (S_ISDIR(info.st_mode)) {
644
- // The symbolic link points to a directory
645
- char resolved_path[FILENAME_MAX + 1];
646
- if (realpath(full_path, resolved_path) != NULL) {
647
- nd_journal_directory_scan_recursively(files, dirs, resolved_path, depth++);
648
- }
649
- } else if (S_ISREG(info.st_mode) && is_journal_file(full_path, len, NULL)) {
650
- if (files)
651
- dictionary_set(files, full_path, NULL, 0);
652
-
653
- send_newline_and_flush(&stdout_mutex);
654
- }
655
- }
656
- }
657
-
658
- closedir(dir);
659
-}
660
-
661
-static size_t nd_journal_files_scans = 0;
662
-bool nd_journal_files_completed_once(void)
663
-{
664
- return nd_journal_files_scans > 0;
665
-}
666
-
667
-int filenames_compar(const void *a, const void *b)
668
-{
669
- const char *p1 = *(const char **)a;
670
- const char *p2 = *(const char **)b;
671
-
672
- const char *at1 = strchr(p1, '@');
673
- const char *at2 = strchr(p2, '@');
674
-
675
- if (!at1 && at2)
676
- return -1;
677
-
678
- if (at1 && !at2)
679
- return 1;
680
-
681
- if (!at1 && !at2)
682
- return strcmp(p1, p2);
683
-
684
- const char *dash1 = strrchr(at1, '-');
685
- const char *dash2 = strrchr(at2, '-');
686
-
687
- if (!dash1 || !dash2)
688
- return strcmp(p1, p2);
689
-
690
- uint64_t ts1 = strtoul(dash1 + 1, NULL, 16);
691
- uint64_t ts2 = strtoul(dash2 + 1, NULL, 16);
692
-
693
- if (ts1 > ts2)
694
- return -1;
695
-
696
- if (ts1 < ts2)
697
- return 1;
698
-
699
- return -strcmp(p1, p2);
700
-}
701
-
702
-void nd_journal_files_registry_update(void)
703
-{
704
- static SPINLOCK spinlock = SPINLOCK_INITIALIZER;
705
-
706
- if (spinlock_trylock(&spinlock)) {
707
- usec_t scan_monotonic_ut = now_monotonic_usec();
708
-
709
- DICTIONARY *files = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
710
- DICTIONARY *dirs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
711
-
712
- for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
713
- if (!journal_directories[i].path)
714
- break;
715
- nd_journal_directory_scan_recursively(files, dirs, string2str(journal_directories[i].path), 0);
716
- }
717
-
718
- const char **array = mallocz(sizeof(const char *) * dictionary_entries(files));
719
- size_t used = 0;
720
-
721
- void *x;
722
- dfe_start_read(files, x)
723
- {
724
- if (used >= dictionary_entries(files))
725
- continue;
726
- array[used++] = x_dfe.name;
727
- }
728
- dfe_done(x);
729
-
730
- qsort(array, used, sizeof(const char *), filenames_compar);
731
-
732
- for (size_t i = 0; i < used; i++) {
733
- const char *full_path = array[i];
734
-
735
- struct stat info;
736
- if (stat(full_path, &info) == -1)
737
- continue;
738
-
739
- struct nd_journal_file njf_tmp = {
740
- .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC,
741
- .last_scan_monotonic_ut = scan_monotonic_ut,
742
- .size = info.st_size,
743
- .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT,
744
- };
745
- struct nd_journal_file *njf =
746
- dictionary_set(nd_journal_files_registry, full_path, &njf_tmp, sizeof(njf_tmp));
747
- nd_journal_file_update_header(njf->filename, njf);
748
- }
749
- freez(array);
750
- dictionary_destroy(files);
751
- dictionary_destroy(dirs);
752
-
753
- struct nd_journal_file *njf;
754
- dfe_start_write(nd_journal_files_registry, njf)
755
- {
756
- if (njf->last_scan_monotonic_ut < scan_monotonic_ut)
757
- dictionary_del(nd_journal_files_registry, njf_dfe.name);
758
- }
759
- dfe_done(njf);
760
- dictionary_garbage_collect(nd_journal_files_registry);
761
-
762
- nd_journal_files_scans++;
763
- spinlock_unlock(&spinlock);
764
-
765
- internal_error(
766
- true,
767
- "Journal library scan completed in %.3f ms",
768
- (double)(now_monotonic_usec() - scan_monotonic_ut) / (double)USEC_PER_MS);
769
- }
770
-}
771
-
772
-// ----------------------------------------------------------------------------
773
-
774
-int nd_journal_file_dict_items_backward_compar(const void *a, const void *b)
775
-{
776
- const DICTIONARY_ITEM **ad = (const DICTIONARY_ITEM **)a, **bd = (const DICTIONARY_ITEM **)b;
777
- struct nd_journal_file *njf_lhs = dictionary_acquired_item_value(*ad);
778
- struct nd_journal_file *njf_rhs = dictionary_acquired_item_value(*bd);
779
-
780
- // compare the last message timestamps
781
- if (njf_lhs->msg_last_ut < njf_rhs->msg_last_ut)
782
- return 1;
783
-
784
- if (njf_lhs->msg_last_ut > njf_rhs->msg_last_ut)
785
- return -1;
786
-
787
- // compare the file last modification timestamps
788
- if (njf_lhs->file_last_modified_ut < njf_rhs->file_last_modified_ut)
789
- return 1;
790
-
791
- if (njf_lhs->file_last_modified_ut > njf_rhs->file_last_modified_ut)
792
- return -1;
793
-
794
- // compare the first message timestamps
795
- if (njf_lhs->msg_first_ut < njf_rhs->msg_first_ut)
796
- return 1;
797
-
798
- if (njf_lhs->msg_first_ut > njf_rhs->msg_first_ut)
799
- return -1;
800
-
801
- return 0;
802
-}
803
-
804
-int nd_journal_file_dict_items_forward_compar(const void *a, const void *b)
805
-{
806
- return -nd_journal_file_dict_items_backward_compar(a, b);
807
-}
808
-
809
-static bool boot_id_conflict_cb(
810
- const DICTIONARY_ITEM *item __maybe_unused,
811
- void *old_value,
812
- void *new_value,
813
- void *data __maybe_unused)
814
-{
815
- usec_t *old_usec = old_value;
816
- usec_t *new_usec = new_value;
817
-
818
- if (*new_usec < *old_usec) {
819
- *old_usec = *new_usec;
820
- return true;
821
- }
822
-
823
- return false;
824
-}
825
-
826
-void nd_journal_init_files_and_directories(void)
827
-{
828
- unsigned d = 0;
829
-
830
- // ------------------------------------------------------------------------
831
- // setup the journal directories
832
-
833
- journal_directories[d++].path = string_strdupz("/run/log/journal");
834
- journal_directories[d++].path = string_strdupz("/var/log/journal");
835
-
836
- if (*netdata_configured_host_prefix) {
837
- char path[PATH_MAX];
838
- snprintfz(path, sizeof(path), "%s/var/log/journal", netdata_configured_host_prefix);
839
- journal_directories[d++].path = string_strdupz(path);
840
- snprintfz(path, sizeof(path), "%s/run/log/journal", netdata_configured_host_prefix);
841
- journal_directories[d++].path = string_strdupz(path);
842
- }
843
-
844
- const char *netdata_configured_log_dir = getenv("NETDATA_LOG_DIR");
845
- if (netdata_configured_log_dir != NULL) {
846
- char path[PATH_MAX];
847
- snprintfz(path, sizeof(path), "%s/otel", netdata_configured_host_prefix);
848
- journal_directories[d++].path = string_strdupz(path);
849
- };
850
-
851
- // terminate the list
852
- journal_directories[d].path = NULL;
853
-
854
- // ------------------------------------------------------------------------
855
- // initialize the used hashes files registry
856
-
857
- used_hashes_registry = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE);
858
-
859
- systemd_journal_session = (now_realtime_usec() / USEC_PER_SEC) * USEC_PER_SEC;
860
-
861
- nd_journal_files_registry = dictionary_create_advanced(
862
- DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(struct nd_journal_file));
863
-
864
- dictionary_register_insert_callback(nd_journal_files_registry, files_registry_insert_cb, NULL);
865
- dictionary_register_delete_callback(nd_journal_files_registry, files_registry_delete_cb, NULL);
866
- dictionary_register_conflict_callback(nd_journal_files_registry, files_registry_conflict_cb, NULL);
867
-
868
- boot_ids_to_first_ut =
869
- dictionary_create_advanced(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_FIXED_SIZE, NULL, sizeof(usec_t));
870
-
871
- dictionary_register_conflict_callback(boot_ids_to_first_ut, boot_id_conflict_cb, NULL);
872
-}
src/collectors/systemd-journal.plugin/systemd-journal-fstat.c
deleted
-93
@@ -1,93 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "systemd-internals.h"
4
-
5
-#if !defined(HAVE_RUST_PROVIDER)
6
-
7
-// ----------------------------------------------------------------------------
8
-// fstat64 overloading to speed up libsystemd
9
-// https://github.com/systemd/systemd/pull/29261
10
-
11
-#include <dlfcn.h>
12
-#include <sys/stat.h>
13
-
14
-#define FSTAT_CACHE_MAX 1024
15
-struct fdstat64_cache_entry {
16
- bool enabled;
17
- bool updated;
18
- int err_no;
19
- struct stat64 stat;
20
- int ret;
21
- size_t cached_count;
22
- size_t session;
23
-};
24
-
25
-struct fdstat64_cache_entry fstat64_cache[FSTAT_CACHE_MAX] = {0};
26
-__thread size_t fstat_thread_calls = 0;
27
-__thread size_t fstat_thread_cached_responses = 0;
28
-static __thread bool enable_thread_fstat = false;
29
-static __thread size_t fstat_caching_thread_session = 0;
30
-static size_t fstat_caching_global_session = 0;
31
-
32
-void fstat_cache_enable_on_thread(void)
33
-{
34
- fstat_caching_thread_session = __atomic_add_fetch(&fstat_caching_global_session, 1, __ATOMIC_ACQUIRE);
35
- enable_thread_fstat = true;
36
-}
37
-
38
-void fstat_cache_disable_on_thread(void)
39
-{
40
- fstat_caching_thread_session = __atomic_add_fetch(&fstat_caching_global_session, 1, __ATOMIC_RELEASE);
41
- enable_thread_fstat = false;
42
-}
43
-
44
-int fstat64(int fd, struct stat64 *buf)
45
-{
46
- static int (*real_fstat)(int, struct stat64 *) = NULL;
47
- if (!real_fstat)
48
- real_fstat = dlsym(RTLD_NEXT, "fstat64");
49
-
50
- fstat_thread_calls++;
51
-
52
- if (fd >= 0 && fd < FSTAT_CACHE_MAX) {
53
- if (enable_thread_fstat && fstat64_cache[fd].session != fstat_caching_thread_session) {
54
- fstat64_cache[fd].session = fstat_caching_thread_session;
55
- fstat64_cache[fd].enabled = true;
56
- fstat64_cache[fd].updated = false;
57
- }
58
-
59
- if (fstat64_cache[fd].enabled && fstat64_cache[fd].updated &&
60
- fstat64_cache[fd].session == fstat_caching_thread_session) {
61
- fstat_thread_cached_responses++;
62
- errno = fstat64_cache[fd].err_no;
63
- *buf = fstat64_cache[fd].stat;
64
- fstat64_cache[fd].cached_count++;
65
- return fstat64_cache[fd].ret;
66
- }
67
- }
68
-
69
- int ret = real_fstat(fd, buf);
70
-
71
- if (fd >= 0 && fd < FSTAT_CACHE_MAX && fstat64_cache[fd].enabled &&
72
- fstat64_cache[fd].session == fstat_caching_thread_session) {
73
- fstat64_cache[fd].ret = ret;
74
- fstat64_cache[fd].updated = true;
75
- fstat64_cache[fd].err_no = errno;
76
- fstat64_cache[fd].stat = *buf;
77
- }
78
-
79
- return ret;
80
-}
81
-
82
-#else // HAVE_RUST_PROVIDER
83
-
84
-// When using Rust provider, disable fstat caching entirely since
85
-// we will not rely on libsystemd.
86
-
87
-__thread size_t fstat_thread_calls = 0;
88
-__thread size_t fstat_thread_cached_responses = 0;
89
-
90
-void fstat_cache_enable_on_thread(void) { }
91
-void fstat_cache_disable_on_thread(void) { }
92
-
93
-#endif
src/collectors/systemd-journal.plugin/systemd-journal-sampling.h
deleted
-415
@@ -1,415 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#ifndef NETDATA_ND_SD_JOURNAL_SAMPLING_H
4
-#define NETDATA_ND_SD_JOURNAL_SAMPLING_H
5
-
6
-// ----------------------------------------------------------------------------
7
-// sampling support
8
-
9
-static inline void sampling_query_init(LOGS_QUERY_STATUS *lqs, FACETS *facets)
10
-{
11
- if (!lqs->rq.sampling)
12
- return;
13
-
14
- if (!lqs->rq.slice) {
15
- // the user is doing a full data query
16
- // disable sampling
17
- lqs->rq.sampling = 0;
18
- return;
19
- }
20
-
21
- if (lqs->rq.data_only) {
22
- // the user is doing a data query
23
- // disable sampling
24
- lqs->rq.sampling = 0;
25
- return;
26
- }
27
-
28
- if (!lqs->c.files_matched) {
29
- // no files have been matched
30
- // disable sampling
31
- lqs->rq.sampling = 0;
32
- return;
33
- }
34
-
35
- lqs->c.samples.slots = facets_histogram_slots(facets);
36
- if (lqs->c.samples.slots < 2)
37
- lqs->c.samples.slots = 2;
38
- if (lqs->c.samples.slots > ND_SD_JOURNAL_SAMPLING_SLOTS)
39
- lqs->c.samples.slots = ND_SD_JOURNAL_SAMPLING_SLOTS;
40
-
41
- if (!lqs->rq.after_ut || !lqs->rq.before_ut || lqs->rq.after_ut >= lqs->rq.before_ut) {
42
- // we don't have enough information for sampling
43
- lqs->rq.sampling = 0;
44
- return;
45
- }
46
-
47
- usec_t delta = lqs->rq.before_ut - lqs->rq.after_ut;
48
- usec_t step = delta / facets_histogram_slots(facets) - 1;
49
- if (step < 1)
50
- step = 1;
51
-
52
- lqs->c.samples_per_time_slot.start_ut = lqs->rq.after_ut;
53
- lqs->c.samples_per_time_slot.end_ut = lqs->rq.before_ut;
54
- lqs->c.samples_per_time_slot.step_ut = step;
55
-
56
- // the minimum number of rows to enable sampling
57
- lqs->c.samples.enable_after_samples = lqs->rq.sampling / 2;
58
-
59
- size_t files_matched = lqs->c.files_matched;
60
- if (!files_matched)
61
- files_matched = 1;
62
-
63
- // the minimum number of rows per file to enable sampling
64
- lqs->c.samples_per_file.enable_after_samples = (lqs->rq.sampling / 4) / files_matched;
65
- if (lqs->c.samples_per_file.enable_after_samples < lqs->rq.entries)
66
- lqs->c.samples_per_file.enable_after_samples = lqs->rq.entries;
67
-
68
- // the minimum number of rows per time slot to enable sampling
69
- lqs->c.samples_per_time_slot.enable_after_samples = (lqs->rq.sampling / 4) / lqs->c.samples.slots;
70
- if (lqs->c.samples_per_time_slot.enable_after_samples < lqs->rq.entries)
71
- lqs->c.samples_per_time_slot.enable_after_samples = lqs->rq.entries;
72
-}
73
-
74
-static inline void sampling_file_init(LOGS_QUERY_STATUS *lqs, struct nd_journal_file *jf __maybe_unused)
75
-{
76
- lqs->c.samples_per_file.sampled = 0;
77
- lqs->c.samples_per_file.unsampled = 0;
78
- lqs->c.samples_per_file.estimated = 0;
79
- lqs->c.samples_per_file.every = 0;
80
- lqs->c.samples_per_file.skipped = 0;
81
- lqs->c.samples_per_file.recalibrate = 0;
82
-}
83
-
84
-static inline size_t sampling_file_lines_scanned_so_far(LOGS_QUERY_STATUS *lqs)
85
-{
86
- size_t sampled = lqs->c.samples_per_file.sampled + lqs->c.samples_per_file.unsampled;
87
- if (!sampled)
88
- sampled = 1;
89
- return sampled;
90
-}
91
-
92
-static inline void sampling_running_file_query_overlapping_timeframe_ut(
93
- LOGS_QUERY_STATUS *lqs,
94
- struct nd_journal_file *jf,
95
- FACETS_ANCHOR_DIRECTION direction,
96
- usec_t msg_ut,
97
- usec_t *after_ut,
98
- usec_t *before_ut)
99
-{
100
- // find the overlap of the query and file timeframes
101
- // taking into account the first message we encountered
102
-
103
- usec_t oldest_ut, newest_ut;
104
- if (direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
105
- // the first message we know (oldest)
106
- oldest_ut = lqs->c.query_file.first_msg_ut ? lqs->c.query_file.first_msg_ut : jf->msg_first_ut;
107
- if (!oldest_ut)
108
- oldest_ut = lqs->c.query_file.start_ut;
109
-
110
- if (jf->msg_last_ut)
111
- newest_ut = MIN(lqs->c.query_file.stop_ut, jf->msg_last_ut);
112
- else if (jf->file_last_modified_ut)
113
- newest_ut = MIN(lqs->c.query_file.stop_ut, jf->file_last_modified_ut);
114
- else
115
- newest_ut = lqs->c.query_file.stop_ut;
116
-
117
- if (msg_ut < oldest_ut)
118
- oldest_ut = msg_ut - 1;
119
- } else /* BACKWARD */ {
120
- // the latest message we know (newest)
121
- newest_ut = lqs->c.query_file.first_msg_ut ? lqs->c.query_file.first_msg_ut : jf->msg_last_ut;
122
- if (!newest_ut)
123
- newest_ut = lqs->c.query_file.start_ut;
124
-
125
- if (jf->msg_first_ut)
126
- oldest_ut = MAX(lqs->c.query_file.stop_ut, jf->msg_first_ut);
127
- else
128
- oldest_ut = lqs->c.query_file.stop_ut;
129
-
130
- if (newest_ut < msg_ut)
131
- newest_ut = msg_ut + 1;
132
- }
133
-
134
- *after_ut = oldest_ut;
135
- *before_ut = newest_ut;
136
-}
137
-
138
-static inline double sampling_running_file_query_progress_by_time(
139
- LOGS_QUERY_STATUS *lqs,
140
- struct nd_journal_file *jf,
141
- FACETS_ANCHOR_DIRECTION direction,
142
- usec_t msg_ut)
143
-{
144
- usec_t after_ut, before_ut, elapsed_ut;
145
- sampling_running_file_query_overlapping_timeframe_ut(lqs, jf, direction, msg_ut, &after_ut, &before_ut);
146
-
147
- if (direction == FACETS_ANCHOR_DIRECTION_FORWARD)
148
- elapsed_ut = msg_ut - after_ut;
149
- else
150
- elapsed_ut = before_ut - msg_ut;
151
-
152
- usec_t total_ut = before_ut - after_ut;
153
- double progress = (double)elapsed_ut / (double)total_ut;
154
-
155
- return progress;
156
-}
157
-
158
-static inline usec_t sampling_running_file_query_remaining_time(
159
- LOGS_QUERY_STATUS *lqs,
160
- struct nd_journal_file *jf,
161
- FACETS_ANCHOR_DIRECTION direction,
162
- usec_t msg_ut,
163
- usec_t *total_time_ut,
164
- usec_t *remaining_start_ut,
165
- usec_t *remaining_end_ut)
166
-{
167
- usec_t after_ut, before_ut;
168
- sampling_running_file_query_overlapping_timeframe_ut(lqs, jf, direction, msg_ut, &after_ut, &before_ut);
169
-
170
- // since we have a timestamp in msg_ut
171
- // this timestamp can extend the overlap
172
- if (msg_ut <= after_ut)
173
- after_ut = msg_ut - 1;
174
-
175
- if (msg_ut >= before_ut)
176
- before_ut = msg_ut + 1;
177
-
178
- // return the remaining duration
179
- usec_t remaining_from_ut, remaining_to_ut;
180
- if (direction == FACETS_ANCHOR_DIRECTION_FORWARD) {
181
- remaining_from_ut = msg_ut;
182
- remaining_to_ut = before_ut;
183
- } else {
184
- remaining_from_ut = after_ut;
185
- remaining_to_ut = msg_ut;
186
- }
187
-
188
- usec_t remaining_ut = remaining_to_ut - remaining_from_ut;
189
-
190
- if (total_time_ut)
191
- *total_time_ut = (before_ut > after_ut) ? before_ut - after_ut : 1;
192
-
193
- if (remaining_start_ut)
194
- *remaining_start_ut = remaining_from_ut;
195
-
196
- if (remaining_end_ut)
197
- *remaining_end_ut = remaining_to_ut;
198
-
199
- return remaining_ut;
200
-}
201
-
202
-static inline size_t sampling_running_file_query_estimate_remaining_lines_by_time(
203
- LOGS_QUERY_STATUS *lqs,
204
- struct nd_journal_file *jf,
205
- FACETS_ANCHOR_DIRECTION direction,
206
- usec_t msg_ut)
207
-{
208
- size_t scanned_lines = sampling_file_lines_scanned_so_far(lqs);
209
-
210
- // Calculate the proportion of time covered
211
- usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
212
- usec_t remaining_time_ut = sampling_running_file_query_remaining_time(
213
- lqs, jf, direction, msg_ut, &total_time_ut, &remaining_start_ut, &remaining_end_ut);
214
- if (total_time_ut == 0)
215
- total_time_ut = 1;
216
-
217
- double proportion_by_time = (double)(total_time_ut - remaining_time_ut) / (double)total_time_ut;
218
-
219
- if (proportion_by_time == 0 || proportion_by_time > 1.0 || !isfinite(proportion_by_time))
220
- proportion_by_time = 1.0;
221
-
222
- // Estimate the total number of lines in the file
223
- size_t expected_matching_logs_by_time = (size_t)((double)scanned_lines / proportion_by_time);
224
-
225
- if (jf->messages_in_file && expected_matching_logs_by_time > jf->messages_in_file)
226
- expected_matching_logs_by_time = jf->messages_in_file;
227
-
228
- // Calculate the estimated number of remaining lines
229
- size_t remaining_logs_by_time = expected_matching_logs_by_time - scanned_lines;
230
- if (remaining_logs_by_time < 1)
231
- remaining_logs_by_time = 1;
232
-
233
- // nd_log(NDLS_COLLECTORS, NDLP_INFO,
234
- // "JOURNAL ESTIMATION: '%s' "
235
- // "scanned_lines=%zu [sampled=%zu, unsampled=%zu, estimated=%zu], "
236
- // "file [%"PRIu64" - %"PRIu64", duration %"PRId64", known lines in file %zu], "
237
- // "query [%"PRIu64" - %"PRIu64", duration %"PRId64"], "
238
- // "first message read from the file at %"PRIu64", current message at %"PRIu64", "
239
- // "proportion of time %.2f %%, "
240
- // "expected total lines in file %zu, "
241
- // "remaining lines %zu, "
242
- // "remaining time %"PRIu64" [%"PRIu64" - %"PRIu64", duration %"PRId64"]"
243
- // , jf->filename
244
- // , scanned_lines, fqs->samples_per_file.sampled, fqs->samples_per_file.unsampled, fqs->samples_per_file.estimated
245
- // , jf->msg_first_ut, jf->msg_last_ut, jf->msg_last_ut - jf->msg_first_ut, jf->messages_in_file
246
- // , fqs->query_file.start_ut, fqs->query_file.stop_ut, fqs->query_file.stop_ut - fqs->query_file.start_ut
247
- // , fqs->query_file.first_msg_ut, msg_ut
248
- // , proportion_by_time * 100.0
249
- // , expected_matching_logs_by_time
250
- // , remaining_logs_by_time
251
- // , remaining_time_ut, remaining_start_ut, remaining_end_ut, remaining_end_ut - remaining_start_ut
252
- // );
253
-
254
- return remaining_logs_by_time;
255
-}
256
-
257
-static inline size_t sampling_running_file_query_estimate_remaining_lines(
258
- NsdJournal *j __maybe_unused,
259
- LOGS_QUERY_STATUS *lqs,
260
- struct nd_journal_file *jf,
261
- FACETS_ANCHOR_DIRECTION direction,
262
- usec_t msg_ut)
263
-{
264
- size_t remaining_logs_by_seqnum = 0;
265
-
266
-#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
267
- size_t expected_matching_logs_by_seqnum = 0;
268
- double proportion_by_seqnum = 0.0;
269
- uint64_t current_msg_seqnum;
270
- NsdId128 current_msg_writer;
271
- if (!lqs->c.query_file.first_msg_seqnum || nsd_journal_get_seqnum(j, ¤t_msg_seqnum, ¤t_msg_writer) < 0) {
272
- lqs->c.query_file.first_msg_seqnum = 0;
273
- lqs->c.query_file.first_msg_writer = NSD_ID128_NULL;
274
- } else if (jf->messages_in_file) {
275
- size_t scanned_lines = sampling_file_lines_scanned_so_far(lqs);
276
-
277
- double proportion_of_all_lines_so_far;
278
- if (direction == FACETS_ANCHOR_DIRECTION_FORWARD)
279
- proportion_of_all_lines_so_far = (double)scanned_lines / (double)(current_msg_seqnum - jf->first_seqnum);
280
- else
281
- proportion_of_all_lines_so_far = (double)scanned_lines / (double)(jf->last_seqnum - current_msg_seqnum);
282
-
283
- if (proportion_of_all_lines_so_far > 1.0)
284
- proportion_of_all_lines_so_far = 1.0;
285
-
286
- expected_matching_logs_by_seqnum = (size_t)(proportion_of_all_lines_so_far * (double)jf->messages_in_file);
287
-
288
- proportion_by_seqnum = (double)scanned_lines / (double)expected_matching_logs_by_seqnum;
289
-
290
- if (proportion_by_seqnum == 0 || proportion_by_seqnum > 1.0 || !isfinite(proportion_by_seqnum))
291
- proportion_by_seqnum = 1.0;
292
-
293
- remaining_logs_by_seqnum = expected_matching_logs_by_seqnum - scanned_lines;
294
- if (!remaining_logs_by_seqnum)
295
- remaining_logs_by_seqnum = 1;
296
- }
297
-#endif
298
-
299
- if (remaining_logs_by_seqnum)
300
- return remaining_logs_by_seqnum;
301
-
302
- return sampling_running_file_query_estimate_remaining_lines_by_time(lqs, jf, direction, msg_ut);
303
-}
304
-
305
-static inline void sampling_decide_file_sampling_every(
306
- NsdJournal *j,
307
- LOGS_QUERY_STATUS *lqs,
308
- struct nd_journal_file *jf,
309
- FACETS_ANCHOR_DIRECTION direction,
310
- usec_t msg_ut)
311
-{
312
- size_t files_matched = lqs->c.files_matched;
313
- if (!files_matched)
314
- files_matched = 1;
315
-
316
- size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, lqs, jf, direction, msg_ut);
317
- size_t wanted_samples = (lqs->rq.sampling / 2) / files_matched;
318
- if (!wanted_samples)
319
- wanted_samples = 1;
320
-
321
- lqs->c.samples_per_file.every = remaining_lines / wanted_samples;
322
-
323
- if (lqs->c.samples_per_file.every < 1)
324
- lqs->c.samples_per_file.every = 1;
325
-}
326
-
327
-typedef enum {
328
- SAMPLING_STOP_AND_ESTIMATE = -1,
329
- SAMPLING_FULL = 0,
330
- SAMPLING_SKIP_FIELDS = 1,
331
-} sampling_t;
332
-
333
-static inline sampling_t is_row_in_sample(
334
- NsdJournal *j,
335
- LOGS_QUERY_STATUS *lqs,
336
- struct nd_journal_file *jf,
337
- usec_t msg_ut,
338
- FACETS_ANCHOR_DIRECTION direction,
339
- bool candidate_to_keep)
340
-{
341
- if (!lqs->rq.sampling || candidate_to_keep)
342
- return SAMPLING_FULL;
343
-
344
- if (unlikely(msg_ut < lqs->c.samples_per_time_slot.start_ut))
345
- msg_ut = lqs->c.samples_per_time_slot.start_ut;
346
- if (unlikely(msg_ut > lqs->c.samples_per_time_slot.end_ut))
347
- msg_ut = lqs->c.samples_per_time_slot.end_ut;
348
-
349
- size_t slot = (msg_ut - lqs->c.samples_per_time_slot.start_ut) / lqs->c.samples_per_time_slot.step_ut;
350
- if (slot >= lqs->c.samples.slots)
351
- slot = lqs->c.samples.slots - 1;
352
-
353
- bool should_sample = false;
354
-
355
- if (lqs->c.samples.sampled < lqs->c.samples.enable_after_samples ||
356
- lqs->c.samples_per_file.sampled < lqs->c.samples_per_file.enable_after_samples ||
357
- lqs->c.samples_per_time_slot.sampled[slot] < lqs->c.samples_per_time_slot.enable_after_samples)
358
- should_sample = true;
359
-
360
- else if (lqs->c.samples_per_file.recalibrate >= ND_SD_JOURNAL_SAMPLING_RECALIBRATE || !lqs->c.samples_per_file.every) {
361
- // this is the first to be unsampled for this file
362
- sampling_decide_file_sampling_every(j, lqs, jf, direction, msg_ut);
363
- lqs->c.samples_per_file.recalibrate = 0;
364
- should_sample = true;
365
- } else {
366
- // we sample 1 every fqs->samples_per_file.every
367
- if (lqs->c.samples_per_file.skipped >= lqs->c.samples_per_file.every) {
368
- lqs->c.samples_per_file.skipped = 0;
369
- should_sample = true;
370
- } else
371
- lqs->c.samples_per_file.skipped++;
372
- }
373
-
374
- if (should_sample) {
375
- lqs->c.samples.sampled++;
376
- lqs->c.samples_per_file.sampled++;
377
- lqs->c.samples_per_time_slot.sampled[slot]++;
378
-
379
- return SAMPLING_FULL;
380
- }
381
-
382
- lqs->c.samples_per_file.recalibrate++;
383
-
384
- lqs->c.samples.unsampled++;
385
- lqs->c.samples_per_file.unsampled++;
386
- lqs->c.samples_per_time_slot.unsampled[slot]++;
387
-
388
- if (lqs->c.samples_per_file.unsampled > lqs->c.samples_per_file.sampled) {
389
- double progress_by_time = sampling_running_file_query_progress_by_time(lqs, jf, direction, msg_ut);
390
-
391
- if (progress_by_time > ND_SD_JOURNAL_ENABLE_ESTIMATIONS_FILE_PERCENTAGE)
392
- return SAMPLING_STOP_AND_ESTIMATE;
393
- }
394
-
395
- return SAMPLING_SKIP_FIELDS;
396
-}
397
-
398
-static inline void sampling_update_running_query_file_estimates(
399
- FACETS *facets,
400
- NsdJournal *j,
401
- LOGS_QUERY_STATUS *lqs,
402
- struct nd_journal_file *jf,
403
- usec_t msg_ut,
404
- FACETS_ANCHOR_DIRECTION direction)
405
-{
406
- usec_t total_time_ut, remaining_start_ut, remaining_end_ut;
407
- sampling_running_file_query_remaining_time(
408
- lqs, jf, direction, msg_ut, &total_time_ut, &remaining_start_ut, &remaining_end_ut);
409
- size_t remaining_lines = sampling_running_file_query_estimate_remaining_lines(j, lqs, jf, direction, msg_ut);
410
- facets_update_estimations(facets, remaining_start_ut, remaining_end_ut, remaining_lines);
411
- lqs->c.samples.estimated += remaining_lines;
412
- lqs->c.samples_per_file.estimated += remaining_lines;
413
-}
414
-
415
-#endif //NETDATA_ND_SD_JOURNAL_SAMPLING_H
src/collectors/systemd-journal.plugin/systemd-journal-watcher.c
deleted
-613
@@ -1,613 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "systemd-internals.h"
4
-#include <sys/inotify.h>
5
-
6
-#define INITIAL_WATCHES 256
7
-
8
-#define WATCH_FOR (IN_CREATE | IN_MODIFY | IN_DELETE | IN_DELETE_SELF | IN_MOVED_FROM | IN_MOVED_TO | IN_UNMOUNT)
9
-
10
-typedef uint32_t INOTIFY_MASK;
11
-
12
-ENUM_STR_MAP_DEFINE(INOTIFY_MASK) = {
13
- // helpers (combine multiple flags)
14
- // must be first in the list
15
- {.id = IN_ALL_EVENTS, .name = "IN_ALL_EVENTS"},
16
- {.id = IN_CLOSE, .name = "IN_CLOSE"},
17
- {.id = IN_MOVE, .name = "IN_MOVE"},
18
-
19
- // individual flags
20
- {.id = IN_ACCESS, .name = "IN_ACCESS"},
21
- {.id = IN_MODIFY, .name = "IN_MODIFY"},
22
- {.id = IN_ATTRIB, .name = "IN_ATTRIB"},
23
- {.id = IN_CLOSE_WRITE, .name = "IN_CLOSE_WRITE"},
24
- {.id = IN_CLOSE_NOWRITE, .name = "IN_CLOSE_NOWRITE"},
25
- {.id = IN_OPEN, .name = "IN_OPEN"},
26
- {.id = IN_MOVED_FROM, .name = "IN_MOVED_FROM"},
27
- {.id = IN_MOVED_TO, .name = "IN_MOVED_TO"},
28
- {.id = IN_CREATE, .name = "IN_CREATE"},
29
- {.id = IN_DELETE, .name = "IN_DELETE"},
30
- {.id = IN_DELETE_SELF, .name = "IN_DELETE_SELF"},
31
- {.id = IN_MOVE_SELF, .name = "IN_MOVE_SELF"},
32
- {.id = IN_UNMOUNT, .name = "IN_UNMOUNT"},
33
- {.id = IN_Q_OVERFLOW, .name = "IN_Q_OVERFLOW"},
34
- {.id = IN_IGNORED, .name = "IN_IGNORED"},
35
- {.id = IN_ONLYDIR, .name = "IN_ONLYDIR"},
36
- {.id = IN_DONT_FOLLOW, .name = "IN_DONT_FOLLOW"},
37
- {.id = IN_EXCL_UNLINK, .name = "IN_EXCL_UNLINK"},
38
-#ifdef IN_MASK_CREATE
39
- {.id = IN_MASK_CREATE, .name = "IN_MASK_CREATE"},
40
-#endif
41
- {.id = IN_MASK_ADD, .name = "IN_MASK_ADD"},
42
- {.id = IN_ISDIR, .name = "IN_ISDIR"},
43
- {.id = IN_ONESHOT, .name = "IN_ONESHOT"},
44
-
45
- // terminator
46
- {.id = 0, .name = NULL}};
47
-
48
-BITMAP_STR_DEFINE_FUNCTIONS(INOTIFY_MASK, 0, "UNKNOWN");
49
-
50
-DEFINE_JUDYL_TYPED(SYMLINKED_DIRS, STRING *);
51
-
52
-typedef struct watch_entry {
53
- int slot;
54
-
55
- int wd; // Watch descriptor
56
- char *path; // Dynamically allocated path
57
-
58
- struct watch_entry *next; // for the free list
59
-} WatchEntry;
60
-
61
-typedef struct {
62
- WatchEntry *watchList;
63
- WatchEntry *freeList;
64
- int watchCount;
65
- int watchListSize;
66
-
67
- size_t errors;
68
-
69
- SYMLINKED_DIRS_JudyLSet symlinkedDirs;
70
- DICTIONARY *pending;
71
-} Watcher;
72
-
73
-static WatchEntry *get_slot(Watcher *watcher)
74
-{
75
- WatchEntry *t;
76
-
77
- if (watcher->freeList != NULL) {
78
- t = watcher->freeList;
79
- watcher->freeList = t->next;
80
- t->next = NULL;
81
- return t;
82
- }
83
-
84
- if (watcher->watchCount == watcher->watchListSize) {
85
- watcher->watchListSize *= 2;
86
- watcher->watchList = reallocz(watcher->watchList, watcher->watchListSize * sizeof(WatchEntry));
87
- }
88
-
89
- watcher->watchList[watcher->watchCount] = (WatchEntry){
90
- .slot = watcher->watchCount,
91
- .wd = -1,
92
- .path = NULL,
93
- .next = NULL,
94
- };
95
- t = &watcher->watchList[watcher->watchCount];
96
- watcher->watchCount++;
97
-
98
- return t;
99
-}
100
-
101
-static void free_slot(Watcher *watcher, WatchEntry *t)
102
-{
103
- t->wd = -1;
104
- freez(t->path);
105
- t->path = NULL;
106
-
107
- // link it to the free list
108
- t->next = watcher->freeList;
109
- watcher->freeList = t;
110
-}
111
-
112
-static int add_watch(Watcher *watcher, int inotifyFd, const char *path)
113
-{
114
- WatchEntry *t = get_slot(watcher);
115
-
116
- errno_clear();
117
- t->wd = inotify_add_watch(inotifyFd, path, WATCH_FOR);
118
- if (t->wd == -1) {
119
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "JOURNAL WATCHER: cannot watch directory: '%s'", path);
120
-
121
- free_slot(watcher, t);
122
-
123
- struct stat info;
124
- if (stat(path, &info) == 0 && S_ISDIR(info.st_mode)) {
125
- // the directory exists, but we failed to add the watch
126
- // increase errors
127
- watcher->errors++;
128
- }
129
- } else {
130
- t->path = strdupz(path);
131
-
132
- nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "JOURNAL WATCHER: watching directory: '%s'", path);
133
- }
134
- return t->wd;
135
-}
136
-
137
-static void remove_watch(Watcher *watcher, int inotifyFd, int wd)
138
-{
139
- errno_clear();
140
-
141
- int i;
142
- for (i = 0; i < watcher->watchCount; ++i) {
143
- if (watcher->watchList[i].wd == wd) {
144
- nd_log(
145
- NDLS_COLLECTORS,
146
- NDLP_DEBUG,
147
- "JOURNAL WATCHER: removing watch from directory: '%s'",
148
- watcher->watchList[i].path);
149
-
150
- if (inotify_rm_watch(inotifyFd, watcher->watchList[i].wd) == -1)
151
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "JOURNAL WATCHER: inotify_rm_watch() returned -1");
152
-
153
- free_slot(watcher, &watcher->watchList[i]);
154
- return;
155
- }
156
- }
157
-
158
- nd_log(NDLS_COLLECTORS, NDLP_WARNING, "JOURNAL WATCHER: cannot find directory watch %d to remove.", wd);
159
-}
160
-
161
-static void free_watches(Watcher *watcher, int inotifyFd)
162
-{
163
- for (int i = 0; i < watcher->watchCount; ++i) {
164
- if (watcher->watchList[i].wd != -1) {
165
- if (inotify_rm_watch(inotifyFd, watcher->watchList[i].wd) == -1)
166
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "JOURNAL WATCHER: inotify_rm_watch() returned -1");
167
- free_slot(watcher, &watcher->watchList[i]);
168
- }
169
- }
170
- freez(watcher->watchList);
171
- watcher->watchList = NULL;
172
-
173
- dictionary_destroy(watcher->pending);
174
- watcher->pending = NULL;
175
-}
176
-
177
-static void free_symlinked_dirs(Watcher *watcher)
178
-{
179
- Word_t idx = 0;
180
- STRING *value;
181
- while ((value = SYMLINKED_DIRS_FIRST(&watcher->symlinkedDirs, &idx))) {
182
- SYMLINKED_DIRS_DEL(&watcher->symlinkedDirs, idx);
183
- STRING *key = (STRING *)idx;
184
- string_freez(key);
185
- string_freez(value);
186
- }
187
-}
188
-
189
-static char *get_path_from_wd(Watcher *watcher, int wd)
190
-{
191
- for (int i = 0; i < watcher->watchCount; ++i) {
192
- if (watcher->watchList[i].wd == wd)
193
- return watcher->watchList[i].path;
194
- }
195
- return NULL;
196
-}
197
-
198
-static bool is_directory_watched(Watcher *watcher, const char *path)
199
-{
200
- for (int i = 0; i < watcher->watchCount; ++i) {
201
- if (watcher->watchList[i].wd != -1 && strcmp(watcher->watchList[i].path, path) == 0) {
202
- return true;
203
- }
204
- }
205
- return false;
206
-}
207
-
208
-static void watch_directory_and_subdirectories(Watcher *watcher, int inotifyFd, const char *basePath)
209
-{
210
- DICTIONARY *dirs = dictionary_create(DICT_OPTION_SINGLE_THREADED | DICT_OPTION_DONT_OVERWRITE_VALUE);
211
-
212
- // First resolve any symlinks in the base path
213
- char real_path[PATH_MAX];
214
- if (realpath(basePath, real_path) == NULL) {
215
- // If realpath fails, try using the original path
216
- strncpyz(real_path, basePath, sizeof(real_path));
217
- }
218
-
219
- nd_journal_directory_scan_recursively(NULL, dirs, real_path, 0);
220
-
221
- void *x;
222
- dfe_start_read(dirs, x)
223
- {
224
- const char *dirname = x_dfe.name;
225
- char resolved_path[PATH_MAX];
226
-
227
- // Resolve symlinks for each subdirectory
228
- if (realpath(dirname, resolved_path) != NULL) {
229
- // Check if this directory is already being watched
230
- if (!is_directory_watched(watcher, resolved_path)) {
231
- add_watch(watcher, inotifyFd, resolved_path);
232
- }
233
- } else {
234
- // If realpath fails, try with original path
235
- if (!is_directory_watched(watcher, dirname)) {
236
- add_watch(watcher, inotifyFd, dirname);
237
- }
238
- }
239
- }
240
- dfe_done(x);
241
-
242
- dictionary_destroy(dirs);
243
-}
244
-
245
-static bool is_subpath(const char *path, const char *subpath)
246
-{
247
- // Use strncmp to compare the paths
248
- if (strncmp(path, subpath, strlen(path)) == 0) {
249
- // Ensure that the next character is a '/' or '\0'
250
- char next_char = subpath[strlen(path)];
251
- return next_char == '/' || next_char == '\0';
252
- }
253
-
254
- return false;
255
-}
256
-
257
-void remove_directory_watch(Watcher *watcher, int inotifyFd, const char *dirPath)
258
-{
259
- for (int i = 0; i < watcher->watchCount; ++i) {
260
- WatchEntry *t = &watcher->watchList[i];
261
- if (t->wd != -1 && is_subpath(dirPath, t->path)) {
262
- if (inotify_rm_watch(inotifyFd, t->wd) == -1)
263
- nd_log(
264
- NDLS_COLLECTORS, NDLP_ERR, "JOURNAL WATCHER: inotify_rm_watch() on path '%s' returned -1", t->path);
265
- else
266
- nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "JOURNAL WATCHER: stopped watching directory '%s'", t->path);
267
- free_slot(watcher, t);
268
- }
269
- }
270
-
271
- struct nd_journal_file *njf;
272
- dfe_start_write(nd_journal_files_registry, njf)
273
- {
274
- if (is_subpath(dirPath, njf->filename))
275
- dictionary_del(nd_journal_files_registry, njf->filename);
276
- }
277
- dfe_done(njf);
278
-
279
- dictionary_garbage_collect(nd_journal_files_registry);
280
-}
281
-
282
-void process_event(Watcher *watcher, int inotifyFd, struct inotify_event *event)
283
-{
284
- errno_clear();
285
-
286
- if (!event->len) {
287
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
288
- INOTIFY_MASK_2buffer(wb, event->mask, ", ");
289
- nd_log(
290
- NDLS_COLLECTORS,
291
- NDLP_NOTICE,
292
- "JOURNAL WATCHER: received event with mask %u (%s) and len %u (this is zero) - ignoring it.",
293
- event->mask,
294
- buffer_tostring(wb),
295
- event->len);
296
- return;
297
- }
298
-
299
- char *dirPath = get_path_from_wd(watcher, event->wd);
300
- if (!dirPath) {
301
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
302
- INOTIFY_MASK_2buffer(wb, event->mask, ", ");
303
- nd_log(
304
- NDLS_COLLECTORS,
305
- NDLP_NOTICE,
306
- "JOURNAL WATCHER: received event with mask %u (%s) and len %u for path: '%s' - "
307
- "but we can't find its watch descriptor - ignoring it.",
308
- event->mask,
309
- buffer_tostring(wb),
310
- event->len,
311
- event->name);
312
- return;
313
- }
314
-
315
-#if 0
316
- {
317
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
318
- INOTIFY_MASK_2buffer(wb, event->mask, ", ");
319
- nd_log(NDLS_COLLECTORS, NDLP_DEBUG,
320
- "JOURNAL WATCHER: received event with mask %u (%s) for path: '%s' inside '%s'",
321
- event->mask, buffer_tostring(wb), event->name, dirPath);
322
- }
323
-#endif
324
-
325
- if (event->mask & IN_DELETE_SELF) {
326
- remove_watch(watcher, inotifyFd, event->wd);
327
- return;
328
- }
329
-
330
- static __thread char fullPath[PATH_MAX];
331
- snprintfz(fullPath, sizeof(fullPath), "%s/%s", dirPath, event->name);
332
-
333
- bool is_dir = event->mask & IN_ISDIR;
334
- char resolved_path[PATH_MAX];
335
- const char *path_to_use = fullPath;
336
-
337
- if (event->mask & (IN_CREATE | IN_MOVED_TO)) {
338
- // Give the system a moment to establish the symlink
339
- sleep_usec(1000); // 1ms sleep
340
-
341
- struct stat st;
342
- if (lstat(fullPath, &st) == 0) {
343
- if (S_ISLNK(st.st_mode)) {
344
- // It's a symlink - resolve it
345
- if (realpath(fullPath, resolved_path) != NULL) {
346
- path_to_use = resolved_path;
347
-
348
- // Check if it points to a directory
349
- if (stat(resolved_path, &st) == 0 && S_ISDIR(st.st_mode)) {
350
- is_dir = true;
351
-
352
- STRING *fullPathString = string_strdupz(fullPath);
353
- STRING *symlinked = SYMLINKED_DIRS_GET(&watcher->symlinkedDirs, (uintptr_t)fullPathString);
354
- if (!symlinked) {
355
- SYMLINKED_DIRS_SET(
356
- &watcher->symlinkedDirs, (uintptr_t)fullPathString, string_strdupz(resolved_path));
357
-
358
- // we leave fullPathString allocated, as it's now in the JudyL set
359
-
360
- nd_log(
361
- NDLS_COLLECTORS,
362
- NDLP_DEBUG,
363
- "JOURNAL WATCHER: New symlinked directory created: '%s' -> '%s'",
364
- fullPath,
365
- resolved_path);
366
- } else if (string_strcmp(symlinked, resolved_path) != 0) {
367
- SYMLINKED_DIRS_SET(
368
- &watcher->symlinkedDirs, (uintptr_t)fullPathString, string_strdupz(resolved_path));
369
-
370
- nd_log(
371
- NDLS_COLLECTORS,
372
- NDLP_DEBUG,
373
- "JOURNAL WATCHER: Updated symlinked directory: '%s' -> '%s' (was '%s')",
374
- fullPath,
375
- resolved_path,
376
- string2str(symlinked));
377
-
378
- string_freez(symlinked);
379
-
380
- // we need to free this, since it was already in the JudyL set
381
- string_freez(fullPathString);
382
- } else
383
- string_freez(fullPathString); // we don't need it anymore
384
- }
385
- }
386
- }
387
- }
388
- } else if (event->mask & IN_DELETE) {
389
- // Check if it was a symlink
390
- STRING *fullPathString = string_strdupz(fullPath);
391
- STRING *symlinked = SYMLINKED_DIRS_GET(&watcher->symlinkedDirs, (uintptr_t)fullPathString);
392
- if (symlinked) {
393
- strncpyz(resolved_path, string2str(symlinked), sizeof(resolved_path) - 1);
394
- path_to_use = resolved_path;
395
- SYMLINKED_DIRS_DEL(&watcher->symlinkedDirs, (uintptr_t)fullPathString);
396
- string_freez(fullPathString); // to remove also the one referenced in the JudyL set
397
- string_freez(symlinked);
398
- is_dir = true;
399
-
400
- nd_log(
401
- NDLS_COLLECTORS,
402
- NDLP_DEBUG,
403
- "JOURNAL WATCHER: Deleted symlinked directory: '%s' -> '%s'",
404
- fullPath,
405
- resolved_path);
406
- }
407
- string_freez(fullPathString); // the one we allocated above
408
- }
409
-
410
- if (is_dir) {
411
- if (event->mask & (IN_DELETE | IN_MOVED_FROM)) {
412
- nd_log(NDLS_COLLECTORS, NDLP_DEBUG, "JOURNAL WATCHER: Directory deleted or moved out: '%s'", path_to_use);
413
-
414
- remove_directory_watch(watcher, inotifyFd, path_to_use);
415
- } else if (event->mask & (IN_CREATE | IN_MOVED_TO)) {
416
- nd_log(
417
- NDLS_COLLECTORS, NDLP_DEBUG, "JOURNAL WATCHER: New directory created or moved in: '%s'", path_to_use);
418
-
419
- watch_directory_and_subdirectories(watcher, inotifyFd, path_to_use);
420
- } else {
421
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
422
- INOTIFY_MASK_2buffer(wb, event->mask, ", ");
423
- nd_log(
424
- NDLS_COLLECTORS,
425
- NDLP_WARNING,
426
- "JOURNAL WATCHER: Received unhandled event with mask %u (%s) for directory '%s'",
427
- event->mask,
428
- buffer_tostring(wb),
429
- path_to_use);
430
- }
431
- } else if (is_journal_file(event->name, (ssize_t)strlen(event->name), NULL)) {
432
- dictionary_set(watcher->pending, path_to_use, NULL, 0);
433
- } else {
434
- CLEAN_BUFFER *wb = buffer_create(0, NULL);
435
- INOTIFY_MASK_2buffer(wb, event->mask, ", ");
436
- nd_log(
437
- NDLS_COLLECTORS,
438
- NDLP_DEBUG,
439
- "JOURNAL WATCHER: ignoring event with mask %u (%s) for file '%s' ('%s')",
440
- event->mask,
441
- buffer_tostring(wb),
442
- path_to_use,
443
- fullPath);
444
- }
445
-}
446
-
447
-static void process_pending(Watcher *watcher)
448
-{
449
- errno_clear();
450
-
451
- void *x;
452
- dfe_start_write(watcher->pending, x)
453
- {
454
- struct stat info;
455
- const char *fullPath = x_dfe.name;
456
-
457
- if (stat(fullPath, &info) != 0) {
458
- nd_log(
459
- NDLS_COLLECTORS,
460
- NDLP_DEBUG,
461
- "JOURNAL WATCHER: file '%s' no longer exists, removing it from the registry",
462
- fullPath);
463
-
464
- dictionary_del(nd_journal_files_registry, fullPath);
465
- } else if (S_ISREG(info.st_mode)) {
466
- // nd_log(NDLS_COLLECTORS, NDLP_DEBUG,
467
- // "JOURNAL WATCHER: file '%s' has been added/updated, updating the registry",
468
- // fullPath);
469
-
470
- struct nd_journal_file t = {
471
- .file_last_modified_ut = info.st_mtim.tv_sec * USEC_PER_SEC + info.st_mtim.tv_nsec / NSEC_PER_USEC,
472
- .last_scan_monotonic_ut = now_monotonic_usec(),
473
- .size = info.st_size,
474
- .max_journal_vs_realtime_delta_ut = JOURNAL_VS_REALTIME_DELTA_DEFAULT_UT,
475
- };
476
- struct nd_journal_file *jf = dictionary_set(nd_journal_files_registry, fullPath, &t, sizeof(t));
477
- nd_journal_file_update_header(jf->filename, jf);
478
- }
479
-
480
- dictionary_del(watcher->pending, fullPath);
481
- }
482
- dfe_done(x);
483
-
484
- dictionary_garbage_collect(watcher->pending);
485
-}
486
-
487
-size_t journal_watcher_wanted_session_id = 0;
488
-
489
-void nd_journal_watcher_restart(void)
490
-{
491
- __atomic_add_fetch(&journal_watcher_wanted_session_id, 1, __ATOMIC_RELAXED);
492
-}
493
-
494
-static bool process_inotify_events(struct buffered_reader *reader, Watcher *watcher, int inotifyFd)
495
-{
496
- errno_clear();
497
-
498
- bool unmount_event = false;
499
- ssize_t processed = 0;
500
-
501
- // Process as many complete events as we can
502
- while (processed + (ssize_t)sizeof(struct inotify_event) <= reader->read_len) {
503
- struct inotify_event *event = (struct inotify_event *)(reader->read_buffer + processed);
504
-
505
- if (event->len > NAME_MAX + 1) {
506
- // The event length is impossibly large
507
- nd_log(
508
- NDLS_COLLECTORS,
509
- NDLP_ERR,
510
- "JOURNAL WATCHER: received impossibly large event length %u - restarting",
511
- event->len);
512
- return true; // force a restart
513
- }
514
-
515
- // Check if we have the complete event including the name
516
- ssize_t total_size = (ssize_t)sizeof(struct inotify_event) + event->len;
517
- if (processed + total_size > reader->read_len)
518
- break; // Wait for more data
519
-
520
- if (event->mask & IN_UNMOUNT) {
521
- unmount_event = true;
522
- break;
523
- }
524
-
525
- process_event(watcher, inotifyFd, event);
526
- processed += total_size;
527
- }
528
-
529
- // If we have unprocessed data, move it to the start
530
- if (processed < reader->read_len) {
531
- memmove(reader->read_buffer, reader->read_buffer + processed, reader->read_len - processed);
532
- reader->read_len -= processed;
533
- } else
534
- reader->read_len = 0;
535
-
536
- reader->read_buffer[reader->read_len] = '\0';
537
- return unmount_event;
538
-}
539
-
540
-void nd_journal_watcher_main(void *arg __maybe_unused)
541
-{
542
- while (1) {
543
- size_t journal_watcher_session_id = __atomic_load_n(&journal_watcher_wanted_session_id, __ATOMIC_RELAXED);
544
-
545
- Watcher watcher = {
546
- .watchList = mallocz(INITIAL_WATCHES * sizeof(WatchEntry)),
547
- .freeList = NULL,
548
- .watchCount = 0,
549
- .watchListSize = INITIAL_WATCHES,
550
- .pending = dictionary_create(DICT_OPTION_DONT_OVERWRITE_VALUE | DICT_OPTION_SINGLE_THREADED),
551
- .errors = 0,
552
- };
553
-
554
- int inotifyFd = inotify_init();
555
- if (inotifyFd < 0) {
556
- nd_log(NDLS_COLLECTORS, NDLP_ERR, "inotify_init() failed.");
557
- free_watches(&watcher, inotifyFd);
558
- return;
559
- }
560
-
561
- for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES; i++) {
562
- if (!journal_directories[i].path)
563
- break;
564
- watch_directory_and_subdirectories(&watcher, inotifyFd, string2str(journal_directories[i].path));
565
- }
566
-
567
- usec_t last_headers_update_ut = now_monotonic_usec();
568
- struct buffered_reader reader;
569
- buffered_reader_init(&reader);
570
-
571
- while (journal_watcher_session_id == __atomic_load_n(&journal_watcher_wanted_session_id, __ATOMIC_RELAXED)) {
572
- buffered_reader_ret_t rc =
573
- buffered_reader_read_timeout(&reader, inotifyFd, ND_SD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS, false);
574
-
575
- if (rc == BUFFERED_READER_READ_OK || rc == BUFFERED_READER_READ_BUFFER_FULL) {
576
- if (process_inotify_events(&reader, &watcher, inotifyFd))
577
- break;
578
- } else if (rc != BUFFERED_READER_READ_POLL_TIMEOUT) {
579
- nd_log(
580
- NDLS_COLLECTORS,
581
- NDLP_ERR,
582
- "JOURNAL WATCHER: cannot read inotify events, buffered_reader_read_timeout() returned %d - "
583
- "restarting the watcher.",
584
- rc);
585
- break;
586
- }
587
-
588
- usec_t ut = now_monotonic_usec();
589
- if (dictionary_entries(watcher.pending) &&
590
- (rc == BUFFERED_READER_READ_POLL_TIMEOUT ||
591
- last_headers_update_ut + (ND_SD_JOURNAL_EXECUTE_WATCHER_PENDING_EVERY_MS * USEC_PER_MS) <= ut)) {
592
- process_pending(&watcher);
593
- last_headers_update_ut = ut;
594
- }
595
-
596
- if (watcher.errors) {
597
- nd_log(
598
- NDLS_COLLECTORS,
599
- NDLP_NOTICE,
600
- "JOURNAL WATCHER: there were errors in setting up inotify watches - restarting the watcher.");
601
- }
602
- }
603
-
604
- close(inotifyFd);
605
- free_watches(&watcher, inotifyFd);
606
- free_symlinked_dirs(&watcher);
607
-
608
- // this will scan the directories and cleanup the registry
609
- nd_journal_files_registry_update();
610
-
611
- sleep_usec(2 * USEC_PER_SEC);
612
- }
613
-}
src/collectors/systemd-journal.plugin/systemd-journal.c
deleted
-1179
@@ -1,1179 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-/*
4
- * TODO
5
- * _UDEV_DEVLINK is frequently set more than once per field - support multi-value faces
6
- *
7
- */
8
-
9
-#include "collectors/systemd-journal.plugin/provider/netdata_provider.h"
10
-#include "systemd-internals.h"
11
-
12
-#define ND_SD_JOURNAL_FUNCTION_DESCRIPTION "View, search and analyze systemd journal entries."
13
-#define ND_SD_JOURNAL_FUNCTION_NAME "systemd-journal"
14
-#define ND_SD_JOURNAL_SAMPLING_SLOTS 1000
15
-#define ND_SD_JOURNAL_SAMPLING_RECALIBRATE 10000
16
-
17
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
18
-#define LQS_DEFAULT_SLICE_MODE 1
19
-#else
20
-#define LQS_DEFAULT_SLICE_MODE 0
21
-#endif
22
-
23
-// functions needed by LQS
24
-static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value);
25
-
26
-// structures needed by LQS
27
-struct lqs_extension {
28
- struct {
29
- usec_t start_ut;
30
- usec_t stop_ut;
31
- usec_t first_msg_ut;
32
-
33
- NsdId128 first_msg_writer;
34
- uint64_t first_msg_seqnum;
35
- } query_file;
36
-
37
- struct {
38
- uint32_t enable_after_samples;
39
- uint32_t slots;
40
- uint32_t sampled;
41
- uint32_t unsampled;
42
- uint32_t estimated;
43
- } samples;
44
-
45
- struct {
46
- uint32_t enable_after_samples;
47
- uint32_t every;
48
- uint32_t skipped;
49
- uint32_t recalibrate;
50
- uint32_t sampled;
51
- uint32_t unsampled;
52
- uint32_t estimated;
53
- } samples_per_file;
54
-
55
- struct {
56
- usec_t start_ut;
57
- usec_t end_ut;
58
- usec_t step_ut;
59
- uint32_t enable_after_samples;
60
- uint32_t sampled[ND_SD_JOURNAL_SAMPLING_SLOTS];
61
- uint32_t unsampled[ND_SD_JOURNAL_SAMPLING_SLOTS];
62
- } samples_per_time_slot;
63
-
64
- // per file progress info
65
- // size_t cached_count;
66
-
67
- // progress statistics
68
- usec_t matches_setup_ut;
69
- size_t rows_useful;
70
- size_t rows_read;
71
- size_t bytes_read;
72
- size_t files_matched;
73
- size_t file_working;
74
-};
75
-
76
-// prepare LQS
77
-#define LQS_FUNCTION_NAME ND_SD_JOURNAL_FUNCTION_NAME
78
-#define LQS_FUNCTION_DESCRIPTION ND_SD_JOURNAL_FUNCTION_DESCRIPTION
79
-#define LQS_DEFAULT_ITEMS_PER_QUERY 200
80
-#define LQS_DEFAULT_ITEMS_SAMPLING 1000000
81
-#define LQS_SOURCE_TYPE SD_JOURNAL_FILE_SOURCE_TYPE
82
-#define LQS_SOURCE_TYPE_ALL ND_SD_JF_ALL
83
-#define LQS_SOURCE_TYPE_NONE ND_SD_JF_NONE
84
-#define LQS_PARAMETER_SOURCE_NAME "Journal Sources" // this is how it is shown to users
85
-#define LQS_FUNCTION_GET_INTERNAL_SOURCE_TYPE(value) get_internal_source_type(value)
86
-#define LQS_FUNCTION_SOURCE_TO_JSON_ARRAY(wb) available_journal_file_sources_to_json_array(wb)
87
-#include "libnetdata/facets/logs_query_status.h"
88
-
89
-#include "systemd-journal-sampling.h"
90
-
91
-#define FACET_MAX_VALUE_LENGTH 8192
92
-#define ND_SD_JOURNAL_DEFAULT_TIMEOUT 60
93
-#define ND_SD_JOURNAL_PROGRESS_EVERY_UT (250 * USEC_PER_MS)
94
-#define JOURNAL_KEY_ND_JOURNAL_FILE "ND_JOURNAL_FILE"
95
-#define JOURNAL_KEY_ND_JOURNAL_PROCESS "ND_JOURNAL_PROCESS"
96
-#define JOURNAL_DEFAULT_DIRECTION FACETS_ANCHOR_DIRECTION_BACKWARD
97
-#define SYSTEMD_ALWAYS_VISIBLE_KEYS NULL
98
-
99
-#define SYSTEMD_KEYS_EXCLUDED_FROM_FACETS \
100
- "!MESSAGE_ID" \
101
- "|*MESSAGE*" \
102
- "|*TIMESTAMP*" \
103
- "|__*" \
104
- ""
105
-
106
-#define SYSTEMD_KEYS_INCLUDED_IN_FACETS \
107
- \
108
- /* --- USER JOURNAL FIELDS --- */ \
109
- \
110
- /* "|MESSAGE" */ \
111
- "|MESSAGE_ID" \
112
- "|PRIORITY" \
113
- "|CODE_FILE" /* "|CODE_LINE" */ \
114
- "|CODE_FUNC" \
115
- "|ERRNO" /* "|INVOCATION_ID" */ /* "|USER_INVOCATION_ID" */ \
116
- "|SYSLOG_FACILITY" \
117
- "|SYSLOG_IDENTIFIER" /* "|SYSLOG_PID" */ /* "|SYSLOG_TIMESTAMP" */ /* "|SYSLOG_RAW" */ /* "!DOCUMENTATION" */ /* "|TID" */ \
118
- "|UNIT" \
119
- "|USER_UNIT" \
120
- "|UNIT_RESULT" /* undocumented */ \
121
- \
122
- /* --- TRUSTED JOURNAL FIELDS --- */ \
123
- \
124
- /* "|_PID" */ \
125
- "|_UID" \
126
- "|_GID" \
127
- "|_COMM" \
128
- "|_EXE" /* "|_CMDLINE" */ \
129
- "|_CAP_EFFECTIVE" /* "|_AUDIT_SESSION" */ \
130
- "|_AUDIT_LOGINUID" \
131
- "|_SYSTEMD_CGROUP" \
132
- "|_SYSTEMD_SLICE" \
133
- "|_SYSTEMD_UNIT" \
134
- "|_SYSTEMD_USER_UNIT" \
135
- "|_SYSTEMD_USER_SLICE" \
136
- "|_SYSTEMD_SESSION" \
137
- "|_SYSTEMD_OWNER_UID" \
138
- "|_SELINUX_CONTEXT" /* "|_SOURCE_REALTIME_TIMESTAMP" */ \
139
- "|_BOOT_ID" \
140
- "|_MACHINE_ID" /* "|_SYSTEMD_INVOCATION_ID" */ \
141
- "|_HOSTNAME" \
142
- "|_TRANSPORT" \
143
- "|_STREAM_ID" /* "|LINE_BREAK" */ \
144
- "|_NAMESPACE" \
145
- "|_RUNTIME_SCOPE" \
146
- \
147
- /* --- KERNEL JOURNAL FIELDS --- */ \
148
- \
149
- /* "|_KERNEL_DEVICE" */ \
150
- "|_KERNEL_SUBSYSTEM" /* "|_UDEV_SYSNAME" */ \
151
- "|_UDEV_DEVNODE" /* "|_UDEV_DEVLINK" */ \
152
- \
153
- /* --- LOGGING ON BEHALF --- */ \
154
- \
155
- "|OBJECT_UID" \
156
- "|OBJECT_GID" \
157
- "|OBJECT_COMM" \
158
- "|OBJECT_EXE" /* "|OBJECT_CMDLINE" */ /* "|OBJECT_AUDIT_SESSION" */ \
159
- "|OBJECT_AUDIT_LOGINUID" \
160
- "|OBJECT_SYSTEMD_CGROUP" \
161
- "|OBJECT_SYSTEMD_SESSION" \
162
- "|OBJECT_SYSTEMD_OWNER_UID" \
163
- "|OBJECT_SYSTEMD_UNIT" \
164
- "|OBJECT_SYSTEMD_USER_UNIT" \
165
- \
166
- /* --- CORE DUMPS --- */ \
167
- \
168
- "|COREDUMP_COMM" \
169
- "|COREDUMP_UNIT" \
170
- "|COREDUMP_USER_UNIT" \
171
- "|COREDUMP_SIGNAL_NAME" \
172
- "|COREDUMP_CGROUP" \
173
- \
174
- /* --- DOCKER --- */ \
175
- \
176
- "|CONTAINER_ID" /* "|CONTAINER_ID_FULL" */ \
177
- "|CONTAINER_NAME" \
178
- "|CONTAINER_TAG" \
179
- "|IMAGE_NAME" /* undocumented */ /* "|CONTAINER_PARTIAL_MESSAGE" */ \
180
- \
181
- /* --- NETDATA --- */ \
182
- \
183
- "|ND_NIDL_NODE" \
184
- "|ND_NIDL_CONTEXT" \
185
- "|ND_LOG_SOURCE" /*"|ND_MODULE" */ \
186
- "|ND_ALERT_NAME" \
187
- "|ND_ALERT_CLASS" \
188
- "|ND_ALERT_COMPONENT" \
189
- "|ND_ALERT_TYPE" \
190
- "|ND_ALERT_STATUS" \
191
- \
192
- ""
193
-
194
-static SD_JOURNAL_FILE_SOURCE_TYPE get_internal_source_type(const char *value)
195
-{
196
- if (strcmp(value, ND_SD_JF_SOURCE_ALL_NAME) == 0)
197
- return ND_SD_JF_ALL;
198
- else if (strcmp(value, ND_SD_JF_SOURCE_LOCAL_NAME) == 0)
199
- return ND_SD_JF_LOCAL_ALL;
200
- else if (strcmp(value, ND_SD_JF_SOURCE_REMOTES_NAME) == 0)
201
- return ND_SD_JF_REMOTE_ALL;
202
- else if (strcmp(value, ND_SD_JF_SOURCE_NAMESPACES_NAME) == 0)
203
- return ND_SD_JF_LOCAL_NAMESPACE;
204
- else if (strcmp(value, ND_SD_JF_SOURCE_LOCAL_SYSTEM_NAME) == 0)
205
- return ND_SD_JF_LOCAL_SYSTEM;
206
- else if (strcmp(value, ND_SD_JF_SOURCE_LOCAL_USERS_NAME) == 0)
207
- return ND_SD_JF_LOCAL_USER;
208
- else if (strcmp(value, ND_SD_JF_SOURCE_LOCAL_OTHER_NAME) == 0)
209
- return ND_SD_JF_LOCAL_OTHER;
210
-
211
- return ND_SD_JF_NONE;
212
-}
213
-
214
-static inline bool nd_sd_journal_seek_to(NsdJournal *j, usec_t timestamp)
215
-{
216
- if (nsd_journal_seek_realtime_usec(j, timestamp) < 0) {
217
- netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to %" PRIu64, timestamp);
218
- if (nsd_journal_seek_tail(j) < 0) {
219
- netdata_log_error("SYSTEMD-JOURNAL: Failed to seek to journal's tail");
220
- return false;
221
- }
222
- }
223
-
224
- return true;
225
-}
226
-
227
-#define JD_SOURCE_REALTIME_TIMESTAMP "_SOURCE_REALTIME_TIMESTAMP"
228
-
229
-static inline size_t
230
-nd_sd_journal_process_row(NsdJournal *j, FACETS *facets, struct nd_journal_file *njf, usec_t *msg_ut)
231
-{
232
- const void *data;
233
- size_t length, bytes = 0;
234
-
235
- facets_add_key_value_length(
236
- facets, JOURNAL_KEY_ND_JOURNAL_FILE, sizeof(JOURNAL_KEY_ND_JOURNAL_FILE) - 1, njf->filename, njf->filename_len);
237
-
238
- NSD_JOURNAL_FOREACH_DATA(j, data, length)
239
- {
240
- const char *key, *value;
241
- size_t key_length, value_length;
242
-
243
- if (!parse_journal_field(data, length, &key, &key_length, &value, &value_length))
244
- continue;
245
-
246
-#ifdef NETDATA_INTERNAL_CHECKS
247
- usec_t origin_journal_ut = *msg_ut;
248
-#endif
249
- if (unlikely(
250
- key_length == sizeof(JD_SOURCE_REALTIME_TIMESTAMP) - 1 &&
251
- memcmp(key, JD_SOURCE_REALTIME_TIMESTAMP, sizeof(JD_SOURCE_REALTIME_TIMESTAMP) - 1) == 0)) {
252
- usec_t ut = str2ull(value, NULL);
253
- if (ut && ut < *msg_ut) {
254
- usec_t delta = *msg_ut - ut;
255
- *msg_ut = ut;
256
-
257
- if (delta > JOURNAL_VS_REALTIME_DELTA_MAX_UT)
258
- delta = JOURNAL_VS_REALTIME_DELTA_MAX_UT;
259
-
260
- // update max_journal_vs_realtime_delta_ut if the delta increased
261
- usec_t expected = njf->max_journal_vs_realtime_delta_ut;
262
- do {
263
- if (delta <= expected)
264
- break;
265
- } while (!__atomic_compare_exchange_n(
266
- &njf->max_journal_vs_realtime_delta_ut, &expected, delta, false, __ATOMIC_RELAXED, __ATOMIC_RELAXED));
267
-
268
- internal_error(
269
- delta > expected,
270
- "increased max_journal_vs_realtime_delta_ut from %" PRIu64 " to %" PRIu64 ", "
271
- "journal %" PRIu64 ", actual %" PRIu64 " (delta %" PRIu64 ")",
272
- expected,
273
- delta,
274
- origin_journal_ut,
275
- *msg_ut,
276
- origin_journal_ut - (*msg_ut));
277
- }
278
- }
279
-
280
- bytes += length;
281
- facets_add_key_value_length(
282
- facets,
283
- key,
284
- key_length,
285
- value,
286
- value_length <= FACET_MAX_VALUE_LENGTH ? value_length : FACET_MAX_VALUE_LENGTH);
287
- }
288
-
289
- return bytes;
290
-}
291
-
292
-#define FUNCTION_PROGRESS_UPDATE_ROWS(rows_read, rows) __atomic_fetch_add(&(rows_read), rows, __ATOMIC_RELAXED)
293
-#define FUNCTION_PROGRESS_UPDATE_BYTES(bytes_read, bytes) __atomic_fetch_add(&(bytes_read), bytes, __ATOMIC_RELAXED)
294
-#define FUNCTION_PROGRESS_EVERY_ROWS (1ULL << 13)
295
-#define FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS (1ULL << 7)
296
-
297
-static inline ND_SD_JOURNAL_STATUS check_stop(const bool *cancelled, const usec_t *stop_monotonic_ut)
298
-{
299
- if (cancelled && __atomic_load_n(cancelled, __ATOMIC_RELAXED)) {
300
- internal_error(true, "Function has been cancelled");
301
- return ND_SD_JOURNAL_CANCELLED;
302
- }
303
-
304
- if (now_monotonic_usec() > __atomic_load_n(stop_monotonic_ut, __ATOMIC_RELAXED)) {
305
- internal_error(true, "Function timed out");
306
- return ND_SD_JOURNAL_TIMED_OUT;
307
- }
308
-
309
- return ND_SD_JOURNAL_OK;
310
-}
311
-
312
-ND_SD_JOURNAL_STATUS nd_sd_journal_query_backward(
313
- NsdJournal *j,
314
- BUFFER *wb __maybe_unused,
315
- FACETS *facets,
316
- struct nd_journal_file *njf,
317
- LOGS_QUERY_STATUS *fqs)
318
-{
319
- usec_t anchor_delta = __atomic_load_n(&njf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
320
- lqs_query_timeframe(fqs, anchor_delta);
321
- usec_t start_ut = fqs->query.start_ut;
322
- usec_t stop_ut = fqs->query.stop_ut;
323
- bool stop_when_full = fqs->query.stop_when_full;
324
-
325
- fqs->c.query_file.start_ut = start_ut;
326
- fqs->c.query_file.stop_ut = stop_ut;
327
-
328
- if (!nd_sd_journal_seek_to(j, start_ut))
329
- return ND_SD_JOURNAL_FAILED_TO_SEEK;
330
-
331
- size_t errors_no_timestamp = 0;
332
- usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
333
- usec_t first_msg_ut = 0; // the first message we got from the db
334
- size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
335
- size_t bytes = 0, last_bytes = 0;
336
-
337
- usec_t last_usec_from = 0;
338
- usec_t last_usec_to = 0;
339
-
340
- ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK;
341
-
342
- facets_rows_begin(facets);
343
- while (status == ND_SD_JOURNAL_OK && nsd_journal_previous(j) > 0) {
344
- usec_t msg_ut = 0;
345
- if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
346
- errors_no_timestamp++;
347
- continue;
348
- }
349
-
350
- if (unlikely(msg_ut > start_ut))
351
- continue;
352
-
353
- if (unlikely(msg_ut < stop_ut))
354
- break;
355
-
356
- if (unlikely(msg_ut > latest_msg_ut))
357
- latest_msg_ut = msg_ut;
358
-
359
- if (unlikely(!first_msg_ut)) {
360
- first_msg_ut = msg_ut;
361
- fqs->c.query_file.first_msg_ut = msg_ut;
362
-
363
-#ifdef HAVE_SD_JOURNAL_GET_SEQNUM
364
- if (nsd_journal_get_seqnum(j, &fqs->c.query_file.first_msg_seqnum, &fqs->c.query_file.first_msg_writer) <
365
- 0) {
366
- fqs->c.query_file.first_msg_seqnum = 0;
367
- fqs->c.query_file.first_msg_writer = NSD_ID128_NULL;
368
- }
369
-#endif
370
- }
371
-
372
- sampling_t sample = is_row_in_sample(
373
- j, fqs, njf, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD, facets_row_candidate_to_keep(facets, msg_ut));
374
-
375
- if (sample == SAMPLING_FULL) {
376
- bytes += nd_sd_journal_process_row(j, facets, njf, &msg_ut);
377
-
378
- // make sure each line gets a unique timestamp
379
- if (unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
380
- msg_ut = --last_usec_from;
381
- else
382
- last_usec_from = last_usec_to = msg_ut;
383
-
384
- if (facets_row_finished(facets, msg_ut))
385
- rows_useful++;
386
-
387
- row_counter++;
388
- if (unlikely(
389
- (row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 && stop_when_full &&
390
- facets_rows(facets) >= fqs->rq.entries)) {
391
- // stop the data only query
392
- usec_t oldest = facets_row_oldest_ut(facets);
393
- if (oldest && msg_ut < (oldest - anchor_delta))
394
- break;
395
- }
396
-
397
- if (unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
398
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
399
- last_row_counter = row_counter;
400
-
401
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
402
- last_bytes = bytes;
403
-
404
- status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
405
- }
406
- } else if (sample == SAMPLING_SKIP_FIELDS)
407
- facets_row_finished_unsampled(facets, msg_ut);
408
- else {
409
- sampling_update_running_query_file_estimates(facets, j, fqs, njf, msg_ut, FACETS_ANCHOR_DIRECTION_BACKWARD);
410
- break;
411
- }
412
- }
413
-
414
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
415
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
416
-
417
- fqs->c.rows_useful += rows_useful;
418
-
419
- if (errors_no_timestamp)
420
- netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);
421
-
422
- if (latest_msg_ut > fqs->last_modified)
423
- fqs->last_modified = latest_msg_ut;
424
-
425
- return status;
426
-}
427
-
428
-ND_SD_JOURNAL_STATUS nd_sd_journal_query_forward(
429
- NsdJournal *j,
430
- BUFFER *wb __maybe_unused,
431
- FACETS *facets,
432
- struct nd_journal_file *njf,
433
- LOGS_QUERY_STATUS *fqs)
434
-{
435
- usec_t anchor_delta = __atomic_load_n(&njf->max_journal_vs_realtime_delta_ut, __ATOMIC_RELAXED);
436
- lqs_query_timeframe(fqs, anchor_delta);
437
- usec_t start_ut = fqs->query.start_ut;
438
- usec_t stop_ut = fqs->query.stop_ut;
439
- bool stop_when_full = fqs->query.stop_when_full;
440
-
441
- fqs->c.query_file.start_ut = start_ut;
442
- fqs->c.query_file.stop_ut = stop_ut;
443
-
444
- if (!nd_sd_journal_seek_to(j, start_ut))
445
- return ND_SD_JOURNAL_FAILED_TO_SEEK;
446
-
447
- size_t errors_no_timestamp = 0;
448
- usec_t latest_msg_ut = 0; // the biggest timestamp we have seen so far
449
- usec_t first_msg_ut = 0; // the first message we got from the db
450
- size_t row_counter = 0, last_row_counter = 0, rows_useful = 0;
451
- size_t bytes = 0, last_bytes = 0;
452
-
453
- usec_t last_usec_from = 0;
454
- usec_t last_usec_to = 0;
455
-
456
- ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_OK;
457
-
458
- facets_rows_begin(facets);
459
- while (status == ND_SD_JOURNAL_OK && nsd_journal_next(j) > 0) {
460
- usec_t msg_ut = 0;
461
- if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0 || !msg_ut) {
462
- errors_no_timestamp++;
463
- continue;
464
- }
465
-
466
- if (unlikely(msg_ut < start_ut))
467
- continue;
468
-
469
- if (unlikely(msg_ut > stop_ut))
470
- break;
471
-
472
- if (likely(msg_ut > latest_msg_ut))
473
- latest_msg_ut = msg_ut;
474
-
475
- if (unlikely(!first_msg_ut)) {
476
- first_msg_ut = msg_ut;
477
- fqs->c.query_file.first_msg_ut = msg_ut;
478
- }
479
-
480
- sampling_t sample = is_row_in_sample(
481
- j, fqs, njf, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD, facets_row_candidate_to_keep(facets, msg_ut));
482
-
483
- if (sample == SAMPLING_FULL) {
484
- bytes += nd_sd_journal_process_row(j, facets, njf, &msg_ut);
485
-
486
- // make sure each line gets a unique timestamp
487
- if (unlikely(msg_ut >= last_usec_from && msg_ut <= last_usec_to))
488
- msg_ut = ++last_usec_to;
489
- else
490
- last_usec_from = last_usec_to = msg_ut;
491
-
492
- if (facets_row_finished(facets, msg_ut))
493
- rows_useful++;
494
-
495
- row_counter++;
496
- if (unlikely(
497
- (row_counter % FUNCTION_DATA_ONLY_CHECK_EVERY_ROWS) == 0 && stop_when_full &&
498
- facets_rows(facets) >= fqs->rq.entries)) {
499
- // stop the data only query
500
- usec_t newest = facets_row_newest_ut(facets);
501
- if (newest && msg_ut > (newest + anchor_delta))
502
- break;
503
- }
504
-
505
- if (unlikely(row_counter % FUNCTION_PROGRESS_EVERY_ROWS == 0)) {
506
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
507
- last_row_counter = row_counter;
508
-
509
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
510
- last_bytes = bytes;
511
-
512
- status = check_stop(fqs->cancelled, fqs->stop_monotonic_ut);
513
- }
514
- } else if (sample == SAMPLING_SKIP_FIELDS)
515
- facets_row_finished_unsampled(facets, msg_ut);
516
- else {
517
- sampling_update_running_query_file_estimates(facets, j, fqs, njf, msg_ut, FACETS_ANCHOR_DIRECTION_FORWARD);
518
- break;
519
- }
520
- }
521
-
522
- FUNCTION_PROGRESS_UPDATE_ROWS(fqs->c.rows_read, row_counter - last_row_counter);
523
- FUNCTION_PROGRESS_UPDATE_BYTES(fqs->c.bytes_read, bytes - last_bytes);
524
-
525
- fqs->c.rows_useful += rows_useful;
526
-
527
- if (errors_no_timestamp)
528
- netdata_log_error("SYSTEMD-JOURNAL: %zu lines did not have timestamps", errors_no_timestamp);
529
-
530
- if (latest_msg_ut > fqs->last_modified)
531
- fqs->last_modified = latest_msg_ut;
532
-
533
- return status;
534
-}
535
-
536
-bool nd_sd_journal_check_if_modified_since(NsdJournal *j, usec_t seek_to, usec_t last_modified)
537
-{
538
- // return true, if data have been modified since the timestamp
539
-
540
- if (!last_modified || !seek_to)
541
- return false;
542
-
543
- if (!nd_sd_journal_seek_to(j, seek_to))
544
- return false;
545
-
546
- usec_t first_msg_ut = 0;
547
- while (nsd_journal_previous(j) > 0) {
548
- usec_t msg_ut;
549
- if (nsd_journal_get_realtime_usec(j, &msg_ut) < 0)
550
- continue;
551
-
552
- first_msg_ut = msg_ut;
553
- break;
554
- }
555
-
556
- return first_msg_ut != last_modified;
557
-}
558
-
559
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
560
-static bool netdata_systemd_filtering_by_journal(NsdJournal *j, FACETS *facets, LOGS_QUERY_STATUS *lqs)
561
-{
562
- const char *field = NULL;
563
- const void *data = NULL;
564
- size_t data_length;
565
- size_t added_keys = 0;
566
- size_t failures = 0;
567
- size_t filters_added = 0;
568
-
569
- NSD_JOURNAL_FOREACH_FIELD(j, field)
570
- { // for each key
571
- bool interesting;
572
-
573
- if (lqs->rq.data_only)
574
- interesting = facets_key_name_is_filter(facets, field);
575
- else
576
- interesting = facets_key_name_is_facet(facets, field);
577
-
578
- if (interesting) {
579
- if (nsd_journal_query_unique(j, field) >= 0) {
580
- bool added_this_key = false;
581
- size_t added_values = 0;
582
-
583
- NSD_JOURNAL_FOREACH_UNIQUE(j, data, data_length)
584
- { // for each value of the key
585
- const char *key, *value;
586
- size_t key_length, value_length;
587
-
588
- if (!parse_journal_field(data, data_length, &key, &key_length, &value, &value_length))
589
- continue;
590
-
591
- facets_add_possible_value_name_to_key(facets, key, key_length, value, value_length);
592
-
593
- if (!facets_key_name_value_length_is_selected(facets, key, key_length, value, value_length))
594
- continue;
595
-
596
- if (added_keys && !added_this_key) {
597
- if (nsd_journal_add_conjunction(j) < 0) // key AND key AND key
598
- failures++;
599
-
600
- added_this_key = true;
601
- added_keys++;
602
- } else if (added_values)
603
- if (nsd_journal_add_disjunction(j) < 0) // value OR value OR value
604
- failures++;
605
-
606
- if (nsd_journal_add_match(j, data, data_length) < 0)
607
- failures++;
608
-
609
- if (!added_keys) {
610
- added_keys++;
611
- added_this_key = true;
612
- }
613
-
614
- added_values++;
615
- filters_added++;
616
- }
617
- }
618
- }
619
- }
620
-
621
- if (failures) {
622
- lqs_log_error(lqs, "failed to setup journal filter, will run the full query.");
623
- nsd_journal_flush_matches(j);
624
- return true;
625
- }
626
-
627
- return filters_added ? true : false;
628
-}
629
-#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
630
-
631
-static ND_SD_JOURNAL_STATUS nd_sd_journal_query_one_file(
632
- const char *filename,
633
- BUFFER *wb,
634
- FACETS *facets,
635
- struct nd_journal_file *njf,
636
- LOGS_QUERY_STATUS *fqs)
637
-{
638
- NsdJournal *j = NULL;
639
- errno_clear();
640
-
641
- fstat_cache_enable_on_thread();
642
-
643
- const char *paths[2] = {
644
- [0] = filename,
645
- [1] = NULL,
646
- };
647
-
648
- if (nsd_journal_open_files(&j, paths, ND_SD_JOURNAL_OPEN_FLAGS) < 0 || !j) {
649
- netdata_log_error("JOURNAL: cannot open file '%s' for query", filename);
650
- fstat_cache_disable_on_thread();
651
- return ND_SD_JOURNAL_FAILED_TO_OPEN;
652
- }
653
-
654
- ND_SD_JOURNAL_STATUS status;
655
- bool matches_filters = true;
656
-
657
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
658
- if (fqs->rq.slice) {
659
- usec_t started = now_monotonic_usec();
660
-
661
- matches_filters = netdata_systemd_filtering_by_journal(j, facets, fqs) || !fqs->rq.filters;
662
- usec_t ended = now_monotonic_usec();
663
-
664
- fqs->c.matches_setup_ut += (ended - started);
665
- }
666
-#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
667
-
668
- if (matches_filters) {
669
- if (fqs->rq.direction == FACETS_ANCHOR_DIRECTION_FORWARD)
670
- status = nd_sd_journal_query_forward(j, wb, facets, njf, fqs);
671
- else
672
- status = nd_sd_journal_query_backward(j, wb, facets, njf, fqs);
673
- } else
674
- status = ND_SD_JOURNAL_NO_FILE_MATCHED;
675
-
676
- nsd_journal_close(j);
677
- fstat_cache_disable_on_thread();
678
-
679
- return status;
680
-}
681
-
682
-static bool jf_is_mine(struct nd_journal_file *njf, LOGS_QUERY_STATUS *fqs)
683
-{
684
- if ((fqs->rq.source_type == ND_SD_JF_NONE && !fqs->rq.sources) || (njf->source_type & fqs->rq.source_type) ||
685
- (fqs->rq.sources && simple_pattern_matches(fqs->rq.sources, string2str(njf->source)))) {
686
- if (!njf->msg_last_ut)
687
- // the file is not scanned yet, or the timestamps have not been updated,
688
- // so we don't know if it can contribute or not - let's add it.
689
- return true;
690
-
691
- usec_t anchor_delta = JOURNAL_VS_REALTIME_DELTA_MAX_UT;
692
- usec_t first_ut = njf->msg_first_ut - anchor_delta;
693
- usec_t last_ut = njf->msg_last_ut + anchor_delta;
694
-
695
- if (last_ut >= fqs->rq.after_ut && first_ut <= fqs->rq.before_ut)
696
- return true;
697
- }
698
-
699
- return false;
700
-}
701
-
702
-static int nd_sd_journal_query(BUFFER *wb, LOGS_QUERY_STATUS *lqs)
703
-{
704
- FACETS *facets = lqs->facets;
705
-
706
- ND_SD_JOURNAL_STATUS status = ND_SD_JOURNAL_NO_FILE_MATCHED;
707
- struct nd_journal_file *njf;
708
-
709
- lqs->c.files_matched = 0;
710
- lqs->c.file_working = 0;
711
- lqs->c.rows_useful = 0;
712
- lqs->c.rows_read = 0;
713
- lqs->c.bytes_read = 0;
714
-
715
- size_t files_used = 0;
716
- size_t files_max = dictionary_entries(nd_journal_files_registry);
717
- const DICTIONARY_ITEM *file_items[files_max];
718
-
719
- // count the files
720
- bool files_are_newer = false;
721
- dfe_start_read(nd_journal_files_registry, njf)
722
- {
723
- if (!jf_is_mine(njf, lqs))
724
- continue;
725
-
726
- file_items[files_used++] = dictionary_acquired_item_dup(nd_journal_files_registry, njf_dfe.item);
727
-
728
- if (njf->msg_last_ut > lqs->rq.if_modified_since)
729
- files_are_newer = true;
730
- }
731
- dfe_done(jf);
732
-
733
- lqs->c.files_matched = files_used;
734
-
735
- if (lqs->rq.if_modified_since && !files_are_newer) {
736
- // release the files
737
- for (size_t f = 0; f < files_used; f++)
738
- dictionary_acquired_item_release(nd_journal_files_registry, file_items[f]);
739
-
740
- return rrd_call_function_error(wb, "No new data since the previous call.", HTTP_RESP_NOT_MODIFIED);
741
- }
742
-
743
- // sort the files, so that they are optimal for facets
744
- if (files_used >= 2) {
745
- if (lqs->rq.direction == FACETS_ANCHOR_DIRECTION_BACKWARD)
746
- qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *), nd_journal_file_dict_items_backward_compar);
747
- else
748
- qsort(file_items, files_used, sizeof(const DICTIONARY_ITEM *), nd_journal_file_dict_items_forward_compar);
749
- }
750
-
751
- bool partial = false;
752
- usec_t query_started_ut = now_monotonic_usec();
753
- usec_t started_ut = query_started_ut;
754
- usec_t ended_ut = started_ut;
755
- usec_t duration_ut = 0, max_duration_ut = 0;
756
- usec_t progress_duration_ut = 0;
757
-
758
- sampling_query_init(lqs, facets);
759
-
760
- buffer_json_member_add_array(wb, "_journal_files");
761
- for (size_t f = 0; f < files_used; f++) {
762
- const char *filename = dictionary_acquired_item_name(file_items[f]);
763
- njf = dictionary_acquired_item_value(file_items[f]);
764
-
765
- if (!jf_is_mine(njf, lqs))
766
- continue;
767
-
768
- started_ut = ended_ut;
769
-
770
- // do not even try to do the query if we expect it to pass the timeout
771
- if (ended_ut + max_duration_ut * 3 >= *lqs->stop_monotonic_ut) {
772
- partial = true;
773
- status = ND_SD_JOURNAL_TIMED_OUT;
774
- break;
775
- }
776
-
777
- lqs->c.file_working++;
778
-
779
- size_t fs_calls = fstat_thread_calls;
780
- size_t fs_cached = fstat_thread_cached_responses;
781
- size_t rows_useful = lqs->c.rows_useful;
782
- size_t rows_read = lqs->c.rows_read;
783
- size_t bytes_read = lqs->c.bytes_read;
784
- size_t matches_setup_ut = lqs->c.matches_setup_ut;
785
-
786
- sampling_file_init(lqs, njf);
787
-
788
- ND_SD_JOURNAL_STATUS tmp_status = nd_sd_journal_query_one_file(filename, wb, facets, njf, lqs);
789
-
790
- rows_useful = lqs->c.rows_useful - rows_useful;
791
- rows_read = lqs->c.rows_read - rows_read;
792
- bytes_read = lqs->c.bytes_read - bytes_read;
793
- matches_setup_ut = lqs->c.matches_setup_ut - matches_setup_ut;
794
- fs_calls = fstat_thread_calls - fs_calls;
795
- fs_cached = fstat_thread_cached_responses - fs_cached;
796
-
797
- ended_ut = now_monotonic_usec();
798
- duration_ut = ended_ut - started_ut;
799
-
800
- if (duration_ut > max_duration_ut)
801
- max_duration_ut = duration_ut;
802
-
803
- progress_duration_ut += duration_ut;
804
- if (progress_duration_ut >= ND_SD_JOURNAL_PROGRESS_EVERY_UT) {
805
- progress_duration_ut = 0;
806
- netdata_mutex_lock(&stdout_mutex);
807
- pluginsd_function_progress_to_stdout(lqs->rq.transaction, f + 1, files_used);
808
- netdata_mutex_unlock(&stdout_mutex);
809
- }
810
-
811
- buffer_json_add_array_item_object(wb); // journal file
812
- {
813
- // information about the file
814
- buffer_json_member_add_string(wb, "_filename", filename);
815
- buffer_json_member_add_uint64(wb, "_source_type", njf->source_type);
816
- buffer_json_member_add_string(wb, "_source", string2str(njf->source));
817
- buffer_json_member_add_uint64(wb, "_last_modified_ut", njf->file_last_modified_ut);
818
- buffer_json_member_add_uint64(wb, "_msg_first_ut", njf->msg_first_ut);
819
- buffer_json_member_add_uint64(wb, "_msg_last_ut", njf->msg_last_ut);
820
- buffer_json_member_add_uint64(wb, "_journal_vs_realtime_delta_ut", njf->max_journal_vs_realtime_delta_ut);
821
-
822
- // information about the current use of the file
823
- buffer_json_member_add_uint64(wb, "duration_ut", ended_ut - started_ut);
824
- buffer_json_member_add_uint64(wb, "rows_read", rows_read);
825
- buffer_json_member_add_uint64(wb, "rows_useful", rows_useful);
826
- buffer_json_member_add_double(
827
- wb, "rows_per_second", (double)rows_read / (double)duration_ut * (double)USEC_PER_SEC);
828
- buffer_json_member_add_uint64(wb, "bytes_read", bytes_read);
829
- buffer_json_member_add_double(
830
- wb, "bytes_per_second", (double)bytes_read / (double)duration_ut * (double)USEC_PER_SEC);
831
- buffer_json_member_add_uint64(wb, "duration_matches_ut", matches_setup_ut);
832
- buffer_json_member_add_uint64(wb, "fstat_query_calls", fs_calls);
833
- buffer_json_member_add_uint64(wb, "fstat_query_cached_responses", fs_cached);
834
-
835
- if (lqs->rq.sampling) {
836
- buffer_json_member_add_object(wb, "_sampling");
837
- {
838
- buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples_per_file.sampled);
839
- buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples_per_file.unsampled);
840
- buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples_per_file.estimated);
841
- }
842
- buffer_json_object_close(wb); // _sampling
843
- }
844
- }
845
- buffer_json_object_close(wb); // journal file
846
-
847
- bool stop = false;
848
- switch (tmp_status) {
849
- case ND_SD_JOURNAL_OK:
850
- case ND_SD_JOURNAL_NO_FILE_MATCHED:
851
- status = (status == ND_SD_JOURNAL_OK) ? ND_SD_JOURNAL_OK : tmp_status;
852
- break;
853
-
854
- case ND_SD_JOURNAL_FAILED_TO_OPEN:
855
- case ND_SD_JOURNAL_FAILED_TO_SEEK:
856
- partial = true;
857
- if (status == ND_SD_JOURNAL_NO_FILE_MATCHED)
858
- status = tmp_status;
859
- break;
860
-
861
- case ND_SD_JOURNAL_CANCELLED:
862
- case ND_SD_JOURNAL_TIMED_OUT:
863
- partial = true;
864
- stop = true;
865
- status = tmp_status;
866
- break;
867
-
868
- case ND_SD_JOURNAL_NOT_MODIFIED:
869
- internal_fatal(true, "this should never be returned here");
870
- break;
871
- }
872
-
873
- if (stop)
874
- break;
875
- }
876
- buffer_json_array_close(wb); // _journal_files
877
-
878
- // release the files
879
- for (size_t f = 0; f < files_used; f++)
880
- dictionary_acquired_item_release(nd_journal_files_registry, file_items[f]);
881
-
882
- switch (status) {
883
- case ND_SD_JOURNAL_OK:
884
- if (lqs->rq.if_modified_since && !lqs->c.rows_useful)
885
- return rrd_call_function_error(
886
- wb, "No additional useful data since the previous call.", HTTP_RESP_NOT_MODIFIED);
887
- break;
888
-
889
- case ND_SD_JOURNAL_TIMED_OUT:
890
- case ND_SD_JOURNAL_NO_FILE_MATCHED:
891
- break;
892
-
893
- case ND_SD_JOURNAL_CANCELLED:
894
- return rrd_call_function_error(wb, "Request cancelled.", HTTP_RESP_CLIENT_CLOSED_REQUEST);
895
-
896
- case ND_SD_JOURNAL_NOT_MODIFIED:
897
- return rrd_call_function_error(wb, "No new data since the previous call.", HTTP_RESP_NOT_MODIFIED);
898
-
899
- case ND_SD_JOURNAL_FAILED_TO_OPEN:
900
- return rrd_call_function_error(wb, "Failed to open systemd journal file.", HTTP_RESP_INTERNAL_SERVER_ERROR);
901
-
902
- case ND_SD_JOURNAL_FAILED_TO_SEEK:
903
- return rrd_call_function_error(
904
- wb, "Failed to seek in systemd journal file.", HTTP_RESP_INTERNAL_SERVER_ERROR);
905
-
906
- default:
907
- return rrd_call_function_error(wb, "Unknown status", HTTP_RESP_INTERNAL_SERVER_ERROR);
908
- }
909
-
910
- buffer_json_member_add_uint64(wb, "status", HTTP_RESP_OK);
911
- buffer_json_member_add_boolean(wb, "partial", partial);
912
- buffer_json_member_add_string(wb, "type", "table");
913
-
914
- // build a message for the query
915
- if (!lqs->rq.data_only) {
916
- CLEAN_BUFFER *msg = buffer_create(0, NULL);
917
- CLEAN_BUFFER *msg_description = buffer_create(0, NULL);
918
- ND_LOG_FIELD_PRIORITY msg_priority = NDLP_INFO;
919
-
920
- if (!nd_journal_files_completed_once()) {
921
- buffer_strcat(msg, "Journals are still being scanned. ");
922
- buffer_strcat(
923
- msg_description,
924
- "LIBRARY SCAN: The journal files are still being scanned, you are probably viewing incomplete data. ");
925
- msg_priority = NDLP_WARNING;
926
- }
927
-
928
- if (partial) {
929
- buffer_strcat(msg, "Query timed-out, incomplete data. ");
930
- buffer_strcat(
931
- msg_description,
932
- "QUERY TIMEOUT: The query timed out and may not include all the data of the selected window. ");
933
- msg_priority = NDLP_WARNING;
934
- }
935
-
936
- if (lqs->c.samples.estimated || lqs->c.samples.unsampled) {
937
- double percent = (double)(lqs->c.samples.sampled * 100.0 /
938
- (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
939
- buffer_sprintf(msg, "%.2f%% real data", percent);
940
- buffer_sprintf(msg_description, "ACTUAL DATA: The filters counters reflect %0.2f%% of the data. ", percent);
941
- msg_priority = MIN(msg_priority, NDLP_NOTICE);
942
- }
943
-
944
- if (lqs->c.samples.unsampled) {
945
- double percent = (double)(lqs->c.samples.unsampled * 100.0 /
946
- (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
947
- buffer_sprintf(msg, ", %.2f%% unsampled", percent);
948
- buffer_sprintf(
949
- msg_description,
950
- "UNSAMPLED DATA: %0.2f%% of the events exist and have been counted, but their values have not been evaluated, so they are not included in the filters counters. ",
951
- percent);
952
- msg_priority = MIN(msg_priority, NDLP_NOTICE);
953
- }
954
-
955
- if (lqs->c.samples.estimated) {
956
- double percent = (double)(lqs->c.samples.estimated * 100.0 /
957
- (lqs->c.samples.estimated + lqs->c.samples.unsampled + lqs->c.samples.sampled));
958
- buffer_sprintf(msg, ", %.2f%% estimated", percent);
959
- buffer_sprintf(
960
- msg_description,
961
- "ESTIMATED DATA: The query selected a large amount of data, so to avoid delaying too much, the presented data are estimated by %0.2f%%. ",
962
- percent);
963
- msg_priority = MIN(msg_priority, NDLP_NOTICE);
964
- }
965
-
966
- buffer_json_member_add_object(wb, "message");
967
- if (buffer_tostring(msg)) {
968
- buffer_json_member_add_string(wb, "title", buffer_tostring(msg));
969
- buffer_json_member_add_string(wb, "description", buffer_tostring(msg_description));
970
- buffer_json_member_add_string(wb, "status", nd_log_id2priority(msg_priority));
971
- }
972
- // else send an empty object if there is nothing to tell
973
- buffer_json_object_close(wb); // message
974
- }
975
-
976
- if (!lqs->rq.data_only) {
977
- buffer_json_member_add_time_t(wb, "update_every", 1);
978
- buffer_json_member_add_string(wb, "help", ND_SD_JOURNAL_FUNCTION_DESCRIPTION);
979
- }
980
-
981
- if (!lqs->rq.data_only || lqs->rq.tail)
982
- buffer_json_member_add_uint64(wb, "last_modified", lqs->last_modified);
983
-
984
- facets_sort_and_reorder_keys(facets);
985
- facets_report(facets, wb, used_hashes_registry);
986
-
987
- wb->expires = now_realtime_sec() + (lqs->rq.data_only ? 3600 : 0);
988
- buffer_json_member_add_time_t(wb, "expires", wb->expires);
989
-
990
- buffer_json_member_add_object(wb, "_fstat_caching");
991
- {
992
- buffer_json_member_add_uint64(wb, "calls", fstat_thread_calls);
993
- buffer_json_member_add_uint64(wb, "cached", fstat_thread_cached_responses);
994
- }
995
- buffer_json_object_close(wb); // _fstat_caching
996
-
997
- if (lqs->rq.sampling) {
998
- buffer_json_member_add_object(wb, "_sampling");
999
- {
1000
- buffer_json_member_add_uint64(wb, "sampled", lqs->c.samples.sampled);
1001
- buffer_json_member_add_uint64(wb, "unsampled", lqs->c.samples.unsampled);
1002
- buffer_json_member_add_uint64(wb, "estimated", lqs->c.samples.estimated);
1003
- }
1004
- buffer_json_object_close(wb); // _sampling
1005
- }
1006
-
1007
- wb->content_type = CT_APPLICATION_JSON;
1008
- wb->response_code = HTTP_RESP_OK;
1009
- return wb->response_code;
1010
-}
1011
-
1012
-static void systemd_journal_register_transformations(LOGS_QUERY_STATUS *lqs)
1013
-{
1014
- FACETS *facets = lqs->facets;
1015
- LOGS_QUERY_REQUEST *rq = &lqs->rq;
1016
-
1017
- // ----------------------------------------------------------------------------------------------------------------
1018
- // register the fields in the order you want them on the dashboard
1019
-
1020
- facets_register_row_severity(facets, syslog_priority_to_facet_severity, NULL);
1021
-
1022
- facets_register_key_name(facets, "_HOSTNAME", rq->default_facet | FACET_KEY_OPTION_VISIBLE);
1023
-
1024
- facets_register_dynamic_key_name(
1025
- facets,
1026
- JOURNAL_KEY_ND_JOURNAL_PROCESS,
1027
- FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_VISIBLE,
1028
- nd_sd_journal_dynamic_row_id,
1029
- NULL);
1030
-
1031
- facets_register_key_name(
1032
- facets,
1033
- "MESSAGE",
1034
- FACET_KEY_OPTION_NEVER_FACET | FACET_KEY_OPTION_MAIN_TEXT | FACET_KEY_OPTION_VISIBLE | FACET_KEY_OPTION_FTS);
1035
-
1036
- facets_register_key_name_transformation(
1037
- facets,
1038
- "PRIORITY",
1039
- rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW | FACET_KEY_OPTION_EXPANDED_FILTER,
1040
- nd_sd_journal_transform_priority,
1041
- NULL);
1042
-
1043
- facets_register_key_name_transformation(
1044
- facets,
1045
- "SYSLOG_FACILITY",
1046
- rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW | FACET_KEY_OPTION_EXPANDED_FILTER,
1047
- nd_sd_journal_transform_syslog_facility,
1048
- NULL);
1049
-
1050
- facets_register_key_name_transformation(
1051
- facets, "ERRNO", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_errno, NULL);
1052
-
1053
- facets_register_key_name(facets, JOURNAL_KEY_ND_JOURNAL_FILE, FACET_KEY_OPTION_NEVER_FACET);
1054
-
1055
- facets_register_key_name(facets, "SYSLOG_IDENTIFIER", rq->default_facet);
1056
-
1057
- facets_register_key_name(facets, "UNIT", rq->default_facet);
1058
-
1059
- facets_register_key_name(facets, "USER_UNIT", rq->default_facet);
1060
-
1061
- facets_register_key_name_transformation(
1062
- facets,
1063
- "MESSAGE_ID",
1064
- rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW | FACET_KEY_OPTION_EXPANDED_FILTER,
1065
- nd_sd_journal_transform_message_id,
1066
- NULL);
1067
-
1068
- facets_register_key_name_transformation(
1069
- facets, "_BOOT_ID", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_boot_id, NULL);
1070
-
1071
- facets_register_key_name_transformation(
1072
- facets,
1073
- "_SYSTEMD_OWNER_UID",
1074
- rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1075
- nd_sd_journal_transform_uid,
1076
- NULL);
1077
-
1078
- facets_register_key_name_transformation(
1079
- facets, "_UID", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_uid, NULL);
1080
-
1081
- facets_register_key_name_transformation(
1082
- facets,
1083
- "OBJECT_SYSTEMD_OWNER_UID",
1084
- rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW,
1085
- nd_sd_journal_transform_uid,
1086
- NULL);
1087
-
1088
- facets_register_key_name_transformation(
1089
- facets, "OBJECT_UID", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_uid, NULL);
1090
-
1091
- facets_register_key_name_transformation(
1092
- facets, "_GID", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_gid, NULL);
1093
-
1094
- facets_register_key_name_transformation(
1095
- facets, "OBJECT_GID", rq->default_facet | FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_gid, NULL);
1096
-
1097
- facets_register_key_name_transformation(
1098
- facets, "_CAP_EFFECTIVE", FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_cap_effective, NULL);
1099
-
1100
- facets_register_key_name_transformation(
1101
- facets, "_AUDIT_LOGINUID", FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_uid, NULL);
1102
-
1103
- facets_register_key_name_transformation(
1104
- facets, "OBJECT_AUDIT_LOGINUID", FACET_KEY_OPTION_TRANSFORM_VIEW, nd_sd_journal_transform_uid, NULL);
1105
-
1106
- facets_register_key_name_transformation(
1107
- facets,
1108
- "_SOURCE_REALTIME_TIMESTAMP",
1109
- FACET_KEY_OPTION_TRANSFORM_VIEW,
1110
- nd_sd_journal_transform_timestamp_usec,
1111
- NULL);
1112
-}
1113
-
1114
-void function_systemd_journal(
1115
- const char *transaction,
1116
- char *function,
1117
- usec_t *stop_monotonic_ut,
1118
- bool *cancelled,
1119
- BUFFER *payload,
1120
- HTTP_ACCESS access __maybe_unused,
1121
- const char *source __maybe_unused,
1122
- void *data __maybe_unused)
1123
-{
1124
- fstat_thread_calls = 0;
1125
- fstat_thread_cached_responses = 0;
1126
-
1127
-#ifdef HAVE_SD_JOURNAL_RESTART_FIELDS
1128
- bool have_slice = true;
1129
-#else
1130
- bool have_slice = false;
1131
-#endif // HAVE_SD_JOURNAL_RESTART_FIELDS
1132
-
1133
- LOGS_QUERY_STATUS tmp_fqs = {
1134
- .facets = lqs_facets_create(
1135
- LQS_DEFAULT_ITEMS_PER_QUERY,
1136
- FACETS_OPTION_ALL_KEYS_FTS | FACETS_OPTION_HASH_IDS,
1137
- SYSTEMD_ALWAYS_VISIBLE_KEYS,
1138
- SYSTEMD_KEYS_INCLUDED_IN_FACETS,
1139
- SYSTEMD_KEYS_EXCLUDED_FROM_FACETS,
1140
- have_slice),
1141
-
1142
- .rq = LOGS_QUERY_REQUEST_DEFAULTS(transaction, LQS_DEFAULT_SLICE_MODE, JOURNAL_DEFAULT_DIRECTION),
1143
-
1144
- .cancelled = cancelled,
1145
- .stop_monotonic_ut = stop_monotonic_ut,
1146
- };
1147
- LOGS_QUERY_STATUS *lqs = &tmp_fqs;
1148
-
1149
- CLEAN_BUFFER *wb = lqs_create_output_buffer();
1150
-
1151
- // ------------------------------------------------------------------------
1152
- // parse the parameters
1153
-
1154
- if (lqs_request_parse_and_validate(lqs, wb, function, payload, have_slice, "PRIORITY")) {
1155
- systemd_journal_register_transformations(lqs);
1156
-
1157
- // ------------------------------------------------------------------------
1158
- // add versions to the response
1159
-
1160
- buffer_json_journal_versions(wb);
1161
-
1162
- // ------------------------------------------------------------------------
1163
- // run the request
1164
-
1165
- if (lqs->rq.info)
1166
- lqs_info_response(wb, lqs->facets);
1167
- else {
1168
- nd_sd_journal_query(wb, lqs);
1169
- if (wb->response_code == HTTP_RESP_OK)
1170
- buffer_json_finalize(wb);
1171
- }
1172
- }
1173
-
1174
- netdata_mutex_lock(&stdout_mutex);
1175
- pluginsd_function_result_to_stdout(transaction, wb);
1176
- netdata_mutex_unlock(&stdout_mutex);
1177
-
1178
- lqs_cleanup(lqs);
1179
-}
src/collectors/systemd-journal.plugin/systemd-main.c
deleted
-126
@@ -1,126 +0,0 @@
1
-// SPDX-License-Identifier: GPL-3.0-or-later
2
-
3
-#include "systemd-internals.h"
4
-#include "libnetdata/required_dummies.h"
5
-
6
-#define ND_SD_JOURNAL_WORKER_THREADS 5
7
-
8
-netdata_mutex_t stdout_mutex;
9
-
10
-static void __attribute__((constructor)) init_mutex(void) {
11
- netdata_mutex_init(&stdout_mutex);
12
-}
13
-
14
-static void __attribute__((destructor)) destroy_mutex(void) {
15
- netdata_mutex_destroy(&stdout_mutex);
16
-}
17
-
18
-static bool plugin_should_exit = false;
19
-
20
-static bool journal_data_directories_exist()
21
-{
22
- struct stat st;
23
- for (unsigned i = 0; i < MAX_JOURNAL_DIRECTORIES && journal_directories[i].path; i++) {
24
- if ((stat(string2str(journal_directories[i].path), &st) == 0) && S_ISDIR(st.st_mode))
25
- return true;
26
- }
27
- return false;
28
-}
29
-
30
-int main(int argc __maybe_unused, char **argv __maybe_unused)
31
-{
32
- nd_thread_tag_set("sd-jrnl.plugin");
33
- nd_log_initialize_for_external_plugins("systemd-journal.plugin");
34
- netdata_threads_init_for_external_plugins(0);
35
-
36
- netdata_configured_host_prefix = getenv("NETDATA_HOST_PREFIX");
37
- if (verify_netdata_host_prefix(true) == -1)
38
- exit(1);
39
-
40
- // ------------------------------------------------------------------------
41
- // initialization
42
-
43
- nd_sd_journal_annotations_init();
44
- nd_journal_init_files_and_directories();
45
-
46
- if (!journal_data_directories_exist()) {
47
- nd_log_collector(NDLP_INFO, "unable to locate journal data directories. Exiting...");
48
- fprintf(stdout, "DISABLE\n");
49
- fflush(stdout);
50
- exit(0);
51
- }
52
-
53
- // ------------------------------------------------------------------------
54
- // debug
55
-
56
- if (argc == 2 && strcmp(argv[1], "debug") == 0) {
57
- nd_journal_files_registry_update();
58
-
59
- bool cancelled = false;
60
- usec_t stop_monotonic_ut = now_monotonic_usec() + 600 * USEC_PER_SEC;
61
- char buf[] =
62
- "systemd-journal after:-8640000 before:0 direction:backward last:200 data_only:false slice:true facets: source:all";
63
- function_systemd_journal("123", buf, &stop_monotonic_ut, &cancelled, NULL, HTTP_ACCESS_ALL, NULL, NULL);
64
- exit(1);
65
- }
66
-
67
- // ------------------------------------------------------------------------
68
- // watcher thread
69
-
70
- nd_thread_create("SDWATCH", NETDATA_THREAD_OPTION_DONT_LOG, nd_journal_watcher_main, NULL);
71
-
72
- // ------------------------------------------------------------------------
73
- // the event loop for functions
74
-
75
- struct functions_evloop_globals *wg =
76
- functions_evloop_init(ND_SD_JOURNAL_WORKER_THREADS, "SDJ", &stdout_mutex, &plugin_should_exit, NULL);
77
-
78
- functions_evloop_add_function(
79
- wg, ND_SD_JOURNAL_FUNCTION_NAME, function_systemd_journal, ND_SD_JOURNAL_DEFAULT_TIMEOUT, NULL);
80
-
81
- nd_systemd_journal_dyncfg_init(wg);
82
-
83
- // ------------------------------------------------------------------------
84
- // register functions to netdata
85
-
86
- netdata_mutex_lock(&stdout_mutex);
87
-
88
- fprintf(
89
- stdout,
90
- PLUGINSD_KEYWORD_FUNCTION " GLOBAL \"%s\" %d \"%s\" \"logs\" " HTTP_ACCESS_FORMAT " %d\n",
91
- ND_SD_JOURNAL_FUNCTION_NAME,
92
- ND_SD_JOURNAL_DEFAULT_TIMEOUT,
93
- ND_SD_JOURNAL_FUNCTION_DESCRIPTION,
94
- (HTTP_ACCESS_FORMAT_CAST)(HTTP_ACCESS_SIGNED_ID | HTTP_ACCESS_SAME_SPACE | HTTP_ACCESS_SENSITIVE_DATA),
95
- RRDFUNCTIONS_PRIORITY_DEFAULT);
96
-
97
- fflush(stdout);
98
- netdata_mutex_unlock(&stdout_mutex);
99
-
100
- // ------------------------------------------------------------------------
101
-
102
- usec_t send_newline_ut = 0;
103
- usec_t since_last_scan_ut =
104
- ND_SD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC * 2; // something big to trigger scanning at start
105
- const bool tty = isatty(fileno(stdout)) == 1;
106
-
107
- heartbeat_t hb;
108
- heartbeat_init(&hb, USEC_PER_SEC);
109
- while (!__atomic_load_n(&plugin_should_exit, __ATOMIC_ACQUIRE)) {
110
- if (since_last_scan_ut > ND_SD_JOURNAL_ALL_FILES_SCAN_EVERY_USEC) {
111
- nd_journal_files_registry_update();
112
- since_last_scan_ut = 0;
113
- }
114
-
115
- usec_t dt_ut = heartbeat_next(&hb);
116
- since_last_scan_ut += dt_ut;
117
- send_newline_ut += dt_ut;
118
-
119
- if (!tty && send_newline_ut > USEC_PER_SEC) {
120
- send_newline_and_flush(&stdout_mutex);
121
- send_newline_ut = 0;
122
- }
123
- }
124
-
125
- exit(0);
126
-}
src/crates/.gitignore
new
+2
@@ -0,0 +1,2 @@
1
+target/
2
+.idea/
src/crates/.helix/languages.toml
new
+5
@@ -0,0 +1,5 @@
1
+[[language]]
2
+name = "rust"
3
+
4
+[language-server.rust-analyzer.config.cargo]
5
+features = [ "allocative" ]
src/crates/Cargo.lock
new
+4073
@@ -0,0 +1,4073 @@
1
+# This file is automatically @generated by Cargo.
2
+# It is not intended for manual editing.
3
+version = 4
4
+
5
+[[package]]
6
+name = "adler2"
7
+version = "2.0.1"
8
+source = "registry+https://github.com/rust-lang/crates.io-index"
9
+checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
10
+
11
+[[package]]
12
+name = "ahash"
13
+version = "0.8.12"
14
+source = "registry+https://github.com/rust-lang/crates.io-index"
15
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
16
+dependencies = [
17
+ "cfg-if",
18
+ "getrandom 0.3.4",
19
+ "once_cell",
20
+ "version_check",
21
+ "zerocopy 0.8.27",
22
+]
23
+
24
+[[package]]
25
+name = "aho-corasick"
26
+version = "1.1.4"
27
+source = "registry+https://github.com/rust-lang/crates.io-index"
28
+checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301"
29
+dependencies = [
30
+ "memchr",
31
+]
32
+
33
+[[package]]
34
+name = "allocative"
35
+version = "0.3.4"
36
+source = "registry+https://github.com/rust-lang/crates.io-index"
37
+checksum = "8fac2ce611db8b8cee9b2aa886ca03c924e9da5e5295d0dbd0526e5d0b0710f7"
38
+dependencies = [
39
+ "allocative_derive",
40
+ "ctor",
41
+]
42
+
43
+[[package]]
44
+name = "allocative_derive"
45
+version = "0.3.3"
46
+source = "registry+https://github.com/rust-lang/crates.io-index"
47
+checksum = "fe233a377643e0fc1a56421d7c90acdec45c291b30345eb9f08e8d0ddce5a4ab"
48
+dependencies = [
49
+ "proc-macro2",
50
+ "quote",
51
+ "syn 2.0.110",
52
+]
53
+
54
+[[package]]
55
+name = "allocator-api2"
56
+version = "0.2.21"
57
+source = "registry+https://github.com/rust-lang/crates.io-index"
58
+checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
59
+
60
+[[package]]
61
+name = "android_system_properties"
62
+version = "0.1.5"
63
+source = "registry+https://github.com/rust-lang/crates.io-index"
64
+checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
65
+dependencies = [
66
+ "libc",
67
+]
68
+
69
+[[package]]
70
+name = "anstream"
71
+version = "0.6.21"
72
+source = "registry+https://github.com/rust-lang/crates.io-index"
73
+checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a"
74
+dependencies = [
75
+ "anstyle",
76
+ "anstyle-parse",
77
+ "anstyle-query",
78
+ "anstyle-wincon",
79
+ "colorchoice",
80
+ "is_terminal_polyfill",
81
+ "utf8parse",
82
+]
83
+
84
+[[package]]
85
+name = "anstyle"
86
+version = "1.0.13"
87
+source = "registry+https://github.com/rust-lang/crates.io-index"
88
+checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
89
+
90
+[[package]]
91
+name = "anstyle-parse"
92
+version = "0.2.7"
93
+source = "registry+https://github.com/rust-lang/crates.io-index"
94
+checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
95
+dependencies = [
96
+ "utf8parse",
97
+]
98
+
99
+[[package]]
100
+name = "anstyle-query"
101
+version = "1.1.5"
102
+source = "registry+https://github.com/rust-lang/crates.io-index"
103
+checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
104
+dependencies = [
105
+ "windows-sys 0.61.2",
106
+]
107
+
108
+[[package]]
109
+name = "anstyle-wincon"
110
+version = "3.0.11"
111
+source = "registry+https://github.com/rust-lang/crates.io-index"
112
+checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
113
+dependencies = [
114
+ "anstyle",
115
+ "once_cell_polyfill",
116
+ "windows-sys 0.61.2",
117
+]
118
+
119
+[[package]]
120
+name = "anyhow"
121
+version = "1.0.100"
122
+source = "registry+https://github.com/rust-lang/crates.io-index"
123
+checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
124
+
125
+[[package]]
126
+name = "arc-swap"
127
+version = "1.7.1"
128
+source = "registry+https://github.com/rust-lang/crates.io-index"
129
+checksum = "69f7f8c3906b62b754cd5326047894316021dcfe5a194c8ea52bdd94934a3457"
130
+
131
+[[package]]
132
+name = "async-channel"
133
+version = "2.5.0"
134
+source = "registry+https://github.com/rust-lang/crates.io-index"
135
+checksum = "924ed96dd52d1b75e9c1a3e6275715fd320f5f9439fb5a4a11fa51f4221158d2"
136
+dependencies = [
137
+ "concurrent-queue",
138
+ "event-listener-strategy",
139
+ "futures-core",
140
+ "pin-project-lite",
141
+]
142
+
143
+[[package]]
144
+name = "async-stream"
145
+version = "0.3.6"
146
+source = "registry+https://github.com/rust-lang/crates.io-index"
147
+checksum = "0b5a71a6f37880a80d1d7f19efd781e4b5de42c88f0722cc13bcb6cc2cfe8476"
148
+dependencies = [
149
+ "async-stream-impl",
150
+ "futures-core",
151
+ "pin-project-lite",
152
+]
153
+
154
+[[package]]
155
+name = "async-stream-impl"
156
+version = "0.3.6"
157
+source = "registry+https://github.com/rust-lang/crates.io-index"
158
+checksum = "c7c24de15d275a1ecfd47a380fb4d5ec9bfe0933f309ed5e705b775596a3574d"
159
+dependencies = [
160
+ "proc-macro2",
161
+ "quote",
162
+ "syn 2.0.110",
163
+]
164
+
165
+[[package]]
166
+name = "async-task"
167
+version = "4.7.1"
168
+source = "registry+https://github.com/rust-lang/crates.io-index"
169
+checksum = "8b75356056920673b02621b35afd0f7dda9306d03c79a30f5c56c44cf256e3de"
170
+
171
+[[package]]
172
+name = "async-trait"
173
+version = "0.1.89"
174
+source = "registry+https://github.com/rust-lang/crates.io-index"
175
+checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
176
+dependencies = [
177
+ "proc-macro2",
178
+ "quote",
179
+ "syn 2.0.110",
180
+]
181
+
182
+[[package]]
183
+name = "atoi"
184
+version = "2.0.0"
185
+source = "registry+https://github.com/rust-lang/crates.io-index"
186
+checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528"
187
+dependencies = [
188
+ "num-traits",
189
+]
190
+
191
+[[package]]
192
+name = "atomic-waker"
193
+version = "1.1.2"
194
+source = "registry+https://github.com/rust-lang/crates.io-index"
195
+checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
196
+
197
+[[package]]
198
+name = "atty"
199
+version = "0.2.14"
200
+source = "registry+https://github.com/rust-lang/crates.io-index"
201
+checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
202
+dependencies = [
203
+ "hermit-abi 0.1.19",
204
+ "libc",
205
+ "winapi",
206
+]
207
+
208
+[[package]]
209
+name = "autocfg"
210
+version = "1.5.0"
211
+source = "registry+https://github.com/rust-lang/crates.io-index"
212
+checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
213
+
214
+[[package]]
215
+name = "axum"
216
+version = "0.7.9"
217
+source = "registry+https://github.com/rust-lang/crates.io-index"
218
+checksum = "edca88bc138befd0323b20752846e6587272d3b03b0343c8ea28a6f819e6e71f"
219
+dependencies = [
220
+ "async-trait",
221
+ "axum-core 0.4.5",
222
+ "bytes",
223
+ "futures-util",
224
+ "http",
225
+ "http-body",
226
+ "http-body-util",
227
+ "itoa",
228
+ "matchit 0.7.3",
229
+ "memchr",
230
+ "mime",
231
+ "percent-encoding",
232
+ "pin-project-lite",
233
+ "rustversion",
234
+ "serde",
235
+ "sync_wrapper",
236
+ "tower 0.5.2",
237
+ "tower-layer",
238
+ "tower-service",
239
+]
240
+
241
+[[package]]
242
+name = "axum"
243
+version = "0.8.6"
244
+source = "registry+https://github.com/rust-lang/crates.io-index"
245
+checksum = "8a18ed336352031311f4e0b4dd2ff392d4fbb370777c9d18d7fc9d7359f73871"
246
+dependencies = [
247
+ "axum-core 0.5.5",
248
+ "bytes",
249
+ "futures-util",
250
+ "http",
251
+ "http-body",
252
+ "http-body-util",
253
+ "itoa",
254
+ "matchit 0.8.4",
255
+ "memchr",
256
+ "mime",
257
+ "percent-encoding",
258
+ "pin-project-lite",
259
+ "serde_core",
260
+ "sync_wrapper",
261
+ "tower 0.5.2",
262
+ "tower-layer",
263
+ "tower-service",
264
+]
265
+
266
+[[package]]
267
+name = "axum-core"
268
+version = "0.4.5"
269
+source = "registry+https://github.com/rust-lang/crates.io-index"
270
+checksum = "09f2bd6146b97ae3359fa0cc6d6b376d9539582c7b4220f041a33ec24c226199"
271
+dependencies = [
272
+ "async-trait",
273
+ "bytes",
274
+ "futures-util",
275
+ "http",
276
+ "http-body",
277
+ "http-body-util",
278
+ "mime",
279
+ "pin-project-lite",
280
+ "rustversion",
281
+ "sync_wrapper",
282
+ "tower-layer",
283
+ "tower-service",
284
+]
285
+
286
+[[package]]
287
+name = "axum-core"
288
+version = "0.5.5"
289
+source = "registry+https://github.com/rust-lang/crates.io-index"
290
+checksum = "59446ce19cd142f8833f856eb31f3eb097812d1479ab224f54d72428ca21ea22"
291
+dependencies = [
292
+ "bytes",
293
+ "futures-core",
294
+ "http",
295
+ "http-body",
296
+ "http-body-util",
297
+ "mime",
298
+ "pin-project-lite",
299
+ "sync_wrapper",
300
+ "tower-layer",
301
+ "tower-service",
302
+]
303
+
304
+[[package]]
305
+name = "base64"
306
+version = "0.21.7"
307
+source = "registry+https://github.com/rust-lang/crates.io-index"
308
+checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
309
+
310
+[[package]]
311
+name = "base64"
312
+version = "0.22.1"
313
+source = "registry+https://github.com/rust-lang/crates.io-index"
314
+checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
315
+
316
+[[package]]
317
+name = "bincode"
318
+version = "1.3.3"
319
+source = "registry+https://github.com/rust-lang/crates.io-index"
320
+checksum = "b1f45e9417d87227c7a56d22e471c6206462cba514c7590c09aff4cf6d1ddcad"
321
+dependencies = [
322
+ "serde",
323
+]
324
+
325
+[[package]]
326
+name = "bitflags"
327
+version = "1.3.2"
328
+source = "registry+https://github.com/rust-lang/crates.io-index"
329
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
330
+
331
+[[package]]
332
+name = "bitflags"
333
+version = "2.10.0"
334
+source = "registry+https://github.com/rust-lang/crates.io-index"
335
+checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3"
336
+
337
+[[package]]
338
+name = "bridge"
339
+version = "0.1.0"
340
+dependencies = [
341
+ "async-trait",
342
+ "console-subscriber",
343
+ "futures",
344
+ "netdata-plugin-error",
345
+ "netdata-plugin-protocol",
346
+ "netdata-plugin-schema",
347
+ "schemars",
348
+ "serde",
349
+ "serde_json",
350
+ "tokio",
351
+ "tokio-util",
352
+ "tracing",
353
+ "tracing-subscriber",
354
+]
355
+
356
+[[package]]
357
+name = "bumpalo"
358
+version = "3.19.0"
359
+source = "registry+https://github.com/rust-lang/crates.io-index"
360
+checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
361
+
362
+[[package]]
363
+name = "bytemuck"
364
+version = "1.24.0"
365
+source = "registry+https://github.com/rust-lang/crates.io-index"
366
+checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4"
367
+
368
+[[package]]
369
+name = "byteorder"
370
+version = "1.5.0"
371
+source = "registry+https://github.com/rust-lang/crates.io-index"
372
+checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
373
+
374
+[[package]]
375
+name = "bytes"
376
+version = "1.10.1"
377
+source = "registry+https://github.com/rust-lang/crates.io-index"
378
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
379
+
380
+[[package]]
381
+name = "bytesize"
382
+version = "1.3.3"
383
+source = "registry+https://github.com/rust-lang/crates.io-index"
384
+checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
385
+
386
+[[package]]
387
+name = "bytesize-serde"
388
+version = "0.2.1"
389
+source = "registry+https://github.com/rust-lang/crates.io-index"
390
+checksum = "86d1eb2fd2668859e9785b99700c66b7ea9fdeda35d99f95827a4e5493440178"
391
+dependencies = [
392
+ "bytesize",
393
+ "serde",
394
+]
395
+
396
+[[package]]
397
+name = "cc"
398
+version = "1.2.45"
399
+source = "registry+https://github.com/rust-lang/crates.io-index"
400
+checksum = "35900b6c8d709fb1d854671ae27aeaa9eec2f8b01b364e1619a40da3e6fe2afe"
401
+dependencies = [
402
+ "find-msvc-tools",
403
+ "jobserver",
404
+ "libc",
405
+ "shlex",
406
+]
407
+
408
+[[package]]
409
+name = "cfg-if"
410
+version = "1.0.4"
411
+source = "registry+https://github.com/rust-lang/crates.io-index"
412
+checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801"
413
+
414
+[[package]]
415
+name = "cfg_aliases"
416
+version = "0.2.1"
417
+source = "registry+https://github.com/rust-lang/crates.io-index"
418
+checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724"
419
+
420
+[[package]]
421
+name = "chrono"
422
+version = "0.4.42"
423
+source = "registry+https://github.com/rust-lang/crates.io-index"
424
+checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
425
+dependencies = [
426
+ "iana-time-zone",
427
+ "js-sys",
428
+ "num-traits",
429
+ "serde",
430
+ "wasm-bindgen",
431
+ "windows-link",
432
+]
433
+
434
+[[package]]
435
+name = "clap"
436
+version = "4.5.51"
437
+source = "registry+https://github.com/rust-lang/crates.io-index"
438
+checksum = "4c26d721170e0295f191a69bd9a1f93efcdb0aff38684b61ab5750468972e5f5"
439
+dependencies = [
440
+ "clap_builder",
441
+ "clap_derive",
442
+]
443
+
444
+[[package]]
445
+name = "clap_builder"
446
+version = "4.5.51"
447
+source = "registry+https://github.com/rust-lang/crates.io-index"
448
+checksum = "75835f0c7bf681bfd05abe44e965760fea999a5286c6eb2d59883634fd02011a"
449
+dependencies = [
450
+ "anstream",
451
+ "anstyle",
452
+ "clap_lex",
453
+ "strsim 0.11.1",
454
+]
455
+
456
+[[package]]
457
+name = "clap_derive"
458
+version = "4.5.49"
459
+source = "registry+https://github.com/rust-lang/crates.io-index"
460
+checksum = "2a0b5487afeab2deb2ff4e03a807ad1a03ac532ff5a2cee5d86884440c7f7671"
461
+dependencies = [
462
+ "heck",
463
+ "proc-macro2",
464
+ "quote",
465
+ "syn 2.0.110",
466
+]
467
+
468
+[[package]]
469
+name = "clap_lex"
470
+version = "0.7.6"
471
+source = "registry+https://github.com/rust-lang/crates.io-index"
472
+checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d"
473
+
474
+[[package]]
475
+name = "cmsketch"
476
+version = "0.2.4"
477
+source = "registry+https://github.com/rust-lang/crates.io-index"
478
+checksum = "d7ee2cfacbd29706479902b06d75ad8f1362900836aa32799eabc7e004bfd854"
479
+
480
+[[package]]
481
+name = "colorchoice"
482
+version = "1.0.4"
483
+source = "registry+https://github.com/rust-lang/crates.io-index"
484
+checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
485
+
486
+[[package]]
487
+name = "concurrent-queue"
488
+version = "2.5.0"
489
+source = "registry+https://github.com/rust-lang/crates.io-index"
490
+checksum = "4ca0197aee26d1ae37445ee532fefce43251d24cc7c166799f4d46817f1d3973"
491
+dependencies = [
492
+ "crossbeam-utils",
493
+]
494
+
495
+[[package]]
496
+name = "console-api"
497
+version = "0.8.1"
498
+source = "registry+https://github.com/rust-lang/crates.io-index"
499
+checksum = "8030735ecb0d128428b64cd379809817e620a40e5001c54465b99ec5feec2857"
500
+dependencies = [
501
+ "futures-core",
502
+ "prost 0.13.5",
503
+ "prost-types",
504
+ "tonic 0.12.3",
505
+ "tracing-core",
506
+]
507
+
508
+[[package]]
509
+name = "console-subscriber"
510
+version = "0.4.1"
511
+source = "registry+https://github.com/rust-lang/crates.io-index"
512
+checksum = "6539aa9c6a4cd31f4b1c040f860a1eac9aa80e7df6b05d506a6e7179936d6a01"
513
+dependencies = [
514
+ "console-api",
515
+ "crossbeam-channel",
516
+ "crossbeam-utils",
517
+ "futures-task",
518
+ "hdrhistogram",
519
+ "humantime",
520
+ "hyper-util",
521
+ "prost 0.13.5",
522
+ "prost-types",
523
+ "serde",
524
+ "serde_json",
525
+ "thread_local",
526
+ "tokio",
527
+ "tokio-stream",
528
+ "tonic 0.12.3",
529
+ "tracing",
530
+ "tracing-core",
531
+ "tracing-subscriber",
532
+]
533
+
534
+[[package]]
535
+name = "const-hex"
536
+version = "1.17.0"
537
+source = "registry+https://github.com/rust-lang/crates.io-index"
538
+checksum = "3bb320cac8a0750d7f25280aa97b09c26edfe161164238ecbbb31092b079e735"
539
+dependencies = [
540
+ "cfg-if",
541
+ "cpufeatures",
542
+ "proptest",
543
+ "serde_core",
544
+]
545
+
546
+[[package]]
547
+name = "core-foundation-sys"
548
+version = "0.8.7"
549
+source = "registry+https://github.com/rust-lang/crates.io-index"
550
+checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
551
+
552
+[[package]]
553
+name = "core_affinity"
554
+version = "0.8.3"
555
+source = "registry+https://github.com/rust-lang/crates.io-index"
556
+checksum = "a034b3a7b624016c6e13f5df875747cc25f884156aad2abd12b6c46797971342"
557
+dependencies = [
558
+ "libc",
559
+ "num_cpus",
560
+ "winapi",
561
+]
562
+
563
+[[package]]
564
+name = "cpufeatures"
565
+version = "0.2.17"
566
+source = "registry+https://github.com/rust-lang/crates.io-index"
567
+checksum = "59ed5838eebb26a2bb2e58f6d5b5316989ae9d08bab10e0e6d103e656d1b0280"
568
+dependencies = [
569
+ "libc",
570
+]
571
+
572
+[[package]]
573
+name = "crc32fast"
574
+version = "1.5.0"
575
+source = "registry+https://github.com/rust-lang/crates.io-index"
576
+checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
577
+dependencies = [
578
+ "cfg-if",
579
+]
580
+
581
+[[package]]
582
+name = "crossbeam-channel"
583
+version = "0.5.15"
584
+source = "registry+https://github.com/rust-lang/crates.io-index"
585
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
586
+dependencies = [
587
+ "crossbeam-utils",
588
+]
589
+
590
+[[package]]
591
+name = "crossbeam-deque"
592
+version = "0.8.6"
593
+source = "registry+https://github.com/rust-lang/crates.io-index"
594
+checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51"
595
+dependencies = [
596
+ "crossbeam-epoch",
597
+ "crossbeam-utils",
598
+]
599
+
600
+[[package]]
601
+name = "crossbeam-epoch"
602
+version = "0.9.18"
603
+source = "registry+https://github.com/rust-lang/crates.io-index"
604
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
605
+dependencies = [
606
+ "crossbeam-utils",
607
+]
608
+
609
+[[package]]
610
+name = "crossbeam-utils"
611
+version = "0.8.21"
612
+source = "registry+https://github.com/rust-lang/crates.io-index"
613
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
614
+
615
+[[package]]
616
+name = "ctor"
617
+version = "0.1.26"
618
+source = "registry+https://github.com/rust-lang/crates.io-index"
619
+checksum = "6d2301688392eb071b0bf1a37be05c469d3cc4dbbd95df672fe28ab021e6a096"
620
+dependencies = [
621
+ "quote",
622
+ "syn 1.0.109",
623
+]
624
+
625
+[[package]]
626
+name = "darling"
627
+version = "0.14.4"
628
+source = "registry+https://github.com/rust-lang/crates.io-index"
629
+checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850"
630
+dependencies = [
631
+ "darling_core",
632
+ "darling_macro",
633
+]
634
+
635
+[[package]]
636
+name = "darling_core"
637
+version = "0.14.4"
638
+source = "registry+https://github.com/rust-lang/crates.io-index"
639
+checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0"
640
+dependencies = [
641
+ "fnv",
642
+ "ident_case",
643
+ "proc-macro2",
644
+ "quote",
645
+ "strsim 0.10.0",
646
+ "syn 1.0.109",
647
+]
648
+
649
+[[package]]
650
+name = "darling_macro"
651
+version = "0.14.4"
652
+source = "registry+https://github.com/rust-lang/crates.io-index"
653
+checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e"
654
+dependencies = [
655
+ "darling_core",
656
+ "quote",
657
+ "syn 1.0.109",
658
+]
659
+
660
+[[package]]
661
+name = "displaydoc"
662
+version = "0.2.5"
663
+source = "registry+https://github.com/rust-lang/crates.io-index"
664
+checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0"
665
+dependencies = [
666
+ "proc-macro2",
667
+ "quote",
668
+ "syn 2.0.110",
669
+]
670
+
671
+[[package]]
672
+name = "downcast-rs"
673
+version = "1.2.1"
674
+source = "registry+https://github.com/rust-lang/crates.io-index"
675
+checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2"
676
+
677
+[[package]]
678
+name = "dyn-clone"
679
+version = "1.0.20"
680
+source = "registry+https://github.com/rust-lang/crates.io-index"
681
+checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555"
682
+
683
+[[package]]
684
+name = "either"
685
+version = "1.15.0"
686
+source = "registry+https://github.com/rust-lang/crates.io-index"
687
+checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
688
+
689
+[[package]]
690
+name = "equivalent"
691
+version = "1.0.2"
692
+source = "registry+https://github.com/rust-lang/crates.io-index"
693
+checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
694
+
695
+[[package]]
696
+name = "errno"
697
+version = "0.3.14"
698
+source = "registry+https://github.com/rust-lang/crates.io-index"
699
+checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
700
+dependencies = [
701
+ "libc",
702
+ "windows-sys 0.61.2",
703
+]
704
+
705
+[[package]]
706
+name = "event-listener"
707
+version = "5.4.1"
708
+source = "registry+https://github.com/rust-lang/crates.io-index"
709
+checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab"
710
+dependencies = [
711
+ "concurrent-queue",
712
+ "parking",
713
+ "pin-project-lite",
714
+]
715
+
716
+[[package]]
717
+name = "event-listener-strategy"
718
+version = "0.5.4"
719
+source = "registry+https://github.com/rust-lang/crates.io-index"
720
+checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93"
721
+dependencies = [
722
+ "event-listener",
723
+ "pin-project-lite",
724
+]
725
+
726
+[[package]]
727
+name = "fastant"
728
+version = "0.1.10"
729
+source = "registry+https://github.com/rust-lang/crates.io-index"
730
+checksum = "62bf7fa928ce0c4a43bd6e7d1235318fc32ac3a3dea06a2208c44e729449471a"
731
+dependencies = [
732
+ "small_ctor",
733
+ "web-time",
734
+]
735
+
736
+[[package]]
737
+name = "fastrand"
738
+version = "2.3.0"
739
+source = "registry+https://github.com/rust-lang/crates.io-index"
740
+checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
741
+
742
+[[package]]
743
+name = "find-msvc-tools"
744
+version = "0.1.4"
745
+source = "registry+https://github.com/rust-lang/crates.io-index"
746
+checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127"
747
+
748
+[[package]]
749
+name = "flate2"
750
+version = "1.1.5"
751
+source = "registry+https://github.com/rust-lang/crates.io-index"
752
+checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb"
753
+dependencies = [
754
+ "crc32fast",
755
+ "miniz_oxide",
756
+]
757
+
758
+[[package]]
759
+name = "flatten-serde-json"
760
+version = "1.22.1"
761
+source = "git+https://github.com/meilisearch/meilisearch?tag=v1.22.1#077ec2ab11bb4daefcb57f89eab9cff16e075fdc"
762
+dependencies = [
763
+ "serde_json",
764
+]
765
+
766
+[[package]]
767
+name = "flatten_otel"
768
+version = "0.1.0"
769
+dependencies = [
770
+ "flatten-serde-json",
771
+ "opentelemetry-proto",
772
+ "serde_json",
773
+ "tracing",
774
+]
775
+
776
+[[package]]
777
+name = "flume"
778
+version = "0.11.1"
779
+source = "registry+https://github.com/rust-lang/crates.io-index"
780
+checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095"
781
+dependencies = [
782
+ "futures-core",
783
+ "futures-sink",
784
+ "nanorand",
785
+ "spin",
786
+]
787
+
788
+[[package]]
789
+name = "fnv"
790
+version = "1.0.7"
791
+source = "registry+https://github.com/rust-lang/crates.io-index"
792
+checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
793
+
794
+[[package]]
795
+name = "foldhash"
796
+version = "0.1.5"
797
+source = "registry+https://github.com/rust-lang/crates.io-index"
798
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
799
+
800
+[[package]]
801
+name = "foldhash"
802
+version = "0.2.0"
803
+source = "registry+https://github.com/rust-lang/crates.io-index"
804
+checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb"
805
+
806
+[[package]]
807
+name = "form_urlencoded"
808
+version = "1.2.2"
809
+source = "registry+https://github.com/rust-lang/crates.io-index"
810
+checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf"
811
+dependencies = [
812
+ "percent-encoding",
813
+]
814
+
815
+[[package]]
816
+name = "foundation"
817
+version = "0.1.0"
818
+dependencies = [
819
+ "tokio",
820
+]
821
+
822
+[[package]]
823
+name = "foyer"
824
+version = "0.20.1"
825
+source = "registry+https://github.com/rust-lang/crates.io-index"
826
+checksum = "50b7b23bdff0e7fecbef83438ce9389c352743ea0b8bda44fabca354f21b19a9"
827
+dependencies = [
828
+ "equivalent",
829
+ "foyer-common",
830
+ "foyer-memory",
831
+ "foyer-storage",
832
+ "madsim-tokio",
833
+ "mixtrics",
834
+ "pin-project",
835
+ "serde",
836
+ "thiserror",
837
+ "tokio",
838
+ "tracing",
839
+]
840
+
841
+[[package]]
842
+name = "foyer-common"
843
+version = "0.20.1"
844
+source = "registry+https://github.com/rust-lang/crates.io-index"
845
+checksum = "1a72f40b0c110e4233f46df0589bf1b2b3b5ed04e4cf504a582374d7548c9c05"
846
+dependencies = [
847
+ "bincode",
848
+ "bytes",
849
+ "cfg-if",
850
+ "itertools",
851
+ "madsim-tokio",
852
+ "mixtrics",
853
+ "parking_lot",
854
+ "pin-project",
855
+ "serde",
856
+ "thiserror",
857
+ "tokio",
858
+ "twox-hash",
859
+]
860
+
861
+[[package]]
862
+name = "foyer-intrusive-collections"
863
+version = "0.10.0-dev"
864
+source = "registry+https://github.com/rust-lang/crates.io-index"
865
+checksum = "6e4fee46bea69e0596130e3210e65d3424e0ac1e6df3bde6636304bdf1ca4a3b"
866
+dependencies = [
867
+ "memoffset",
868
+]
869
+
870
+[[package]]
871
+name = "foyer-memory"
872
+version = "0.20.1"
873
+source = "registry+https://github.com/rust-lang/crates.io-index"
874
+checksum = "6b1b23cefdcd78b4d729679d75bb6fb96a57f7dd94f141dcc373a2db1c04fbaa"
875
+dependencies = [
876
+ "arc-swap",
877
+ "bitflags 2.10.0",
878
+ "cmsketch",
879
+ "equivalent",
880
+ "foyer-common",
881
+ "foyer-intrusive-collections",
882
+ "hashbrown 0.15.5",
883
+ "itertools",
884
+ "madsim-tokio",
885
+ "mixtrics",
886
+ "parking_lot",
887
+ "pin-project",
888
+ "serde",
889
+ "thiserror",
890
+ "tokio",
891
+ "tracing",
892
+]
893
+
894
+[[package]]
895
+name = "foyer-storage"
896
+version = "0.20.1"
897
+source = "registry+https://github.com/rust-lang/crates.io-index"
898
+checksum = "f1ea5872e0073e3c37bec0aa99923f03bcec3951a056c77cfdfcc8001a370f9a"
899
+dependencies = [
900
+ "allocator-api2",
901
+ "anyhow",
902
+ "bytes",
903
+ "core_affinity",
904
+ "equivalent",
905
+ "fastant",
906
+ "flume",
907
+ "foyer-common",
908
+ "foyer-memory",
909
+ "fs4",
910
+ "futures-core",
911
+ "futures-util",
912
+ "hashbrown 0.15.5",
913
+ "io-uring",
914
+ "itertools",
915
+ "libc",
916
+ "lz4",
917
+ "madsim-tokio",
918
+ "parking_lot",
919
+ "pin-project",
920
+ "rand 0.9.2",
921
+ "serde",
922
+ "thiserror",
923
+ "tokio",
924
+ "tracing",
925
+ "twox-hash",
926
+ "zstd",
927
+]
928
+
929
+[[package]]
930
+name = "fs4"
931
+version = "0.13.1"
932
+source = "registry+https://github.com/rust-lang/crates.io-index"
933
+checksum = "8640e34b88f7652208ce9e88b1a37a2ae95227d84abec377ccd3c5cfeb141ed4"
934
+dependencies = [
935
+ "rustix",
936
+ "windows-sys 0.59.0",
937
+]
938
+
939
+[[package]]
940
+name = "fsevent-sys"
941
+version = "4.1.0"
942
+source = "registry+https://github.com/rust-lang/crates.io-index"
943
+checksum = "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2"
944
+dependencies = [
945
+ "libc",
946
+]
947
+
948
+[[package]]
949
+name = "futures"
950
+version = "0.3.31"
951
+source = "registry+https://github.com/rust-lang/crates.io-index"
952
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
953
+dependencies = [
954
+ "futures-channel",
955
+ "futures-core",
956
+ "futures-executor",
957
+ "futures-io",
958
+ "futures-sink",
959
+ "futures-task",
960
+ "futures-util",
961
+]
962
+
963
+[[package]]
964
+name = "futures-channel"
965
+version = "0.3.31"
966
+source = "registry+https://github.com/rust-lang/crates.io-index"
967
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
968
+dependencies = [
969
+ "futures-core",
970
+ "futures-sink",
971
+]
972
+
973
+[[package]]
974
+name = "futures-core"
975
+version = "0.3.31"
976
+source = "registry+https://github.com/rust-lang/crates.io-index"
977
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
978
+
979
+[[package]]
980
+name = "futures-executor"
981
+version = "0.3.31"
982
+source = "registry+https://github.com/rust-lang/crates.io-index"
983
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
984
+dependencies = [
985
+ "futures-core",
986
+ "futures-task",
987
+ "futures-util",
988
+]
989
+
990
+[[package]]
991
+name = "futures-io"
992
+version = "0.3.31"
993
+source = "registry+https://github.com/rust-lang/crates.io-index"
994
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
995
+
996
+[[package]]
997
+name = "futures-macro"
998
+version = "0.3.31"
999
+source = "registry+https://github.com/rust-lang/crates.io-index"
1000
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
1001
+dependencies = [
1002
+ "proc-macro2",
1003
+ "quote",
1004
+ "syn 2.0.110",
1005
+]
1006
+
1007
+[[package]]
1008
+name = "futures-sink"
1009
+version = "0.3.31"
1010
+source = "registry+https://github.com/rust-lang/crates.io-index"
1011
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
1012
+
1013
+[[package]]
1014
+name = "futures-task"
1015
+version = "0.3.31"
1016
+source = "registry+https://github.com/rust-lang/crates.io-index"
1017
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
1018
+
1019
+[[package]]
1020
+name = "futures-util"
1021
+version = "0.3.31"
1022
+source = "registry+https://github.com/rust-lang/crates.io-index"
1023
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
1024
+dependencies = [
1025
+ "futures-channel",
1026
+ "futures-core",
1027
+ "futures-io",
1028
+ "futures-macro",
1029
+ "futures-sink",
1030
+ "futures-task",
1031
+ "memchr",
1032
+ "pin-project-lite",
1033
+ "pin-utils",
1034
+ "slab",
1035
+]
1036
+
1037
+[[package]]
1038
+name = "fxhash"
1039
+version = "0.2.1"
1040
+source = "registry+https://github.com/rust-lang/crates.io-index"
1041
+checksum = "c31b6d751ae2c7f11320402d34e41349dd1016f8d5d45e48c4312bc8625af50c"
1042
+dependencies = [
1043
+ "byteorder",
1044
+]
1045
+
1046
+[[package]]
1047
+name = "getrandom"
1048
+version = "0.2.16"
1049
+source = "registry+https://github.com/rust-lang/crates.io-index"
1050
+checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
1051
+dependencies = [
1052
+ "cfg-if",
1053
+ "js-sys",
1054
+ "libc",
1055
+ "wasi",
1056
+ "wasm-bindgen",
1057
+]
1058
+
1059
+[[package]]
1060
+name = "getrandom"
1061
+version = "0.3.4"
1062
+source = "registry+https://github.com/rust-lang/crates.io-index"
1063
+checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd"
1064
+dependencies = [
1065
+ "cfg-if",
1066
+ "libc",
1067
+ "r-efi",
1068
+ "wasip2",
1069
+]
1070
+
1071
+[[package]]
1072
+name = "h2"
1073
+version = "0.4.12"
1074
+source = "registry+https://github.com/rust-lang/crates.io-index"
1075
+checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
1076
+dependencies = [
1077
+ "atomic-waker",
1078
+ "bytes",
1079
+ "fnv",
1080
+ "futures-core",
1081
+ "futures-sink",
1082
+ "http",
1083
+ "indexmap 2.12.0",
1084
+ "slab",
1085
+ "tokio",
1086
+ "tokio-util",
1087
+ "tracing",
1088
+]
1089
+
1090
+[[package]]
1091
+name = "hashbrown"
1092
+version = "0.12.3"
1093
+source = "registry+https://github.com/rust-lang/crates.io-index"
1094
+checksum = "8a9ee70c43aaf417c914396645a0fa852624801b24ebb7ae78fe8272889ac888"
1095
+
1096
+[[package]]
1097
+name = "hashbrown"
1098
+version = "0.15.5"
1099
+source = "registry+https://github.com/rust-lang/crates.io-index"
1100
+checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
1101
+dependencies = [
1102
+ "allocator-api2",
1103
+ "equivalent",
1104
+ "foldhash 0.1.5",
1105
+]
1106
+
1107
+[[package]]
1108
+name = "hashbrown"
1109
+version = "0.16.0"
1110
+source = "registry+https://github.com/rust-lang/crates.io-index"
1111
+checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
1112
+dependencies = [
1113
+ "allocator-api2",
1114
+ "equivalent",
1115
+ "foldhash 0.2.0",
1116
+]
1117
+
1118
+[[package]]
1119
+name = "hashers"
1120
+version = "1.0.1"
1121
+source = "registry+https://github.com/rust-lang/crates.io-index"
1122
+checksum = "b2bca93b15ea5a746f220e56587f71e73c6165eab783df9e26590069953e3c30"
1123
+dependencies = [
1124
+ "fxhash",
1125
+]
1126
+
1127
+[[package]]
1128
+name = "hdrhistogram"
1129
+version = "7.5.4"
1130
+source = "registry+https://github.com/rust-lang/crates.io-index"
1131
+checksum = "765c9198f173dd59ce26ff9f95ef0aafd0a0fe01fb9d72841bc5066a4c06511d"
1132
+dependencies = [
1133
+ "base64 0.21.7",
1134
+ "byteorder",
1135
+ "flate2",
1136
+ "nom",
1137
+ "num-traits",
1138
+]
1139
+
1140
+[[package]]
1141
+name = "heck"
1142
+version = "0.5.0"
1143
+source = "registry+https://github.com/rust-lang/crates.io-index"
1144
+checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
1145
+
1146
+[[package]]
1147
+name = "hermit-abi"
1148
+version = "0.1.19"
1149
+source = "registry+https://github.com/rust-lang/crates.io-index"
1150
+checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
1151
+dependencies = [
1152
+ "libc",
1153
+]
1154
+
1155
+[[package]]
1156
+name = "hermit-abi"
1157
+version = "0.5.2"
1158
+source = "registry+https://github.com/rust-lang/crates.io-index"
1159
+checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c"
1160
+
1161
+[[package]]
1162
+name = "http"
1163
+version = "1.3.1"
1164
+source = "registry+https://github.com/rust-lang/crates.io-index"
1165
+checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
1166
+dependencies = [
1167
+ "bytes",
1168
+ "fnv",
1169
+ "itoa",
1170
+]
1171
+
1172
+[[package]]
1173
+name = "http-body"
1174
+version = "1.0.1"
1175
+source = "registry+https://github.com/rust-lang/crates.io-index"
1176
+checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
1177
+dependencies = [
1178
+ "bytes",
1179
+ "http",
1180
+]
1181
+
1182
+[[package]]
1183
+name = "http-body-util"
1184
+version = "0.1.3"
1185
+source = "registry+https://github.com/rust-lang/crates.io-index"
1186
+checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
1187
+dependencies = [
1188
+ "bytes",
1189
+ "futures-core",
1190
+ "http",
1191
+ "http-body",
1192
+ "pin-project-lite",
1193
+]
1194
+
1195
+[[package]]
1196
+name = "httparse"
1197
+version = "1.10.1"
1198
+source = "registry+https://github.com/rust-lang/crates.io-index"
1199
+checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
1200
+
1201
+[[package]]
1202
+name = "httpdate"
1203
+version = "1.0.3"
1204
+source = "registry+https://github.com/rust-lang/crates.io-index"
1205
+checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
1206
+
1207
+[[package]]
1208
+name = "humantime"
1209
+version = "2.3.0"
1210
+source = "registry+https://github.com/rust-lang/crates.io-index"
1211
+checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
1212
+
1213
+[[package]]
1214
+name = "humantime-serde"
1215
+version = "1.1.1"
1216
+source = "registry+https://github.com/rust-lang/crates.io-index"
1217
+checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c"
1218
+dependencies = [
1219
+ "humantime",
1220
+ "serde",
1221
+]
1222
+
1223
+[[package]]
1224
+name = "hyper"
1225
+version = "1.8.0"
1226
+source = "registry+https://github.com/rust-lang/crates.io-index"
1227
+checksum = "1744436df46f0bde35af3eda22aeaba453aada65d8f1c171cd8a5f59030bd69f"
1228
+dependencies = [
1229
+ "atomic-waker",
1230
+ "bytes",
1231
+ "futures-channel",
1232
+ "futures-core",
1233
+ "h2",
1234
+ "http",
1235
+ "http-body",
1236
+ "httparse",
1237
+ "httpdate",
1238
+ "itoa",
1239
+ "pin-project-lite",
1240
+ "pin-utils",
1241
+ "smallvec",
1242
+ "tokio",
1243
+ "want",
1244
+]
1245
+
1246
+[[package]]
1247
+name = "hyper-timeout"
1248
+version = "0.5.2"
1249
+source = "registry+https://github.com/rust-lang/crates.io-index"
1250
+checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
1251
+dependencies = [
1252
+ "hyper",
1253
+ "hyper-util",
1254
+ "pin-project-lite",
1255
+ "tokio",
1256
+ "tower-service",
1257
+]
1258
+
1259
+[[package]]
1260
+name = "hyper-util"
1261
+version = "0.1.18"
1262
+source = "registry+https://github.com/rust-lang/crates.io-index"
1263
+checksum = "52e9a2a24dc5c6821e71a7030e1e14b7b632acac55c40e9d2e082c621261bb56"
1264
+dependencies = [
1265
+ "base64 0.22.1",
1266
+ "bytes",
1267
+ "futures-channel",
1268
+ "futures-core",
1269
+ "futures-util",
1270
+ "http",
1271
+ "http-body",
1272
+ "hyper",
1273
+ "ipnet",
1274
+ "libc",
1275
+ "percent-encoding",
1276
+ "pin-project-lite",
1277
+ "socket2 0.6.1",
1278
+ "tokio",
1279
+ "tower-service",
1280
+ "tracing",
1281
+]
1282
+
1283
+[[package]]
1284
+name = "iana-time-zone"
1285
+version = "0.1.64"
1286
+source = "registry+https://github.com/rust-lang/crates.io-index"
1287
+checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
1288
+dependencies = [
1289
+ "android_system_properties",
1290
+ "core-foundation-sys",
1291
+ "iana-time-zone-haiku",
1292
+ "js-sys",
1293
+ "log",
1294
+ "wasm-bindgen",
1295
+ "windows-core",
1296
+]
1297
+
1298
+[[package]]
1299
+name = "iana-time-zone-haiku"
1300
+version = "0.1.2"
1301
+source = "registry+https://github.com/rust-lang/crates.io-index"
1302
+checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
1303
+dependencies = [
1304
+ "cc",
1305
+]
1306
+
1307
+[[package]]
1308
+name = "icu_collections"
1309
+version = "2.1.1"
1310
+source = "registry+https://github.com/rust-lang/crates.io-index"
1311
+checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43"
1312
+dependencies = [
1313
+ "displaydoc",
1314
+ "potential_utf",
1315
+ "yoke",
1316
+ "zerofrom",
1317
+ "zerovec",
1318
+]
1319
+
1320
+[[package]]
1321
+name = "icu_locale_core"
1322
+version = "2.1.1"
1323
+source = "registry+https://github.com/rust-lang/crates.io-index"
1324
+checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6"
1325
+dependencies = [
1326
+ "displaydoc",
1327
+ "litemap",
1328
+ "tinystr",
1329
+ "writeable",
1330
+ "zerovec",
1331
+]
1332
+
1333
+[[package]]
1334
+name = "icu_normalizer"
1335
+version = "2.1.1"
1336
+source = "registry+https://github.com/rust-lang/crates.io-index"
1337
+checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599"
1338
+dependencies = [
1339
+ "icu_collections",
1340
+ "icu_normalizer_data",
1341
+ "icu_properties",
1342
+ "icu_provider",
1343
+ "smallvec",
1344
+ "zerovec",
1345
+]
1346
+
1347
+[[package]]
1348
+name = "icu_normalizer_data"
1349
+version = "2.1.1"
1350
+source = "registry+https://github.com/rust-lang/crates.io-index"
1351
+checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a"
1352
+
1353
+[[package]]
1354
+name = "icu_properties"
1355
+version = "2.1.1"
1356
+source = "registry+https://github.com/rust-lang/crates.io-index"
1357
+checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99"
1358
+dependencies = [
1359
+ "icu_collections",
1360
+ "icu_locale_core",
1361
+ "icu_properties_data",
1362
+ "icu_provider",
1363
+ "zerotrie",
1364
+ "zerovec",
1365
+]
1366
+
1367
+[[package]]
1368
+name = "icu_properties_data"
1369
+version = "2.1.1"
1370
+source = "registry+https://github.com/rust-lang/crates.io-index"
1371
+checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899"
1372
+
1373
+[[package]]
1374
+name = "icu_provider"
1375
+version = "2.1.1"
1376
+source = "registry+https://github.com/rust-lang/crates.io-index"
1377
+checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614"
1378
+dependencies = [
1379
+ "displaydoc",
1380
+ "icu_locale_core",
1381
+ "writeable",
1382
+ "yoke",
1383
+ "zerofrom",
1384
+ "zerotrie",
1385
+ "zerovec",
1386
+]
1387
+
1388
+[[package]]
1389
+name = "ident_case"
1390
+version = "1.0.1"
1391
+source = "registry+https://github.com/rust-lang/crates.io-index"
1392
+checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39"
1393
+
1394
+[[package]]
1395
+name = "idna"
1396
+version = "1.1.0"
1397
+source = "registry+https://github.com/rust-lang/crates.io-index"
1398
+checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de"
1399
+dependencies = [
1400
+ "idna_adapter",
1401
+ "smallvec",
1402
+ "utf8_iter",
1403
+]
1404
+
1405
+[[package]]
1406
+name = "idna_adapter"
1407
+version = "1.2.1"
1408
+source = "registry+https://github.com/rust-lang/crates.io-index"
1409
+checksum = "3acae9609540aa318d1bc588455225fb2085b9ed0c4f6bd0d9d5bcd86f1a0344"
1410
+dependencies = [
1411
+ "icu_normalizer",
1412
+ "icu_properties",
1413
+]
1414
+
1415
+[[package]]
1416
+name = "indexmap"
1417
+version = "1.9.3"
1418
+source = "registry+https://github.com/rust-lang/crates.io-index"
1419
+checksum = "bd070e393353796e801d209ad339e89596eb4c8d430d18ede6a1cced8fafbd99"
1420
+dependencies = [
1421
+ "autocfg",
1422
+ "hashbrown 0.12.3",
1423
+]
1424
+
1425
+[[package]]
1426
+name = "indexmap"
1427
+version = "2.12.0"
1428
+source = "registry+https://github.com/rust-lang/crates.io-index"
1429
+checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f"
1430
+dependencies = [
1431
+ "equivalent",
1432
+ "hashbrown 0.16.0",
1433
+]
1434
+
1435
+[[package]]
1436
+name = "inotify"
1437
+version = "0.11.0"
1438
+source = "registry+https://github.com/rust-lang/crates.io-index"
1439
+checksum = "f37dccff2791ab604f9babef0ba14fbe0be30bd368dc541e2b08d07c8aa908f3"
1440
+dependencies = [
1441
+ "bitflags 2.10.0",
1442
+ "inotify-sys",
1443
+ "libc",
1444
+]
1445
+
1446
+[[package]]
1447
+name = "inotify-sys"
1448
+version = "0.1.5"
1449
+source = "registry+https://github.com/rust-lang/crates.io-index"
1450
+checksum = "e05c02b5e89bff3b946cedeca278abc628fe811e604f027c45a8aa3cf793d0eb"
1451
+dependencies = [
1452
+ "libc",
1453
+]
1454
+
1455
+[[package]]
1456
+name = "io-uring"
1457
+version = "0.7.11"
1458
+source = "registry+https://github.com/rust-lang/crates.io-index"
1459
+checksum = "fdd7bddefd0a8833b88a4b68f90dae22c7450d11b354198baee3874fd811b344"
1460
+dependencies = [
1461
+ "bitflags 2.10.0",
1462
+ "cfg-if",
1463
+ "libc",
1464
+]
1465
+
1466
+[[package]]
1467
+name = "ipnet"
1468
+version = "2.11.0"
1469
+source = "registry+https://github.com/rust-lang/crates.io-index"
1470
+checksum = "469fb0b9cefa57e3ef31275ee7cacb78f2fdca44e4765491884a2b119d4eb130"
1471
+
1472
+[[package]]
1473
+name = "iri-string"
1474
+version = "0.7.9"
1475
+source = "registry+https://github.com/rust-lang/crates.io-index"
1476
+checksum = "4f867b9d1d896b67beb18518eda36fdb77a32ea590de864f1325b294a6d14397"
1477
+dependencies = [
1478
+ "memchr",
1479
+ "serde",
1480
+]
1481
+
1482
+[[package]]
1483
+name = "is_terminal_polyfill"
1484
+version = "1.70.2"
1485
+source = "registry+https://github.com/rust-lang/crates.io-index"
1486
+checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695"
1487
+
1488
+[[package]]
1489
+name = "itertools"
1490
+version = "0.14.0"
1491
+source = "registry+https://github.com/rust-lang/crates.io-index"
1492
+checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
1493
+dependencies = [
1494
+ "either",
1495
+]
1496
+
1497
+[[package]]
1498
+name = "itoa"
1499
+version = "1.0.15"
1500
+source = "registry+https://github.com/rust-lang/crates.io-index"
1501
+checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
1502
+
1503
+[[package]]
1504
+name = "jobserver"
1505
+version = "0.1.34"
1506
+source = "registry+https://github.com/rust-lang/crates.io-index"
1507
+checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33"
1508
+dependencies = [
1509
+ "getrandom 0.3.4",
1510
+ "libc",
1511
+]
1512
+
1513
+[[package]]
1514
+name = "journal-common"
1515
+version = "0.1.0"
1516
+dependencies = [
1517
+ "allocative",
1518
+ "nix",
1519
+ "rustc-hash",
1520
+ "serde",
1521
+ "uuid",
1522
+]
1523
+
1524
+[[package]]
1525
+name = "journal-core"
1526
+version = "0.1.0"
1527
+dependencies = [
1528
+ "allocative",
1529
+ "anyhow",
1530
+ "chrono",
1531
+ "clap",
1532
+ "crossbeam-channel",
1533
+ "hashers",
1534
+ "journal-common",
1535
+ "journal-registry",
1536
+ "libc",
1537
+ "lz4",
1538
+ "md5",
1539
+ "memmap2",
1540
+ "nix",
1541
+ "notify",
1542
+ "parking_lot",
1543
+ "rand 0.9.2",
1544
+ "rayon",
1545
+ "rdp",
1546
+ "regex",
1547
+ "rustc-hash",
1548
+ "ruzstd",
1549
+ "serde",
1550
+ "serde_json",
1551
+ "siphasher",
1552
+ "static_assertions",
1553
+ "tempfile",
1554
+ "thiserror",
1555
+ "tokio",
1556
+ "tracing",
1557
+ "tracing-subscriber",
1558
+ "twox-hash",
1559
+ "uuid",
1560
+ "walkdir",
1561
+ "zerocopy 0.9.0-alpha.0",
1562
+ "zstd",
1563
+]
1564
+
1565
+[[package]]
1566
+name = "journal-engine"
1567
+version = "0.1.0"
1568
+dependencies = [
1569
+ "allocative",
1570
+ "async-stream",
1571
+ "async-trait",
1572
+ "chrono",
1573
+ "foundation",
1574
+ "foyer",
1575
+ "futures",
1576
+ "journal-common",
1577
+ "journal-core",
1578
+ "journal-index",
1579
+ "journal-registry",
1580
+ "lru",
1581
+ "parking_lot",
1582
+ "rayon",
1583
+ "serde",
1584
+ "serde_json",
1585
+ "static_assertions",
1586
+ "tempfile",
1587
+ "thiserror",
1588
+ "tokio",
1589
+ "tracing",
1590
+ "tracing-subscriber",
1591
+ "uuid",
1592
+]
1593
+
1594
+[[package]]
1595
+name = "journal-function"
1596
+version = "0.1.0"
1597
+dependencies = [
1598
+ "allocative",
1599
+ "async-stream",
1600
+ "async-trait",
1601
+ "chrono",
1602
+ "clap",
1603
+ "foyer",
1604
+ "futures",
1605
+ "journal-core",
1606
+ "journal-engine",
1607
+ "journal-index",
1608
+ "journal-registry",
1609
+ "nix",
1610
+ "notify",
1611
+ "parking_lot",
1612
+ "rt",
1613
+ "schemars",
1614
+ "serde",
1615
+ "serde_json",
1616
+ "thiserror",
1617
+ "tokio",
1618
+ "tracing",
1619
+ "tracing-subscriber",
1620
+]
1621
+
1622
+[[package]]
1623
+name = "journal-index"
1624
+version = "0.1.0"
1625
+dependencies = [
1626
+ "allocative",
1627
+ "journal-common",
1628
+ "journal-core",
1629
+ "journal-registry",
1630
+ "regex",
1631
+ "roaring",
1632
+ "serde",
1633
+ "static_assertions",
1634
+ "tempfile",
1635
+ "thiserror",
1636
+ "tracing",
1637
+ "uuid",
1638
+]
1639
+
1640
+[[package]]
1641
+name = "journal-log-writer"
1642
+version = "0.1.0"
1643
+dependencies = [
1644
+ "flatten-serde-json",
1645
+ "journal-common",
1646
+ "journal-core",
1647
+ "journal-registry",
1648
+ "nix",
1649
+ "rdp",
1650
+ "serde",
1651
+ "serde_json",
1652
+ "tempfile",
1653
+ "thiserror",
1654
+ "tracing",
1655
+ "uuid",
1656
+]
1657
+
1658
+[[package]]
1659
+name = "journal-registry"
1660
+version = "0.1.0"
1661
+dependencies = [
1662
+ "allocative",
1663
+ "journal-common",
1664
+ "notify",
1665
+ "parking_lot",
1666
+ "serde",
1667
+ "thiserror",
1668
+ "tokio",
1669
+ "tracing",
1670
+ "uuid",
1671
+ "walkdir",
1672
+]
1673
+
1674
+[[package]]
1675
+name = "journal-viewer-plugin"
1676
+version = "0.1.0"
1677
+dependencies = [
1678
+ "anyhow",
1679
+ "async-trait",
1680
+ "bytesize",
1681
+ "bytesize-serde",
1682
+ "foyer",
1683
+ "journal-core",
1684
+ "journal-function",
1685
+ "journal-index",
1686
+ "journal-registry",
1687
+ "netdata-plugin-error",
1688
+ "netdata-plugin-protocol",
1689
+ "netdata-plugin-schema",
1690
+ "notify",
1691
+ "num_cpus",
1692
+ "parking_lot",
1693
+ "rt",
1694
+ "schemars",
1695
+ "serde",
1696
+ "serde_json",
1697
+ "serde_yaml",
1698
+ "thiserror",
1699
+ "tokio",
1700
+ "tracing",
1701
+]
1702
+
1703
+[[package]]
1704
+name = "js-sys"
1705
+version = "0.3.82"
1706
+source = "registry+https://github.com/rust-lang/crates.io-index"
1707
+checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65"
1708
+dependencies = [
1709
+ "once_cell",
1710
+ "wasm-bindgen",
1711
+]
1712
+
1713
+[[package]]
1714
+name = "kqueue"
1715
+version = "1.1.1"
1716
+source = "registry+https://github.com/rust-lang/crates.io-index"
1717
+checksum = "eac30106d7dce88daf4a3fcb4879ea939476d5074a9b7ddd0fb97fa4bed5596a"
1718
+dependencies = [
1719
+ "kqueue-sys",
1720
+ "libc",
1721
+]
1722
+
1723
+[[package]]
1724
+name = "kqueue-sys"
1725
+version = "1.0.4"
1726
+source = "registry+https://github.com/rust-lang/crates.io-index"
1727
+checksum = "ed9625ffda8729b85e45cf04090035ac368927b8cebc34898e7c120f52e4838b"
1728
+dependencies = [
1729
+ "bitflags 1.3.2",
1730
+ "libc",
1731
+]
1732
+
1733
+[[package]]
1734
+name = "lazy_static"
1735
+version = "1.5.0"
1736
+source = "registry+https://github.com/rust-lang/crates.io-index"
1737
+checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
1738
+
1739
+[[package]]
1740
+name = "libc"
1741
+version = "0.2.177"
1742
+source = "registry+https://github.com/rust-lang/crates.io-index"
1743
+checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976"
1744
+
1745
+[[package]]
1746
+name = "linux-raw-sys"
1747
+version = "0.11.0"
1748
+source = "registry+https://github.com/rust-lang/crates.io-index"
1749
+checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
1750
+
1751
+[[package]]
1752
+name = "litemap"
1753
+version = "0.8.1"
1754
+source = "registry+https://github.com/rust-lang/crates.io-index"
1755
+checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77"
1756
+
1757
+[[package]]
1758
+name = "lock_api"
1759
+version = "0.4.14"
1760
+source = "registry+https://github.com/rust-lang/crates.io-index"
1761
+checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965"
1762
+dependencies = [
1763
+ "scopeguard",
1764
+]
1765
+
1766
+[[package]]
1767
+name = "log"
1768
+version = "0.4.28"
1769
+source = "registry+https://github.com/rust-lang/crates.io-index"
1770
+checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
1771
+
1772
+[[package]]
1773
+name = "lru"
1774
+version = "0.16.2"
1775
+source = "registry+https://github.com/rust-lang/crates.io-index"
1776
+checksum = "96051b46fc183dc9cd4a223960ef37b9af631b55191852a8274bfef064cda20f"
1777
+dependencies = [
1778
+ "hashbrown 0.16.0",
1779
+]
1780
+
1781
+[[package]]
1782
+name = "lz4"
1783
+version = "1.28.1"
1784
+source = "registry+https://github.com/rust-lang/crates.io-index"
1785
+checksum = "a20b523e860d03443e98350ceaac5e71c6ba89aea7d960769ec3ce37f4de5af4"
1786
+dependencies = [
1787
+ "lz4-sys",
1788
+]
1789
+
1790
+[[package]]
1791
+name = "lz4-sys"
1792
+version = "1.11.1+lz4-1.10.0"
1793
+source = "registry+https://github.com/rust-lang/crates.io-index"
1794
+checksum = "6bd8c0d6c6ed0cd30b3652886bb8711dc4bb01d637a68105a3d5158039b418e6"
1795
+dependencies = [
1796
+ "cc",
1797
+ "libc",
1798
+]
1799
+
1800
+[[package]]
1801
+name = "madsim"
1802
+version = "0.2.34"
1803
+source = "registry+https://github.com/rust-lang/crates.io-index"
1804
+checksum = "18351aac4194337d6ea9ffbd25b3d1540ecc0754142af1bff5ba7392d1f6f771"
1805
+dependencies = [
1806
+ "ahash",
1807
+ "async-channel",
1808
+ "async-stream",
1809
+ "async-task",
1810
+ "bincode",
1811
+ "bytes",
1812
+ "downcast-rs",
1813
+ "errno",
1814
+ "futures-util",
1815
+ "lazy_static",
1816
+ "libc",
1817
+ "madsim-macros",
1818
+ "naive-timer",
1819
+ "panic-message",
1820
+ "rand 0.8.5",
1821
+ "rand_xoshiro",
1822
+ "rustversion",
1823
+ "serde",
1824
+ "spin",
1825
+ "tokio",
1826
+ "tokio-util",
1827
+ "toml",
1828
+ "tracing",
1829
+ "tracing-subscriber",
1830
+]
1831
+
1832
+[[package]]
1833
+name = "madsim-macros"
1834
+version = "0.2.12"
1835
+source = "registry+https://github.com/rust-lang/crates.io-index"
1836
+checksum = "f3d248e97b1a48826a12c3828d921e8548e714394bf17274dd0a93910dc946e1"
1837
+dependencies = [
1838
+ "darling",
1839
+ "proc-macro2",
1840
+ "quote",
1841
+ "syn 1.0.109",
1842
+]
1843
+
1844
+[[package]]
1845
+name = "madsim-tokio"
1846
+version = "0.2.30"
1847
+source = "registry+https://github.com/rust-lang/crates.io-index"
1848
+checksum = "7d3eb2acc57c82d21d699119b859e2df70a91dbdb84734885a1e72be83bdecb5"
1849
+dependencies = [
1850
+ "madsim",
1851
+ "spin",
1852
+ "tokio",
1853
+]
1854
+
1855
+[[package]]
1856
+name = "matchers"
1857
+version = "0.2.0"
1858
+source = "registry+https://github.com/rust-lang/crates.io-index"
1859
+checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9"
1860
+dependencies = [
1861
+ "regex-automata",
1862
+]
1863
+
1864
+[[package]]
1865
+name = "matchit"
1866
+version = "0.7.3"
1867
+source = "registry+https://github.com/rust-lang/crates.io-index"
1868
+checksum = "0e7465ac9959cc2b1404e8e2367b43684a6d13790fe23056cc8c6c5a6b7bcb94"
1869
+
1870
+[[package]]
1871
+name = "matchit"
1872
+version = "0.8.4"
1873
+source = "registry+https://github.com/rust-lang/crates.io-index"
1874
+checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
1875
+
1876
+[[package]]
1877
+name = "md5"
1878
+version = "0.7.0"
1879
+source = "registry+https://github.com/rust-lang/crates.io-index"
1880
+checksum = "490cc448043f947bae3cbee9c203358d62dbee0db12107a74be5c30ccfd09771"
1881
+
1882
+[[package]]
1883
+name = "memchr"
1884
+version = "2.7.6"
1885
+source = "registry+https://github.com/rust-lang/crates.io-index"
1886
+checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
1887
+
1888
+[[package]]
1889
+name = "memmap2"
1890
+version = "0.9.9"
1891
+source = "registry+https://github.com/rust-lang/crates.io-index"
1892
+checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490"
1893
+dependencies = [
1894
+ "libc",
1895
+]
1896
+
1897
+[[package]]
1898
+name = "memoffset"
1899
+version = "0.9.1"
1900
+source = "registry+https://github.com/rust-lang/crates.io-index"
1901
+checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a"
1902
+dependencies = [
1903
+ "autocfg",
1904
+]
1905
+
1906
+[[package]]
1907
+name = "mime"
1908
+version = "0.3.17"
1909
+source = "registry+https://github.com/rust-lang/crates.io-index"
1910
+checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
1911
+
1912
+[[package]]
1913
+name = "minimal-lexical"
1914
+version = "0.2.1"
1915
+source = "registry+https://github.com/rust-lang/crates.io-index"
1916
+checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
1917
+
1918
+[[package]]
1919
+name = "miniz_oxide"
1920
+version = "0.8.9"
1921
+source = "registry+https://github.com/rust-lang/crates.io-index"
1922
+checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
1923
+dependencies = [
1924
+ "adler2",
1925
+ "simd-adler32",
1926
+]
1927
+
1928
+[[package]]
1929
+name = "mio"
1930
+version = "1.1.0"
1931
+source = "registry+https://github.com/rust-lang/crates.io-index"
1932
+checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873"
1933
+dependencies = [
1934
+ "libc",
1935
+ "log",
1936
+ "wasi",
1937
+ "windows-sys 0.61.2",
1938
+]
1939
+
1940
+[[package]]
1941
+name = "mixtrics"
1942
+version = "0.2.3"
1943
+source = "registry+https://github.com/rust-lang/crates.io-index"
1944
+checksum = "fb252c728b9d77c6ef9103f0c81524fa0a3d3b161d0a936295d7fbeff6e04c11"
1945
+dependencies = [
1946
+ "itertools",
1947
+ "parking_lot",
1948
+]
1949
+
1950
+[[package]]
1951
+name = "naive-timer"
1952
+version = "0.2.0"
1953
+source = "registry+https://github.com/rust-lang/crates.io-index"
1954
+checksum = "034a0ad7deebf0c2abcf2435950a6666c3c15ea9d8fad0c0f48efa8a7f843fed"
1955
+
1956
+[[package]]
1957
+name = "nanorand"
1958
+version = "0.7.0"
1959
+source = "registry+https://github.com/rust-lang/crates.io-index"
1960
+checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3"
1961
+dependencies = [
1962
+ "getrandom 0.2.16",
1963
+]
1964
+
1965
+[[package]]
1966
+name = "netdata-plugin-charts-derive"
1967
+version = "0.1.0"
1968
+dependencies = [
1969
+ "proc-macro2",
1970
+ "quote",
1971
+ "syn 2.0.110",
1972
+]
1973
+
1974
+[[package]]
1975
+name = "netdata-plugin-error"
1976
+version = "0.1.0"
1977
+dependencies = [
1978
+ "thiserror",
1979
+]
1980
+
1981
+[[package]]
1982
+name = "netdata-plugin-protocol"
1983
+version = "0.1.0"
1984
+dependencies = [
1985
+ "atoi",
1986
+ "bytes",
1987
+ "futures",
1988
+ "futures-util",
1989
+ "netdata-plugin-error",
1990
+ "netdata-plugin-types",
1991
+ "phf",
1992
+ "phf_codegen",
1993
+ "serde",
1994
+ "serde_json",
1995
+ "tokio",
1996
+ "tokio-util",
1997
+ "tracing",
1998
+]
1999
+
2000
+[[package]]
2001
+name = "netdata-plugin-schema"
2002
+version = "0.1.0"
2003
+dependencies = [
2004
+ "netdata-plugin-error",
2005
+ "netdata-plugin-types",
2006
+ "schemars",
2007
+ "serde",
2008
+ "serde_json",
2009
+]
2010
+
2011
+[[package]]
2012
+name = "netdata-plugin-types"
2013
+version = "0.1.0"
2014
+dependencies = [
2015
+ "bitflags 2.10.0",
2016
+ "netdata-plugin-error",
2017
+ "serde_json",
2018
+]
2019
+
2020
+[[package]]
2021
+name = "nix"
2022
+version = "0.30.1"
2023
+source = "registry+https://github.com/rust-lang/crates.io-index"
2024
+checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6"
2025
+dependencies = [
2026
+ "bitflags 2.10.0",
2027
+ "cfg-if",
2028
+ "cfg_aliases",
2029
+ "libc",
2030
+]
2031
+
2032
+[[package]]
2033
+name = "nom"
2034
+version = "7.1.3"
2035
+source = "registry+https://github.com/rust-lang/crates.io-index"
2036
+checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a"
2037
+dependencies = [
2038
+ "memchr",
2039
+ "minimal-lexical",
2040
+]
2041
+
2042
+[[package]]
2043
+name = "notify"
2044
+version = "8.2.0"
2045
+source = "registry+https://github.com/rust-lang/crates.io-index"
2046
+checksum = "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3"
2047
+dependencies = [
2048
+ "bitflags 2.10.0",
2049
+ "fsevent-sys",
2050
+ "inotify",
2051
+ "kqueue",
2052
+ "libc",
2053
+ "log",
2054
+ "mio",
2055
+ "notify-types",
2056
+ "walkdir",
2057
+ "windows-sys 0.60.2",
2058
+]
2059
+
2060
+[[package]]
2061
+name = "notify-types"
2062
+version = "2.0.0"
2063
+source = "registry+https://github.com/rust-lang/crates.io-index"
2064
+checksum = "5e0826a989adedc2a244799e823aece04662b66609d96af8dff7ac6df9a8925d"
2065
+
2066
+[[package]]
2067
+name = "nu-ansi-term"
2068
+version = "0.50.3"
2069
+source = "registry+https://github.com/rust-lang/crates.io-index"
2070
+checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
2071
+dependencies = [
2072
+ "windows-sys 0.61.2",
2073
+]
2074
+
2075
+[[package]]
2076
+name = "num-traits"
2077
+version = "0.2.19"
2078
+source = "registry+https://github.com/rust-lang/crates.io-index"
2079
+checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
2080
+dependencies = [
2081
+ "autocfg",
2082
+]
2083
+
2084
+[[package]]
2085
+name = "num_cpus"
2086
+version = "1.17.0"
2087
+source = "registry+https://github.com/rust-lang/crates.io-index"
2088
+checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b"
2089
+dependencies = [
2090
+ "hermit-abi 0.5.2",
2091
+ "libc",
2092
+]
2093
+
2094
+[[package]]
2095
+name = "once_cell"
2096
+version = "1.21.3"
2097
+source = "registry+https://github.com/rust-lang/crates.io-index"
2098
+checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
2099
+
2100
+[[package]]
2101
+name = "once_cell_polyfill"
2102
+version = "1.70.2"
2103
+source = "registry+https://github.com/rust-lang/crates.io-index"
2104
+checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe"
2105
+
2106
+[[package]]
2107
+name = "opentelemetry"
2108
+version = "0.31.0"
2109
+source = "registry+https://github.com/rust-lang/crates.io-index"
2110
+checksum = "b84bcd6ae87133e903af7ef497404dda70c60d0ea14895fc8a5e6722754fc2a0"
2111
+dependencies = [
2112
+ "futures-core",
2113
+ "futures-sink",
2114
+ "js-sys",
2115
+ "pin-project-lite",
2116
+ "thiserror",
2117
+ "tracing",
2118
+]
2119
+
2120
+[[package]]
2121
+name = "opentelemetry-http"
2122
+version = "0.31.0"
2123
+source = "registry+https://github.com/rust-lang/crates.io-index"
2124
+checksum = "d7a6d09a73194e6b66df7c8f1b680f156d916a1a942abf2de06823dd02b7855d"
2125
+dependencies = [
2126
+ "async-trait",
2127
+ "bytes",
2128
+ "http",
2129
+ "opentelemetry",
2130
+ "reqwest",
2131
+]
2132
+
2133
+[[package]]
2134
+name = "opentelemetry-otlp"
2135
+version = "0.31.0"
2136
+source = "registry+https://github.com/rust-lang/crates.io-index"
2137
+checksum = "7a2366db2dca4d2ad033cad11e6ee42844fd727007af5ad04a1730f4cb8163bf"
2138
+dependencies = [
2139
+ "http",
2140
+ "opentelemetry",
2141
+ "opentelemetry-http",
2142
+ "opentelemetry-proto",
2143
+ "opentelemetry_sdk",
2144
+ "prost 0.14.1",
2145
+ "reqwest",
2146
+ "thiserror",
2147
+ "tokio",
2148
+ "tonic 0.14.2",
2149
+ "tracing",
2150
+]
2151
+
2152
+[[package]]
2153
+name = "opentelemetry-proto"
2154
+version = "0.31.0"
2155
+source = "registry+https://github.com/rust-lang/crates.io-index"
2156
+checksum = "a7175df06de5eaee9909d4805a3d07e28bb752c34cab57fa9cff549da596b30f"
2157
+dependencies = [
2158
+ "base64 0.22.1",
2159
+ "const-hex",
2160
+ "opentelemetry",
2161
+ "opentelemetry_sdk",
2162
+ "prost 0.14.1",
2163
+ "serde",
2164
+ "serde_json",
2165
+ "tonic 0.14.2",
2166
+ "tonic-prost",
2167
+]
2168
+
2169
+[[package]]
2170
+name = "opentelemetry_sdk"
2171
+version = "0.31.0"
2172
+source = "registry+https://github.com/rust-lang/crates.io-index"
2173
+checksum = "e14ae4f5991976fd48df6d843de219ca6d31b01daaab2dad5af2badeded372bd"
2174
+dependencies = [
2175
+ "futures-channel",
2176
+ "futures-executor",
2177
+ "futures-util",
2178
+ "opentelemetry",
2179
+ "percent-encoding",
2180
+ "rand 0.9.2",
2181
+ "thiserror",
2182
+ "tokio",
2183
+ "tokio-stream",
2184
+]
2185
+
2186
+[[package]]
2187
+name = "otel-plugin"
2188
+version = "0.1.0"
2189
+dependencies = [
2190
+ "anyhow",
2191
+ "atty",
2192
+ "bytesize",
2193
+ "bytesize-serde",
2194
+ "clap",
2195
+ "flatten_otel",
2196
+ "humantime",
2197
+ "humantime-serde",
2198
+ "journal-common",
2199
+ "journal-core",
2200
+ "journal-log-writer",
2201
+ "journal-registry",
2202
+ "opentelemetry",
2203
+ "opentelemetry-otlp",
2204
+ "opentelemetry-proto",
2205
+ "opentelemetry_sdk",
2206
+ "regex",
2207
+ "rt",
2208
+ "serde",
2209
+ "serde_json",
2210
+ "serde_regex",
2211
+ "serde_yaml",
2212
+ "tokio",
2213
+ "tonic 0.14.2",
2214
+ "tracing",
2215
+]
2216
+
2217
+[[package]]
2218
+name = "panic-message"
2219
+version = "0.3.0"
2220
+source = "registry+https://github.com/rust-lang/crates.io-index"
2221
+checksum = "384e52fd8fbd4cbe3c317e8216260c21a0f9134de108cea8a4dd4e7e152c472d"
2222
+
2223
+[[package]]
2224
+name = "parking"
2225
+version = "2.2.1"
2226
+source = "registry+https://github.com/rust-lang/crates.io-index"
2227
+checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba"
2228
+
2229
+[[package]]
2230
+name = "parking_lot"
2231
+version = "0.12.5"
2232
+source = "registry+https://github.com/rust-lang/crates.io-index"
2233
+checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a"
2234
+dependencies = [
2235
+ "lock_api",
2236
+ "parking_lot_core",
2237
+]
2238
+
2239
+[[package]]
2240
+name = "parking_lot_core"
2241
+version = "0.9.12"
2242
+source = "registry+https://github.com/rust-lang/crates.io-index"
2243
+checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1"
2244
+dependencies = [
2245
+ "cfg-if",
2246
+ "libc",
2247
+ "redox_syscall",
2248
+ "smallvec",
2249
+ "windows-link",
2250
+]
2251
+
2252
+[[package]]
2253
+name = "percent-encoding"
2254
+version = "2.3.2"
2255
+source = "registry+https://github.com/rust-lang/crates.io-index"
2256
+checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
2257
+
2258
+[[package]]
2259
+name = "phf"
2260
+version = "0.12.1"
2261
+source = "registry+https://github.com/rust-lang/crates.io-index"
2262
+checksum = "913273894cec178f401a31ec4b656318d95473527be05c0752cc41cdc32be8b7"
2263
+dependencies = [
2264
+ "phf_shared",
2265
+]
2266
+
2267
+[[package]]
2268
+name = "phf_codegen"
2269
+version = "0.12.1"
2270
+source = "registry+https://github.com/rust-lang/crates.io-index"
2271
+checksum = "efbdcb6f01d193b17f0b9c3360fa7e0e620991b193ff08702f78b3ce365d7e61"
2272
+dependencies = [
2273
+ "phf_generator",
2274
+ "phf_shared",
2275
+]
2276
+
2277
+[[package]]
2278
+name = "phf_generator"
2279
+version = "0.12.1"
2280
+source = "registry+https://github.com/rust-lang/crates.io-index"
2281
+checksum = "2cbb1126afed61dd6368748dae63b1ee7dc480191c6262a3b4ff1e29d86a6c5b"
2282
+dependencies = [
2283
+ "fastrand",
2284
+ "phf_shared",
2285
+]
2286
+
2287
+[[package]]
2288
+name = "phf_shared"
2289
+version = "0.12.1"
2290
+source = "registry+https://github.com/rust-lang/crates.io-index"
2291
+checksum = "06005508882fb681fd97892ecff4b7fd0fee13ef1aa569f8695dae7ab9099981"
2292
+dependencies = [
2293
+ "siphasher",
2294
+]
2295
+
2296
+[[package]]
2297
+name = "pin-project"
2298
+version = "1.1.10"
2299
+source = "registry+https://github.com/rust-lang/crates.io-index"
2300
+checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
2301
+dependencies = [
2302
+ "pin-project-internal",
2303
+]
2304
+
2305
+[[package]]
2306
+name = "pin-project-internal"
2307
+version = "1.1.10"
2308
+source = "registry+https://github.com/rust-lang/crates.io-index"
2309
+checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
2310
+dependencies = [
2311
+ "proc-macro2",
2312
+ "quote",
2313
+ "syn 2.0.110",
2314
+]
2315
+
2316
+[[package]]
2317
+name = "pin-project-lite"
2318
+version = "0.2.16"
2319
+source = "registry+https://github.com/rust-lang/crates.io-index"
2320
+checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
2321
+
2322
+[[package]]
2323
+name = "pin-utils"
2324
+version = "0.1.0"
2325
+source = "registry+https://github.com/rust-lang/crates.io-index"
2326
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
2327
+
2328
+[[package]]
2329
+name = "pkg-config"
2330
+version = "0.3.32"
2331
+source = "registry+https://github.com/rust-lang/crates.io-index"
2332
+checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c"
2333
+
2334
+[[package]]
2335
+name = "potential_utf"
2336
+version = "0.1.4"
2337
+source = "registry+https://github.com/rust-lang/crates.io-index"
2338
+checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77"
2339
+dependencies = [
2340
+ "zerovec",
2341
+]
2342
+
2343
+[[package]]
2344
+name = "ppv-lite86"
2345
+version = "0.2.21"
2346
+source = "registry+https://github.com/rust-lang/crates.io-index"
2347
+checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
2348
+dependencies = [
2349
+ "zerocopy 0.8.27",
2350
+]
2351
+
2352
+[[package]]
2353
+name = "proc-macro2"
2354
+version = "1.0.103"
2355
+source = "registry+https://github.com/rust-lang/crates.io-index"
2356
+checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8"
2357
+dependencies = [
2358
+ "unicode-ident",
2359
+]
2360
+
2361
+[[package]]
2362
+name = "proptest"
2363
+version = "1.9.0"
2364
+source = "registry+https://github.com/rust-lang/crates.io-index"
2365
+checksum = "bee689443a2bd0a16ab0348b52ee43e3b2d1b1f931c8aa5c9f8de4c86fbe8c40"
2366
+dependencies = [
2367
+ "bitflags 2.10.0",
2368
+ "num-traits",
2369
+ "rand 0.9.2",
2370
+ "rand_chacha 0.9.0",
2371
+ "rand_xorshift",
2372
+ "regex-syntax",
2373
+ "unarray",
2374
+]
2375
+
2376
+[[package]]
2377
+name = "prost"
2378
+version = "0.13.5"
2379
+source = "registry+https://github.com/rust-lang/crates.io-index"
2380
+checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
2381
+dependencies = [
2382
+ "bytes",
2383
+ "prost-derive 0.13.5",
2384
+]
2385
+
2386
+[[package]]
2387
+name = "prost"
2388
+version = "0.14.1"
2389
+source = "registry+https://github.com/rust-lang/crates.io-index"
2390
+checksum = "7231bd9b3d3d33c86b58adbac74b5ec0ad9f496b19d22801d773636feaa95f3d"
2391
+dependencies = [
2392
+ "bytes",
2393
+ "prost-derive 0.14.1",
2394
+]
2395
+
2396
+[[package]]
2397
+name = "prost-derive"
2398
+version = "0.13.5"
2399
+source = "registry+https://github.com/rust-lang/crates.io-index"
2400
+checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
2401
+dependencies = [
2402
+ "anyhow",
2403
+ "itertools",
2404
+ "proc-macro2",
2405
+ "quote",
2406
+ "syn 2.0.110",
2407
+]
2408
+
2409
+[[package]]
2410
+name = "prost-derive"
2411
+version = "0.14.1"
2412
+source = "registry+https://github.com/rust-lang/crates.io-index"
2413
+checksum = "9120690fafc389a67ba3803df527d0ec9cbbc9cc45e4cc20b332996dfb672425"
2414
+dependencies = [
2415
+ "anyhow",
2416
+ "itertools",
2417
+ "proc-macro2",
2418
+ "quote",
2419
+ "syn 2.0.110",
2420
+]
2421
+
2422
+[[package]]
2423
+name = "prost-types"
2424
+version = "0.13.5"
2425
+source = "registry+https://github.com/rust-lang/crates.io-index"
2426
+checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
2427
+dependencies = [
2428
+ "prost 0.13.5",
2429
+]
2430
+
2431
+[[package]]
2432
+name = "quote"
2433
+version = "1.0.42"
2434
+source = "registry+https://github.com/rust-lang/crates.io-index"
2435
+checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f"
2436
+dependencies = [
2437
+ "proc-macro2",
2438
+]
2439
+
2440
+[[package]]
2441
+name = "r-efi"
2442
+version = "5.3.0"
2443
+source = "registry+https://github.com/rust-lang/crates.io-index"
2444
+checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
2445
+
2446
+[[package]]
2447
+name = "rand"
2448
+version = "0.8.5"
2449
+source = "registry+https://github.com/rust-lang/crates.io-index"
2450
+checksum = "34af8d1a0e25924bc5b7c43c079c942339d8f0a8b57c39049bef581b46327404"
2451
+dependencies = [
2452
+ "libc",
2453
+ "rand_chacha 0.3.1",
2454
+ "rand_core 0.6.4",
2455
+]
2456
+
2457
+[[package]]
2458
+name = "rand"
2459
+version = "0.9.2"
2460
+source = "registry+https://github.com/rust-lang/crates.io-index"
2461
+checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
2462
+dependencies = [
2463
+ "rand_chacha 0.9.0",
2464
+ "rand_core 0.9.3",
2465
+]
2466
+
2467
+[[package]]
2468
+name = "rand_chacha"
2469
+version = "0.3.1"
2470
+source = "registry+https://github.com/rust-lang/crates.io-index"
2471
+checksum = "e6c10a63a0fa32252be49d21e7709d4d4baf8d231c2dbce1eaa8141b9b127d88"
2472
+dependencies = [
2473
+ "ppv-lite86",
2474
+ "rand_core 0.6.4",
2475
+]
2476
+
2477
+[[package]]
2478
+name = "rand_chacha"
2479
+version = "0.9.0"
2480
+source = "registry+https://github.com/rust-lang/crates.io-index"
2481
+checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
2482
+dependencies = [
2483
+ "ppv-lite86",
2484
+ "rand_core 0.9.3",
2485
+]
2486
+
2487
+[[package]]
2488
+name = "rand_core"
2489
+version = "0.6.4"
2490
+source = "registry+https://github.com/rust-lang/crates.io-index"
2491
+checksum = "ec0be4795e2f6a28069bec0b5ff3e2ac9bafc99e6a9a7dc3547996c5c816922c"
2492
+dependencies = [
2493
+ "getrandom 0.2.16",
2494
+]
2495
+
2496
+[[package]]
2497
+name = "rand_core"
2498
+version = "0.9.3"
2499
+source = "registry+https://github.com/rust-lang/crates.io-index"
2500
+checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
2501
+dependencies = [
2502
+ "getrandom 0.3.4",
2503
+]
2504
+
2505
+[[package]]
2506
+name = "rand_xorshift"
2507
+version = "0.4.0"
2508
+source = "registry+https://github.com/rust-lang/crates.io-index"
2509
+checksum = "513962919efc330f829edb2535844d1b912b0fbe2ca165d613e4e8788bb05a5a"
2510
+dependencies = [
2511
+ "rand_core 0.9.3",
2512
+]
2513
+
2514
+[[package]]
2515
+name = "rand_xoshiro"
2516
+version = "0.6.0"
2517
+source = "registry+https://github.com/rust-lang/crates.io-index"
2518
+checksum = "6f97cdb2a36ed4183de61b2f824cc45c9f1037f28afe0a322e9fff4c108b5aaa"
2519
+dependencies = [
2520
+ "rand_core 0.6.4",
2521
+]
2522
+
2523
+[[package]]
2524
+name = "rayon"
2525
+version = "1.11.0"
2526
+source = "registry+https://github.com/rust-lang/crates.io-index"
2527
+checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f"
2528
+dependencies = [
2529
+ "either",
2530
+ "rayon-core",
2531
+]
2532
+
2533
+[[package]]
2534
+name = "rayon-core"
2535
+version = "1.13.0"
2536
+source = "registry+https://github.com/rust-lang/crates.io-index"
2537
+checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91"
2538
+dependencies = [
2539
+ "crossbeam-deque",
2540
+ "crossbeam-utils",
2541
+]
2542
+
2543
+[[package]]
2544
+name = "rdp"
2545
+version = "0.1.0"
2546
+dependencies = [
2547
+ "md5",
2548
+]
2549
+
2550
+[[package]]
2551
+name = "redox_syscall"
2552
+version = "0.5.18"
2553
+source = "registry+https://github.com/rust-lang/crates.io-index"
2554
+checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d"
2555
+dependencies = [
2556
+ "bitflags 2.10.0",
2557
+]
2558
+
2559
+[[package]]
2560
+name = "ref-cast"
2561
+version = "1.0.25"
2562
+source = "registry+https://github.com/rust-lang/crates.io-index"
2563
+checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d"
2564
+dependencies = [
2565
+ "ref-cast-impl",
2566
+]
2567
+
2568
+[[package]]
2569
+name = "ref-cast-impl"
2570
+version = "1.0.25"
2571
+source = "registry+https://github.com/rust-lang/crates.io-index"
2572
+checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da"
2573
+dependencies = [
2574
+ "proc-macro2",
2575
+ "quote",
2576
+ "syn 2.0.110",
2577
+]
2578
+
2579
+[[package]]
2580
+name = "regex"
2581
+version = "1.12.2"
2582
+source = "registry+https://github.com/rust-lang/crates.io-index"
2583
+checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4"
2584
+dependencies = [
2585
+ "aho-corasick",
2586
+ "memchr",
2587
+ "regex-automata",
2588
+ "regex-syntax",
2589
+]
2590
+
2591
+[[package]]
2592
+name = "regex-automata"
2593
+version = "0.4.13"
2594
+source = "registry+https://github.com/rust-lang/crates.io-index"
2595
+checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c"
2596
+dependencies = [
2597
+ "aho-corasick",
2598
+ "memchr",
2599
+ "regex-syntax",
2600
+]
2601
+
2602
+[[package]]
2603
+name = "regex-syntax"
2604
+version = "0.8.8"
2605
+source = "registry+https://github.com/rust-lang/crates.io-index"
2606
+checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58"
2607
+
2608
+[[package]]
2609
+name = "reqwest"
2610
+version = "0.12.24"
2611
+source = "registry+https://github.com/rust-lang/crates.io-index"
2612
+checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f"
2613
+dependencies = [
2614
+ "base64 0.22.1",
2615
+ "bytes",
2616
+ "futures-channel",
2617
+ "futures-core",
2618
+ "futures-util",
2619
+ "http",
2620
+ "http-body",
2621
+ "http-body-util",
2622
+ "hyper",
2623
+ "hyper-util",
2624
+ "js-sys",
2625
+ "log",
2626
+ "percent-encoding",
2627
+ "pin-project-lite",
2628
+ "serde",
2629
+ "serde_json",
2630
+ "serde_urlencoded",
2631
+ "sync_wrapper",
2632
+ "tokio",
2633
+ "tower 0.5.2",
2634
+ "tower-http",
2635
+ "tower-service",
2636
+ "url",
2637
+ "wasm-bindgen",
2638
+ "wasm-bindgen-futures",
2639
+ "web-sys",
2640
+]
2641
+
2642
+[[package]]
2643
+name = "ring"
2644
+version = "0.17.14"
2645
+source = "registry+https://github.com/rust-lang/crates.io-index"
2646
+checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
2647
+dependencies = [
2648
+ "cc",
2649
+ "cfg-if",
2650
+ "getrandom 0.2.16",
2651
+ "libc",
2652
+ "untrusted",
2653
+ "windows-sys 0.52.0",
2654
+]
2655
+
2656
+[[package]]
2657
+name = "roaring"
2658
+version = "0.11.2"
2659
+source = "git+https://github.com/netdata/roaring-rs.git?branch=allocative#3ee69b1db5a2a4fc909076962c0159ddae893f8c"
2660
+dependencies = [
2661
+ "allocative",
2662
+ "bytemuck",
2663
+ "byteorder",
2664
+ "serde",
2665
+]
2666
+
2667
+[[package]]
2668
+name = "rt"
2669
+version = "0.1.0"
2670
+dependencies = [
2671
+ "async-trait",
2672
+ "bytes",
2673
+ "console-subscriber",
2674
+ "foundation",
2675
+ "futures",
2676
+ "itoa",
2677
+ "netdata-plugin-charts-derive",
2678
+ "netdata-plugin-error",
2679
+ "netdata-plugin-protocol",
2680
+ "netdata-plugin-schema",
2681
+ "parking_lot",
2682
+ "schemars",
2683
+ "serde",
2684
+ "serde_json",
2685
+ "tokio",
2686
+ "tokio-util",
2687
+ "tracing",
2688
+ "tracing-futures",
2689
+ "tracing-journald",
2690
+ "tracing-subscriber",
2691
+]
2692
+
2693
+[[package]]
2694
+name = "rustc-hash"
2695
+version = "2.1.1"
2696
+source = "registry+https://github.com/rust-lang/crates.io-index"
2697
+checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d"
2698
+
2699
+[[package]]
2700
+name = "rustix"
2701
+version = "1.1.2"
2702
+source = "registry+https://github.com/rust-lang/crates.io-index"
2703
+checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
2704
+dependencies = [
2705
+ "bitflags 2.10.0",
2706
+ "errno",
2707
+ "libc",
2708
+ "linux-raw-sys",
2709
+ "windows-sys 0.61.2",
2710
+]
2711
+
2712
+[[package]]
2713
+name = "rustls"
2714
+version = "0.23.35"
2715
+source = "registry+https://github.com/rust-lang/crates.io-index"
2716
+checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f"
2717
+dependencies = [
2718
+ "log",
2719
+ "once_cell",
2720
+ "ring",
2721
+ "rustls-pki-types",
2722
+ "rustls-webpki",
2723
+ "subtle",
2724
+ "zeroize",
2725
+]
2726
+
2727
+[[package]]
2728
+name = "rustls-pki-types"
2729
+version = "1.13.0"
2730
+source = "registry+https://github.com/rust-lang/crates.io-index"
2731
+checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a"
2732
+dependencies = [
2733
+ "zeroize",
2734
+]
2735
+
2736
+[[package]]
2737
+name = "rustls-webpki"
2738
+version = "0.103.8"
2739
+source = "registry+https://github.com/rust-lang/crates.io-index"
2740
+checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52"
2741
+dependencies = [
2742
+ "ring",
2743
+ "rustls-pki-types",
2744
+ "untrusted",
2745
+]
2746
+
2747
+[[package]]
2748
+name = "rustversion"
2749
+version = "1.0.22"
2750
+source = "registry+https://github.com/rust-lang/crates.io-index"
2751
+checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
2752
+
2753
+[[package]]
2754
+name = "ruzstd"
2755
+version = "0.8.2"
2756
+source = "registry+https://github.com/rust-lang/crates.io-index"
2757
+checksum = "e5ff0cc5e135c8870a775d3320910cd9b564ec036b4dc0b8741629020be63f01"
2758
+dependencies = [
2759
+ "twox-hash",
2760
+]
2761
+
2762
+[[package]]
2763
+name = "ryu"
2764
+version = "1.0.20"
2765
+source = "registry+https://github.com/rust-lang/crates.io-index"
2766
+checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
2767
+
2768
+[[package]]
2769
+name = "same-file"
2770
+version = "1.0.6"
2771
+source = "registry+https://github.com/rust-lang/crates.io-index"
2772
+checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502"
2773
+dependencies = [
2774
+ "winapi-util",
2775
+]
2776
+
2777
+[[package]]
2778
+name = "schemars"
2779
+version = "1.1.0"
2780
+source = "registry+https://github.com/rust-lang/crates.io-index"
2781
+checksum = "9558e172d4e8533736ba97870c4b2cd63f84b382a3d6eb063da41b91cce17289"
2782
+dependencies = [
2783
+ "dyn-clone",
2784
+ "ref-cast",
2785
+ "schemars_derive",
2786
+ "serde",
2787
+ "serde_json",
2788
+]
2789
+
2790
+[[package]]
2791
+name = "schemars_derive"
2792
+version = "1.1.0"
2793
+source = "registry+https://github.com/rust-lang/crates.io-index"
2794
+checksum = "301858a4023d78debd2353c7426dc486001bddc91ae31a76fb1f55132f7e2633"
2795
+dependencies = [
2796
+ "proc-macro2",
2797
+ "quote",
2798
+ "serde_derive_internals",
2799
+ "syn 2.0.110",
2800
+]
2801
+
2802
+[[package]]
2803
+name = "scopeguard"
2804
+version = "1.2.0"
2805
+source = "registry+https://github.com/rust-lang/crates.io-index"
2806
+checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49"
2807
+
2808
+[[package]]
2809
+name = "serde"
2810
+version = "1.0.228"
2811
+source = "registry+https://github.com/rust-lang/crates.io-index"
2812
+checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
2813
+dependencies = [
2814
+ "serde_core",
2815
+ "serde_derive",
2816
+]
2817
+
2818
+[[package]]
2819
+name = "serde_core"
2820
+version = "1.0.228"
2821
+source = "registry+https://github.com/rust-lang/crates.io-index"
2822
+checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
2823
+dependencies = [
2824
+ "serde_derive",
2825
+]
2826
+
2827
+[[package]]
2828
+name = "serde_derive"
2829
+version = "1.0.228"
2830
+source = "registry+https://github.com/rust-lang/crates.io-index"
2831
+checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
2832
+dependencies = [
2833
+ "proc-macro2",
2834
+ "quote",
2835
+ "syn 2.0.110",
2836
+]
2837
+
2838
+[[package]]
2839
+name = "serde_derive_internals"
2840
+version = "0.29.1"
2841
+source = "registry+https://github.com/rust-lang/crates.io-index"
2842
+checksum = "18d26a20a969b9e3fdf2fc2d9f21eda6c40e2de84c9408bb5d3b05d499aae711"
2843
+dependencies = [
2844
+ "proc-macro2",
2845
+ "quote",
2846
+ "syn 2.0.110",
2847
+]
2848
+
2849
+[[package]]
2850
+name = "serde_json"
2851
+version = "1.0.145"
2852
+source = "registry+https://github.com/rust-lang/crates.io-index"
2853
+checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
2854
+dependencies = [
2855
+ "indexmap 2.12.0",
2856
+ "itoa",
2857
+ "memchr",
2858
+ "ryu",
2859
+ "serde",
2860
+ "serde_core",
2861
+]
2862
+
2863
+[[package]]
2864
+name = "serde_regex"
2865
+version = "1.1.0"
2866
+source = "registry+https://github.com/rust-lang/crates.io-index"
2867
+checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf"
2868
+dependencies = [
2869
+ "regex",
2870
+ "serde",
2871
+]
2872
+
2873
+[[package]]
2874
+name = "serde_spanned"
2875
+version = "1.0.3"
2876
+source = "registry+https://github.com/rust-lang/crates.io-index"
2877
+checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392"
2878
+dependencies = [
2879
+ "serde_core",
2880
+]
2881
+
2882
+[[package]]
2883
+name = "serde_urlencoded"
2884
+version = "0.7.1"
2885
+source = "registry+https://github.com/rust-lang/crates.io-index"
2886
+checksum = "d3491c14715ca2294c4d6a88f15e84739788c1d030eed8c110436aafdaa2f3fd"
2887
+dependencies = [
2888
+ "form_urlencoded",
2889
+ "itoa",
2890
+ "ryu",
2891
+ "serde",
2892
+]
2893
+
2894
+[[package]]
2895
+name = "serde_yaml"
2896
+version = "0.9.34+deprecated"
2897
+source = "registry+https://github.com/rust-lang/crates.io-index"
2898
+checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
2899
+dependencies = [
2900
+ "indexmap 2.12.0",
2901
+ "itoa",
2902
+ "ryu",
2903
+ "serde",
2904
+ "unsafe-libyaml",
2905
+]
2906
+
2907
+[[package]]
2908
+name = "sharded-slab"
2909
+version = "0.1.7"
2910
+source = "registry+https://github.com/rust-lang/crates.io-index"
2911
+checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
2912
+dependencies = [
2913
+ "lazy_static",
2914
+]
2915
+
2916
+[[package]]
2917
+name = "shlex"
2918
+version = "1.3.0"
2919
+source = "registry+https://github.com/rust-lang/crates.io-index"
2920
+checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
2921
+
2922
+[[package]]
2923
+name = "signal-hook-registry"
2924
+version = "1.4.6"
2925
+source = "registry+https://github.com/rust-lang/crates.io-index"
2926
+checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b"
2927
+dependencies = [
2928
+ "libc",
2929
+]
2930
+
2931
+[[package]]
2932
+name = "simd-adler32"
2933
+version = "0.3.7"
2934
+source = "registry+https://github.com/rust-lang/crates.io-index"
2935
+checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
2936
+
2937
+[[package]]
2938
+name = "siphasher"
2939
+version = "1.0.1"
2940
+source = "registry+https://github.com/rust-lang/crates.io-index"
2941
+checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
2942
+
2943
+[[package]]
2944
+name = "slab"
2945
+version = "0.4.11"
2946
+source = "registry+https://github.com/rust-lang/crates.io-index"
2947
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
2948
+
2949
+[[package]]
2950
+name = "small_ctor"
2951
+version = "0.1.2"
2952
+source = "registry+https://github.com/rust-lang/crates.io-index"
2953
+checksum = "88414a5ca1f85d82cc34471e975f0f74f6aa54c40f062efa42c0080e7f763f81"
2954
+
2955
+[[package]]
2956
+name = "smallvec"
2957
+version = "1.15.1"
2958
+source = "registry+https://github.com/rust-lang/crates.io-index"
2959
+checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
2960
+
2961
+[[package]]
2962
+name = "socket2"
2963
+version = "0.5.10"
2964
+source = "registry+https://github.com/rust-lang/crates.io-index"
2965
+checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
2966
+dependencies = [
2967
+ "libc",
2968
+ "windows-sys 0.52.0",
2969
+]
2970
+
2971
+[[package]]
2972
+name = "socket2"
2973
+version = "0.6.1"
2974
+source = "registry+https://github.com/rust-lang/crates.io-index"
2975
+checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881"
2976
+dependencies = [
2977
+ "libc",
2978
+ "windows-sys 0.60.2",
2979
+]
2980
+
2981
+[[package]]
2982
+name = "spin"
2983
+version = "0.9.8"
2984
+source = "registry+https://github.com/rust-lang/crates.io-index"
2985
+checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
2986
+dependencies = [
2987
+ "lock_api",
2988
+]
2989
+
2990
+[[package]]
2991
+name = "stable_deref_trait"
2992
+version = "1.2.1"
2993
+source = "registry+https://github.com/rust-lang/crates.io-index"
2994
+checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596"
2995
+
2996
+[[package]]
2997
+name = "static_assertions"
2998
+version = "1.1.0"
2999
+source = "registry+https://github.com/rust-lang/crates.io-index"
3000
+checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
3001
+
3002
+[[package]]
3003
+name = "strsim"
3004
+version = "0.10.0"
3005
+source = "registry+https://github.com/rust-lang/crates.io-index"
3006
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
3007
+
3008
+[[package]]
3009
+name = "strsim"
3010
+version = "0.11.1"
3011
+source = "registry+https://github.com/rust-lang/crates.io-index"
3012
+checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
3013
+
3014
+[[package]]
3015
+name = "subtle"
3016
+version = "2.6.1"
3017
+source = "registry+https://github.com/rust-lang/crates.io-index"
3018
+checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
3019
+
3020
+[[package]]
3021
+name = "syn"
3022
+version = "1.0.109"
3023
+source = "registry+https://github.com/rust-lang/crates.io-index"
3024
+checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237"
3025
+dependencies = [
3026
+ "proc-macro2",
3027
+ "quote",
3028
+ "unicode-ident",
3029
+]
3030
+
3031
+[[package]]
3032
+name = "syn"
3033
+version = "2.0.110"
3034
+source = "registry+https://github.com/rust-lang/crates.io-index"
3035
+checksum = "a99801b5bd34ede4cf3fc688c5919368fea4e4814a4664359503e6015b280aea"
3036
+dependencies = [
3037
+ "proc-macro2",
3038
+ "quote",
3039
+ "unicode-ident",
3040
+]
3041
+
3042
+[[package]]
3043
+name = "sync_wrapper"
3044
+version = "1.0.2"
3045
+source = "registry+https://github.com/rust-lang/crates.io-index"
3046
+checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
3047
+dependencies = [
3048
+ "futures-core",
3049
+]
3050
+
3051
+[[package]]
3052
+name = "synstructure"
3053
+version = "0.13.2"
3054
+source = "registry+https://github.com/rust-lang/crates.io-index"
3055
+checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2"
3056
+dependencies = [
3057
+ "proc-macro2",
3058
+ "quote",
3059
+ "syn 2.0.110",
3060
+]
3061
+
3062
+[[package]]
3063
+name = "tempfile"
3064
+version = "3.23.0"
3065
+source = "registry+https://github.com/rust-lang/crates.io-index"
3066
+checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
3067
+dependencies = [
3068
+ "fastrand",
3069
+ "getrandom 0.3.4",
3070
+ "once_cell",
3071
+ "rustix",
3072
+ "windows-sys 0.61.2",
3073
+]
3074
+
3075
+[[package]]
3076
+name = "thiserror"
3077
+version = "2.0.17"
3078
+source = "registry+https://github.com/rust-lang/crates.io-index"
3079
+checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
3080
+dependencies = [
3081
+ "thiserror-impl",
3082
+]
3083
+
3084
+[[package]]
3085
+name = "thiserror-impl"
3086
+version = "2.0.17"
3087
+source = "registry+https://github.com/rust-lang/crates.io-index"
3088
+checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
3089
+dependencies = [
3090
+ "proc-macro2",
3091
+ "quote",
3092
+ "syn 2.0.110",
3093
+]
3094
+
3095
+[[package]]
3096
+name = "thread_local"
3097
+version = "1.1.9"
3098
+source = "registry+https://github.com/rust-lang/crates.io-index"
3099
+checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
3100
+dependencies = [
3101
+ "cfg-if",
3102
+]
3103
+
3104
+[[package]]
3105
+name = "tinystr"
3106
+version = "0.8.2"
3107
+source = "registry+https://github.com/rust-lang/crates.io-index"
3108
+checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869"
3109
+dependencies = [
3110
+ "displaydoc",
3111
+ "zerovec",
3112
+]
3113
+
3114
+[[package]]
3115
+name = "tokio"
3116
+version = "1.48.0"
3117
+source = "registry+https://github.com/rust-lang/crates.io-index"
3118
+checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
3119
+dependencies = [
3120
+ "bytes",
3121
+ "libc",
3122
+ "mio",
3123
+ "parking_lot",
3124
+ "pin-project-lite",
3125
+ "signal-hook-registry",
3126
+ "socket2 0.6.1",
3127
+ "tokio-macros",
3128
+ "tracing",
3129
+ "windows-sys 0.61.2",
3130
+]
3131
+
3132
+[[package]]
3133
+name = "tokio-macros"
3134
+version = "2.6.0"
3135
+source = "registry+https://github.com/rust-lang/crates.io-index"
3136
+checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
3137
+dependencies = [
3138
+ "proc-macro2",
3139
+ "quote",
3140
+ "syn 2.0.110",
3141
+]
3142
+
3143
+[[package]]
3144
+name = "tokio-rustls"
3145
+version = "0.26.4"
3146
+source = "registry+https://github.com/rust-lang/crates.io-index"
3147
+checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
3148
+dependencies = [
3149
+ "rustls",
3150
+ "tokio",
3151
+]
3152
+
3153
+[[package]]
3154
+name = "tokio-stream"
3155
+version = "0.1.17"
3156
+source = "registry+https://github.com/rust-lang/crates.io-index"
3157
+checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
3158
+dependencies = [
3159
+ "futures-core",
3160
+ "pin-project-lite",
3161
+ "tokio",
3162
+]
3163
+
3164
+[[package]]
3165
+name = "tokio-util"
3166
+version = "0.7.17"
3167
+source = "registry+https://github.com/rust-lang/crates.io-index"
3168
+checksum = "2efa149fe76073d6e8fd97ef4f4eca7b67f599660115591483572e406e165594"
3169
+dependencies = [
3170
+ "bytes",
3171
+ "futures-core",
3172
+ "futures-sink",
3173
+ "pin-project-lite",
3174
+ "tokio",
3175
+]
3176
+
3177
+[[package]]
3178
+name = "toml"
3179
+version = "0.9.8"
3180
+source = "registry+https://github.com/rust-lang/crates.io-index"
3181
+checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8"
3182
+dependencies = [
3183
+ "indexmap 2.12.0",
3184
+ "serde_core",
3185
+ "serde_spanned",
3186
+ "toml_datetime",
3187
+ "toml_parser",
3188
+ "toml_writer",
3189
+ "winnow",
3190
+]
3191
+
3192
+[[package]]
3193
+name = "toml_datetime"
3194
+version = "0.7.3"
3195
+source = "registry+https://github.com/rust-lang/crates.io-index"
3196
+checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533"
3197
+dependencies = [
3198
+ "serde_core",
3199
+]
3200
+
3201
+[[package]]
3202
+name = "toml_parser"
3203
+version = "1.0.4"
3204
+source = "registry+https://github.com/rust-lang/crates.io-index"
3205
+checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e"
3206
+dependencies = [
3207
+ "winnow",
3208
+]
3209
+
3210
+[[package]]
3211
+name = "toml_writer"
3212
+version = "1.0.4"
3213
+source = "registry+https://github.com/rust-lang/crates.io-index"
3214
+checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2"
3215
+
3216
+[[package]]
3217
+name = "tonic"
3218
+version = "0.12.3"
3219
+source = "registry+https://github.com/rust-lang/crates.io-index"
3220
+checksum = "877c5b330756d856ffcc4553ab34a5684481ade925ecc54bcd1bf02b1d0d4d52"
3221
+dependencies = [
3222
+ "async-stream",
3223
+ "async-trait",
3224
+ "axum 0.7.9",
3225
+ "base64 0.22.1",
3226
+ "bytes",
3227
+ "h2",
3228
+ "http",
3229
+ "http-body",
3230
+ "http-body-util",
3231
+ "hyper",
3232
+ "hyper-timeout",
3233
+ "hyper-util",
3234
+ "percent-encoding",
3235
+ "pin-project",
3236
+ "prost 0.13.5",
3237
+ "socket2 0.5.10",
3238
+ "tokio",
3239
+ "tokio-stream",
3240
+ "tower 0.4.13",
3241
+ "tower-layer",
3242
+ "tower-service",
3243
+ "tracing",
3244
+]
3245
+
3246
+[[package]]
3247
+name = "tonic"
3248
+version = "0.14.2"
3249
+source = "registry+https://github.com/rust-lang/crates.io-index"
3250
+checksum = "eb7613188ce9f7df5bfe185db26c5814347d110db17920415cf2fbcad85e7203"
3251
+dependencies = [
3252
+ "async-trait",
3253
+ "axum 0.8.6",
3254
+ "base64 0.22.1",
3255
+ "bytes",
3256
+ "flate2",
3257
+ "h2",
3258
+ "http",
3259
+ "http-body",
3260
+ "http-body-util",
3261
+ "hyper",
3262
+ "hyper-timeout",
3263
+ "hyper-util",
3264
+ "percent-encoding",
3265
+ "pin-project",
3266
+ "socket2 0.6.1",
3267
+ "sync_wrapper",
3268
+ "tokio",
3269
+ "tokio-rustls",
3270
+ "tokio-stream",
3271
+ "tower 0.5.2",
3272
+ "tower-layer",
3273
+ "tower-service",
3274
+ "tracing",
3275
+]
3276
+
3277
+[[package]]
3278
+name = "tonic-prost"
3279
+version = "0.14.2"
3280
+source = "registry+https://github.com/rust-lang/crates.io-index"
3281
+checksum = "66bd50ad6ce1252d87ef024b3d64fe4c3cf54a86fb9ef4c631fdd0ded7aeaa67"
3282
+dependencies = [
3283
+ "bytes",
3284
+ "prost 0.14.1",
3285
+ "tonic 0.14.2",
3286
+]
3287
+
3288
+[[package]]
3289
+name = "tower"
3290
+version = "0.4.13"
3291
+source = "registry+https://github.com/rust-lang/crates.io-index"
3292
+checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c"
3293
+dependencies = [
3294
+ "futures-core",
3295
+ "futures-util",
3296
+ "indexmap 1.9.3",
3297
+ "pin-project",
3298
+ "pin-project-lite",
3299
+ "rand 0.8.5",
3300
+ "slab",
3301
+ "tokio",
3302
+ "tokio-util",
3303
+ "tower-layer",
3304
+ "tower-service",
3305
+ "tracing",
3306
+]
3307
+
3308
+[[package]]
3309
+name = "tower"
3310
+version = "0.5.2"
3311
+source = "registry+https://github.com/rust-lang/crates.io-index"
3312
+checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
3313
+dependencies = [
3314
+ "futures-core",
3315
+ "futures-util",
3316
+ "indexmap 2.12.0",
3317
+ "pin-project-lite",
3318
+ "slab",
3319
+ "sync_wrapper",
3320
+ "tokio",
3321
+ "tokio-util",
3322
+ "tower-layer",
3323
+ "tower-service",
3324
+ "tracing",
3325
+]
3326
+
3327
+[[package]]
3328
+name = "tower-http"
3329
+version = "0.6.6"
3330
+source = "registry+https://github.com/rust-lang/crates.io-index"
3331
+checksum = "adc82fd73de2a9722ac5da747f12383d2bfdb93591ee6c58486e0097890f05f2"
3332
+dependencies = [
3333
+ "bitflags 2.10.0",
3334
+ "bytes",
3335
+ "futures-util",
3336
+ "http",
3337
+ "http-body",
3338
+ "iri-string",
3339
+ "pin-project-lite",
3340
+ "tower 0.5.2",
3341
+ "tower-layer",
3342
+ "tower-service",
3343
+]
3344
+
3345
+[[package]]
3346
+name = "tower-layer"
3347
+version = "0.3.3"
3348
+source = "registry+https://github.com/rust-lang/crates.io-index"
3349
+checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
3350
+
3351
+[[package]]
3352
+name = "tower-service"
3353
+version = "0.3.3"
3354
+source = "registry+https://github.com/rust-lang/crates.io-index"
3355
+checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
3356
+
3357
+[[package]]
3358
+name = "tracing"
3359
+version = "0.1.41"
3360
+source = "registry+https://github.com/rust-lang/crates.io-index"
3361
+checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
3362
+dependencies = [
3363
+ "pin-project-lite",
3364
+ "tracing-attributes",
3365
+ "tracing-core",
3366
+]
3367
+
3368
+[[package]]
3369
+name = "tracing-attributes"
3370
+version = "0.1.30"
3371
+source = "registry+https://github.com/rust-lang/crates.io-index"
3372
+checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
3373
+dependencies = [
3374
+ "proc-macro2",
3375
+ "quote",
3376
+ "syn 2.0.110",
3377
+]
3378
+
3379
+[[package]]
3380
+name = "tracing-core"
3381
+version = "0.1.34"
3382
+source = "registry+https://github.com/rust-lang/crates.io-index"
3383
+checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
3384
+dependencies = [
3385
+ "once_cell",
3386
+ "valuable",
3387
+]
3388
+
3389
+[[package]]
3390
+name = "tracing-futures"
3391
+version = "0.2.5"
3392
+source = "registry+https://github.com/rust-lang/crates.io-index"
3393
+checksum = "97d095ae15e245a057c8e8451bab9b3ee1e1f68e9ba2b4fbc18d0ac5237835f2"
3394
+dependencies = [
3395
+ "pin-project",
3396
+ "tracing",
3397
+]
3398
+
3399
+[[package]]
3400
+name = "tracing-journald"
3401
+version = "0.3.1"
3402
+source = "registry+https://github.com/rust-lang/crates.io-index"
3403
+checksum = "fc0b4143302cf1022dac868d521e36e8b27691f72c84b3311750d5188ebba657"
3404
+dependencies = [
3405
+ "libc",
3406
+ "tracing-core",
3407
+ "tracing-subscriber",
3408
+]
3409
+
3410
+[[package]]
3411
+name = "tracing-log"
3412
+version = "0.2.0"
3413
+source = "registry+https://github.com/rust-lang/crates.io-index"
3414
+checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
3415
+dependencies = [
3416
+ "log",
3417
+ "once_cell",
3418
+ "tracing-core",
3419
+]
3420
+
3421
+[[package]]
3422
+name = "tracing-serde"
3423
+version = "0.2.0"
3424
+source = "registry+https://github.com/rust-lang/crates.io-index"
3425
+checksum = "704b1aeb7be0d0a84fc9828cae51dab5970fee5088f83d1dd7ee6f6246fc6ff1"
3426
+dependencies = [
3427
+ "serde",
3428
+ "tracing-core",
3429
+]
3430
+
3431
+[[package]]
3432
+name = "tracing-subscriber"
3433
+version = "0.3.20"
3434
+source = "registry+https://github.com/rust-lang/crates.io-index"
3435
+checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
3436
+dependencies = [
3437
+ "matchers",
3438
+ "nu-ansi-term",
3439
+ "once_cell",
3440
+ "regex-automata",
3441
+ "serde",
3442
+ "serde_json",
3443
+ "sharded-slab",
3444
+ "smallvec",
3445
+ "thread_local",
3446
+ "tracing",
3447
+ "tracing-core",
3448
+ "tracing-log",
3449
+ "tracing-serde",
3450
+]
3451
+
3452
+[[package]]
3453
+name = "try-lock"
3454
+version = "0.2.5"
3455
+source = "registry+https://github.com/rust-lang/crates.io-index"
3456
+checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
3457
+
3458
+[[package]]
3459
+name = "twox-hash"
3460
+version = "2.1.2"
3461
+source = "registry+https://github.com/rust-lang/crates.io-index"
3462
+checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
3463
+dependencies = [
3464
+ "rand 0.9.2",
3465
+]
3466
+
3467
+[[package]]
3468
+name = "unarray"
3469
+version = "0.1.4"
3470
+source = "registry+https://github.com/rust-lang/crates.io-index"
3471
+checksum = "eaea85b334db583fe3274d12b4cd1880032beab409c0d774be044d4480ab9a94"
3472
+
3473
+[[package]]
3474
+name = "unicode-ident"
3475
+version = "1.0.22"
3476
+source = "registry+https://github.com/rust-lang/crates.io-index"
3477
+checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5"
3478
+
3479
+[[package]]
3480
+name = "unsafe-libyaml"
3481
+version = "0.2.11"
3482
+source = "registry+https://github.com/rust-lang/crates.io-index"
3483
+checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
3484
+
3485
+[[package]]
3486
+name = "untrusted"
3487
+version = "0.9.0"
3488
+source = "registry+https://github.com/rust-lang/crates.io-index"
3489
+checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
3490
+
3491
+[[package]]
3492
+name = "url"
3493
+version = "2.5.7"
3494
+source = "registry+https://github.com/rust-lang/crates.io-index"
3495
+checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b"
3496
+dependencies = [
3497
+ "form_urlencoded",
3498
+ "idna",
3499
+ "percent-encoding",
3500
+ "serde",
3501
+]
3502
+
3503
+[[package]]
3504
+name = "utf8_iter"
3505
+version = "1.0.4"
3506
+source = "registry+https://github.com/rust-lang/crates.io-index"
3507
+checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"
3508
+
3509
+[[package]]
3510
+name = "utf8parse"
3511
+version = "0.2.2"
3512
+source = "registry+https://github.com/rust-lang/crates.io-index"
3513
+checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
3514
+
3515
+[[package]]
3516
+name = "uuid"
3517
+version = "1.18.1"
3518
+source = "registry+https://github.com/rust-lang/crates.io-index"
3519
+checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
3520
+dependencies = [
3521
+ "getrandom 0.3.4",
3522
+ "js-sys",
3523
+ "serde",
3524
+ "wasm-bindgen",
3525
+]
3526
+
3527
+[[package]]
3528
+name = "valuable"
3529
+version = "0.1.1"
3530
+source = "registry+https://github.com/rust-lang/crates.io-index"
3531
+checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
3532
+
3533
+[[package]]
3534
+name = "version_check"
3535
+version = "0.9.5"
3536
+source = "registry+https://github.com/rust-lang/crates.io-index"
3537
+checksum = "0b928f33d975fc6ad9f86c8f283853ad26bdd5b10b7f1542aa2fa15e2289105a"
3538
+
3539
+[[package]]
3540
+name = "walkdir"
3541
+version = "2.5.0"
3542
+source = "registry+https://github.com/rust-lang/crates.io-index"
3543
+checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b"
3544
+dependencies = [
3545
+ "same-file",
3546
+ "winapi-util",
3547
+]
3548
+
3549
+[[package]]
3550
+name = "want"
3551
+version = "0.3.1"
3552
+source = "registry+https://github.com/rust-lang/crates.io-index"
3553
+checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
3554
+dependencies = [
3555
+ "try-lock",
3556
+]
3557
+
3558
+[[package]]
3559
+name = "wasi"
3560
+version = "0.11.1+wasi-snapshot-preview1"
3561
+source = "registry+https://github.com/rust-lang/crates.io-index"
3562
+checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
3563
+
3564
+[[package]]
3565
+name = "wasip2"
3566
+version = "1.0.1+wasi-0.2.4"
3567
+source = "registry+https://github.com/rust-lang/crates.io-index"
3568
+checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
3569
+dependencies = [
3570
+ "wit-bindgen",
3571
+]
3572
+
3573
+[[package]]
3574
+name = "wasm-bindgen"
3575
+version = "0.2.105"
3576
+source = "registry+https://github.com/rust-lang/crates.io-index"
3577
+checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60"
3578
+dependencies = [
3579
+ "cfg-if",
3580
+ "once_cell",
3581
+ "rustversion",
3582
+ "wasm-bindgen-macro",
3583
+ "wasm-bindgen-shared",
3584
+]
3585
+
3586
+[[package]]
3587
+name = "wasm-bindgen-futures"
3588
+version = "0.4.55"
3589
+source = "registry+https://github.com/rust-lang/crates.io-index"
3590
+checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0"
3591
+dependencies = [
3592
+ "cfg-if",
3593
+ "js-sys",
3594
+ "once_cell",
3595
+ "wasm-bindgen",
3596
+ "web-sys",
3597
+]
3598
+
3599
+[[package]]
3600
+name = "wasm-bindgen-macro"
3601
+version = "0.2.105"
3602
+source = "registry+https://github.com/rust-lang/crates.io-index"
3603
+checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2"
3604
+dependencies = [
3605
+ "quote",
3606
+ "wasm-bindgen-macro-support",
3607
+]
3608
+
3609
+[[package]]
3610
+name = "wasm-bindgen-macro-support"
3611
+version = "0.2.105"
3612
+source = "registry+https://github.com/rust-lang/crates.io-index"
3613
+checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc"
3614
+dependencies = [
3615
+ "bumpalo",
3616
+ "proc-macro2",
3617
+ "quote",
3618
+ "syn 2.0.110",
3619
+ "wasm-bindgen-shared",
3620
+]
3621
+
3622
+[[package]]
3623
+name = "wasm-bindgen-shared"
3624
+version = "0.2.105"
3625
+source = "registry+https://github.com/rust-lang/crates.io-index"
3626
+checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76"
3627
+dependencies = [
3628
+ "unicode-ident",
3629
+]
3630
+
3631
+[[package]]
3632
+name = "web-sys"
3633
+version = "0.3.82"
3634
+source = "registry+https://github.com/rust-lang/crates.io-index"
3635
+checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1"
3636
+dependencies = [
3637
+ "js-sys",
3638
+ "wasm-bindgen",
3639
+]
3640
+
3641
+[[package]]
3642
+name = "web-time"
3643
+version = "1.1.0"
3644
+source = "registry+https://github.com/rust-lang/crates.io-index"
3645
+checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb"
3646
+dependencies = [
3647
+ "js-sys",
3648
+ "wasm-bindgen",
3649
+]
3650
+
3651
+[[package]]
3652
+name = "winapi"
3653
+version = "0.3.9"
3654
+source = "registry+https://github.com/rust-lang/crates.io-index"
3655
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
3656
+dependencies = [
3657
+ "winapi-i686-pc-windows-gnu",
3658
+ "winapi-x86_64-pc-windows-gnu",
3659
+]
3660
+
3661
+[[package]]
3662
+name = "winapi-i686-pc-windows-gnu"
3663
+version = "0.4.0"
3664
+source = "registry+https://github.com/rust-lang/crates.io-index"
3665
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
3666
+
3667
+[[package]]
3668
+name = "winapi-util"
3669
+version = "0.1.11"
3670
+source = "registry+https://github.com/rust-lang/crates.io-index"
3671
+checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
3672
+dependencies = [
3673
+ "windows-sys 0.61.2",
3674
+]
3675
+
3676
+[[package]]
3677
+name = "winapi-x86_64-pc-windows-gnu"
3678
+version = "0.4.0"
3679
+source = "registry+https://github.com/rust-lang/crates.io-index"
3680
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
3681
+
3682
+[[package]]
3683
+name = "windows-core"
3684
+version = "0.62.2"
3685
+source = "registry+https://github.com/rust-lang/crates.io-index"
3686
+checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb"
3687
+dependencies = [
3688
+ "windows-implement",
3689
+ "windows-interface",
3690
+ "windows-link",
3691
+ "windows-result",
3692
+ "windows-strings",
3693
+]
3694
+
3695
+[[package]]
3696
+name = "windows-implement"
3697
+version = "0.60.2"
3698
+source = "registry+https://github.com/rust-lang/crates.io-index"
3699
+checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf"
3700
+dependencies = [
3701
+ "proc-macro2",
3702
+ "quote",
3703
+ "syn 2.0.110",
3704
+]
3705
+
3706
+[[package]]
3707
+name = "windows-interface"
3708
+version = "0.59.3"
3709
+source = "registry+https://github.com/rust-lang/crates.io-index"
3710
+checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358"
3711
+dependencies = [
3712
+ "proc-macro2",
3713
+ "quote",
3714
+ "syn 2.0.110",
3715
+]
3716
+
3717
+[[package]]
3718
+name = "windows-link"
3719
+version = "0.2.1"
3720
+source = "registry+https://github.com/rust-lang/crates.io-index"
3721
+checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5"
3722
+
3723
+[[package]]
3724
+name = "windows-result"
3725
+version = "0.4.1"
3726
+source = "registry+https://github.com/rust-lang/crates.io-index"
3727
+checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5"
3728
+dependencies = [
3729
+ "windows-link",
3730
+]
3731
+
3732
+[[package]]
3733
+name = "windows-strings"
3734
+version = "0.5.1"
3735
+source = "registry+https://github.com/rust-lang/crates.io-index"
3736
+checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091"
3737
+dependencies = [
3738
+ "windows-link",
3739
+]
3740
+
3741
+[[package]]
3742
+name = "windows-sys"
3743
+version = "0.52.0"
3744
+source = "registry+https://github.com/rust-lang/crates.io-index"
3745
+checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
3746
+dependencies = [
3747
+ "windows-targets 0.52.6",
3748
+]
3749
+
3750
+[[package]]
3751
+name = "windows-sys"
3752
+version = "0.59.0"
3753
+source = "registry+https://github.com/rust-lang/crates.io-index"
3754
+checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
3755
+dependencies = [
3756
+ "windows-targets 0.52.6",
3757
+]
3758
+
3759
+[[package]]
3760
+name = "windows-sys"
3761
+version = "0.60.2"
3762
+source = "registry+https://github.com/rust-lang/crates.io-index"
3763
+checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
3764
+dependencies = [
3765
+ "windows-targets 0.53.5",
3766
+]
3767
+
3768
+[[package]]
3769
+name = "windows-sys"
3770
+version = "0.61.2"
3771
+source = "registry+https://github.com/rust-lang/crates.io-index"
3772
+checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc"
3773
+dependencies = [
3774
+ "windows-link",
3775
+]
3776
+
3777
+[[package]]
3778
+name = "windows-targets"
3779
+version = "0.52.6"
3780
+source = "registry+https://github.com/rust-lang/crates.io-index"
3781
+checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
3782
+dependencies = [
3783
+ "windows_aarch64_gnullvm 0.52.6",
3784
+ "windows_aarch64_msvc 0.52.6",
3785
+ "windows_i686_gnu 0.52.6",
3786
+ "windows_i686_gnullvm 0.52.6",
3787
+ "windows_i686_msvc 0.52.6",
3788
+ "windows_x86_64_gnu 0.52.6",
3789
+ "windows_x86_64_gnullvm 0.52.6",
3790
+ "windows_x86_64_msvc 0.52.6",
3791
+]
3792
+
3793
+[[package]]
3794
+name = "windows-targets"
3795
+version = "0.53.5"
3796
+source = "registry+https://github.com/rust-lang/crates.io-index"
3797
+checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3"
3798
+dependencies = [
3799
+ "windows-link",
3800
+ "windows_aarch64_gnullvm 0.53.1",
3801
+ "windows_aarch64_msvc 0.53.1",
3802
+ "windows_i686_gnu 0.53.1",
3803
+ "windows_i686_gnullvm 0.53.1",
3804
+ "windows_i686_msvc 0.53.1",
3805
+ "windows_x86_64_gnu 0.53.1",
3806
+ "windows_x86_64_gnullvm 0.53.1",
3807
+ "windows_x86_64_msvc 0.53.1",
3808
+]
3809
+
3810
+[[package]]
3811
+name = "windows_aarch64_gnullvm"
3812
+version = "0.52.6"
3813
+source = "registry+https://github.com/rust-lang/crates.io-index"
3814
+checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
3815
+
3816
+[[package]]
3817
+name = "windows_aarch64_gnullvm"
3818
+version = "0.53.1"
3819
+source = "registry+https://github.com/rust-lang/crates.io-index"
3820
+checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53"
3821
+
3822
+[[package]]
3823
+name = "windows_aarch64_msvc"
3824
+version = "0.52.6"
3825
+source = "registry+https://github.com/rust-lang/crates.io-index"
3826
+checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
3827
+
3828
+[[package]]
3829
+name = "windows_aarch64_msvc"
3830
+version = "0.53.1"
3831
+source = "registry+https://github.com/rust-lang/crates.io-index"
3832
+checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006"
3833
+
3834
+[[package]]
3835
+name = "windows_i686_gnu"
3836
+version = "0.52.6"
3837
+source = "registry+https://github.com/rust-lang/crates.io-index"
3838
+checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
3839
+
3840
+[[package]]
3841
+name = "windows_i686_gnu"
3842
+version = "0.53.1"
3843
+source = "registry+https://github.com/rust-lang/crates.io-index"
3844
+checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3"
3845
+
3846
+[[package]]
3847
+name = "windows_i686_gnullvm"
3848
+version = "0.52.6"
3849
+source = "registry+https://github.com/rust-lang/crates.io-index"
3850
+checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
3851
+
3852
+[[package]]
3853
+name = "windows_i686_gnullvm"
3854
+version = "0.53.1"
3855
+source = "registry+https://github.com/rust-lang/crates.io-index"
3856
+checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c"
3857
+
3858
+[[package]]
3859
+name = "windows_i686_msvc"
3860
+version = "0.52.6"
3861
+source = "registry+https://github.com/rust-lang/crates.io-index"
3862
+checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
3863
+
3864
+[[package]]
3865
+name = "windows_i686_msvc"
3866
+version = "0.53.1"
3867
+source = "registry+https://github.com/rust-lang/crates.io-index"
3868
+checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2"
3869
+
3870
+[[package]]
3871
+name = "windows_x86_64_gnu"
3872
+version = "0.52.6"
3873
+source = "registry+https://github.com/rust-lang/crates.io-index"
3874
+checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
3875
+
3876
+[[package]]
3877
+name = "windows_x86_64_gnu"
3878
+version = "0.53.1"
3879
+source = "registry+https://github.com/rust-lang/crates.io-index"
3880
+checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499"
3881
+
3882
+[[package]]
3883
+name = "windows_x86_64_gnullvm"
3884
+version = "0.52.6"
3885
+source = "registry+https://github.com/rust-lang/crates.io-index"
3886
+checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
3887
+
3888
+[[package]]
3889
+name = "windows_x86_64_gnullvm"
3890
+version = "0.53.1"
3891
+source = "registry+https://github.com/rust-lang/crates.io-index"
3892
+checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1"
3893
+
3894
+[[package]]
3895
+name = "windows_x86_64_msvc"
3896
+version = "0.52.6"
3897
+source = "registry+https://github.com/rust-lang/crates.io-index"
3898
+checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
3899
+
3900
+[[package]]
3901
+name = "windows_x86_64_msvc"
3902
+version = "0.53.1"
3903
+source = "registry+https://github.com/rust-lang/crates.io-index"
3904
+checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650"
3905
+
3906
+[[package]]
3907
+name = "winnow"
3908
+version = "0.7.13"
3909
+source = "registry+https://github.com/rust-lang/crates.io-index"
3910
+checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
3911
+
3912
+[[package]]
3913
+name = "wit-bindgen"
3914
+version = "0.46.0"
3915
+source = "registry+https://github.com/rust-lang/crates.io-index"
3916
+checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
3917
+
3918
+[[package]]
3919
+name = "writeable"
3920
+version = "0.6.2"
3921
+source = "registry+https://github.com/rust-lang/crates.io-index"
3922
+checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9"
3923
+
3924
+[[package]]
3925
+name = "yoke"
3926
+version = "0.8.1"
3927
+source = "registry+https://github.com/rust-lang/crates.io-index"
3928
+checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954"
3929
+dependencies = [
3930
+ "stable_deref_trait",
3931
+ "yoke-derive",
3932
+ "zerofrom",
3933
+]
3934
+
3935
+[[package]]
3936
+name = "yoke-derive"
3937
+version = "0.8.1"
3938
+source = "registry+https://github.com/rust-lang/crates.io-index"
3939
+checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d"
3940
+dependencies = [
3941
+ "proc-macro2",
3942
+ "quote",
3943
+ "syn 2.0.110",
3944
+ "synstructure",
3945
+]
3946
+
3947
+[[package]]
3948
+name = "zerocopy"
3949
+version = "0.8.27"
3950
+source = "registry+https://github.com/rust-lang/crates.io-index"
3951
+checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
3952
+dependencies = [
3953
+ "zerocopy-derive 0.8.27",
3954
+]
3955
+
3956
+[[package]]
3957
+name = "zerocopy"
3958
+version = "0.9.0-alpha.0"
3959
+source = "registry+https://github.com/rust-lang/crates.io-index"
3960
+checksum = "3c9c378180b968f2e98efa0c4bba8c49fd4cd247dc82a7f01e72dc137d447d8c"
3961
+dependencies = [
3962
+ "zerocopy-derive 0.9.0-alpha.0",
3963
+]
3964
+
3965
+[[package]]
3966
+name = "zerocopy-derive"
3967
+version = "0.8.27"
3968
+source = "registry+https://github.com/rust-lang/crates.io-index"
3969
+checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
3970
+dependencies = [
3971
+ "proc-macro2",
3972
+ "quote",
3973
+ "syn 2.0.110",
3974
+]
3975
+
3976
+[[package]]
3977
+name = "zerocopy-derive"
3978
+version = "0.9.0-alpha.0"
3979
+source = "registry+https://github.com/rust-lang/crates.io-index"
3980
+checksum = "36811c5ea90dd441f941919fc99163e702f150a8e1495d72a268106a0100df09"
3981
+dependencies = [
3982
+ "proc-macro2",
3983
+ "quote",
3984
+ "syn 2.0.110",
3985
+]
3986
+
3987
+[[package]]
3988
+name = "zerofrom"
3989
+version = "0.1.6"
3990
+source = "registry+https://github.com/rust-lang/crates.io-index"
3991
+checksum = "50cc42e0333e05660c3587f3bf9d0478688e15d870fab3346451ce7f8c9fbea5"
3992
+dependencies = [
3993
+ "zerofrom-derive",
3994
+]
3995
+
3996
+[[package]]
3997
+name = "zerofrom-derive"
3998
+version = "0.1.6"
3999
+source = "registry+https://github.com/rust-lang/crates.io-index"
4000
+checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502"
4001
+dependencies = [
4002
+ "proc-macro2",
4003
+ "quote",
4004
+ "syn 2.0.110",
4005
+ "synstructure",
4006
+]
4007
+
4008
+[[package]]
4009
+name = "zeroize"
4010
+version = "1.8.2"
4011
+source = "registry+https://github.com/rust-lang/crates.io-index"
4012
+checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
4013
+
4014
+[[package]]
4015
+name = "zerotrie"
4016
+version = "0.2.3"
4017
+source = "registry+https://github.com/rust-lang/crates.io-index"
4018
+checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851"
4019
+dependencies = [
4020
+ "displaydoc",
4021
+ "yoke",
4022
+ "zerofrom",
4023
+]
4024
+
4025
+[[package]]
4026
+name = "zerovec"
4027
+version = "0.11.5"
4028
+source = "registry+https://github.com/rust-lang/crates.io-index"
4029
+checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002"
4030
+dependencies = [
4031
+ "yoke",
4032
+ "zerofrom",
4033
+ "zerovec-derive",
4034
+]
4035
+
4036
+[[package]]
4037
+name = "zerovec-derive"
4038
+version = "0.11.2"
4039
+source = "registry+https://github.com/rust-lang/crates.io-index"
4040
+checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3"
4041
+dependencies = [
4042
+ "proc-macro2",
4043
+ "quote",
4044
+ "syn 2.0.110",
4045
+]
4046
+
4047
+[[package]]
4048
+name = "zstd"
4049
+version = "0.13.3"
4050
+source = "registry+https://github.com/rust-lang/crates.io-index"
4051
+checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a"
4052
+dependencies = [
4053
+ "zstd-safe",
4054
+]
4055
+
4056
+[[package]]
4057
+name = "zstd-safe"
4058
+version = "7.2.4"
4059
+source = "registry+https://github.com/rust-lang/crates.io-index"
4060
+checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d"
4061
+dependencies = [
4062
+ "zstd-sys",
4063
+]
4064
+
4065
+[[package]]
4066
+name = "zstd-sys"
4067
+version = "2.0.16+zstd.1.5.7"
4068
+source = "registry+https://github.com/rust-lang/crates.io-index"
4069
+checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748"
4070
+dependencies = [
4071
+ "cc",
4072
+ "pkg-config",
4073
+]
src/crates/Cargo.toml
new
+179
@@ -0,0 +1,179 @@
1
+[workspace]
2
+resolver = "2"
3
+members = [
4
+ # Individual crates
5
+ "journal-common",
6
+ "journal-core",
7
+ "journal-index",
8
+ "journal-log-writer",
9
+ "journal-engine",
10
+ "journal-registry",
11
+ "rdp",
12
+
13
+ # Netdata plugin workspace members
14
+ "netdata-plugin/foundation",
15
+ "netdata-plugin/error",
16
+ "netdata-plugin/protocol",
17
+ "netdata-plugin/rt",
18
+ "netdata-plugin/schema",
19
+ "netdata-plugin/types",
20
+ "netdata-plugin/bridge",
21
+ "netdata-plugin/charts-derive",
22
+
23
+ # Netdata log viewer workspace members
24
+ "netdata-log-viewer/journal-function",
25
+ "netdata-log-viewer/journal-viewer-plugin",
26
+
27
+ # Netdata OTEL workspace members
28
+ "netdata-otel/otel-plugin",
29
+ "netdata-otel/flatten_otel",
30
+]
31
+
32
+[workspace.package]
33
+version = "0.1.0"
34
+edition = "2024"
35
+rust-version = "1.85"
36
+
37
+[workspace.lints.clippy]
38
+collapsible_if = "allow"
39
+
40
+[workspace.dependencies]
41
+static_assertions = "1.1"
42
+thiserror = "2"
43
+zerocopy = { version = "0.9.0-alpha.0" }
44
+libc = "0.2"
45
+memmap2 = "0.9"
46
+uuid = { version = "1.0"}
47
+nix = { version = "0.30" }
48
+
49
+ruzstd = "0.8"
50
+zstd = "0.13"
51
+siphasher = "1.0"
52
+hashers = "1.0"
53
+twox-hash = { version = "2.1", default-features = false }
54
+rand = "0.9"
55
+lz4 = "1.25"
56
+md5 = "0.7"
57
+
58
+roaring = { git = "https://github.com/netdata/roaring-rs.git", branch="allocative" }
59
+serde = { version = "1.0" }
60
+rustc-hash = "2.1"
61
+bincode = { version = "1.3" }
62
+notify = "8.2"
63
+
64
+walkdir = "2.4"
65
+
66
+async-stream = "0.3"
67
+crossbeam-channel = "0.5"
68
+regex = "1.10"
69
+bitvec = "1.0.1"
70
+serde_json = "1.0"
71
+anyhow = "1.0"
72
+arbitrary = "1.3"
73
+chrono = { version = "0.4" }
74
+
75
+
76
+async-trait = "0.1.89"
77
+serde_yaml = "0.9.34"
78
+serde_regex = "1.1.0"
79
+tokio = { version = "1.45", features = ["full"] }
80
+
81
+# Tracing
82
+tracing = { version = "0.1" }
83
+tracing-journald = "0.3"
84
+tracing-futures = { version = "0.2.5" }
85
+tracing-opentelemetry = "0.32.0"
86
+tracing-subscriber = "0.3"
87
+opentelemetry = "0.31.0"
88
+opentelemetry_sdk = { version = "0.31.0", features = ["rt-tokio", "logs"] }
89
+opentelemetry-otlp = { version = "0.31.0", features = ["grpc-tonic", "logs"] }
90
+opentelemetry-proto = "0.31"
91
+
92
+# Netdata plugin
93
+schemars = "1.0"
94
+
95
+# Journal histogram dependencies
96
+atoi = "2.0"
97
+bitflags = "2.9"
98
+bytes = "1"
99
+futures = "0.3"
100
+phf = { version = "0.12", default-features = false }
101
+phf_codegen = "0.12"
102
+tokio-stream = "0.1"
103
+tokio-util = { version = "0.7", features = ["codec"] }
104
+foyer = { version = "0.20.0", features = ["serde"] }
105
+lru = "0.16"
106
+parking_lot = "0.12"
107
+allocative = { version = "0.3.4" }
108
+
109
+hdrhistogram = { version = "7.5" }
110
+rayon = "1.11.0"
111
+itoa = "1.0"
112
+
113
+# OTEL-specific dependencies
114
+atty = "0.2"
115
+bytesize = "1.3"
116
+bytesize-serde = "0.2"
117
+clap = { version = "4", features = ["derive"] }
118
+duct = "0.13"
119
+humantime = "2.2"
120
+humantime-serde = "1.1"
121
+hyper-util = "0.1"
122
+indexmap = "2.9"
123
+tonic = "0.14"
124
+tower = "0.5"
125
+
126
+flatten-serde-json = { git = "https://github.com/meilisearch/meilisearch", tag = "v1.22.1" }
127
+
128
+# Internal workspace crates
129
+journal-common = { path = "journal-common" }
130
+journal-core = { path = "journal-core" }
131
+journal-index = { path = "journal-index" }
132
+journal-log-writer = { path = "journal-log-writer" }
133
+journal-engine = { path = "journal-engine" }
134
+journal-registry = { path = "journal-registry" }
135
+journalctl = { path = "journalctl" }
136
+rdp = { path = "rdp" }
137
+
138
+# Netdata plugin crates
139
+foundation = { path = "netdata-plugin/foundation" }
140
+netdata-plugin-error = { path = "netdata-plugin/error" }
141
+netdata-plugin-protocol = { path = "netdata-plugin/protocol" }
142
+netdata-plugin-rt = { path = "netdata-plugin/rt" }
143
+netdata-plugin-schema = { path = "netdata-plugin/schema" }
144
+netdata-plugin-types = { path = "netdata-plugin/types" }
145
+netdata-plugin-bridge = { path = "netdata-plugin/bridge" }
146
+netdata-plugin-charts-derive = { path = "netdata-plugin/charts-derive" }
147
+rt = { path = "netdata-plugin/rt" }
148
+
149
+# Netdata log viewer crates
150
+journal-function = { path = "netdata-log-viewer/journal-function" }
151
+journal-viewer-plugin = { path = "netdata-log-viewer/journal-viewer-plugin" }
152
+
153
+# Netdata OTEL crates
154
+otel-plugin = { path = "netdata-otel/otel-plugin" }
155
+flatten_otel = { path = "netdata-otel/flatten_otel" }
156
+
157
+tempfile = { version = "3" }
158
+
159
+proptest = "1.9"
160
+
161
+[profile.dev]
162
+opt-level = 1
163
+debug = true
164
+debug-assertions = true
165
+
166
+[profile.release]
167
+debug = true
168
+# lto = true
169
+opt-level = "z"
170
+# strip = true
171
+# codegen-units = 1
172
+
173
+[profile.release-min]
174
+inherits = "release"
175
+opt-level = "z"
176
+lto = "fat"
177
+codegen-units = 1
178
+strip = true
179
+panic = "abort"
src/crates/LICENSE
renamed
src/crates/jf/.gitignore
deleted
-3
@@ -1,3 +0,0 @@
1
-target/
2
-.idea/
3
-journal_reader_ffi/journal_reader_ffi.h
src/crates/jf/Cargo.lock
deleted
-2221
@@ -1,2221 +0,0 @@
1
-# This file is automatically @generated by Cargo.
2
-# It is not intended for manual editing.
3
-version = 4
4
-
5
-[[package]]
6
-name = "addr2line"
7
-version = "0.25.1"
8
-source = "registry+https://github.com/rust-lang/crates.io-index"
9
-checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b"
10
-dependencies = [
11
- "gimli",
12
-]
13
-
14
-[[package]]
15
-name = "adler2"
16
-version = "2.0.1"
17
-source = "registry+https://github.com/rust-lang/crates.io-index"
18
-checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
19
-
20
-[[package]]
21
-name = "aho-corasick"
22
-version = "1.1.3"
23
-source = "registry+https://github.com/rust-lang/crates.io-index"
24
-checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"
25
-dependencies = [
26
- "memchr",
27
-]
28
-
29
-[[package]]
30
-name = "android_system_properties"
31
-version = "0.1.5"
32
-source = "registry+https://github.com/rust-lang/crates.io-index"
33
-checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311"
34
-dependencies = [
35
- "libc",
36
-]
37
-
38
-[[package]]
39
-name = "anstream"
40
-version = "0.6.20"
41
-source = "registry+https://github.com/rust-lang/crates.io-index"
42
-checksum = "3ae563653d1938f79b1ab1b5e668c87c76a9930414574a6583a7b7e11a8e6192"
43
-dependencies = [
44
- "anstyle",
45
- "anstyle-parse",
46
- "anstyle-query",
47
- "anstyle-wincon",
48
- "colorchoice",
49
- "is_terminal_polyfill",
50
- "utf8parse",
51
-]
52
-
53
-[[package]]
54
-name = "anstyle"
55
-version = "1.0.13"
56
-source = "registry+https://github.com/rust-lang/crates.io-index"
57
-checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78"
58
-
59
-[[package]]
60
-name = "anstyle-parse"
61
-version = "0.2.7"
62
-source = "registry+https://github.com/rust-lang/crates.io-index"
63
-checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2"
64
-dependencies = [
65
- "utf8parse",
66
-]
67
-
68
-[[package]]
69
-name = "anstyle-query"
70
-version = "1.1.4"
71
-source = "registry+https://github.com/rust-lang/crates.io-index"
72
-checksum = "9e231f6134f61b71076a3eab506c379d4f36122f2af15a9ff04415ea4c3339e2"
73
-dependencies = [
74
- "windows-sys 0.60.2",
75
-]
76
-
77
-[[package]]
78
-name = "anstyle-wincon"
79
-version = "3.0.10"
80
-source = "registry+https://github.com/rust-lang/crates.io-index"
81
-checksum = "3e0633414522a32ffaac8ac6cc8f748e090c5717661fddeea04219e2344f5f2a"
82
-dependencies = [
83
- "anstyle",
84
- "once_cell_polyfill",
85
- "windows-sys 0.60.2",
86
-]
87
-
88
-[[package]]
89
-name = "anyhow"
90
-version = "1.0.100"
91
-source = "registry+https://github.com/rust-lang/crates.io-index"
92
-checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61"
93
-
94
-[[package]]
95
-name = "async-trait"
96
-version = "0.1.89"
97
-source = "registry+https://github.com/rust-lang/crates.io-index"
98
-checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb"
99
-dependencies = [
100
- "proc-macro2",
101
- "quote",
102
- "syn",
103
-]
104
-
105
-[[package]]
106
-name = "atomic-waker"
107
-version = "1.1.2"
108
-source = "registry+https://github.com/rust-lang/crates.io-index"
109
-checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0"
110
-
111
-[[package]]
112
-name = "atty"
113
-version = "0.2.14"
114
-source = "registry+https://github.com/rust-lang/crates.io-index"
115
-checksum = "d9b39be18770d11421cdb1b9947a45dd3f37e93092cbf377614828a319d5fee8"
116
-dependencies = [
117
- "hermit-abi",
118
- "libc",
119
- "winapi",
120
-]
121
-
122
-[[package]]
123
-name = "autocfg"
124
-version = "1.5.0"
125
-source = "registry+https://github.com/rust-lang/crates.io-index"
126
-checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8"
127
-
128
-[[package]]
129
-name = "axum"
130
-version = "0.8.5"
131
-source = "registry+https://github.com/rust-lang/crates.io-index"
132
-checksum = "98e529aee37b5c8206bb4bf4c44797127566d72f76952c970bd3d1e85de8f4e2"
133
-dependencies = [
134
- "axum-core",
135
- "bytes",
136
- "futures-util",
137
- "http",
138
- "http-body",
139
- "http-body-util",
140
- "itoa",
141
- "matchit",
142
- "memchr",
143
- "mime",
144
- "percent-encoding",
145
- "pin-project-lite",
146
- "serde_core",
147
- "sync_wrapper",
148
- "tower",
149
- "tower-layer",
150
- "tower-service",
151
-]
152
-
153
-[[package]]
154
-name = "axum-core"
155
-version = "0.5.4"
156
-source = "registry+https://github.com/rust-lang/crates.io-index"
157
-checksum = "0ac7a6beb1182c7e30253ee75c3e918080bfb83f5a3023bcdf7209d85fd147e6"
158
-dependencies = [
159
- "bytes",
160
- "futures-core",
161
- "http",
162
- "http-body",
163
- "http-body-util",
164
- "mime",
165
- "pin-project-lite",
166
- "sync_wrapper",
167
- "tower-layer",
168
- "tower-service",
169
-]
170
-
171
-[[package]]
172
-name = "backtrace"
173
-version = "0.3.76"
174
-source = "registry+https://github.com/rust-lang/crates.io-index"
175
-checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6"
176
-dependencies = [
177
- "addr2line",
178
- "cfg-if",
179
- "libc",
180
- "miniz_oxide",
181
- "object",
182
- "rustc-demangle",
183
- "windows-link",
184
-]
185
-
186
-[[package]]
187
-name = "base64"
188
-version = "0.21.7"
189
-source = "registry+https://github.com/rust-lang/crates.io-index"
190
-checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567"
191
-
192
-[[package]]
193
-name = "base64"
194
-version = "0.22.1"
195
-source = "registry+https://github.com/rust-lang/crates.io-index"
196
-checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6"
197
-
198
-[[package]]
199
-name = "bitflags"
200
-version = "2.9.4"
201
-source = "registry+https://github.com/rust-lang/crates.io-index"
202
-checksum = "2261d10cca569e4643e526d8dc2e62e433cc8aba21ab764233731f8d369bf394"
203
-
204
-[[package]]
205
-name = "bumpalo"
206
-version = "3.19.0"
207
-source = "registry+https://github.com/rust-lang/crates.io-index"
208
-checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43"
209
-
210
-[[package]]
211
-name = "bytes"
212
-version = "1.10.1"
213
-source = "registry+https://github.com/rust-lang/crates.io-index"
214
-checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
215
-
216
-[[package]]
217
-name = "bytesize"
218
-version = "1.3.3"
219
-source = "registry+https://github.com/rust-lang/crates.io-index"
220
-checksum = "2e93abca9e28e0a1b9877922aacb20576e05d4679ffa78c3d6dc22a26a216659"
221
-
222
-[[package]]
223
-name = "bytesize-serde"
224
-version = "0.2.1"
225
-source = "registry+https://github.com/rust-lang/crates.io-index"
226
-checksum = "86d1eb2fd2668859e9785b99700c66b7ea9fdeda35d99f95827a4e5493440178"
227
-dependencies = [
228
- "bytesize",
229
- "serde",
230
-]
231
-
232
-[[package]]
233
-name = "cbindgen"
234
-version = "0.28.0"
235
-source = "registry+https://github.com/rust-lang/crates.io-index"
236
-checksum = "eadd868a2ce9ca38de7eeafdcec9c7065ef89b42b32f0839278d55f35c54d1ff"
237
-dependencies = [
238
- "clap",
239
- "heck 0.4.1",
240
- "indexmap",
241
- "log",
242
- "proc-macro2",
243
- "quote",
244
- "serde",
245
- "serde_json",
246
- "syn",
247
- "tempfile",
248
- "toml",
249
-]
250
-
251
-[[package]]
252
-name = "cc"
253
-version = "1.2.39"
254
-source = "registry+https://github.com/rust-lang/crates.io-index"
255
-checksum = "e1354349954c6fc9cb0deab020f27f783cf0b604e8bb754dc4658ecf0d29c35f"
256
-dependencies = [
257
- "find-msvc-tools",
258
- "shlex",
259
-]
260
-
261
-[[package]]
262
-name = "cfg-if"
263
-version = "1.0.3"
264
-source = "registry+https://github.com/rust-lang/crates.io-index"
265
-checksum = "2fd1289c04a9ea8cb22300a459a72a385d7c73d3259e2ed7dcb2af674838cfa9"
266
-
267
-[[package]]
268
-name = "chrono"
269
-version = "0.4.42"
270
-source = "registry+https://github.com/rust-lang/crates.io-index"
271
-checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2"
272
-dependencies = [
273
- "iana-time-zone",
274
- "js-sys",
275
- "num-traits",
276
- "serde",
277
- "wasm-bindgen",
278
- "windows-link",
279
-]
280
-
281
-[[package]]
282
-name = "clap"
283
-version = "4.5.48"
284
-source = "registry+https://github.com/rust-lang/crates.io-index"
285
-checksum = "e2134bb3ea021b78629caa971416385309e0131b351b25e01dc16fb54e1b5fae"
286
-dependencies = [
287
- "clap_builder",
288
- "clap_derive",
289
-]
290
-
291
-[[package]]
292
-name = "clap_builder"
293
-version = "4.5.48"
294
-source = "registry+https://github.com/rust-lang/crates.io-index"
295
-checksum = "c2ba64afa3c0a6df7fa517765e31314e983f51dda798ffba27b988194fb65dc9"
296
-dependencies = [
297
- "anstream",
298
- "anstyle",
299
- "clap_lex",
300
- "strsim",
301
-]
302
-
303
-[[package]]
304
-name = "clap_derive"
305
-version = "4.5.47"
306
-source = "registry+https://github.com/rust-lang/crates.io-index"
307
-checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c"
308
-dependencies = [
309
- "heck 0.5.0",
310
- "proc-macro2",
311
- "quote",
312
- "syn",
313
-]
314
-
315
-[[package]]
316
-name = "clap_lex"
317
-version = "0.7.5"
318
-source = "registry+https://github.com/rust-lang/crates.io-index"
319
-checksum = "b94f61472cee1439c0b966b47e3aca9ae07e45d070759512cd390ea2bebc6675"
320
-
321
-[[package]]
322
-name = "colorchoice"
323
-version = "1.0.4"
324
-source = "registry+https://github.com/rust-lang/crates.io-index"
325
-checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75"
326
-
327
-[[package]]
328
-name = "core-foundation-sys"
329
-version = "0.8.7"
330
-source = "registry+https://github.com/rust-lang/crates.io-index"
331
-checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b"
332
-
333
-[[package]]
334
-name = "crc32fast"
335
-version = "1.5.0"
336
-source = "registry+https://github.com/rust-lang/crates.io-index"
337
-checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511"
338
-dependencies = [
339
- "cfg-if",
340
-]
341
-
342
-[[package]]
343
-name = "either"
344
-version = "1.15.0"
345
-source = "registry+https://github.com/rust-lang/crates.io-index"
346
-checksum = "48c757948c5ede0e46177b7add2e67155f70e33c07fea8284df6576da70b3719"
347
-
348
-[[package]]
349
-name = "equivalent"
350
-version = "1.0.2"
351
-source = "registry+https://github.com/rust-lang/crates.io-index"
352
-checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f"
353
-
354
-[[package]]
355
-name = "errno"
356
-version = "0.3.14"
357
-source = "registry+https://github.com/rust-lang/crates.io-index"
358
-checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
359
-dependencies = [
360
- "libc",
361
- "windows-sys 0.61.1",
362
-]
363
-
364
-[[package]]
365
-name = "error"
366
-version = "0.1.0"
367
-dependencies = [
368
- "static_assertions",
369
- "thiserror",
370
- "zerocopy 0.9.0-alpha.0",
371
-]
372
-
373
-[[package]]
374
-name = "fastrand"
375
-version = "2.3.0"
376
-source = "registry+https://github.com/rust-lang/crates.io-index"
377
-checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be"
378
-
379
-[[package]]
380
-name = "find-msvc-tools"
381
-version = "0.1.2"
382
-source = "registry+https://github.com/rust-lang/crates.io-index"
383
-checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959"
384
-
385
-[[package]]
386
-name = "fixedbitset"
387
-version = "0.5.7"
388
-source = "registry+https://github.com/rust-lang/crates.io-index"
389
-checksum = "1d674e81391d1e1ab681a28d99df07927c6d4aa5b027d7da16ba32d1d21ecd99"
390
-
391
-[[package]]
392
-name = "flate2"
393
-version = "1.1.2"
394
-source = "registry+https://github.com/rust-lang/crates.io-index"
395
-checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d"
396
-dependencies = [
397
- "crc32fast",
398
- "miniz_oxide",
399
-]
400
-
401
-[[package]]
402
-name = "flatten-serde-json"
403
-version = "1.22.1"
404
-source = "git+https://github.com/meilisearch/meilisearch?tag=v1.22.1#077ec2ab11bb4daefcb57f89eab9cff16e075fdc"
405
-dependencies = [
406
- "serde_json",
407
-]
408
-
409
-[[package]]
410
-name = "flatten_otel"
411
-version = "0.1.0"
412
-dependencies = [
413
- "flatten-serde-json",
414
- "opentelemetry-proto",
415
- "serde_json",
416
-]
417
-
418
-[[package]]
419
-name = "flog-otel"
420
-version = "0.1.0"
421
-dependencies = [
422
- "anyhow",
423
- "chrono",
424
- "clap",
425
- "opentelemetry-proto",
426
- "prost",
427
- "serde",
428
- "serde_json",
429
- "tokio",
430
- "tonic",
431
- "tonic-build",
432
- "tracing",
433
- "tracing-subscriber",
434
- "uuid",
435
-]
436
-
437
-[[package]]
438
-name = "fnv"
439
-version = "1.0.7"
440
-source = "registry+https://github.com/rust-lang/crates.io-index"
441
-checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
442
-
443
-[[package]]
444
-name = "futures-channel"
445
-version = "0.3.31"
446
-source = "registry+https://github.com/rust-lang/crates.io-index"
447
-checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
448
-dependencies = [
449
- "futures-core",
450
-]
451
-
452
-[[package]]
453
-name = "futures-core"
454
-version = "0.3.31"
455
-source = "registry+https://github.com/rust-lang/crates.io-index"
456
-checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
457
-
458
-[[package]]
459
-name = "futures-executor"
460
-version = "0.3.31"
461
-source = "registry+https://github.com/rust-lang/crates.io-index"
462
-checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
463
-dependencies = [
464
- "futures-core",
465
- "futures-task",
466
- "futures-util",
467
-]
468
-
469
-[[package]]
470
-name = "futures-macro"
471
-version = "0.3.31"
472
-source = "registry+https://github.com/rust-lang/crates.io-index"
473
-checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
474
-dependencies = [
475
- "proc-macro2",
476
- "quote",
477
- "syn",
478
-]
479
-
480
-[[package]]
481
-name = "futures-sink"
482
-version = "0.3.31"
483
-source = "registry+https://github.com/rust-lang/crates.io-index"
484
-checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
485
-
486
-[[package]]
487
-name = "futures-task"
488
-version = "0.3.31"
489
-source = "registry+https://github.com/rust-lang/crates.io-index"
490
-checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
491
-
492
-[[package]]
493
-name = "futures-util"
494
-version = "0.3.31"
495
-source = "registry+https://github.com/rust-lang/crates.io-index"
496
-checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
497
-dependencies = [
498
- "futures-core",
499
- "futures-macro",
500
- "futures-sink",
501
- "futures-task",
502
- "pin-project-lite",
503
- "pin-utils",
504
- "slab",
505
-]
506
-
507
-[[package]]
508
-name = "getrandom"
509
-version = "0.2.16"
510
-source = "registry+https://github.com/rust-lang/crates.io-index"
511
-checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592"
512
-dependencies = [
513
- "cfg-if",
514
- "libc",
515
- "wasi 0.11.1+wasi-snapshot-preview1",
516
-]
517
-
518
-[[package]]
519
-name = "getrandom"
520
-version = "0.3.3"
521
-source = "registry+https://github.com/rust-lang/crates.io-index"
522
-checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4"
523
-dependencies = [
524
- "cfg-if",
525
- "libc",
526
- "r-efi",
527
- "wasi 0.14.7+wasi-0.2.4",
528
-]
529
-
530
-[[package]]
531
-name = "gimli"
532
-version = "0.32.3"
533
-source = "registry+https://github.com/rust-lang/crates.io-index"
534
-checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7"
535
-
536
-[[package]]
537
-name = "h2"
538
-version = "0.4.12"
539
-source = "registry+https://github.com/rust-lang/crates.io-index"
540
-checksum = "f3c0b69cfcb4e1b9f1bf2f53f95f766e4661169728ec61cd3fe5a0166f2d1386"
541
-dependencies = [
542
- "atomic-waker",
543
- "bytes",
544
- "fnv",
545
- "futures-core",
546
- "futures-sink",
547
- "http",
548
- "indexmap",
549
- "slab",
550
- "tokio",
551
- "tokio-util",
552
- "tracing",
553
-]
554
-
555
-[[package]]
556
-name = "hashbrown"
557
-version = "0.16.0"
558
-source = "registry+https://github.com/rust-lang/crates.io-index"
559
-checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d"
560
-
561
-[[package]]
562
-name = "heck"
563
-version = "0.4.1"
564
-source = "registry+https://github.com/rust-lang/crates.io-index"
565
-checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8"
566
-
567
-[[package]]
568
-name = "heck"
569
-version = "0.5.0"
570
-source = "registry+https://github.com/rust-lang/crates.io-index"
571
-checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea"
572
-
573
-[[package]]
574
-name = "hermit-abi"
575
-version = "0.1.19"
576
-source = "registry+https://github.com/rust-lang/crates.io-index"
577
-checksum = "62b467343b94ba476dcb2500d242dadbb39557df889310ac77c5d99100aaac33"
578
-dependencies = [
579
- "libc",
580
-]
581
-
582
-[[package]]
583
-name = "hex"
584
-version = "0.4.3"
585
-source = "registry+https://github.com/rust-lang/crates.io-index"
586
-checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70"
587
-
588
-[[package]]
589
-name = "http"
590
-version = "1.3.1"
591
-source = "registry+https://github.com/rust-lang/crates.io-index"
592
-checksum = "f4a85d31aea989eead29a3aaf9e1115a180df8282431156e533de47660892565"
593
-dependencies = [
594
- "bytes",
595
- "fnv",
596
- "itoa",
597
-]
598
-
599
-[[package]]
600
-name = "http-body"
601
-version = "1.0.1"
602
-source = "registry+https://github.com/rust-lang/crates.io-index"
603
-checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184"
604
-dependencies = [
605
- "bytes",
606
- "http",
607
-]
608
-
609
-[[package]]
610
-name = "http-body-util"
611
-version = "0.1.3"
612
-source = "registry+https://github.com/rust-lang/crates.io-index"
613
-checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a"
614
-dependencies = [
615
- "bytes",
616
- "futures-core",
617
- "http",
618
- "http-body",
619
- "pin-project-lite",
620
-]
621
-
622
-[[package]]
623
-name = "httparse"
624
-version = "1.10.1"
625
-source = "registry+https://github.com/rust-lang/crates.io-index"
626
-checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87"
627
-
628
-[[package]]
629
-name = "httpdate"
630
-version = "1.0.3"
631
-source = "registry+https://github.com/rust-lang/crates.io-index"
632
-checksum = "df3b46402a9d5adb4c86a0cf463f42e19994e3ee891101b1841f30a545cb49a9"
633
-
634
-[[package]]
635
-name = "humantime"
636
-version = "2.3.0"
637
-source = "registry+https://github.com/rust-lang/crates.io-index"
638
-checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424"
639
-
640
-[[package]]
641
-name = "humantime-serde"
642
-version = "1.1.1"
643
-source = "registry+https://github.com/rust-lang/crates.io-index"
644
-checksum = "57a3db5ea5923d99402c94e9feb261dc5ee9b4efa158b0315f788cf549cc200c"
645
-dependencies = [
646
- "humantime",
647
- "serde",
648
-]
649
-
650
-[[package]]
651
-name = "hyper"
652
-version = "1.7.0"
653
-source = "registry+https://github.com/rust-lang/crates.io-index"
654
-checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e"
655
-dependencies = [
656
- "atomic-waker",
657
- "bytes",
658
- "futures-channel",
659
- "futures-core",
660
- "h2",
661
- "http",
662
- "http-body",
663
- "httparse",
664
- "httpdate",
665
- "itoa",
666
- "pin-project-lite",
667
- "pin-utils",
668
- "smallvec",
669
- "tokio",
670
- "want",
671
-]
672
-
673
-[[package]]
674
-name = "hyper-timeout"
675
-version = "0.5.2"
676
-source = "registry+https://github.com/rust-lang/crates.io-index"
677
-checksum = "2b90d566bffbce6a75bd8b09a05aa8c2cb1fabb6cb348f8840c9e4c90a0d83b0"
678
-dependencies = [
679
- "hyper",
680
- "hyper-util",
681
- "pin-project-lite",
682
- "tokio",
683
- "tower-service",
684
-]
685
-
686
-[[package]]
687
-name = "hyper-util"
688
-version = "0.1.17"
689
-source = "registry+https://github.com/rust-lang/crates.io-index"
690
-checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8"
691
-dependencies = [
692
- "bytes",
693
- "futures-channel",
694
- "futures-core",
695
- "futures-util",
696
- "http",
697
- "http-body",
698
- "hyper",
699
- "libc",
700
- "pin-project-lite",
701
- "socket2 0.6.0",
702
- "tokio",
703
- "tower-service",
704
- "tracing",
705
-]
706
-
707
-[[package]]
708
-name = "iana-time-zone"
709
-version = "0.1.64"
710
-source = "registry+https://github.com/rust-lang/crates.io-index"
711
-checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb"
712
-dependencies = [
713
- "android_system_properties",
714
- "core-foundation-sys",
715
- "iana-time-zone-haiku",
716
- "js-sys",
717
- "log",
718
- "wasm-bindgen",
719
- "windows-core",
720
-]
721
-
722
-[[package]]
723
-name = "iana-time-zone-haiku"
724
-version = "0.1.2"
725
-source = "registry+https://github.com/rust-lang/crates.io-index"
726
-checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f"
727
-dependencies = [
728
- "cc",
729
-]
730
-
731
-[[package]]
732
-name = "indexmap"
733
-version = "2.11.4"
734
-source = "registry+https://github.com/rust-lang/crates.io-index"
735
-checksum = "4b0f83760fb341a774ed326568e19f5a863af4a952def8c39f9ab92fd95b88e5"
736
-dependencies = [
737
- "equivalent",
738
- "hashbrown",
739
-]
740
-
741
-[[package]]
742
-name = "io-uring"
743
-version = "0.7.10"
744
-source = "registry+https://github.com/rust-lang/crates.io-index"
745
-checksum = "046fa2d4d00aea763528b4950358d0ead425372445dc8ff86312b3c69ff7727b"
746
-dependencies = [
747
- "bitflags",
748
- "cfg-if",
749
- "libc",
750
-]
751
-
752
-[[package]]
753
-name = "is_terminal_polyfill"
754
-version = "1.70.1"
755
-source = "registry+https://github.com/rust-lang/crates.io-index"
756
-checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf"
757
-
758
-[[package]]
759
-name = "itertools"
760
-version = "0.14.0"
761
-source = "registry+https://github.com/rust-lang/crates.io-index"
762
-checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285"
763
-dependencies = [
764
- "either",
765
-]
766
-
767
-[[package]]
768
-name = "itoa"
769
-version = "1.0.15"
770
-source = "registry+https://github.com/rust-lang/crates.io-index"
771
-checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c"
772
-
773
-[[package]]
774
-name = "journal_file"
775
-version = "0.1.0"
776
-dependencies = [
777
- "error",
778
- "hex",
779
- "memmap2",
780
- "rand",
781
- "ruzstd",
782
- "siphasher",
783
- "tempfile",
784
- "twox-hash",
785
- "window_manager",
786
- "zerocopy 0.9.0-alpha.0",
787
-]
788
-
789
-[[package]]
790
-name = "journal_log"
791
-version = "0.1.0"
792
-dependencies = [
793
- "error",
794
- "journal_file",
795
- "memmap2",
796
- "tempfile",
797
- "uuid",
798
-]
799
-
800
-[[package]]
801
-name = "journal_reader_ffi"
802
-version = "0.1.0"
803
-dependencies = [
804
- "cbindgen",
805
- "error",
806
- "journal_file",
807
- "memmap2",
808
- "serde_json",
809
- "sigbus",
810
-]
811
-
812
-[[package]]
813
-name = "js-sys"
814
-version = "0.3.81"
815
-source = "registry+https://github.com/rust-lang/crates.io-index"
816
-checksum = "ec48937a97411dcb524a265206ccd4c90bb711fca92b2792c407f268825b9305"
817
-dependencies = [
818
- "once_cell",
819
- "wasm-bindgen",
820
-]
821
-
822
-[[package]]
823
-name = "lazy_static"
824
-version = "1.5.0"
825
-source = "registry+https://github.com/rust-lang/crates.io-index"
826
-checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe"
827
-
828
-[[package]]
829
-name = "libc"
830
-version = "0.2.176"
831
-source = "registry+https://github.com/rust-lang/crates.io-index"
832
-checksum = "58f929b4d672ea937a23a1ab494143d968337a5f47e56d0815df1e0890ddf174"
833
-
834
-[[package]]
835
-name = "linux-raw-sys"
836
-version = "0.11.0"
837
-source = "registry+https://github.com/rust-lang/crates.io-index"
838
-checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039"
839
-
840
-[[package]]
841
-name = "log"
842
-version = "0.4.28"
843
-source = "registry+https://github.com/rust-lang/crates.io-index"
844
-checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432"
845
-
846
-[[package]]
847
-name = "matchit"
848
-version = "0.8.4"
849
-source = "registry+https://github.com/rust-lang/crates.io-index"
850
-checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3"
851
-
852
-[[package]]
853
-name = "memchr"
854
-version = "2.7.6"
855
-source = "registry+https://github.com/rust-lang/crates.io-index"
856
-checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273"
857
-
858
-[[package]]
859
-name = "memmap2"
860
-version = "0.9.8"
861
-source = "registry+https://github.com/rust-lang/crates.io-index"
862
-checksum = "843a98750cd611cc2965a8213b53b43e715f13c37a9e096c6408e69990961db7"
863
-dependencies = [
864
- "libc",
865
-]
866
-
867
-[[package]]
868
-name = "mime"
869
-version = "0.3.17"
870
-source = "registry+https://github.com/rust-lang/crates.io-index"
871
-checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a"
872
-
873
-[[package]]
874
-name = "miniz_oxide"
875
-version = "0.8.9"
876
-source = "registry+https://github.com/rust-lang/crates.io-index"
877
-checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316"
878
-dependencies = [
879
- "adler2",
880
-]
881
-
882
-[[package]]
883
-name = "mio"
884
-version = "1.0.4"
885
-source = "registry+https://github.com/rust-lang/crates.io-index"
886
-checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c"
887
-dependencies = [
888
- "libc",
889
- "wasi 0.11.1+wasi-snapshot-preview1",
890
- "windows-sys 0.59.0",
891
-]
892
-
893
-[[package]]
894
-name = "multimap"
895
-version = "0.10.1"
896
-source = "registry+https://github.com/rust-lang/crates.io-index"
897
-checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084"
898
-
899
-[[package]]
900
-name = "nu-ansi-term"
901
-version = "0.50.1"
902
-source = "registry+https://github.com/rust-lang/crates.io-index"
903
-checksum = "d4a28e057d01f97e61255210fcff094d74ed0466038633e95017f5beb68e4399"
904
-dependencies = [
905
- "windows-sys 0.52.0",
906
-]
907
-
908
-[[package]]
909
-name = "num-traits"
910
-version = "0.2.19"
911
-source = "registry+https://github.com/rust-lang/crates.io-index"
912
-checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841"
913
-dependencies = [
914
- "autocfg",
915
-]
916
-
917
-[[package]]
918
-name = "object"
919
-version = "0.37.3"
920
-source = "registry+https://github.com/rust-lang/crates.io-index"
921
-checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe"
922
-dependencies = [
923
- "memchr",
924
-]
925
-
926
-[[package]]
927
-name = "once_cell"
928
-version = "1.21.3"
929
-source = "registry+https://github.com/rust-lang/crates.io-index"
930
-checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d"
931
-
932
-[[package]]
933
-name = "once_cell_polyfill"
934
-version = "1.70.1"
935
-source = "registry+https://github.com/rust-lang/crates.io-index"
936
-checksum = "a4895175b425cb1f87721b59f0f286c2092bd4af812243672510e1ac53e2e0ad"
937
-
938
-[[package]]
939
-name = "opentelemetry"
940
-version = "0.30.0"
941
-source = "registry+https://github.com/rust-lang/crates.io-index"
942
-checksum = "aaf416e4cb72756655126f7dd7bb0af49c674f4c1b9903e80c009e0c37e552e6"
943
-dependencies = [
944
- "futures-core",
945
- "futures-sink",
946
- "js-sys",
947
- "pin-project-lite",
948
- "thiserror",
949
- "tracing",
950
-]
951
-
952
-[[package]]
953
-name = "opentelemetry-proto"
954
-version = "0.30.0"
955
-source = "registry+https://github.com/rust-lang/crates.io-index"
956
-checksum = "2e046fd7660710fe5a05e8748e70d9058dc15c94ba914e7c4faa7c728f0e8ddc"
957
-dependencies = [
958
- "base64 0.22.1",
959
- "hex",
960
- "opentelemetry",
961
- "opentelemetry_sdk",
962
- "prost",
963
- "serde",
964
- "tonic",
965
-]
966
-
967
-[[package]]
968
-name = "opentelemetry_sdk"
969
-version = "0.30.0"
970
-source = "registry+https://github.com/rust-lang/crates.io-index"
971
-checksum = "11f644aa9e5e31d11896e024305d7e3c98a88884d9f8919dbf37a9991bc47a4b"
972
-dependencies = [
973
- "futures-channel",
974
- "futures-executor",
975
- "futures-util",
976
- "opentelemetry",
977
- "percent-encoding",
978
- "rand",
979
- "serde_json",
980
- "thiserror",
981
-]
982
-
983
-[[package]]
984
-name = "otel-plugin"
985
-version = "0.1.0"
986
-dependencies = [
987
- "anyhow",
988
- "atty",
989
- "base64 0.21.7",
990
- "bytesize",
991
- "bytesize-serde",
992
- "clap",
993
- "flatten-serde-json",
994
- "flatten_otel",
995
- "humantime",
996
- "humantime-serde",
997
- "journal_file",
998
- "journal_log",
999
- "memmap2",
1000
- "opentelemetry-proto",
1001
- "prost",
1002
- "regex",
1003
- "serde",
1004
- "serde_json",
1005
- "serde_regex",
1006
- "serde_yaml",
1007
- "tokio",
1008
- "tonic",
1009
- "uuid",
1010
-]
1011
-
1012
-[[package]]
1013
-name = "percent-encoding"
1014
-version = "2.3.2"
1015
-source = "registry+https://github.com/rust-lang/crates.io-index"
1016
-checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220"
1017
-
1018
-[[package]]
1019
-name = "petgraph"
1020
-version = "0.7.1"
1021
-source = "registry+https://github.com/rust-lang/crates.io-index"
1022
-checksum = "3672b37090dbd86368a4145bc067582552b29c27377cad4e0a306c97f9bd7772"
1023
-dependencies = [
1024
- "fixedbitset",
1025
- "indexmap",
1026
-]
1027
-
1028
-[[package]]
1029
-name = "pin-project"
1030
-version = "1.1.10"
1031
-source = "registry+https://github.com/rust-lang/crates.io-index"
1032
-checksum = "677f1add503faace112b9f1373e43e9e054bfdd22ff1a63c1bc485eaec6a6a8a"
1033
-dependencies = [
1034
- "pin-project-internal",
1035
-]
1036
-
1037
-[[package]]
1038
-name = "pin-project-internal"
1039
-version = "1.1.10"
1040
-source = "registry+https://github.com/rust-lang/crates.io-index"
1041
-checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861"
1042
-dependencies = [
1043
- "proc-macro2",
1044
- "quote",
1045
- "syn",
1046
-]
1047
-
1048
-[[package]]
1049
-name = "pin-project-lite"
1050
-version = "0.2.16"
1051
-source = "registry+https://github.com/rust-lang/crates.io-index"
1052
-checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"
1053
-
1054
-[[package]]
1055
-name = "pin-utils"
1056
-version = "0.1.0"
1057
-source = "registry+https://github.com/rust-lang/crates.io-index"
1058
-checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
1059
-
1060
-[[package]]
1061
-name = "ppv-lite86"
1062
-version = "0.2.21"
1063
-source = "registry+https://github.com/rust-lang/crates.io-index"
1064
-checksum = "85eae3c4ed2f50dcfe72643da4befc30deadb458a9b590d720cde2f2b1e97da9"
1065
-dependencies = [
1066
- "zerocopy 0.8.27",
1067
-]
1068
-
1069
-[[package]]
1070
-name = "prettyplease"
1071
-version = "0.2.37"
1072
-source = "registry+https://github.com/rust-lang/crates.io-index"
1073
-checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b"
1074
-dependencies = [
1075
- "proc-macro2",
1076
- "syn",
1077
-]
1078
-
1079
-[[package]]
1080
-name = "proc-macro2"
1081
-version = "1.0.101"
1082
-source = "registry+https://github.com/rust-lang/crates.io-index"
1083
-checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de"
1084
-dependencies = [
1085
- "unicode-ident",
1086
-]
1087
-
1088
-[[package]]
1089
-name = "prost"
1090
-version = "0.13.5"
1091
-source = "registry+https://github.com/rust-lang/crates.io-index"
1092
-checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5"
1093
-dependencies = [
1094
- "bytes",
1095
- "prost-derive",
1096
-]
1097
-
1098
-[[package]]
1099
-name = "prost-build"
1100
-version = "0.13.5"
1101
-source = "registry+https://github.com/rust-lang/crates.io-index"
1102
-checksum = "be769465445e8c1474e9c5dac2018218498557af32d9ed057325ec9a41ae81bf"
1103
-dependencies = [
1104
- "heck 0.5.0",
1105
- "itertools",
1106
- "log",
1107
- "multimap",
1108
- "once_cell",
1109
- "petgraph",
1110
- "prettyplease",
1111
- "prost",
1112
- "prost-types",
1113
- "regex",
1114
- "syn",
1115
- "tempfile",
1116
-]
1117
-
1118
-[[package]]
1119
-name = "prost-derive"
1120
-version = "0.13.5"
1121
-source = "registry+https://github.com/rust-lang/crates.io-index"
1122
-checksum = "8a56d757972c98b346a9b766e3f02746cde6dd1cd1d1d563472929fdd74bec4d"
1123
-dependencies = [
1124
- "anyhow",
1125
- "itertools",
1126
- "proc-macro2",
1127
- "quote",
1128
- "syn",
1129
-]
1130
-
1131
-[[package]]
1132
-name = "prost-types"
1133
-version = "0.13.5"
1134
-source = "registry+https://github.com/rust-lang/crates.io-index"
1135
-checksum = "52c2c1bf36ddb1a1c396b3601a3cec27c2462e45f07c386894ec3ccf5332bd16"
1136
-dependencies = [
1137
- "prost",
1138
-]
1139
-
1140
-[[package]]
1141
-name = "quote"
1142
-version = "1.0.41"
1143
-source = "registry+https://github.com/rust-lang/crates.io-index"
1144
-checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1"
1145
-dependencies = [
1146
- "proc-macro2",
1147
-]
1148
-
1149
-[[package]]
1150
-name = "r-efi"
1151
-version = "5.3.0"
1152
-source = "registry+https://github.com/rust-lang/crates.io-index"
1153
-checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f"
1154
-
1155
-[[package]]
1156
-name = "rand"
1157
-version = "0.9.2"
1158
-source = "registry+https://github.com/rust-lang/crates.io-index"
1159
-checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1"
1160
-dependencies = [
1161
- "rand_chacha",
1162
- "rand_core",
1163
-]
1164
-
1165
-[[package]]
1166
-name = "rand_chacha"
1167
-version = "0.9.0"
1168
-source = "registry+https://github.com/rust-lang/crates.io-index"
1169
-checksum = "d3022b5f1df60f26e1ffddd6c66e8aa15de382ae63b3a0c1bfc0e4d3e3f325cb"
1170
-dependencies = [
1171
- "ppv-lite86",
1172
- "rand_core",
1173
-]
1174
-
1175
-[[package]]
1176
-name = "rand_core"
1177
-version = "0.9.3"
1178
-source = "registry+https://github.com/rust-lang/crates.io-index"
1179
-checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38"
1180
-dependencies = [
1181
- "getrandom 0.3.3",
1182
-]
1183
-
1184
-[[package]]
1185
-name = "regex"
1186
-version = "1.11.3"
1187
-source = "registry+https://github.com/rust-lang/crates.io-index"
1188
-checksum = "8b5288124840bee7b386bc413c487869b360b2b4ec421ea56425128692f2a82c"
1189
-dependencies = [
1190
- "aho-corasick",
1191
- "memchr",
1192
- "regex-automata",
1193
- "regex-syntax",
1194
-]
1195
-
1196
-[[package]]
1197
-name = "regex-automata"
1198
-version = "0.4.11"
1199
-source = "registry+https://github.com/rust-lang/crates.io-index"
1200
-checksum = "833eb9ce86d40ef33cb1306d8accf7bc8ec2bfea4355cbdebb3df68b40925cad"
1201
-dependencies = [
1202
- "aho-corasick",
1203
- "memchr",
1204
- "regex-syntax",
1205
-]
1206
-
1207
-[[package]]
1208
-name = "regex-syntax"
1209
-version = "0.8.6"
1210
-source = "registry+https://github.com/rust-lang/crates.io-index"
1211
-checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001"
1212
-
1213
-[[package]]
1214
-name = "ring"
1215
-version = "0.17.14"
1216
-source = "registry+https://github.com/rust-lang/crates.io-index"
1217
-checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7"
1218
-dependencies = [
1219
- "cc",
1220
- "cfg-if",
1221
- "getrandom 0.2.16",
1222
- "libc",
1223
- "untrusted",
1224
- "windows-sys 0.52.0",
1225
-]
1226
-
1227
-[[package]]
1228
-name = "rustc-demangle"
1229
-version = "0.1.26"
1230
-source = "registry+https://github.com/rust-lang/crates.io-index"
1231
-checksum = "56f7d92ca342cea22a06f2121d944b4fd82af56988c270852495420f961d4ace"
1232
-
1233
-[[package]]
1234
-name = "rustix"
1235
-version = "1.1.2"
1236
-source = "registry+https://github.com/rust-lang/crates.io-index"
1237
-checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e"
1238
-dependencies = [
1239
- "bitflags",
1240
- "errno",
1241
- "libc",
1242
- "linux-raw-sys",
1243
- "windows-sys 0.61.1",
1244
-]
1245
-
1246
-[[package]]
1247
-name = "rustls"
1248
-version = "0.23.32"
1249
-source = "registry+https://github.com/rust-lang/crates.io-index"
1250
-checksum = "cd3c25631629d034ce7cd9940adc9d45762d46de2b0f57193c4443b92c6d4d40"
1251
-dependencies = [
1252
- "log",
1253
- "once_cell",
1254
- "ring",
1255
- "rustls-pki-types",
1256
- "rustls-webpki",
1257
- "subtle",
1258
- "zeroize",
1259
-]
1260
-
1261
-[[package]]
1262
-name = "rustls-pki-types"
1263
-version = "1.12.0"
1264
-source = "registry+https://github.com/rust-lang/crates.io-index"
1265
-checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79"
1266
-dependencies = [
1267
- "zeroize",
1268
-]
1269
-
1270
-[[package]]
1271
-name = "rustls-webpki"
1272
-version = "0.103.6"
1273
-source = "registry+https://github.com/rust-lang/crates.io-index"
1274
-checksum = "8572f3c2cb9934231157b45499fc41e1f58c589fdfb81a844ba873265e80f8eb"
1275
-dependencies = [
1276
- "ring",
1277
- "rustls-pki-types",
1278
- "untrusted",
1279
-]
1280
-
1281
-[[package]]
1282
-name = "rustversion"
1283
-version = "1.0.22"
1284
-source = "registry+https://github.com/rust-lang/crates.io-index"
1285
-checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d"
1286
-
1287
-[[package]]
1288
-name = "ruzstd"
1289
-version = "0.8.1"
1290
-source = "registry+https://github.com/rust-lang/crates.io-index"
1291
-checksum = "3640bec8aad418d7d03c72ea2de10d5c646a598f9883c7babc160d91e3c1b26c"
1292
-dependencies = [
1293
- "twox-hash",
1294
-]
1295
-
1296
-[[package]]
1297
-name = "ryu"
1298
-version = "1.0.20"
1299
-source = "registry+https://github.com/rust-lang/crates.io-index"
1300
-checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f"
1301
-
1302
-[[package]]
1303
-name = "serde"
1304
-version = "1.0.228"
1305
-source = "registry+https://github.com/rust-lang/crates.io-index"
1306
-checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e"
1307
-dependencies = [
1308
- "serde_core",
1309
- "serde_derive",
1310
-]
1311
-
1312
-[[package]]
1313
-name = "serde_core"
1314
-version = "1.0.228"
1315
-source = "registry+https://github.com/rust-lang/crates.io-index"
1316
-checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad"
1317
-dependencies = [
1318
- "serde_derive",
1319
-]
1320
-
1321
-[[package]]
1322
-name = "serde_derive"
1323
-version = "1.0.228"
1324
-source = "registry+https://github.com/rust-lang/crates.io-index"
1325
-checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79"
1326
-dependencies = [
1327
- "proc-macro2",
1328
- "quote",
1329
- "syn",
1330
-]
1331
-
1332
-[[package]]
1333
-name = "serde_json"
1334
-version = "1.0.145"
1335
-source = "registry+https://github.com/rust-lang/crates.io-index"
1336
-checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c"
1337
-dependencies = [
1338
- "indexmap",
1339
- "itoa",
1340
- "memchr",
1341
- "ryu",
1342
- "serde",
1343
- "serde_core",
1344
-]
1345
-
1346
-[[package]]
1347
-name = "serde_regex"
1348
-version = "1.1.0"
1349
-source = "registry+https://github.com/rust-lang/crates.io-index"
1350
-checksum = "a8136f1a4ea815d7eac4101cfd0b16dc0cb5e1fe1b8609dfd728058656b7badf"
1351
-dependencies = [
1352
- "regex",
1353
- "serde",
1354
-]
1355
-
1356
-[[package]]
1357
-name = "serde_spanned"
1358
-version = "0.6.9"
1359
-source = "registry+https://github.com/rust-lang/crates.io-index"
1360
-checksum = "bf41e0cfaf7226dca15e8197172c295a782857fcb97fad1808a166870dee75a3"
1361
-dependencies = [
1362
- "serde",
1363
-]
1364
-
1365
-[[package]]
1366
-name = "serde_yaml"
1367
-version = "0.9.34+deprecated"
1368
-source = "registry+https://github.com/rust-lang/crates.io-index"
1369
-checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47"
1370
-dependencies = [
1371
- "indexmap",
1372
- "itoa",
1373
- "ryu",
1374
- "serde",
1375
- "unsafe-libyaml",
1376
-]
1377
-
1378
-[[package]]
1379
-name = "sharded-slab"
1380
-version = "0.1.7"
1381
-source = "registry+https://github.com/rust-lang/crates.io-index"
1382
-checksum = "f40ca3c46823713e0d4209592e8d6e826aa57e928f09752619fc696c499637f6"
1383
-dependencies = [
1384
- "lazy_static",
1385
-]
1386
-
1387
-[[package]]
1388
-name = "shlex"
1389
-version = "1.3.0"
1390
-source = "registry+https://github.com/rust-lang/crates.io-index"
1391
-checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64"
1392
-
1393
-[[package]]
1394
-name = "sigbus"
1395
-version = "0.1.0"
1396
-dependencies = [
1397
- "error",
1398
- "libc",
1399
-]
1400
-
1401
-[[package]]
1402
-name = "signal-hook-registry"
1403
-version = "1.4.6"
1404
-source = "registry+https://github.com/rust-lang/crates.io-index"
1405
-checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b"
1406
-dependencies = [
1407
- "libc",
1408
-]
1409
-
1410
-[[package]]
1411
-name = "siphasher"
1412
-version = "1.0.1"
1413
-source = "registry+https://github.com/rust-lang/crates.io-index"
1414
-checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d"
1415
-
1416
-[[package]]
1417
-name = "slab"
1418
-version = "0.4.11"
1419
-source = "registry+https://github.com/rust-lang/crates.io-index"
1420
-checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
1421
-
1422
-[[package]]
1423
-name = "smallvec"
1424
-version = "1.15.1"
1425
-source = "registry+https://github.com/rust-lang/crates.io-index"
1426
-checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03"
1427
-
1428
-[[package]]
1429
-name = "socket2"
1430
-version = "0.5.10"
1431
-source = "registry+https://github.com/rust-lang/crates.io-index"
1432
-checksum = "e22376abed350d73dd1cd119b57ffccad95b4e585a7cda43e286245ce23c0678"
1433
-dependencies = [
1434
- "libc",
1435
- "windows-sys 0.52.0",
1436
-]
1437
-
1438
-[[package]]
1439
-name = "socket2"
1440
-version = "0.6.0"
1441
-source = "registry+https://github.com/rust-lang/crates.io-index"
1442
-checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807"
1443
-dependencies = [
1444
- "libc",
1445
- "windows-sys 0.59.0",
1446
-]
1447
-
1448
-[[package]]
1449
-name = "static_assertions"
1450
-version = "1.1.0"
1451
-source = "registry+https://github.com/rust-lang/crates.io-index"
1452
-checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f"
1453
-
1454
-[[package]]
1455
-name = "strsim"
1456
-version = "0.11.1"
1457
-source = "registry+https://github.com/rust-lang/crates.io-index"
1458
-checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f"
1459
-
1460
-[[package]]
1461
-name = "subtle"
1462
-version = "2.6.1"
1463
-source = "registry+https://github.com/rust-lang/crates.io-index"
1464
-checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292"
1465
-
1466
-[[package]]
1467
-name = "syn"
1468
-version = "2.0.106"
1469
-source = "registry+https://github.com/rust-lang/crates.io-index"
1470
-checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6"
1471
-dependencies = [
1472
- "proc-macro2",
1473
- "quote",
1474
- "unicode-ident",
1475
-]
1476
-
1477
-[[package]]
1478
-name = "sync_wrapper"
1479
-version = "1.0.2"
1480
-source = "registry+https://github.com/rust-lang/crates.io-index"
1481
-checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263"
1482
-
1483
-[[package]]
1484
-name = "tempfile"
1485
-version = "3.23.0"
1486
-source = "registry+https://github.com/rust-lang/crates.io-index"
1487
-checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16"
1488
-dependencies = [
1489
- "fastrand",
1490
- "getrandom 0.3.3",
1491
- "once_cell",
1492
- "rustix",
1493
- "windows-sys 0.61.1",
1494
-]
1495
-
1496
-[[package]]
1497
-name = "thiserror"
1498
-version = "2.0.17"
1499
-source = "registry+https://github.com/rust-lang/crates.io-index"
1500
-checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8"
1501
-dependencies = [
1502
- "thiserror-impl",
1503
-]
1504
-
1505
-[[package]]
1506
-name = "thiserror-impl"
1507
-version = "2.0.17"
1508
-source = "registry+https://github.com/rust-lang/crates.io-index"
1509
-checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913"
1510
-dependencies = [
1511
- "proc-macro2",
1512
- "quote",
1513
- "syn",
1514
-]
1515
-
1516
-[[package]]
1517
-name = "thread_local"
1518
-version = "1.1.9"
1519
-source = "registry+https://github.com/rust-lang/crates.io-index"
1520
-checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185"
1521
-dependencies = [
1522
- "cfg-if",
1523
-]
1524
-
1525
-[[package]]
1526
-name = "tokio"
1527
-version = "1.47.1"
1528
-source = "registry+https://github.com/rust-lang/crates.io-index"
1529
-checksum = "89e49afdadebb872d3145a5638b59eb0691ea23e46ca484037cfab3b76b95038"
1530
-dependencies = [
1531
- "backtrace",
1532
- "bytes",
1533
- "io-uring",
1534
- "libc",
1535
- "mio",
1536
- "pin-project-lite",
1537
- "signal-hook-registry",
1538
- "slab",
1539
- "socket2 0.6.0",
1540
- "tokio-macros",
1541
- "windows-sys 0.59.0",
1542
-]
1543
-
1544
-[[package]]
1545
-name = "tokio-macros"
1546
-version = "2.5.0"
1547
-source = "registry+https://github.com/rust-lang/crates.io-index"
1548
-checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
1549
-dependencies = [
1550
- "proc-macro2",
1551
- "quote",
1552
- "syn",
1553
-]
1554
-
1555
-[[package]]
1556
-name = "tokio-rustls"
1557
-version = "0.26.4"
1558
-source = "registry+https://github.com/rust-lang/crates.io-index"
1559
-checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61"
1560
-dependencies = [
1561
- "rustls",
1562
- "tokio",
1563
-]
1564
-
1565
-[[package]]
1566
-name = "tokio-stream"
1567
-version = "0.1.17"
1568
-source = "registry+https://github.com/rust-lang/crates.io-index"
1569
-checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047"
1570
-dependencies = [
1571
- "futures-core",
1572
- "pin-project-lite",
1573
- "tokio",
1574
-]
1575
-
1576
-[[package]]
1577
-name = "tokio-util"
1578
-version = "0.7.16"
1579
-source = "registry+https://github.com/rust-lang/crates.io-index"
1580
-checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5"
1581
-dependencies = [
1582
- "bytes",
1583
- "futures-core",
1584
- "futures-sink",
1585
- "pin-project-lite",
1586
- "tokio",
1587
-]
1588
-
1589
-[[package]]
1590
-name = "toml"
1591
-version = "0.8.23"
1592
-source = "registry+https://github.com/rust-lang/crates.io-index"
1593
-checksum = "dc1beb996b9d83529a9e75c17a1686767d148d70663143c7854d8b4a09ced362"
1594
-dependencies = [
1595
- "serde",
1596
- "serde_spanned",
1597
- "toml_datetime",
1598
- "toml_edit",
1599
-]
1600
-
1601
-[[package]]
1602
-name = "toml_datetime"
1603
-version = "0.6.11"
1604
-source = "registry+https://github.com/rust-lang/crates.io-index"
1605
-checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c"
1606
-dependencies = [
1607
- "serde",
1608
-]
1609
-
1610
-[[package]]
1611
-name = "toml_edit"
1612
-version = "0.22.27"
1613
-source = "registry+https://github.com/rust-lang/crates.io-index"
1614
-checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a"
1615
-dependencies = [
1616
- "indexmap",
1617
- "serde",
1618
- "serde_spanned",
1619
- "toml_datetime",
1620
- "toml_write",
1621
- "winnow",
1622
-]
1623
-
1624
-[[package]]
1625
-name = "toml_write"
1626
-version = "0.1.2"
1627
-source = "registry+https://github.com/rust-lang/crates.io-index"
1628
-checksum = "5d99f8c9a7727884afe522e9bd5edbfc91a3312b36a77b5fb8926e4c31a41801"
1629
-
1630
-[[package]]
1631
-name = "tonic"
1632
-version = "0.13.1"
1633
-source = "registry+https://github.com/rust-lang/crates.io-index"
1634
-checksum = "7e581ba15a835f4d9ea06c55ab1bd4dce26fc53752c69a04aac00703bfb49ba9"
1635
-dependencies = [
1636
- "async-trait",
1637
- "axum",
1638
- "base64 0.22.1",
1639
- "bytes",
1640
- "flate2",
1641
- "h2",
1642
- "http",
1643
- "http-body",
1644
- "http-body-util",
1645
- "hyper",
1646
- "hyper-timeout",
1647
- "hyper-util",
1648
- "percent-encoding",
1649
- "pin-project",
1650
- "prost",
1651
- "socket2 0.5.10",
1652
- "tokio",
1653
- "tokio-rustls",
1654
- "tokio-stream",
1655
- "tower",
1656
- "tower-layer",
1657
- "tower-service",
1658
- "tracing",
1659
-]
1660
-
1661
-[[package]]
1662
-name = "tonic-build"
1663
-version = "0.13.1"
1664
-source = "registry+https://github.com/rust-lang/crates.io-index"
1665
-checksum = "eac6f67be712d12f0b41328db3137e0d0757645d8904b4cb7d51cd9c2279e847"
1666
-dependencies = [
1667
- "prettyplease",
1668
- "proc-macro2",
1669
- "prost-build",
1670
- "prost-types",
1671
- "quote",
1672
- "syn",
1673
-]
1674
-
1675
-[[package]]
1676
-name = "tower"
1677
-version = "0.5.2"
1678
-source = "registry+https://github.com/rust-lang/crates.io-index"
1679
-checksum = "d039ad9159c98b70ecfd540b2573b97f7f52c3e8d9f8ad57a24b916a536975f9"
1680
-dependencies = [
1681
- "futures-core",
1682
- "futures-util",
1683
- "indexmap",
1684
- "pin-project-lite",
1685
- "slab",
1686
- "sync_wrapper",
1687
- "tokio",
1688
- "tokio-util",
1689
- "tower-layer",
1690
- "tower-service",
1691
- "tracing",
1692
-]
1693
-
1694
-[[package]]
1695
-name = "tower-layer"
1696
-version = "0.3.3"
1697
-source = "registry+https://github.com/rust-lang/crates.io-index"
1698
-checksum = "121c2a6cda46980bb0fcd1647ffaf6cd3fc79a013de288782836f6df9c48780e"
1699
-
1700
-[[package]]
1701
-name = "tower-service"
1702
-version = "0.3.3"
1703
-source = "registry+https://github.com/rust-lang/crates.io-index"
1704
-checksum = "8df9b6e13f2d32c91b9bd719c00d1958837bc7dec474d94952798cc8e69eeec3"
1705
-
1706
-[[package]]
1707
-name = "tracing"
1708
-version = "0.1.41"
1709
-source = "registry+https://github.com/rust-lang/crates.io-index"
1710
-checksum = "784e0ac535deb450455cbfa28a6f0df145ea1bb7ae51b821cf5e7927fdcfbdd0"
1711
-dependencies = [
1712
- "pin-project-lite",
1713
- "tracing-attributes",
1714
- "tracing-core",
1715
-]
1716
-
1717
-[[package]]
1718
-name = "tracing-attributes"
1719
-version = "0.1.30"
1720
-source = "registry+https://github.com/rust-lang/crates.io-index"
1721
-checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903"
1722
-dependencies = [
1723
- "proc-macro2",
1724
- "quote",
1725
- "syn",
1726
-]
1727
-
1728
-[[package]]
1729
-name = "tracing-core"
1730
-version = "0.1.34"
1731
-source = "registry+https://github.com/rust-lang/crates.io-index"
1732
-checksum = "b9d12581f227e93f094d3af2ae690a574abb8a2b9b7a96e7cfe9647b2b617678"
1733
-dependencies = [
1734
- "once_cell",
1735
- "valuable",
1736
-]
1737
-
1738
-[[package]]
1739
-name = "tracing-log"
1740
-version = "0.2.0"
1741
-source = "registry+https://github.com/rust-lang/crates.io-index"
1742
-checksum = "ee855f1f400bd0e5c02d150ae5de3840039a3f54b025156404e34c23c03f47c3"
1743
-dependencies = [
1744
- "log",
1745
- "once_cell",
1746
- "tracing-core",
1747
-]
1748
-
1749
-[[package]]
1750
-name = "tracing-subscriber"
1751
-version = "0.3.20"
1752
-source = "registry+https://github.com/rust-lang/crates.io-index"
1753
-checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5"
1754
-dependencies = [
1755
- "nu-ansi-term",
1756
- "sharded-slab",
1757
- "smallvec",
1758
- "thread_local",
1759
- "tracing-core",
1760
- "tracing-log",
1761
-]
1762
-
1763
-[[package]]
1764
-name = "try-lock"
1765
-version = "0.2.5"
1766
-source = "registry+https://github.com/rust-lang/crates.io-index"
1767
-checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b"
1768
-
1769
-[[package]]
1770
-name = "twox-hash"
1771
-version = "2.1.2"
1772
-source = "registry+https://github.com/rust-lang/crates.io-index"
1773
-checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c"
1774
-
1775
-[[package]]
1776
-name = "unicode-ident"
1777
-version = "1.0.19"
1778
-source = "registry+https://github.com/rust-lang/crates.io-index"
1779
-checksum = "f63a545481291138910575129486daeaf8ac54aee4387fe7906919f7830c7d9d"
1780
-
1781
-[[package]]
1782
-name = "unsafe-libyaml"
1783
-version = "0.2.11"
1784
-source = "registry+https://github.com/rust-lang/crates.io-index"
1785
-checksum = "673aac59facbab8a9007c7f6108d11f63b603f7cabff99fabf650fea5c32b861"
1786
-
1787
-[[package]]
1788
-name = "untrusted"
1789
-version = "0.9.0"
1790
-source = "registry+https://github.com/rust-lang/crates.io-index"
1791
-checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1"
1792
-
1793
-[[package]]
1794
-name = "utf8parse"
1795
-version = "0.2.2"
1796
-source = "registry+https://github.com/rust-lang/crates.io-index"
1797
-checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821"
1798
-
1799
-[[package]]
1800
-name = "uuid"
1801
-version = "1.18.1"
1802
-source = "registry+https://github.com/rust-lang/crates.io-index"
1803
-checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2"
1804
-dependencies = [
1805
- "getrandom 0.3.3",
1806
- "js-sys",
1807
- "wasm-bindgen",
1808
-]
1809
-
1810
-[[package]]
1811
-name = "valuable"
1812
-version = "0.1.1"
1813
-source = "registry+https://github.com/rust-lang/crates.io-index"
1814
-checksum = "ba73ea9cf16a25df0c8caa16c51acb937d5712a8429db78a3ee29d5dcacd3a65"
1815
-
1816
-[[package]]
1817
-name = "want"
1818
-version = "0.3.1"
1819
-source = "registry+https://github.com/rust-lang/crates.io-index"
1820
-checksum = "bfa7760aed19e106de2c7c0b581b509f2f25d3dacaf737cb82ac61bc6d760b0e"
1821
-dependencies = [
1822
- "try-lock",
1823
-]
1824
-
1825
-[[package]]
1826
-name = "wasi"
1827
-version = "0.11.1+wasi-snapshot-preview1"
1828
-source = "registry+https://github.com/rust-lang/crates.io-index"
1829
-checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b"
1830
-
1831
-[[package]]
1832
-name = "wasi"
1833
-version = "0.14.7+wasi-0.2.4"
1834
-source = "registry+https://github.com/rust-lang/crates.io-index"
1835
-checksum = "883478de20367e224c0090af9cf5f9fa85bed63a95c1abf3afc5c083ebc06e8c"
1836
-dependencies = [
1837
- "wasip2",
1838
-]
1839
-
1840
-[[package]]
1841
-name = "wasip2"
1842
-version = "1.0.1+wasi-0.2.4"
1843
-source = "registry+https://github.com/rust-lang/crates.io-index"
1844
-checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7"
1845
-dependencies = [
1846
- "wit-bindgen",
1847
-]
1848
-
1849
-[[package]]
1850
-name = "wasm-bindgen"
1851
-version = "0.2.104"
1852
-source = "registry+https://github.com/rust-lang/crates.io-index"
1853
-checksum = "c1da10c01ae9f1ae40cbfac0bac3b1e724b320abfcf52229f80b547c0d250e2d"
1854
-dependencies = [
1855
- "cfg-if",
1856
- "once_cell",
1857
- "rustversion",
1858
- "wasm-bindgen-macro",
1859
- "wasm-bindgen-shared",
1860
-]
1861
-
1862
-[[package]]
1863
-name = "wasm-bindgen-backend"
1864
-version = "0.2.104"
1865
-source = "registry+https://github.com/rust-lang/crates.io-index"
1866
-checksum = "671c9a5a66f49d8a47345ab942e2cb93c7d1d0339065d4f8139c486121b43b19"
1867
-dependencies = [
1868
- "bumpalo",
1869
- "log",
1870
- "proc-macro2",
1871
- "quote",
1872
- "syn",
1873
- "wasm-bindgen-shared",
1874
-]
1875
-
1876
-[[package]]
1877
-name = "wasm-bindgen-macro"
1878
-version = "0.2.104"
1879
-source = "registry+https://github.com/rust-lang/crates.io-index"
1880
-checksum = "7ca60477e4c59f5f2986c50191cd972e3a50d8a95603bc9434501cf156a9a119"
1881
-dependencies = [
1882
- "quote",
1883
- "wasm-bindgen-macro-support",
1884
-]
1885
-
1886
-[[package]]
1887
-name = "wasm-bindgen-macro-support"
1888
-version = "0.2.104"
1889
-source = "registry+https://github.com/rust-lang/crates.io-index"
1890
-checksum = "9f07d2f20d4da7b26400c9f4a0511e6e0345b040694e8a75bd41d578fa4421d7"
1891
-dependencies = [
1892
- "proc-macro2",
1893
- "quote",
1894
- "syn",
1895
- "wasm-bindgen-backend",
1896
- "wasm-bindgen-shared",
1897
-]
1898
-
1899
-[[package]]
1900
-name = "wasm-bindgen-shared"
1901
-version = "0.2.104"
1902
-source = "registry+https://github.com/rust-lang/crates.io-index"
1903
-checksum = "bad67dc8b2a1a6e5448428adec4c3e84c43e561d8c9ee8a9e5aabeb193ec41d1"
1904
-dependencies = [
1905
- "unicode-ident",
1906
-]
1907
-
1908
-[[package]]
1909
-name = "winapi"
1910
-version = "0.3.9"
1911
-source = "registry+https://github.com/rust-lang/crates.io-index"
1912
-checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
1913
-dependencies = [
1914
- "winapi-i686-pc-windows-gnu",
1915
- "winapi-x86_64-pc-windows-gnu",
1916
-]
1917
-
1918
-[[package]]
1919
-name = "winapi-i686-pc-windows-gnu"
1920
-version = "0.4.0"
1921
-source = "registry+https://github.com/rust-lang/crates.io-index"
1922
-checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
1923
-
1924
-[[package]]
1925
-name = "winapi-x86_64-pc-windows-gnu"
1926
-version = "0.4.0"
1927
-source = "registry+https://github.com/rust-lang/crates.io-index"
1928
-checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
1929
-
1930
-[[package]]
1931
-name = "window_manager"
1932
-version = "0.1.0"
1933
-dependencies = [
1934
- "error",
1935
- "memmap2",
1936
-]
1937
-
1938
-[[package]]
1939
-name = "windows-core"
1940
-version = "0.62.1"
1941
-source = "registry+https://github.com/rust-lang/crates.io-index"
1942
-checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9"
1943
-dependencies = [
1944
- "windows-implement",
1945
- "windows-interface",
1946
- "windows-link",
1947
- "windows-result",
1948
- "windows-strings",
1949
-]
1950
-
1951
-[[package]]
1952
-name = "windows-implement"
1953
-version = "0.60.1"
1954
-source = "registry+https://github.com/rust-lang/crates.io-index"
1955
-checksum = "edb307e42a74fb6de9bf3a02d9712678b22399c87e6fa869d6dfcd8c1b7754e0"
1956
-dependencies = [
1957
- "proc-macro2",
1958
- "quote",
1959
- "syn",
1960
-]
1961
-
1962
-[[package]]
1963
-name = "windows-interface"
1964
-version = "0.59.2"
1965
-source = "registry+https://github.com/rust-lang/crates.io-index"
1966
-checksum = "c0abd1ddbc6964ac14db11c7213d6532ef34bd9aa042c2e5935f59d7908b46a5"
1967
-dependencies = [
1968
- "proc-macro2",
1969
- "quote",
1970
- "syn",
1971
-]
1972
-
1973
-[[package]]
1974
-name = "windows-link"
1975
-version = "0.2.0"
1976
-source = "registry+https://github.com/rust-lang/crates.io-index"
1977
-checksum = "45e46c0661abb7180e7b9c281db115305d49ca1709ab8242adf09666d2173c65"
1978
-
1979
-[[package]]
1980
-name = "windows-result"
1981
-version = "0.4.0"
1982
-source = "registry+https://github.com/rust-lang/crates.io-index"
1983
-checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f"
1984
-dependencies = [
1985
- "windows-link",
1986
-]
1987
-
1988
-[[package]]
1989
-name = "windows-strings"
1990
-version = "0.5.0"
1991
-source = "registry+https://github.com/rust-lang/crates.io-index"
1992
-checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda"
1993
-dependencies = [
1994
- "windows-link",
1995
-]
1996
-
1997
-[[package]]
1998
-name = "windows-sys"
1999
-version = "0.52.0"
2000
-source = "registry+https://github.com/rust-lang/crates.io-index"
2001
-checksum = "282be5f36a8ce781fad8c8ae18fa3f9beff57ec1b52cb3de0789201425d9a33d"
2002
-dependencies = [
2003
- "windows-targets 0.52.6",
2004
-]
2005
-
2006
-[[package]]
2007
-name = "windows-sys"
2008
-version = "0.59.0"
2009
-source = "registry+https://github.com/rust-lang/crates.io-index"
2010
-checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b"
2011
-dependencies = [
2012
- "windows-targets 0.52.6",
2013
-]
2014
-
2015
-[[package]]
2016
-name = "windows-sys"
2017
-version = "0.60.2"
2018
-source = "registry+https://github.com/rust-lang/crates.io-index"
2019
-checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb"
2020
-dependencies = [
2021
- "windows-targets 0.53.4",
2022
-]
2023
-
2024
-[[package]]
2025
-name = "windows-sys"
2026
-version = "0.61.1"
2027
-source = "registry+https://github.com/rust-lang/crates.io-index"
2028
-checksum = "6f109e41dd4a3c848907eb83d5a42ea98b3769495597450cf6d153507b166f0f"
2029
-dependencies = [
2030
- "windows-link",
2031
-]
2032
-
2033
-[[package]]
2034
-name = "windows-targets"
2035
-version = "0.52.6"
2036
-source = "registry+https://github.com/rust-lang/crates.io-index"
2037
-checksum = "9b724f72796e036ab90c1021d4780d4d3d648aca59e491e6b98e725b84e99973"
2038
-dependencies = [
2039
- "windows_aarch64_gnullvm 0.52.6",
2040
- "windows_aarch64_msvc 0.52.6",
2041
- "windows_i686_gnu 0.52.6",
2042
- "windows_i686_gnullvm 0.52.6",
2043
- "windows_i686_msvc 0.52.6",
2044
- "windows_x86_64_gnu 0.52.6",
2045
- "windows_x86_64_gnullvm 0.52.6",
2046
- "windows_x86_64_msvc 0.52.6",
2047
-]
2048
-
2049
-[[package]]
2050
-name = "windows-targets"
2051
-version = "0.53.4"
2052
-source = "registry+https://github.com/rust-lang/crates.io-index"
2053
-checksum = "2d42b7b7f66d2a06854650af09cfdf8713e427a439c97ad65a6375318033ac4b"
2054
-dependencies = [
2055
- "windows-link",
2056
- "windows_aarch64_gnullvm 0.53.0",
2057
- "windows_aarch64_msvc 0.53.0",
2058
- "windows_i686_gnu 0.53.0",
2059
- "windows_i686_gnullvm 0.53.0",
2060
- "windows_i686_msvc 0.53.0",
2061
- "windows_x86_64_gnu 0.53.0",
2062
- "windows_x86_64_gnullvm 0.53.0",
2063
- "windows_x86_64_msvc 0.53.0",
2064
-]
2065
-
2066
-[[package]]
2067
-name = "windows_aarch64_gnullvm"
2068
-version = "0.52.6"
2069
-source = "registry+https://github.com/rust-lang/crates.io-index"
2070
-checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3"
2071
-
2072
-[[package]]
2073
-name = "windows_aarch64_gnullvm"
2074
-version = "0.53.0"
2075
-source = "registry+https://github.com/rust-lang/crates.io-index"
2076
-checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764"
2077
-
2078
-[[package]]
2079
-name = "windows_aarch64_msvc"
2080
-version = "0.52.6"
2081
-source = "registry+https://github.com/rust-lang/crates.io-index"
2082
-checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469"
2083
-
2084
-[[package]]
2085
-name = "windows_aarch64_msvc"
2086
-version = "0.53.0"
2087
-source = "registry+https://github.com/rust-lang/crates.io-index"
2088
-checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c"
2089
-
2090
-[[package]]
2091
-name = "windows_i686_gnu"
2092
-version = "0.52.6"
2093
-source = "registry+https://github.com/rust-lang/crates.io-index"
2094
-checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b"
2095
-
2096
-[[package]]
2097
-name = "windows_i686_gnu"
2098
-version = "0.53.0"
2099
-source = "registry+https://github.com/rust-lang/crates.io-index"
2100
-checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3"
2101
-
2102
-[[package]]
2103
-name = "windows_i686_gnullvm"
2104
-version = "0.52.6"
2105
-source = "registry+https://github.com/rust-lang/crates.io-index"
2106
-checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66"
2107
-
2108
-[[package]]
2109
-name = "windows_i686_gnullvm"
2110
-version = "0.53.0"
2111
-source = "registry+https://github.com/rust-lang/crates.io-index"
2112
-checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11"
2113
-
2114
-[[package]]
2115
-name = "windows_i686_msvc"
2116
-version = "0.52.6"
2117
-source = "registry+https://github.com/rust-lang/crates.io-index"
2118
-checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66"
2119
-
2120
-[[package]]
2121
-name = "windows_i686_msvc"
2122
-version = "0.53.0"
2123
-source = "registry+https://github.com/rust-lang/crates.io-index"
2124
-checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d"
2125
-
2126
-[[package]]
2127
-name = "windows_x86_64_gnu"
2128
-version = "0.52.6"
2129
-source = "registry+https://github.com/rust-lang/crates.io-index"
2130
-checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78"
2131
-
2132
-[[package]]
2133
-name = "windows_x86_64_gnu"
2134
-version = "0.53.0"
2135
-source = "registry+https://github.com/rust-lang/crates.io-index"
2136
-checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba"
2137
-
2138
-[[package]]
2139
-name = "windows_x86_64_gnullvm"
2140
-version = "0.52.6"
2141
-source = "registry+https://github.com/rust-lang/crates.io-index"
2142
-checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d"
2143
-
2144
-[[package]]
2145
-name = "windows_x86_64_gnullvm"
2146
-version = "0.53.0"
2147
-source = "registry+https://github.com/rust-lang/crates.io-index"
2148
-checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57"
2149
-
2150
-[[package]]
2151
-name = "windows_x86_64_msvc"
2152
-version = "0.52.6"
2153
-source = "registry+https://github.com/rust-lang/crates.io-index"
2154
-checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
2155
-
2156
-[[package]]
2157
-name = "windows_x86_64_msvc"
2158
-version = "0.53.0"
2159
-source = "registry+https://github.com/rust-lang/crates.io-index"
2160
-checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486"
2161
-
2162
-[[package]]
2163
-name = "winnow"
2164
-version = "0.7.13"
2165
-source = "registry+https://github.com/rust-lang/crates.io-index"
2166
-checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf"
2167
-dependencies = [
2168
- "memchr",
2169
-]
2170
-
2171
-[[package]]
2172
-name = "wit-bindgen"
2173
-version = "0.46.0"
2174
-source = "registry+https://github.com/rust-lang/crates.io-index"
2175
-checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59"
2176
-
2177
-[[package]]
2178
-name = "zerocopy"
2179
-version = "0.8.27"
2180
-source = "registry+https://github.com/rust-lang/crates.io-index"
2181
-checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c"
2182
-dependencies = [
2183
- "zerocopy-derive 0.8.27",
2184
-]
2185
-
2186
-[[package]]
2187
-name = "zerocopy"
2188
-version = "0.9.0-alpha.0"
2189
-source = "registry+https://github.com/rust-lang/crates.io-index"
2190
-checksum = "3c9c378180b968f2e98efa0c4bba8c49fd4cd247dc82a7f01e72dc137d447d8c"
2191
-dependencies = [
2192
- "zerocopy-derive 0.9.0-alpha.0",
2193
-]
2194
-
2195
-[[package]]
2196
-name = "zerocopy-derive"
2197
-version = "0.8.27"
2198
-source = "registry+https://github.com/rust-lang/crates.io-index"
2199
-checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831"
2200
-dependencies = [
2201
- "proc-macro2",
2202
- "quote",
2203
- "syn",
2204
-]
2205
-
2206
-[[package]]
2207
-name = "zerocopy-derive"
2208
-version = "0.9.0-alpha.0"
2209
-source = "registry+https://github.com/rust-lang/crates.io-index"
2210
-checksum = "36811c5ea90dd441f941919fc99163e702f150a8e1495d72a268106a0100df09"
2211
-dependencies = [
2212
- "proc-macro2",
2213
- "quote",
2214
- "syn",
2215
-]
2216
-
2217
-[[package]]
2218
-name = "zeroize"
2219
-version = "1.8.2"
2220
-source = "registry+https://github.com/rust-lang/crates.io-index"
2221
-checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0"
src/crates/jf/Cargo.toml
deleted
-66
@@ -1,66 +0,0 @@
1
-[workspace]
2
-resolver = "2"
3
-members = [
4
- "error",
5
- "journal_file",
6
- "journal_reader_ffi",
7
- "journal_log",
8
- "window_manager",
9
- "sigbus",
10
- "otel-plugin",
11
- "flatten_otel",
12
- "flog-otel",
13
-]
14
-
15
-[workspace.package]
16
-version = "0.1.0"
17
-edition = "2021"
18
-rust-version = "1.85"
19
-
20
-[workspace.dependencies]
21
-memmap2 = "0.9"
22
-zerocopy = { version = "0.9.0-alpha.0", features = ["derive"] }
23
-thiserror = "2"
24
-rand = "0.9"
25
-siphasher = "1.0"
26
-twox-hash = { version = "2.1", default-features = false, features = ["std"] }
27
-static_assertions = "1.1"
28
-duct = "0.13"
29
-serde = { version = "1.0", features = ["derive"] }
30
-serde_json = "1.0"
31
-walkdir = "2"
32
-ruzstd = "0.8"
33
-libc = "0.2"
34
-hex = "0.4"
35
-tempfile = "3"
36
-indexmap = "2.9"
37
-uuid = { version = "1.0", features = ["v4", "rng"] }
38
-chrono = { version = "0.4", features = ["serde"] }
39
-tracing = "0.1"
40
-tracing-subscriber = "0.3"
41
-
42
-# otel-plugin
43
-prost = "0.13"
44
-regex = "1.10"
45
-tokio = { version = "1.45", features = ["rt-multi-thread", "process", "io-util"] }
46
-tokio-stream = "0.1"
47
-tonic = "0.13"
48
-tonic-build = "0.13"
49
-prost-build = "0.13"
50
-hyper-util = "0.1"
51
-tower = "0.5"
52
-opentelemetry-proto = "0.30"
53
-atty = "0.2"
54
-anyhow = "1.0"
55
-humantime-serde = "1.1"
56
-humantime = "2.2"
57
-bytesize = "1.0"
58
-bytesize-serde = "0.2"
59
-
60
-clap = { version = "4.5", features = ["derive", "env"] }
61
-
62
-flatten-serde-json = { git = "https://github.com/meilisearch/meilisearch", tag = "v1.22.1" }
63
-
64
-[profile.release]
65
-lto = true
66
-codegen-units = 1
src/crates/jf/error/Cargo.toml
deleted
-10
@@ -1,10 +0,0 @@
1
-[package]
2
-name = "error"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-static_assertions = { workspace = true }
9
-thiserror = { workspace = true }
10
-zerocopy = { workspace = true }
src/crates/jf/flog-otel/Cargo.toml
deleted
-22
@@ -1,22 +0,0 @@
1
-[package]
2
-name = "flog-otel"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-tokio.workspace = true
9
-serde.workspace = true
10
-serde_json.workspace = true
11
-chrono.workspace = true
12
-clap.workspace = true
13
-uuid.workspace = true
14
-anyhow.workspace = true
15
-tracing.workspace = true
16
-tracing-subscriber.workspace = true
17
-tonic.workspace = true
18
-prost.workspace = true
19
-opentelemetry-proto.workspace = true
20
-
21
-[build-dependencies]
22
-tonic-build.workspace = true
src/crates/jf/flog-otel/src/main.rs
deleted
-312
@@ -1,312 +0,0 @@
1
-use anyhow::Result;
2
-use chrono::{DateTime, Utc};
3
-use clap::Parser;
4
-use opentelemetry_proto::tonic::collector::logs::v1::{
5
- logs_service_client::LogsServiceClient, ExportLogsServiceRequest,
6
-};
7
-use opentelemetry_proto::tonic::common::v1::{any_value, AnyValue, KeyValue};
8
-use opentelemetry_proto::tonic::logs::v1::{LogRecord, ResourceLogs, ScopeLogs};
9
-use opentelemetry_proto::tonic::resource::v1::Resource;
10
-use serde::Deserialize;
11
-use std::process::Stdio;
12
-use std::sync::Arc;
13
-use tokio::io::{AsyncBufReadExt, BufReader};
14
-use tokio::sync::Mutex;
15
-use tokio::time::{Duration, Instant};
16
-use tracing::{error, info, warn};
17
-use tracing_subscriber;
18
-use uuid::Uuid;
19
-
20
-#[derive(Parser, Debug)]
21
-#[command(author, version, about, long_about = None)]
22
-struct Args {
23
- #[arg(short, long, default_value = "1048576")]
24
- rate_limit_bytes: u64,
25
-
26
- #[arg(short, long, default_value = "http://127.0.0.1:4317")]
27
- otel_endpoint: String,
28
-
29
- #[arg(short, long, default_value = "1000")]
30
- log_count: u32,
31
-
32
- #[arg(long, default_value = "false")]
33
- loop_forever: bool,
34
-}
35
-
36
-#[derive(Deserialize, Debug)]
37
-struct FlogEntry {
38
- host: String,
39
- #[serde(rename = "user-identifier")]
40
- user_identifier: String,
41
- datetime: String,
42
- method: String,
43
- request: String,
44
- protocol: String,
45
- status: u16,
46
- bytes: u64,
47
- referer: String,
48
-}
49
-
50
-struct RateLimiter {
51
- bytes_sent: u64,
52
- last_reset: Instant,
53
- limit: u64,
54
-}
55
-
56
-impl RateLimiter {
57
- fn new(limit: u64) -> Self {
58
- Self {
59
- bytes_sent: 0,
60
- last_reset: Instant::now(),
61
- limit,
62
- }
63
- }
64
-
65
- async fn can_send(&mut self, size: u64) -> bool {
66
- if self.last_reset.elapsed() >= Duration::from_secs(1) {
67
- self.bytes_sent = 0;
68
- self.last_reset = Instant::now();
69
- }
70
-
71
- if self.bytes_sent + size <= self.limit {
72
- self.bytes_sent += size;
73
- true
74
- } else {
75
- false
76
- }
77
- }
78
-}
79
-
80
-fn parse_flog_datetime(datetime_str: &str) -> Result<DateTime<Utc>> {
81
- let dt = chrono::DateTime::parse_from_str(datetime_str, "%d/%b/%Y:%H:%M:%S %z")?;
82
- Ok(dt.with_timezone(&Utc))
83
-}
84
-
85
-fn flog_to_otel(entry: FlogEntry) -> Result<LogRecord> {
86
- let dt = parse_flog_datetime(&entry.datetime)?;
87
- let time_unix_nano = dt.timestamp_nanos_opt().unwrap_or(0) as u64;
88
-
89
- let severity_number = match entry.status {
90
- 200..=299 => 9, // INFO
91
- 300..=399 => 5, // DEBUG
92
- 400..=499 => 13, // WARN
93
- 500..=599 => 17, // ERROR
94
- _ => 9, // INFO
95
- };
96
-
97
- let log_message = format!(
98
- "{} {} {} {} {} {}",
99
- entry.host, entry.method, entry.request, entry.protocol, entry.status, entry.bytes
100
- );
101
-
102
- let attributes = vec![
103
- KeyValue {
104
- key: "host".to_string(),
105
- value: Some(AnyValue {
106
- value: Some(any_value::Value::StringValue(entry.host)),
107
- }),
108
- },
109
- KeyValue {
110
- key: "method".to_string(),
111
- value: Some(AnyValue {
112
- value: Some(any_value::Value::StringValue(entry.method)),
113
- }),
114
- },
115
- KeyValue {
116
- key: "request".to_string(),
117
- value: Some(AnyValue {
118
- value: Some(any_value::Value::StringValue(entry.request)),
119
- }),
120
- },
121
- KeyValue {
122
- key: "protocol".to_string(),
123
- value: Some(AnyValue {
124
- value: Some(any_value::Value::StringValue(entry.protocol)),
125
- }),
126
- },
127
- KeyValue {
128
- key: "status".to_string(),
129
- value: Some(AnyValue {
130
- value: Some(any_value::Value::IntValue(entry.status as i64)),
131
- }),
132
- },
133
- KeyValue {
134
- key: "response_bytes".to_string(),
135
- value: Some(AnyValue {
136
- value: Some(any_value::Value::IntValue(entry.bytes as i64)),
137
- }),
138
- },
139
- KeyValue {
140
- key: "referer".to_string(),
141
- value: Some(AnyValue {
142
- value: Some(any_value::Value::StringValue(entry.referer)),
143
- }),
144
- },
145
- KeyValue {
146
- key: "user_identifier".to_string(),
147
- value: Some(AnyValue {
148
- value: Some(any_value::Value::StringValue(entry.user_identifier)),
149
- }),
150
- },
151
- ];
152
-
153
- let trace_id_bytes = Uuid::new_v4().as_u128().to_be_bytes().to_vec();
154
- let span_id_bytes = (Uuid::new_v4().as_u128() as u64).to_be_bytes().to_vec();
155
-
156
- Ok(LogRecord {
157
- time_unix_nano,
158
- severity_number: severity_number as i32,
159
- severity_text: String::new(),
160
- body: Some(AnyValue {
161
- value: Some(any_value::Value::StringValue(log_message)),
162
- }),
163
- attributes,
164
- dropped_attributes_count: 0,
165
- flags: 0,
166
- trace_id: trace_id_bytes,
167
- span_id: span_id_bytes,
168
- ..Default::default()
169
- })
170
-}
171
-
172
-async fn send_otel_logs(logs: Vec<LogRecord>, endpoint: &str) -> Result<()> {
173
- let mut client = LogsServiceClient::connect(endpoint.to_string()).await?;
174
-
175
- let resource_logs = vec![ResourceLogs {
176
- resource: Some(Resource {
177
- attributes: vec![
178
- KeyValue {
179
- key: "service.name".to_string(),
180
- value: Some(AnyValue {
181
- value: Some(any_value::Value::StringValue(
182
- "flog-otel-wrapper".to_string(),
183
- )),
184
- }),
185
- },
186
- KeyValue {
187
- key: "service.version".to_string(),
188
- value: Some(AnyValue {
189
- value: Some(any_value::Value::StringValue("0.1.0".to_string())),
190
- }),
191
- },
192
- ],
193
- dropped_attributes_count: 0,
194
- entity_refs: vec![],
195
- }),
196
- scope_logs: vec![ScopeLogs {
197
- scope: Some(
198
- opentelemetry_proto::tonic::common::v1::InstrumentationScope {
199
- name: "flog-otel-wrapper".to_string(),
200
- version: "0.1.0".to_string(),
201
- attributes: vec![],
202
- dropped_attributes_count: 0,
203
- },
204
- ),
205
- log_records: logs,
206
- schema_url: String::new(),
207
- }],
208
- schema_url: String::new(),
209
- }];
210
-
211
- let request = tonic::Request::new(ExportLogsServiceRequest { resource_logs });
212
-
213
- let response = client.export(request).await?;
214
-
215
- if response.get_ref().partial_success.is_some() {
216
- let partial_success = response.get_ref().partial_success.as_ref().unwrap();
217
- if partial_success.rejected_log_records > 0 {
218
- warn!("Some logs were rejected: {}", partial_success.error_message);
219
- }
220
- }
221
-
222
- info!("Successfully sent logs to OTEL collector via gRPC");
223
- Ok(())
224
-}
225
-
226
-#[tokio::main]
227
-async fn main() -> Result<()> {
228
- tracing_subscriber::fmt::init();
229
-
230
- let args = Args::parse();
231
- let rate_limiter = Arc::new(Mutex::new(RateLimiter::new(args.rate_limit_bytes)));
232
-
233
- info!(
234
- "Starting flog-otel-wrapper with rate limit: {} bytes/sec",
235
- args.rate_limit_bytes
236
- );
237
- info!("OTEL endpoint: {}", args.otel_endpoint);
238
-
239
- loop {
240
- let log_count_str = args.log_count.to_string();
241
- let mut flog_args = vec!["-f", "json", "-t", "stdout"];
242
-
243
- if args.loop_forever {
244
- flog_args.extend(&["-l"]);
245
- } else {
246
- flog_args.extend(&["-n", &log_count_str]);
247
- }
248
-
249
- let mut child = tokio::process::Command::new("flog")
250
- .args(&flog_args)
251
- .stdout(Stdio::piped())
252
- .spawn()?;
253
-
254
- let stdout = child.stdout.take().unwrap();
255
- let reader = BufReader::new(stdout);
256
- let mut lines = reader.lines();
257
-
258
- let mut log_batch = Vec::new();
259
- const BATCH_SIZE: usize = 100;
260
-
261
- while let Some(line) = lines.next_line().await? {
262
- match serde_json::from_str::<FlogEntry>(&line) {
263
- Ok(entry) => match flog_to_otel(entry) {
264
- Ok(otel_log) => {
265
- log_batch.push(otel_log);
266
-
267
- if log_batch.len() >= BATCH_SIZE {
268
- let batch_size = (log_batch.len() * 1024) as u64;
269
-
270
- let mut limiter = rate_limiter.lock().await;
271
- if limiter.can_send(batch_size).await {
272
- drop(limiter);
273
-
274
- if let Err(e) =
275
- send_otel_logs(log_batch.clone(), &args.otel_endpoint).await
276
- {
277
- error!("Failed to send OTEL logs: {}", e);
278
- }
279
- log_batch.clear();
280
- } else {
281
- drop(limiter);
282
- info!("Rate limit reached, waiting 1 second...");
283
- tokio::time::sleep(Duration::from_secs(1)).await;
284
- }
285
- }
286
- }
287
- Err(e) => error!("Failed to convert to OTEL format: {}", e),
288
- },
289
- Err(e) => error!("Failed to parse JSON: {}", e),
290
- }
291
- }
292
-
293
- if !log_batch.is_empty() {
294
- let batch_size = (log_batch.len() * 1024) as u64;
295
- let mut limiter = rate_limiter.lock().await;
296
- if limiter.can_send(batch_size).await {
297
- drop(limiter);
298
- if let Err(e) = send_otel_logs(log_batch, &args.otel_endpoint).await {
299
- error!("Failed to send final OTEL logs: {}", e);
300
- }
301
- }
302
- }
303
-
304
- child.wait().await?;
305
-
306
- if !args.loop_forever {
307
- break;
308
- }
309
- }
310
-
311
- Ok(())
312
-}
src/crates/jf/journal_file/Cargo.toml
deleted
-19
@@ -1,19 +0,0 @@
1
-[package]
2
-name = "journal_file"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-window_manager = { path = "../window_manager" }
10
-memmap2 = { workspace = true }
11
-ruzstd = { workspace = true }
12
-siphasher = { workspace = true }
13
-twox-hash = { workspace = true }
14
-zerocopy = { workspace = true }
15
-hex = { workspace = true }
16
-rand = { workspace = true }
17
-
18
-[dev-dependencies]
19
-tempfile = { workspace = true }
src/crates/jf/journal_file/src/filter.rs
deleted
-438
@@ -1,438 +0,0 @@
1
-use crate::{
2
- file::JournalFile,
3
- offset_array::{Direction, InlinedCursor},
4
-};
5
-use error::{JournalError, Result};
6
-use std::num::NonZeroU64;
7
-use window_manager::MemoryMap;
8
-
9
-#[derive(Clone, Debug)]
10
-pub enum FilterExpr {
11
- Match(u64, Option<InlinedCursor>),
12
- Conjunction(Vec<FilterExpr>),
13
- Disjunction(Vec<FilterExpr>),
14
-}
15
-
16
-impl FilterExpr {
17
- pub fn lookup<M: MemoryMap>(
18
- &self,
19
- journal_file: &JournalFile<M>,
20
- needle_offset: u64,
21
- direction: Direction,
22
- ) -> Result<Option<u64>> {
23
- let Some(needle_offset) = NonZeroU64::new(needle_offset) else {
24
- return Err(JournalError::InvalidOffset);
25
- };
26
-
27
- let predicate =
28
- move |entry_offset: NonZeroU64| -> Result<bool> { Ok(entry_offset < needle_offset) };
29
-
30
- match self {
31
- FilterExpr::Match(data_offset, _) => {
32
- let Some(data_offset) = NonZeroU64::new(*data_offset) else {
33
- return Err(JournalError::InvalidOffset);
34
- };
35
- let entry_offset = journal_file.data_object_directed_partition_point(
36
- data_offset,
37
- predicate,
38
- direction,
39
- )?;
40
- Ok(entry_offset.map(|x| x.get()))
41
- }
42
- FilterExpr::Conjunction(filter_exprs) => {
43
- let mut current_offset = needle_offset;
44
-
45
- loop {
46
- let previous_offset = current_offset;
47
-
48
- for filter_expr in filter_exprs {
49
- if direction == Direction::Backward {
50
- current_offset = current_offset.saturating_add(1);
51
- }
52
-
53
- match filter_expr.lookup(journal_file, current_offset.get(), direction)? {
54
- Some(new_offset) => {
55
- if new_offset == 0 {
56
- panic!("Wtf");
57
- }
58
- current_offset = NonZeroU64::new(new_offset).unwrap();
59
- }
60
- None => return Ok(None),
61
- }
62
- }
63
-
64
- if current_offset == previous_offset {
65
- return Ok(Some(current_offset.get()));
66
- }
67
- }
68
- }
69
- FilterExpr::Disjunction(filter_exprs) => {
70
- let cmp = match direction {
71
- Direction::Forward => std::cmp::min,
72
- Direction::Backward => std::cmp::max,
73
- };
74
-
75
- filter_exprs.iter().try_fold(None, |acc, expr| {
76
- let result = expr.lookup(journal_file, needle_offset.get(), direction)?;
77
-
78
- Ok(match (acc, result) {
79
- (None, Some(offset)) => Some(offset),
80
- (Some(best), Some(offset)) => Some(cmp(best, offset)),
81
- (acc, None) => acc,
82
- })
83
- })
84
- }
85
- }
86
- }
87
-
88
- pub fn head(&mut self) -> &mut Self {
89
- match self {
90
- FilterExpr::Match(_, None) => (),
91
- FilterExpr::Match(_, Some(ic)) => {
92
- *ic = ic.head();
93
- }
94
- FilterExpr::Conjunction(filter_exprs) => {
95
- for filter_expr in filter_exprs.iter_mut() {
96
- filter_expr.head();
97
- }
98
- }
99
- FilterExpr::Disjunction(filter_exprs) => {
100
- for filter_expr in filter_exprs.iter_mut() {
101
- filter_expr.head();
102
- }
103
- }
104
- }
105
-
106
- self
107
- }
108
-
109
- pub fn tail<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<&mut Self> {
110
- match self {
111
- FilterExpr::Match(_, None) => (),
112
- FilterExpr::Match(_, Some(ic)) => {
113
- *ic = ic.tail(journal_file)?;
114
- }
115
- FilterExpr::Conjunction(filter_exprs) => {
116
- for filter_expr in filter_exprs.iter_mut() {
117
- filter_expr.tail(journal_file)?;
118
- }
119
- }
120
- FilterExpr::Disjunction(filter_exprs) => {
121
- for filter_expr in filter_exprs.iter_mut() {
122
- filter_expr.tail(journal_file)?;
123
- }
124
- }
125
- }
126
-
127
- Ok(self)
128
- }
129
-
130
- // Returns the offset of the next matching entry, if any, with an offset
131
- // greater or equal to the needle offset.
132
- pub fn next<M: MemoryMap>(
133
- &mut self,
134
- journal_file: &JournalFile<M>,
135
- needle_offset: NonZeroU64,
136
- ) -> Result<Option<NonZeroU64>> {
137
- match self {
138
- FilterExpr::Match(_, None) => Ok(None),
139
- FilterExpr::Match(_, Some(ic)) => ic.next_until(journal_file, needle_offset),
140
- FilterExpr::Conjunction(filter_exprs) => {
141
- let mut needle_offset = needle_offset;
142
-
143
- loop {
144
- let previous_offset = needle_offset;
145
-
146
- for fe in filter_exprs.iter_mut() {
147
- if let Some(new_offset) = fe.next(journal_file, needle_offset)? {
148
- needle_offset = new_offset;
149
- } else {
150
- return Ok(None);
151
- }
152
- }
153
-
154
- if needle_offset == previous_offset {
155
- return Ok(Some(needle_offset));
156
- }
157
- }
158
- }
159
- FilterExpr::Disjunction(filter_exprs) => {
160
- let mut best_offset: Option<NonZeroU64> = None;
161
-
162
- for fe in filter_exprs.iter_mut() {
163
- if let Some(fe_offset) = fe.next(journal_file, needle_offset)? {
164
- best_offset = match best_offset {
165
- Some(offset) => Some(fe_offset.min(offset)),
166
- None => Some(fe_offset),
167
- };
168
- }
169
- }
170
-
171
- Ok(best_offset)
172
- }
173
- }
174
- }
175
-
176
- // Returns the offset of the previous matching entry, if any, with an offset
177
- // less or equal to the needle offset.
178
- pub fn previous<M: MemoryMap>(
179
- &mut self,
180
- journal_file: &JournalFile<M>,
181
- needle_offset: NonZeroU64,
182
- ) -> Result<Option<NonZeroU64>> {
183
- match self {
184
- FilterExpr::Match(_, None) => Ok(None),
185
- FilterExpr::Match(_, Some(ic)) => ic.previous_until(journal_file, needle_offset),
186
- FilterExpr::Conjunction(filter_exprs) => {
187
- let mut needle_offset = needle_offset;
188
-
189
- loop {
190
- let previous_offset = needle_offset;
191
-
192
- for fe in filter_exprs.iter_mut().rev() {
193
- if let Some(new_offset) = fe.previous(journal_file, needle_offset)? {
194
- needle_offset = new_offset;
195
- } else {
196
- return Ok(None);
197
- }
198
- }
199
-
200
- if needle_offset == previous_offset {
201
- return Ok(Some(needle_offset));
202
- }
203
- }
204
- }
205
- FilterExpr::Disjunction(filter_exprs) => {
206
- let mut best_offset: Option<NonZeroU64> = None;
207
-
208
- for fe in filter_exprs.iter_mut() {
209
- if let Some(fe_offset) = fe.previous(journal_file, needle_offset)? {
210
- best_offset = match best_offset {
211
- Some(offset) => Some(fe_offset.max(offset)),
212
- None => Some(fe_offset),
213
- };
214
- }
215
- }
216
-
217
- Ok(best_offset)
218
- }
219
- }
220
- }
221
-
222
- pub fn dump<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<String> {
223
- let mut output = String::new();
224
- self.dump_internal(journal_file, 0, &mut output)?;
225
- Ok(output)
226
- }
227
-
228
- /// Helper function for format_data_objects that handles nested expressions and indentation
229
- fn dump_internal<M: MemoryMap>(
230
- &self,
231
- journal_file: &JournalFile<M>,
232
- indent_level: usize,
233
- output: &mut String,
234
- ) -> Result<()> {
235
- let indent = " ".repeat(indent_level);
236
-
237
- match self {
238
- FilterExpr::Match(data_offset, inlined_cursor) => {
239
- // Load the data object
240
- let Some(data_offset) = NonZeroU64::new(*data_offset) else {
241
- return Err(JournalError::InvalidOffset);
242
- };
243
- let data_object = journal_file.data_ref(data_offset)?;
244
-
245
- // Get the payload as a string if possible
246
- let payload_bytes = data_object.payload_bytes();
247
- let payload_str = String::from_utf8_lossy(payload_bytes);
248
-
249
- // Format the data offset and payload
250
- output.push_str(&format!(
251
- "{}Match[offset=0x{:x}]: {}\n",
252
- indent, data_offset, payload_str
253
- ));
254
-
255
- // Format cursor information if available
256
- if let Some(ic) = inlined_cursor {
257
- output.push_str(&format!("{} Cursor: {:?}\n", indent, ic));
258
- }
259
- }
260
- FilterExpr::Conjunction(filter_exprs) => {
261
- output.push_str(&format!("{}Conjunction (AND) {{\n", indent));
262
- for expr in filter_exprs {
263
- expr.dump_internal(journal_file, indent_level + 1, output)?;
264
- }
265
- output.push_str(&format!("{}}}\n", indent));
266
- }
267
- FilterExpr::Disjunction(filter_exprs) => {
268
- output.push_str(&format!("{}Disjunction (OR) {{\n", indent));
269
- for expr in filter_exprs {
270
- expr.dump_internal(journal_file, indent_level + 1, output)?;
271
- }
272
- output.push_str(&format!("{}}}\n", indent));
273
- }
274
- }
275
-
276
- Ok(())
277
- }
278
-}
279
-
280
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
281
-pub enum LogicalOp {
282
- Conjunction,
283
- Disjunction,
284
-}
285
-
286
-#[derive(Debug)]
287
-pub struct JournalFilter {
288
- filter_expr: Option<FilterExpr>,
289
- current_matches: Vec<Vec<u8>>,
290
- current_op: LogicalOp,
291
-}
292
-
293
-impl Default for JournalFilter {
294
- fn default() -> Self {
295
- Self {
296
- filter_expr: None,
297
- current_matches: Vec::new(),
298
- current_op: LogicalOp::Conjunction,
299
- }
300
- }
301
-}
302
-
303
-impl JournalFilter {
304
- fn extract_key(kv_pair: &[u8]) -> Option<&[u8]> {
305
- if let Some(equal_pos) = kv_pair.iter().position(|&b| b == b'=') {
306
- Some(&kv_pair[..equal_pos])
307
- } else {
308
- None
309
- }
310
- }
311
-
312
- fn convert_current_matches<M: MemoryMap>(
313
- &mut self,
314
- journal_file: &JournalFile<M>,
315
- ) -> Result<Option<FilterExpr>> {
316
- if self.current_matches.is_empty() {
317
- return Ok(None);
318
- }
319
-
320
- let mut elements = Vec::new();
321
- let mut i = 0;
322
-
323
- while i < self.current_matches.len() {
324
- let current_key = Self::extract_key(&self.current_matches[i]).unwrap_or(&[]);
325
- let start = i;
326
-
327
- // Find all matches with the same key
328
- while i < self.current_matches.len()
329
- && Self::extract_key(&self.current_matches[i]).unwrap_or(&[]) == current_key
330
- {
331
- i += 1;
332
- }
333
-
334
- // If we have multiple values for this key, create a disjunction
335
- if i - start > 1 {
336
- let mut matches = Vec::with_capacity(i - start);
337
- for idx in start..i {
338
- let data = self.current_matches[idx].as_slice();
339
- let hash = journal_file.hash(data);
340
- let Some(offset) = journal_file.find_data_offset(hash, data)? else {
341
- // TODO: should we really return an error?
342
- return Err(JournalError::InvalidOffset);
343
- };
344
-
345
- let ic = journal_file.data_ref(offset)?.inlined_cursor();
346
- matches.push(FilterExpr::Match(offset.get(), ic));
347
- }
348
- elements.push(FilterExpr::Disjunction(matches));
349
- } else {
350
- let data = self.current_matches[start].as_slice();
351
- let hash = journal_file.hash(data);
352
- let Some(offset) = journal_file.find_data_offset(hash, data)? else {
353
- // TODO: should we really return an error?
354
- return Err(JournalError::InvalidOffset);
355
- };
356
-
357
- let ic = journal_file.data_ref(offset)?.inlined_cursor();
358
- elements.push(FilterExpr::Match(offset.get(), ic));
359
- }
360
- }
361
-
362
- self.current_matches.clear();
363
-
364
- match elements.len() {
365
- 0 => panic!("Could not create filter elements from current matches"),
366
- 1 => Ok(Some(elements.remove(0))),
367
- _ => Ok(Some(FilterExpr::Conjunction(elements))),
368
- }
369
- }
370
-
371
- pub fn add_match(&mut self, kv_pair: &[u8]) {
372
- if kv_pair.contains(&b'=') {
373
- let new_item = kv_pair.to_vec();
374
- let new_key = Self::extract_key(&new_item).unwrap_or(&[]);
375
-
376
- // Find the insertion position using binary search
377
- let pos = self
378
- .current_matches
379
- .binary_search_by(|item| {
380
- let key = Self::extract_key(item).unwrap_or(&[]);
381
- key.cmp(new_key)
382
- })
383
- .unwrap_or_else(|e| e);
384
-
385
- // Insert at the found position
386
- self.current_matches.insert(pos, new_item);
387
- }
388
- }
389
-
390
- pub fn set_operation<M: MemoryMap>(
391
- &mut self,
392
- journal_file: &JournalFile<M>,
393
- op: LogicalOp,
394
- ) -> Result<()> {
395
- let new_expr = self.convert_current_matches(journal_file)?;
396
- if new_expr.is_none() {
397
- self.current_op = op;
398
- return Ok(());
399
- }
400
-
401
- if self.filter_expr.is_none() {
402
- self.filter_expr = new_expr;
403
- self.current_op = op;
404
- return Ok(());
405
- }
406
-
407
- let new_expr = new_expr.unwrap();
408
- let current_expr = self.filter_expr.take().unwrap();
409
-
410
- self.filter_expr = Some(match (current_expr, self.current_op) {
411
- (FilterExpr::Disjunction(mut exprs), LogicalOp::Disjunction) => {
412
- exprs.push(new_expr);
413
- FilterExpr::Disjunction(exprs)
414
- }
415
- (FilterExpr::Conjunction(mut exprs), LogicalOp::Conjunction) => {
416
- exprs.push(new_expr);
417
- FilterExpr::Conjunction(exprs)
418
- }
419
- (current_expr, LogicalOp::Disjunction) => {
420
- FilterExpr::Disjunction(vec![current_expr, new_expr])
421
- }
422
- (current_expr, LogicalOp::Conjunction) => {
423
- FilterExpr::Conjunction(vec![current_expr, new_expr])
424
- }
425
- });
426
-
427
- self.current_op = op;
428
- Ok(())
429
- }
430
-
431
- pub fn build<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<FilterExpr> {
432
- self.set_operation(journal_file, self.current_op)?;
433
-
434
- self.current_matches.clear();
435
- self.current_op = LogicalOp::Conjunction;
436
- self.filter_expr.take().ok_or(JournalError::MalformedFilter)
437
- }
438
-}
src/crates/jf/journal_file/src/journal_file.rs
deleted
-816
@@ -1,816 +0,0 @@
1
-#![allow(clippy::field_reassign_with_default)]
2
-
3
-use crate::hash;
4
-use crate::object::*;
5
-use crate::offset_array;
6
-use error::{JournalError, Result};
7
-use std::cell::{RefCell, UnsafeCell};
8
-use std::fs::OpenOptions;
9
-use std::path::Path;
10
-use window_manager::{MemoryMap, MemoryMapMut, WindowManager};
11
-use zerocopy::FromBytes;
12
-
13
-#[cfg(debug_assertions)]
14
-use std::backtrace::Backtrace;
15
-
16
-use crate::value_guard::ValueGuard;
17
-
18
-// Size to pad objects to (8 bytes)
19
-const OBJECT_ALIGNMENT: u64 = 8;
20
-
21
-/// A reader for systemd journal files that efficiently maps small regions of the file into memory.
22
-///
23
-/// # Memory Management
24
-///
25
-/// This implementation uses a window-based memory mapping strategy similar to systemd's original
26
-/// implementation. Instead of mapping the entire file, it maintains a small set of memory-mapped
27
-/// windows and reuses them as needed.
28
-///
29
-/// # Concurrency and Safety
30
-///
31
-/// `JournalFile` uses interior mutability to provide a safe API with the following characteristics:
32
-///
33
-/// - The window manager is wrapped in an `UnsafeCell` to allow mutation through a shared reference.
34
-/// - A single `RefCell<bool>` guards access to ensure only one object can be active at a time.
35
-/// - Methods like `data_object()` return a `ValueGuard<T>` that automatically releases the lock
36
-/// when dropped.
37
-///
38
-/// This design ensures that memory safety is maintained even though references to memory-mapped
39
-/// regions could be invalidated when new objects are created.
40
-pub struct JournalFile<M: MemoryMap> {
41
- // Persistent memory maps for journal header and data/field hash tables
42
- header_map: M,
43
- data_hash_table_map: Option<M>,
44
- field_hash_table_map: Option<M>,
45
-
46
- // Window manager for other objects
47
- window_manager: UnsafeCell<WindowManager<M>>,
48
-
49
- // Flag to track if any object is in use
50
- object_in_use: RefCell<bool>,
51
-
52
- #[cfg(debug_assertions)]
53
- prev_backtrace: RefCell<Backtrace>,
54
- #[cfg(debug_assertions)]
55
- backtrace: RefCell<Backtrace>,
56
-}
57
-
58
-impl<M: MemoryMapMut> JournalFile<M> {
59
- pub fn create(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
60
- debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
61
-
62
- let file = OpenOptions::new()
63
- .create(true)
64
- .truncate(true)
65
- .read(true)
66
- .write(true)
67
- .open(&path)?;
68
-
69
- let mut header = JournalHeader::default();
70
- header.signature = *b"LPKSHHRH";
71
-
72
- header.data_hash_table_offset = std::mem::size_of::<JournalHeader>() as u64
73
- + std::mem::size_of::<ObjectHeader>() as u64;
74
- header.data_hash_table_size = 4096 * std::mem::size_of::<HashItem>() as u64;
75
-
76
- header.field_hash_table_offset = header.data_hash_table_offset
77
- + header.data_hash_table_size
78
- + std::mem::size_of::<ObjectHeader>() as u64;
79
- header.field_hash_table_size = 512 * std::mem::size_of::<HashItem>() as u64;
80
-
81
- debug_assert_eq!(header.data_hash_table_offset % OBJECT_ALIGNMENT, 0);
82
- debug_assert_eq!(header.data_hash_table_size % OBJECT_ALIGNMENT, 0);
83
- let data_hash_table_map = header.map_data_hash_table(&file)?;
84
-
85
- debug_assert_eq!(header.field_hash_table_offset % OBJECT_ALIGNMENT, 0);
86
- debug_assert_eq!(header.field_hash_table_size % OBJECT_ALIGNMENT, 0);
87
- let field_hash_table_map = header.map_field_hash_table(&file)?;
88
-
89
- header.tail_object_offset = header.data_hash_table_offset + header.data_hash_table_size;
90
- header.n_objects = 2;
91
-
92
- debug_assert_eq!(header.tail_object_offset % OBJECT_ALIGNMENT, 0);
93
-
94
- let header_size = std::mem::size_of::<JournalHeader>() as u64;
95
- let mut header_map = M::create(&file, 0, header_size)?;
96
- {
97
- let header_mut = JournalHeader::mut_from_prefix(&mut header_map).unwrap().0;
98
- *header_mut = header;
99
- }
100
-
101
- // Create window manager for the rest of the objects
102
- let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
103
-
104
- Ok(JournalFile {
105
- header_map,
106
- data_hash_table_map,
107
- field_hash_table_map,
108
- window_manager,
109
- object_in_use: RefCell::new(false),
110
-
111
- #[cfg(debug_assertions)]
112
- prev_backtrace: RefCell::new(Backtrace::capture()),
113
- #[cfg(debug_assertions)]
114
- backtrace: RefCell::new(Backtrace::capture()),
115
- })
116
- }
117
-
118
- pub fn journal_header_mut(&mut self) -> &mut JournalHeader {
119
- JournalHeader::mut_from_prefix(&mut self.header_map)
120
- .unwrap()
121
- .0
122
- }
123
-
124
- pub fn data_hash_table_mut(&mut self) -> Option<HashTableObject<&mut [u8]>> {
125
- self.data_hash_table_map
126
- .as_mut()
127
- .map(|m| HashTableObject::<&mut [u8]>::from_data_mut(m, false))
128
- }
129
-
130
- pub fn field_hash_table_mut(&mut self) -> Option<HashTableObject<&mut [u8]>> {
131
- self.field_hash_table_map
132
- .as_mut()
133
- .map(|m| HashTableObject::<&mut [u8]>::from_data_mut(m, false))
134
- }
135
-
136
- fn object_header_mut(&self, position: u64) -> Result<&mut ObjectHeader> {
137
- let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
138
- let window_manager = unsafe { &mut *self.window_manager.get() };
139
- let header_slice = window_manager.get_slice_mut(position, size_needed)?;
140
- Ok(ObjectHeader::mut_from_bytes(header_slice).unwrap())
141
- }
142
-
143
- fn object_data_mut(&self, position: u64, size_needed: u64) -> Result<&mut [u8]> {
144
- let window_manager = unsafe { &mut *self.window_manager.get() };
145
- let object_slice = window_manager.get_slice_mut(position, size_needed)?;
146
- Ok(object_slice)
147
- }
148
-
149
- fn journal_object_mut<'a, T>(
150
- &'a self,
151
- type_: ObjectType,
152
- position: u64,
153
- size: Option<u64>,
154
- ) -> Result<ValueGuard<'a, T>>
155
- where
156
- T: JournalObjectMut<&'a mut [u8]>,
157
- {
158
- // Check if any object is already in use
159
- let mut is_in_use = self.object_in_use.borrow_mut();
160
- if *is_in_use {
161
- #[cfg(debug_assertions)]
162
- {
163
- eprintln!(
164
- "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
165
- self.backtrace.borrow().to_string(),
166
- self.prev_backtrace.borrow().to_string()
167
- );
168
- }
169
- return Err(JournalError::ValueGuardInUse);
170
- }
171
-
172
- #[cfg(debug_assertions)]
173
- {
174
- self.backtrace.swap(&self.prev_backtrace);
175
- let _ = self.backtrace.replace(Backtrace::force_capture());
176
- }
177
-
178
- let is_compact = self
179
- .journal_header_ref()
180
- .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
181
-
182
- let size_needed = match size {
183
- Some(size) => {
184
- let header = self.object_header_mut(position)?;
185
- header.type_ = type_ as u8;
186
- header.size = size;
187
- size
188
- }
189
- None => {
190
- let header = self.object_header_ref(position)?;
191
- if header.type_ != type_ as u8 {
192
- return Err(JournalError::InvalidObjectType);
193
- }
194
- header.size
195
- }
196
- };
197
-
198
- let data = self.object_data_mut(position, size_needed)?;
199
- let object = T::from_data_mut(data, is_compact);
200
-
201
- // Mark as in use
202
- *is_in_use = true;
203
- Ok(ValueGuard::new(object, &self.object_in_use))
204
- }
205
-
206
- pub fn offset_array_mut(
207
- &self,
208
- position: u64,
209
- capacity: Option<u64>,
210
- ) -> Result<ValueGuard<OffsetArrayObject<&mut [u8]>>> {
211
- let size = capacity.map(|c| {
212
- let mut size = std::mem::size_of::<OffsetArrayObjectHeader>() as u64;
213
-
214
- let is_compact = self
215
- .journal_header_ref()
216
- .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
217
- if is_compact {
218
- size += c * std::mem::size_of::<u32>() as u64;
219
- } else {
220
- size += c * std::mem::size_of::<u64>() as u64;
221
- }
222
-
223
- size
224
- });
225
-
226
- let offset_array = self.journal_object_mut(ObjectType::EntryArray, position, size);
227
- offset_array
228
- }
229
-
230
- pub fn field_mut(
231
- &self,
232
- position: u64,
233
- size: Option<u64>,
234
- ) -> Result<ValueGuard<FieldObject<&mut [u8]>>> {
235
- let size = size.map(|n| std::mem::size_of::<FieldObjectHeader>() as u64 + n);
236
- self.journal_object_mut(ObjectType::Field, position, size)
237
- }
238
-
239
- pub fn entry_mut(&self, position: u64) -> Result<ValueGuard<EntryObject<&mut [u8]>>> {
240
- self.journal_object_mut(ObjectType::Entry, position, None)
241
- }
242
-
243
- pub fn data_mut(
244
- &self,
245
- position: u64,
246
- size: Option<u64>,
247
- ) -> Result<ValueGuard<DataObject<&mut [u8]>>> {
248
- let size = size.map(|n| std::mem::size_of::<DataObjectHeader>() as u64 + n);
249
- self.journal_object_mut(ObjectType::Data, position, size)
250
- }
251
-
252
- pub fn tag_mut(&self, position: u64, new: bool) -> Result<ValueGuard<TagObject<&mut [u8]>>> {
253
- let size = if new {
254
- Some(std::mem::size_of::<TagObjectHeader>() as u64)
255
- } else {
256
- None
257
- };
258
- self.journal_object_mut(ObjectType::Tag, position, size)
259
- }
260
-}
261
-
262
-impl<M: MemoryMap> JournalFile<M> {
263
- pub fn open(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
264
- debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
265
-
266
- // Open file and check its size
267
- let file = OpenOptions::new().read(true).write(false).open(&path)?;
268
-
269
- // Create a memory map for the header
270
- let header_size = std::mem::size_of::<JournalHeader>() as u64;
271
- let header_map = M::create(&file, 0, header_size)?;
272
- let header = JournalHeader::ref_from_prefix(&header_map).unwrap().0;
273
- if header.signature != *b"LPKSHHRH" {
274
- return Err(JournalError::InvalidMagicNumber);
275
- }
276
-
277
- // Initialize the hash table maps if they exist
278
- let data_hash_table_map = header.map_data_hash_table(&file)?;
279
- let field_hash_table_map = header.map_field_hash_table(&file)?;
280
-
281
- // Create window manager for the rest of the objects
282
- let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
283
-
284
- Ok(JournalFile {
285
- header_map,
286
- data_hash_table_map,
287
- field_hash_table_map,
288
- window_manager,
289
- object_in_use: RefCell::new(false),
290
-
291
- #[cfg(debug_assertions)]
292
- prev_backtrace: RefCell::new(Backtrace::capture()),
293
- #[cfg(debug_assertions)]
294
- backtrace: RefCell::new(Backtrace::capture()),
295
- })
296
- }
297
-
298
- pub fn hash(&self, data: &[u8]) -> u64 {
299
- let is_keyed_hash = self
300
- .journal_header_ref()
301
- .has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash);
302
-
303
- hash::journal_hash_data(
304
- data,
305
- is_keyed_hash,
306
- if is_keyed_hash {
307
- Some(&self.journal_header_ref().file_id)
308
- } else {
309
- None
310
- },
311
- )
312
- }
313
-
314
- pub fn entry_list(&self) -> Option<offset_array::List> {
315
- let head_offset = std::num::NonZeroU64::new(self.journal_header_ref().entry_array_offset)?;
316
- let total_items =
317
- std::num::NonZeroUsize::new(self.journal_header_ref().n_entries as usize)?;
318
- Some(offset_array::List::new(head_offset, total_items))
319
- }
320
-
321
- pub fn journal_header_ref(&self) -> &JournalHeader {
322
- JournalHeader::ref_from_prefix(&self.header_map).unwrap().0
323
- }
324
-
325
- pub fn data_hash_table_ref(&self) -> Option<HashTableObject<&[u8]>> {
326
- self.data_hash_table_map
327
- .as_ref()
328
- .and_then(|m| HashTableObject::<&[u8]>::from_data(m, false))
329
- }
330
-
331
- pub fn field_hash_table_ref(&self) -> Option<HashTableObject<&[u8]>> {
332
- self.field_hash_table_map
333
- .as_ref()
334
- .and_then(|m| HashTableObject::<&[u8]>::from_data(m, false))
335
- }
336
-
337
- fn object_header_ref(&self, position: u64) -> Result<&ObjectHeader> {
338
- let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
339
- let window_manager = unsafe { &mut *self.window_manager.get() };
340
- let header_slice = window_manager.get_slice(position, size_needed)?;
341
- Ok(ObjectHeader::ref_from_bytes(header_slice).unwrap())
342
- }
343
-
344
- fn object_data_ref(&self, position: u64, size_needed: u64) -> Result<&[u8]> {
345
- let window_manager = unsafe { &mut *self.window_manager.get() };
346
- let object_slice = window_manager.get_slice(position, size_needed)?;
347
- Ok(object_slice)
348
- }
349
-
350
- fn journal_object_ref<'a, T>(&'a self, position: u64) -> Result<ValueGuard<'a, T>>
351
- where
352
- T: JournalObject<&'a [u8]>,
353
- {
354
- // Check if any object is already in use
355
- let mut is_in_use = self.object_in_use.borrow_mut();
356
- if *is_in_use {
357
- #[cfg(debug_assertions)]
358
- {
359
- eprintln!(
360
- "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
361
- self.backtrace.borrow().to_string(),
362
- self.prev_backtrace.borrow().to_string()
363
- );
364
- }
365
- return Err(JournalError::ValueGuardInUse);
366
- }
367
-
368
- #[cfg(debug_assertions)]
369
- {
370
- self.backtrace.swap(&self.prev_backtrace);
371
- let _ = self.backtrace.replace(Backtrace::force_capture());
372
- }
373
-
374
- let is_compact = self
375
- .journal_header_ref()
376
- .has_incompatible_flag(HeaderIncompatibleFlags::Compact);
377
-
378
- let size_needed = {
379
- let header = self.object_header_ref(position)?;
380
- header.size
381
- };
382
-
383
- let data = self.object_data_ref(position, size_needed)?;
384
- let Some(object) = T::from_data(data, is_compact) else {
385
- return Err(JournalError::ZerocopyFailure);
386
- };
387
-
388
- // Mark as in use
389
- *is_in_use = true;
390
-
391
- Ok(ValueGuard::new(object, &self.object_in_use))
392
- }
393
-
394
- pub fn offset_array_ref(&self, position: u64) -> Result<ValueGuard<OffsetArrayObject<&[u8]>>> {
395
- self.journal_object_ref(position)
396
- }
397
-
398
- pub fn field_ref(&self, position: u64) -> Result<ValueGuard<FieldObject<&[u8]>>> {
399
- self.journal_object_ref(position)
400
- }
401
-
402
- pub fn entry_ref(&self, position: u64) -> Result<ValueGuard<EntryObject<&[u8]>>> {
403
- self.journal_object_ref(position)
404
- }
405
-
406
- pub fn data_ref(&self, position: u64) -> Result<ValueGuard<DataObject<&[u8]>>> {
407
- self.journal_object_ref(position)
408
- }
409
-
410
- pub fn tag_ref(&self, position: u64) -> Result<ValueGuard<TagObject<&[u8]>>> {
411
- self.journal_object_ref(position)
412
- }
413
-
414
- fn lookup_hash_table<'a, T, F>(
415
- &'a self,
416
- hash_table: Option<HashTableObject<&[u8]>>,
417
- data: &[u8],
418
- hash: u64,
419
- fetch_fn: F,
420
- ) -> Result<u64>
421
- where
422
- T: HashableObject,
423
- F: Fn(u64) -> Result<ValueGuard<'a, T>>,
424
- {
425
- let hash_table = hash_table.ok_or(JournalError::MissingHashTable)?;
426
-
427
- // Find the right bucket in the hash table
428
- let hash_table_size = hash_table.items.len();
429
- let bucket_idx = (hash % hash_table_size as u64) as usize;
430
-
431
- // Get the head object offset from the bucket
432
- let bucket = hash_table.items[bucket_idx];
433
- let mut object_offset = bucket.head_hash_offset;
434
-
435
- // Traverse the linked list of objects in this bucket
436
- while object_offset != 0 {
437
- match fetch_fn(object_offset) {
438
- Ok(object_guard) => {
439
- // Check if this is the object we're looking for
440
- if object_guard.hash() == hash && object_guard.get_payload() == data {
441
- return Ok(object_offset);
442
- }
443
-
444
- // Move to the next object in the chain
445
- object_offset = object_guard.next_hash_offset();
446
- }
447
- Err(e) => {
448
- return Err(e);
449
- }
450
- }
451
- }
452
-
453
- Err(JournalError::MissingObjectFromHashTable)
454
- }
455
-
456
- /// Finds a field object by name and returns its offset
457
- pub fn find_field_offset_by_name(&self, field_name: &[u8], hash: u64) -> Result<u64> {
458
- self.lookup_hash_table::<FieldObject<&[u8]>, _>(
459
- self.field_hash_table_ref(),
460
- field_name,
461
- hash,
462
- |offset| self.field_ref(offset),
463
- )
464
- }
465
-
466
- /// Finds a data object by payload and returns its offset
467
- pub fn find_data_offset_by_payload(&self, payload: &[u8], hash: u64) -> Result<u64> {
468
- self.lookup_hash_table::<DataObject<&[u8]>, _>(
469
- self.data_hash_table_ref(),
470
- payload,
471
- hash,
472
- |offset| self.data_ref(offset),
473
- )
474
- }
475
-
476
- /// Run a directed partition point query on a data object's entry array
477
- ///
478
- /// This finds the first/last entry (depending on direction) that satisfies the given predicate
479
- /// in the entry array chain of the data object.
480
- pub fn data_object_directed_partition_point<F>(
481
- &self,
482
- data_offset: u64,
483
- predicate: F,
484
- direction: offset_array::Direction,
485
- ) -> Result<Option<u64>>
486
- where
487
- F: Fn(u64) -> Result<bool>,
488
- {
489
- let Some(cursor) = self.data_ref(data_offset)?.inlined_cursor() else {
490
- return Ok(None);
491
- };
492
-
493
- let best_match = cursor.directed_partition_point(self, predicate, direction)?;
494
-
495
- // Convert the result to an entry offset
496
- best_match.map(|c| c.value(self)).transpose()
497
- }
498
-
499
- /// Creates an iterator over all offsets of an offset array list
500
- pub fn array_offsets(&self, position: u64) -> Result<OffsetArrayListIterator<'_, M>> {
501
- Ok(OffsetArrayListIterator {
502
- journal: self,
503
- offset: position,
504
- capacity: if position == 0 {
505
- 0
506
- } else {
507
- self.offset_array_ref(position)?.capacity()
508
- },
509
- index: 0,
510
- })
511
- }
512
-
513
- /// Creates an iterator over all entry offsets in the journal
514
- pub fn entry_offsets(&self) -> Result<OffsetArrayListIterator<'_, M>> {
515
- self.array_offsets(self.journal_header_ref().entry_array_offset)
516
- }
517
-
518
- /// Creates an iterator over all field objects in the field hash table
519
- pub fn fields(&self) -> FieldIterator<'_, M> {
520
- // Get the field hash table
521
- let field_hash_table = self.field_hash_table_ref();
522
-
523
- // Initialize with the first bucket
524
- let mut iterator = FieldIterator {
525
- journal: self,
526
- field_hash_table,
527
- current_bucket_index: 0,
528
- next_field_offset: 0,
529
- };
530
-
531
- // Find the first non-empty bucket
532
- iterator.advance_to_next_nonempty_bucket();
533
-
534
- iterator
535
- }
536
-
537
- /// Creates an iterator over all DATA objects for the specified field
538
- pub fn field_data_objects<'a>(
539
- &'a self,
540
- field_name: &'a [u8],
541
- ) -> Result<FieldDataIterator<'a, M>> {
542
- // Find the field offset by name
543
- let field_hash = self.hash(field_name);
544
- let field_offset = self.find_field_offset_by_name(field_name, field_hash)?;
545
-
546
- // Get the field object to access its head_data_offset
547
- let field_guard = self.field_ref(field_offset)?;
548
- let head_data_offset = field_guard.header.head_data_offset;
549
-
550
- // Create the iterator
551
- Ok(FieldDataIterator {
552
- journal: self,
553
- current_data_offset: head_data_offset,
554
- })
555
- }
556
-
557
- /// Creates an iterator over all DATA objects for a specific entry
558
- pub fn entry_data_objects(&self, entry_offset: u64) -> Result<EntryDataIterator<'_, M>> {
559
- // Get the entry object to determine how many data items it has
560
- let entry_guard = self.entry_ref(entry_offset)?;
561
-
562
- // Get the total number of items
563
- let total_items = match &entry_guard.items {
564
- EntryItemsType::Regular(items) => items.len(),
565
- EntryItemsType::Compact(items) => items.len(),
566
- };
567
-
568
- // Create the iterator
569
- Ok(EntryDataIterator {
570
- journal: self,
571
- entry_offset,
572
- current_index: 0,
573
- total_items,
574
- })
575
- }
576
-}
577
-
578
-/*
579
- * Offset array iteration
580
-*/
581
-
582
-/// Iterator that returns all offsets in an offset array list
583
-pub struct OffsetArrayListIterator<'a, M: MemoryMap> {
584
- journal: &'a JournalFile<M>,
585
- offset: u64,
586
- capacity: usize,
587
- index: usize,
588
-}
589
-
590
-impl<M: MemoryMap> Iterator for OffsetArrayListIterator<'_, M> {
591
- type Item = Result<u64>;
592
-
593
- fn next(&mut self) -> Option<Self::Item> {
594
- // If we've reached the end of offsets in the offset array list, return None
595
- if self.offset == 0 {
596
- return None;
597
- }
598
-
599
- // Check if we need to move to the next offset array
600
- if self.index >= self.capacity {
601
- // Get the next offset array offset
602
- let next_offset = match self.journal.offset_array_ref(self.offset) {
603
- Ok(array_guard) => array_guard.header.next_offset_array,
604
- Err(e) => return Some(Err(e)),
605
- };
606
-
607
- // If there's no next offset array, we're done
608
- if next_offset == 0 {
609
- self.offset = 0;
610
- return None;
611
- }
612
-
613
- // Set up the next offset array
614
- match self.journal.offset_array_ref(next_offset) {
615
- Ok(array_guard) => {
616
- self.offset = next_offset;
617
- self.capacity = array_guard.capacity();
618
- self.index = 0;
619
- }
620
- Err(e) => return Some(Err(e)),
621
- }
622
- }
623
-
624
- // Get the current offset from the offset array
625
- let offset = match self.journal.offset_array_ref(self.offset) {
626
- Ok(array_guard) => match &array_guard.items {
627
- OffsetsType::Regular(offsets) => {
628
- if self.index < offsets.len() {
629
- offsets[self.index]
630
- } else {
631
- 0
632
- }
633
- }
634
- OffsetsType::Compact(offsets) => {
635
- if self.index < offsets.len() {
636
- offsets[self.index] as u64
637
- } else {
638
- 0
639
- }
640
- }
641
- },
642
- Err(e) => return Some(Err(e)),
643
- };
644
-
645
- // Increment index for next iteration
646
- self.index += 1;
647
-
648
- // If offset is zero, we've reached the end of all entries
649
- if offset == 0 {
650
- self.offset = 0;
651
- return None;
652
- }
653
-
654
- Some(Ok(offset))
655
- }
656
-}
657
-
658
-/// Iterator that walks through all field objects in the field hash table
659
-pub struct FieldIterator<'a, M: MemoryMap> {
660
- journal: &'a JournalFile<M>,
661
- field_hash_table: Option<HashTableObject<&'a [u8]>>,
662
- current_bucket_index: usize,
663
- next_field_offset: u64,
664
-}
665
-
666
-impl<M: MemoryMap> FieldIterator<'_, M> {
667
- /// Advances to the next non-empty bucket
668
- fn advance_to_next_nonempty_bucket(&mut self) {
669
- // If we don't have a hash table, there's nothing to iterate
670
- let Some(hash_table) = &self.field_hash_table else {
671
- return;
672
- };
673
-
674
- let items = &hash_table.items;
675
- let num_buckets = items.len();
676
-
677
- // Find the next non-empty bucket
678
- while self.current_bucket_index < num_buckets {
679
- let bucket = items[self.current_bucket_index];
680
- if bucket.head_hash_offset != 0 {
681
- // Found a non-empty bucket
682
- self.next_field_offset = bucket.head_hash_offset;
683
- return;
684
- }
685
- self.current_bucket_index += 1;
686
- }
687
-
688
- // No more non-empty buckets
689
- self.next_field_offset = 0;
690
- }
691
-}
692
-
693
-impl<'a, M: MemoryMap> Iterator for FieldIterator<'a, M> {
694
- type Item = Result<ValueGuard<'a, FieldObject<&'a [u8]>>>;
695
-
696
- fn next(&mut self) -> Option<Self::Item> {
697
- // If we've reached the end, return None
698
- if self.next_field_offset == 0 {
699
- return None;
700
- }
701
-
702
- // Save the current offset to return
703
- let offset = self.next_field_offset;
704
-
705
- // Try to get the field object
706
- match self.journal.field_ref(offset) {
707
- Ok(field_guard) => {
708
- // Get the next field offset before we return the guard
709
- self.next_field_offset = field_guard.header.next_hash_offset;
710
-
711
- // If we've reached the end of the chain, move to the next bucket
712
- if self.next_field_offset == 0 {
713
- self.current_bucket_index += 1;
714
- self.advance_to_next_nonempty_bucket();
715
- }
716
-
717
- Some(Ok(field_guard))
718
- }
719
- Err(e) => {
720
- // If we can't read the field, return the error and stop iteration
721
- self.next_field_offset = 0;
722
- Some(Err(e))
723
- }
724
- }
725
- }
726
-}
727
-
728
-/// Iterator that walks through all DATA objects for a specific field
729
-pub struct FieldDataIterator<'a, M: MemoryMap> {
730
- journal: &'a JournalFile<M>,
731
- current_data_offset: u64,
732
-}
733
-
734
-impl<'a, M: MemoryMap> Iterator for FieldDataIterator<'a, M> {
735
- type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
736
-
737
- fn next(&mut self) -> Option<Self::Item> {
738
- // If we've reached the end, return None
739
- if self.current_data_offset == 0 {
740
- return None;
741
- }
742
-
743
- // Save the current offset to return
744
- let offset = self.current_data_offset;
745
-
746
- // Try to get the data object
747
- match self.journal.data_ref(offset) {
748
- Ok(data_guard) => {
749
- // Get the next data offset before we return the guard
750
- self.current_data_offset = data_guard.header.next_field_offset;
751
- Some(Ok(data_guard))
752
- }
753
- Err(e) => {
754
- // If we can't read the data object, return the error and stop iteration
755
- self.current_data_offset = 0;
756
- Some(Err(e))
757
- }
758
- }
759
- }
760
-}
761
-
762
-/// Iterator that walks through all DATA objects for a specific entry
763
-pub struct EntryDataIterator<'a, M: MemoryMap> {
764
- journal: &'a JournalFile<M>,
765
- entry_offset: u64,
766
- current_index: usize,
767
- total_items: usize,
768
-}
769
-
770
-impl<'a, M: MemoryMap> Iterator for EntryDataIterator<'a, M> {
771
- type Item = Result<ValueGuard<'a, DataObject<&'a [u8]>>>;
772
-
773
- fn next(&mut self) -> Option<Self::Item> {
774
- // If we've reached the end of the data indices, return None
775
- if self.current_index >= self.total_items {
776
- return None;
777
- }
778
-
779
- // Get the entry object to access the data offset
780
- match self.journal.entry_ref(self.entry_offset) {
781
- Ok(entry_guard) => {
782
- let idx = self.current_index;
783
- self.current_index += 1;
784
-
785
- let data_offset = match &entry_guard.items {
786
- EntryItemsType::Regular(items) => {
787
- if idx >= items.len() {
788
- return None;
789
- }
790
- items[idx].object_offset
791
- }
792
- EntryItemsType::Compact(items) => {
793
- if idx >= items.len() {
794
- return None;
795
- }
796
- items[idx].object_offset as u64
797
- }
798
- };
799
-
800
- // Drop the entry guard before obtaining the data object
801
- drop(entry_guard);
802
-
803
- // Try to get the data object
804
- match self.journal.data_ref(data_offset) {
805
- Ok(data_guard) => Some(Ok(data_guard)),
806
- Err(e) => Some(Err(e)),
807
- }
808
- }
809
- Err(e) => {
810
- // If we can't read the entry, return the error and stop iteration
811
- self.current_index = self.total_items;
812
- Some(Err(e))
813
- }
814
- }
815
- }
816
-}
src/crates/jf/journal_file/src/reader.rs
deleted
-194
@@ -1,194 +0,0 @@
1
-use crate::{
2
- cursor::{JournalCursor, Location},
3
- file::{EntryDataIterator, FieldDataIterator, FieldIterator, JournalFile},
4
- filter::{JournalFilter, LogicalOp},
5
- object::{DataObject, FieldObject},
6
- offset_array::Direction,
7
- value_guard::ValueGuard,
8
-};
9
-use error::Result;
10
-use std::num::NonZeroU64;
11
-use window_manager::MemoryMap;
12
-
13
-pub struct JournalReader<'a, M: MemoryMap> {
14
- cursor: JournalCursor,
15
-
16
- filter: Option<JournalFilter>,
17
- field_iterator: Option<FieldIterator<'a, M>>,
18
- field_data_iterator: Option<FieldDataIterator<'a, M>>,
19
- entry_data_iterator: Option<EntryDataIterator<'a, M>>,
20
-
21
- field_guard: Option<ValueGuard<'a, FieldObject<&'a [u8]>>>,
22
- data_guard: Option<ValueGuard<'a, DataObject<&'a [u8]>>>,
23
-}
24
-
25
-impl<M: MemoryMap> std::fmt::Debug for JournalReader<'_, M> {
26
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
27
- f.debug_struct("JournalReader")
28
- // .field("cursor", &self.cursor)
29
- .field("field_guard", &self.field_guard)
30
- .field("data_guard", &self.data_guard)
31
- .finish()
32
- }
33
-}
34
-
35
-impl<M: MemoryMap> Default for JournalReader<'_, M> {
36
- fn default() -> Self {
37
- Self {
38
- cursor: JournalCursor::new(),
39
- filter: None,
40
- field_iterator: None,
41
- field_data_iterator: None,
42
- entry_data_iterator: None,
43
- field_guard: None,
44
- data_guard: None,
45
- }
46
- }
47
-}
48
-
49
-impl<'a, M: MemoryMap> JournalReader<'a, M> {
50
- pub fn dump(&self, journal_file: &'a JournalFile<M>) -> Result<String> {
51
- if let Some(filter_expr) = self.cursor.filter_expr.as_ref() {
52
- filter_expr.dump(journal_file)
53
- } else {
54
- Ok(String::from("no filter expr"))
55
- }
56
- }
57
-
58
- pub fn set_location(&mut self, location: Location) {
59
- self.cursor.set_location(location)
60
- }
61
-
62
- pub fn step(&mut self, journal_file: &'a JournalFile<M>, direction: Direction) -> Result<bool> {
63
- self.drop_guards();
64
-
65
- if let Some(filter) = self.filter.as_mut() {
66
- let filter_expr = filter.build(journal_file)?;
67
- self.cursor.set_filter(filter_expr);
68
- self.filter = None;
69
- }
70
-
71
- self.cursor.step(journal_file, direction)
72
- }
73
-
74
- pub fn add_match(&mut self, data: &[u8]) {
75
- self.filter.get_or_insert_default().add_match(data);
76
- }
77
-
78
- pub fn add_conjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
79
- self.filter
80
- .get_or_insert_default()
81
- .set_operation(journal_file, LogicalOp::Conjunction)
82
- }
83
-
84
- pub fn add_disjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
85
- self.filter
86
- .get_or_insert_default()
87
- .set_operation(journal_file, LogicalOp::Disjunction)
88
- }
89
-
90
- pub fn flush_matches(&mut self) {
91
- self.cursor.clear_filter();
92
- self.filter = None;
93
- }
94
-
95
- pub fn get_realtime_usec(&self, journal_file: &'a JournalFile<M>) -> Result<u64> {
96
- let entry_offset = self.cursor.position()?;
97
- let entry_object = journal_file.entry_ref(entry_offset)?;
98
- Ok(entry_object.header.realtime)
99
- }
100
-
101
- pub fn get_seqnum(&self, journal_file: &'a JournalFile<M>) -> Result<(u64, [u8; 16])> {
102
- let entry_offset = self.cursor.position()?;
103
- let entry_object = journal_file.entry_ref(entry_offset)?;
104
- Ok((
105
- entry_object.header.seqnum,
106
- journal_file.journal_header_ref().seqnum_id,
107
- ))
108
- }
109
-
110
- pub fn get_entry_offset(&self) -> Result<NonZeroU64> {
111
- self.cursor.position()
112
- }
113
-
114
- fn drop_guards(&mut self) {
115
- self.field_guard.take();
116
- self.data_guard.take();
117
- }
118
-
119
- pub fn fields_restart(&mut self) {
120
- self.drop_guards();
121
- self.field_iterator = None;
122
- }
123
-
124
- pub fn fields_enumerate(
125
- &mut self,
126
- journal_file: &'a JournalFile<M>,
127
- ) -> Result<Option<&ValueGuard<'_, FieldObject<&'a [u8]>>>> {
128
- self.drop_guards();
129
-
130
- if self.field_iterator.is_none() {
131
- self.field_iterator = Some(journal_file.fields());
132
- }
133
-
134
- if let Some(iter) = &mut self.field_iterator {
135
- self.field_guard = iter.next().transpose()?;
136
- Ok(self.field_guard.as_ref())
137
- } else {
138
- Ok(None)
139
- }
140
- }
141
-
142
- pub fn field_data_query_unique(
143
- &mut self,
144
- journal_file: &'a JournalFile<M>,
145
- field_name: &'a [u8],
146
- ) -> Result<()> {
147
- self.drop_guards();
148
-
149
- self.field_data_iterator = Some(journal_file.field_data_objects(field_name)?);
150
- Ok(())
151
- }
152
-
153
- pub fn field_data_restart(&mut self) {
154
- self.drop_guards();
155
- }
156
-
157
- pub fn field_data_enumerate(
158
- &mut self,
159
- _: &'a JournalFile<M>,
160
- ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
161
- self.drop_guards();
162
-
163
- if let Some(iter) = &mut self.field_data_iterator {
164
- self.data_guard = iter.next().transpose()?;
165
- Ok(self.data_guard.as_ref())
166
- } else {
167
- Ok(None)
168
- }
169
- }
170
-
171
- pub fn entry_data_restart(&mut self) {
172
- self.drop_guards();
173
- self.entry_data_iterator = None;
174
- }
175
-
176
- pub fn entry_data_enumerate(
177
- &mut self,
178
- journal_file: &'a JournalFile<M>,
179
- ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
180
- self.drop_guards();
181
-
182
- if self.entry_data_iterator.is_none() {
183
- let entry_offset = self.cursor.position()?;
184
- self.entry_data_iterator = Some(journal_file.entry_data_objects(entry_offset)?);
185
- }
186
-
187
- if let Some(iter) = &mut self.entry_data_iterator {
188
- self.data_guard = iter.next().transpose()?;
189
- Ok(self.data_guard.as_ref())
190
- } else {
191
- Ok(None)
192
- }
193
- }
194
-}
src/crates/jf/journal_log/Cargo.toml
deleted
-14
@@ -1,14 +0,0 @@
1
-[package]
2
-name = "journal_log"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-journal_file = { path = "../journal_file" }
10
-memmap2 = { workspace = true }
11
-uuid = { version = "1.0", features = ["v4", "rng"] }
12
-
13
-[dev-dependencies]
14
-tempfile = { workspace = true }
src/crates/jf/journal_log/src/lib.rs
deleted
-632
@@ -1,632 +0,0 @@
1
-use error::{JournalError, Result};
2
-use journal_file::{
3
- load_boot_id, BucketUtilization, JournalFile, JournalFileOptions, JournalWriter,
4
-};
5
-use memmap2::MmapMut;
6
-use std::cmp::Ordering;
7
-use std::ffi::OsStr;
8
-use std::path::{Path, PathBuf};
9
-use std::time::{Duration, SystemTime, UNIX_EPOCH};
10
-
11
-#[derive(Debug, Clone, PartialEq, Eq)]
12
-pub struct JournalFileInfo {
13
- pub path: PathBuf,
14
- pub timestamp: SystemTime,
15
- pub counter: u64,
16
- pub size: Option<u64>,
17
-}
18
-
19
-impl JournalFileInfo {
20
- pub fn from_path(path: impl AsRef<Path>) -> Result<Self> {
21
- let path = path.as_ref();
22
- let filename = path
23
- .file_name()
24
- .and_then(OsStr::to_str)
25
- .ok_or(JournalError::InvalidFilename)?;
26
-
27
- let (timestamp, counter) = Self::parse_filename(filename)?;
28
-
29
- Ok(Self {
30
- path: path.to_path_buf(),
31
- timestamp,
32
- counter,
33
- size: None,
34
- })
35
- }
36
-
37
- pub fn from_parts(
38
- timestamp: SystemTime,
39
- counter: u64,
40
- size: Option<u64>,
41
- ) -> Result<JournalFileInfo> {
42
- let duration = timestamp
43
- .duration_since(UNIX_EPOCH)
44
- .map_err(|_| JournalError::SystemTimeError)?;
45
- let micros = duration.as_secs() * 1_000_000 + duration.subsec_micros() as u64;
46
- let path = PathBuf::from(format!("journal-{}-{}.journal", micros, counter));
47
-
48
- Ok(Self {
49
- path,
50
- timestamp,
51
- counter,
52
- size,
53
- })
54
- }
55
-
56
- /// Parse timestamp and counter from filename
57
- /// Expected format: "journal-{timestamp_micros}-{counter}.journal"
58
- fn parse_filename(filename: &str) -> Result<(SystemTime, u64)> {
59
- let name = filename.strip_suffix(".journal").unwrap_or(filename);
60
-
61
- if let Some(stripped) = name.strip_prefix("journal-") {
62
- let parts: Vec<&str> = stripped.split('-').collect();
63
- if parts.len() == 2 {
64
- let timestamp_micros: u64 = parts[0]
65
- .parse()
66
- .map_err(|_| JournalError::InvalidFilename)?;
67
- let counter: u64 = parts[1]
68
- .parse()
69
- .map_err(|_| JournalError::InvalidFilename)?;
70
-
71
- let timestamp = UNIX_EPOCH + Duration::from_micros(timestamp_micros);
72
- return Ok((timestamp, counter));
73
- }
74
- }
75
-
76
- Err(JournalError::InvalidFilename)
77
- }
78
-
79
- /// Get file size, loading from filesystem if not cached
80
- pub fn get_size(&mut self) -> Result<u64> {
81
- if let Some(size) = self.size {
82
- Ok(size)
83
- } else {
84
- let metadata = std::fs::metadata(&self.path)?;
85
- let size = metadata.len();
86
- self.size = Some(size);
87
- Ok(size)
88
- }
89
- }
90
-}
91
-
92
-// Implement ordering based on counter (for detecting duplicates and ordering)
93
-impl Ord for JournalFileInfo {
94
- fn cmp(&self, other: &Self) -> Ordering {
95
- // Order by counter only - this enables duplicate detection
96
- self.counter.cmp(&other.counter)
97
- }
98
-}
99
-
100
-impl PartialOrd for JournalFileInfo {
101
- fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
102
- Some(self.cmp(other))
103
- }
104
-}
105
-
106
-/// Determines when an active journal file should be sealed
107
-#[derive(Debug, Copy, Clone, Default)]
108
-pub struct RotationPolicy {
109
- /// Maximum file size before rotating (in bytes)
110
- pub size_of_journal_file: Option<u64>,
111
- /// Maximum duration that entries in a single file can span
112
- pub duration_of_journal_file: Option<Duration>,
113
-}
114
-
115
-impl RotationPolicy {
116
- pub fn with_size_of_journal_file(mut self, size_of_journal_file: u64) -> Self {
117
- self.size_of_journal_file = Some(size_of_journal_file);
118
- self
119
- }
120
-
121
- pub fn with_duration_of_journal_file(mut self, duration_of_journal_file: Duration) -> Self {
122
- self.duration_of_journal_file = Some(duration_of_journal_file);
123
- self
124
- }
125
-}
126
-
127
-/// Retention policy - determines when files should be removed
128
-#[derive(Debug, Copy, Clone, Default)]
129
-pub struct RetentionPolicy {
130
- /// Maximum number of journal files to keep
131
- pub number_of_journal_files: Option<usize>,
132
- /// Maximum total size of all journal files (in bytes)
133
- pub size_of_journal_files: Option<u64>,
134
- /// Maximum age of files to keep
135
- pub duration_of_journal_files: Option<Duration>,
136
-}
137
-
138
-impl RetentionPolicy {
139
- pub fn with_number_of_journal_files(mut self, number_of_journal_files: usize) -> Self {
140
- self.number_of_journal_files = Some(number_of_journal_files);
141
- self
142
- }
143
-
144
- pub fn with_size_of_journal_files(mut self, size_of_journal_files: u64) -> Self {
145
- self.size_of_journal_files = Some(size_of_journal_files);
146
- self
147
- }
148
-
149
- pub fn with_duration_of_journal_files(mut self, duration_of_journal_files: Duration) -> Self {
150
- self.duration_of_journal_files = Some(duration_of_journal_files);
151
- self
152
- }
153
-}
154
-
155
-/// Configuration for journal directory management
156
-#[derive(Debug, Clone)]
157
-pub struct JournalDirectoryConfig {
158
- /// Directory path where journal files are stored
159
- pub directory: PathBuf,
160
- /// Policy for when to rotate active files
161
- pub rotation_policy: RotationPolicy,
162
- /// Policy for when to remove old files
163
- pub retention_policy: RetentionPolicy,
164
-}
165
-
166
-impl JournalDirectoryConfig {
167
- pub fn new(directory: impl Into<PathBuf>) -> Self {
168
- Self {
169
- directory: directory.into(),
170
- rotation_policy: RotationPolicy::default(),
171
- retention_policy: RetentionPolicy::default(),
172
- }
173
- }
174
-
175
- pub fn with_sealing_policy(mut self, policy: RotationPolicy) -> Self {
176
- self.rotation_policy = policy;
177
- self
178
- }
179
-
180
- pub fn with_retention_policy(mut self, policy: RetentionPolicy) -> Self {
181
- self.retention_policy = policy;
182
- self
183
- }
184
-}
185
-
186
-/// Manages a directory of journal files with automatic cleanup and sealing
187
-#[derive(Debug)]
188
-pub struct JournalDirectory {
189
- config: JournalDirectoryConfig,
190
- /// Files ordered by counter (oldest counter first)
191
- files: Vec<JournalFileInfo>,
192
- /// Next counter value for new files
193
- next_counter: u64,
194
- /// Cached total size of all files
195
- total_size: u64,
196
-}
197
-
198
-impl JournalDirectory {
199
- /// Scan the directory and load existing journal files
200
- pub fn with_config(config: JournalDirectoryConfig) -> Result<Self> {
201
- // Create directory if it does not already exist.
202
- if !config.directory.exists() {
203
- std::fs::create_dir_all(&config.directory)?;
204
- } else if !config.directory.is_dir() {
205
- return Err(JournalError::NotADirectory);
206
- }
207
-
208
- let mut journal_directory = Self {
209
- config,
210
- files: Vec::new(),
211
- next_counter: 0,
212
- total_size: 0,
213
- };
214
-
215
- // Read all .journal files from directory
216
- for entry in std::fs::read_dir(&journal_directory.config.directory)? {
217
- let entry = entry?;
218
- let file_path = entry.path();
219
-
220
- if file_path.extension() != Some(OsStr::new("journal")) {
221
- continue;
222
- }
223
-
224
- match JournalFileInfo::from_path(&file_path) {
225
- Ok(mut file_info) => {
226
- // Load the actual file size from filesystem
227
- let file_size = file_info.get_size().unwrap_or(0);
228
- journal_directory.total_size += file_size;
229
- journal_directory.next_counter =
230
- journal_directory.next_counter.max(file_info.counter + 1);
231
- journal_directory.files.push(file_info);
232
- }
233
- Err(_) => {
234
- // Skip files with invalid names
235
- continue;
236
- }
237
- }
238
- }
239
-
240
- // Sort files by counter to maintain order
241
- journal_directory.files.sort();
242
-
243
- Ok(journal_directory)
244
- }
245
-
246
- pub fn directory_path(&self) -> &Path {
247
- &self.config.directory
248
- }
249
-
250
- pub fn get_full_path(&self, file_info: &JournalFileInfo) -> PathBuf {
251
- if file_info.path.is_absolute() {
252
- file_info.path.clone()
253
- } else {
254
- self.config.directory.join(&file_info.path)
255
- }
256
- }
257
-
258
- // Get information about all the files in the journal directory
259
- pub fn files(&self) -> Vec<JournalFileInfo> {
260
- self.files.clone()
261
- }
262
-
263
- /// Add a new journal file to the directory representation
264
- pub fn new_file(&mut self, existing_file: Option<JournalFileInfo>) -> Result<JournalFileInfo> {
265
- let timestamp = SystemTime::now();
266
- let new_file = JournalFileInfo::from_parts(timestamp, self.next_counter, None)?;
267
-
268
- self.files.push(new_file.clone());
269
- self.next_counter += 1;
270
-
271
- if let Some(existing_file) = existing_file {
272
- self.total_size += existing_file.size.unwrap_or(0);
273
- }
274
-
275
- Ok(new_file)
276
- }
277
-
278
- /// Remove the oldest file (by counter) from both filesystem and tracking
279
- fn remove_oldest_file(&mut self) -> Result<()> {
280
- if let Some(oldest_file) = self.files.first() {
281
- let file_path = self.get_full_path(oldest_file);
282
- let file_size = oldest_file.size.unwrap_or(0);
283
-
284
- // Remove from filesystem
285
- if let Err(e) = std::fs::remove_file(&file_path) {
286
- // Log error but continue cleanup - file might already be deleted
287
- eprintln!(
288
- "Warning: Failed to remove journal file {:?}: {}",
289
- file_path, e
290
- );
291
- }
292
-
293
- // Remove from tracking and update total size
294
- self.files.remove(0);
295
- self.total_size = self.total_size.saturating_sub(file_size);
296
- }
297
-
298
- Ok(())
299
- }
300
-
301
- /// Remove files older than the specified cutoff time
302
- fn remove_files_older_than(&mut self, cutoff_time: SystemTime) -> Result<()> {
303
- // Find files older than cutoff time
304
- let mut files_to_remove = Vec::new();
305
- for (index, file) in self.files.iter().enumerate() {
306
- if file.timestamp <= cutoff_time {
307
- files_to_remove.push(index);
308
- }
309
- }
310
-
311
- // Remove files in reverse order to maintain indices
312
- for &index in files_to_remove.iter().rev() {
313
- let file = &self.files[index];
314
- let file_path = self.get_full_path(file);
315
- let file_size = file.size.unwrap_or(0);
316
-
317
- // Remove from filesystem
318
- if let Err(e) = std::fs::remove_file(&file_path) {
319
- // Log error but continue cleanup
320
- eprintln!(
321
- "Warning: Failed to remove journal file {:?}: {}",
322
- file_path, e
323
- );
324
- }
325
-
326
- // Remove from tracking and update total size
327
- self.files.remove(index);
328
- self.total_size = self.total_size.saturating_sub(file_size);
329
- }
330
-
331
- Ok(())
332
- }
333
-
334
- /// Enforce the retention policy by removing old files
335
- pub fn enforce_retention_policy(&mut self) -> Result<()> {
336
- let policy = self.config.retention_policy;
337
-
338
- // 1. Remove by file count limit
339
- if let Some(max_files) = policy.number_of_journal_files {
340
- while self.files.len() > max_files {
341
- self.remove_oldest_file()?;
342
- }
343
- }
344
-
345
- // 2. Remove by total size limit
346
- if let Some(max_total_size) = policy.size_of_journal_files {
347
- while self.total_size > max_total_size && !self.files.is_empty() {
348
- self.remove_oldest_file()?;
349
- }
350
- }
351
-
352
- // 3. Remove by entry age limit
353
- if let Some(max_entry_age) = policy.duration_of_journal_files {
354
- let cutoff_time = SystemTime::now()
355
- .checked_sub(max_entry_age)
356
- .unwrap_or(SystemTime::UNIX_EPOCH);
357
- self.remove_files_older_than(cutoff_time)?;
358
- }
359
-
360
- Ok(())
361
- }
362
-}
363
-
364
-fn generate_uuid() -> [u8; 16] {
365
- uuid::Uuid::new_v4().into_bytes()
366
-}
367
-
368
-/// Configuration for JournalLog
369
-#[derive(Debug, Clone)]
370
-pub struct JournalLogConfig {
371
- /// Directory where journal files are stored
372
- pub journal_dir: PathBuf,
373
- /// Policy for when to rotate active files
374
- pub rotation_policy: RotationPolicy,
375
- /// Policy for when to remove old files
376
- pub retention_policy: RetentionPolicy,
377
-}
378
-
379
-impl JournalLogConfig {
380
- pub fn new(journal_dir: impl Into<PathBuf>) -> Self {
381
- Self {
382
- journal_dir: journal_dir.into(),
383
- rotation_policy: RotationPolicy::default()
384
- .with_size_of_journal_file(100 * 1024 * 1024) // 100MB
385
- .with_duration_of_journal_file(Duration::from_secs(2 * 3600)), // 2 hours
386
- retention_policy: RetentionPolicy::default()
387
- .with_number_of_journal_files(10) // 10 files
388
- .with_size_of_journal_files(1024 * 1024 * 1024) // 1GB
389
- .with_duration_of_journal_files(Duration::from_secs(7 * 24 * 3600)), // 7 days
390
- }
391
- }
392
-
393
- pub fn with_rotation_policy(mut self, policy: RotationPolicy) -> Self {
394
- self.rotation_policy = policy;
395
- self
396
- }
397
-
398
- pub fn with_retention_policy(mut self, policy: RetentionPolicy) -> Self {
399
- self.retention_policy = policy;
400
- self
401
- }
402
-}
403
-
404
-pub struct JournalLog {
405
- directory: JournalDirectory,
406
- current_file: Option<JournalFile<MmapMut>>,
407
- current_writer: Option<JournalWriter>,
408
- current_file_info: Option<JournalFileInfo>,
409
- machine_id: [u8; 16],
410
- boot_id: [u8; 16],
411
- seqnum_id: [u8; 16],
412
- previous_bucket_utilization: Option<BucketUtilization>,
413
-}
414
-
415
-/// Calculate optimal bucket sizes based on previous file utilization or rotation policy
416
-fn calculate_bucket_sizes(
417
- previous_utilization: Option<&BucketUtilization>,
418
- rotation_policy: &RotationPolicy,
419
-) -> (usize, usize) {
420
- if let Some(utilization) = previous_utilization {
421
- let data_utilization = utilization.data_utilization();
422
- let field_utilization = utilization.field_utilization();
423
-
424
- let data_buckets = if data_utilization > 0.75 {
425
- (utilization.data_total * 2).next_power_of_two()
426
- } else if data_utilization < 0.25 && utilization.data_total > 4096 {
427
- (utilization.data_total / 2).next_power_of_two()
428
- } else {
429
- utilization.data_total
430
- };
431
-
432
- let field_buckets = if field_utilization > 0.75 {
433
- (utilization.field_total * 2).next_power_of_two()
434
- } else if field_utilization < 0.25 && utilization.field_total > 512 {
435
- (utilization.field_total / 2).next_power_of_two()
436
- } else {
437
- utilization.field_total
438
- };
439
-
440
- (data_buckets, field_buckets)
441
- } else {
442
- // Initial sizing based on rotation policy max file size
443
- let max_file_size = rotation_policy
444
- .size_of_journal_file
445
- .unwrap_or(8 * 1024 * 1024);
446
-
447
- // 16 MiB -> 4096 data buckets
448
- let data_buckets = (max_file_size / 4096).max(1024).next_power_of_two() as usize;
449
- let field_buckets = 128; // Assume ~8:1 data:field ratio
450
-
451
- (data_buckets, field_buckets)
452
- }
453
-}
454
-
455
-impl JournalLog {
456
- pub fn new(config: JournalLogConfig) -> Result<Self> {
457
- let journal_config = JournalDirectoryConfig::new(&config.journal_dir)
458
- .with_sealing_policy(config.rotation_policy)
459
- .with_retention_policy(config.retention_policy);
460
-
461
- let mut directory = JournalDirectory::with_config(journal_config)?;
462
-
463
- // Enforce retention policy on startup to clean up any old files
464
- directory.enforce_retention_policy()?;
465
-
466
- let machine_id = journal_file::file::load_machine_id()?;
467
- let boot_id = load_boot_id()?;
468
- // TODO: Use NETDATA_INVOCATION_ID
469
- let seqnum_id = generate_uuid();
470
-
471
- Ok(JournalLog {
472
- directory,
473
- current_file: None,
474
- current_writer: None,
475
- current_file_info: None,
476
- machine_id,
477
- boot_id,
478
- seqnum_id,
479
- previous_bucket_utilization: None,
480
- })
481
- }
482
-
483
- fn ensure_active_journal(&mut self) -> Result<()> {
484
- // Check if rotation is needed before writing
485
- if let Some(writer) = &self.current_writer {
486
- if self.should_rotate(writer) {
487
- self.rotate_current_file()?;
488
- }
489
- }
490
-
491
- if self.current_file.is_none() {
492
- // Create a new journal file
493
- let file_info = self.directory.new_file(None)?;
494
-
495
- // Get the full path for the journal file
496
- let file_path = self.directory.get_full_path(&file_info);
497
-
498
- // Calculate optimal bucket sizes based on previous file utilization
499
- let (data_buckets, field_buckets) = calculate_bucket_sizes(
500
- self.previous_bucket_utilization.as_ref(),
501
- &self.directory.config.rotation_policy,
502
- );
503
-
504
- let options = JournalFileOptions::new(
505
- self.machine_id,
506
- self.boot_id,
507
- self.seqnum_id,
508
- generate_uuid(),
509
- )
510
- .with_window_size(8 * 1024 * 1024)
511
- .with_data_hash_table_buckets(data_buckets)
512
- .with_field_hash_table_buckets(field_buckets)
513
- .with_keyed_hash(true);
514
-
515
- let mut journal_file = JournalFile::create(&file_path, options)?;
516
- let writer = JournalWriter::new(&mut journal_file)?;
517
-
518
- self.current_file = Some(journal_file);
519
- self.current_writer = Some(writer);
520
- self.current_file_info = Some(file_info);
521
-
522
- // Enforce retention policy after creating new file to account for the new file count
523
- self.directory.enforce_retention_policy()?;
524
- }
525
-
526
- Ok(())
527
- }
528
-
529
- /// Checks if we have to rotate. Prioritizes file size over file creation
530
- /// time.
531
- fn should_rotate(&self, writer: &JournalWriter) -> bool {
532
- let policy = self.directory.config.rotation_policy;
533
-
534
- // Check if the file size went over the limit
535
- if let Some(max_size) = policy.size_of_journal_file {
536
- if writer.current_file_size() >= max_size {
537
- return true;
538
- }
539
- }
540
-
541
- // Check if the time span between first and last entries exceeds the limit
542
- let Some(file) = &self.current_file else {
543
- return false;
544
- };
545
- let Some(max_entry_span) = policy.duration_of_journal_file else {
546
- return false;
547
- };
548
- let Some(first_monotonic) = writer.first_entry_monotonic() else {
549
- return false;
550
- };
551
-
552
- let header = file.journal_header_ref();
553
- let last_monotonic = header.tail_entry_monotonic;
554
-
555
- // Convert monotonic timestamps (microseconds) to duration
556
- let entry_span = if last_monotonic >= first_monotonic {
557
- Duration::from_micros(last_monotonic - first_monotonic)
558
- } else {
559
- return false;
560
- };
561
-
562
- if entry_span >= max_entry_span {
563
- return true;
564
- }
565
-
566
- false
567
- }
568
-
569
- fn rotate_current_file(&mut self) -> Result<()> {
570
- // Capture bucket utilization before closing the file
571
- if let Some(file) = &self.current_file {
572
- self.previous_bucket_utilization = file.bucket_utilization();
573
- }
574
-
575
- // Update the current file's size in our tracking before closing
576
- if let (Some(file_info), Some(writer)) = (&mut self.current_file_info, &self.current_writer)
577
- {
578
- let current_size = writer.current_file_size();
579
- file_info.size = Some(current_size);
580
-
581
- // Update the size in the directory's file list
582
- if let Some(tracked_file) = self
583
- .directory
584
- .files
585
- .iter_mut()
586
- .find(|f| f.counter == file_info.counter)
587
- {
588
- let old_size = tracked_file.size.unwrap_or(0);
589
- tracked_file.size = Some(current_size);
590
-
591
- // Update total size tracking
592
- self.directory.total_size = self
593
- .directory
594
- .total_size
595
- .saturating_sub(old_size)
596
- .saturating_add(current_size);
597
- }
598
- }
599
-
600
- // Close current file
601
- self.current_file = None;
602
- self.current_writer = None;
603
- self.current_file_info = None;
604
-
605
- // Next call to ensure_active_journal() will create new file
606
- Ok(())
607
- }
608
-
609
- pub fn write_entry(&mut self, items: &[&[u8]]) -> Result<()> {
610
- if items.is_empty() {
611
- return Ok(());
612
- }
613
-
614
- self.ensure_active_journal()?;
615
-
616
- let journal_file = self.current_file.as_mut().unwrap();
617
- let writer = self.current_writer.as_mut().unwrap();
618
-
619
- let now = SystemTime::now();
620
- let realtime = now
621
- .duration_since(UNIX_EPOCH)
622
- .unwrap_or_default()
623
- .as_micros() as u64;
624
-
625
- // Use realtime for monotonic as well for simplicity
626
- let monotonic = realtime;
627
-
628
- writer.add_entry(journal_file, items, realtime, monotonic, self.boot_id)?;
629
-
630
- Ok(())
631
- }
632
-}
src/crates/jf/journal_reader/Cargo.toml
deleted
-10
@@ -1,10 +0,0 @@
1
-[package]
2
-name = "journal_reader"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-journal_file = { path = "../journal_file" }
10
-window_manager = { path = "../window_manager" }
src/crates/jf/journal_reader/src/journal_cursor.rs
deleted
-214
@@ -1,214 +0,0 @@
1
-use crate::journal_filter::FilterExpr;
2
-use error::{JournalError, Result};
3
-use journal_file::{offset_array, offset_array::Direction, JournalFile};
4
-use window_manager::MemoryMap;
5
-
6
-#[derive(Debug, Copy, Clone, PartialEq, Eq)]
7
-pub enum Location {
8
- Head,
9
- Tail,
10
- Realtime(u64),
11
- Monotonic(u64, [u8; 16]),
12
- Seqnum(u64, Option<[u8; 16]>),
13
- XorHash(u64),
14
- ResolvedEntry(u64),
15
-}
16
-
17
-impl Default for Location {
18
- fn default() -> Self {
19
- Self::Head
20
- }
21
-}
22
-
23
-#[derive(Debug)]
24
-pub struct JournalCursor {
25
- pub location: Location,
26
- pub filter_expr: Option<FilterExpr>,
27
- pub array_cursor: Option<offset_array::Cursor>,
28
-}
29
-
30
-impl JournalCursor {
31
- #[allow(clippy::new_without_default)]
32
- pub fn new() -> Self {
33
- Self {
34
- location: Location::Head,
35
- filter_expr: None,
36
- array_cursor: None,
37
- }
38
- }
39
-
40
- pub fn set_location(&mut self, location: Location) {
41
- self.location = location;
42
- self.array_cursor = None;
43
- }
44
-
45
- pub fn set_filter(&mut self, filter_expr: FilterExpr) {
46
- self.filter_expr = Some(filter_expr);
47
- // FIXME: should we set cursor to None?
48
- }
49
-
50
- pub fn clear_filter(&mut self) {
51
- self.filter_expr = None;
52
- self.array_cursor = None;
53
- self.set_location(Location::Head);
54
- }
55
-
56
- pub fn step<M: MemoryMap>(
57
- &mut self,
58
- journal_file: &JournalFile<M>,
59
- direction: Direction,
60
- ) -> Result<bool> {
61
- let new_location = if self.filter_expr.is_some() {
62
- self.resolve_filter_location(journal_file, direction)?
63
- } else {
64
- self.resolve_array_cursor(journal_file, direction)?
65
- };
66
-
67
- if let Some(location) = new_location {
68
- self.location = location;
69
- Ok(true)
70
- } else {
71
- Ok(false)
72
- }
73
- }
74
-
75
- pub fn position(&self) -> Result<u64> {
76
- match self.location {
77
- Location::ResolvedEntry(entry_offset) => Ok(entry_offset),
78
- _ => Err(JournalError::UnsetCursor),
79
- }
80
- }
81
-
82
- fn resolve_array_cursor<M: MemoryMap>(
83
- &mut self,
84
- journal_file: &JournalFile<M>,
85
- direction: Direction,
86
- ) -> Result<Option<Location>> {
87
- let new_location = match (self.location, direction) {
88
- (Location::Head, Direction::Forward) => {
89
- let entry_list = journal_file
90
- .entry_list()
91
- .ok_or(JournalError::InvalidOffsetArrayOffset)?;
92
-
93
- let cursor = entry_list.cursor_head();
94
- let offset = cursor.value(journal_file)?;
95
- self.array_cursor = Some(cursor);
96
- Some(Location::ResolvedEntry(offset))
97
- }
98
- (Location::Head, Direction::Backward) => None,
99
- (Location::Tail, Direction::Forward) => None,
100
- (Location::Tail, Direction::Backward) => {
101
- let entry_list = journal_file
102
- .entry_list()
103
- .ok_or(JournalError::InvalidOffsetArrayOffset)?;
104
-
105
- let cursor = entry_list.cursor_tail(journal_file)?;
106
- let offset = cursor.value(journal_file)?;
107
- self.array_cursor = Some(cursor);
108
- Some(Location::ResolvedEntry(offset))
109
- }
110
- (Location::Realtime(realtime), _) => {
111
- let entry_list = journal_file
112
- .entry_list()
113
- .ok_or(JournalError::InvalidOffsetArrayOffset)?;
114
-
115
- let predicate = |entry_offset| {
116
- let entry_object = journal_file.entry_ref(entry_offset)?;
117
- Ok(entry_object.header.realtime < realtime)
118
- };
119
-
120
- let cursor = entry_list
121
- .directed_partition_point(journal_file, predicate, Direction::Forward)?
122
- .map(Ok)
123
- .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
124
-
125
- let offset = cursor.value(journal_file)?;
126
- self.array_cursor = Some(cursor);
127
- Some(Location::ResolvedEntry(offset))
128
- }
129
- (Location::ResolvedEntry(_), Direction::Forward) => {
130
- let Some(cursor) = self.array_cursor.unwrap().next(journal_file)? else {
131
- return Ok(None);
132
- };
133
-
134
- let offset = cursor.value(journal_file)?;
135
- self.array_cursor = Some(cursor);
136
- Some(Location::ResolvedEntry(offset))
137
- }
138
- (Location::ResolvedEntry(_), Direction::Backward) => {
139
- let Some(cursor) = self.array_cursor.unwrap().previous(journal_file)? else {
140
- return Ok(None);
141
- };
142
-
143
- let offset = cursor.value(journal_file)?;
144
- self.array_cursor = Some(cursor);
145
- Some(Location::ResolvedEntry(offset))
146
- }
147
- _ => {
148
- unimplemented!()
149
- }
150
- };
151
-
152
- Ok(new_location)
153
- }
154
-
155
- fn resolve_filter_location<M: MemoryMap>(
156
- &mut self,
157
- journal_file: &JournalFile<M>,
158
- direction: Direction,
159
- ) -> Result<Option<Location>> {
160
- let filter_expr = self.filter_expr.as_mut().unwrap();
161
-
162
- let resolved_location = match (self.location, direction) {
163
- (Location::Head, Direction::Forward) => filter_expr
164
- .head()
165
- .next(journal_file, u64::MIN)?
166
- .map(Location::ResolvedEntry),
167
- (Location::Head, Direction::Backward) => None,
168
- (Location::Tail, Direction::Forward) => None,
169
- (Location::Tail, Direction::Backward) => filter_expr
170
- .tail(journal_file)?
171
- .previous(journal_file, u64::MAX)?
172
- .map(Location::ResolvedEntry),
173
- (Location::Realtime(realtime), direction) => {
174
- let entry_list = journal_file
175
- .entry_list()
176
- .ok_or(JournalError::InvalidOffsetArrayOffset)?;
177
-
178
- let predicate = |entry_offset| {
179
- let entry_object = journal_file.entry_ref(entry_offset)?;
180
- Ok(entry_object.header.realtime < realtime)
181
- };
182
-
183
- let cursor = entry_list
184
- .directed_partition_point(journal_file, predicate, Direction::Forward)?
185
- .map(Ok)
186
- .unwrap_or_else(|| entry_list.cursor_tail(journal_file))?;
187
-
188
- let entry_offset = cursor.value(journal_file)?;
189
-
190
- match direction {
191
- Direction::Forward => filter_expr
192
- .head()
193
- .next(journal_file, entry_offset)?
194
- .map(Location::ResolvedEntry),
195
- Direction::Backward => filter_expr
196
- .tail(journal_file)?
197
- .previous(journal_file, entry_offset)?
198
- .map(Location::ResolvedEntry),
199
- }
200
- }
201
- (Location::ResolvedEntry(location_offset), Direction::Forward) => filter_expr
202
- .next(journal_file, location_offset + 1)?
203
- .map(Location::ResolvedEntry),
204
- (Location::ResolvedEntry(location_offset), Direction::Backward) => filter_expr
205
- .previous(journal_file, location_offset - 1)?
206
- .map(Location::ResolvedEntry),
207
- _ => {
208
- unimplemented!();
209
- }
210
- };
211
-
212
- Ok(resolved_location)
213
- }
214
-}
src/crates/jf/journal_reader/src/lib.rs
deleted
-197
@@ -1,197 +0,0 @@
1
-use error::Result;
2
-use journal_file::{
3
- offset_array, DataObject, EntryDataIterator, FieldDataIterator, FieldIterator, FieldObject,
4
- JournalFile, ValueGuard,
5
-};
6
-use window_manager::MemoryMap;
7
-
8
-pub mod journal_filter;
9
-use journal_filter::{JournalFilter, LogicalOp};
10
-pub mod journal_cursor;
11
-use journal_cursor::JournalCursor;
12
-pub use journal_cursor::Location;
13
-
14
-pub use offset_array::Direction;
15
-
16
-pub struct JournalReader<'a, M: MemoryMap> {
17
- cursor: JournalCursor,
18
-
19
- filter: Option<JournalFilter>,
20
- field_iterator: Option<FieldIterator<'a, M>>,
21
- field_data_iterator: Option<FieldDataIterator<'a, M>>,
22
- entry_data_iterator: Option<EntryDataIterator<'a, M>>,
23
-
24
- field_guard: Option<ValueGuard<'a, FieldObject<&'a [u8]>>>,
25
- data_guard: Option<ValueGuard<'a, DataObject<&'a [u8]>>>,
26
-}
27
-
28
-impl<M: MemoryMap> std::fmt::Debug for JournalReader<'_, M> {
29
- fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
30
- f.debug_struct("JournalReader")
31
- // .field("cursor", &self.cursor)
32
- .field("field_guard", &self.field_guard)
33
- .field("data_guard", &self.data_guard)
34
- .finish()
35
- }
36
-}
37
-
38
-impl<M: MemoryMap> Default for JournalReader<'_, M> {
39
- fn default() -> Self {
40
- Self {
41
- cursor: JournalCursor::new(),
42
- filter: None,
43
- field_iterator: None,
44
- field_data_iterator: None,
45
- entry_data_iterator: None,
46
- field_guard: None,
47
- data_guard: None,
48
- }
49
- }
50
-}
51
-
52
-impl<'a, M: MemoryMap> JournalReader<'a, M> {
53
- pub fn dump(&self, journal_file: &'a JournalFile<M>) -> Result<String> {
54
- if let Some(filter_expr) = self.cursor.filter_expr.as_ref() {
55
- filter_expr.dump(journal_file)
56
- } else {
57
- Ok(String::from("no filter expr"))
58
- }
59
- }
60
-
61
- pub fn set_location(&mut self, location: Location) {
62
- self.cursor.set_location(location)
63
- }
64
-
65
- pub fn step(&mut self, journal_file: &'a JournalFile<M>, direction: Direction) -> Result<bool> {
66
- self.drop_guards();
67
-
68
- if let Some(filter) = self.filter.as_mut() {
69
- let filter_expr = filter.build(journal_file)?;
70
- self.cursor.set_filter(filter_expr);
71
- self.filter = None;
72
- }
73
-
74
- self.cursor.step(journal_file, direction)
75
- }
76
-
77
- pub fn add_match(&mut self, data: &[u8]) {
78
- self.filter.get_or_insert_default().add_match(data);
79
- }
80
-
81
- pub fn add_conjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
82
- self.filter
83
- .get_or_insert_default()
84
- .set_operation(journal_file, LogicalOp::Conjunction)
85
- }
86
-
87
- pub fn add_disjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
88
- self.filter
89
- .get_or_insert_default()
90
- .set_operation(journal_file, LogicalOp::Disjunction)
91
- }
92
-
93
- pub fn flush_matches(&mut self) {
94
- self.cursor.clear_filter();
95
- self.filter = None;
96
- }
97
-
98
- pub fn get_realtime_usec(&self, journal_file: &'a JournalFile<M>) -> Result<u64> {
99
- let entry_offset = self.cursor.position()?;
100
- let entry_object = journal_file.entry_ref(entry_offset)?;
101
- Ok(entry_object.header.realtime)
102
- }
103
-
104
- pub fn get_seqnum(&self, journal_file: &'a JournalFile<M>) -> Result<(u64, [u8; 16])> {
105
- let entry_offset = self.cursor.position()?;
106
- let entry_object = journal_file.entry_ref(entry_offset)?;
107
- Ok((
108
- entry_object.header.seqnum,
109
- journal_file.journal_header_ref().seqnum_id,
110
- ))
111
- }
112
-
113
- pub fn get_entry_offset(&self) -> Result<u64> {
114
- self.cursor.position()
115
- }
116
-
117
- fn drop_guards(&mut self) {
118
- self.field_guard.take();
119
- self.data_guard.take();
120
- }
121
-
122
- pub fn fields_restart(&mut self) {
123
- self.drop_guards();
124
- self.field_iterator = None;
125
- }
126
-
127
- pub fn fields_enumerate(
128
- &mut self,
129
- journal_file: &'a JournalFile<M>,
130
- ) -> Result<Option<&ValueGuard<FieldObject<&'a [u8]>>>> {
131
- self.drop_guards();
132
-
133
- if self.field_iterator.is_none() {
134
- self.field_iterator = Some(journal_file.fields());
135
- }
136
-
137
- if let Some(iter) = &mut self.field_iterator {
138
- self.field_guard = iter.next().transpose()?;
139
- Ok(self.field_guard.as_ref())
140
- } else {
141
- Ok(None)
142
- }
143
- }
144
-
145
- pub fn field_data_query_unique(
146
- &mut self,
147
- journal_file: &'a JournalFile<M>,
148
- field_name: &'a [u8],
149
- ) -> Result<()> {
150
- self.drop_guards();
151
-
152
- self.field_data_iterator = Some(journal_file.field_data_objects(field_name)?);
153
- Ok(())
154
- }
155
-
156
- pub fn field_data_restart(&mut self) {
157
- self.drop_guards();
158
- }
159
-
160
- pub fn field_data_enumerate(
161
- &mut self,
162
- _: &'a JournalFile<M>,
163
- ) -> Result<Option<&ValueGuard<DataObject<&'a [u8]>>>> {
164
- self.drop_guards();
165
-
166
- if let Some(iter) = &mut self.field_data_iterator {
167
- self.data_guard = iter.next().transpose()?;
168
- Ok(self.data_guard.as_ref())
169
- } else {
170
- Ok(None)
171
- }
172
- }
173
-
174
- pub fn entry_data_restart(&mut self) {
175
- self.drop_guards();
176
- self.entry_data_iterator = None;
177
- }
178
-
179
- pub fn entry_data_enumerate(
180
- &mut self,
181
- journal_file: &'a JournalFile<M>,
182
- ) -> Result<Option<&ValueGuard<DataObject<&'a [u8]>>>> {
183
- self.drop_guards();
184
-
185
- if self.entry_data_iterator.is_none() {
186
- let entry_offset = self.cursor.position()?;
187
- self.entry_data_iterator = Some(journal_file.entry_data_objects(entry_offset)?);
188
- }
189
-
190
- if let Some(iter) = &mut self.entry_data_iterator {
191
- self.data_guard = iter.next().transpose()?;
192
- Ok(self.data_guard.as_ref())
193
- } else {
194
- Ok(None)
195
- }
196
- }
197
-}
src/crates/jf/journal_reader_ffi/Cargo.toml
deleted
-19
@@ -1,19 +0,0 @@
1
-[package]
2
-name = "journal_reader_ffi"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-# journal_reader integrated into journal_file
10
-journal_file = { path = "../journal_file" }
11
-memmap2 = { workspace = true }
12
-serde_json = { workspace = true }
13
-sigbus = { path = "../sigbus" }
14
-
15
-[build-dependencies]
16
-cbindgen = "0.28.0"
17
-
18
-[lib]
19
-crate-type = ["staticlib"]
src/crates/jf/journal_reader_ffi/build.rs
deleted
-21
@@ -1,21 +0,0 @@
1
-extern crate cbindgen;
2
-
3
-use std::env;
4
-
5
-fn main() {
6
- let crate_dir = env::var("CARGO_MANIFEST_DIR").unwrap();
7
-
8
- cbindgen::Builder::new()
9
- .with_crate(crate_dir)
10
- .with_language(cbindgen::Language::C)
11
- .with_cpp_compat(true)
12
- .with_include_guard("JOURNAL_READER_FFI_H")
13
- .rename_item("SdJournal", "sd_journal")
14
- .rename_item("SdId128", "sd_id128_t")
15
- .generate()
16
- .expect("Unable to generate bindings")
17
- .write_to_file("journal_reader_ffi.h");
18
-
19
- println!("cargo:rerun-if-changed=cbindgen.toml");
20
- println!("cargo:rerun-if-changed=src/");
21
-}
src/crates/jf/journal_reader_ffi/src/lib.rs
deleted
-448
@@ -1,448 +0,0 @@
1
-use journal_file::{Direction, HashableObject, JournalFile, JournalReader, Location};
2
-use memmap2::Mmap;
3
-use std::ffi::{c_char, c_int, c_void, CStr};
4
-
5
-#[repr(C)]
6
-#[derive(Debug, Clone, Copy)]
7
-pub struct RsdId128 {
8
- pub bytes: [u8; 16],
9
-}
10
-
11
-fn unhexchar(c: u8) -> Result<u8, i32> {
12
- match c {
13
- b'0'..=b'9' => Ok(c - b'0'),
14
- b'a'..=b'f' => Ok(c - b'a' + 10),
15
- b'A'..=b'F' => Ok(c - b'A' + 10),
16
- _ => Err(-22), // -EINVAL
17
- }
18
-}
19
-
20
-#[no_mangle]
21
-unsafe extern "C" fn rsd_id128_from_string(s: *const c_char, ret: *mut RsdId128) -> i32 {
22
- debug_assert!(!s.is_null());
23
- debug_assert!(!ret.is_null());
24
-
25
- let c_str = match CStr::from_ptr(s).to_str() {
26
- Ok(s) => s,
27
- Err(_) => return -1,
28
- };
29
-
30
- let res = &mut *ret;
31
- let mut n: usize = 0;
32
- let mut i: usize = 0;
33
- let mut is_guid = false;
34
-
35
- let bytes = c_str.as_bytes();
36
-
37
- while n < 16 {
38
- if i >= bytes.len() {
39
- return -1;
40
- }
41
-
42
- if bytes[i] == b'-' {
43
- if i == 8 {
44
- is_guid = true;
45
- } else if i == 13 || i == 18 || i == 23 {
46
- if !is_guid {
47
- return -1;
48
- }
49
- } else {
50
- return -1;
51
- }
52
-
53
- i += 1;
54
- continue;
55
- }
56
-
57
- if i + 1 >= bytes.len() {
58
- return -1;
59
- }
60
-
61
- let a = match unhexchar(bytes[i]) {
62
- Ok(val) => val,
63
- Err(e) => return e,
64
- };
65
- i += 1;
66
-
67
- let b = match unhexchar(bytes[i]) {
68
- Ok(val) => val,
69
- Err(e) => return e,
70
- };
71
- i += 1;
72
-
73
- res.bytes[n] = (a << 4) | b;
74
- n += 1;
75
- }
76
-
77
- let expected_len = if is_guid { 36 } else { 32 };
78
- if i != expected_len || i >= bytes.len() || bytes[i] != 0 {
79
- return -1;
80
- }
81
-
82
- 0
83
-}
84
-
85
-#[no_mangle]
86
-pub extern "C" fn rsd_id128_equal(a: RsdId128, b: RsdId128) -> i32 {
87
- (a.bytes == b.bytes) as i32
88
-}
89
-
90
-impl PartialEq for RsdId128 {
91
- fn eq(&self, other: &Self) -> bool {
92
- self.bytes == other.bytes
93
- }
94
-}
95
-
96
-impl Eq for RsdId128 {}
97
-
98
-struct RsdJournal<'a> {
99
- journal_file: Box<JournalFile<Mmap>>,
100
- reader: JournalReader<'a, Mmap>,
101
- field_buffer: Vec<u8>,
102
- decompressed_payload: Vec<u8>,
103
-}
104
-
105
-#[no_mangle]
106
-unsafe extern "C" fn rsd_journal_open_files(
107
- ret: *mut *mut RsdJournal,
108
- paths: *const *const c_char,
109
- _flags: c_int,
110
-) -> c_int {
111
- debug_assert!(!ret.is_null());
112
- debug_assert!(!paths.is_null());
113
-
114
- if sigbus::install_handler().is_err() {
115
- eprintln!("Failed to install sigbus handler");
116
- }
117
-
118
- // Get the first path
119
- let path_ptr = *paths;
120
- if path_ptr.is_null() {
121
- return error::JournalError::InvalidFfiOp.to_error_code();
122
- }
123
-
124
- // Convert C string to Rust string
125
- let path = match CStr::from_ptr(path_ptr).to_str() {
126
- Ok(s) => s,
127
- Err(_) => {
128
- return error::JournalError::InvalidFfiOp.to_error_code();
129
- }
130
- };
131
-
132
- // Create the ObjectFile
133
- let window_size = 512 * 1024 * 1024;
134
- let journal_file = match JournalFile::<Mmap>::open(path, window_size) {
135
- Ok(f) => Box::new(f),
136
- Err(e) => {
137
- return e.to_error_code();
138
- }
139
- };
140
-
141
- let journal = Box::new(RsdJournal {
142
- reader: JournalReader::default(),
143
- journal_file,
144
- field_buffer: Vec::with_capacity(256),
145
- decompressed_payload: Vec::new(),
146
- });
147
-
148
- // Pass ownership to the caller
149
- *ret = Box::into_raw(journal);
150
-
151
- 0
152
-}
153
-
154
-#[no_mangle]
155
-unsafe extern "C" fn rsd_journal_close(j: *mut RsdJournal) {
156
- debug_assert!(!j.is_null());
157
- let _ = Box::from_raw(j);
158
-}
159
-
160
-#[no_mangle]
161
-unsafe extern "C" fn rsd_journal_seek_head(j: *mut RsdJournal) -> c_int {
162
- debug_assert!(!j.is_null());
163
- let journal = &mut *j;
164
- journal.reader.set_location(Location::Head);
165
- 0
166
-}
167
-
168
-#[no_mangle]
169
-unsafe extern "C" fn rsd_journal_seek_tail(j: *mut RsdJournal) -> c_int {
170
- debug_assert!(!j.is_null());
171
- let journal = &mut *j;
172
- journal.reader.set_location(Location::Tail);
173
- 0
174
-}
175
-
176
-#[no_mangle]
177
-unsafe extern "C" fn rsd_journal_seek_realtime_usec(j: *mut RsdJournal, usec: u64) -> c_int {
178
- debug_assert!(!j.is_null());
179
- let journal = &mut *j;
180
- journal.reader.set_location(Location::Realtime(usec));
181
- 0
182
-}
183
-
184
-#[no_mangle]
185
-unsafe extern "C" fn rsd_journal_next(j: *mut RsdJournal) -> c_int {
186
- debug_assert!(!j.is_null());
187
- let journal = &mut *j;
188
-
189
- match journal
190
- .reader
191
- .step(&journal.journal_file, Direction::Forward)
192
- {
193
- Ok(has_entry) => {
194
- if has_entry {
195
- 1
196
- } else {
197
- 0
198
- }
199
- }
200
- Err(e) => e.to_error_code(),
201
- }
202
-}
203
-
204
-#[no_mangle]
205
-unsafe extern "C" fn rsd_journal_previous(j: *mut RsdJournal) -> c_int {
206
- debug_assert!(!j.is_null());
207
- let journal = &mut *j;
208
-
209
- match journal
210
- .reader
211
- .step(&journal.journal_file, Direction::Backward)
212
- {
213
- Ok(has_entry) => {
214
- if has_entry {
215
- 1
216
- } else {
217
- 0
218
- }
219
- }
220
- Err(e) => e.to_error_code(),
221
- }
222
-}
223
-
224
-#[no_mangle]
225
-unsafe extern "C" fn rsd_journal_get_seqnum(
226
- j: *mut RsdJournal,
227
- ret_seqnum: *mut u64,
228
- ret_seqnum_id: *mut RsdId128,
229
-) -> c_int {
230
- debug_assert!(!j.is_null());
231
- debug_assert!(!ret_seqnum.is_null());
232
- debug_assert!(!ret_seqnum_id.is_null());
233
-
234
- let journal = &mut *j;
235
- match journal.reader.get_seqnum(&journal.journal_file) {
236
- Ok((seqnum, boot_id)) => {
237
- *ret_seqnum = seqnum;
238
-
239
- if !ret_seqnum_id.is_null() {
240
- *ret_seqnum_id = RsdId128 { bytes: boot_id };
241
- }
242
-
243
- 0
244
- }
245
- Err(e) => e.to_error_code(),
246
- }
247
-}
248
-
249
-#[no_mangle]
250
-unsafe extern "C" fn rsd_journal_get_realtime_usec(j: *mut RsdJournal, ret: *mut u64) -> c_int {
251
- debug_assert!(!j.is_null());
252
- debug_assert!(!ret.is_null());
253
-
254
- let journal = &mut *j;
255
-
256
- match journal.reader.get_realtime_usec(&journal.journal_file) {
257
- Ok(realtime) => {
258
- *ret = realtime;
259
- 0
260
- }
261
- Err(e) => e.to_error_code(),
262
- }
263
-}
264
-
265
-#[no_mangle]
266
-unsafe extern "C" fn rsd_journal_restart_data(j: *mut RsdJournal) {
267
- debug_assert!(!j.is_null());
268
-
269
- let journal = &mut *j;
270
- journal.reader.entry_data_restart();
271
-}
272
-
273
-#[no_mangle]
274
-unsafe extern "C" fn rsd_journal_enumerate_available_data(
275
- j: *mut RsdJournal,
276
- data: *mut *const c_void,
277
- l: *mut usize,
278
-) -> c_int {
279
- debug_assert!(!j.is_null());
280
- debug_assert!(!data.is_null());
281
- debug_assert!(!l.is_null());
282
-
283
- let journal = &mut *j;
284
-
285
- match journal.reader.entry_data_enumerate(&journal.journal_file) {
286
- Ok(Some(data_guard)) => {
287
- if data_guard.is_compressed() {
288
- return match data_guard.decompress(&mut journal.decompressed_payload) {
289
- Ok(n) => {
290
- *l = n;
291
- *data = journal.decompressed_payload.as_ptr() as *const c_void;
292
- 1
293
- }
294
- Err(error::JournalError::UnknownCompressionMethod) => {
295
- eprintln!("unknown compression method");
296
- -1
297
- }
298
- Err(_) => -1,
299
- };
300
- } else {
301
- let payload = data_guard.payload_bytes();
302
- *l = payload.len();
303
- *data = payload.as_ptr() as *const c_void;
304
- }
305
- 1
306
- }
307
- Ok(None) => 0,
308
- Err(e) => e.to_error_code(),
309
- }
310
-}
311
-
312
-#[no_mangle]
313
-unsafe extern "C" fn rsd_journal_restart_fields(j: *mut RsdJournal) {
314
- debug_assert!(!j.is_null());
315
-
316
- let journal = &mut *j;
317
- journal.reader.fields_restart();
318
-}
319
-
320
-#[no_mangle]
321
-unsafe extern "C" fn rsd_journal_enumerate_fields(
322
- j: *mut RsdJournal,
323
- field: *mut *const c_char,
324
-) -> c_int {
325
- debug_assert!(!j.is_null());
326
- debug_assert!(!field.is_null());
327
-
328
- let journal = &mut *j;
329
-
330
- match journal.reader.fields_enumerate(&journal.journal_file) {
331
- Ok(Some(field_guard)) => {
332
- let field_name = field_guard.get_payload();
333
-
334
- journal.field_buffer.clear();
335
- journal.field_buffer.extend_from_slice(field_name);
336
- journal.field_buffer.push(0);
337
- *field = journal.field_buffer.as_ptr() as *const c_char;
338
-
339
- 1
340
- }
341
- Ok(None) => 0,
342
- Err(e) => e.to_error_code(),
343
- }
344
-}
345
-
346
-#[no_mangle]
347
-unsafe extern "C" fn rsd_journal_query_unique(j: *mut RsdJournal, field: *const c_char) -> c_int {
348
- debug_assert!(!j.is_null());
349
- debug_assert!(!field.is_null());
350
-
351
- let journal = &mut *j;
352
- let field_cstr = CStr::from_ptr(field);
353
- let field_name = field_cstr.to_bytes();
354
-
355
- match journal
356
- .reader
357
- .field_data_query_unique(&journal.journal_file, field_name)
358
- {
359
- Ok(_) => 0,
360
- Err(e) => e.to_error_code(),
361
- }
362
-}
363
-
364
-#[no_mangle]
365
-unsafe extern "C" fn rsd_journal_restart_unique(j: *mut RsdJournal) {
366
- debug_assert!(!j.is_null());
367
- let journal = &mut *j;
368
- journal.reader.field_data_restart();
369
-}
370
-
371
-#[no_mangle]
372
-unsafe extern "C" fn rsd_journal_enumerate_available_unique(
373
- j: *mut RsdJournal,
374
- data: *mut *const c_void,
375
- l: *mut usize,
376
-) -> c_int {
377
- debug_assert!(!j.is_null());
378
- debug_assert!(!data.is_null());
379
- debug_assert!(!l.is_null());
380
-
381
- let journal = &mut *j;
382
-
383
- match journal.reader.field_data_enumerate(&journal.journal_file) {
384
- Ok(Some(data_guard)) => {
385
- let payload = data_guard.payload_bytes();
386
- *data = payload.as_ptr() as *const c_void;
387
- *l = payload.len();
388
-
389
- 1
390
- }
391
- Ok(None) => 0,
392
- Err(e) => e.to_error_code(),
393
- }
394
-}
395
-
396
-#[no_mangle]
397
-unsafe extern "C" fn rsd_journal_add_match(
398
- j: *mut RsdJournal,
399
- data: *const c_void,
400
- size: usize,
401
-) -> c_int {
402
- debug_assert!(!j.is_null());
403
- debug_assert!(!data.is_null());
404
-
405
- let journal = &mut *j;
406
-
407
- let data_slice = if size == 0 {
408
- let mut len = 0;
409
- let data_ptr = data as *const u8;
410
- while *data_ptr.add(len) != 0 {
411
- len += 1;
412
- }
413
- std::slice::from_raw_parts(data as *const u8, len)
414
- } else {
415
- std::slice::from_raw_parts(data as *const u8, size)
416
- };
417
-
418
- journal.reader.add_match(data_slice);
419
- 0
420
-}
421
-
422
-#[no_mangle]
423
-unsafe extern "C" fn rsd_journal_add_conjunction(j: *mut RsdJournal) -> c_int {
424
- debug_assert!(!j.is_null());
425
- let journal = &mut *j;
426
- match journal.reader.add_conjunction(&journal.journal_file) {
427
- Ok(_) => 0,
428
- Err(e) => e.to_error_code(),
429
- }
430
-}
431
-
432
-#[no_mangle]
433
-unsafe extern "C" fn rsd_journal_add_disjunction(j: *mut RsdJournal) -> c_int {
434
- debug_assert!(!j.is_null());
435
-
436
- let journal = &mut *j;
437
- match journal.reader.add_disjunction(&journal.journal_file) {
438
- Ok(_) => 0,
439
- Err(e) => e.to_error_code(),
440
- }
441
-}
442
-
443
-#[no_mangle]
444
-unsafe extern "C" fn rsd_journal_flush_matches(j: *mut RsdJournal) {
445
- debug_assert!(!j.is_null());
446
- let journal = &mut *j;
447
- journal.reader.flush_matches();
448
-}
src/crates/jf/otel-plugin/Cargo.toml
deleted
-31
@@ -1,31 +0,0 @@
1
-[package]
2
-name = "otel-plugin"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-tokio = { workspace = true }
9
-prost = { workspace = true }
10
-tonic = { workspace = true, features = ["gzip", "tls-ring"] }
11
-opentelemetry-proto = { workspace = true, features = ["metrics", "logs"] }
12
-
13
-flatten_otel = { path = "../flatten_otel" }
14
-journal_file = { path = "../journal_file" }
15
-journal_log = { path = "../journal_log" }
16
-memmap2 = { workspace = true }
17
-serde_json = { workspace = true, features = ["preserve_order"] }
18
-uuid = { version = "1.0", features = ["v4", "rng"] }
19
-flatten-serde-json = { workspace = true }
20
-base64 = "0.21"
21
-regex = { workspace = true }
22
-serde_yaml = "0.9.34"
23
-serde_regex = "1.1.0"
24
-serde = { version = "1.0.219", features = ["derive"] }
25
-atty = { workspace = true }
26
-clap = { workspace = true }
27
-anyhow = { workspace = true }
28
-humantime-serde = { workspace = true }
29
-humantime = { workspace = true }
30
-bytesize = { workspace = true }
31
-bytesize-serde = { workspace = true }
src/crates/jf/otel-plugin/src/logs_service.rs
deleted
-99
@@ -1,99 +0,0 @@
1
-use anyhow::{Context, Result};
2
-use flatten_otel::json_from_export_logs_service_request;
3
-use journal_log::{JournalLog, JournalLogConfig, RetentionPolicy, RotationPolicy};
4
-use opentelemetry_proto::tonic::collector::logs::v1::{
5
- logs_service_server::LogsService, ExportLogsServiceRequest, ExportLogsServiceResponse,
6
-};
7
-use serde_json::Value;
8
-use std::sync::{Arc, Mutex};
9
-use tonic::{Request, Response, Status};
10
-
11
-use crate::plugin_config::PluginConfig;
12
-
13
-pub struct NetdataLogsService {
14
- journal_log: Arc<Mutex<JournalLog>>,
15
-}
16
-
17
-impl NetdataLogsService {
18
- pub fn new(plugin_config: PluginConfig) -> Result<Self> {
19
- let logs_config = plugin_config.logs;
20
-
21
- let rotation_policy = RotationPolicy::default()
22
- .with_size_of_journal_file(logs_config.size_of_journal_file.as_u64())
23
- .with_duration_of_journal_file(logs_config.duration_of_journal_file);
24
-
25
- let retention_policy = RetentionPolicy::default()
26
- .with_number_of_journal_files(logs_config.number_of_journal_files)
27
- .with_size_of_journal_files(logs_config.size_of_journal_files.as_u64())
28
- .with_duration_of_journal_files(logs_config.duration_of_journal_files);
29
-
30
- let journal_config = JournalLogConfig::new(&logs_config.journal_dir)
31
- .with_rotation_policy(rotation_policy)
32
- .with_retention_policy(retention_policy);
33
-
34
- let journal_log = Arc::new(Mutex::new(JournalLog::new(journal_config).with_context(
35
- || {
36
- format!(
37
- "Failed to create journal log for directory: {}",
38
- logs_config.journal_dir
39
- )
40
- },
41
- )?));
42
- Ok(NetdataLogsService { journal_log })
43
- }
44
-
45
- fn json_to_entry_data(&self, json_value: &Value) -> Vec<Vec<u8>> {
46
- let mut entry_data = Vec::new();
47
-
48
- if let Value::Object(obj) = json_value {
49
- for (key, value) in obj {
50
- let value_str = match value {
51
- Value::String(s) => s.clone(),
52
- Value::Number(n) => n.to_string(),
53
- Value::Bool(b) => b.to_string(),
54
- Value::Null => "null".to_string(),
55
- _ => serde_json::to_string(value).unwrap_or_default(),
56
- };
57
-
58
- let kv_pair = format!("{}={}", key, value_str);
59
- entry_data.push(kv_pair.into_bytes());
60
- }
61
- }
62
-
63
- entry_data
64
- }
65
-}
66
-
67
-#[tonic::async_trait]
68
-impl LogsService for NetdataLogsService {
69
- async fn export(
70
- &self,
71
- request: Request<ExportLogsServiceRequest>,
72
- ) -> Result<Response<ExportLogsServiceResponse>, Status> {
73
- let req = request.into_inner();
74
-
75
- let json_array = json_from_export_logs_service_request(&req);
76
-
77
- if let Value::Array(entries) = json_array {
78
- for entry in entries {
79
- let entry_data = self.json_to_entry_data(&entry);
80
- if !entry_data.is_empty() {
81
- let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
82
- if let Err(e) = self.journal_log.lock().unwrap().write_entry(&entry_refs) {
83
- eprintln!("Failed to write log entry: {}", e);
84
- return Err(Status::internal(format!(
85
- "Failed to write log entry: {}",
86
- e
87
- )));
88
- }
89
- }
90
- }
91
- }
92
-
93
- let reply = ExportLogsServiceResponse {
94
- partial_success: None,
95
- };
96
-
97
- Ok(Response::new(reply))
98
- }
99
-}
src/crates/jf/otel-plugin/src/main.rs
deleted
-104
@@ -1,104 +0,0 @@
1
-use anyhow::{Context, Result};
2
-use opentelemetry_proto::tonic::collector::{
3
- logs::v1::logs_service_server::LogsServiceServer,
4
- metrics::v1::metrics_service_server::MetricsServiceServer,
5
-};
6
-use tonic::transport::{Identity, Server, ServerTlsConfig};
7
-
8
-mod chart_config;
9
-mod flattened_point;
10
-mod netdata_chart;
11
-mod netdata_env;
12
-mod regex_cache;
13
-mod samples_table;
14
-
15
-mod plugin_config;
16
-use crate::plugin_config::PluginConfig;
17
-
18
-mod logs_service;
19
-use crate::logs_service::NetdataLogsService;
20
-
21
-mod metrics_service;
22
-use crate::metrics_service::NetdataMetricsService;
23
-
24
-async fn send_keepalive_periodically() {
25
- let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(60));
26
-
27
- loop {
28
- interval.tick().await;
29
- println!("PLUGIN_KEEPALIVE");
30
- }
31
-}
32
-
33
-#[tokio::main]
34
-async fn main() -> Result<()> {
35
- let config = PluginConfig::new().context("Failed to initialize plugin configuration")?;
36
-
37
- let addr =
38
- config.endpoint.path.parse().with_context(|| {
39
- format!("Failed to parse endpoint address: {}", config.endpoint.path)
40
- })?;
41
- let metrics_service =
42
- NetdataMetricsService::new(config.clone()).context("Failed to create metrics service")?;
43
- let logs_service =
44
- NetdataLogsService::new(config.clone()).context("Failed to create logs service")?;
45
-
46
- println!("TRUST_DURATIONS 1");
47
-
48
- let mut server_builder = Server::builder();
49
-
50
- // Configure TLS if provided
51
- if let (Some(cert_path), Some(key_path)) = (
52
- &config.endpoint.tls_cert_path,
53
- &config.endpoint.tls_key_path,
54
- ) {
55
- let cert = std::fs::read(cert_path)
56
- .with_context(|| format!("Failed to read TLS certificate from: {}", cert_path))?;
57
- let key = std::fs::read(key_path)
58
- .with_context(|| format!("Failed to read TLS private key from: {}", key_path))?;
59
- let identity = Identity::from_pem(cert, key);
60
-
61
- let mut tls_config_builder = ServerTlsConfig::new().identity(identity);
62
-
63
- // If CA certificate is provided, enable client authentication
64
- if let Some(ref ca_cert_path) = config.endpoint.tls_ca_cert_path {
65
- let ca_cert = std::fs::read(ca_cert_path)
66
- .with_context(|| format!("Failed to read CA certificate from: {}", ca_cert_path))?;
67
- tls_config_builder =
68
- tls_config_builder.client_ca_root(tonic::transport::Certificate::from_pem(ca_cert));
69
- }
70
-
71
- server_builder = server_builder
72
- .tls_config(tls_config_builder)
73
- .context("Failed to configure TLS")?;
74
- } else {
75
- eprintln!(
76
- "TLS disabled, using insecure connection on endpoint: {}",
77
- config.endpoint.path
78
- );
79
- }
80
-
81
- let server = server_builder
82
- .add_service(
83
- MetricsServiceServer::new(metrics_service)
84
- .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
85
- )
86
- .add_service(
87
- LogsServiceServer::new(logs_service)
88
- .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
89
- )
90
- .serve(addr);
91
-
92
- let keepalive = send_keepalive_periodically();
93
-
94
- tokio::select! {
95
- result = server => {
96
- result.with_context(|| format!("Failed to serve gRPC server on {}", addr))?;
97
- }
98
- _ = keepalive => {
99
- // This should never complete
100
- }
101
- }
102
-
103
- Ok(())
104
-}
src/crates/jf/sigbus/Cargo.toml
deleted
-9
@@ -1,9 +0,0 @@
1
-[package]
2
-name = "sigbus"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-libc = { workspace = true }
src/crates/jf/window_manager/Cargo.toml
deleted
-9
@@ -1,9 +0,0 @@
1
-[package]
2
-name = "window_manager"
3
-version.workspace = true
4
-edition.workspace = true
5
-rust-version.workspace = true
6
-
7
-[dependencies]
8
-error = { path = "../error" }
9
-memmap2 = { workspace = true }
src/crates/journal-common/Cargo.toml
new
+18
@@ -0,0 +1,18 @@
1
+[package]
2
+name = "journal-common"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[features]
11
+allocative = ["dep:allocative"]
12
+
13
+[dependencies]
14
+allocative = { workspace = true, optional = true }
15
+rustc-hash = { workspace = true }
16
+uuid = { workspace = true }
17
+serde = { workspace = true, features = ["derive"] }
18
+nix = { workspace = true, features = ["time"] }
src/crates/journal-common/src/collections.rs
new
+9
@@ -0,0 +1,9 @@
1
+//! Collection type aliases.
2
+//!
3
+//! This module provides convenient aliases for the hash-based collections
4
+//! used throughout the journal crates. We use `rustc_hash::FxHashMap` and
5
+//! `FxHashSet` for their performance characteristics.
6
+
7
+pub type HashMap<K, V> = rustc_hash::FxHashMap<K, V>;
8
+pub type HashSet<T> = rustc_hash::FxHashSet<T>;
9
+pub type VecDeque<T> = std::collections::VecDeque<T>;
src/crates/journal-common/src/compat.rs
new
+64
@@ -0,0 +1,64 @@
1
+//! MSRV compatibility helpers.
2
+//!
3
+//! This module provides backports of newer Rust standard library features
4
+//! to maintain compatibility with our current MSRV (Minimum Supported Rust Version).
5
+
6
+/// Returns `true` if `value` is an integer multiple of `divisor`, and `false` otherwise.
7
+///
8
+/// This is a compatibility function for MSRV < 1.87.0 that provides the same functionality
9
+/// as the built-in `u32::is_multiple_of()` method.
10
+///
11
+/// # Migration path
12
+///
13
+/// When MSRV >= 1.87.0, replace calls to this function with the standard library's
14
+/// `is_multiple_of()` method:
15
+/// ```ignore
16
+/// // Old (MSRV < 1.87.0):
17
+/// use journal_common::compat::is_multiple_of;
18
+/// is_multiple_of(value, divisor)
19
+///
20
+/// // New (MSRV >= 1.87.0):
21
+/// value.is_multiple_of(divisor)
22
+/// ```
23
+///
24
+/// # Examples
25
+///
26
+/// ```
27
+/// use journal_common::compat::is_multiple_of;
28
+///
29
+/// assert!(is_multiple_of(100u32, 10));
30
+/// assert!(is_multiple_of(100u32, 20));
31
+/// assert!(!is_multiple_of(100u32, 7));
32
+/// ```
33
+#[inline]
34
+pub fn is_multiple_of<T>(value: T, divisor: T) -> bool
35
+where
36
+ T: std::ops::Rem<Output = T> + PartialEq + From<u8>,
37
+{
38
+ value % divisor == T::from(0)
39
+}
40
+
41
+#[cfg(test)]
42
+mod tests {
43
+ use super::*;
44
+
45
+ #[test]
46
+ fn test_is_multiple_of_u32() {
47
+ assert!(is_multiple_of(100u32, 10));
48
+ assert!(is_multiple_of(100u32, 20));
49
+ assert!(is_multiple_of(100u32, 100));
50
+ assert!(is_multiple_of(0u32, 10));
51
+
52
+ assert!(!is_multiple_of(100u32, 7));
53
+ assert!(!is_multiple_of(100u32, 30));
54
+ }
55
+
56
+ #[test]
57
+ fn test_is_multiple_of_u64() {
58
+ assert!(is_multiple_of(1000u64, 10));
59
+ assert!(is_multiple_of(1000u64, 100));
60
+ assert!(is_multiple_of(0u64, 100));
61
+
62
+ assert!(!is_multiple_of(1000u64, 7));
63
+ }
64
+}
src/crates/journal-common/src/lib.rs
new
+17
@@ -0,0 +1,17 @@
1
+//! Common types and utilities shared across journal crates.
2
+//!
3
+//! This crate provides foundational types and utilities used by multiple
4
+//! journal-related crates, avoiding code duplication and circular dependencies.
5
+
6
+pub mod collections;
7
+pub mod compat;
8
+pub mod system;
9
+pub mod time;
10
+
11
+pub use time::{Microseconds, RealtimeClock, Seconds, monotonic_now};
12
+
13
+// Re-export collection types for convenience
14
+pub use collections::{HashMap, HashSet, VecDeque};
15
+
16
+// Re-export system utilities for convenience
17
+pub use system::{load_boot_id, load_machine_id};
src/crates/journal-common/src/system.rs
new
+136
@@ -0,0 +1,136 @@
1
+//! System utilities for loading machine and boot identifiers.
2
+//!
3
+//! This module provides platform-specific functions to load system identifiers
4
+//! that are used for journal file creation and identification.
5
+
6
+use std::io;
7
+
8
+/// Reads a file from the host filesystem, trying both the normal path and /host/ prefix.
9
+///
10
+/// This is useful when running in containers where the host filesystem may be mounted at /host.
11
+fn read_host_file(filename: &str) -> io::Result<String> {
12
+ match std::fs::read_to_string(filename) {
13
+ Ok(contents) => Ok(contents),
14
+ Err(e) if e.kind() == io::ErrorKind::NotFound => {
15
+ let filename = format!("/host/{}", filename);
16
+ std::fs::read_to_string(filename)
17
+ }
18
+ Err(e) => Err(e),
19
+ }
20
+}
21
+
22
+/// Loads the machine ID from the system.
23
+///
24
+/// On Linux, this reads from `/etc/machine-id`.
25
+/// On macOS, this uses `system_profiler` to get the hardware UUID.
26
+/// On other platforms, this returns an error.
27
+#[cfg(target_os = "linux")]
28
+pub fn load_machine_id() -> io::Result<uuid::Uuid> {
29
+ let content = read_host_file("/etc/machine-id")?;
30
+ uuid::Uuid::try_parse(content.trim()).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
31
+}
32
+
33
+#[cfg(target_os = "macos")]
34
+pub fn load_machine_id() -> io::Result<uuid::Uuid> {
35
+ use std::process::Command;
36
+
37
+ let output = Command::new("system_profiler")
38
+ .arg("SPHardwareDataType")
39
+ .output()?;
40
+
41
+ if output.status.success() {
42
+ let output_str = String::from_utf8_lossy(&output.stdout);
43
+ for line in output_str.lines() {
44
+ if line.contains("Hardware UUID:") {
45
+ if let Some(uuid_str) = line.split("Hardware UUID:").nth(1) {
46
+ let uuid_str = uuid_str.trim();
47
+ let hex_str: String = uuid_str.chars().filter(|c| *c != '-').collect();
48
+
49
+ if hex_str.len() == 32 {
50
+ let mut bytes = [0u8; 16];
51
+ for i in 0..16 {
52
+ let hex_pair = &hex_str[i * 2..i * 2 + 2];
53
+ bytes[i] = u8::from_str_radix(hex_pair, 16)
54
+ .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?;
55
+ }
56
+ return Ok(uuid::Uuid::from_bytes(bytes));
57
+ }
58
+ }
59
+ }
60
+ }
61
+ }
62
+
63
+ Err(io::Error::new(
64
+ io::ErrorKind::NotFound,
65
+ "Could not find Hardware UUID",
66
+ ))
67
+}
68
+
69
+#[cfg(not(any(target_os = "linux", target_os = "macos")))]
70
+pub fn load_machine_id() -> io::Result<uuid::Uuid> {
71
+ Err(io::Error::new(
72
+ io::ErrorKind::Unsupported,
73
+ "Machine ID loading not supported on this platform",
74
+ ))
75
+}
76
+
77
+/// Loads the boot ID from the system.
78
+///
79
+/// On Linux, this reads from `/proc/sys/kernel/random/boot_id`.
80
+/// On macOS, this derives a deterministic ID from the boot time.
81
+/// On other platforms, this returns an error.
82
+#[cfg(target_os = "linux")]
83
+pub fn load_boot_id() -> io::Result<uuid::Uuid> {
84
+ let content = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?;
85
+ uuid::Uuid::try_parse(content.trim()).map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))
86
+}
87
+
88
+#[cfg(target_os = "macos")]
89
+pub fn load_boot_id() -> io::Result<uuid::Uuid> {
90
+ use std::process::Command;
91
+
92
+ let output = Command::new("sysctl")
93
+ .arg("-n")
94
+ .arg("kern.boottime")
95
+ .output()?;
96
+
97
+ if output.status.success() {
98
+ let output_str = String::from_utf8_lossy(&output.stdout);
99
+ // Parse "{ sec = 1753988677, usec = 131097 } Thu Jul 31 22:04:37 2025"
100
+ // Extract sec and usec values
101
+ if let (Some(sec_start), Some(usec_start)) =
102
+ (output_str.find("sec = "), output_str.find("usec = "))
103
+ {
104
+ let sec_str = &output_str[sec_start + 6..];
105
+ let sec_end = sec_str.find(',').unwrap_or(sec_str.len());
106
+ let sec_str = &sec_str[..sec_end].trim();
107
+
108
+ let usec_str = &output_str[usec_start + 7..];
109
+ let usec_end = usec_str.find(' ').unwrap_or(usec_str.len());
110
+ let usec_str = &usec_str[..usec_end].trim();
111
+
112
+ if let (Ok(sec), Ok(usec)) = (sec_str.parse::<u64>(), usec_str.parse::<u64>()) {
113
+ // Create a deterministic UUID from boot time
114
+ // Use sec in first 8 bytes, usec in next 4 bytes, pad remaining with zeros
115
+ let mut bytes = [0u8; 16];
116
+ bytes[0..8].copy_from_slice(&sec.to_be_bytes());
117
+ bytes[8..12].copy_from_slice(&(usec as u32).to_be_bytes());
118
+ // bytes[12..16] remain zero-filled for consistency
119
+ return Ok(uuid::Uuid::from_bytes(bytes));
120
+ }
121
+ }
122
+ }
123
+
124
+ Err(io::Error::new(
125
+ io::ErrorKind::NotFound,
126
+ "Could not parse boot time",
127
+ ))
128
+}
129
+
130
+#[cfg(not(any(target_os = "linux", target_os = "macos")))]
131
+pub fn load_boot_id() -> io::Result<uuid::Uuid> {
132
+ Err(io::Error::new(
133
+ io::ErrorKind::Unsupported,
134
+ "Boot ID loading not supported on this platform",
135
+ ))
136
+}
src/crates/journal-common/src/time.rs
new
+546
@@ -0,0 +1,546 @@
1
+//! Time units for journal timestamps.
2
+//!
3
+//! Provides type-safe wrappers for seconds and microseconds to prevent unit confusion.
4
+
5
+use serde::{Deserialize, Serialize};
6
+use std::cell::Cell;
7
+use std::ops::{Add, Rem, Sub};
8
+
9
+/// Timestamp in seconds since Unix epoch.
10
+///
11
+/// Used for histogram buckets, time ranges, and coarse-grained time operations.
12
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
13
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
14
+pub struct Seconds(pub u32);
15
+
16
+/// Timestamp in microseconds since Unix epoch.
17
+///
18
+/// Used for journal entry timestamps and fine-grained time operations.
19
+#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
20
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
21
+pub struct Microseconds(pub u64);
22
+
23
+impl Seconds {
24
+ /// Create a timestamp from seconds.
25
+ pub fn new(seconds: u32) -> Self {
26
+ Self(seconds)
27
+ }
28
+
29
+ /// Get the current time as seconds since Unix epoch.
30
+ pub fn now() -> Self {
31
+ Self(
32
+ std::time::SystemTime::now()
33
+ .duration_since(std::time::UNIX_EPOCH)
34
+ .expect("system time must be after UNIX_EPOCH")
35
+ .as_secs() as u32,
36
+ )
37
+ }
38
+
39
+ /// Get the raw seconds value.
40
+ pub fn get(self) -> u32 {
41
+ self.0
42
+ }
43
+
44
+ /// Convert to microseconds.
45
+ pub fn to_microseconds(self) -> Microseconds {
46
+ Microseconds(self.0 as u64 * 1_000_000)
47
+ }
48
+
49
+ /// Add two durations with saturation at the numeric bounds.
50
+ pub fn saturating_add(self, other: Self) -> Self {
51
+ Seconds(self.0.saturating_add(other.0))
52
+ }
53
+
54
+ /// Subtract two durations with saturation at the numeric bounds.
55
+ pub fn saturating_sub(self, other: Self) -> Self {
56
+ Seconds(self.0.saturating_sub(other.0))
57
+ }
58
+
59
+ /// Checked addition. Returns None if overflow occurred.
60
+ pub fn checked_add(self, other: Self) -> Option<Self> {
61
+ self.0.checked_add(other.0).map(Seconds)
62
+ }
63
+
64
+ /// Checked subtraction. Returns None if overflow occurred.
65
+ pub fn checked_sub(self, other: Self) -> Option<Self> {
66
+ self.0.checked_sub(other.0).map(Seconds)
67
+ }
68
+
69
+ /// Returns true if this duration is a multiple of the other duration.
70
+ ///
71
+ /// Useful for checking if bucket durations align.
72
+ pub fn is_multiple_of(self, other: Self) -> bool {
73
+ other.0 != 0 && self.0 % other.0 == 0
74
+ }
75
+}
76
+
77
+impl Microseconds {
78
+ /// Create a timestamp from microseconds.
79
+ pub fn new(microseconds: u64) -> Self {
80
+ Self(microseconds)
81
+ }
82
+
83
+ /// Get the current time as microseconds since Unix epoch.
84
+ pub fn now() -> Self {
85
+ Self(
86
+ std::time::SystemTime::now()
87
+ .duration_since(std::time::UNIX_EPOCH)
88
+ .expect("system time must be after UNIX_EPOCH")
89
+ .as_micros() as u64,
90
+ )
91
+ }
92
+
93
+ /// Get the raw microseconds value.
94
+ pub fn get(self) -> u64 {
95
+ self.0
96
+ }
97
+
98
+ /// Convert to seconds (truncates).
99
+ pub fn to_seconds(self) -> Seconds {
100
+ Seconds((self.0 / 1_000_000) as u32)
101
+ }
102
+
103
+ /// Add two durations with saturation at the numeric bounds.
104
+ pub fn saturating_add(self, other: Self) -> Self {
105
+ Microseconds(self.0.saturating_add(other.0))
106
+ }
107
+
108
+ /// Subtract two durations with saturation at the numeric bounds.
109
+ pub fn saturating_sub(self, other: Self) -> Self {
110
+ Microseconds(self.0.saturating_sub(other.0))
111
+ }
112
+
113
+ /// Checked addition. Returns None if overflow occurred.
114
+ pub fn checked_add(self, other: Self) -> Option<Self> {
115
+ self.0.checked_add(other.0).map(Microseconds)
116
+ }
117
+
118
+ /// Checked subtraction. Returns None if overflow occurred.
119
+ pub fn checked_sub(self, other: Self) -> Option<Self> {
120
+ self.0.checked_sub(other.0).map(Microseconds)
121
+ }
122
+
123
+ /// Returns true if this duration is a multiple of the other duration.
124
+ ///
125
+ /// Useful for checking if bucket durations align.
126
+ pub fn is_multiple_of(self, other: Self) -> bool {
127
+ other.0 != 0 && self.0 % other.0 == 0
128
+ }
129
+}
130
+
131
+impl From<Seconds> for Microseconds {
132
+ fn from(s: Seconds) -> Self {
133
+ s.to_microseconds()
134
+ }
135
+}
136
+
137
+impl From<u32> for Seconds {
138
+ fn from(s: u32) -> Self {
139
+ Seconds(s)
140
+ }
141
+}
142
+
143
+impl From<u64> for Microseconds {
144
+ fn from(us: u64) -> Self {
145
+ Microseconds(us)
146
+ }
147
+}
148
+
149
+// Arithmetic operators for Seconds
150
+impl Add for Seconds {
151
+ type Output = Self;
152
+
153
+ fn add(self, other: Self) -> Self {
154
+ Seconds(self.0 + other.0)
155
+ }
156
+}
157
+
158
+impl Sub for Seconds {
159
+ type Output = Self;
160
+
161
+ fn sub(self, other: Self) -> Self {
162
+ Seconds(self.0 - other.0)
163
+ }
164
+}
165
+
166
+impl Rem for Seconds {
167
+ type Output = Self;
168
+
169
+ fn rem(self, other: Self) -> Self {
170
+ Seconds(self.0 % other.0)
171
+ }
172
+}
173
+
174
+// Arithmetic operators for Microseconds
175
+impl Add for Microseconds {
176
+ type Output = Self;
177
+
178
+ fn add(self, other: Self) -> Self {
179
+ Microseconds(self.0 + other.0)
180
+ }
181
+}
182
+
183
+impl Sub for Microseconds {
184
+ type Output = Self;
185
+
186
+ fn sub(self, other: Self) -> Self {
187
+ Microseconds(self.0 - other.0)
188
+ }
189
+}
190
+
191
+impl Rem for Microseconds {
192
+ type Output = Self;
193
+
194
+ fn rem(self, other: Self) -> Self {
195
+ Microseconds(self.0 % other.0)
196
+ }
197
+}
198
+
199
+impl std::fmt::Display for Seconds {
200
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
201
+ write!(f, "{}s", self.0)
202
+ }
203
+}
204
+
205
+impl std::fmt::Display for Microseconds {
206
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
207
+ write!(f, "{}µs", self.0)
208
+ }
209
+}
210
+
211
+/// A monotonic realtime clock that ensures timestamps always move forward.
212
+///
213
+/// Wraps `SystemTime` but guarantees each `now()` call returns a timestamp
214
+/// strictly greater than all previous calls, even if the system clock jumps
215
+/// backwards. When the clock goes backwards, it increments from the last seen
216
+/// timestamp by one microsecond.
217
+#[derive(Debug)]
218
+pub struct RealtimeClock {
219
+ max_seen: Cell<u64>,
220
+}
221
+
222
+impl RealtimeClock {
223
+ /// Create a new realtime clock initialized with the current system time.
224
+ pub fn new() -> Self {
225
+ Self::with_initial(Microseconds::now())
226
+ }
227
+
228
+ /// Create a realtime clock initialized with a specific timestamp.
229
+ ///
230
+ /// Useful for resuming from a persisted state (e.g., last journal entry).
231
+ pub fn with_initial(initial: Microseconds) -> Self {
232
+ Self {
233
+ max_seen: Cell::new(initial.get()),
234
+ }
235
+ }
236
+
237
+ /// Get the current monotonic timestamp in microseconds since Unix epoch.
238
+ ///
239
+ /// Returns system time if it moved forward, otherwise returns last seen + 1µs.
240
+ pub fn now(&self) -> Microseconds {
241
+ let current = Microseconds::now();
242
+ let max = self.max_seen.get();
243
+
244
+ let next = if current.get() > max {
245
+ current.get()
246
+ } else {
247
+ max.saturating_add(1)
248
+ };
249
+
250
+ self.max_seen.set(next);
251
+ Microseconds::new(next)
252
+ }
253
+
254
+ /// Get the last seen timestamp without advancing the clock.
255
+ pub fn last_seen(&self) -> Microseconds {
256
+ Microseconds::new(self.max_seen.get())
257
+ }
258
+}
259
+
260
+impl Default for RealtimeClock {
261
+ fn default() -> Self {
262
+ Self::new()
263
+ }
264
+}
265
+
266
+/// Gets the current monotonic timestamp in microseconds since boot.
267
+///
268
+/// Uses CLOCK_MONOTONIC which provides a monotonically increasing timestamp
269
+/// that is not affected by system clock adjustments but does not count time
270
+/// when the system is suspended.
271
+///
272
+/// This matches systemd's behavior for journal entry monotonic timestamps.
273
+pub fn monotonic_now() -> std::io::Result<Microseconds> {
274
+ use nix::sys::time::TimeValLike;
275
+ use nix::time::ClockId;
276
+
277
+ let ts = ClockId::CLOCK_MONOTONIC
278
+ .now()
279
+ .map_err(|e| std::io::Error::from_raw_os_error(e as i32))?;
280
+
281
+ Ok(Microseconds::new(ts.num_microseconds() as u64))
282
+}
283
+
284
+#[cfg(test)]
285
+mod tests {
286
+ use super::*;
287
+
288
+ #[test]
289
+ fn test_seconds_to_microseconds() {
290
+ let seconds = Seconds::new(42);
291
+ let micros = seconds.to_microseconds();
292
+ assert_eq!(micros.get(), 42_000_000);
293
+ }
294
+
295
+ #[test]
296
+ fn test_microseconds_to_seconds() {
297
+ let micros = Microseconds::new(42_500_000);
298
+ let seconds = micros.to_seconds();
299
+ assert_eq!(seconds.get(), 42);
300
+ }
301
+
302
+ #[test]
303
+ fn test_conversion_roundtrip() {
304
+ let original = Seconds::new(100);
305
+ let roundtrip = original.to_microseconds().to_seconds();
306
+ assert_eq!(original, roundtrip);
307
+ }
308
+
309
+ #[test]
310
+ fn test_from_conversions() {
311
+ let s: Seconds = 42u32.into();
312
+ assert_eq!(s.get(), 42);
313
+
314
+ let us: Microseconds = 42000u64.into();
315
+ assert_eq!(us.get(), 42000);
316
+ }
317
+
318
+ // Arithmetic operator tests for Seconds
319
+ #[test]
320
+ fn test_seconds_add() {
321
+ let a = Seconds::new(10);
322
+ let b = Seconds::new(20);
323
+ assert_eq!(a + b, Seconds::new(30));
324
+ }
325
+
326
+ #[test]
327
+ fn test_seconds_sub() {
328
+ let a = Seconds::new(30);
329
+ let b = Seconds::new(10);
330
+ assert_eq!(a - b, Seconds::new(20));
331
+ }
332
+
333
+ #[test]
334
+ #[should_panic]
335
+ fn test_seconds_sub_underflow() {
336
+ let a = Seconds::new(10);
337
+ let b = Seconds::new(20);
338
+ let _ = a - b; // Should panic
339
+ }
340
+
341
+ #[test]
342
+ fn test_seconds_rem() {
343
+ let a = Seconds::new(10);
344
+ let b = Seconds::new(3);
345
+ assert_eq!(a % b, Seconds::new(1));
346
+ }
347
+
348
+ #[test]
349
+ fn test_seconds_saturating_add() {
350
+ let a = Seconds::new(u32::MAX - 5);
351
+ let b = Seconds::new(10);
352
+ assert_eq!(a.saturating_add(b), Seconds::new(u32::MAX));
353
+ }
354
+
355
+ #[test]
356
+ fn test_seconds_saturating_sub() {
357
+ let a = Seconds::new(10);
358
+ let b = Seconds::new(20);
359
+ assert_eq!(a.saturating_sub(b), Seconds::new(0));
360
+ }
361
+
362
+ #[test]
363
+ fn test_seconds_checked_add() {
364
+ let a = Seconds::new(10);
365
+ let b = Seconds::new(20);
366
+ assert_eq!(a.checked_add(b), Some(Seconds::new(30)));
367
+
368
+ let c = Seconds::new(u32::MAX);
369
+ let d = Seconds::new(1);
370
+ assert_eq!(c.checked_add(d), None);
371
+ }
372
+
373
+ #[test]
374
+ fn test_seconds_checked_sub() {
375
+ let a = Seconds::new(30);
376
+ let b = Seconds::new(10);
377
+ assert_eq!(a.checked_sub(b), Some(Seconds::new(20)));
378
+
379
+ let c = Seconds::new(10);
380
+ let d = Seconds::new(20);
381
+ assert_eq!(c.checked_sub(d), None);
382
+ }
383
+
384
+ #[test]
385
+ fn test_seconds_is_multiple_of() {
386
+ let a = Seconds::new(60);
387
+ let b = Seconds::new(15);
388
+ assert!(a.is_multiple_of(b));
389
+
390
+ let c = Seconds::new(60);
391
+ let d = Seconds::new(17);
392
+ assert!(!c.is_multiple_of(d));
393
+
394
+ let e = Seconds::new(0);
395
+ let f = Seconds::new(10);
396
+ assert!(e.is_multiple_of(f));
397
+
398
+ let g = Seconds::new(10);
399
+ let h = Seconds::new(0);
400
+ assert!(!g.is_multiple_of(h)); // Division by zero case
401
+ }
402
+
403
+ // Arithmetic operator tests for Microseconds
404
+ #[test]
405
+ fn test_microseconds_add() {
406
+ let a = Microseconds::new(1000);
407
+ let b = Microseconds::new(2000);
408
+ assert_eq!(a + b, Microseconds::new(3000));
409
+ }
410
+
411
+ #[test]
412
+ fn test_microseconds_sub() {
413
+ let a = Microseconds::new(3000);
414
+ let b = Microseconds::new(1000);
415
+ assert_eq!(a - b, Microseconds::new(2000));
416
+ }
417
+
418
+ #[test]
419
+ #[should_panic]
420
+ fn test_microseconds_sub_underflow() {
421
+ let a = Microseconds::new(1000);
422
+ let b = Microseconds::new(2000);
423
+ let _ = a - b; // Should panic
424
+ }
425
+
426
+ #[test]
427
+ fn test_microseconds_rem() {
428
+ let a = Microseconds::new(1000);
429
+ let b = Microseconds::new(300);
430
+ assert_eq!(a % b, Microseconds::new(100));
431
+ }
432
+
433
+ #[test]
434
+ fn test_microseconds_saturating_add() {
435
+ let a = Microseconds::new(u64::MAX - 5);
436
+ let b = Microseconds::new(10);
437
+ assert_eq!(a.saturating_add(b), Microseconds::new(u64::MAX));
438
+ }
439
+
440
+ #[test]
441
+ fn test_microseconds_saturating_sub() {
442
+ let a = Microseconds::new(1000);
443
+ let b = Microseconds::new(2000);
444
+ assert_eq!(a.saturating_sub(b), Microseconds::new(0));
445
+ }
446
+
447
+ #[test]
448
+ fn test_microseconds_checked_add() {
449
+ let a = Microseconds::new(1000);
450
+ let b = Microseconds::new(2000);
451
+ assert_eq!(a.checked_add(b), Some(Microseconds::new(3000)));
452
+
453
+ let c = Microseconds::new(u64::MAX);
454
+ let d = Microseconds::new(1);
455
+ assert_eq!(c.checked_add(d), None);
456
+ }
457
+
458
+ #[test]
459
+ fn test_microseconds_checked_sub() {
460
+ let a = Microseconds::new(3000);
461
+ let b = Microseconds::new(1000);
462
+ assert_eq!(a.checked_sub(b), Some(Microseconds::new(2000)));
463
+
464
+ let c = Microseconds::new(1000);
465
+ let d = Microseconds::new(2000);
466
+ assert_eq!(c.checked_sub(d), None);
467
+ }
468
+
469
+ #[test]
470
+ fn test_microseconds_is_multiple_of() {
471
+ let a = Microseconds::new(60000);
472
+ let b = Microseconds::new(15000);
473
+ assert!(a.is_multiple_of(b));
474
+
475
+ let c = Microseconds::new(60000);
476
+ let d = Microseconds::new(17000);
477
+ assert!(!c.is_multiple_of(d));
478
+
479
+ let e = Microseconds::new(0);
480
+ let f = Microseconds::new(10000);
481
+ assert!(e.is_multiple_of(f));
482
+
483
+ let g = Microseconds::new(10000);
484
+ let h = Microseconds::new(0);
485
+ assert!(!g.is_multiple_of(h)); // Division by zero case
486
+ }
487
+
488
+ // RealtimeClock tests
489
+ #[test]
490
+ fn test_realtime_clock_monotonic() {
491
+ let clock = RealtimeClock::new();
492
+ let t1 = clock.now();
493
+ let t2 = clock.now();
494
+ let t3 = clock.now();
495
+
496
+ assert!(t2 > t1);
497
+ assert!(t3 > t2);
498
+ }
499
+
500
+ #[test]
501
+ fn test_realtime_clock_with_initial() {
502
+ let initial = Microseconds::new(1000000);
503
+ let clock = RealtimeClock::with_initial(initial);
504
+
505
+ assert_eq!(clock.last_seen(), initial);
506
+
507
+ let t1 = clock.now();
508
+ assert!(t1 >= initial);
509
+ }
510
+
511
+ #[test]
512
+ fn test_realtime_clock_handles_same_time() {
513
+ // Start with a specific timestamp
514
+ let initial = Microseconds::new(1000000);
515
+ let clock = RealtimeClock::with_initial(initial);
516
+
517
+ // Even if system time doesn't advance, clock should increment
518
+ let t1 = clock.now();
519
+ let t2 = clock.now();
520
+
521
+ assert!(t2 > t1);
522
+ assert_eq!(t2.get() - t1.get(), 1); // Should increment by 1 microsecond
523
+ }
524
+
525
+ #[test]
526
+ fn test_realtime_clock_last_seen() {
527
+ let clock = RealtimeClock::new();
528
+
529
+ let t1 = clock.now();
530
+ assert_eq!(clock.last_seen(), t1);
531
+
532
+ let t2 = clock.now();
533
+ assert_eq!(clock.last_seen(), t2);
534
+ }
535
+
536
+ #[test]
537
+ fn test_realtime_clock_forward_jump() {
538
+ // Start with a timestamp in the past
539
+ let past = Microseconds::new(1000000);
540
+ let clock = RealtimeClock::with_initial(past);
541
+
542
+ // When system time is ahead, it should use system time
543
+ let t1 = clock.now();
544
+ assert!(t1.get() > past.get());
545
+ }
546
+}
src/crates/journal-core/Cargo.toml
new
+61
@@ -0,0 +1,61 @@
1
+[package]
2
+name = "journal-core"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[lints]
7
+workspace = true
8
+
9
+[features]
10
+allocative = ["dep:allocative"]
11
+
12
+[dependencies]
13
+journal-common = { workspace = true }
14
+journal-registry = { workspace = true }
15
+rdp = { workspace = true }
16
+
17
+# Core dependencies
18
+static_assertions = { workspace = true }
19
+thiserror = { workspace = true }
20
+zerocopy = { workspace = true , features = ["derive"] }
21
+libc = { workspace = true }
22
+memmap2 = { workspace = true }
23
+uuid = { workspace = true , features = ["v4", "rng", "serde"] }
24
+nix = { workspace = true }
25
+
26
+# File format dependencies
27
+ruzstd = { workspace = true }
28
+zstd = { workspace = true }
29
+siphasher = { workspace = true }
30
+md5 = { workspace = true }
31
+hashers = { workspace = true }
32
+
33
+twox-hash = { workspace = true, features = ["std"] }
34
+rand = { workspace = true }
35
+
36
+lz4 = { workspace = true }
37
+
38
+serde = { workspace = true, features = ["derive"] }
39
+
40
+rustc-hash = { workspace = true }
41
+
42
+# Registry dependencies
43
+parking_lot = { workspace = true }
44
+walkdir = { workspace = true }
45
+crossbeam-channel = { workspace = true }
46
+regex = { workspace = true }
47
+chrono = { workspace = true, features = ["serde"] }
48
+serde_json = { workspace = true }
49
+anyhow = { workspace = true }
50
+notify = { workspace = true }
51
+
52
+tracing = { workspace = true }
53
+allocative = { workspace = true, optional = true }
54
+
55
+tokio = { workspace = true, features = ["rt", "sync", "macros", "rt-multi-thread"] }
56
+
57
+[dev-dependencies]
58
+tempfile = { workspace = true }
59
+rayon = { workspace = true }
60
+clap = { workspace = true, features = ["derive"] }
61
+tracing-subscriber = { workspace = true }
src/crates/journal-core/src/collections.rs
new
+6
@@ -0,0 +1,6 @@
1
+//! Collection type aliases re-exported from journal-common.
2
+//!
3
+//! This module re-exports the collection types from `journal-common` for
4
+//! backwards compatibility and convenience.
5
+
6
+pub use journal_common::collections::{HashMap, HashSet, VecDeque};
src/crates/journal-core/src/error.rs
renamed
+4
-46
@@ -1,6 +1,3 @@
1
-#[macro_use]
2
-extern crate static_assertions;
3
-
1
use std::io;
2
use thiserror::Error;
3
@@ -75,63 +72,24 @@ pub enum JournalError {
72
#[error("unknown compression method")]
73
UnknownCompressionMethod,
74
78
- #[error("ffi error")]
79
- InvalidFfiOp,
80
-
75
#[error("uuid encoding/decoding")]
76
UuidSerde,
77
78
#[error("invalid filename")]
79
InvalidFilename,
80
87
- #[error("system time error")]
88
- SystemTimeError,
89
-
81
#[error("directory not found")]
82
DirectoryNotFound,
83
84
#[error("not a directory")]
85
NotADirectory,
95
-}
86
97
-const_assert!(std::mem::size_of::<JournalError>() <= 16);
98
-
99
-impl JournalError {
100
- pub fn to_error_code(&self) -> i32 {
101
- match self {
102
- JournalError::InvalidMagicNumber => -1,
103
- JournalError::InvalidJournalFileState => -2,
104
- JournalError::InvalidObjectType => -3,
105
- JournalError::InvalidObjectLocation => -4,
106
- JournalError::InvalidZeroCopySize => -5,
107
- JournalError::ValueGuardInUse => -6,
108
- JournalError::Io(_) => -7,
109
- JournalError::MissingHashTable => -8,
110
- JournalError::MissingObjectFromHashTable => -9,
111
- JournalError::InvalidOffsetArrayOffset => -10,
112
- JournalError::InvalidOffsetArrayIndex => -11,
113
- JournalError::EmptyOffsetArrayList => -12,
114
- JournalError::EmptyOffsetArrayNode => -13,
115
- JournalError::EmptyInlineCursor => -14,
116
- JournalError::UnsetCursor => -15,
117
- JournalError::MalformedFilter => -16,
118
- JournalError::InvalidField => -17,
119
- JournalError::DecompressorError => -18,
120
- JournalError::OutOfBoundsIndex => -19,
121
- JournalError::InvalidOffset => -20,
122
- JournalError::ZerocopyFailure => -21,
123
- JournalError::SigbusHandlerError => -22,
124
- JournalError::UnknownCompressionMethod => -23,
125
- JournalError::InvalidFfiOp => -24,
126
- JournalError::UuidSerde => -25,
127
- JournalError::InvalidFilename => -26,
128
- JournalError::SystemTimeError => -27,
129
- JournalError::DirectoryNotFound => -28,
130
- JournalError::NotADirectory => -29,
131
- }
132
- }
87
+ #[error("invalid query configuration")]
88
+ InvalidQueryConfiguration,
89
}
90
91
+static_assertions::const_assert!(std::mem::size_of::<JournalError>() <= 16);
92
+
93
impl<T: zerocopy::KnownLayout> From<zerocopy::SizeError<&[u8], T>> for JournalError {
94
fn from(_: zerocopy::SizeError<&[u8], T>) -> Self {
95
JournalError::InvalidZeroCopySize
src/crates/journal-core/src/field_map.rs
new
+204
@@ -0,0 +1,204 @@
1
+use crate::collections::{HashMap, HashSet};
2
+
3
+pub const REMAPPING_MARKER: &[u8] = b"ND_REMAPPING=1";
4
+
5
+/// Validates if a field name is compatible with systemd journal requirements.
6
+pub fn is_systemd_compatible(field_name: &[u8]) -> bool {
7
+ if field_name.is_empty() || field_name.len() > 64 {
8
+ return false;
9
+ }
10
+
11
+ // First byte must be uppercase A-Z
12
+ if !field_name[0].is_ascii_uppercase() {
13
+ return false;
14
+ }
15
+
16
+ // All bytes must be uppercase A-Z, digit 0-9, or underscore
17
+ field_name
18
+ .iter()
19
+ .all(|&b| b.is_ascii_uppercase() || b.is_ascii_digit() || b == b'_')
20
+}
21
+
22
+/// Extracts the field name from a `KEY=VALUE` pair.
23
+pub fn extract_field_name(item: &[u8]) -> Option<&[u8]> {
24
+ item.iter().position(|&b| b == b'=').map(|pos| &item[..pos])
25
+}
26
+
27
+/// Bidirectional mapping registry between original field names and their systemd-compatible versions.
28
+#[derive(Debug, Default)]
29
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
30
+pub struct FieldMap {
31
+ /// Maps original field name → remapped name (ND_<md5>)
32
+ otel_to_systemd: HashMap<Vec<u8>, String>,
33
+ /// Maps remapped name (ND_<md5>) → original field name
34
+ systemd_to_otel: HashMap<String, Vec<u8>>,
35
+}
36
+
37
+impl FieldMap {
38
+ /// Creates a new empty remapping registry.
39
+ pub fn new() -> Self {
40
+ Self {
41
+ otel_to_systemd: HashMap::default(),
42
+ systemd_to_otel: HashMap::default(),
43
+ }
44
+ }
45
+
46
+ /// Adds a new mapping to the registry.
47
+ ///
48
+ /// Returns `true` if this is a new mapping, `false` if it already existed.
49
+ pub fn add_otel_mapping(&mut self, otel_name: Vec<u8>, systemd_name: String) -> bool {
50
+ if self.otel_to_systemd.contains_key(&otel_name) {
51
+ return false;
52
+ }
53
+
54
+ self.systemd_to_otel
55
+ .insert(systemd_name.clone(), otel_name.clone());
56
+ self.otel_to_systemd.insert(otel_name, systemd_name);
57
+ true
58
+ }
59
+
60
+ /// Gets the systemd-compatible name for an original field name.
61
+ ///
62
+ /// Returns `None` if no mapping exists for this field name.
63
+ pub fn get_systemd_name(&self, otel_name: &[u8]) -> Option<&str> {
64
+ self.otel_to_systemd.get(otel_name).map(|s| s.as_str())
65
+ }
66
+
67
+ /// Gets the original field name for a systemd-compatible name.
68
+ ///
69
+ /// Returns `None` if no mapping exists for this systemd name.
70
+ pub fn get_otel_name(&self, systemd_name: &str) -> Option<&[u8]> {
71
+ self.systemd_to_otel.get(systemd_name).map(|v| v.as_slice())
72
+ }
73
+
74
+ /// Returns `true` if the registry contains a mapping for this original field name.
75
+ pub fn contains_otel_name(&self, otel_name: &[u8]) -> bool {
76
+ self.otel_to_systemd.contains_key(otel_name)
77
+ }
78
+
79
+ /// Returns `true` if the registry is empty.
80
+ pub fn is_empty(&self) -> bool {
81
+ self.otel_to_systemd.is_empty()
82
+ }
83
+
84
+ /// Returns the number of mappings in the registry.
85
+ pub fn len(&self) -> usize {
86
+ self.otel_to_systemd.len()
87
+ }
88
+
89
+ pub fn clear(&mut self) {
90
+ self.otel_to_systemd.clear();
91
+ self.systemd_to_otel.clear();
92
+ }
93
+
94
+ pub fn fields(self) -> HashSet<String> {
95
+ self.otel_to_systemd
96
+ .into_keys()
97
+ .map(|x| unsafe { String::from_utf8_unchecked(x) })
98
+ .collect()
99
+ }
100
+}
101
+
102
+#[cfg(test)]
103
+mod tests {
104
+ use super::*;
105
+
106
+ #[test]
107
+ fn test_is_systemd_compatible() {
108
+ // Valid field names
109
+ assert!(is_systemd_compatible(b"MESSAGE"));
110
+ assert!(is_systemd_compatible(b"PRIORITY"));
111
+ assert!(is_systemd_compatible(b"USER_ID"));
112
+ assert!(is_systemd_compatible(b"A"));
113
+ assert!(is_systemd_compatible(b"A1"));
114
+ assert!(is_systemd_compatible(b"A_B_C"));
115
+ assert!(is_systemd_compatible(b"Z9_"));
116
+ assert!(is_systemd_compatible(b"ND_REMAPPING")); // Our marker field
117
+
118
+ // Invalid field names - lowercase
119
+ assert!(!is_systemd_compatible(b"message"));
120
+ assert!(!is_systemd_compatible(b"Message"));
121
+ assert!(!is_systemd_compatible(b"mESSAGE"));
122
+
123
+ // Invalid field names - special chars
124
+ assert!(!is_systemd_compatible(b"my.field"));
125
+ assert!(!is_systemd_compatible(b"my-field"));
126
+ assert!(!is_systemd_compatible(b"my:field"));
127
+ assert!(!is_systemd_compatible(b"my field"));
128
+
129
+ // Invalid field names - doesn't start with uppercase
130
+ assert!(!is_systemd_compatible(b"1MESSAGE"));
131
+ assert!(!is_systemd_compatible(b"_MESSAGE"));
132
+
133
+ // Invalid field names - empty or too long
134
+ assert!(!is_systemd_compatible(b""));
135
+ assert!(!is_systemd_compatible(&[b'A'; 65]));
136
+ }
137
+
138
+ #[test]
139
+ fn test_extract_field_name() {
140
+ assert_eq!(
141
+ extract_field_name(b"MESSAGE=hello"),
142
+ Some(b"MESSAGE".as_ref())
143
+ );
144
+ assert_eq!(
145
+ extract_field_name(b"PRIORITY=5"),
146
+ Some(b"PRIORITY".as_ref())
147
+ );
148
+ assert_eq!(extract_field_name(b"A="), Some(b"A".as_ref()));
149
+ assert_eq!(extract_field_name(b"=value"), Some(b"".as_ref()));
150
+ assert_eq!(extract_field_name(b"NO_EQUALS"), None);
151
+ assert_eq!(extract_field_name(b""), None);
152
+ }
153
+
154
+ #[test]
155
+ fn test_remapping_registry() {
156
+ let mut registry = FieldMap::new();
157
+
158
+ assert!(registry.is_empty());
159
+ assert_eq!(registry.len(), 0);
160
+
161
+ // Add first mapping
162
+ let otel_name = b"my.field.name".to_vec();
163
+ let systemd_name = rdp::encode_full(&otel_name);
164
+ assert!(registry.add_otel_mapping(otel_name.clone(), systemd_name.clone()));
165
+
166
+ assert!(!registry.is_empty());
167
+ assert_eq!(registry.len(), 1);
168
+
169
+ // Lookup works both ways
170
+ assert_eq!(
171
+ registry.get_systemd_name(&otel_name),
172
+ Some(systemd_name.as_str())
173
+ );
174
+ assert_eq!(
175
+ registry.get_otel_name(&systemd_name),
176
+ Some(otel_name.as_slice())
177
+ );
178
+
179
+ // Adding same mapping again returns false
180
+ assert!(!registry.add_otel_mapping(otel_name.clone(), systemd_name.clone()));
181
+ assert_eq!(registry.len(), 1);
182
+
183
+ // Add different mapping
184
+ let otel_name2 = b"trace-id".to_vec();
185
+ let systemd_name2 = rdp::encode_full(&otel_name2);
186
+ assert!(registry.add_otel_mapping(otel_name2.clone(), systemd_name2.clone()));
187
+
188
+ assert_eq!(registry.len(), 2);
189
+
190
+ // Both mappings exist
191
+ assert!(registry.contains_otel_name(&otel_name));
192
+ assert!(registry.contains_otel_name(&otel_name2));
193
+
194
+ // Lookups still work
195
+ assert_eq!(
196
+ registry.get_systemd_name(&otel_name2),
197
+ Some(systemd_name2.as_str())
198
+ );
199
+ assert_eq!(
200
+ registry.get_otel_name(&systemd_name2),
201
+ Some(otel_name2.as_slice())
202
+ );
203
+ }
204
+}
src/crates/journal-core/src/file/cursor.rs
renamed
+3
-3
@@ -1,7 +1,7 @@
1
-use crate::{file::JournalFile, filter::FilterExpr, offset_array, offset_array::Direction};
2
-use error::{JournalError, Result};
1
+use crate::file::{file::JournalFile, filter::FilterExpr, offset_array, offset_array::Direction};
2
+use crate::error::{JournalError, Result};
3
use std::num::NonZeroU64;
4
-use window_manager::MemoryMap;
4
+use super::mmap::MemoryMap;
5
6
#[derive(Debug, Copy, Clone, PartialEq, Eq)]
7
pub enum Location {
src/crates/journal-core/src/file/file.rs
renamed
+296
-284
@@ -1,146 +1,19 @@
1
-#![allow(unused_imports, clippy::field_reassign_with_default)]
2
-
3
-use crate::hash;
4
-use crate::object::*;
5
-use crate::offset_array;
6
-use error::{JournalError, Result};
7
-use std::cell::{RefCell, UnsafeCell};
1
+#![allow(clippy::field_reassign_with_default)]
2
+
3
+use super::mmap::{MemoryMap, MemoryMapMut, WindowManager};
4
+use crate::collections::HashMap;
5
+use crate::error::{JournalError, Result};
6
+use crate::file::guarded_cell::GuardedCell;
7
+use crate::file::hash;
8
+use crate::file::object::*;
9
+use crate::file::offset_array;
10
use std::fs::{File, OpenOptions};
11
use std::marker::PhantomData;
10
-use std::num::NonZero;
11
-use std::num::NonZeroI128;
12
use std::num::NonZeroU64;
13
-use std::path::Path;
14
-use window_manager::{MemoryMap, MemoryMapMut, WindowManager};
15
-use zerocopy::{ByteSlice, FromBytes, SplitByteSlice, SplitByteSliceMut};
16
-
17
-#[cfg(debug_assertions)]
18
-use std::backtrace::Backtrace;
19
-
20
-use crate::value_guard::ValueGuard;
21
-
22
-fn read_host_file(filename: &str) -> Result<String> {
23
- match std::fs::read_to_string(filename) {
24
- Ok(contents) => Ok(contents),
25
- Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
26
- let filename = format!("/host/{}", filename);
27
- Ok(std::fs::read_to_string(filename)?)
28
- }
29
- Err(e) => Err(e.into()),
30
- }
31
-}
13
+use std::time::Duration;
14
+use zerocopy::{ByteSlice, FromBytes};
15
33
-#[cfg(target_os = "linux")]
34
-pub fn load_machine_id() -> Result<[u8; 16]> {
35
- let content = read_host_file("/etc/machine-id")?;
36
- let decoded = hex::decode(content.trim()).map_err(|_| JournalError::UuidSerde)?;
37
- let bytes: [u8; 16] = decoded.try_into().map_err(|_| JournalError::UuidSerde)?;
38
- Ok(bytes)
39
-}
40
-
41
-#[cfg(target_os = "macos")]
42
-pub fn load_machine_id() -> Result<[u8; 16]> {
43
- use std::process::Command;
44
-
45
- let output = Command::new("system_profiler")
46
- .arg("SPHardwareDataType")
47
- .output()
48
- .map_err(|_| JournalError::UuidSerde)?;
49
-
50
- if output.status.success() {
51
- let output_str = String::from_utf8_lossy(&output.stdout);
52
- for line in output_str.lines() {
53
- if line.contains("Hardware UUID:") {
54
- if let Some(uuid_str) = line.split("Hardware UUID:").nth(1) {
55
- let uuid_str = uuid_str.trim();
56
- let hex_str: String = uuid_str.chars().filter(|c| *c != '-').collect();
57
-
58
- if hex_str.len() == 32 {
59
- let mut bytes = [0u8; 16];
60
- for i in 0..16 {
61
- let hex_pair = &hex_str[i * 2..i * 2 + 2];
62
- bytes[i] = u8::from_str_radix(hex_pair, 16)
63
- .map_err(|_| JournalError::UuidSerde)?;
64
- }
65
- return Ok(bytes);
66
- }
67
- }
68
- }
69
- }
70
- }
71
-
72
- Err(JournalError::UuidSerde)
73
-}
74
-
75
-#[cfg(not(any(target_os = "linux", target_os = "macos")))]
76
-pub fn load_machine_id() -> Result<[u8; 16]> {
77
- Err(JournalError::UuidSerde)
78
-}
79
-
80
-#[cfg(target_os = "linux")]
81
-pub fn load_boot_id() -> Result<[u8; 16]> {
82
- let content = std::fs::read_to_string("/proc/sys/kernel/random/boot_id")?;
83
-
84
- let uuid_str = content.trim();
85
- let hex_str: String = uuid_str.chars().filter(|c| *c != '-').collect();
86
-
87
- if hex_str.len() != 32 {
88
- return Err(JournalError::UuidSerde);
89
- }
90
-
91
- let mut bytes = [0u8; 16];
92
- for i in 0..16 {
93
- let hex_pair = &hex_str[i * 2..i * 2 + 2];
94
- bytes[i] = u8::from_str_radix(hex_pair, 16).map_err(|_| JournalError::UuidSerde)?;
95
- }
96
-
97
- Ok(bytes)
98
-}
99
-
100
-#[cfg(target_os = "macos")]
101
-pub fn load_boot_id() -> Result<[u8; 16]> {
102
- use std::process::Command;
103
-
104
- let output = Command::new("sysctl")
105
- .arg("-n")
106
- .arg("kern.boottime")
107
- .output()
108
- .map_err(|_| JournalError::UuidSerde)?;
109
-
110
- if output.status.success() {
111
- let output_str = String::from_utf8_lossy(&output.stdout);
112
- // Parse "{ sec = 1753988677, usec = 131097 } Thu Jul 31 22:04:37 2025"
113
- // Extract sec and usec values
114
- if let (Some(sec_start), Some(usec_start)) =
115
- (output_str.find("sec = "), output_str.find("usec = "))
116
- {
117
- let sec_str = &output_str[sec_start + 6..];
118
- let sec_end = sec_str.find(',').unwrap_or(sec_str.len());
119
- let sec_str = &sec_str[..sec_end].trim();
120
-
121
- let usec_str = &output_str[usec_start + 7..];
122
- let usec_end = usec_str.find(' ').unwrap_or(usec_str.len());
123
- let usec_str = &usec_str[..usec_end].trim();
124
-
125
- if let (Ok(sec), Ok(usec)) = (sec_str.parse::<u64>(), usec_str.parse::<u64>()) {
126
- // Create a deterministic UUID from boot time
127
- // Use sec in first 8 bytes, usec in next 4 bytes, pad remaining with zeros
128
- let mut bytes = [0u8; 16];
129
- bytes[0..8].copy_from_slice(&sec.to_be_bytes());
130
- bytes[8..12].copy_from_slice(&(usec as u32).to_be_bytes());
131
- // bytes[12..16] remain zero-filled for consistency
132
- return Ok(bytes);
133
- }
134
- }
135
- }
136
-
137
- Err(JournalError::UuidSerde)
138
-}
139
-
140
-#[cfg(not(any(target_os = "linux", target_os = "macos")))]
141
-pub fn load_boot_id() -> Result<[u8; 16]> {
142
- Err(JournalError::UuidSerde)
143
-}
16
+use crate::file::value_guard::ValueGuard;
17
18
// Size to pad objects to (8 bytes)
19
const OBJECT_ALIGNMENT: u64 = 8;
@@ -198,10 +71,10 @@ where
71
72
#[derive(Debug, Clone)]
73
pub struct JournalFileOptions {
201
- machine_id: [u8; 16],
202
- boot_id: [u8; 16],
203
- seqnum_id: [u8; 16],
204
- file_id: [u8; 16],
74
+ machine_id: uuid::Uuid,
75
+ boot_id: uuid::Uuid,
76
+ seqnum_id: uuid::Uuid,
77
+ file_id: uuid::Uuid,
78
window_size: u64,
79
data_hash_table_buckets: usize,
80
field_hash_table_buckets: usize,
@@ -209,12 +82,9 @@ pub struct JournalFileOptions {
82
}
83
84
impl JournalFileOptions {
212
- pub fn new(
213
- machine_id: [u8; 16],
214
- boot_id: [u8; 16],
215
- seqnum_id: [u8; 16],
216
- file_id: [u8; 16],
217
- ) -> Self {
85
+ pub fn new(machine_id: uuid::Uuid, boot_id: uuid::Uuid, seqnum_id: uuid::Uuid) -> Self {
86
+ let file_id = uuid::Uuid::new_v4();
87
+
88
Self {
89
machine_id,
90
boot_id,
@@ -227,6 +97,49 @@ impl JournalFileOptions {
97
}
98
}
99
100
+ /// Creates options with bucket sizes optimized based on previous utilization
101
+ pub fn with_optimized_buckets(
102
+ mut self,
103
+ previous_utilization: Option<BucketUtilization>,
104
+ max_file_size: Option<u64>,
105
+ ) -> Self {
106
+ let (data_buckets, field_buckets) = if let Some(utilization) = previous_utilization {
107
+ let data_utilization = utilization.data_utilization();
108
+ let field_utilization = utilization.field_utilization();
109
+
110
+ let data_buckets = if data_utilization > 0.75 {
111
+ (utilization.data_total * 2).next_power_of_two()
112
+ } else if data_utilization < 0.25 && utilization.data_total > 4096 {
113
+ (utilization.data_total / 2).next_power_of_two()
114
+ } else {
115
+ utilization.data_total
116
+ };
117
+
118
+ let field_buckets = if field_utilization > 0.75 {
119
+ (utilization.field_total * 2).next_power_of_two()
120
+ } else if field_utilization < 0.25 && utilization.field_total > 512 {
121
+ (utilization.field_total / 2).next_power_of_two()
122
+ } else {
123
+ utilization.field_total
124
+ };
125
+
126
+ (data_buckets, field_buckets)
127
+ } else {
128
+ // Initial sizing based on rotation policy max file size
129
+ let max_file_size = max_file_size.unwrap_or(8 * 1024 * 1024);
130
+
131
+ // 16 MiB -> 4096 data buckets
132
+ let data_buckets = (max_file_size / 4096).max(1024).next_power_of_two() as usize;
133
+ let field_buckets = 128; // Assume ~8:1 data:field ratio
134
+
135
+ (data_buckets, field_buckets)
136
+ };
137
+
138
+ self.data_hash_table_buckets = data_buckets;
139
+ self.field_hash_table_buckets = field_buckets;
140
+ self
141
+ }
142
+
143
pub fn with_window_size(mut self, size: u64) -> Self {
144
assert_eq!(size % OBJECT_ALIGNMENT, 0);
145
assert_eq!(size % 4096, 0, "Window size must be page-aligned");
@@ -257,8 +170,8 @@ impl JournalFileOptions {
170
self
171
}
172
260
- pub fn create<M: MemoryMapMut>(self, path: impl AsRef<Path>) -> Result<JournalFile<M>> {
261
- JournalFile::create(path, self)
173
+ pub fn create<M: MemoryMapMut>(self, file: &crate::repository::File) -> Result<JournalFile<M>> {
174
+ JournalFile::create(file, self)
175
}
176
}
177
@@ -302,29 +215,25 @@ impl BucketUtilization {
215
///
216
/// `JournalFile` uses interior mutability to provide a safe API with the following characteristics:
217
///
305
-/// - The window manager is wrapped in an `UnsafeCell` to allow mutation through a shared reference.
306
-/// - A single `RefCell<bool>` guards access to ensure only one object can be active at a time.
307
-/// - Methods like `data_object()` return a `ValueGuard<T>` that automatically releases the lock
218
+/// - The window manager is wrapped in a `GuardedCell` which owns both the `WindowManager` and
219
+/// its guard flag, providing interior mutability with integrated guard-based exclusion.
220
+/// - The guard flag ensures only one object can be active at a time.
221
+/// - Methods like `data_object()` return a `ValueGuard<T>` that automatically releases the guard
222
/// when dropped.
223
///
224
/// This design ensures that memory safety is maintained even though references to memory-mapped
225
/// regions could be invalidated when new objects are created.
226
pub struct JournalFile<M: MemoryMap> {
227
+ // The validated File this journal represents
228
+ file: crate::repository::File,
229
+
230
// Persistent memory maps for journal header and data/field hash tables
231
header_map: M,
232
data_hash_table_map: Option<M>,
233
field_hash_table_map: Option<M>,
234
318
- // Window manager for other objects
319
- window_manager: UnsafeCell<WindowManager<M>>,
320
-
321
- // Flag to track if any object is in use
322
- object_in_use: RefCell<bool>,
323
-
324
- #[cfg(debug_assertions)]
325
- prev_backtrace: RefCell<Backtrace>,
326
- #[cfg(debug_assertions)]
327
- backtrace: RefCell<Backtrace>,
235
+ // Window manager for other objects (owns the guard flag internally)
236
+ window_manager: GuardedCell<WindowManager<M>>,
237
}
238
239
fn map_hash_table<M: MemoryMap>(
@@ -376,15 +285,18 @@ impl<M: MemoryMap> JournalFile<M> {
285
Ok(None)
286
}
287
379
- pub fn open(path: impl AsRef<Path>, window_size: u64) -> Result<Self> {
288
+ pub fn open(file: &crate::repository::File, window_size: u64) -> Result<Self> {
289
debug_assert_eq!(window_size % OBJECT_ALIGNMENT, 0);
290
291
// Open file and check its size
383
- let file = OpenOptions::new().read(true).write(false).open(&path)?;
292
+ let fd = OpenOptions::new()
293
+ .read(true)
294
+ .write(false)
295
+ .open(file.path())?;
296
297
// Create a memory map for the header
298
let header_size = std::mem::size_of::<JournalHeader>() as u64;
387
- let header_map = M::create(&file, 0, header_size)?;
299
+ let header_map = M::create(&fd, 0, header_size)?;
300
let header = JournalHeader::ref_from_prefix(&header_map).unwrap().0;
301
if header.signature != *b"LPKSHHRH" {
302
return Err(JournalError::InvalidMagicNumber);
@@ -392,33 +304,32 @@ impl<M: MemoryMap> JournalFile<M> {
304
305
// Initialize the hash table maps if they exist
306
let data_hash_table_map = map_hash_table(
395
- &file,
307
+ &fd,
308
header.data_hash_table_offset,
309
header.data_hash_table_size,
310
)?;
311
let field_hash_table_map = map_hash_table(
400
- &file,
312
+ &fd,
313
header.field_hash_table_offset,
314
header.field_hash_table_size,
315
)?;
316
317
// Create window manager for the rest of the objects
406
- let window_manager = UnsafeCell::new(WindowManager::new(file, window_size, 32)?);
318
+ let window_manager = GuardedCell::new(WindowManager::new(fd, window_size, 16)?);
319
320
Ok(JournalFile {
321
+ file: file.clone(),
322
header_map,
323
data_hash_table_map,
324
field_hash_table_map,
325
window_manager,
413
- object_in_use: RefCell::new(false),
414
-
415
- #[cfg(debug_assertions)]
416
- prev_backtrace: RefCell::new(Backtrace::capture()),
417
- #[cfg(debug_assertions)]
418
- backtrace: RefCell::new(Backtrace::capture()),
326
})
327
}
328
329
+ pub fn file(&self) -> &crate::repository::File {
330
+ &self.file
331
+ }
332
+
333
pub fn hash(&self, data: &[u8]) -> u64 {
334
let is_keyed_hash = self
335
.journal_header_ref()
@@ -436,10 +347,31 @@ impl<M: MemoryMap> JournalFile<M> {
347
}
348
349
pub fn entry_list(&self) -> Option<offset_array::List> {
439
- let head_offset = self.journal_header_ref().entry_array_offset?;
440
- let total_items =
441
- std::num::NonZeroUsize::new(self.journal_header_ref().n_entries as usize)?;
442
- Some(offset_array::List::new(head_offset, total_items))
350
+ let header = self.journal_header_ref();
351
+
352
+ header.entry_array_offset.and_then(|head_offset| {
353
+ std::num::NonZeroUsize::new(header.n_entries as usize)
354
+ .map(|total_items| offset_array::List::new(head_offset, total_items))
355
+ })
356
+ }
357
+
358
+ pub fn entry_offsets(&self, offsets: &mut Vec<NonZeroU64>) -> Result<()> {
359
+ if let Some(entry_list) = self.entry_list() {
360
+ entry_list.collect_offsets(self, offsets)?;
361
+ }
362
+
363
+ Ok(())
364
+ }
365
+
366
+ // Returns the data object offsets of the entry object at the specified
367
+ // offset
368
+ pub fn entry_data_object_offsets(
369
+ &self,
370
+ entry_offset: NonZeroU64,
371
+ offsets: &mut Vec<NonZeroU64>,
372
+ ) -> Result<()> {
373
+ let entry_guard = self.entry_ref(entry_offset)?;
374
+ entry_guard.collect_offsets(offsets)
375
}
376
377
pub fn journal_header_ref(&self) -> &JournalHeader {
@@ -467,59 +399,36 @@ impl<M: MemoryMap> JournalFile<M> {
399
400
pub fn object_header_ref(&self, position: NonZeroU64) -> Result<&ObjectHeader> {
401
let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
470
- let window_manager = unsafe { &mut *self.window_manager.get() };
402
+ let window_manager = self.window_manager.borrow_mut_checked()?;
403
let header_slice = window_manager.get_slice(position.get(), size_needed)?;
404
Ok(ObjectHeader::ref_from_bytes(header_slice).unwrap())
405
}
406
475
- fn object_data_ref(&self, offset: NonZeroU64, size_needed: u64) -> Result<&[u8]> {
476
- let window_manager = unsafe { &mut *self.window_manager.get() };
477
- let object_slice = window_manager.get_slice(offset.get(), size_needed)?;
478
- Ok(object_slice)
479
- }
480
-
407
fn journal_object_ref<'a, T>(&'a self, offset: NonZeroU64) -> Result<ValueGuard<'a, T>>
408
where
409
T: JournalObject<&'a [u8]>,
410
{
485
- // Check if any object is already in use
486
- let mut is_in_use = self.object_in_use.borrow_mut();
487
- if *is_in_use {
488
- #[cfg(debug_assertions)]
489
- {
490
- eprintln!(
491
- "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
492
- self.backtrace.borrow().to_string(),
493
- self.prev_backtrace.borrow().to_string()
494
- );
495
- }
496
- return Err(JournalError::ValueGuardInUse);
497
- }
498
-
499
- #[cfg(debug_assertions)]
500
- {
501
- self.backtrace.swap(&self.prev_backtrace);
502
- let _ = self.backtrace.replace(Backtrace::force_capture());
503
- }
504
-
411
let is_compact = self
412
.journal_header_ref()
413
.has_incompatible_flag(HeaderIncompatibleFlags::Compact);
414
509
- let size_needed = {
510
- let header = self.object_header_ref(offset)?;
511
- header.size
512
- };
415
+ self.window_manager.with_guarded(offset, |wm| {
416
+ // Get the object header to determine size
417
+ let size_needed = {
418
+ let header_slice =
419
+ wm.get_slice(offset.get(), std::mem::size_of::<ObjectHeader>() as u64)?;
420
+ let header = ObjectHeader::ref_from_bytes(header_slice).unwrap();
421
+ header.size
422
+ };
423
514
- let data = self.object_data_ref(offset, size_needed)?;
515
- let Some(value) = T::from_data(data, is_compact) else {
516
- return Err(JournalError::ZerocopyFailure);
517
- };
424
+ // Get the full object data
425
+ let data = wm.get_slice(offset.get(), size_needed)?;
426
519
- // Mark as in use
520
- *is_in_use = true;
427
+ // Parse the object
428
+ let value = T::from_data(data, is_compact).ok_or(JournalError::ZerocopyFailure)?;
429
522
- Ok(ValueGuard::new(offset, value, &self.object_in_use))
430
+ Ok(value)
431
+ })
432
}
433
434
pub fn offset_array_ref(
@@ -598,6 +507,70 @@ impl<M: MemoryMap> JournalFile<M> {
507
iterator
508
}
509
510
+ pub fn load_fields(&self) -> Result<HashMap<String, String>> {
511
+ let remapping_payload = b"ND_REMAPPING=1".as_slice();
512
+ let hash = self.hash(remapping_payload);
513
+
514
+ let mut field_map = HashMap::default();
515
+
516
+ match self.find_data_offset(hash, remapping_payload) {
517
+ Ok(Some(offset)) => {
518
+ let Some(ic) = self.data_ref(offset)?.header.inlined_cursor() else {
519
+ return Err(JournalError::EmptyInlineCursor);
520
+ };
521
+
522
+ let mut entry_offsets = Vec::new();
523
+ ic.collect_offsets(self, &mut entry_offsets)?;
524
+
525
+ let mut data_offsets = Vec::new();
526
+ for entry_offset in entry_offsets {
527
+ {
528
+ let entry_object = self.entry_ref(entry_offset)?;
529
+ data_offsets.clear();
530
+ entry_object.collect_offsets(&mut data_offsets)?;
531
+ }
532
+
533
+ for data_offset in data_offsets.iter().copied() {
534
+ let data_object = self.data_ref(data_offset)?;
535
+ let payload = data_object.payload_bytes();
536
+
537
+ if payload == remapping_payload {
538
+ continue;
539
+ }
540
+
541
+ let s = std::str::from_utf8(payload).expect("utf8 data");
542
+
543
+ let Some((field, value)) = s.split_once('=') else {
544
+ return Err(JournalError::InvalidField);
545
+ };
546
+
547
+ let systemd_name = String::from(field);
548
+ let otel_name = String::from(value);
549
+
550
+ field_map.insert(otel_name, systemd_name);
551
+ }
552
+ }
553
+ }
554
+ Ok(None) => {
555
+ // Just load fields from the field hash table
556
+ }
557
+ Err(e) => {
558
+ return Err(e);
559
+ }
560
+ };
561
+
562
+ for value_guard in self.fields() {
563
+ let field = value_guard?;
564
+ if field.payload.starts_with(b"ND") {
565
+ continue;
566
+ }
567
+ let s = String::from_utf8(field.get_payload().to_vec()).expect("utf8 data");
568
+ field_map.insert(s.clone(), s);
569
+ }
570
+
571
+ Ok(field_map)
572
+ }
573
+
574
/// Creates an iterator over all DATA objects for the specified field
575
pub fn field_data_objects<'a>(
576
&'a self,
@@ -668,16 +641,71 @@ impl<M: MemoryMap> JournalFile<M> {
641
field_total,
642
})
643
}
644
+
645
+ /// Get the duration covered by all entries in the journal
646
+ /// Returns None if the journal is empty or contains only one entry
647
+ pub fn duration(&self) -> Option<Duration> {
648
+ let header = self.journal_header_ref();
649
+
650
+ if header.head_entry_realtime == 0 || header.tail_entry_realtime == 0 {
651
+ return None;
652
+ }
653
+
654
+ if header.tail_entry_realtime <= header.head_entry_realtime {
655
+ // Single entry or invalid state
656
+ return None;
657
+ }
658
+
659
+ let duration_micros = header.tail_entry_realtime - header.head_entry_realtime;
660
+ Some(Duration::from_micros(duration_micros))
661
+ }
662
}
663
664
impl<M: MemoryMapMut> JournalFile<M> {
674
- pub fn create(path: impl AsRef<Path>, options: JournalFileOptions) -> Result<Self> {
675
- let file = OpenOptions::new()
665
+ /// Syncs all file data to disk, ensuring all changes are persisted
666
+ ///
667
+ /// This performs a two-step sync process:
668
+ /// 1. Flushes memory-mapped regions to the file page cache (msync)
669
+ /// 2. Syncs the file page cache to physical disk (fdatasync)
670
+ pub fn sync(&mut self) -> Result<()> {
671
+ // Flush memory-mapped header to file page cache
672
+ self.header_map.flush()?;
673
+
674
+ // Sync file page cache to disk
675
+ let window_manager = self.window_manager.get_mut();
676
+ window_manager.sync()?;
677
+
678
+ Ok(())
679
+ }
680
+
681
+ /// Creates a successor journal file with optimized bucket sizes based on this file's utilization
682
+ pub fn create_successor(
683
+ &self,
684
+ file: &crate::repository::File,
685
+ max_file_size: Option<u64>,
686
+ ) -> Result<Self> {
687
+ let header = self.journal_header_ref();
688
+ let bucket_utilization = self.bucket_utilization();
689
+
690
+ let options = JournalFileOptions::new(
691
+ uuid::Uuid::from_bytes(header.machine_id),
692
+ uuid::Uuid::from_bytes(header.tail_entry_boot_id),
693
+ uuid::Uuid::from_bytes(header.seqnum_id),
694
+ )
695
+ .with_window_size(8 * 1024 * 1024)
696
+ .with_optimized_buckets(bucket_utilization, max_file_size)
697
+ .with_keyed_hash(header.has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash));
698
+
699
+ Self::create(file, options)
700
+ }
701
+
702
+ pub fn create(file: &crate::repository::File, options: JournalFileOptions) -> Result<Self> {
703
+ let fd = OpenOptions::new()
704
.create(true)
705
.truncate(true)
706
.read(true)
707
.write(true)
680
- .open(&path)?;
708
+ .open(file.path())?;
709
710
// Calculate hash table sizes
711
let data_hash_table_size =
@@ -716,45 +744,42 @@ impl<M: MemoryMapMut> JournalFile<M> {
744
field_hash_table_offset + field_hash_table_size as u64 - header.header_size;
745
746
// Set IDs from options
719
- header.machine_id = options.machine_id;
720
- header.tail_entry_boot_id = options.boot_id;
721
- header.file_id = options.file_id;
722
- header.seqnum_id = options.seqnum_id;
747
+ header.machine_id = *options.machine_id.as_bytes();
748
+ header.tail_entry_boot_id = *options.boot_id.as_bytes();
749
+ header.file_id = *options.file_id.as_bytes();
750
+ header.seqnum_id = *options.seqnum_id.as_bytes();
751
752
// Create memory maps for hash tables
753
let data_hash_table_map = map_hash_table(
726
- &file,
754
+ &fd,
755
header.data_hash_table_offset,
756
header.data_hash_table_size,
757
)?;
758
let field_hash_table_map = map_hash_table(
731
- &file,
759
+ &fd,
760
header.field_hash_table_offset,
761
header.field_hash_table_size,
762
)?;
763
764
// Create header memory map and write header
765
let header_size = std::mem::size_of::<JournalHeader>() as u64;
738
- let mut header_map = M::create(&file, 0, header_size)?;
766
+ let mut header_map = M::create(&fd, 0, header_size)?;
767
{
768
let header_mut = JournalHeader::mut_from_prefix(&mut header_map).unwrap().0;
769
*header_mut = header;
770
+ // Set state to ONLINE as per journal file format spec
771
+ header_mut.state = JournalState::Online as u8;
772
}
773
774
// Create window manager for the rest of the objects
745
- let window_manager = UnsafeCell::new(WindowManager::new(file, options.window_size, 32)?);
775
+ let window_manager = GuardedCell::new(WindowManager::new(fd, options.window_size, 32)?);
776
747
- let jf = JournalFile {
777
+ let mut jf = JournalFile {
778
+ file: file.clone(),
779
header_map,
780
data_hash_table_map,
781
field_hash_table_map,
782
window_manager,
752
- object_in_use: RefCell::new(false),
753
-
754
- #[cfg(debug_assertions)]
755
- prev_backtrace: RefCell::new(Backtrace::capture()),
756
- #[cfg(debug_assertions)]
757
- backtrace: RefCell::new(Backtrace::capture()),
783
};
784
785
// write data hash table object header info
@@ -787,6 +812,9 @@ impl<M: MemoryMapMut> JournalFile<M> {
812
object_header.size = size
813
}
814
815
+ // Sync to ensure the ONLINE state is persisted to disk
816
+ jf.sync()?;
817
+
818
Ok(jf)
819
}
820
@@ -808,19 +836,14 @@ impl<M: MemoryMapMut> JournalFile<M> {
836
.and_then(|m| FieldHashTable::<&mut [u8]>::from_data_mut(m, false))
837
}
838
839
+ #[allow(clippy::mut_from_ref)]
840
fn object_header_mut(&self, offset: NonZeroU64) -> Result<&mut ObjectHeader> {
841
let size_needed = std::mem::size_of::<ObjectHeader>() as u64;
813
- let window_manager = unsafe { &mut *self.window_manager.get() };
842
+ let window_manager = self.window_manager.borrow_mut_checked()?;
843
let header_slice = window_manager.get_slice_mut(offset.get(), size_needed)?;
844
Ok(ObjectHeader::mut_from_bytes(header_slice).unwrap())
845
}
846
818
- fn object_data_mut(&self, offset: NonZeroU64, size_needed: u64) -> Result<&mut [u8]> {
819
- let window_manager = unsafe { &mut *self.window_manager.get() };
820
- let object_slice = window_manager.get_slice_mut(offset.get(), size_needed)?;
821
- Ok(object_slice)
822
- }
823
-
847
fn journal_object_mut<'a, T>(
848
&'a self,
849
type_: ObjectType,
@@ -830,52 +853,42 @@ impl<M: MemoryMapMut> JournalFile<M> {
853
where
854
T: JournalObjectMut<&'a mut [u8]>,
855
{
833
- // Check if any object is already in use
834
- let mut is_in_use = self.object_in_use.borrow_mut();
835
- if *is_in_use {
836
- #[cfg(debug_assertions)]
837
- {
838
- eprintln!(
839
- "Value is in use. Current Backtrace: {:?}, Previous Backtrace: {:?}",
840
- self.backtrace.borrow().to_string(),
841
- self.prev_backtrace.borrow().to_string()
842
- );
843
- }
844
- return Err(JournalError::ValueGuardInUse);
845
- }
846
-
847
- #[cfg(debug_assertions)]
848
- {
849
- self.backtrace.swap(&self.prev_backtrace);
850
- let _ = self.backtrace.replace(Backtrace::force_capture());
851
- }
852
-
856
let is_compact = self
857
.journal_header_ref()
858
.has_incompatible_flag(HeaderIncompatibleFlags::Compact);
859
857
- let size_needed = match size {
858
- Some(size) => {
859
- let header = self.object_header_mut(offset)?;
860
- header.type_ = type_ as u8;
861
- header.size = size;
862
- size
863
- }
864
- None => {
865
- let header = self.object_header_ref(offset)?;
866
- if header.type_ != type_ as u8 {
867
- return Err(JournalError::InvalidObjectType);
860
+ self.window_manager.with_guarded(offset, |wm| {
861
+ // Get or set the size
862
+ let size_needed = match size {
863
+ Some(size) => {
864
+ // Setting object header for a new object
865
+ let header_slice =
866
+ wm.get_slice_mut(offset.get(), std::mem::size_of::<ObjectHeader>() as u64)?;
867
+ let header = ObjectHeader::mut_from_bytes(header_slice).unwrap();
868
+ header.type_ = type_ as u8;
869
+ header.size = size;
870
+ size
871
}
869
- header.size
870
- }
871
- };
872
+ None => {
873
+ // Reading existing object header
874
+ let header_slice =
875
+ wm.get_slice(offset.get(), std::mem::size_of::<ObjectHeader>() as u64)?;
876
+ let header = ObjectHeader::ref_from_bytes(header_slice).unwrap();
877
+ if header.type_ != type_ as u8 {
878
+ return Err(JournalError::InvalidObjectType);
879
+ }
880
+ header.size
881
+ }
882
+ };
883
873
- let data = self.object_data_mut(offset, size_needed)?;
874
- let value = T::from_data_mut(data, is_compact).ok_or(JournalError::ZerocopyFailure)?;
884
+ // Get mutable object data
885
+ let data = wm.get_slice_mut(offset.get(), size_needed)?;
886
876
- // Mark as in use
877
- *is_in_use = true;
878
- Ok(ValueGuard::new(offset, value, &self.object_in_use))
887
+ // Parse the mutable object
888
+ let value = T::from_data_mut(data, is_compact).ok_or(JournalError::ZerocopyFailure)?;
889
+
890
+ Ok(value)
891
+ })
892
}
893
894
pub fn offset_array_mut(
@@ -898,8 +911,7 @@ impl<M: MemoryMapMut> JournalFile<M> {
911
size
912
});
913
901
- let offset_array = self.journal_object_mut(ObjectType::EntryArray, offset, size);
902
- offset_array
914
+ self.journal_object_mut(ObjectType::EntryArray, offset, size)
915
}
916
917
pub fn field_mut(
src/crates/journal-core/src/file/filter.rs
renamed
+107
-139
@@ -1,81 +1,93 @@
1
-use error::{JournalError, Result};
2
-use journal_file::{
3
- offset_array::{Direction, InlinedCursor},
4
- JournalFile,
5
-};
6
-use window_manager::MemoryMap;
1
+use crate::file::{file::JournalFile, offset_array::InlinedCursor};
2
+use crate::error::{JournalError, Result};
3
+use std::num::NonZeroU64;
4
+use super::mmap::MemoryMap;
5
6
#[derive(Clone, Debug)]
7
pub enum FilterExpr {
10
- Match(u64, Option<InlinedCursor>),
8
+ None,
9
+ Match(NonZeroU64, InlinedCursor),
10
Conjunction(Vec<FilterExpr>),
11
Disjunction(Vec<FilterExpr>),
12
}
13
14
impl FilterExpr {
16
- pub fn lookup<M: MemoryMap>(
17
- &self,
18
- journal_file: &JournalFile<M>,
19
- needle_offset: u64,
20
- direction: Direction,
21
- ) -> Result<Option<u64>> {
22
- let predicate =
23
- move |entry_offset: u64| -> Result<bool> { Ok(entry_offset < needle_offset) };
24
-
25
- match self {
26
- FilterExpr::Match(data_offset, _) => {
27
- let entry_offset = journal_file.data_object_directed_partition_point(
28
- *data_offset,
29
- predicate,
30
- direction,
31
- )?;
32
- Ok(entry_offset)
33
- }
34
- FilterExpr::Conjunction(filter_exprs) => {
35
- let mut current_offset = needle_offset;
36
-
37
- loop {
38
- let previous_offset = current_offset;
39
-
40
- for filter_expr in filter_exprs {
41
- if direction == Direction::Backward {
42
- current_offset = current_offset.saturating_add(1);
43
- }
44
-
45
- match filter_expr.lookup(journal_file, current_offset, direction)? {
46
- Some(new_offset) => current_offset = new_offset,
47
- None => return Ok(None),
48
- }
49
- }
50
-
51
- if current_offset == previous_offset {
52
- return Ok(Some(current_offset));
53
- }
54
- }
55
- }
56
- FilterExpr::Disjunction(filter_exprs) => {
57
- let cmp = match direction {
58
- Direction::Forward => std::cmp::min,
59
- Direction::Backward => std::cmp::max,
60
- };
61
-
62
- filter_exprs.iter().try_fold(None, |acc, expr| {
63
- let result = expr.lookup(journal_file, needle_offset, direction)?;
64
-
65
- Ok(match (acc, result) {
66
- (None, Some(offset)) => Some(offset),
67
- (Some(best), Some(offset)) => Some(cmp(best, offset)),
68
- (acc, None) => acc,
69
- })
70
- })
71
- }
72
- }
73
- }
15
+ // pub fn lookup<M: MemoryMap>(
16
+ // &self,
17
+ // journal_file: &JournalFile<M>,
18
+ // needle_offset: u64,
19
+ // direction: Direction,
20
+ // ) -> Result<Option<u64>> {
21
+ // let Some(needle_offset) = NonZeroU64::new(needle_offset) else {
22
+ // return Err(JournalError::InvalidOffset);
23
+ // };
24
+
25
+ // let predicate =
26
+ // move |entry_offset: NonZeroU64| -> Result<bool> { Ok(entry_offset < needle_offset) };
27
+
28
+ // match self {
29
+ // FilterExpr::Match(data_offset, _) => {
30
+ // let Some(data_offset) = NonZeroU64::new(*data_offset) else {
31
+ // return Err(JournalError::InvalidOffset);
32
+ // };
33
+ // let entry_offset = journal_file.data_object_directed_partition_point(
34
+ // data_offset,
35
+ // predicate,
36
+ // direction,
37
+ // )?;
38
+ // Ok(entry_offset.map(|x| x.get()))
39
+ // }
40
+ // FilterExpr::Conjunction(filter_exprs) => {
41
+ // let mut current_offset = needle_offset;
42
+
43
+ // loop {
44
+ // let previous_offset = current_offset;
45
+
46
+ // for filter_expr in filter_exprs {
47
+ // if direction == Direction::Backward {
48
+ // current_offset = current_offset.saturating_add(1);
49
+ // }
50
+
51
+ // match filter_expr.lookup(journal_file, current_offset.get(), direction)? {
52
+ // Some(new_offset) => {
53
+ // if new_offset == 0 {
54
+ // panic!("Wtf");
55
+ // }
56
+ // current_offset = NonZeroU64::new(new_offset).unwrap();
57
+ // }
58
+ // None => return Ok(None),
59
+ // }
60
+ // }
61
+
62
+ // if current_offset == previous_offset {
63
+ // return Ok(Some(current_offset.get()));
64
+ // }
65
+ // }
66
+ // }
67
+ // FilterExpr::Disjunction(filter_exprs) => {
68
+ // let cmp = match direction {
69
+ // Direction::Forward => std::cmp::min,
70
+ // Direction::Backward => std::cmp::max,
71
+ // };
72
+
73
+ // filter_exprs.iter().try_fold(None, |acc, expr| {
74
+ // let result = expr.lookup(journal_file, needle_offset.get(), direction)?;
75
+
76
+ // Ok(match (acc, result) {
77
+ // (None, Some(offset)) => Some(offset),
78
+ // (Some(best), Some(offset)) => Some(cmp(best, offset)),
79
+ // (acc, None) => acc,
80
+ // })
81
+ // })
82
+ // }
83
+ // FilterExpr::None => Ok(None),
84
+ // }
85
+ // }
86
87
pub fn head(&mut self) -> &mut Self {
88
match self {
77
- FilterExpr::Match(_, None) => (),
78
- FilterExpr::Match(_, Some(ic)) => {
89
+ FilterExpr::None => (),
90
+ FilterExpr::Match(_, ic) => {
91
*ic = ic.head();
92
}
93
FilterExpr::Conjunction(filter_exprs) => {
@@ -95,8 +107,8 @@ impl FilterExpr {
107
108
pub fn tail<M: MemoryMap>(&mut self, journal_file: &JournalFile<M>) -> Result<&mut Self> {
109
match self {
98
- FilterExpr::Match(_, None) => (),
99
- FilterExpr::Match(_, Some(ic)) => {
110
+ FilterExpr::None => {}
111
+ FilterExpr::Match(_, ic) => {
112
*ic = ic.tail(journal_file)?;
113
}
114
FilterExpr::Conjunction(filter_exprs) => {
@@ -119,11 +131,11 @@ impl FilterExpr {
131
pub fn next<M: MemoryMap>(
132
&mut self,
133
journal_file: &JournalFile<M>,
122
- needle_offset: u64,
123
- ) -> Result<Option<u64>> {
134
+ needle_offset: NonZeroU64,
135
+ ) -> Result<Option<NonZeroU64>> {
136
match self {
125
- FilterExpr::Match(_, None) => Ok(None),
126
- FilterExpr::Match(_, Some(ic)) => ic.next_until(journal_file, needle_offset),
137
+ FilterExpr::None => Ok(None),
138
+ FilterExpr::Match(_, ic) => ic.next_until(journal_file, needle_offset),
139
FilterExpr::Conjunction(filter_exprs) => {
140
let mut needle_offset = needle_offset;
141
@@ -144,7 +156,7 @@ impl FilterExpr {
156
}
157
}
158
FilterExpr::Disjunction(filter_exprs) => {
147
- let mut best_offset: Option<u64> = None;
159
+ let mut best_offset: Option<NonZeroU64> = None;
160
161
for fe in filter_exprs.iter_mut() {
162
if let Some(fe_offset) = fe.next(journal_file, needle_offset)? {
@@ -165,11 +177,11 @@ impl FilterExpr {
177
pub fn previous<M: MemoryMap>(
178
&mut self,
179
journal_file: &JournalFile<M>,
168
- needle_offset: u64,
169
- ) -> Result<Option<u64>> {
180
+ needle_offset: NonZeroU64,
181
+ ) -> Result<Option<NonZeroU64>> {
182
match self {
171
- FilterExpr::Match(_, None) => Ok(None),
172
- FilterExpr::Match(_, Some(ic)) => ic.previous_until(journal_file, needle_offset),
183
+ FilterExpr::None => Ok(None),
184
+ FilterExpr::Match(_, ic) => ic.previous_until(journal_file, needle_offset),
185
FilterExpr::Conjunction(filter_exprs) => {
186
let mut needle_offset = needle_offset;
187
@@ -190,7 +202,7 @@ impl FilterExpr {
202
}
203
}
204
FilterExpr::Disjunction(filter_exprs) => {
193
- let mut best_offset: Option<u64> = None;
205
+ let mut best_offset: Option<NonZeroU64> = None;
206
207
for fe in filter_exprs.iter_mut() {
208
if let Some(fe_offset) = fe.previous(journal_file, needle_offset)? {
@@ -205,60 +217,6 @@ impl FilterExpr {
217
}
218
}
219
}
208
-
209
- pub fn dump<M: MemoryMap>(&self, journal_file: &JournalFile<M>) -> Result<String> {
210
- let mut output = String::new();
211
- self.dump_internal(journal_file, 0, &mut output)?;
212
- Ok(output)
213
- }
214
-
215
- /// Helper function for format_data_objects that handles nested expressions and indentation
216
- fn dump_internal<M: MemoryMap>(
217
- &self,
218
- journal_file: &JournalFile<M>,
219
- indent_level: usize,
220
- output: &mut String,
221
- ) -> Result<()> {
222
- let indent = " ".repeat(indent_level);
223
-
224
- match self {
225
- FilterExpr::Match(data_offset, inlined_cursor) => {
226
- // Load the data object
227
- let data_object = journal_file.data_ref(*data_offset)?;
228
-
229
- // Get the payload as a string if possible
230
- let payload_bytes = data_object.payload_bytes();
231
- let payload_str = String::from_utf8_lossy(payload_bytes);
232
-
233
- // Format the data offset and payload
234
- output.push_str(&format!(
235
- "{}Match[offset=0x{:x}]: {}\n",
236
- indent, data_offset, payload_str
237
- ));
238
-
239
- // Format cursor information if available
240
- if let Some(ic) = inlined_cursor {
241
- output.push_str(&format!("{} Cursor: {:?}\n", indent, ic));
242
- }
243
- }
244
- FilterExpr::Conjunction(filter_exprs) => {
245
- output.push_str(&format!("{}Conjunction (AND) {{\n", indent));
246
- for expr in filter_exprs {
247
- expr.dump_internal(journal_file, indent_level + 1, output)?;
248
- }
249
- output.push_str(&format!("{}}}\n", indent));
250
- }
251
- FilterExpr::Disjunction(filter_exprs) => {
252
- output.push_str(&format!("{}Disjunction (OR) {{\n", indent));
253
- for expr in filter_exprs {
254
- expr.dump_internal(journal_file, indent_level + 1, output)?;
255
- }
256
- output.push_str(&format!("{}}}\n", indent));
257
- }
258
- }
259
-
260
- Ok(())
261
- }
220
}
221
222
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -321,19 +279,29 @@ impl JournalFilter {
279
for idx in start..i {
280
let data = self.current_matches[idx].as_slice();
281
let hash = journal_file.hash(data);
324
- let offset = journal_file.find_data_offset_by_payload(data, hash)?;
282
326
- let ic = journal_file.data_ref(offset)?.inlined_cursor();
327
- matches.push(FilterExpr::Match(offset, ic));
283
+ let match_expr = match journal_file.find_data_offset(hash, data)? {
284
+ Some(offset) => match journal_file.data_ref(offset)?.inlined_cursor() {
285
+ Some(ic) => FilterExpr::Match(offset, ic),
286
+ None => FilterExpr::None,
287
+ },
288
+ None => FilterExpr::None,
289
+ };
290
+ matches.push(match_expr);
291
}
292
elements.push(FilterExpr::Disjunction(matches));
293
} else {
294
let data = self.current_matches[start].as_slice();
295
let hash = journal_file.hash(data);
333
- let offset = journal_file.find_data_offset_by_payload(data, hash)?;
296
335
- let ic = journal_file.data_ref(offset)?.inlined_cursor();
336
- elements.push(FilterExpr::Match(offset, ic));
297
+ let match_expr = match journal_file.find_data_offset(hash, data)? {
298
+ Some(offset) => match journal_file.data_ref(offset)?.inlined_cursor() {
299
+ Some(ic) => FilterExpr::Match(offset, ic),
300
+ None => FilterExpr::None,
301
+ },
302
+ None => FilterExpr::None,
303
+ };
304
+ elements.push(match_expr);
305
}
306
}
307
src/crates/journal-core/src/file/guarded_cell.rs
new
+330
@@ -0,0 +1,330 @@
1
+use crate::error::{JournalError, Result};
2
+use crate::file::value_guard::ValueGuard;
3
+use std::cell::{RefCell, UnsafeCell};
4
+use std::num::NonZeroU64;
5
+
6
+/// A cell type that provides interior mutability with integrated guard-based exclusion.
7
+///
8
+/// # Purpose
9
+///
10
+/// `GuardedCell` is designed for situations where:
11
+/// - You need interior mutability (get `&mut T` from `&self`)
12
+/// - References to the inner data must outlive the borrowing function scope
13
+/// - You need to ensure only one "logical borrow" is active at a time
14
+/// - `RefCell` cannot be used because `RefMut` guards must be dropped before returning
15
+///
16
+/// # Design
17
+///
18
+/// `GuardedCell` owns both the value and a `RefCell<bool>` guard flag. When borrowing,
19
+/// it checks that the guard is clear (no active borrow), then returns a mutable reference.
20
+/// The caller is responsible for managing the guard's lifecycle, typically via a separate
21
+/// RAII type (like `ValueGuard`) that sets the flag and clears it on drop.
22
+///
23
+/// # Example Use Case
24
+///
25
+/// This type is used for the `WindowManager` in journal files:
26
+/// - The window manager maps/unmaps memory windows dynamically
27
+/// - Methods return slices into these windows that must outlive the borrow scope
28
+/// - Only one object slice can be active at a time (enforced by the guard)
29
+/// - A `ValueGuard` RAII type manages the guard lifecycle
30
+///
31
+/// # Safety Model
32
+///
33
+/// The safety relies on two cooperating parts:
34
+/// 1. `GuardedCell` - Owns the guard and checks it before borrowing
35
+/// 2. External RAII guard (like `ValueGuard`) - Manages the guard flag lifecycle
36
+///
37
+/// The contract is:
38
+/// - Before calling `borrow_mut_checked()`, the guard must be `false`
39
+/// - After getting a reference, the caller must set the guard to `true`
40
+/// - The guard must stay `true` while any reference derived from the borrow is live
41
+/// - When all references are done, the guard must be set back to `false`
42
+///
43
+/// # Example
44
+///
45
+/// ```ignore
46
+/// struct Container {
47
+/// window_manager: GuardedCell<WindowManager>,
48
+/// }
49
+///
50
+/// impl Container {
51
+/// fn get_slice(&self, offset: u64, size: u64) -> Result<ValueGuard<&[u8]>> {
52
+/// // Check if guard is already held
53
+/// let mut is_in_use = self.window_manager.guard().borrow_mut();
54
+/// if *is_in_use {
55
+/// return Err(Error::AlreadyBorrowed);
56
+/// }
57
+///
58
+/// // Borrow the window manager
59
+/// let wm = self.window_manager.borrow_mut_checked()?;
60
+/// let slice = wm.get_slice(offset, size)?;
61
+///
62
+/// // Set guard and return RAII guard that clears it on drop
63
+/// *is_in_use = true;
64
+/// Ok(ValueGuard::new(slice, self.window_manager.guard()))
65
+/// }
66
+/// }
67
+/// ```
68
+pub struct GuardedCell<T> {
69
+ value: UnsafeCell<T>,
70
+ guard: RefCell<bool>,
71
+}
72
+
73
+impl<T> GuardedCell<T> {
74
+ /// Creates a new `GuardedCell` containing the given value.
75
+ ///
76
+ /// The guard is initialized to `false` (not in use).
77
+ pub fn new(value: T) -> Self {
78
+ Self {
79
+ value: UnsafeCell::new(value),
80
+ guard: RefCell::new(false),
81
+ }
82
+ }
83
+
84
+ /// Returns a reference to the guard flag.
85
+ ///
86
+ /// This allows external RAII types (like `ValueGuard`) to manage the guard's
87
+ /// lifecycle by setting it to `true` when borrowing and `false` when done.
88
+ #[allow(dead_code)]
89
+ pub fn guard(&self) -> &RefCell<bool> {
90
+ &self.guard
91
+ }
92
+
93
+ /// Attempts to borrow the inner value mutably.
94
+ ///
95
+ /// # Returns
96
+ ///
97
+ /// - `Ok(&mut T)` if the guard is not currently held
98
+ /// - `Err(JournalError::ValueGuardInUse)` if the guard is held (another borrow is active)
99
+ ///
100
+ /// # Safety Contract
101
+ ///
102
+ /// After calling this method successfully, the caller MUST:
103
+ /// 1. Set the guard to `true` (via `guard().borrow_mut()`) before using the returned reference
104
+ /// 2. Keep the guard `true` for the entire lifetime of the returned reference
105
+ /// 3. Set the guard back to `false` when the reference is no longer needed
106
+ ///
107
+ /// This is typically enforced via a RAII guard type that manages the flag automatically.
108
+ ///
109
+ /// # Example
110
+ ///
111
+ /// ```ignore
112
+ /// let cell = GuardedCell::new(WindowManager::new(...));
113
+ ///
114
+ /// // Check and borrow
115
+ /// let mut is_in_use = cell.guard().borrow_mut();
116
+ /// if *is_in_use {
117
+ /// return Err(JournalError::ValueGuardInUse);
118
+ /// }
119
+ ///
120
+ /// let wm = cell.borrow_mut_checked()?;
121
+ /// let slice = wm.get_slice(offset, size)?;
122
+ ///
123
+ /// // Set guard before using the reference
124
+ /// *is_in_use = true;
125
+ ///
126
+ /// // Use slice...
127
+ /// // (guard will be cleared by RAII when done)
128
+ /// ```
129
+ #[allow(clippy::mut_from_ref)]
130
+ pub fn borrow_mut_checked(&self) -> Result<&mut T> {
131
+ // Check the guard to ensure no other borrow is active
132
+ let is_in_use = self.guard.borrow();
133
+ if *is_in_use {
134
+ return Err(JournalError::ValueGuardInUse);
135
+ }
136
+ drop(is_in_use);
137
+
138
+ // SAFETY: We've verified via the guard that no other mutable reference exists.
139
+ // The caller is responsible for:
140
+ // 1. Setting the guard to true before using the returned reference
141
+ // 2. Keeping the guard true while the reference (or data derived from it) is live
142
+ // 3. Clearing the guard when done (typically via RAII Drop)
143
+ //
144
+ // This manual lifetime management is necessary because:
145
+ // - We need references that outlive this function's scope
146
+ // - RefCell's RefMut cannot express this pattern (it must drop before returning)
147
+ // - The guard flag provides the runtime safety check for exclusive access
148
+ unsafe { Ok(&mut *self.value.get()) }
149
+ }
150
+
151
+ /// Gets a mutable reference to the inner value.
152
+ ///
153
+ /// This is safe because it requires `&mut self`, guaranteeing unique access.
154
+ /// No guard checking is needed.
155
+ pub fn get_mut(&mut self) -> &mut T {
156
+ self.value.get_mut()
157
+ }
158
+
159
+ /// Consumes the cell and returns the inner value.
160
+ #[allow(dead_code)]
161
+ pub fn into_inner(self) -> T {
162
+ self.value.into_inner()
163
+ }
164
+
165
+ /// Executes a closure with mutable access to the inner value and wraps the result in a `ValueGuard`.
166
+ ///
167
+ /// This is the primary method for working with `GuardedCell` in a safe, ergonomic way.
168
+ /// It handles all guard management automatically:
169
+ /// 1. Checks that the guard is not currently held
170
+ /// 2. Provides mutable access to the inner value via the closure
171
+ /// 3. Sets the guard flag
172
+ /// 4. Wraps the closure's result in a `ValueGuard` that will clear the flag on drop
173
+ ///
174
+ /// # Parameters
175
+ ///
176
+ /// - `offset`: Domain-specific metadata (e.g., file offset for journal objects) that will be
177
+ /// stored in the `ValueGuard` for later retrieval
178
+ /// - `f`: A closure that takes mutable access to `T` and returns a `Result<R>` where `R` is
179
+ /// the value to be wrapped in the guard
180
+ ///
181
+ /// # Returns
182
+ ///
183
+ /// - `Ok(ValueGuard<R>)` on success, which automatically manages the guard lifecycle
184
+ /// - `Err(JournalError::ValueGuardInUse)` if the guard is already held
185
+ /// - `Err(...)` if the closure returns an error
186
+ ///
187
+ /// # Example
188
+ ///
189
+ /// ```ignore
190
+ /// let window_manager = GuardedCell::new(WindowManager::new(...));
191
+ ///
192
+ /// let object_guard = window_manager.with_guarded(object_offset, |wm| {
193
+ /// // Access window manager mutably
194
+ /// let slice = wm.get_slice(offset, size)?;
195
+ /// let object = parse_object(slice)?;
196
+ /// Ok(object)
197
+ /// })?;
198
+ ///
199
+ /// // object_guard automatically clears the guard when dropped
200
+ /// ```
201
+ ///
202
+ /// # Safety
203
+ ///
204
+ /// This method encapsulates all the safety requirements:
205
+ /// - Guard checking is done automatically
206
+ /// - The guard is set before the `ValueGuard` is returned
207
+ /// - The guard is cleared automatically when `ValueGuard` is dropped
208
+ /// - No `unsafe` code is exposed to the caller
209
+ pub fn with_guarded<'a, R, F>(&'a self, offset: NonZeroU64, f: F) -> Result<ValueGuard<'a, R>>
210
+ where
211
+ F: FnOnce(&'a mut T) -> Result<R>,
212
+ {
213
+ // Check if the guard is already held
214
+ let mut is_in_use = self.guard.borrow_mut();
215
+ if *is_in_use {
216
+ return Err(JournalError::ValueGuardInUse);
217
+ }
218
+
219
+ // SAFETY: We've verified via the guard that no other mutable reference exists.
220
+ // The closure gets temporary mutable access, but we ensure the guard is set
221
+ // before returning the ValueGuard.
222
+ let value_ref = unsafe { &mut *self.value.get() };
223
+
224
+ // Execute the user's closure to extract/create the result value
225
+ let result = f(value_ref)?;
226
+
227
+ // Mark the guard as in use
228
+ *is_in_use = true;
229
+
230
+ // Return a ValueGuard that will automatically clear the guard on drop
231
+ Ok(ValueGuard::new(offset, result, &self.guard))
232
+ }
233
+}
234
+
235
+// GuardedCell is NOT Send or Sync by default (inherited from UnsafeCell).
236
+// This is correct for single-threaded use in journal file reading.
237
+
238
+#[cfg(test)]
239
+mod tests {
240
+ use super::*;
241
+
242
+ struct TestData {
243
+ value: Vec<u8>,
244
+ }
245
+
246
+ impl TestData {
247
+ fn get_slice(&mut self, start: usize, len: usize) -> &[u8] {
248
+ &self.value[start..start + len]
249
+ }
250
+ }
251
+
252
+ #[test]
253
+ fn test_basic_borrow() {
254
+ let cell = GuardedCell::new(TestData {
255
+ value: vec![1, 2, 3, 4, 5],
256
+ });
257
+
258
+ // First borrow
259
+ {
260
+ // Check guard is not in use
261
+ {
262
+ let is_in_use = cell.guard().borrow();
263
+ assert!(!*is_in_use);
264
+ }
265
+
266
+ // Borrow the data
267
+ let data = cell.borrow_mut_checked().unwrap();
268
+ let slice = data.get_slice(0, 3);
269
+ assert_eq!(slice, &[1, 2, 3]);
270
+
271
+ // Mark as in use
272
+ *cell.guard().borrow_mut() = true;
273
+ }
274
+
275
+ // Clear guard
276
+ *cell.guard().borrow_mut() = false;
277
+
278
+ // Second borrow after clearing
279
+ {
280
+ // Check guard is not in use
281
+ {
282
+ let is_in_use = cell.guard().borrow();
283
+ assert!(!*is_in_use);
284
+ }
285
+
286
+ // Borrow the data
287
+ let data = cell.borrow_mut_checked().unwrap();
288
+ let slice = data.get_slice(2, 3);
289
+ assert_eq!(slice, &[3, 4, 5]);
290
+
291
+ // Mark as in use
292
+ *cell.guard().borrow_mut() = true;
293
+ }
294
+ }
295
+
296
+ #[test]
297
+ fn test_guard_prevents_double_borrow() {
298
+ let cell = GuardedCell::new(TestData {
299
+ value: vec![1, 2, 3],
300
+ });
301
+
302
+ // Set guard to true (simulating active borrow)
303
+ *cell.guard().borrow_mut() = true;
304
+
305
+ // Attempt to borrow while guard is held should fail
306
+ let result = cell.borrow_mut_checked();
307
+ assert!(matches!(result, Err(JournalError::ValueGuardInUse)));
308
+ }
309
+
310
+ #[test]
311
+ fn test_get_mut() {
312
+ let mut cell = GuardedCell::new(TestData {
313
+ value: vec![1, 2, 3],
314
+ });
315
+
316
+ let data = cell.get_mut();
317
+ data.value.push(4);
318
+ assert_eq!(data.value, vec![1, 2, 3, 4]);
319
+ }
320
+
321
+ #[test]
322
+ fn test_into_inner() {
323
+ let cell = GuardedCell::new(TestData {
324
+ value: vec![1, 2, 3],
325
+ });
326
+
327
+ let data = cell.into_inner();
328
+ assert_eq!(data.value, vec![1, 2, 3]);
329
+ }
330
+}
src/crates/journal-core/src/file/hash.rs
renamed
+13
-5
@@ -1,14 +1,22 @@
1
use siphasher::sip::SipHasher24;
2
use std::hash::Hasher;
3
4
-fn jenkins_hash64(data: &[u8]) -> u64 {
5
- // FIXME: user real jenkins hasher
6
- let mut hasher = twox_hash::XxHash64::default();
4
+pub fn jenkins_hash64(data: &[u8]) -> u64 {
5
+ use hashers::jenkins::Lookup3Hasher;
6
+
7
+ let mut hasher = Lookup3Hasher::default();
8
hasher.write(data);
8
- hasher.finish()
9
+ let hash = hasher.finish();
10
+
11
+ // Jenkins lookup3 returns two 32-bit values (pc, pb)
12
+ // systemd expects: first 32-bit as high part, second 32-bit as low part
13
+ // But Lookup3Hasher::finish() returns them in opposite order, so swap them
14
+ let low = (hash & 0xFFFFFFFF) as u32;
15
+ let high = (hash >> 32) as u32;
16
+ ((low as u64) << 32) | (high as u64)
17
}
18
11
-fn siphash24(data: &[u8], key: &[u8; 16]) -> u64 {
19
+pub fn siphash24(data: &[u8], key: &[u8; 16]) -> u64 {
20
let k0 = u64::from_le_bytes(key[0..8].try_into().unwrap());
21
let k1 = u64::from_le_bytes(key[8..16].try_into().unwrap());
22
src/crates/journal-core/src/file/index_filter.rs
new
+369
@@ -0,0 +1,369 @@
1
+use super::index::FileIndex;
2
+use roaring::RoaringBitmap;
3
+
4
+#[derive(Clone, Debug)]
5
+pub enum IndexFilterExpr {
6
+ None,
7
+ Match(RoaringBitmap),
8
+ Conjunction(Vec<IndexFilterExpr>),
9
+ Disjunction(Vec<IndexFilterExpr>),
10
+}
11
+
12
+impl IndexFilterExpr {
13
+ /// Get all entry indices that match this filter expression
14
+ pub fn matching_indices(&self) -> RoaringBitmap {
15
+ match self {
16
+ IndexFilterExpr::None => RoaringBitmap::new(),
17
+ IndexFilterExpr::Match(bitmap) => bitmap.clone(),
18
+ IndexFilterExpr::Conjunction(filter_exprs) => {
19
+ if filter_exprs.is_empty() {
20
+ return RoaringBitmap::new();
21
+ }
22
+
23
+ let mut result = filter_exprs[0].matching_indices();
24
+ for expr in filter_exprs.iter().skip(1) {
25
+ result &= expr.matching_indices();
26
+ if result.is_empty() {
27
+ break; // Early termination for empty conjunction
28
+ }
29
+ }
30
+ result
31
+ }
32
+ IndexFilterExpr::Disjunction(filter_exprs) => {
33
+ let mut result = RoaringBitmap::new();
34
+ for expr in filter_exprs.iter() {
35
+ result |= expr.matching_indices();
36
+ }
37
+ result
38
+ }
39
+ }
40
+ }
41
+
42
+ /// Count the number of matching entries
43
+ pub fn count(&self) -> u64 {
44
+ self.matching_indices().len()
45
+ }
46
+
47
+ /// Check if there are any matching entries
48
+ pub fn has_matches(&self) -> bool {
49
+ match self {
50
+ IndexFilterExpr::None => false,
51
+ IndexFilterExpr::Match(bitmap) => !bitmap.is_empty(),
52
+ IndexFilterExpr::Conjunction(filter_exprs) => {
53
+ filter_exprs.iter().all(|expr| expr.has_matches())
54
+ }
55
+ IndexFilterExpr::Disjunction(filter_exprs) => {
56
+ filter_exprs.iter().any(|expr| expr.has_matches())
57
+ }
58
+ }
59
+ }
60
+
61
+ /// Get matching indices within a specific range
62
+ pub fn matching_indices_in_range(&self, start: u32, end: u32) -> RoaringBitmap {
63
+ let mut result = self.matching_indices();
64
+ result.remove_range(..start);
65
+ result.remove_range((end + 1)..);
66
+ result
67
+ }
68
+
69
+ /// Get matching indices within specific histogram bucket
70
+ pub fn matching_indices_in_bucket(
71
+ &self,
72
+ file_index: &FileIndex,
73
+ bucket_index: usize,
74
+ ) -> Option<RoaringBitmap> {
75
+ let (start, end) = file_index.file_histogram.get_entry_range(bucket_index)?;
76
+ Some(self.matching_indices_in_range(start, end))
77
+ }
78
+}
79
+
80
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
81
+pub enum LogicalOp {
82
+ Conjunction,
83
+ Disjunction,
84
+}
85
+
86
+#[derive(Debug)]
87
+pub struct IndexFilter {
88
+ filter_expr: Option<IndexFilterExpr>,
89
+ current_matches: Vec<String>,
90
+ current_op: LogicalOp,
91
+}
92
+
93
+impl Default for IndexFilter {
94
+ fn default() -> Self {
95
+ Self {
96
+ filter_expr: None,
97
+ current_matches: Vec::new(),
98
+ current_op: LogicalOp::Conjunction,
99
+ }
100
+ }
101
+}
102
+
103
+impl IndexFilter {
104
+ /// Create a new empty filter
105
+ pub fn new() -> Self {
106
+ Self::default()
107
+ }
108
+
109
+ /// Extract the field key from a field=value pair
110
+ fn extract_key(field_value: &str) -> Option<&str> {
111
+ field_value.split('=').next()
112
+ }
113
+
114
+ /// Convert current matches to a filter expression
115
+ fn convert_current_matches(&mut self, file_index: &FileIndex) -> Option<IndexFilterExpr> {
116
+ if self.current_matches.is_empty() {
117
+ return None;
118
+ }
119
+
120
+ let mut elements = Vec::new();
121
+ let mut i = 0;
122
+
123
+ // Sort current matches by key for grouping
124
+ self.current_matches.sort_by(|a, b| {
125
+ let key_a = Self::extract_key(a).unwrap_or("");
126
+ let key_b = Self::extract_key(b).unwrap_or("");
127
+ key_a.cmp(key_b)
128
+ });
129
+
130
+ while i < self.current_matches.len() {
131
+ let current_key = Self::extract_key(&self.current_matches[i]).unwrap_or("");
132
+ let start = i;
133
+
134
+ // Find all matches with the same key
135
+ while i < self.current_matches.len()
136
+ && Self::extract_key(&self.current_matches[i]).unwrap_or("") == current_key
137
+ {
138
+ i += 1;
139
+ }
140
+
141
+ // If we have multiple values for this key, create a disjunction
142
+ if i - start > 1 {
143
+ let mut matches = Vec::with_capacity(i - start);
144
+ for idx in start..i {
145
+ let field_value = &self.current_matches[idx];
146
+ if let Some(bitmap) = file_index.entries_index.get(field_value) {
147
+ matches.push(IndexFilterExpr::Match(bitmap.clone()));
148
+ } else {
149
+ matches.push(IndexFilterExpr::None);
150
+ }
151
+ }
152
+ elements.push(IndexFilterExpr::Disjunction(matches));
153
+ } else {
154
+ let field_value = &self.current_matches[start];
155
+ if let Some(bitmap) = file_index.entries_index.get(field_value) {
156
+ elements.push(IndexFilterExpr::Match(bitmap.clone()));
157
+ } else {
158
+ elements.push(IndexFilterExpr::None);
159
+ }
160
+ }
161
+ }
162
+
163
+ self.current_matches.clear();
164
+
165
+ match elements.len() {
166
+ 0 => None,
167
+ 1 => Some(elements.into_iter().next().unwrap()),
168
+ _ => Some(IndexFilterExpr::Conjunction(elements)),
169
+ }
170
+ }
171
+
172
+ /// Add a field=value match to the current filter being built
173
+ ///
174
+ /// # Examples
175
+ /// ```
176
+ /// filter.add_match("_SYSTEMD_UNIT=ssh.service");
177
+ /// filter.add_match("PRIORITY=6");
178
+ /// ```
179
+ pub fn add_match(&mut self, field_value: &str) {
180
+ if field_value.contains('=') {
181
+ // Insert in sorted order to group by field name
182
+ let key = Self::extract_key(field_value).unwrap_or("");
183
+ let pos = self
184
+ .current_matches
185
+ .binary_search_by(|item| {
186
+ let item_key = Self::extract_key(item).unwrap_or("");
187
+ item_key.cmp(key)
188
+ })
189
+ .unwrap_or_else(|e| e);
190
+
191
+ self.current_matches.insert(pos, field_value.to_string());
192
+ }
193
+ }
194
+
195
+ /// Set the logical operation for combining the current matches with the existing filter
196
+ pub fn set_operation(&mut self, file_index: &FileIndex, op: LogicalOp) {
197
+ let new_expr = self.convert_current_matches(file_index);
198
+ if new_expr.is_none() {
199
+ self.current_op = op;
200
+ return;
201
+ }
202
+
203
+ if self.filter_expr.is_none() {
204
+ self.filter_expr = new_expr;
205
+ self.current_op = op;
206
+ return;
207
+ }
208
+
209
+ let new_expr = new_expr.unwrap();
210
+ let current_expr = self.filter_expr.take().unwrap();
211
+
212
+ self.filter_expr = Some(match (current_expr, self.current_op) {
213
+ (IndexFilterExpr::Disjunction(mut exprs), LogicalOp::Disjunction) => {
214
+ exprs.push(new_expr);
215
+ IndexFilterExpr::Disjunction(exprs)
216
+ }
217
+ (IndexFilterExpr::Conjunction(mut exprs), LogicalOp::Conjunction) => {
218
+ exprs.push(new_expr);
219
+ IndexFilterExpr::Conjunction(exprs)
220
+ }
221
+ (current_expr, LogicalOp::Disjunction) => {
222
+ IndexFilterExpr::Disjunction(vec![current_expr, new_expr])
223
+ }
224
+ (current_expr, LogicalOp::Conjunction) => {
225
+ IndexFilterExpr::Conjunction(vec![current_expr, new_expr])
226
+ }
227
+ });
228
+
229
+ self.current_op = op;
230
+ }
231
+
232
+ /// Add conjunction (AND) operation
233
+ pub fn add_conjunction(&mut self, file_index: &FileIndex) {
234
+ self.set_operation(file_index, LogicalOp::Conjunction);
235
+ }
236
+
237
+ /// Add disjunction (OR) operation
238
+ pub fn add_disjunction(&mut self, file_index: &FileIndex) {
239
+ self.set_operation(file_index, LogicalOp::Disjunction);
240
+ }
241
+
242
+ /// Build the final filter expression
243
+ pub fn build(&mut self, file_index: &FileIndex) -> IndexFilterExpr {
244
+ self.set_operation(file_index, self.current_op);
245
+
246
+ self.current_matches.clear();
247
+ self.current_op = LogicalOp::Conjunction;
248
+ self.filter_expr.take().unwrap_or(IndexFilterExpr::None)
249
+ }
250
+
251
+ /// Convenience method to create a simple match filter
252
+ pub fn simple_match(file_index: &FileIndex, field_value: &str) -> IndexFilterExpr {
253
+ if let Some(bitmap) = file_index.entries_index.get(field_value) {
254
+ IndexFilterExpr::Match(bitmap.clone())
255
+ } else {
256
+ IndexFilterExpr::None
257
+ }
258
+ }
259
+
260
+ /// Convenience method to create a conjunction of multiple field=value pairs
261
+ pub fn conjunction(file_index: &FileIndex, field_values: &[&str]) -> IndexFilterExpr {
262
+ let mut filter = IndexFilter::new();
263
+ for field_value in field_values {
264
+ filter.add_match(field_value);
265
+ }
266
+ filter.build(file_index)
267
+ }
268
+
269
+ /// Convenience method to create a disjunction of multiple field=value pairs
270
+ pub fn disjunction(file_index: &FileIndex, field_values: &[&str]) -> IndexFilterExpr {
271
+ let matches: Vec<_> = field_values
272
+ .iter()
273
+ .map(|fv| Self::simple_match(file_index, fv))
274
+ .collect();
275
+
276
+ if matches.is_empty() {
277
+ IndexFilterExpr::None
278
+ } else if matches.len() == 1 {
279
+ matches.into_iter().next().unwrap()
280
+ } else {
281
+ IndexFilterExpr::Disjunction(matches)
282
+ }
283
+ }
284
+}
285
+
286
+#[cfg(test)]
287
+mod tests {
288
+ use super::*;
289
+ use crate::index::{FileHistogram, FileIndex};
290
+
291
+ fn create_test_file_index() -> FileIndex {
292
+ let mut entry_indices = FxHashMap::default();
293
+
294
+ // Add some test data
295
+ entry_indices.insert(
296
+ "_SYSTEMD_UNIT=ssh.service".to_string(),
297
+ RoaringBitmap::from_sorted_iter([1, 3, 5, 7]).unwrap(),
298
+ );
299
+ entry_indices.insert(
300
+ "_SYSTEMD_UNIT=nginx.service".to_string(),
301
+ RoaringBitmap::from_sorted_iter([2, 4, 6, 8]).unwrap(),
302
+ );
303
+ entry_indices.insert(
304
+ "PRIORITY=6".to_string(),
305
+ RoaringBitmap::from_sorted_iter([1, 2, 9, 10]).unwrap(),
306
+ );
307
+ entry_indices.insert(
308
+ "PRIORITY=3".to_string(),
309
+ RoaringBitmap::from_sorted_iter([3, 4, 5]).unwrap(),
310
+ );
311
+
312
+ FileIndex {
313
+ file_histogram: FileHistogram::default(),
314
+ entries_index: entry_indices,
315
+ }
316
+ }
317
+
318
+ #[test]
319
+ fn test_simple_match() {
320
+ let file_index = create_test_file_index();
321
+ let filter = IndexFilter::simple_match(&file_index, "_SYSTEMD_UNIT=ssh.service");
322
+
323
+ let matches = filter.matching_indices();
324
+ assert_eq!(matches.iter().collect::<Vec<_>>(), vec![1, 3, 5, 7]);
325
+ }
326
+
327
+ #[test]
328
+ fn test_conjunction() {
329
+ let file_index = create_test_file_index();
330
+ let filter =
331
+ IndexFilter::conjunction(&file_index, &["_SYSTEMD_UNIT=ssh.service", "PRIORITY=6"]);
332
+
333
+ let matches = filter.matching_indices();
334
+ assert_eq!(matches.iter().collect::<Vec<_>>(), vec![1]); // Only entry 1 matches both
335
+ }
336
+
337
+ #[test]
338
+ fn test_disjunction() {
339
+ let file_index = create_test_file_index();
340
+ let filter = IndexFilter::disjunction(
341
+ &file_index,
342
+ &["_SYSTEMD_UNIT=ssh.service", "_SYSTEMD_UNIT=nginx.service"],
343
+ );
344
+
345
+ let matches = filter.matching_indices();
346
+ assert_eq!(
347
+ matches.iter().collect::<Vec<_>>(),
348
+ vec![1, 2, 3, 4, 5, 6, 7, 8]
349
+ );
350
+ }
351
+
352
+ #[test]
353
+ fn test_complex_filter() {
354
+ let file_index = create_test_file_index();
355
+ let mut filter = IndexFilter::new();
356
+
357
+ // Add matches for same key (will be OR'd)
358
+ filter.add_match("_SYSTEMD_UNIT=ssh.service");
359
+ filter.add_match("_SYSTEMD_UNIT=nginx.service");
360
+ filter.add_conjunction(&file_index);
361
+
362
+ // Add another condition (will be AND'd with above)
363
+ filter.add_match("PRIORITY=6");
364
+
365
+ let result = filter.build(&file_index);
366
+ let matches = result.matching_indices();
367
+ assert_eq!(matches.iter().collect::<Vec<_>>(), vec![1, 2]); // Entries that match PRIORITY=6 AND (ssh OR nginx)
368
+ }
369
+}
src/crates/journal-core/src/file/mmap.rs
renamed
+22
-5
@@ -1,8 +1,11 @@
1
-use error::Result;
2
-use memmap2::{Mmap, MmapMut, MmapOptions};
1
+use crate::error::Result;
2
+use journal_common::compat::is_multiple_of;
3
use std::fs::File;
4
use std::ops::{Deref, DerefMut};
5
6
+// Re-export memmap2 types for other crates and import for internal use
7
+pub use memmap2::{Mmap, MmapMut, MmapOptions};
8
+
9
const PAGE_SIZE: u64 = 4096;
10
11
pub trait MemoryMap: Deref<Target = [u8]> {
@@ -11,7 +14,10 @@ pub trait MemoryMap: Deref<Target = [u8]> {
14
Self: Sized;
15
}
16
14
-pub trait MemoryMapMut: MemoryMap + DerefMut {}
17
+pub trait MemoryMapMut: MemoryMap + DerefMut {
18
+ /// Flushes outstanding memory map modifications to disk
19
+ fn flush(&self) -> Result<()>;
20
+}
21
22
impl MemoryMap for Mmap {
23
fn create(file: &File, offset: u64, size: u64) -> Result<Self> {
@@ -45,7 +51,12 @@ impl MemoryMap for MmapMut {
51
}
52
}
53
48
-impl MemoryMapMut for MmapMut {}
54
+impl MemoryMapMut for MmapMut {
55
+ fn flush(&self) -> Result<()> {
56
+ MmapMut::flush(self)?;
57
+ Ok(())
58
+ }
59
+}
60
61
struct Window<M: MemoryMap> {
62
offset: u64,
@@ -103,7 +114,7 @@ pub struct WindowManager<M: MemoryMap> {
114
115
impl<M: MemoryMap> WindowManager<M> {
116
pub fn new(file: File, chunk_size: u64, max_windows: usize) -> Result<Self> {
106
- debug_assert!(chunk_size != 0 && (chunk_size % PAGE_SIZE) == 0);
117
+ debug_assert!(chunk_size != 0 && is_multiple_of(chunk_size, PAGE_SIZE));
118
debug_assert!(max_windows != 0);
119
120
let _file_size = file.metadata()?.len();
@@ -232,4 +243,10 @@ impl<M: MemoryMapMut> WindowManager<M> {
243
let window = self.get_window(position, size)?;
244
Ok(window.get_mut_slice(position, size))
245
}
246
+
247
+ /// Syncs all file data to disk
248
+ pub fn sync(&self) -> Result<()> {
249
+ self.file.sync_data()?;
250
+ Ok(())
251
+ }
252
}
src/crates/journal-core/src/file/mod.rs
renamed
+13
-5
@@ -2,15 +2,18 @@
2
pub mod cursor;
3
pub mod file;
4
pub mod filter;
5
-mod hash;
5
+mod guarded_cell;
6
+pub mod hash;
7
+pub mod mmap;
8
mod object;
9
pub mod offset_array;
10
pub mod reader;
11
+pub(crate) mod sigbus;
12
mod value_guard;
13
pub mod writer;
14
15
// Core functionality
13
-pub use file::{load_boot_id, BucketUtilization, JournalFile, JournalFileOptions};
16
+pub use file::{BucketUtilization, JournalFile, JournalFileOptions};
17
pub use reader::JournalReader;
18
pub use writer::JournalWriter;
19
@@ -23,13 +26,18 @@ pub use cursor::JournalCursor;
26
pub use filter::{FilterExpr, JournalFilter, LogicalOp};
27
28
// For FFI compatibility and advanced object manipulation
26
-pub use object::HashableObject;
29
+pub use object::{EntryItemsType, HashableObject, JournalState, HeaderIncompatibleFlags};
30
31
// Re-export commonly needed external types
29
-pub use memmap2::{Mmap, MmapMut};
32
+pub use mmap::{Mmap, MmapMut};
33
34
// Internal utilities that might be needed
32
-pub use crate::hash::journal_hash_data;
35
+pub use crate::file::hash::journal_hash_data;
36
37
// Internal re-exports needed by the crate itself (not part of public API)
38
pub(crate) use object::*;
39
+
40
+// Re-export DataObject for journal-index
41
+pub use object::DataObject;
42
+
43
+pub type JournalFileMap = JournalFile<Mmap>;
src/crates/journal-core/src/file/object.rs
renamed
+67
-2
@@ -1,5 +1,5 @@
1
-use crate::offset_array::{Cursor, InlinedCursor, List};
2
-use error::{JournalError, Result};
1
+use crate::error::{JournalError, Result};
2
+use crate::file::offset_array::{Cursor, InlinedCursor, List};
3
use std::num::{NonZeroU32, NonZeroU64, NonZeroUsize};
4
use zerocopy::{
5
ByteSlice, ByteSliceMut, FromBytes, Immutable, IntoBytes, KnownLayout, Ref, SplitByteSlice,
@@ -260,6 +260,16 @@ impl TryFrom<u8> for JournalState {
260
}
261
}
262
263
+impl std::fmt::Display for JournalState {
264
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
265
+ match self {
266
+ JournalState::Offline => write!(f, "OFFLINE"),
267
+ JournalState::Online => write!(f, "ONLINE"),
268
+ JournalState::Archived => write!(f, "ARCHIVED"),
269
+ }
270
+ }
271
+}
272
+
273
#[derive(Default, Debug, Clone, Copy, FromBytes, IntoBytes, Immutable, KnownLayout)]
274
#[repr(C)]
275
pub struct JournalHeader {
@@ -489,6 +499,34 @@ impl<B: ByteSlice> OffsetArrayObject<B> {
499
500
Ok(self.items.get(index))
501
}
502
+
503
+ pub fn collect_offsets(
504
+ &self,
505
+ start_index: usize,
506
+ remaining_items: usize,
507
+ offsets: &mut Vec<NonZeroU64>,
508
+ ) -> Result<()> {
509
+ let len = self.len(remaining_items);
510
+
511
+ if start_index >= len {
512
+ return Err(JournalError::InvalidOffsetArrayIndex);
513
+ }
514
+
515
+ match &self.items {
516
+ OffsetsType::Regular(s) => {
517
+ offsets.extend(s[start_index..len].iter().filter_map(|&opt| opt));
518
+ }
519
+ OffsetsType::Compact(s) => {
520
+ offsets.extend(
521
+ s[start_index..len]
522
+ .iter()
523
+ .filter_map(|&opt| opt.map(NonZeroU64::from)),
524
+ );
525
+ }
526
+ }
527
+
528
+ Ok(())
529
+ }
530
}
531
532
impl<B: ByteSliceMut> OffsetArrayObject<B> {
@@ -640,6 +678,33 @@ pub struct EntryObject<B: ByteSlice> {
678
pub items: EntryItemsType<B>,
679
}
680
681
+impl<B: ByteSlice> EntryObject<B> {
682
+ pub fn collect_offsets(&self, offsets: &mut Vec<NonZeroU64>) -> Result<()> {
683
+ match &self.items {
684
+ EntryItemsType::Regular(items) => {
685
+ offsets.reserve(items.len());
686
+
687
+ for item in items.iter() {
688
+ let offset =
689
+ NonZeroU64::new(item.object_offset).ok_or(JournalError::InvalidOffset)?;
690
+ offsets.push(offset);
691
+ }
692
+ }
693
+ EntryItemsType::Compact(items) => {
694
+ offsets.reserve(items.len());
695
+
696
+ for item in items.iter() {
697
+ let offset = NonZeroU64::new(item.object_offset as u64)
698
+ .ok_or(JournalError::InvalidOffset)?;
699
+ offsets.push(offset);
700
+ }
701
+ }
702
+ }
703
+
704
+ Ok(())
705
+ }
706
+}
707
+
708
impl<B: ByteSlice> std::fmt::Debug for EntryObject<B> {
709
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
710
f.debug_struct("EntryObject")
src/crates/journal-core/src/file/offset_array.rs
renamed
+77
-2
@@ -1,7 +1,7 @@
1
+use super::mmap::MemoryMap;
2
+use crate::error::{JournalError, Result};
3
use crate::file::JournalFile;
2
-use error::{JournalError, Result};
4
use std::num::{NonZeroU64, NonZeroUsize};
4
-use window_manager::MemoryMap;
5
6
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7
pub enum Direction {
@@ -172,6 +172,7 @@ impl std::fmt::Debug for Node {
172
173
/// A linked list of offset arrays
174
#[derive(Copy, Clone)]
175
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
176
pub struct List {
177
head_offset: NonZeroU64,
178
total_items: NonZeroUsize,
@@ -294,10 +295,37 @@ impl List {
295
// No match found in any array
296
Ok(None)
297
}
298
+
299
+ /// Collect all offsets in the entire list into the given vector
300
+ pub fn collect_offsets<M: MemoryMap>(
301
+ &self,
302
+ journal_file: &JournalFile<M>,
303
+ offsets: &mut Vec<NonZeroU64>,
304
+ ) -> Result<()> {
305
+ offsets.reserve(self.total_items.get());
306
+
307
+ let mut node = self.head(journal_file)?;
308
+
309
+ loop {
310
+ {
311
+ let array = journal_file.offset_array_ref(node.offset())?;
312
+ let remaining_items = node.remaining_items.get();
313
+ array.collect_offsets(0, remaining_items, offsets)?;
314
+ }
315
+
316
+ match node.next(journal_file)? {
317
+ Some(next) => node = next,
318
+ None => break,
319
+ }
320
+ }
321
+
322
+ Ok(())
323
+ }
324
}
325
326
/// A cursor pointing to a specific position within an offset array chain
327
#[derive(Clone, Copy)]
328
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
329
pub struct Cursor {
330
list: List,
331
array_offset: NonZeroU64,
@@ -439,6 +467,31 @@ impl Cursor {
467
468
Err(JournalError::InvalidOffsetArrayOffset)
469
}
470
+
471
+ pub fn collect_offsets<M: MemoryMap>(
472
+ &self,
473
+ journal_file: &JournalFile<M>,
474
+ offsets: &mut Vec<NonZeroU64>,
475
+ ) -> Result<()> {
476
+ let mut node = self.node(journal_file)?;
477
+
478
+ // Copy from position in the current array
479
+ {
480
+ let array = journal_file.offset_array_ref(node.offset())?;
481
+ let remaining_items = node.remaining_items.get();
482
+ array.collect_offsets(self.array_index, remaining_items, offsets)?;
483
+ }
484
+
485
+ // Copy from subsequent arrays
486
+ while let Some(next_node) = node.next(journal_file)? {
487
+ let array = journal_file.offset_array_ref(next_node.offset())?;
488
+ let remaining_items = node.remaining_items.get();
489
+ array.collect_offsets(0, remaining_items, offsets)?;
490
+ node = next_node;
491
+ }
492
+
493
+ Ok(())
494
+ }
495
}
496
497
impl std::fmt::Debug for Cursor {
@@ -452,6 +505,7 @@ impl std::fmt::Debug for Cursor {
505
}
506
507
#[derive(Debug, Copy, Clone)]
508
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
509
pub struct InlinedCursor {
510
inlined_offset: NonZeroU64,
511
cursor: Option<Cursor>,
@@ -683,4 +737,25 @@ impl InlinedCursor {
737
738
Ok(best_match)
739
}
740
+
741
+ pub fn collect_offsets<M: MemoryMap>(
742
+ &self,
743
+ journal_file: &JournalFile<M>,
744
+ offsets: &mut Vec<NonZeroU64>,
745
+ ) -> Result<()> {
746
+ // Handle the inline offset first if we're at it
747
+ if self.at_inlined_offset {
748
+ offsets.push(self.inlined_offset);
749
+
750
+ // If we have a cursor, collect all offsets from the beginning
751
+ if let Some(cursor) = self.cursor {
752
+ cursor.list.collect_offsets(journal_file, offsets)?;
753
+ }
754
+ } else if let Some(cursor) = self.cursor {
755
+ // We're somewhere in the array chain, collect from current position
756
+ cursor.collect_offsets(journal_file, offsets)?;
757
+ }
758
+
759
+ Ok(())
760
+ }
761
}
src/crates/journal-core/src/file/reader.rs
new
+466
@@ -0,0 +1,466 @@
1
+use super::mmap::MemoryMap;
2
+use crate::error::Result;
3
+use crate::field_map::{FieldMap, REMAPPING_MARKER, extract_field_name};
4
+use crate::file::{
5
+ EntryItemsType,
6
+ cursor::{JournalCursor, Location},
7
+ file::{EntryDataIterator, FieldDataIterator, FieldIterator, JournalFile},
8
+ filter::{JournalFilter, LogicalOp},
9
+ object::{DataObject, FieldObject, HashableObject},
10
+ offset_array::Direction,
11
+ value_guard::ValueGuard,
12
+};
13
+use std::num::NonZeroU64;
14
+
15
+pub struct JournalReader<'a, M: MemoryMap> {
16
+ cursor: JournalCursor,
17
+
18
+ filter: Option<JournalFilter>,
19
+ field_iterator: Option<FieldIterator<'a, M>>,
20
+ field_data_iterator: Option<FieldDataIterator<'a, M>>,
21
+ entry_data_iterator: Option<EntryDataIterator<'a, M>>,
22
+
23
+ field_guard: Option<ValueGuard<'a, FieldObject<&'a [u8]>>>,
24
+ data_guard: Option<ValueGuard<'a, DataObject<&'a [u8]>>>,
25
+
26
+ // Field name remapping support
27
+ remapping_registry: FieldMap,
28
+ translated_payload: Vec<u8>,
29
+}
30
+
31
+impl<M: MemoryMap> std::fmt::Debug for JournalReader<'_, M> {
32
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
33
+ f.debug_struct("JournalReader")
34
+ // .field("cursor", &self.cursor)
35
+ .field("field_guard", &self.field_guard)
36
+ .field("data_guard", &self.data_guard)
37
+ .finish()
38
+ }
39
+}
40
+
41
+impl<M: MemoryMap> Default for JournalReader<'_, M> {
42
+ fn default() -> Self {
43
+ Self {
44
+ cursor: JournalCursor::new(),
45
+ filter: None,
46
+ field_iterator: None,
47
+ field_data_iterator: None,
48
+ entry_data_iterator: None,
49
+ field_guard: None,
50
+ data_guard: None,
51
+ remapping_registry: FieldMap::new(),
52
+ translated_payload: Vec::new(),
53
+ }
54
+ }
55
+}
56
+
57
+impl<'a, M: MemoryMap> JournalReader<'a, M> {
58
+ pub fn dump(&self, _journal_file: &'a JournalFile<M>) -> Result<String> {
59
+ if let Some(_filter_expr) = self.cursor.filter_expr.as_ref() {
60
+ todo!();
61
+ } else {
62
+ Ok(String::from("no filter expr"))
63
+ }
64
+ }
65
+
66
+ pub fn set_location(&mut self, location: Location) {
67
+ self.cursor.set_location(location)
68
+ }
69
+
70
+ pub fn step(&mut self, journal_file: &'a JournalFile<M>, direction: Direction) -> Result<bool> {
71
+ self.drop_guards();
72
+
73
+ if let Some(filter) = self.filter.as_mut() {
74
+ let filter_expr = filter.build(journal_file)?;
75
+ self.cursor.set_filter(filter_expr);
76
+ self.filter = None;
77
+ }
78
+
79
+ self.cursor.step(journal_file, direction)
80
+ }
81
+
82
+ /// Adds a match filter for the given field=value pair.
83
+ ///
84
+ /// If the field name is an original (otel) name that has been remapped,
85
+ /// this automatically translates it to the systemd-compatible name before
86
+ /// applying the filter.
87
+ ///
88
+ /// # Example
89
+ ///
90
+ /// ```no_run
91
+ /// # use journal_core::{JournalReader, JournalFile};
92
+ /// # use journal_core::file::Mmap;
93
+ /// # fn example(reader: &mut JournalReader<Mmap>, file: &JournalFile<Mmap>) {
94
+ /// // Even if "my.field.name" was remapped to "ND_ABC123...", this works:
95
+ /// reader.add_match(b"my.field.name=some_value");
96
+ /// # }
97
+ /// ```
98
+ pub fn add_match(&mut self, data: &[u8]) {
99
+ // Check if the field name needs translation
100
+ if let Some(field_name) = extract_field_name(data) {
101
+ if let Some(systemd_name) = self.remapping_registry.get_systemd_name(field_name) {
102
+ // Field has been remapped - translate the query
103
+ let eq_pos = data.iter().position(|&b| b == b'=').unwrap();
104
+ let value = &data[eq_pos..]; // includes '='
105
+
106
+ let mut translated_query = Vec::with_capacity(systemd_name.len() + value.len());
107
+ translated_query.extend_from_slice(systemd_name.as_bytes());
108
+ translated_query.extend_from_slice(value);
109
+
110
+ self.filter
111
+ .get_or_insert_default()
112
+ .add_match(&translated_query);
113
+ return;
114
+ }
115
+ }
116
+
117
+ // No translation needed - use original
118
+ self.filter.get_or_insert_default().add_match(data);
119
+ }
120
+
121
+ pub fn add_conjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
122
+ self.filter
123
+ .get_or_insert_default()
124
+ .set_operation(journal_file, LogicalOp::Conjunction)
125
+ }
126
+
127
+ pub fn add_disjunction(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
128
+ self.filter
129
+ .get_or_insert_default()
130
+ .set_operation(journal_file, LogicalOp::Disjunction)
131
+ }
132
+
133
+ pub fn flush_matches(&mut self) {
134
+ self.cursor.clear_filter();
135
+ self.filter = None;
136
+ }
137
+
138
+ pub fn get_realtime_usec(&self, journal_file: &'a JournalFile<M>) -> Result<u64> {
139
+ let entry_offset = self.cursor.position()?;
140
+ let entry_object = journal_file.entry_ref(entry_offset)?;
141
+ Ok(entry_object.header.realtime)
142
+ }
143
+
144
+ pub fn get_seqnum(&self, journal_file: &'a JournalFile<M>) -> Result<(u64, [u8; 16])> {
145
+ let entry_offset = self.cursor.position()?;
146
+ let entry_object = journal_file.entry_ref(entry_offset)?;
147
+ Ok((
148
+ entry_object.header.seqnum,
149
+ journal_file.journal_header_ref().seqnum_id,
150
+ ))
151
+ }
152
+
153
+ pub fn get_entry_offset(&self) -> Result<NonZeroU64> {
154
+ self.cursor.position()
155
+ }
156
+
157
+ fn drop_guards(&mut self) {
158
+ self.field_guard.take();
159
+ self.data_guard.take();
160
+ }
161
+
162
+ pub fn fields_restart(&mut self) {
163
+ self.drop_guards();
164
+ self.field_iterator = None;
165
+ }
166
+
167
+ pub fn fields_enumerate(
168
+ &mut self,
169
+ journal_file: &'a JournalFile<M>,
170
+ ) -> Result<Option<&ValueGuard<'_, FieldObject<&'a [u8]>>>> {
171
+ self.drop_guards();
172
+
173
+ if self.field_iterator.is_none() {
174
+ self.field_iterator = Some(journal_file.fields());
175
+ }
176
+
177
+ if let Some(iter) = &mut self.field_iterator {
178
+ self.field_guard = iter.next().transpose()?;
179
+ Ok(self.field_guard.as_ref())
180
+ } else {
181
+ Ok(None)
182
+ }
183
+ }
184
+
185
+ pub fn field_data_query_unique(
186
+ &mut self,
187
+ journal_file: &'a JournalFile<M>,
188
+ field_name: &'a [u8],
189
+ ) -> Result<()> {
190
+ self.drop_guards();
191
+
192
+ self.field_data_iterator = Some(journal_file.field_data_objects(field_name)?);
193
+ Ok(())
194
+ }
195
+
196
+ pub fn field_data_restart(&mut self) {
197
+ self.drop_guards();
198
+ }
199
+
200
+ pub fn field_data_enumerate(
201
+ &mut self,
202
+ _: &'a JournalFile<M>,
203
+ ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
204
+ self.drop_guards();
205
+
206
+ if let Some(iter) = &mut self.field_data_iterator {
207
+ self.data_guard = iter.next().transpose()?;
208
+ Ok(self.data_guard.as_ref())
209
+ } else {
210
+ Ok(None)
211
+ }
212
+ }
213
+
214
+ pub fn entry_data_restart(&mut self) {
215
+ self.drop_guards();
216
+ self.entry_data_iterator = None;
217
+ }
218
+
219
+ pub fn entry_data_enumerate(
220
+ &mut self,
221
+ journal_file: &'a JournalFile<M>,
222
+ ) -> Result<Option<&ValueGuard<'_, DataObject<&'a [u8]>>>> {
223
+ self.drop_guards();
224
+
225
+ if self.entry_data_iterator.is_none() {
226
+ let entry_offset = self.cursor.position()?;
227
+ self.entry_data_iterator = Some(journal_file.entry_data_objects(entry_offset)?);
228
+ }
229
+
230
+ if let Some(iter) = &mut self.entry_data_iterator {
231
+ self.data_guard = iter.next().transpose()?;
232
+
233
+ // Translate field name if needed
234
+ if let Some(data_guard) = &self.data_guard {
235
+ let payload = data_guard.get_payload();
236
+
237
+ // Check if this field needs translation
238
+ if let Some(field_name) = extract_field_name(payload) {
239
+ if field_name.starts_with(b"ND_") {
240
+ // This looks like a remapped field
241
+ if let Ok(systemd_name) = std::str::from_utf8(field_name) {
242
+ if let Some(otel_name) =
243
+ self.remapping_registry.get_otel_name(systemd_name)
244
+ {
245
+ // Translate: build new payload with original field name
246
+ let eq_pos = payload.iter().position(|&b| b == b'=').unwrap();
247
+ let value = &payload[eq_pos..]; // includes '='
248
+
249
+ self.translated_payload.clear();
250
+ self.translated_payload.extend_from_slice(otel_name);
251
+ self.translated_payload.extend_from_slice(value);
252
+ } else {
253
+ // No mapping found - clear buffer
254
+ self.translated_payload.clear();
255
+ }
256
+ } else {
257
+ // Invalid UTF-8 - clear buffer
258
+ self.translated_payload.clear();
259
+ }
260
+ } else {
261
+ // Not a remapped field - clear buffer
262
+ self.translated_payload.clear();
263
+ }
264
+ } else {
265
+ // No '=' found - clear buffer
266
+ self.translated_payload.clear();
267
+ }
268
+ }
269
+
270
+ Ok(self.data_guard.as_ref())
271
+ } else {
272
+ Ok(None)
273
+ }
274
+ }
275
+
276
+ pub fn entry_data_offsets(
277
+ &self,
278
+ journal_file: &'a JournalFile<M>,
279
+ data_offsets: &mut Vec<NonZeroU64>,
280
+ ) -> Result<()> {
281
+ let entry_offset = self.cursor.position()?;
282
+ let entry_guard = journal_file.entry_ref(entry_offset)?;
283
+
284
+ match &entry_guard.items {
285
+ EntryItemsType::Regular(items) => {
286
+ for item in items.iter() {
287
+ if let Some(offset) = NonZeroU64::new(item.object_offset) {
288
+ data_offsets.push(offset);
289
+ }
290
+ }
291
+ }
292
+ EntryItemsType::Compact(items) => {
293
+ for item in items.iter() {
294
+ if let Some(offset) = NonZeroU64::new(item.object_offset as u64) {
295
+ data_offsets.push(offset);
296
+ }
297
+ }
298
+ }
299
+ }
300
+
301
+ Ok(())
302
+ }
303
+
304
+ /// Loads field name remappings from the journal file.
305
+ ///
306
+ /// This method scans the journal for entries tagged with `ND_REMAPPING=1`,
307
+ /// extracts the field name mappings, and builds the internal remapping registry.
308
+ ///
309
+ /// This is automatically called by convenience wrappers, but can be called
310
+ /// explicitly if needed for manual reader setup.
311
+ ///
312
+ /// # Performance
313
+ ///
314
+ /// This uses the field indexing optimization (O(k) where k = number of mapping entries)
315
+ /// rather than scanning all entries (O(n)).
316
+ pub fn load_remappings(&mut self, journal_file: &'a JournalFile<M>) -> Result<()> {
317
+ // Look up all entries with ND_REMAPPING=1 using field indexing
318
+ let marker_field_name = {
319
+ let marker_str = std::str::from_utf8(REMAPPING_MARKER)
320
+ .map_err(|_| crate::error::JournalError::InvalidField)?;
321
+ let eq_pos = marker_str
322
+ .find('=')
323
+ .ok_or(crate::error::JournalError::InvalidField)?;
324
+ &marker_str.as_bytes()[..eq_pos]
325
+ };
326
+
327
+ // Collect entry information first to avoid iterator conflicts
328
+ let mut entry_info: Vec<(Option<NonZeroU64>, Option<(NonZeroU64, u64)>)> = Vec::new();
329
+
330
+ {
331
+ // Get iterator for all data objects with this field
332
+ let Ok(mut data_iter) = journal_file.field_data_objects(marker_field_name) else {
333
+ // No remapping entries found - this is fine
334
+ return Ok(());
335
+ };
336
+
337
+ // Process each data object (should all have value "1")
338
+ while let Some(data_guard) = data_iter.next().transpose()? {
339
+ // Get all entries that contain this data object
340
+ let n_entries = data_guard.header.n_entries;
341
+
342
+ if let Some(entry_count) = n_entries {
343
+ match entry_count.get() {
344
+ 0 => {
345
+ // Should not happen
346
+ continue;
347
+ }
348
+ 1 => {
349
+ // Single entry - stored directly
350
+ if let Some(entry_offset) = data_guard.header.entry_offset {
351
+ entry_info.push((Some(entry_offset), None));
352
+ }
353
+ }
354
+ n => {
355
+ // Multiple entries - first is inlined, rest in entry array
356
+ // Process the first entry (inlined)
357
+ if let Some(entry_offset) = data_guard.header.entry_offset {
358
+ entry_info.push((Some(entry_offset), None));
359
+ }
360
+ // Process remaining entries from array
361
+ if let Some(array_offset) = data_guard.header.entry_array_offset {
362
+ entry_info.push((None, Some((array_offset, n))));
363
+ }
364
+ }
365
+ }
366
+ }
367
+ }
368
+ }
369
+
370
+ // Now parse the collected entries
371
+ for (single_entry, array_entry) in entry_info {
372
+ if let Some(entry_offset) = single_entry {
373
+ self.parse_remapping_entry(journal_file, entry_offset)?;
374
+ } else if let Some((array_offset, n_entries)) = array_entry {
375
+ self.parse_remapping_entries_from_array(journal_file, array_offset, n_entries)?;
376
+ }
377
+ }
378
+
379
+ Ok(())
380
+ }
381
+
382
+ fn parse_remapping_entry(
383
+ &mut self,
384
+ journal_file: &'a JournalFile<M>,
385
+ entry_offset: NonZeroU64,
386
+ ) -> Result<()> {
387
+ // Collect all payloads first to avoid guard lifetime issues
388
+ let mut payloads: Vec<Vec<u8>> = Vec::new();
389
+
390
+ {
391
+ let data_iter = journal_file.entry_data_objects(entry_offset)?;
392
+ for data_result in data_iter {
393
+ let data_guard = data_result?;
394
+ let payload = data_guard.get_payload();
395
+ payloads.push(payload.to_vec());
396
+ }
397
+ }
398
+
399
+ // Now parse the collected payloads
400
+ for payload in payloads {
401
+ // Skip the marker field itself
402
+ if payload == REMAPPING_MARKER {
403
+ continue;
404
+ }
405
+
406
+ // Parse ND_<md5>=<original_name>
407
+ if let Some(field_name) = extract_field_name(&payload) {
408
+ if field_name.starts_with(b"ND_") && field_name.len() == 35 {
409
+ // This is a remapping field
410
+ let eq_pos = payload.iter().position(|&b| b == b'=').unwrap();
411
+ let systemd_name = std::str::from_utf8(field_name)
412
+ .map_err(|_| crate::error::JournalError::InvalidField)?
413
+ .to_string();
414
+ let otel_name = payload[eq_pos + 1..].to_vec();
415
+
416
+ self.remapping_registry
417
+ .add_otel_mapping(otel_name, systemd_name);
418
+ }
419
+ }
420
+ }
421
+
422
+ Ok(())
423
+ }
424
+
425
+ fn parse_remapping_entries_from_array(
426
+ &mut self,
427
+ journal_file: &'a JournalFile<M>,
428
+ array_offset: NonZeroU64,
429
+ n_entries: u64,
430
+ ) -> Result<()> {
431
+ // Get all entry offsets from the array chain
432
+ let mut entry_offsets = Vec::new();
433
+
434
+ // Create list and collect all offsets
435
+ // n_entries includes the inlined entry (count=1) plus array entries (count=n-1)
436
+ // So the array has n-1 entries
437
+ let array_count = n_entries.saturating_sub(1);
438
+
439
+ if let Some(total_items_nz) = std::num::NonZeroUsize::new(array_count as usize) {
440
+ use crate::file::offset_array::List;
441
+ let list = List::new(array_offset, total_items_nz);
442
+ list.collect_offsets(journal_file, &mut entry_offsets)?;
443
+ }
444
+
445
+ // Parse each remapping entry
446
+ for entry_offset in entry_offsets {
447
+ self.parse_remapping_entry(journal_file, entry_offset)?;
448
+ }
449
+
450
+ Ok(())
451
+ }
452
+
453
+ /// Gets the current entry data payload, translating remapped field names if applicable.
454
+ ///
455
+ /// This should be called after `entry_data_enumerate()` to get the translated version
456
+ /// of the field name. If the field name doesn't need translation, returns the original.
457
+ pub fn get_entry_data_payload(&self) -> &[u8] {
458
+ if !self.translated_payload.is_empty() {
459
+ &self.translated_payload
460
+ } else if let Some(data_guard) = &self.data_guard {
461
+ data_guard.get_payload()
462
+ } else {
463
+ &[]
464
+ }
465
+ }
466
+}
src/crates/journal-core/src/file/sigbus.rs
renamed
+4
-2
@@ -1,6 +1,8 @@
1
-use error::{JournalError, Result};
2
-use std::sync::atomic::{AtomicBool, Ordering};
1
+#![allow(dead_code)]
2
+
3
+use crate::error::{JournalError, Result};
4
use std::sync::OnceLock;
5
+use std::sync::atomic::{AtomicBool, Ordering};
6
7
static SIGBUS_OCCURRED: AtomicBool = AtomicBool::new(false);
8
static HANDLER_INSTALLED: OnceLock<i32> = OnceLock::new();
src/crates/journal-core/src/file/value_guard.rs
renamed
+2
-2
@@ -67,7 +67,7 @@ impl<T> Drop for ValueGuard<'_, T> {
67
}
68
}
69
70
-use crate::{HashableObject, HashableObjectMut};
70
+use crate::file::{HashableObject, HashableObjectMut};
71
use std::num::NonZeroU64;
72
73
impl<T: HashableObject> HashableObject for ValueGuard<'_, T> {
@@ -83,7 +83,7 @@ impl<T: HashableObject> HashableObject for ValueGuard<'_, T> {
83
self.value.next_hash_offset()
84
}
85
86
- fn object_type() -> crate::ObjectType {
86
+ fn object_type() -> super::object::ObjectType {
87
T::object_type()
88
}
89
}
src/crates/journal-core/src/file/writer.rs
renamed
+41
-270
@@ -1,18 +1,18 @@
1
#![allow(unused_imports, dead_code)]
2
3
-use crate::{
4
- journal_hash_data, CompactEntryItem, DataHashTable, DataObject, DataObjectHeader,
5
- DataPayloadType, EntryObject, EntryObjectHeader, FieldHashTable, FieldObject,
6
- FieldObjectHeader, HashItem, HashTable, HashTableMut, HashableObject, HashableObjectMut,
7
- HeaderIncompatibleFlags, JournalFile, JournalFileOptions, JournalHeader, JournalState,
8
- ObjectHeader, ObjectType, RegularEntryItem,
3
+use super::mmap::MemoryMapMut;
4
+use super::mmap::MmapMut;
5
+use crate::error::{JournalError, Result};
6
+use crate::file::{
7
+ CompactEntryItem, DataHashTable, DataObject, DataObjectHeader, DataPayloadType, EntryObject,
8
+ EntryObjectHeader, FieldHashTable, FieldObject, FieldObjectHeader, HashItem, HashTable,
9
+ HashTableMut, HashableObject, HashableObjectMut, HeaderIncompatibleFlags, JournalFile,
10
+ JournalFileOptions, JournalHeader, JournalState, ObjectHeader, ObjectType, RegularEntryItem,
11
+ hash::jenkins_hash64, journal_hash_data,
12
};
10
-use error::{JournalError, Result};
11
-use memmap2::MmapMut;
12
-use rand::{seq::IndexedRandom, Rng};
13
+use rand::{Rng, seq::IndexedRandom};
14
use std::num::{NonZeroU64, NonZeroUsize};
15
use std::path::Path;
15
-use window_manager::MemoryMapMut;
16
use zerocopy::{FromBytes, IntoBytes};
17
18
const OBJECT_ALIGNMENT: u64 = 8;
@@ -30,6 +30,7 @@ pub struct JournalWriter {
30
num_written_objects: u64,
31
entry_items: Vec<EntryItem>,
32
first_entry_monotonic: Option<u64>,
33
+ boot_id: uuid::Uuid,
34
}
35
36
impl JournalWriter {
@@ -43,8 +44,22 @@ impl JournalWriter {
44
self.first_entry_monotonic
45
}
46
46
- pub fn new(journal_file: &mut JournalFile<MmapMut>) -> Result<Self> {
47
- let (append_offset, next_seqnum) = {
47
+ /// Get the next sequence number that will be written
48
+ pub fn next_seqnum(&self) -> u64 {
49
+ self.next_seqnum
50
+ }
51
+
52
+ /// Get the boot ID for this writer
53
+ pub fn boot_id(&self) -> uuid::Uuid {
54
+ self.boot_id
55
+ }
56
+
57
+ pub fn new(
58
+ journal_file: &mut JournalFile<MmapMut>,
59
+ next_seqnum: u64,
60
+ boot_id: uuid::Uuid,
61
+ ) -> Result<Self> {
62
+ let append_offset = {
63
let header = journal_file.journal_header_ref();
64
65
let Some(tail_object_offset) = header.tail_object_offset else {
@@ -53,10 +68,7 @@ impl JournalWriter {
68
69
let tail_object = journal_file.object_header_ref(tail_object_offset)?;
70
56
- (
57
- tail_object_offset.saturating_add(tail_object.size),
58
- header.tail_entry_seqnum + 1,
59
- )
71
+ tail_object_offset.saturating_add(tail_object.size)
72
};
73
74
Ok(Self {
@@ -69,16 +81,21 @@ impl JournalWriter {
81
num_written_objects: 0,
82
entry_items: Vec::with_capacity(128),
83
first_entry_monotonic: None,
84
+ boot_id,
85
})
86
}
87
88
+ /// Creates a successor writer for a new journal file
89
+ pub fn create_successor(&self, journal_file: &mut JournalFile<MmapMut>) -> Result<Self> {
90
+ Self::new(journal_file, self.next_seqnum, self.boot_id)
91
+ }
92
+
93
pub fn add_entry(
94
&mut self,
95
journal_file: &mut JournalFile<MmapMut>,
96
items: &[&[u8]],
97
realtime: u64,
98
monotonic: u64,
81
- boot_id: [u8; 16],
99
) -> Result<()> {
100
let header = journal_file.journal_header_ref();
101
assert!(header.has_incompatible_flag(HeaderIncompatibleFlags::KeyedHash));
@@ -98,7 +115,9 @@ impl JournalWriter {
115
let entry_item = EntryItem { offset, hash };
116
self.entry_items.push(entry_item);
117
101
- xor_hash ^= journal_hash_data(payload, true, None);
118
+ // Per journal file format spec: xor_hash always uses Jenkins lookup3,
119
+ // even for files with HEADER_INCOMPATIBLE_KEYED_HASH flag set
120
+ xor_hash ^= jenkins_hash64(payload);
121
}
122
123
self.entry_items
@@ -114,7 +133,7 @@ impl JournalWriter {
133
134
entry_guard.header.seqnum = self.next_seqnum;
135
entry_guard.header.xor_hash = xor_hash;
117
- entry_guard.header.boot_id = boot_id;
136
+ entry_guard.header.boot_id = *self.boot_id.as_bytes();
137
entry_guard.header.monotonic = monotonic;
138
entry_guard.header.realtime = realtime;
139
@@ -134,12 +153,7 @@ impl JournalWriter {
153
self.link_data_to_entry(journal_file, entry_offset, entry_item_index)?;
154
}
155
137
- self.entry_added(
138
- journal_file.journal_header_mut(),
139
- realtime,
140
- monotonic,
141
- boot_id,
142
- );
156
+ self.entry_added(journal_file.journal_header_mut(), realtime, monotonic);
157
158
Ok(())
159
}
@@ -150,13 +164,7 @@ impl JournalWriter {
164
self.num_written_objects += 1;
165
}
166
153
- fn entry_added(
154
- &mut self,
155
- header: &mut JournalHeader,
156
- realtime: u64,
157
- monotonic: u64,
158
- boot_id: [u8; 16],
159
- ) {
167
+ fn entry_added(&mut self, header: &mut JournalHeader, realtime: u64, monotonic: u64) {
168
header.n_entries += 1;
169
header.n_objects += self.num_written_objects;
170
header.tail_object_offset = Some(self.tail_object_offset);
@@ -175,7 +183,7 @@ impl JournalWriter {
183
header.tail_entry_seqnum = self.next_seqnum;
184
header.tail_entry_realtime = realtime;
185
header.tail_entry_monotonic = monotonic;
178
- header.tail_entry_boot_id = boot_id;
186
+ header.tail_entry_boot_id = *self.boot_id.as_bytes();
187
188
self.next_seqnum += 1;
189
self.num_written_objects = 0;
@@ -461,240 +469,3 @@ impl JournalWriter {
469
Ok(())
470
}
471
}
464
-
465
-#[cfg(test)]
466
-mod tests {
467
- use super::*;
468
- use crate::{load_boot_id, Direction, JournalFile, JournalReader, Location};
469
- use memmap2::Mmap;
470
- use std::collections::HashMap;
471
- use tempfile::NamedTempFile;
472
-
473
- fn generate_uuid() -> [u8; 16] {
474
- use rand::Rng;
475
- let mut rng = rand::rng();
476
- rng.random()
477
- }
478
-
479
- #[test]
480
- fn test_write_and_read_journal_entries() -> Result<()> {
481
- // Create test data - a hash map with key/values to add to the journal file
482
- let mut test_data = HashMap::new();
483
- test_data.insert(
484
- "MESSAGE",
485
- vec!["Hello, world!", "Another message", "Final message"],
486
- );
487
- test_data.insert("PRIORITY", vec!["6", "4", "3"]);
488
- test_data.insert(
489
- "_SYSTEMD_UNIT",
490
- vec!["test.service", "other.service", "test.service"],
491
- );
492
- test_data.insert("_PID", vec!["1234", "5678", "9999"]);
493
-
494
- // Create a temporary file for the journal
495
- let temp_file = NamedTempFile::new().map_err(JournalError::Io).unwrap();
496
- let journal_path = temp_file.path();
497
-
498
- // Step 1: Create and write to the journal
499
- let boot_id = load_boot_id().unwrap_or([1; 16]); // Use real boot_id or fallback
500
- let num_entries = test_data.values().next().unwrap().len();
501
-
502
- let options = JournalFileOptions::new(
503
- generate_uuid(),
504
- generate_uuid(),
505
- generate_uuid(),
506
- generate_uuid(),
507
- );
508
- let mut journal_file = JournalFile::create(journal_path, options)?;
509
- let iterations = 5000;
510
- for _ in 0..iterations {
511
- let mut writer = JournalWriter::new(&mut journal_file)?;
512
-
513
- // Write entries to the journal
514
- for i in 0..num_entries {
515
- let mut entry_data = Vec::new();
516
-
517
- // Build the entry data for this index
518
- for (key, values) in &test_data {
519
- let kv_pair = format!("{}={}", key, values[i]);
520
- entry_data.push(kv_pair.into_bytes());
521
- }
522
-
523
- // Convert to slice references for the writer
524
- let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
525
-
526
- // Write the entry with timestamps
527
- let realtime = 1000000 + (i as u64 * 1000); // Mock realtime in microseconds
528
- let monotonic = 500000 + (i as u64 * 1000); // Mock monotonic time
529
-
530
- writer.add_entry(&mut journal_file, &entry_refs, realtime, monotonic, boot_id)?;
531
- }
532
- }
533
-
534
- // Step 2: Read back and verify the journal contents
535
- {
536
- let journal_file = JournalFile::<Mmap>::open(journal_path, 8 * 1024)?;
537
- let mut reader = JournalReader::default();
538
-
539
- let hdr = journal_file.journal_header_ref();
540
- println!("Header: {:#?}", hdr);
541
-
542
- // Start from the head
543
- reader.set_location(Location::Head);
544
-
545
- let mut entries_read = 0;
546
- while reader.step(&journal_file, Direction::Forward)? {
547
- println!("Reading entry {}", entries_read);
548
-
549
- // Verify timestamps
550
- let realtime = reader.get_realtime_usec(&journal_file)?;
551
- let expected_realtime = 1000000 + ((entries_read % 3) * 1000);
552
- assert_eq!(
553
- realtime, expected_realtime,
554
- "Realtime mismatch for entry {}",
555
- entries_read
556
- );
557
-
558
- let (seqnum, _seqnum_id) = reader.get_seqnum(&journal_file)?;
559
- assert_eq!(
560
- seqnum,
561
- entries_read + 1,
562
- "Sequence number mismatch for entry {}",
563
- entries_read
564
- );
565
-
566
- // Read all data for this entry
567
- let mut entry_fields = HashMap::new();
568
- reader.entry_data_restart();
569
-
570
- while let Some(data_guard) = reader.entry_data_enumerate(&journal_file)? {
571
- let payload = data_guard.payload_bytes();
572
- let payload_str = String::from_utf8_lossy(payload);
573
-
574
- if let Some(eq_pos) = payload_str.find('=') {
575
- let key = &payload_str[..eq_pos];
576
- let value = &payload_str[eq_pos + 1..];
577
- entry_fields.insert(key.to_string(), value.to_string());
578
- }
579
- }
580
-
581
- // Verify the data matches what we wrote
582
- for (key, values) in &test_data {
583
- let expected_value = &values[entries_read as usize % 3];
584
- let actual_value = entry_fields.get(*key).unwrap_or_else(|| {
585
- panic!("Missing key '{}' in entry {}", key, entries_read)
586
- });
587
-
588
- assert_eq!(
589
- actual_value, expected_value,
590
- "Value mismatch for key '{}' in entry {}",
591
- key, entries_read
592
- );
593
- }
594
-
595
- println!("Read entry {}", entries_read);
596
- entries_read += 1;
597
- }
598
-
599
- assert_eq!(
600
- entries_read as usize,
601
- num_entries * iterations,
602
- "Number of entries read doesn't match written"
603
- );
604
- }
605
-
606
- // Step 3: Test filtering by specific fields
607
- {
608
- let journal_file = JournalFile::<Mmap>::open(journal_path, 64 * 1024)?;
609
- let mut reader = JournalReader::default();
610
-
611
- // Test filtering by _SYSTEMD_UNIT=test.service
612
- reader.add_match(b"_SYSTEMD_UNIT=test.service");
613
- reader.set_location(Location::Head);
614
-
615
- let mut filtered_entries = 0;
616
- while reader.step(&journal_file, Direction::Forward)? {
617
- // Verify this entry actually contains the filter match
618
- reader.entry_data_restart();
619
- let mut found_match = false;
620
-
621
- while let Some(data_guard) = reader.entry_data_enumerate(&journal_file)? {
622
- let payload = data_guard.payload_bytes();
623
- if payload == b"_SYSTEMD_UNIT=test.service" {
624
- found_match = true;
625
- break;
626
- }
627
- }
628
-
629
- assert!(
630
- found_match,
631
- "Filtered entry doesn't contain the expected field"
632
- );
633
- filtered_entries += 1;
634
- }
635
-
636
- // Should find 2 entries with _SYSTEMD_UNIT=test.service (entries 0 and 2)
637
- assert_eq!(filtered_entries, 2, "Expected 2 filtered entries");
638
- }
639
-
640
- println!("✅ All tests passed!");
641
- Ok(())
642
- }
643
-
644
- #[test]
645
- fn test_field_enumeration() -> Result<()> {
646
- // Create a simple journal with known fields
647
- let temp_file = NamedTempFile::new().map_err(JournalError::Io)?;
648
- let journal_path = temp_file.path();
649
-
650
- let test_fields = vec!["MESSAGE", "PRIORITY", "_SYSTEMD_UNIT"];
651
- let boot_id = [1; 16];
652
-
653
- // Write a single entry with multiple fields
654
- {
655
- let options = JournalFileOptions::new(
656
- generate_uuid(),
657
- generate_uuid(),
658
- generate_uuid(),
659
- generate_uuid(),
660
- );
661
-
662
- let mut journal_file = JournalFile::create(journal_path, options)?;
663
- let mut writer = JournalWriter::new(&mut journal_file)?;
664
-
665
- let entry_data = vec![
666
- b"MESSAGE=Test message".as_slice(),
667
- b"PRIORITY=6".as_slice(),
668
- b"_SYSTEMD_UNIT=test.service".as_slice(),
669
- ];
670
-
671
- writer.add_entry(&mut journal_file, &entry_data, 1000000, 500000, boot_id)?;
672
- }
673
-
674
- // Read back and enumerate fields
675
- {
676
- let journal_file = JournalFile::<Mmap>::open(journal_path, 8 * 1024)?;
677
- let mut reader = JournalReader::default();
678
-
679
- let mut found_fields = Vec::new();
680
- reader.fields_restart();
681
-
682
- while let Some(field_guard) = reader.fields_enumerate(&journal_file)? {
683
- let field_name = String::from_utf8_lossy(field_guard.payload);
684
- found_fields.push(field_name.to_string());
685
- }
686
-
687
- // Verify all expected fields were found
688
- for expected_field in &test_fields {
689
- assert!(
690
- found_fields.contains(&expected_field.to_string()),
691
- "Expected field '{}' not found. Found: {:?}",
692
- expected_field,
693
- found_fields
694
- );
695
- }
696
- }
697
-
698
- Ok(())
699
- }
700
-}
src/crates/journal-core/src/lib.rs
new
+34
@@ -0,0 +1,34 @@
1
+//! Core functionality for working with systemd journal files.
2
+//!
3
+//! This crate provides low-level file I/O for systemd journal files.
4
+//!
5
+//! For related functionality:
6
+//! - High-level journaling with rotation and retention: see `journal-log-writer` crate
7
+//! - File tracking and monitoring: see `journal-registry` crate
8
+//! - Indexing and querying: see `journal-index` crate
9
+
10
+// Core error types used throughout the crate
11
+pub mod error;
12
+
13
+// Collection type aliases
14
+pub mod collections;
15
+
16
+// Low-level journal file format I/O
17
+pub mod file;
18
+
19
+// Field name mapping for systemd compatibility
20
+pub mod field_map;
21
+
22
+// Re-export repository types from journal-registry for convenience
23
+pub mod repository {
24
+ pub use journal_registry::repository::*;
25
+}
26
+
27
+// Re-export commonly used types for convenience
28
+pub use error::{JournalError, Result};
29
+
30
+// File module re-exports
31
+pub use file::{
32
+ BucketUtilization, Direction, JournalCursor, JournalFile, JournalFileOptions, JournalReader,
33
+ JournalWriter, Location,
34
+};
src/crates/journal-engine/Cargo.toml
new
+41
@@ -0,0 +1,41 @@
1
+[package]
2
+name = "journal-engine"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[features]
11
+allocative = ["dep:allocative"]
12
+
13
+[dependencies]
14
+async-stream = { workspace = true }
15
+async-trait = { workspace = true }
16
+futures = { workspace = true }
17
+serde = { workspace = true }
18
+serde_json = { workspace = true }
19
+static_assertions = { workspace = true }
20
+thiserror = { workspace = true }
21
+tokio = { workspace = true }
22
+tracing = { workspace = true }
23
+chrono = { workspace = true, features = ["serde"] }
24
+foyer = { workspace = true }
25
+parking_lot = { workspace = true, features = ["send_guard"] }
26
+rayon = { workspace = true }
27
+
28
+journal-core = { workspace = true }
29
+journal-index = { workspace = true }
30
+journal-registry = { workspace = true }
31
+foundation = { workspace = true }
32
+lru = { workspace = true }
33
+
34
+allocative = { workspace = true, optional = true }
35
+
36
+[dev-dependencies]
37
+chrono = { workspace = true }
38
+tracing-subscriber = { workspace = true }
39
+tempfile = { workspace = true }
40
+uuid = { workspace = true }
41
+journal-common = { workspace = true }
src/crates/journal-engine/examples/cgroup-run.sh
new
+13
@@ -0,0 +1,13 @@
1
+#!/usr/bin/env bash
2
+
3
+set -exu -o pipefail
4
+
5
+cargo build --example index
6
+
7
+sudo find /mnt/slow-disk/foyer-cache -type f -delete
8
+
9
+sudo sync
10
+echo 3 | sudo tee /proc/sys/vm/drop_caches
11
+sleep 1
12
+
13
+sudo cgexec -g io:/slow-io ../../target/debug/examples/index
src/crates/journal-engine/examples/index.rs
new
+134
@@ -0,0 +1,134 @@
1
+//! Simple test binary for batch_compute_file_indexes.
2
+//!
3
+//! Points at a directory of journal files and indexes them.
4
+
5
+// # 1. Create a mount point
6
+//
7
+// dd if=/dev/zero of=/tmp/slow-disk.img bs=1G count=100
8
+// LOOP=$(sudo losetup -f --show /tmp/slow-disk.img)
9
+// sudo mkfs.ext4 $LOOP
10
+// sudo mkdir -p /mnt/slow-disk
11
+// sudo mount $LOOP /mnt/slow-disk
12
+// sudo chown $USER:$USER /mnt/slow-disk
13
+//
14
+// # 2. Copy journal files
15
+// cp -r ~/repos/tmp/otel-aws /mnt/slow-disk/
16
+//
17
+// # 3. Unmount and recreate with delay
18
+// sudo umount /mnt/slow-disk
19
+// SIZE=$(sudo blockdev --getsz $LOOP)
20
+// sudo dmsetup create slow-disk --table "0 $SIZE delay $LOOP 0 50 $LOOP 0 50"
21
+// sudo mount /dev/mapper/slow-disk /mnt/slow-disk
22
+//
23
+// # 4. Now /mnt/slow-disk/otel-aws has your journals on a "slow" disk
24
+//
25
+// # 5. Create slow-io cgroup
26
+// sudo mkdir -p /sys/fs/cgroup/slow-io
27
+// echo "+io" | sudo tee /sys/fs/cgroup/cgroup.controllers
28
+// # Find your device's major:minor (e.g., for nvme0n1)
29
+// cat /sys/block/nvme0n1/dev
30
+// # Let's say it's 259:0, Set a 10MB/s read and write limit
31
+// echo "259:0 rbps=10485760 wbps=10485760" | sudo tee /sys/fs/cgroup/slow-io/io.max
32
+
33
+use foundation::Timeout;
34
+use journal_engine::{
35
+ Facets, FileIndexCacheBuilder, FileIndexKey, QueryTimeRange, batch_compute_file_indexes,
36
+};
37
+use journal_index::FieldName;
38
+use journal_registry::{Monitor, Registry};
39
+use std::env;
40
+use std::path::PathBuf;
41
+use std::time::Duration;
42
+
43
+#[allow(unused_imports)]
44
+use tracing::{info, warn};
45
+
46
+#[tokio::main]
47
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
48
+ // Initialize tracing
49
+ tracing_subscriber::fmt()
50
+ .with_max_level(tracing::Level::DEBUG)
51
+ .init();
52
+
53
+ // Get directory from args or use default
54
+ let dir = if let Some(arg) = env::args().nth(1) {
55
+ PathBuf::from(arg)
56
+ } else {
57
+ PathBuf::from("/mnt/slow-disk/otel-aws")
58
+ };
59
+
60
+ info!("scanning directory: {}", dir.display());
61
+
62
+ // Create registry and scan directory
63
+ let (monitor, _event_receiver) = Monitor::new()?;
64
+ let registry = Registry::new(monitor);
65
+
66
+ registry.watch_directory(dir.to_str().unwrap())?;
67
+
68
+ // Find all files
69
+ let files = registry.find_files_in_range(
70
+ journal_common::Seconds(0),
71
+ journal_common::Seconds(u32::MAX),
72
+ )?;
73
+
74
+ info!("found {} journal files", files.len());
75
+ if files.is_empty() {
76
+ return Ok(());
77
+ }
78
+ // files.truncate(1);
79
+
80
+ // Create file index cache
81
+ let cache = FileIndexCacheBuilder::new()
82
+ // .with_cache_path("/mnt/slow-disk/foyer-cache")
83
+ .with_cache_path("/tmp/foyer-cache")
84
+ .with_memory_capacity(1000)
85
+ .with_disk_capacity(2048 * 1024 * 1024)
86
+ .with_block_size(4 * 1024 * 1024)
87
+ .build()
88
+ .await?;
89
+
90
+ info!("created file index cache");
91
+
92
+ // Configure indexing parameters (modify these as needed)
93
+ let facets = Facets::new(&["log.severity_number".to_string()]);
94
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
95
+
96
+ let keys: Vec<FileIndexKey> = files
97
+ .iter()
98
+ .map(|file_info| {
99
+ FileIndexKey::new(
100
+ &file_info.file,
101
+ &facets,
102
+ Some(source_timestamp_field.clone()),
103
+ )
104
+ })
105
+ .collect();
106
+
107
+ // Create a time range for indexing (24 hours)
108
+ let now = std::time::SystemTime::now()
109
+ .duration_since(std::time::UNIX_EPOCH)?
110
+ .as_secs() as u32;
111
+ let time_range = QueryTimeRange::new(now - 86400, now)?;
112
+ let timeout = Timeout::new(Duration::from_secs(60));
113
+
114
+ info!(
115
+ "computing {} file indexes with timeout {:?}, bucket duration: {}s",
116
+ keys.len(),
117
+ timeout.remaining(),
118
+ time_range.bucket_duration()
119
+ );
120
+
121
+ // Run batch indexing
122
+ let start = std::time::Instant::now();
123
+ let responses =
124
+ batch_compute_file_indexes(&cache, ®istry, keys, &time_range, timeout).await?;
125
+
126
+ let elapsed = start.elapsed();
127
+
128
+ info!("responses={}, duration={:?}", responses.len(), elapsed);
129
+
130
+ // Close the cache to flush and shut down I/O tasks gracefully
131
+ cache.close().await?;
132
+
133
+ Ok(())
134
+}
src/crates/journal-engine/src/cache.rs
new
+38
@@ -0,0 +1,38 @@
1
+//! Cache types for journal file indexes
2
+
3
+use crate::facets::Facets;
4
+use foyer::HybridCache;
5
+use journal_index::{FieldName, FileIndex};
6
+use journal_registry::File;
7
+use serde::{Deserialize, Serialize};
8
+
9
+/// Cache version number. Increment this when the FileIndex or FileIndexKey
10
+/// schema changes to automatically invalidate old cache entries.
11
+const CACHE_VERSION: u32 = 1;
12
+
13
+/// Cache key for file indexes that includes the file, facets, source timestamp
14
+/// field, and cache version. Different facet configurations or timestamp fields
15
+/// produce different indexes, so all are needed to uniquely identify a cached
16
+/// index. The version ensures that schema changes automatically invalidate old
17
+/// cache entries.
18
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
19
+pub struct FileIndexKey {
20
+ version: u32,
21
+ pub file: File,
22
+ pub(crate) facets: Facets,
23
+ pub(crate) source_timestamp_field: Option<FieldName>,
24
+}
25
+
26
+impl FileIndexKey {
27
+ pub fn new(file: &File, facets: &Facets, source_timestamp_field: Option<FieldName>) -> Self {
28
+ Self {
29
+ version: CACHE_VERSION,
30
+ file: file.clone(),
31
+ facets: facets.clone(),
32
+ source_timestamp_field,
33
+ }
34
+ }
35
+}
36
+
37
+/// Type alias for file index cache using Foyer's HybridCache.
38
+pub type FileIndexCache = HybridCache<FileIndexKey, FileIndex>;
src/crates/journal-engine/src/error.rs
new
+61
@@ -0,0 +1,61 @@
1
+//! Error types for journal engine operations
2
+
3
+use std::path::PathBuf;
4
+use thiserror::Error;
5
+
6
+/// Errors that can occur during engine operations
7
+#[derive(Debug, Error)]
8
+pub enum EngineError {
9
+ /// I/O error when reading files
10
+ #[error("I/O error: {0}")]
11
+ Io(#[from] std::io::Error),
12
+
13
+ /// Error from journal core operations
14
+ #[error("Journal error: {0}")]
15
+ Journal(#[from] journal_core::JournalError),
16
+
17
+ /// Error from journal indexing operations
18
+ #[error("Index error: {0}")]
19
+ Index(#[from] journal_index::IndexError),
20
+
21
+ /// Error from repository operations
22
+ #[error("Repository error: {0}")]
23
+ Repository(#[from] journal_registry::repository::RepositoryError),
24
+
25
+ /// Error from registry operations
26
+ #[error("Registry error: {0}")]
27
+ Registry(#[from] journal_registry::RegistryError),
28
+
29
+ /// Error when parsing a journal file path
30
+ #[error("Failed to parse journal file path: {path}")]
31
+ InvalidPath { path: String },
32
+
33
+ /// Error when a path contains invalid UTF-8
34
+ #[error("Path contains invalid UTF-8: {}", .path.display())]
35
+ InvalidUtf8 { path: PathBuf },
36
+
37
+ /// Channel closed error
38
+ #[error("Channel closed")]
39
+ ChannelClosed,
40
+
41
+ /// Foyer cache error
42
+ #[error("Cache error: {0}")]
43
+ Foyer(#[from] foyer::Error),
44
+
45
+ /// Foyer IO engine error
46
+ #[error("Foyer IO error: {0}")]
47
+ FoyerIo(#[from] foyer::IoError),
48
+
49
+ /// Time budget exceeded during batch processing
50
+ #[error("Time budget exceeded")]
51
+ TimeBudgetExceeded,
52
+
53
+ /// Invalid time range (start >= end)
54
+ #[error("Invalid time range: start={start} >= end={end}")]
55
+ InvalidTimeRange { start: u32, end: u32 },
56
+}
57
+
58
+static_assertions::const_assert!(std::mem::size_of::<EngineError>() <= 64);
59
+
60
+/// A specialized Result type for engine operations
61
+pub type Result<T> = std::result::Result<T, EngineError>;
src/crates/journal-engine/src/facets.rs
new
+175
@@ -0,0 +1,175 @@
1
+//! Field facets configuration for indexing
2
+//!
3
+//! Facets determine which fields should be indexed when processing journal files.
4
+
5
+use journal_index::FieldName;
6
+use std::hash::Hash;
7
+use std::sync::Arc;
8
+
9
+/// Configuration specifying which fields should be indexed.
10
+///
11
+/// Facets are used as part of the cache key for file indexes, since different
12
+/// field selections produce different indexes.
13
+///
14
+/// # Serialization
15
+///
16
+/// Facets serializes as a sequence of field name strings. The precomputed hash
17
+/// is NOT serialized - it is recomputed during deserialization to maintain the
18
+/// invariant that `precomputed_hash == hash(fields)`.
19
+#[derive(Debug, Clone)]
20
+pub struct Facets {
21
+ fields: Arc<Vec<FieldName>>,
22
+ precomputed_hash: u64,
23
+}
24
+
25
+impl Hash for Facets {
26
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
27
+ state.write_u64(self.precomputed_hash);
28
+ }
29
+}
30
+
31
+impl PartialEq for Facets {
32
+ fn eq(&self, other: &Self) -> bool {
33
+ if self.precomputed_hash != other.precomputed_hash {
34
+ return false;
35
+ }
36
+
37
+ Arc::ptr_eq(&self.fields, &other.fields) || self.fields == other.fields
38
+ }
39
+}
40
+
41
+impl Eq for Facets {}
42
+
43
+impl Facets {
44
+ fn default_facets() -> Vec<FieldName> {
45
+ let v: Vec<&str> = vec![
46
+ "_HOSTNAME",
47
+ "PRIORITY",
48
+ "SYSLOG_FACILITY",
49
+ "ERRNO",
50
+ "SYSLOG_IDENTIFIER",
51
+ // "UNIT",
52
+ "USER_UNIT",
53
+ "MESSAGE_ID",
54
+ "_BOOT_ID",
55
+ "_SYSTEMD_OWNER_UID",
56
+ "_UID",
57
+ "OBJECT_SYSTEMD_OWNER_UID",
58
+ "OBJECT_UID",
59
+ "_GID",
60
+ "OBJECT_GID",
61
+ "_CAP_EFFECTIVE",
62
+ "_AUDIT_LOGINUID",
63
+ "OBJECT_AUDIT_LOGINUID",
64
+ "CODE_FUNC",
65
+ "ND_LOG_SOURCE",
66
+ "CODE_FILE",
67
+ "ND_ALERT_NAME",
68
+ "ND_ALERT_CLASS",
69
+ "_SELINUX_CONTEXT",
70
+ "_MACHINE_ID",
71
+ "ND_ALERT_TYPE",
72
+ "_SYSTEMD_SLICE",
73
+ "_EXE",
74
+ // "_SYSTEMD_UNIT",
75
+ "_NAMESPACE",
76
+ "_TRANSPORT",
77
+ "_RUNTIME_SCOPE",
78
+ "_STREAM_ID",
79
+ "ND_NIDL_CONTEXT",
80
+ "ND_ALERT_STATUS",
81
+ // "_SYSTEMD_CGROUP",
82
+ "ND_NIDL_NODE",
83
+ "ND_ALERT_COMPONENT",
84
+ "_COMM",
85
+ "_SYSTEMD_USER_UNIT",
86
+ "_SYSTEMD_USER_SLICE",
87
+ // "_SYSTEMD_SESSION",
88
+ "__logs_sources",
89
+ "log.severity_number",
90
+ ];
91
+
92
+ v.into_iter().map(FieldName::new_unchecked).collect()
93
+ }
94
+
95
+ pub fn new(facets: &[String]) -> Self {
96
+ let mut facets = if facets.is_empty() {
97
+ Self::default_facets()
98
+ } else {
99
+ // Parse and validate each facet string into FieldName
100
+ facets
101
+ .iter()
102
+ .filter_map(|s| FieldName::new(s.clone()))
103
+ .collect()
104
+ };
105
+
106
+ // Sort in order to get the same hash for the same set of fields
107
+ facets.sort();
108
+
109
+ use std::hash::Hasher;
110
+ let mut hasher = std::hash::DefaultHasher::new();
111
+ // Hash the string representation for consistency
112
+ for field in &facets {
113
+ field.as_str().hash(&mut hasher);
114
+ }
115
+ let precomputed_hash = hasher.finish();
116
+
117
+ Self {
118
+ fields: Arc::new(facets),
119
+ precomputed_hash,
120
+ }
121
+ }
122
+
123
+ /// Returns an iterator over the facet field names
124
+ pub fn iter(&self) -> impl Iterator<Item = &FieldName> {
125
+ self.fields.iter()
126
+ }
127
+
128
+ /// Returns the facet fields as a slice
129
+ pub fn as_slice(&self) -> &[FieldName] {
130
+ &self.fields
131
+ }
132
+
133
+ /// Returns the number of facet fields
134
+ pub fn len(&self) -> usize {
135
+ self.fields.len()
136
+ }
137
+
138
+ /// Returns true if there are no facet fields
139
+ #[allow(dead_code)]
140
+ pub fn is_empty(&self) -> bool {
141
+ self.fields.is_empty()
142
+ }
143
+
144
+ pub fn precomputed_hash(&self) -> u64 {
145
+ self.precomputed_hash
146
+ }
147
+}
148
+
149
+impl serde::Serialize for Facets {
150
+ fn serialize<S>(&self, serializer: S) -> Result<S::Ok, S::Error>
151
+ where
152
+ S: serde::Serializer,
153
+ {
154
+ // Serialize as a sequence of field name strings.
155
+ // The precomputed_hash is NOT serialized - it will be recomputed on deserialization.
156
+ use serde::ser::SerializeSeq;
157
+ let mut seq = serializer.serialize_seq(Some(self.fields.len()))?;
158
+ for field in self.fields.iter() {
159
+ seq.serialize_element(field.as_str())?;
160
+ }
161
+ seq.end()
162
+ }
163
+}
164
+
165
+impl<'de> serde::Deserialize<'de> for Facets {
166
+ fn deserialize<D>(deserializer: D) -> Result<Self, D::Error>
167
+ where
168
+ D: serde::Deserializer<'de>,
169
+ {
170
+ // Deserialize as a sequence of strings, then reconstruct via Facets::new()
171
+ // which will recompute the hash, maintaining the invariant.
172
+ let fields: Vec<String> = Vec::deserialize(deserializer)?;
173
+ Ok(Facets::new(&fields))
174
+ }
175
+}
src/crates/journal-engine/src/histogram.rs
new
+370
@@ -0,0 +1,370 @@
1
+//! Histogram functionality for generating time-series data from journal files.
2
+//!
3
+//! This module provides types and services for computing histograms of journal log entries
4
+//! over time ranges, with support for filtering and faceted field indexing.
5
+
6
+use crate::{cache::FileIndexKey, error::Result, facets::Facets};
7
+use journal_core::collections::HashSet;
8
+use journal_index::{FieldName, FieldValuePair, FileIndex, Filter, Seconds};
9
+use lru::LruCache;
10
+use parking_lot::RwLock;
11
+use std::collections::HashMap;
12
+use std::num::NonZeroUsize;
13
+use std::time::Duration;
14
+
15
+#[allow(unused_imports)]
16
+use tracing::{debug, error};
17
+
18
+/// Calculate the appropriate bucket duration for a given time range.
19
+///
20
+/// This function determines the bucket size that will result in approximately
21
+/// 50-100 buckets for the given time range. The bucket durations are selected
22
+/// from a predefined set of "nice" values (1s, 2s, 5s, 10s, 1m, 5m, 1h, etc.)
23
+/// to make the resulting histograms easy to interpret.
24
+///
25
+/// # Arguments
26
+/// * `time_range_duration` - The duration of the time range in seconds
27
+///
28
+/// # Returns
29
+/// The bucket duration in seconds
30
+pub fn calculate_bucket_duration(time_range_duration: u32) -> u32 {
31
+ const MINUTE: Duration = Duration::from_secs(60);
32
+ const HOUR: Duration = Duration::from_secs(60 * MINUTE.as_secs());
33
+ const DAY: Duration = Duration::from_secs(24 * HOUR.as_secs());
34
+
35
+ const VALID_DURATIONS: &[Duration] = &[
36
+ // Seconds
37
+ Duration::from_secs(1),
38
+ Duration::from_secs(2),
39
+ Duration::from_secs(5),
40
+ Duration::from_secs(10),
41
+ Duration::from_secs(15),
42
+ Duration::from_secs(30),
43
+ // Minutes
44
+ MINUTE,
45
+ Duration::from_secs(2 * MINUTE.as_secs()),
46
+ Duration::from_secs(3 * MINUTE.as_secs()),
47
+ Duration::from_secs(5 * MINUTE.as_secs()),
48
+ Duration::from_secs(10 * MINUTE.as_secs()),
49
+ Duration::from_secs(15 * MINUTE.as_secs()),
50
+ Duration::from_secs(30 * MINUTE.as_secs()),
51
+ // Hours
52
+ HOUR,
53
+ Duration::from_secs(2 * HOUR.as_secs()),
54
+ Duration::from_secs(6 * HOUR.as_secs()),
55
+ Duration::from_secs(8 * HOUR.as_secs()),
56
+ Duration::from_secs(12 * HOUR.as_secs()),
57
+ // Days
58
+ DAY,
59
+ Duration::from_secs(2 * DAY.as_secs()),
60
+ Duration::from_secs(3 * DAY.as_secs()),
61
+ Duration::from_secs(5 * DAY.as_secs()),
62
+ Duration::from_secs(7 * DAY.as_secs()),
63
+ Duration::from_secs(14 * DAY.as_secs()),
64
+ Duration::from_secs(30 * DAY.as_secs()),
65
+ ];
66
+
67
+ VALID_DURATIONS
68
+ .iter()
69
+ .rev()
70
+ .find(|&&bucket_width| time_range_duration as u64 / bucket_width.as_secs() >= 50)
71
+ .map(|d| d.as_secs())
72
+ .unwrap_or(1) as u32
73
+}
74
+
75
+/// A bucket request contains a [start, end) time range along with the
76
+/// filter that should be applied.
77
+#[derive(Debug, Clone, Eq, PartialEq, Hash)]
78
+pub struct BucketRequest {
79
+ /// Start time of the bucket request
80
+ pub start: Seconds,
81
+ /// End time of the bucket request
82
+ pub end: Seconds,
83
+ /// Facets to use for file index
84
+ pub facets: Facets,
85
+ /// Applied filter expression
86
+ pub filter_expr: Filter,
87
+}
88
+
89
+impl BucketRequest {
90
+ /// The duration of the bucket request in seconds
91
+ pub fn duration(&self) -> Seconds {
92
+ self.end - self.start
93
+ }
94
+}
95
+
96
+/// A bucket response containing aggregated field value counts.
97
+#[derive(Debug, Clone)]
98
+pub struct BucketResponse {
99
+ /// Maps field=value pairs to (unfiltered, filtered) counts
100
+ pub fv_counts: HashMap<FieldValuePair, (usize, usize)>,
101
+ /// Set of fields that are not indexed
102
+ pub unindexed_fields: HashSet<FieldName>,
103
+}
104
+
105
+impl BucketResponse {
106
+ /// Creates a new empty bucket response.
107
+ pub(crate) fn new() -> Self {
108
+ Self {
109
+ fv_counts: HashMap::default(),
110
+ unindexed_fields: HashSet::default(),
111
+ }
112
+ }
113
+
114
+ /// Get all indexed field names from this bucket response.
115
+ pub fn indexed_fields(&self) -> HashSet<FieldName> {
116
+ self.fv_counts
117
+ .keys()
118
+ .map(|pair| pair.extract_field())
119
+ .collect()
120
+ }
121
+}
122
+
123
+/// Represents a histogram of journal log entries over time.
124
+///
125
+/// A histogram contains bucketed data where each bucket represents a time range
126
+/// and holds aggregated counts of field values and filtering results.
127
+#[derive(Debug, Clone)]
128
+pub struct Histogram {
129
+ pub buckets: Vec<(BucketRequest, BucketResponse)>,
130
+}
131
+
132
+impl Histogram {
133
+ /// Returns the start time of the histogram (first bucket's start time).
134
+ pub fn start_time(&self) -> Seconds {
135
+ let bucket_request = &self
136
+ .buckets
137
+ .first()
138
+ .expect("histogram with at least one bucket")
139
+ .0;
140
+ bucket_request.start
141
+ }
142
+
143
+ /// Returns the end time of the histogram (last bucket's end time).
144
+ pub fn end_time(&self) -> Seconds {
145
+ let bucket_request = &self
146
+ .buckets
147
+ .last()
148
+ .expect("histogram with at least one bucket")
149
+ .0;
150
+ bucket_request.end
151
+ }
152
+
153
+ /// Returns the duration of each bucket in seconds.
154
+ pub fn bucket_duration(&self) -> Seconds {
155
+ self.buckets
156
+ .first()
157
+ .expect("histogram with at least one bucket")
158
+ .0
159
+ .duration()
160
+ }
161
+
162
+ /// Returns all discovered field names from the histogram buckets in a deterministic order.
163
+ pub fn discovered_fields(&self) -> Vec<FieldName> {
164
+ // Collect all unique fields from all buckets
165
+ let mut fields = HashSet::default();
166
+ for (_, bucket_response) in &self.buckets {
167
+ fields.extend(bucket_response.indexed_fields());
168
+ fields.extend(bucket_response.unindexed_fields.iter().cloned());
169
+ }
170
+
171
+ let mut v: Vec<FieldName> = fields.into_iter().collect();
172
+ v.sort();
173
+ v
174
+ }
175
+}
176
+
177
+/// Engine for computing histograms from journal files.
178
+///
179
+/// The engine maintains caches and resources for efficiently computing histograms
180
+/// across multiple queries. It can be reused for multiple histogram computations.
181
+pub struct HistogramEngine {
182
+ responses: RwLock<LruCache<BucketRequest, BucketResponse>>,
183
+}
184
+
185
+impl HistogramEngine {
186
+ /// Creates a new HistogramEngine with a default capacity of 1000 bucket responses.
187
+ pub fn new() -> Self {
188
+ Self::with_capacity(1000)
189
+ }
190
+
191
+ /// Creates a new HistogramEngine with the specified cache capacity.
192
+ ///
193
+ /// The capacity determines how many bucket responses will be cached before
194
+ /// old entries are evicted using an LRU policy.
195
+ pub fn with_capacity(capacity: usize) -> Self {
196
+ Self {
197
+ responses: RwLock::new(LruCache::new(
198
+ NonZeroUsize::new(capacity).expect("capacity must be non-zero")
199
+ )),
200
+ }
201
+ }
202
+
203
+ /// Compute a histogram from pre-indexed files.
204
+ ///
205
+ /// This method allows you to compute histograms from file indexes that have
206
+ /// already been loaded, avoiding redundant cache lookups and file discoveries.
207
+ ///
208
+ /// # Arguments
209
+ /// * `indexed_files` - Pre-computed file indexes
210
+ /// * `time_range` - Query time range with aligned boundaries and bucket duration
211
+ /// * `facets` - Fields to index
212
+ /// * `filter_expr` - Filter expression to apply
213
+ pub fn compute_from_indexes(
214
+ &self,
215
+ indexed_files: &[(FileIndexKey, FileIndex)],
216
+ time_range: &crate::QueryTimeRange,
217
+ facets: &[String],
218
+ filter_expr: &Filter,
219
+ ) -> Result<Histogram> {
220
+ // Generate bucket requests from time range
221
+ let facets = Facets::new(facets);
222
+ let bucket_requests: Vec<BucketRequest> = time_range
223
+ .buckets()
224
+ .map(|(start, end)| BucketRequest {
225
+ start: Seconds(start),
226
+ end: Seconds(end),
227
+ facets: facets.clone(),
228
+ filter_expr: filter_expr.clone(),
229
+ })
230
+ .collect();
231
+
232
+ // Find buckets that need computation
233
+ let buckets_to_compute: Vec<BucketRequest> = {
234
+ let responses = self.responses.read();
235
+
236
+ bucket_requests
237
+ .iter()
238
+ .filter(|br| !responses.contains(br))
239
+ .cloned()
240
+ .collect()
241
+ };
242
+
243
+ if !buckets_to_compute.is_empty() {
244
+ // Initialize responses for buckets we need to compute
245
+ let mut new_responses: HashMap<BucketRequest, BucketResponse> = buckets_to_compute
246
+ .iter()
247
+ .map(|br| (br.clone(), BucketResponse::new()))
248
+ .collect();
249
+
250
+ // Track which buckets can be cached (no online file contributions)
251
+ let mut bucket_cacheable: HashMap<BucketRequest, bool> = buckets_to_compute
252
+ .iter()
253
+ .map(|br| (br.clone(), true))
254
+ .collect();
255
+
256
+ // Process all file indexes and update responses
257
+ for (_, file_index) in indexed_files {
258
+ let is_online = file_index.online();
259
+ // Get file's time range from the index
260
+ let file_start = file_index.start_time();
261
+ let file_end = file_index.end_time();
262
+
263
+ // Find all bucket requests that need data from this file
264
+ for bucket_request in &buckets_to_compute {
265
+ let response = match new_responses.get_mut(bucket_request) {
266
+ Some(r) => r,
267
+ None => continue,
268
+ };
269
+
270
+ // Skip if file's time range doesn't overlap with bucket's time range
271
+ if file_start >= bucket_request.end || file_end <= bucket_request.start {
272
+ continue;
273
+ }
274
+
275
+ // If this file is online and overlaps with the bucket, mark bucket as non-cacheable
276
+ if is_online {
277
+ bucket_cacheable.insert(bucket_request.clone(), false);
278
+ }
279
+
280
+ // Evaluate filter to bitmap
281
+ let filter_bitmap = if !bucket_request.filter_expr.is_none() {
282
+ Some(bucket_request.filter_expr.evaluate(file_index))
283
+ } else {
284
+ None
285
+ };
286
+
287
+ // Track unindexed fields
288
+ for field in file_index.fields() {
289
+ if !file_index.is_indexed(field) {
290
+ if let Some(field_name) = FieldName::new(field) {
291
+ response.unindexed_fields.insert(field_name);
292
+ }
293
+ }
294
+ }
295
+
296
+ // Count field=value pairs in this file for this bucket's time range
297
+ for (indexed_field, field_bitmap) in file_index.bitmaps() {
298
+ let unfiltered_count = file_index
299
+ .count_entries_in_time_range(
300
+ field_bitmap,
301
+ bucket_request.start,
302
+ bucket_request.end,
303
+ )
304
+ .unwrap_or(0);
305
+
306
+ let filtered_count = if let Some(ref filter_bitmap) = filter_bitmap {
307
+ let filtered_bitmap = field_bitmap & filter_bitmap;
308
+ file_index
309
+ .count_entries_in_time_range(
310
+ &filtered_bitmap,
311
+ bucket_request.start,
312
+ bucket_request.end,
313
+ )
314
+ .unwrap_or(0)
315
+ } else {
316
+ unfiltered_count
317
+ };
318
+
319
+ // Update counts
320
+ if let Some(pair) = FieldValuePair::parse(indexed_field) {
321
+ let counts = response.fv_counts.entry(pair).or_insert((0, 0));
322
+ counts.0 += unfiltered_count;
323
+ counts.1 += filtered_count;
324
+ }
325
+ }
326
+ }
327
+ }
328
+
329
+ // Cache only the responses that are safe to cache (no online file contributions)
330
+ let mut responses_guard = self.responses.write();
331
+ for (bucket_request, response) in &new_responses {
332
+ if bucket_cacheable.get(bucket_request).copied().unwrap_or(false) {
333
+ responses_guard.put(bucket_request.clone(), response.clone());
334
+ }
335
+ }
336
+ drop(responses_guard);
337
+
338
+ // Build histogram from all responses (cached + newly computed non-cacheable)
339
+ let mut responses_guard = self.responses.write();
340
+ let buckets = bucket_requests
341
+ .into_iter()
342
+ .filter_map(|bucket_request| {
343
+ // Try to get from cache first (updates LRU), then from newly computed responses
344
+ let response = responses_guard
345
+ .get(&bucket_request)
346
+ .cloned()
347
+ .or_else(|| new_responses.get(&bucket_request).cloned());
348
+
349
+ response.map(|r| (bucket_request, r))
350
+ })
351
+ .collect();
352
+
353
+ Ok(Histogram { buckets })
354
+ } else {
355
+ // All buckets were cached, just build histogram from cache
356
+ let mut responses = self.responses.write();
357
+ let buckets = bucket_requests
358
+ .into_iter()
359
+ .filter_map(|bucket_request| {
360
+ // Use get() to update LRU order for accessed entries
361
+ responses
362
+ .get(&bucket_request)
363
+ .map(|response| (bucket_request, response.clone()))
364
+ })
365
+ .collect();
366
+
367
+ Ok(Histogram { buckets })
368
+ }
369
+ }
370
+}
src/crates/journal-engine/src/indexing.rs
new
+300
@@ -0,0 +1,300 @@
1
+//! Journal file indexing infrastructure.
2
+//!
3
+//! This module provides infrastructure for indexing journal files:
4
+//! - Batch parallel indexing with time budget enforcement
5
+//! - Cache builder for file indexes
6
+
7
+use crate::{
8
+ cache::{FileIndexCache, FileIndexKey},
9
+ error::{EngineError, Result},
10
+ query_time_range::QueryTimeRange,
11
+};
12
+use foundation::Timeout;
13
+use journal_index::{FileIndex, FileIndexer};
14
+use journal_registry::Registry;
15
+use tracing::{error, trace};
16
+
17
+// ============================================================================
18
+// File Index Cache Builder
19
+// ============================================================================
20
+
21
+/// Builder for constructing a FileIndexCache with custom configuration.
22
+pub struct FileIndexCacheBuilder {
23
+ cache_path: Option<std::path::PathBuf>,
24
+ memory_capacity: Option<usize>,
25
+ disk_capacity: Option<usize>,
26
+ block_size: Option<usize>,
27
+}
28
+
29
+impl FileIndexCacheBuilder {
30
+ /// Creates a new builder with no configuration.
31
+ ///
32
+ /// All options use defaults if not explicitly set:
33
+ /// - Cache path: temp directory + "journal-engine-cache"
34
+ /// - Memory capacity: 128 entries
35
+ /// - Disk capacity: 16 MB
36
+ /// - Block size: 4 MB
37
+ pub fn new() -> Self {
38
+ Self {
39
+ cache_path: None,
40
+ memory_capacity: None,
41
+ disk_capacity: None,
42
+ block_size: None,
43
+ }
44
+ }
45
+
46
+ /// Sets the cache directory path.
47
+ pub fn with_cache_path(mut self, path: impl Into<std::path::PathBuf>) -> Self {
48
+ self.cache_path = Some(path.into());
49
+ self
50
+ }
51
+
52
+ /// Sets the memory capacity (number of items to keep in memory).
53
+ pub fn with_memory_capacity(mut self, capacity: usize) -> Self {
54
+ self.memory_capacity = Some(capacity);
55
+ self
56
+ }
57
+
58
+ /// Sets the disk capacity in bytes.
59
+ pub fn with_disk_capacity(mut self, capacity: usize) -> Self {
60
+ self.disk_capacity = Some(capacity);
61
+ self
62
+ }
63
+
64
+ /// Sets the block size in bytes.
65
+ pub fn with_block_size(mut self, size: usize) -> Self {
66
+ self.block_size = Some(size);
67
+ self
68
+ }
69
+
70
+ /// Builds the FileIndexCache with the configured settings.
71
+ pub async fn build(self) -> Result<FileIndexCache> {
72
+ use foyer::{
73
+ BlockEngineBuilder, DeviceBuilder, FsDeviceBuilder, HybridCacheBuilder,
74
+ IoEngineBuilder, PsyncIoEngineBuilder,
75
+ };
76
+
77
+ // Compute defaults
78
+ let cache_path = self
79
+ .cache_path
80
+ .unwrap_or_else(|| std::env::temp_dir().join("journal-engine-cache"));
81
+ let memory_capacity = self.memory_capacity.unwrap_or(128);
82
+ let disk_capacity = self.disk_capacity.unwrap_or(16 * 1024 * 1024);
83
+ let block_size = self.block_size.unwrap_or(4 * 1024 * 1024);
84
+
85
+ // Ensure cache directory exists
86
+ std::fs::create_dir_all(&cache_path).map_err(|e| {
87
+ EngineError::Io(std::io::Error::other(format!(
88
+ "Failed to create cache directory: {}",
89
+ e
90
+ )))
91
+ })?;
92
+
93
+ // Build Foyer hybrid cache
94
+ let cache = HybridCacheBuilder::new()
95
+ .with_name("file-index-cache")
96
+ .with_policy(foyer::HybridCachePolicy::WriteOnInsertion)
97
+ .memory(memory_capacity)
98
+ .with_shards(4)
99
+ .storage()
100
+ .with_io_engine(PsyncIoEngineBuilder::new().build().await?)
101
+ .with_engine_config(
102
+ BlockEngineBuilder::new(
103
+ FsDeviceBuilder::new(&cache_path)
104
+ .with_capacity(disk_capacity)
105
+ .build()?,
106
+ )
107
+ .with_block_size(block_size),
108
+ )
109
+ .build()
110
+ .await?;
111
+
112
+ Ok(cache)
113
+ }
114
+}
115
+
116
+impl Default for FileIndexCacheBuilder {
117
+ fn default() -> Self {
118
+ Self::new()
119
+ }
120
+}
121
+
122
+// ============================================================================
123
+// Batch Processing
124
+// ============================================================================
125
+
126
+/// Batch computes file indexes in parallel using rayon, with cache checking and time budget enforcement.
127
+///
128
+/// This function:
129
+/// 1. Checks cache for all keys upfront
130
+/// 2. Identifies cache misses
131
+/// 3. Uses tokio::task to compute missing indexes in parallel
132
+/// 4. Inserts newly computed indexes into cache
133
+/// 5. Returns all results (cached + newly computed)
134
+///
135
+/// # Arguments
136
+/// * `cache` - The file index cache
137
+/// * `registry` - Registry to update with file metadata
138
+/// * `keys` - Vector of (file, facets, source_timestamp_field) to fetch/compute indexes for
139
+/// * `bucket_duration` - Duration of histogram buckets in seconds
140
+/// * `timeout` - Timeout for the entire operation (can be extended dynamically)
141
+///
142
+/// # Returns
143
+/// Vector of responses for each key. Successful responses contain the file index.
144
+/// If timeout expires, returns TimeBudgetExceeded error.
145
+pub async fn batch_compute_file_indexes(
146
+ cache: &FileIndexCache,
147
+ registry: &Registry,
148
+ keys: Vec<FileIndexKey>,
149
+ time_range: &QueryTimeRange,
150
+ timeout: Timeout,
151
+) -> Result<Vec<(FileIndexKey, FileIndex)>> {
152
+ let bucket_duration = time_range.bucket_duration_seconds();
153
+ // Phase 1: Batch check cache for all keys upfront
154
+ let cache_lookup_futures = keys.iter().map(|key| {
155
+ let key_clone = key.clone();
156
+ async move {
157
+ let cached = cache
158
+ .get(&key_clone)
159
+ .await
160
+ .map(|entry| entry.map(|e| e.value().clone()))
161
+ .map_err(|e| e.into());
162
+ (key_clone, cached)
163
+ }
164
+ });
165
+
166
+ let cache_lookup_results: Vec<(FileIndexKey, Result<Option<FileIndex>>)> =
167
+ tokio::time::timeout(
168
+ timeout.remaining(),
169
+ futures::future::join_all(cache_lookup_futures),
170
+ )
171
+ .await
172
+ .map_err(|_| EngineError::TimeBudgetExceeded)?;
173
+
174
+ // Phase 2: Separate cache hits from misses, check freshness and compatibility
175
+ let mut responses = Vec::with_capacity(keys.len());
176
+ let mut keys_to_compute = Vec::new();
177
+ let mut cache_hits = 0;
178
+ let mut cache_misses = 0;
179
+ let mut stale_entries = 0;
180
+ let mut incompatible_bucket = 0;
181
+
182
+ for (key, cache_lookup_result) in cache_lookup_results {
183
+ match cache_lookup_result {
184
+ Ok(Some(file_index)) => {
185
+ let fresh = file_index.is_fresh();
186
+ let bucket_ok = file_index.bucket_duration() <= bucket_duration
187
+ && bucket_duration.is_multiple_of(file_index.bucket_duration());
188
+
189
+ if fresh && bucket_ok {
190
+ // Cache hit with fresh data and compatible granularity
191
+ cache_hits += 1;
192
+ responses.push((key, file_index));
193
+ } else {
194
+ if !fresh {
195
+ stale_entries += 1;
196
+ }
197
+ if !bucket_ok {
198
+ incompatible_bucket += 1;
199
+ }
200
+ keys_to_compute.push(key);
201
+ }
202
+ }
203
+ Ok(None) => {
204
+ // Cache miss - need to compute
205
+ cache_misses += 1;
206
+ keys_to_compute.push(key);
207
+ }
208
+ Err(e) => {
209
+ error!("cached file index lookup error {}", e);
210
+ }
211
+ }
212
+ }
213
+
214
+ if timeout.is_expired() {
215
+ return Err(EngineError::TimeBudgetExceeded);
216
+ }
217
+
218
+ trace!(
219
+ "phase 2 summary: hits={}, misses={}, stale={}, incompatible_bucket={}",
220
+ cache_hits, cache_misses, stale_entries, incompatible_bucket
221
+ );
222
+
223
+ // Phase 3: Spawn single blocking task with rayon for parallel computation
224
+ let time_budget_remaining = timeout.remaining();
225
+
226
+ let compute_task = tokio::task::spawn_blocking(move || {
227
+ use rayon::prelude::*;
228
+ use std::sync::Arc;
229
+ use std::sync::atomic::{AtomicBool, Ordering};
230
+
231
+ let deadline = std::time::Instant::now() + time_budget_remaining;
232
+ let timed_out = Arc::new(AtomicBool::new(false));
233
+
234
+ keys_to_compute
235
+ .into_par_iter()
236
+ .map(|key| {
237
+ // Check time budget before processing
238
+ if std::time::Instant::now() >= deadline || timed_out.load(Ordering::Relaxed) {
239
+ timed_out.store(true, Ordering::Relaxed);
240
+ return (key, Err(EngineError::TimeBudgetExceeded));
241
+ }
242
+
243
+ let mut file_indexer = FileIndexer::default();
244
+ let result = file_indexer
245
+ .index(
246
+ &key.file,
247
+ key.source_timestamp_field.as_ref(),
248
+ key.facets.as_slice(),
249
+ bucket_duration,
250
+ )
251
+ .map_err(|e| e.into());
252
+
253
+ (key, result)
254
+ })
255
+ .collect::<Vec<(FileIndexKey, Result<FileIndex>)>>()
256
+ });
257
+
258
+ let computed_results = match tokio::time::timeout(time_budget_remaining, compute_task).await {
259
+ Ok(Ok(results)) => results,
260
+ Ok(Err(e)) => {
261
+ return Err(EngineError::Io(std::io::Error::new(
262
+ std::io::ErrorKind::Other,
263
+ format!("Blocking task panicked: {}", e),
264
+ )));
265
+ }
266
+ Err(_timeout) => {
267
+ // Note: the blocking task will continue running in background but
268
+ // we will ignore the results
269
+ return Err(EngineError::TimeBudgetExceeded);
270
+ }
271
+ };
272
+
273
+ // Phase 4: Update registry and cache, then collect responses
274
+ for (key, response) in computed_results {
275
+ match response {
276
+ Ok(index) => {
277
+ // Update registry and cache on success
278
+ registry.update_time_range(
279
+ &key.file,
280
+ index.start_time(),
281
+ index.end_time(),
282
+ index.indexed_at(),
283
+ index.online(),
284
+ );
285
+
286
+ cache.insert(key.clone(), index.clone());
287
+ responses.push((key, index));
288
+ }
289
+ Err(e) => {
290
+ error!(
291
+ "file index computation failed for file={}: {}",
292
+ key.file.path(),
293
+ e
294
+ );
295
+ }
296
+ }
297
+ }
298
+
299
+ Ok(responses)
300
+}
src/crates/journal-engine/src/lib.rs
new
+38
@@ -0,0 +1,38 @@
1
+//! Journal processing engine
2
+//!
3
+//! This crate provides the core engine for processing systemd journal files, including
4
+//! indexing, caching, querying, and histogram computation. It's designed to be independent
5
+//! of specific metrics implementations or UI frameworks.
6
+//!
7
+//! # Key Components
8
+//!
9
+//! - **IndexingEngine**: Background file indexing with worker pool and cache management
10
+//! - **HistogramEngine**: Time-series histogram computation over journal entries
11
+//! - **Log Queries**: Flexible log entry retrieval with filtering
12
+//! - **Facets**: Field selection configuration for indexing
13
+//!
14
+//! # Architecture
15
+//!
16
+//! The engine layer provides the foundational infrastructure that higher-level crates
17
+//! (like `journal-sql` and `journal-function`) build upon. It handles all the complexity
18
+//! of file indexing, caching strategies, and query execution.
19
+
20
+// Public modules
21
+pub mod cache;
22
+pub mod error;
23
+pub mod facets;
24
+pub mod histogram;
25
+pub mod indexing;
26
+pub mod logs;
27
+pub mod query_time_range;
28
+
29
+// Re-export key types for convenience
30
+pub use cache::{FileIndexCache, FileIndexKey};
31
+pub use error::{EngineError, Result};
32
+pub use facets::Facets;
33
+pub use histogram::{
34
+ BucketRequest, BucketResponse, Histogram, HistogramEngine, calculate_bucket_duration,
35
+};
36
+pub use indexing::{FileIndexCacheBuilder, batch_compute_file_indexes};
37
+pub use logs::{CellValue, ColumnInfo, LogEntryData, LogQuery, Table, entry_data_to_table};
38
+pub use query_time_range::QueryTimeRange;
src/crates/journal-engine/src/logs/mod.rs
new
+10
@@ -0,0 +1,10 @@
1
+//! Log entry formatting and display.
2
+//!
3
+//! This module provides generic types for converting log entries
4
+//! into formatted tables.
5
+
6
+pub mod query;
7
+pub mod table;
8
+
9
+pub use query::{LogEntryData, LogQuery};
10
+pub use table::{CellValue, ColumnInfo, Table, entry_data_to_table};
src/crates/journal-engine/src/logs/query.rs
new
+526
@@ -0,0 +1,526 @@
1
+//! Log querying from indexed journal files.
2
+//!
3
+//! This module provides the `LogQuery` builder for efficiently querying and
4
+//! merging log entries from multiple indexed journal files, as well as
5
+//! functions for extracting raw field data from journal entries.
6
+
7
+use crate::error::Result;
8
+use journal_core::file::{JournalFile, Mmap};
9
+use journal_index::{
10
+ Anchor, Direction, FieldName, FieldValuePair, FileIndex, Filter, LogEntryId, LogQueryParams,
11
+ LogQueryParamsBuilder, Microseconds,
12
+};
13
+use journal_registry::File;
14
+use std::collections::HashMap;
15
+use std::num::NonZeroU64;
16
+
17
+/// Pagination state for multi-file log queries.
18
+///
19
+/// This tracks the position in each file where we stopped reading,
20
+/// allowing queries to resume efficiently without re-scanning entries.
21
+///
22
+/// The state is tied to a specific query configuration (filter, anchor, direction, etc).
23
+/// Changing the query parameters while using the same pagination state will produce
24
+/// undefined results.
25
+#[derive(Debug, Clone, Default)]
26
+pub struct PaginationState {
27
+ /// Maps each file to the last position we read from it
28
+ pub file_positions: HashMap<File, usize>,
29
+}
30
+
31
+/// Builder for configuring and executing log queries from indexed journal files.
32
+///
33
+/// This builder allows you to specify:
34
+/// - Direction (forward/backward in time)
35
+/// - Anchor timestamp (starting point)
36
+/// - Limit (maximum entries to retrieve)
37
+/// - Source timestamp field (which field to use for timestamps)
38
+/// - Filter (to match specific entries)
39
+///
40
+/// # Example
41
+///
42
+/// ```ignore
43
+/// use journal_index::{Anchor, Direction};
44
+/// use journal_function::logs::LogQuery;
45
+///
46
+/// let entries = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
47
+/// .with_limit(100)
48
+/// .execute();
49
+/// ```
50
+pub struct LogQuery<'a> {
51
+ file_indexes: &'a [FileIndex],
52
+ builder: LogQueryParamsBuilder,
53
+}
54
+
55
+impl<'a> LogQuery<'a> {
56
+ /// Create a new log query builder with required parameters.
57
+ ///
58
+ /// # Arguments
59
+ ///
60
+ /// * `file_indexes` - Journal file indexes to query
61
+ /// * `anchor` - Starting point for the query (Head, Tail, or specific timestamp)
62
+ /// * `direction` - Direction to iterate (Forward or Backward)
63
+ ///
64
+ /// # Optional Configuration
65
+ ///
66
+ /// Use builder methods to set optional parameters:
67
+ /// - Limit: None (unlimited)
68
+ /// - Source timestamp field: _SOURCE_REALTIME_TIMESTAMP
69
+ /// - Filter: None
70
+ pub fn new(file_indexes: &'a [FileIndex], anchor: Anchor, direction: Direction) -> Self {
71
+ Self {
72
+ file_indexes,
73
+ builder: LogQueryParamsBuilder::new(anchor, direction).with_source_timestamp_field(
74
+ Some(FieldName::new_unchecked("_SOURCE_REALTIME_TIMESTAMP")),
75
+ ),
76
+ }
77
+ }
78
+
79
+ /// Set the maximum number of log entries to retrieve (optional).
80
+ ///
81
+ /// If not set (None), all matching entries will be retrieved.
82
+ pub fn with_limit(mut self, limit: usize) -> Self {
83
+ self.builder = self.builder.with_limit(limit);
84
+ self
85
+ }
86
+
87
+ /// Set the source timestamp field to use for entry timestamps (optional).
88
+ ///
89
+ /// Pass `None` to use the entry's realtime timestamp from the journal header.
90
+ /// Pass `Some(field_name)` to use a custom timestamp field from the entry data.
91
+ pub fn with_source_timestamp_field(mut self, field: Option<FieldName>) -> Self {
92
+ self.builder = self.builder.with_source_timestamp_field(field);
93
+ self
94
+ }
95
+
96
+ /// Set a filter to apply to log entries (optional).
97
+ ///
98
+ /// Only entries matching the filter will be included in the results.
99
+ pub fn with_filter(mut self, filter: Filter) -> Self {
100
+ self.builder = self.builder.with_filter(filter);
101
+ self
102
+ }
103
+
104
+ /// Set the lower time boundary (inclusive) in microseconds (optional).
105
+ ///
106
+ /// Only entries with timestamp >= after_usec will be included.
107
+ /// This enforces a hard boundary regardless of anchor or limit.
108
+ pub fn with_after_usec(mut self, after: u64) -> Self {
109
+ self.builder = self.builder.with_after(Microseconds(after));
110
+ self
111
+ }
112
+
113
+ /// Set the upper time boundary (exclusive) in microseconds (optional).
114
+ ///
115
+ /// Only entries with timestamp < before_usec will be included.
116
+ /// This enforces a hard boundary regardless of anchor or limit.
117
+ pub fn with_before_usec(mut self, before: u64) -> Self {
118
+ self.builder = self.builder.with_before(Microseconds(before));
119
+ self
120
+ }
121
+
122
+ /// Set a regex pattern for full-text search (optional).
123
+ ///
124
+ /// Only entries where at least one data object (in "FIELD=value" format)
125
+ /// matches the regex will be included in the results.
126
+ ///
127
+ /// The pattern will be compiled when the query is executed. Invalid patterns
128
+ /// will cause execute() to return an error.
129
+ pub fn with_regex(mut self, pattern: impl Into<String>) -> Self {
130
+ self.builder = self.builder.with_regex(pattern);
131
+ self
132
+ }
133
+
134
+ /// Execute the query and return log entries.
135
+ ///
136
+ /// This consumes the builder and returns a vector of log entries sorted by timestamp
137
+ /// according to the configured direction.
138
+ ///
139
+ /// # Errors
140
+ ///
141
+ /// Returns an error if anchor or direction were not set, or if time boundaries are invalid.
142
+ pub fn execute(self) -> Result<Vec<LogEntryData>> {
143
+ let params = self.builder.build()?;
144
+ let (log_entry_ids, _state) =
145
+ retrieve_log_entries(self.file_indexes.to_vec(), params, None);
146
+
147
+ extract_entry_data(&log_entry_ids)
148
+ }
149
+
150
+ /// Execute the query with pagination support.
151
+ ///
152
+ /// This consumes the builder and returns a page of log entries along with
153
+ /// pagination state that can be used to retrieve the next page.
154
+ ///
155
+ /// # Arguments
156
+ ///
157
+ /// * `state` - Optional pagination state from a previous query. Pass `None` for the first page.
158
+ ///
159
+ /// # Returns
160
+ ///
161
+ /// Returns a tuple of (log entry data, new pagination state). If the pagination state
162
+ /// is empty (no file positions tracked), there are no more results.
163
+ ///
164
+ /// # Errors
165
+ ///
166
+ /// Returns an error if anchor or direction were not set, or if time boundaries are invalid.
167
+ pub fn execute_page(
168
+ self,
169
+ state: Option<&PaginationState>,
170
+ ) -> Result<(Vec<LogEntryData>, PaginationState)> {
171
+ let params = self.builder.build()?;
172
+ let (log_entry_ids, new_state) =
173
+ retrieve_log_entries(self.file_indexes.to_vec(), params, state);
174
+
175
+ let data = extract_entry_data(&log_entry_ids)?;
176
+ Ok((data, new_state))
177
+ }
178
+}
179
+
180
+/// Retrieve and merge log entries from multiple indexed journal files.
181
+///
182
+/// This function efficiently retrieves log entries from multiple journal files,
183
+/// merging them in timestamp order while respecting the limit constraint.
184
+///
185
+/// # Arguments
186
+///
187
+/// * `file_indexes` - Vector of indexed journal files to retrieve from
188
+/// * `params` - Query parameters (anchor, direction, limit, filter, boundaries)
189
+/// * `state` - Optional pagination state to resume from previous query
190
+///
191
+/// # Returns
192
+///
193
+/// A tuple of (log entries, new pagination state). The entries are sorted by timestamp
194
+/// and limited to `params.limit`. The new state can be used to resume the query.
195
+fn retrieve_log_entries(
196
+ file_indexes: Vec<FileIndex>,
197
+ params: LogQueryParams,
198
+ state: Option<&PaginationState>,
199
+) -> (Vec<LogEntryId>, PaginationState) {
200
+ // Handle edge cases
201
+ if params.limit() == Some(0) || file_indexes.is_empty() {
202
+ return (Vec::new(), PaginationState::default());
203
+ }
204
+
205
+ // Resolve anchor to concrete timestamp for multi-file queries
206
+ let anchor_usec = match params.anchor() {
207
+ Anchor::Timestamp(ts) => ts.get(),
208
+ Anchor::Head => {
209
+ // For Head: use minimum start time across all files
210
+ file_indexes
211
+ .iter()
212
+ .map(|fi| fi.start_time().to_microseconds().get())
213
+ .min()
214
+ .unwrap_or(0)
215
+ }
216
+ Anchor::Tail => {
217
+ // For Tail: use maximum end time across all files
218
+ file_indexes
219
+ .iter()
220
+ .map(|fi| fi.end_time().to_microseconds().get())
221
+ .max()
222
+ .unwrap_or(0)
223
+ }
224
+ };
225
+
226
+ // Filter to FileIndex instances that could contain relevant entries
227
+ let mut relevant_indexes: Vec<&FileIndex> = match params.direction() {
228
+ Direction::Forward => {
229
+ // For forward: end timestamp must be at or after the anchor
230
+ file_indexes
231
+ .iter()
232
+ .filter(|fi| fi.end_time().to_microseconds().get() >= anchor_usec)
233
+ .collect()
234
+ }
235
+ Direction::Backward => {
236
+ // For backward: start timestamp must be at or before the anchor
237
+ file_indexes
238
+ .iter()
239
+ .filter(|fi| fi.start_time().to_microseconds().get() <= anchor_usec)
240
+ .collect()
241
+ }
242
+ };
243
+
244
+ if relevant_indexes.is_empty() {
245
+ return (Vec::new(), PaginationState::default());
246
+ }
247
+
248
+ // Sort files to process them in temporal order
249
+ match params.direction() {
250
+ Direction::Forward => {
251
+ // Sort by start timestamp ascending to process files in temporal order
252
+ relevant_indexes.sort_by_key(|fi| fi.start_time());
253
+ }
254
+ Direction::Backward => {
255
+ // Sort by end timestamp descending to process files in reverse temporal order
256
+ relevant_indexes.sort_by_key(|fi| std::cmp::Reverse(fi.end_time()));
257
+ }
258
+ }
259
+
260
+ // Initialize result vector with capacity for efficiency
261
+ let (limit, mut collected_entries) = match params.limit() {
262
+ Some(limit) => (limit, Vec::with_capacity(limit)),
263
+ None => (usize::MAX, Vec::with_capacity(200)),
264
+ };
265
+
266
+ // Track the new pagination state, starting from the previous state if available
267
+ let mut new_state = state.cloned().unwrap_or_default();
268
+
269
+ for file_index in relevant_indexes {
270
+ // Pruning optimization: if we have a full result set, check if we can skip
271
+ // remaining files based on their time ranges
272
+ if collected_entries.len() >= limit {
273
+ if let Some(should_break) =
274
+ can_prune_file(file_index, &collected_entries, params.direction())
275
+ {
276
+ if should_break {
277
+ break;
278
+ }
279
+ }
280
+ }
281
+
282
+ // Perform I/O to retrieve entries from this FileIndex
283
+ let file = file_index.file();
284
+
285
+ // Check if we have a resume position for this file
286
+ let resume_position = state.and_then(|s| s.file_positions.get(file).copied());
287
+
288
+ // Create params with resume position if available
289
+ let file_params = if let Some(pos) = resume_position {
290
+ let mut builder = LogQueryParamsBuilder::new(params.anchor(), params.direction());
291
+ if let Some(limit) = params.limit() {
292
+ builder = builder.with_limit(limit);
293
+ }
294
+ if let Some(field) = params.source_timestamp_field() {
295
+ builder = builder.with_source_timestamp_field(Some(field.clone()));
296
+ }
297
+ if let Some(filter) = params.filter() {
298
+ builder = builder.with_filter(filter.clone());
299
+ }
300
+ if let Some(after) = params.after() {
301
+ builder = builder.with_after(after);
302
+ }
303
+ if let Some(before) = params.before() {
304
+ builder = builder.with_before(before);
305
+ }
306
+ if let Some(regex) = params.regex() {
307
+ builder = builder.with_regex(regex.as_str());
308
+ }
309
+ builder = builder.with_resume_position(pos);
310
+ builder.build().unwrap() // Safe because we're copying from valid params
311
+ } else {
312
+ params.clone()
313
+ };
314
+
315
+ let new_entries = match file_index.find_log_entries(file, &file_params) {
316
+ Ok(entries) => entries,
317
+ Err(_) => continue, // Skip files that fail to read
318
+ };
319
+
320
+ if new_entries.is_empty() {
321
+ continue;
322
+ }
323
+
324
+ // Merge the new entries with our existing results, maintaining
325
+ // sorted order and respecting the limit constraint
326
+ collected_entries =
327
+ merge_log_entries(collected_entries, new_entries, limit, params.direction());
328
+ }
329
+
330
+ // Update pagination state based on the last position for each file in collected_entries
331
+ // For forward direction: track the maximum position (we're progressing upward)
332
+ // For backward direction: track the minimum position (we're progressing downward)
333
+ for entry in &collected_entries {
334
+ new_state
335
+ .file_positions
336
+ .entry(entry.file.clone())
337
+ .and_modify(|pos| {
338
+ *pos = match params.direction() {
339
+ Direction::Forward => (*pos).max(entry.position),
340
+ Direction::Backward => (*pos).min(entry.position),
341
+ }
342
+ })
343
+ .or_insert(entry.position);
344
+ }
345
+
346
+ (collected_entries, new_state)
347
+}
348
+
349
+/// Check if we can prune (skip) a file based on its time range and current results.
350
+///
351
+/// Returns Some(true) if we should break early, Some(false) if we should continue,
352
+/// or None if we can't determine (shouldn't happen with a full result set).
353
+fn can_prune_file(
354
+ file_index: &FileIndex,
355
+ result: &[LogEntryId],
356
+ direction: Direction,
357
+) -> Option<bool> {
358
+ match direction {
359
+ Direction::Forward => {
360
+ // For forward: if file starts after our latest entry, skip all remaining files
361
+ let max_timestamp = result.last()?.timestamp.get();
362
+ Some(file_index.start_time().to_microseconds().get() > max_timestamp)
363
+ }
364
+ Direction::Backward => {
365
+ // For backward: if file ends before our earliest entry, skip all remaining files
366
+ let min_timestamp = result.first()?.timestamp.get();
367
+ Some(file_index.end_time().to_microseconds().get() < min_timestamp)
368
+ }
369
+ }
370
+}
371
+
372
+/// Merges two sorted vectors into a single sorted vector with at most `limit` elements.
373
+///
374
+/// This function performs a two-pointer merge, which is efficient for combining
375
+/// sorted sequences. It only retains the smallest/largest `limit` entries by timestamp
376
+/// depending on the direction.
377
+///
378
+/// # Arguments
379
+///
380
+/// * `a` - First sorted vector
381
+/// * `b` - Second sorted vector
382
+/// * `limit` - Maximum number of elements in the result
383
+/// * `direction` - Direction determines ascending (Forward) or descending (Backward) order
384
+///
385
+/// # Returns
386
+///
387
+/// A new vector containing the merged and limited results
388
+fn merge_log_entries(
389
+ a: Vec<LogEntryId>,
390
+ b: Vec<LogEntryId>,
391
+ limit: usize,
392
+ direction: Direction,
393
+) -> Vec<LogEntryId> {
394
+ // Handle simple cases
395
+ if a.is_empty() {
396
+ return b.into_iter().take(limit).collect();
397
+ }
398
+ if b.is_empty() {
399
+ return a.into_iter().take(limit).collect();
400
+ }
401
+
402
+ // Allocate result vector with appropriate capacity
403
+ let mut result = Vec::with_capacity(limit);
404
+ let mut i = 0;
405
+ let mut j = 0;
406
+
407
+ // Two-pointer merge: always take the appropriate element based on direction
408
+ while result.len() < limit {
409
+ let take_from_a = match (i < a.len(), j < b.len()) {
410
+ (true, false) => true,
411
+ (false, true) => false,
412
+ (false, false) => break,
413
+ (true, true) => match direction {
414
+ Direction::Forward => a[i].timestamp <= b[j].timestamp,
415
+ Direction::Backward => a[i].timestamp >= b[j].timestamp,
416
+ },
417
+ };
418
+
419
+ if take_from_a {
420
+ result.push(a[i].clone());
421
+ i += 1;
422
+ } else {
423
+ result.push(b[j].clone());
424
+ j += 1;
425
+ }
426
+ }
427
+
428
+ result
429
+}
430
+
431
+/// Raw field data extracted from a journal entry.
432
+///
433
+/// This is an intermediate representation between a `LogEntryId` (which only contains
434
+/// a file offset) and format-specific structures like `Table`, Arrow `RecordBatch`,
435
+/// or columnar data.
436
+///
437
+/// The fields are stored as `FieldValuePair` objects, which efficiently store the
438
+/// field name and value with a cached split position for fast access.
439
+#[derive(Debug, Clone)]
440
+pub struct LogEntryData {
441
+ /// Timestamp of the entry in microseconds since epoch
442
+ pub timestamp: u64,
443
+ /// All field=value pairs in this entry
444
+ pub fields: Vec<FieldValuePair>,
445
+}
446
+
447
+/// Extracts raw field data from multiple log entries efficiently.
448
+///
449
+/// This function groups entries by file and processes them in batches,
450
+/// minimizing file open/close overhead. It reads the journal files and
451
+/// extracts all field=value pairs without applying any transformations.
452
+///
453
+/// # Arguments
454
+///
455
+/// * `log_entries` - Slice of log entry IDs to extract data from
456
+///
457
+/// # Returns
458
+///
459
+/// A vector of `LogEntryData` in the same order as the input entries
460
+fn extract_entry_data(log_entries: &[LogEntryId]) -> Result<Vec<LogEntryData>> {
461
+ // Group entries by file to minimize file open/close operations
462
+ let mut entries_by_file: HashMap<&File, Vec<(usize, &LogEntryId)>> = HashMap::new();
463
+ for (idx, entry) in log_entries.iter().enumerate() {
464
+ entries_by_file
465
+ .entry(&entry.file)
466
+ .or_default()
467
+ .push((idx, entry));
468
+ }
469
+
470
+ // Pre-allocate result vector with exact capacity
471
+ let mut result = vec![None; log_entries.len()];
472
+
473
+ // Process each file's entries
474
+ for (file, file_entries) in entries_by_file {
475
+ let journal_file = JournalFile::<Mmap>::open(file, 8 * 1024 * 1024)?;
476
+
477
+ // Load and reverse the field mapping (systemd -> OTEL)
478
+ // This allows us to reverse-map systemd field names back to their original OTEL names
479
+ let field_map = journal_file.load_fields()?;
480
+ let reverse_map: HashMap<String, String> = field_map
481
+ .into_iter()
482
+ .map(|(otel, systemd)| (systemd, otel))
483
+ .collect();
484
+
485
+ let mut data_offsets = Vec::new();
486
+
487
+ for (original_idx, entry) in file_entries {
488
+ // Read the entry at the specified offset
489
+ let entry_offset =
490
+ NonZeroU64::new(entry.offset).ok_or(journal_core::JournalError::InvalidOffset)?;
491
+ let entry_guard = journal_file.entry_ref(entry_offset)?;
492
+
493
+ // Collect all data object offsets for this entry
494
+ data_offsets.clear();
495
+ entry_guard.collect_offsets(&mut data_offsets)?;
496
+ drop(entry_guard);
497
+
498
+ // Extract all field=value pairs
499
+ let mut fields = Vec::new();
500
+ for data_offset in data_offsets.iter().copied() {
501
+ let data_guard = journal_file.data_ref(data_offset)?;
502
+ let payload_bytes = data_guard.payload_bytes();
503
+ let payload_str = String::from_utf8_lossy(payload_bytes);
504
+
505
+ if let Some(mut pair) = FieldValuePair::parse(&payload_str) {
506
+ // Reverse-map systemd field name back to OTEL name if needed
507
+ if let Some(otel_name) = reverse_map.get(pair.field()) {
508
+ pair = FieldValuePair::new_unchecked(
509
+ FieldName::new_unchecked(otel_name),
510
+ pair.value().to_string(),
511
+ );
512
+ }
513
+ fields.push(pair);
514
+ }
515
+ }
516
+
517
+ result[original_idx] = Some(LogEntryData {
518
+ timestamp: entry.timestamp.get(),
519
+ fields,
520
+ });
521
+ }
522
+ }
523
+
524
+ // Unwrap all Options (they're all Some at this point)
525
+ Ok(result.into_iter().map(|opt| opt.unwrap()).collect())
526
+}
src/crates/journal-engine/src/logs/table.rs
new
+209
@@ -0,0 +1,209 @@
1
+use super::query::LogEntryData;
2
+use journal_core::Result;
3
+use std::collections::HashMap;
4
+use std::fmt;
5
+
6
+/// A cell value with both raw and display representations
7
+#[derive(Debug, Clone)]
8
+pub struct CellValue {
9
+ pub raw: Option<String>,
10
+ pub display: Option<String>,
11
+}
12
+
13
+impl CellValue {
14
+ /// Create a new cell value with no transformation
15
+ pub fn new(value: Option<String>) -> Self {
16
+ Self {
17
+ raw: value.clone(),
18
+ display: value,
19
+ }
20
+ }
21
+
22
+ /// Create a new cell value with separate raw and display representations
23
+ pub fn with_display(raw: Option<String>, display: Option<String>) -> Self {
24
+ Self { raw, display }
25
+ }
26
+}
27
+
28
+/// Column metadata for a table, compatible with the JSON response format
29
+#[derive(Debug, Clone)]
30
+pub struct ColumnInfo {
31
+ pub name: String,
32
+ pub index: usize,
33
+}
34
+
35
+impl ColumnInfo {
36
+ pub fn new(name: String, index: usize) -> Self {
37
+ Self { name, index }
38
+ }
39
+}
40
+
41
+/// A table representation of log entries with extracted field values
42
+#[derive(Debug, Clone)]
43
+pub struct Table {
44
+ pub columns: Vec<ColumnInfo>,
45
+ pub data: Vec<Vec<CellValue>>,
46
+}
47
+
48
+impl Table {
49
+ /// Create a new empty table with the given column names
50
+ pub fn new(column_names: Vec<String>) -> Self {
51
+ let columns = column_names
52
+ .into_iter()
53
+ .enumerate()
54
+ .map(|(index, name)| ColumnInfo::new(name, index))
55
+ .collect();
56
+
57
+ Self {
58
+ columns,
59
+ data: Vec::new(),
60
+ }
61
+ }
62
+
63
+ /// Add a row to the table
64
+ pub fn add_row(&mut self, row: Vec<CellValue>) {
65
+ self.data.push(row);
66
+ }
67
+
68
+ /// Get the number of rows in the table
69
+ pub fn row_count(&self) -> usize {
70
+ self.data.len()
71
+ }
72
+
73
+ /// Get the number of columns in the table
74
+ pub fn column_count(&self) -> usize {
75
+ self.columns.len()
76
+ }
77
+
78
+ /// Get the column metadata
79
+ pub fn columns(&self) -> &[ColumnInfo] {
80
+ &self.columns
81
+ }
82
+
83
+ /// Get the table rows
84
+ pub fn rows(&self) -> &[Vec<CellValue>] {
85
+ &self.data
86
+ }
87
+
88
+ /// Calculate the optimal column widths for display
89
+ fn calculate_column_widths(&self) -> Vec<usize> {
90
+ const MESSAGE_MAX_WIDTH: usize = 80;
91
+
92
+ let mut widths: Vec<usize> = self.columns.iter().map(|col| col.name.len()).collect();
93
+
94
+ // Check each row to find the maximum width needed for each column
95
+ for row in &self.data {
96
+ for (col_idx, cell) in row.iter().enumerate() {
97
+ let display_len = cell.display.as_deref().unwrap_or("-").len();
98
+ if display_len > widths[col_idx] {
99
+ widths[col_idx] = display_len;
100
+ }
101
+ }
102
+ }
103
+
104
+ // Cap the MESSAGE column width at MESSAGE_MAX_WIDTH
105
+ for (col_idx, col) in self.columns.iter().enumerate() {
106
+ if col.name == "MESSAGE" && widths[col_idx] > MESSAGE_MAX_WIDTH {
107
+ widths[col_idx] = MESSAGE_MAX_WIDTH;
108
+ }
109
+ }
110
+
111
+ widths
112
+ }
113
+}
114
+
115
+impl fmt::Display for Table {
116
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
117
+ if self.columns.is_empty() {
118
+ return writeln!(f, "(empty table)");
119
+ }
120
+
121
+ let widths = self.calculate_column_widths();
122
+ let total_width: usize = widths.iter().sum::<usize>() + (widths.len() - 1) * 3 + 2;
123
+
124
+ // Print top border
125
+ writeln!(f, "{}", "=".repeat(total_width))?;
126
+
127
+ // Print header
128
+ write!(f, "|")?;
129
+ for (col, width) in self.columns.iter().zip(&widths) {
130
+ write!(f, " {:<width$} |", col.name, width = width)?;
131
+ }
132
+ writeln!(f)?;
133
+
134
+ // Print separator
135
+ writeln!(f, "{}", "=".repeat(total_width))?;
136
+
137
+ // Print rows
138
+ for row in &self.data {
139
+ write!(f, "|")?;
140
+ for (cell, width) in row.iter().zip(&widths) {
141
+ let display = cell.display.as_deref().unwrap_or("-");
142
+ // Truncate if the value is longer than the column width
143
+ if display.len() > *width {
144
+ let truncated = &display[..*width];
145
+ write!(f, " {:<width$} |", truncated, width = width)?;
146
+ } else {
147
+ write!(f, " {:<width$} |", display, width = width)?;
148
+ }
149
+ }
150
+ writeln!(f)?;
151
+ }
152
+
153
+ // Print bottom border
154
+ writeln!(f, "{}", "=".repeat(total_width))?;
155
+
156
+ Ok(())
157
+ }
158
+}
159
+
160
+/// Converts extracted entry data into a table with specified columns.
161
+///
162
+/// This function takes raw field data and builds a table structure.
163
+/// It only extracts fields that are in the requested columns list.
164
+///
165
+/// # Arguments
166
+///
167
+/// * `entry_data` - Vector of extracted entry data
168
+/// * `column_names` - Names of fields to include (timestamp is always prepended)
169
+///
170
+/// # Returns
171
+///
172
+/// A `Table` containing the raw field values
173
+pub fn entry_data_to_table(
174
+ entry_data: &[LogEntryData],
175
+ column_names: Vec<String>,
176
+) -> Result<Table> {
177
+ // Always prepend "timestamp" as the first column
178
+ let mut all_columns = vec!["timestamp".to_string()];
179
+ all_columns.extend(column_names.clone());
180
+
181
+ let mut table = Table::new(all_columns);
182
+
183
+ // Create a mapping from column name to index for fast lookup
184
+ let column_map: HashMap<&str, usize> = column_names
185
+ .iter()
186
+ .enumerate()
187
+ .map(|(idx, name)| (name.as_str(), idx + 1)) // +1 because timestamp is at index 0
188
+ .collect();
189
+
190
+ // Process each entry
191
+ for data in entry_data {
192
+ let num_cols = column_names.len() + 1;
193
+ let mut row = vec![CellValue::new(None); num_cols];
194
+
195
+ // First column: timestamp
196
+ row[0] = CellValue::new(Some(data.timestamp.to_string()));
197
+
198
+ // Extract requested fields
199
+ for pair in &data.fields {
200
+ if let Some(&col_idx) = column_map.get(pair.field()) {
201
+ row[col_idx] = CellValue::new(Some(pair.value().to_string()));
202
+ }
203
+ }
204
+
205
+ table.add_row(row);
206
+ }
207
+
208
+ Ok(table)
209
+}
src/crates/journal-engine/src/logs/transformations.rs
new
+589
@@ -0,0 +1,589 @@
1
+use std::collections::HashMap;
2
+use std::sync::Arc;
3
+
4
+/// Trait for field transformations
5
+pub trait FieldTransformation: Send + Sync {
6
+ /// Transform a raw field value to a display representation
7
+ fn transform(&self, raw_value: &str) -> String;
8
+}
9
+
10
+/// Registry of field transformations
11
+#[derive(Clone)]
12
+pub struct TransformationRegistry {
13
+ transformations: HashMap<String, Arc<dyn FieldTransformation>>,
14
+}
15
+
16
+impl TransformationRegistry {
17
+ /// Create a new empty registry
18
+ pub fn new() -> Self {
19
+ Self {
20
+ transformations: HashMap::new(),
21
+ }
22
+ }
23
+
24
+ /// Register a transformation for a specific field
25
+ pub fn register(&mut self, field_name: impl Into<String>, transform: Arc<dyn FieldTransformation>) {
26
+ self.transformations.insert(field_name.into(), transform);
27
+ }
28
+
29
+ /// Transform a field value using the registered transformation
30
+ pub fn transform_field(&self, field_name: &str, raw: Option<String>) -> super::table::CellValue {
31
+ match raw {
32
+ None => super::table::CellValue::new(None),
33
+ Some(raw_str) => {
34
+ let display = self
35
+ .transformations
36
+ .get(field_name)
37
+ .map(|t| t.transform(&raw_str))
38
+ .unwrap_or_else(|| raw_str.clone());
39
+
40
+ super::table::CellValue::with_display(Some(raw_str), Some(display))
41
+ }
42
+ }
43
+ }
44
+}
45
+
46
+impl Default for TransformationRegistry {
47
+ fn default() -> Self {
48
+ Self::new()
49
+ }
50
+}
51
+
52
+/// PRIORITY: 0-7 → human-readable names
53
+pub struct PriorityTransformation;
54
+
55
+impl FieldTransformation for PriorityTransformation {
56
+ fn transform(&self, raw_value: &str) -> String {
57
+ match raw_value {
58
+ "0" => "panic".to_string(),
59
+ "1" => "alert".to_string(),
60
+ "2" => "critical".to_string(),
61
+ "3" => "error".to_string(),
62
+ "4" => "warning".to_string(),
63
+ "5" => "notice".to_string(),
64
+ "6" => "info".to_string(),
65
+ "7" => "debug".to_string(),
66
+ _ => raw_value.to_string(),
67
+ }
68
+ }
69
+}
70
+
71
+/// SYSLOG_FACILITY: 0-23 → facility names
72
+pub struct SyslogFacilityTransformation;
73
+
74
+impl FieldTransformation for SyslogFacilityTransformation {
75
+ fn transform(&self, raw_value: &str) -> String {
76
+ match raw_value {
77
+ "0" => "kern".to_string(),
78
+ "1" => "user".to_string(),
79
+ "2" => "mail".to_string(),
80
+ "3" => "daemon".to_string(),
81
+ "4" => "auth".to_string(),
82
+ "5" => "syslog".to_string(),
83
+ "6" => "lpr".to_string(),
84
+ "7" => "news".to_string(),
85
+ "8" => "uucp".to_string(),
86
+ "9" => "cron".to_string(),
87
+ "10" => "authpriv".to_string(),
88
+ "11" => "ftp".to_string(),
89
+ "12" => "ntp".to_string(),
90
+ "13" => "security".to_string(),
91
+ "14" => "console".to_string(),
92
+ "15" => "solaris-cron".to_string(),
93
+ "16" => "local0".to_string(),
94
+ "17" => "local1".to_string(),
95
+ "18" => "local2".to_string(),
96
+ "19" => "local3".to_string(),
97
+ "20" => "local4".to_string(),
98
+ "21" => "local5".to_string(),
99
+ "22" => "local6".to_string(),
100
+ "23" => "local7".to_string(),
101
+ _ => raw_value.to_string(),
102
+ }
103
+ }
104
+}
105
+
106
+/// ERRNO: numeric → "code (name)"
107
+pub struct ErrnoTransformation;
108
+
109
+impl FieldTransformation for ErrnoTransformation {
110
+ fn transform(&self, raw_value: &str) -> String {
111
+ let name = match raw_value {
112
+ "1" => "EPERM",
113
+ "2" => "ENOENT",
114
+ "3" => "ESRCH",
115
+ "4" => "EINTR",
116
+ "5" => "EIO",
117
+ "6" => "ENXIO",
118
+ "7" => "E2BIG",
119
+ "8" => "ENOEXEC",
120
+ "9" => "EBADF",
121
+ "10" => "ECHILD",
122
+ "11" => "EAGAIN",
123
+ "12" => "ENOMEM",
124
+ "13" => "EACCES",
125
+ "14" => "EFAULT",
126
+ "15" => "ENOTBLK",
127
+ "16" => "EBUSY",
128
+ "17" => "EEXIST",
129
+ "18" => "EXDEV",
130
+ "19" => "ENODEV",
131
+ "20" => "ENOTDIR",
132
+ "21" => "EISDIR",
133
+ "22" => "EINVAL",
134
+ "23" => "ENFILE",
135
+ "24" => "EMFILE",
136
+ "25" => "ENOTTY",
137
+ "26" => "ETXTBSY",
138
+ "27" => "EFBIG",
139
+ "28" => "ENOSPC",
140
+ "29" => "ESPIPE",
141
+ "30" => "EROFS",
142
+ "31" => "EMLINK",
143
+ "32" => "EPIPE",
144
+ "33" => "EDOM",
145
+ "34" => "ERANGE",
146
+ "35" => "EDEADLK",
147
+ "36" => "ENAMETOOLONG",
148
+ "37" => "ENOLCK",
149
+ "38" => "ENOSYS",
150
+ "39" => "ENOTEMPTY",
151
+ "40" => "ELOOP",
152
+ "42" => "ENOMSG",
153
+ "43" => "EIDRM",
154
+ "44" => "ECHRNG",
155
+ "45" => "EL2NSYNC",
156
+ "46" => "EL3HLT",
157
+ "47" => "EL3RST",
158
+ "48" => "ELNRNG",
159
+ "49" => "EUNATCH",
160
+ "50" => "ENOCSI",
161
+ "51" => "EL2HLT",
162
+ "52" => "EBADE",
163
+ "53" => "EBADR",
164
+ "54" => "EXFULL",
165
+ "55" => "ENOANO",
166
+ "56" => "EBADRQC",
167
+ "57" => "EBADSLT",
168
+ "59" => "EBFONT",
169
+ "60" => "ENOSTR",
170
+ "61" => "ENODATA",
171
+ "62" => "ETIME",
172
+ "63" => "ENOSR",
173
+ "64" => "ENONET",
174
+ "65" => "ENOPKG",
175
+ "66" => "EREMOTE",
176
+ "67" => "ENOLINK",
177
+ "68" => "EADV",
178
+ "69" => "ESRMNT",
179
+ "70" => "ECOMM",
180
+ "71" => "EPROTO",
181
+ "72" => "EMULTIHOP",
182
+ "73" => "EDOTDOT",
183
+ "74" => "EBADMSG",
184
+ "75" => "EOVERFLOW",
185
+ "76" => "ENOTUNIQ",
186
+ "77" => "EBADFD",
187
+ "78" => "EREMCHG",
188
+ "79" => "ELIBACC",
189
+ "80" => "ELIBBAD",
190
+ "81" => "ELIBSCN",
191
+ "82" => "ELIBMAX",
192
+ "83" => "ELIBEXEC",
193
+ "84" => "EILSEQ",
194
+ "85" => "ERESTART",
195
+ "86" => "ESTRPIPE",
196
+ "87" => "EUSERS",
197
+ "88" => "ENOTSOCK",
198
+ "89" => "EDESTADDRREQ",
199
+ "90" => "EMSGSIZE",
200
+ "91" => "EPROTOTYPE",
201
+ "92" => "ENOPROTOOPT",
202
+ "93" => "EPROTONOSUPPORT",
203
+ "94" => "ESOCKTNOSUPPORT",
204
+ "95" => "EOPNOTSUPP",
205
+ "96" => "EPFNOSUPPORT",
206
+ "97" => "EAFNOSUPPORT",
207
+ "98" => "EADDRINUSE",
208
+ "99" => "EADDRNOTAVAIL",
209
+ "100" => "ENETDOWN",
210
+ "101" => "ENETUNREACH",
211
+ "102" => "ENETRESET",
212
+ "103" => "ECONNABORTED",
213
+ "104" => "ECONNRESET",
214
+ "105" => "ENOBUFS",
215
+ "106" => "EISCONN",
216
+ "107" => "ENOTCONN",
217
+ "108" => "ESHUTDOWN",
218
+ "109" => "ETOOMANYREFS",
219
+ "110" => "ETIMEDOUT",
220
+ "111" => "ECONNREFUSED",
221
+ "112" => "EHOSTDOWN",
222
+ "113" => "EHOSTUNREACH",
223
+ "114" => "EALREADY",
224
+ "115" => "EINPROGRESS",
225
+ "116" => "ESTALE",
226
+ "117" => "EUCLEAN",
227
+ "118" => "ENOTNAM",
228
+ "119" => "ENAVAIL",
229
+ "120" => "EISNAM",
230
+ "121" => "EREMOTEIO",
231
+ "122" => "EDQUOT",
232
+ "123" => "ENOMEDIUM",
233
+ "124" => "EMEDIUMTYPE",
234
+ "125" => "ECANCELED",
235
+ "126" => "ENOKEY",
236
+ "127" => "EKEYEXPIRED",
237
+ "128" => "EKEYREVOKED",
238
+ "129" => "EKEYREJECTED",
239
+ "130" => "EOWNERDEAD",
240
+ "131" => "ENOTRECOVERABLE",
241
+ "132" => "ERFKILL",
242
+ "133" => "EHWPOISON",
243
+ _ => return raw_value.to_string(),
244
+ };
245
+
246
+ format!("{} ({})", raw_value, name)
247
+ }
248
+}
249
+
250
+/// _BOOT_ID: UUID → "UUID (timestamp)"
251
+/// Note: Full implementation would require boot_id cache lookup
252
+pub struct BootIdTransformation;
253
+
254
+impl FieldTransformation for BootIdTransformation {
255
+ fn transform(&self, raw_value: &str) -> String {
256
+ // For now, just return the UUID
257
+ // Full implementation would look up first boot timestamp from cache
258
+ raw_value.to_string()
259
+ }
260
+}
261
+
262
+/// _UID: numeric → username
263
+pub struct UidTransformation;
264
+
265
+impl FieldTransformation for UidTransformation {
266
+ fn transform(&self, raw_value: &str) -> String {
267
+ use nix::unistd::{Uid, User};
268
+
269
+ // Parse the UID
270
+ let Ok(uid_num) = raw_value.parse::<u32>() else {
271
+ return raw_value.to_string();
272
+ };
273
+
274
+ let uid = Uid::from_raw(uid_num);
275
+
276
+ // Look up the user
277
+ match User::from_uid(uid) {
278
+ Ok(Some(user)) => format!("{} ({})", raw_value, user.name),
279
+ Ok(None) => raw_value.to_string(), // User not found
280
+ Err(_) => raw_value.to_string(), // Lookup error
281
+ }
282
+ }
283
+}
284
+
285
+/// _GID: numeric → groupname
286
+pub struct GidTransformation;
287
+
288
+impl FieldTransformation for GidTransformation {
289
+ fn transform(&self, raw_value: &str) -> String {
290
+ use nix::unistd::{Gid, Group};
291
+
292
+ // Parse the GID
293
+ let Ok(gid_num) = raw_value.parse::<u32>() else {
294
+ return raw_value.to_string();
295
+ };
296
+
297
+ let gid = Gid::from_raw(gid_num);
298
+
299
+ // Look up the group
300
+ match Group::from_gid(gid) {
301
+ Ok(Some(group)) => format!("{} ({})", raw_value, group.name),
302
+ Ok(None) => raw_value.to_string(), // Group not found
303
+ Err(_) => raw_value.to_string(), // Lookup error
304
+ }
305
+ }
306
+}
307
+
308
+/// _CAP_EFFECTIVE: hex → "hex (capability names)"
309
+pub struct CapEffectiveTransformation;
310
+
311
+impl FieldTransformation for CapEffectiveTransformation {
312
+ fn transform(&self, raw_value: &str) -> String {
313
+ // Parse hex value
314
+ let caps_value = if let Some(hex) = raw_value.strip_prefix("0x") {
315
+ u64::from_str_radix(hex, 16).ok()
316
+ } else {
317
+ raw_value.parse::<u64>().ok()
318
+ };
319
+
320
+ let Some(caps) = caps_value else {
321
+ return raw_value.to_string();
322
+ };
323
+
324
+ // Linux capabilities (41 capabilities as of Linux 5.x)
325
+ const CAPABILITIES: &[&str] = &[
326
+ "CAP_CHOWN",
327
+ "CAP_DAC_OVERRIDE",
328
+ "CAP_DAC_READ_SEARCH",
329
+ "CAP_FOWNER",
330
+ "CAP_FSETID",
331
+ "CAP_KILL",
332
+ "CAP_SETGID",
333
+ "CAP_SETUID",
334
+ "CAP_SETPCAP",
335
+ "CAP_LINUX_IMMUTABLE",
336
+ "CAP_NET_BIND_SERVICE",
337
+ "CAP_NET_BROADCAST",
338
+ "CAP_NET_ADMIN",
339
+ "CAP_NET_RAW",
340
+ "CAP_IPC_LOCK",
341
+ "CAP_IPC_OWNER",
342
+ "CAP_SYS_MODULE",
343
+ "CAP_SYS_RAWIO",
344
+ "CAP_SYS_CHROOT",
345
+ "CAP_SYS_PTRACE",
346
+ "CAP_SYS_PACCT",
347
+ "CAP_SYS_ADMIN",
348
+ "CAP_SYS_BOOT",
349
+ "CAP_SYS_NICE",
350
+ "CAP_SYS_RESOURCE",
351
+ "CAP_SYS_TIME",
352
+ "CAP_SYS_TTY_CONFIG",
353
+ "CAP_MKNOD",
354
+ "CAP_LEASE",
355
+ "CAP_AUDIT_WRITE",
356
+ "CAP_AUDIT_CONTROL",
357
+ "CAP_SETFCAP",
358
+ "CAP_MAC_OVERRIDE",
359
+ "CAP_MAC_ADMIN",
360
+ "CAP_SYSLOG",
361
+ "CAP_WAKE_ALARM",
362
+ "CAP_BLOCK_SUSPEND",
363
+ "CAP_AUDIT_READ",
364
+ "CAP_PERFMON",
365
+ "CAP_BPF",
366
+ "CAP_CHECKPOINT_RESTORE",
367
+ ];
368
+
369
+ let mut cap_names = Vec::new();
370
+ for (i, &cap_name) in CAPABILITIES.iter().enumerate() {
371
+ if caps & (1u64 << i) != 0 {
372
+ cap_names.push(cap_name);
373
+ }
374
+ }
375
+
376
+ if cap_names.is_empty() {
377
+ format!("{} (none)", raw_value)
378
+ } else {
379
+ format!("{} ({})", raw_value, cap_names.join(", "))
380
+ }
381
+ }
382
+}
383
+
384
+/// _SOURCE_REALTIME_TIMESTAMP: microseconds → "microseconds (ISO8601)"
385
+pub struct SourceRealtimeTimestampTransformation;
386
+
387
+impl FieldTransformation for SourceRealtimeTimestampTransformation {
388
+ fn transform(&self, raw_value: &str) -> String {
389
+ // Parse microseconds since epoch
390
+ let Ok(usec) = raw_value.parse::<i64>() else {
391
+ return raw_value.to_string();
392
+ };
393
+
394
+ // Convert to seconds and nanoseconds
395
+ let secs = usec / 1_000_000;
396
+ let nsecs = ((usec % 1_000_000) * 1000) as u32;
397
+
398
+ // Create DateTime in UTC, then convert to local timezone
399
+ use chrono::{Local, TimeZone, Utc};
400
+ let dt_utc = match Utc.timestamp_opt(secs, nsecs) {
401
+ chrono::LocalResult::Single(dt) => dt,
402
+ _ => return raw_value.to_string(),
403
+ };
404
+
405
+ // Convert to local time
406
+ let dt_local = dt_utc.with_timezone(&Local);
407
+
408
+ // Format as RFC3339 with microsecond precision in local timezone
409
+ format!("{} ({})", raw_value, dt_local.to_rfc3339_opts(chrono::SecondsFormat::Micros, true))
410
+ }
411
+}
412
+
413
+/// MESSAGE_ID: UUID → "UUID (description)"
414
+pub struct MessageIdTransformation;
415
+
416
+impl FieldTransformation for MessageIdTransformation {
417
+ fn transform(&self, raw_value: &str) -> String {
418
+ // Known journal message IDs from systemd and other sources
419
+ let description = match raw_value {
420
+ "f77379a8490b408bbe5f6940505a777b" => "Journal started",
421
+ "d93fb3c9c24d451a97cea615ce59c00b" => "Journal stopped",
422
+ "a596d6fe7bfa4994828e72309e95d61e" => "Journal messages suppressed",
423
+ "e9bf28e6e834481bb6f48f548ad13606" => "Journal messages missed",
424
+ "ec387f577b844b8fa948f33cad9a75e6" => "Journal disk space usage",
425
+ "fc2e22bc6ee647b6b90729ab34a250b1" => "Coredump",
426
+ "5aadd8e954dc4b1a8c954d63fd9e1137" => "Coredump truncated",
427
+ "1f4e0a44a88649939aaea34fc6da8c95" => "Backtrace",
428
+ "8d45620c1a4348dbb17410da57c60c66" => "User Session created",
429
+ "3354939424b4456d9802ca8333ed424a" => "User Session terminated",
430
+ "fcbefc5da23d428093f97c82a9290f7b" => "Seat started",
431
+ "e7852bfe46784ed0accde04bc864c2d5" => "Seat removed",
432
+ "24d8d4452573402496068381a6312df2" => "VM or container started",
433
+ "58432bd3bace477cb514b56381b8a758" => "VM or container stopped",
434
+ "c7a787079b354eaaa9e77b371893cd27" => "Time change",
435
+ "45f82f4aef7a4bbf942ce861d1f20990" => "Timezone change",
436
+ "50876a9db00f4c40bde1a2ad381c3a1b" => "System configuration issues",
437
+ "b07a249cd024414a82dd00cd181378ff" => "System start-up completed",
438
+ "eed00a68ffd84e31882105fd973abdd1" => "User start-up completed",
439
+ "6bbd95ee977941e497c48be27c254128" => "Sleep start",
440
+ "8811e6df2a8e40f58a94cea26f8ebf14" => "Sleep stop",
441
+ "98268866d1d54a499c4e98921d93bc40" => "System shutdown initiated",
442
+ "c14aaf76ec284a5fa1f105f88dfb061c" => "System factory reset initiated",
443
+ "d9ec5e95e4b646aaaea2fd05214edbda" => "Container init crashed",
444
+ "3ed0163e868a4417ab8b9e210407a96c" => "System reboot failed after crash",
445
+ "645c735537634ae0a32b15a7c6cba7d4" => "Init execution froze",
446
+ "5addb3a06a734d3396b794bf98fb2d01" => "Init crashed no coredump",
447
+ "5c9e98de4ab94c6a9d04d0ad793bd903" => "Init crashed no fork",
448
+ "5e6f1f5e4db64a0eaee3368249d20b94" => "Init crashed unknown signal",
449
+ "83f84b35ee264f74a3896a9717af34cb" => "Init crashed systemd signal",
450
+ "3a73a98baf5b4b199929e3226c0be783" => "Init crashed process signal",
451
+ "2ed18d4f78ca47f0a9bc25271c26adb4" => "Init crashed waitpid failed",
452
+ "56b1cd96f24246c5b607666fda952356" => "Init crashed coredump failed",
453
+ "4ac7566d4d7548f4981f629a28f0f829" => "Init crashed coredump",
454
+ "38e8b1e039ad469291b18b44c553a5b7" => "Crash shell failed to fork",
455
+ "872729b47dbe473eb768ccecd477beda" => "Crash shell failed to execute",
456
+ "658a67adc1c940b3b3316e7e8628834a" => "Selinux failed",
457
+ "e6f456bd92004d9580160b2207555186" => "Battery low warning",
458
+ "267437d33fdd41099ad76221cc24a335" => "Battery low powering off",
459
+ "79e05b67bc4545d1922fe47107ee60c5" => "Manager mainloop failed",
460
+ "dbb136b10ef4457ba47a795d62f108c9" => "Manager no xdgdir path",
461
+ "ed158c2df8884fa584eead2d902c1032" => "Init failed to drop capability bounding set of usermode",
462
+ "42695b500df048298bee37159caa9f2e" => "Init failed to drop capability bounding set",
463
+ "bfc2430724ab44499735b4f94cca9295" => "User manager can't disable new privileges",
464
+ "59288af523be43a28d494e41e26e4510" => "Manager failed to start default target",
465
+ "689b4fcc97b4486ea5da92db69c9e314" => "Manager failed to isolate default target",
466
+ "5ed836f1766f4a8a9fc5da45aae23b29" => "Manager failed to collect passed file descriptors",
467
+ "6a40fbfbd2ba4b8db02fb40c9cd090d7" => "Init failed to fix up environment variables",
468
+ "0e54470984ac419689743d957a119e2e" => "Manager failed to allocate",
469
+ "d67fa9f847aa4b048a2ae33535331adb" => "Manager failed to write Smack",
470
+ "af55a6f75b544431b72649f36ff6d62c" => "System shutdown critical error",
471
+ "d18e0339efb24a068d9c1060221048c2" => "Init failed to fork off valgrind",
472
+ "7d4958e842da4a758f6c1cdc7b36dcc5" => "Unit starting",
473
+ "39f53479d3a045ac8e11786248231fbf" => "Unit started",
474
+ "be02cf6855d2428ba40df7e9d022f03d" => "Unit failed",
475
+ "de5b426a63be47a7b6ac3eaac82e2f6f" => "Unit stopping",
476
+ "9d1aaa27d60140bd96365438aad20286" => "Unit stopped",
477
+ "d34d037fff1847e6ae669a370e694725" => "Unit reloading",
478
+ "7b05ebc668384222baa8881179cfda54" => "Unit reloaded",
479
+ "5eb03494b6584870a536b337290809b3" => "Unit restart scheduled",
480
+ "ae8f7b866b0347b9af31fe1c80b127c0" => "Unit resources",
481
+ "7ad2d189f7e94e70a38c781354912448" => "Unit success",
482
+ "0e4284a0caca4bfc81c0bb6786972673" => "Unit skipped",
483
+ "d9b373ed55a64feb8242e02dbe79a49c" => "Unit failure result",
484
+ "641257651c1b4ec9a8624d7a40a9e1e7" => "Process execution failed",
485
+ "98e322203f7a4ed290d09fe03c09fe15" => "Unit process exited",
486
+ "0027229ca0644181a76c4e92458afa2e" => "Syslog forward missed",
487
+ "1dee0369c7fc4736b7099b38ecb46ee7" => "Mount point is not empty",
488
+ "d989611b15e44c9dbf31e3c81256e4ed" => "Unit oomd kill",
489
+ "fe6faa94e7774663a0da52717891d8ef" => "Unit out of memory",
490
+ "b72ea4a2881545a0b50e200e55b9b06f" => "Lid opened",
491
+ "b72ea4a2881545a0b50e200e55b9b070" => "Lid closed",
492
+ "f5f416b862074b28927a48c3ba7d51ff" => "System docked",
493
+ "51e171bd585248568110144c517cca53" => "System undocked",
494
+ "b72ea4a2881545a0b50e200e55b9b071" => "Power key",
495
+ "3e0117101eb243c1b9a50db3494ab10b" => "Power key long press",
496
+ "9fa9d2c012134ec385451ffe316f97d0" => "Reboot key",
497
+ "f1c59a58c9d943668965c337caec5975" => "Reboot key long press",
498
+ "b72ea4a2881545a0b50e200e55b9b072" => "Suspend key",
499
+ "bfdaf6d312ab4007bc1fe40a15df78e8" => "Suspend key long press",
500
+ "b72ea4a2881545a0b50e200e55b9b073" => "Hibernate key",
501
+ "167836df6f7f428e98147227b2dc8945" => "Hibernate key long press",
502
+ "c772d24e9a884cbeb9ea12625c306c01" => "Invalid configuration",
503
+ "1675d7f172174098b1108bf8c7dc8f5d" => "DNSSEC validation failed",
504
+ "4d4408cfd0d144859184d1e65d7c8a65" => "DNSSEC trust anchor revoked",
505
+ "36db2dfa5a9045e1bd4af5f93e1cf057" => "DNSSEC turned off",
506
+ "b61fdac612e94b9182285b998843061f" => "Username unsafe",
507
+ "1b3bb94037f04bbf81028e135a12d293" => "Mount point path not suitable",
508
+ "010190138f494e29a0ef6669749531aa" => "Device path not suitable",
509
+ "b480325f9c394a7b802c231e51a2752c" => "Nobody user unsuitable",
510
+ "1c0454c1bd2241e0ac6fefb4bc631433" => "Systemd udev settle deprecated",
511
+ "7c8a41f37b764941a0e1780b1be2f037" => "Time initial sync",
512
+ "7db73c8af0d94eeb822ae04323fe6ab6" => "Time initial bump",
513
+ "9e7066279dc8403da79ce4b1a69064b2" => "Shutdown scheduled",
514
+ "249f6fb9e6e2428c96f3f0875681ffa3" => "Shutdown canceled",
515
+ "3f7d5ef3e54f4302b4f0b143bb270cab" => "TPM PCR Extended",
516
+ "f9b0be465ad540d0850ad32172d57c21" => "Memory Trimmed",
517
+ "a8fa8dacdb1d443e9503b8be367a6adb" => "SysV Service Found",
518
+ "187c62eb1e7f463bb530394f52cb090f" => "Portable Service attached",
519
+ "76c5c754d628490d8ecba4c9d042112b" => "Portable Service detached",
520
+ "9cf56b8baf9546cf9478783a8de42113" => "systemd-networkd sysctl changed by foreign process",
521
+ "ad7089f928ac4f7ea00c07457d47ba8a" => "SRK into TPM authorization failure",
522
+ "b2bcbaf5edf948e093ce50bbea0e81ec" => "Secure Attention Key (SAK) was pressed",
523
+ "7fc63312330b479bb32e598d47cef1a8" => "dbus activate no unit",
524
+ "ee9799dab1e24d81b7bee7759a543e1b" => "dbus activate masked unit",
525
+ "a0fa58cafd6f4f0c8d003d16ccf9e797" => "dbus broker exited",
526
+ "c8c6cde1c488439aba371a664353d9d8" => "dbus dirwatch",
527
+ "8af3357071af4153af414daae07d38e7" => "dbus dispatch stats",
528
+ "199d4300277f495f84ba4028c984214c" => "dbus no sopeergroup",
529
+ "b209c0d9d1764ab38d13b8e00d1784d6" => "dbus protocol violation",
530
+ "6fa70fa776044fa28be7a21daf42a108" => "dbus receive failed",
531
+ "0ce0fa61d1a9433dabd67417f6b8e535" => "dbus service failed open",
532
+ "24dc708d9e6a4226a3efe2033bb744de" => "dbus service invalid",
533
+ "f15d2347662d483ea9bcd8aa1a691d28" => "dbus sighup",
534
+ "0ce153587afa4095832d233c17a88001" => "Gnome SM startup succeeded",
535
+ "10dd2dc188b54a5e98970f56499d1f73" => "Gnome SM unrecoverable failure",
536
+ "f3ea493c22934e26811cd62abe8e203a" => "Gnome shell started",
537
+ "c7b39b1e006b464599465e105b361485" => "Flatpak cache",
538
+ "75ba3deb0af041a9a46272ff85d9e73e" => "Flathub pulls",
539
+ "f02bce89a54e4efab3a94a797d26204a" => "Flathub pull errors",
540
+ "dd11929c788e48bdbb6276fb5f26b08a" => "Boltd starting",
541
+ "1e6061a9fbd44501b3ccc368119f2b69" => "Netdata startup",
542
+ "ed4cdb8f1beb4ad3b57cb3cae2d162fa" => "Netdata connection from child",
543
+ "6e2e3839067648968b646045dbf28d66" => "Netdata connection to parent",
544
+ "9ce0cb58ab8b44df82c4bf1ad9ee22de" => "Netdata alert transition",
545
+ "6db0018e83e34320ae2a659d78019fb7" => "Netdata alert notification",
546
+ "23e93dfccbf64e11aac858b9410d8a82" => "Netdata fatal message",
547
+ "8ddaf5ba33a74078b609250db1e951f3" => "Sensor state transition",
548
+ "ec87a56120d5431bace51e2fb8bba243" => "Netdata log flood protection",
549
+ "acb33cb95778476baac702eb7e4e151d" => "Netdata Cloud connection",
550
+ "d1f59606dd4d41e3b217a0cfcae8e632" => "Netdata extreme cardinality",
551
+ "02f47d350af5449197bf7a95b605a468" => "Netdata exit reason",
552
+ "4fdf40816c124623a032b7fe73beacb8" => "Netdata dynamic configuration",
553
+ _ => return raw_value.to_string(),
554
+ };
555
+
556
+ format!("{} ({})", raw_value, description)
557
+ }
558
+}
559
+
560
+/// Create a transformation registry with all systemd journal transformations
561
+pub fn create_systemd_journal_transformations() -> TransformationRegistry {
562
+ let mut registry = TransformationRegistry::new();
563
+
564
+ // Timestamp transformation (used for the first column)
565
+ registry.register("timestamp", Arc::new(SourceRealtimeTimestampTransformation));
566
+
567
+ registry.register("PRIORITY", Arc::new(PriorityTransformation));
568
+ registry.register("SYSLOG_FACILITY", Arc::new(SyslogFacilityTransformation));
569
+ registry.register("ERRNO", Arc::new(ErrnoTransformation));
570
+ registry.register("_BOOT_ID", Arc::new(BootIdTransformation));
571
+ registry.register("_UID", Arc::new(UidTransformation));
572
+ registry.register("_GID", Arc::new(GidTransformation));
573
+ registry.register("_CAP_EFFECTIVE", Arc::new(CapEffectiveTransformation));
574
+ registry.register(
575
+ "_SOURCE_REALTIME_TIMESTAMP",
576
+ Arc::new(SourceRealtimeTimestampTransformation),
577
+ );
578
+ registry.register("MESSAGE_ID", Arc::new(MessageIdTransformation));
579
+
580
+ // Also register variations that exist in the wild
581
+ registry.register("OBJECT_UID", Arc::new(UidTransformation));
582
+ registry.register("OBJECT_GID", Arc::new(GidTransformation));
583
+ registry.register("_SYSTEMD_OWNER_UID", Arc::new(UidTransformation));
584
+ registry.register("OBJECT_SYSTEMD_OWNER_UID", Arc::new(UidTransformation));
585
+ registry.register("_AUDIT_LOGINUID", Arc::new(UidTransformation));
586
+ registry.register("OBJECT_AUDIT_LOGINUID", Arc::new(UidTransformation));
587
+
588
+ registry
589
+}
src/crates/journal-engine/src/query_time_range.rs
new
+209
@@ -0,0 +1,209 @@
1
+//! Query time range with automatic alignment for histogram bucketing
2
+
3
+use crate::histogram::calculate_bucket_duration;
4
+use crate::EngineError;
5
+use journal_index::Seconds;
6
+
7
+/// A time range for querying journal entries with automatic alignment.
8
+///
9
+/// This type encapsulates:
10
+/// - The original requested time boundaries
11
+/// - The computed bucket duration based on the range
12
+/// - The aligned boundaries for consistent indexing and querying
13
+///
14
+/// All alignment logic is handled internally, ensuring consistency between
15
+/// histogram computation, file indexing, and log queries.
16
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
17
+pub struct QueryTimeRange {
18
+ /// Original requested start time (seconds)
19
+ requested_start: u32,
20
+ /// Original requested end time (seconds)
21
+ requested_end: u32,
22
+ /// Computed bucket duration in seconds
23
+ bucket_duration: u32,
24
+ /// Aligned start time (rounds down to bucket boundary)
25
+ aligned_start: u32,
26
+ /// Aligned end time (rounds up to bucket boundary)
27
+ aligned_end: u32,
28
+}
29
+
30
+impl QueryTimeRange {
31
+ /// Create a new query time range with automatic alignment.
32
+ ///
33
+ /// The bucket duration is computed based on the range duration, and
34
+ /// the boundaries are aligned to bucket boundaries:
35
+ /// - `aligned_start` rounds down to the nearest bucket boundary
36
+ /// - `aligned_end` rounds up to the nearest bucket boundary
37
+ ///
38
+ /// # Arguments
39
+ /// * `start` - Start time in seconds (inclusive)
40
+ /// * `end` - End time in seconds (exclusive)
41
+ ///
42
+ /// # Returns
43
+ /// * `Ok(QueryTimeRange)` if the range is valid
44
+ /// * `Err(EngineError::InvalidTimeRange)` if start >= end
45
+ ///
46
+ /// # Example
47
+ /// ```
48
+ /// use journal_engine::QueryTimeRange;
49
+ ///
50
+ /// let range = QueryTimeRange::new(100, 500).unwrap();
51
+ /// assert_eq!(range.requested_start(), 100);
52
+ /// assert_eq!(range.requested_end(), 500);
53
+ /// assert!(range.aligned_start() <= 100);
54
+ /// assert!(range.aligned_end() >= 500);
55
+ /// ```
56
+ pub fn new(start: u32, end: u32) -> Result<Self, EngineError> {
57
+ if start >= end {
58
+ return Err(EngineError::InvalidTimeRange {
59
+ start,
60
+ end,
61
+ });
62
+ }
63
+
64
+ let duration = end - start;
65
+ let bucket_duration = calculate_bucket_duration(duration);
66
+ let aligned_start = (start / bucket_duration) * bucket_duration;
67
+ let aligned_end = end.div_ceil(bucket_duration) * bucket_duration;
68
+
69
+ Ok(Self {
70
+ requested_start: start,
71
+ requested_end: end,
72
+ bucket_duration,
73
+ aligned_start,
74
+ aligned_end,
75
+ })
76
+ }
77
+
78
+ /// Get the original requested start time (seconds).
79
+ pub fn requested_start(&self) -> u32 {
80
+ self.requested_start
81
+ }
82
+
83
+ /// Get the original requested end time (seconds).
84
+ pub fn requested_end(&self) -> u32 {
85
+ self.requested_end
86
+ }
87
+
88
+ /// Get the computed bucket duration (seconds).
89
+ ///
90
+ /// This is used for file indexing to ensure all files are indexed
91
+ /// with the same bucket size.
92
+ pub fn bucket_duration(&self) -> u32 {
93
+ self.bucket_duration
94
+ }
95
+
96
+ /// Get the bucket duration as `Seconds`.
97
+ pub fn bucket_duration_seconds(&self) -> Seconds {
98
+ Seconds(self.bucket_duration)
99
+ }
100
+
101
+ /// Get the aligned start time (seconds).
102
+ ///
103
+ /// This is the start time rounded down to the nearest bucket boundary.
104
+ pub fn aligned_start(&self) -> u32 {
105
+ self.aligned_start
106
+ }
107
+
108
+ /// Get the aligned end time (seconds).
109
+ ///
110
+ /// This is the end time rounded up to the nearest bucket boundary.
111
+ pub fn aligned_end(&self) -> u32 {
112
+ self.aligned_end
113
+ }
114
+
115
+ /// Get the duration of the aligned range (seconds).
116
+ pub fn aligned_duration(&self) -> u32 {
117
+ self.aligned_end - self.aligned_start
118
+ }
119
+
120
+ /// Get the duration of the requested range (seconds).
121
+ pub fn requested_duration(&self) -> u32 {
122
+ self.requested_end - self.requested_start
123
+ }
124
+
125
+ /// Returns an iterator over the bucket time ranges.
126
+ ///
127
+ /// Each bucket is a `(start, end)` tuple in seconds, where:
128
+ /// - `start` is inclusive
129
+ /// - `end` is exclusive
130
+ /// - `end - start == bucket_duration`
131
+ ///
132
+ /// # Example
133
+ /// ```
134
+ /// use journal_engine::QueryTimeRange;
135
+ ///
136
+ /// let range = QueryTimeRange::new(0, 1000).unwrap();
137
+ /// for (start, end) in range.buckets() {
138
+ /// println!("Bucket: [{}, {})", start, end);
139
+ /// }
140
+ /// ```
141
+ pub fn buckets(&self) -> impl Iterator<Item = (u32, u32)> + '_ {
142
+ let bucket_duration = self.bucket_duration;
143
+ let num_buckets = (self.aligned_end - self.aligned_start) / bucket_duration;
144
+
145
+ (0..num_buckets).map(move |i| {
146
+ let start = self.aligned_start + (i * bucket_duration);
147
+ let end = start + bucket_duration;
148
+ (start, end)
149
+ })
150
+ }
151
+}
152
+
153
+#[cfg(test)]
154
+mod tests {
155
+ use super::*;
156
+
157
+ #[test]
158
+ fn test_invalid_range() {
159
+ assert!(QueryTimeRange::new(100, 100).is_err());
160
+ assert!(QueryTimeRange::new(100, 50).is_err());
161
+ }
162
+
163
+ #[test]
164
+ fn test_alignment() {
165
+ let range = QueryTimeRange::new(100, 500).unwrap();
166
+
167
+ // Aligned boundaries should encompass requested boundaries
168
+ assert!(range.aligned_start() <= range.requested_start());
169
+ assert!(range.aligned_end() >= range.requested_end());
170
+
171
+ // Aligned boundaries should be multiples of bucket duration
172
+ assert_eq!(range.aligned_start() % range.bucket_duration(), 0);
173
+ assert_eq!(range.aligned_end() % range.bucket_duration(), 0);
174
+ }
175
+
176
+ #[test]
177
+ fn test_accessors() {
178
+ let range = QueryTimeRange::new(100, 500).unwrap();
179
+
180
+ assert_eq!(range.requested_start(), 100);
181
+ assert_eq!(range.requested_end(), 500);
182
+ assert_eq!(range.requested_duration(), 400);
183
+ assert!(range.bucket_duration() > 0);
184
+ assert_eq!(range.aligned_duration(), range.aligned_end() - range.aligned_start());
185
+ }
186
+
187
+ #[test]
188
+ fn test_buckets_iterator() {
189
+ let range = QueryTimeRange::new(0, 1000).unwrap();
190
+ let buckets: Vec<(u32, u32)> = range.buckets().collect();
191
+
192
+ // Check that we have at least one bucket
193
+ assert!(!buckets.is_empty());
194
+
195
+ // Check that buckets are contiguous and cover the aligned range
196
+ let mut expected_start = range.aligned_start();
197
+ for (start, end) in &buckets {
198
+ assert_eq!(*start, expected_start);
199
+ assert_eq!(end - start, range.bucket_duration());
200
+ expected_start = *end;
201
+ }
202
+ assert_eq!(expected_start, range.aligned_end());
203
+
204
+ // Check that all buckets have the same duration
205
+ for (start, end) in &buckets {
206
+ assert_eq!(end - start, range.bucket_duration());
207
+ }
208
+ }
209
+}
src/crates/journal-engine/tests/multi_file_pagination.rs
new
+3345
@@ -0,0 +1,3345 @@
1
+//! Integration tests for multi-file pagination with PaginationState.
2
+
3
+use journal_common::Seconds;
4
+use journal_core::file::{JournalFile, JournalFileOptions, JournalWriter};
5
+use journal_core::repository::File;
6
+use journal_engine::logs::query::LogQuery;
7
+use journal_index::{
8
+ Anchor, Direction, FieldName, FieldValuePair, FileIndexer, Filter, Microseconds,
9
+};
10
+use std::collections::HashSet;
11
+use std::fs;
12
+use std::path::PathBuf;
13
+use tempfile::TempDir;
14
+use uuid::Uuid;
15
+
16
+/// Test journal entry specification
17
+struct TestEntry {
18
+ timestamp: Microseconds,
19
+ fields: Vec<(String, String)>,
20
+}
21
+
22
+impl TestEntry {
23
+ fn new(timestamp: Microseconds) -> Self {
24
+ Self {
25
+ timestamp,
26
+ fields: Vec::new(),
27
+ }
28
+ }
29
+
30
+ fn with_field(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
31
+ self.fields.push((name.into(), value.into()));
32
+ self
33
+ }
34
+}
35
+
36
+/// Create a test journal file path with a specific name
37
+fn create_test_journal_path(temp_dir: &TempDir, filename: &str) -> PathBuf {
38
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
39
+ let machine_dir = temp_dir.path().join(machine_id.to_string());
40
+ fs::create_dir_all(&machine_dir).expect("create machine dir");
41
+ machine_dir.join(filename)
42
+}
43
+
44
+/// Helper to create a test journal file with specified entries
45
+fn create_test_journal(
46
+ temp_dir: &TempDir,
47
+ filename: &str,
48
+ entries: Vec<TestEntry>,
49
+) -> Result<File, Box<dyn std::error::Error>> {
50
+ let journal_path = create_test_journal_path(temp_dir, filename);
51
+
52
+ let file =
53
+ File::from_path(&journal_path).ok_or("Failed to create repository File from path")?;
54
+
55
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
56
+ let boot_id = Uuid::from_u128(0x11111111_1111_1111_1111_111111111111);
57
+ let seqnum_id = Uuid::from_u128(0x22222222_2222_2222_2222_222222222222);
58
+
59
+ let options = JournalFileOptions::new(machine_id, boot_id, seqnum_id);
60
+
61
+ let mut journal_file = JournalFile::create(&file, options)?;
62
+ let mut writer = JournalWriter::new(&mut journal_file, 1, boot_id)?;
63
+
64
+ for entry in entries {
65
+ let mut entry_data = Vec::new();
66
+
67
+ // Add _SOURCE_REALTIME_TIMESTAMP first
68
+ entry_data.push(format!("_SOURCE_REALTIME_TIMESTAMP={}", entry.timestamp.0).into_bytes());
69
+
70
+ // Add all other fields
71
+ for (field, value) in entry.fields {
72
+ entry_data.push(format!("{}={}", field, value).into_bytes());
73
+ }
74
+
75
+ let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
76
+
77
+ writer.add_entry(
78
+ &mut journal_file,
79
+ &entry_refs,
80
+ entry.timestamp.0,
81
+ entry.timestamp.0,
82
+ )?;
83
+ }
84
+
85
+ Ok(file)
86
+}
87
+
88
+#[test]
89
+fn test_multi_file_pagination_forward_non_overlapping() {
90
+ // Create temporary directory
91
+ let temp_dir = TempDir::new().unwrap();
92
+
93
+ // File 1: entries at t=100..200 microseconds (100 entries)
94
+ let entries_file1: Vec<TestEntry> = (100..200)
95
+ .map(|i| {
96
+ TestEntry::new(Microseconds(i))
97
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
98
+ .with_field("FILE", "1")
99
+ })
100
+ .collect();
101
+
102
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
103
+
104
+ // File 2: entries at t=200..300 microseconds (100 entries)
105
+ let entries_file2: Vec<TestEntry> = (200..300)
106
+ .map(|i| {
107
+ TestEntry::new(Microseconds(i))
108
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
109
+ .with_field("FILE", "2")
110
+ })
111
+ .collect();
112
+
113
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
114
+
115
+ // Index both files
116
+ let mut indexer = FileIndexer::default();
117
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
118
+ let file_field = FieldName::new("FILE").unwrap();
119
+
120
+ let index1 = indexer
121
+ .index(
122
+ &file1,
123
+ Some(&source_timestamp_field),
124
+ &[file_field.clone()],
125
+ Seconds(3600),
126
+ )
127
+ .unwrap();
128
+
129
+ let index2 = indexer
130
+ .index(
131
+ &file2,
132
+ Some(&source_timestamp_field),
133
+ &[file_field],
134
+ Seconds(3600),
135
+ )
136
+ .unwrap();
137
+
138
+ let file_indexes = vec![index1, index2];
139
+
140
+ // First page: limit=150, should get all 100 from file1 + 50 from file2
141
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
142
+ .with_limit(150)
143
+ .execute_page(None)
144
+ .unwrap();
145
+
146
+ assert_eq!(
147
+ first_page.len(),
148
+ 150,
149
+ "First page should contain exactly 150 entries"
150
+ );
151
+
152
+ // Verify timestamps are in ascending order
153
+ for i in 1..first_page.len() {
154
+ assert!(
155
+ first_page[i - 1].timestamp <= first_page[i].timestamp,
156
+ "Entries should be in ascending timestamp order"
157
+ );
158
+ }
159
+
160
+ // First entry should be at timestamp 100
161
+ assert_eq!(
162
+ first_page.first().unwrap().timestamp,
163
+ 100,
164
+ "First entry should be at timestamp 100"
165
+ );
166
+
167
+ // Last entry should be at timestamp 249 (100-199 from file1, then 200-249 from file2)
168
+ assert_eq!(
169
+ first_page.last().unwrap().timestamp,
170
+ 249,
171
+ "Last entry of first page should be at timestamp 249"
172
+ );
173
+
174
+ // State should track positions for files we read from
175
+ assert!(
176
+ !state1.file_positions.is_empty(),
177
+ "State should track positions"
178
+ );
179
+
180
+ // Second page: use state to get remaining 50 entries
181
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
182
+ .with_limit(150)
183
+ .execute_page(Some(&state1))
184
+ .unwrap();
185
+
186
+ assert_eq!(
187
+ second_page.len(),
188
+ 50,
189
+ "Second page should contain remaining 50 entries"
190
+ );
191
+
192
+ // Verify timestamps continue from where first page left off
193
+ assert_eq!(
194
+ second_page.first().unwrap().timestamp,
195
+ 250,
196
+ "Second page should start at timestamp 250"
197
+ );
198
+
199
+ assert_eq!(
200
+ second_page.last().unwrap().timestamp,
201
+ 299,
202
+ "Second page should end at timestamp 299"
203
+ );
204
+
205
+ // Verify no duplicates across both pages
206
+ let mut all_timestamps = HashSet::new();
207
+ for entry in &first_page {
208
+ assert!(
209
+ all_timestamps.insert(entry.timestamp),
210
+ "Found duplicate timestamp: {}",
211
+ entry.timestamp
212
+ );
213
+ }
214
+ for entry in &second_page {
215
+ assert!(
216
+ all_timestamps.insert(entry.timestamp),
217
+ "Found duplicate timestamp: {}",
218
+ entry.timestamp
219
+ );
220
+ }
221
+
222
+ // Verify we got all 200 unique entries
223
+ assert_eq!(
224
+ all_timestamps.len(),
225
+ 200,
226
+ "Should have retrieved all 200 unique entries"
227
+ );
228
+
229
+ // Third page should be empty
230
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
231
+ .with_limit(150)
232
+ .execute_page(Some(&state2))
233
+ .unwrap();
234
+
235
+ assert_eq!(
236
+ third_page.len(),
237
+ 0,
238
+ "Third page should be empty (no more entries)"
239
+ );
240
+}
241
+
242
+#[test]
243
+fn test_multi_file_pagination_same_timestamps() {
244
+ // Create temporary directory
245
+ let temp_dir = TempDir::new().unwrap();
246
+
247
+ // File 1: 150 entries all at timestamp 1000
248
+ let entries_file1: Vec<TestEntry> = (0..150)
249
+ .map(|i| {
250
+ TestEntry::new(Microseconds(1000))
251
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
252
+ .with_field("ENTRY_ID", format!("file1_{}", i))
253
+ .with_field("FILE", "1")
254
+ })
255
+ .collect();
256
+
257
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
258
+
259
+ // File 2: 150 entries all at timestamp 1000
260
+ let entries_file2: Vec<TestEntry> = (0..150)
261
+ .map(|i| {
262
+ TestEntry::new(Microseconds(1000))
263
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
264
+ .with_field("ENTRY_ID", format!("file2_{}", i))
265
+ .with_field("FILE", "2")
266
+ })
267
+ .collect();
268
+
269
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
270
+
271
+ // Index both files
272
+ let mut indexer = FileIndexer::default();
273
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
274
+ let file_field = FieldName::new("FILE").unwrap();
275
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
276
+
277
+ let index1 = indexer
278
+ .index(
279
+ &file1,
280
+ Some(&source_timestamp_field),
281
+ &[file_field.clone(), entry_id_field.clone()],
282
+ Seconds(3600),
283
+ )
284
+ .unwrap();
285
+
286
+ let index2 = indexer
287
+ .index(
288
+ &file2,
289
+ Some(&source_timestamp_field),
290
+ &[file_field, entry_id_field],
291
+ Seconds(3600),
292
+ )
293
+ .unwrap();
294
+
295
+ let file_indexes = vec![index1, index2];
296
+
297
+ // First page: limit=200, should get all 150 from file1 + 50 from file2
298
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
299
+ .with_limit(200)
300
+ .execute_page(None)
301
+ .unwrap();
302
+
303
+ assert_eq!(
304
+ first_page.len(),
305
+ 200,
306
+ "First page should contain exactly 200 entries"
307
+ );
308
+
309
+ // All timestamps should be 1000
310
+ for entry in &first_page {
311
+ assert_eq!(
312
+ entry.timestamp, 1000,
313
+ "All entries should have timestamp 1000"
314
+ );
315
+ }
316
+
317
+ // State should track positions for both files
318
+ assert_eq!(
319
+ state1.file_positions.len(),
320
+ 2,
321
+ "State should track positions for both files"
322
+ );
323
+
324
+ // Second page: use state to get remaining 100 entries
325
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
326
+ .with_limit(200)
327
+ .execute_page(Some(&state1))
328
+ .unwrap();
329
+
330
+ assert_eq!(
331
+ second_page.len(),
332
+ 100,
333
+ "Second page should contain remaining 100 entries"
334
+ );
335
+
336
+ // All timestamps should still be 1000
337
+ for entry in &second_page {
338
+ assert_eq!(
339
+ entry.timestamp, 1000,
340
+ "All entries should have timestamp 1000"
341
+ );
342
+ }
343
+
344
+ // Collect all ENTRY_ID values to verify uniqueness
345
+ let mut all_entry_ids = HashSet::new();
346
+
347
+ for entry in &first_page {
348
+ for field in &entry.fields {
349
+ if field.field() == "ENTRY_ID" {
350
+ assert!(
351
+ all_entry_ids.insert(field.value().to_string()),
352
+ "Found duplicate ENTRY_ID: {}",
353
+ field.value()
354
+ );
355
+ }
356
+ }
357
+ }
358
+
359
+ for entry in &second_page {
360
+ for field in &entry.fields {
361
+ if field.field() == "ENTRY_ID" {
362
+ assert!(
363
+ all_entry_ids.insert(field.value().to_string()),
364
+ "Found duplicate ENTRY_ID: {}",
365
+ field.value()
366
+ );
367
+ }
368
+ }
369
+ }
370
+
371
+ // Verify we got all 300 unique entries
372
+ assert_eq!(
373
+ all_entry_ids.len(),
374
+ 300,
375
+ "Should have retrieved all 300 unique entries"
376
+ );
377
+
378
+ // Verify we have entries from both files
379
+ let file1_entries: usize = all_entry_ids
380
+ .iter()
381
+ .filter(|id| id.starts_with("file1_"))
382
+ .count();
383
+ let file2_entries: usize = all_entry_ids
384
+ .iter()
385
+ .filter(|id| id.starts_with("file2_"))
386
+ .count();
387
+
388
+ assert_eq!(file1_entries, 150, "Should have 150 entries from file1");
389
+ assert_eq!(file2_entries, 150, "Should have 150 entries from file2");
390
+
391
+ // Third page should be empty
392
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
393
+ .with_limit(200)
394
+ .execute_page(Some(&state2))
395
+ .unwrap();
396
+
397
+ assert_eq!(
398
+ third_page.len(),
399
+ 0,
400
+ "Third page should be empty (no more entries)"
401
+ );
402
+}
403
+
404
+#[test]
405
+fn test_multi_file_pagination_overlapping_timestamps() {
406
+ // Create temporary directory
407
+ let temp_dir = TempDir::new().unwrap();
408
+
409
+ // File 1: entries at t=100..200 (100 entries)
410
+ let entries_file1: Vec<TestEntry> = (100..200)
411
+ .map(|i| {
412
+ TestEntry::new(Microseconds(i))
413
+ .with_field("MESSAGE", format!("File1 Entry at {}", i))
414
+ .with_field("ENTRY_ID", format!("file1_{}", i))
415
+ .with_field("FILE", "1")
416
+ })
417
+ .collect();
418
+
419
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
420
+
421
+ // File 2: entries at t=150..250 (100 entries) - overlaps with file1 from 150-199
422
+ let entries_file2: Vec<TestEntry> = (150..250)
423
+ .map(|i| {
424
+ TestEntry::new(Microseconds(i))
425
+ .with_field("MESSAGE", format!("File2 Entry at {}", i))
426
+ .with_field("ENTRY_ID", format!("file2_{}", i))
427
+ .with_field("FILE", "2")
428
+ })
429
+ .collect();
430
+
431
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
432
+
433
+ // Index both files
434
+ let mut indexer = FileIndexer::default();
435
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
436
+ let file_field = FieldName::new("FILE").unwrap();
437
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
438
+
439
+ let index1 = indexer
440
+ .index(
441
+ &file1,
442
+ Some(&source_timestamp_field),
443
+ &[file_field.clone(), entry_id_field.clone()],
444
+ Seconds(3600),
445
+ )
446
+ .unwrap();
447
+
448
+ let index2 = indexer
449
+ .index(
450
+ &file2,
451
+ Some(&source_timestamp_field),
452
+ &[file_field, entry_id_field],
453
+ Seconds(3600),
454
+ )
455
+ .unwrap();
456
+
457
+ let file_indexes = vec![index1, index2];
458
+
459
+ // First page: limit=120
460
+ // Expected: 50 from file1 (100-149) + 50 interleaved from both (150-199) + 20 from file2 (200-219)
461
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
462
+ .with_limit(120)
463
+ .execute_page(None)
464
+ .unwrap();
465
+
466
+ assert_eq!(
467
+ first_page.len(),
468
+ 120,
469
+ "First page should contain exactly 120 entries"
470
+ );
471
+
472
+ // Verify timestamps are in ascending order
473
+ for i in 1..first_page.len() {
474
+ assert!(
475
+ first_page[i - 1].timestamp <= first_page[i].timestamp,
476
+ "Entries should be in ascending timestamp order"
477
+ );
478
+ }
479
+
480
+ // First entry should be at timestamp 100
481
+ assert_eq!(
482
+ first_page.first().unwrap().timestamp,
483
+ 100,
484
+ "First entry should be at timestamp 100"
485
+ );
486
+
487
+ // State should track positions for both files
488
+ assert!(
489
+ !state1.file_positions.is_empty(),
490
+ "State should track positions"
491
+ );
492
+
493
+ // Second page: get remaining entries
494
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
495
+ .with_limit(120)
496
+ .execute_page(Some(&state1))
497
+ .unwrap();
498
+
499
+ assert_eq!(
500
+ second_page.len(),
501
+ 80,
502
+ "Second page should contain remaining 80 entries"
503
+ );
504
+
505
+ // Verify timestamps continue in order
506
+ for i in 1..second_page.len() {
507
+ assert!(
508
+ second_page[i - 1].timestamp <= second_page[i].timestamp,
509
+ "Entries should be in ascending timestamp order"
510
+ );
511
+ }
512
+
513
+ // Verify no timestamp gap between pages
514
+ if !first_page.is_empty() && !second_page.is_empty() {
515
+ assert!(
516
+ first_page.last().unwrap().timestamp <= second_page.first().unwrap().timestamp,
517
+ "Second page should continue from first page timestamp"
518
+ );
519
+ }
520
+
521
+ // Last entry should be at timestamp 249
522
+ assert_eq!(
523
+ second_page.last().unwrap().timestamp,
524
+ 249,
525
+ "Last entry should be at timestamp 249"
526
+ );
527
+
528
+ // Collect all ENTRY_ID values to verify uniqueness and completeness
529
+ let mut all_entry_ids = HashSet::new();
530
+
531
+ for entry in &first_page {
532
+ for field in &entry.fields {
533
+ if field.field() == "ENTRY_ID" {
534
+ assert!(
535
+ all_entry_ids.insert(field.value().to_string()),
536
+ "Found duplicate ENTRY_ID: {}",
537
+ field.value()
538
+ );
539
+ }
540
+ }
541
+ }
542
+
543
+ for entry in &second_page {
544
+ for field in &entry.fields {
545
+ if field.field() == "ENTRY_ID" {
546
+ assert!(
547
+ all_entry_ids.insert(field.value().to_string()),
548
+ "Found duplicate ENTRY_ID: {}",
549
+ field.value()
550
+ );
551
+ }
552
+ }
553
+ }
554
+
555
+ // Verify we got all 200 unique entries (100 from each file)
556
+ assert_eq!(
557
+ all_entry_ids.len(),
558
+ 200,
559
+ "Should have retrieved all 200 unique entries"
560
+ );
561
+
562
+ // Verify we have entries from both files
563
+ let file1_entries: usize = all_entry_ids
564
+ .iter()
565
+ .filter(|id| id.starts_with("file1_"))
566
+ .count();
567
+ let file2_entries: usize = all_entry_ids
568
+ .iter()
569
+ .filter(|id| id.starts_with("file2_"))
570
+ .count();
571
+
572
+ assert_eq!(file1_entries, 100, "Should have 100 entries from file1");
573
+ assert_eq!(file2_entries, 100, "Should have 100 entries from file2");
574
+
575
+ // Verify all timestamps from 100-249 are represented
576
+ let mut all_timestamps = HashSet::new();
577
+ for entry in first_page.iter().chain(second_page.iter()) {
578
+ all_timestamps.insert(entry.timestamp);
579
+ }
580
+
581
+ // We should have entries at all timestamps from 100-249 (150 unique timestamps)
582
+ for ts in 100..250 {
583
+ assert!(all_timestamps.contains(&ts), "Missing timestamp: {}", ts);
584
+ }
585
+
586
+ // Third page should be empty
587
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
588
+ .with_limit(120)
589
+ .execute_page(Some(&state2))
590
+ .unwrap();
591
+
592
+ assert_eq!(
593
+ third_page.len(),
594
+ 0,
595
+ "Third page should be empty (no more entries)"
596
+ );
597
+}
598
+
599
+#[test]
600
+fn test_multi_file_pagination_three_files() {
601
+ // Create temporary directory
602
+ let temp_dir = TempDir::new().unwrap();
603
+
604
+ // File 1: entries at t=100..200 (100 entries)
605
+ let entries_file1: Vec<TestEntry> = (100..200)
606
+ .map(|i| {
607
+ TestEntry::new(Microseconds(i))
608
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
609
+ .with_field("ENTRY_ID", format!("file1_{}", i))
610
+ .with_field("FILE", "1")
611
+ })
612
+ .collect();
613
+
614
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
615
+
616
+ // File 2: entries at t=200..300 (100 entries)
617
+ let entries_file2: Vec<TestEntry> = (200..300)
618
+ .map(|i| {
619
+ TestEntry::new(Microseconds(i))
620
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
621
+ .with_field("ENTRY_ID", format!("file2_{}", i))
622
+ .with_field("FILE", "2")
623
+ })
624
+ .collect();
625
+
626
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
627
+
628
+ // File 3: entries at t=300..400 (100 entries)
629
+ let entries_file3: Vec<TestEntry> = (300..400)
630
+ .map(|i| {
631
+ TestEntry::new(Microseconds(i))
632
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
633
+ .with_field("ENTRY_ID", format!("file3_{}", i))
634
+ .with_field("FILE", "3")
635
+ })
636
+ .collect();
637
+
638
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
639
+
640
+ // Index all three files
641
+ let mut indexer = FileIndexer::default();
642
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
643
+ let file_field = FieldName::new("FILE").unwrap();
644
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
645
+
646
+ let index1 = indexer
647
+ .index(
648
+ &file1,
649
+ Some(&source_timestamp_field),
650
+ &[file_field.clone(), entry_id_field.clone()],
651
+ Seconds(3600),
652
+ )
653
+ .unwrap();
654
+
655
+ let index2 = indexer
656
+ .index(
657
+ &file2,
658
+ Some(&source_timestamp_field),
659
+ &[file_field.clone(), entry_id_field.clone()],
660
+ Seconds(3600),
661
+ )
662
+ .unwrap();
663
+
664
+ let index3 = indexer
665
+ .index(
666
+ &file3,
667
+ Some(&source_timestamp_field),
668
+ &[file_field, entry_id_field],
669
+ Seconds(3600),
670
+ )
671
+ .unwrap();
672
+
673
+ let file_indexes = vec![index1, index2, index3];
674
+
675
+ // First page: limit=125, should get all 100 from file1 + 25 from file2
676
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
677
+ .with_limit(125)
678
+ .execute_page(None)
679
+ .unwrap();
680
+
681
+ assert_eq!(
682
+ first_page.len(),
683
+ 125,
684
+ "First page should contain exactly 125 entries"
685
+ );
686
+
687
+ assert_eq!(first_page.first().unwrap().timestamp, 100);
688
+ assert_eq!(first_page.last().unwrap().timestamp, 224);
689
+
690
+ // State should track positions for file1 and file2
691
+ assert_eq!(
692
+ state1.file_positions.len(),
693
+ 2,
694
+ "State should track positions for 2 files"
695
+ );
696
+
697
+ // Second page: limit=125, should get 75 from file2 + 50 from file3
698
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
699
+ .with_limit(125)
700
+ .execute_page(Some(&state1))
701
+ .unwrap();
702
+
703
+ assert_eq!(
704
+ second_page.len(),
705
+ 125,
706
+ "Second page should contain exactly 125 entries"
707
+ );
708
+
709
+ assert_eq!(second_page.first().unwrap().timestamp, 225);
710
+ assert_eq!(second_page.last().unwrap().timestamp, 349);
711
+
712
+ // State should now track all 3 files
713
+ assert_eq!(
714
+ state2.file_positions.len(),
715
+ 3,
716
+ "State should track positions for all 3 files"
717
+ );
718
+
719
+ // Third page: remaining 50 from file3
720
+ let (third_page, state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
721
+ .with_limit(125)
722
+ .execute_page(Some(&state2))
723
+ .unwrap();
724
+
725
+ assert_eq!(
726
+ third_page.len(),
727
+ 50,
728
+ "Third page should contain remaining 50 entries"
729
+ );
730
+
731
+ assert_eq!(third_page.first().unwrap().timestamp, 350);
732
+ assert_eq!(third_page.last().unwrap().timestamp, 399);
733
+
734
+ // Collect all ENTRY_ID values to verify uniqueness
735
+ let mut all_entry_ids = HashSet::new();
736
+
737
+ for entry in first_page
738
+ .iter()
739
+ .chain(second_page.iter())
740
+ .chain(third_page.iter())
741
+ {
742
+ for field in &entry.fields {
743
+ if field.field() == "ENTRY_ID" {
744
+ assert!(
745
+ all_entry_ids.insert(field.value().to_string()),
746
+ "Found duplicate ENTRY_ID: {}",
747
+ field.value()
748
+ );
749
+ }
750
+ }
751
+ }
752
+
753
+ // Verify we got all 300 unique entries
754
+ assert_eq!(
755
+ all_entry_ids.len(),
756
+ 300,
757
+ "Should have retrieved all 300 unique entries"
758
+ );
759
+
760
+ // Verify distribution: 100 from each file
761
+ let file1_count = all_entry_ids
762
+ .iter()
763
+ .filter(|id| id.starts_with("file1_"))
764
+ .count();
765
+ let file2_count = all_entry_ids
766
+ .iter()
767
+ .filter(|id| id.starts_with("file2_"))
768
+ .count();
769
+ let file3_count = all_entry_ids
770
+ .iter()
771
+ .filter(|id| id.starts_with("file3_"))
772
+ .count();
773
+
774
+ assert_eq!(file1_count, 100, "Should have 100 entries from file1");
775
+ assert_eq!(file2_count, 100, "Should have 100 entries from file2");
776
+ assert_eq!(file3_count, 100, "Should have 100 entries from file3");
777
+
778
+ // Fourth page should be empty
779
+ let (fourth_page, _state4) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
780
+ .with_limit(125)
781
+ .execute_page(Some(&state3))
782
+ .unwrap();
783
+
784
+ assert_eq!(
785
+ fourth_page.len(),
786
+ 0,
787
+ "Fourth page should be empty (no more entries)"
788
+ );
789
+}
790
+
791
+#[test]
792
+fn test_multi_file_pagination_small_limit() {
793
+ // Create temporary directory
794
+ let temp_dir = TempDir::new().unwrap();
795
+
796
+ // File 1: 100 entries at t=100..200
797
+ let entries_file1: Vec<TestEntry> = (100..200)
798
+ .map(|i| {
799
+ TestEntry::new(Microseconds(i))
800
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
801
+ .with_field("ENTRY_ID", format!("file1_{}", i))
802
+ })
803
+ .collect();
804
+
805
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
806
+
807
+ // File 2: 100 entries at t=200..300
808
+ let entries_file2: Vec<TestEntry> = (200..300)
809
+ .map(|i| {
810
+ TestEntry::new(Microseconds(i))
811
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
812
+ .with_field("ENTRY_ID", format!("file2_{}", i))
813
+ })
814
+ .collect();
815
+
816
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
817
+
818
+ // Index both files
819
+ let mut indexer = FileIndexer::default();
820
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
821
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
822
+
823
+ let index1 = indexer
824
+ .index(
825
+ &file1,
826
+ Some(&source_timestamp_field),
827
+ &[entry_id_field.clone()],
828
+ Seconds(3600),
829
+ )
830
+ .unwrap();
831
+
832
+ let index2 = indexer
833
+ .index(
834
+ &file2,
835
+ Some(&source_timestamp_field),
836
+ &[entry_id_field],
837
+ Seconds(3600),
838
+ )
839
+ .unwrap();
840
+
841
+ let file_indexes = vec![index1, index2];
842
+
843
+ // Use very small limit=30, need multiple pages for file1 alone
844
+ let mut all_entry_ids = HashSet::new();
845
+ let mut state = None;
846
+ let mut page_count = 0;
847
+
848
+ // Paginate through all entries with small page size
849
+ loop {
850
+ let (page, new_state) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
851
+ .with_limit(30)
852
+ .execute_page(state.as_ref())
853
+ .unwrap();
854
+
855
+ if page.is_empty() {
856
+ break;
857
+ }
858
+
859
+ page_count += 1;
860
+
861
+ // Verify order within page
862
+ for i in 1..page.len() {
863
+ assert!(
864
+ page[i - 1].timestamp <= page[i].timestamp,
865
+ "Page {} entries should be in ascending order",
866
+ page_count
867
+ );
868
+ }
869
+
870
+ // Collect ENTRY_IDs
871
+ for entry in &page {
872
+ for field in &entry.fields {
873
+ if field.field() == "ENTRY_ID" {
874
+ assert!(
875
+ all_entry_ids.insert(field.value().to_string()),
876
+ "Found duplicate ENTRY_ID: {}",
877
+ field.value()
878
+ );
879
+ }
880
+ }
881
+ }
882
+
883
+ state = Some(new_state);
884
+ }
885
+
886
+ // Should need 7 pages: 30+30+30+30+30+30+20 = 200 entries
887
+ assert_eq!(page_count, 7, "Should need exactly 7 pages");
888
+
889
+ // Verify we got all 200 unique entries
890
+ assert_eq!(
891
+ all_entry_ids.len(),
892
+ 200,
893
+ "Should have retrieved all 200 unique entries"
894
+ );
895
+
896
+ // Verify distribution
897
+ let file1_count = all_entry_ids
898
+ .iter()
899
+ .filter(|id| id.starts_with("file1_"))
900
+ .count();
901
+ let file2_count = all_entry_ids
902
+ .iter()
903
+ .filter(|id| id.starts_with("file2_"))
904
+ .count();
905
+
906
+ assert_eq!(file1_count, 100, "Should have 100 entries from file1");
907
+ assert_eq!(file2_count, 100, "Should have 100 entries from file2");
908
+}
909
+
910
+#[test]
911
+fn test_multi_file_pagination_limit_one() {
912
+ // Create temporary directory
913
+ let temp_dir = TempDir::new().unwrap();
914
+
915
+ // File 1: 10 entries at t=100..110
916
+ let entries_file1: Vec<TestEntry> = (100..110)
917
+ .map(|i| {
918
+ TestEntry::new(Microseconds(i))
919
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
920
+ .with_field("ENTRY_ID", format!("file1_{}", i))
921
+ })
922
+ .collect();
923
+
924
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
925
+
926
+ // File 2: 10 entries at t=110..120
927
+ let entries_file2: Vec<TestEntry> = (110..120)
928
+ .map(|i| {
929
+ TestEntry::new(Microseconds(i))
930
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
931
+ .with_field("ENTRY_ID", format!("file2_{}", i))
932
+ })
933
+ .collect();
934
+
935
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
936
+
937
+ // Index both files
938
+ let mut indexer = FileIndexer::default();
939
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
940
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
941
+
942
+ let index1 = indexer
943
+ .index(
944
+ &file1,
945
+ Some(&source_timestamp_field),
946
+ &[entry_id_field.clone()],
947
+ Seconds(3600),
948
+ )
949
+ .unwrap();
950
+
951
+ let index2 = indexer
952
+ .index(
953
+ &file2,
954
+ Some(&source_timestamp_field),
955
+ &[entry_id_field],
956
+ Seconds(3600),
957
+ )
958
+ .unwrap();
959
+
960
+ let file_indexes = vec![index1, index2];
961
+
962
+ // Paginate with limit=1 (extreme case)
963
+ let mut all_entry_ids = HashSet::new();
964
+ let mut all_timestamps = Vec::new();
965
+ let mut state = None;
966
+ let mut page_count = 0;
967
+
968
+ loop {
969
+ let (page, new_state) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
970
+ .with_limit(1)
971
+ .execute_page(state.as_ref())
972
+ .unwrap();
973
+
974
+ if page.is_empty() {
975
+ break;
976
+ }
977
+
978
+ page_count += 1;
979
+
980
+ // Each page should have exactly 1 entry
981
+ assert_eq!(page.len(), 1, "Each page should have exactly 1 entry");
982
+
983
+ // Collect ENTRY_ID and timestamp
984
+ for entry in &page {
985
+ all_timestamps.push(entry.timestamp);
986
+ for field in &entry.fields {
987
+ if field.field() == "ENTRY_ID" {
988
+ assert!(
989
+ all_entry_ids.insert(field.value().to_string()),
990
+ "Found duplicate ENTRY_ID: {}",
991
+ field.value()
992
+ );
993
+ }
994
+ }
995
+ }
996
+
997
+ state = Some(new_state);
998
+ }
999
+
1000
+ // Should need 20 pages for 20 entries
1001
+ assert_eq!(page_count, 20, "Should need exactly 20 pages");
1002
+
1003
+ // Verify we got all 20 unique entries
1004
+ assert_eq!(
1005
+ all_entry_ids.len(),
1006
+ 20,
1007
+ "Should have retrieved all 20 unique entries"
1008
+ );
1009
+
1010
+ // Verify timestamps are in ascending order
1011
+ for i in 1..all_timestamps.len() {
1012
+ assert!(
1013
+ all_timestamps[i - 1] <= all_timestamps[i],
1014
+ "Timestamps should be in ascending order"
1015
+ );
1016
+ }
1017
+
1018
+ // Verify we got all timestamps from 100-119
1019
+ let unique_timestamps: HashSet<_> = all_timestamps.into_iter().collect();
1020
+ assert_eq!(
1021
+ unique_timestamps.len(),
1022
+ 20,
1023
+ "Should have 20 unique timestamps"
1024
+ );
1025
+ for ts in 100..120 {
1026
+ assert!(unique_timestamps.contains(&ts), "Missing timestamp: {}", ts);
1027
+ }
1028
+}
1029
+
1030
+#[test]
1031
+fn test_multi_file_pagination_with_empty_file() {
1032
+ // Create temporary directory
1033
+ let temp_dir = TempDir::new().unwrap();
1034
+
1035
+ // File 1: 50 entries at t=100..150
1036
+ let entries_file1: Vec<TestEntry> = (100..150)
1037
+ .map(|i| {
1038
+ TestEntry::new(Microseconds(i))
1039
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1040
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1041
+ })
1042
+ .collect();
1043
+
1044
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1045
+
1046
+ // File 2: Empty (0 entries) - created but not indexed since empty files cannot be indexed
1047
+ let entries_file2: Vec<TestEntry> = vec![];
1048
+ let _file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1049
+
1050
+ // File 3: 50 entries at t=150..200
1051
+ let entries_file3: Vec<TestEntry> = (150..200)
1052
+ .map(|i| {
1053
+ TestEntry::new(Microseconds(i))
1054
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
1055
+ .with_field("ENTRY_ID", format!("file3_{}", i))
1056
+ })
1057
+ .collect();
1058
+
1059
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
1060
+
1061
+ // Index files - note that empty file cannot be indexed (returns EmptyHistogramInput error)
1062
+ // In practice, the system would skip files with no entries
1063
+ let mut indexer = FileIndexer::default();
1064
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1065
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1066
+
1067
+ let index1 = indexer
1068
+ .index(
1069
+ &file1,
1070
+ Some(&source_timestamp_field),
1071
+ &[entry_id_field.clone()],
1072
+ Seconds(3600),
1073
+ )
1074
+ .unwrap();
1075
+
1076
+ // Skip empty file - it cannot be indexed
1077
+ // let index2 = indexer.index(&file2, ...) would fail with EmptyHistogramInput
1078
+
1079
+ let index3 = indexer
1080
+ .index(
1081
+ &file3,
1082
+ Some(&source_timestamp_field),
1083
+ &[entry_id_field],
1084
+ Seconds(3600),
1085
+ )
1086
+ .unwrap();
1087
+
1088
+ let file_indexes = vec![index1, index3];
1089
+
1090
+ // First page: limit=60, should get 50 from file1 + 10 from file3 (skipping empty file2)
1091
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1092
+ .with_limit(60)
1093
+ .execute_page(None)
1094
+ .unwrap();
1095
+
1096
+ assert_eq!(first_page.len(), 60, "First page should contain 60 entries");
1097
+
1098
+ assert_eq!(first_page.first().unwrap().timestamp, 100);
1099
+ assert_eq!(first_page.last().unwrap().timestamp, 159);
1100
+
1101
+ // Second page: remaining 40 from file3
1102
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1103
+ .with_limit(60)
1104
+ .execute_page(Some(&state1))
1105
+ .unwrap();
1106
+
1107
+ assert_eq!(
1108
+ second_page.len(),
1109
+ 40,
1110
+ "Second page should contain remaining 40 entries"
1111
+ );
1112
+
1113
+ assert_eq!(second_page.first().unwrap().timestamp, 160);
1114
+ assert_eq!(second_page.last().unwrap().timestamp, 199);
1115
+
1116
+ // Collect all ENTRY_IDs
1117
+ let mut all_entry_ids = HashSet::new();
1118
+ for entry in first_page.iter().chain(second_page.iter()) {
1119
+ for field in &entry.fields {
1120
+ if field.field() == "ENTRY_ID" {
1121
+ assert!(
1122
+ all_entry_ids.insert(field.value().to_string()),
1123
+ "Found duplicate ENTRY_ID: {}",
1124
+ field.value()
1125
+ );
1126
+ }
1127
+ }
1128
+ }
1129
+
1130
+ // Verify we got all 100 unique entries (none from empty file)
1131
+ assert_eq!(
1132
+ all_entry_ids.len(),
1133
+ 100,
1134
+ "Should have retrieved all 100 unique entries"
1135
+ );
1136
+
1137
+ let file1_count = all_entry_ids
1138
+ .iter()
1139
+ .filter(|id| id.starts_with("file1_"))
1140
+ .count();
1141
+ let file3_count = all_entry_ids
1142
+ .iter()
1143
+ .filter(|id| id.starts_with("file3_"))
1144
+ .count();
1145
+
1146
+ assert_eq!(file1_count, 50, "Should have 50 entries from file1");
1147
+ assert_eq!(file3_count, 50, "Should have 50 entries from file3");
1148
+
1149
+ // Third page should be empty
1150
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1151
+ .with_limit(60)
1152
+ .execute_page(Some(&state2))
1153
+ .unwrap();
1154
+
1155
+ assert_eq!(
1156
+ third_page.len(),
1157
+ 0,
1158
+ "Third page should be empty (no more entries)"
1159
+ );
1160
+}
1161
+
1162
+#[test]
1163
+fn test_multi_file_pagination_reverse_file_order() {
1164
+ // Create temporary directory
1165
+ let temp_dir = TempDir::new().unwrap();
1166
+
1167
+ // File 1: entries at t=100..200 (oldest)
1168
+ let entries_file1: Vec<TestEntry> = (100..200)
1169
+ .map(|i| {
1170
+ TestEntry::new(Microseconds(i))
1171
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1172
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1173
+ })
1174
+ .collect();
1175
+
1176
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1177
+
1178
+ // File 2: entries at t=200..300 (middle)
1179
+ let entries_file2: Vec<TestEntry> = (200..300)
1180
+ .map(|i| {
1181
+ TestEntry::new(Microseconds(i))
1182
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1183
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1184
+ })
1185
+ .collect();
1186
+
1187
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1188
+
1189
+ // File 3: entries at t=300..400 (newest)
1190
+ let entries_file3: Vec<TestEntry> = (300..400)
1191
+ .map(|i| {
1192
+ TestEntry::new(Microseconds(i))
1193
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
1194
+ .with_field("ENTRY_ID", format!("file3_{}", i))
1195
+ })
1196
+ .collect();
1197
+
1198
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
1199
+
1200
+ // Index all three files
1201
+ let mut indexer = FileIndexer::default();
1202
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1203
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1204
+
1205
+ let index1 = indexer
1206
+ .index(
1207
+ &file1,
1208
+ Some(&source_timestamp_field),
1209
+ &[entry_id_field.clone()],
1210
+ Seconds(3600),
1211
+ )
1212
+ .unwrap();
1213
+
1214
+ let index2 = indexer
1215
+ .index(
1216
+ &file2,
1217
+ Some(&source_timestamp_field),
1218
+ &[entry_id_field.clone()],
1219
+ Seconds(3600),
1220
+ )
1221
+ .unwrap();
1222
+
1223
+ let index3 = indexer
1224
+ .index(
1225
+ &file3,
1226
+ Some(&source_timestamp_field),
1227
+ &[entry_id_field],
1228
+ Seconds(3600),
1229
+ )
1230
+ .unwrap();
1231
+
1232
+ // Pass files in REVERSE chronological order (newest first)
1233
+ // The query should still process them in correct temporal order
1234
+ let file_indexes = vec![index3, index2, index1];
1235
+
1236
+ // Query should still return entries in ascending timestamp order
1237
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1238
+ .with_limit(150)
1239
+ .execute_page(None)
1240
+ .unwrap();
1241
+
1242
+ assert_eq!(first_page.len(), 150);
1243
+
1244
+ // First entry should be from file1 (oldest timestamp)
1245
+ assert_eq!(
1246
+ first_page.first().unwrap().timestamp,
1247
+ 100,
1248
+ "First entry should be at timestamp 100 from file1"
1249
+ );
1250
+
1251
+ // Verify ascending order
1252
+ for i in 1..first_page.len() {
1253
+ assert!(
1254
+ first_page[i - 1].timestamp <= first_page[i].timestamp,
1255
+ "Entries should be in ascending order"
1256
+ );
1257
+ }
1258
+
1259
+ // Continue pagination
1260
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1261
+ .with_limit(150)
1262
+ .execute_page(Some(&state1))
1263
+ .unwrap();
1264
+
1265
+ assert_eq!(second_page.len(), 150);
1266
+
1267
+ for i in 1..second_page.len() {
1268
+ assert!(
1269
+ second_page[i - 1].timestamp <= second_page[i].timestamp,
1270
+ "Entries should be in ascending order"
1271
+ );
1272
+ }
1273
+
1274
+ // Verify continuity between pages
1275
+ assert!(
1276
+ first_page.last().unwrap().timestamp <= second_page.first().unwrap().timestamp,
1277
+ "Second page should continue from first page"
1278
+ );
1279
+
1280
+ // Collect all ENTRY_IDs
1281
+ let mut all_entry_ids = HashSet::new();
1282
+ for entry in first_page.iter().chain(second_page.iter()) {
1283
+ for field in &entry.fields {
1284
+ if field.field() == "ENTRY_ID" {
1285
+ assert!(
1286
+ all_entry_ids.insert(field.value().to_string()),
1287
+ "Found duplicate ENTRY_ID: {}",
1288
+ field.value()
1289
+ );
1290
+ }
1291
+ }
1292
+ }
1293
+
1294
+ // Should have 300 unique entries
1295
+ assert_eq!(
1296
+ all_entry_ids.len(),
1297
+ 300,
1298
+ "Should have retrieved all 300 unique entries"
1299
+ );
1300
+
1301
+ // Third page should be empty
1302
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
1303
+ .with_limit(150)
1304
+ .execute_page(Some(&state2))
1305
+ .unwrap();
1306
+
1307
+ assert_eq!(third_page.len(), 0, "Third page should be empty");
1308
+}
1309
+
1310
+#[test]
1311
+fn test_multi_file_pagination_backward_non_overlapping() {
1312
+ // Create temporary directory
1313
+ let temp_dir = TempDir::new().unwrap();
1314
+
1315
+ // File 1: entries at t=100..200 (100 entries)
1316
+ let entries_file1: Vec<TestEntry> = (100..200)
1317
+ .map(|i| {
1318
+ TestEntry::new(Microseconds(i))
1319
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1320
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1321
+ .with_field("FILE", "1")
1322
+ })
1323
+ .collect();
1324
+
1325
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1326
+
1327
+ // File 2: entries at t=200..300 (100 entries)
1328
+ let entries_file2: Vec<TestEntry> = (200..300)
1329
+ .map(|i| {
1330
+ TestEntry::new(Microseconds(i))
1331
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1332
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1333
+ .with_field("FILE", "2")
1334
+ })
1335
+ .collect();
1336
+
1337
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1338
+
1339
+ // Index both files
1340
+ let mut indexer = FileIndexer::default();
1341
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1342
+ let file_field = FieldName::new("FILE").unwrap();
1343
+
1344
+ let index1 = indexer
1345
+ .index(
1346
+ &file1,
1347
+ Some(&source_timestamp_field),
1348
+ &[file_field.clone()],
1349
+ Seconds(3600),
1350
+ )
1351
+ .unwrap();
1352
+
1353
+ let index2 = indexer
1354
+ .index(
1355
+ &file2,
1356
+ Some(&source_timestamp_field),
1357
+ &[file_field],
1358
+ Seconds(3600),
1359
+ )
1360
+ .unwrap();
1361
+
1362
+ let file_indexes = vec![index1, index2];
1363
+
1364
+ // First page: limit=150, backward from tail, should get all 100 from file2 + 50 from file1
1365
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1366
+ .with_limit(150)
1367
+ .execute_page(None)
1368
+ .unwrap();
1369
+
1370
+ assert_eq!(
1371
+ first_page.len(),
1372
+ 150,
1373
+ "First page should contain exactly 150 entries"
1374
+ );
1375
+
1376
+ // Verify timestamps are in descending order
1377
+ for i in 1..first_page.len() {
1378
+ assert!(
1379
+ first_page[i - 1].timestamp >= first_page[i].timestamp,
1380
+ "Entries should be in descending timestamp order"
1381
+ );
1382
+ }
1383
+
1384
+ // First entry should be at timestamp 299 (highest)
1385
+ assert_eq!(
1386
+ first_page.first().unwrap().timestamp,
1387
+ 299,
1388
+ "First entry should be at timestamp 299"
1389
+ );
1390
+
1391
+ // Last entry should be at timestamp 150 (100 from file2: 299-200, then 50 from file1: 199-150)
1392
+ assert_eq!(
1393
+ first_page.last().unwrap().timestamp,
1394
+ 150,
1395
+ "Last entry of first page should be at timestamp 150"
1396
+ );
1397
+
1398
+ // State should track positions for files we read from
1399
+ assert!(
1400
+ !state1.file_positions.is_empty(),
1401
+ "State should track positions"
1402
+ );
1403
+
1404
+ // Second page: use state to get remaining 50 entries
1405
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1406
+ .with_limit(150)
1407
+ .execute_page(Some(&state1))
1408
+ .unwrap();
1409
+
1410
+ assert_eq!(
1411
+ second_page.len(),
1412
+ 50,
1413
+ "Second page should contain remaining 50 entries"
1414
+ );
1415
+
1416
+ // Verify timestamps continue in descending order
1417
+ assert_eq!(
1418
+ second_page.first().unwrap().timestamp,
1419
+ 149,
1420
+ "Second page should start at timestamp 149"
1421
+ );
1422
+
1423
+ assert_eq!(
1424
+ second_page.last().unwrap().timestamp,
1425
+ 100,
1426
+ "Second page should end at timestamp 100"
1427
+ );
1428
+
1429
+ // Verify no duplicates across both pages
1430
+ let mut all_timestamps = HashSet::new();
1431
+ for entry in &first_page {
1432
+ assert!(
1433
+ all_timestamps.insert(entry.timestamp),
1434
+ "Found duplicate timestamp: {}",
1435
+ entry.timestamp
1436
+ );
1437
+ }
1438
+ for entry in &second_page {
1439
+ assert!(
1440
+ all_timestamps.insert(entry.timestamp),
1441
+ "Found duplicate timestamp: {}",
1442
+ entry.timestamp
1443
+ );
1444
+ }
1445
+
1446
+ // Verify we got all 200 unique entries
1447
+ assert_eq!(
1448
+ all_timestamps.len(),
1449
+ 200,
1450
+ "Should have retrieved all 200 unique entries"
1451
+ );
1452
+
1453
+ // Third page should be empty
1454
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1455
+ .with_limit(150)
1456
+ .execute_page(Some(&state2))
1457
+ .unwrap();
1458
+
1459
+ assert_eq!(
1460
+ third_page.len(),
1461
+ 0,
1462
+ "Third page should be empty (no more entries)"
1463
+ );
1464
+}
1465
+
1466
+#[test]
1467
+fn test_multi_file_pagination_backward_same_timestamps() {
1468
+ // Create temporary directory
1469
+ let temp_dir = TempDir::new().unwrap();
1470
+
1471
+ // File 1: 150 entries all at timestamp 1000
1472
+ let entries_file1: Vec<TestEntry> = (0..150)
1473
+ .map(|i| {
1474
+ TestEntry::new(Microseconds(1000))
1475
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1476
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1477
+ .with_field("FILE", "1")
1478
+ })
1479
+ .collect();
1480
+
1481
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1482
+
1483
+ // File 2: 150 entries all at timestamp 1000
1484
+ let entries_file2: Vec<TestEntry> = (0..150)
1485
+ .map(|i| {
1486
+ TestEntry::new(Microseconds(1000))
1487
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1488
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1489
+ .with_field("FILE", "2")
1490
+ })
1491
+ .collect();
1492
+
1493
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1494
+
1495
+ // Index both files
1496
+ let mut indexer = FileIndexer::default();
1497
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1498
+ let file_field = FieldName::new("FILE").unwrap();
1499
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1500
+
1501
+ let index1 = indexer
1502
+ .index(
1503
+ &file1,
1504
+ Some(&source_timestamp_field),
1505
+ &[file_field.clone(), entry_id_field.clone()],
1506
+ Seconds(3600),
1507
+ )
1508
+ .unwrap();
1509
+
1510
+ let index2 = indexer
1511
+ .index(
1512
+ &file2,
1513
+ Some(&source_timestamp_field),
1514
+ &[file_field, entry_id_field],
1515
+ Seconds(3600),
1516
+ )
1517
+ .unwrap();
1518
+
1519
+ let file_indexes = vec![index1, index2];
1520
+
1521
+ // First page: limit=200, backward from tail
1522
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1523
+ .with_limit(200)
1524
+ .execute_page(None)
1525
+ .unwrap();
1526
+
1527
+ assert_eq!(
1528
+ first_page.len(),
1529
+ 200,
1530
+ "First page should contain exactly 200 entries"
1531
+ );
1532
+
1533
+ // All timestamps should be 1000
1534
+ for entry in &first_page {
1535
+ assert_eq!(
1536
+ entry.timestamp, 1000,
1537
+ "All entries should have timestamp 1000"
1538
+ );
1539
+ }
1540
+
1541
+ // State should track positions for both files
1542
+ assert_eq!(
1543
+ state1.file_positions.len(),
1544
+ 2,
1545
+ "State should track positions for both files"
1546
+ );
1547
+
1548
+ // Second page: use state to get remaining 100 entries
1549
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1550
+ .with_limit(200)
1551
+ .execute_page(Some(&state1))
1552
+ .unwrap();
1553
+
1554
+ assert_eq!(
1555
+ second_page.len(),
1556
+ 100,
1557
+ "Second page should contain remaining 100 entries"
1558
+ );
1559
+
1560
+ // All timestamps should still be 1000
1561
+ for entry in &second_page {
1562
+ assert_eq!(
1563
+ entry.timestamp, 1000,
1564
+ "All entries should have timestamp 1000"
1565
+ );
1566
+ }
1567
+
1568
+ // Collect all ENTRY_ID values to verify uniqueness
1569
+ let mut all_entry_ids = HashSet::new();
1570
+
1571
+ for entry in &first_page {
1572
+ for field in &entry.fields {
1573
+ if field.field() == "ENTRY_ID" {
1574
+ assert!(
1575
+ all_entry_ids.insert(field.value().to_string()),
1576
+ "Found duplicate ENTRY_ID: {}",
1577
+ field.value()
1578
+ );
1579
+ }
1580
+ }
1581
+ }
1582
+
1583
+ for entry in &second_page {
1584
+ for field in &entry.fields {
1585
+ if field.field() == "ENTRY_ID" {
1586
+ assert!(
1587
+ all_entry_ids.insert(field.value().to_string()),
1588
+ "Found duplicate ENTRY_ID: {}",
1589
+ field.value()
1590
+ );
1591
+ }
1592
+ }
1593
+ }
1594
+
1595
+ // Verify we got all 300 unique entries
1596
+ assert_eq!(
1597
+ all_entry_ids.len(),
1598
+ 300,
1599
+ "Should have retrieved all 300 unique entries"
1600
+ );
1601
+
1602
+ // Verify we have entries from both files
1603
+ let file1_entries: usize = all_entry_ids
1604
+ .iter()
1605
+ .filter(|id| id.starts_with("file1_"))
1606
+ .count();
1607
+ let file2_entries: usize = all_entry_ids
1608
+ .iter()
1609
+ .filter(|id| id.starts_with("file2_"))
1610
+ .count();
1611
+
1612
+ assert_eq!(file1_entries, 150, "Should have 150 entries from file1");
1613
+ assert_eq!(file2_entries, 150, "Should have 150 entries from file2");
1614
+
1615
+ // Third page should be empty
1616
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1617
+ .with_limit(200)
1618
+ .execute_page(Some(&state2))
1619
+ .unwrap();
1620
+
1621
+ assert_eq!(
1622
+ third_page.len(),
1623
+ 0,
1624
+ "Third page should be empty (no more entries)"
1625
+ );
1626
+}
1627
+
1628
+#[test]
1629
+fn test_multi_file_pagination_backward_limit_one() {
1630
+ // Create temporary directory
1631
+ let temp_dir = TempDir::new().unwrap();
1632
+
1633
+ // File 1: 10 entries at t=100..110
1634
+ let entries_file1: Vec<TestEntry> = (100..110)
1635
+ .map(|i| {
1636
+ TestEntry::new(Microseconds(i))
1637
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1638
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1639
+ })
1640
+ .collect();
1641
+
1642
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1643
+
1644
+ // File 2: 10 entries at t=110..120
1645
+ let entries_file2: Vec<TestEntry> = (110..120)
1646
+ .map(|i| {
1647
+ TestEntry::new(Microseconds(i))
1648
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1649
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1650
+ })
1651
+ .collect();
1652
+
1653
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1654
+
1655
+ // Index both files
1656
+ let mut indexer = FileIndexer::default();
1657
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1658
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1659
+
1660
+ let index1 = indexer
1661
+ .index(
1662
+ &file1,
1663
+ Some(&source_timestamp_field),
1664
+ &[entry_id_field.clone()],
1665
+ Seconds(3600),
1666
+ )
1667
+ .unwrap();
1668
+
1669
+ let index2 = indexer
1670
+ .index(
1671
+ &file2,
1672
+ Some(&source_timestamp_field),
1673
+ &[entry_id_field],
1674
+ Seconds(3600),
1675
+ )
1676
+ .unwrap();
1677
+
1678
+ let file_indexes = vec![index1, index2];
1679
+
1680
+ // Paginate backward with limit=1 (extreme case)
1681
+ let mut all_entry_ids = HashSet::new();
1682
+ let mut all_timestamps = Vec::new();
1683
+ let mut state = None;
1684
+ let mut page_count = 0;
1685
+
1686
+ loop {
1687
+ let (page, new_state) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
1688
+ .with_limit(1)
1689
+ .execute_page(state.as_ref())
1690
+ .unwrap();
1691
+
1692
+ if page.is_empty() {
1693
+ break;
1694
+ }
1695
+
1696
+ page_count += 1;
1697
+
1698
+ // Each page should have exactly 1 entry
1699
+ assert_eq!(page.len(), 1, "Each page should have exactly 1 entry");
1700
+
1701
+ // Collect ENTRY_ID and timestamp
1702
+ for entry in &page {
1703
+ all_timestamps.push(entry.timestamp);
1704
+ for field in &entry.fields {
1705
+ if field.field() == "ENTRY_ID" {
1706
+ assert!(
1707
+ all_entry_ids.insert(field.value().to_string()),
1708
+ "Found duplicate ENTRY_ID: {}",
1709
+ field.value()
1710
+ );
1711
+ }
1712
+ }
1713
+ }
1714
+
1715
+ state = Some(new_state);
1716
+ }
1717
+
1718
+ // Should need 20 pages for 20 entries
1719
+ assert_eq!(page_count, 20, "Should need exactly 20 pages");
1720
+
1721
+ // Verify we got all 20 unique entries
1722
+ assert_eq!(
1723
+ all_entry_ids.len(),
1724
+ 20,
1725
+ "Should have retrieved all 20 unique entries"
1726
+ );
1727
+
1728
+ // Verify timestamps are in descending order
1729
+ for i in 1..all_timestamps.len() {
1730
+ assert!(
1731
+ all_timestamps[i - 1] >= all_timestamps[i],
1732
+ "Timestamps should be in descending order"
1733
+ );
1734
+ }
1735
+
1736
+ // Verify we got all timestamps from 119 down to 100
1737
+ let unique_timestamps: HashSet<_> = all_timestamps.into_iter().collect();
1738
+ assert_eq!(
1739
+ unique_timestamps.len(),
1740
+ 20,
1741
+ "Should have 20 unique timestamps"
1742
+ );
1743
+ for ts in 100..120 {
1744
+ assert!(unique_timestamps.contains(&ts), "Missing timestamp: {}", ts);
1745
+ }
1746
+}
1747
+
1748
+#[test]
1749
+fn test_multi_file_pagination_anchor_timestamp_forward() {
1750
+ // Create temporary directory
1751
+ let temp_dir = TempDir::new().unwrap();
1752
+
1753
+ // File 1: entries at t=100..200 (100 entries)
1754
+ let entries_file1: Vec<TestEntry> = (100..200)
1755
+ .map(|i| {
1756
+ TestEntry::new(Microseconds(i))
1757
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1758
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1759
+ })
1760
+ .collect();
1761
+
1762
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1763
+
1764
+ // File 2: entries at t=200..300 (100 entries)
1765
+ let entries_file2: Vec<TestEntry> = (200..300)
1766
+ .map(|i| {
1767
+ TestEntry::new(Microseconds(i))
1768
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1769
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1770
+ })
1771
+ .collect();
1772
+
1773
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1774
+
1775
+ // Index both files
1776
+ let mut indexer = FileIndexer::default();
1777
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1778
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1779
+
1780
+ let index1 = indexer
1781
+ .index(
1782
+ &file1,
1783
+ Some(&source_timestamp_field),
1784
+ &[entry_id_field.clone()],
1785
+ Seconds(3600),
1786
+ )
1787
+ .unwrap();
1788
+
1789
+ let index2 = indexer
1790
+ .index(
1791
+ &file2,
1792
+ Some(&source_timestamp_field),
1793
+ &[entry_id_field],
1794
+ Seconds(3600),
1795
+ )
1796
+ .unwrap();
1797
+
1798
+ let file_indexes = vec![index1, index2];
1799
+
1800
+ // Start from middle timestamp 150 (in file1), forward direction
1801
+ let anchor = Anchor::Timestamp(Microseconds(150));
1802
+
1803
+ // First page: limit=80, should get 50 from file1 (150-199) + 30 from file2 (200-229)
1804
+ let (first_page, state1) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
1805
+ .with_limit(80)
1806
+ .execute_page(None)
1807
+ .unwrap();
1808
+
1809
+ assert_eq!(
1810
+ first_page.len(),
1811
+ 80,
1812
+ "First page should contain exactly 80 entries"
1813
+ );
1814
+
1815
+ // First entry should be at timestamp 150
1816
+ assert_eq!(
1817
+ first_page.first().unwrap().timestamp,
1818
+ 150,
1819
+ "First entry should be at timestamp 150 (anchor)"
1820
+ );
1821
+
1822
+ // Last entry should be at timestamp 229
1823
+ assert_eq!(
1824
+ first_page.last().unwrap().timestamp,
1825
+ 229,
1826
+ "Last entry should be at timestamp 229"
1827
+ );
1828
+
1829
+ // Verify ascending order
1830
+ for i in 1..first_page.len() {
1831
+ assert!(
1832
+ first_page[i - 1].timestamp <= first_page[i].timestamp,
1833
+ "Entries should be in ascending order"
1834
+ );
1835
+ }
1836
+
1837
+ // Second page: get remaining entries
1838
+ let (second_page, state2) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
1839
+ .with_limit(80)
1840
+ .execute_page(Some(&state1))
1841
+ .unwrap();
1842
+
1843
+ assert_eq!(
1844
+ second_page.len(),
1845
+ 70,
1846
+ "Second page should contain remaining 70 entries (230-299)"
1847
+ );
1848
+
1849
+ assert_eq!(
1850
+ second_page.first().unwrap().timestamp,
1851
+ 230,
1852
+ "Second page should start at timestamp 230"
1853
+ );
1854
+
1855
+ assert_eq!(
1856
+ second_page.last().unwrap().timestamp,
1857
+ 299,
1858
+ "Second page should end at timestamp 299"
1859
+ );
1860
+
1861
+ // Verify no duplicates
1862
+ let mut all_entry_ids = HashSet::new();
1863
+ for entry in first_page.iter().chain(second_page.iter()) {
1864
+ for field in &entry.fields {
1865
+ if field.field() == "ENTRY_ID" {
1866
+ assert!(
1867
+ all_entry_ids.insert(field.value().to_string()),
1868
+ "Found duplicate ENTRY_ID: {}",
1869
+ field.value()
1870
+ );
1871
+ }
1872
+ }
1873
+ }
1874
+
1875
+ // Should have 150 entries total (from 150-299)
1876
+ assert_eq!(
1877
+ all_entry_ids.len(),
1878
+ 150,
1879
+ "Should have retrieved 150 unique entries from timestamp 150 onwards"
1880
+ );
1881
+
1882
+ // Third page should be empty
1883
+ let (third_page, _state3) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
1884
+ .with_limit(80)
1885
+ .execute_page(Some(&state2))
1886
+ .unwrap();
1887
+
1888
+ assert_eq!(
1889
+ third_page.len(),
1890
+ 0,
1891
+ "Third page should be empty (no more entries)"
1892
+ );
1893
+}
1894
+
1895
+#[test]
1896
+fn test_multi_file_pagination_anchor_timestamp_backward() {
1897
+ // Create temporary directory
1898
+ let temp_dir = TempDir::new().unwrap();
1899
+
1900
+ // File 1: entries at t=100..200 (100 entries)
1901
+ let entries_file1: Vec<TestEntry> = (100..200)
1902
+ .map(|i| {
1903
+ TestEntry::new(Microseconds(i))
1904
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
1905
+ .with_field("ENTRY_ID", format!("file1_{}", i))
1906
+ })
1907
+ .collect();
1908
+
1909
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
1910
+
1911
+ // File 2: entries at t=200..300 (100 entries)
1912
+ let entries_file2: Vec<TestEntry> = (200..300)
1913
+ .map(|i| {
1914
+ TestEntry::new(Microseconds(i))
1915
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
1916
+ .with_field("ENTRY_ID", format!("file2_{}", i))
1917
+ })
1918
+ .collect();
1919
+
1920
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
1921
+
1922
+ // Index both files
1923
+ let mut indexer = FileIndexer::default();
1924
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
1925
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
1926
+
1927
+ let index1 = indexer
1928
+ .index(
1929
+ &file1,
1930
+ Some(&source_timestamp_field),
1931
+ &[entry_id_field.clone()],
1932
+ Seconds(3600),
1933
+ )
1934
+ .unwrap();
1935
+
1936
+ let index2 = indexer
1937
+ .index(
1938
+ &file2,
1939
+ Some(&source_timestamp_field),
1940
+ &[entry_id_field],
1941
+ Seconds(3600),
1942
+ )
1943
+ .unwrap();
1944
+
1945
+ let file_indexes = vec![index1, index2];
1946
+
1947
+ // Start from middle timestamp 250 (in file2), backward direction
1948
+ let anchor = Anchor::Timestamp(Microseconds(250));
1949
+
1950
+ // First page: limit=80, should get 51 from file2 (250-200) + 29 from file1 (199-171)
1951
+ let (first_page, state1) = LogQuery::new(&file_indexes, anchor, Direction::Backward)
1952
+ .with_limit(80)
1953
+ .execute_page(None)
1954
+ .unwrap();
1955
+
1956
+ assert_eq!(
1957
+ first_page.len(),
1958
+ 80,
1959
+ "First page should contain exactly 80 entries"
1960
+ );
1961
+
1962
+ // First entry should be at timestamp 250
1963
+ assert_eq!(
1964
+ first_page.first().unwrap().timestamp,
1965
+ 250,
1966
+ "First entry should be at timestamp 250 (anchor)"
1967
+ );
1968
+
1969
+ // Last entry should be at timestamp 171
1970
+ assert_eq!(
1971
+ first_page.last().unwrap().timestamp,
1972
+ 171,
1973
+ "Last entry should be at timestamp 171"
1974
+ );
1975
+
1976
+ // Verify descending order
1977
+ for i in 1..first_page.len() {
1978
+ assert!(
1979
+ first_page[i - 1].timestamp >= first_page[i].timestamp,
1980
+ "Entries should be in descending order"
1981
+ );
1982
+ }
1983
+
1984
+ // Second page: get remaining entries
1985
+ let (second_page, state2) = LogQuery::new(&file_indexes, anchor, Direction::Backward)
1986
+ .with_limit(80)
1987
+ .execute_page(Some(&state1))
1988
+ .unwrap();
1989
+
1990
+ assert_eq!(
1991
+ second_page.len(),
1992
+ 71,
1993
+ "Second page should contain remaining 71 entries (170-100)"
1994
+ );
1995
+
1996
+ assert_eq!(
1997
+ second_page.first().unwrap().timestamp,
1998
+ 170,
1999
+ "Second page should start at timestamp 170"
2000
+ );
2001
+
2002
+ assert_eq!(
2003
+ second_page.last().unwrap().timestamp,
2004
+ 100,
2005
+ "Second page should end at timestamp 100"
2006
+ );
2007
+
2008
+ // Verify no duplicates
2009
+ let mut all_entry_ids = HashSet::new();
2010
+ for entry in first_page.iter().chain(second_page.iter()) {
2011
+ for field in &entry.fields {
2012
+ if field.field() == "ENTRY_ID" {
2013
+ assert!(
2014
+ all_entry_ids.insert(field.value().to_string()),
2015
+ "Found duplicate ENTRY_ID: {}",
2016
+ field.value()
2017
+ );
2018
+ }
2019
+ }
2020
+ }
2021
+
2022
+ // Should have 151 entries total (from 250 down to 100)
2023
+ assert_eq!(
2024
+ all_entry_ids.len(),
2025
+ 151,
2026
+ "Should have retrieved 151 unique entries from timestamp 250 backwards to 100"
2027
+ );
2028
+
2029
+ // Third page should be empty
2030
+ let (third_page, _state3) = LogQuery::new(&file_indexes, anchor, Direction::Backward)
2031
+ .with_limit(80)
2032
+ .execute_page(Some(&state2))
2033
+ .unwrap();
2034
+
2035
+ assert_eq!(
2036
+ third_page.len(),
2037
+ 0,
2038
+ "Third page should be empty (no more entries)"
2039
+ );
2040
+}
2041
+
2042
+#[test]
2043
+fn test_multi_file_pagination_anchor_timestamp_same_timestamps() {
2044
+ // Create temporary directory
2045
+ let temp_dir = TempDir::new().unwrap();
2046
+
2047
+ // File 1: 100 entries all at timestamp 150
2048
+ let entries_file1: Vec<TestEntry> = (0..100)
2049
+ .map(|i| {
2050
+ TestEntry::new(Microseconds(150))
2051
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
2052
+ .with_field("ENTRY_ID", format!("file1_{}", i))
2053
+ })
2054
+ .collect();
2055
+
2056
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
2057
+
2058
+ // File 2: 100 entries all at timestamp 150
2059
+ let entries_file2: Vec<TestEntry> = (0..100)
2060
+ .map(|i| {
2061
+ TestEntry::new(Microseconds(150))
2062
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
2063
+ .with_field("ENTRY_ID", format!("file2_{}", i))
2064
+ })
2065
+ .collect();
2066
+
2067
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
2068
+
2069
+ // Index both files
2070
+ let mut indexer = FileIndexer::default();
2071
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
2072
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
2073
+
2074
+ let index1 = indexer
2075
+ .index(
2076
+ &file1,
2077
+ Some(&source_timestamp_field),
2078
+ &[entry_id_field.clone()],
2079
+ Seconds(3600),
2080
+ )
2081
+ .unwrap();
2082
+
2083
+ let index2 = indexer
2084
+ .index(
2085
+ &file2,
2086
+ Some(&source_timestamp_field),
2087
+ &[entry_id_field],
2088
+ Seconds(3600),
2089
+ )
2090
+ .unwrap();
2091
+
2092
+ let file_indexes = vec![index1, index2];
2093
+
2094
+ // Anchor at timestamp 150 (all entries have this timestamp), forward direction
2095
+ let anchor = Anchor::Timestamp(Microseconds(150));
2096
+
2097
+ // First page: limit=80
2098
+ let (first_page, state1) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
2099
+ .with_limit(80)
2100
+ .execute_page(None)
2101
+ .unwrap();
2102
+
2103
+ assert_eq!(
2104
+ first_page.len(),
2105
+ 80,
2106
+ "First page should contain exactly 80 entries"
2107
+ );
2108
+
2109
+ // All timestamps should be 150
2110
+ for entry in &first_page {
2111
+ assert_eq!(
2112
+ entry.timestamp, 150,
2113
+ "All entries should have timestamp 150"
2114
+ );
2115
+ }
2116
+
2117
+ // Second page: get remaining entries
2118
+ let (second_page, state2) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
2119
+ .with_limit(80)
2120
+ .execute_page(Some(&state1))
2121
+ .unwrap();
2122
+
2123
+ assert_eq!(
2124
+ second_page.len(),
2125
+ 80,
2126
+ "Second page should contain 80 entries"
2127
+ );
2128
+
2129
+ for entry in &second_page {
2130
+ assert_eq!(
2131
+ entry.timestamp, 150,
2132
+ "All entries should have timestamp 150"
2133
+ );
2134
+ }
2135
+
2136
+ // Third page: remaining entries
2137
+ let (third_page, state3) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
2138
+ .with_limit(80)
2139
+ .execute_page(Some(&state2))
2140
+ .unwrap();
2141
+
2142
+ assert_eq!(
2143
+ third_page.len(),
2144
+ 40,
2145
+ "Third page should contain remaining 40 entries"
2146
+ );
2147
+
2148
+ for entry in &third_page {
2149
+ assert_eq!(
2150
+ entry.timestamp, 150,
2151
+ "All entries should have timestamp 150"
2152
+ );
2153
+ }
2154
+
2155
+ // Verify no duplicates
2156
+ let mut all_entry_ids = HashSet::new();
2157
+ for entry in first_page
2158
+ .iter()
2159
+ .chain(second_page.iter())
2160
+ .chain(third_page.iter())
2161
+ {
2162
+ for field in &entry.fields {
2163
+ if field.field() == "ENTRY_ID" {
2164
+ assert!(
2165
+ all_entry_ids.insert(field.value().to_string()),
2166
+ "Found duplicate ENTRY_ID: {}",
2167
+ field.value()
2168
+ );
2169
+ }
2170
+ }
2171
+ }
2172
+
2173
+ // Should have all 200 entries
2174
+ assert_eq!(
2175
+ all_entry_ids.len(),
2176
+ 200,
2177
+ "Should have retrieved all 200 unique entries"
2178
+ );
2179
+
2180
+ // Fourth page should be empty
2181
+ let (fourth_page, _state4) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
2182
+ .with_limit(80)
2183
+ .execute_page(Some(&state3))
2184
+ .unwrap();
2185
+
2186
+ assert_eq!(
2187
+ fourth_page.len(),
2188
+ 0,
2189
+ "Fourth page should be empty (no more entries)"
2190
+ );
2191
+}
2192
+
2193
+#[test]
2194
+fn test_multi_file_pagination_forward_with_time_boundaries() {
2195
+ // Create temporary directory
2196
+ let temp_dir = TempDir::new().unwrap();
2197
+
2198
+ // File 1: entries at t=100..200 (100 entries)
2199
+ let entries_file1: Vec<TestEntry> = (100..200)
2200
+ .map(|i| {
2201
+ TestEntry::new(Microseconds(i))
2202
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
2203
+ .with_field("ENTRY_ID", format!("file1_{}", i))
2204
+ })
2205
+ .collect();
2206
+
2207
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
2208
+
2209
+ // File 2: entries at t=200..300 (100 entries)
2210
+ let entries_file2: Vec<TestEntry> = (200..300)
2211
+ .map(|i| {
2212
+ TestEntry::new(Microseconds(i))
2213
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
2214
+ .with_field("ENTRY_ID", format!("file2_{}", i))
2215
+ })
2216
+ .collect();
2217
+
2218
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
2219
+
2220
+ // File 3: entries at t=300..400 (100 entries)
2221
+ let entries_file3: Vec<TestEntry> = (300..400)
2222
+ .map(|i| {
2223
+ TestEntry::new(Microseconds(i))
2224
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
2225
+ .with_field("ENTRY_ID", format!("file3_{}", i))
2226
+ })
2227
+ .collect();
2228
+
2229
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
2230
+
2231
+ // Index all three files
2232
+ let mut indexer = FileIndexer::default();
2233
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
2234
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
2235
+
2236
+ let index1 = indexer
2237
+ .index(
2238
+ &file1,
2239
+ Some(&source_timestamp_field),
2240
+ &[entry_id_field.clone()],
2241
+ Seconds(3600),
2242
+ )
2243
+ .unwrap();
2244
+
2245
+ let index2 = indexer
2246
+ .index(
2247
+ &file2,
2248
+ Some(&source_timestamp_field),
2249
+ &[entry_id_field.clone()],
2250
+ Seconds(3600),
2251
+ )
2252
+ .unwrap();
2253
+
2254
+ let index3 = indexer
2255
+ .index(
2256
+ &file3,
2257
+ Some(&source_timestamp_field),
2258
+ &[entry_id_field],
2259
+ Seconds(3600),
2260
+ )
2261
+ .unwrap();
2262
+
2263
+ let file_indexes = vec![index1, index2, index3];
2264
+
2265
+ // Query with time boundaries: after=150, before=350
2266
+ // This should return entries from 150-349 (200 entries total)
2267
+ // File1: 150-199 (50 entries)
2268
+ // File2: 200-299 (100 entries)
2269
+ // File3: 300-349 (50 entries)
2270
+
2271
+ // First page: limit=80
2272
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2273
+ .with_after_usec(150)
2274
+ .with_before_usec(350)
2275
+ .with_limit(80)
2276
+ .execute_page(None)
2277
+ .unwrap();
2278
+
2279
+ assert_eq!(first_page.len(), 80, "First page should contain 80 entries");
2280
+
2281
+ // Should start at timestamp 150
2282
+ assert_eq!(
2283
+ first_page.first().unwrap().timestamp,
2284
+ 150,
2285
+ "First entry should be at timestamp 150"
2286
+ );
2287
+
2288
+ // Should end at timestamp 229 (50 from file1 + 30 from file2)
2289
+ assert_eq!(
2290
+ first_page.last().unwrap().timestamp,
2291
+ 229,
2292
+ "Last entry should be at timestamp 229"
2293
+ );
2294
+
2295
+ // Verify all timestamps are within boundaries
2296
+ for entry in &first_page {
2297
+ assert!(
2298
+ entry.timestamp >= 150 && entry.timestamp < 350,
2299
+ "Entry timestamp {} should be within [150, 350)",
2300
+ entry.timestamp
2301
+ );
2302
+ }
2303
+
2304
+ // Second page: limit=80
2305
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2306
+ .with_after_usec(150)
2307
+ .with_before_usec(350)
2308
+ .with_limit(80)
2309
+ .execute_page(Some(&state1))
2310
+ .unwrap();
2311
+
2312
+ assert_eq!(
2313
+ second_page.len(),
2314
+ 80,
2315
+ "Second page should contain 80 entries"
2316
+ );
2317
+
2318
+ assert_eq!(
2319
+ second_page.first().unwrap().timestamp,
2320
+ 230,
2321
+ "Second page should start at timestamp 230"
2322
+ );
2323
+
2324
+ assert_eq!(
2325
+ second_page.last().unwrap().timestamp,
2326
+ 309,
2327
+ "Second page should end at timestamp 309"
2328
+ );
2329
+
2330
+ // Verify all timestamps are within boundaries
2331
+ for entry in &second_page {
2332
+ assert!(
2333
+ entry.timestamp >= 150 && entry.timestamp < 350,
2334
+ "Entry timestamp {} should be within [150, 350)",
2335
+ entry.timestamp
2336
+ );
2337
+ }
2338
+
2339
+ // Third page: remaining 40 entries
2340
+ let (third_page, state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2341
+ .with_after_usec(150)
2342
+ .with_before_usec(350)
2343
+ .with_limit(80)
2344
+ .execute_page(Some(&state2))
2345
+ .unwrap();
2346
+
2347
+ assert_eq!(
2348
+ third_page.len(),
2349
+ 40,
2350
+ "Third page should contain remaining 40 entries"
2351
+ );
2352
+
2353
+ assert_eq!(
2354
+ third_page.first().unwrap().timestamp,
2355
+ 310,
2356
+ "Third page should start at timestamp 310"
2357
+ );
2358
+
2359
+ assert_eq!(
2360
+ third_page.last().unwrap().timestamp,
2361
+ 349,
2362
+ "Third page should end at timestamp 349 (before boundary)"
2363
+ );
2364
+
2365
+ // Verify all timestamps are within boundaries
2366
+ for entry in &third_page {
2367
+ assert!(
2368
+ entry.timestamp >= 150 && entry.timestamp < 350,
2369
+ "Entry timestamp {} should be within [150, 350)",
2370
+ entry.timestamp
2371
+ );
2372
+ }
2373
+
2374
+ // Verify no duplicates and correct total count
2375
+ let mut all_entry_ids = HashSet::new();
2376
+ for entry in first_page
2377
+ .iter()
2378
+ .chain(second_page.iter())
2379
+ .chain(third_page.iter())
2380
+ {
2381
+ for field in &entry.fields {
2382
+ if field.field() == "ENTRY_ID" {
2383
+ assert!(
2384
+ all_entry_ids.insert(field.value().to_string()),
2385
+ "Found duplicate ENTRY_ID: {}",
2386
+ field.value()
2387
+ );
2388
+ }
2389
+ }
2390
+ }
2391
+
2392
+ // Should have exactly 200 entries (150-349)
2393
+ assert_eq!(
2394
+ all_entry_ids.len(),
2395
+ 200,
2396
+ "Should have retrieved exactly 200 entries within time boundaries"
2397
+ );
2398
+
2399
+ // Fourth page should be empty
2400
+ let (fourth_page, _state4) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2401
+ .with_after_usec(150)
2402
+ .with_before_usec(350)
2403
+ .with_limit(80)
2404
+ .execute_page(Some(&state3))
2405
+ .unwrap();
2406
+
2407
+ assert_eq!(
2408
+ fourth_page.len(),
2409
+ 0,
2410
+ "Fourth page should be empty (no more entries)"
2411
+ );
2412
+}
2413
+
2414
+#[test]
2415
+fn test_multi_file_pagination_backward_overlapping_timestamps() {
2416
+ // Create temporary directory
2417
+ let temp_dir = TempDir::new().unwrap();
2418
+
2419
+ // File 1: entries at t=100..200 (100 entries)
2420
+ let entries_file1: Vec<TestEntry> = (100..200)
2421
+ .map(|i| {
2422
+ TestEntry::new(Microseconds(i))
2423
+ .with_field("MESSAGE", format!("File1 Entry at {}", i))
2424
+ .with_field("ENTRY_ID", format!("file1_{}", i))
2425
+ .with_field("FILE", "1")
2426
+ })
2427
+ .collect();
2428
+
2429
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
2430
+
2431
+ // File 2: entries at t=150..250 (100 entries) - overlaps with file1 from 150-199
2432
+ let entries_file2: Vec<TestEntry> = (150..250)
2433
+ .map(|i| {
2434
+ TestEntry::new(Microseconds(i))
2435
+ .with_field("MESSAGE", format!("File2 Entry at {}", i))
2436
+ .with_field("ENTRY_ID", format!("file2_{}", i))
2437
+ .with_field("FILE", "2")
2438
+ })
2439
+ .collect();
2440
+
2441
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
2442
+
2443
+ // Index both files
2444
+ let mut indexer = FileIndexer::default();
2445
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
2446
+ let file_field = FieldName::new("FILE").unwrap();
2447
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
2448
+
2449
+ let index1 = indexer
2450
+ .index(
2451
+ &file1,
2452
+ Some(&source_timestamp_field),
2453
+ &[file_field.clone(), entry_id_field.clone()],
2454
+ Seconds(3600),
2455
+ )
2456
+ .unwrap();
2457
+
2458
+ let index2 = indexer
2459
+ .index(
2460
+ &file2,
2461
+ Some(&source_timestamp_field),
2462
+ &[file_field, entry_id_field],
2463
+ Seconds(3600),
2464
+ )
2465
+ .unwrap();
2466
+
2467
+ let file_indexes = vec![index1, index2];
2468
+
2469
+ // First page: limit=120, backward from tail
2470
+ // Expected: from timestamp 249 going backward
2471
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2472
+ .with_limit(120)
2473
+ .execute_page(None)
2474
+ .unwrap();
2475
+
2476
+ assert_eq!(
2477
+ first_page.len(),
2478
+ 120,
2479
+ "First page should contain exactly 120 entries"
2480
+ );
2481
+
2482
+ // Verify timestamps are in descending order
2483
+ for i in 1..first_page.len() {
2484
+ assert!(
2485
+ first_page[i - 1].timestamp >= first_page[i].timestamp,
2486
+ "Entries should be in descending timestamp order"
2487
+ );
2488
+ }
2489
+
2490
+ // First entry should be at timestamp 249 (highest)
2491
+ assert_eq!(
2492
+ first_page.first().unwrap().timestamp,
2493
+ 249,
2494
+ "First entry should be at timestamp 249"
2495
+ );
2496
+
2497
+ // State should track positions for both files
2498
+ assert!(
2499
+ !state1.file_positions.is_empty(),
2500
+ "State should track positions"
2501
+ );
2502
+
2503
+ // Second page: get remaining entries
2504
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2505
+ .with_limit(120)
2506
+ .execute_page(Some(&state1))
2507
+ .unwrap();
2508
+
2509
+ assert_eq!(
2510
+ second_page.len(),
2511
+ 80,
2512
+ "Second page should contain remaining 80 entries"
2513
+ );
2514
+
2515
+ // Verify timestamps continue in descending order
2516
+ for i in 1..second_page.len() {
2517
+ assert!(
2518
+ second_page[i - 1].timestamp >= second_page[i].timestamp,
2519
+ "Entries should be in descending timestamp order"
2520
+ );
2521
+ }
2522
+
2523
+ // Verify no timestamp gap between pages
2524
+ if !first_page.is_empty() && !second_page.is_empty() {
2525
+ assert!(
2526
+ first_page.last().unwrap().timestamp >= second_page.first().unwrap().timestamp,
2527
+ "Second page should continue from first page timestamp"
2528
+ );
2529
+ }
2530
+
2531
+ // Last entry should be at timestamp 100
2532
+ assert_eq!(
2533
+ second_page.last().unwrap().timestamp,
2534
+ 100,
2535
+ "Last entry should be at timestamp 100"
2536
+ );
2537
+
2538
+ // Collect all ENTRY_ID values to verify uniqueness and completeness
2539
+ let mut all_entry_ids = HashSet::new();
2540
+
2541
+ for entry in &first_page {
2542
+ for field in &entry.fields {
2543
+ if field.field() == "ENTRY_ID" {
2544
+ assert!(
2545
+ all_entry_ids.insert(field.value().to_string()),
2546
+ "Found duplicate ENTRY_ID: {}",
2547
+ field.value()
2548
+ );
2549
+ }
2550
+ }
2551
+ }
2552
+
2553
+ for entry in &second_page {
2554
+ for field in &entry.fields {
2555
+ if field.field() == "ENTRY_ID" {
2556
+ assert!(
2557
+ all_entry_ids.insert(field.value().to_string()),
2558
+ "Found duplicate ENTRY_ID: {}",
2559
+ field.value()
2560
+ );
2561
+ }
2562
+ }
2563
+ }
2564
+
2565
+ // Verify we got all 200 unique entries (100 from each file)
2566
+ assert_eq!(
2567
+ all_entry_ids.len(),
2568
+ 200,
2569
+ "Should have retrieved all 200 unique entries"
2570
+ );
2571
+
2572
+ // Verify we have entries from both files
2573
+ let file1_entries: usize = all_entry_ids
2574
+ .iter()
2575
+ .filter(|id| id.starts_with("file1_"))
2576
+ .count();
2577
+ let file2_entries: usize = all_entry_ids
2578
+ .iter()
2579
+ .filter(|id| id.starts_with("file2_"))
2580
+ .count();
2581
+
2582
+ assert_eq!(file1_entries, 100, "Should have 100 entries from file1");
2583
+ assert_eq!(file2_entries, 100, "Should have 100 entries from file2");
2584
+
2585
+ // Verify all timestamps from 100-249 are represented
2586
+ let mut all_timestamps = HashSet::new();
2587
+ for entry in first_page.iter().chain(second_page.iter()) {
2588
+ all_timestamps.insert(entry.timestamp);
2589
+ }
2590
+
2591
+ // We should have entries at all timestamps from 100-249 (150 unique timestamps)
2592
+ for ts in 100..250 {
2593
+ assert!(all_timestamps.contains(&ts), "Missing timestamp: {}", ts);
2594
+ }
2595
+
2596
+ // Third page should be empty
2597
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2598
+ .with_limit(120)
2599
+ .execute_page(Some(&state2))
2600
+ .unwrap();
2601
+
2602
+ assert_eq!(
2603
+ third_page.len(),
2604
+ 0,
2605
+ "Third page should be empty (no more entries)"
2606
+ );
2607
+}
2608
+
2609
+#[test]
2610
+fn test_multi_file_pagination_backward_three_files() {
2611
+ // Create temporary directory
2612
+ let temp_dir = TempDir::new().unwrap();
2613
+
2614
+ // File 1: entries at t=100..200 (100 entries)
2615
+ let entries_file1: Vec<TestEntry> = (100..200)
2616
+ .map(|i| {
2617
+ TestEntry::new(Microseconds(i))
2618
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
2619
+ .with_field("ENTRY_ID", format!("file1_{}", i))
2620
+ .with_field("FILE", "1")
2621
+ })
2622
+ .collect();
2623
+
2624
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
2625
+
2626
+ // File 2: entries at t=200..300 (100 entries)
2627
+ let entries_file2: Vec<TestEntry> = (200..300)
2628
+ .map(|i| {
2629
+ TestEntry::new(Microseconds(i))
2630
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
2631
+ .with_field("ENTRY_ID", format!("file2_{}", i))
2632
+ .with_field("FILE", "2")
2633
+ })
2634
+ .collect();
2635
+
2636
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
2637
+
2638
+ // File 3: entries at t=300..400 (100 entries)
2639
+ let entries_file3: Vec<TestEntry> = (300..400)
2640
+ .map(|i| {
2641
+ TestEntry::new(Microseconds(i))
2642
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
2643
+ .with_field("ENTRY_ID", format!("file3_{}", i))
2644
+ .with_field("FILE", "3")
2645
+ })
2646
+ .collect();
2647
+
2648
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
2649
+
2650
+ // Index all three files
2651
+ let mut indexer = FileIndexer::default();
2652
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
2653
+ let file_field = FieldName::new("FILE").unwrap();
2654
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
2655
+
2656
+ let index1 = indexer
2657
+ .index(
2658
+ &file1,
2659
+ Some(&source_timestamp_field),
2660
+ &[file_field.clone(), entry_id_field.clone()],
2661
+ Seconds(3600),
2662
+ )
2663
+ .unwrap();
2664
+
2665
+ let index2 = indexer
2666
+ .index(
2667
+ &file2,
2668
+ Some(&source_timestamp_field),
2669
+ &[file_field.clone(), entry_id_field.clone()],
2670
+ Seconds(3600),
2671
+ )
2672
+ .unwrap();
2673
+
2674
+ let index3 = indexer
2675
+ .index(
2676
+ &file3,
2677
+ Some(&source_timestamp_field),
2678
+ &[file_field, entry_id_field],
2679
+ Seconds(3600),
2680
+ )
2681
+ .unwrap();
2682
+
2683
+ let file_indexes = vec![index1, index2, index3];
2684
+
2685
+ // First page: limit=125, backward from tail, should get all 100 from file3 + 25 from file2
2686
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2687
+ .with_limit(125)
2688
+ .execute_page(None)
2689
+ .unwrap();
2690
+
2691
+ assert_eq!(
2692
+ first_page.len(),
2693
+ 125,
2694
+ "First page should contain exactly 125 entries"
2695
+ );
2696
+
2697
+ // Verify descending order
2698
+ for i in 1..first_page.len() {
2699
+ assert!(
2700
+ first_page[i - 1].timestamp >= first_page[i].timestamp,
2701
+ "Entries should be in descending order"
2702
+ );
2703
+ }
2704
+
2705
+ assert_eq!(first_page.first().unwrap().timestamp, 399);
2706
+ assert_eq!(first_page.last().unwrap().timestamp, 275);
2707
+
2708
+ // State should track positions for file3 and file2
2709
+ assert_eq!(
2710
+ state1.file_positions.len(),
2711
+ 2,
2712
+ "State should track positions for 2 files"
2713
+ );
2714
+
2715
+ // Second page: limit=125, should get 75 from file2 + 50 from file1
2716
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2717
+ .with_limit(125)
2718
+ .execute_page(Some(&state1))
2719
+ .unwrap();
2720
+
2721
+ assert_eq!(
2722
+ second_page.len(),
2723
+ 125,
2724
+ "Second page should contain exactly 125 entries"
2725
+ );
2726
+
2727
+ // Verify descending order
2728
+ for i in 1..second_page.len() {
2729
+ assert!(
2730
+ second_page[i - 1].timestamp >= second_page[i].timestamp,
2731
+ "Entries should be in descending order"
2732
+ );
2733
+ }
2734
+
2735
+ assert_eq!(second_page.first().unwrap().timestamp, 274);
2736
+ assert_eq!(second_page.last().unwrap().timestamp, 150);
2737
+
2738
+ // State should now track all 3 files
2739
+ assert_eq!(
2740
+ state2.file_positions.len(),
2741
+ 3,
2742
+ "State should track positions for all 3 files"
2743
+ );
2744
+
2745
+ // Third page: remaining 50 from file1
2746
+ let (third_page, state3) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2747
+ .with_limit(125)
2748
+ .execute_page(Some(&state2))
2749
+ .unwrap();
2750
+
2751
+ assert_eq!(
2752
+ third_page.len(),
2753
+ 50,
2754
+ "Third page should contain remaining 50 entries"
2755
+ );
2756
+
2757
+ // Verify descending order
2758
+ for i in 1..third_page.len() {
2759
+ assert!(
2760
+ third_page[i - 1].timestamp >= third_page[i].timestamp,
2761
+ "Entries should be in descending order"
2762
+ );
2763
+ }
2764
+
2765
+ assert_eq!(third_page.first().unwrap().timestamp, 149);
2766
+ assert_eq!(third_page.last().unwrap().timestamp, 100);
2767
+
2768
+ // Collect all ENTRY_ID values to verify uniqueness
2769
+ let mut all_entry_ids = HashSet::new();
2770
+
2771
+ for entry in first_page
2772
+ .iter()
2773
+ .chain(second_page.iter())
2774
+ .chain(third_page.iter())
2775
+ {
2776
+ for field in &entry.fields {
2777
+ if field.field() == "ENTRY_ID" {
2778
+ assert!(
2779
+ all_entry_ids.insert(field.value().to_string()),
2780
+ "Found duplicate ENTRY_ID: {}",
2781
+ field.value()
2782
+ );
2783
+ }
2784
+ }
2785
+ }
2786
+
2787
+ // Verify we got all 300 unique entries
2788
+ assert_eq!(
2789
+ all_entry_ids.len(),
2790
+ 300,
2791
+ "Should have retrieved all 300 unique entries"
2792
+ );
2793
+
2794
+ // Verify distribution: 100 from each file
2795
+ let file1_count = all_entry_ids
2796
+ .iter()
2797
+ .filter(|id| id.starts_with("file1_"))
2798
+ .count();
2799
+ let file2_count = all_entry_ids
2800
+ .iter()
2801
+ .filter(|id| id.starts_with("file2_"))
2802
+ .count();
2803
+ let file3_count = all_entry_ids
2804
+ .iter()
2805
+ .filter(|id| id.starts_with("file3_"))
2806
+ .count();
2807
+
2808
+ assert_eq!(file1_count, 100, "Should have 100 entries from file1");
2809
+ assert_eq!(file2_count, 100, "Should have 100 entries from file2");
2810
+ assert_eq!(file3_count, 100, "Should have 100 entries from file3");
2811
+
2812
+ // Fourth page should be empty
2813
+ let (fourth_page, _state4) = LogQuery::new(&file_indexes, Anchor::Tail, Direction::Backward)
2814
+ .with_limit(125)
2815
+ .execute_page(Some(&state3))
2816
+ .unwrap();
2817
+
2818
+ assert_eq!(
2819
+ fourth_page.len(),
2820
+ 0,
2821
+ "Fourth page should be empty (no more entries)"
2822
+ );
2823
+}
2824
+
2825
+#[test]
2826
+fn test_multi_file_pagination_with_filter() {
2827
+ // Create temporary directory
2828
+ let temp_dir = TempDir::new().unwrap();
2829
+
2830
+ // File 1: 50 entries at t=100..150 with LEVEL=ERROR
2831
+ // 50 entries at t=150..200 with LEVEL=INFO
2832
+ let mut entries_file1: Vec<TestEntry> = (100..150)
2833
+ .map(|i| {
2834
+ TestEntry::new(Microseconds(i))
2835
+ .with_field("MESSAGE", format!("File1 Error {}", i))
2836
+ .with_field("ENTRY_ID", format!("file1_error_{}", i))
2837
+ .with_field("LEVEL", "ERROR")
2838
+ })
2839
+ .collect();
2840
+
2841
+ entries_file1.extend((150..200).map(|i| {
2842
+ TestEntry::new(Microseconds(i))
2843
+ .with_field("MESSAGE", format!("File1 Info {}", i))
2844
+ .with_field("ENTRY_ID", format!("file1_info_{}", i))
2845
+ .with_field("LEVEL", "INFO")
2846
+ }));
2847
+
2848
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
2849
+
2850
+ // File 2: 50 entries at t=200..250 with LEVEL=ERROR
2851
+ // 50 entries at t=250..300 with LEVEL=INFO
2852
+ let mut entries_file2: Vec<TestEntry> = (200..250)
2853
+ .map(|i| {
2854
+ TestEntry::new(Microseconds(i))
2855
+ .with_field("MESSAGE", format!("File2 Error {}", i))
2856
+ .with_field("ENTRY_ID", format!("file2_error_{}", i))
2857
+ .with_field("LEVEL", "ERROR")
2858
+ })
2859
+ .collect();
2860
+
2861
+ entries_file2.extend((250..300).map(|i| {
2862
+ TestEntry::new(Microseconds(i))
2863
+ .with_field("MESSAGE", format!("File2 Info {}", i))
2864
+ .with_field("ENTRY_ID", format!("file2_info_{}", i))
2865
+ .with_field("LEVEL", "INFO")
2866
+ }));
2867
+
2868
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
2869
+
2870
+ // File 3: 50 entries at t=300..350 with LEVEL=ERROR
2871
+ // 50 entries at t=350..400 with LEVEL=INFO
2872
+ let mut entries_file3: Vec<TestEntry> = (300..350)
2873
+ .map(|i| {
2874
+ TestEntry::new(Microseconds(i))
2875
+ .with_field("MESSAGE", format!("File3 Error {}", i))
2876
+ .with_field("ENTRY_ID", format!("file3_error_{}", i))
2877
+ .with_field("LEVEL", "ERROR")
2878
+ })
2879
+ .collect();
2880
+
2881
+ entries_file3.extend((350..400).map(|i| {
2882
+ TestEntry::new(Microseconds(i))
2883
+ .with_field("MESSAGE", format!("File3 Info {}", i))
2884
+ .with_field("ENTRY_ID", format!("file3_info_{}", i))
2885
+ .with_field("LEVEL", "INFO")
2886
+ }));
2887
+
2888
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
2889
+
2890
+ // Index all three files
2891
+ let mut indexer = FileIndexer::default();
2892
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
2893
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
2894
+ let level_field = FieldName::new("LEVEL").unwrap();
2895
+
2896
+ let index1 = indexer
2897
+ .index(
2898
+ &file1,
2899
+ Some(&source_timestamp_field),
2900
+ &[entry_id_field.clone(), level_field.clone()],
2901
+ Seconds(3600),
2902
+ )
2903
+ .unwrap();
2904
+
2905
+ let index2 = indexer
2906
+ .index(
2907
+ &file2,
2908
+ Some(&source_timestamp_field),
2909
+ &[entry_id_field.clone(), level_field.clone()],
2910
+ Seconds(3600),
2911
+ )
2912
+ .unwrap();
2913
+
2914
+ let index3 = indexer
2915
+ .index(
2916
+ &file3,
2917
+ Some(&source_timestamp_field),
2918
+ &[entry_id_field, level_field],
2919
+ Seconds(3600),
2920
+ )
2921
+ .unwrap();
2922
+
2923
+ let file_indexes = vec![index1, index2, index3];
2924
+
2925
+ // Create filter to match only LEVEL=ERROR entries
2926
+ // This should match 50 entries per file (150 total)
2927
+ let filter = Filter::match_field_value_pair(FieldValuePair::parse("LEVEL=ERROR").unwrap());
2928
+
2929
+ // First page: limit=80, should get 50 from file1 + 30 from file2
2930
+ let (first_page, state1) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2931
+ .with_filter(filter.clone())
2932
+ .with_limit(80)
2933
+ .execute_page(None)
2934
+ .unwrap();
2935
+
2936
+ assert_eq!(
2937
+ first_page.len(),
2938
+ 80,
2939
+ "First page should contain 80 ERROR entries"
2940
+ );
2941
+
2942
+ // All entries should have LEVEL=ERROR
2943
+ for entry in &first_page {
2944
+ let level_values: Vec<_> = entry
2945
+ .fields
2946
+ .iter()
2947
+ .filter(|f| f.field() == "LEVEL")
2948
+ .map(|f| f.value())
2949
+ .collect();
2950
+ assert_eq!(
2951
+ level_values,
2952
+ vec!["ERROR"],
2953
+ "All entries should have LEVEL=ERROR"
2954
+ );
2955
+ }
2956
+
2957
+ // First entry should be at timestamp 100
2958
+ assert_eq!(
2959
+ first_page.first().unwrap().timestamp,
2960
+ 100,
2961
+ "First entry should be at timestamp 100"
2962
+ );
2963
+
2964
+ // Last entry should be at timestamp 229
2965
+ assert_eq!(
2966
+ first_page.last().unwrap().timestamp,
2967
+ 229,
2968
+ "Last entry should be at timestamp 229"
2969
+ );
2970
+
2971
+ // Second page: get remaining ERROR entries
2972
+ let (second_page, state2) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
2973
+ .with_filter(filter.clone())
2974
+ .with_limit(80)
2975
+ .execute_page(Some(&state1))
2976
+ .unwrap();
2977
+
2978
+ assert_eq!(
2979
+ second_page.len(),
2980
+ 70,
2981
+ "Second page should contain remaining 70 ERROR entries"
2982
+ );
2983
+
2984
+ // All entries should have LEVEL=ERROR
2985
+ for entry in &second_page {
2986
+ let level_values: Vec<_> = entry
2987
+ .fields
2988
+ .iter()
2989
+ .filter(|f| f.field() == "LEVEL")
2990
+ .map(|f| f.value())
2991
+ .collect();
2992
+ assert_eq!(
2993
+ level_values,
2994
+ vec!["ERROR"],
2995
+ "All entries should have LEVEL=ERROR"
2996
+ );
2997
+ }
2998
+
2999
+ // Should continue from timestamp 230 and go to 349
3000
+ assert_eq!(
3001
+ second_page.first().unwrap().timestamp,
3002
+ 230,
3003
+ "Second page should start at timestamp 230"
3004
+ );
3005
+
3006
+ assert_eq!(
3007
+ second_page.last().unwrap().timestamp,
3008
+ 349,
3009
+ "Second page should end at timestamp 349"
3010
+ );
3011
+
3012
+ // Verify no duplicates
3013
+ let mut all_entry_ids = HashSet::new();
3014
+ for entry in first_page.iter().chain(second_page.iter()) {
3015
+ for field in &entry.fields {
3016
+ if field.field() == "ENTRY_ID" {
3017
+ assert!(
3018
+ all_entry_ids.insert(field.value().to_string()),
3019
+ "Found duplicate ENTRY_ID: {}",
3020
+ field.value()
3021
+ );
3022
+ }
3023
+ }
3024
+ }
3025
+
3026
+ // Should have exactly 150 ERROR entries (50 from each file)
3027
+ assert_eq!(
3028
+ all_entry_ids.len(),
3029
+ 150,
3030
+ "Should have retrieved exactly 150 ERROR entries"
3031
+ );
3032
+
3033
+ // Verify all are error entries
3034
+ let error_count = all_entry_ids
3035
+ .iter()
3036
+ .filter(|id| id.contains("_error_"))
3037
+ .count();
3038
+ assert_eq!(error_count, 150, "All entries should be error entries");
3039
+
3040
+ // Verify distribution across files
3041
+ let file1_count = all_entry_ids
3042
+ .iter()
3043
+ .filter(|id| id.starts_with("file1_"))
3044
+ .count();
3045
+ let file2_count = all_entry_ids
3046
+ .iter()
3047
+ .filter(|id| id.starts_with("file2_"))
3048
+ .count();
3049
+ let file3_count = all_entry_ids
3050
+ .iter()
3051
+ .filter(|id| id.starts_with("file3_"))
3052
+ .count();
3053
+
3054
+ assert_eq!(file1_count, 50, "Should have 50 ERROR entries from file1");
3055
+ assert_eq!(file2_count, 50, "Should have 50 ERROR entries from file2");
3056
+ assert_eq!(file3_count, 50, "Should have 50 ERROR entries from file3");
3057
+
3058
+ // Third page should be empty
3059
+ let (third_page, _state3) = LogQuery::new(&file_indexes, Anchor::Head, Direction::Forward)
3060
+ .with_filter(filter)
3061
+ .with_limit(80)
3062
+ .execute_page(Some(&state2))
3063
+ .unwrap();
3064
+
3065
+ assert_eq!(
3066
+ third_page.len(),
3067
+ 0,
3068
+ "Third page should be empty (no more ERROR entries)"
3069
+ );
3070
+}
3071
+
3072
+#[test]
3073
+fn test_multi_file_pagination_anchor_at_file_boundary() {
3074
+ // Create temporary directory
3075
+ let temp_dir = TempDir::new().unwrap();
3076
+
3077
+ // File 1: entries at t=100..200 (100 entries)
3078
+ let entries_file1: Vec<TestEntry> = (100..200)
3079
+ .map(|i| {
3080
+ TestEntry::new(Microseconds(i))
3081
+ .with_field("MESSAGE", format!("File1 Entry {}", i))
3082
+ .with_field("ENTRY_ID", format!("file1_{}", i))
3083
+ })
3084
+ .collect();
3085
+
3086
+ let file1 = create_test_journal(&temp_dir, "file1.journal", entries_file1).unwrap();
3087
+
3088
+ // File 2: entries at t=200..300 (100 entries) - starts exactly where file1 ends
3089
+ let entries_file2: Vec<TestEntry> = (200..300)
3090
+ .map(|i| {
3091
+ TestEntry::new(Microseconds(i))
3092
+ .with_field("MESSAGE", format!("File2 Entry {}", i))
3093
+ .with_field("ENTRY_ID", format!("file2_{}", i))
3094
+ })
3095
+ .collect();
3096
+
3097
+ let file2 = create_test_journal(&temp_dir, "file2.journal", entries_file2).unwrap();
3098
+
3099
+ // File 3: entries at t=300..400 (100 entries) - starts exactly where file2 ends
3100
+ let entries_file3: Vec<TestEntry> = (300..400)
3101
+ .map(|i| {
3102
+ TestEntry::new(Microseconds(i))
3103
+ .with_field("MESSAGE", format!("File3 Entry {}", i))
3104
+ .with_field("ENTRY_ID", format!("file3_{}", i))
3105
+ })
3106
+ .collect();
3107
+
3108
+ let file3 = create_test_journal(&temp_dir, "file3.journal", entries_file3).unwrap();
3109
+
3110
+ // Index all three files
3111
+ let mut indexer = FileIndexer::default();
3112
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
3113
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
3114
+
3115
+ let index1 = indexer
3116
+ .index(
3117
+ &file1,
3118
+ Some(&source_timestamp_field),
3119
+ &[entry_id_field.clone()],
3120
+ Seconds(3600),
3121
+ )
3122
+ .unwrap();
3123
+
3124
+ let index2 = indexer
3125
+ .index(
3126
+ &file2,
3127
+ Some(&source_timestamp_field),
3128
+ &[entry_id_field.clone()],
3129
+ Seconds(3600),
3130
+ )
3131
+ .unwrap();
3132
+
3133
+ let index3 = indexer
3134
+ .index(
3135
+ &file3,
3136
+ Some(&source_timestamp_field),
3137
+ &[entry_id_field],
3138
+ Seconds(3600),
3139
+ )
3140
+ .unwrap();
3141
+
3142
+ let file_indexes = vec![index1, index2, index3];
3143
+
3144
+ // Test forward from exact boundary timestamp 200 (where file1 ends and file2 starts)
3145
+ let anchor = Anchor::Timestamp(Microseconds(200));
3146
+
3147
+ let (first_page_fwd, state1_fwd) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
3148
+ .with_limit(80)
3149
+ .execute_page(None)
3150
+ .unwrap();
3151
+
3152
+ assert_eq!(
3153
+ first_page_fwd.len(),
3154
+ 80,
3155
+ "Forward from boundary should return 80 entries"
3156
+ );
3157
+
3158
+ // Should start at timestamp 200 (first entry of file2)
3159
+ assert_eq!(
3160
+ first_page_fwd.first().unwrap().timestamp,
3161
+ 200,
3162
+ "Forward should start at timestamp 200"
3163
+ );
3164
+
3165
+ // Should end at timestamp 279
3166
+ assert_eq!(
3167
+ first_page_fwd.last().unwrap().timestamp,
3168
+ 279,
3169
+ "Forward should end at timestamp 279"
3170
+ );
3171
+
3172
+ // Continue forward pagination
3173
+ let (second_page_fwd, _state2_fwd) = LogQuery::new(&file_indexes, anchor, Direction::Forward)
3174
+ .with_limit(80)
3175
+ .execute_page(Some(&state1_fwd))
3176
+ .unwrap();
3177
+
3178
+ assert_eq!(
3179
+ second_page_fwd.len(),
3180
+ 80,
3181
+ "Second forward page should return 80 entries"
3182
+ );
3183
+
3184
+ assert_eq!(
3185
+ second_page_fwd.first().unwrap().timestamp,
3186
+ 280,
3187
+ "Second page should start at 280"
3188
+ );
3189
+
3190
+ assert_eq!(
3191
+ second_page_fwd.last().unwrap().timestamp,
3192
+ 359,
3193
+ "Second page should end at 359"
3194
+ );
3195
+
3196
+ // Test backward from exact boundary timestamp 200
3197
+ let (first_page_bwd, state1_bwd) = LogQuery::new(&file_indexes, anchor, Direction::Backward)
3198
+ .with_limit(80)
3199
+ .execute_page(None)
3200
+ .unwrap();
3201
+
3202
+ assert_eq!(
3203
+ first_page_bwd.len(),
3204
+ 80,
3205
+ "Backward from boundary should return 80 entries"
3206
+ );
3207
+
3208
+ // Should start at timestamp 200 (inclusive for backward)
3209
+ assert_eq!(
3210
+ first_page_bwd.first().unwrap().timestamp,
3211
+ 200,
3212
+ "Backward should start at timestamp 200 (inclusive)"
3213
+ );
3214
+
3215
+ // Should end at timestamp 121
3216
+ assert_eq!(
3217
+ first_page_bwd.last().unwrap().timestamp,
3218
+ 121,
3219
+ "Backward should end at timestamp 121"
3220
+ );
3221
+
3222
+ // Continue backward pagination
3223
+ let (second_page_bwd, _state2_bwd) = LogQuery::new(&file_indexes, anchor, Direction::Backward)
3224
+ .with_limit(80)
3225
+ .execute_page(Some(&state1_bwd))
3226
+ .unwrap();
3227
+
3228
+ assert_eq!(
3229
+ second_page_bwd.len(),
3230
+ 21,
3231
+ "Second backward page should return remaining 21 entries"
3232
+ );
3233
+
3234
+ assert_eq!(
3235
+ second_page_bwd.first().unwrap().timestamp,
3236
+ 120,
3237
+ "Second backward page should start at 120"
3238
+ );
3239
+
3240
+ assert_eq!(
3241
+ second_page_bwd.last().unwrap().timestamp,
3242
+ 100,
3243
+ "Second backward page should end at 100"
3244
+ );
3245
+
3246
+ // Test anchor at boundary 300 (between file2 and file3)
3247
+ let anchor_300 = Anchor::Timestamp(Microseconds(300));
3248
+
3249
+ let (page_fwd_300, _) = LogQuery::new(&file_indexes, anchor_300, Direction::Forward)
3250
+ .with_limit(50)
3251
+ .execute_page(None)
3252
+ .unwrap();
3253
+
3254
+ assert_eq!(
3255
+ page_fwd_300.len(),
3256
+ 50,
3257
+ "Forward from 300 should return 50 entries"
3258
+ );
3259
+
3260
+ assert_eq!(
3261
+ page_fwd_300.first().unwrap().timestamp,
3262
+ 300,
3263
+ "Should start at 300"
3264
+ );
3265
+
3266
+ assert_eq!(
3267
+ page_fwd_300.last().unwrap().timestamp,
3268
+ 349,
3269
+ "Should end at 349"
3270
+ );
3271
+
3272
+ let (page_bwd_300, _) = LogQuery::new(&file_indexes, anchor_300, Direction::Backward)
3273
+ .with_limit(50)
3274
+ .execute_page(None)
3275
+ .unwrap();
3276
+
3277
+ assert_eq!(
3278
+ page_bwd_300.len(),
3279
+ 50,
3280
+ "Backward from 300 should return 50 entries"
3281
+ );
3282
+
3283
+ assert_eq!(
3284
+ page_bwd_300.first().unwrap().timestamp,
3285
+ 300,
3286
+ "Should start at 300"
3287
+ );
3288
+
3289
+ assert_eq!(
3290
+ page_bwd_300.last().unwrap().timestamp,
3291
+ 251,
3292
+ "Should end at 251"
3293
+ );
3294
+
3295
+ // Verify no duplicates within each query direction from anchor 200
3296
+ let mut fwd_200_ids = HashSet::new();
3297
+ for entry in first_page_fwd.iter().chain(second_page_fwd.iter()) {
3298
+ for field in &entry.fields {
3299
+ if field.field() == "ENTRY_ID" {
3300
+ assert!(
3301
+ fwd_200_ids.insert(field.value().to_string()),
3302
+ "Found duplicate in forward from 200: {}",
3303
+ field.value()
3304
+ );
3305
+ }
3306
+ }
3307
+ }
3308
+
3309
+ let mut bwd_200_ids = HashSet::new();
3310
+ for entry in first_page_bwd.iter().chain(second_page_bwd.iter()) {
3311
+ for field in &entry.fields {
3312
+ if field.field() == "ENTRY_ID" {
3313
+ assert!(
3314
+ bwd_200_ids.insert(field.value().to_string()),
3315
+ "Found duplicate in backward from 200: {}",
3316
+ field.value()
3317
+ );
3318
+ }
3319
+ }
3320
+ }
3321
+
3322
+ // Forward from 200: two pages of 80 = 160 entries (200-359)
3323
+ assert_eq!(
3324
+ fwd_200_ids.len(),
3325
+ 160,
3326
+ "Forward from boundary 200 should return 160 unique entries (2 pages of 80)"
3327
+ );
3328
+
3329
+ // Backward from 200: 80 + 21 = 101 entries (100-200, inclusive)
3330
+ assert_eq!(
3331
+ bwd_200_ids.len(),
3332
+ 101,
3333
+ "Backward from boundary 200 should return 101 unique entries"
3334
+ );
3335
+
3336
+ // The boundary entry (200) should appear in both forward and backward results
3337
+ assert!(
3338
+ fwd_200_ids.contains("file2_200"),
3339
+ "Forward should include boundary entry 200"
3340
+ );
3341
+ assert!(
3342
+ bwd_200_ids.contains("file2_200"),
3343
+ "Backward should include boundary entry 200"
3344
+ );
3345
+}
src/crates/journal-index/Cargo.toml
new
+33
@@ -0,0 +1,33 @@
1
+[package]
2
+name = "journal-index"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[features]
11
+allocative = [
12
+ "dep:allocative",
13
+ "roaring/allocative",
14
+ "journal-core/allocative",
15
+ "journal-common/allocative"
16
+]
17
+
18
+[dependencies]
19
+allocative = { workspace = true, optional = true }
20
+regex = { workspace = true }
21
+roaring = { workspace = true , features = ["serde"] }
22
+serde = { workspace = true, features = ["derive"] }
23
+static_assertions = { workspace = true }
24
+thiserror = { workspace = true }
25
+tracing = { workspace = true }
26
+
27
+journal-core = { path = "../journal-core" }
28
+journal-common = { workspace = true }
29
+journal-registry = { workspace = true }
30
+
31
+[dev-dependencies]
32
+tempfile = { workspace = true }
33
+uuid = { workspace = true }
src/crates/journal-index/src/bitmap.rs
new
+161
@@ -0,0 +1,161 @@
1
+//! Compressed bitmap for efficient set operations on entry indices.
2
+
3
+use roaring::RoaringBitmap;
4
+use serde::{Deserialize, Serialize};
5
+
6
+/// A compressed bitmap representing a set of journal entry indices.
7
+///
8
+/// Wraps [`RoaringBitmap`] and supports bitwise AND/OR operations for combining filters.
9
+#[derive(Debug, Clone, Default, Serialize, Deserialize)]
10
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
11
+#[serde(transparent)]
12
+pub struct Bitmap(pub RoaringBitmap);
13
+
14
+impl Bitmap {
15
+ /// Create an empty bitmap.
16
+ pub fn new() -> Self {
17
+ Self(RoaringBitmap::new())
18
+ }
19
+
20
+ /// Create a bitmap from a sorted iterator of entry indices.
21
+ pub fn from_sorted_iter<I: IntoIterator<Item = u32>>(
22
+ iterator: I,
23
+ ) -> Result<Bitmap, roaring::NonSortedIntegers> {
24
+ RoaringBitmap::from_sorted_iter(iterator).map(Bitmap)
25
+ }
26
+
27
+ /// Create a bitmap containing all integers in the given range.
28
+ pub fn insert_range<R>(range: R) -> Self
29
+ where
30
+ R: std::ops::RangeBounds<u32>,
31
+ {
32
+ let mut bitmap = Self::new();
33
+ RoaringBitmap::insert_range(&mut bitmap, range);
34
+ bitmap
35
+ }
36
+}
37
+
38
+impl std::ops::Deref for Bitmap {
39
+ type Target = RoaringBitmap;
40
+
41
+ fn deref(&self) -> &Self::Target {
42
+ &self.0
43
+ }
44
+}
45
+
46
+impl std::ops::DerefMut for Bitmap {
47
+ fn deref_mut(&mut self) -> &mut Self::Target {
48
+ &mut self.0
49
+ }
50
+}
51
+
52
+impl From<RoaringBitmap> for Bitmap {
53
+ fn from(bitmap: RoaringBitmap) -> Self {
54
+ Self(bitmap)
55
+ }
56
+}
57
+
58
+impl From<Bitmap> for RoaringBitmap {
59
+ fn from(wrapper: Bitmap) -> Self {
60
+ wrapper.0
61
+ }
62
+}
63
+
64
+impl std::ops::BitAndAssign<&Bitmap> for Bitmap {
65
+ fn bitand_assign(&mut self, rhs: &Bitmap) {
66
+ self.0 &= &rhs.0;
67
+ }
68
+}
69
+
70
+impl std::ops::BitAndAssign<Bitmap> for Bitmap {
71
+ fn bitand_assign(&mut self, rhs: Bitmap) {
72
+ self.0 &= rhs.0;
73
+ }
74
+}
75
+
76
+impl std::ops::BitOrAssign<&Bitmap> for Bitmap {
77
+ fn bitor_assign(&mut self, rhs: &Bitmap) {
78
+ self.0 |= &rhs.0;
79
+ }
80
+}
81
+
82
+impl std::ops::BitOrAssign<Bitmap> for Bitmap {
83
+ fn bitor_assign(&mut self, rhs: Bitmap) {
84
+ self.0 |= rhs.0;
85
+ }
86
+}
87
+
88
+impl std::ops::BitAnd for &Bitmap {
89
+ type Output = Bitmap;
90
+
91
+ fn bitand(self, rhs: &Bitmap) -> Bitmap {
92
+ Bitmap(&self.0 & &rhs.0)
93
+ }
94
+}
95
+
96
+impl std::ops::BitAnd<Bitmap> for &Bitmap {
97
+ type Output = Bitmap;
98
+
99
+ fn bitand(self, rhs: Bitmap) -> Bitmap {
100
+ Bitmap(&self.0 & rhs.0)
101
+ }
102
+}
103
+
104
+impl std::ops::BitAnd<&Bitmap> for Bitmap {
105
+ type Output = Bitmap;
106
+
107
+ fn bitand(self, rhs: &Bitmap) -> Bitmap {
108
+ Bitmap(self.0 & &rhs.0)
109
+ }
110
+}
111
+
112
+impl std::ops::BitAnd for Bitmap {
113
+ type Output = Bitmap;
114
+
115
+ fn bitand(self, rhs: Bitmap) -> Bitmap {
116
+ Bitmap(self.0 & rhs.0)
117
+ }
118
+}
119
+
120
+#[cfg(test)]
121
+mod tests {
122
+ use super::*;
123
+
124
+ #[test]
125
+ fn test_from_sorted_iter() {
126
+ let bitmap = Bitmap::from_sorted_iter([0, 5, 10, 15]).expect("sorted iterator");
127
+
128
+ assert_eq!(bitmap.len(), 4);
129
+ assert!(bitmap.contains(5));
130
+ assert!(!bitmap.contains(6));
131
+ }
132
+
133
+ #[test]
134
+ fn test_from_sorted_iter_rejects_unsorted() {
135
+ let result = Bitmap::from_sorted_iter([10, 5, 15]);
136
+ assert!(result.is_err());
137
+ }
138
+
139
+ #[test]
140
+ fn test_insert_range() {
141
+ let bitmap = Bitmap::insert_range(10..15);
142
+
143
+ assert_eq!(bitmap.len(), 5);
144
+ assert!(bitmap.contains(10));
145
+ assert!(bitmap.contains(14));
146
+ assert!(!bitmap.contains(15));
147
+ }
148
+
149
+ #[test]
150
+ fn test_bitwise_operations() {
151
+ let bitmap1 = Bitmap::from_sorted_iter([1, 2, 3]).expect("sorted");
152
+ let bitmap2 = Bitmap::from_sorted_iter([2, 3, 4]).expect("sorted");
153
+
154
+ let intersection = &bitmap1 & &bitmap2;
155
+ assert_eq!(intersection.len(), 2);
156
+
157
+ let mut union = bitmap1.clone();
158
+ union |= bitmap2;
159
+ assert_eq!(union.len(), 4);
160
+ }
161
+}
src/crates/journal-index/src/error.rs
new
+49
@@ -0,0 +1,49 @@
1
+use thiserror::Error;
2
+
3
+/// Errors that can occur during journal indexing operations.
4
+#[derive(Error, Debug)]
5
+pub enum IndexError {
6
+ /// Bucket duration cannot be zero
7
+ #[error("bucket duration must not be zero")]
8
+ ZeroBucketDuration,
9
+
10
+ /// Cannot create a histogram from empty input
11
+ #[error("cannot create histogram from empty input")]
12
+ EmptyHistogramInput,
13
+
14
+ /// Invalid query time range
15
+ #[error("invalid query time range")]
16
+ InvalidQueryTimeRange,
17
+
18
+ /// Invalid regex pattern
19
+ #[error("invalid regex pattern:")]
20
+ InvalidRegex,
21
+
22
+ /// Data payload does not contain this field prefix
23
+ #[error("invalid field prefix")]
24
+ InvalidFieldPrefix,
25
+
26
+ /// Field value not utf8
27
+ #[error("non-utf8 payload")]
28
+ NonUtf8Payload,
29
+
30
+ /// Field value can not be parsed as integer
31
+ #[error("non-integer payload")]
32
+ NonIntegerPayload,
33
+
34
+ /// Log entry does not have this field
35
+ #[error("missing field name")]
36
+ MissingFieldName,
37
+
38
+ /// Missing required offset in journal file
39
+ #[error("missing required offset in journal file")]
40
+ MissingOffset,
41
+
42
+ /// Underlying journal file error
43
+ #[error("journal error: {0}")]
44
+ Journal(#[from] journal_core::error::JournalError),
45
+}
46
+
47
+static_assertions::const_assert!(std::mem::size_of::<IndexError>() <= 32);
48
+
49
+pub type Result<T> = std::result::Result<T, IndexError>;
src/crates/journal-index/src/field_types.rs
new
+366
@@ -0,0 +1,366 @@
1
+//! Type-safe wrappers for field names and field=value pairs.
2
+//!
3
+//! This module provides newtypes that distinguish between:
4
+//! - Field names (e.g., "PRIORITY")
5
+//! - Field=value pairs (e.g., "PRIORITY=error")
6
+//!
7
+//! These types are used throughout the journal indexing system to ensure
8
+//! type safety and prevent mixing different concepts.
9
+
10
+use serde::{Deserialize, Serialize};
11
+use std::fmt;
12
+
13
+/// A field name (e.g., "PRIORITY", "SYSLOG_IDENTIFIER").
14
+///
15
+/// This represents just the field name without any associated value.
16
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Ord, PartialOrd)]
17
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
18
+pub struct FieldName(String);
19
+
20
+impl FieldName {
21
+ /// Create a new FieldName without validation.
22
+ ///
23
+ /// Use this when you know the string is a valid field name
24
+ /// (e.g., from trusted sources like hardcoded constants).
25
+ pub fn new_unchecked(name: impl Into<String>) -> Self {
26
+ Self(name.into())
27
+ }
28
+
29
+ /// Create a FieldName with validation.
30
+ ///
31
+ /// Returns None if the name contains '=' or is empty.
32
+ pub fn new(name: impl Into<String>) -> Option<Self> {
33
+ let name = name.into();
34
+ if name.is_empty() || name.contains('=') {
35
+ None
36
+ } else {
37
+ Some(Self(name))
38
+ }
39
+ }
40
+
41
+ /// Get the field name as a string slice.
42
+ pub fn as_str(&self) -> &str {
43
+ &self.0
44
+ }
45
+
46
+ /// Get the field name as a byte slice.
47
+ pub fn as_bytes(&self) -> &[u8] {
48
+ self.0.as_bytes()
49
+ }
50
+
51
+ /// Convert into the inner String.
52
+ pub fn into_inner(self) -> String {
53
+ self.0
54
+ }
55
+
56
+ /// Combine this field name with a value to create a FieldValuePair.
57
+ pub fn with_value(&self, value: impl AsRef<str>) -> FieldValuePair {
58
+ FieldValuePair::new_unchecked(self.clone(), value.as_ref().to_string())
59
+ }
60
+}
61
+
62
+impl fmt::Display for FieldName {
63
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
64
+ write!(f, "{}", self.0)
65
+ }
66
+}
67
+
68
+impl AsRef<str> for FieldName {
69
+ fn as_ref(&self) -> &str {
70
+ &self.0
71
+ }
72
+}
73
+
74
+/// A field=value pair (e.g., "PRIORITY=error", "SYSLOG_IDENTIFIER=systemd").
75
+///
76
+/// Invariant: Always in the format "field=value". The value portion may contain '=' characters.
77
+/// The split is always at the first '=' character.
78
+#[derive(Debug, Clone, Eq, PartialEq, Hash, Serialize, Deserialize, Ord, PartialOrd)]
79
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
80
+pub struct FieldValuePair {
81
+ // Store the formatted string for efficient HashMap lookups
82
+ key: String,
83
+ // Cache the split position for fast field/value extraction
84
+ split_pos: usize,
85
+}
86
+
87
+impl FieldValuePair {
88
+ /// Create a new FieldValuePair from field and value components.
89
+ ///
90
+ /// This is unchecked - assumes field doesn't contain '='.
91
+ pub fn new_unchecked(field: FieldName, value: String) -> Self {
92
+ let split_pos = field.as_str().len();
93
+ let key = format!("{}={}", field.as_str(), value);
94
+ Self { key, split_pos }
95
+ }
96
+
97
+ /// Parse a "field=value" string into a FieldValuePair.
98
+ ///
99
+ /// Returns None if the string doesn't contain '=' or if the field name is empty.
100
+ /// The value portion may contain '=' characters - parsing splits on the first '=' only.
101
+ pub fn parse(s: impl AsRef<str>) -> Option<Self> {
102
+ let s = s.as_ref();
103
+ let split_pos = s.find('=')?;
104
+
105
+ if split_pos == 0 {
106
+ // Empty field name
107
+ return None;
108
+ }
109
+
110
+ Some(Self {
111
+ key: s.to_string(),
112
+ split_pos,
113
+ })
114
+ }
115
+
116
+ /// Get the field name portion.
117
+ pub fn field(&self) -> &str {
118
+ &self.key[..self.split_pos]
119
+ }
120
+
121
+ /// Get the value portion.
122
+ pub fn value(&self) -> &str {
123
+ &self.key[self.split_pos + 1..]
124
+ }
125
+
126
+ /// Get the full "field=value" string.
127
+ pub fn as_str(&self) -> &str {
128
+ &self.key
129
+ }
130
+
131
+ /// Get the full "field=value" as a byte slice.
132
+ pub fn as_bytes(&self) -> &[u8] {
133
+ self.key.as_bytes()
134
+ }
135
+
136
+ /// Convert into the inner String.
137
+ pub fn into_inner(self) -> String {
138
+ self.key
139
+ }
140
+
141
+ /// Extract the field name as a FieldName.
142
+ pub fn extract_field(&self) -> FieldName {
143
+ FieldName::new_unchecked(self.field())
144
+ }
145
+
146
+ /// Decompose into (field_name, value).
147
+ pub fn decompose(self) -> (FieldName, String) {
148
+ let field = FieldName::new_unchecked(self.field());
149
+ let value = self.value().to_string();
150
+ (field, value)
151
+ }
152
+
153
+ /// Extract the value portion from a field=value byte slice.
154
+ ///
155
+ /// Returns the value bytes if the payload is in the format "field_name=value",
156
+ /// where `field_name` matches the provided bytes. This is a zero-copy operation.
157
+ ///
158
+ /// # Examples
159
+ ///
160
+ /// ```
161
+ /// # use journal_index::FieldValuePair;
162
+ /// let payload = b"PRIORITY=6";
163
+ /// let field_name = b"PRIORITY";
164
+ /// let value = FieldValuePair::strip_field_prefix(field_name, payload);
165
+ /// assert_eq!(value, Some(&b"6"[..]));
166
+ ///
167
+ /// // Wrong field name
168
+ /// assert_eq!(FieldValuePair::strip_field_prefix(b"MESSAGE", payload), None);
169
+ ///
170
+ /// // Missing '='
171
+ /// assert_eq!(FieldValuePair::strip_field_prefix(b"PRIORITY", b"PRIORITY6"), None);
172
+ /// ```
173
+ pub fn strip_field_prefix<'a>(field_name: &[u8], payload: &'a [u8]) -> Option<&'a [u8]> {
174
+ // Check that payload starts with field_name
175
+ if !payload.starts_with(field_name) {
176
+ return None;
177
+ }
178
+
179
+ let offset = field_name.len();
180
+
181
+ // Check that there's an '=' after the field name
182
+ if payload.len() <= offset || payload[offset] != b'=' {
183
+ return None;
184
+ }
185
+
186
+ // Return the value portion after '='
187
+ Some(&payload[offset + 1..])
188
+ }
189
+}
190
+
191
+impl fmt::Display for FieldValuePair {
192
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
193
+ write!(f, "{}", self.key)
194
+ }
195
+}
196
+
197
+impl AsRef<str> for FieldValuePair {
198
+ fn as_ref(&self) -> &str {
199
+ &self.key
200
+ }
201
+}
202
+
203
+// Conversion helpers for backward compatibility
204
+impl From<FieldValuePair> for String {
205
+ fn from(pair: FieldValuePair) -> String {
206
+ pair.into_inner()
207
+ }
208
+}
209
+
210
+impl From<&FieldValuePair> for String {
211
+ fn from(pair: &FieldValuePair) -> String {
212
+ pair.to_string()
213
+ }
214
+}
215
+
216
+impl From<FieldName> for String {
217
+ fn from(name: FieldName) -> String {
218
+ name.into_inner()
219
+ }
220
+}
221
+
222
+impl From<&FieldName> for String {
223
+ fn from(name: &FieldName) -> String {
224
+ name.to_string()
225
+ }
226
+}
227
+
228
+/// Parse a u64 timestamp from a field data object.
229
+pub fn parse_timestamp(
230
+ field_name: &[u8],
231
+ data_object: &journal_core::file::DataObject<&[u8]>,
232
+) -> crate::Result<u64> {
233
+ let payload = data_object.payload_bytes();
234
+
235
+ let value_bytes = FieldValuePair::strip_field_prefix(field_name, payload)
236
+ .ok_or_else(|| crate::IndexError::InvalidFieldPrefix)?;
237
+
238
+ let timestamp_str =
239
+ std::str::from_utf8(value_bytes).map_err(|_| crate::IndexError::NonUtf8Payload)?;
240
+
241
+ timestamp_str
242
+ .parse::<u64>()
243
+ .map_err(|_| crate::IndexError::NonIntegerPayload)
244
+}
245
+
246
+#[cfg(test)]
247
+mod tests {
248
+ use super::*;
249
+
250
+ #[test]
251
+ fn test_field_name_creation() {
252
+ assert!(FieldName::new("PRIORITY").is_some());
253
+ assert!(FieldName::new("SYSLOG_IDENTIFIER").is_some());
254
+ assert!(FieldName::new("").is_none());
255
+ assert!(FieldName::new("PRIORITY=error").is_none());
256
+ }
257
+
258
+ #[test]
259
+ fn test_field_name_as_bytes() {
260
+ let field = FieldName::new("PRIORITY").unwrap();
261
+ assert_eq!(field.as_bytes(), b"PRIORITY");
262
+ assert_eq!(field.as_str(), "PRIORITY");
263
+ }
264
+
265
+ #[test]
266
+ fn test_field_value_pair_parsing() {
267
+ let pair = FieldValuePair::parse("PRIORITY=error").unwrap();
268
+ assert_eq!(pair.field(), "PRIORITY");
269
+ assert_eq!(pair.value(), "error");
270
+ assert_eq!(pair.as_str(), "PRIORITY=error");
271
+ assert_eq!(pair.as_bytes(), b"PRIORITY=error");
272
+
273
+ // Values can contain '=' characters
274
+ let pair = FieldValuePair::parse("MESSAGE=IN=eth0 OUT= MAC=aa:bb:cc").unwrap();
275
+ assert_eq!(pair.field(), "MESSAGE");
276
+ assert_eq!(pair.value(), "IN=eth0 OUT= MAC=aa:bb:cc");
277
+
278
+ assert!(FieldValuePair::parse("PRIORITY").is_none());
279
+ assert!(FieldValuePair::parse("=error").is_none());
280
+ }
281
+
282
+ #[test]
283
+ fn test_field_with_value() {
284
+ let field = FieldName::new("PRIORITY").unwrap();
285
+ let pair = field.with_value("error");
286
+
287
+ assert_eq!(pair.field(), "PRIORITY");
288
+ assert_eq!(pair.value(), "error");
289
+ assert_eq!(pair.as_str(), "PRIORITY=error");
290
+ }
291
+
292
+ #[test]
293
+ fn test_field_name_ordering() {
294
+ let mut fields = vec![
295
+ FieldName::new("PRIORITY").unwrap(),
296
+ FieldName::new("_HOSTNAME").unwrap(),
297
+ FieldName::new("SYSLOG_IDENTIFIER").unwrap(),
298
+ FieldName::new("ERRNO").unwrap(),
299
+ ];
300
+
301
+ fields.sort();
302
+
303
+ assert_eq!(fields[0].as_str(), "ERRNO");
304
+ assert_eq!(fields[1].as_str(), "PRIORITY");
305
+ assert_eq!(fields[2].as_str(), "SYSLOG_IDENTIFIER");
306
+ assert_eq!(fields[3].as_str(), "_HOSTNAME");
307
+ }
308
+
309
+ #[test]
310
+ fn test_field_value_pair_ordering() {
311
+ let mut pairs = vec![
312
+ FieldValuePair::parse("PRIORITY=error").unwrap(),
313
+ FieldValuePair::parse("PRIORITY=debug").unwrap(),
314
+ FieldValuePair::parse("_HOSTNAME=server2").unwrap(),
315
+ FieldValuePair::parse("_HOSTNAME=server1").unwrap(),
316
+ ];
317
+
318
+ pairs.sort();
319
+
320
+ assert_eq!(pairs[0].as_str(), "PRIORITY=debug");
321
+ assert_eq!(pairs[1].as_str(), "PRIORITY=error");
322
+ assert_eq!(pairs[2].as_str(), "_HOSTNAME=server1");
323
+ assert_eq!(pairs[3].as_str(), "_HOSTNAME=server2");
324
+ }
325
+
326
+ #[test]
327
+ fn test_strip_field_prefix() {
328
+ // Valid field=value
329
+ let payload = b"PRIORITY=6";
330
+ let value = FieldValuePair::strip_field_prefix(b"PRIORITY", payload);
331
+ assert_eq!(value, Some(&b"6"[..]));
332
+
333
+ // Value with special characters
334
+ let payload = b"MESSAGE=error: connection=failed";
335
+ let value = FieldValuePair::strip_field_prefix(b"MESSAGE", payload);
336
+ assert_eq!(value, Some(&b"error: connection=failed"[..]));
337
+
338
+ // Empty value
339
+ let payload = b"FIELD=";
340
+ let value = FieldValuePair::strip_field_prefix(b"FIELD", payload);
341
+ assert_eq!(value, Some(&b""[..]));
342
+
343
+ // Wrong field name
344
+ let payload = b"PRIORITY=6";
345
+ let value = FieldValuePair::strip_field_prefix(b"MESSAGE", payload);
346
+ assert_eq!(value, None);
347
+
348
+ // Missing '='
349
+ let payload = b"PRIORITY6";
350
+ let value = FieldValuePair::strip_field_prefix(b"PRIORITY", payload);
351
+ assert_eq!(value, None);
352
+
353
+ // Field name matches but continues (no '=')
354
+ let payload = b"PRIORITYX=6";
355
+ let value = FieldValuePair::strip_field_prefix(b"PRIORITY", payload);
356
+ assert_eq!(value, None);
357
+
358
+ // Empty payload
359
+ let value = FieldValuePair::strip_field_prefix(b"PRIORITY", b"");
360
+ assert_eq!(value, None);
361
+
362
+ // Payload shorter than field name
363
+ let value = FieldValuePair::strip_field_prefix(b"PRIORITY", b"PRI");
364
+ assert_eq!(value, None);
365
+ }
366
+}
src/crates/journal-index/src/file_index.rs
new
+818
@@ -0,0 +1,818 @@
1
+use crate::{
2
+ Bitmap, FieldName, FieldValuePair, Histogram, IndexError, Microseconds, Result, Seconds,
3
+};
4
+use journal_core::collections::{HashMap, HashSet};
5
+use journal_core::file::{JournalFile, Mmap};
6
+use journal_core::repository::File;
7
+use regex::Regex;
8
+use serde::{Deserialize, Serialize};
9
+use std::num::NonZeroU64;
10
+use tracing::{debug, error};
11
+
12
+/// Index for a single journal file, enabling efficient querying and filtering.
13
+///
14
+/// A `FileIndex` contains pre-computed metadata about a journal file:
15
+/// - Time-based histogram for quick time-range queries
16
+/// - Entry offsets sorted by timestamp for binary search
17
+/// - Bitmaps for indexed field=value pairs enabling fast filtering
18
+/// - Field names present in the file
19
+///
20
+/// The index is immutable after creation and represents a snapshot of the journal
21
+/// file at the time it was indexed. For actively-written files, the index may
22
+/// become stale and need rebuilding.
23
+#[derive(Debug, Clone, Serialize, Deserialize)]
24
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
25
+pub struct FileIndex {
26
+ // The file this index was created for
27
+ file: File,
28
+ // Unix timestamp (seconds since epoch) when this index was created
29
+ indexed_at: Seconds,
30
+ // True if the journal file was online (state=1) when indexed
31
+ was_online: bool,
32
+ // The journal file's histogram
33
+ histogram: Histogram,
34
+ // Entry offsets sorted by time
35
+ entry_offsets: Vec<u32>,
36
+ // Set of fields in the file
37
+ file_fields: HashSet<FieldName>,
38
+ // Set of fields that were requested to be indexed
39
+ indexed_fields: HashSet<FieldName>,
40
+ // Bitmap for each indexed field=value pair
41
+ bitmaps: HashMap<FieldValuePair, Bitmap>,
42
+}
43
+
44
+impl FileIndex {
45
+ /// Create a new file index.
46
+ #[allow(clippy::too_many_arguments)]
47
+ pub fn new(
48
+ file: File,
49
+ indexed_at: Seconds,
50
+ was_online: bool,
51
+ histogram: Histogram,
52
+ entry_offsets: Vec<u32>,
53
+ fields: HashSet<FieldName>,
54
+ indexed_fields: HashSet<FieldName>,
55
+ bitmaps: HashMap<FieldValuePair, Bitmap>,
56
+ ) -> Self {
57
+ Self {
58
+ file,
59
+ indexed_at,
60
+ was_online,
61
+ histogram,
62
+ entry_offsets,
63
+ file_fields: fields,
64
+ indexed_fields,
65
+ bitmaps,
66
+ }
67
+ }
68
+
69
+ /// Get the bucket duration granularity of the file's histogram.
70
+ pub fn bucket_duration(&self) -> Seconds {
71
+ Seconds(self.histogram.bucket_duration.get())
72
+ }
73
+
74
+ /// Get a reference to the journal file this index represents.
75
+ pub fn file(&self) -> &File {
76
+ &self.file
77
+ }
78
+
79
+ /// Get the timestamp when this index was created.
80
+ pub fn indexed_at(&self) -> Seconds {
81
+ self.indexed_at
82
+ }
83
+
84
+ /// Check if the journal file was online (actively being written) when indexed.
85
+ pub fn online(&self) -> bool {
86
+ self.was_online
87
+ }
88
+
89
+ /// Check if this index is still fresh.
90
+ ///
91
+ /// For files that were online (actively being written) when indexed, the cache
92
+ /// is considered stale after 1 second. For archived/offline files, the cache
93
+ /// is always fresh since they never change.
94
+ pub fn is_fresh(&self) -> bool {
95
+ if self.was_online {
96
+ let now = Seconds::now();
97
+ let age = now.get().saturating_sub(self.indexed_at.get());
98
+ age < 1
99
+ } else {
100
+ // Archived/offline file: always fresh
101
+ true
102
+ }
103
+ }
104
+
105
+ /// Get the start time of this file's indexed time range.
106
+ pub fn start_time(&self) -> Seconds {
107
+ self.histogram.start_time()
108
+ }
109
+
110
+ /// Get the end time of this file's indexed time range.
111
+ pub fn end_time(&self) -> Seconds {
112
+ self.histogram.end_time()
113
+ }
114
+
115
+ /// Get the number of time buckets.
116
+ pub fn num_buckets(&self) -> usize {
117
+ self.histogram.num_buckets()
118
+ }
119
+
120
+ /// Get the total count of entries indexed.
121
+ pub fn total_entries(&self) -> usize {
122
+ self.histogram.total_entries()
123
+ }
124
+
125
+ /// Get all field names present in this file.
126
+ pub fn fields(&self) -> &HashSet<FieldName> {
127
+ &self.file_fields
128
+ }
129
+
130
+ /// Get all indexed field=value pairs with their bitmaps.
131
+ pub fn bitmaps(&self) -> &HashMap<FieldValuePair, Bitmap> {
132
+ &self.bitmaps
133
+ }
134
+
135
+ /// Check if a field is indexed.
136
+ pub fn is_indexed(&self, field: &FieldName) -> bool {
137
+ self.indexed_fields.contains(field)
138
+ }
139
+
140
+ /// Count entries (from a bitmap) that fall within a time range.
141
+ pub fn count_entries_in_time_range(
142
+ &self,
143
+ bitmap: &Bitmap,
144
+ start_time: Seconds,
145
+ end_time: Seconds,
146
+ ) -> Option<usize> {
147
+ self.histogram
148
+ .count_entries_in_time_range(bitmap, start_time, end_time)
149
+ }
150
+}
151
+
152
+/// Direction for iterating through entries
153
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
154
+#[serde(rename_all = "lowercase")]
155
+pub enum Direction {
156
+ /// Iterate forward in time (from older to newer entries)
157
+ #[default]
158
+ Forward,
159
+ /// Iterate backward in time (from newer to older entries)
160
+ Backward,
161
+}
162
+
163
+/// Anchor point for starting a log query
164
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
165
+#[serde(rename_all = "lowercase")]
166
+pub enum Anchor {
167
+ /// Explicit timestamp in microseconds since epoch
168
+ Timestamp(Microseconds),
169
+ /// Start from the earliest timestamp (minimum start time from file indexes)
170
+ Head,
171
+ /// Start from the latest timestamp (maximum end time from file indexes)
172
+ Tail,
173
+}
174
+
175
+/// Parameters for querying log entries from journal files.
176
+///
177
+/// This struct encapsulates all the configuration needed to query log entries,
178
+/// whether from a single file or multiple files.
179
+///
180
+/// Use `LogQueryParamsBuilder` to construct instances of this type.
181
+#[derive(Debug, Clone)]
182
+pub struct LogQueryParams {
183
+ /// Starting point for the query
184
+ anchor: Anchor,
185
+ /// Direction to iterate (Forward or Backward)
186
+ direction: Direction,
187
+ /// Maximum number of entries to return (None means unlimited)
188
+ limit: Option<usize>,
189
+ /// Optional field to use for timestamps (None uses realtime)
190
+ source_timestamp_field: Option<super::FieldName>,
191
+ /// Optional filter to apply to entries
192
+ filter: Option<super::Filter>,
193
+ /// Optional lower time boundary (inclusive) in microseconds
194
+ after: Option<Microseconds>,
195
+ /// Optional upper time boundary (exclusive) in microseconds
196
+ before: Option<Microseconds>,
197
+ /// Optional position to resume from for pagination.
198
+ /// When set, the query will skip the binary search and continue from this position.
199
+ /// The filter must remain unchanged between paginated queries.
200
+ resume_position: Option<usize>,
201
+ /// Optional regex for free text search against entry data objects.
202
+ /// If set, only entries where at least one data object's full payload matches will be returned.
203
+ regex: Option<Regex>,
204
+}
205
+
206
+impl LogQueryParams {
207
+ /// Get the anchor point for the query
208
+ pub fn anchor(&self) -> Anchor {
209
+ self.anchor
210
+ }
211
+
212
+ /// Get the direction for iterating through entries
213
+ pub fn direction(&self) -> Direction {
214
+ self.direction
215
+ }
216
+
217
+ /// Get the maximum number of entries to return
218
+ pub fn limit(&self) -> Option<usize> {
219
+ self.limit
220
+ }
221
+
222
+ /// Get the source timestamp field
223
+ pub fn source_timestamp_field(&self) -> Option<&super::FieldName> {
224
+ self.source_timestamp_field.as_ref()
225
+ }
226
+
227
+ /// Get the filter to apply to entries
228
+ pub fn filter(&self) -> Option<&super::Filter> {
229
+ self.filter.as_ref()
230
+ }
231
+
232
+ /// Get the lower time boundary
233
+ pub fn after(&self) -> Option<Microseconds> {
234
+ self.after
235
+ }
236
+
237
+ /// Get the upper time boundary
238
+ pub fn before(&self) -> Option<Microseconds> {
239
+ self.before
240
+ }
241
+
242
+ /// Get the resume position for pagination
243
+ pub fn resume_position(&self) -> Option<usize> {
244
+ self.resume_position
245
+ }
246
+
247
+ /// Get the regex pattern for free text search
248
+ pub fn regex(&self) -> Option<&Regex> {
249
+ self.regex.as_ref()
250
+ }
251
+}
252
+
253
+/// Builder for constructing `LogQueryParams` with validation.
254
+///
255
+/// Anchor and direction are required at construction time.
256
+/// Other fields are optional and can be set via builder methods.
257
+#[derive(Debug, Clone)]
258
+pub struct LogQueryParamsBuilder {
259
+ anchor: Anchor,
260
+ direction: Direction,
261
+ limit: Option<usize>,
262
+ source_timestamp_field: Option<super::FieldName>,
263
+ filter: Option<super::Filter>,
264
+ after: Option<Microseconds>,
265
+ before: Option<Microseconds>,
266
+ resume_position: Option<usize>,
267
+ regex_pattern: Option<String>,
268
+}
269
+
270
+impl LogQueryParamsBuilder {
271
+ /// Create a new builder with required fields
272
+ ///
273
+ /// # Arguments
274
+ ///
275
+ /// * `anchor` - Starting point for the query
276
+ /// * `direction` - Direction to iterate through entries
277
+ pub fn new(anchor: Anchor, direction: Direction) -> Self {
278
+ Self {
279
+ anchor,
280
+ direction,
281
+ limit: None,
282
+ source_timestamp_field: None,
283
+ filter: None,
284
+ after: None,
285
+ before: None,
286
+ resume_position: None,
287
+ regex_pattern: None,
288
+ }
289
+ }
290
+
291
+ /// Set the maximum number of entries to return
292
+ pub fn with_limit(mut self, limit: usize) -> Self {
293
+ self.limit = Some(limit);
294
+ self
295
+ }
296
+
297
+ /// Set the source timestamp field
298
+ pub fn with_source_timestamp_field(mut self, field: Option<super::FieldName>) -> Self {
299
+ self.source_timestamp_field = field;
300
+ self
301
+ }
302
+
303
+ /// Set the filter
304
+ pub fn with_filter(mut self, filter: super::Filter) -> Self {
305
+ self.filter = Some(filter);
306
+ self
307
+ }
308
+
309
+ /// Set the lower time boundary
310
+ pub fn with_after(mut self, after: Microseconds) -> Self {
311
+ self.after = Some(after);
312
+ self
313
+ }
314
+
315
+ /// Set the upper time boundary
316
+ pub fn with_before(mut self, before: Microseconds) -> Self {
317
+ self.before = Some(before);
318
+ self
319
+ }
320
+
321
+ /// Set the resume position for pagination
322
+ pub fn with_resume_position(mut self, position: usize) -> Self {
323
+ self.resume_position = Some(position);
324
+ self
325
+ }
326
+
327
+ /// Set the regex pattern for free text search.
328
+ ///
329
+ /// The regex will be matched against the full payload of each data object
330
+ /// (in "FIELD=value" format). Only entries where at least one data object
331
+ /// matches will be returned.
332
+ ///
333
+ /// The pattern will be compiled during `build()`. Invalid patterns will
334
+ /// cause `build()` to return an error.
335
+ pub fn with_regex(mut self, pattern: impl Into<String>) -> Self {
336
+ self.regex_pattern = Some(pattern.into());
337
+ self
338
+ }
339
+
340
+ /// Build the LogQueryParams, validating optional constraints
341
+ pub fn build(self) -> Result<LogQueryParams> {
342
+ // Validate time boundaries if both are set
343
+ if let (Some(after), Some(before)) = (self.after, self.before) {
344
+ if after >= before {
345
+ return Err(IndexError::InvalidQueryTimeRange);
346
+ }
347
+ }
348
+
349
+ // Compile regex pattern if provided
350
+ let regex = if let Some(pattern) = self.regex_pattern {
351
+ debug!("compiling regex pattern for log query: {:?}", pattern);
352
+ match Regex::new(&pattern) {
353
+ Ok(regex) => {
354
+ debug!("regex pattern compiled successfully");
355
+ Some(regex)
356
+ }
357
+ Err(e) => {
358
+ error!("failed to compile regex pattern {:?}: {}", pattern, e);
359
+ return Err(IndexError::InvalidRegex);
360
+ }
361
+ }
362
+ } else {
363
+ None
364
+ };
365
+
366
+ Ok(LogQueryParams {
367
+ anchor: self.anchor,
368
+ direction: self.direction,
369
+ limit: self.limit,
370
+ source_timestamp_field: self.source_timestamp_field,
371
+ filter: self.filter,
372
+ after: self.after,
373
+ before: self.before,
374
+ resume_position: self.resume_position,
375
+ regex,
376
+ })
377
+ }
378
+}
379
+
380
+/// Read a timestamp field value from an entry's data objects.
381
+fn get_timestamp_field(
382
+ journal_file: &JournalFile<Mmap>,
383
+ field_name: &super::FieldName,
384
+ entry_offset: NonZeroU64,
385
+) -> Result<u64> {
386
+ let data_iter = journal_file.entry_data_objects(entry_offset)?;
387
+
388
+ for data_result in data_iter {
389
+ let data_object = data_result?;
390
+ match crate::field_types::parse_timestamp(field_name.as_bytes(), &data_object) {
391
+ Ok(timestamp) => return Ok(timestamp),
392
+ Err(IndexError::InvalidFieldPrefix) => {
393
+ continue;
394
+ }
395
+ Err(e) => return Err(e),
396
+ };
397
+ }
398
+
399
+ Err(IndexError::MissingFieldName)
400
+}
401
+
402
+/// Get the timestamp for an entry at the given offset.
403
+///
404
+/// Attempts to read the source_timestamp_field from the entry's data objects.
405
+/// Falls back to the entry's realtime timestamp if the field is not found.
406
+fn get_entry_timestamp(
407
+ journal_file: &JournalFile<Mmap>,
408
+ source_timestamp_field: Option<&super::FieldName>,
409
+ entry_offset: NonZeroU64,
410
+) -> Result<u64> {
411
+ // Try to read the source timestamp field if specified
412
+ if let Some(field_name) = source_timestamp_field {
413
+ if let Ok(timestamp) = get_timestamp_field(journal_file, field_name, entry_offset) {
414
+ return Ok(timestamp);
415
+ }
416
+ }
417
+
418
+ // Fall back to realtime timestamp
419
+ let entry = journal_file.entry_ref(entry_offset)?;
420
+ Ok(entry.header.realtime)
421
+}
422
+
423
+/// Binary search to find the partition point in a slice of entry offsets.
424
+///
425
+/// Returns the index of the first element for which the predicate returns false.
426
+/// The predicate may perform I/O and return errors, which are propagated.
427
+fn partition_point_entries<F>(
428
+ entry_offsets: &[NonZeroU64],
429
+ left: usize,
430
+ right: usize,
431
+ predicate: F,
432
+) -> Result<usize>
433
+where
434
+ F: Fn(NonZeroU64) -> Result<bool>,
435
+{
436
+ let mut left = left;
437
+ let mut right = right;
438
+
439
+ debug_assert!(left <= right);
440
+ debug_assert!(right <= entry_offsets.len());
441
+
442
+ while left != right {
443
+ let mid = left.midpoint(right);
444
+
445
+ if predicate(entry_offsets[mid])? {
446
+ left = mid + 1;
447
+ } else {
448
+ right = mid;
449
+ }
450
+ }
451
+
452
+ Ok(left)
453
+}
454
+
455
+/// Check if an entry matches a regex pattern without using a cache (for benchmarking).
456
+///
457
+/// This is the original implementation that loads and checks each data object
458
+/// for every entry, without caching results.
459
+#[doc(hidden)]
460
+pub fn entry_matches_regex_uncached(
461
+ journal_file: &JournalFile<Mmap>,
462
+ entry_offset: NonZeroU64,
463
+ regex: &Regex,
464
+) -> Result<bool> {
465
+ let data_iter = journal_file.entry_data_objects(entry_offset)?;
466
+
467
+ for data_result in data_iter {
468
+ let data_object = data_result?;
469
+ let payload = data_object.payload_bytes();
470
+
471
+ // Try to match as UTF-8 string
472
+ if let Ok(payload_str) = std::str::from_utf8(payload) {
473
+ if regex.is_match(payload_str) {
474
+ return Ok(true);
475
+ }
476
+ }
477
+ }
478
+
479
+ Ok(false)
480
+}
481
+
482
+/// Check if an entry matches a regex pattern
483
+fn entry_matches_regex(
484
+ journal_file: &JournalFile<Mmap>,
485
+ entry_offset: NonZeroU64,
486
+ regex: &Regex,
487
+ data_match_cache: &mut HashMap<NonZeroU64, bool>,
488
+ data_offsets_scratch: &mut Vec<NonZeroU64>,
489
+) -> Result<bool> {
490
+ // Collect all data object offsets for this entry
491
+ data_offsets_scratch.clear();
492
+ {
493
+ let entry = journal_file.entry_ref(entry_offset)?;
494
+ entry.collect_offsets(data_offsets_scratch)?;
495
+ }
496
+
497
+ // Check each data object offset
498
+ for data_offset in data_offsets_scratch.iter().copied() {
499
+ // Check cache first
500
+ if let Some(&matches) = data_match_cache.get(&data_offset) {
501
+ if matches {
502
+ return Ok(true);
503
+ }
504
+ continue;
505
+ }
506
+
507
+ // Cache miss - load the data object and check if it matches
508
+ let data_object = journal_file.data_ref(data_offset)?;
509
+ let payload = data_object.payload_bytes();
510
+
511
+ let matches = if let Ok(payload_str) = std::str::from_utf8(payload) {
512
+ regex.is_match(payload_str)
513
+ } else {
514
+ false
515
+ };
516
+
517
+ // Update cache
518
+ data_match_cache.insert(data_offset, matches);
519
+
520
+ if matches {
521
+ return Ok(true);
522
+ }
523
+ }
524
+
525
+ Ok(false)
526
+}
527
+
528
+/// Identifies a specific log entry within a journal file.
529
+#[derive(Debug, Clone)]
530
+pub struct LogEntryId {
531
+ /// The journal file containing this entry.
532
+ pub file: File,
533
+ /// Byte offset of the entry within the file.
534
+ pub offset: u64,
535
+ /// Timestamp of the entry in microseconds since epoch.
536
+ pub timestamp: Microseconds,
537
+ /// Position in the filtered entry_offsets vector.
538
+ /// Used for pagination to resume queries at the exact position.
539
+ pub position: usize,
540
+}
541
+
542
+impl FileIndex {
543
+ /// Retrieve log entries with filtering.
544
+ ///
545
+ /// This method efficiently retrieves journal entries based on the provided query
546
+ /// parameters. It uses binary search (partition point) to find the starting position,
547
+ /// then iterates in the specified direction.
548
+ ///
549
+ /// # Arguments
550
+ ///
551
+ /// * `file` - The journal file to read timestamps and entries from
552
+ /// * `params` - Query parameters (anchor, direction, limit, filter, boundaries)
553
+ ///
554
+ /// # Returns
555
+ ///
556
+ /// A vector of `LogEntryId` items sorted by time according to direction:
557
+ /// - Forward: Returns entries in ascending time order (oldest to newest after anchor)
558
+ /// - Backward: Returns entries in descending time order (newest to oldest before/at anchor)
559
+ ///
560
+ /// The vector length will not exceed `params.limit`. Returns an empty vector if no
561
+ /// entries match the criteria or if limit is 0.
562
+ pub fn find_log_entries(
563
+ &self,
564
+ file: &File,
565
+ params: &LogQueryParams,
566
+ ) -> Result<Vec<LogEntryId>> {
567
+ // Resolve anchor to concrete timestamp
568
+ // For single file queries, Head uses file start time and Tail uses file end time
569
+ let anchor_usec = match params.anchor() {
570
+ Anchor::Timestamp(ts) => ts,
571
+ Anchor::Head => self.start_time().to_microseconds(),
572
+ Anchor::Tail => self.end_time().to_microseconds(),
573
+ };
574
+
575
+ // Use filter's bitmap or one that fully covers all entries
576
+ let bitmap = params
577
+ .filter()
578
+ .map(|f| f.evaluate(self))
579
+ .unwrap_or_else(|| Bitmap::insert_range(0..self.entry_offsets.len() as u32));
580
+
581
+ if bitmap.is_empty() {
582
+ // Nothing matches
583
+ return Ok(Vec::new());
584
+ }
585
+
586
+ let window_size = 32 * 1024 * 1024;
587
+ let journal_file = JournalFile::open(file, window_size)?;
588
+
589
+ // Collect the entry offsets in the bitmap
590
+ // TODO: How should we handle zero offsets?
591
+ let entry_offsets: Vec<_> = bitmap
592
+ .iter()
593
+ .map(|idx| self.entry_offsets[idx as usize])
594
+ .filter(|offset| *offset != 0)
595
+ .map(|x| NonZeroU64::new(x as u64).expect("non-zero offset"))
596
+ .collect();
597
+
598
+ // Figure out what the limit should be
599
+ if let Some(limit) = params.limit() {
600
+ if limit == 0 {
601
+ return Ok(Vec::new());
602
+ }
603
+ }
604
+ let limit = params.limit().unwrap_or(entry_offsets.len());
605
+
606
+ let mut log_entry_ids = Vec::with_capacity(limit.min(entry_offsets.len()));
607
+
608
+ // Cache for regex matching. We use a scratch buffer for collecting
609
+ // the data offsets of an entry, and a hash map that stores the
610
+ // evaluation of each data object against the provided regex. This
611
+ // ensures that we will evaluate each data object only once.
612
+ let mut data_offsets_scratch = Vec::new();
613
+ let mut data_match_cache = HashMap::default();
614
+
615
+ // Log if regex filtering is active
616
+ let mut regex_filtered_count = 0usize;
617
+ if params.regex().is_some() {
618
+ debug!(
619
+ "regex filtering enabled for query, will filter {} candidate entries",
620
+ entry_offsets.len()
621
+ );
622
+ }
623
+
624
+ match params.direction() {
625
+ Direction::Forward => {
626
+ // Determine starting index: use resume_position or binary search
627
+ let start_idx = if let Some(resume_pos) = params.resume_position() {
628
+ // Resume from next position after the last returned entry
629
+ resume_pos + 1
630
+ } else {
631
+ // Find the partition point: first index where timestamp >= anchor_timestamp
632
+ // Predicate returns true while timestamp < anchor_timestamp
633
+ // Result is the index of the first entry with timestamp >= anchor_timestamp
634
+ partition_point_entries(
635
+ &entry_offsets,
636
+ 0,
637
+ entry_offsets.len(),
638
+ |entry_offset| {
639
+ let entry_timestamp = get_entry_timestamp(
640
+ &journal_file,
641
+ params.source_timestamp_field(),
642
+ entry_offset,
643
+ )?;
644
+ Ok(entry_timestamp < anchor_usec.get())
645
+ },
646
+ )?
647
+ };
648
+
649
+ // Edge cases for forward iteration:
650
+ // - start_idx == 0: anchor is <= all entries, start from first entry
651
+ // - start_idx == len: anchor is > all entries, no results
652
+ // - Otherwise: start from entry at start_idx (first entry >= anchor)
653
+
654
+ // Check bounds before slicing to avoid panic
655
+ if start_idx >= entry_offsets.len() {
656
+ // No entries to return
657
+ return Ok(log_entry_ids);
658
+ }
659
+
660
+ for (idx, &entry_offset) in entry_offsets[start_idx..].iter().enumerate() {
661
+ let timestamp = get_entry_timestamp(
662
+ &journal_file,
663
+ params.source_timestamp_field(),
664
+ entry_offset,
665
+ )?;
666
+
667
+ // Enforce time boundaries
668
+ if let Some(after) = params.after() {
669
+ if timestamp < after.get() {
670
+ continue;
671
+ }
672
+ }
673
+ if let Some(before) = params.before() {
674
+ if timestamp >= before.get() {
675
+ break; // Stop when we hit or exceed upper boundary
676
+ }
677
+ }
678
+
679
+ // Check regex filter if present
680
+ if let Some(regex) = params.regex() {
681
+ if !entry_matches_regex(
682
+ &journal_file,
683
+ entry_offset,
684
+ regex,
685
+ &mut data_match_cache,
686
+ &mut data_offsets_scratch,
687
+ )? {
688
+ regex_filtered_count += 1;
689
+ continue;
690
+ }
691
+ }
692
+
693
+ log_entry_ids.push(LogEntryId {
694
+ file: self.file.clone(),
695
+ offset: entry_offset.get(),
696
+ timestamp: Microseconds(timestamp),
697
+ position: start_idx + idx,
698
+ });
699
+
700
+ // Stop when we reach the limit
701
+ if log_entry_ids.len() >= limit {
702
+ break;
703
+ }
704
+ }
705
+ }
706
+ Direction::Backward => {
707
+ // Determine starting index: use resume_position or binary search
708
+ let start_idx = if let Some(resume_pos) = params.resume_position() {
709
+ // Resume from previous position before the last returned entry
710
+ if resume_pos == 0 {
711
+ // No more entries to return
712
+ return Ok(log_entry_ids);
713
+ }
714
+ // Check if resume_pos is out of bounds
715
+ if resume_pos >= entry_offsets.len() {
716
+ // Resume position is beyond valid range
717
+ return Ok(log_entry_ids);
718
+ }
719
+ resume_pos - 1
720
+ } else {
721
+ // Find the partition point: first index where timestamp > anchor_timestamp
722
+ // We want the LAST entry with timestamp <= anchor_timestamp
723
+ // which is at index (partition_point - 1)
724
+ let partition_idx = partition_point_entries(
725
+ &entry_offsets,
726
+ 0,
727
+ entry_offsets.len(),
728
+ |entry_offset| {
729
+ let entry_timestamp = get_entry_timestamp(
730
+ &journal_file,
731
+ params.source_timestamp_field(),
732
+ entry_offset,
733
+ )?;
734
+ Ok(entry_timestamp <= anchor_usec.get())
735
+ },
736
+ )?;
737
+
738
+ // Edge cases for backward iteration:
739
+ // - partition_idx == 0: all entries are > anchor, no results
740
+ // - partition_idx == len: anchor is >= all entries, start from last entry
741
+ // - Otherwise: start from entry at (partition_idx - 1), last entry <= anchor
742
+
743
+ if partition_idx == 0 {
744
+ // All entries have timestamp > anchor, no results
745
+ return Ok(log_entry_ids);
746
+ }
747
+
748
+ // Start from the last entry <= anchor (at partition_idx - 1)
749
+ partition_idx - 1
750
+ };
751
+
752
+ // Check bounds before slicing to avoid panic
753
+ if start_idx >= entry_offsets.len() {
754
+ // No entries to return
755
+ return Ok(log_entry_ids);
756
+ }
757
+
758
+ // Iterate backwards: from start_idx down to 0
759
+ for (idx, &entry_offset) in entry_offsets[..=start_idx].iter().rev().enumerate() {
760
+ let timestamp = get_entry_timestamp(
761
+ &journal_file,
762
+ params.source_timestamp_field(),
763
+ entry_offset,
764
+ )?;
765
+
766
+ // Enforce time boundaries
767
+ if let Some(before) = params.before() {
768
+ if timestamp >= before.get() {
769
+ continue;
770
+ }
771
+ }
772
+ if let Some(after) = params.after() {
773
+ if timestamp < after.get() {
774
+ break; // Stop when we go below lower boundary
775
+ }
776
+ }
777
+
778
+ // Check regex filter if present
779
+ if let Some(regex) = params.regex() {
780
+ if !entry_matches_regex(
781
+ &journal_file,
782
+ entry_offset,
783
+ regex,
784
+ &mut data_match_cache,
785
+ &mut data_offsets_scratch,
786
+ )? {
787
+ regex_filtered_count += 1;
788
+ continue;
789
+ }
790
+ }
791
+
792
+ log_entry_ids.push(LogEntryId {
793
+ file: self.file.clone(),
794
+ offset: entry_offset.get(),
795
+ timestamp: Microseconds(timestamp),
796
+ position: start_idx - idx,
797
+ });
798
+
799
+ // Stop when we reach the limit
800
+ if log_entry_ids.len() >= limit {
801
+ break;
802
+ }
803
+ }
804
+ }
805
+ }
806
+
807
+ // Log regex filtering statistics if regex was used
808
+ if params.regex().is_some() {
809
+ debug!(
810
+ "regex filtering complete: {} entries matched, {} entries filtered out",
811
+ log_entry_ids.len(),
812
+ regex_filtered_count
813
+ );
814
+ }
815
+
816
+ Ok(log_entry_ids)
817
+ }
818
+}
src/crates/journal-index/src/file_indexer.rs
new
+446
@@ -0,0 +1,446 @@
1
+//! Journal file indexing functionality.
2
+//!
3
+//! This module provides the [`FileIndexer`] type which creates searchable
4
+//! indexes from journal files. The indexing process extracts:
5
+//!
6
+//! - Time-based histograms for efficient range queries
7
+//! - Bitmap indexes for fast field=value lookups
8
+//! - Metadata about available and indexed fields
9
+
10
+use crate::{
11
+ Bitmap, FieldName, FieldValuePair, FileIndex, Histogram, IndexError, Microseconds, Result,
12
+ Seconds,
13
+};
14
+use journal_core::collections::{HashMap, HashSet};
15
+use journal_core::file::{JournalFile, Mmap, offset_array::InlinedCursor};
16
+use journal_registry::File;
17
+use std::num::NonZeroU64;
18
+use tracing::{error, warn};
19
+
20
+/// Reusable indexer for creating searchable indexes from journal files.
21
+///
22
+/// # Indexing Process
23
+///
24
+/// The indexer performs three main tasks:
25
+///
26
+/// 1. **Histogram Construction**: Creates time-based buckets for efficient
27
+/// range queries. Entries are ordered by their source timestamp (if
28
+/// available) or realtime timestamp.
29
+///
30
+/// 2. **Bitmap Index Creation**: For each specified field, creates bitmap
31
+/// indexes mapping field=value pairs to entry indices, enabling fast
32
+/// filtered queries.
33
+///
34
+/// 3. **Metadata Collection**: Tracks which fields are available in the file
35
+/// and which were indexed.
36
+///
37
+/// # Concurrent Write Handling
38
+///
39
+/// The indexer captures the journal file's `tail_object_offset` at the start of indexing
40
+/// to create a consistent snapshot. Any entries written to the file after indexing begins
41
+/// are ignored, preventing race conditions with concurrent writers.
42
+#[derive(Debug, Default)]
43
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
44
+pub struct FileIndexer {
45
+ // Associates a source timestamp value with its inlined cursor
46
+ source_timestamp_cursor_pairs: Vec<(Microseconds, InlinedCursor)>,
47
+
48
+ // Scratch buffer to collect entry offsets from the inlined cursor of a
49
+ // source timestamp value, or the global entry offset array
50
+ entry_offsets: Vec<NonZeroU64>,
51
+
52
+ // Associates a source timestamp value with its entry offset
53
+ source_timestamp_entry_offset_pairs: Vec<(Microseconds, NonZeroU64)>,
54
+
55
+ // Associates a journal file's entry realtime value with its offset
56
+ realtime_entry_offset_pairs: Vec<(Microseconds, NonZeroU64)>,
57
+
58
+ // Scratch buffer to collect the indices of entries in which a data
59
+ // object appears
60
+ entry_indices: Vec<u32>,
61
+
62
+ // Maps entry offsets to an index of an implicitly defined time-ordered
63
+ // array of entries
64
+ entry_offset_index: HashMap<NonZeroU64, u64>,
65
+}
66
+
67
+impl FileIndexer {
68
+ /// Create a searchable index from a journal file.
69
+ pub fn index(
70
+ &mut self,
71
+ file: &File,
72
+ source_timestamp_field: Option<&FieldName>,
73
+ field_names: &[FieldName],
74
+ bucket_duration: Seconds,
75
+ ) -> Result<FileIndex> {
76
+ self.source_timestamp_cursor_pairs = Vec::new();
77
+ self.source_timestamp_entry_offset_pairs = Vec::new();
78
+ self.realtime_entry_offset_pairs = Vec::new();
79
+ self.entry_indices = Vec::new();
80
+ self.entry_offsets = Vec::new();
81
+ self.entry_offset_index = HashMap::default();
82
+
83
+ let window_size = 32 * 1024 * 1024;
84
+ let journal_file = JournalFile::<Mmap>::open(file, window_size)?;
85
+
86
+ // NOTE: Capture the maximum valid entry offset at the start of
87
+ // indexing.
88
+ //
89
+ // This prevents race conditions when the journal file is being
90
+ // actively written to. The `tail_object_offset` from the header tells
91
+ // us the offset of the last object in the file at this moment. Any
92
+ // entry offset beyond this was added after we started indexing and
93
+ // should be ignored.
94
+ let Some(tail_object_offset) = journal_file.journal_header_ref().tail_object_offset else {
95
+ return Err(IndexError::MissingOffset);
96
+ };
97
+
98
+ // Capture indexing timestamp
99
+ let indexed_at = Seconds::now();
100
+
101
+ // Capture whether the file was online when indexed
102
+ let was_online = journal_file.journal_header_ref().state == 1;
103
+
104
+ let field_map = journal_file.load_fields()?;
105
+
106
+ // Build the file histogram
107
+ let histogram = self.build_histogram(
108
+ &journal_file,
109
+ source_timestamp_field,
110
+ bucket_duration,
111
+ tail_object_offset,
112
+ )?;
113
+
114
+ // Use the (timestamp, entry-offset) pairs to construct a vector that
115
+ // will contain entry offsets sorted by time
116
+ let entry_offsets = self
117
+ .source_timestamp_entry_offset_pairs
118
+ .iter()
119
+ .map(|(_, entry_offset)| entry_offset.get() as u32)
120
+ .collect();
121
+
122
+ // Create the bitmaps for field=value pairs
123
+ let entries =
124
+ self.build_entries_index(&journal_file, &field_map, field_names, tail_object_offset)?;
125
+
126
+ // Convert field_names to HashSet<FieldName> for indexed_fields
127
+ let indexed_fields: HashSet<FieldName> = field_names.iter().cloned().collect();
128
+
129
+ let mut file_fields = HashSet::default();
130
+ for field in field_map.keys() {
131
+ file_fields.insert(FieldName::new_unchecked(field));
132
+ }
133
+
134
+ Ok(FileIndex::new(
135
+ file.clone(),
136
+ indexed_at,
137
+ was_online,
138
+ histogram,
139
+ entry_offsets,
140
+ file_fields,
141
+ indexed_fields,
142
+ entries,
143
+ ))
144
+ }
145
+
146
+ /// Build bitmap indexes for field=value pairs.
147
+ ///
148
+ /// For each field in `field_names`, this iterates through all data objects
149
+ /// for that field and creates a bitmap mapping each unique field=value pair
150
+ /// to the entry indices where it appears.
151
+ ///
152
+ /// Only entries with offsets <= `tail_object_offset` are included in the
153
+ /// bitmaps, ensuring a consistent snapshot.
154
+ fn build_entries_index(
155
+ &mut self,
156
+ journal_file: &JournalFile<Mmap>,
157
+ field_map: &HashMap<String, String>,
158
+ field_names: &[FieldName],
159
+ tail_object_offset: NonZeroU64,
160
+ ) -> Result<HashMap<FieldValuePair, Bitmap>> {
161
+ let mut entries_index = HashMap::default();
162
+
163
+ for field_name in field_names {
164
+ let Some(systemd_field) = field_map.get(field_name.as_str()) else {
165
+ continue;
166
+ };
167
+
168
+ // Get the data object iterator for this field
169
+ let field_data_iterator =
170
+ match journal_file.field_data_objects(systemd_field.as_bytes()) {
171
+ Ok(field_data_iterator) => field_data_iterator,
172
+ Err(e) => {
173
+ warn!(
174
+ "failed to iterate field data objects for field '{}' in file {}: {:#?}",
175
+ systemd_field,
176
+ journal_file.file().path(),
177
+ e
178
+ );
179
+ continue;
180
+ }
181
+ };
182
+
183
+ for data_object in field_data_iterator {
184
+ // Get the payload and the inlined cursor for this data object
185
+ let (data_payload, inlined_cursor) = {
186
+ let Ok(data_object) = data_object else {
187
+ continue;
188
+ };
189
+
190
+ // Skip the remapping value
191
+ if data_object.payload_bytes().ends_with(field_name.as_bytes()) {
192
+ continue;
193
+ };
194
+
195
+ let data_payload =
196
+ String::from_utf8_lossy(data_object.payload_bytes()).into_owned();
197
+ let Some(inlined_cursor) = data_object.inlined_cursor() else {
198
+ continue;
199
+ };
200
+
201
+ (data_payload, inlined_cursor)
202
+ };
203
+
204
+ // Parse the payload into a FieldValuePair (format is "FIELD=value")
205
+ let Some(pair) = FieldValuePair::parse(&data_payload) else {
206
+ warn!("Invalid field=value format: {}", data_payload);
207
+ continue;
208
+ };
209
+
210
+ // Collect the offset of entries where this data object appears
211
+ self.entry_offsets.clear();
212
+ if inlined_cursor
213
+ .collect_offsets(journal_file, &mut self.entry_offsets)
214
+ .is_err()
215
+ {
216
+ continue;
217
+ }
218
+
219
+ // Map entry offsets where this data object appears to entry indices.
220
+ // Filter out any offsets that are beyond our initial snapshot's maximum
221
+ self.entry_indices.clear();
222
+ for entry_offset in self
223
+ .entry_offsets
224
+ .iter()
225
+ .copied()
226
+ .filter(|offset| *offset <= tail_object_offset)
227
+ {
228
+ let Some(entry_index) = self.entry_offset_index.get(&entry_offset) else {
229
+ // This should never happen given that we filter by the tail object offset.
230
+ panic!(
231
+ "missing entry offset {} from index (total offsets: {})",
232
+ entry_offset,
233
+ self.entry_offset_index.len()
234
+ );
235
+ };
236
+ self.entry_indices.push(*entry_index as u32);
237
+ }
238
+ self.entry_indices.sort_unstable();
239
+
240
+ // Create the bitmap for the entry indices
241
+ let mut bitmap = Bitmap::from_sorted_iter(self.entry_indices.iter().copied())
242
+ .expect("sorted entry indices");
243
+ bitmap.optimize();
244
+
245
+ let field_name = FieldName::new_unchecked(field_name);
246
+ let k = FieldValuePair::new_unchecked(field_name, String::from(pair.value()));
247
+ entries_index.insert(k, bitmap);
248
+ }
249
+ }
250
+
251
+ Ok(entries_index)
252
+ }
253
+
254
+ /// Collect timestamp information from a source timestamp field.
255
+ ///
256
+ /// This extracts (timestamp, entry_offset) pairs from the specified source
257
+ /// field (typically `_SOURCE_REALTIME_TIMESTAMP`). The pairs are sorted by
258
+ /// timestamp and used to build `entry_offset_index`, which maps each entry
259
+ /// offset to its position in the time-ordered sequence.
260
+ ///
261
+ /// If the source field is missing for some entries, those entries will be
262
+ /// handled later using the journal file's realtime timestamp.
263
+ fn collect_source_field_info(
264
+ &mut self,
265
+ journal_file: &JournalFile<Mmap>,
266
+ source_field_name: &[u8],
267
+ ) -> Result<()> {
268
+ // Create an iterator over all the different values the field can take
269
+ let field_data_iterator = journal_file.field_data_objects(source_field_name)?;
270
+
271
+ // Collect all the inlined cursors of the source timestamp field
272
+ self.source_timestamp_cursor_pairs.clear();
273
+ for data_object_result in field_data_iterator {
274
+ let Ok(data_object) = data_object_result else {
275
+ warn!("loading data object failed");
276
+ continue;
277
+ };
278
+
279
+ let Ok(source_timestamp) =
280
+ crate::field_types::parse_timestamp(source_field_name, &data_object)
281
+ else {
282
+ warn!("parsing source timestamp failed");
283
+ continue;
284
+ };
285
+
286
+ let Some(ic) = data_object.inlined_cursor() else {
287
+ use journal_core::file::JournalState;
288
+
289
+ let file_state = JournalState::try_from(journal_file.journal_header_ref().state)
290
+ .map(|s| s.to_string())
291
+ .unwrap_or_else(|_| "UNKNOWN".to_string());
292
+
293
+ warn!(
294
+ "orphaned data object (no entries) for _SOURCE_REALTIME_TIMESTAMP={} in {} (state: {})",
295
+ source_timestamp,
296
+ journal_file.file().path(),
297
+ file_state
298
+ );
299
+ continue;
300
+ };
301
+
302
+ self.source_timestamp_cursor_pairs
303
+ .push((Microseconds(source_timestamp), ic));
304
+ }
305
+
306
+ // Collect all the [source_timestamp, entry-offset] pairs
307
+ self.source_timestamp_entry_offset_pairs.clear();
308
+ for (ts, ic) in self.source_timestamp_cursor_pairs.iter() {
309
+ self.entry_offsets.clear();
310
+
311
+ match ic.collect_offsets(journal_file, &mut self.entry_offsets) {
312
+ Ok(_) => {}
313
+ Err(e) => {
314
+ error!("failed to collect offsets from source timestamp: {}", e);
315
+ continue;
316
+ }
317
+ }
318
+
319
+ for entry_offset in &self.entry_offsets {
320
+ self.source_timestamp_entry_offset_pairs
321
+ .push((*ts, *entry_offset));
322
+ }
323
+ }
324
+ // Sort the [source_timestamp, entry-offset] pairs
325
+ self.source_timestamp_entry_offset_pairs.sort_unstable();
326
+
327
+ // Map each entry offset to its position in the pair vector
328
+ for (idx, (_, entry_offset)) in self.source_timestamp_entry_offset_pairs.iter().enumerate()
329
+ {
330
+ self.entry_offset_index.insert(*entry_offset, idx as _);
331
+ }
332
+
333
+ Ok(())
334
+ }
335
+
336
+ /// Build a time-based histogram for the journal file.
337
+ ///
338
+ /// This creates a histogram that maps time ranges to entry counts, enabling
339
+ /// efficient time-range queries. The histogram uses the source timestamp field
340
+ /// if available, falling back to the journal file's realtime timestamp for
341
+ /// entries where the source field is missing.
342
+ ///
343
+ /// The method:
344
+ /// 1. Collects timestamps from the source field (if specified)
345
+ /// 2. Loads the global entry offset array
346
+ /// 3. Fills in missing timestamps using realtime values
347
+ /// 4. Sorts all (timestamp, entry_offset) pairs by time
348
+ /// 5. Constructs the histogram with the specified bucket duration
349
+ fn build_histogram(
350
+ &mut self,
351
+ journal_file: &JournalFile<Mmap>,
352
+ source_timestamp_field_name: Option<&FieldName>,
353
+ bucket_duration: Seconds,
354
+ tail_object_offset: NonZeroU64,
355
+ ) -> Result<Histogram> {
356
+ // Collect information from the source timestamp field
357
+ if let Some(source_field_name) = source_timestamp_field_name {
358
+ self.collect_source_field_info(journal_file, source_field_name.as_bytes())?;
359
+ }
360
+
361
+ // At this point:
362
+ //
363
+ // - `self.source_timestamp_entry_offset_pairs`: contains a vector of
364
+ // (timestamp, entry-offset) pairs sorted by time,
365
+ // - `self.entry_offset_index`: maps an entry offset to a number
366
+ // with the following invariant:
367
+ // if (e1.offset < e2.offset) then e1.number < e2.number.
368
+
369
+ // Load the global entry offset array from the file
370
+ self.entry_offsets.clear();
371
+ journal_file.entry_offsets(&mut self.entry_offsets)?;
372
+
373
+ // Iterate through entry offsets and find entries for which we could
374
+ // not collect a timestamp. In this case, fall-back to using the journal
375
+ // file's realtime timestamp. Filter out offsets beyond our maximum.
376
+ self.realtime_entry_offset_pairs.clear();
377
+ for entry_offset in self
378
+ .entry_offsets
379
+ .iter()
380
+ .copied()
381
+ .filter(|offset| *offset <= tail_object_offset)
382
+ {
383
+ if self.entry_offset_index.contains_key(&entry_offset) {
384
+ // We have the timestamp of this entry offset
385
+ continue;
386
+ }
387
+
388
+ // We don't know the timestamp of this entry offset, use
389
+ // the journal's file realtime timestamp.
390
+
391
+ let timestamp = {
392
+ let entry = journal_file.entry_ref(entry_offset)?;
393
+ entry.header.realtime
394
+ };
395
+
396
+ // Add the new (timestamp, entry-offset) pair
397
+ self.realtime_entry_offset_pairs
398
+ .push((Microseconds(timestamp), entry_offset));
399
+ }
400
+
401
+ // At this point:
402
+ //
403
+ // - `self.realtime_entry_offset_pairs`: contains (timestamp, entry-offset)
404
+ // pairs of all the entries for which we had to use the journal file's
405
+ // realtime timestamp.
406
+
407
+ // Reconstruct our indexes if we have entries whose time does not
408
+ // come from the source timestamp
409
+ if !self.realtime_entry_offset_pairs.is_empty() {
410
+ // Extend the vector holding pairs collected from the source timestamp
411
+ // with the pairs collected from the realtime timestamp and
412
+ // sort it by time again.
413
+ self.source_timestamp_entry_offset_pairs
414
+ .append(&mut self.realtime_entry_offset_pairs);
415
+ self.source_timestamp_entry_offset_pairs.sort_unstable();
416
+
417
+ // We need to rebuild the `self.entry_offset_index` because
418
+ // we found entry offsets from the global entry offset array
419
+ // whose timestamp is assume to be equal to the realtime timestamp
420
+ // of the journal file
421
+ self.entry_offset_index.clear();
422
+ for (idx, (_, entry_offset)) in
423
+ self.source_timestamp_entry_offset_pairs.iter().enumerate()
424
+ {
425
+ self.entry_offset_index.insert(*entry_offset, idx as _);
426
+ }
427
+ }
428
+
429
+ // At this point, we have information about the order and the time
430
+ // of all entries in the journal file:
431
+ //
432
+ // - `self.source_timestamp_entry_offset_pairs`: contains a vector of
433
+ // (timestamp, entry-offset) pairs sorted by time,
434
+ // - `self.entry_offset_index`: maps an entry offset to a number
435
+ // with the following invariant:
436
+ // if (e1.offset < e2.offset) then e1.number < e2.number.
437
+ //
438
+ // We can proceed with building the histogram
439
+
440
+ // Now we can build the file histogram
441
+ Histogram::from_timestamp_offset_pairs(
442
+ bucket_duration,
443
+ self.source_timestamp_entry_offset_pairs.as_slice(),
444
+ )
445
+ }
446
+}
src/crates/journal-index/src/filter.rs
new
+297
@@ -0,0 +1,297 @@
1
+use crate::{Bitmap, FieldName, FieldValuePair, FileIndex};
2
+use std::hash::{Hash, Hasher};
3
+use std::sync::Arc;
4
+
5
+/// Represents what a filter expression can match against.
6
+///
7
+/// This enum distinguishes between:
8
+/// - Matching a field name (e.g., "PRIORITY" matches any PRIORITY value)
9
+/// - Matching a specific field=value pair (e.g., "PRIORITY=error")
10
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
11
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
12
+enum FilterTarget {
13
+ /// Match any entry that has this field, regardless of value
14
+ Field(FieldName),
15
+ /// Match entries where this specific field=value pair exists
16
+ Pair(FieldValuePair),
17
+}
18
+
19
+/// High-level filter expression that operates on field names and field=value pairs.
20
+///
21
+/// This is the primary type used when constructing filters from user queries.
22
+/// Use [`Filter::match_field_name()`] to match any entry with a specific field,
23
+/// or [`Filter::match_field_value_pair()`] to match a specific field=value combination.
24
+///
25
+/// Filters can be combined using [`Filter::and()`] and [`Filter::or()`] for complex queries.
26
+#[derive(Clone, Debug)]
27
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
28
+pub struct Filter {
29
+ inner: Arc<FilterExpr<FilterTarget>>,
30
+}
31
+
32
+impl Filter {
33
+ /// Create a filter that matches any entry with the given field name.
34
+ pub fn match_field_name(name: FieldName) -> Self {
35
+ Self {
36
+ inner: Arc::new(FilterExpr::Match(FilterTarget::Field(name))),
37
+ }
38
+ }
39
+
40
+ /// Create a filter that matches a specific field=value pair.
41
+ pub fn match_field_value_pair(pair: FieldValuePair) -> Self {
42
+ Self {
43
+ inner: Arc::new(FilterExpr::Match(FilterTarget::Pair(pair))),
44
+ }
45
+ }
46
+
47
+ /// Combine multiple filters with AND logic.
48
+ pub fn and(filters: Vec<Self>) -> Self {
49
+ let inner_filters: Vec<FilterExpr<FilterTarget>> =
50
+ filters.into_iter().map(|f| (*f.inner).clone()).collect();
51
+
52
+ Self {
53
+ inner: Arc::new(FilterExpr::and(inner_filters)),
54
+ }
55
+ }
56
+
57
+ /// Combine multiple filters with OR logic.
58
+ pub fn or(filters: Vec<Self>) -> Self {
59
+ let inner_filters: Vec<FilterExpr<FilterTarget>> =
60
+ filters.into_iter().map(|f| (*f.inner).clone()).collect();
61
+
62
+ Self {
63
+ inner: Arc::new(FilterExpr::or(inner_filters)),
64
+ }
65
+ }
66
+
67
+ /// Create a filter that matches nothing.
68
+ pub fn none() -> Self {
69
+ Self {
70
+ inner: Arc::new(FilterExpr::None),
71
+ }
72
+ }
73
+
74
+ /// Check if this is a None filter.
75
+ pub fn is_none(&self) -> bool {
76
+ matches!(self.inner.as_ref(), FilterExpr::None)
77
+ }
78
+
79
+ /// Evaluate this filter against a file index to get matching entry indices.
80
+ pub fn evaluate(&self, file_index: &FileIndex) -> Bitmap {
81
+ self.inner.resolve(file_index).evaluate()
82
+ }
83
+}
84
+
85
+impl PartialEq for Filter {
86
+ fn eq(&self, other: &Self) -> bool {
87
+ // Quick pointer equality check first
88
+ if Arc::ptr_eq(&self.inner, &other.inner) {
89
+ return true;
90
+ }
91
+
92
+ // Fall back to value equality
93
+ self.inner == other.inner
94
+ }
95
+}
96
+
97
+impl Eq for Filter {}
98
+
99
+impl std::hash::Hash for Filter {
100
+ fn hash<H: std::hash::Hasher>(&self, state: &mut H) {
101
+ self.inner.hash(state);
102
+ }
103
+}
104
+
105
+impl std::fmt::Display for Filter {
106
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
107
+ write!(f, "{}", self.inner)
108
+ }
109
+}
110
+
111
+#[derive(Clone, Debug, PartialEq)]
112
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
113
+enum FilterExpr<T> {
114
+ None,
115
+ Match(T),
116
+ Conjunction(Vec<Self>),
117
+ Disjunction(Vec<Self>),
118
+}
119
+
120
+impl Eq for FilterExpr<FilterTarget> {}
121
+
122
+impl Hash for FilterExpr<FilterTarget> {
123
+ fn hash<H: Hasher>(&self, state: &mut H) {
124
+ std::mem::discriminant(self).hash(state);
125
+
126
+ match self {
127
+ FilterExpr::None => {}
128
+ FilterExpr::Match(target) => target.hash(state),
129
+ FilterExpr::Conjunction(filters) => filters.hash(state),
130
+ FilterExpr::Disjunction(filters) => filters.hash(state),
131
+ }
132
+ }
133
+}
134
+
135
+impl FilterExpr<FilterTarget> {
136
+ fn and(filters: Vec<Self>) -> Self {
137
+ // Flatten any nested conjunctions and remove None filters
138
+ let mut flattened = Vec::new();
139
+ for filter in filters {
140
+ match filter {
141
+ FilterExpr::Conjunction(inner) => flattened.extend(inner),
142
+ FilterExpr::None => continue,
143
+ other => flattened.push(other),
144
+ }
145
+ }
146
+
147
+ match flattened.len() {
148
+ 0 => FilterExpr::None,
149
+ 1 => flattened.into_iter().next().unwrap(),
150
+ _ => FilterExpr::Conjunction(flattened),
151
+ }
152
+ }
153
+
154
+ fn or(filters: Vec<Self>) -> Self {
155
+ // Flatten any nested disjunctions and remove None filters
156
+ let mut flattened = Vec::new();
157
+ for filter in filters {
158
+ match filter {
159
+ FilterExpr::Disjunction(inner) => flattened.extend(inner),
160
+ FilterExpr::None => continue,
161
+ other => flattened.push(other),
162
+ }
163
+ }
164
+
165
+ match flattened.len() {
166
+ 0 => FilterExpr::None,
167
+ 1 => flattened.into_iter().next().unwrap(),
168
+ _ => FilterExpr::Disjunction(flattened),
169
+ }
170
+ }
171
+
172
+ /// Convert a [`FilterExpr<FilterTarget>`] to [`FilterExpr<Bitmap>`] using the file index
173
+ fn resolve(&self, file_index: &FileIndex) -> FilterExpr<Bitmap> {
174
+ match self {
175
+ FilterExpr::None => FilterExpr::None,
176
+ FilterExpr::Match(target) => match target {
177
+ FilterTarget::Field(field_name) => {
178
+ // Find all field=value pairs with matching field name
179
+ let matches: Vec<_> = file_index
180
+ .bitmaps()
181
+ .iter()
182
+ .filter(|(pair, _)| pair.field() == field_name.as_str())
183
+ .map(|(_, bitmap)| FilterExpr::Match(bitmap.clone()))
184
+ .collect();
185
+
186
+ match matches.len() {
187
+ 0 => FilterExpr::None,
188
+ 1 => matches.into_iter().next().unwrap(),
189
+ _ => FilterExpr::Disjunction(matches),
190
+ }
191
+ }
192
+ FilterTarget::Pair(pair) => {
193
+ // Lookup specific field=value pair
194
+ if let Some(bitmap) = file_index.bitmaps().get(pair) {
195
+ FilterExpr::Match(bitmap.clone())
196
+ } else {
197
+ FilterExpr::None
198
+ }
199
+ }
200
+ },
201
+ FilterExpr::Conjunction(filters) => {
202
+ let mut resolved = Vec::with_capacity(filters.len());
203
+ for filter in filters {
204
+ let r = filter.resolve(file_index);
205
+ if matches!(r, FilterExpr::None) {
206
+ return FilterExpr::None;
207
+ }
208
+ resolved.push(r);
209
+ }
210
+
211
+ match resolved.len() {
212
+ 0 => FilterExpr::None,
213
+ 1 => resolved.into_iter().next().unwrap(),
214
+ _ => FilterExpr::Conjunction(resolved),
215
+ }
216
+ }
217
+ FilterExpr::Disjunction(filters) => {
218
+ let mut resolved = Vec::with_capacity(filters.len());
219
+ for filter in filters {
220
+ let r = filter.resolve(file_index);
221
+ if !matches!(r, FilterExpr::None) {
222
+ resolved.push(r);
223
+ }
224
+ }
225
+
226
+ match resolved.len() {
227
+ 0 => FilterExpr::None,
228
+ 1 => resolved.into_iter().next().unwrap(),
229
+ _ => FilterExpr::Disjunction(resolved),
230
+ }
231
+ }
232
+ }
233
+ }
234
+}
235
+
236
+impl std::fmt::Display for FilterExpr<FilterTarget> {
237
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
238
+ match self {
239
+ FilterExpr::None => write!(f, "None"),
240
+ FilterExpr::Match(target) => match target {
241
+ FilterTarget::Field(name) => write!(f, "{}", name),
242
+ FilterTarget::Pair(pair) => write!(f, "{}", pair),
243
+ },
244
+ FilterExpr::Conjunction(filters) => {
245
+ write!(f, "(")?;
246
+ for (i, filter) in filters.iter().enumerate() {
247
+ if i > 0 {
248
+ write!(f, " AND ")?;
249
+ }
250
+ write!(f, "{}", filter)?;
251
+ }
252
+ write!(f, ")")
253
+ }
254
+ FilterExpr::Disjunction(filters) => {
255
+ write!(f, "(")?;
256
+ for (i, filter) in filters.iter().enumerate() {
257
+ if i > 0 {
258
+ write!(f, " OR ")?;
259
+ }
260
+ write!(f, "{}", filter)?;
261
+ }
262
+ write!(f, ")")
263
+ }
264
+ }
265
+ }
266
+}
267
+
268
+impl FilterExpr<Bitmap> {
269
+ /// Get all entry indices that match this filter expression
270
+ fn evaluate(&self) -> Bitmap {
271
+ match self {
272
+ Self::None => Bitmap::new(),
273
+ Self::Match(bitmap) => bitmap.clone(),
274
+ Self::Conjunction(filter_exprs) => {
275
+ if filter_exprs.is_empty() {
276
+ return Bitmap::new();
277
+ }
278
+
279
+ let mut result = filter_exprs[0].evaluate();
280
+ for expr in filter_exprs.iter().skip(1) {
281
+ result &= expr.evaluate();
282
+ if result.is_empty() {
283
+ break; // Early termination for empty conjunction
284
+ }
285
+ }
286
+ result
287
+ }
288
+ Self::Disjunction(filter_exprs) => {
289
+ let mut result = Bitmap::new();
290
+ for expr in filter_exprs.iter() {
291
+ result |= expr.evaluate();
292
+ }
293
+ result
294
+ }
295
+ }
296
+ }
297
+}
src/crates/journal-index/src/histogram.rs
new
+649
@@ -0,0 +1,649 @@
1
+//! Sparse histogram with running counts for time-based aggregation.
2
+//!
3
+//! The histogram stores only bucket boundaries where entries exist. Each
4
+//! bucket contains a running count: the total number of entries from the
5
+//! start up to and including that bucket. This enables efficient range
6
+//! queries via binary search on bucket boundaries.
7
+
8
+use crate::{Bitmap, IndexError, Microseconds, Result, Seconds};
9
+use journal_common::compat::is_multiple_of;
10
+
11
+use serde::{Deserialize, Serialize};
12
+use std::num::NonZeroU32;
13
+
14
+/// A bucket boundary storing a time and running count.
15
+///
16
+/// The `count` is the 0-based index of the last entry in this bucket. For
17
+/// example, if bucket at time 0 has count=4 and bucket at time 60 has count=9,
18
+/// then:
19
+/// - Bucket [0, 60) contains entries with indices 0-4 (5 entries)
20
+/// - Bucket [60, 120) contains entries with indices 5-9 (5 entries)
21
+#[derive(Debug, Clone, Copy, Serialize, Deserialize)]
22
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
23
+pub struct Bucket {
24
+ /// Start time of this bucket (aligned to bucket_duration)
25
+ pub start_time: Seconds,
26
+ /// 0-based index of the last entry in this bucket
27
+ pub count: u32,
28
+}
29
+
30
+/// Sparse histogram storing only bucket boundaries with running counts.
31
+///
32
+/// Invariants:
33
+/// - `buckets` is sorted by `start_time`
34
+/// - `start_time % bucket_duration == 0` for all buckets
35
+/// - Always contains at least one bucket
36
+#[derive(Clone, Debug, Serialize, Deserialize)]
37
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
38
+pub struct Histogram {
39
+ /// Fixed size of each time bucket in seconds
40
+ pub bucket_duration: NonZeroU32,
41
+ /// Sorted sparse vector of bucket boundaries
42
+ pub buckets: Vec<Bucket>,
43
+}
44
+
45
+impl Histogram {
46
+ /// Constructs a histogram from sorted timestamp-offset pairs.
47
+ ///
48
+ /// # Algorithm
49
+ ///
50
+ /// 1. Convert each timestamp to seconds and compute its bucket:
51
+ /// `(timestamp_secs / bucket_duration) * bucket_duration`
52
+ /// 2. Track bucket boundaries: when an entry falls into a new bucket,
53
+ /// store the previous bucket with the index of its last entry (running
54
+ /// count)
55
+ /// 3. Only buckets containing entries are stored (sparse representation)
56
+ ///
57
+ /// # Errors
58
+ ///
59
+ /// - `ZeroBucketDuration` if `bucket_duration` is 0
60
+ /// - `EmptyHistogramInput` if `timestamp_offset_pairs` is empty
61
+ ///
62
+ /// # Panics
63
+ ///
64
+ /// Debug builds panic if `timestamp_offset_pairs` is not sorted.
65
+ pub fn from_timestamp_offset_pairs(
66
+ bucket_duration: Seconds,
67
+ timestamp_offset_pairs: &[(Microseconds, std::num::NonZeroU64)],
68
+ ) -> Result<Histogram> {
69
+ if bucket_duration.0 == 0 {
70
+ return Err(IndexError::ZeroBucketDuration);
71
+ }
72
+
73
+ if timestamp_offset_pairs.is_empty() {
74
+ return Err(IndexError::EmptyHistogramInput);
75
+ }
76
+
77
+ debug_assert!(timestamp_offset_pairs.is_sorted());
78
+
79
+ let mut buckets = Vec::new();
80
+ let mut current_bucket = None;
81
+
82
+ for (offset_index, &(timestamp, _offset)) in timestamp_offset_pairs.iter().enumerate() {
83
+ // Calculate which bucket this timestamp falls into
84
+ let bucket =
85
+ Seconds((timestamp.to_seconds().0 / bucket_duration.0) * bucket_duration.0);
86
+
87
+ match current_bucket {
88
+ None => {
89
+ // First entry - don't create bucket yet, just track the bucket
90
+ debug_assert_eq!(offset_index, 0);
91
+ current_bucket = Some(bucket);
92
+ }
93
+ Some(prev_bucket) if bucket.0 > prev_bucket.0 => {
94
+ // New bucket boundary - save the LAST index of the previous bucket
95
+ buckets.push(Bucket {
96
+ start_time: prev_bucket,
97
+ count: offset_index as u32 - 1,
98
+ });
99
+ current_bucket = Some(bucket);
100
+ }
101
+ _ => {} // Same bucket, continue
102
+ }
103
+ }
104
+
105
+ // Handle last bucket
106
+ if let Some(last_bucket) = current_bucket {
107
+ buckets.push(Bucket {
108
+ start_time: last_bucket,
109
+ count: timestamp_offset_pairs.len() as u32 - 1,
110
+ });
111
+ }
112
+
113
+ // Now that we are done, we can convert to non-zero
114
+ let bucket_duration = NonZeroU32::new(bucket_duration.0).expect("non-zero bucket duration");
115
+
116
+ Ok(Histogram {
117
+ bucket_duration,
118
+ buckets,
119
+ })
120
+ }
121
+
122
+ /// Get the start time of the histogram.
123
+ pub fn start_time(&self) -> Seconds {
124
+ let first_bucket = self.buckets.first().expect("histogram to have buckets");
125
+ first_bucket.start_time
126
+ }
127
+
128
+ /// Get the end time of the histogram.
129
+ pub fn end_time(&self) -> Seconds {
130
+ let last_bucket = self.buckets.last().expect("histogram to have buckets");
131
+ Seconds(last_bucket.start_time.0 + self.bucket_duration.get())
132
+ }
133
+
134
+ /// Get the time range covered by the histogram.
135
+ pub fn time_range(&self) -> (Seconds, Seconds) {
136
+ (self.start_time(), self.end_time())
137
+ }
138
+
139
+ /// Returns the number of buckets in the histogram.
140
+ pub fn num_buckets(&self) -> usize {
141
+ self.buckets.len()
142
+ }
143
+
144
+ /// Get the total number of entries in the histogram.
145
+ pub fn total_entries(&self) -> usize {
146
+ let last_bucket = self.buckets.last().expect("histogram to have buckets");
147
+ // FIXME: Off-by-one error
148
+ last_bucket.count as usize + 1
149
+ }
150
+
151
+ /// Check if the file histogram is empty.
152
+ pub fn is_empty(&self) -> bool {
153
+ self.buckets.is_empty()
154
+ }
155
+
156
+ /// Count entries (from a bitmap) that fall within a time range using the histogram's bucket structure.
157
+ ///
158
+ /// # Algorithm
159
+ ///
160
+ /// 1. Binary search to find the first bucket at or after `start_time`
161
+ /// 2. Binary search to find the last bucket before `end_time`
162
+ /// 3. Extract entry index range from running counts:
163
+ /// - Start index: running count of previous bucket + 1 (or 0 if first bucket)
164
+ /// - End index: running count of last bucket (inclusive)
165
+ /// 4. Count bitmap entries in the index range
166
+ ///
167
+ /// Returns `None` if the time range is not aligned to `bucket_duration` or
168
+ /// invalid.
169
+ pub fn count_entries_in_time_range(
170
+ &self,
171
+ bitmap: &Bitmap,
172
+ start_time: Seconds,
173
+ end_time: Seconds,
174
+ ) -> Option<usize> {
175
+ // Validate inputs
176
+ if start_time >= end_time {
177
+ return None;
178
+ }
179
+
180
+ // Verify alignment to bucket_duration
181
+ if !is_multiple_of(start_time.0, self.bucket_duration.get())
182
+ || !is_multiple_of(end_time.0, self.bucket_duration.get())
183
+ {
184
+ return None;
185
+ }
186
+
187
+ // Handle empty histogram or bitmap
188
+ if self.buckets.is_empty() || bitmap.is_empty() {
189
+ return Some(0);
190
+ }
191
+
192
+ // Find the bucket indices for start and end times using binary search
193
+ // partition_point returns the index of the first bucket with start_time >= start_time
194
+ let start_bucket_idx = self.buckets.partition_point(|b| b.start_time < start_time);
195
+
196
+ // If start_bucket_idx is beyond all buckets, no matches possible
197
+ if start_bucket_idx >= self.buckets.len() {
198
+ return Some(0);
199
+ }
200
+
201
+ // Find the last bucket that starts before end_time
202
+ // partition_point returns the index of the first bucket with start_time >= end_time,
203
+ // so we need to subtract 1 to get the last bucket before end_time
204
+ let end_bucket_idx = self
205
+ .buckets
206
+ .partition_point(|b| b.start_time < end_time)
207
+ .saturating_sub(1);
208
+
209
+ // If start is after end, the range doesn't contain any buckets
210
+ if start_bucket_idx > end_bucket_idx {
211
+ return Some(0);
212
+ }
213
+
214
+ // Get the running count boundaries
215
+ // For start: we want entries AFTER the previous bucket's running count
216
+ let start_running_count = if start_bucket_idx == 0 {
217
+ 0
218
+ } else {
219
+ self.buckets[start_bucket_idx - 1].count + 1
220
+ };
221
+
222
+ // For end: we want entries UP TO AND INCLUDING this bucket's running count
223
+ let end_running_count = self.buckets[end_bucket_idx].count;
224
+
225
+ // Range is [start_running_count, end_running_count + 1) since range_cardinality is exclusive on the end
226
+ let count = bitmap.range_cardinality(start_running_count..(end_running_count + 1));
227
+
228
+ Some(count as usize)
229
+ }
230
+
231
+ #[deprecated(since = "0.1.0", note = "Use count_entries_in_time_range() instead")]
232
+ pub fn count_bitmap_entries_in_range(
233
+ &self,
234
+ bitmap: &Bitmap,
235
+ start_time: Seconds,
236
+ end_time: Seconds,
237
+ ) -> Option<usize> {
238
+ self.count_entries_in_time_range(bitmap, start_time, end_time)
239
+ }
240
+}
241
+
242
+#[cfg(test)]
243
+mod tests {
244
+ use super::*;
245
+
246
+ /// Helper to create a test histogram with known buckets
247
+ ///
248
+ /// Creates a histogram with:
249
+ /// - bucket_duration: 60 seconds
250
+ /// - Entries at indices 0-4 in bucket starting at time 0
251
+ /// - Entries at indices 5-9 in bucket starting at time 60
252
+ /// - Entries at indices 10-14 in bucket starting at time 120
253
+ /// - Entries at indices 15-19 in bucket starting at time 180
254
+ fn create_test_histogram() -> Histogram {
255
+ // Create 20 entries across 4 buckets (60 second buckets)
256
+ let pairs: Vec<(Microseconds, std::num::NonZeroU64)> = (0..20)
257
+ .map(|i| {
258
+ // Distribute entries: 0-4 -> [0,60), 5-9 -> [60,120), etc.
259
+ let bucket_index = i / 5;
260
+ let offset_in_bucket = i % 5;
261
+ // Spread entries within each bucket at 10 second intervals
262
+ let timestamp_secs = bucket_index * 60 + offset_in_bucket * 10;
263
+ (
264
+ Microseconds(timestamp_secs * 1_000_000),
265
+ std::num::NonZeroU64::new(i as u64 + 1).unwrap(),
266
+ )
267
+ })
268
+ .collect();
269
+
270
+ Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap()
271
+ }
272
+
273
+ // Tests for from_timestamp_offset_pairs construction
274
+
275
+ #[test]
276
+ fn test_from_timestamp_offset_pairs_single_entry() {
277
+ let pairs = vec![(
278
+ Seconds(1).to_microseconds(),
279
+ std::num::NonZeroU64::new(1).unwrap(),
280
+ )];
281
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap();
282
+
283
+ assert_eq!(histogram.bucket_duration.get(), 60);
284
+ assert_eq!(histogram.num_buckets(), 1);
285
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
286
+ assert_eq!(histogram.buckets[0].count, 0);
287
+ assert_eq!(histogram.total_entries(), 1);
288
+ }
289
+
290
+ #[test]
291
+ fn test_from_timestamp_offset_pairs_all_in_one_bucket() {
292
+ let pairs: Vec<_> = (0..5)
293
+ .map(|i| {
294
+ (
295
+ Seconds(i * 10).to_microseconds(),
296
+ std::num::NonZeroU64::new(i as u64 + 1).unwrap(),
297
+ )
298
+ })
299
+ .collect();
300
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap();
301
+
302
+ assert_eq!(histogram.num_buckets(), 1);
303
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
304
+ assert_eq!(histogram.buckets[0].count, 4);
305
+ assert_eq!(histogram.total_entries(), 5);
306
+ }
307
+
308
+ #[test]
309
+ fn test_from_timestamp_offset_pairs_exact_boundaries() {
310
+ // Test entries at exact bucket boundaries
311
+ let pairs = vec![
312
+ (Microseconds(0), std::num::NonZeroU64::new(1).unwrap()),
313
+ (
314
+ Seconds(60).to_microseconds(),
315
+ std::num::NonZeroU64::new(2).unwrap(),
316
+ ),
317
+ (
318
+ Seconds(120).to_microseconds(),
319
+ std::num::NonZeroU64::new(3).unwrap(),
320
+ ),
321
+ (
322
+ Seconds(180).to_microseconds(),
323
+ std::num::NonZeroU64::new(4).unwrap(),
324
+ ),
325
+ ];
326
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap();
327
+
328
+ assert_eq!(histogram.num_buckets(), 4);
329
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
330
+ assert_eq!(histogram.buckets[0].count, 0);
331
+ assert_eq!(histogram.buckets[1].start_time, Seconds(60));
332
+ assert_eq!(histogram.buckets[1].count, 1);
333
+ assert_eq!(histogram.buckets[2].start_time, Seconds(120));
334
+ assert_eq!(histogram.buckets[2].count, 2);
335
+ assert_eq!(histogram.buckets[3].start_time, Seconds(180));
336
+ assert_eq!(histogram.buckets[3].count, 3);
337
+ assert_eq!(histogram.total_entries(), 4);
338
+ }
339
+
340
+ #[test]
341
+ fn test_from_timestamp_offset_pairs_multiple_buckets() {
342
+ // 2 entries in first bucket, 3 in second bucket
343
+ let pairs = vec![
344
+ (
345
+ Seconds(10).to_microseconds(),
346
+ std::num::NonZeroU64::new(1).unwrap(),
347
+ ),
348
+ (
349
+ Seconds(20).to_microseconds(),
350
+ std::num::NonZeroU64::new(2).unwrap(),
351
+ ),
352
+ (
353
+ Seconds(70).to_microseconds(),
354
+ std::num::NonZeroU64::new(3).unwrap(),
355
+ ),
356
+ (
357
+ Seconds(80).to_microseconds(),
358
+ std::num::NonZeroU64::new(4).unwrap(),
359
+ ),
360
+ (
361
+ Seconds(90).to_microseconds(),
362
+ std::num::NonZeroU64::new(5).unwrap(),
363
+ ),
364
+ ];
365
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap();
366
+
367
+ assert_eq!(histogram.num_buckets(), 2);
368
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
369
+ assert_eq!(histogram.buckets[0].count, 1); // Entries 0-1
370
+ assert_eq!(histogram.buckets[1].start_time, Seconds(60));
371
+ assert_eq!(histogram.buckets[1].count, 4); // Entries 0-4
372
+ assert_eq!(histogram.total_entries(), 5);
373
+ }
374
+
375
+ #[test]
376
+ fn test_from_timestamp_offset_pairs_sparse_buckets() {
377
+ // Test with gaps between buckets
378
+ let pairs = vec![
379
+ (Microseconds(0), std::num::NonZeroU64::new(1).unwrap()),
380
+ (
381
+ Seconds(180).to_microseconds(),
382
+ std::num::NonZeroU64::new(2).unwrap(),
383
+ ), // Skip buckets 60 and 120
384
+ ];
385
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(60), &pairs).unwrap();
386
+
387
+ assert_eq!(histogram.num_buckets(), 2);
388
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
389
+ assert_eq!(histogram.buckets[0].count, 0);
390
+ assert_eq!(histogram.buckets[1].start_time, Seconds(180));
391
+ assert_eq!(histogram.buckets[1].count, 1);
392
+ assert_eq!(histogram.total_entries(), 2);
393
+ }
394
+
395
+ #[test]
396
+ fn test_from_timestamp_offset_pairs_large_bucket_duration() {
397
+ // Test with larger bucket duration
398
+ let pairs = vec![
399
+ (Microseconds(0), std::num::NonZeroU64::new(1).unwrap()),
400
+ (
401
+ Seconds(500).to_microseconds(),
402
+ std::num::NonZeroU64::new(2).unwrap(),
403
+ ),
404
+ (
405
+ Seconds(1000).to_microseconds(),
406
+ std::num::NonZeroU64::new(3).unwrap(),
407
+ ),
408
+ ];
409
+ let histogram = Histogram::from_timestamp_offset_pairs(Seconds(600), &pairs).unwrap();
410
+
411
+ assert_eq!(histogram.bucket_duration.get(), 600);
412
+ assert_eq!(histogram.num_buckets(), 2);
413
+ assert_eq!(histogram.buckets[0].start_time, Seconds(0));
414
+ assert_eq!(histogram.buckets[0].count, 1); // Entries 0-1 (0s and 500s both in [0, 600))
415
+ assert_eq!(histogram.buckets[1].start_time, Seconds(600));
416
+ assert_eq!(histogram.buckets[1].count, 2); // Entry 2 (1000s in [600, 1200))
417
+ assert_eq!(histogram.total_entries(), 3);
418
+ }
419
+
420
+ #[test]
421
+ fn test_from_timestamp_offset_pairs_zero_bucket_duration() {
422
+ let pairs = vec![(Microseconds(0), std::num::NonZeroU64::new(1).unwrap())];
423
+ let result = Histogram::from_timestamp_offset_pairs(Seconds(0), &pairs);
424
+ assert!(matches!(result, Err(IndexError::ZeroBucketDuration)));
425
+ }
426
+
427
+ #[test]
428
+ fn test_from_empty_timestamp_offset_pairs() {
429
+ let pairs = Vec::new();
430
+ let result = Histogram::from_timestamp_offset_pairs(Seconds(1), &pairs);
431
+ assert!(matches!(result, Err(IndexError::EmptyHistogramInput)));
432
+ }
433
+
434
+ #[test]
435
+ fn test_count_entries_in_time_range_full_bucket() {
436
+ let histogram = create_test_histogram();
437
+ // Bitmap contains entries 5, 6, 7, 8, 9 (all in bucket starting at 60)
438
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7, 8, 9]).unwrap();
439
+
440
+ // Query for the full bucket from 60 to 120
441
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(120));
442
+ assert_eq!(count, Some(5));
443
+ }
444
+
445
+ #[test]
446
+ fn test_count_entries_in_time_range_partial_match() {
447
+ let histogram = create_test_histogram();
448
+ // Bitmap contains some entries in bucket 60-120 and some in 120-180
449
+ let bitmap = Bitmap::from_sorted_iter([7, 8, 9, 10, 11]).unwrap();
450
+
451
+ // Query for bucket 60-120 should only count entries 7, 8, 9
452
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(120));
453
+ assert_eq!(count, Some(3));
454
+ }
455
+
456
+ #[test]
457
+ fn test_count_entries_in_time_range_multiple_buckets() {
458
+ let histogram = create_test_histogram();
459
+ // Bitmap spans multiple buckets
460
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 10, 11, 15, 16]).unwrap();
461
+
462
+ // Query for buckets 60-180 (includes buckets at 60 and 120)
463
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(180));
464
+ assert_eq!(count, Some(4)); // 5, 6, 10, 11
465
+ }
466
+
467
+ #[test]
468
+ fn test_count_entries_in_time_range_no_matches() {
469
+ let histogram = create_test_histogram();
470
+ // Bitmap contains entries in bucket 0-60
471
+ let bitmap = Bitmap::from_sorted_iter([0, 1, 2]).unwrap();
472
+
473
+ // Query for bucket 120-180 should find no matches
474
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(120), Seconds(180));
475
+ assert_eq!(count, Some(0));
476
+ }
477
+
478
+ #[test]
479
+ fn test_count_entries_in_time_range_empty_bitmap() {
480
+ let histogram = create_test_histogram();
481
+ let bitmap = Bitmap::new();
482
+
483
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(0), Seconds(60));
484
+ assert_eq!(count, Some(0));
485
+ }
486
+
487
+ #[test]
488
+ fn test_count_entries_in_time_range_unaligned_start() {
489
+ let histogram = create_test_histogram();
490
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7]).unwrap();
491
+
492
+ // Start time not aligned to bucket_duration (60)
493
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(30), Seconds(120));
494
+ assert_eq!(count, None);
495
+ }
496
+
497
+ #[test]
498
+ fn test_count_entries_in_time_range_unaligned_end() {
499
+ let histogram = create_test_histogram();
500
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7]).unwrap();
501
+
502
+ // End time not aligned to bucket_duration (60)
503
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(100));
504
+ assert_eq!(count, None);
505
+ }
506
+
507
+ #[test]
508
+ fn test_count_entries_in_time_range_invalid_range() {
509
+ let histogram = create_test_histogram();
510
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7]).unwrap();
511
+
512
+ // start >= end
513
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(120), Seconds(60));
514
+ assert_eq!(count, None);
515
+
516
+ // start == end
517
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(60));
518
+ assert_eq!(count, None);
519
+ }
520
+
521
+ #[test]
522
+ fn test_count_entries_in_time_range_outside_histogram() {
523
+ let histogram = create_test_histogram();
524
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7]).unwrap();
525
+
526
+ // Range completely before histogram
527
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(0), Seconds(60));
528
+ // This will actually work since 0-60 is the first bucket
529
+ assert!(count.is_some());
530
+
531
+ // Range completely after histogram (histogram ends at 240)
532
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(240), Seconds(300));
533
+ assert_eq!(count, Some(0));
534
+ }
535
+
536
+ #[test]
537
+ fn test_count_entries_in_time_range_first_bucket() {
538
+ let histogram = create_test_histogram();
539
+ // Entries in first bucket (0-60)
540
+ let bitmap = Bitmap::from_sorted_iter([0, 1, 2, 3, 4]).unwrap();
541
+
542
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(0), Seconds(60));
543
+ assert_eq!(count, Some(5));
544
+ }
545
+
546
+ #[test]
547
+ fn test_count_entries_in_time_range_last_bucket() {
548
+ let histogram = create_test_histogram();
549
+ // Entries in last bucket (180-240)
550
+ let bitmap = Bitmap::from_sorted_iter([15, 16, 17, 18, 19]).unwrap();
551
+
552
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(180), Seconds(240));
553
+ assert_eq!(count, Some(5));
554
+ }
555
+
556
+ #[test]
557
+ fn test_count_entries_in_time_range_all_buckets() {
558
+ let histogram = create_test_histogram();
559
+ // Entries spanning all buckets
560
+ let bitmap = Bitmap::from_sorted_iter([0, 5, 10, 15]).unwrap();
561
+
562
+ // Query for entire histogram range
563
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(0), Seconds(240));
564
+ assert_eq!(count, Some(4));
565
+ }
566
+
567
+ #[test]
568
+ fn test_histogram_properties() {
569
+ let histogram = create_test_histogram();
570
+
571
+ assert_eq!(histogram.start_time(), Seconds(0));
572
+ assert_eq!(histogram.end_time(), Seconds(240));
573
+ assert_eq!(histogram.time_range(), (Seconds(0), Seconds(240)));
574
+ assert_eq!(histogram.num_buckets(), 4);
575
+ assert!(!histogram.is_empty());
576
+ assert_eq!(histogram.total_entries(), 20);
577
+ }
578
+
579
+ // Bitmap edge case tests
580
+
581
+ #[test]
582
+ fn test_bitmap_with_indices_beyond_histogram_range() {
583
+ let histogram = create_test_histogram();
584
+ // Histogram has entries 0-19, bitmap has indices beyond that
585
+ let bitmap = Bitmap::from_sorted_iter([5, 6, 7, 25, 30, 100]).unwrap();
586
+
587
+ // Query bucket 60-120 (entries 5-9)
588
+ // Should only count the valid indices (5, 6, 7) that fall in range
589
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(120));
590
+ assert_eq!(count, Some(3));
591
+ }
592
+
593
+ #[test]
594
+ fn test_bitmap_all_indices_outside_queried_range() {
595
+ let histogram = create_test_histogram();
596
+ // Bitmap has valid indices but none in the queried range
597
+ let bitmap = Bitmap::from_sorted_iter([0, 1, 2, 3, 4]).unwrap();
598
+
599
+ // Query bucket 120-180 (entries 10-14)
600
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(120), Seconds(180));
601
+ assert_eq!(count, Some(0));
602
+ }
603
+
604
+ #[test]
605
+ fn test_bitmap_with_sparse_scattered_indices() {
606
+ let histogram = create_test_histogram();
607
+ // Very sparse bitmap with indices scattered across all buckets
608
+ let bitmap = Bitmap::from_sorted_iter([1, 7, 11, 18]).unwrap();
609
+
610
+ // Query middle buckets 60-180 (entries 5-14)
611
+ // Should count indices 7 and 11
612
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(180));
613
+ assert_eq!(count, Some(2));
614
+ }
615
+
616
+ #[test]
617
+ fn test_bitmap_at_range_boundaries() {
618
+ let histogram = create_test_histogram();
619
+ // Bitmap with entries exactly at the boundaries of the queried range
620
+ let bitmap = Bitmap::from_sorted_iter([4, 5, 9, 10]).unwrap();
621
+
622
+ // Query bucket 60-120 (entries 5-9)
623
+ // Should include 5, 9 but not 4 (in previous bucket) or 10 (in next bucket)
624
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(120));
625
+ assert_eq!(count, Some(2));
626
+ }
627
+
628
+ #[test]
629
+ fn test_bitmap_with_only_out_of_range_indices() {
630
+ let histogram = create_test_histogram();
631
+ // Bitmap with indices all beyond the histogram's range
632
+ let bitmap = Bitmap::from_sorted_iter([25, 30, 50, 100]).unwrap();
633
+
634
+ // Query any valid range
635
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(0), Seconds(60));
636
+ assert_eq!(count, Some(0));
637
+ }
638
+
639
+ #[test]
640
+ fn test_bitmap_single_index_in_range() {
641
+ let histogram = create_test_histogram();
642
+ // Bitmap with many indices but only one in the queried range
643
+ let bitmap = Bitmap::from_sorted_iter([0, 1, 2, 7, 15, 16, 17]).unwrap();
644
+
645
+ // Query bucket 60-120 (entries 5-9), only index 7 matches
646
+ let count = histogram.count_entries_in_time_range(&bitmap, Seconds(60), Seconds(120));
647
+ assert_eq!(count, Some(1));
648
+ }
649
+}
src/crates/journal-index/src/lib.rs
new
+32
@@ -0,0 +1,32 @@
1
+//! Indexing functionality for systemd journal files.
2
+//!
3
+//! This crate provides:
4
+//! - Histograms for time-based aggregation
5
+//! - File indexing for fast lookups
6
+//! - Bitmap-based filtering
7
+//! - Field type definitions
8
+
9
+pub use journal_common::{Microseconds, Seconds};
10
+
11
+pub mod error;
12
+pub use error::{IndexError, Result};
13
+
14
+pub mod histogram;
15
+pub use histogram::{Bucket, Histogram};
16
+
17
+pub mod file_index;
18
+pub use file_index::{
19
+ Anchor, Direction, FileIndex, LogEntryId, LogQueryParams, LogQueryParamsBuilder,
20
+};
21
+
22
+pub mod file_indexer;
23
+pub use file_indexer::FileIndexer;
24
+
25
+pub mod bitmap;
26
+pub use bitmap::Bitmap;
27
+
28
+pub mod filter;
29
+pub use filter::Filter;
30
+
31
+pub mod field_types;
32
+pub use field_types::{FieldName, FieldValuePair};
src/crates/journal-index/tests/filter_evaluation.rs
new
+555
@@ -0,0 +1,555 @@
1
+//! Integration tests for filter evaluation.
2
+//!
3
+//! These tests create actual journal files, index them, and verify that
4
+//! filter evaluation produces correct results.
5
+
6
+use journal_common::Seconds;
7
+use journal_core::file::{JournalFile, JournalFileOptions, JournalWriter};
8
+use journal_core::repository::File;
9
+use journal_index::{FieldName, FieldValuePair, FileIndexer, Filter, Microseconds};
10
+use std::fs;
11
+use std::path::PathBuf;
12
+use tempfile::TempDir;
13
+use uuid::Uuid;
14
+
15
+// Helper constants and functions for creating readable timestamps
16
+const JAN_1_2024_MIDNIGHT: Microseconds = Microseconds(1704067200_000_000);
17
+
18
+fn hours(n: u64) -> Microseconds {
19
+ Microseconds(n * 3600_000_000)
20
+}
21
+
22
+fn add_time(base: Microseconds, offset: Microseconds) -> Microseconds {
23
+ Microseconds(base.0 + offset.0)
24
+}
25
+
26
+/// Test journal entry specification
27
+struct TestEntry {
28
+ timestamp: Microseconds,
29
+ fields: Vec<(String, String)>,
30
+}
31
+
32
+impl TestEntry {
33
+ fn new(timestamp: Microseconds) -> Self {
34
+ Self {
35
+ timestamp,
36
+ fields: Vec::new(),
37
+ }
38
+ }
39
+
40
+ fn with_field(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
41
+ self.fields.push((name.into(), value.into()));
42
+ self
43
+ }
44
+}
45
+
46
+/// Create a test journal file path that conforms to the expected format
47
+fn create_test_journal_path(temp_dir: &TempDir) -> PathBuf {
48
+ // Create a machine ID subdirectory
49
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
50
+ let machine_dir = temp_dir.path().join(machine_id.to_string());
51
+ fs::create_dir_all(&machine_dir).expect("create machine dir");
52
+
53
+ // Create journal file path in the format: <dir>/<machine_id>/system.journal
54
+ machine_dir.join("system.journal")
55
+}
56
+
57
+/// Helper to create a test journal file with specified entries
58
+fn create_test_journal(
59
+ entries: Vec<TestEntry>,
60
+) -> Result<(TempDir, File), Box<dyn std::error::Error>> {
61
+ let temp_dir = TempDir::new()?;
62
+ let journal_path = create_test_journal_path(&temp_dir);
63
+
64
+ // Create a File object from the path
65
+ let file =
66
+ File::from_path(&journal_path).ok_or("Failed to create repository File from path")?;
67
+
68
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
69
+ let boot_id = Uuid::from_u128(0x11111111_1111_1111_1111_111111111111);
70
+ let seqnum_id = Uuid::from_u128(0x22222222_2222_2222_2222_222222222222);
71
+
72
+ let options = JournalFileOptions::new(machine_id, boot_id, seqnum_id);
73
+
74
+ let mut journal_file = JournalFile::create(&file, options)?;
75
+ let mut writer = JournalWriter::new(&mut journal_file, 1, boot_id)?;
76
+
77
+ for entry in entries {
78
+ let mut entry_data = Vec::new();
79
+
80
+ // Add _SOURCE_REALTIME_TIMESTAMP first
81
+ entry_data.push(format!("_SOURCE_REALTIME_TIMESTAMP={}", entry.timestamp.0).into_bytes());
82
+
83
+ // Add all other fields
84
+ for (field, value) in entry.fields {
85
+ entry_data.push(format!("{}={}", field, value).into_bytes());
86
+ }
87
+
88
+ let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
89
+
90
+ writer.add_entry(
91
+ &mut journal_file,
92
+ &entry_refs,
93
+ entry.timestamp.0,
94
+ entry.timestamp.0,
95
+ )?;
96
+ }
97
+
98
+ // Return TempDir to keep it alive and the File for opening
99
+ Ok((temp_dir, file))
100
+}
101
+
102
+#[test]
103
+fn test_filter_field_value_pair_single_match() {
104
+ // Create a journal with known entries
105
+ let entries = vec![
106
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3"),
107
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1))).with_field("PRIORITY", "6"),
108
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2))).with_field("PRIORITY", "3"),
109
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(3))).with_field("PRIORITY", "7"),
110
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(4))).with_field("PRIORITY", "3"),
111
+ ];
112
+
113
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
114
+
115
+ let mut indexer = FileIndexer::default();
116
+
117
+ let priority_field = FieldName::new("PRIORITY").unwrap();
118
+ let file_index = indexer
119
+ .index(&file, None, &[priority_field], Seconds(3600))
120
+ .unwrap();
121
+
122
+ // Create and evaluate filter for PRIORITY=3
123
+ let pair = FieldValuePair::parse("PRIORITY=3").unwrap();
124
+ let filter = Filter::match_field_value_pair(pair);
125
+ let bitmap = filter.evaluate(&file_index);
126
+
127
+ // Should match entries 0, 2, and 4
128
+ assert_eq!(bitmap.len(), 3);
129
+ assert!(bitmap.contains(0));
130
+ assert!(bitmap.contains(2));
131
+ assert!(bitmap.contains(4));
132
+ assert!(!bitmap.contains(1));
133
+ assert!(!bitmap.contains(3));
134
+}
135
+
136
+#[test]
137
+fn test_filter_field_name_matches_all_values() {
138
+ let entries = vec![
139
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3"),
140
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1))).with_field("PRIORITY", "6"),
141
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2))).with_field("PRIORITY", "7"),
142
+ ];
143
+
144
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
145
+
146
+ let mut indexer = FileIndexer::default();
147
+
148
+ let priority_field = FieldName::new("PRIORITY").unwrap();
149
+ let file_index = indexer
150
+ .index(&file, None, &[priority_field.clone()], Seconds(3600))
151
+ .unwrap();
152
+
153
+ // Create filter that matches any PRIORITY field
154
+ let filter = Filter::match_field_name(priority_field);
155
+ let bitmap = filter.evaluate(&file_index);
156
+
157
+ // Should match all entries (0, 1, 2)
158
+ assert_eq!(bitmap.len(), 3);
159
+ assert!(bitmap.contains(0));
160
+ assert!(bitmap.contains(1));
161
+ assert!(bitmap.contains(2));
162
+}
163
+
164
+#[test]
165
+fn test_filter_and_combination() {
166
+ let entries = vec![
167
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0)))
168
+ .with_field("PRIORITY", "3")
169
+ .with_field("_HOSTNAME", "server1"),
170
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1)))
171
+ .with_field("PRIORITY", "6")
172
+ .with_field("_HOSTNAME", "server1"),
173
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2)))
174
+ .with_field("PRIORITY", "3")
175
+ .with_field("_HOSTNAME", "server2"),
176
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(3)))
177
+ .with_field("PRIORITY", "3")
178
+ .with_field("_HOSTNAME", "server1"),
179
+ ];
180
+
181
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
182
+
183
+ let mut indexer = FileIndexer::default();
184
+
185
+ let priority_field = FieldName::new("PRIORITY").unwrap();
186
+ let hostname_field = FieldName::new("_HOSTNAME").unwrap();
187
+ let file_index = indexer
188
+ .index(
189
+ &file,
190
+ None,
191
+ &[priority_field, hostname_field],
192
+ Seconds(3600),
193
+ )
194
+ .unwrap();
195
+
196
+ // Filter: PRIORITY=3 AND _HOSTNAME=server1
197
+ let filter = Filter::and(vec![
198
+ Filter::match_field_value_pair(FieldValuePair::parse("PRIORITY=3").unwrap()),
199
+ Filter::match_field_value_pair(FieldValuePair::parse("_HOSTNAME=server1").unwrap()),
200
+ ]);
201
+
202
+ let bitmap = filter.evaluate(&file_index);
203
+
204
+ // Should match entries 0 and 3 (both have PRIORITY=3 AND _HOSTNAME=server1)
205
+ assert_eq!(bitmap.len(), 2);
206
+ assert!(bitmap.contains(0));
207
+ assert!(bitmap.contains(3));
208
+ assert!(!bitmap.contains(1)); // Has server1 but PRIORITY=6
209
+ assert!(!bitmap.contains(2)); // Has PRIORITY=3 but server2
210
+}
211
+
212
+#[test]
213
+fn test_filter_or_combination() {
214
+ let entries = vec![
215
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3"),
216
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1))).with_field("PRIORITY", "6"),
217
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2))).with_field("PRIORITY", "3"),
218
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(3))).with_field("PRIORITY", "7"),
219
+ ];
220
+
221
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
222
+
223
+ let mut indexer = FileIndexer::default();
224
+
225
+ let priority_field = FieldName::new("PRIORITY").unwrap();
226
+ let file_index = indexer
227
+ .index(&file, None, &[priority_field], Seconds(3600))
228
+ .unwrap();
229
+
230
+ // Filter: PRIORITY=3 OR PRIORITY=6
231
+ let filter = Filter::or(vec![
232
+ Filter::match_field_value_pair(FieldValuePair::parse("PRIORITY=3").unwrap()),
233
+ Filter::match_field_value_pair(FieldValuePair::parse("PRIORITY=6").unwrap()),
234
+ ]);
235
+
236
+ let bitmap = filter.evaluate(&file_index);
237
+
238
+ // Should match entries 0, 1, and 2
239
+ assert_eq!(bitmap.len(), 3);
240
+ assert!(bitmap.contains(0));
241
+ assert!(bitmap.contains(1));
242
+ assert!(bitmap.contains(2));
243
+ assert!(!bitmap.contains(3)); // PRIORITY=7
244
+}
245
+
246
+#[test]
247
+fn test_filter_none() {
248
+ let entries =
249
+ vec![TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3")];
250
+
251
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
252
+
253
+ let mut indexer = FileIndexer::default();
254
+
255
+ let priority_field = FieldName::new("PRIORITY").unwrap();
256
+ let file_index = indexer
257
+ .index(&file, None, &[priority_field], Seconds(3600))
258
+ .unwrap();
259
+
260
+ // Create a None filter
261
+ let filter = Filter::none();
262
+ let bitmap = filter.evaluate(&file_index);
263
+
264
+ // Should match nothing
265
+ assert_eq!(bitmap.len(), 0);
266
+ assert!(filter.is_none());
267
+}
268
+
269
+#[test]
270
+fn test_filter_nonexistent_field() {
271
+ let entries =
272
+ vec![TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3")];
273
+
274
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
275
+
276
+ let mut indexer = FileIndexer::default();
277
+
278
+ let priority_field = FieldName::new("PRIORITY").unwrap();
279
+ let file_index = indexer
280
+ .index(&file, None, &[priority_field], Seconds(3600))
281
+ .unwrap();
282
+
283
+ // Filter for a field that wasn't indexed
284
+ let filter =
285
+ Filter::match_field_value_pair(FieldValuePair::parse("NONEXISTENT_FIELD=value").unwrap());
286
+ let bitmap = filter.evaluate(&file_index);
287
+
288
+ // Should match nothing
289
+ assert_eq!(bitmap.len(), 0);
290
+}
291
+
292
+#[test]
293
+fn test_filter_complex_nested() {
294
+ let entries = vec![
295
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0)))
296
+ .with_field("PRIORITY", "3")
297
+ .with_field("_HOSTNAME", "server1")
298
+ .with_field("SYSLOG_IDENTIFIER", "systemd"),
299
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1)))
300
+ .with_field("PRIORITY", "6")
301
+ .with_field("_HOSTNAME", "server1")
302
+ .with_field("SYSLOG_IDENTIFIER", "kernel"),
303
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2)))
304
+ .with_field("PRIORITY", "3")
305
+ .with_field("_HOSTNAME", "server2")
306
+ .with_field("SYSLOG_IDENTIFIER", "systemd"),
307
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(3)))
308
+ .with_field("PRIORITY", "3")
309
+ .with_field("_HOSTNAME", "server1")
310
+ .with_field("SYSLOG_IDENTIFIER", "kernel"),
311
+ ];
312
+
313
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
314
+
315
+ let mut indexer = FileIndexer::default();
316
+
317
+ let fields = vec![
318
+ FieldName::new("PRIORITY").unwrap(),
319
+ FieldName::new("_HOSTNAME").unwrap(),
320
+ FieldName::new("SYSLOG_IDENTIFIER").unwrap(),
321
+ ];
322
+ let file_index = indexer.index(&file, None, &fields, Seconds(3600)).unwrap();
323
+
324
+ // Complex filter: (PRIORITY=3 AND _HOSTNAME=server1) OR SYSLOG_IDENTIFIER=kernel
325
+ let filter = Filter::or(vec![
326
+ Filter::and(vec![
327
+ Filter::match_field_value_pair(FieldValuePair::parse("PRIORITY=3").unwrap()),
328
+ Filter::match_field_value_pair(FieldValuePair::parse("_HOSTNAME=server1").unwrap()),
329
+ ]),
330
+ Filter::match_field_value_pair(FieldValuePair::parse("SYSLOG_IDENTIFIER=kernel").unwrap()),
331
+ ]);
332
+
333
+ let bitmap = filter.evaluate(&file_index);
334
+
335
+ // Should match:
336
+ // - Entry 0: PRIORITY=3 AND _HOSTNAME=server1
337
+ // - Entry 1: SYSLOG_IDENTIFIER=kernel
338
+ // - Entry 3: PRIORITY=3 AND _HOSTNAME=server1
339
+ assert_eq!(bitmap.len(), 3);
340
+ assert!(bitmap.contains(0));
341
+ assert!(bitmap.contains(1));
342
+ assert!(!bitmap.contains(2));
343
+ assert!(bitmap.contains(3));
344
+}
345
+
346
+#[test]
347
+fn test_filter_empty_and() {
348
+ let entries =
349
+ vec![TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3")];
350
+
351
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
352
+
353
+ let mut indexer = FileIndexer::default();
354
+
355
+ let priority_field = FieldName::new("PRIORITY").unwrap();
356
+ let file_index = indexer
357
+ .index(&file, None, &[priority_field], Seconds(3600))
358
+ .unwrap();
359
+
360
+ // Empty AND should produce a None filter
361
+ let filter = Filter::and(vec![]);
362
+ let bitmap = filter.evaluate(&file_index);
363
+
364
+ assert_eq!(bitmap.len(), 0);
365
+ assert!(filter.is_none());
366
+}
367
+
368
+#[test]
369
+fn test_filter_empty_or() {
370
+ let entries =
371
+ vec![TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3")];
372
+
373
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
374
+
375
+ let mut indexer = FileIndexer::default();
376
+
377
+ let priority_field = FieldName::new("PRIORITY").unwrap();
378
+ let file_index = indexer
379
+ .index(&file, None, &[priority_field], Seconds(3600))
380
+ .unwrap();
381
+
382
+ // Empty OR should produce a None filter
383
+ let filter = Filter::or(vec![]);
384
+ let bitmap = filter.evaluate(&file_index);
385
+
386
+ assert_eq!(bitmap.len(), 0);
387
+ assert!(filter.is_none());
388
+}
389
+
390
+#[test]
391
+fn test_file_index_metadata() {
392
+ let entries = vec![
393
+ TestEntry::new(JAN_1_2024_MIDNIGHT)
394
+ .with_field("PRIORITY", "3")
395
+ .with_field("_HOSTNAME", "server1"),
396
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1)))
397
+ .with_field("PRIORITY", "6")
398
+ .with_field("_HOSTNAME", "server1")
399
+ .with_field("MESSAGE", "test message"),
400
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2)))
401
+ .with_field("PRIORITY", "3")
402
+ .with_field("MESSAGE", "another message"),
403
+ ];
404
+
405
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
406
+
407
+ let mut indexer = FileIndexer::default();
408
+
409
+ let priority_field = FieldName::new("PRIORITY").unwrap();
410
+ let hostname_field = FieldName::new("_HOSTNAME").unwrap();
411
+ let file_index = indexer
412
+ .index(
413
+ &file,
414
+ None,
415
+ &[priority_field.clone(), hostname_field.clone()],
416
+ Seconds(3600),
417
+ )
418
+ .unwrap();
419
+
420
+ // Verify file reference
421
+ assert_eq!(file_index.file(), &file);
422
+
423
+ // Verify time range (stored in seconds, with 1-hour bucket duration)
424
+ // Start time should be rounded down to the nearest bucket
425
+ assert_eq!(file_index.start_time().0, 1704067200); // Exact start
426
+ // End time is start of last bucket + bucket_duration
427
+ assert_eq!(file_index.end_time().0, 1704074400 + 3600); // 2 hours + bucket size
428
+
429
+ // Verify file fields (all fields present in the journal)
430
+ let file_fields = file_index.fields();
431
+ assert!(file_fields.contains(&FieldName::new("PRIORITY").unwrap()));
432
+ assert!(file_fields.contains(&FieldName::new("_HOSTNAME").unwrap()));
433
+ assert!(file_fields.contains(&FieldName::new("MESSAGE").unwrap()));
434
+ assert!(file_fields.contains(&FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap()));
435
+
436
+ // Verify indexed fields (only fields we asked to index)
437
+ assert!(file_index.is_indexed(&priority_field));
438
+ assert!(file_index.is_indexed(&hostname_field));
439
+ assert!(!file_index.is_indexed(&FieldName::new("MESSAGE").unwrap()));
440
+
441
+ // Verify entry count
442
+ assert_eq!(file_index.total_entries(), 3);
443
+
444
+ // Verify bitmaps exist for indexed field values
445
+ let bitmaps = file_index.bitmaps();
446
+ assert!(bitmaps.contains_key(&FieldValuePair::parse("PRIORITY=3").unwrap()));
447
+ assert!(bitmaps.contains_key(&FieldValuePair::parse("PRIORITY=6").unwrap()));
448
+ assert!(bitmaps.contains_key(&FieldValuePair::parse("_HOSTNAME=server1").unwrap()));
449
+
450
+ // MESSAGE field values should not be indexed
451
+ assert!(!bitmaps.contains_key(&FieldValuePair::parse("MESSAGE=test message").unwrap()));
452
+}
453
+
454
+#[test]
455
+fn test_source_timestamp_ordering() {
456
+ // Create entries where source timestamp ordering differs from creation order
457
+ let entries = vec![
458
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(3))).with_field("PRIORITY", "3"), // +3 hours (created first, but should be ordered last)
459
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1))).with_field("PRIORITY", "6"), // +1 hour (created second, but should be ordered first)
460
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2))).with_field("PRIORITY", "7"), // +2 hours (created third, but should be ordered middle)
461
+ ];
462
+
463
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
464
+
465
+ let mut indexer = FileIndexer::default();
466
+
467
+ let priority_field = FieldName::new("PRIORITY").unwrap();
468
+ let source_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
469
+
470
+ // Index WITH source timestamp field - entries should be ordered by source time
471
+ let file_index = indexer
472
+ .index(&file, Some(&source_field), &[priority_field], Seconds(3600))
473
+ .unwrap();
474
+
475
+ // After indexing with source timestamp, entries should be reordered:
476
+ // Index 0: +1 hour (original entry 1) - PRIORITY=6
477
+ // Index 1: +2 hours (original entry 2) - PRIORITY=7
478
+ // Index 2: +3 hours (original entry 0) - PRIORITY=3
479
+
480
+ // Verify time range reflects source timestamp order (in seconds)
481
+ let expected_start = (add_time(JAN_1_2024_MIDNIGHT, hours(1)).0 / 1_000_000) as u32; // +1 hour in seconds
482
+ let expected_end = ((add_time(JAN_1_2024_MIDNIGHT, hours(3)).0 / 1_000_000) + 3600) as u32; // +3 hours + bucket duration
483
+ assert_eq!(file_index.start_time().0, expected_start);
484
+ assert_eq!(file_index.end_time().0, expected_end);
485
+
486
+ // Verify bitmaps reflect the new ordering
487
+ let bitmaps = file_index.bitmaps();
488
+
489
+ // PRIORITY=6 should be at index 0 (entry with source_time=1_000_000)
490
+ let priority_6_bitmap = bitmaps
491
+ .get(&FieldValuePair::parse("PRIORITY=6").unwrap())
492
+ .unwrap();
493
+ assert_eq!(priority_6_bitmap.len(), 1);
494
+ assert!(priority_6_bitmap.contains(0));
495
+
496
+ // PRIORITY=7 should be at index 1 (entry with source_time=2_000_000)
497
+ let priority_7_bitmap = bitmaps
498
+ .get(&FieldValuePair::parse("PRIORITY=7").unwrap())
499
+ .unwrap();
500
+ assert_eq!(priority_7_bitmap.len(), 1);
501
+ assert!(priority_7_bitmap.contains(1));
502
+
503
+ // PRIORITY=3 should be at index 2 (entry with source_time=3_000_000)
504
+ let priority_3_bitmap = bitmaps
505
+ .get(&FieldValuePair::parse("PRIORITY=3").unwrap())
506
+ .unwrap();
507
+ assert_eq!(priority_3_bitmap.len(), 1);
508
+ assert!(priority_3_bitmap.contains(2));
509
+}
510
+
511
+#[test]
512
+fn test_indexing_without_source_timestamp() {
513
+ // Create entries without specifying source timestamp field
514
+ // They should be ordered by the journal's realtime timestamp instead
515
+ let entries = vec![
516
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(0))).with_field("PRIORITY", "3"),
517
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(1))).with_field("PRIORITY", "6"),
518
+ TestEntry::new(add_time(JAN_1_2024_MIDNIGHT, hours(2))).with_field("PRIORITY", "7"),
519
+ ];
520
+
521
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
522
+
523
+ let mut indexer = FileIndexer::default();
524
+
525
+ let priority_field = FieldName::new("PRIORITY").unwrap();
526
+
527
+ // Index WITHOUT source timestamp field (None)
528
+ let file_index = indexer
529
+ .index(&file, None, &[priority_field], Seconds(3600))
530
+ .unwrap();
531
+
532
+ // Verify entries maintain their natural order
533
+ let bitmaps = file_index.bitmaps();
534
+
535
+ // PRIORITY=3 should be at index 0
536
+ let priority_3_bitmap = bitmaps
537
+ .get(&FieldValuePair::parse("PRIORITY=3").unwrap())
538
+ .unwrap();
539
+ assert_eq!(priority_3_bitmap.len(), 1);
540
+ assert!(priority_3_bitmap.contains(0));
541
+
542
+ // PRIORITY=6 should be at index 1
543
+ let priority_6_bitmap = bitmaps
544
+ .get(&FieldValuePair::parse("PRIORITY=6").unwrap())
545
+ .unwrap();
546
+ assert_eq!(priority_6_bitmap.len(), 1);
547
+ assert!(priority_6_bitmap.contains(1));
548
+
549
+ // PRIORITY=7 should be at index 2
550
+ let priority_7_bitmap = bitmaps
551
+ .get(&FieldValuePair::parse("PRIORITY=7").unwrap())
552
+ .unwrap();
553
+ assert_eq!(priority_7_bitmap.len(), 1);
554
+ assert!(priority_7_bitmap.contains(2));
555
+}
src/crates/journal-index/tests/pagination.rs
new
+1169
@@ -0,0 +1,1169 @@
1
+//! Integration tests for query pagination.
2
+//!
3
+//! These tests verify that pagination works correctly when querying log entries,
4
+//! especially in the edge case where many entries share the same timestamp.
5
+
6
+use journal_common::Seconds;
7
+use journal_core::file::{JournalFile, JournalFileOptions, JournalWriter};
8
+use journal_core::repository::File;
9
+use journal_index::{
10
+ Anchor, Direction, FieldName, FileIndexer, LogQueryParamsBuilder, Microseconds,
11
+};
12
+use std::collections::HashSet;
13
+use std::fs;
14
+use std::path::PathBuf;
15
+use tempfile::TempDir;
16
+use uuid::Uuid;
17
+
18
+// Helper constants
19
+const JAN_1_2024_MIDNIGHT: Microseconds = Microseconds(1704067200_000_000);
20
+
21
+/// Test journal entry specification
22
+struct TestEntry {
23
+ timestamp: Microseconds,
24
+ fields: Vec<(String, String)>,
25
+}
26
+
27
+impl TestEntry {
28
+ fn new(timestamp: Microseconds) -> Self {
29
+ Self {
30
+ timestamp,
31
+ fields: Vec::new(),
32
+ }
33
+ }
34
+
35
+ fn with_field(mut self, name: impl Into<String>, value: impl Into<String>) -> Self {
36
+ self.fields.push((name.into(), value.into()));
37
+ self
38
+ }
39
+}
40
+
41
+/// Create a test journal file path that conforms to the expected format
42
+fn create_test_journal_path(temp_dir: &TempDir) -> PathBuf {
43
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
44
+ let machine_dir = temp_dir.path().join(machine_id.to_string());
45
+ fs::create_dir_all(&machine_dir).expect("create machine dir");
46
+ machine_dir.join("system.journal")
47
+}
48
+
49
+/// Helper to create a test journal file with specified entries
50
+fn create_test_journal(
51
+ entries: Vec<TestEntry>,
52
+) -> Result<(TempDir, File), Box<dyn std::error::Error>> {
53
+ let temp_dir = TempDir::new()?;
54
+ let journal_path = create_test_journal_path(&temp_dir);
55
+
56
+ let file =
57
+ File::from_path(&journal_path).ok_or("Failed to create repository File from path")?;
58
+
59
+ let machine_id = Uuid::from_u128(0x12345678_1234_1234_1234_123456789abc);
60
+ let boot_id = Uuid::from_u128(0x11111111_1111_1111_1111_111111111111);
61
+ let seqnum_id = Uuid::from_u128(0x22222222_2222_2222_2222_222222222222);
62
+
63
+ let options = JournalFileOptions::new(machine_id, boot_id, seqnum_id);
64
+
65
+ let mut journal_file = JournalFile::create(&file, options)?;
66
+ let mut writer = JournalWriter::new(&mut journal_file, 1, boot_id)?;
67
+
68
+ for entry in entries {
69
+ let mut entry_data = Vec::new();
70
+
71
+ // Add _SOURCE_REALTIME_TIMESTAMP first
72
+ entry_data.push(format!("_SOURCE_REALTIME_TIMESTAMP={}", entry.timestamp.0).into_bytes());
73
+
74
+ // Add all other fields
75
+ for (field, value) in entry.fields {
76
+ entry_data.push(format!("{}={}", field, value).into_bytes());
77
+ }
78
+
79
+ let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
80
+
81
+ writer.add_entry(
82
+ &mut journal_file,
83
+ &entry_refs,
84
+ entry.timestamp.0,
85
+ entry.timestamp.0,
86
+ )?;
87
+ }
88
+
89
+ Ok((temp_dir, file))
90
+}
91
+
92
+#[test]
93
+fn test_pagination_forward_with_same_timestamps() {
94
+ // Create 300 entries all with the same timestamp
95
+ const TOTAL_ENTRIES: usize = 300;
96
+ const PAGE_SIZE: usize = 200;
97
+ let same_timestamp = JAN_1_2024_MIDNIGHT;
98
+
99
+ let entries: Vec<TestEntry> = (0..TOTAL_ENTRIES)
100
+ .map(|i| {
101
+ TestEntry::new(same_timestamp)
102
+ .with_field("MESSAGE", format!("Entry {}", i))
103
+ .with_field("ENTRY_ID", i.to_string())
104
+ })
105
+ .collect();
106
+
107
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
108
+
109
+ let mut indexer = FileIndexer::default();
110
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
111
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
112
+ let file_index = indexer
113
+ .index(
114
+ &file,
115
+ Some(&source_timestamp_field),
116
+ &[entry_id_field],
117
+ Seconds(3600),
118
+ )
119
+ .unwrap();
120
+
121
+ let mut all_offsets = Vec::new();
122
+ let mut all_positions = HashSet::new();
123
+ let mut resume_position = None;
124
+
125
+ // First page
126
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
127
+ .with_limit(PAGE_SIZE)
128
+ .build()
129
+ .unwrap();
130
+
131
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
132
+ println!("First page: {} entries", results.len());
133
+ assert_eq!(results.len(), PAGE_SIZE, "First page should return PAGE_SIZE entries");
134
+
135
+ // Verify all have the same timestamp
136
+ for entry in &results {
137
+ assert_eq!(entry.timestamp, same_timestamp);
138
+ all_offsets.push(entry.offset);
139
+ assert!(all_positions.insert(entry.position), "Position {} appeared twice", entry.position);
140
+ }
141
+
142
+ if let Some(last_entry) = results.last() {
143
+ resume_position = Some(last_entry.position);
144
+ }
145
+
146
+ // Second page - should get remaining 100 entries
147
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
148
+ .with_limit(PAGE_SIZE)
149
+ .with_resume_position(resume_position.unwrap())
150
+ .build()
151
+ .unwrap();
152
+
153
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
154
+ println!("Second page: {} entries", results.len());
155
+ assert_eq!(results.len(), TOTAL_ENTRIES - PAGE_SIZE, "Second page should return remaining entries");
156
+
157
+ // Verify all have the same timestamp
158
+ for entry in &results {
159
+ assert_eq!(entry.timestamp, same_timestamp);
160
+ all_offsets.push(entry.offset);
161
+ assert!(all_positions.insert(entry.position), "Position {} appeared twice", entry.position);
162
+ }
163
+
164
+ if let Some(last_entry) = results.last() {
165
+ resume_position = Some(last_entry.position);
166
+ }
167
+
168
+ // Third page - should be empty
169
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
170
+ .with_limit(PAGE_SIZE)
171
+ .with_resume_position(resume_position.unwrap())
172
+ .build()
173
+ .unwrap();
174
+
175
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
176
+ println!("Third page: {} entries", results.len());
177
+ assert_eq!(results.len(), 0, "Third page should be empty");
178
+
179
+ // Verify we got all entries
180
+ assert_eq!(all_offsets.len(), TOTAL_ENTRIES, "Should have retrieved all entries");
181
+
182
+ // Verify all offsets are unique (no duplicates)
183
+ let unique_offsets: HashSet<_> = all_offsets.iter().collect();
184
+ assert_eq!(unique_offsets.len(), TOTAL_ENTRIES, "All offsets should be unique");
185
+
186
+ // Verify all positions are unique and contiguous
187
+ assert_eq!(all_positions.len(), TOTAL_ENTRIES, "Should have unique positions");
188
+ for i in 0..TOTAL_ENTRIES {
189
+ assert!(all_positions.contains(&i), "Position {} missing", i);
190
+ }
191
+}
192
+
193
+#[test]
194
+fn test_pagination_backward_with_same_timestamps() {
195
+ // Create 300 entries all with the same timestamp
196
+ const TOTAL_ENTRIES: usize = 300;
197
+ const PAGE_SIZE: usize = 200;
198
+ let same_timestamp = JAN_1_2024_MIDNIGHT;
199
+
200
+ let entries: Vec<TestEntry> = (0..TOTAL_ENTRIES)
201
+ .map(|i| {
202
+ TestEntry::new(same_timestamp)
203
+ .with_field("MESSAGE", format!("Entry {}", i))
204
+ .with_field("ENTRY_ID", i.to_string())
205
+ })
206
+ .collect();
207
+
208
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
209
+
210
+ let mut indexer = FileIndexer::default();
211
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
212
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
213
+ let file_index = indexer
214
+ .index(
215
+ &file,
216
+ Some(&source_timestamp_field),
217
+ &[entry_id_field],
218
+ Seconds(3600),
219
+ )
220
+ .unwrap();
221
+
222
+ let mut all_offsets = Vec::new();
223
+ let mut all_positions = HashSet::new();
224
+ let mut resume_position = None;
225
+
226
+ // First page (from tail, going backward)
227
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
228
+ .with_limit(PAGE_SIZE)
229
+ .build()
230
+ .unwrap();
231
+
232
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
233
+ println!("First page: {} entries", results.len());
234
+ assert_eq!(results.len(), PAGE_SIZE, "First page should return PAGE_SIZE entries");
235
+
236
+ // Verify all have the same timestamp
237
+ for entry in &results {
238
+ assert_eq!(entry.timestamp, same_timestamp);
239
+ all_offsets.push(entry.offset);
240
+ assert!(all_positions.insert(entry.position), "Position {} appeared twice", entry.position);
241
+ }
242
+
243
+ if let Some(last_entry) = results.last() {
244
+ resume_position = Some(last_entry.position);
245
+ }
246
+
247
+ // Second page - should get remaining 100 entries
248
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
249
+ .with_limit(PAGE_SIZE)
250
+ .with_resume_position(resume_position.unwrap())
251
+ .build()
252
+ .unwrap();
253
+
254
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
255
+ println!("Second page: {} entries", results.len());
256
+ assert_eq!(results.len(), TOTAL_ENTRIES - PAGE_SIZE, "Second page should return remaining entries");
257
+
258
+ // Verify all have the same timestamp
259
+ for entry in &results {
260
+ assert_eq!(entry.timestamp, same_timestamp);
261
+ all_offsets.push(entry.offset);
262
+ assert!(all_positions.insert(entry.position), "Position {} appeared twice", entry.position);
263
+ }
264
+
265
+ if let Some(last_entry) = results.last() {
266
+ resume_position = Some(last_entry.position);
267
+ }
268
+
269
+ // Third page - should be empty
270
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
271
+ .with_limit(PAGE_SIZE)
272
+ .with_resume_position(resume_position.unwrap())
273
+ .build()
274
+ .unwrap();
275
+
276
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
277
+ println!("Third page: {} entries", results.len());
278
+ assert_eq!(results.len(), 0, "Third page should be empty");
279
+
280
+ // Verify we got all entries
281
+ assert_eq!(all_offsets.len(), TOTAL_ENTRIES, "Should have retrieved all entries");
282
+
283
+ // Verify all offsets are unique (no duplicates)
284
+ let unique_offsets: HashSet<_> = all_offsets.iter().collect();
285
+ assert_eq!(unique_offsets.len(), TOTAL_ENTRIES, "All offsets should be unique");
286
+
287
+ // Verify all positions are unique and contiguous
288
+ assert_eq!(all_positions.len(), TOTAL_ENTRIES, "Should have unique positions");
289
+ for i in 0..TOTAL_ENTRIES {
290
+ assert!(all_positions.contains(&i), "Position {} missing", i);
291
+ }
292
+}
293
+
294
+#[test]
295
+fn test_pagination_forward_with_mixed_timestamps() {
296
+ // Create entries with varying timestamps to ensure pagination works across different timestamps too
297
+ const ENTRIES_PER_TIMESTAMP: usize = 150;
298
+ const PAGE_SIZE: usize = 200;
299
+
300
+ let mut entries = Vec::new();
301
+
302
+ // First 150 entries at timestamp T
303
+ let timestamp1 = JAN_1_2024_MIDNIGHT;
304
+ for i in 0..ENTRIES_PER_TIMESTAMP {
305
+ entries.push(
306
+ TestEntry::new(timestamp1)
307
+ .with_field("MESSAGE", format!("Batch 1 Entry {}", i))
308
+ .with_field("ENTRY_ID", format!("1-{}", i))
309
+ );
310
+ }
311
+
312
+ // Next 150 entries at timestamp T+1
313
+ let timestamp2 = Microseconds(timestamp1.0 + 1_000_000);
314
+ for i in 0..ENTRIES_PER_TIMESTAMP {
315
+ entries.push(
316
+ TestEntry::new(timestamp2)
317
+ .with_field("MESSAGE", format!("Batch 2 Entry {}", i))
318
+ .with_field("ENTRY_ID", format!("2-{}", i))
319
+ );
320
+ }
321
+
322
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
323
+
324
+ let mut indexer = FileIndexer::default();
325
+ let entry_id_field = FieldName::new("ENTRY_ID").unwrap();
326
+ let source_timestamp_field = FieldName::new("_SOURCE_REALTIME_TIMESTAMP").unwrap();
327
+ let file_index = indexer
328
+ .index(
329
+ &file,
330
+ Some(&source_timestamp_field),
331
+ &[entry_id_field],
332
+ Seconds(3600),
333
+ )
334
+ .unwrap();
335
+
336
+ let mut all_offsets = Vec::new();
337
+ let mut all_positions = HashSet::new();
338
+
339
+ // First page - should get 200 entries (all 150 from timestamp1 + 50 from timestamp2)
340
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
341
+ .with_limit(PAGE_SIZE)
342
+ .build()
343
+ .unwrap();
344
+
345
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
346
+ println!("First page: {} entries", results.len());
347
+ assert_eq!(results.len(), PAGE_SIZE);
348
+
349
+ for entry in &results {
350
+ all_offsets.push(entry.offset);
351
+ assert!(all_positions.insert(entry.position));
352
+ }
353
+
354
+ let resume_position = results.last().unwrap().position;
355
+
356
+ // Second page - should get remaining 100 entries (all from timestamp2)
357
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
358
+ .with_limit(PAGE_SIZE)
359
+ .with_resume_position(resume_position)
360
+ .build()
361
+ .unwrap();
362
+
363
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
364
+ println!("Second page: {} entries", results.len());
365
+ assert_eq!(results.len(), 100);
366
+
367
+ // All entries in second page should have timestamp2
368
+ for entry in &results {
369
+ assert_eq!(entry.timestamp, timestamp2);
370
+ all_offsets.push(entry.offset);
371
+ assert!(all_positions.insert(entry.position));
372
+ }
373
+
374
+ // Verify we got all entries without duplicates
375
+ assert_eq!(all_offsets.len(), 300);
376
+ let unique_offsets: HashSet<_> = all_offsets.iter().collect();
377
+ assert_eq!(unique_offsets.len(), 300);
378
+}
379
+
380
+#[test]
381
+fn test_pagination_empty_journal() {
382
+ // Create an empty journal file
383
+ let entries = Vec::new();
384
+
385
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
386
+
387
+ let mut indexer = FileIndexer::default();
388
+
389
+ // Indexing an empty journal should fail with EmptyHistogramInput
390
+ let result = indexer.index(&file, None, &[], Seconds(3600));
391
+ assert!(result.is_err(), "Empty journal should fail to index");
392
+}
393
+
394
+#[test]
395
+fn test_pagination_single_entry() {
396
+ let timestamp = JAN_1_2024_MIDNIGHT;
397
+ let entries = vec![
398
+ TestEntry::new(timestamp).with_field("MESSAGE", "Single entry"),
399
+ ];
400
+
401
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
402
+
403
+ let mut indexer = FileIndexer::default();
404
+ let file_index = indexer
405
+ .index(&file, None, &[], Seconds(3600))
406
+ .unwrap();
407
+
408
+ // Query forward with large limit
409
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
410
+ .with_limit(100)
411
+ .build()
412
+ .unwrap();
413
+
414
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
415
+ assert_eq!(results.len(), 1, "Should return single entry");
416
+ assert_eq!(results[0].timestamp, timestamp);
417
+ assert_eq!(results[0].position, 0);
418
+
419
+ // Try to paginate from that position (should return empty)
420
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
421
+ .with_limit(100)
422
+ .with_resume_position(results[0].position)
423
+ .build()
424
+ .unwrap();
425
+
426
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
427
+ assert_eq!(results.len(), 0, "No more entries after single entry");
428
+
429
+ // Query backward
430
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
431
+ .with_limit(100)
432
+ .build()
433
+ .unwrap();
434
+
435
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
436
+ assert_eq!(results.len(), 1, "Should return single entry");
437
+ assert_eq!(results[0].timestamp, timestamp);
438
+ assert_eq!(results[0].position, 0);
439
+
440
+ // Try to paginate backward from position 0 (should return empty)
441
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
442
+ .with_limit(100)
443
+ .with_resume_position(0)
444
+ .build()
445
+ .unwrap();
446
+
447
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
448
+ assert_eq!(results.len(), 0, "Backward from position 0 should return empty");
449
+}
450
+
451
+#[test]
452
+fn test_pagination_two_entries() {
453
+ let timestamp = JAN_1_2024_MIDNIGHT;
454
+ let entries = vec![
455
+ TestEntry::new(timestamp).with_field("MESSAGE", "Entry 1"),
456
+ TestEntry::new(timestamp).with_field("MESSAGE", "Entry 2"),
457
+ ];
458
+
459
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
460
+
461
+ let mut indexer = FileIndexer::default();
462
+ let file_index = indexer
463
+ .index(&file, None, &[], Seconds(3600))
464
+ .unwrap();
465
+
466
+ // Forward: Get both entries at once with limit 10
467
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
468
+ .with_limit(10)
469
+ .build()
470
+ .unwrap();
471
+
472
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
473
+ assert_eq!(results.len(), 2, "Should return both entries");
474
+ assert_eq!(results[0].position, 0);
475
+ assert_eq!(results[1].position, 1);
476
+
477
+ // Forward: Get first entry with limit 1, then paginate
478
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
479
+ .with_limit(1)
480
+ .build()
481
+ .unwrap();
482
+
483
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
484
+ assert_eq!(results.len(), 1, "Should return first entry");
485
+ assert_eq!(results[0].position, 0);
486
+
487
+ let first_position = results[0].position;
488
+
489
+ // Get second entry
490
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
491
+ .with_limit(1)
492
+ .with_resume_position(first_position)
493
+ .build()
494
+ .unwrap();
495
+
496
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
497
+ assert_eq!(results.len(), 1, "Should return second entry");
498
+ assert_eq!(results[0].position, 1);
499
+
500
+ // Try to get third entry (should be empty)
501
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
502
+ .with_limit(1)
503
+ .with_resume_position(1)
504
+ .build()
505
+ .unwrap();
506
+
507
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
508
+ assert_eq!(results.len(), 0, "No third entry");
509
+
510
+ // Backward: Get both entries at once
511
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
512
+ .with_limit(10)
513
+ .build()
514
+ .unwrap();
515
+
516
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
517
+ assert_eq!(results.len(), 2, "Should return both entries backward");
518
+
519
+ // Backward: Get last entry with limit 1, then paginate
520
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
521
+ .with_limit(1)
522
+ .build()
523
+ .unwrap();
524
+
525
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
526
+ assert_eq!(results.len(), 1, "Should return last entry");
527
+ assert_eq!(results[0].position, 1);
528
+
529
+ // Get first entry going backward
530
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
531
+ .with_limit(1)
532
+ .with_resume_position(1)
533
+ .build()
534
+ .unwrap();
535
+
536
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
537
+ assert_eq!(results.len(), 1, "Should return first entry");
538
+ assert_eq!(results[0].position, 0);
539
+}
540
+
541
+#[test]
542
+fn test_pagination_limit_zero() {
543
+ let timestamp = JAN_1_2024_MIDNIGHT;
544
+ let entries = vec![
545
+ TestEntry::new(timestamp).with_field("MESSAGE", "Entry 1"),
546
+ TestEntry::new(timestamp).with_field("MESSAGE", "Entry 2"),
547
+ TestEntry::new(timestamp).with_field("MESSAGE", "Entry 3"),
548
+ ];
549
+
550
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
551
+
552
+ let mut indexer = FileIndexer::default();
553
+ let file_index = indexer
554
+ .index(&file, None, &[], Seconds(3600))
555
+ .unwrap();
556
+
557
+ // Query with limit 0 should return empty results
558
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
559
+ .with_limit(0)
560
+ .build()
561
+ .unwrap();
562
+
563
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
564
+ assert_eq!(results.len(), 0, "Limit 0 should return no results");
565
+
566
+ // Same for backward
567
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
568
+ .with_limit(0)
569
+ .build()
570
+ .unwrap();
571
+
572
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
573
+ assert_eq!(results.len(), 0, "Limit 0 should return no results");
574
+}
575
+
576
+#[test]
577
+fn test_pagination_limit_exact_match() {
578
+ // Create exactly 50 entries
579
+ const TOTAL_ENTRIES: usize = 50;
580
+ let timestamp = JAN_1_2024_MIDNIGHT;
581
+
582
+ let entries: Vec<TestEntry> = (0..TOTAL_ENTRIES)
583
+ .map(|i| TestEntry::new(timestamp).with_field("ENTRY_ID", i.to_string()))
584
+ .collect();
585
+
586
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
587
+
588
+ let mut indexer = FileIndexer::default();
589
+ let file_index = indexer
590
+ .index(&file, None, &[], Seconds(3600))
591
+ .unwrap();
592
+
593
+ // Query with limit exactly equal to total entries
594
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
595
+ .with_limit(TOTAL_ENTRIES)
596
+ .build()
597
+ .unwrap();
598
+
599
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
600
+ assert_eq!(results.len(), TOTAL_ENTRIES, "Should return all entries");
601
+
602
+ // Try to paginate from last position (should return empty)
603
+ let last_position = results.last().unwrap().position;
604
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
605
+ .with_limit(TOTAL_ENTRIES)
606
+ .with_resume_position(last_position)
607
+ .build()
608
+ .unwrap();
609
+
610
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
611
+ assert_eq!(results.len(), 0, "No more entries after exact match");
612
+}
613
+
614
+#[test]
615
+fn test_pagination_limit_exceeds_total() {
616
+ // Create 10 entries but query with limit 1000
617
+ const TOTAL_ENTRIES: usize = 10;
618
+ const LARGE_LIMIT: usize = 1000;
619
+ let timestamp = JAN_1_2024_MIDNIGHT;
620
+
621
+ let entries: Vec<TestEntry> = (0..TOTAL_ENTRIES)
622
+ .map(|i| TestEntry::new(timestamp).with_field("ENTRY_ID", i.to_string()))
623
+ .collect();
624
+
625
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
626
+
627
+ let mut indexer = FileIndexer::default();
628
+ let file_index = indexer
629
+ .index(&file, None, &[], Seconds(3600))
630
+ .unwrap();
631
+
632
+ // Query with limit much larger than total entries
633
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
634
+ .with_limit(LARGE_LIMIT)
635
+ .build()
636
+ .unwrap();
637
+
638
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
639
+ assert_eq!(
640
+ results.len(),
641
+ TOTAL_ENTRIES,
642
+ "Should return all entries (not more than available)"
643
+ );
644
+
645
+ // Verify all positions are present
646
+ for (i, entry) in results.iter().enumerate() {
647
+ assert_eq!(entry.position, i);
648
+ }
649
+
650
+ // Same for backward
651
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
652
+ .with_limit(LARGE_LIMIT)
653
+ .build()
654
+ .unwrap();
655
+
656
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
657
+ assert_eq!(results.len(), TOTAL_ENTRIES, "Should return all entries backward");
658
+}
659
+
660
+#[test]
661
+fn test_pagination_resume_out_of_bounds() {
662
+ // Create 10 entries
663
+ const TOTAL_ENTRIES: usize = 10;
664
+ let timestamp = JAN_1_2024_MIDNIGHT;
665
+
666
+ let entries: Vec<TestEntry> = (0..TOTAL_ENTRIES)
667
+ .map(|i| TestEntry::new(timestamp).with_field("ENTRY_ID", i.to_string()))
668
+ .collect();
669
+
670
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
671
+
672
+ let mut indexer = FileIndexer::default();
673
+ let file_index = indexer
674
+ .index(&file, None, &[], Seconds(3600))
675
+ .unwrap();
676
+
677
+ // Forward: Resume from position equal to total entries (at boundary)
678
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
679
+ .with_limit(10)
680
+ .with_resume_position(TOTAL_ENTRIES - 1)
681
+ .build()
682
+ .unwrap();
683
+
684
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
685
+ assert_eq!(results.len(), 0, "Resume from last position should return empty");
686
+
687
+ // Forward: Resume from position beyond total entries
688
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
689
+ .with_limit(10)
690
+ .with_resume_position(TOTAL_ENTRIES)
691
+ .build()
692
+ .unwrap();
693
+
694
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
695
+ assert_eq!(
696
+ results.len(),
697
+ 0,
698
+ "Resume from beyond last position should return empty"
699
+ );
700
+
701
+ // Forward: Resume from way beyond total entries
702
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
703
+ .with_limit(10)
704
+ .with_resume_position(999)
705
+ .build()
706
+ .unwrap();
707
+
708
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
709
+ assert_eq!(
710
+ results.len(),
711
+ 0,
712
+ "Resume from way beyond should return empty (not panic)"
713
+ );
714
+
715
+ // Backward: Resume from position 0 returns empty (already tested but for completeness)
716
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
717
+ .with_limit(10)
718
+ .with_resume_position(0)
719
+ .build()
720
+ .unwrap();
721
+
722
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
723
+ assert_eq!(results.len(), 0, "Backward from position 0 should return empty");
724
+
725
+ // Backward: Resume from position equal to total entries
726
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
727
+ .with_limit(10)
728
+ .with_resume_position(TOTAL_ENTRIES)
729
+ .build()
730
+ .unwrap();
731
+
732
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
733
+ assert_eq!(
734
+ results.len(),
735
+ 0,
736
+ "Backward from position equal to total should return empty (not panic)"
737
+ );
738
+
739
+ // Backward: Resume from position beyond total entries
740
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
741
+ .with_limit(10)
742
+ .with_resume_position(TOTAL_ENTRIES + 5)
743
+ .build()
744
+ .unwrap();
745
+
746
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
747
+ assert_eq!(
748
+ results.len(),
749
+ 0,
750
+ "Backward from beyond total should return empty (not panic)"
751
+ );
752
+
753
+ // Backward: Resume from way beyond total entries
754
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
755
+ .with_limit(10)
756
+ .with_resume_position(999)
757
+ .build()
758
+ .unwrap();
759
+
760
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
761
+ assert_eq!(
762
+ results.len(),
763
+ 0,
764
+ "Backward from way beyond should return empty (not panic)"
765
+ );
766
+}
767
+
768
+#[test]
769
+fn test_pagination_anchor_before_all_entries() {
770
+ // Create entries at timestamps 10:00, 11:00, 12:00
771
+ let base_timestamp = JAN_1_2024_MIDNIGHT;
772
+ let entries = vec![
773
+ TestEntry::new(Microseconds(base_timestamp.0 + 10 * 3600_000_000))
774
+ .with_field("ENTRY_ID", "0"),
775
+ TestEntry::new(Microseconds(base_timestamp.0 + 11 * 3600_000_000))
776
+ .with_field("ENTRY_ID", "1"),
777
+ TestEntry::new(Microseconds(base_timestamp.0 + 12 * 3600_000_000))
778
+ .with_field("ENTRY_ID", "2"),
779
+ ];
780
+
781
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
782
+
783
+ let mut indexer = FileIndexer::default();
784
+ let file_index = indexer
785
+ .index(&file, None, &[], Seconds(3600))
786
+ .unwrap();
787
+
788
+ // Anchor at 09:00 (before all entries), going forward
789
+ let anchor_timestamp = Microseconds(base_timestamp.0 + 9 * 3600_000_000);
790
+ let params = LogQueryParamsBuilder::new(
791
+ Anchor::Timestamp(anchor_timestamp),
792
+ Direction::Forward,
793
+ )
794
+ .with_limit(10)
795
+ .build()
796
+ .unwrap();
797
+
798
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
799
+ assert_eq!(
800
+ results.len(),
801
+ 3,
802
+ "Forward from before all entries should return all entries"
803
+ );
804
+
805
+ // Anchor at 09:00, going backward
806
+ let params = LogQueryParamsBuilder::new(
807
+ Anchor::Timestamp(anchor_timestamp),
808
+ Direction::Backward,
809
+ )
810
+ .with_limit(10)
811
+ .build()
812
+ .unwrap();
813
+
814
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
815
+ assert_eq!(
816
+ results.len(),
817
+ 0,
818
+ "Backward from before all entries should return no entries"
819
+ );
820
+}
821
+
822
+#[test]
823
+fn test_pagination_anchor_after_all_entries() {
824
+ // Create entries at timestamps 10:00, 11:00, 12:00
825
+ let base_timestamp = JAN_1_2024_MIDNIGHT;
826
+ let entries = vec![
827
+ TestEntry::new(Microseconds(base_timestamp.0 + 10 * 3600_000_000))
828
+ .with_field("ENTRY_ID", "0"),
829
+ TestEntry::new(Microseconds(base_timestamp.0 + 11 * 3600_000_000))
830
+ .with_field("ENTRY_ID", "1"),
831
+ TestEntry::new(Microseconds(base_timestamp.0 + 12 * 3600_000_000))
832
+ .with_field("ENTRY_ID", "2"),
833
+ ];
834
+
835
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
836
+
837
+ let mut indexer = FileIndexer::default();
838
+ let file_index = indexer
839
+ .index(&file, None, &[], Seconds(3600))
840
+ .unwrap();
841
+
842
+ // Anchor at 13:00 (after all entries), going forward
843
+ let anchor_timestamp = Microseconds(base_timestamp.0 + 13 * 3600_000_000);
844
+ let params = LogQueryParamsBuilder::new(
845
+ Anchor::Timestamp(anchor_timestamp),
846
+ Direction::Forward,
847
+ )
848
+ .with_limit(10)
849
+ .build()
850
+ .unwrap();
851
+
852
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
853
+ assert_eq!(
854
+ results.len(),
855
+ 0,
856
+ "Forward from after all entries should return no entries"
857
+ );
858
+
859
+ // Anchor at 13:00, going backward
860
+ let params = LogQueryParamsBuilder::new(
861
+ Anchor::Timestamp(anchor_timestamp),
862
+ Direction::Backward,
863
+ )
864
+ .with_limit(10)
865
+ .build()
866
+ .unwrap();
867
+
868
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
869
+ assert_eq!(
870
+ results.len(),
871
+ 3,
872
+ "Backward from after all entries should return all entries"
873
+ );
874
+}
875
+
876
+#[test]
877
+fn test_pagination_anchor_in_middle_with_pagination() {
878
+ // Create entries at timestamps 10:00, 11:00, 12:00, 13:00, 14:00
879
+ let base_timestamp = JAN_1_2024_MIDNIGHT;
880
+ let entries = vec![
881
+ TestEntry::new(Microseconds(base_timestamp.0 + 10 * 3600_000_000))
882
+ .with_field("ENTRY_ID", "0"),
883
+ TestEntry::new(Microseconds(base_timestamp.0 + 11 * 3600_000_000))
884
+ .with_field("ENTRY_ID", "1"),
885
+ TestEntry::new(Microseconds(base_timestamp.0 + 12 * 3600_000_000))
886
+ .with_field("ENTRY_ID", "2"),
887
+ TestEntry::new(Microseconds(base_timestamp.0 + 13 * 3600_000_000))
888
+ .with_field("ENTRY_ID", "3"),
889
+ TestEntry::new(Microseconds(base_timestamp.0 + 14 * 3600_000_000))
890
+ .with_field("ENTRY_ID", "4"),
891
+ ];
892
+
893
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
894
+
895
+ let mut indexer = FileIndexer::default();
896
+ let file_index = indexer
897
+ .index(&file, None, &[], Seconds(3600))
898
+ .unwrap();
899
+
900
+ // Anchor at 12:00 (middle), going forward with limit 2
901
+ let anchor_timestamp = Microseconds(base_timestamp.0 + 12 * 3600_000_000);
902
+ let params = LogQueryParamsBuilder::new(
903
+ Anchor::Timestamp(anchor_timestamp),
904
+ Direction::Forward,
905
+ )
906
+ .with_limit(2)
907
+ .build()
908
+ .unwrap();
909
+
910
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
911
+ assert_eq!(
912
+ results.len(),
913
+ 2,
914
+ "Should return 2 entries starting from anchor"
915
+ );
916
+ // Should get entries at 12:00 and 13:00 (positions 2 and 3)
917
+ assert_eq!(results[0].position, 2);
918
+ assert_eq!(results[1].position, 3);
919
+
920
+ // Paginate forward to get the rest
921
+ let params = LogQueryParamsBuilder::new(
922
+ Anchor::Timestamp(anchor_timestamp),
923
+ Direction::Forward,
924
+ )
925
+ .with_limit(2)
926
+ .with_resume_position(results[1].position)
927
+ .build()
928
+ .unwrap();
929
+
930
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
931
+ assert_eq!(results.len(), 1, "Should return remaining 1 entry");
932
+ assert_eq!(results[0].position, 4);
933
+
934
+ // Anchor at 12:00, going backward with limit 2
935
+ let params = LogQueryParamsBuilder::new(
936
+ Anchor::Timestamp(anchor_timestamp),
937
+ Direction::Backward,
938
+ )
939
+ .with_limit(2)
940
+ .build()
941
+ .unwrap();
942
+
943
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
944
+ assert_eq!(
945
+ results.len(),
946
+ 2,
947
+ "Should return 2 entries backward from anchor"
948
+ );
949
+ // Should get entries at 12:00 and 11:00 (positions 2 and 1)
950
+ assert_eq!(results[0].position, 2);
951
+ assert_eq!(results[1].position, 1);
952
+
953
+ // Paginate backward to get the rest
954
+ let params = LogQueryParamsBuilder::new(
955
+ Anchor::Timestamp(anchor_timestamp),
956
+ Direction::Backward,
957
+ )
958
+ .with_limit(2)
959
+ .with_resume_position(results[1].position)
960
+ .build()
961
+ .unwrap();
962
+
963
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
964
+ assert_eq!(results.len(), 1, "Should return remaining 1 entry");
965
+ assert_eq!(results[0].position, 0);
966
+}
967
+
968
+#[test]
969
+fn test_pagination_with_time_boundaries() {
970
+ // Create entries at different timestamps
971
+ let base_timestamp = JAN_1_2024_MIDNIGHT;
972
+ let entries: Vec<TestEntry> = (0..20)
973
+ .map(|i| {
974
+ // Entry at hour i
975
+ TestEntry::new(Microseconds(base_timestamp.0 + i * 3600_000_000))
976
+ .with_field("ENTRY_ID", i.to_string())
977
+ })
978
+ .collect();
979
+
980
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
981
+
982
+ let mut indexer = FileIndexer::default();
983
+ let file_index = indexer
984
+ .index(&file, None, &[], Seconds(3600))
985
+ .unwrap();
986
+
987
+ // Query with after and before boundaries: entries from hour 5 to hour 15 (exclusive)
988
+ // That's entries 5, 6, 7, 8, 9, 10, 11, 12, 13, 14 (10 entries total)
989
+ let after = Microseconds(base_timestamp.0 + 5 * 3600_000_000);
990
+ let before = Microseconds(base_timestamp.0 + 15 * 3600_000_000);
991
+
992
+ // First page with limit 4
993
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
994
+ .with_after(after)
995
+ .with_before(before)
996
+ .with_limit(4)
997
+ .build()
998
+ .unwrap();
999
+
1000
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1001
+ assert_eq!(results.len(), 4, "First page should return 4 entries");
1002
+ // Should get entries 5, 6, 7, 8
1003
+ assert_eq!(results[0].position, 5);
1004
+ assert_eq!(results[3].position, 8);
1005
+
1006
+ let mut all_results = results.clone();
1007
+
1008
+ // Second page
1009
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
1010
+ .with_after(after)
1011
+ .with_before(before)
1012
+ .with_limit(4)
1013
+ .with_resume_position(results.last().unwrap().position)
1014
+ .build()
1015
+ .unwrap();
1016
+
1017
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1018
+ assert_eq!(results.len(), 4, "Second page should return 4 entries");
1019
+ // Should get entries 9, 10, 11, 12
1020
+ assert_eq!(results[0].position, 9);
1021
+ assert_eq!(results[3].position, 12);
1022
+
1023
+ all_results.extend(results.clone());
1024
+
1025
+ // Third page
1026
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
1027
+ .with_after(after)
1028
+ .with_before(before)
1029
+ .with_limit(4)
1030
+ .with_resume_position(results.last().unwrap().position)
1031
+ .build()
1032
+ .unwrap();
1033
+
1034
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1035
+ assert_eq!(results.len(), 2, "Third page should return remaining 2 entries");
1036
+ // Should get entries 13, 14
1037
+ assert_eq!(results[0].position, 13);
1038
+ assert_eq!(results[1].position, 14);
1039
+
1040
+ all_results.extend(results.clone());
1041
+
1042
+ // Fourth page (should be empty)
1043
+ let params = LogQueryParamsBuilder::new(Anchor::Head, Direction::Forward)
1044
+ .with_after(after)
1045
+ .with_before(before)
1046
+ .with_limit(4)
1047
+ .with_resume_position(results.last().unwrap().position)
1048
+ .build()
1049
+ .unwrap();
1050
+
1051
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1052
+ assert_eq!(results.len(), 0, "Fourth page should be empty");
1053
+
1054
+ // Verify we got exactly 10 entries total
1055
+ assert_eq!(all_results.len(), 10);
1056
+
1057
+ // Verify all timestamps are within boundaries
1058
+ for entry in &all_results {
1059
+ assert!(
1060
+ entry.timestamp.0 >= after.0,
1061
+ "Entry timestamp should be >= after boundary"
1062
+ );
1063
+ assert!(
1064
+ entry.timestamp.0 < before.0,
1065
+ "Entry timestamp should be < before boundary"
1066
+ );
1067
+ }
1068
+}
1069
+
1070
+#[test]
1071
+fn test_pagination_backward_with_time_boundaries() {
1072
+ // Create entries at different timestamps
1073
+ let base_timestamp = JAN_1_2024_MIDNIGHT;
1074
+ let entries: Vec<TestEntry> = (0..20)
1075
+ .map(|i| {
1076
+ TestEntry::new(Microseconds(base_timestamp.0 + i * 3600_000_000))
1077
+ .with_field("ENTRY_ID", i.to_string())
1078
+ })
1079
+ .collect();
1080
+
1081
+ let (_temp_dir, file) = create_test_journal(entries).unwrap();
1082
+
1083
+ let mut indexer = FileIndexer::default();
1084
+ let file_index = indexer
1085
+ .index(&file, None, &[], Seconds(3600))
1086
+ .unwrap();
1087
+
1088
+ // Query backward with boundaries: entries from hour 5 to hour 15 (exclusive)
1089
+ // That's entries 5-14 (10 entries total), going backward from 14 to 5
1090
+ let after = Microseconds(base_timestamp.0 + 5 * 3600_000_000);
1091
+ let before = Microseconds(base_timestamp.0 + 15 * 3600_000_000);
1092
+
1093
+ // First page with limit 4
1094
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
1095
+ .with_after(after)
1096
+ .with_before(before)
1097
+ .with_limit(4)
1098
+ .build()
1099
+ .unwrap();
1100
+
1101
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1102
+ assert_eq!(results.len(), 4, "First page should return 4 entries");
1103
+ // Going backward, should get 14, 13, 12, 11
1104
+ assert_eq!(results[0].position, 14);
1105
+ assert_eq!(results[3].position, 11);
1106
+
1107
+ let mut all_results = results.clone();
1108
+
1109
+ // Second page
1110
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
1111
+ .with_after(after)
1112
+ .with_before(before)
1113
+ .with_limit(4)
1114
+ .with_resume_position(results.last().unwrap().position)
1115
+ .build()
1116
+ .unwrap();
1117
+
1118
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1119
+ assert_eq!(results.len(), 4, "Second page should return 4 entries");
1120
+ // Should get 10, 9, 8, 7
1121
+ assert_eq!(results[0].position, 10);
1122
+ assert_eq!(results[3].position, 7);
1123
+
1124
+ all_results.extend(results.clone());
1125
+
1126
+ // Third page
1127
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
1128
+ .with_after(after)
1129
+ .with_before(before)
1130
+ .with_limit(4)
1131
+ .with_resume_position(results.last().unwrap().position)
1132
+ .build()
1133
+ .unwrap();
1134
+
1135
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1136
+ assert_eq!(results.len(), 2, "Third page should return remaining 2 entries");
1137
+ // Should get 6, 5
1138
+ assert_eq!(results[0].position, 6);
1139
+ assert_eq!(results[1].position, 5);
1140
+
1141
+ all_results.extend(results.clone());
1142
+
1143
+ // Fourth page (should be empty)
1144
+ let params = LogQueryParamsBuilder::new(Anchor::Tail, Direction::Backward)
1145
+ .with_after(after)
1146
+ .with_before(before)
1147
+ .with_limit(4)
1148
+ .with_resume_position(results.last().unwrap().position)
1149
+ .build()
1150
+ .unwrap();
1151
+
1152
+ let results = file_index.find_log_entries(&file, ¶ms).unwrap();
1153
+ assert_eq!(results.len(), 0, "Fourth page should be empty");
1154
+
1155
+ // Verify we got exactly 10 entries total
1156
+ assert_eq!(all_results.len(), 10);
1157
+
1158
+ // Verify all timestamps are within boundaries
1159
+ for entry in &all_results {
1160
+ assert!(
1161
+ entry.timestamp.0 >= after.0,
1162
+ "Entry timestamp should be >= after boundary"
1163
+ );
1164
+ assert!(
1165
+ entry.timestamp.0 < before.0,
1166
+ "Entry timestamp should be < before boundary"
1167
+ );
1168
+ }
1169
+}
src/crates/journal-log-writer/Cargo.toml
new
+30
@@ -0,0 +1,30 @@
1
+[package]
2
+name = "journal-log-writer"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dependencies]
11
+journal-common = { workspace = true }
12
+journal-core = { workspace = true }
13
+journal-registry = { workspace = true }
14
+rdp = { workspace = true }
15
+
16
+uuid = { workspace = true, features = ["v4"] }
17
+tracing = { workspace = true }
18
+thiserror = { workspace = true }
19
+nix = { workspace = true, features = ["time"] }
20
+
21
+# Optional serde support for structured logging
22
+serde = { workspace = true, optional = true }
23
+serde_json = { workspace = true, optional = true }
24
+flatten-serde-json = { workspace = true, optional = true }
25
+
26
+[features]
27
+serde-api = ["serde", "serde_json", "flatten-serde-json"]
28
+
29
+[dev-dependencies]
30
+tempfile = { workspace = true }
src/crates/journal-log-writer/src/error.rs
new
+39
@@ -0,0 +1,39 @@
1
+use thiserror::Error;
2
+
3
+/// Errors that can occur during journal writing operations.
4
+#[derive(Error, Debug)]
5
+pub enum WriterError {
6
+ /// Failed to serialize value to journal entry format
7
+ #[error("serialization error: {0}")]
8
+ Serialization(String),
9
+
10
+ /// Invalid path for journal directory
11
+ #[error("invalid path: {0}")]
12
+ InvalidPath(String),
13
+
14
+ /// Path is not a directory
15
+ #[error("not a directory: {0}")]
16
+ NotADirectory(String),
17
+
18
+ /// Failed to create journal file
19
+ #[error("failed to create journal file: {0}")]
20
+ FileCreation(String),
21
+
22
+ /// Machine ID could not be loaded or validated
23
+ #[error("machine ID error: {0}")]
24
+ MachineId(String),
25
+
26
+ /// I/O error when interacting with filesystem
27
+ #[error("I/O error: {0}")]
28
+ Io(#[from] std::io::Error),
29
+
30
+ /// Underlying journal file error
31
+ #[error("journal error: {0}")]
32
+ Journal(#[from] journal_core::error::JournalError),
33
+
34
+ /// Repository/registry error
35
+ #[error("registry error: {0}")]
36
+ Registry(#[from] journal_registry::RegistryError),
37
+}
38
+
39
+pub type Result<T> = std::result::Result<T, WriterError>;
src/crates/journal-log-writer/src/lib.rs
new
+47
@@ -0,0 +1,47 @@
1
+//! High-level journal log writer with rotation and retention policies
2
+//!
3
+//! This crate provides a high-level interface for writing to systemd journal files
4
+//! in a directory, with automatic rotation and retention management.
5
+//!
6
+//! ## Usage
7
+//!
8
+//! ```no_run
9
+//! use journal_log_writer::{Log, Config, RotationPolicy, RetentionPolicy};
10
+//! use journal_registry::Origin;
11
+//! use std::path::Path;
12
+//!
13
+//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
14
+//! // Configure rotation and retention policies
15
+//! let rotation = RotationPolicy::default()
16
+//! .with_size_of_journal_file(100 * 1024 * 1024); // 100 MB per file
17
+//!
18
+//! let retention = RetentionPolicy::default()
19
+//! .with_number_of_journal_files(10); // Keep 10 files max
20
+//!
21
+//! let origin = Origin {
22
+//! machine_id: None,
23
+//! namespace: None,
24
+//! source: journal_registry::Source::System,
25
+//! };
26
+//!
27
+//! let config = Config::new(origin, rotation, retention);
28
+//!
29
+//! // Create a log writer
30
+//! let mut log = Log::new(Path::new("/var/log/myapp"), config)?;
31
+//!
32
+//! // Write entries
33
+//! let entry = [
34
+//! b"MESSAGE=Hello, journal!" as &[u8],
35
+//! b"PRIORITY=6",
36
+//! ];
37
+//! log.write_entry(&entry, None)?;
38
+//! log.sync()?;
39
+//! # Ok(())
40
+//! # }
41
+//! ```
42
+
43
+mod error;
44
+mod log;
45
+
46
+pub use error::{Result, WriterError};
47
+pub use log::{Config, Log, RetentionPolicy, RotationPolicy};
src/crates/journal-log-writer/src/log/chain.rs
new
+221
@@ -0,0 +1,221 @@
1
+use crate::error::{Result, WriterError};
2
+use crate::log::RetentionPolicy;
3
+use journal_common::Microseconds;
4
+use journal_core::JournalFile;
5
+use journal_core::collections::HashMap;
6
+use journal_core::file::Mmap;
7
+use journal_registry::repository;
8
+use journal_registry::repository::File;
9
+use std::path::PathBuf;
10
+use uuid::Uuid;
11
+
12
+#[allow(unused_imports)]
13
+use tracing::{error, info, instrument};
14
+
15
+// Helper function to create a File with archived status
16
+fn create_chain_file(
17
+ path: &PathBuf,
18
+ seqnum_id: Uuid,
19
+ head_seqnum: u64,
20
+ head_realtime: u64,
21
+) -> Option<repository::File> {
22
+ // Format the path using the same logic as journal_registry
23
+ let filename = format!(
24
+ "system@{}-{:016x}-{:016x}.journal",
25
+ seqnum_id.simple(),
26
+ head_seqnum,
27
+ head_realtime
28
+ );
29
+
30
+ let path = path.join(filename);
31
+
32
+ repository::File::from_path(&path)
33
+}
34
+
35
+/// Manages a directory of journal files with automatic cleanup.
36
+///
37
+/// Scans the directory for existing files, tracks their sizes, and enforces retention
38
+/// policies. Typically not used directly - see [`JournalLog`](crate::JournalLog) instead.
39
+#[derive(Debug)]
40
+pub(super) struct OwnedChain {
41
+ pub(super) path: PathBuf,
42
+ pub(super) machine_id: Uuid,
43
+
44
+ pub(super) inner: repository::Chain,
45
+ pub(super) file_sizes: HashMap<File, u64>,
46
+ pub(super) total_size: u64,
47
+}
48
+
49
+impl OwnedChain {
50
+ pub(super) fn new(path: PathBuf, machine_id: Uuid) -> Result<Self> {
51
+ #[cfg(debug_assertions)]
52
+ {
53
+ use std::os::unix::ffi::OsStrExt;
54
+
55
+ debug_assert!(path.exists() && path.is_dir());
56
+
57
+ let filename = path.file_name().unwrap().as_bytes();
58
+ debug_assert_eq!(Ok(machine_id), Uuid::try_parse_ascii(filename));
59
+ }
60
+
61
+ let mut chain = Self {
62
+ path,
63
+ machine_id,
64
+ inner: repository::Chain::default(),
65
+ file_sizes: HashMap::default(),
66
+ total_size: 0,
67
+ };
68
+
69
+ for entry in std::fs::read_dir(&chain.path)? {
70
+ let Ok(file_path) = entry.map(|e| e.path()) else {
71
+ continue;
72
+ };
73
+
74
+ let Some(file) = repository::File::from_path(&file_path) else {
75
+ continue;
76
+ };
77
+
78
+ let Ok(size) = std::fs::metadata(file.path()).map(|m| m.len()) else {
79
+ continue;
80
+ };
81
+
82
+ chain.total_size += size;
83
+ chain.file_sizes.insert(file.clone(), size);
84
+ chain.inner.insert_file(file);
85
+ }
86
+
87
+ Ok(chain)
88
+ }
89
+
90
+ pub(super) fn tail_seqnum(&self) -> Result<u64> {
91
+ let Some(file) = self.inner.back() else {
92
+ return Ok(0);
93
+ };
94
+
95
+ let window_size = 4096;
96
+ let jf = JournalFile::<Mmap>::open(file, window_size)?;
97
+
98
+ Ok(jf.journal_header_ref().tail_entry_seqnum)
99
+ }
100
+
101
+ pub(super) fn tail_realtime(&self) -> Result<Option<Microseconds>> {
102
+ let Some(file) = self.inner.back() else {
103
+ return Ok(None);
104
+ };
105
+
106
+ let window_size = 4096;
107
+ let jf = JournalFile::<Mmap>::open(file, window_size)?;
108
+
109
+ let realtime = jf.journal_header_ref().tail_entry_realtime;
110
+ if realtime == 0 {
111
+ Ok(None)
112
+ } else {
113
+ Ok(Some(Microseconds::new(realtime)))
114
+ }
115
+ }
116
+
117
+ /// Registers a new journal file with the directory.
118
+ pub(super) fn create_file(
119
+ &mut self,
120
+ seqnum_id: Uuid,
121
+ head_seqnum: u64,
122
+ head_realtime: u64,
123
+ ) -> Result<repository::File> {
124
+ let Some(file) = create_chain_file(&self.path, seqnum_id, head_seqnum, head_realtime)
125
+ else {
126
+ return Err(WriterError::FileCreation(format!(
127
+ "failed to create journal file in {}",
128
+ self.path.display()
129
+ )));
130
+ };
131
+ self.inner.insert_file(file.clone());
132
+ Ok(file)
133
+ }
134
+
135
+ /// Updates the tracked size of a file in the chain
136
+ pub(super) fn update_file_size(&mut self, file: &File, new_size: u64) {
137
+ let old_size = self.file_sizes.get(file).copied().unwrap_or(0);
138
+ self.file_sizes.insert(file.clone(), new_size);
139
+ self.total_size = self
140
+ .total_size
141
+ .saturating_sub(old_size)
142
+ .saturating_add(new_size);
143
+ }
144
+
145
+ /// Retains the files that satisfy retention policy limits.
146
+ #[tracing::instrument(skip_all, fields(reason))]
147
+ pub(super) fn retain(&mut self, retention_policy: &RetentionPolicy) -> Result<()> {
148
+ // Remove by file count limit
149
+ if let Some(max_files) = retention_policy.number_of_journal_files {
150
+ while self.inner.len() > max_files {
151
+ let reason = format!("num_files({}) > max_files({})", self.inner.len(), max_files);
152
+ tracing::Span::current().record("reason", reason);
153
+ self.delete_oldest_file()?;
154
+ }
155
+ }
156
+
157
+ // Remove by total size limit
158
+ if let Some(max_total_size) = retention_policy.size_of_journal_files {
159
+ while self.total_size > max_total_size && !self.inner.is_empty() {
160
+ let reason = format!(
161
+ "total_size({}) > max_size({})",
162
+ self.total_size, max_total_size
163
+ );
164
+ tracing::Span::current().record("reason", reason);
165
+ self.delete_oldest_file()?;
166
+ }
167
+ }
168
+
169
+ // Remove by entry age limit
170
+ if let Some(max_entry_age) = retention_policy.duration_of_journal_files {
171
+ self.delete_files_older_than(max_entry_age)?;
172
+ }
173
+
174
+ Ok(())
175
+ }
176
+
177
+ /// Remove the oldest file
178
+ #[tracing::instrument(skip_all)]
179
+ fn delete_oldest_file(&mut self) -> Result<()> {
180
+ let Some(file) = self.inner.pop_front() else {
181
+ return Ok(());
182
+ };
183
+
184
+ info!("deleting {}", file.path());
185
+
186
+ let file_size = self.file_sizes.get(&file).copied().unwrap_or(0);
187
+
188
+ // Remove from filesystem
189
+ if let Err(e) = std::fs::remove_file(file.path()) {
190
+ // Log error but continue cleanup - file might already be deleted
191
+ error!("failed to remove journal file {:?}: {}", file.path(), e);
192
+ }
193
+
194
+ self.file_sizes.remove(&file);
195
+ self.total_size = self.total_size.saturating_sub(file_size);
196
+ Ok(())
197
+ }
198
+
199
+ /// Remove files older than the specified cutoff time
200
+ #[tracing::instrument(skip(self))]
201
+ fn delete_files_older_than(&mut self, max_entry_age: std::time::Duration) -> Result<()> {
202
+ let cutoff_time = Microseconds::now()
203
+ .get()
204
+ .saturating_sub(max_entry_age.as_micros() as u64);
205
+
206
+ for file in self.inner.drain(cutoff_time) {
207
+ info!("deleting {}", file.path());
208
+ let file_size = self.file_sizes.get(&file).copied().unwrap_or(0);
209
+
210
+ if let Err(e) = std::fs::remove_file(file.path()) {
211
+ error!("failed to remove journal file {:?}: {}", file.path(), e);
212
+ continue;
213
+ }
214
+
215
+ self.file_sizes.remove(&file);
216
+ self.total_size = self.total_size.saturating_sub(file_size);
217
+ }
218
+
219
+ Ok(())
220
+ }
221
+}
src/crates/journal-log-writer/src/log/config.rs
new
+107
@@ -0,0 +1,107 @@
1
+use journal_registry::Origin;
2
+use std::time::Duration;
3
+
4
+/// Controls when journal files should be rotated
5
+///
6
+/// A file rotates when *any* configured limit is exceeded. If all fields are `None`,
7
+/// files never rotate automatically.
8
+#[derive(Debug, Copy, Clone, Default)]
9
+pub struct RotationPolicy {
10
+ /// Maximum file size
11
+ pub size_of_journal_file: Option<u64>,
12
+ /// Maximum duration of head/tail entries
13
+ pub duration_of_journal_file: Option<Duration>,
14
+ /// Maximum number of log entries
15
+ pub number_of_entries: Option<usize>,
16
+}
17
+
18
+impl RotationPolicy {
19
+ /// Specifies the maximum journal file size.
20
+ pub fn with_size_of_journal_file(mut self, size_of_journal_file: u64) -> Self {
21
+ self.size_of_journal_file = Some(size_of_journal_file);
22
+ self
23
+ }
24
+
25
+ /// Specifies the maximum duration between head/tail entry.
26
+ pub fn with_duration_of_journal_file(mut self, duration_of_journal_file: Duration) -> Self {
27
+ self.duration_of_journal_file = Some(duration_of_journal_file);
28
+ self
29
+ }
30
+
31
+ /// Specifies maximum number of entries.
32
+ pub fn with_number_of_entries(mut self, number_of_entries: usize) -> Self {
33
+ self.number_of_entries = Some(number_of_entries);
34
+ self
35
+ }
36
+}
37
+
38
+/// Controls when old journal files should be deleted.
39
+///
40
+/// Old files are removed to satisfy *all* configured limits. Removal starts with
41
+/// the oldest files first. If all fields are `None`, files are never deleted.
42
+#[derive(Debug, Copy, Clone, Default)]
43
+pub struct RetentionPolicy {
44
+ /// Maximum number of journal files to keep
45
+ pub number_of_journal_files: Option<usize>,
46
+ /// Maximum total size of all journal files (in bytes)
47
+ pub size_of_journal_files: Option<u64>,
48
+ /// Maximum age of files to keep
49
+ pub duration_of_journal_files: Option<Duration>,
50
+}
51
+
52
+impl RetentionPolicy {
53
+ /// Specifies maximum number of journal files.
54
+ pub fn with_number_of_journal_files(mut self, number_of_journal_files: usize) -> Self {
55
+ self.number_of_journal_files = Some(number_of_journal_files);
56
+ self
57
+ }
58
+
59
+ /// Specifies maximum size of journal files.
60
+ pub fn with_size_of_journal_files(mut self, size_of_journal_files: u64) -> Self {
61
+ self.size_of_journal_files = Some(size_of_journal_files);
62
+ self
63
+ }
64
+
65
+ /// Specifies maximum duration of journal files.
66
+ pub fn with_duration_of_journal_files(mut self, duration_of_journal_files: Duration) -> Self {
67
+ self.duration_of_journal_files = Some(duration_of_journal_files);
68
+ self
69
+ }
70
+}
71
+
72
+/// Configuration for a journal log.
73
+#[derive(Debug, Clone)]
74
+pub struct Config {
75
+ pub origin: Origin,
76
+ /// Policy for when to rotate active files
77
+ pub rotation_policy: RotationPolicy,
78
+ /// Policy for when to remove old files
79
+ pub retention_policy: RetentionPolicy,
80
+}
81
+
82
+impl Config {
83
+ /// Creates a new log configuration.
84
+ pub fn new(
85
+ origin: Origin,
86
+ rotation_policy: RotationPolicy,
87
+ retention_policy: RetentionPolicy,
88
+ ) -> Self {
89
+ Self {
90
+ origin,
91
+ rotation_policy,
92
+ retention_policy,
93
+ }
94
+ }
95
+
96
+ /// Specifies the rotation policy of the log directory
97
+ pub fn with_rotation_policy(mut self, policy: RotationPolicy) -> Self {
98
+ self.rotation_policy = policy;
99
+ self
100
+ }
101
+
102
+ /// Specifies the retention policy of the log directory
103
+ pub fn with_retention_policy(mut self, policy: RetentionPolicy) -> Self {
104
+ self.retention_policy = policy;
105
+ self
106
+ }
107
+}
src/crates/journal-log-writer/src/log/mod.rs
new
+527
@@ -0,0 +1,527 @@
1
+mod chain;
2
+use chain::OwnedChain;
3
+
4
+mod config;
5
+pub use config::{Config, RetentionPolicy, RotationPolicy};
6
+
7
+use crate::{Result, WriterError};
8
+use journal_common::{RealtimeClock, load_boot_id, load_machine_id, monotonic_now};
9
+use journal_core::field_map::{
10
+ FieldMap, REMAPPING_MARKER, extract_field_name, is_systemd_compatible,
11
+};
12
+use journal_core::file::mmap::MmapMut;
13
+use journal_core::file::{JournalFile, JournalFileOptions, JournalWriter};
14
+use journal_registry::repository;
15
+use std::path::{Path, PathBuf};
16
+
17
+#[allow(unused_imports)]
18
+use tracing::{debug, error, info, instrument, span, warn};
19
+
20
+fn create_chain(path: &Path) -> Result<OwnedChain> {
21
+ let machine_id = load_machine_id()
22
+ .map_err(|e| WriterError::MachineId(format!("failed to load machine ID: {}", e)))?;
23
+
24
+ if path.exists() && !path.is_dir() {
25
+ return Err(WriterError::NotADirectory(path.display().to_string()));
26
+ }
27
+
28
+ if path.to_str().is_none() {
29
+ return Err(WriterError::InvalidPath(
30
+ "path contains invalid UTF-8".to_string(),
31
+ ));
32
+ }
33
+
34
+ let path = PathBuf::from(path).join(machine_id.as_simple().to_string());
35
+ if path.to_str().is_none() {
36
+ return Err(WriterError::InvalidPath(
37
+ "path with machine ID contains invalid UTF-8".to_string(),
38
+ ));
39
+ }
40
+
41
+ std::fs::create_dir_all(&path)?;
42
+
43
+ path.canonicalize()
44
+ .map_err(|e| WriterError::NotADirectory(format!("failed to canonicalize path: {}", e)))?;
45
+ if path.to_str().is_none() {
46
+ return Err(WriterError::InvalidPath(
47
+ "canonicalized path contains invalid UTF-8".to_string(),
48
+ ));
49
+ }
50
+
51
+ OwnedChain::new(path, machine_id)
52
+}
53
+
54
+/// Tracks rotation state for size and count limits
55
+struct RotationState {
56
+ size: Option<(u64, u64)>, // (max, current)
57
+ count: Option<(usize, usize)>, // (max, current)
58
+}
59
+
60
+impl RotationState {
61
+ fn new(rotation_policy: &RotationPolicy) -> Self {
62
+ Self {
63
+ size: rotation_policy.size_of_journal_file.map(|max| (max, 0)),
64
+ count: rotation_policy.number_of_entries.map(|max| (max, 0)),
65
+ }
66
+ }
67
+
68
+ fn should_rotate(&self) -> bool {
69
+ self.size.is_some_and(|(max, current)| current >= max)
70
+ || self.count.is_some_and(|(max, current)| current >= max)
71
+ }
72
+
73
+ fn update(&mut self, journal_writer: &JournalWriter) {
74
+ if let Some((_, ref mut current)) = self.size {
75
+ *current = journal_writer.current_file_size();
76
+ }
77
+ if let Some((_, ref mut current)) = self.count {
78
+ *current += 1;
79
+ }
80
+ }
81
+
82
+ fn reset(&mut self) {
83
+ if let Some((_, ref mut current)) = self.size {
84
+ *current = 0;
85
+ }
86
+ if let Some((_, ref mut current)) = self.count {
87
+ *current = 0;
88
+ }
89
+ }
90
+}
91
+
92
+/// Groups a journal file and its writer together
93
+struct ActiveFile {
94
+ repository_file: repository::File,
95
+ journal_file: JournalFile<MmapMut>,
96
+ writer: JournalWriter,
97
+}
98
+
99
+impl ActiveFile {
100
+ /// Creates a new journal file with the given parameters
101
+ fn create(
102
+ chain: &mut OwnedChain,
103
+ seqnum_id: uuid::Uuid,
104
+ boot_id: uuid::Uuid,
105
+ next_seqnum: u64,
106
+ max_file_size: Option<u64>,
107
+ head_realtime: u64,
108
+ ) -> Result<Self> {
109
+ let head_seqnum = next_seqnum;
110
+
111
+ let repository_file = chain.create_file(seqnum_id, head_seqnum, head_realtime)?;
112
+
113
+ let options = JournalFileOptions::new(chain.machine_id, boot_id, seqnum_id)
114
+ .with_window_size(8 * 1024 * 1024)
115
+ .with_optimized_buckets(None, max_file_size)
116
+ .with_keyed_hash(true);
117
+
118
+ let mut journal_file = JournalFile::create(&repository_file, options)?;
119
+ let writer = JournalWriter::new(&mut journal_file, head_seqnum, boot_id)?;
120
+
121
+ Ok(Self {
122
+ repository_file,
123
+ journal_file,
124
+ writer,
125
+ })
126
+ }
127
+
128
+ /// Creates a successor file, inheriting settings from this file
129
+ fn rotate(
130
+ self,
131
+ chain: &mut OwnedChain,
132
+ max_file_size: Option<u64>,
133
+ head_realtime: u64,
134
+ ) -> Result<Self> {
135
+ let next_seqnum = self.writer.next_seqnum();
136
+ let boot_id = self.writer.boot_id();
137
+
138
+ let head_seqnum = next_seqnum;
139
+
140
+ let seqnum_id = uuid::Uuid::from_bytes(self.journal_file.journal_header_ref().seqnum_id);
141
+ let repository_file = chain.create_file(seqnum_id, head_seqnum, head_realtime)?;
142
+
143
+ let mut journal_file = self
144
+ .journal_file
145
+ .create_successor(&repository_file, max_file_size)?;
146
+ let writer = JournalWriter::new(&mut journal_file, head_seqnum, boot_id)?;
147
+
148
+ Ok(Self {
149
+ repository_file,
150
+ journal_file,
151
+ writer,
152
+ })
153
+ }
154
+
155
+ /// Writes a journal entry
156
+ fn write_entry(&mut self, items: &[&[u8]], realtime: u64, monotonic: u64) -> Result<()> {
157
+ self.writer
158
+ .add_entry(&mut self.journal_file, items, realtime, monotonic)?;
159
+ Ok(())
160
+ }
161
+
162
+ /// Gets the current file size
163
+ fn current_file_size(&self) -> u64 {
164
+ self.writer.current_file_size()
165
+ }
166
+}
167
+
168
+pub struct Log {
169
+ chain: OwnedChain,
170
+ config: Config,
171
+ active_file: Option<ActiveFile>,
172
+ rotation_state: RotationState,
173
+ boot_id: uuid::Uuid,
174
+ seqnum_id: uuid::Uuid,
175
+ current_seqnum: u64,
176
+ remapping_registry: FieldMap,
177
+ clock: RealtimeClock,
178
+}
179
+
180
+impl Log {
181
+ /// Captures both realtime and monotonic timestamps, similar to systemd's dual_timestamp_now().
182
+ ///
183
+ /// Returns (realtime_usec, monotonic_usec) where:
184
+ /// - realtime: microseconds since Unix epoch (CLOCK_REALTIME), monotonically increasing
185
+ /// - monotonic: microseconds since boot (CLOCK_MONOTONIC)
186
+ fn capture_dual_timestamp(&self) -> Result<(u64, u64)> {
187
+ let realtime = self.clock.now().get();
188
+ let monotonic = monotonic_now().map_err(|e| WriterError::Io(e))?.get();
189
+ Ok((realtime, monotonic))
190
+ }
191
+
192
+ /// Creates a new journal log.
193
+ pub fn new(path: &Path, config: Config) -> Result<Self> {
194
+ let chain = create_chain(path)?;
195
+
196
+ let current_seqnum = chain.tail_seqnum()?;
197
+ let boot_id = load_boot_id()?;
198
+ let seqnum_id = uuid::Uuid::new_v4();
199
+ let rotation_state = RotationState::new(&config.rotation_policy);
200
+
201
+ // Initialize clock with last entry timestamp if available
202
+ let clock = if let Some(tail_realtime) = chain.tail_realtime()? {
203
+ RealtimeClock::with_initial(tail_realtime)
204
+ } else {
205
+ RealtimeClock::new()
206
+ };
207
+
208
+ Ok(Log {
209
+ chain,
210
+ config,
211
+ active_file: None,
212
+ rotation_state,
213
+ boot_id,
214
+ seqnum_id,
215
+ current_seqnum,
216
+ remapping_registry: FieldMap::new(),
217
+ clock,
218
+ })
219
+ }
220
+
221
+ /// Writes a journal entry.
222
+ ///
223
+ /// If `source_realtime_usec` is provided, a `_SOURCE_REALTIME_TIMESTAMP` field will be added
224
+ /// to record the original timestamp from the source (in microseconds since Unix epoch).
225
+ /// This is useful when ingesting logs from external sources that have their own timestamps.
226
+ pub fn write_entry(&mut self, items: &[&[u8]], source_realtime_usec: Option<u64>) -> Result<()> {
227
+ if items.is_empty() {
228
+ return Ok(());
229
+ }
230
+
231
+ if self.should_rotate() {
232
+ self.rotate()?;
233
+ self.remapping_registry.clear();
234
+ }
235
+
236
+ // Collect new incompatible field names that need remapping
237
+ let mut new_mappings: Vec<(Vec<u8>, String)> = Vec::new();
238
+
239
+ for item in items {
240
+ if let Some(field_name) = extract_field_name(item) {
241
+ // Skip if already systemd-compatible
242
+ if is_systemd_compatible(field_name) {
243
+ continue;
244
+ }
245
+
246
+ // Skip if already in registry
247
+ if self.remapping_registry.contains_otel_name(field_name) {
248
+ continue;
249
+ }
250
+
251
+ // Generate remapped name and add to list
252
+ let remapped_name = rdp::encode_full(field_name);
253
+ new_mappings.push((field_name.to_vec(), remapped_name));
254
+ }
255
+ }
256
+
257
+ // Write remapping entry if we have new mappings
258
+ if !new_mappings.is_empty() {
259
+ self.write_remapping_entry(&new_mappings)?;
260
+
261
+ // Update registry
262
+ for (otel_name, systemd_name) in new_mappings.iter() {
263
+ self.remapping_registry
264
+ .add_otel_mapping(otel_name.clone(), systemd_name.clone());
265
+ }
266
+ }
267
+
268
+ // Inject _BOOT_ID field - this is required for journalctl boot filtering to work
269
+ let boot_id_field = format!("_BOOT_ID={}", self.boot_id.as_simple());
270
+
271
+ // Transform items to use remapped field names, prepending _BOOT_ID
272
+ let mut transformed_items: Vec<Vec<u8>> = Vec::with_capacity(items.len() + 2);
273
+ let mut items_refs: Vec<&[u8]> = Vec::with_capacity(items.len() + 2);
274
+
275
+ // Prepend _BOOT_ID field first
276
+ transformed_items.push(boot_id_field.into_bytes());
277
+
278
+ // Add _SOURCE_REALTIME_TIMESTAMP if provided
279
+ if let Some(timestamp_usec) = source_realtime_usec {
280
+ let source_timestamp_field = format!("_SOURCE_REALTIME_TIMESTAMP={}", timestamp_usec);
281
+ transformed_items.push(source_timestamp_field.into_bytes());
282
+ }
283
+
284
+ for item in items {
285
+ if let Some(field_name) = extract_field_name(item) {
286
+ if let Some(remapped_name) = self.remapping_registry.get_systemd_name(field_name) {
287
+ // Need to remap: create new item with remapped field name
288
+ let equals_pos = item.iter().position(|&b| b == b'=').unwrap();
289
+ let value = &item[equals_pos..]; // includes '='
290
+ let mut new_item = Vec::with_capacity(remapped_name.len() + value.len());
291
+ new_item.extend_from_slice(remapped_name.as_bytes());
292
+ new_item.extend_from_slice(value);
293
+ transformed_items.push(new_item);
294
+ } else {
295
+ // No remapping needed, use original
296
+ transformed_items.push(item.to_vec());
297
+ }
298
+ } else {
299
+ // No field name (shouldn't happen with valid items)
300
+ transformed_items.push(item.to_vec());
301
+ }
302
+ }
303
+
304
+ // Build references for the underlying write
305
+ for item in &transformed_items {
306
+ items_refs.push(item.as_slice());
307
+ }
308
+
309
+ let (realtime, monotonic) = self.capture_dual_timestamp()?;
310
+
311
+ let active_file = self.active_file.as_mut().unwrap();
312
+ active_file.write_entry(&items_refs, realtime, monotonic)?;
313
+
314
+ self.rotation_state.update(&active_file.writer);
315
+ self.current_seqnum += 1;
316
+
317
+ Ok(())
318
+ }
319
+
320
+ /// Writes a remapping entry containing field name mappings.
321
+ ///
322
+ /// Format:
323
+ /// _BOOT_ID=<boot_id>
324
+ /// ND_REMAPPING=1
325
+ /// ND_<md5_1>=<otel_key_1>
326
+ /// ND_<md5_2>=<otel_key_2>
327
+ /// ...
328
+ fn write_remapping_entry(&mut self, mappings: &[(Vec<u8>, String)]) -> Result<()> {
329
+ let mut remapping_items: Vec<Vec<u8>> = Vec::with_capacity(mappings.len() + 2);
330
+
331
+ // Inject _BOOT_ID field first
332
+ let boot_id_field = format!("_BOOT_ID={}", self.boot_id.as_simple());
333
+ remapping_items.push(boot_id_field.into_bytes());
334
+
335
+ // Add marker field
336
+ remapping_items.push(REMAPPING_MARKER.to_vec());
337
+
338
+ // Add each mapping as ND_<md5>=<otel_key>
339
+ for (otel_name, systemd_name) in mappings {
340
+ let mut item = Vec::with_capacity(systemd_name.len() + 1 + otel_name.len());
341
+ item.extend_from_slice(systemd_name.as_bytes());
342
+ item.push(b'=');
343
+ item.extend_from_slice(otel_name);
344
+ remapping_items.push(item);
345
+ }
346
+
347
+ // Build references
348
+ let items_refs: Vec<&[u8]> = remapping_items.iter().map(|v| v.as_slice()).collect();
349
+
350
+ let (realtime, monotonic) = self.capture_dual_timestamp()?;
351
+
352
+ let active_file = self.active_file.as_mut().unwrap();
353
+ active_file.write_entry(&items_refs, realtime, monotonic)?;
354
+
355
+ self.rotation_state.update(&active_file.writer);
356
+ self.current_seqnum += 1;
357
+
358
+ Ok(())
359
+ }
360
+
361
+ /// Syncs all written data to disk, ensuring durability.
362
+ ///
363
+ /// This should be called after writing a batch of log entries to ensure
364
+ /// they are persisted to disk before acknowledging the request.
365
+ pub fn sync(&mut self) -> Result<()> {
366
+ if let Some(active_file) = &mut self.active_file {
367
+ active_file.journal_file.sync()?;
368
+ }
369
+ Ok(())
370
+ }
371
+
372
+ fn should_rotate(&self) -> bool {
373
+ self.active_file.is_none() || self.rotation_state.should_rotate()
374
+ }
375
+
376
+ #[tracing::instrument(skip_all, fields(active_file))]
377
+ fn rotate(&mut self) -> Result<()> {
378
+ use journal_core::file::JournalState;
379
+
380
+ // Update chain with current file size before rotating
381
+ if let Some(active_file) = &self.active_file {
382
+ self.chain.update_file_size(
383
+ &active_file.repository_file,
384
+ active_file.current_file_size(),
385
+ );
386
+ }
387
+
388
+ // Respect retention policy
389
+ self.chain.retain(&self.config.retention_policy)?;
390
+
391
+ // Create new file (either initial or rotated)
392
+ let max_file_size = self.config.rotation_policy.size_of_journal_file;
393
+ let head_realtime = self.clock.now().get();
394
+ let new_file = if let Some(mut old_file) = self.active_file.take() {
395
+ // Set the old file's state to ARCHIVED before creating successor
396
+ old_file.journal_file.journal_header_mut().state = JournalState::Archived as u8;
397
+ old_file.journal_file.sync()?;
398
+
399
+ old_file.rotate(&mut self.chain, max_file_size, head_realtime)?
400
+ } else {
401
+ ActiveFile::create(
402
+ &mut self.chain,
403
+ self.seqnum_id,
404
+ self.boot_id,
405
+ self.current_seqnum + 1,
406
+ max_file_size,
407
+ head_realtime,
408
+ )?
409
+ };
410
+
411
+ tracing::Span::current().record("new_file", new_file.repository_file.path());
412
+
413
+ self.active_file = Some(new_file);
414
+ self.rotation_state.reset();
415
+
416
+ Ok(())
417
+ }
418
+
419
+ /// Writes a journal entry from a serializable value.
420
+ ///
421
+ /// This method serializes the value to JSON, flattens it, and writes it to the journal.
422
+ /// The flattened structure converts nested JSON into KEY=VALUE pairs suitable for journal entries.
423
+ ///
424
+ /// # Example
425
+ ///
426
+ /// ```no_run
427
+ /// use serde::Serialize;
428
+ /// use journal_log_writer::{Log, Config, RotationPolicy, RetentionPolicy};
429
+ /// use journal_registry::Origin;
430
+ /// use std::path::Path;
431
+ ///
432
+ /// #[derive(Serialize)]
433
+ /// struct LogEntry {
434
+ /// message: String,
435
+ /// level: String,
436
+ /// user: User,
437
+ /// }
438
+ ///
439
+ /// #[derive(Serialize)]
440
+ /// struct User {
441
+ /// id: u64,
442
+ /// name: String,
443
+ /// }
444
+ ///
445
+ /// # fn main() -> Result<(), Box<dyn std::error::Error>> {
446
+ /// let origin = Origin {
447
+ /// machine_id: None,
448
+ /// namespace: None,
449
+ /// source: journal_registry::Source::System,
450
+ /// };
451
+ /// let config = Config::new(origin, RotationPolicy::default(), RetentionPolicy::default());
452
+ /// let mut log = Log::new(Path::new("/tmp/test-journal"), config)?;
453
+ ///
454
+ /// let entry = LogEntry {
455
+ /// message: "User logged in".to_string(),
456
+ /// level: "INFO".to_string(),
457
+ /// user: User {
458
+ /// id: 42,
459
+ /// name: "alice".to_string(),
460
+ /// },
461
+ /// };
462
+ ///
463
+ /// // This will write fields like:
464
+ /// // MESSAGE=User logged in
465
+ /// // LEVEL=INFO
466
+ /// // USER_ID=42
467
+ /// // USER_NAME=alice
468
+ /// log.write_structured(&entry)?;
469
+ /// # Ok(())
470
+ /// # }
471
+ /// ```
472
+ #[cfg(feature = "serde-api")]
473
+ pub fn write_structured<T: serde::Serialize>(&mut self, value: &T) -> Result<()> {
474
+ use flatten_serde_json::flatten;
475
+
476
+ // Serialize to JSON value
477
+ let json_value = serde_json::to_value(value).map_err(|e| {
478
+ WriterError::Serialization(format!("failed to serialize to JSON: {}", e))
479
+ })?;
480
+
481
+ // Flatten the JSON structure - requires a JSON object (Map)
482
+ let flattened = if let serde_json::Value::Object(map) = json_value {
483
+ flatten(&map)
484
+ } else {
485
+ // If not an object, return error
486
+ return Err(WriterError::Serialization(
487
+ "value must be a JSON object, not a primitive or array".to_string(),
488
+ ));
489
+ };
490
+
491
+ // Convert to journal field format (KEY=VALUE)
492
+ let mut fields: Vec<Vec<u8>> = Vec::with_capacity(flattened.len());
493
+
494
+ for (key, value) in flattened.iter() {
495
+ // Convert key to uppercase and replace dots with underscores
496
+ // (journal convention)
497
+ let journal_key = key.to_uppercase().replace('.', "_");
498
+
499
+ // Format as KEY=VALUE
500
+ let field = match value {
501
+ serde_json::Value::String(s) => {
502
+ format!("{}={}", journal_key, s)
503
+ }
504
+ serde_json::Value::Number(n) => {
505
+ format!("{}={}", journal_key, n)
506
+ }
507
+ serde_json::Value::Bool(b) => {
508
+ format!("{}={}", journal_key, if *b { "true" } else { "false" })
509
+ }
510
+ serde_json::Value::Null => {
511
+ format!("{}=", journal_key)
512
+ }
513
+ // Arrays and objects should be flattened already, but just in case
514
+ _ => {
515
+ format!("{}={}", journal_key, value)
516
+ }
517
+ };
518
+
519
+ fields.push(field.into_bytes());
520
+ }
521
+
522
+ // Convert Vec<Vec<u8>> to Vec<&[u8]> for write_entry
523
+ let field_refs: Vec<&[u8]> = fields.iter().map(|f| f.as_slice()).collect();
524
+
525
+ self.write_entry(&field_refs, None)
526
+ }
527
+}
src/crates/journal-log-writer/tests/log_writer.rs
new
+273
@@ -0,0 +1,273 @@
1
+//! Integration tests for journal log writer
2
+//!
3
+//! Tests cover:
4
+//! - Basic entry writing
5
+//! - File rotation (size-based, count-based)
6
+//! - Retention policies
7
+
8
+use journal_common::load_machine_id;
9
+use journal_log_writer::{Config, Log, RetentionPolicy, RotationPolicy};
10
+use journal_registry::Origin;
11
+use std::fs;
12
+use tempfile::TempDir;
13
+
14
+/// Helper to create a default test config
15
+fn test_config() -> Config {
16
+ let origin = Origin {
17
+ machine_id: None,
18
+ namespace: None,
19
+ source: journal_registry::Source::System,
20
+ };
21
+
22
+ Config::new(
23
+ origin,
24
+ RotationPolicy::default(),
25
+ RetentionPolicy::default(),
26
+ )
27
+}
28
+
29
+/// Helper to count journal files in a directory
30
+fn count_journal_files(dir: &TempDir) -> usize {
31
+ let machine_id = load_machine_id().unwrap();
32
+ let journal_dir = dir.path().join(machine_id.as_simple().to_string());
33
+
34
+ fs::read_dir(&journal_dir)
35
+ .unwrap()
36
+ .filter_map(|e| e.ok())
37
+ .filter(|e| {
38
+ e.path()
39
+ .extension()
40
+ .and_then(|s| s.to_str())
41
+ .map(|s| s == "journal")
42
+ .unwrap_or(false)
43
+ })
44
+ .count()
45
+}
46
+
47
+#[test]
48
+fn test_write_single_entry() {
49
+ let dir = TempDir::new().unwrap();
50
+ let config = test_config();
51
+
52
+ let mut log = Log::new(dir.path(), config).unwrap();
53
+
54
+ let entry = [b"MESSAGE=Hello, World!" as &[u8], b"PRIORITY=6"];
55
+
56
+ log.write_entry(&entry, None).unwrap();
57
+ log.sync().unwrap();
58
+
59
+ // Verify file was created
60
+ assert_eq!(count_journal_files(&dir), 1);
61
+}
62
+
63
+#[test]
64
+fn test_write_multiple_entries() {
65
+ let dir = TempDir::new().unwrap();
66
+ let config = test_config();
67
+
68
+ let mut log = Log::new(dir.path(), config).unwrap();
69
+
70
+ // Write 10 entries
71
+ for i in 0..10 {
72
+ let message = format!("MESSAGE=Entry {}", i);
73
+ let entry = [message.as_bytes(), b"PRIORITY=6"];
74
+ log.write_entry(&entry, None).unwrap();
75
+ }
76
+
77
+ log.sync().unwrap();
78
+
79
+ // Should still be 1 file
80
+ assert_eq!(count_journal_files(&dir), 1);
81
+}
82
+
83
+#[test]
84
+fn test_rotation_by_entry_count() {
85
+ let dir = TempDir::new().unwrap();
86
+
87
+ // Rotate after 5 entries
88
+ let rotation = RotationPolicy::default().with_number_of_entries(5);
89
+ let config = test_config().with_rotation_policy(rotation);
90
+
91
+ let mut log = Log::new(dir.path(), config).unwrap();
92
+
93
+ // Write 12 entries (should create 3 files: 5 + 5 + 2)
94
+ for i in 0..12 {
95
+ let message = format!("MESSAGE=Entry {}", i);
96
+ let entry = [message.as_bytes(), b"PRIORITY=6"];
97
+ log.write_entry(&entry, None).unwrap();
98
+ }
99
+
100
+ log.sync().unwrap();
101
+
102
+ assert_eq!(count_journal_files(&dir), 3);
103
+}
104
+
105
+#[test]
106
+fn test_rotation_by_file_size() {
107
+ let dir = TempDir::new().unwrap();
108
+
109
+ // Rotate at ~50KB (small for testing)
110
+ let rotation = RotationPolicy::default().with_size_of_journal_file(50 * 1024);
111
+ let config = test_config().with_rotation_policy(rotation);
112
+
113
+ let mut log = Log::new(dir.path(), config).unwrap();
114
+
115
+ // Write entries with large messages to trigger size-based rotation
116
+ for i in 0..100 {
117
+ let message = format!(
118
+ "MESSAGE=Entry {} with lots of padding: {}",
119
+ i,
120
+ "x".repeat(1000)
121
+ );
122
+ let entry = [message.as_bytes(), b"PRIORITY=6"];
123
+ log.write_entry(&entry, None).unwrap();
124
+ }
125
+
126
+ log.sync().unwrap();
127
+
128
+ // Should have rotated at least once
129
+ assert!(count_journal_files(&dir) > 1);
130
+}
131
+
132
+#[test]
133
+fn test_retention_by_file_count() {
134
+ let dir = TempDir::new().unwrap();
135
+
136
+ // Rotate after 3 entries, keep max 2 files
137
+ let rotation = RotationPolicy::default().with_number_of_entries(3);
138
+ let retention = RetentionPolicy::default().with_number_of_journal_files(2);
139
+ let config = test_config()
140
+ .with_rotation_policy(rotation)
141
+ .with_retention_policy(retention);
142
+
143
+ let mut log = Log::new(dir.path(), config).unwrap();
144
+
145
+ // Write 10 entries (should create 4 files, but keep only 2)
146
+ for i in 0..10 {
147
+ let message = format!("MESSAGE=Entry {}", i);
148
+ let entry = [message.as_bytes(), b"PRIORITY=6"];
149
+ log.write_entry(&entry, None).unwrap();
150
+ }
151
+
152
+ log.sync().unwrap();
153
+
154
+ // Retention is enforced during rotation, so there might be 1 extra file
155
+ // (the active file + retention limit). Check that we're at or near the limit.
156
+ let file_count = count_journal_files(&dir);
157
+ assert!(
158
+ file_count <= 3,
159
+ "Should have at most 3 files (active + retention limit), got {}",
160
+ file_count
161
+ );
162
+}
163
+
164
+#[test]
165
+fn test_retention_by_total_size() {
166
+ let dir = TempDir::new().unwrap();
167
+
168
+ // Rotate after 5 entries, keep max 2 files based on actual data size
169
+ // Note: Journal files pre-allocate space (sparse files), but retention
170
+ // is based on actual data written (append_offset), not logical file size
171
+ let rotation = RotationPolicy::default().with_number_of_entries(5);
172
+
173
+ // Each small entry is ~50-100 bytes, plus journal overhead (~4KB per file)
174
+ // Set limit to ~12KB to allow 2-3 files before triggering retention
175
+ let retention = RetentionPolicy::default().with_size_of_journal_files(12 * 1024);
176
+
177
+ let config = test_config()
178
+ .with_rotation_policy(rotation)
179
+ .with_retention_policy(retention);
180
+
181
+ let mut log = Log::new(dir.path(), config).unwrap();
182
+
183
+ // Write 20 entries (creates 4 files of 5 entries each)
184
+ for i in 0..20 {
185
+ let message = format!("MESSAGE=Entry {}", i);
186
+ let entry = [message.as_bytes(), b"PRIORITY=6"];
187
+ log.write_entry(&entry, None).unwrap();
188
+ }
189
+
190
+ log.sync().unwrap();
191
+
192
+ let file_count = count_journal_files(&dir);
193
+
194
+ // Should have rotated (4 files), but retention should limit to 3
195
+ // (oldest file deleted when total data size exceeds 12KB limit)
196
+ assert!(
197
+ file_count <= 3,
198
+ "Size-based retention should limit files, got {}",
199
+ file_count
200
+ );
201
+}
202
+
203
+#[test]
204
+fn test_empty_entry() {
205
+ let dir = TempDir::new().unwrap();
206
+ let config = test_config();
207
+
208
+ let mut log = Log::new(dir.path(), config).unwrap();
209
+
210
+ // Write empty entry (should be no-op)
211
+ let entry: [&[u8]; 0] = [];
212
+ log.write_entry(&entry, None).unwrap();
213
+
214
+ // Should not create any files (no rotation triggered)
215
+ assert_eq!(count_journal_files(&dir), 0);
216
+}
217
+
218
+#[test]
219
+fn test_boot_id_injection() {
220
+ use journal_common::load_boot_id;
221
+ use std::process::Command;
222
+
223
+ let dir = TempDir::new().unwrap();
224
+ let config = test_config();
225
+
226
+ let mut log = Log::new(dir.path(), config).unwrap();
227
+
228
+ // Write a single entry
229
+ let entry = [b"MESSAGE=Test entry" as &[u8], b"PRIORITY=6"];
230
+ log.write_entry(&entry, None).unwrap();
231
+ log.sync().unwrap();
232
+
233
+ // Find the created journal file
234
+ let machine_id = load_machine_id().unwrap();
235
+ let journal_dir = dir.path().join(machine_id.as_simple().to_string());
236
+ let journal_files: Vec<_> = fs::read_dir(&journal_dir)
237
+ .unwrap()
238
+ .filter_map(|e| e.ok())
239
+ .filter(|e| {
240
+ e.path()
241
+ .extension()
242
+ .and_then(|s| s.to_str())
243
+ .map(|s| s == "journal")
244
+ .unwrap_or(false)
245
+ })
246
+ .collect();
247
+
248
+ assert_eq!(journal_files.len(), 1, "Should have created exactly one journal file");
249
+
250
+ let journal_path = journal_files[0].path();
251
+ let boot_id = load_boot_id().unwrap();
252
+ let expected_boot_id = boot_id.as_simple().to_string();
253
+
254
+ // Use journalctl to verify _BOOT_ID field is present
255
+ let output = Command::new("journalctl")
256
+ .arg("--output=json")
257
+ .arg("--file")
258
+ .arg(&journal_path)
259
+ .output()
260
+ .expect("Failed to run journalctl");
261
+
262
+ assert!(output.status.success(), "journalctl should succeed");
263
+
264
+ let output_str = String::from_utf8_lossy(&output.stdout);
265
+
266
+ // Check that the output contains the expected _BOOT_ID field
267
+ let boot_id_field = format!("\"_BOOT_ID\":\"{}\"", expected_boot_id);
268
+ assert!(
269
+ output_str.contains(&boot_id_field),
270
+ "_BOOT_ID field with value {} should be present in journal entry output",
271
+ expected_boot_id
272
+ );
273
+}
src/crates/journal-registry/Cargo.toml
new
+24
@@ -0,0 +1,24 @@
1
+[package]
2
+name = "journal-registry"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dependencies]
11
+allocative = { workspace = true, optional = true }
12
+notify = { workspace = true }
13
+parking_lot = { workspace = true }
14
+serde = { workspace = true, features = ["derive"] }
15
+thiserror = { workspace = true }
16
+tokio = { workspace = true, features = ["sync"] }
17
+tracing = { workspace = true }
18
+uuid = { workspace = true, features = ["v4", "serde"] }
19
+walkdir = { workspace = true }
20
+
21
+journal-common = { workspace = true }
22
+
23
+[features]
24
+allocative = ["dep:allocative", "journal-common/allocative"]
src/crates/journal-registry/README.md
new
+69
@@ -0,0 +1,69 @@
1
+# journal-registry
2
+
3
+This crate watches directories for journal files, parses their metadata from
4
+filenames, organizes them efficiently, and keeps the collection updated as
5
+files are created, rotated, or deleted.
6
+
7
+## How to use it
8
+
9
+Start by creating a monitor and registry, then watch directories:
10
+
11
+```rust
12
+use journal_registry::{Registry, Monitor};
13
+
14
+#[tokio::main]
15
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
16
+ let (monitor, mut event_receiver) = Monitor::new()?;
17
+ let registry = Registry::new(monitor);
18
+
19
+ registry.watch_directory("/var/log/journal")?;
20
+
21
+ // Process filesystem events in the background
22
+ let registry_clone = registry.clone();
23
+ tokio::spawn(async move {
24
+ while let Some(event) = event_receiver.recv().await {
25
+ registry_clone.process_event(event).ok();
26
+ }
27
+ });
28
+
29
+ Ok(())
30
+}
31
+```
32
+
33
+Query for files in a time range:
34
+
35
+```rust
36
+let files = registry.find_files_in_range(start_sec, end_sec)?;
37
+
38
+for file_info in files {
39
+ println!("{}", file_info.file.path());
40
+}
41
+```
42
+
43
+Update metadata after indexing:
44
+
45
+```rust
46
+registry.update_time_range(&file, start_time, end_time, indexed_at, online);
47
+```
48
+
49
+## How it works
50
+
51
+Journal files follow systemd's naming convention. Active files are named
52
+like `system.journal` or `user-1000.journal`. Archived files append
53
+metadata: `system@<seqnum_id>-<head_seqnum>-<head_realtime>.journal`.
54
+Corrupted files end with `.journal~`.
55
+
56
+The registry organizes files into a three-level hierarchy:
57
+directories contain origins (system, user, remote), and each origin has a
58
+chain of files sorted by status and time. Disposed files come first,
59
+followed by archived files in chronological order, with the active file
60
+last. This ordering makes time-range queries efficient.
61
+
62
+Files start with unknown time ranges. After you index them and call
63
+`update_time_range`, the registry uses this metadata to filter queries
64
+appropriately. Files with bounded ranges are included only if they overlap
65
+the requested time window, while unknown and active files are always included.
66
+
67
+The monitor watches directories recursively and sends create, delete, and
68
+rename events through an async channel. The registry processes these to keep
69
+the collection current.
src/crates/journal-registry/src/lib.rs
new
+46
@@ -0,0 +1,46 @@
1
+//! Journal file registry and repository
2
+//!
3
+//! This crate provides functionality for discovering, tracking, and monitoring
4
+//! systemd journal files in directories.
5
+//!
6
+//! ## Key Components
7
+//!
8
+//! - **Repository**: Types for representing journal files and organizing them into chains
9
+//! - **Registry**: High-level interface for watching directories and tracking file changes
10
+//!
11
+//! ## Usage
12
+//!
13
+//! ```no_run
14
+//! use journal_registry::{Registry, Monitor};
15
+//! use journal_common::Seconds;
16
+//!
17
+//! # fn main() -> Result<(), Box<dyn std::error::Error>> {
18
+//! let (monitor, mut event_receiver) = Monitor::new()?;
19
+//! let registry = Registry::new(monitor);
20
+//!
21
+//! // Watch a directory for journal files
22
+//! registry.watch_directory("/var/log/journal")?;
23
+//!
24
+//! // Process file system events in background
25
+//! let registry_clone = registry.clone();
26
+//! tokio::spawn(async move {
27
+//! while let Some(event) = event_receiver.recv().await {
28
+//! if let Err(e) = registry_clone.process_event(event) {
29
+//! eprintln!("Error processing event: {}", e);
30
+//! }
31
+//! }
32
+//! });
33
+//!
34
+//! // Find files in a time range (seconds since epoch)
35
+//! let files = registry.find_files_in_range(Seconds(1000000), Seconds(2000000))?;
36
+//! # Ok(())
37
+//! # }
38
+//! ```
39
+
40
+pub mod registry;
41
+pub mod repository;
42
+pub mod time_range;
43
+
44
+pub use registry::{Monitor, Registry, RegistryError};
45
+pub use repository::{File, FileInfo, Origin, Source, Status};
46
+pub use time_range::TimeRange;
src/crates/journal-registry/src/registry/error.rs
new
+21
@@ -0,0 +1,21 @@
1
+use crate::repository::RepositoryError;
2
+use thiserror::Error;
3
+
4
+/// Errors that can occur when working with the journal registry
5
+#[derive(Debug, Error)]
6
+pub enum RegistryError {
7
+ /// Error from the file system watcher
8
+ #[error("File system watcher error: {0}")]
9
+ Notify(#[from] notify::Error),
10
+
11
+ /// I/O error when reading or scanning directories
12
+ #[error("I/O error: {0}")]
13
+ Io(#[from] std::io::Error),
14
+
15
+ /// Error from the underlying repository
16
+ #[error("Repository error: {0}")]
17
+ Repository(#[from] RepositoryError),
18
+}
19
+
20
+/// A specialized Result type for journal registry operations
21
+pub type Result<T> = std::result::Result<T, RegistryError>;
src/crates/journal-registry/src/registry/mod.rs
new
+332
@@ -0,0 +1,332 @@
1
+//! Journal file registry with monitoring and metadata tracking
2
+//!
3
+//! This module provides the complete infrastructure for tracking journal files,
4
+//! including file system monitoring, metadata management, and file collection.
5
+
6
+pub mod error;
7
+pub use error::RegistryError;
8
+
9
+use crate::registry::error::Result;
10
+use crate::repository::{Repository as BaseRepository, scan_journal_files};
11
+use crate::{File, FileInfo, TimeRange};
12
+use journal_common::Seconds;
13
+use journal_common::collections::{HashMap, HashSet};
14
+use notify::{
15
+ Event,
16
+ event::{EventKind, ModifyKind, RenameMode},
17
+};
18
+use parking_lot::RwLock;
19
+use std::sync::Arc;
20
+use tracing::{debug, error, info, trace, warn};
21
+
22
+mod monitor;
23
+pub use monitor::Monitor;
24
+
25
+// ============================================================================
26
+// Repository with Metadata
27
+// ============================================================================
28
+
29
+/// Repository that tracks journal files with metadata
30
+///
31
+/// This wraps the base repository and automatically maintains time range metadata
32
+/// for each file. Metadata starts as Unknown and can be updated when computed.
33
+struct Repository {
34
+ base: BaseRepository,
35
+ file_metadata: HashMap<File, FileInfo>,
36
+}
37
+
38
+impl Repository {
39
+ /// Create a new empty repository
40
+ fn new() -> Self {
41
+ Self {
42
+ base: BaseRepository::default(),
43
+ file_metadata: HashMap::default(),
44
+ }
45
+ }
46
+
47
+ /// Insert a file to the repository
48
+ fn insert(&mut self, file: File) -> Result<()> {
49
+ let file_info = FileInfo {
50
+ file: file.clone(),
51
+ time_range: TimeRange::Unknown,
52
+ };
53
+
54
+ self.base.insert(file.clone())?;
55
+ self.file_metadata.insert(file, file_info);
56
+
57
+ Ok(())
58
+ }
59
+
60
+ /// Remove a file from the repository
61
+ fn remove(&mut self, file: &File) -> Result<()> {
62
+ self.base.remove(file)?;
63
+ self.file_metadata.remove(file);
64
+ Ok(())
65
+ }
66
+
67
+ /// Remove all files from a directory
68
+ fn remove_directory(&mut self, path: &str) {
69
+ self.base.remove_directory(path);
70
+ self.file_metadata
71
+ .retain(|file, _| file.dir().ok().map(|dir| dir != path).unwrap_or(true));
72
+ }
73
+
74
+ /// Find files in a time range
75
+ ///
76
+ /// Uses indexed metadata when available to filter out files that don't
77
+ /// overlap with the requested time range. Falls back to base repository
78
+ /// logic for files with unknown time ranges.
79
+ fn find_files_in_range(&self, start: Seconds, end: Seconds) -> Vec<FileInfo> {
80
+ let files: Vec<File> = self.base.find_files_in_range(start, end);
81
+
82
+ files
83
+ .into_iter()
84
+ .filter_map(|file| {
85
+ let file_info =
86
+ self.file_metadata
87
+ .get(&file)
88
+ .cloned()
89
+ .unwrap_or_else(|| FileInfo {
90
+ file: file.clone(),
91
+ time_range: TimeRange::Unknown,
92
+ });
93
+
94
+ // Filter based on time range metadata if available
95
+ let include = match file_info.time_range {
96
+ TimeRange::Unknown => {
97
+ // Don't know the time range yet, include it to be safe
98
+ true
99
+ }
100
+ TimeRange::Active { end: _file_end, .. } => {
101
+ // We could have the following check: file_end >= start
102
+ // However, imagine someone panning outside of the
103
+ // active's file time range and then going back to
104
+ // `now` with a small histogram time-range...
105
+ true
106
+ }
107
+ TimeRange::Bounded {
108
+ start: file_start,
109
+ end: file_end,
110
+ ..
111
+ } => {
112
+ // Archived file: check for exact overlap
113
+ // File range [file_start, file_end) overlaps with [start, end) if:
114
+ // file_start < end && file_end > start
115
+ file_start.0 < end.0 && file_end.0 > start.0
116
+ }
117
+ };
118
+
119
+ if include { Some(file_info) } else { None }
120
+ })
121
+ .collect()
122
+ }
123
+
124
+ /// Update time range metadata for a file
125
+ fn update_file_info(&mut self, file_info: FileInfo) {
126
+ let file = file_info.file.clone();
127
+ self.file_metadata.insert(file, file_info);
128
+ }
129
+}
130
+
131
+impl Default for Repository {
132
+ fn default() -> Self {
133
+ Self::new()
134
+ }
135
+}
136
+
137
+// ============================================================================
138
+// Registry
139
+// ============================================================================
140
+
141
+/// Internal state for Registry
142
+struct RegistryInner {
143
+ repository: Repository,
144
+ watched_directories: HashSet<String>,
145
+ monitor: Monitor,
146
+}
147
+
148
+/// Coordinates file monitoring and repository management (thread-safe)
149
+#[derive(Clone)]
150
+pub struct Registry {
151
+ inner: Arc<RwLock<RegistryInner>>,
152
+}
153
+
154
+impl Registry {
155
+ /// Create a new registry with the given monitor
156
+ pub fn new(monitor: Monitor) -> Self {
157
+ let inner = RegistryInner {
158
+ repository: Repository::new(),
159
+ watched_directories: HashSet::default(),
160
+ monitor,
161
+ };
162
+
163
+ Self {
164
+ inner: Arc::new(RwLock::new(inner)),
165
+ }
166
+ }
167
+
168
+ /// Watch a directory for journal files
169
+ ///
170
+ /// Performs an initial scan to discover existing files, then monitors for changes.
171
+ pub fn watch_directory(&self, path: &str) -> Result<()> {
172
+ let mut inner = self.inner.write();
173
+
174
+ if inner.watched_directories.contains(path) {
175
+ warn!("Directory {} is already being watched", path);
176
+ return Ok(());
177
+ }
178
+
179
+ info!("scanning directory: {}", path);
180
+ let files = scan_journal_files(path)?;
181
+ info!("found {} journal files in {}", files.len(), path);
182
+
183
+ // Start watching with notify
184
+ inner.monitor.watch_directory(path)?;
185
+ inner.watched_directories.insert(String::from(path));
186
+
187
+ // Insert all discovered files into repository (automatically initializes metadata)
188
+ for file in files {
189
+ debug!("adding file to repository: {:?}", file.path());
190
+
191
+ if let Err(e) = inner.repository.insert(file) {
192
+ error!("failed to insert file into repository: {}", e);
193
+ }
194
+ }
195
+
196
+ info!(
197
+ "now watching directory: {} (total directories: {})",
198
+ path,
199
+ inner.watched_directories.len()
200
+ );
201
+ Ok(())
202
+ }
203
+
204
+ /// Stop watching a directory and remove its files from the repository
205
+ pub fn unwatch_directory(&self, path: &str) -> Result<()> {
206
+ let mut inner = self.inner.write();
207
+
208
+ if !inner.watched_directories.contains(path) {
209
+ warn!("directory {} is not being watched", path);
210
+ return Ok(());
211
+ }
212
+
213
+ inner.monitor.unwatch_directory(path)?;
214
+ inner.repository.remove_directory(path); // Handles both repository and metadata cleanup
215
+ inner.watched_directories.remove(path);
216
+
217
+ info!("stopped watching directory: {}", path);
218
+ Ok(())
219
+ }
220
+
221
+ /// Process a filesystem event from the monitor
222
+ ///
223
+ /// Handles creates, deletes, and renames. Call this for each event from the receiver.
224
+ pub fn process_event(&self, event: Event) -> Result<()> {
225
+ let mut inner = self.inner.write();
226
+
227
+ match event.kind {
228
+ EventKind::Create(_) => {
229
+ for path in &event.paths {
230
+ debug!("adding file to repository: {:?}", path);
231
+
232
+ if let Some(file) = File::from_path(path) {
233
+ if let Err(e) = inner.repository.insert(file) {
234
+ error!("failed to insert file: {}", e);
235
+ }
236
+ } else {
237
+ warn!("path is not a valid journal file: {:?}", path);
238
+ }
239
+ }
240
+ }
241
+ EventKind::Remove(_) => {
242
+ for path in &event.paths {
243
+ debug!("removing file from repository: {:?}", path);
244
+
245
+ if let Some(file) = File::from_path(path) {
246
+ if let Err(e) = inner.repository.remove(&file) {
247
+ error!("failed to remove file: {}", e);
248
+ }
249
+ } else {
250
+ warn!("path is not a valid journal file: {:?}", path);
251
+ }
252
+ }
253
+ }
254
+ EventKind::Modify(ModifyKind::Name(RenameMode::Both)) => {
255
+ // Handle renames: remove old, add new
256
+ if event.paths.len() >= 2 {
257
+ let old_path = &event.paths[0];
258
+ let new_path = &event.paths[1];
259
+ info!("rename event: {:?} -> {:?}", old_path, new_path);
260
+
261
+ if let Some(old_file) = File::from_path(old_path) {
262
+ info!("removing old file: {:?}", old_file.path());
263
+ if let Err(e) = inner.repository.remove(&old_file) {
264
+ error!("failed to remove old file: {}", e);
265
+ }
266
+ }
267
+
268
+ if let Some(new_file) = File::from_path(new_path) {
269
+ info!("inserting new file: {:?}", new_file.path());
270
+ if let Err(e) = inner.repository.insert(new_file) {
271
+ error!("failed to insert new file: {}", e);
272
+ }
273
+ }
274
+ } else {
275
+ error!(
276
+ "rename event with unexpected path count: {:#?}",
277
+ event.paths
278
+ );
279
+ }
280
+ }
281
+ EventKind::Modify(ModifyKind::Name(rename_mode)) => {
282
+ error!("unhandled rename mode: {:?}", rename_mode);
283
+ }
284
+ event_kind => {
285
+ // Ignore other events (content modifications, access, etc.)
286
+ trace!("ignoring notify event kind: {:?}", event_kind);
287
+ }
288
+ }
289
+ Ok(())
290
+ }
291
+
292
+ /// Find files overlapping with a time range
293
+ ///
294
+ /// Uses indexed metadata to filter efficiently. Files with unknown or active time
295
+ /// ranges are always included.
296
+ pub fn find_files_in_range(&self, start: Seconds, end: Seconds) -> Result<Vec<FileInfo>> {
297
+ let inner = self.inner.read();
298
+ Ok(inner.repository.find_files_in_range(start, end))
299
+ }
300
+
301
+ /// Update time range metadata after indexing a file
302
+ pub fn update_time_range(
303
+ &self,
304
+ file: &File,
305
+ start_time: Seconds,
306
+ end_time: Seconds,
307
+ indexed_at: Seconds,
308
+ online: bool,
309
+ ) {
310
+ let mut inner = self.inner.write();
311
+
312
+ let time_range = if online {
313
+ TimeRange::Active {
314
+ start: start_time,
315
+ end: end_time,
316
+ indexed_at: indexed_at,
317
+ }
318
+ } else {
319
+ TimeRange::Bounded {
320
+ start: start_time,
321
+ end: end_time,
322
+ indexed_at: indexed_at,
323
+ }
324
+ };
325
+
326
+ let file_info = FileInfo {
327
+ file: file.clone(),
328
+ time_range,
329
+ };
330
+ inner.repository.update_file_info(file_info);
331
+ }
332
+}
src/crates/journal-registry/src/registry/monitor.rs
new
+46
@@ -0,0 +1,46 @@
1
+use super::error::Result;
2
+use notify::{Event, RecommendedWatcher, RecursiveMode, Watcher};
3
+use std::path::Path;
4
+use tokio::sync::mpsc;
5
+use tracing::warn;
6
+
7
+/// File system watcher that sends events through an async channel
8
+#[derive(Debug)]
9
+pub struct Monitor {
10
+ /// The watcher instance
11
+ watcher: RecommendedWatcher,
12
+}
13
+
14
+impl Monitor {
15
+ /// Create a new monitor with an event receiver channel
16
+ pub fn new() -> Result<(Self, mpsc::UnboundedReceiver<Event>)> {
17
+ let (event_sender, event_receiver) = mpsc::unbounded_channel();
18
+
19
+ let watcher = RecommendedWatcher::new(
20
+ move |res| {
21
+ if let Ok(event) = res {
22
+ if let Err(e) = event_sender.send(event) {
23
+ warn!("Failed to send file system event: receiver dropped ({})", e);
24
+ }
25
+ }
26
+ },
27
+ notify::Config::default(),
28
+ )?;
29
+
30
+ Ok((Self { watcher }, event_receiver))
31
+ }
32
+
33
+ /// Start watching a directory recursively
34
+ pub fn watch_directory(&mut self, path: &str) -> Result<()> {
35
+ self.watcher
36
+ .watch(Path::new(path), RecursiveMode::Recursive)?;
37
+
38
+ Ok(())
39
+ }
40
+
41
+ /// Stop watching a directory
42
+ pub fn unwatch_directory(&mut self, path: &str) -> Result<()> {
43
+ self.watcher.unwatch(Path::new(path))?;
44
+ Ok(())
45
+ }
46
+}
src/crates/journal-registry/src/repository/collection.rs
new
+273
@@ -0,0 +1,273 @@
1
+use crate::repository::error::Result;
2
+use crate::repository::{File, Origin, Status};
3
+use journal_common::Seconds;
4
+use journal_common::collections::{HashMap, VecDeque};
5
+use tracing::error;
6
+
7
+/// An ordered collection of journal files from the same origin
8
+///
9
+/// Files are kept sorted by status and time:
10
+/// - Disposed files (corrupted) come first
11
+/// - Archived files follow in chronological order (by head_realtime)
12
+/// - Active file (if any) comes last
13
+///
14
+/// This ordering is maintained automatically by [`insert_file()`](Self::insert_file)
15
+/// and is critical for correct time-range queries.
16
+#[derive(Debug, Clone, Default)]
17
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
18
+pub struct Chain {
19
+ /// Ordered collection of files maintaining the sorting invariant
20
+ pub(crate) files: VecDeque<File>,
21
+}
22
+
23
+impl Chain {
24
+ /// Insert a file maintaining sorted order (disposed → archived → active)
25
+ pub fn insert_file(&mut self, file: File) {
26
+ let pos = self.files.partition_point(|f| *f < file);
27
+
28
+ if pos < self.files.len() && self.files[pos] == file {
29
+ return;
30
+ }
31
+
32
+ self.files.insert(pos, file.clone());
33
+ }
34
+
35
+ /// Remove a file from the chain
36
+ pub fn remove_file(&mut self, file: &File) {
37
+ // Use partition_point to find where the file would be
38
+ let pos = self.files.partition_point(|f| f < file);
39
+
40
+ // Check if the file at this position matches the one we want to remove
41
+ if pos < self.files.len() && self.files[pos] == *file {
42
+ self.files.remove(pos);
43
+ }
44
+ }
45
+
46
+ pub fn pop_front(&mut self) -> Option<File> {
47
+ self.files.pop_front()
48
+ }
49
+
50
+ pub fn back(&self) -> Option<&File> {
51
+ self.files.back()
52
+ }
53
+
54
+ pub fn is_empty(&self) -> bool {
55
+ self.files.is_empty()
56
+ }
57
+
58
+ pub fn len(&self) -> usize {
59
+ self.files.len()
60
+ }
61
+
62
+ /// Remove files older than cutoff time (microseconds since epoch)
63
+ ///
64
+ /// Active files are never drained.
65
+ pub fn drain(&mut self, cutoff_time: u64) -> impl Iterator<Item = File> + '_ {
66
+ let pos = self.files.partition_point(|file| match file.status() {
67
+ Status::Active => false,
68
+ Status::Archived { head_realtime, .. } => *head_realtime <= cutoff_time,
69
+ Status::Disposed { timestamp, .. } => *timestamp <= cutoff_time,
70
+ });
71
+
72
+ self.files.drain(..pos)
73
+ }
74
+
75
+ /// Find files that overlap with the time range [start, end)
76
+ ///
77
+ /// Extends the provided collection with matching files.
78
+ pub fn find_files_in_range<C>(&self, start: Seconds, end: Seconds, files: &mut C)
79
+ where
80
+ C: Extend<File>,
81
+ {
82
+ if self.files.is_empty() || start >= end {
83
+ return;
84
+ }
85
+
86
+ const USEC_PER_SEC: u64 = std::time::Duration::from_secs(1).as_micros() as u64;
87
+ let start = start.0 as u64 * USEC_PER_SEC;
88
+ let end = end.0 as u64 * USEC_PER_SEC;
89
+
90
+ let pos = self
91
+ .files
92
+ .partition_point(|f| match f.status() {
93
+ Status::Active => false,
94
+ Status::Archived { head_realtime, .. } => *head_realtime < start,
95
+ Status::Disposed { .. } => true,
96
+ })
97
+ .saturating_sub(1);
98
+
99
+ let mut prev_head_realtime = match self.files.get(pos).map(|f| f.status()) {
100
+ Some(Status::Archived { head_realtime, .. }) => Some(*head_realtime),
101
+ _ => None,
102
+ };
103
+
104
+ let mut iter = self.files.iter().skip(pos).peekable();
105
+
106
+ while let Some(file) = iter.next() {
107
+ match file.status() {
108
+ Status::Archived { head_realtime, .. } => {
109
+ if *head_realtime >= end {
110
+ break;
111
+ }
112
+
113
+ // Peek at the next file to determine tail_realtime
114
+ let tail_realtime = if let Some(next_file) = iter.peek() {
115
+ match next_file.status() {
116
+ Status::Active => {
117
+ // We don't know the tail_realtime of the active file
118
+ u64::MAX
119
+ }
120
+ Status::Archived {
121
+ head_realtime: tail_realtime,
122
+ ..
123
+ } => *tail_realtime,
124
+ Status::Disposed { .. } => {
125
+ // This violates chain ordering invariant (disposed should be at front)
126
+ // Cannot determine where current archived file ends, so treat as unbounded
127
+ // to avoid excluding valid data from the current file
128
+ error!(
129
+ "Disposed file found after archived file, violating chain ordering: {:?}",
130
+ next_file.path()
131
+ );
132
+ u64::MAX
133
+ }
134
+ }
135
+ } else {
136
+ // This is the last file and it's archived
137
+ u64::MAX
138
+ };
139
+
140
+ // Check if [head_realtime, tail_realtime) overlaps with [start, end)
141
+ // Overlap occurs when: head_realtime < end && tail_realtime > start
142
+ if *head_realtime < end && tail_realtime > start {
143
+ files.extend(std::iter::once(file.clone()));
144
+ }
145
+
146
+ // Remember this head_realtime for potential active file
147
+ prev_head_realtime = Some(*head_realtime);
148
+ }
149
+ Status::Active => {
150
+ // For active files:
151
+ // - tail_realtime is assumed to be u64::MAX (still being written)
152
+ // - head_realtime is either the previous archived file's head_realtime or u64::MIN
153
+
154
+ let head_realtime = prev_head_realtime.unwrap_or(u64::MIN);
155
+ let tail_realtime = u64::MAX;
156
+
157
+ // Check overlap: active_head < end && active_tail > start
158
+ if head_realtime < end && tail_realtime > start {
159
+ files.extend(std::iter::once(file.clone()));
160
+ }
161
+
162
+ // There should only be one active file at the end
163
+ break;
164
+ }
165
+ Status::Disposed { .. } => {
166
+ // This might happen if the partition point moved
167
+ // us in a disposed file position.
168
+ continue;
169
+ }
170
+ }
171
+ }
172
+ }
173
+}
174
+
175
+#[derive(Default, Debug)]
176
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
177
+pub(super) struct Directory {
178
+ pub(super) chains: HashMap<Origin, Chain>,
179
+}
180
+
181
+/// A repository that organizes journal files by directory and origin
182
+///
183
+/// The repository maintains a three-level hierarchy:
184
+/// ```text
185
+/// Repository
186
+/// └─ Directory (/var/log/journal)
187
+/// └─ Origin (System, User(1000), etc.)
188
+/// └─ Chain (ordered list of files)
189
+/// ```
190
+///
191
+/// This structure allows efficient querying and management of journal files
192
+/// from multiple directories and origins.
193
+#[derive(Default)]
194
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
195
+pub struct Repository {
196
+ /// Maps journal directory paths to their contents
197
+ pub(super) directories: HashMap<String, Directory>,
198
+}
199
+
200
+impl Repository {
201
+ /// Insert a file into the appropriate directory/origin/chain
202
+ pub fn insert(&mut self, file: File) -> Result<()> {
203
+ let dir = file.dir()?.to_string();
204
+
205
+ if let Some(directory) = self.directories.get_mut(&dir) {
206
+ if let Some(chain) = directory.chains.get_mut(file.origin()) {
207
+ chain.insert_file(file);
208
+ } else {
209
+ let origin = file.origin().clone();
210
+ let mut chain = Chain::default();
211
+ chain.insert_file(file);
212
+ directory.chains.insert(origin, chain);
213
+ }
214
+ } else {
215
+ let origin = file.origin().clone();
216
+ let mut chain = Chain::default();
217
+ chain.insert_file(file);
218
+
219
+ let mut directory = Directory::default();
220
+ directory.chains.insert(origin, chain);
221
+
222
+ self.directories.insert(dir, directory);
223
+ }
224
+ Ok(())
225
+ }
226
+
227
+ /// Remove a file and clean up empty chains/directories
228
+ pub fn remove(&mut self, file: &File) -> Result<()> {
229
+ let dir = file.dir()?;
230
+ let mut remove_directory = false;
231
+
232
+ if let Some(directory) = self.directories.get_mut(dir) {
233
+ let mut remove_chain = false;
234
+
235
+ if let Some(chain) = directory.chains.get_mut(file.origin()) {
236
+ chain.remove_file(file);
237
+ remove_chain = chain.is_empty();
238
+ };
239
+
240
+ if remove_chain {
241
+ directory.chains.remove(file.origin());
242
+ }
243
+
244
+ remove_directory = directory.chains.is_empty();
245
+ };
246
+
247
+ if remove_directory {
248
+ self.directories.remove(dir);
249
+ }
250
+ Ok(())
251
+ }
252
+
253
+ /// Remove all files from a directory
254
+ pub fn remove_directory(&mut self, path: &str) {
255
+ self.directories.remove(path);
256
+ }
257
+
258
+ /// Collect all files in the given time range
259
+ pub fn find_files_in_range<C>(&self, start: Seconds, end: Seconds) -> C
260
+ where
261
+ C: FromIterator<File> + Extend<File> + Default,
262
+ {
263
+ let mut files = C::default();
264
+
265
+ for directory in self.directories.values() {
266
+ for chain in directory.chains.values() {
267
+ chain.find_files_in_range(start, end, &mut files);
268
+ }
269
+ }
270
+
271
+ files
272
+ }
273
+}
src/crates/journal-registry/src/repository/error.rs
new
+25
@@ -0,0 +1,25 @@
1
+use std::path::PathBuf;
2
+use thiserror::Error;
3
+
4
+/// Errors that can occur when working with a journal repository
5
+#[derive(Debug, Error)]
6
+pub enum RepositoryError {
7
+ /// I/O error when reading or scanning directories
8
+ #[error("I/O error: {0}")]
9
+ Io(#[from] std::io::Error),
10
+
11
+ /// Error when parsing a journal file path
12
+ #[error("Failed to parse journal file path: {path}")]
13
+ InvalidPath { path: String },
14
+
15
+ /// Error when a path contains invalid UTF-8
16
+ #[error("Path contains invalid UTF-8: {}", .path.display())]
17
+ InvalidUtf8 { path: PathBuf },
18
+
19
+ /// Error from walkdir when scanning directories
20
+ #[error("Directory walk error: {0}")]
21
+ WalkDir(#[from] walkdir::Error),
22
+}
23
+
24
+/// A specialized Result type for journal registry operations
25
+pub type Result<T> = std::result::Result<T, RepositoryError>;
src/crates/journal-registry/src/repository/file.rs
new
+389
@@ -0,0 +1,389 @@
1
+use crate::repository::RepositoryError;
2
+use crate::repository::error::Result;
3
+use serde::{Deserialize, Serialize};
4
+use std::cmp::Ordering;
5
+use std::path::Path;
6
+use std::sync::Arc;
7
+use uuid::Uuid;
8
+
9
+/// Status of a journal file
10
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
11
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
12
+pub enum Status {
13
+ /// Active journal file currently being written to
14
+ Active,
15
+ /// Archived journal file that has been rotated and is no longer being written to
16
+ Archived {
17
+ /// Sequence number ID for ordering entries across files
18
+ #[cfg_attr(feature = "allocative", allocative(skip))]
19
+ seqnum_id: Uuid,
20
+ /// Sequence number of the first entry in this file
21
+ head_seqnum: u64,
22
+ /// Realtime timestamp (microseconds since epoch) of the first entry
23
+ head_realtime: u64,
24
+ },
25
+ /// Disposed (corrupted or incomplete) journal file marked for cleanup
26
+ Disposed {
27
+ /// Timestamp when the file was disposed (microseconds since epoch)
28
+ timestamp: u64,
29
+ /// Sequence number for ordering multiple disposed files
30
+ number: u64,
31
+ },
32
+}
33
+
34
+impl Ord for Status {
35
+ fn cmp(&self, other: &Self) -> Ordering {
36
+ match (self, other) {
37
+ // Disposed files come first, sorted by timestamp then number
38
+ (
39
+ Status::Disposed {
40
+ timestamp: t1,
41
+ number: n1,
42
+ },
43
+ Status::Disposed {
44
+ timestamp: t2,
45
+ number: n2,
46
+ },
47
+ ) => t1.cmp(t2).then_with(|| n1.cmp(n2)),
48
+
49
+ // Disposed always comes before non-disposed
50
+ (Status::Disposed { .. }, _) => Ordering::Less,
51
+ (_, Status::Disposed { .. }) => Ordering::Greater,
52
+
53
+ // Archived files sorted by head_realtime (then seqnum for stability)
54
+ (
55
+ Status::Archived {
56
+ seqnum_id: lhs_seqnum_id,
57
+ head_seqnum: lhs_head_seqnum,
58
+ head_realtime: lhs_head_realtime,
59
+ },
60
+ Status::Archived {
61
+ seqnum_id: rhs_seqnum_id,
62
+ head_seqnum: rhs_head_seqnum,
63
+ head_realtime: rhs_head_realtime,
64
+ },
65
+ ) => lhs_head_realtime
66
+ .cmp(rhs_head_realtime)
67
+ .then_with(|| lhs_seqnum_id.cmp(rhs_seqnum_id))
68
+ .then_with(|| lhs_head_seqnum.cmp(rhs_head_seqnum)),
69
+
70
+ // Archived comes before Active
71
+ (Status::Archived { .. }, Status::Active) => Ordering::Less,
72
+ (Status::Active, Status::Archived { .. }) => Ordering::Greater,
73
+
74
+ // Active files are equal in terms of status ordering
75
+ (Status::Active, Status::Active) => Ordering::Equal,
76
+ }
77
+ }
78
+}
79
+
80
+impl PartialOrd for Status {
81
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
82
+ Some(self.cmp(other))
83
+ }
84
+}
85
+
86
+impl Status {
87
+ /// Parse the journal file status from the end of the path, returning the status and the remaining path
88
+ pub(super) fn parse(path: &str) -> Option<(Self, &str)> {
89
+ if let Some(stem) = path.strip_suffix(".journal") {
90
+ // Check if it's archived (has @ suffix) or active
91
+ if let Some((prefix, suffix)) = stem.rsplit_once('@') {
92
+ // Parse archived format: @seqnum_id-head_seqnum-head_realtime
93
+ let mut parts = suffix.split('-');
94
+
95
+ let seqnum_id = parts.next()?;
96
+ let head_seqnum = parts.next()?;
97
+ let head_realtime = parts.next()?;
98
+
99
+ if parts.next().is_some() {
100
+ return None; // Too many parts
101
+ }
102
+
103
+ let seqnum_id = Uuid::try_parse(seqnum_id).ok()?;
104
+ let head_seqnum = u64::from_str_radix(head_seqnum, 16).ok()?;
105
+ let head_realtime = u64::from_str_radix(head_realtime, 16).ok()?;
106
+
107
+ Some((
108
+ Status::Archived {
109
+ seqnum_id,
110
+ head_seqnum,
111
+ head_realtime,
112
+ },
113
+ prefix,
114
+ ))
115
+ } else {
116
+ // Active journal
117
+ Some((Status::Active, stem))
118
+ }
119
+ } else if let Some(stem) = path.strip_suffix(".journal~") {
120
+ // Disposed format: @timestamp-number.journal~
121
+ let (prefix, suffix) = stem.rsplit_once('@')?;
122
+ let (timestamp, number) = suffix.rsplit_once('-')?;
123
+
124
+ let timestamp = u64::from_str_radix(timestamp, 16).ok()?;
125
+ let number = u64::from_str_radix(number, 16).ok()?;
126
+
127
+ Some((Status::Disposed { timestamp, number }, prefix))
128
+ } else {
129
+ None
130
+ }
131
+ }
132
+}
133
+
134
+/// Source of journal entries
135
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
136
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
137
+pub enum Source {
138
+ /// System-wide journal (system.journal)
139
+ System,
140
+ /// User-specific journal with the given UID
141
+ User(u32),
142
+ /// Journal from a remote host
143
+ Remote(String),
144
+ /// Unknown or non-standard journal type
145
+ Unknown(String),
146
+}
147
+
148
+impl Source {
149
+ /// Parse the journal basename from the end of the path, returning the basename and the remaining path
150
+ pub(super) fn parse(path: &str) -> Option<(Self, &str)> {
151
+ // Split on the last '/' to get directory and basename
152
+ let (dir_path, basename) = path.rsplit_once('/')?;
153
+
154
+ let journal_type = if basename == "system" {
155
+ Source::System
156
+ } else if let Some(uid_str) = basename.strip_prefix("user-") {
157
+ if let Ok(uid) = uid_str.parse::<u32>() {
158
+ Source::User(uid)
159
+ } else {
160
+ Source::Unknown(basename.to_string())
161
+ }
162
+ } else if let Some(remote_host) = basename.strip_prefix("remote-") {
163
+ Source::Remote(remote_host.to_string())
164
+ } else {
165
+ Source::Unknown(basename.to_string())
166
+ };
167
+
168
+ Some((journal_type, dir_path))
169
+ }
170
+}
171
+
172
+/// Origin identifies where a journal file comes from
173
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
174
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
175
+pub struct Origin {
176
+ /// Machine ID from which the journal originates
177
+ #[cfg_attr(feature = "allocative", allocative(skip))]
178
+ pub machine_id: Option<Uuid>,
179
+ /// Optional namespace for isolated journal instances
180
+ pub namespace: Option<String>,
181
+ /// Source type (system, user, remote, or unknown)
182
+ pub source: Source,
183
+}
184
+
185
+#[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)]
186
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
187
+pub(crate) struct FileInner {
188
+ pub(crate) path: String,
189
+ pub(crate) origin: Origin,
190
+ pub(crate) status: Status,
191
+}
192
+
193
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
194
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
195
+pub struct File {
196
+ pub(super) inner: Arc<FileInner>,
197
+}
198
+
199
+impl serde::Serialize for File {
200
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
201
+ where
202
+ S: serde::Serializer,
203
+ {
204
+ self.inner.as_ref().serialize(serializer)
205
+ }
206
+}
207
+
208
+impl<'de> serde::Deserialize<'de> for File {
209
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
210
+ where
211
+ D: serde::Deserializer<'de>,
212
+ {
213
+ let inner = FileInner::deserialize(deserializer)?;
214
+ Ok(File {
215
+ inner: Arc::new(inner),
216
+ })
217
+ }
218
+}
219
+
220
+impl File {
221
+ pub fn path(&self) -> &str {
222
+ &self.inner.path
223
+ }
224
+
225
+ pub fn origin(&self) -> &Origin {
226
+ &self.inner.origin
227
+ }
228
+
229
+ pub fn status(&self) -> &Status {
230
+ &self.inner.status
231
+ }
232
+
233
+ pub fn from_path(path: &Path) -> Option<Self> {
234
+ Self::from_str(path.to_str()?)
235
+ }
236
+
237
+ #[allow(clippy::should_implement_trait)]
238
+ pub fn from_str(path: &str) -> Option<Self> {
239
+ // We only accept absolute paths
240
+ if !path.starts_with("/") {
241
+ return None;
242
+ }
243
+
244
+ // Parse from right to left
245
+ let (status, path_after_status) = Status::parse(path)?;
246
+ let (source, path_after_source) = Source::parse(path_after_status)?;
247
+
248
+ // Try to parse machine ID and namespace from the directory name
249
+ let (machine_id, namespace) = if !path_after_source.is_empty() {
250
+ // Get the last directory component
251
+ let dirname = if let Some((_parent, dir)) = path_after_source.rsplit_once('/') {
252
+ dir
253
+ } else {
254
+ path_after_source
255
+ };
256
+
257
+ if let Some((id_str, ns)) = dirname.split_once('.') {
258
+ // Has namespace
259
+ let machine_id = Uuid::try_parse(id_str).ok()?;
260
+ (Some(machine_id), Some(ns.to_string()))
261
+ } else {
262
+ // No namespace, just machine ID
263
+ let machine_id = Uuid::try_parse(dirname).ok();
264
+ (machine_id, None)
265
+ }
266
+ } else {
267
+ (None, None)
268
+ };
269
+
270
+ let origin = Origin {
271
+ machine_id,
272
+ namespace,
273
+ source,
274
+ };
275
+
276
+ let inner = Arc::new(FileInner {
277
+ path: String::from(path),
278
+ origin,
279
+ status,
280
+ });
281
+
282
+ Some(File { inner })
283
+ }
284
+
285
+ pub fn dir(&self) -> Result<&str> {
286
+ Path::new(&self.inner.path)
287
+ .parent()
288
+ .and_then(|p| {
289
+ if self.inner.origin.machine_id.is_some() {
290
+ p.parent()
291
+ } else {
292
+ Some(p)
293
+ }
294
+ })
295
+ .and_then(|p| p.to_str())
296
+ .ok_or_else(|| RepositoryError::InvalidUtf8 {
297
+ path: Path::new(&self.inner.path).to_path_buf(),
298
+ })
299
+ }
300
+
301
+ /// Check if a path looks like a journal file
302
+ pub fn is_journal_file(path: &str) -> bool {
303
+ path.ends_with(".journal") || path.ends_with(".journal~")
304
+ }
305
+
306
+ /// Check if this is an active journal file that's currently being written to
307
+ pub fn is_active(&self) -> bool {
308
+ matches!(self.inner.status, Status::Active)
309
+ }
310
+
311
+ /// Check if this is an archived journal file
312
+ pub fn is_archived(&self) -> bool {
313
+ matches!(self.inner.status, Status::Archived { .. })
314
+ }
315
+
316
+ /// Check if this is a corrupted/disposed journal file
317
+ pub fn is_disposed(&self) -> bool {
318
+ matches!(self.inner.status, Status::Disposed { .. })
319
+ }
320
+
321
+ /// Check if this contains logs from users
322
+ pub fn is_user(&self) -> bool {
323
+ matches!(self.inner.origin.source, Source::User(_))
324
+ }
325
+
326
+ /// Check if this contains logs from system
327
+ pub fn is_system(&self) -> bool {
328
+ matches!(self.inner.origin.source, Source::System)
329
+ }
330
+
331
+ pub fn is_remote(&self) -> bool {
332
+ matches!(self.inner.origin.source, Source::Remote(_))
333
+ }
334
+
335
+ /// Get the user ID if this is a user journal
336
+ pub fn user_id(&self) -> Option<u32> {
337
+ match &self.inner.origin.source {
338
+ Source::User(uid) => Some(*uid),
339
+ _ => None,
340
+ }
341
+ }
342
+
343
+ /// Get the remote host if this is a remote journal
344
+ pub fn remote_host(&self) -> Option<&str> {
345
+ match &self.inner.origin.source {
346
+ Source::Remote(host) => Some(host.as_str()),
347
+ _ => None,
348
+ }
349
+ }
350
+
351
+ /// Get the namespace if this journal belongs to a namespace
352
+ pub fn namespace(&self) -> Option<&str> {
353
+ self.inner.origin.namespace.as_deref()
354
+ }
355
+}
356
+
357
+impl Ord for File {
358
+ fn cmp(&self, other: &Self) -> Ordering {
359
+ // First compare by status, then by path for stability
360
+ self.inner
361
+ .status
362
+ .cmp(&other.inner.status)
363
+ .then_with(|| self.inner.path.cmp(&other.inner.path))
364
+ }
365
+}
366
+
367
+impl PartialOrd for File {
368
+ fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
369
+ Some(self.cmp(other))
370
+ }
371
+}
372
+
373
+/// Scan a directory recursively for journal files
374
+pub fn scan_journal_files(path: &str) -> Result<Vec<File>> {
375
+ let mut files = Vec::new();
376
+
377
+ for entry in walkdir::WalkDir::new(path).follow_links(false) {
378
+ let entry = entry?;
379
+ let path = entry.path();
380
+
381
+ if path.is_file() {
382
+ if let Some(file) = File::from_path(path) {
383
+ files.push(file);
384
+ }
385
+ }
386
+ }
387
+
388
+ Ok(files)
389
+}
src/crates/journal-registry/src/repository/metadata.rs
new
+16
@@ -0,0 +1,16 @@
1
+//! Metadata types for journal files
2
+//!
3
+//! This module provides metadata tracking for journal files, including time ranges
4
+//! derived from indexing operations.
5
+
6
+use crate::TimeRange;
7
+use crate::repository::File;
8
+
9
+/// Pairs a File with its TimeRange.
10
+#[derive(Debug, Clone)]
11
+pub struct FileInfo {
12
+ /// The journal file
13
+ pub file: File,
14
+ /// Time range from its file index
15
+ pub time_range: TimeRange,
16
+}
src/crates/journal-registry/src/repository/mod.rs
new
+417
@@ -0,0 +1,417 @@
1
+//! Journal file repository
2
+//!
3
+//! This module provides types and data structures for representing and organizing
4
+//! systemd journal files into a queryable repository.
5
+//!
6
+//! ## Key Components
7
+//!
8
+//! - **File**: Represents a journal file with parsed metadata (origin, status)
9
+//! - **Origin**: Identifies where a journal file comes from (system, user, remote)
10
+//! - **Status**: Indicates whether a file is active, archived, or disposed
11
+//! - **Chain**: An ordered collection of journal files from the same origin
12
+//! - **Repository**: The top-level container organizing chains by directory and origin
13
+//! - **FileInfo**: Associates a file with its time range metadata
14
+//! - **TimeRange**: Tracks the temporal bounds of indexed journal files
15
+//!
16
+//! ## Architecture
17
+//!
18
+//! The repository organizes files in a three-level hierarchy:
19
+//! ```text
20
+//! Repository
21
+//! └─ Directory (/var/log/journal)
22
+//! └─ Origin (System, User(1000), Remote("host"))
23
+//! └─ Chain (ordered list of files)
24
+//! ```
25
+//!
26
+//! Files within a chain are kept sorted:
27
+//! - Disposed files (corrupted) come first
28
+//! - Archived files follow in chronological order
29
+//! - Active file (if any) comes last
30
+
31
+// Public modules - accessible to workspace crates via full paths
32
+pub mod collection;
33
+pub mod error;
34
+pub mod file;
35
+pub mod metadata;
36
+
37
+// Re-export only the public API types
38
+pub use crate::repository::file::{File, Origin, Source, Status};
39
+pub use crate::repository::metadata::FileInfo;
40
+
41
+// Re-export workspace-internal types (hidden from public docs)
42
+// These are not in lib.rs exports but accessible via full paths for workspace crates
43
+#[doc(hidden)]
44
+pub use crate::repository::collection::{Chain, Repository};
45
+#[doc(hidden)]
46
+pub use crate::repository::error::RepositoryError;
47
+
48
+// Crate-internal only
49
+pub(crate) use crate::repository::file::scan_journal_files;
50
+
51
+#[cfg(test)]
52
+mod tests {
53
+ use super::*;
54
+ use crate::repository::collection::Chain;
55
+ use crate::repository::file::FileInner;
56
+ use journal_common::Seconds;
57
+ use journal_common::collections::VecDeque;
58
+ use std::sync::Arc;
59
+ use uuid::Uuid;
60
+
61
+ const USEC_PER_SEC: u64 = std::time::Duration::from_secs(1).as_micros() as u64;
62
+
63
+ fn create_test_origin() -> Origin {
64
+ Origin {
65
+ machine_id: Some(Uuid::new_v4()),
66
+ namespace: None,
67
+ source: Source::System,
68
+ }
69
+ }
70
+
71
+ fn create_archived_file(origin: &Origin, head_realtime: u64) -> File {
72
+ let inner = FileInner {
73
+ path: format!("/var/log/journal/system@{}.journal", head_realtime),
74
+ origin: origin.clone(),
75
+ status: Status::Archived {
76
+ seqnum_id: Uuid::new_v4(),
77
+ head_seqnum: 1000 + head_realtime,
78
+ head_realtime,
79
+ },
80
+ };
81
+
82
+ File {
83
+ inner: Arc::new(inner),
84
+ }
85
+ }
86
+
87
+ fn create_active_file(origin: &Origin) -> File {
88
+ let inner = FileInner {
89
+ path: "/var/log/journal/system.journal".to_string(),
90
+ origin: origin.clone(),
91
+ status: Status::Active,
92
+ };
93
+
94
+ File {
95
+ inner: Arc::new(inner),
96
+ }
97
+ }
98
+
99
+ fn create_disposed_file(origin: &Origin, timestamp: u64, number: u64) -> File {
100
+ let inner = FileInner {
101
+ path: format!("/var/log/journal/system@{}-{}.journal~", timestamp, number),
102
+ origin: origin.clone(),
103
+ status: Status::Disposed { timestamp, number },
104
+ };
105
+
106
+ File {
107
+ inner: Arc::new(inner),
108
+ }
109
+ }
110
+
111
+ #[test]
112
+ fn test_find_files_in_range_empty_chain() {
113
+ let chain = Chain {
114
+ files: VecDeque::new(),
115
+ };
116
+
117
+ let mut files = Vec::new();
118
+ chain.find_files_in_range(Seconds(100), Seconds(200), &mut files);
119
+ assert!(files.is_empty());
120
+ }
121
+
122
+ #[test]
123
+ fn test_find_files_in_range_invalid_range() {
124
+ let origin = create_test_origin();
125
+ let mut chain = Chain {
126
+ files: VecDeque::new(),
127
+ };
128
+
129
+ // Add some files
130
+ chain
131
+ .files
132
+ .push_back(create_archived_file(&origin, 100 * USEC_PER_SEC));
133
+ chain
134
+ .files
135
+ .push_back(create_archived_file(&origin, 200 * USEC_PER_SEC));
136
+
137
+ let mut files = Vec::new();
138
+ // Test with start >= end
139
+ chain.find_files_in_range(Seconds(200), Seconds(200), &mut files);
140
+ assert!(files.is_empty());
141
+
142
+ chain.find_files_in_range(Seconds(200), Seconds(100), &mut files);
143
+ assert!(files.is_empty());
144
+ }
145
+
146
+ #[test]
147
+ fn test_find_files_in_range_single_archived() {
148
+ let origin = create_test_origin();
149
+ let mut chain = Chain {
150
+ files: VecDeque::new(),
151
+ };
152
+
153
+ // Single archived file with head_realtime = 150
154
+ let file = create_archived_file(&origin, 150 * USEC_PER_SEC);
155
+ chain.files.push_back(file.clone());
156
+
157
+ // Test range that starts before and ends after the file
158
+ let mut files = Vec::new();
159
+ chain.find_files_in_range(Seconds(100), Seconds(200), &mut files);
160
+ assert_eq!(files.len(), 1);
161
+ assert_eq!(files[0], file);
162
+
163
+ // Test range that starts exactly at head_realtime
164
+ files.clear();
165
+ chain.find_files_in_range(Seconds(150), Seconds(200), &mut files);
166
+ assert_eq!(files.len(), 1);
167
+ assert_eq!(files[0], file);
168
+
169
+ // Test range that ends exactly at head_realtime (should not include)
170
+ files.clear();
171
+ chain.find_files_in_range(Seconds(100), Seconds(150), &mut files);
172
+ assert!(files.is_empty());
173
+
174
+ // Test range entirely before the file
175
+ files.clear();
176
+ chain.find_files_in_range(Seconds(50), Seconds(100), &mut files);
177
+ assert!(files.is_empty());
178
+
179
+ // Test range entirely after (single archived file extends to infinity)
180
+ files.clear();
181
+ chain.find_files_in_range(Seconds(200), Seconds(300), &mut files);
182
+ assert_eq!(files.len(), 1);
183
+ assert_eq!(files[0], file);
184
+ }
185
+
186
+ #[test]
187
+ fn test_find_files_in_range_multiple_archived() {
188
+ let origin = create_test_origin();
189
+ let mut chain = Chain {
190
+ files: VecDeque::new(),
191
+ };
192
+
193
+ // Multiple archived files: 100, 200, 300, 400
194
+ let file1 = create_archived_file(&origin, 100 * USEC_PER_SEC);
195
+ let file2 = create_archived_file(&origin, 200 * USEC_PER_SEC);
196
+ let file3 = create_archived_file(&origin, 300 * USEC_PER_SEC);
197
+ let file4 = create_archived_file(&origin, 400 * USEC_PER_SEC);
198
+
199
+ chain.files.push_back(file1.clone());
200
+ chain.files.push_back(file2.clone());
201
+ chain.files.push_back(file3.clone());
202
+ chain.files.push_back(file4.clone());
203
+
204
+ // Range [150, 350) should include files at 100, 200, and 300 in order
205
+ let mut files = Vec::new();
206
+ chain.find_files_in_range(Seconds(150), Seconds(350), &mut files);
207
+ assert_eq!(files.len(), 3);
208
+ assert_eq!(files[0], file1); // Files returned in chronological order
209
+ assert_eq!(files[1], file2);
210
+ assert_eq!(files[2], file3);
211
+
212
+ // Range [200, 300) should include only file at 200
213
+ files.clear();
214
+ chain.find_files_in_range(Seconds(200), Seconds(300), &mut files);
215
+ assert_eq!(files.len(), 1);
216
+ assert_eq!(files[0], file2);
217
+
218
+ // Range [250, 350) should include files at 200 and 300 in order
219
+ files.clear();
220
+ chain.find_files_in_range(Seconds(250), Seconds(350), &mut files);
221
+ assert_eq!(files.len(), 2);
222
+ assert_eq!(files[0], file2);
223
+ assert_eq!(files[1], file3);
224
+
225
+ // Range [450, 500) should include file at 400 (last file extends to infinity)
226
+ files.clear();
227
+ chain.find_files_in_range(Seconds(450), Seconds(500), &mut files);
228
+ assert_eq!(files.len(), 1);
229
+ assert_eq!(files[0], file4);
230
+ }
231
+
232
+ #[test]
233
+ fn test_find_files_in_range_with_active() {
234
+ let origin = create_test_origin();
235
+ let mut chain = Chain {
236
+ files: VecDeque::new(),
237
+ };
238
+
239
+ // Archived files at 100, 200, then active
240
+ let file1 = create_archived_file(&origin, 100 * USEC_PER_SEC);
241
+ let file2 = create_archived_file(&origin, 200 * USEC_PER_SEC);
242
+ let active = create_active_file(&origin);
243
+
244
+ chain.files.push_back(file1.clone());
245
+ chain.files.push_back(file2.clone());
246
+ chain.files.push_back(active.clone());
247
+
248
+ // Range [150, 250) should include files at 100, 200, and active in order
249
+ let mut files = Vec::new();
250
+ chain.find_files_in_range(Seconds(150), Seconds(250), &mut files);
251
+ assert_eq!(files.len(), 3);
252
+ assert_eq!(files[0], file1); // Archived files in chronological order
253
+ assert_eq!(files[1], file2);
254
+ assert_eq!(files[2], active); // Active file comes last
255
+
256
+ // Range [250, 350) should include only file at 200 and active in order
257
+ files.clear();
258
+ chain.find_files_in_range(Seconds(250), Seconds(350), &mut files);
259
+ assert_eq!(files.len(), 2);
260
+ assert_eq!(files[0], file2);
261
+ assert_eq!(files[1], active);
262
+
263
+ // Range [50, 150) should include file at 100
264
+ files.clear();
265
+ chain.find_files_in_range(Seconds(50), Seconds(150), &mut files);
266
+ assert_eq!(files.len(), 1);
267
+ assert_eq!(files[0], file1);
268
+ }
269
+
270
+ #[test]
271
+ fn test_find_files_in_range_only_active() {
272
+ let origin = create_test_origin();
273
+ let mut chain = Chain {
274
+ files: VecDeque::new(),
275
+ };
276
+
277
+ let active = create_active_file(&origin);
278
+ chain.files.push_back(active.clone());
279
+
280
+ // Active file with no archived files should span from u64::MIN to u64::MAX
281
+ let mut files = Vec::new();
282
+ chain.find_files_in_range(Seconds(0), Seconds(100), &mut files);
283
+ assert_eq!(files.len(), 1);
284
+ assert_eq!(files[0], active);
285
+
286
+ files.clear();
287
+ let start = Seconds(u32::MAX - 100);
288
+ let end = Seconds(u32::MAX);
289
+ chain.find_files_in_range(start, end, &mut files);
290
+ assert_eq!(files.len(), 1);
291
+ assert_eq!(files[0], active);
292
+ }
293
+
294
+ #[test]
295
+ fn test_find_files_in_range_with_disposed() {
296
+ let origin = create_test_origin();
297
+ let mut chain = Chain {
298
+ files: VecDeque::new(),
299
+ };
300
+
301
+ // Disposed files should be at the beginning and should be skipped
302
+ let disposed1 = create_disposed_file(&origin, 50 * USEC_PER_SEC, 1);
303
+ let disposed2 = create_disposed_file(&origin, 60 * USEC_PER_SEC, 2);
304
+ let file1 = create_archived_file(&origin, 100 * USEC_PER_SEC);
305
+ let file2 = create_archived_file(&origin, 200 * USEC_PER_SEC);
306
+
307
+ chain.files.push_back(disposed1);
308
+ chain.files.push_back(disposed2);
309
+ chain.files.push_back(file1.clone());
310
+ chain.files.push_back(file2.clone());
311
+
312
+ // Disposed files should not appear in output, only archived files in order
313
+ let mut files = Vec::new();
314
+ chain.find_files_in_range(Seconds(0), Seconds(300), &mut files);
315
+ assert_eq!(files.len(), 2);
316
+ assert_eq!(files[0], file1); // Files in chronological order
317
+ assert_eq!(files[1], file2);
318
+ }
319
+
320
+ #[test]
321
+ fn test_find_files_in_range_edge_cases() {
322
+ let origin = create_test_origin();
323
+ let mut chain = Chain {
324
+ files: VecDeque::new(),
325
+ };
326
+
327
+ // Files at 100, 200, 300
328
+ let file1 = create_archived_file(&origin, 100 * USEC_PER_SEC);
329
+ let file2 = create_archived_file(&origin, 200 * USEC_PER_SEC);
330
+ let file3 = create_archived_file(&origin, 300 * USEC_PER_SEC);
331
+
332
+ chain.files.push_back(file1.clone());
333
+ chain.files.push_back(file2.clone());
334
+ chain.files.push_back(file3.clone());
335
+
336
+ // Test exact boundaries
337
+ let mut files = Vec::new();
338
+
339
+ // Range [100, 200) should include only file at 100
340
+ chain.find_files_in_range(Seconds(100), Seconds(200), &mut files);
341
+ assert_eq!(files.len(), 1);
342
+ assert_eq!(files[0], file1);
343
+
344
+ // Range [200, 300) should include only file at 200
345
+ files.clear();
346
+ chain.find_files_in_range(Seconds(200), Seconds(300), &mut files);
347
+ assert_eq!(files.len(), 1);
348
+ assert_eq!(files[0], file2);
349
+
350
+ // Range [300, 400) should include only file at 300
351
+ files.clear();
352
+ chain.find_files_in_range(Seconds(300), Seconds(400), &mut files);
353
+ assert_eq!(files.len(), 1);
354
+ assert_eq!(files[0], file3);
355
+
356
+ // Range [199, 201) should include files at 100 and 200 in order
357
+ files.clear();
358
+ chain.find_files_in_range(Seconds(199), Seconds(201), &mut files);
359
+ assert_eq!(files.len(), 2);
360
+ assert_eq!(files[0], file1);
361
+ assert_eq!(files[1], file2);
362
+ }
363
+
364
+ #[test]
365
+ fn test_find_files_in_range_complex_scenario() {
366
+ let origin = create_test_origin();
367
+ let mut chain = Chain {
368
+ files: VecDeque::new(),
369
+ };
370
+
371
+ // Complex scenario with disposed, archived, and active files
372
+ let disposed = create_disposed_file(&origin, 10 * USEC_PER_SEC, 1);
373
+ let file1 = create_archived_file(&origin, 1000 * USEC_PER_SEC);
374
+ let file2 = create_archived_file(&origin, 2000 * USEC_PER_SEC);
375
+ let file3 = create_archived_file(&origin, 3000 * USEC_PER_SEC);
376
+ let file4 = create_archived_file(&origin, 4000 * USEC_PER_SEC);
377
+ let active = create_active_file(&origin);
378
+
379
+ chain.files.push_back(disposed);
380
+ chain.files.push_back(file1.clone());
381
+ chain.files.push_back(file2.clone());
382
+ chain.files.push_back(file3.clone());
383
+ chain.files.push_back(file4.clone());
384
+ chain.files.push_back(active.clone());
385
+
386
+ // Range [1500, 3500) should include files at 1000, 2000, 3000 in chronological order
387
+ let mut files = Vec::new();
388
+ chain.find_files_in_range(Seconds(1500), Seconds(3500), &mut files);
389
+ assert_eq!(files.len(), 3);
390
+ assert_eq!(files[0], file1); // Files returned in chronological order
391
+ assert_eq!(files[1], file2);
392
+ assert_eq!(files[2], file3);
393
+
394
+ // Range [4500, 5000) should include file4 and active in order
395
+ files.clear();
396
+ chain.find_files_in_range(Seconds(4500), Seconds(5000), &mut files);
397
+ assert_eq!(files.len(), 2);
398
+ assert_eq!(files[0], file4); // Last archived file
399
+ assert_eq!(files[1], active); // Active file comes last
400
+
401
+ // Range [500, 1500) should include file at 1000
402
+ files.clear();
403
+ chain.find_files_in_range(Seconds(500), Seconds(1500), &mut files);
404
+ assert_eq!(files.len(), 1);
405
+ assert_eq!(files[0], file1);
406
+
407
+ // Range covering everything - all files in chronological order
408
+ files.clear();
409
+ chain.find_files_in_range(Seconds(0), Seconds(u32::MAX), &mut files);
410
+ assert_eq!(files.len(), 5); // All except disposed
411
+ assert_eq!(files[0], file1); // Chronological order: 1000, 2000, 3000, 4000, active
412
+ assert_eq!(files[1], file2);
413
+ assert_eq!(files[2], file3);
414
+ assert_eq!(files[3], file4);
415
+ assert_eq!(files[4], active);
416
+ }
417
+}
src/crates/journal-registry/src/time_range.rs
new
+27
@@ -0,0 +1,27 @@
1
+//! Time range metadata for indexed journal files
2
+
3
+use journal_common::Seconds;
4
+
5
+/// Time range information for a journal file derived from indexing it.
6
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
7
+pub enum TimeRange {
8
+ /// File has not been indexed yet, time range unknown. These files will
9
+ /// be queued for indexing and reported in subsequent poll cycles.
10
+ Unknown,
11
+
12
+ /// Active file currently being written to. The end time represents
13
+ /// the latest entry seen when the file was indexed, but new entries
14
+ /// may have been written since.
15
+ Active {
16
+ start: Seconds,
17
+ end: Seconds,
18
+ indexed_at: Seconds,
19
+ },
20
+
21
+ /// Archived file with known start and end times.
22
+ Bounded {
23
+ start: Seconds,
24
+ end: Seconds,
25
+ indexed_at: Seconds,
26
+ },
27
+}
src/crates/netdata-log-viewer/README.md
new
+258
@@ -0,0 +1,258 @@
1
+# Netdata Log Viewer Plugin
2
+
3
+A Netdata external plugin for querying and visualizing systemd journal entries with histogram analysis and faceted search.
4
+
5
+## Overview
6
+
7
+This plugin provides a `systemd-journal` function that Netdata can call to query journal entries, compute histograms, and return faceted data for visualization in the Netdata dashboard.
8
+
9
+## Architecture
10
+
11
+```
12
+┌─────────────┐
13
+│ Netdata │
14
+│ Agent │
15
+└──────┬──────┘
16
+ │ stdin/stdout
17
+ │ (plugin protocol)
18
+ ↓
19
+┌──────────────────────────┐
20
+│ journal-viewer-plugin │
21
+│ │
22
+│ ┌───────────────┐ │
23
+│ │ Journal │ │
24
+│ │ Handler │ │
25
+│ └───────────────┘ │
26
+│ ↓ │
27
+│ ┌───────────────┐ │
28
+│ │ Shared State │ │
29
+│ │ (AppState) │ │
30
+│ └───────────────┘ │
31
+│ ↓ │
32
+│ ┌───────────────┐ │
33
+│ │ histogram- │ │
34
+│ │ service │ │
35
+│ └───────────────┘ │
36
+│ ↓ │
37
+│ ┌───────────────┐ │
38
+│ │ journal │ │
39
+│ │ (indexing) │ │
40
+│ └───────────────┘ │
41
+└──────────────────────────┘
42
+ ↓
43
+ ┌─────────┐
44
+ │ Jaeger │ (tracing)
45
+ └─────────┘
46
+```
47
+
48
+## Key Features
49
+
50
+- **Fast histogram computation** using pre-built indexes
51
+- **Faceted search** across journal fields (PRIORITY, HOSTNAME, etc.)
52
+- **Caching** with memory + disk tiers for performance
53
+- **Distributed tracing** via OpenTelemetry/Jaeger
54
+- **Metrics tracking** for function call success/failure rates
55
+
56
+## Quick Start
57
+
58
+### Prerequisites
59
+
60
+1. **Jaeger** (optional, for development visibility):
61
+```bash
62
+# NOTE: Port 4318 avoids conflict with Netdata's otel-plugin on 4317
63
+docker run -d --name jaeger \
64
+ -p 16686:16686 \
65
+ -p 4318:4317 \
66
+ jaegertracing/all-in-one:latest
67
+```
68
+
69
+### Building
70
+
71
+```bash
72
+cargo build --bin journal-viewer-plugin --release
73
+```
74
+
75
+### Running
76
+
77
+The plugin is designed to be spawned by Netdata:
78
+
79
+```bash
80
+# Netdata spawns the plugin automatically when configured
81
+# Configure in /etc/netdata/netdata.conf:
82
+
83
+[plugins]
84
+ journal-viewer = yes
85
+```
86
+
87
+For development/testing:
88
+
89
+```bash
90
+# Set log level
91
+export RUST_LOG="debug"
92
+
93
+# Run Netdata in foreground
94
+sudo netdata -D
95
+
96
+# View traces
97
+open http://localhost:16686
98
+```
99
+
100
+## Development
101
+
102
+See [QUICKSTART.md](./QUICKSTART.md) for the fast development loop.
103
+
104
+See [DEVELOPMENT.md](./DEVELOPMENT.md) for comprehensive documentation.
105
+
106
+### Development Workflow
107
+
108
+1. **Make changes** to the plugin code
109
+2. **Rebuild** (fast, seconds): `cargo build --bin journal-viewer-plugin`
110
+3. **Restart** Netdata: `sudo systemctl restart netdata`
111
+4. **View traces** in Jaeger: http://localhost:16686
112
+5. **Check logs**: `sudo journalctl -u netdata -f`
113
+
114
+### Key Benefits of Current Architecture
115
+
116
+✅ **Simple** - Single binary, single mode (production-only)
117
+✅ **Fast iteration** - Rebuild only the plugin, not all of Netdata
118
+✅ **Observable** - Rich tracing and logging via Jaeger + stderr
119
+✅ **Production parity** - Develop with exact production setup
120
+✅ **No mock infrastructure** - No TCP bridges or test harnesses needed
121
+
122
+## Project Structure
123
+
124
+```
125
+netdata-log-viewer/
126
+├── histogram-service/ # Core business logic (library)
127
+├── journal-viewer-plugin/ # Netdata plugin (binary)
128
+├── types/ # Shared request/response types
129
+├── watcher-plugin/ # DEPRECATED - no longer needed
130
+├── lv/ # DEPRECATED - no longer needed
131
+├── DEVELOPMENT.md # Detailed development guide
132
+├── QUICKSTART.md # Fast reference guide
133
+└── README.md # This file
134
+```
135
+
136
+## Configuration
137
+
138
+The plugin is configured at compile time with sensible defaults:
139
+
140
+- **Journal path**: `/var/log/journal`
141
+- **Cache directory**: `/mnt/ramfs/foyer-storage`
142
+- **Memory cache**: 10,000 entries
143
+- **Disk cache**: 64 MiB
144
+
145
+To customize, edit `create_shared_state()` in `journal-viewer-plugin/src/main.rs`.
146
+
147
+## Observability
148
+
149
+### Tracing (Jaeger)
150
+
151
+View request traces at http://localhost:16686:
152
+- Function call timelines
153
+- Histogram computation duration
154
+- Lock acquisition times
155
+- Error traces
156
+
157
+### Logging (Stderr)
158
+
159
+Control log verbosity with `RUST_LOG`:
160
+```bash
161
+# Debug everything
162
+export RUST_LOG="debug"
163
+
164
+# Selective logging
165
+export RUST_LOG="journal_viewer_plugin=trace,histogram_service=debug,journal=info"
166
+```
167
+
168
+### Metrics (Netdata)
169
+
170
+The plugin reports its own metrics:
171
+- `journal_viewer.journal_calls` - Successful/failed/cancelled function calls
172
+
173
+## Function Interface
174
+
175
+The plugin exposes the `systemd-journal` function:
176
+
177
+**Request**:
178
+```json
179
+{
180
+ "after": 1699000000,
181
+ "before": 1699100000,
182
+ "selections": {
183
+ "PRIORITY": ["3", "4"],
184
+ "_HOSTNAME": ["server1"]
185
+ }
186
+}
187
+```
188
+
189
+**Response**:
190
+```json
191
+{
192
+ "status": 200,
193
+ "facets": [...],
194
+ "histogram": [...],
195
+ "available_histograms": [...],
196
+ "columns": {...},
197
+ "data": [...]
198
+}
199
+```
200
+
201
+## Dependencies
202
+
203
+### Core
204
+- `rt` - Netdata plugin runtime
205
+- `histogram-service` - Histogram computation
206
+- `journal` - Journal file indexing
207
+
208
+### Tracing
209
+- `tracing` - Structured logging
210
+- `opentelemetry` - Distributed tracing
211
+- `opentelemetry-otlp` - OTLP exporter for Jaeger
212
+
213
+### Async Runtime
214
+- `tokio` - Async runtime
215
+
216
+## Performance
217
+
218
+- **Index caching**: Avoids re-reading journal files
219
+- **Parallel processing**: Uses Rayon for CPU-bound work
220
+- **Lock-free where possible**: RwLock allows concurrent reads
221
+- **Efficient filtering**: Pre-built indexes for fast queries
222
+
223
+## Troubleshooting
224
+
225
+### Plugin not starting
226
+
227
+```bash
228
+# Check logs
229
+sudo tail -f /var/log/netdata/error.log
230
+
231
+# Run manually
232
+sudo -u netdata /path/to/journal-viewer-plugin
233
+```
234
+
235
+### No traces in Jaeger
236
+
237
+```bash
238
+# Verify Jaeger is running
239
+docker ps | grep jaeger
240
+
241
+# Check connectivity
242
+nc -zv localhost 4317
243
+```
244
+
245
+### Slow queries
246
+
247
+Check Jaeger traces for:
248
+- Lock contention on shared state
249
+- Cache misses in IndexCache
250
+- Large time ranges
251
+
252
+## License
253
+
254
+[Your license here]
255
+
256
+## Contributing
257
+
258
+[Contributing guidelines here]
src/crates/netdata-log-viewer/journal-function/Cargo.toml
new
+40
@@ -0,0 +1,40 @@
1
+[package]
2
+name = "journal-function"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[features]
11
+allocative = ["dep:allocative"]
12
+
13
+[dependencies]
14
+async-stream = { workspace = true }
15
+async-trait = { workspace = true }
16
+futures = { workspace = true }
17
+serde = { workspace = true }
18
+serde_json = { workspace = true, features = ["preserve_order"] }
19
+thiserror = { workspace = true }
20
+tokio = { workspace = true }
21
+tracing = { workspace = true }
22
+notify = { workspace = true }
23
+foyer = { workspace = true }
24
+parking_lot = { workspace = true, features = ["send_guard"] }
25
+schemars = { workspace = true }
26
+chrono = { workspace = true, features = ["serde"] }
27
+nix = { workspace = true, features = ["user"] }
28
+
29
+journal-core = { workspace = true }
30
+journal-index = { workspace = true }
31
+journal-engine = { workspace = true }
32
+journal-registry = { workspace = true }
33
+rt = { path = "../../netdata-plugin/rt" }
34
+
35
+allocative = { workspace = true, optional = true }
36
+
37
+[dev-dependencies]
38
+chrono = { workspace = true }
39
+clap = { workspace = true, features = ["derive"] }
40
+tracing-subscriber = { workspace = true }
src/crates/netdata-log-viewer/journal-function/src/charts.rs
new
+96
@@ -0,0 +1,96 @@
1
+//! Chart definitions for journal-function metrics
2
+//!
3
+//! This module contains Netdata chart metric structures that track
4
+//! file indexing performance and cache utilization.
5
+
6
+use rt::{ChartHandle, NetdataChart, StdPluginRuntime};
7
+use schemars::JsonSchema;
8
+use serde::{Deserialize, Serialize};
9
+use std::time::Duration;
10
+
11
+/// Container for all journal-function metrics chart handles
12
+pub struct JournalMetrics {
13
+ pub file_indexing: ChartHandle<FileIndexingMetrics>,
14
+ pub bucket_cache: ChartHandle<BucketCacheMetrics>,
15
+ pub bucket_operations: ChartHandle<BucketOperationsMetrics>,
16
+}
17
+
18
+impl JournalMetrics {
19
+ /// Register all metric charts with the plugin runtime
20
+ pub fn new(runtime: &mut StdPluginRuntime) -> Self {
21
+ Self {
22
+ file_indexing: runtime
23
+ .register_chart(FileIndexingMetrics::default(), Duration::from_secs(1)),
24
+ bucket_cache: runtime
25
+ .register_chart(BucketCacheMetrics::default(), Duration::from_secs(1)),
26
+ bucket_operations: runtime
27
+ .register_chart(BucketOperationsMetrics::default(), Duration::from_secs(1)),
28
+ }
29
+ }
30
+}
31
+
32
+/// Metrics for tracking file indexing operations
33
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
34
+#[schemars(
35
+ extend("x-chart-id" = "journal.file_indexing"),
36
+ extend("x-chart-title" = "Journal File Indexing Operations"),
37
+ extend("x-chart-units" = "indexes/s"),
38
+ extend("x-chart-type" = "line"),
39
+ extend("x-chart-family" = "indexing"),
40
+ extend("x-chart-context" = "journal.file_indexing"),
41
+)]
42
+pub struct FileIndexingMetrics {
43
+ /// Number of new file indexes computed (cache miss)
44
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
45
+ pub computed: u64,
46
+ /// Number of file indexes retrieved from cache (cache hit)
47
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
48
+ pub cached: u64,
49
+}
50
+
51
+/// Metrics for tracking bucket response cache state
52
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
53
+#[schemars(
54
+ extend("x-chart-id" = "journal.bucket_cache"),
55
+ extend("x-chart-title" = "Bucket Response Cache"),
56
+ extend("x-chart-units" = "buckets"),
57
+ extend("x-chart-type" = "stacked"),
58
+ extend("x-chart-family" = "cache"),
59
+ extend("x-chart-context" = "journal.bucket_cache"),
60
+)]
61
+pub struct BucketCacheMetrics {
62
+ /// Number of partial bucket responses in cache (still indexing)
63
+ #[schemars(extend("x-dimension-algorithm" = "absolute"))]
64
+ pub partial: u64,
65
+ /// Number of complete bucket responses in cache (fully indexed)
66
+ #[schemars(extend("x-dimension-algorithm" = "absolute"))]
67
+ pub complete: u64,
68
+}
69
+
70
+/// Metrics for tracking bucket response operations
71
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
72
+#[schemars(
73
+ extend("x-chart-id" = "journal.bucket_operations"),
74
+ extend("x-chart-title" = "Bucket Response Operations"),
75
+ extend("x-chart-units" = "buckets/s"),
76
+ extend("x-chart-type" = "line"),
77
+ extend("x-chart-family" = "operations"),
78
+ extend("x-chart-context" = "journal.bucket_operations"),
79
+)]
80
+pub struct BucketOperationsMetrics {
81
+ /// Buckets served as complete from cache
82
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
83
+ pub served_complete: u64,
84
+ /// Buckets served as partial (still indexing)
85
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
86
+ pub served_partial: u64,
87
+ /// Partial buckets promoted to complete
88
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
89
+ pub promoted: u64,
90
+ /// Buckets created (new partial responses)
91
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
92
+ pub created: u64,
93
+ /// Buckets invalidated (removed because covering current time)
94
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
95
+ pub invalidated: u64,
96
+}
src/crates/netdata-log-viewer/journal-function/src/lib.rs
new
+28
@@ -0,0 +1,28 @@
1
+//! Systemd journal function implementation crate.
2
+//!
3
+//! This crate provides the Netdata-specific integration layer for systemd journal querying,
4
+//! including charts/metrics, protocol types, and UI formatting.
5
+//!
6
+//! Core query functionality has been moved to the `journal-query` crate.
7
+
8
+pub mod charts;
9
+pub mod netdata;
10
+
11
+// Re-export types from journal-engine for convenience
12
+pub use journal_engine::{
13
+ BucketRequest, BucketResponse, CellValue, ColumnInfo, Facets, FileIndexCache,
14
+ FileIndexCacheBuilder, FileIndexKey, Histogram, HistogramEngine, LogEntryData, LogQuery,
15
+ QueryTimeRange, Result, Table, batch_compute_file_indexes, calculate_bucket_duration,
16
+ entry_data_to_table,
17
+};
18
+
19
+// Re-export Timeout from foundation (via rt for backward compatibility)
20
+pub use rt::Timeout;
21
+
22
+// Re-export Netdata-specific charts/metrics
23
+pub use charts::{
24
+ BucketCacheMetrics, BucketOperationsMetrics, FileIndexingMetrics, JournalMetrics,
25
+};
26
+
27
+// Re-export registry types from journal_registry
28
+pub use journal_registry::{File, FileInfo, Monitor, Registry, TimeRange};
src/crates/netdata-log-viewer/journal-function/src/netdata/builder.rs
new
+110
@@ -0,0 +1,110 @@
1
+//! High-level builder for Netdata UI responses.
2
+//!
3
+//! This module provides convenience functions for building complete Netdata UI
4
+//! responses from log data and histogram information.
5
+
6
+use crate::netdata::transformations::{TransformationRegistry, systemd_transformations};
7
+use journal_core::Result;
8
+use journal_engine::{CellValue, Histogram, LogEntryData, Table};
9
+use serde_json;
10
+use std::collections::HashMap;
11
+use tracing::warn;
12
+
13
+/// Wrapper around entry_data_to_table that applies Netdata transformations.
14
+fn entry_data_to_table_with_transformations(
15
+ entry_data: &[LogEntryData],
16
+ column_names: Vec<String>,
17
+ transformations: &TransformationRegistry,
18
+) -> Result<Table> {
19
+ // Create table with same column structure
20
+ let mut all_columns = vec!["timestamp".to_string()];
21
+ all_columns.extend(column_names.clone());
22
+ let mut transformed_table = Table::new(all_columns);
23
+
24
+ // Create a mapping from column name to index for fast lookup
25
+ let column_map: HashMap<&str, usize> = column_names
26
+ .iter()
27
+ .enumerate()
28
+ .map(|(idx, name)| (name.as_str(), idx + 1)) // +1 because timestamp is at index 0
29
+ .collect();
30
+
31
+ // Process each entry with transformations
32
+ for data in entry_data {
33
+ let num_cols = column_names.len() + 1;
34
+ let mut row = vec![CellValue::new(None); num_cols];
35
+
36
+ // First column: timestamp (transformed)
37
+ row[0] = transformations.transform_field("timestamp", Some(data.timestamp.to_string()));
38
+
39
+ // Extract and transform requested fields
40
+ for pair in &data.fields {
41
+ if let Some(&col_idx) = column_map.get(pair.field()) {
42
+ row[col_idx] =
43
+ transformations.transform_field(pair.field(), Some(pair.value().to_string()));
44
+ }
45
+ }
46
+
47
+ transformed_table.add_row(row);
48
+ }
49
+
50
+ Ok(transformed_table)
51
+}
52
+
53
+/// Build a complete Netdata UI response from log entries and histogram data.
54
+///
55
+/// This is a high-level convenience function that:
56
+/// 1. Generates column schema from histogram response
57
+/// 2. Builds a table with the log entries and discovered fields
58
+/// 3. Transforms the table to Netdata UI JSON format
59
+/// 4. Returns both the column schema and data for the UI
60
+///
61
+/// # Arguments
62
+///
63
+/// * `histogram` - The histogram containing discovered fields
64
+/// * `log_entries` - The log entry data to format
65
+///
66
+/// # Returns
67
+///
68
+/// A tuple of `(columns, data)` where:
69
+/// - `columns` is the JSON serialization of the column schema
70
+/// - `data` is the JSON array of formatted log rows
71
+///
72
+/// # Example
73
+///
74
+/// ```ignore
75
+/// use journal_function::netdata::build_ui_response;
76
+///
77
+/// let (columns, data) = build_ui_response(&histogram, &log_entries);
78
+/// ```
79
+pub fn build_ui_response(
80
+ histogram: &Histogram,
81
+ log_entries: &[LogEntryData],
82
+) -> (serde_json::Value, serde_json::Value) {
83
+ if log_entries.is_empty() {
84
+ return (serde_json::json!([]), serde_json::json!([]));
85
+ }
86
+
87
+ // Generate column schema from histogram
88
+ let field_names: Vec<String> = histogram
89
+ .discovered_fields()
90
+ .iter()
91
+ .map(|f| f.to_string())
92
+ .collect();
93
+ let column_schema = super::columns::generate_column_schema(&field_names);
94
+ // Convert to JSON with keys sorted by index (required by UI)
95
+ let columns = super::columns::columns_to_sorted_json(&column_schema);
96
+
97
+ let transformations = systemd_transformations();
98
+
99
+ match entry_data_to_table_with_transformations(log_entries, field_names, &transformations) {
100
+ Ok(table) => {
101
+ // Transform to UI format
102
+ let ui_data_rows = super::response::table_to_netdata_response(&table, &column_schema);
103
+ (columns, serde_json::json!(ui_data_rows))
104
+ }
105
+ Err(e) => {
106
+ warn!("failed to create table from log entries: {}", e);
107
+ (columns, serde_json::json!([]))
108
+ }
109
+ }
110
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/columns.rs
new
+295
@@ -0,0 +1,295 @@
1
+//! Column schema generation for the logs table UI.
2
+//!
3
+//! This module provides types and functions for generating the column schema
4
+//! that defines how log entries should be displayed in the Netdata dashboard.
5
+
6
+use serde::{Deserialize, Serialize};
7
+use serde_json::{Map, Value};
8
+use std::collections::HashMap as StdHashMap;
9
+
10
+/// Filter type for a column in the logs table.
11
+#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)]
12
+#[serde(rename_all = "snake_case")]
13
+pub enum FilterType {
14
+ /// Faceted filtering (for fields with enumerable values)
15
+ Facet,
16
+ /// Range filtering (for numeric/timestamp values)
17
+ Range,
18
+ /// No filtering available
19
+ None,
20
+}
21
+
22
+/// Value transformation options for a column.
23
+#[derive(Debug, Clone, Serialize, Deserialize)]
24
+pub struct ValueOptions {
25
+ /// Transformation to apply (e.g., "datetime_usec", "none")
26
+ pub transform: String,
27
+ /// Number of decimal points for numeric values
28
+ pub decimal_points: u32,
29
+ /// Default value when field is missing
30
+ pub default_value: Option<String>,
31
+}
32
+
33
+impl Default for ValueOptions {
34
+ fn default() -> Self {
35
+ Self {
36
+ transform: "none".to_string(),
37
+ decimal_points: 0,
38
+ default_value: Some("-".to_string()),
39
+ }
40
+ }
41
+}
42
+
43
+/// Complete schema for a single column in the logs table.
44
+#[derive(Debug, Clone, Serialize, Deserialize)]
45
+pub struct ColumnSchema {
46
+ /// Column index (position in the table)
47
+ pub index: usize,
48
+
49
+ /// Column identifier/key
50
+ #[serde(skip)]
51
+ pub key: String,
52
+
53
+ pub id: String,
54
+
55
+ /// Whether this column is a unique key
56
+ pub unique_key: bool,
57
+
58
+ /// Display name for the column
59
+ pub name: String,
60
+
61
+ /// Whether the column is visible by default
62
+ pub visible: bool,
63
+
64
+ /// Data type of the column
65
+ #[serde(rename = "type")]
66
+ pub column_type: String,
67
+
68
+ /// How to visualize the column
69
+ pub visualization: String,
70
+
71
+ /// Value transformation options
72
+ pub value_options: ValueOptions,
73
+
74
+ /// Sort direction
75
+ pub sort: String,
76
+
77
+ /// Whether the column is sortable
78
+ pub sortable: bool,
79
+
80
+ /// Whether the column is sticky (stays visible when scrolling)
81
+ pub sticky: bool,
82
+
83
+ /// Summary function for aggregation
84
+ pub summary: String,
85
+
86
+ /// Filter type for this column
87
+ pub filter: FilterType,
88
+
89
+ /// Whether the column should take full width
90
+ pub full_width: bool,
91
+
92
+ /// Whether text should wrap in the column
93
+ pub wrap: bool,
94
+
95
+ /// Whether the filter should be expanded by default
96
+ pub default_expanded_filter: bool,
97
+
98
+ /// Whether this is a dummy column (for rowOptions)
99
+ #[serde(skip_serializing_if = "Option::is_none")]
100
+ pub dummy: Option<bool>,
101
+}
102
+
103
+impl ColumnSchema {
104
+ /// Create the special "timestamp" column (index 0).
105
+ pub fn timestamp() -> Self {
106
+ Self {
107
+ index: 0,
108
+ id: "timestamp".to_string(),
109
+ key: "timestamp".to_string(),
110
+ unique_key: true,
111
+ name: "Timestamp".to_string(),
112
+ visible: true,
113
+ column_type: "timestamp".to_string(),
114
+ visualization: "value".to_string(),
115
+ value_options: ValueOptions {
116
+ transform: "datetime_usec".to_string(),
117
+ decimal_points: 0,
118
+ default_value: None,
119
+ },
120
+ sort: "ascending".to_string(),
121
+ sortable: false,
122
+ sticky: false,
123
+ summary: "count".to_string(),
124
+ filter: FilterType::Range,
125
+ full_width: false,
126
+ wrap: true,
127
+ default_expanded_filter: false,
128
+ dummy: None,
129
+ }
130
+ }
131
+
132
+ /// Create the special "rowOptions" column (index 1).
133
+ pub fn row_options() -> Self {
134
+ Self {
135
+ index: 1,
136
+ id: "rowOptions".to_string(),
137
+ key: "rowOptions".to_string(),
138
+ unique_key: false,
139
+ name: "rowOptions".to_string(),
140
+ visible: false,
141
+ column_type: "none".to_string(),
142
+ visualization: "rowOptions".to_string(),
143
+ value_options: ValueOptions {
144
+ transform: "none".to_string(),
145
+ decimal_points: 0,
146
+ default_value: None,
147
+ },
148
+ sort: "ascending".to_string(),
149
+ sortable: false,
150
+ sticky: false,
151
+ summary: "count".to_string(),
152
+ filter: FilterType::None,
153
+ full_width: false,
154
+ wrap: false,
155
+ default_expanded_filter: false,
156
+ dummy: Some(true),
157
+ }
158
+ }
159
+
160
+ /// Create a schema for a regular field column.
161
+ ///
162
+ /// # Arguments
163
+ ///
164
+ /// * `index` - Column index (must be >= 2, as 0 and 1 are special columns)
165
+ /// * `field_name` - The journal field name
166
+ pub fn for_field(index: usize, field_name: &str) -> Self {
167
+ debug_assert!(index >= 2);
168
+
169
+ let name = field_name.to_string();
170
+
171
+ // Determine filter type - matches C implementation (facets.c:2733)
172
+ // Default is FACET for all fields, EXCEPT those marked NEVER_FACET
173
+ let filter = match name.as_str() {
174
+ // Fields with FACET_KEY_OPTION_NEVER_FACET in C code
175
+ "MESSAGE" | "ND_JOURNAL_PROCESS" | "ND_JOURNAL_FILE" => FilterType::None,
176
+ // All other fields get facet filter by default
177
+ _ => FilterType::Facet,
178
+ };
179
+
180
+ // Determine visibility - most fields hidden by default
181
+ // Only MESSAGE is visible by default (timestamp is always visible)
182
+ let visible = name == "MESSAGE";
183
+
184
+ // Determine if filter should be expanded by default
185
+ let default_expanded_filter =
186
+ matches!(name.as_str(), "PRIORITY" | "SYSLOG_FACILITY" | "MESSAGE_ID");
187
+
188
+ // MESSAGE gets full width
189
+ let full_width = (name == "MESSAGE") || (name == "log.body");
190
+
191
+ // log.body should not wrap
192
+ let wrap = (name == "MESSAGE") || (name == "log.body");
193
+
194
+ Self {
195
+ index,
196
+ id: name.clone(),
197
+ key: name.clone(),
198
+ unique_key: false,
199
+ name,
200
+ visible,
201
+ column_type: "string".to_string(),
202
+ visualization: "value".to_string(),
203
+ value_options: ValueOptions::default(),
204
+ sort: "ascending".to_string(),
205
+ sortable: false,
206
+ sticky: false,
207
+ summary: "count".to_string(),
208
+ filter,
209
+ full_width,
210
+ wrap,
211
+ default_expanded_filter,
212
+ dummy: None,
213
+ }
214
+ }
215
+}
216
+
217
+/// Generate the complete column schema map for the logs table UI.
218
+///
219
+/// This function creates the full schema including:
220
+/// 1. Special "timestamp" column (index 0) - with range filter
221
+/// 2. Special "rowOptions" column (index 1) - UI-only, no filter
222
+/// 3. All discovered fields from the histogram (index 2+) - with facet or no filter
223
+///
224
+/// Filter type logic matches C implementation (facets.c:2733):
225
+/// - Default: `filter: "facet"` for all fields
226
+/// - Exception: MESSAGE, ND_JOURNAL_PROCESS, ND_JOURNAL_FILE → `filter: "none"`
227
+///
228
+/// **IMPORTANT**: When serializing to JSON, the keys must be sorted by their `index`
229
+/// field to match the order expected by the UI. Use `columns_to_sorted_json()` helper.
230
+///
231
+/// # Arguments
232
+///
233
+/// * `discovered_fields` - Ordered list of field names from HistogramResponse::discovered_fields()
234
+///
235
+/// # Returns
236
+///
237
+/// A HashMap mapping column keys to their schemas.
238
+pub fn generate_column_schema(discovered_fields: &[String]) -> StdHashMap<String, ColumnSchema> {
239
+ let mut columns = StdHashMap::new();
240
+
241
+ // Add special columns first (in index order)
242
+ let timestamp = ColumnSchema::timestamp();
243
+ columns.insert(timestamp.key.clone(), timestamp);
244
+
245
+ let row_options = ColumnSchema::row_options();
246
+ columns.insert(row_options.key.clone(), row_options);
247
+
248
+ // Add discovered fields starting at index 2 (in index order)
249
+ // Skip special columns (timestamp, rowOptions) if they appear in discovered_fields
250
+ let mut index = 2;
251
+ for field_name in discovered_fields.iter() {
252
+ // Skip special column names to avoid overwriting them
253
+ if field_name == "timestamp" || field_name == "rowOptions" {
254
+ continue;
255
+ }
256
+
257
+ let schema = ColumnSchema::for_field(index, field_name);
258
+ columns.insert(schema.key.clone(), schema);
259
+ index += 1;
260
+ }
261
+
262
+ columns
263
+}
264
+
265
+/// Convert column schema HashMap to JSON with keys sorted by index.
266
+///
267
+/// The UI expects column keys in the JSON object to appear in index order (0, 1, 2, ...).
268
+/// This function sorts the HashMap entries by their `index` field before creating the JSON.
269
+///
270
+/// **IMPORTANT**: This requires serde_json's "preserve_order" feature to be enabled,
271
+/// so that `serde_json::Map` uses IndexMap internally and preserves insertion order.
272
+///
273
+/// # Arguments
274
+///
275
+/// * `columns` - HashMap of column schemas
276
+///
277
+/// # Returns
278
+///
279
+/// A JSON Value with columns as an object where keys are ordered by index.
280
+pub fn columns_to_sorted_json(columns: &StdHashMap<String, ColumnSchema>) -> Value {
281
+ // Collect entries and sort by index
282
+ let mut entries: Vec<_> = columns.iter().collect();
283
+ entries.sort_by_key(|(_, schema)| schema.index);
284
+
285
+ // Build JSON object with keys in index order
286
+ // With preserve_order feature, serde_json::Map preserves insertion order
287
+ let mut map = Map::new();
288
+ for (key, schema) in entries {
289
+ if let Ok(value) = serde_json::to_value(schema) {
290
+ map.insert(key.clone(), value);
291
+ }
292
+ }
293
+
294
+ Value::Object(map)
295
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/facets.rs
new
+74
@@ -0,0 +1,74 @@
1
+//! Facet generation for Netdata UI.
2
+//!
3
+//! This module converts histogram responses into facet structures for the
4
+//! Netdata dashboard filtering UI.
5
+
6
+use journal_engine::Histogram;
7
+use super::transformations::TransformationRegistry;
8
+use super::ui_types::{Facet, FacetOption};
9
+use journal_index::FieldValuePair;
10
+use journal_core::collections::HashMap;
11
+
12
+/// Creates a list of facets from a Histogram.
13
+///
14
+/// Aggregates field=value counts across all buckets and groups them by field.
15
+/// Applies transformations to facet option names for display.
16
+pub fn facets(
17
+ histogram_response: &Histogram,
18
+ transformations: &TransformationRegistry,
19
+) -> Vec<Facet> {
20
+ // Aggregate filtered counts for each field=value pair across all buckets
21
+ let mut field_value_counts: HashMap<FieldValuePair, usize> = HashMap::default();
22
+
23
+ for (_, bucket_response) in &histogram_response.buckets {
24
+ for (pair, (_unfiltered, filtered)) in &bucket_response.fv_counts {
25
+ *field_value_counts.entry(pair.clone()).or_insert(0) += filtered;
26
+ }
27
+ }
28
+
29
+ // Group values by field
30
+ let mut field_to_values: HashMap<String, Vec<(String, usize)>> = HashMap::default();
31
+
32
+ for (pair, count) in field_value_counts {
33
+ field_to_values
34
+ .entry(pair.field().to_string())
35
+ .or_default()
36
+ .push((pair.value().to_string(), count));
37
+ }
38
+
39
+ // Create facets with sorted fields and options
40
+ let mut facets = Vec::new();
41
+ let mut field_names: Vec<String> = field_to_values.keys().cloned().collect();
42
+ field_names.sort();
43
+
44
+ for (order, field_name) in field_names.into_iter().enumerate() {
45
+ let Some(values) = field_to_values.get(&field_name) else {
46
+ continue;
47
+ };
48
+ let mut values = values.clone();
49
+ values.sort_by(|a, b| a.0.cmp(&b.0));
50
+
51
+ let options: Vec<FacetOption> = values
52
+ .into_iter()
53
+ .enumerate()
54
+ .map(|(opt_order, (value, count))| {
55
+ let display_name = transformations.transform_value(&field_name, &value);
56
+ FacetOption {
57
+ id: value,
58
+ name: display_name,
59
+ order: opt_order,
60
+ count,
61
+ }
62
+ })
63
+ .collect();
64
+
65
+ facets.push(Facet {
66
+ id: field_name.clone(),
67
+ name: field_name.clone(),
68
+ order,
69
+ options,
70
+ });
71
+ }
72
+
73
+ facets
74
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/histogram.rs
new
+166
@@ -0,0 +1,166 @@
1
+//! Histogram and chart generation for Netdata UI.
2
+//!
3
+//! This module converts histogram responses into chart structures for the
4
+//! Netdata dashboard visualization.
5
+
6
+use super::transformations::TransformationRegistry;
7
+use super::ui_types::{
8
+ AvailableHistogram, Chart, ChartDimensions, ChartPoint, ChartResult, ChartView, DataPoint,
9
+ Histogram,
10
+};
11
+use journal_core::collections::HashSet;
12
+use journal_engine::Histogram as QueryHistogram;
13
+use journal_index::FieldName;
14
+
15
+/// Creates a list of available histograms from a query histogram.
16
+///
17
+/// Returns one available histogram for each indexed field found in the buckets.
18
+pub fn available_histograms(histogram_response: &QueryHistogram) -> Vec<AvailableHistogram> {
19
+ let mut indexed_fields = HashSet::default();
20
+
21
+ for (_, bucket) in &histogram_response.buckets {
22
+ indexed_fields.extend(bucket.indexed_fields());
23
+ }
24
+
25
+ let mut available_histograms = Vec::with_capacity(indexed_fields.len());
26
+ for field_name in indexed_fields {
27
+ let id = field_name.to_string();
28
+ available_histograms.push(AvailableHistogram {
29
+ id: id.clone(),
30
+ name: id,
31
+ order: 0,
32
+ });
33
+ }
34
+
35
+ available_histograms.sort_by(|a, b| a.id.cmp(&b.id));
36
+
37
+ for (order, available_histogram) in available_histograms.iter_mut().enumerate() {
38
+ available_histogram.order = order;
39
+ }
40
+
41
+ available_histograms
42
+}
43
+
44
+/// Creates a Histogram for the given field from a query histogram.
45
+///
46
+/// # Arguments
47
+/// * `histogram_response` - The query histogram to convert
48
+/// * `field` - The field to generate the histogram for
49
+/// * `transformations` - Transformation registry for field value display
50
+pub fn histogram(
51
+ histogram_response: &QueryHistogram,
52
+ field: &FieldName,
53
+ transformations: &TransformationRegistry,
54
+) -> Histogram {
55
+ let field_str = field.as_str();
56
+ Histogram {
57
+ id: String::from(field_str),
58
+ name: String::from(field_str),
59
+ chart: chart_from_histogram(histogram_response, field, transformations),
60
+ }
61
+}
62
+
63
+/// Creates a Chart for the given field from a query histogram.
64
+fn chart_from_histogram(
65
+ histogram_response: &QueryHistogram,
66
+ field: &FieldName,
67
+ transformations: &TransformationRegistry,
68
+) -> Chart {
69
+ let result = chart_result_from_histogram(histogram_response, field, transformations);
70
+ let view = chart_view_from_histogram(histogram_response, field, &result.labels);
71
+
72
+ Chart { view, result }
73
+}
74
+
75
+/// Creates chart result data for the given field from a query histogram.
76
+fn chart_result_from_histogram(
77
+ histogram_response: &QueryHistogram,
78
+ field: &FieldName,
79
+ transformations: &TransformationRegistry,
80
+) -> ChartResult {
81
+ let field_str = field.as_str();
82
+
83
+ // Collect all unique values for the field across all buckets
84
+ let mut values = HashSet::default();
85
+
86
+ for (_, bucket_response) in &histogram_response.buckets {
87
+ for pair in bucket_response.fv_counts.keys() {
88
+ if pair.field() == field_str {
89
+ values.insert(pair.value().to_string());
90
+ }
91
+ }
92
+ }
93
+
94
+ // Sort raw values for consistent ordering
95
+ let mut raw_values: Vec<String> = values.into_iter().collect();
96
+ raw_values.sort();
97
+
98
+ // Transform values for display
99
+ let mut labels: Vec<String> = raw_values
100
+ .iter()
101
+ .map(|v| transformations.transform_value(field_str, v))
102
+ .collect();
103
+
104
+ // Build data array using raw values for lookups
105
+ let mut data = Vec::new();
106
+
107
+ for (request, bucket_response) in &histogram_response.buckets {
108
+ let timestamp = request.start;
109
+ let mut counts = Vec::with_capacity(raw_values.len());
110
+
111
+ for raw_value in &raw_values {
112
+ // Create FieldValuePair for lookup using raw (untransformed) value
113
+ let pair = field.with_value(raw_value);
114
+
115
+ let count = bucket_response
116
+ .fv_counts
117
+ .get(&pair)
118
+ .map(|(_, filtered)| *filtered)
119
+ .unwrap_or(0);
120
+
121
+ counts.push([count, 0, 0]);
122
+ }
123
+
124
+ data.push(DataPoint {
125
+ timestamp: timestamp.0 as u64 * std::time::Duration::from_secs(1).as_millis() as u64,
126
+ items: counts,
127
+ });
128
+ }
129
+
130
+ let point = ChartPoint {
131
+ value: 0,
132
+ arp: 1,
133
+ pa: 2,
134
+ };
135
+
136
+ labels.insert(0, String::from("time"));
137
+
138
+ ChartResult {
139
+ labels,
140
+ point,
141
+ data,
142
+ }
143
+}
144
+
145
+/// Creates chart view metadata for the given field from a query histogram.
146
+fn chart_view_from_histogram(
147
+ histogram_response: &QueryHistogram,
148
+ field: &FieldName,
149
+ labels: &[String],
150
+) -> ChartView {
151
+ let ids: Vec<String> = labels.iter().skip(1).cloned().collect();
152
+ let names = ids.clone();
153
+ let units = std::iter::repeat_n("events".to_string(), ids.len()).collect();
154
+
155
+ let dimensions = ChartDimensions { ids, names, units };
156
+
157
+ ChartView {
158
+ title: format!("Events distribution by {}", field.as_str()),
159
+ after: histogram_response.start_time().0,
160
+ before: histogram_response.end_time().0,
161
+ update_every: histogram_response.bucket_duration().get(),
162
+ units: String::from("units"),
163
+ chart_type: String::from("stackedBar"),
164
+ dimensions,
165
+ }
166
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/mod.rs
new
+38
@@ -0,0 +1,38 @@
1
+//! Netdata-specific formatting and protocol types.
2
+//!
3
+//! This module contains all Netdata-specific logic for the systemd-journal function plugin,
4
+//! including protocol types, UI formatting for logs and histograms, and field transformations.
5
+
6
+pub mod builder;
7
+pub mod columns;
8
+pub mod facets;
9
+pub mod histogram;
10
+pub mod response;
11
+pub mod severity;
12
+pub mod transformations;
13
+pub mod types;
14
+pub mod ui_types;
15
+
16
+// High-level builders
17
+pub use builder::build_ui_response;
18
+
19
+// Protocol types
20
+pub use types::{
21
+ Items, JournalRequest, JournalResponse, MultiSelection, MultiSelectionOption, Pagination,
22
+ RequestParam, RequiredParam, Version,
23
+};
24
+
25
+// Log formatting
26
+pub use columns::FilterType;
27
+pub use severity::Severity;
28
+pub use transformations::{FieldTransformation, TransformationRegistry, systemd_transformations};
29
+
30
+// Histogram/facet formatting
31
+pub use facets::facets;
32
+pub use histogram::{available_histograms, histogram};
33
+
34
+// UI types
35
+pub use ui_types::{
36
+ AvailableHistogram, Chart, ChartDimensions, ChartPoint, ChartResult, ChartView, DataPoint,
37
+ Facet, FacetOption, Histogram, Response,
38
+};
src/crates/netdata-log-viewer/journal-function/src/netdata/response.rs
new
+118
@@ -0,0 +1,118 @@
1
+//! Netdata response formatting.
2
+//!
3
+//! This module converts generic Table structures into the format expected
4
+//! by the Netdata dashboard UI.
5
+
6
+use super::columns::ColumnSchema;
7
+use super::severity::Severity;
8
+use journal_engine::Table;
9
+use serde_json::json;
10
+use std::collections::HashMap;
11
+
12
+/// Transform a table to the Netdata UI data format.
13
+///
14
+/// Converts the table to the format expected by the Netdata dashboard:
15
+/// ```json
16
+/// [
17
+/// [timestamp_usec, {severity: "info"}, val1, val2, ...],
18
+/// [timestamp_usec, {severity: "error"}, val1, val2, ...],
19
+/// ...
20
+/// ]
21
+/// ```
22
+///
23
+/// # Arguments
24
+///
25
+/// * `table` - The table to convert
26
+/// * `column_schema` - The column schema, used for ordering and visibility
27
+///
28
+/// # Returns
29
+///
30
+/// A vector of JSON values, where each value is an array representing one row.
31
+///
32
+/// # Format
33
+///
34
+/// Each row array contains:
35
+/// 1. Timestamp (u64, microseconds) - from first column
36
+/// 2. rowOptions object with severity - calculated from PRIORITY field
37
+/// 3. Field values in schema order - one per non-hidden column
38
+///
39
+/// # Example
40
+///
41
+/// ```ignore
42
+/// use journal_function::logs::entry_data_to_table;
43
+/// use journal_function::netdata::table_to_netdata_response;
44
+///
45
+/// let table = entry_data_to_table(&log_entries, columns, &transformations)?;
46
+/// let column_schema = netdata::generate_column_schema(&field_names);
47
+/// let ui_data = table_to_netdata_response(&table, &column_schema);
48
+/// response.data = serde_json::to_value(ui_data)?;
49
+/// ```
50
+pub fn table_to_netdata_response(
51
+ table: &Table,
52
+ column_schema: &HashMap<String, ColumnSchema>,
53
+) -> Vec<serde_json::Value> {
54
+ let mut rows = Vec::with_capacity(table.row_count());
55
+
56
+ // Build mapping: table column name → table column index
57
+ let col_map: HashMap<&str, usize> = table
58
+ .columns()
59
+ .iter()
60
+ .map(|col| (col.name.as_str(), col.index))
61
+ .collect();
62
+
63
+ // Get timestamp column index (should always be present)
64
+ let timestamp_idx = col_map.get("timestamp").copied().unwrap_or(0);
65
+
66
+ // Get PRIORITY column index (optional, for severity calculation)
67
+ let priority_idx = col_map.get("PRIORITY").copied();
68
+
69
+ // Get non-special columns from schema in index order
70
+ // Exclude timestamp (index 0), rowOptions (index 1), and hidden columns
71
+ let mut schema_cols: Vec<_> = column_schema
72
+ .values()
73
+ .filter(|col| col.key != "timestamp" && col.key != "rowOptions")
74
+ .filter(|col| {
75
+ // Only include visible columns or those explicitly in the table
76
+ col.visible || col_map.contains_key(col.key.as_str())
77
+ })
78
+ .collect();
79
+ schema_cols.sort_by_key(|col| col.index);
80
+
81
+ // Transform each row
82
+ for table_row in table.rows() {
83
+ let mut ui_row = Vec::with_capacity(2 + schema_cols.len());
84
+
85
+ // Element 0: timestamp (as u64, microseconds)
86
+ let timestamp = table_row
87
+ .get(timestamp_idx)
88
+ .and_then(|cell| cell.raw.as_ref())
89
+ .and_then(|s| s.parse::<u64>().ok())
90
+ .unwrap_or(0);
91
+ ui_row.push(json!(timestamp));
92
+
93
+ // Element 1: rowOptions with severity
94
+ let priority_value = priority_idx
95
+ .and_then(|idx| table_row.get(idx))
96
+ .and_then(|cell| cell.raw.as_deref());
97
+ let severity = Severity::from_priority(priority_value);
98
+ ui_row.push(json!({"severity": severity}));
99
+
100
+ // Elements 2+: field values in schema order
101
+ for schema_col in &schema_cols {
102
+ if let Some(&table_idx) = col_map.get(schema_col.key.as_str()) {
103
+ // Use display value from the table
104
+ let value = table_row
105
+ .get(table_idx)
106
+ .and_then(|cell| cell.display.clone());
107
+ ui_row.push(json!(value));
108
+ } else {
109
+ // Column is in schema but not in table data (shouldn't happen normally)
110
+ ui_row.push(json!(null));
111
+ }
112
+ }
113
+
114
+ rows.push(json!(ui_row));
115
+ }
116
+
117
+ rows
118
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/severity.rs
new
+142
@@ -0,0 +1,142 @@
1
+//! Severity levels for log entries.
2
+//!
3
+//! This module provides severity classification based on syslog PRIORITY values,
4
+//! matching the C implementation in systemd-journal-annotations.c.
5
+
6
+use serde::{Deserialize, Serialize};
7
+
8
+/// Log entry severity level.
9
+///
10
+/// Maps syslog PRIORITY values to severity levels for UI display.
11
+/// Implementation matches systemd-journal-annotations.c:256-281.
12
+#[derive(Default, Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
13
+#[serde(rename_all = "lowercase")]
14
+pub enum Severity {
15
+ /// Critical errors (PRIORITY <= 3, LOG_ERR)
16
+ Critical,
17
+
18
+ /// Warnings (PRIORITY == 4, LOG_WARNING)
19
+ Warning,
20
+
21
+ /// Notices (PRIORITY == 5, LOG_NOTICE)
22
+ Notice,
23
+
24
+ /// Debug messages (PRIORITY >= 7, LOG_DEBUG)
25
+ Debug,
26
+
27
+ /// Normal/Info messages (PRIORITY == 6, LOG_INFO, or missing)
28
+ #[default]
29
+ Normal,
30
+}
31
+
32
+impl Severity {
33
+ /// Convert syslog PRIORITY value to severity level.
34
+ ///
35
+ /// Implements the same logic as `syslog_priority_to_facet_severity()`
36
+ /// in systemd-journal-annotations.c:256-281.
37
+ ///
38
+ /// Priority mapping (from syslog.h):
39
+ /// - 0 = LOG_EMERG
40
+ /// - 1 = LOG_ALERT
41
+ /// - 2 = LOG_CRIT
42
+ /// - 3 = LOG_ERR
43
+ /// - 4 = LOG_WARNING
44
+ /// - 5 = LOG_NOTICE
45
+ /// - 6 = LOG_INFO (default)
46
+ /// - 7 = LOG_DEBUG
47
+ ///
48
+ /// # Arguments
49
+ ///
50
+ /// * `priority` - The PRIORITY field value (as string), or None if missing
51
+ ///
52
+ /// # Examples
53
+ ///
54
+ /// ```
55
+ /// use journal_function::netdata::Severity;
56
+ ///
57
+ /// assert_eq!(Severity::from_priority(Some("0")), Severity::Critical);
58
+ /// assert_eq!(Severity::from_priority(Some("3")), Severity::Critical);
59
+ /// assert_eq!(Severity::from_priority(Some("4")), Severity::Warning);
60
+ /// assert_eq!(Severity::from_priority(Some("5")), Severity::Notice);
61
+ /// assert_eq!(Severity::from_priority(Some("6")), Severity::Normal);
62
+ /// assert_eq!(Severity::from_priority(Some("7")), Severity::Debug);
63
+ /// assert_eq!(Severity::from_priority(None), Severity::Normal);
64
+ /// ```
65
+ pub fn from_priority(priority: Option<&str>) -> Self {
66
+ let priority_num = priority.and_then(|s| s.parse::<i32>().ok()).unwrap_or(6); // Default to LOG_INFO if missing or invalid
67
+
68
+ // Matches C code logic exactly
69
+ if priority_num <= 3 {
70
+ // LOG_ERR and below (EMERG, ALERT, CRIT, ERR)
71
+ Severity::Critical
72
+ } else if priority_num <= 4 {
73
+ // LOG_WARNING
74
+ Severity::Warning
75
+ } else if priority_num <= 5 {
76
+ // LOG_NOTICE
77
+ Severity::Notice
78
+ } else if priority_num >= 7 {
79
+ // LOG_DEBUG
80
+ Severity::Debug
81
+ } else {
82
+ // LOG_INFO (6) or anything else
83
+ Severity::Normal
84
+ }
85
+ }
86
+}
87
+
88
+#[cfg(test)]
89
+mod tests {
90
+ use super::*;
91
+
92
+ #[test]
93
+ fn test_severity_from_priority() {
94
+ // Critical: 0-3 (EMERG, ALERT, CRIT, ERR)
95
+ assert_eq!(Severity::from_priority(Some("0")), Severity::Critical);
96
+ assert_eq!(Severity::from_priority(Some("1")), Severity::Critical);
97
+ assert_eq!(Severity::from_priority(Some("2")), Severity::Critical);
98
+ assert_eq!(Severity::from_priority(Some("3")), Severity::Critical);
99
+
100
+ // Warning: 4
101
+ assert_eq!(Severity::from_priority(Some("4")), Severity::Warning);
102
+
103
+ // Notice: 5
104
+ assert_eq!(Severity::from_priority(Some("5")), Severity::Notice);
105
+
106
+ // Normal: 6 (INFO)
107
+ assert_eq!(Severity::from_priority(Some("6")), Severity::Normal);
108
+
109
+ // Debug: 7
110
+ assert_eq!(Severity::from_priority(Some("7")), Severity::Debug);
111
+
112
+ // Default: missing or invalid → Normal
113
+ assert_eq!(Severity::from_priority(None), Severity::Normal);
114
+ assert_eq!(Severity::from_priority(Some("invalid")), Severity::Normal);
115
+ assert_eq!(Severity::from_priority(Some("")), Severity::Normal);
116
+ }
117
+
118
+ #[test]
119
+ fn test_severity_serialization() {
120
+ // Test that severity serializes to lowercase strings
121
+ assert_eq!(
122
+ serde_json::to_string(&Severity::Critical).unwrap(),
123
+ "\"critical\""
124
+ );
125
+ assert_eq!(
126
+ serde_json::to_string(&Severity::Warning).unwrap(),
127
+ "\"warning\""
128
+ );
129
+ assert_eq!(
130
+ serde_json::to_string(&Severity::Notice).unwrap(),
131
+ "\"notice\""
132
+ );
133
+ assert_eq!(
134
+ serde_json::to_string(&Severity::Debug).unwrap(),
135
+ "\"debug\""
136
+ );
137
+ assert_eq!(
138
+ serde_json::to_string(&Severity::Normal).unwrap(),
139
+ "\"normal\""
140
+ );
141
+ }
142
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/transformations.rs
new
+639
@@ -0,0 +1,639 @@
1
+use std::collections::HashMap;
2
+use std::sync::Arc;
3
+
4
+/// Trait for field transformations
5
+pub trait FieldTransformation: Send + Sync {
6
+ /// Transform a raw field value to a display representation
7
+ fn transform(&self, raw_value: &str) -> String;
8
+}
9
+
10
+/// Registry of field transformations
11
+#[derive(Clone)]
12
+pub struct TransformationRegistry {
13
+ transformations: HashMap<String, Arc<dyn FieldTransformation>>,
14
+}
15
+
16
+impl TransformationRegistry {
17
+ /// Create a new empty registry
18
+ pub fn new() -> Self {
19
+ Self {
20
+ transformations: HashMap::new(),
21
+ }
22
+ }
23
+
24
+ /// Register a transformation for a specific field
25
+ pub fn register(
26
+ &mut self,
27
+ field_name: impl Into<String>,
28
+ transform: Arc<dyn FieldTransformation>,
29
+ ) {
30
+ self.transformations.insert(field_name.into(), transform);
31
+ }
32
+
33
+ /// Transform a field value using the registered transformation
34
+ pub fn transform_field(
35
+ &self,
36
+ field_name: &str,
37
+ raw: Option<String>,
38
+ ) -> journal_engine::CellValue {
39
+ match raw {
40
+ None => journal_engine::CellValue::new(None),
41
+ Some(raw_str) => {
42
+ let display = self
43
+ .transformations
44
+ .get(field_name)
45
+ .map(|t| t.transform(&raw_str))
46
+ .unwrap_or_else(|| raw_str.clone());
47
+
48
+ journal_engine::CellValue::with_display(Some(raw_str), Some(display))
49
+ }
50
+ }
51
+ }
52
+
53
+ /// Transform a string value for a field, returning the transformed string.
54
+ ///
55
+ /// If no transformation is registered for the field, returns the original value.
56
+ pub fn transform_value(&self, field_name: &str, value: &str) -> String {
57
+ self.transformations
58
+ .get(field_name)
59
+ .map(|t| t.transform(value))
60
+ .unwrap_or_else(|| value.to_string())
61
+ }
62
+}
63
+
64
+impl Default for TransformationRegistry {
65
+ fn default() -> Self {
66
+ Self::new()
67
+ }
68
+}
69
+
70
+/// PRIORITY: 0-7 → human-readable names
71
+pub struct PriorityTransformation;
72
+
73
+impl FieldTransformation for PriorityTransformation {
74
+ fn transform(&self, raw_value: &str) -> String {
75
+ match raw_value {
76
+ "0" => "panic".to_string(),
77
+ "1" => "alert".to_string(),
78
+ "2" => "critical".to_string(),
79
+ "3" => "error".to_string(),
80
+ "4" => "warning".to_string(),
81
+ "5" => "notice".to_string(),
82
+ "6" => "info".to_string(),
83
+ "7" => "debug".to_string(),
84
+ _ => raw_value.to_string(),
85
+ }
86
+ }
87
+}
88
+
89
+/// log.severity_number: OpenTelemetry severity number → short names
90
+/// Maps according to OpenTelemetry specification ranges:
91
+/// https://opentelemetry.io/docs/specs/otel/logs/data-model/#displaying-severity
92
+pub struct OtelSeverityNumberTransformation;
93
+
94
+impl FieldTransformation for OtelSeverityNumberTransformation {
95
+ fn transform(&self, raw_value: &str) -> String {
96
+ match raw_value.parse::<u32>() {
97
+ Ok(num) => match num {
98
+ 1..=4 => "TRACE".to_string(),
99
+ 5..=8 => "DEBUG".to_string(),
100
+ 9..=12 => "INFO".to_string(),
101
+ 13..=16 => "WARN".to_string(),
102
+ 17..=20 => "ERROR".to_string(),
103
+ 21..=24 => "FATAL".to_string(),
104
+ 0 => "UNSPECIFIED".to_string(),
105
+ _ => raw_value.to_string(),
106
+ },
107
+ Err(_) => raw_value.to_string(),
108
+ }
109
+ }
110
+}
111
+
112
+/// SYSLOG_FACILITY: 0-23 → facility names
113
+pub struct SyslogFacilityTransformation;
114
+
115
+impl FieldTransformation for SyslogFacilityTransformation {
116
+ fn transform(&self, raw_value: &str) -> String {
117
+ match raw_value {
118
+ "0" => "kern".to_string(),
119
+ "1" => "user".to_string(),
120
+ "2" => "mail".to_string(),
121
+ "3" => "daemon".to_string(),
122
+ "4" => "auth".to_string(),
123
+ "5" => "syslog".to_string(),
124
+ "6" => "lpr".to_string(),
125
+ "7" => "news".to_string(),
126
+ "8" => "uucp".to_string(),
127
+ "9" => "cron".to_string(),
128
+ "10" => "authpriv".to_string(),
129
+ "11" => "ftp".to_string(),
130
+ "12" => "ntp".to_string(),
131
+ "13" => "security".to_string(),
132
+ "14" => "console".to_string(),
133
+ "15" => "solaris-cron".to_string(),
134
+ "16" => "local0".to_string(),
135
+ "17" => "local1".to_string(),
136
+ "18" => "local2".to_string(),
137
+ "19" => "local3".to_string(),
138
+ "20" => "local4".to_string(),
139
+ "21" => "local5".to_string(),
140
+ "22" => "local6".to_string(),
141
+ "23" => "local7".to_string(),
142
+ _ => raw_value.to_string(),
143
+ }
144
+ }
145
+}
146
+
147
+/// ERRNO: numeric → "code (name)"
148
+pub struct ErrnoTransformation;
149
+
150
+impl FieldTransformation for ErrnoTransformation {
151
+ fn transform(&self, raw_value: &str) -> String {
152
+ let name = match raw_value {
153
+ "1" => "EPERM",
154
+ "2" => "ENOENT",
155
+ "3" => "ESRCH",
156
+ "4" => "EINTR",
157
+ "5" => "EIO",
158
+ "6" => "ENXIO",
159
+ "7" => "E2BIG",
160
+ "8" => "ENOEXEC",
161
+ "9" => "EBADF",
162
+ "10" => "ECHILD",
163
+ "11" => "EAGAIN",
164
+ "12" => "ENOMEM",
165
+ "13" => "EACCES",
166
+ "14" => "EFAULT",
167
+ "15" => "ENOTBLK",
168
+ "16" => "EBUSY",
169
+ "17" => "EEXIST",
170
+ "18" => "EXDEV",
171
+ "19" => "ENODEV",
172
+ "20" => "ENOTDIR",
173
+ "21" => "EISDIR",
174
+ "22" => "EINVAL",
175
+ "23" => "ENFILE",
176
+ "24" => "EMFILE",
177
+ "25" => "ENOTTY",
178
+ "26" => "ETXTBSY",
179
+ "27" => "EFBIG",
180
+ "28" => "ENOSPC",
181
+ "29" => "ESPIPE",
182
+ "30" => "EROFS",
183
+ "31" => "EMLINK",
184
+ "32" => "EPIPE",
185
+ "33" => "EDOM",
186
+ "34" => "ERANGE",
187
+ "35" => "EDEADLK",
188
+ "36" => "ENAMETOOLONG",
189
+ "37" => "ENOLCK",
190
+ "38" => "ENOSYS",
191
+ "39" => "ENOTEMPTY",
192
+ "40" => "ELOOP",
193
+ "42" => "ENOMSG",
194
+ "43" => "EIDRM",
195
+ "44" => "ECHRNG",
196
+ "45" => "EL2NSYNC",
197
+ "46" => "EL3HLT",
198
+ "47" => "EL3RST",
199
+ "48" => "ELNRNG",
200
+ "49" => "EUNATCH",
201
+ "50" => "ENOCSI",
202
+ "51" => "EL2HLT",
203
+ "52" => "EBADE",
204
+ "53" => "EBADR",
205
+ "54" => "EXFULL",
206
+ "55" => "ENOANO",
207
+ "56" => "EBADRQC",
208
+ "57" => "EBADSLT",
209
+ "59" => "EBFONT",
210
+ "60" => "ENOSTR",
211
+ "61" => "ENODATA",
212
+ "62" => "ETIME",
213
+ "63" => "ENOSR",
214
+ "64" => "ENONET",
215
+ "65" => "ENOPKG",
216
+ "66" => "EREMOTE",
217
+ "67" => "ENOLINK",
218
+ "68" => "EADV",
219
+ "69" => "ESRMNT",
220
+ "70" => "ECOMM",
221
+ "71" => "EPROTO",
222
+ "72" => "EMULTIHOP",
223
+ "73" => "EDOTDOT",
224
+ "74" => "EBADMSG",
225
+ "75" => "EOVERFLOW",
226
+ "76" => "ENOTUNIQ",
227
+ "77" => "EBADFD",
228
+ "78" => "EREMCHG",
229
+ "79" => "ELIBACC",
230
+ "80" => "ELIBBAD",
231
+ "81" => "ELIBSCN",
232
+ "82" => "ELIBMAX",
233
+ "83" => "ELIBEXEC",
234
+ "84" => "EILSEQ",
235
+ "85" => "ERESTART",
236
+ "86" => "ESTRPIPE",
237
+ "87" => "EUSERS",
238
+ "88" => "ENOTSOCK",
239
+ "89" => "EDESTADDRREQ",
240
+ "90" => "EMSGSIZE",
241
+ "91" => "EPROTOTYPE",
242
+ "92" => "ENOPROTOOPT",
243
+ "93" => "EPROTONOSUPPORT",
244
+ "94" => "ESOCKTNOSUPPORT",
245
+ "95" => "EOPNOTSUPP",
246
+ "96" => "EPFNOSUPPORT",
247
+ "97" => "EAFNOSUPPORT",
248
+ "98" => "EADDRINUSE",
249
+ "99" => "EADDRNOTAVAIL",
250
+ "100" => "ENETDOWN",
251
+ "101" => "ENETUNREACH",
252
+ "102" => "ENETRESET",
253
+ "103" => "ECONNABORTED",
254
+ "104" => "ECONNRESET",
255
+ "105" => "ENOBUFS",
256
+ "106" => "EISCONN",
257
+ "107" => "ENOTCONN",
258
+ "108" => "ESHUTDOWN",
259
+ "109" => "ETOOMANYREFS",
260
+ "110" => "ETIMEDOUT",
261
+ "111" => "ECONNREFUSED",
262
+ "112" => "EHOSTDOWN",
263
+ "113" => "EHOSTUNREACH",
264
+ "114" => "EALREADY",
265
+ "115" => "EINPROGRESS",
266
+ "116" => "ESTALE",
267
+ "117" => "EUCLEAN",
268
+ "118" => "ENOTNAM",
269
+ "119" => "ENAVAIL",
270
+ "120" => "EISNAM",
271
+ "121" => "EREMOTEIO",
272
+ "122" => "EDQUOT",
273
+ "123" => "ENOMEDIUM",
274
+ "124" => "EMEDIUMTYPE",
275
+ "125" => "ECANCELED",
276
+ "126" => "ENOKEY",
277
+ "127" => "EKEYEXPIRED",
278
+ "128" => "EKEYREVOKED",
279
+ "129" => "EKEYREJECTED",
280
+ "130" => "EOWNERDEAD",
281
+ "131" => "ENOTRECOVERABLE",
282
+ "132" => "ERFKILL",
283
+ "133" => "EHWPOISON",
284
+ _ => return raw_value.to_string(),
285
+ };
286
+
287
+ name.to_string()
288
+ }
289
+}
290
+
291
+/// _BOOT_ID: UUID → "UUID (timestamp)"
292
+/// Note: Full implementation would require boot_id cache lookup
293
+pub struct BootIdTransformation;
294
+
295
+impl FieldTransformation for BootIdTransformation {
296
+ fn transform(&self, raw_value: &str) -> String {
297
+ // For now, just return the UUID
298
+ // Full implementation would look up first boot timestamp from cache
299
+ raw_value.to_string()
300
+ }
301
+}
302
+
303
+/// _UID: numeric → username
304
+pub struct UidTransformation;
305
+
306
+impl FieldTransformation for UidTransformation {
307
+ fn transform(&self, raw_value: &str) -> String {
308
+ use nix::unistd::{Uid, User};
309
+
310
+ // Parse the UID
311
+ let Ok(uid_num) = raw_value.parse::<u32>() else {
312
+ return raw_value.to_string();
313
+ };
314
+
315
+ let uid = Uid::from_raw(uid_num);
316
+
317
+ // Look up the user
318
+ match User::from_uid(uid) {
319
+ Ok(Some(user)) => user.name,
320
+ Ok(None) => raw_value.to_string(), // User not found
321
+ Err(_) => raw_value.to_string(), // Lookup error
322
+ }
323
+ }
324
+}
325
+
326
+/// _GID: numeric → groupname
327
+pub struct GidTransformation;
328
+
329
+impl FieldTransformation for GidTransformation {
330
+ fn transform(&self, raw_value: &str) -> String {
331
+ use nix::unistd::{Gid, Group};
332
+
333
+ // Parse the GID
334
+ let Ok(gid_num) = raw_value.parse::<u32>() else {
335
+ return raw_value.to_string();
336
+ };
337
+
338
+ let gid = Gid::from_raw(gid_num);
339
+
340
+ // Look up the group
341
+ match Group::from_gid(gid) {
342
+ Ok(Some(group)) => group.name,
343
+ Ok(None) => raw_value.to_string(), // Group not found
344
+ Err(_) => raw_value.to_string(), // Lookup error
345
+ }
346
+ }
347
+}
348
+
349
+/// _CAP_EFFECTIVE: hex → "hex (capability names)"
350
+pub struct CapEffectiveTransformation;
351
+
352
+impl FieldTransformation for CapEffectiveTransformation {
353
+ fn transform(&self, raw_value: &str) -> String {
354
+ // Parse hex value
355
+ let caps_value = if let Some(hex) = raw_value.strip_prefix("0x") {
356
+ u64::from_str_radix(hex, 16).ok()
357
+ } else {
358
+ raw_value.parse::<u64>().ok()
359
+ };
360
+
361
+ let Some(caps) = caps_value else {
362
+ return raw_value.to_string();
363
+ };
364
+
365
+ // Linux capabilities (41 capabilities as of Linux 5.x)
366
+ const CAPABILITIES: &[&str] = &[
367
+ "CAP_CHOWN",
368
+ "CAP_DAC_OVERRIDE",
369
+ "CAP_DAC_READ_SEARCH",
370
+ "CAP_FOWNER",
371
+ "CAP_FSETID",
372
+ "CAP_KILL",
373
+ "CAP_SETGID",
374
+ "CAP_SETUID",
375
+ "CAP_SETPCAP",
376
+ "CAP_LINUX_IMMUTABLE",
377
+ "CAP_NET_BIND_SERVICE",
378
+ "CAP_NET_BROADCAST",
379
+ "CAP_NET_ADMIN",
380
+ "CAP_NET_RAW",
381
+ "CAP_IPC_LOCK",
382
+ "CAP_IPC_OWNER",
383
+ "CAP_SYS_MODULE",
384
+ "CAP_SYS_RAWIO",
385
+ "CAP_SYS_CHROOT",
386
+ "CAP_SYS_PTRACE",
387
+ "CAP_SYS_PACCT",
388
+ "CAP_SYS_ADMIN",
389
+ "CAP_SYS_BOOT",
390
+ "CAP_SYS_NICE",
391
+ "CAP_SYS_RESOURCE",
392
+ "CAP_SYS_TIME",
393
+ "CAP_SYS_TTY_CONFIG",
394
+ "CAP_MKNOD",
395
+ "CAP_LEASE",
396
+ "CAP_AUDIT_WRITE",
397
+ "CAP_AUDIT_CONTROL",
398
+ "CAP_SETFCAP",
399
+ "CAP_MAC_OVERRIDE",
400
+ "CAP_MAC_ADMIN",
401
+ "CAP_SYSLOG",
402
+ "CAP_WAKE_ALARM",
403
+ "CAP_BLOCK_SUSPEND",
404
+ "CAP_AUDIT_READ",
405
+ "CAP_PERFMON",
406
+ "CAP_BPF",
407
+ "CAP_CHECKPOINT_RESTORE",
408
+ ];
409
+
410
+ let mut cap_names = Vec::new();
411
+ for (i, &cap_name) in CAPABILITIES.iter().enumerate() {
412
+ if caps & (1u64 << i) != 0 {
413
+ cap_names.push(cap_name);
414
+ }
415
+ }
416
+
417
+ if cap_names.is_empty() {
418
+ "none".to_string()
419
+ } else {
420
+ cap_names.join(", ")
421
+ }
422
+ }
423
+}
424
+
425
+/// _SOURCE_REALTIME_TIMESTAMP: microseconds → "microseconds (ISO8601)"
426
+pub struct SourceRealtimeTimestampTransformation;
427
+
428
+impl FieldTransformation for SourceRealtimeTimestampTransformation {
429
+ fn transform(&self, raw_value: &str) -> String {
430
+ // Parse microseconds since epoch
431
+ let Ok(usec) = raw_value.parse::<i64>() else {
432
+ return raw_value.to_string();
433
+ };
434
+
435
+ // Convert to seconds and nanoseconds
436
+ let secs = usec / 1_000_000;
437
+ let nsecs = ((usec % 1_000_000) * 1000) as u32;
438
+
439
+ // Create DateTime in UTC, then convert to local timezone
440
+ use chrono::{Local, TimeZone, Utc};
441
+ let dt_utc = match Utc.timestamp_opt(secs, nsecs) {
442
+ chrono::LocalResult::Single(dt) => dt,
443
+ _ => return raw_value.to_string(),
444
+ };
445
+
446
+ // Convert to local time
447
+ let dt_local = dt_utc.with_timezone(&Local);
448
+
449
+ // Format as RFC3339 with microsecond precision in local timezone
450
+ dt_local.to_rfc3339_opts(chrono::SecondsFormat::Micros, true)
451
+ }
452
+}
453
+
454
+/// MESSAGE_ID: UUID → "UUID (description)"
455
+pub struct MessageIdTransformation;
456
+
457
+impl FieldTransformation for MessageIdTransformation {
458
+ fn transform(&self, raw_value: &str) -> String {
459
+ // Known journal message IDs from systemd and other sources
460
+ let description = match raw_value {
461
+ "f77379a8490b408bbe5f6940505a777b" => "Journal started",
462
+ "d93fb3c9c24d451a97cea615ce59c00b" => "Journal stopped",
463
+ "a596d6fe7bfa4994828e72309e95d61e" => "Journal messages suppressed",
464
+ "e9bf28e6e834481bb6f48f548ad13606" => "Journal messages missed",
465
+ "ec387f577b844b8fa948f33cad9a75e6" => "Journal disk space usage",
466
+ "fc2e22bc6ee647b6b90729ab34a250b1" => "Coredump",
467
+ "5aadd8e954dc4b1a8c954d63fd9e1137" => "Coredump truncated",
468
+ "1f4e0a44a88649939aaea34fc6da8c95" => "Backtrace",
469
+ "8d45620c1a4348dbb17410da57c60c66" => "User Session created",
470
+ "3354939424b4456d9802ca8333ed424a" => "User Session terminated",
471
+ "fcbefc5da23d428093f97c82a9290f7b" => "Seat started",
472
+ "e7852bfe46784ed0accde04bc864c2d5" => "Seat removed",
473
+ "24d8d4452573402496068381a6312df2" => "VM or container started",
474
+ "58432bd3bace477cb514b56381b8a758" => "VM or container stopped",
475
+ "c7a787079b354eaaa9e77b371893cd27" => "Time change",
476
+ "45f82f4aef7a4bbf942ce861d1f20990" => "Timezone change",
477
+ "50876a9db00f4c40bde1a2ad381c3a1b" => "System configuration issues",
478
+ "b07a249cd024414a82dd00cd181378ff" => "System start-up completed",
479
+ "eed00a68ffd84e31882105fd973abdd1" => "User start-up completed",
480
+ "6bbd95ee977941e497c48be27c254128" => "Sleep start",
481
+ "8811e6df2a8e40f58a94cea26f8ebf14" => "Sleep stop",
482
+ "98268866d1d54a499c4e98921d93bc40" => "System shutdown initiated",
483
+ "c14aaf76ec284a5fa1f105f88dfb061c" => "System factory reset initiated",
484
+ "d9ec5e95e4b646aaaea2fd05214edbda" => "Container init crashed",
485
+ "3ed0163e868a4417ab8b9e210407a96c" => "System reboot failed after crash",
486
+ "645c735537634ae0a32b15a7c6cba7d4" => "Init execution froze",
487
+ "5addb3a06a734d3396b794bf98fb2d01" => "Init crashed no coredump",
488
+ "5c9e98de4ab94c6a9d04d0ad793bd903" => "Init crashed no fork",
489
+ "5e6f1f5e4db64a0eaee3368249d20b94" => "Init crashed unknown signal",
490
+ "83f84b35ee264f74a3896a9717af34cb" => "Init crashed systemd signal",
491
+ "3a73a98baf5b4b199929e3226c0be783" => "Init crashed process signal",
492
+ "2ed18d4f78ca47f0a9bc25271c26adb4" => "Init crashed waitpid failed",
493
+ "56b1cd96f24246c5b607666fda952356" => "Init crashed coredump failed",
494
+ "4ac7566d4d7548f4981f629a28f0f829" => "Init crashed coredump",
495
+ "38e8b1e039ad469291b18b44c553a5b7" => "Crash shell failed to fork",
496
+ "872729b47dbe473eb768ccecd477beda" => "Crash shell failed to execute",
497
+ "658a67adc1c940b3b3316e7e8628834a" => "Selinux failed",
498
+ "e6f456bd92004d9580160b2207555186" => "Battery low warning",
499
+ "267437d33fdd41099ad76221cc24a335" => "Battery low powering off",
500
+ "79e05b67bc4545d1922fe47107ee60c5" => "Manager mainloop failed",
501
+ "dbb136b10ef4457ba47a795d62f108c9" => "Manager no xdgdir path",
502
+ "ed158c2df8884fa584eead2d902c1032" => {
503
+ "Init failed to drop capability bounding set of usermode"
504
+ }
505
+ "42695b500df048298bee37159caa9f2e" => "Init failed to drop capability bounding set",
506
+ "bfc2430724ab44499735b4f94cca9295" => "User manager can't disable new privileges",
507
+ "59288af523be43a28d494e41e26e4510" => "Manager failed to start default target",
508
+ "689b4fcc97b4486ea5da92db69c9e314" => "Manager failed to isolate default target",
509
+ "5ed836f1766f4a8a9fc5da45aae23b29" => {
510
+ "Manager failed to collect passed file descriptors"
511
+ }
512
+ "6a40fbfbd2ba4b8db02fb40c9cd090d7" => "Init failed to fix up environment variables",
513
+ "0e54470984ac419689743d957a119e2e" => "Manager failed to allocate",
514
+ "d67fa9f847aa4b048a2ae33535331adb" => "Manager failed to write Smack",
515
+ "af55a6f75b544431b72649f36ff6d62c" => "System shutdown critical error",
516
+ "d18e0339efb24a068d9c1060221048c2" => "Init failed to fork off valgrind",
517
+ "7d4958e842da4a758f6c1cdc7b36dcc5" => "Unit starting",
518
+ "39f53479d3a045ac8e11786248231fbf" => "Unit started",
519
+ "be02cf6855d2428ba40df7e9d022f03d" => "Unit failed",
520
+ "de5b426a63be47a7b6ac3eaac82e2f6f" => "Unit stopping",
521
+ "9d1aaa27d60140bd96365438aad20286" => "Unit stopped",
522
+ "d34d037fff1847e6ae669a370e694725" => "Unit reloading",
523
+ "7b05ebc668384222baa8881179cfda54" => "Unit reloaded",
524
+ "5eb03494b6584870a536b337290809b3" => "Unit restart scheduled",
525
+ "ae8f7b866b0347b9af31fe1c80b127c0" => "Unit resources",
526
+ "7ad2d189f7e94e70a38c781354912448" => "Unit success",
527
+ "0e4284a0caca4bfc81c0bb6786972673" => "Unit skipped",
528
+ "d9b373ed55a64feb8242e02dbe79a49c" => "Unit failure result",
529
+ "641257651c1b4ec9a8624d7a40a9e1e7" => "Process execution failed",
530
+ "98e322203f7a4ed290d09fe03c09fe15" => "Unit process exited",
531
+ "0027229ca0644181a76c4e92458afa2e" => "Syslog forward missed",
532
+ "1dee0369c7fc4736b7099b38ecb46ee7" => "Mount point is not empty",
533
+ "d989611b15e44c9dbf31e3c81256e4ed" => "Unit oomd kill",
534
+ "fe6faa94e7774663a0da52717891d8ef" => "Unit out of memory",
535
+ "b72ea4a2881545a0b50e200e55b9b06f" => "Lid opened",
536
+ "b72ea4a2881545a0b50e200e55b9b070" => "Lid closed",
537
+ "f5f416b862074b28927a48c3ba7d51ff" => "System docked",
538
+ "51e171bd585248568110144c517cca53" => "System undocked",
539
+ "b72ea4a2881545a0b50e200e55b9b071" => "Power key",
540
+ "3e0117101eb243c1b9a50db3494ab10b" => "Power key long press",
541
+ "9fa9d2c012134ec385451ffe316f97d0" => "Reboot key",
542
+ "f1c59a58c9d943668965c337caec5975" => "Reboot key long press",
543
+ "b72ea4a2881545a0b50e200e55b9b072" => "Suspend key",
544
+ "bfdaf6d312ab4007bc1fe40a15df78e8" => "Suspend key long press",
545
+ "b72ea4a2881545a0b50e200e55b9b073" => "Hibernate key",
546
+ "167836df6f7f428e98147227b2dc8945" => "Hibernate key long press",
547
+ "c772d24e9a884cbeb9ea12625c306c01" => "Invalid configuration",
548
+ "1675d7f172174098b1108bf8c7dc8f5d" => "DNSSEC validation failed",
549
+ "4d4408cfd0d144859184d1e65d7c8a65" => "DNSSEC trust anchor revoked",
550
+ "36db2dfa5a9045e1bd4af5f93e1cf057" => "DNSSEC turned off",
551
+ "b61fdac612e94b9182285b998843061f" => "Username unsafe",
552
+ "1b3bb94037f04bbf81028e135a12d293" => "Mount point path not suitable",
553
+ "010190138f494e29a0ef6669749531aa" => "Device path not suitable",
554
+ "b480325f9c394a7b802c231e51a2752c" => "Nobody user unsuitable",
555
+ "1c0454c1bd2241e0ac6fefb4bc631433" => "Systemd udev settle deprecated",
556
+ "7c8a41f37b764941a0e1780b1be2f037" => "Time initial sync",
557
+ "7db73c8af0d94eeb822ae04323fe6ab6" => "Time initial bump",
558
+ "9e7066279dc8403da79ce4b1a69064b2" => "Shutdown scheduled",
559
+ "249f6fb9e6e2428c96f3f0875681ffa3" => "Shutdown canceled",
560
+ "3f7d5ef3e54f4302b4f0b143bb270cab" => "TPM PCR Extended",
561
+ "f9b0be465ad540d0850ad32172d57c21" => "Memory Trimmed",
562
+ "a8fa8dacdb1d443e9503b8be367a6adb" => "SysV Service Found",
563
+ "187c62eb1e7f463bb530394f52cb090f" => "Portable Service attached",
564
+ "76c5c754d628490d8ecba4c9d042112b" => "Portable Service detached",
565
+ "9cf56b8baf9546cf9478783a8de42113" => {
566
+ "systemd-networkd sysctl changed by foreign process"
567
+ }
568
+ "ad7089f928ac4f7ea00c07457d47ba8a" => "SRK into TPM authorization failure",
569
+ "b2bcbaf5edf948e093ce50bbea0e81ec" => "Secure Attention Key (SAK) was pressed",
570
+ "7fc63312330b479bb32e598d47cef1a8" => "dbus activate no unit",
571
+ "ee9799dab1e24d81b7bee7759a543e1b" => "dbus activate masked unit",
572
+ "a0fa58cafd6f4f0c8d003d16ccf9e797" => "dbus broker exited",
573
+ "c8c6cde1c488439aba371a664353d9d8" => "dbus dirwatch",
574
+ "8af3357071af4153af414daae07d38e7" => "dbus dispatch stats",
575
+ "199d4300277f495f84ba4028c984214c" => "dbus no sopeergroup",
576
+ "b209c0d9d1764ab38d13b8e00d1784d6" => "dbus protocol violation",
577
+ "6fa70fa776044fa28be7a21daf42a108" => "dbus receive failed",
578
+ "0ce0fa61d1a9433dabd67417f6b8e535" => "dbus service failed open",
579
+ "24dc708d9e6a4226a3efe2033bb744de" => "dbus service invalid",
580
+ "f15d2347662d483ea9bcd8aa1a691d28" => "dbus sighup",
581
+ "0ce153587afa4095832d233c17a88001" => "Gnome SM startup succeeded",
582
+ "10dd2dc188b54a5e98970f56499d1f73" => "Gnome SM unrecoverable failure",
583
+ "f3ea493c22934e26811cd62abe8e203a" => "Gnome shell started",
584
+ "c7b39b1e006b464599465e105b361485" => "Flatpak cache",
585
+ "75ba3deb0af041a9a46272ff85d9e73e" => "Flathub pulls",
586
+ "f02bce89a54e4efab3a94a797d26204a" => "Flathub pull errors",
587
+ "dd11929c788e48bdbb6276fb5f26b08a" => "Boltd starting",
588
+ "1e6061a9fbd44501b3ccc368119f2b69" => "Netdata startup",
589
+ "ed4cdb8f1beb4ad3b57cb3cae2d162fa" => "Netdata connection from child",
590
+ "6e2e3839067648968b646045dbf28d66" => "Netdata connection to parent",
591
+ "9ce0cb58ab8b44df82c4bf1ad9ee22de" => "Netdata alert transition",
592
+ "6db0018e83e34320ae2a659d78019fb7" => "Netdata alert notification",
593
+ "23e93dfccbf64e11aac858b9410d8a82" => "Netdata fatal message",
594
+ "8ddaf5ba33a74078b609250db1e951f3" => "Sensor state transition",
595
+ "ec87a56120d5431bace51e2fb8bba243" => "Netdata log flood protection",
596
+ "acb33cb95778476baac702eb7e4e151d" => "Netdata Cloud connection",
597
+ "d1f59606dd4d41e3b217a0cfcae8e632" => "Netdata extreme cardinality",
598
+ "02f47d350af5449197bf7a95b605a468" => "Netdata exit reason",
599
+ "4fdf40816c124623a032b7fe73beacb8" => "Netdata dynamic configuration",
600
+ _ => return raw_value.to_string(),
601
+ };
602
+
603
+ description.to_string()
604
+ }
605
+}
606
+
607
+/// Create a transformation registry with all systemd journal transformations
608
+pub fn systemd_transformations() -> TransformationRegistry {
609
+ let mut registry = TransformationRegistry::new();
610
+
611
+ // Timestamp transformation (used for the first column)
612
+ registry.register("timestamp", Arc::new(SourceRealtimeTimestampTransformation));
613
+
614
+ registry.register("PRIORITY", Arc::new(PriorityTransformation));
615
+ registry.register("SYSLOG_FACILITY", Arc::new(SyslogFacilityTransformation));
616
+ registry.register("ERRNO", Arc::new(ErrnoTransformation));
617
+ registry.register("_BOOT_ID", Arc::new(BootIdTransformation));
618
+ registry.register("_UID", Arc::new(UidTransformation));
619
+ registry.register("_GID", Arc::new(GidTransformation));
620
+ registry.register("_CAP_EFFECTIVE", Arc::new(CapEffectiveTransformation));
621
+ registry.register(
622
+ "_SOURCE_REALTIME_TIMESTAMP",
623
+ Arc::new(SourceRealtimeTimestampTransformation),
624
+ );
625
+ registry.register("MESSAGE_ID", Arc::new(MessageIdTransformation));
626
+
627
+ // OpenTelemetry log fields
628
+ registry.register("log.severity_number", Arc::new(OtelSeverityNumberTransformation));
629
+
630
+ // Also register variations that exist in the wild
631
+ registry.register("OBJECT_UID", Arc::new(UidTransformation));
632
+ registry.register("OBJECT_GID", Arc::new(GidTransformation));
633
+ registry.register("_SYSTEMD_OWNER_UID", Arc::new(UidTransformation));
634
+ registry.register("OBJECT_SYSTEMD_OWNER_UID", Arc::new(UidTransformation));
635
+ registry.register("_AUDIT_LOGINUID", Arc::new(UidTransformation));
636
+ registry.register("OBJECT_AUDIT_LOGINUID", Arc::new(UidTransformation));
637
+
638
+ registry
639
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/types.rs
new
+219
@@ -0,0 +1,219 @@
1
+//! Request and response types for the systemd-journal function.
2
+//!
3
+//! This module defines the API types used for communication between the Netdata
4
+//! dashboard and the systemd-journal function plugin.
5
+
6
+use super::ui_types as ui; // ui_types is a sibling module in netdata
7
+use journal_index::Direction;
8
+use serde::{Deserialize, Serialize};
9
+use std::collections::HashMap;
10
+
11
+#[derive(Debug, Serialize, Deserialize, Clone)]
12
+pub struct JournalRequest {
13
+ #[serde(default)]
14
+ pub info: bool,
15
+
16
+ /// Unix timestamp for the start of the time range (seconds)
17
+ pub after: u32,
18
+
19
+ /// Unix timestamp for the end of the time range (seconds)
20
+ pub before: u32,
21
+
22
+ /// Anchor timestamp in microseconds for pagination
23
+ pub anchor: Option<u64>,
24
+
25
+ /// Maximum number of results to return
26
+ pub last: Option<usize>,
27
+
28
+ /// List of facets to include in the response
29
+ #[serde(default)]
30
+ pub facets: Vec<String>,
31
+
32
+ /// Field name to use for histogram visualization
33
+ #[serde(default)]
34
+ pub histogram: String,
35
+
36
+ /// Direction for log retrieval (forward = oldest to newest, backward = newest to oldest)
37
+ #[serde(default = "JournalRequest::default_direction")]
38
+ pub direction: Direction,
39
+
40
+ /// Whether to slice the results
41
+ pub slice: Option<bool>,
42
+
43
+ /// Text search query
44
+ #[serde(default)]
45
+ pub query: String,
46
+
47
+ /// Selection filters
48
+ #[serde(default)]
49
+ pub selections: HashMap<String, Vec<String>>,
50
+
51
+ /// Timeout in milliseconds
52
+ pub timeout: Option<u32>,
53
+}
54
+
55
+impl Default for JournalRequest {
56
+ fn default() -> Self {
57
+ Self {
58
+ info: true,
59
+ after: 0,
60
+ before: 0,
61
+ anchor: None,
62
+ last: Some(200),
63
+ facets: Vec::new(),
64
+ histogram: String::new(),
65
+ direction: Direction::Backward,
66
+ slice: None,
67
+ query: String::new(),
68
+ selections: HashMap::new(),
69
+ timeout: None,
70
+ }
71
+ }
72
+}
73
+
74
+impl JournalRequest {
75
+ /// Default direction for journal log retrieval (backward = newest to oldest)
76
+ fn default_direction() -> Direction {
77
+ Direction::Backward
78
+ }
79
+}
80
+
81
+#[derive(Debug, Copy, Clone, Serialize, Deserialize)]
82
+#[serde(rename_all = "snake_case")]
83
+pub enum RequestParam {
84
+ Info,
85
+ After,
86
+ Before,
87
+ Anchor,
88
+ Direction,
89
+ Last,
90
+ Query,
91
+ Facets,
92
+ Histogram,
93
+ IfModifiedSince,
94
+ DataOnly,
95
+ Delta,
96
+ Tail,
97
+ Sampling,
98
+ Slice,
99
+ #[serde(rename = "_auxiliary")]
100
+ Auxiliary,
101
+}
102
+
103
+#[derive(Debug, Serialize, Deserialize)]
104
+pub struct MultiSelectionOption {
105
+ pub id: String,
106
+ pub name: String,
107
+ pub pill: String,
108
+ pub info: String,
109
+}
110
+
111
+#[derive(Debug, Serialize, Deserialize)]
112
+pub struct MultiSelection {
113
+ pub id: RequestParam,
114
+ pub name: String,
115
+ pub help: String,
116
+ #[serde(rename = "type", default = "MultiSelection::default_type")]
117
+ pub type_: String,
118
+ pub options: Vec<MultiSelectionOption>,
119
+}
120
+
121
+impl MultiSelection {
122
+ fn default_type() -> String {
123
+ "multiselect".to_string()
124
+ }
125
+}
126
+
127
+#[derive(Debug, Serialize, Deserialize)]
128
+#[serde(untagged)]
129
+pub enum RequiredParam {
130
+ MultiSelection(MultiSelection),
131
+}
132
+
133
+#[derive(Debug, Serialize, Deserialize)]
134
+pub struct Version(u32);
135
+
136
+impl Default for Version {
137
+ fn default() -> Self {
138
+ Self(3)
139
+ }
140
+}
141
+
142
+#[derive(Debug, Serialize, Deserialize)]
143
+pub struct Pagination {
144
+ enabled: bool,
145
+ key: RequestParam,
146
+ column: String,
147
+ units: String,
148
+}
149
+
150
+impl Default for Pagination {
151
+ fn default() -> Self {
152
+ Self {
153
+ enabled: true,
154
+ key: RequestParam::Anchor,
155
+ column: String::from("timestamp"),
156
+ units: String::from("timestamp_usec"),
157
+ }
158
+ }
159
+}
160
+
161
+// #[derive(Debug, Serialize, Deserialize)]
162
+// struct Versions {
163
+// sources: u64,
164
+// }
165
+
166
+// #[derive(Debug, Serialize, Deserialize)]
167
+// pub struct Columns {}
168
+
169
+use serde_json::Value;
170
+
171
+#[derive(Debug, Serialize, Deserialize)]
172
+pub struct Items {
173
+ #[serde(default)]
174
+ pub evaluated: usize,
175
+
176
+ #[serde(default)]
177
+ pub unsampled: usize,
178
+
179
+ #[serde(default)]
180
+ pub estimated: usize,
181
+
182
+ pub matched: usize,
183
+ pub before: usize,
184
+ pub after: usize,
185
+ pub returned: usize,
186
+
187
+ pub max_to_return: usize,
188
+}
189
+
190
+#[derive(Debug, Serialize, Deserialize)]
191
+pub struct JournalResponse {
192
+ pub progress: u32,
193
+
194
+ #[serde(rename = "v")]
195
+ pub version: Version,
196
+
197
+ pub accepted_params: Vec<RequestParam>,
198
+ pub required_params: Vec<RequiredParam>,
199
+
200
+ pub facets: Vec<ui::Facet>,
201
+
202
+ pub available_histograms: Vec<ui::AvailableHistogram>,
203
+ pub histogram: ui::Histogram,
204
+ pub columns: Value,
205
+ pub data: Value,
206
+ pub default_charts: Vec<u32>,
207
+
208
+ pub items: Items,
209
+
210
+ // Hard-coded stuff
211
+ pub show_ids: bool,
212
+ pub has_history: bool,
213
+ pub status: u32,
214
+ #[serde(rename = "type")]
215
+ pub response_type: String,
216
+ pub help: String,
217
+ pub pagination: Pagination,
218
+ // versions: Versions,
219
+}
src/crates/netdata-log-viewer/journal-function/src/netdata/ui_types.rs
new
+205
@@ -0,0 +1,205 @@
1
+//! UI response types for rendering data in the Netdata dashboard.
2
+//!
3
+//! This module provides types for converting histogram responses into UI-friendly formats,
4
+//! including facets, charts, and data points formatted for the Netdata dashboard.
5
+
6
+use serde::{Deserialize, Serialize};
7
+
8
+// ============================================================================
9
+// UI Response Types (flat structure)
10
+// ============================================================================
11
+
12
+/// Top-level response containing facets, available histograms, and a histogram.
13
+#[derive(Debug, Serialize, Deserialize)]
14
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
15
+pub struct Response {
16
+ pub facets: Vec<Facet>,
17
+ pub available_histograms: Vec<AvailableHistogram>,
18
+ pub histogram: Histogram,
19
+}
20
+
21
+/// Represents an available histogram option.
22
+#[derive(Debug, Serialize, Deserialize)]
23
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
24
+pub struct AvailableHistogram {
25
+ pub id: String,
26
+ pub name: String,
27
+ pub order: usize,
28
+}
29
+
30
+/// A facet represents a field with multiple value options.
31
+#[derive(Debug, Serialize, Deserialize)]
32
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
33
+pub struct Facet {
34
+ pub id: String,
35
+ pub name: String,
36
+ pub order: usize,
37
+ pub options: Vec<FacetOption>,
38
+}
39
+
40
+/// A single option within a facet.
41
+#[derive(Debug, Serialize, Deserialize)]
42
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
43
+pub struct FacetOption {
44
+ pub id: String,
45
+ pub name: String,
46
+ pub order: usize,
47
+ pub count: usize,
48
+}
49
+
50
+/// A histogram for a specific field.
51
+#[derive(Debug, Serialize, Deserialize)]
52
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
53
+pub struct Histogram {
54
+ pub id: String,
55
+ pub name: String,
56
+ pub chart: Chart,
57
+}
58
+
59
+impl Histogram {
60
+ pub fn count(&self) -> usize {
61
+ self.chart.count()
62
+ }
63
+}
64
+
65
+/// A chart containing view metadata and result data.
66
+#[derive(Debug, Serialize, Deserialize)]
67
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
68
+pub struct Chart {
69
+ pub view: ChartView,
70
+ pub result: ChartResult,
71
+}
72
+
73
+impl Chart {
74
+ pub fn count(&self) -> usize {
75
+ self.result.count()
76
+ }
77
+}
78
+
79
+/// Chart view metadata describing how to display the chart.
80
+#[derive(Debug, Serialize, Deserialize)]
81
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
82
+pub struct ChartView {
83
+ pub title: String,
84
+ pub after: u32,
85
+ pub before: u32,
86
+ pub update_every: u32,
87
+ pub units: String,
88
+ pub chart_type: String,
89
+ pub dimensions: ChartDimensions,
90
+}
91
+
92
+/// Dimensions for the chart view.
93
+#[derive(Debug, Serialize, Deserialize)]
94
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
95
+pub struct ChartDimensions {
96
+ pub ids: Vec<String>,
97
+ pub names: Vec<String>,
98
+ pub units: Vec<String>,
99
+}
100
+
101
+/// Chart result data containing labels, point metadata, and time series data.
102
+#[derive(Debug, Serialize, Deserialize)]
103
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
104
+pub struct ChartResult {
105
+ pub labels: Vec<String>,
106
+ pub point: ChartPoint,
107
+ pub data: Vec<DataPoint>,
108
+}
109
+
110
+impl ChartResult {
111
+ pub fn count(&self) -> usize {
112
+ let mut n = 0;
113
+
114
+ for dp in self.data.iter() {
115
+ for item in dp.items.iter() {
116
+ n += item[0];
117
+ }
118
+ }
119
+
120
+ n
121
+ }
122
+}
123
+
124
+/// Point metadata for chart result.
125
+#[derive(Debug, Serialize, Deserialize)]
126
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
127
+pub struct ChartPoint {
128
+ pub value: u64,
129
+ pub arp: u64,
130
+ pub pa: u64,
131
+}
132
+
133
+/// A single data point in a time series.
134
+#[derive(Debug)]
135
+#[cfg_attr(feature = "allocative", derive(allocative::Allocative))]
136
+pub struct DataPoint {
137
+ pub timestamp: u64,
138
+ pub items: Vec<[usize; 3]>,
139
+}
140
+
141
+/// Custom serialization for DataPoint to flatten the structure.
142
+///
143
+/// Serialize as: [timestamp, [val1, arp1, pa1], [val2, arp2, pa2], ...]
144
+/// This format matches the expected Netdata chart data format where the first
145
+/// element is the timestamp followed by dimension data arrays.
146
+impl Serialize for DataPoint {
147
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
148
+ where
149
+ S: serde::Serializer,
150
+ {
151
+ use serde::ser::SerializeSeq;
152
+
153
+ // Create a sequence with length = 1 (timestamp) + number of items
154
+ let mut seq = serializer.serialize_seq(Some(1 + self.items.len()))?;
155
+
156
+ // First element: timestamp
157
+ seq.serialize_element(&self.timestamp)?;
158
+
159
+ // Remaining elements: each [usize; 3] array
160
+ for item in &self.items {
161
+ seq.serialize_element(item)?;
162
+ }
163
+
164
+ seq.end()
165
+ }
166
+}
167
+
168
+impl<'de> Deserialize<'de> for DataPoint {
169
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
170
+ where
171
+ D: serde::Deserializer<'de>,
172
+ {
173
+ use serde::de::{SeqAccess, Visitor};
174
+
175
+ struct DataPointVisitor;
176
+
177
+ impl<'de> Visitor<'de> for DataPointVisitor {
178
+ type Value = DataPoint;
179
+
180
+ fn expecting(&self, formatter: &mut std::fmt::Formatter) -> std::fmt::Result {
181
+ formatter.write_str("an array with timestamp followed by data items")
182
+ }
183
+
184
+ fn visit_seq<A>(self, mut seq: A) -> std::result::Result<Self::Value, A::Error>
185
+ where
186
+ A: SeqAccess<'de>,
187
+ {
188
+ // First element: timestamp
189
+ let timestamp = seq
190
+ .next_element()?
191
+ .ok_or_else(|| serde::de::Error::invalid_length(0, &self))?;
192
+
193
+ // Remaining elements: collect all [usize; 3] arrays
194
+ let mut items = Vec::new();
195
+ while let Some(item) = seq.next_element()? {
196
+ items.push(item);
197
+ }
198
+
199
+ Ok(DataPoint { timestamp, items })
200
+ }
201
+ }
202
+
203
+ deserializer.deserialize_seq(DataPointVisitor)
204
+ }
205
+}
src/crates/netdata-log-viewer/journal-viewer-plugin/Cargo.toml
new
+45
@@ -0,0 +1,45 @@
1
+[package]
2
+name = "journal-viewer-plugin"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[[bin]]
11
+name = "journal-viewer-plugin"
12
+path = "src/main.rs"
13
+
14
+[dependencies]
15
+async-trait = { workspace = true }
16
+
17
+rt = { workspace = true }
18
+netdata-plugin-error = { workspace = true }
19
+netdata-plugin-protocol = { workspace = true }
20
+netdata-plugin-schema = { workspace = true }
21
+schemars = { workspace = true }
22
+serde = { workspace = true }
23
+serde_json = { workspace = true }
24
+tracing = { workspace = true}
25
+tokio = { workspace = true }
26
+notify = { workspace = true }
27
+thiserror = { workspace = true }
28
+foyer = { workspace = true }
29
+
30
+# Configuration support
31
+anyhow = { workspace = true }
32
+bytesize = { workspace = true }
33
+bytesize-serde = { workspace = true }
34
+num_cpus = "1"
35
+serde_yaml = { workspace = true }
36
+parking_lot = { workspace = true }
37
+
38
+# Business logic
39
+# journal-engine = { workspace = true }
40
+# types = { path = "../types" }
41
+
42
+journal-core = { workspace = true }
43
+journal-index = { workspace = true }
44
+journal-function = { workspace = true }
45
+journal-registry = { workspace = true }
src/crates/netdata-log-viewer/journal-viewer-plugin/configs/journal-viewer.yaml.in
new
+45
@@ -0,0 +1,45 @@
1
+# Log Viewer Plugin Configuration
2
+# This file configures the systemd journal log viewer plugin for Netdata
3
+
4
+journal:
5
+ # Paths to journal directories to watch and index. Multiple directories can
6
+ # be specified to monitor different journal sources. At least one path must
7
+ # be specified
8
+ paths:
9
+ - "/var/log/journal"
10
+ - "/run/log/journal"
11
+ - "@localstatedir_POST@/log/netdata/otel/v1"
12
+
13
+cache:
14
+ # Directory to store the hybrid cache (memory + disk backed by Foyer)
15
+ # Relative paths are resolved based on Netdata's cache directory
16
+ # Default: /var/cache/netdata/log-viewer
17
+ directory: "@cachedir_POST@/log-viewer"
18
+
19
+ # Memory cache capacity (number of indexed journal entries to keep in memory)
20
+ # Higher values improve query performance but use more RAM
21
+ # Default: 1000
22
+ memory_capacity: 1000
23
+
24
+ # Disk cache size (maximum size of the disk-backed cache)
25
+ # Accepts human-readable sizes: "32MB", "64MB", "1GB", etc.
26
+ # Higher values improve performance for large journal files
27
+ # Default: 32MB
28
+ disk_capacity: "32MB"
29
+
30
+ # Cache block size (size of individual cache blocks in Foyer)
31
+ # Accepts human-readable sizes: "4MB", "8MB", "16MB", etc.
32
+ # Trade-off between I/O efficiency and memory granularity
33
+ # Default: 4MB
34
+ block_size: "4MB"
35
+
36
+ # Number of background workers for indexing journal files
37
+ # Higher values speed up indexing but use more CPU
38
+ # Default: number of CPU cores (auto-detected)
39
+ # Uncomment to override:
40
+ # workers: 24
41
+
42
+ # Queue capacity for pending indexing requests
43
+ # Controls backpressure on the indexing system
44
+ # Default: 100
45
+ queue_capacity: 100
src/crates/netdata-log-viewer/journal-viewer-plugin/src/catalog.rs
new
+764
@@ -0,0 +1,764 @@
1
+//! Journal catalog functionality with file monitoring and metadata tracking
2
+
3
+use async_trait::async_trait;
4
+use netdata_plugin_error::Result;
5
+use netdata_plugin_protocol::FunctionDeclaration;
6
+use netdata_plugin_schema::HttpAccess;
7
+use parking_lot::RwLock;
8
+use rt::FunctionHandler;
9
+use std::sync::Arc;
10
+use tracing::{debug, error, info, instrument, warn};
11
+
12
+// Import types from journal-function crate
13
+use journal_function::{
14
+ Facets, FileIndexCache, FileIndexCacheBuilder, FileIndexKey, HistogramEngine, Monitor,
15
+ Registry, Result as CatalogResult, netdata,
16
+};
17
+
18
+/*
19
+ * CatalogFunction
20
+*/
21
+use std::collections::HashMap;
22
+
23
+/// Request parameters for the catalog function (uses journal request structure)
24
+pub type CatalogRequest = netdata::JournalRequest;
25
+
26
+/// Response from the catalog function (uses journal response structure)
27
+pub type CatalogResponse = netdata::JournalResponse;
28
+
29
+use journal_index::Filter;
30
+use journal_index::{FieldName, FieldValuePair, Microseconds, Seconds};
31
+
32
+/// Builds a Filter from the selections HashMap
33
+#[instrument(skip(selections))]
34
+fn build_filter_from_selections(selections: &HashMap<String, Vec<String>>) -> Filter {
35
+ if selections.is_empty() {
36
+ return Filter::none();
37
+ }
38
+
39
+ let mut field_filters = Vec::new();
40
+
41
+ for (field, values) in selections {
42
+ if values.is_empty() {
43
+ continue;
44
+ }
45
+
46
+ // Build OR filter for all values of this field
47
+ let value_filters: Vec<_> = values
48
+ .iter()
49
+ .filter_map(|value| {
50
+ let pair_str = format!("{}={}", field, value);
51
+ FieldValuePair::parse(&pair_str).map(Filter::match_field_value_pair)
52
+ })
53
+ .collect();
54
+
55
+ if value_filters.is_empty() {
56
+ warn!("All values failed to parse for field '{}'", field);
57
+ continue;
58
+ }
59
+
60
+ let field_filter = Filter::or(value_filters);
61
+ field_filters.push(field_filter);
62
+ }
63
+
64
+ if field_filters.is_empty() {
65
+ Filter::none()
66
+ } else {
67
+ Filter::and(field_filters)
68
+ }
69
+}
70
+
71
+fn accepted_params() -> Vec<netdata::RequestParam> {
72
+ use netdata::RequestParam;
73
+
74
+ vec![
75
+ RequestParam::Info,
76
+ RequestParam::After,
77
+ RequestParam::Before,
78
+ RequestParam::Anchor,
79
+ RequestParam::Direction,
80
+ RequestParam::Last,
81
+ RequestParam::Query,
82
+ RequestParam::Facets,
83
+ RequestParam::Histogram,
84
+ RequestParam::IfModifiedSince,
85
+ RequestParam::DataOnly,
86
+ RequestParam::Delta,
87
+ RequestParam::Tail,
88
+ RequestParam::Sampling,
89
+ RequestParam::Slice,
90
+ ]
91
+}
92
+
93
+fn required_params() -> Vec<netdata::RequiredParam> {
94
+ Vec::new()
95
+}
96
+
97
+#[derive(Debug)]
98
+struct TransactionInner {
99
+ id: String,
100
+ start_time: tokio::time::Instant,
101
+ report_progress: bool,
102
+ cancel_call: bool,
103
+ timeout: Option<journal_function::Timeout>,
104
+}
105
+
106
+/// Represents a tracked transaction for a function call.
107
+///
108
+/// Transactions track the lifecycle and state of individual function calls,
109
+/// allowing for cancellation checks, progress reporting, and timeout detection.
110
+#[derive(Debug, Clone)]
111
+struct Transaction {
112
+ inner: Arc<RwLock<TransactionInner>>,
113
+}
114
+
115
+impl Transaction {
116
+ /// Create a new transaction with the given ID.
117
+ fn new(id: String, timeout: Option<journal_function::Timeout>) -> Self {
118
+ Self {
119
+ inner: Arc::new(RwLock::new(TransactionInner {
120
+ id,
121
+ start_time: tokio::time::Instant::now(),
122
+ report_progress: false,
123
+ cancel_call: false,
124
+ timeout,
125
+ })),
126
+ }
127
+ }
128
+
129
+ /// Get the transaction ID.
130
+ fn id(&self) -> String {
131
+ self.inner.read().id.clone()
132
+ }
133
+
134
+ /// Check if the transaction has been marked for cancellation.
135
+ #[allow(dead_code)]
136
+ fn is_cancelled(&self) -> bool {
137
+ self.inner.read().cancel_call
138
+ }
139
+
140
+ /// Mark the transaction for cancellation.
141
+ fn cancel(&self) {
142
+ self.inner.write().cancel_call = true;
143
+ }
144
+
145
+ /// Check if progress reporting is requested for this transaction.
146
+ #[allow(dead_code)]
147
+ fn should_report_progress(&self) -> bool {
148
+ self.inner.read().report_progress
149
+ }
150
+
151
+ /// Set the progress reporting flag.
152
+ fn set_report_progress(&self, report: bool) {
153
+ self.inner.write().report_progress = report;
154
+ }
155
+
156
+ /// Get the elapsed time since the transaction started.
157
+ fn elapsed(&self) -> std::time::Duration {
158
+ self.inner.read().start_time.elapsed()
159
+ }
160
+
161
+ /// Reset the timeout to the initial budget from the current time.
162
+ ///
163
+ /// This is called when progress is reported to give the operation its full timeout budget again.
164
+ fn reset_timeout(&self) {
165
+ if let Some(timeout) = &self.inner.read().timeout {
166
+ timeout.reset();
167
+ }
168
+ }
169
+}
170
+
171
+/// Registry for managing active transactions.
172
+///
173
+/// Provides thread-safe storage and lookup for transactions, allowing
174
+/// multiple parts of the application to track and manage ongoing operations.
175
+#[derive(Debug, Clone)]
176
+struct TransactionRegistry {
177
+ transactions: Arc<RwLock<HashMap<String, Transaction>>>,
178
+}
179
+
180
+impl TransactionRegistry {
181
+ /// Create a new transaction registry.
182
+ fn new() -> Self {
183
+ Self {
184
+ transactions: Arc::new(RwLock::new(HashMap::new())),
185
+ }
186
+ }
187
+
188
+ /// Create and register a new transaction with the given ID.
189
+ ///
190
+ /// Returns None if a transaction with this ID already exists.
191
+ fn create(
192
+ &self,
193
+ id: String,
194
+ timeout: Option<journal_function::Timeout>,
195
+ ) -> Option<Transaction> {
196
+ let mut transactions = self.transactions.write();
197
+
198
+ if transactions.contains_key(&id) {
199
+ warn!("Transaction {} already exists in registry", id);
200
+ return None;
201
+ }
202
+
203
+ let transaction = Transaction::new(id.clone(), timeout);
204
+ transactions.insert(id, transaction.clone());
205
+
206
+ Some(transaction)
207
+ }
208
+
209
+ /// Get an existing transaction by ID.
210
+ fn get(&self, id: &str) -> Option<Transaction> {
211
+ self.transactions.read().get(id).cloned()
212
+ }
213
+
214
+ /// Remove a transaction from the registry.
215
+ ///
216
+ /// Returns the removed transaction if it existed.
217
+ fn remove(&self, id: &str) -> Option<Transaction> {
218
+ let transaction = self.transactions.write().remove(id);
219
+ transaction
220
+ }
221
+
222
+ /// Cancel a transaction by ID.
223
+ ///
224
+ /// Returns true if the transaction was found and cancelled.
225
+ fn cancel(&self, id: &str) -> bool {
226
+ if let Some(transaction) = self.get(id) {
227
+ transaction.cancel();
228
+ info!("Cancelled transaction {}", id);
229
+ true
230
+ } else {
231
+ warn!("Cannot cancel non-existent transaction {}", id);
232
+ false
233
+ }
234
+ }
235
+
236
+ /// Get the number of active transactions.
237
+ #[allow(dead_code)]
238
+ fn len(&self) -> usize {
239
+ self.transactions.read().len()
240
+ }
241
+
242
+ /// Check if the registry is empty.
243
+ #[allow(dead_code)]
244
+ fn is_empty(&self) -> bool {
245
+ self.transactions.read().is_empty()
246
+ }
247
+}
248
+
249
+/// Inner state for CatalogFunction
250
+struct CatalogFunctionInner {
251
+ registry: Registry,
252
+ cache: FileIndexCache,
253
+ histogram_engine: Arc<HistogramEngine>,
254
+ transaction_registry: TransactionRegistry,
255
+}
256
+
257
+/// Function handler that provides catalog information about journal files
258
+#[derive(Clone)]
259
+pub struct CatalogFunction {
260
+ inner: Arc<CatalogFunctionInner>,
261
+}
262
+
263
+impl CatalogFunction {
264
+ /// Query log entries from pre-indexed files.
265
+ ///
266
+ /// This method:
267
+ /// 1. Queries log entries using LogQuery from pre-indexed files
268
+ /// 2. Returns raw log entry data and pagination flags
269
+ ///
270
+ /// Returns: (entries, has_before, has_after)
271
+ /// - entries: The log entries matching the query
272
+ /// - has_before: true if there are more entries before the returned window
273
+ /// - has_after: true if there are more entries after the returned window
274
+ fn query_logs_from_indexes(
275
+ &self,
276
+ indexed_files: &[journal_index::FileIndex],
277
+ time_range: &journal_function::QueryTimeRange,
278
+ anchor: Option<u64>,
279
+ filter: &Filter,
280
+ search_query: &str,
281
+ limit: usize,
282
+ direction: journal_index::Direction,
283
+ ) -> (Vec<journal_function::LogEntryData>, bool, bool) {
284
+ use journal_function::LogQuery;
285
+
286
+ if indexed_files.is_empty() {
287
+ return (Vec::new(), false, false);
288
+ }
289
+
290
+ // Convert time range boundaries to microseconds
291
+ let after_usec = time_range.aligned_start() as u64 * 1_000_000;
292
+ let before_usec = time_range.aligned_end() as u64 * 1_000_000;
293
+
294
+ // Query log entries
295
+ // Determine anchor point: use explicit anchor if provided, otherwise use time range boundary
296
+ let query_anchor = if let Some(anchor_usec) = anchor {
297
+ match direction {
298
+ journal_index::Direction::Forward => {
299
+ // Forward: start from lower boundary
300
+ journal_index::Anchor::Timestamp(Microseconds(anchor_usec + 1))
301
+ }
302
+ journal_index::Direction::Backward => {
303
+ // Backward: start from upper boundary
304
+ journal_index::Anchor::Timestamp(Microseconds(anchor_usec - 1))
305
+ }
306
+ }
307
+ } else {
308
+ // No explicit anchor: start from time range boundary based on direction
309
+ match direction {
310
+ journal_index::Direction::Forward => {
311
+ // Forward: start from lower boundary
312
+ journal_index::Anchor::Timestamp(Microseconds(after_usec))
313
+ }
314
+ journal_index::Direction::Backward => {
315
+ // Backward: start from upper boundary
316
+ journal_index::Anchor::Timestamp(Microseconds(before_usec))
317
+ }
318
+ }
319
+ };
320
+
321
+ // Query with limit + 1 to detect if there are more entries in this direction
322
+ let mut query = LogQuery::new(&indexed_files, query_anchor, direction)
323
+ .with_limit(limit + 1)
324
+ .with_after_usec(after_usec)
325
+ .with_before_usec(before_usec);
326
+
327
+ // Only apply filter if it's not Filter::none() (which matches nothing)
328
+ if !filter.is_none() {
329
+ query = query.with_filter(filter.clone());
330
+ }
331
+
332
+ // Apply regex search if search_query is not empty
333
+ if !search_query.is_empty() {
334
+ query = query.with_regex(search_query);
335
+ }
336
+
337
+ let mut log_entries = match query.execute() {
338
+ Ok(entries) => entries,
339
+ Err(e) => {
340
+ error!("log query execution failed: {}", e);
341
+ if !search_query.is_empty() {
342
+ error!(
343
+ "query may have failed due to invalid regex pattern: {:?}",
344
+ search_query
345
+ );
346
+ }
347
+ return (Vec::new(), false, false);
348
+ }
349
+ };
350
+
351
+ // Check if we got more than limit entries (meaning there are more in this direction)
352
+ let has_more_in_query_direction = log_entries.len() > limit;
353
+ if has_more_in_query_direction {
354
+ log_entries.truncate(limit);
355
+ }
356
+
357
+ // Query 1 entry in the opposite direction to check if there are entries there
358
+ // However, if there's no anchor (initial query), we're starting from the time range
359
+ // boundary, so there are no entries in the opposite direction by definition.
360
+ let has_more_in_opposite_direction = if anchor.is_none() {
361
+ // No anchor: we're at the time range boundary, no entries in opposite direction
362
+ false
363
+ } else {
364
+ let opposite_direction = match direction {
365
+ journal_index::Direction::Forward => journal_index::Direction::Backward,
366
+ journal_index::Direction::Backward => journal_index::Direction::Forward,
367
+ };
368
+
369
+ let opposite_anchor = match opposite_direction {
370
+ journal_index::Direction::Forward => {
371
+ journal_index::Anchor::Timestamp(Microseconds(anchor.unwrap() + 1))
372
+ }
373
+ journal_index::Direction::Backward => {
374
+ journal_index::Anchor::Timestamp(Microseconds(anchor.unwrap() - 1))
375
+ }
376
+ };
377
+
378
+ let mut opposite_query =
379
+ LogQuery::new(&indexed_files, opposite_anchor, opposite_direction)
380
+ .with_limit(1)
381
+ .with_after_usec(after_usec)
382
+ .with_before_usec(before_usec);
383
+
384
+ // Only apply filter if it's not Filter::none() (which matches nothing)
385
+ if !filter.is_none() {
386
+ opposite_query = opposite_query.with_filter(filter.clone());
387
+ }
388
+
389
+ // Apply regex search if search_query is not empty
390
+ if !search_query.is_empty() {
391
+ debug!("applying regex filter to opposite direction query");
392
+ opposite_query = opposite_query.with_regex(search_query);
393
+ }
394
+
395
+ match opposite_query.execute() {
396
+ Ok(entries) => !entries.is_empty(),
397
+ Err(e) => {
398
+ warn!("opposite direction query error: {}", e);
399
+ if !search_query.is_empty() {
400
+ warn!(
401
+ "opposite direction query may have failed due to invalid regex pattern"
402
+ );
403
+ }
404
+ false
405
+ }
406
+ }
407
+ };
408
+
409
+ // Calculate has_before and has_after based on the query direction
410
+ let (has_before, has_after) = match direction {
411
+ journal_index::Direction::Forward => {
412
+ // Forward query: has_more means has_after, opposite check gives has_before
413
+ (has_more_in_opposite_direction, has_more_in_query_direction)
414
+ }
415
+ journal_index::Direction::Backward => {
416
+ // Backward query: has_more means has_before, opposite check gives has_after
417
+ (has_more_in_query_direction, has_more_in_opposite_direction)
418
+ }
419
+ };
420
+
421
+ // UI always expects logs sorted descending by timestamp (newest first)
422
+ // regardless of query direction
423
+ log_entries.sort_by(|a, b| b.timestamp.cmp(&a.timestamp));
424
+
425
+ (log_entries, has_before, has_after)
426
+ }
427
+
428
+ /// Create a new catalog function with the given monitor and cache configuration.
429
+ ///
430
+ /// # Arguments
431
+ /// * `monitor` - File system monitor for watching journal directories
432
+ /// * `cache_dir` - Directory path for disk cache storage
433
+ /// * `memory_capacity` - Number of file indexes to keep in memory
434
+ /// * `disk_capacity` - Disk cache size in bytes
435
+ /// * `file_indexing_metrics` - Metrics chart for file indexing operations
436
+ /// * `bucket_cache_metrics` - Metrics chart for bucket cache operations
437
+ /// * `bucket_operations_metrics` - Metrics chart for bucket operations
438
+ pub async fn new(
439
+ monitor: Monitor,
440
+ cache_dir: impl Into<std::path::PathBuf>,
441
+ memory_capacity: usize,
442
+ disk_capacity: usize,
443
+ ) -> CatalogResult<Self> {
444
+ let registry = Registry::new(monitor);
445
+
446
+ // Create file index cache with disk-backed storage
447
+ let cache = FileIndexCacheBuilder::new()
448
+ .with_cache_path(cache_dir)
449
+ .with_memory_capacity(memory_capacity)
450
+ .with_disk_capacity(disk_capacity)
451
+ .with_block_size(4 * 1024 * 1024)
452
+ .build()
453
+ .await?;
454
+
455
+ // Create histogram engine
456
+ let histogram_engine = HistogramEngine::new();
457
+
458
+ let inner = CatalogFunctionInner {
459
+ registry,
460
+ cache,
461
+ histogram_engine: Arc::new(histogram_engine),
462
+ transaction_registry: TransactionRegistry::new(),
463
+ };
464
+
465
+ Ok(Self {
466
+ inner: Arc::new(inner),
467
+ })
468
+ }
469
+
470
+ /// Watch a directory for journal files
471
+ pub fn watch_directory(&self, path: &str) -> Result<()> {
472
+ self.inner.registry.watch_directory(path).map_err(|e| {
473
+ netdata_plugin_error::NetdataPluginError::Other {
474
+ message: format!("failed to watch directory: {}", e),
475
+ }
476
+ })
477
+ }
478
+
479
+ /// Process a notify event
480
+ pub fn process_notify_event(&self, event: notify::Event) {
481
+ if let Err(e) = self.inner.registry.process_event(event) {
482
+ error!("failed to process notify event: {}", e);
483
+ }
484
+ }
485
+}
486
+
487
+#[async_trait]
488
+impl FunctionHandler for CatalogFunction {
489
+ type Request = CatalogRequest;
490
+ type Response = CatalogResponse;
491
+
492
+ async fn on_call(&self, transaction: String, request: Self::Request) -> Result<Self::Response> {
493
+ // Register the transaction with the timeout
494
+ let timeout = journal_function::Timeout::new(std::time::Duration::from_secs(10));
495
+ let Some(txn) = self
496
+ .inner
497
+ .transaction_registry
498
+ .create(transaction.clone(), Some(timeout.clone()))
499
+ else {
500
+ return Err(netdata_plugin_error::NetdataPluginError::Other {
501
+ message: format!("[{}] transaction already exists", transaction),
502
+ });
503
+ };
504
+ info!("[{}] started transaction", txn.id());
505
+
506
+ // Create query time range with automatic alignment
507
+ let time_range = journal_function::QueryTimeRange::new(request.after, request.before)
508
+ .map_err(|e| netdata_plugin_error::NetdataPluginError::Other {
509
+ message: format!("[{}] {}", txn.id(), e),
510
+ })?;
511
+
512
+ info!(
513
+ "[{}] time range: [{}, {}), aligned: [{}, {}), bucket duration: {} seconds",
514
+ txn.id(),
515
+ time_range.requested_start(),
516
+ time_range.requested_end(),
517
+ time_range.aligned_start(),
518
+ time_range.aligned_end(),
519
+ time_range.bucket_duration()
520
+ );
521
+
522
+ // Find files in the time range
523
+ let op_start = std::time::Instant::now();
524
+ let files = self
525
+ .inner
526
+ .registry
527
+ .find_files_in_range(Seconds(request.after), Seconds(request.before))
528
+ .map_err(|e| netdata_plugin_error::NetdataPluginError::Other {
529
+ message: format!("[{}] failed to find files in range: {}", txn.id(), e),
530
+ })?;
531
+ let find_files_duration = op_start.elapsed();
532
+ info!("[{}] found {} files in time range", txn.id(), files.len(),);
533
+ if tracing::enabled!(tracing::Level::DEBUG) {
534
+ for (idx, file_info) in files.iter().enumerate() {
535
+ debug!(
536
+ "[{}] file[{}/{}]: {}",
537
+ txn.id(),
538
+ idx + 1,
539
+ files.len(),
540
+ file_info.file.path(),
541
+ );
542
+ }
543
+ }
544
+
545
+ // Build fiter expression
546
+ let filter_expr = build_filter_from_selections(&request.selections);
547
+ info!("[{}] filter expression: {}", txn.id(), filter_expr);
548
+
549
+ // Build facets for file indexes
550
+ let facets = Facets::new(&request.facets);
551
+ info!(
552
+ "[{}] using {} facets with precomputed hash {}",
553
+ txn.id(),
554
+ facets.len(),
555
+ facets.precomputed_hash()
556
+ );
557
+ if tracing::enabled!(tracing::Level::DEBUG) {
558
+ for (idx, facet) in facets.iter().enumerate() {
559
+ debug!(
560
+ "[{}] facet[{}/{}]: {}",
561
+ txn.id(),
562
+ idx + 1,
563
+ facets.len(),
564
+ facet.as_str(),
565
+ );
566
+ }
567
+ }
568
+
569
+ // Build file index keys
570
+ let source_timestamp_field = FieldName::new_unchecked("_SOURCE_REALTIME_TIMESTAMP");
571
+ let keys: Vec<FileIndexKey> = files
572
+ .iter()
573
+ .map(|f| FileIndexKey::new(&f.file, &facets, Some(source_timestamp_field.clone())))
574
+ .collect();
575
+
576
+ // Index all files
577
+ let op_start = std::time::Instant::now();
578
+ let indexed_files = journal_function::batch_compute_file_indexes(
579
+ &self.inner.cache,
580
+ &self.inner.registry,
581
+ keys,
582
+ &time_range,
583
+ timeout,
584
+ )
585
+ .await
586
+ .map_err(|e| netdata_plugin_error::NetdataPluginError::Other {
587
+ message: format!("[{}] failed to index files: {}", txn.id(), e),
588
+ })?;
589
+ let indexing_duration = op_start.elapsed();
590
+
591
+ info!(
592
+ "[{}] retrieved {}/{} file indexes for histogram buckets and log entries",
593
+ txn.id(),
594
+ indexed_files.len(),
595
+ files.len(),
596
+ );
597
+ if tracing::enabled!(tracing::Level::DEBUG) {
598
+ for (idx, (key, file_index)) in indexed_files.iter().enumerate() {
599
+ debug!(
600
+ "[{}] file index[{}/{}]: {}, indexed at: {}, online: {}, bucket duration: {}",
601
+ txn.id(),
602
+ idx + 1,
603
+ files.len(),
604
+ key.file.path(),
605
+ file_index.indexed_at().0,
606
+ file_index.online(),
607
+ file_index.bucket_duration().0
608
+ );
609
+ }
610
+ }
611
+
612
+ // Compute histogram from pre-indexed files
613
+ let op_start = std::time::Instant::now();
614
+ let histogram = self
615
+ .inner
616
+ .histogram_engine
617
+ .compute_from_indexes(&indexed_files, &time_range, &request.facets, &filter_expr)
618
+ .map_err(|e| netdata_plugin_error::NetdataPluginError::Other {
619
+ message: format!("failed to compute histogram: {}", e),
620
+ })?;
621
+ let histogram_duration = op_start.elapsed();
622
+
623
+ // Query logs from pre-indexed files
624
+ let op_start = std::time::Instant::now();
625
+ let limit = request.last.unwrap_or(200);
626
+ let file_indexes: Vec<_> = indexed_files.iter().map(|(_, idx)| idx.clone()).collect();
627
+ let (log_entries, has_before, has_after) = self.query_logs_from_indexes(
628
+ &file_indexes,
629
+ &time_range,
630
+ request.anchor,
631
+ &filter_expr,
632
+ &request.query,
633
+ limit,
634
+ request.direction,
635
+ );
636
+ let query_logs_duration = op_start.elapsed();
637
+ info!(
638
+ "[{}] retrieved {} log entries (has before: {}, has after: {})",
639
+ txn.id(),
640
+ log_entries.len(),
641
+ has_before,
642
+ has_after
643
+ );
644
+
645
+ // Build Netdata UI response (columns + data)
646
+ let (columns, data) = netdata::build_ui_response(&histogram, &log_entries);
647
+
648
+ // Get transformations for histogram chart labels
649
+ let transformations = netdata::systemd_transformations();
650
+
651
+ // Determine which field to use for the histogram (default to PRIORITY if not specified)
652
+ let histogram_field_name = if request.histogram.is_empty() {
653
+ "PRIORITY"
654
+ } else {
655
+ &request.histogram
656
+ };
657
+ let histogram_field = FieldName::new_unchecked(histogram_field_name);
658
+
659
+ let ui_histogram = netdata::histogram(&histogram, &histogram_field, &transformations);
660
+
661
+ let items = netdata::Items {
662
+ evaluated: u32::MAX as usize,
663
+ unsampled: u32::MAX as usize,
664
+ estimated: u32::MAX as usize,
665
+ matched: ui_histogram.count(),
666
+ // UI treats these as booleans: 0 = false, >0 = true
667
+ before: if has_after { 1 } else { 0 },
668
+ after: if has_before { 1 } else { 0 },
669
+ returned: log_entries.len(),
670
+ max_to_return: limit,
671
+ };
672
+
673
+ let response = CatalogResponse {
674
+ progress: 100, // All responses are now complete
675
+ version: netdata::Version::default(),
676
+ accepted_params: accepted_params(),
677
+ required_params: required_params(),
678
+ facets: netdata::facets(&histogram, &transformations),
679
+ histogram: ui_histogram,
680
+ available_histograms: netdata::available_histograms(&histogram),
681
+ columns,
682
+ data,
683
+ default_charts: Vec::new(),
684
+ items,
685
+ show_ids: false,
686
+ has_history: true,
687
+ status: 200,
688
+ response_type: String::from("table"),
689
+ help: String::from("View, search and analyze systemd journal entries."),
690
+ pagination: netdata::Pagination::default(),
691
+ };
692
+
693
+ let Some(txn) = self.inner.transaction_registry.remove(&transaction) else {
694
+ return Err(netdata_plugin_error::NetdataPluginError::Other {
695
+ message: format!("[{}] transaction does not exist", transaction),
696
+ });
697
+ };
698
+ info!(
699
+ "[{}] completed transaction (find_files: {:?}, indexing: {:?}, histogram: {:?}, query_logs: {:?}, total: {:?})",
700
+ txn.id(),
701
+ find_files_duration,
702
+ indexing_duration,
703
+ histogram_duration,
704
+ query_logs_duration,
705
+ txn.elapsed()
706
+ );
707
+
708
+ Ok(response)
709
+ }
710
+
711
+ async fn on_cancellation(&self, transaction: String) -> Result<Self::Response> {
712
+ warn!("catalog function call {} cancelled by Netdata", transaction);
713
+
714
+ // Mark the transaction as cancelled
715
+ self.inner.transaction_registry.cancel(&transaction);
716
+
717
+ // Remove the transaction from the registry
718
+ self.inner.transaction_registry.remove(&transaction);
719
+
720
+ Err(netdata_plugin_error::NetdataPluginError::Other {
721
+ message: "catalog function cancelled by user".to_string(),
722
+ })
723
+ }
724
+
725
+ async fn on_progress(&self, transaction: String) {
726
+ info!(
727
+ "progress report requested for catalog function call {}",
728
+ transaction
729
+ );
730
+
731
+ // Mark the transaction for progress reporting and reset the timeout
732
+ if let Some(txn) = self.inner.transaction_registry.get(&transaction) {
733
+ txn.set_report_progress(true);
734
+ txn.reset_timeout();
735
+ info!(
736
+ "Transaction {} marked for progress reporting and timeout reset to initial budget (elapsed: {:?})",
737
+ transaction,
738
+ txn.elapsed()
739
+ );
740
+ } else {
741
+ warn!(
742
+ "Progress requested for non-existent transaction {}",
743
+ transaction
744
+ );
745
+ }
746
+ }
747
+
748
+ fn declaration(&self) -> FunctionDeclaration {
749
+ // NOTE: `rt` special cases this function call to handle GET/POST
750
+ // calls in a consistent way. If you rename this function, you should
751
+ // update the `rt` crate as well.
752
+
753
+ info!("generating journal-viewer function declaration");
754
+ let mut func_decl = FunctionDeclaration::new(
755
+ "journal-viewer",
756
+ "Query and visualize journal log entries with histograms and facets",
757
+ );
758
+ func_decl.global = true;
759
+ func_decl.tags = Some(String::from("logs"));
760
+ func_decl.access =
761
+ Some(HttpAccess::SIGNED_ID | HttpAccess::SAME_SPACE | HttpAccess::SENSITIVE_DATA);
762
+ func_decl
763
+ }
764
+}
src/crates/netdata-log-viewer/journal-viewer-plugin/src/charts.rs
new
+103
@@ -0,0 +1,103 @@
1
+//! Chart definitions for the journal-viewer plugin
2
+//!
3
+//! This module contains all Netdata chart metric structures that track
4
+//! plugin performance, cache utilization, and request characteristics.
5
+
6
+use rt::{ChartHandle, NetdataChart, StdPluginRuntime};
7
+use schemars::JsonSchema;
8
+use serde::{Deserialize, Serialize};
9
+use std::time::Duration;
10
+
11
+/// Container for all plugin metrics chart handles
12
+pub struct Metrics {
13
+ pub call_metrics: ChartHandle<JournalCallMetrics>,
14
+ pub cache_size: ChartHandle<BucketCacheSizeMetrics>,
15
+ pub bucket_responses: ChartHandle<BucketResponseMetrics>,
16
+ pub histogram_requests: ChartHandle<HistogramRequestMetrics>,
17
+}
18
+
19
+impl Metrics {
20
+ /// Register all metric charts with the plugin runtime
21
+ pub fn new(runtime: &mut StdPluginRuntime) -> Self {
22
+ Self {
23
+ call_metrics: runtime
24
+ .register_chart(JournalCallMetrics::default(), Duration::from_secs(1)),
25
+ cache_size: runtime
26
+ .register_chart(BucketCacheSizeMetrics::default(), Duration::from_secs(1)),
27
+ bucket_responses: runtime
28
+ .register_chart(BucketResponseMetrics::default(), Duration::from_secs(1)),
29
+ histogram_requests: runtime
30
+ .register_chart(HistogramRequestMetrics::default(), Duration::from_secs(1)),
31
+ }
32
+ }
33
+}
34
+
35
+/// Metrics for tracking journal function calls
36
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
37
+#[schemars(
38
+ extend("x-chart-id" = "journal_viewer.journal_calls"),
39
+ extend("x-chart-title" = "Journal Function Calls"),
40
+ extend("x-chart-units" = "calls/s"),
41
+ extend("x-chart-type" = "line"),
42
+ extend("x-chart-family" = "requests"),
43
+ extend("x-chart-context" = "journal_viewer.journal_calls"),
44
+)]
45
+pub struct JournalCallMetrics {
46
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
47
+ pub successful: u64,
48
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
49
+ pub failed: u64,
50
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
51
+ pub cancelled: u64,
52
+}
53
+
54
+/// Metrics for tracking cache sizes
55
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
56
+#[schemars(
57
+ extend("x-chart-id" = "journal_viewer.bucket_lru_cache_size"),
58
+ extend("x-chart-title" = "LRU Bucket Cache Size"),
59
+ extend("x-chart-units" = "entries"),
60
+ extend("x-chart-type" = "line"),
61
+ extend("x-chart-family" = "cache"),
62
+ extend("x-chart-context" = "journal_viewer.bucket_lru_cache_size"),
63
+)]
64
+pub struct BucketCacheSizeMetrics {
65
+ #[schemars(extend("x-dimension-algorithm" = "absolute"))]
66
+ pub partial: u64,
67
+ #[schemars(extend("x-dimension-algorithm" = "absolute"))]
68
+ pub complete: u64,
69
+}
70
+
71
+/// Metrics for tracking bucket response types
72
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
73
+#[schemars(
74
+ extend("x-chart-id" = "journal_viewer.bucket_responses"),
75
+ extend("x-chart-title" = "Bucket Response Types"),
76
+ extend("x-chart-units" = "buckets/s"),
77
+ extend("x-chart-type" = "stacked"),
78
+ extend("x-chart-family" = "responses"),
79
+ extend("x-chart-context" = "journal_viewer.bucket_responses"),
80
+)]
81
+pub struct BucketResponseMetrics {
82
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
83
+ pub complete: u64,
84
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
85
+ pub partial: u64,
86
+}
87
+
88
+/// Metrics for tracking histogram request characteristics
89
+#[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize, Deserialize)]
90
+#[schemars(
91
+ extend("x-chart-id" = "journal_viewer.histogram_requests"),
92
+ extend("x-chart-title" = "Histogram Request Details"),
93
+ extend("x-chart-units" = "count/s"),
94
+ extend("x-chart-type" = "line"),
95
+ extend("x-chart-family" = "requests"),
96
+ extend("x-chart-context" = "journal_viewer.histogram_requests"),
97
+)]
98
+pub struct HistogramRequestMetrics {
99
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
100
+ pub total_buckets: u64,
101
+ #[schemars(extend("x-dimension-algorithm" = "incremental"))]
102
+ pub pending_files: u64,
103
+}
src/crates/netdata-log-viewer/journal-viewer-plugin/src/main.rs
new
+119
@@ -0,0 +1,119 @@
1
+//! journal-viewer-plugin standalone binary
2
+
3
+use journal_registry::Monitor;
4
+
5
+mod catalog;
6
+use catalog::CatalogFunction;
7
+
8
+mod plugin_config;
9
+use plugin_config::PluginConfig;
10
+
11
+use rt::PluginRuntime;
12
+use tracing::{error, info};
13
+
14
+#[tokio::main]
15
+async fn main() {
16
+ println!("TRUST_DURATIONS 1");
17
+
18
+ rt::init_tracing();
19
+
20
+ let result = run_plugin().await;
21
+
22
+ match result {
23
+ Ok(()) => {
24
+ info!("plugin runtime stopped");
25
+ }
26
+ Err(e) => {
27
+ error!("plugin error: {:#}", e);
28
+ std::process::exit(1);
29
+ }
30
+ }
31
+}
32
+
33
+async fn run_plugin() -> std::result::Result<(), Box<dyn std::error::Error>> {
34
+ // Load configuration
35
+ let plugin_config = PluginConfig::new()?;
36
+ let config = &plugin_config.config;
37
+
38
+ info!(
39
+ "configuration loaded: journal_paths={:?}, cache_dir={}, memory_capacity={}, disk_capacity={}, workers={}",
40
+ config.journal.paths,
41
+ config.cache.directory,
42
+ config.cache.memory_capacity,
43
+ config.cache.disk_capacity,
44
+ config.cache.workers
45
+ );
46
+
47
+ let mut runtime = PluginRuntime::new("journal-viewer");
48
+ info!("plugin runtime created");
49
+
50
+ let (monitor, notify_rx) = match Monitor::new() {
51
+ Ok(t) => t,
52
+ Err(e) => {
53
+ error!("failed to setup notify monitoring: {}", e);
54
+ return Ok(());
55
+ }
56
+ };
57
+
58
+ // Create catalog function with disk-backed cache
59
+ info!("creating catalog function with Foyer hybrid cache");
60
+ let catalog_function = CatalogFunction::new(
61
+ monitor,
62
+ &config.cache.directory,
63
+ config.cache.memory_capacity,
64
+ config.cache.disk_capacity.as_u64() as usize,
65
+ )
66
+ .await?;
67
+ info!("catalog function initialized");
68
+
69
+ // Watch configured journal directories
70
+ for path in &config.journal.paths {
71
+ match catalog_function.watch_directory(path) {
72
+ Ok(()) => {
73
+ info!("watching journal directory: {}", path);
74
+ }
75
+ Err(e) => {
76
+ error!("failed to watch directory {}: {:#?}", path, e);
77
+ }
78
+ }
79
+ }
80
+
81
+ runtime.register_handler(catalog_function.clone());
82
+ info!("catalog function handler registered");
83
+
84
+ // Spawn task to process notify events
85
+ let catalog_function_clone = catalog_function.clone();
86
+ tokio::spawn(async move {
87
+ let mut notify_rx = notify_rx;
88
+ while let Some(event) = notify_rx.recv().await {
89
+ catalog_function_clone.process_notify_event(event);
90
+ }
91
+ info!("notify event processing task terminated");
92
+ });
93
+
94
+ // Keepalive future to prevent Netdata from killing the plugin
95
+ let writer = runtime.writer();
96
+ let keepalive = async move {
97
+ let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(60));
98
+ loop {
99
+ interval.tick().await;
100
+ if let Ok(mut w) = writer.try_lock() {
101
+ let _ = w.write_raw(b"PLUGIN_KEEPALIVE\n").await;
102
+ }
103
+ }
104
+ };
105
+
106
+ info!("starting plugin runtime");
107
+
108
+ // Run plugin runtime and keepalive concurrently
109
+ tokio::select! {
110
+ result = runtime.run() => {
111
+ result?;
112
+ }
113
+ _ = keepalive => {
114
+ // Keepalive loop never completes normally
115
+ }
116
+ }
117
+
118
+ Ok(())
119
+}
src/crates/netdata-log-viewer/journal-viewer-plugin/src/plugin_config.rs
new
+209
@@ -0,0 +1,209 @@
1
+use anyhow::{Context, Result};
2
+use bytesize::ByteSize;
3
+use rt::NetdataEnv;
4
+use serde::{Deserialize, Serialize};
5
+use std::fs;
6
+use std::path::Path;
7
+use tracing::warn;
8
+
9
+/// Default value for workers (number of CPU cores)
10
+fn default_workers() -> usize {
11
+ num_cpus::get()
12
+}
13
+
14
+#[derive(Debug, Clone, Serialize, Deserialize)]
15
+#[serde(deny_unknown_fields)]
16
+pub struct JournalConfig {
17
+ /// Paths to systemd journal directories to watch
18
+ pub paths: Vec<String>,
19
+}
20
+
21
+impl Default for JournalConfig {
22
+ fn default() -> Self {
23
+ Self {
24
+ paths: vec![String::from("/var/log/journal")],
25
+ }
26
+ }
27
+}
28
+
29
+#[derive(Debug, Clone, Serialize, Deserialize)]
30
+#[serde(deny_unknown_fields)]
31
+pub struct CacheConfig {
32
+ /// Directory to store the hybrid cache (memory + disk)
33
+ pub directory: String,
34
+
35
+ /// Memory cache capacity (number of entries to cache in memory)
36
+ pub memory_capacity: usize,
37
+
38
+ /// Disk cache size (total size of disk-backed cache)
39
+ #[serde(with = "bytesize_serde")]
40
+ pub disk_capacity: ByteSize,
41
+
42
+ /// Cache block size (size of cache blocks)
43
+ #[serde(with = "bytesize_serde")]
44
+ pub block_size: ByteSize,
45
+
46
+ /// Number of background workers for indexing journal files
47
+ #[serde(default = "default_workers")]
48
+ pub workers: usize,
49
+
50
+ /// Queue capacity for pending indexing requests
51
+ pub queue_capacity: usize,
52
+}
53
+
54
+impl Default for CacheConfig {
55
+ fn default() -> Self {
56
+ Self {
57
+ directory: String::from("/var/cache/netdata/log-viewer"),
58
+ memory_capacity: 1000,
59
+ disk_capacity: ByteSize::mb(32),
60
+ block_size: ByteSize::mb(4),
61
+ workers: default_workers(),
62
+ queue_capacity: 100,
63
+ }
64
+ }
65
+}
66
+
67
+#[derive(Default, Debug, Clone, Serialize, Deserialize)]
68
+#[serde(deny_unknown_fields)]
69
+pub struct Config {
70
+ /// Journal source configuration
71
+ #[serde(rename = "journal")]
72
+ pub journal: JournalConfig,
73
+
74
+ /// Cache configuration
75
+ #[serde(rename = "cache")]
76
+ pub cache: CacheConfig,
77
+}
78
+
79
+pub struct PluginConfig {
80
+ pub config: Config,
81
+ pub _netdata_env: NetdataEnv,
82
+}
83
+
84
+impl PluginConfig {
85
+ /// Load configuration from Netdata environment
86
+ pub fn new() -> Result<Self> {
87
+ let netdata_env = NetdataEnv::from_environment();
88
+
89
+ let mut config = if netdata_env.running_under_netdata() {
90
+ // Running under Netdata - try user config first, fallback to stock config
91
+ let user_config = netdata_env
92
+ .user_config_dir
93
+ .as_ref()
94
+ .map(|path| path.join("journal-viewer.yaml"))
95
+ .and_then(|path| {
96
+ if path.exists() {
97
+ Config::from_yaml_file(&path)
98
+ .with_context(|| format!("Loading user config from {}", path.display()))
99
+ .ok()
100
+ } else {
101
+ None
102
+ }
103
+ });
104
+
105
+ if let Some(config) = user_config {
106
+ config
107
+ } else if let Some(stock_path) = netdata_env
108
+ .stock_config_dir
109
+ .as_ref()
110
+ .map(|p| p.join("journal-viewer.yaml"))
111
+ {
112
+ if stock_path.exists() {
113
+ Config::from_yaml_file(&stock_path).with_context(|| {
114
+ format!("Loading stock config from {}", stock_path.display())
115
+ })?
116
+ } else {
117
+ // No config files found, use defaults
118
+ Config::default()
119
+ }
120
+ } else {
121
+ // No config directories available, use defaults
122
+ Config::default()
123
+ }
124
+ } else {
125
+ // Not running under Netdata, use defaults
126
+ Config::default()
127
+ };
128
+
129
+ // Resolve relative paths
130
+ config.cache.directory =
131
+ resolve_relative_path(&config.cache.directory, netdata_env.cache_dir.as_deref());
132
+
133
+ // Validate configuration (also performs deduplication)
134
+ Self::validate(&mut config)?;
135
+
136
+ Ok(PluginConfig {
137
+ config,
138
+ _netdata_env: netdata_env,
139
+ })
140
+ }
141
+
142
+ /// Validate configuration values
143
+ fn validate(config: &mut Config) -> Result<()> {
144
+ // Validate journal paths
145
+ if config.journal.paths.is_empty() {
146
+ anyhow::bail!("journal.paths must contain at least one path");
147
+ }
148
+
149
+ // Deduplicate paths while preserving order
150
+ let mut seen = std::collections::HashSet::new();
151
+ config
152
+ .journal
153
+ .paths
154
+ .retain(|path| seen.insert(path.clone()));
155
+
156
+ // Validate that journal paths exist (warning only)
157
+ for path in &config.journal.paths {
158
+ if !Path::new(path).exists() {
159
+ warn!("journal path does not exist: {}", path);
160
+ }
161
+ }
162
+
163
+ if config.cache.memory_capacity == 0 {
164
+ anyhow::bail!("cache.memory_capacity must be greater than 0");
165
+ }
166
+
167
+ if config.cache.disk_capacity.as_u64() == 0 {
168
+ anyhow::bail!("cache.disk_capacity must be greater than 0");
169
+ }
170
+
171
+ if config.cache.block_size.as_u64() == 0 {
172
+ anyhow::bail!("cache.block_size must be greater than 0");
173
+ }
174
+
175
+ if config.cache.workers == 0 {
176
+ anyhow::bail!("cache.workers must be greater than 0");
177
+ }
178
+
179
+ if config.cache.queue_capacity == 0 {
180
+ anyhow::bail!("cache.queue_capacity must be greater than 0");
181
+ }
182
+
183
+ Ok(())
184
+ }
185
+}
186
+
187
+impl Config {
188
+ /// Load configuration from a YAML file
189
+ pub fn from_yaml_file<P: AsRef<Path>>(path: P) -> Result<Self> {
190
+ let path = path.as_ref();
191
+ let contents = fs::read_to_string(path)
192
+ .with_context(|| format!("Failed to read config file: {}", path.display()))?;
193
+ let config: Config = serde_yaml::from_str(&contents)
194
+ .with_context(|| format!("Failed to parse YAML config file: {}", path.display()))?;
195
+ Ok(config)
196
+ }
197
+}
198
+
199
+/// Helper function to resolve relative paths against a base directory
200
+fn resolve_relative_path(path: &str, base_dir: Option<&Path>) -> String {
201
+ let path = Path::new(path);
202
+ if path.is_absolute() {
203
+ path.to_string_lossy().to_string()
204
+ } else if let Some(base) = base_dir {
205
+ base.join(path).to_string_lossy().to_string()
206
+ } else {
207
+ path.to_string_lossy().to_string()
208
+ }
209
+}
src/crates/netdata-otel/flatten_otel/Cargo.toml
renamed
+4
@@ -4,7 +4,11 @@ version.workspace = true
4
edition.workspace = true
5
rust-version.workspace = true
6
7
+[lints]
8
+workspace = true
9
+
10
[dependencies]
11
opentelemetry-proto = { workspace = true, features = ["logs", "metrics", "with-serde"] }
12
flatten-serde-json = { workspace = true }
13
serde_json = { workspace = true }
14
+tracing = { workspace = true }
src/crates/netdata-otel/flatten_otel/src/lib.rs
renamed
+3
-3
@@ -11,7 +11,7 @@ mod metrics;
11
pub use logs::{json_from_export_logs_service_request, json_from_log_record};
12
pub use metrics::flatten_metrics_request;
13
14
-fn json_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValue> {
14
+pub fn json_from_key_value_list(kvl: &Vec<KeyValue>) -> JsonMap<String, JsonValue> {
15
let mut map = JsonMap::new();
16
17
for kv in kvl {
@@ -49,7 +49,7 @@ fn json_from_any_value(any_value: &AnyValue) -> JsonValue {
49
}
50
}
51
52
-fn json_from_resource(jm: &mut JsonMap<String, JsonValue>, resource: &Resource) {
52
+pub fn json_from_resource(jm: &mut JsonMap<String, JsonValue>, resource: &Resource) {
53
if !resource.attributes.is_empty() {
54
let resource_attrs = json_from_key_value_list(&resource.attributes);
55
for (key, value) in resource_attrs {
@@ -58,7 +58,7 @@ fn json_from_resource(jm: &mut JsonMap<String, JsonValue>, resource: &Resource)
58
}
59
}
60
61
-fn json_from_instrumentation_scope(
61
+pub fn json_from_instrumentation_scope(
62
jm: &mut JsonMap<String, JsonValue>,
63
scope: &InstrumentationScope,
64
) {
src/crates/netdata-otel/flatten_otel/src/logs.rs
renamed
+45
-7
@@ -30,13 +30,50 @@ pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &Lo
30
31
// Add body if present
32
if let Some(body) = &log_record.body {
33
- let mut temp_map = JsonMap::new();
34
- temp_map.insert("body".to_string(), json_from_any_value(body));
35
-
36
- let flattened_body = flatten_serde_json::flatten(&temp_map);
37
- for (key, value) in flattened_body {
38
- if key != "body" {
39
- jm.insert(format!("log.{}", key), value);
33
+ let body_json = json_from_any_value(body);
34
+
35
+ match &body_json {
36
+ // If body is a string, try to parse it as JSON first
37
+ JsonValue::String(s) => {
38
+ // Try to parse the string as JSON
39
+ if let Ok(parsed) = serde_json::from_str::<JsonValue>(s) {
40
+ // Successfully parsed JSON - check if it's an object
41
+ if let JsonValue::Object(_) = parsed {
42
+ // Flatten the parsed JSON object
43
+ let mut temp_map = JsonMap::new();
44
+ temp_map.insert("body".to_string(), parsed);
45
+
46
+ let flattened_body = flatten_serde_json::flatten(&temp_map);
47
+ for (key, value) in flattened_body {
48
+ jm.insert(format!("log.{}", key), value);
49
+ }
50
+ } else {
51
+ // Parsed JSON but not an object (array, primitive, etc.)
52
+ // Store the original string
53
+ jm.insert("log.body".to_string(), body_json);
54
+ }
55
+ } else {
56
+ // Not valid JSON, store as-is
57
+ jm.insert("log.body".to_string(), body_json);
58
+ }
59
+ }
60
+ // If body is a number or bool, add it directly
61
+ JsonValue::Number(_) | JsonValue::Bool(_) => {
62
+ jm.insert("log.body".to_string(), body_json);
63
+ }
64
+ // If body is structured, flatten it
65
+ JsonValue::Object(_) => {
66
+ let mut temp_map = JsonMap::new();
67
+ temp_map.insert("body".to_string(), body_json);
68
+
69
+ let flattened_body = flatten_serde_json::flatten(&temp_map);
70
+ for (key, value) in flattened_body {
71
+ jm.insert(format!("log.{}", key), value);
72
+ }
73
+ }
74
+ _ => {
75
+ // Arrays, null, etc. - add as-is
76
+ jm.insert("log.body".to_string(), body_json);
77
}
78
}
79
}
@@ -80,6 +117,7 @@ pub fn json_from_log_record(jm: &mut JsonMap<String, JsonValue>, log_record: &Lo
117
}
118
119
// TODO: this does not belong here, it should be part of the service.
120
+#[tracing::instrument(skip_all)]
121
pub fn json_from_export_logs_service_request(request: &ExportLogsServiceRequest) -> JsonValue {
122
let mut items = Vec::new();
123
src/crates/netdata-otel/flatten_otel/src/metrics.rs
renamed
src/crates/netdata-otel/otel-plugin/Cargo.toml
new
+46
@@ -0,0 +1,46 @@
1
+[package]
2
+name = "otel-plugin"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[lib]
11
+name = "otel_plugin"
12
+path = "src/lib.rs"
13
+
14
+[[bin]]
15
+name = "otel-plugin"
16
+path = "src/main.rs"
17
+
18
+[dependencies]
19
+anyhow = { workspace = true }
20
+atty = { workspace = true }
21
+bytesize-serde = { workspace = true }
22
+bytesize = { workspace = true }
23
+clap = { workspace = true, features = ["derive"] }
24
+humantime-serde = { workspace = true }
25
+humantime = { workspace = true }
26
+regex = { workspace = true }
27
+serde_json = { workspace = true, features = ["preserve_order"] }
28
+serde_regex = { workspace = true }
29
+serde = { workspace=true }
30
+serde_yaml = { workspace = true }
31
+tokio = { workspace = true }
32
+tonic = { workspace = true, features = ["gzip", "tls-ring"] }
33
+
34
+journal-common = { workspace = true }
35
+journal-core = { workspace = true }
36
+journal-log-writer = { workspace = true }
37
+journal-registry = { workspace = true }
38
+flatten_otel = { path = "../flatten_otel" }
39
+rt = { workspace = true }
40
+
41
+tracing = { workspace = true }
42
+
43
+opentelemetry = { workspace = true }
44
+opentelemetry_sdk = { workspace = true , features = ["rt-tokio", "logs"] }
45
+opentelemetry-otlp = { workspace = true, features = ["grpc-tonic", "logs"] }
46
+opentelemetry-proto = { workspace = true }
src/crates/netdata-otel/otel-plugin/README.md
renamed
+3
-3
@@ -5,14 +5,14 @@ enabling users to ingest, store and visualize OpenTelemetry metrics in charts.
5
6
## Configuration
7
8
-Edit the [otel.yml](https://github.com/netdata/netdata/blob/master/src/crates/jf/otel-plugin/configs/otel.yml)
8
+Edit the [otel.yaml](https://github.com/netdata/netdata/blob/master/src/crates/jf/otel-plugin/configs/otel.yaml)
9
configuration file using `edit-config` from the Netdata
10
[config directory](/docs/netdata-agent/configuration/README.md#locate-your-config-directory),
11
which is typically located under `/etc/netdata`.
12
13
```bash
14
cd /etc/netdata # Replace this path with your Netdata config directory
15
-sudo ./edit-config otel.yml
15
+sudo ./edit-config otel.yaml
16
```
17
18
### gRPC Endpoint
@@ -65,7 +65,7 @@ the attributes that the `otel.plugin` will use when creating new chart
65
instances and dimension names.
66
67
For example, the following bit from the
68
-[otel.d/v1/metrics/hostmetrics.yml](https://github.com/netdata/netdata/blob/master/src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yml)
68
+[otel.d/v1/metrics/hostmetrics.yaml](https://github.com/netdata/netdata/blob/master/src/crates/jf/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yaml)
69
configuration file for the [hostmetrics](https://github.com/open-telemetry/opentelemetry-collector-contrib/blob/main/receiver/hostmetricsreceiver/internal/scraper/networkscraper/documentation.md) receiver:
70
```yaml
71
select:
src/crates/netdata-otel/otel-plugin/configs/otel.d/v1/metrics/hostmetrics-receiver.yaml
renamed
src/crates/netdata-otel/otel-plugin/configs/otel.yaml
renamed
+4
@@ -47,3 +47,7 @@ logs:
47
48
# Maximum time span that entries in a single journal file can cover (e.g., "2 hours", "1h", "30m").
49
duration_of_journal_file: "2 hours"
50
+
51
+ # Store the complete OTLP JSON representation in the OTLP_JSON field.
52
+ # When enabled, each log entry includes the full original JSON message for debugging and reprocessing.
53
+ store_otlp_json: false
src/crates/netdata-otel/otel-plugin/src/chart_config.rs
renamed
+1
-1
@@ -95,7 +95,7 @@ impl ChartConfigManager {
95
96
fn load_stock_config(&mut self) {
97
const DEFAULT_CONFIGS_YAML: &str =
98
- include_str!("../configs/otel.d/v1/metrics/hostmetrics-receiver.yml");
98
+ include_str!("../configs/otel.d/v1/metrics/hostmetrics-receiver.yaml");
99
100
match serde_yaml::from_str::<ChartConfigs>(DEFAULT_CONFIGS_YAML) {
101
Ok(configs) => {
src/crates/netdata-otel/otel-plugin/src/flattened_point.rs
renamed
src/crates/netdata-otel/otel-plugin/src/lib.rs
new
+152
@@ -0,0 +1,152 @@
1
+//! otel-plugin library - can be called from multi-call binaries or standalone
2
+
3
+use anyhow::{Context, Result};
4
+use opentelemetry_proto::tonic::collector::{
5
+ logs::v1::logs_service_server::LogsServiceServer,
6
+ metrics::v1::metrics_service_server::MetricsServiceServer,
7
+};
8
+use rt::PluginRuntime;
9
+use tonic::transport::{Identity, Server, ServerTlsConfig};
10
+
11
+mod chart_config;
12
+mod flattened_point;
13
+mod netdata_chart;
14
+mod regex_cache;
15
+mod samples_table;
16
+
17
+mod plugin_config;
18
+use crate::plugin_config::PluginConfig;
19
+
20
+mod logs_service;
21
+use crate::logs_service::NetdataLogsService;
22
+
23
+mod metrics_service;
24
+use crate::metrics_service::NetdataMetricsService;
25
+
26
+/// Entry point for otel-plugin - can be called from multi-call binary
27
+///
28
+/// # Arguments
29
+/// * `args` - Command-line arguments (should include argv[0] as "otel-plugin")
30
+///
31
+/// # Returns
32
+/// Exit code (0 for success, non-zero for errors)
33
+pub fn run(args: Vec<String>) -> i32 {
34
+ // otel-plugin is async, so we need a tokio runtime
35
+ let runtime = tokio::runtime::Runtime::new().unwrap();
36
+ runtime.block_on(async_run(args))
37
+}
38
+
39
+async fn async_run(_args: Vec<String>) -> i32 {
40
+ rt::init_tracing();
41
+
42
+ match run_internal().await {
43
+ Ok(()) => 0,
44
+ Err(e) => {
45
+ eprintln!("Error: {:#}", e);
46
+ 1
47
+ }
48
+ }
49
+}
50
+
51
+async fn run_internal() -> Result<()> {
52
+ // 1. Create plugin runtime
53
+ let runtime = PluginRuntime::new("otel");
54
+
55
+ // 2. Get shared writer for protocol coordination
56
+ let writer = runtime.writer();
57
+
58
+ // 3. Write initial protocol messages
59
+ {
60
+ let mut w = writer.lock().await;
61
+ w.write_raw(b"TRUST_DURATIONS 1\n")
62
+ .await
63
+ .context("Failed to write TRUST_DURATIONS")?;
64
+ }
65
+
66
+ // 4. Load configuration
67
+ let config = PluginConfig::new().context("Failed to initialize plugin configuration")?;
68
+
69
+ // 5. Create gRPC services
70
+ let metrics_service = NetdataMetricsService::new(config.clone())
71
+ .context("Failed to create metrics service")?;
72
+ let logs_service = NetdataLogsService::new(config.clone())
73
+ .context("Failed to create logs service")?;
74
+
75
+ // 7. Parse gRPC endpoint address
76
+ let addr = config
77
+ .endpoint
78
+ .path
79
+ .parse()
80
+ .with_context(|| format!("Failed to parse endpoint address: {}", config.endpoint.path))?;
81
+
82
+ // 8. Build gRPC server (with TLS if configured)
83
+ let mut server_builder = Server::builder();
84
+
85
+ if let (Some(cert_path), Some(key_path)) = (
86
+ &config.endpoint.tls_cert_path,
87
+ &config.endpoint.tls_key_path,
88
+ ) {
89
+ let cert = std::fs::read(cert_path)
90
+ .with_context(|| format!("Failed to read TLS certificate from: {}", cert_path))?;
91
+ let key = std::fs::read(key_path)
92
+ .with_context(|| format!("Failed to read TLS private key from: {}", key_path))?;
93
+ let identity = Identity::from_pem(cert, key);
94
+
95
+ let mut tls_config_builder = ServerTlsConfig::new().identity(identity);
96
+
97
+ if let Some(ref ca_cert_path) = config.endpoint.tls_ca_cert_path {
98
+ let ca_cert = std::fs::read(ca_cert_path)
99
+ .with_context(|| format!("Failed to read CA certificate from: {}", ca_cert_path))?;
100
+ tls_config_builder =
101
+ tls_config_builder.client_ca_root(tonic::transport::Certificate::from_pem(ca_cert));
102
+ }
103
+
104
+ server_builder = server_builder
105
+ .tls_config(tls_config_builder)
106
+ .context("Failed to configure TLS")?;
107
+ } else {
108
+ eprintln!(
109
+ "TLS disabled, using insecure connection on endpoint: {}",
110
+ config.endpoint.path
111
+ );
112
+ }
113
+
114
+ // 9. Build gRPC server future
115
+ let grpc_server = server_builder
116
+ .add_service(
117
+ MetricsServiceServer::new(metrics_service)
118
+ .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
119
+ )
120
+ .add_service(
121
+ LogsServiceServer::new(logs_service)
122
+ .accept_compressed(tonic::codec::CompressionEncoding::Gzip),
123
+ )
124
+ .serve(addr);
125
+
126
+ // 10. Keepalive future (PluginRuntime doesn't send keepalive automatically)
127
+ let writer_clone = writer.clone();
128
+ let keepalive = async move {
129
+ let mut interval = tokio::time::interval(tokio::time::Duration::from_secs(60));
130
+ loop {
131
+ interval.tick().await;
132
+ if let Ok(mut w) = writer_clone.try_lock() {
133
+ let _ = w.write_raw(b"PLUGIN_KEEPALIVE\n").await;
134
+ }
135
+ }
136
+ };
137
+
138
+ // 11. Run gRPC server, plugin runtime, and keepalive concurrently
139
+ tokio::select! {
140
+ result = grpc_server => {
141
+ result.with_context(|| format!("gRPC server error on {}", config.endpoint.path))?;
142
+ }
143
+ result = runtime.run() => {
144
+ result.context("PluginRuntime error")?;
145
+ }
146
+ _ = keepalive => {
147
+ // Keepalive loop never completes normally
148
+ }
149
+ }
150
+
151
+ Ok(())
152
+}
src/crates/netdata-otel/otel-plugin/src/logs_service.rs
new
+180
@@ -0,0 +1,180 @@
1
+use anyhow::{Context, Result};
2
+use flatten_otel::json_from_export_logs_service_request;
3
+use journal_common::load_machine_id;
4
+use journal_log_writer::{Config, Log, RetentionPolicy, RotationPolicy};
5
+use journal_registry::Origin;
6
+use opentelemetry_proto::tonic::collector::logs::v1::{
7
+ ExportLogsServiceRequest, ExportLogsServiceResponse, logs_service_server::LogsService,
8
+};
9
+use serde_json::Value;
10
+use std::sync::{Arc, Mutex};
11
+use tonic::{Request, Response, Status};
12
+
13
+use crate::plugin_config::PluginConfig;
14
+
15
+pub struct NetdataLogsService {
16
+ log: Arc<Mutex<Log>>,
17
+ store_otlp_json: bool,
18
+}
19
+
20
+impl NetdataLogsService {
21
+ pub fn new(plugin_config: PluginConfig) -> Result<Self> {
22
+ let logs_config = plugin_config.logs;
23
+
24
+ let rotation_policy = RotationPolicy::default()
25
+ .with_size_of_journal_file(logs_config.size_of_journal_file.as_u64())
26
+ .with_duration_of_journal_file(logs_config.duration_of_journal_file)
27
+ .with_number_of_entries(logs_config.entries_of_journal_file);
28
+
29
+ let retention_policy = RetentionPolicy::default()
30
+ .with_number_of_journal_files(logs_config.number_of_journal_files)
31
+ .with_size_of_journal_files(logs_config.size_of_journal_files.as_u64())
32
+ .with_duration_of_journal_files(logs_config.duration_of_journal_files);
33
+
34
+ let machine_id = load_machine_id()?;
35
+ let origin = Origin {
36
+ machine_id: Some(machine_id),
37
+ namespace: None,
38
+ source: journal_registry::Source::System,
39
+ };
40
+
41
+ let path = std::path::Path::new(&logs_config.journal_dir);
42
+ let journal_config = Config::new(origin, rotation_policy, retention_policy);
43
+
44
+ let journal_log = Arc::new(Mutex::new(Log::new(path, journal_config).with_context(
45
+ || {
46
+ format!(
47
+ "Failed to create journal log for directory: {}",
48
+ logs_config.journal_dir
49
+ )
50
+ },
51
+ )?));
52
+ Ok(NetdataLogsService {
53
+ log: journal_log,
54
+ store_otlp_json: logs_config.store_otlp_json,
55
+ })
56
+ }
57
+
58
+ fn extract_timestamp_for_sorting(json_value: &Value) -> u64 {
59
+ if let Value::Object(obj) = json_value {
60
+ // Extract timestamp for sorting (same logic as json_to_entry_data)
61
+ // Per OTLP spec: Use time_unix_nano if present (non-zero), otherwise use observed_time_unix_nano
62
+ let time_unix_nano = obj
63
+ .get("log.time_unix_nano")
64
+ .and_then(|v| v.as_u64())
65
+ .filter(|&t| t != 0);
66
+
67
+ let observed_time_unix_nano = obj
68
+ .get("log.observed_time_unix_nano")
69
+ .and_then(|v| v.as_u64())
70
+ .filter(|&t| t != 0);
71
+
72
+ time_unix_nano.or(observed_time_unix_nano).unwrap_or(0)
73
+ } else {
74
+ 0
75
+ }
76
+ }
77
+
78
+ fn json_to_entry_data(&self, json_value: &Value) -> (Vec<Vec<u8>>, Option<u64>) {
79
+ let mut entry_data = Vec::new();
80
+ let mut source_timestamp_usec = None;
81
+
82
+ if let Value::Object(obj) = json_value {
83
+ // Extract timestamps for source realtime timestamp
84
+ // Per OTLP spec: Use time_unix_nano if present (non-zero), otherwise use observed_time_unix_nano
85
+ let time_unix_nano = obj
86
+ .get("log.time_unix_nano")
87
+ .and_then(|v| v.as_u64())
88
+ .filter(|&t| t != 0);
89
+
90
+ let observed_time_unix_nano = obj
91
+ .get("log.observed_time_unix_nano")
92
+ .and_then(|v| v.as_u64())
93
+ .filter(|&t| t != 0);
94
+
95
+ // Convert from nanoseconds to microseconds (systemd journal uses microseconds)
96
+ source_timestamp_usec = time_unix_nano
97
+ .or(observed_time_unix_nano)
98
+ .map(|nano| nano / 1000);
99
+
100
+ // Add OTLP_JSON field containing the complete JSON representation if enabled
101
+ // This preserves the full original message for debugging and reprocessing
102
+ if self.store_otlp_json {
103
+ if let Ok(json_str) = serde_json::to_string(json_value) {
104
+ let kv_pair = format!("OTLP_JSON={}", json_str);
105
+ entry_data.push(kv_pair.into_bytes());
106
+ }
107
+ }
108
+
109
+ for (key, value) in obj {
110
+ let value_str = match value {
111
+ Value::String(s) => s.clone(),
112
+ Value::Number(n) => n.to_string(),
113
+ Value::Bool(b) => b.to_string(),
114
+ Value::Null => "null".to_string(),
115
+ _ => serde_json::to_string(value).unwrap_or_default(),
116
+ };
117
+
118
+ let kv_pair = format!("{}={}", key, value_str);
119
+ entry_data.push(kv_pair.into_bytes());
120
+ }
121
+ }
122
+
123
+ (entry_data, source_timestamp_usec)
124
+ }
125
+}
126
+
127
+#[tonic::async_trait]
128
+impl LogsService for NetdataLogsService {
129
+ #[tracing::instrument(skip_all, fields(received_logs))]
130
+ async fn export(
131
+ &self,
132
+ request: Request<ExportLogsServiceRequest>,
133
+ ) -> Result<Response<ExportLogsServiceResponse>, Status> {
134
+ let req = request.into_inner();
135
+
136
+ let json_array = json_from_export_logs_service_request(&req);
137
+
138
+ if let Value::Array(mut entries) = json_array {
139
+ tracing::Span::current().record("received_logs", entries.len());
140
+
141
+ // Sort entries by their creation timestamp before writing to journal
142
+ // This ensures journal entries are written in chronological order, which:
143
+ // - Optimizes journal file structure and indexing
144
+ // - Improves query performance
145
+ // - Enhances compression efficiency
146
+ entries.sort_by_key(Self::extract_timestamp_for_sorting);
147
+
148
+ for entry in entries {
149
+ let (entry_data, source_timestamp_usec) = self.json_to_entry_data(&entry);
150
+
151
+ if entry_data.is_empty() {
152
+ continue;
153
+ }
154
+
155
+ let entry_refs: Vec<&[u8]> = entry_data.iter().map(|v| v.as_slice()).collect();
156
+ if let Err(e) = self.log.lock().unwrap().write_entry(&entry_refs, source_timestamp_usec) {
157
+ eprintln!("Failed to write log entry: {}", e);
158
+ return Err(Status::internal(format!(
159
+ "Failed to write log entry: {}",
160
+ e
161
+ )));
162
+ }
163
+ }
164
+
165
+ if let Err(e) = self.log.lock().unwrap().sync() {
166
+ eprintln!("Failed to sync journal file: {}", e);
167
+ return Err(Status::internal(format!(
168
+ "Failed to sync journal file: {}",
169
+ e
170
+ )));
171
+ }
172
+ }
173
+
174
+ let reply = ExportLogsServiceResponse {
175
+ partial_success: None,
176
+ };
177
+
178
+ Ok(Response::new(reply))
179
+ }
180
+}
src/crates/netdata-otel/otel-plugin/src/main.rs
new
+7
@@ -0,0 +1,7 @@
1
+//! otel-plugin standalone binary
2
+
3
+fn main() {
4
+ let args: Vec<String> = std::env::args().collect();
5
+ let exit_code = otel_plugin::run(args);
6
+ std::process::exit(exit_code);
7
+}
src/crates/netdata-otel/otel-plugin/src/metrics_service.rs
renamed
+5
-2
@@ -15,7 +15,6 @@ use crate::netdata_chart::NetdataChart;
15
use crate::plugin_config::PluginConfig;
16
use crate::regex_cache::RegexCache;
17
18
-#[derive(Default)]
18
pub struct NetdataMetricsService {
19
regex_cache: RegexCache,
20
charts: Arc<RwLock<HashMap<String, NetdataChart>>>,
@@ -115,10 +114,14 @@ impl MetricsService for NetdataMetricsService {
114
// process
115
{
116
let mut guard = self.charts.write().await;
117
+ let mut output_buffer = String::new();
118
119
for netdata_chart in guard.values_mut() {
120
- netdata_chart.process();
120
+ netdata_chart.process(&mut output_buffer);
121
}
122
+
123
+ // Write chart data to stdout
124
+ print!("{}", output_buffer);
125
}
126
127
// cleanup stale charts
src/crates/netdata-otel/otel-plugin/src/netdata_chart.rs
renamed
+33
-28
@@ -3,6 +3,9 @@ use serde_json::{Map as JsonMap, Value as JsonValue};
3
use crate::flattened_point::FlattenedPoint;
4
use crate::samples_table::{CollectionInterval, SamplesTable};
5
6
+// Use a simple string buffer for chart protocol output
7
+pub type ChartOutputBuffer = String;
8
+
9
#[derive(Debug, Default, Clone)]
10
enum ChartState {
11
#[default]
@@ -73,7 +76,7 @@ impl NetdataChart {
76
}
77
}
78
76
- fn initialize(&mut self) -> bool {
79
+ fn initialize(&mut self, buffer: &mut ChartOutputBuffer) -> bool {
80
// Clean up stale samples if we have a previous interval
81
if let Some(ci) = &self.last_samples_table_interval {
82
self.samples_table.drop_stale_samples(ci);
@@ -107,30 +110,30 @@ impl NetdataChart {
110
if let Some(old_lci) = old_lci {
111
if old_lci.update_every != new_lci.update_every {
112
// Update every changed, emit the chart definition again
110
- self.emit_chart_definition();
113
+ self.emit_chart_definition(buffer);
114
}
115
} else {
116
// No previous collection interval, we need to emit the
117
// chart definition first
115
- self.emit_chart_definition();
118
+ self.emit_chart_definition(buffer);
119
}
120
}
121
122
true
123
}
124
122
- pub fn process(&mut self) {
125
+ pub fn process(&mut self, buffer: &mut ChartOutputBuffer) {
126
loop {
127
match &self.chart_state {
128
ChartState::Uninitialized | ChartState::InGap => {
126
- if !self.initialize() {
129
+ if !self.initialize(buffer) {
130
return;
131
}
132
133
self.chart_state = ChartState::Initialized;
134
}
135
ChartState::Initialized => {
133
- self.chart_state = self.process_next_interval();
136
+ self.chart_state = self.process_next_interval(buffer);
137
}
138
ChartState::Empty => {
139
self.chart_state = ChartState::Initialized;
@@ -140,7 +143,7 @@ impl NetdataChart {
143
}
144
}
145
143
- fn emit_chart_definition(&self) {
146
+ fn emit_chart_definition(&self, buffer: &mut ChartOutputBuffer) {
147
let ci = self.last_collection_interval.unwrap();
148
let ue = ci.update_every;
149
@@ -158,10 +161,12 @@ impl NetdataChart {
161
let priority = 1;
162
let update_every = std::time::Duration::from_nanos(ue.get()).as_secs();
163
161
- println!(
162
- "CHART {type_id} '{name}' '{title}' '{units}' '{family}' '{context}' {chart_type} {priority} {update_every}"
163
- );
164
+ // CHART command
165
+ buffer.push_str(&format!(
166
+ "CHART {type_id} '{name}' '{title}' '{units}' '{family}' '{context}' {chart_type} {priority} {update_every}\n"
167
+ ));
168
169
+ // CLABEL commands
170
for (key, value) in self.attributes.iter() {
171
let value_str = match value {
172
JsonValue::String(s) => s.clone(),
@@ -170,9 +175,9 @@ impl NetdataChart {
175
_ => continue,
176
};
177
173
- println!("CLABEL '{key}' '{value_str}' 1");
178
+ buffer.push_str(&format!("CLABEL '{key}' '{value_str}' 1\n"));
179
}
175
- println!("CLABEL_COMMIT");
180
+ buffer.push_str("CLABEL_COMMIT\n");
181
182
// Emit dimensions
183
if self.is_histogram() {
@@ -197,10 +202,10 @@ impl NetdataChart {
202
Some(true) => "incremental",
203
_ => "absolute",
204
};
200
- println!(
201
- "DIMENSION {} {} {} 1 {}",
205
+ buffer.push_str(&format!(
206
+ "DIMENSION {} {} {} 1 {}\n",
207
dimension_name, dimension_name, algorithm, self.divisor
203
- );
208
+ ));
209
}
210
} else {
211
for dimension_name in self.samples_table.iter_dimensions() {
@@ -208,15 +213,15 @@ impl NetdataChart {
213
Some(true) => "incremental",
214
_ => "absolute",
215
};
211
- println!(
212
- "DIMENSION {} {} {} 1 {}",
216
+ buffer.push_str(&format!(
217
+ "DIMENSION {} {} {} 1 {}\n",
218
dimension_name, dimension_name, algorithm, self.divisor
214
- );
219
+ ));
220
}
221
}
222
}
223
219
- fn process_next_interval(&mut self) -> ChartState {
224
+ fn process_next_interval(&mut self, buffer: &mut ChartOutputBuffer) -> ChartState {
225
let lsti = match &self.last_samples_table_interval {
226
Some(interval) => interval,
227
None => return ChartState::Empty,
@@ -257,11 +262,11 @@ impl NetdataChart {
262
263
// Emit data if we have samples
264
if !samples_to_emit.is_empty() {
260
- self.emit_begin(lci.update_every.get());
265
+ self.emit_begin(buffer, lci.update_every.get());
266
for (dimension_name, value) in samples_to_emit {
262
- self.emit_set(&dimension_name, value);
267
+ self.emit_set(buffer, &dimension_name, value);
268
}
264
- self.emit_end();
269
+ self.emit_end(buffer);
270
}
271
272
// Move to next interval
@@ -271,21 +276,21 @@ impl NetdataChart {
276
ChartState::Initialized
277
}
278
274
- fn emit_begin(&self, update_every: u64) {
279
+ fn emit_begin(&self, buffer: &mut ChartOutputBuffer, update_every: u64) {
280
let ue = std::time::Duration::from_nanos(update_every).as_micros() as u64;
276
- println!("BEGIN {} {}", self.chart_id, ue);
281
+ buffer.push_str(&format!("BEGIN {} {}\n", self.chart_id, ue));
282
}
283
279
- fn emit_set(&self, dimension_name: &str, value: f64) {
280
- println!("SET {} {}", dimension_name, value * self.divisor as f64);
284
+ fn emit_set(&self, buffer: &mut ChartOutputBuffer, dimension_name: &str, value: f64) {
285
+ buffer.push_str(&format!("SET {} {}\n", dimension_name, value * self.divisor as f64));
286
}
287
283
- fn emit_end(&self) {
288
+ fn emit_end(&self, buffer: &mut ChartOutputBuffer) {
289
let collection_time = std::time::Duration::from_nanos(
290
self.last_collection_interval.unwrap().collection_time(),
291
)
292
.as_secs();
288
- println!("END {collection_time}");
293
+ buffer.push_str(&format!("END {collection_time}\n"));
294
}
295
296
pub fn last_collection_time(&self) -> Option<std::time::SystemTime> {
src/crates/netdata-otel/otel-plugin/src/plugin_config.rs
renamed
+25
-4
@@ -1,8 +1,7 @@
1
-use crate::netdata_env::NetdataEnv;
2
-
1
use anyhow::{Context, Result};
2
use bytesize::ByteSize;
3
use clap::Parser;
4
+use rt::NetdataEnv;
5
use serde::{Deserialize, Serialize};
6
use std::fs;
7
use std::path::Path;
@@ -90,6 +89,11 @@ fn parse_bytesize(s: &str) -> Result<ByteSize, String> {
89
})
90
}
91
92
+/// Default value for entries_of_journal_file
93
+fn default_entries_of_journal_file() -> usize {
94
+ 50000
95
+}
96
+
97
#[derive(Parser, Debug, Clone, Serialize, Deserialize)]
98
#[serde(deny_unknown_fields)]
99
pub struct LogsConfig {
@@ -106,6 +110,14 @@ pub struct LogsConfig {
110
#[serde(with = "bytesize_serde")]
111
pub size_of_journal_file: ByteSize,
112
113
+ /// Maximum number of entries in journal files
114
+ #[arg(
115
+ long = "otel-logs-rotation-entries-of-journal-file",
116
+ default_value = "50000"
117
+ )]
118
+ #[serde(default = "default_entries_of_journal_file")]
119
+ pub entries_of_journal_file: usize,
120
+
121
/// Maximum number of journal files to keep
122
#[arg(
123
long = "otel-logs-retention-number-of-journal-files",
@@ -139,6 +151,13 @@ pub struct LogsConfig {
151
)]
152
#[serde(with = "humantime_serde")]
153
pub duration_of_journal_file: Duration,
154
+
155
+ /// Store the complete OTLP JSON representation in the OTLP_JSON field
156
+ /// This preserves the full original message for debugging and reprocessing,
157
+ /// but increases storage usage and write overhead
158
+ #[arg(long = "otel-logs-store-otlp-json", default_value = "false")]
159
+ #[serde(default)]
160
+ pub store_otlp_json: bool,
161
}
162
163
impl Default for LogsConfig {
@@ -146,10 +165,12 @@ impl Default for LogsConfig {
165
Self {
166
journal_dir: String::from("/tmp/netdata-journals"),
167
size_of_journal_file: ByteSize::mb(100),
168
+ entries_of_journal_file: 50000,
169
number_of_journal_files: 10,
170
size_of_journal_files: ByteSize::gb(1),
171
duration_of_journal_files: Duration::from_secs(7 * 24 * 60 * 60), // 7 days
172
duration_of_journal_file: Duration::from_secs(2 * 60 * 60), // 2 hours
173
+ store_otlp_json: false,
174
}
175
}
176
}
@@ -195,7 +216,7 @@ impl PluginConfig {
216
let user_config = netdata_env
217
.user_config_dir
218
.as_ref()
198
- .map(|path| path.join("otel.yml"))
219
+ .map(|path| path.join("otel.yaml"))
220
.and_then(|path| {
221
Self::from_yaml_file(&path)
222
.with_context(|| format!("Loading user config from {}", path.display()))
@@ -207,7 +228,7 @@ impl PluginConfig {
228
} else if let Some(stock_path) = netdata_env
229
.stock_config_dir
230
.as_ref()
210
- .map(|p| p.join("otel.yml"))
231
+ .map(|p| p.join("otel.yaml"))
232
{
233
Self::from_yaml_file(&stock_path).with_context(|| {
234
format!("Loading stock config from {}", stock_path.display())
src/crates/netdata-otel/otel-plugin/src/regex_cache.rs
renamed
src/crates/netdata-otel/otel-plugin/src/samples_table.rs
renamed
+4
-9
@@ -137,7 +137,8 @@ impl SamplesTable {
137
pub fn insert(&mut self, dimension: &str, unix_time: u64, value: f64) -> bool {
138
let sp = SamplePoint { unix_time, value };
139
140
- let is_new_dimension = if let Some(sb) = self.dimensions.get_mut(dimension) {
140
+ // returns true if this we added a new dimension
141
+ if let Some(sb) = self.dimensions.get_mut(dimension) {
142
sb.push(sp);
143
false
144
} else {
@@ -145,9 +146,7 @@ impl SamplesTable {
146
sb.push(sp);
147
self.dimensions.insert(dimension.to_string(), sb);
148
true
148
- };
149
-
150
- is_new_dimension
149
+ }
150
}
151
152
pub fn is_empty(&self) -> bool {
@@ -212,10 +211,6 @@ impl SamplesTable {
211
}
212
213
// Return 1/1000 scaling if all values are in range and at least one is non-zero
215
- if has_nonzero {
216
- (1, 1000)
217
- } else {
218
- (1, 1)
219
- }
214
+ if has_nonzero { (1, 1000) } else { (1, 1) }
215
}
216
}
src/crates/netdata-plugin/bridge/Cargo.toml
new
+30
@@ -0,0 +1,30 @@
1
+[package]
2
+name = "bridge"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dependencies]
11
+# Local crates
12
+netdata-plugin-error = { path = "../error" }
13
+netdata-plugin-protocol = { path = "../protocol" }
14
+netdata-plugin-schema = { path = "../schema" }
15
+
16
+# Async runtime and tokio utilities
17
+tokio = { version = "1.0", features = ["full"] }
18
+tokio-util = { version = "0.7", features = ["codec"] }
19
+
20
+async-trait = { workspace = true }
21
+futures = { workspace = true }
22
+tracing = { workspace = true }
23
+schemars = { workspace = true }
24
+serde_json = { workspace = true }
25
+serde = { workspace = true }
26
+tracing-subscriber = { version = "0.3", features = ["env-filter"] }
27
+
28
+[dev-dependencies]
29
+console-subscriber = "0.4"
30
+
src/crates/netdata-plugin/bridge/examples/add_example.rs
new
+9
@@ -0,0 +1,9 @@
1
+use bridge::add;
2
+
3
+fn main() {
4
+ let result = add(5, 7);
5
+ println!("5 + 7 = {}", result);
6
+
7
+ let result = add(100, 250);
8
+ println!("100 + 250 = {}", result);
9
+}
src/crates/netdata-plugin/bridge/src/lib.rs
new
+14
@@ -0,0 +1,14 @@
1
+pub fn add(left: u64, right: u64) -> u64 {
2
+ left + right
3
+}
4
+
5
+#[cfg(test)]
6
+mod tests {
7
+ use super::*;
8
+
9
+ #[test]
10
+ fn it_works() {
11
+ let result = add(2, 2);
12
+ assert_eq!(result, 4);
13
+ }
14
+}
src/crates/netdata-plugin/charts-derive/Cargo.toml
new
+18
@@ -0,0 +1,18 @@
1
+[package]
2
+name = "netdata-plugin-charts-derive"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+description = "Derive macros for netdata-plugin-charts"
7
+license = "MIT OR Apache-2.0"
8
+
9
+[lib]
10
+proc-macro = true
11
+
12
+[lints]
13
+workspace = true
14
+
15
+[dependencies]
16
+syn = { version = "2.0", features = ["full", "extra-traits"] }
17
+quote = "1.0"
18
+proc-macro2 = "1.0"
src/crates/netdata-plugin/charts-derive/src/lib.rs
new
+97
@@ -0,0 +1,97 @@
1
+//! Derive macro for NetdataChart trait
2
+//!
3
+//! This macro generates efficient code for writing chart dimensions directly
4
+//! to the ChartWriter without JSON serialization overhead.
5
+
6
+use proc_macro::TokenStream;
7
+use quote::quote;
8
+use syn::{parse_macro_input, Data, DeriveInput, Fields};
9
+
10
+/// Derive macro for NetdataChart trait
11
+///
12
+/// Generates a `write_dimensions` method that directly writes dimension values
13
+/// to the ChartWriter without JSON serialization.
14
+///
15
+/// # Example
16
+///
17
+/// ```ignore
18
+/// #[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq, Serialize)]
19
+/// #[schemars(
20
+/// extend("x-chart-id" = "cpu.usage"),
21
+/// extend("x-chart-title" = "CPU Usage"),
22
+/// )]
23
+/// struct CpuMetrics {
24
+/// user: u64,
25
+/// system: u64,
26
+/// idle: u64,
27
+/// }
28
+/// ```
29
+///
30
+/// The macro skips fields marked with `x-chart-instance`:
31
+///
32
+/// ```ignore
33
+/// struct CpuCoreMetrics {
34
+/// #[schemars(extend("x-chart-instance" = true))]
35
+/// core_id: String, // Skipped in dimension output
36
+/// user: u64,
37
+/// system: u64,
38
+/// }
39
+/// ```
40
+#[proc_macro_derive(NetdataChart)]
41
+pub fn derive_netdata_chart(input: TokenStream) -> TokenStream {
42
+ let input = parse_macro_input!(input as DeriveInput);
43
+ let name = &input.ident;
44
+
45
+ // Extract fields from the struct
46
+ let fields = match &input.data {
47
+ Data::Struct(data) => match &data.fields {
48
+ Fields::Named(fields) => &fields.named,
49
+ _ => {
50
+ return syn::Error::new_spanned(
51
+ &input,
52
+ "NetdataChart can only be derived for structs with named fields",
53
+ )
54
+ .to_compile_error()
55
+ .into();
56
+ }
57
+ },
58
+ _ => {
59
+ return syn::Error::new_spanned(&input, "NetdataChart can only be derived for structs")
60
+ .to_compile_error()
61
+ .into();
62
+ }
63
+ };
64
+
65
+ // Generate write_dimension calls for each field
66
+ let dimension_writes = fields.iter().filter_map(|field| {
67
+ let field_name = field.ident.as_ref()?;
68
+ let field_name_str = field_name.to_string();
69
+
70
+ // Check if this field is marked as the instance field by looking for x-chart-instance in any attribute
71
+ let is_instance_field = field.attrs.iter().any(|attr| {
72
+ // Convert attribute to string and check if it contains x-chart-instance
73
+ let attr_str = quote!(#attr).to_string();
74
+ attr_str.contains("x-chart-instance")
75
+ });
76
+
77
+ // Skip instance fields
78
+ if is_instance_field {
79
+ return None;
80
+ }
81
+
82
+ // Generate the write call
83
+ Some(quote! {
84
+ __writer.write_dimension(#field_name_str, self.#field_name as i64);
85
+ })
86
+ });
87
+
88
+ let expanded = quote! {
89
+ impl rt::charts::ChartDimensions for #name {
90
+ fn write_dimensions(&self, __writer: &mut rt::charts::ChartWriter) {
91
+ #(#dimension_writes)*
92
+ }
93
+ }
94
+ };
95
+
96
+ TokenStream::from(expanded)
97
+}
src/crates/netdata-plugin/docs/DYNCFG.md
new
+468
@@ -0,0 +1,468 @@
1
+# Dynamic Configuration for External Plugins
2
+
3
+External plugins in Netdata can expose dynamic configuration capabilities through the DynCfg system. This document explains how to implement DynCfg in external plugins using the plugins.d protocol.
4
+
5
+## Overview
6
+
7
+The DynCfg system allows external plugins to:
8
+
9
+1. Register configurable entities (both single configurations and templates for creating jobs)
10
+2. Receive configuration commands from users
11
+3. Validate and apply configurations
12
+4. Persist configurations between Netdata agent restarts
13
+
14
+## Protocol Commands
15
+
16
+DynCfg for external plugins uses the following plugins.d protocol commands:
17
+
18
+1. `CONFIG`: Sent from the plugin to Netdata to register, update status, or delete configurations
19
+2. `FUNCTION`/`FUNCTION_PAYLOAD_BEGIN`: Received by the plugin to handle configuration commands
20
+3. `FUNCTION_RESULT_BEGIN`: Sent from the plugin to respond to commands
21
+
22
+## Implementing DynCfg in External Plugins
23
+
24
+### 1. Register a Configuration
25
+
26
+To register a configuration, the plugin sends the CONFIG command:
27
+
28
+```
29
+CONFIG <id> CREATE <status> <type> <path> <source_type> <source> <cmds> <view_access> <edit_access>
30
+```
31
+
32
+Where:
33
+
34
+- `id` is a unique identifier for the configurable entity (e.g., "go.d:nginx")
35
+- `status` can be:
36
+ - `accepted`: Configuration is accepted but not running
37
+ - `running`: Configuration is accepted and running
38
+ - `failed`: Plugin fails to run the configuration
39
+ - `incomplete`: Plugin needs additional settings
40
+ - `disabled`: Configuration is disabled by a user
41
+- `type` can be:
42
+ - `single`: A single configuration object (not addable or removable by users)
43
+ - `template`: A template for creating multiple job configurations
44
+ - `job`: A specific job configuration (derived from a template)
45
+- `path` is the UI organization path (usually "/collectors") that determines where in the configuration tree the item will appear in the UI. This is separate from the ID and controls the hierarchical navigation structure.
46
+- `source_type` can be:
47
+ - `internal`: Based on internal code settings
48
+ - `stock`: Default configurations
49
+ - `user`: User configurations via a file
50
+ - `dyncfg`: Configuration received via this mechanism
51
+ - `discovered`: Dynamically discovered by the plugin
52
+- `source` provides more details about the exact source
53
+- `cmds` is a space or pipe (|) separated list of supported commands:
54
+ - `schema`: Get JSON schema for the configuration
55
+ - `get`: Get current configuration values
56
+ - `update`: Receive configuration updates
57
+ - `add`: Receive job creation commands (templates only)
58
+ - `remove`: Remove a configuration (jobs only)
59
+ - `enable`/`disable`: Enable or disable the configuration
60
+ - `test`: Test a configuration without applying it
61
+ - `restart`: Restart the configuration
62
+ - `userconfig`: Get user-friendly configuration format
63
+- `view_access` and `edit_access` are permission bitmaps (use 0 for default permissions)
64
+
65
+Example:
66
+
67
+```
68
+CONFIG go.d:nginx CREATE accepted template /collectors internal internal schema|add|enable|disable 0 0
69
+CONFIG go.d:nginx:local_server CREATE running job /collectors dyncfg user schema|get|update|remove|enable|disable|restart 0 0
70
+```
71
+
72
+### 2. Respond to Configuration Commands
73
+
74
+The plugin receives configuration commands from Netdata as plugin functions. These come in two forms:
75
+
76
+#### Without Payload:
77
+
78
+```
79
+FUNCTION <transaction_id> <timeout_ms> "config <id> <command>" "<http_access>" "<source>"
80
+```
81
+
82
+Used for commands like: `schema`, `get`, `remove`, `enable`, `disable`, `restart`
83
+
84
+Example:
85
+
86
+```
87
+FUNCTION abcd1234 60 "config go.d:nginx:local_server get" "member" "netdata-cli"
88
+```
89
+
90
+#### With Payload:
91
+
92
+```
93
+FUNCTION_PAYLOAD_BEGIN <transaction_id> <timeout_ms> "config <id> <command>" "<http_access>" "<source>" "<content_type>"
94
+<payload_data>
95
+FUNCTION_PAYLOAD_END
96
+```
97
+
98
+Used for commands like: `update`, `add`, `test` that require additional data.
99
+
100
+Example:
101
+
102
+```
103
+FUNCTION_PAYLOAD_BEGIN abcd1234 60 "config go.d:nginx:local_server update" "member" "netdata-cli" "application/json"
104
+{
105
+ "url": "http://localhost:80/stub_status",
106
+ "timeout": 5,
107
+ "update_every": 10
108
+}
109
+FUNCTION_PAYLOAD_END
110
+```
111
+
112
+### 3. Process Commands and Respond
113
+
114
+After receiving a command, the plugin should process it and respond with a function result:
115
+
116
+```
117
+FUNCTION_RESULT_BEGIN <transaction_id> <http_status_code> <content_type> <expiration>
118
+<result_data>
119
+FUNCTION_RESULT_END
120
+```
121
+
122
+Where:
123
+
124
+- `transaction_id` is the same ID received in the original command
125
+- `http_status_code` is the standard HTTP response code:
126
+ - `200`: Success (DYNCFG_RESP_RUNNING) - Configuration accepted and running
127
+ - `202`: Accepted (DYNCFG_RESP_ACCEPTED) - Configuration accepted but not running yet
128
+ - `298`: Accepted but disabled (DYNCFG_RESP_ACCEPTED_DISABLED)
129
+ - `299`: Accepted but restart required (DYNCFG_RESP_ACCEPTED_RESTART_REQUIRED)
130
+ - `400`: Bad request - Invalid configuration
131
+ - `404`: Not found - Configuration not found
132
+ - `500`: Internal server error
133
+- `content_type` is typically "application/json"
134
+- `expiration` is the absolute timestamp (unix epoch) for result expiration
135
+
136
+The result data depends on the command:
137
+
138
+- `schema`: Return JSON Schema document
139
+- `get`: Return current configuration values
140
+- Other commands: Return a success or error message
141
+
142
+Success response example:
143
+
144
+```
145
+FUNCTION_RESULT_BEGIN abcd1234 200 application/json 0
146
+{
147
+ "status": 200,
148
+ "message": "Configuration updated successfully"
149
+}
150
+FUNCTION_RESULT_END
151
+```
152
+
153
+Error response example:
154
+
155
+```
156
+FUNCTION_RESULT_BEGIN abcd1234 400 application/json 0
157
+{
158
+ "status": 400,
159
+ "error_message": "Invalid URL format"
160
+}
161
+FUNCTION_RESULT_END
162
+```
163
+
164
+### 4. Update Configuration Status
165
+
166
+To update the status of a configuration after it's been created:
167
+
168
+```
169
+CONFIG <id> STATUS <new_status>
170
+```
171
+
172
+Example:
173
+
174
+```
175
+CONFIG go.d:nginx:local_server STATUS running
176
+```
177
+
178
+This is useful when a configuration transitions from "accepted" to "running" or "failed" after being tested.
179
+
180
+### 5. Delete a Configuration
181
+
182
+When a configuration is no longer available (e.g., the monitored service is removed):
183
+
184
+```
185
+CONFIG <id> DELETE
186
+```
187
+
188
+Example:
189
+
190
+```
191
+CONFIG go.d:nginx:local_server DELETE
192
+```
193
+
194
+## JSON Schema for Configuration UI
195
+
196
+DynCfg uses JSON Schema to define the structure of configuration objects, which is used to generate the UI.
197
+
198
+### Static Schema Files (Optional)
199
+
200
+Before calling the plugin, Netdata will first attempt to find a static schema file. You can provide static schema files in:
201
+
202
+- `CONFIG_DIR/schema.d/` (user-provided schemas, typically `/etc/netdata/schema.d/`)
203
+- `LIBCONFIG_DIR/schema.d/` (stock schemas, typically `/usr/lib/netdata/conf.d/schema.d/`)
204
+
205
+Schema files should be named after the configuration ID with `.json` extension:
206
+
207
+```
208
+/etc/netdata/schema.d/go.d:nginx.json
209
+```
210
+
211
+This approach is useful for stable schemas that don't change frequently.
212
+
213
+### Dynamic Schema Generation
214
+
215
+If no static schema file is found, Netdata will send a `schema` command to the plugin. When handling a `schema` request, the plugin should return a JSON Schema document:
216
+
217
+```json
218
+{
219
+ "type": "object",
220
+ "properties": {
221
+ "url": {
222
+ "type": "string",
223
+ "format": "uri",
224
+ "title": "Server URL",
225
+ "description": "The URL of the Nginx stub_status endpoint"
226
+ },
227
+ "timeout": {
228
+ "type": "integer",
229
+ "minimum": 1,
230
+ "maximum": 60,
231
+ "title": "Timeout",
232
+ "description": "Connection timeout in seconds"
233
+ },
234
+ "update_every": {
235
+ "type": "integer",
236
+ "minimum": 1,
237
+ "title": "Update Every",
238
+ "description": "Data collection frequency in seconds"
239
+ }
240
+ },
241
+ "required": [
242
+ "url"
243
+ ]
244
+}
245
+```
246
+
247
+For templates, the schema will be used when users add new jobs based on the template.
248
+
249
+## Action Behavior Reference
250
+
251
+When implementing DynCfg in your external plugin, be aware of how actions should behave based on the configuration type:
252
+
253
+| Action | TEMPLATE | JOB |
254
+|----------------|-----------------------------------------|-----------------------------------------|
255
+| **SCHEMA** | Return schema for creating new jobs | Use template's schema |
256
+| **GET** | Not applicable | Return current configuration |
257
+| **UPDATE** | Not applicable | Update configuration and apply if valid |
258
+| **ADD** | Create new job from template | Not applicable |
259
+| **REMOVE** | Not supported | Remove job (only for user-created jobs) |
260
+| **ENABLE** | Enable template and all its jobs | Enable specific job |
261
+| **DISABLE** | Disable template and all its jobs | Disable specific job |
262
+| **RESTART** | Restart all jobs based on template | Restart specific job |
263
+| **TEST** | Test a potential job configuration | Test configuration changes |
264
+| **USERCONFIG** | Return template in user-friendly format | Return job in user-friendly format |
265
+
266
+**Important Implementation Notes:**
267
+
268
+- When a template is disabled, send DISABLE commands to all jobs of that template
269
+- Reject ENABLE commands for jobs if their template is disabled
270
+- For job SCHEMA requests, return the same schema as the template
271
+- REMOVE should only work on dynamically added jobs, not ones from static configurations
272
+- Return appropriate response codes to indicate the status (running, accepted, disabled)
273
+
274
+## External Plugin Examples
275
+
276
+### C-based External Plugin (systemd-journal.plugin)
277
+
278
+The systemd-journal.plugin is a C-based external plugin that uses DynCfg to manage journal directory configurations. It implements a SINGLE configuration type to manage the list of journald directories to monitor:
279
+
280
+```c
281
+// Register the configuration
282
+functions_evloop_dyncfg_add(
283
+ wg,
284
+ "systemd-journal:monitored-directories", // ID
285
+ "/logs/systemd-journal", // UI Path
286
+ DYNCFG_STATUS_RUNNING, // Status
287
+ DYNCFG_TYPE_SINGLE, // Type - single configuration
288
+ DYNCFG_SOURCE_TYPE_INTERNAL, // Source type
289
+ "internal", // Source
290
+ DYNCFG_CMD_SCHEMA | DYNCFG_CMD_GET | DYNCFG_CMD_UPDATE, // Supported commands
291
+ HTTP_ACCESS_NONE, // View permissions
292
+ HTTP_ACCESS_NONE, // Edit permissions
293
+ systemd_journal_directories_dyncfg_cb, // Callback function
294
+ NULL // User data
295
+);
296
+```
297
+
298
+Key points about its implementation:
299
+
300
+- Uses a single, non-removable configuration object
301
+- Supports schema, get, and update commands
302
+- Validates directory paths for security
303
+- Updates the systemd-journal watcher when configuration changes
304
+
305
+### Go-based External Plugin (go.d.plugin)
306
+
307
+Here's a complete example showing how a Go-based external plugin might implement DynCfg for an Nginx module:
308
+
309
+### 1. Register the Template and Jobs on Startup
310
+
311
+```
312
+# Register the template for Nginx configurations
313
+CONFIG go.d:nginx CREATE accepted template /collectors internal internal schema|add|enable|disable 0 0
314
+
315
+# Register existing jobs
316
+CONFIG go.d:nginx:local_server CREATE running job /collectors user /etc/netdata/go.d/nginx.conf schema|get|update|remove|enable|disable|restart 0 0
317
+CONFIG go.d:nginx:production CREATE running job /collectors user /etc/netdata/go.d/nginx.conf schema|get|update|remove|enable|disable|restart 0 0
318
+```
319
+
320
+### 2. Handle Schema Command
321
+
322
+When receiving:
323
+
324
+```
325
+FUNCTION abcd1234 60 "config go.d:nginx schema" "member" "netdata-cli"
326
+```
327
+
328
+Respond with:
329
+
330
+```
331
+FUNCTION_RESULT_BEGIN abcd1234 200 application/json 0
332
+{
333
+ "type": "object",
334
+ "properties": {
335
+ "url": {
336
+ "type": "string",
337
+ "format": "uri",
338
+ "title": "Server URL",
339
+ "description": "The URL of the Nginx stub_status endpoint"
340
+ },
341
+ "timeout": {
342
+ "type": "integer",
343
+ "minimum": 1,
344
+ "maximum": 60,
345
+ "title": "Timeout",
346
+ "description": "Connection timeout in seconds"
347
+ },
348
+ "update_every": {
349
+ "type": "integer",
350
+ "minimum": 1,
351
+ "title": "Update Every",
352
+ "description": "Data collection frequency in seconds"
353
+ }
354
+ },
355
+ "required": ["url"]
356
+}
357
+FUNCTION_RESULT_END
358
+```
359
+
360
+### 3. Handle Get Command
361
+
362
+When receiving:
363
+
364
+```
365
+FUNCTION abcd1234 60 "config go.d:nginx:local_server get" "member" "netdata-cli"
366
+```
367
+
368
+Respond with:
369
+
370
+```
371
+FUNCTION_RESULT_BEGIN abcd1234 200 application/json 0
372
+{
373
+ "url": "http://localhost:80/stub_status",
374
+ "timeout": 5,
375
+ "update_every": 10
376
+}
377
+FUNCTION_RESULT_END
378
+```
379
+
380
+### 4. Handle Update Command
381
+
382
+When receiving:
383
+
384
+```
385
+FUNCTION_PAYLOAD_BEGIN abcd1234 60 "config go.d:nginx:local_server update" "member" "netdata-cli" "application/json"
386
+{
387
+ "url": "http://localhost:8080/stub_status",
388
+ "timeout": 3,
389
+ "update_every": 5
390
+}
391
+FUNCTION_PAYLOAD_END
392
+```
393
+
394
+Process the update and respond:
395
+
396
+```
397
+FUNCTION_RESULT_BEGIN abcd1234 200 application/json 0
398
+{
399
+ "status": 200,
400
+ "message": "Configuration updated successfully"
401
+}
402
+FUNCTION_RESULT_END
403
+```
404
+
405
+If a restart is required:
406
+
407
+```
408
+FUNCTION_RESULT_BEGIN abcd1234 299 application/json 0
409
+{
410
+ "status": 299,
411
+ "message": "Configuration updated, restart required to apply changes"
412
+}
413
+FUNCTION_RESULT_END
414
+```
415
+
416
+### 5. Handle Add Command (for templates)
417
+
418
+When receiving:
419
+
420
+```
421
+FUNCTION_PAYLOAD_BEGIN abcd1234 60 "config go.d:nginx add" "member" "netdata-cli" "application/json"
422
+{
423
+ "name": "staging",
424
+ "url": "http://staging:80/stub_status",
425
+ "timeout": 5,
426
+ "update_every": 10
427
+}
428
+FUNCTION_PAYLOAD_END
429
+```
430
+
431
+Process the new job and respond:
432
+
433
+```
434
+FUNCTION_RESULT_BEGIN abcd1234 200 application/json 0
435
+{
436
+ "status": 200,
437
+ "message": "Job 'staging' created successfully"
438
+}
439
+FUNCTION_RESULT_END
440
+```
441
+
442
+Then register the new job:
443
+
444
+```
445
+CONFIG go.d:nginx:staging CREATE running job /collectors dyncfg netdata-cli schema|get|update|remove|enable|disable|restart 0 0
446
+```
447
+
448
+## Best Practices
449
+
450
+1. **Use Consistent IDs**: Follow the pattern `component:template_name` for templates and `component:template_name:job_name` for jobs
451
+2. **Validate Thoroughly**: Always validate configuration changes before accepting them
452
+3. **Include Descriptive Messages**: Provide helpful error messages when rejections occur
453
+4. **Document Your Schema**: Include clear titles and descriptions for all properties in your JSON Schema
454
+5. **Handle Errors Gracefully**: Return appropriate HTTP status codes and error messages
455
+6. **Update Status Promptly**: When a configuration changes state (e.g., from "accepted" to "running"), update its status
456
+7. **Clean Up Configurations**: When a monitored resource is gone, delete its configuration with `CONFIG id DELETE`
457
+
458
+## Debugging Tips
459
+
460
+1. Set `NETDATA_DEBUG_DYNCFG=1` environment variable when running Netdata to see detailed logs
461
+2. If configurations aren't being registered, check for errors in the plugin output
462
+3. Verify configuration files are saved in `/var/lib/netdata/config/`
463
+4. Test configurations via the API: `/api/v3/config?id=<your-config-id>`
464
+
465
+## Related Documentation
466
+
467
+- [Main DynCfg Documentation](/src/daemon/dyncfg/README.md) - Core DynCfg system concepts and APIs
468
+- [Plugins.d Protocol](/src/plugins.d/README.md) - Complete documentation of the plugins.d protocol
src/crates/netdata-plugin/docs/FUNCTION_UI_DEVELOPER_GUIDE.md
new
+953
@@ -0,0 +1,953 @@
1
+# Netdata Functions: Developer Guide
2
+
3
+> **Note**: This is the practical developer guide. For the complete technical specification, see [FUNCTIONS_REFERENCE.md](FUNCTIONS_REFERENCE.md).
4
+
5
+## Overview
6
+
7
+This guide teaches you how to create Netdata functions that provide interactive data through the web UI. You'll learn to build both simple tables and advanced log explorers.
8
+
9
+**What You'll Learn:**
10
+- How to create simple table functions for system monitoring
11
+- How to build log explorer functions with faceted search
12
+- All column types, options, and their UI effects
13
+- Query patterns, filtering, and aggregation
14
+- Real-world examples and best practices
15
+
16
+**Quick Navigation:**
17
+- [Part 1: Simple Table Functions](#part-1-simple-table-functions) - Basic monitoring data
18
+- [Part 2: Log Explorer Functions](#part-2-log-explorer-functions) - Historical data with search
19
+- [Part 3: Complete Options Reference](#part-3-complete-options-reference) - Every option explained
20
+
21
+---
22
+
23
+# Part 1: Simple Table Functions
24
+
25
+Simple table functions display current system state - processes, connections, services, etc. They're perfect for "top-like" views and system monitoring.
26
+
27
+**Implementation Architecture:**
28
+- **Frontend handles**: Filtering, search, facet counting, sorting
29
+- **Backend provides**: Raw data, column definitions, optional charts
30
+- **Performance**: Limited by browser processing (good for \<10k rows)
31
+- **Query parameter**: Processed in frontend only (substring search)
32
+- **Histograms**: Not supported
33
+
34
+## Your First Simple Table Function
35
+
36
+Start with this minimal working example:
37
+
38
+```json
39
+{
40
+ "status": 200,
41
+ "type": "table",
42
+ "has_history": false,
43
+ "help": "Shows system services",
44
+ "data": [
45
+ ["nginx", 25, "Running"],
46
+ ["mysql", 67, "Stopped"],
47
+ ["redis", 91, "Running"]
48
+ ],
49
+ "columns": {
50
+ "name": {
51
+ "index": 0,
52
+ "name": "Service",
53
+ "type": "string",
54
+ "unique_key": true
55
+ },
56
+ "cpu": {
57
+ "index": 1,
58
+ "name": "CPU %",
59
+ "type": "bar-with-integer",
60
+ "max": 100
61
+ },
62
+ "status": {
63
+ "index": 2,
64
+ "name": "Status",
65
+ "type": "string"
66
+ }
67
+ }
68
+}
69
+```
70
+
71
+This creates a basic 3-column table showing service names, CPU usage bars, and status.
72
+
73
+## Essential Features for Simple Tables
74
+
75
+### 1. Required Fields
76
+
77
+Every simple table function needs:
78
+
79
+```json
80
+{
81
+ "status": 200, // HTTP status (200 = success)
82
+ "type": "table", // Always "table"
83
+ "has_history": false, // Simple table (not log explorer)
84
+ "columns": {...}, // Column definitions
85
+ "data": [...] // Your actual data
86
+}
87
+```
88
+
89
+### 2. Column Definitions
90
+
91
+Each column must have:
92
+
93
+```json
94
+"column_id": {
95
+ "index": 0, // Position in data arrays (0-based)
96
+ "name": "Display Name", // Column header shown to users
97
+ "type": "string" // How to render the data
98
+}
99
+```
100
+
101
+### 3. The Data Array
102
+
103
+Data is an array of arrays - each inner array is one row:
104
+
105
+```json
106
+"data": [
107
+ ["nginx", 25, "Running"], // Row 1: index 0=name, 1=cpu, 2=status
108
+ ["mysql", 67, "Stopped"], // Row 2
109
+ ["redis", 91, "Running"] // Row 3
110
+]
111
+```
112
+
113
+**Important**: Values must be in the same order as column `index` fields.
114
+
115
+## Adding Visual Polish
116
+
117
+Make your table more useful with these enhancements:
118
+
119
+```json
120
+{
121
+ "status": 200,
122
+ "type": "table",
123
+ "has_history": false,
124
+ "help": "System services with CPU usage and status",
125
+ "data": [
126
+ ["nginx", 25, "Running", {"rowOptions": {"severity": "normal"}}],
127
+ ["mysql", 67, "Stopped", {"rowOptions": {"severity": "error"}}],
128
+ ["redis", 91, "Running", {"rowOptions": {"severity": "warning"}}]
129
+ ],
130
+ "columns": {
131
+ "name": {
132
+ "index": 0,
133
+ "name": "Service Name",
134
+ "type": "string",
135
+ "unique_key": true,
136
+ "sticky": true,
137
+ "filter": "multiselect"
138
+ },
139
+ "cpu": {
140
+ "index": 1,
141
+ "name": "CPU Usage",
142
+ "type": "bar-with-integer",
143
+ "units": "%",
144
+ "max": 100,
145
+ "sort": "descending",
146
+ "filter": "range"
147
+ },
148
+ "status": {
149
+ "index": 2,
150
+ "name": "Status",
151
+ "type": "string",
152
+ "visualization": "pill",
153
+ "filter": "multiselect"
154
+ }
155
+ },
156
+ "default_sort_column": "cpu"
157
+}
158
+```
159
+
160
+**New Features Added:**
161
+- **Row coloring**: `rowOptions` with severity levels
162
+- **Sticky columns**: Pin important columns when scrolling
163
+- **Filters**: Let users filter by categories or numeric ranges
164
+- **Progress bars**: Visual CPU usage display
165
+- **Pills**: Status badges with colors
166
+- **Default sorting**: Start sorted by CPU usage
167
+
168
+## Limitations of Simple Tables
169
+
170
+Simple tables have certain limitations due to their frontend-only processing architecture:
171
+
172
+- **No histograms**: Time-based visualization is not supported
173
+- **Performance limits**: Frontend processing is limited by browser memory (recommend \<10k rows)
174
+- **No backend search**: Query parameter is not sent to backend - search happens client-side only
175
+- **Static facet counts**: Counts are computed by frontend from all received data
176
+
177
+For large datasets or advanced log analysis features, consider using log explorers (`has_history: true`).
178
+
179
+## Advanced Simple Table Features
180
+
181
+### Aggregated Views
182
+
183
+Some functions can show both detailed and aggregated data. When enabled, facet pills show both counts:
184
+
185
+```json
186
+{
187
+ "aggregated_view": {
188
+ "column": "Count",
189
+ "results_label": "unique combinations",
190
+ "aggregated_label": "connections"
191
+ }
192
+}
193
+```
194
+
195
+This enables smart facet pills like `"15 ⊃ 42"` meaning "15 connections aggregated into 42 rows".
196
+
197
+### Charts Integration
198
+
199
+Simple tables support interactive charts computed from your table data. The backend defines available charts, and the frontend computes and renders them.
200
+
201
+**Chart Types Available:**
202
+- `"bar"` - Basic bar chart
203
+- `"stacked-bar"` - Multi-column stacked bars (most common)
204
+- `"doughnut"` - Pie/doughnut chart
205
+- `"value"` - Simple numeric display
206
+
207
+**Example Configuration:**
208
+```json
209
+{
210
+ "charts": {
211
+ "cpu_usage": {
212
+ "name": "CPU Usage by Service",
213
+ "type": "stacked-bar",
214
+ "columns": ["user_cpu", "system_cpu", "guest_cpu"],
215
+ "groupBy": "column",
216
+ "aggregation": "sum"
217
+ },
218
+ "memory_breakdown": {
219
+ "name": "Memory Types",
220
+ "type": "doughnut",
221
+ "columns": ["resident", "virtual", "shared"],
222
+ "groupBy": "all",
223
+ "aggregation": "sum"
224
+ }
225
+ },
226
+ "default_charts": [
227
+ ["cpu_usage", "status"],
228
+ ["memory_breakdown", "status"]
229
+ ]
230
+}
231
+```
232
+
233
+**Chart Options:**
234
+- **`groupBy`**:
235
+ - `"column"` (default): Group by selected filter column
236
+ - `"all"`: Aggregate all data together
237
+- **`aggregation`**: `sum`, `mean`, `max`, `min`, `count`
238
+
239
+**How It Works:**
240
+1. Frontend takes your table data
241
+2. Groups by selected column (e.g., "service type")
242
+3. Aggregates values using specified method (e.g., sum CPU values)
243
+4. Renders chart with one bar/slice per group
244
+
245
+### Table Row Grouping
246
+
247
+Simple tables support grouping rows with customizable aggregation. When users group by a column, rows with the same value are combined using the aggregation method you specify.
248
+
249
+**Enable Grouping Options:**
250
+```json
251
+{
252
+ "group_by": {
253
+ "aggregated": [{
254
+ "id": "by_status",
255
+ "name": "By Status",
256
+ "column": "status"
257
+ }, {
258
+ "id": "by_user",
259
+ "name": "By User",
260
+ "column": "user"
261
+ }]
262
+ }
263
+}
264
+```
265
+
266
+**Define Column Aggregation:**
267
+Each column needs a summary type to control how values are aggregated when grouping:
268
+
269
+```c
270
+// In your C backend code
271
+buffer_rrdf_table_add_field(
272
+ wb, field_id++, "cpu_percent", "CPU %",
273
+ RRDF_FIELD_TYPE_BAR_WITH_INTEGER,
274
+ RRDF_FIELD_VISUAL_BAR,
275
+ RRDF_FIELD_TRANSFORM_NUMBER,
276
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
277
+ RRDF_FIELD_SUMMARY_SUM, // Sum CPU values when grouping
278
+ RRDF_FIELD_FILTER_RANGE,
279
+ RRDF_FIELD_OPTS_VISIBLE, NULL
280
+);
281
+
282
+buffer_rrdf_table_add_field(
283
+ wb, field_id++, "process_count", "Processes",
284
+ RRDF_FIELD_TYPE_INTEGER,
285
+ RRDF_FIELD_VISUAL_VALUE,
286
+ RRDF_FIELD_TRANSFORM_NUMBER,
287
+ 0, "processes", NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
288
+ RRDF_FIELD_SUMMARY_COUNT, // Count processes when grouping
289
+ RRDF_FIELD_FILTER_RANGE,
290
+ RRDF_FIELD_OPTS_VISIBLE, NULL
291
+);
292
+```
293
+
294
+**Available Summary/Aggregation Types:**
295
+- `RRDF_FIELD_SUMMARY_COUNT` - Count rows in group
296
+- `RRDF_FIELD_SUMMARY_SUM` - Sum numeric values
297
+- `RRDF_FIELD_SUMMARY_MEAN` - Average values
298
+- `RRDF_FIELD_SUMMARY_MIN` - Minimum value
299
+- `RRDF_FIELD_SUMMARY_MAX` - Maximum value
300
+- `RRDF_FIELD_SUMMARY_UNIQUECOUNT` - Count unique values
301
+- `RRDF_FIELD_SUMMARY_MEDIAN` - Median value
302
+
303
+**Example Result:**
304
+When user groups by "service type", rows like:
305
+```
306
+nginx_worker1 25% Running
307
+nginx_worker2 30% Running
308
+mysql_main 45% Running
309
+```
310
+
311
+Become:
312
+```
313
+nginx 55% 2 processes (25% + 30% CPU, 2 processes counted)
314
+mysql 45% 1 process (45% CPU, 1 process counted)
315
+```
316
+
317
+---
318
+
319
+# Part 2: Log Explorer Functions
320
+
321
+Log explorer functions (`has_history: true`) provide advanced log analysis with full-text search, faceted filtering, time navigation, and histograms. Perfect for systemd journals, event logs, and audit trails.
322
+
323
+**Implementation Architecture:**
324
+- **Backend handles**: Query processing, pattern matching, facet counting, filtering
325
+- **Frontend provides**: UI for facets, histogram display, infinite scroll
326
+- **Performance**: Scales to millions of records with sampling and server-side filtering
327
+- **Query parameter**: Processed by backend using Netdata simple patterns
328
+- **Histograms**: Optional, generated by backend in Netdata chart format
329
+- **Uses facets library**: All processing leverages `libnetdata/facets/`
330
+
331
+### Complete Log Explorer Example
332
+
333
+```json
334
+{
335
+ "status": 200,
336
+ "type": "table",
337
+ "has_history": true,
338
+ "help": "System journal with real-time updates",
339
+ "accepted_params": [
340
+ "info", "after", "before", "direction", "last", "anchor",
341
+ "query", "facets", "histogram", "if_modified_since",
342
+ "data_only", "delta", "tail", "sampling"
343
+ ],
344
+ "table": {"id": "journal", "has_history": true},
345
+ "columns": {
346
+ "timestamp": {
347
+ "index": 0,
348
+ "name": "Time",
349
+ "type": "timestamp",
350
+ "transform": "datetime_usec",
351
+ "sort": "descending|fixed",
352
+ "sticky": true
353
+ },
354
+ "priority": {
355
+ "index": 1,
356
+ "name": "Level",
357
+ "type": "string",
358
+ "visualization": "pill",
359
+ "filter": "facet",
360
+ "options": ["facet", "visible", "sticky"]
361
+ },
362
+ "message": {
363
+ "index": 2,
364
+ "name": "Message",
365
+ "type": "string",
366
+ "full_width": true,
367
+ "options": ["full_width", "wrap", "visible", "main_text", "fts"]
368
+ }
369
+ },
370
+ "data": [
371
+ [1697644320000000, {"rowOptions": {"severity": "error"}}, "ERROR", "Service failed"],
372
+ [1697644319000000, {"rowOptions": {"severity": "normal"}}, "INFO", "Service started"]
373
+ ],
374
+ "facets": [
375
+ {
376
+ "id": "priority",
377
+ "name": "Log Level",
378
+ "order": 1,
379
+ "defaultExpanded": true,
380
+ "options": [
381
+ {"id": "ERROR", "name": "ERROR", "count": 45, "order": 1},
382
+ {"id": "INFO", "name": "INFO", "count": 234, "order": 3}
383
+ ]
384
+ }
385
+ ],
386
+ "items": {
387
+ "evaluated": 50000,
388
+ "matched": 2520,
389
+ "returned": 100,
390
+ "max_to_return": 100
391
+ },
392
+ "anchor": {
393
+ "last_modified": 1697644320000000,
394
+ "direction": "backward"
395
+ }
396
+}
397
+```
398
+
399
+**Key Features Demonstrated:**
400
+- `has_history: true` - Enables log explorer UI with infinite scroll
401
+- `accepted_params` - Full parameter support for advanced features
402
+- **Faceted search** - Real-time counts computed by backend
403
+- **Anchor pagination** - Efficient navigation through large datasets
404
+- **Microsecond timestamps** - High precision for log entries
405
+- **Full-text search** - Backend pattern matching with `"fts"` fields
406
+- **Row coloring** - Severity-based visual indicators
407
+
408
+## Essential Log Explorer Features
409
+
410
+### 1. Faceted Search Sidebar
411
+
412
+Facets provide dynamic filtering with real-time counts computed by the backend:
413
+
414
+```json
415
+{
416
+ "facets": [
417
+ {
418
+ "id": "level",
419
+ "name": "Log Level",
420
+ "order": 1,
421
+ "defaultExpanded": true,
422
+ "options": [
423
+ {"id": "ERROR", "name": "ERROR", "count": 45, "order": 1},
424
+ {"id": "WARN", "name": "WARN", "count": 123, "order": 2},
425
+ {"id": "INFO", "name": "INFO", "count": 2341, "order": 3}
426
+ ]
427
+ }
428
+ ]
429
+}
430
+```
431
+
432
+**Important**: These counts are calculated by the backend facets library during query execution, not by the frontend. The backend scans through matching records and maintains counters for each facet value.
433
+
434
+### 2. Full-Text Search
435
+
436
+Enable powerful query search across all fields (processed by backend):
437
+
438
+```json
439
+{
440
+ "columns": {
441
+ "message": {
442
+ "options": ["fts"], // Make field full-text searchable
443
+ // ... other options
444
+ }
445
+ }
446
+}
447
+```
448
+
449
+**Important**: Unlike simple tables, the query parameter is sent to the backend and processed server-side using the facets library. The backend performs pattern matching and only returns matching records.
450
+
451
+**Query Patterns** (using Netdata simple patterns):
452
+- `"error"` - Find "error" anywhere (substring match)
453
+- `"error|warning"` - Find "error" OR "warning" (pipe separator)
454
+- `"!debug"` - Exclude logs containing "debug"
455
+- `"!*debugging*|*debug*"` - Include "debug" but exclude "debugging"
456
+- `"connection failed"` - Find exact phrase (spaces are literal)
457
+
458
+**Pattern Evaluation Rules:**
459
+1. **Within field**: Left-to-right, first match wins
460
+2. **Across fields**: ALL fields evaluated, ANY negative match excludes row
461
+3. **Case-insensitive** matching throughout
462
+
463
+### 3. Time-Based Histograms
464
+
465
+Visualize log distribution over time (log explorers only):
466
+
467
+```json
468
+{
469
+ "available_histograms": [
470
+ {"id": "level", "name": "Log Level", "order": 1},
471
+ {"id": "source", "name": "Source", "order": 2}
472
+ ],
473
+ "histogram": {
474
+ "id": "level",
475
+ "name": "Log Level",
476
+ "chart": {
477
+ "result": {
478
+ "labels": ["time", "ERROR", "WARN", "INFO"],
479
+ "data": [
480
+ [1697644200, 5, 12, 234],
481
+ [1697644260, 3, 8, 198]
482
+ ]
483
+ }
484
+ }
485
+ }
486
+}
487
+```
488
+
489
+**Important**: Histograms are NOT supported for simple tables (`has_history: false`). They are optional for log explorers and use the same format as Netdata's `/api/v3/data` endpoint. The backend generates histogram data using the facets library.
490
+
491
+### 4. Anchor-Based Navigation
492
+
493
+Handle large datasets with efficient pagination:
494
+
495
+```json
496
+{
497
+ "items": {
498
+ "evaluated": 50000, // Total scanned
499
+ "matched": 2520, // Match filters
500
+ "returned": 100, // In this response
501
+ "max_to_return": 100
502
+ },
503
+ "anchor": {
504
+ "last_modified": 1697644320000000,
505
+ "direction": "backward"
506
+ }
507
+}
508
+```
509
+
510
+### Parameter Integration Patterns
511
+
512
+**Basic Monitoring Function:**
513
+```json
514
+{
515
+ "accepted_params": ["info", "after", "before"]
516
+}
517
+```
518
+
519
+**Advanced Log Function:**
520
+```json
521
+{
522
+ "accepted_params": [
523
+ "info", "after", "before", "direction", "last", "anchor",
524
+ "query", "facets", "histogram", "if_modified_since",
525
+ "data_only", "delta", "tail", "sampling", "slice"
526
+ ]
527
+}
528
+```
529
+
530
+**UI Feature Enablement:**
531
+- `"slice"` → Enables "Full data queries" toggle
532
+- `"direction"` → Enables bidirectional pagination
533
+- `"tail"` → Enables streaming mode
534
+- `"delta"` → Enables incremental updates
535
+- `"query"` → Enables full-text search
536
+
537
+## Advanced Log Explorer Features
538
+
539
+### Column Options for Logs
540
+
541
+Log explorers support special column options:
542
+
543
+| Option | Effect |
544
+|--------|--------|
545
+| `"fts"` | Full-text searchable by query |
546
+| `"facet"` | Appears in sidebar filters with counts |
547
+| `"main_text"` | Primary content (usually message) |
548
+| `"rich_text"` | May contain formatting |
549
+| `"hidden"` | Hide by default |
550
+
551
+Example:
552
+```json
553
+{
554
+ "message": {
555
+ "type": "string",
556
+ "full_width": true,
557
+ "options": ["full_width", "wrap", "visible", "main_text", "fts", "rich_text"]
558
+ }
559
+}
560
+```
561
+
562
+### Log-Specific UI Behavior
563
+
564
+When `has_history: true`:
565
+- **Sidebar**: Shows faceted filters instead of simple filters
566
+- **Search box**: Queries all `fts` fields using pattern matching
567
+- **Infinite scroll**: Loads more data as you scroll
568
+- **Time navigation**: Jump to specific time periods
569
+- **Sampling**: For very large datasets
570
+
571
+---
572
+
573
+# Part 3: Complete Options Reference
574
+
575
+This section documents every field type, option, and behavior for quick reference while developing.
576
+
577
+## Field Types and UI Rendering
578
+
579
+### Text and Categories
580
+
581
+```json
582
+{
583
+ "type": "string",
584
+ "visualization": "value" // Default: plain text, left-aligned
585
+}
586
+```
587
+
588
+```json
589
+{
590
+ "type": "string",
591
+ "visualization": "pill" // Colored badges for status/categories
592
+}
593
+```
594
+
595
+### Numbers and Metrics
596
+
597
+```json
598
+{
599
+ "type": "integer", // Right-aligned numbers
600
+ "transform": "number", // Respect decimal_points
601
+ "decimal_points": 2
602
+}
603
+```
604
+
605
+```json
606
+{
607
+ "type": "bar-with-integer", // Progress bars with values
608
+ "max": 100, // Required for bars
609
+ "units": "%"
610
+}
611
+```
612
+
613
+### Timestamp Format Requirements
614
+
615
+**Simple Tables**: Use milliseconds with `datetime` transform
616
+```json
617
+{
618
+ "type": "timestamp",
619
+ "transform": "datetime", // Expects milliseconds
620
+ "data": [1697644320000] // JavaScript Date format
621
+}
622
+```
623
+
624
+**Log Explorers**: Use microseconds with `datetime_usec` transform
625
+```json
626
+{
627
+ "type": "timestamp",
628
+ "transform": "datetime_usec", // Expects microseconds
629
+ "data": [1697644320000000] // Microsecond precision
630
+}
631
+```
632
+
633
+**Frontend Conversion:**
634
+```javascript
635
+// datetime_usec automatically converts to milliseconds
636
+if (usec) {
637
+ epoch = epoch ? Math.floor(epoch / 1000) : epoch
638
+}
639
+```
640
+
641
+**API Parameters**: `after` and `before` are automatically converted from milliseconds to seconds when sent to functions.
642
+
643
+```json
644
+{
645
+ "type": "duration",
646
+ "transform": "duration_s" // Formats seconds as "1d 2h 3m"
647
+}
648
+```
649
+
650
+### Rich Content
651
+
652
+```json
653
+{
654
+ "type": "feedTemplate", // Full-width rich content
655
+ "full_width": true // Automatically applied
656
+}
657
+```
658
+
659
+## Essential Column Options
660
+
661
+### Layout Control
662
+
663
+```json
664
+{
665
+ "unique_key": true, // Row identifier (exactly one required)
666
+ "sticky": true, // Pin when scrolling horizontally
667
+ "visible": true, // Show by default
668
+ "full_width": true, // Expand to fill available space
669
+ "wrap": true // Enable text wrapping
670
+}
671
+```
672
+
673
+### Filtering Options
674
+
675
+```json
676
+{
677
+ "filter": "multiselect" // Checkboxes (default for simple tables)
678
+}
679
+```
680
+
681
+```json
682
+{
683
+ "filter": "range" // Min/max sliders for numbers
684
+}
685
+```
686
+
687
+```json
688
+{
689
+ "filter": "facet" // Sidebar with counts (log explorers only)
690
+}
691
+```
692
+
693
+### Sorting Options
694
+
695
+```json
696
+{
697
+ "sort": "descending", // Default sort direction
698
+ "sortable": true // Allow user sorting (default)
699
+}
700
+```
701
+
702
+```json
703
+{
704
+ "sort": "descending|fixed", // Prevent user from changing sort
705
+ "sortable": false
706
+}
707
+```
708
+
709
+## Row-Level Features
710
+
711
+### Row Coloring by Severity
712
+
713
+Add row coloring by including `rowOptions` as the last element:
714
+
715
+```json
716
+{
717
+ "data": [
718
+ ["normal data", "values", {"rowOptions": {"severity": "normal"}}],
719
+ ["warning data", "values", {"rowOptions": {"severity": "warning"}}],
720
+ ["error data", "values", {"rowOptions": {"severity": "error"}}],
721
+ ["notice data", "values", {"rowOptions": {"severity": "notice"}}]
722
+ ]
723
+}
724
+```
725
+
726
+**Severity Levels:**
727
+- `"normal"` - Default appearance
728
+- `"warning"` - Yellow background
729
+- `"error"` - Red background
730
+- `"notice"` - Blue background
731
+
732
+## Chart and Grouping Configuration
733
+
734
+### Chart Definition
735
+
736
+```json
737
+{
738
+ "charts": {
739
+ "resource_usage": {
740
+ "name": "Resource Usage",
741
+ "type": "stacked-bar",
742
+ "columns": ["cpu", "memory", "disk"],
743
+ "groupBy": "column",
744
+ "aggregation": "sum"
745
+ },
746
+ "status_distribution": {
747
+ "name": "Status Distribution",
748
+ "type": "doughnut",
749
+ "columns": ["count"],
750
+ "groupBy": "all",
751
+ "aggregation": "count"
752
+ }
753
+ },
754
+ "default_charts": [
755
+ ["resource_usage", "service_type"],
756
+ ["status_distribution", "status"]
757
+ ]
758
+}
759
+```
760
+
761
+### Grouping Configuration
762
+
763
+```json
764
+{
765
+ "group_by": {
766
+ "aggregated": [
767
+ {
768
+ "id": "by_service",
769
+ "name": "By Service Type",
770
+ "column": "service_type"
771
+ },
772
+ {
773
+ "id": "by_status",
774
+ "name": "By Status",
775
+ "column": "status"
776
+ }
777
+ ]
778
+ }
779
+}
780
+```
781
+
782
+## Backend Implementation Examples
783
+
784
+### C Code for Simple Tables
785
+
786
+```c
787
+// Add a progress bar column
788
+buffer_rrdf_table_add_field(
789
+ wb, field_id++, "cpu", "CPU Usage",
790
+ RRDF_FIELD_TYPE_BAR_WITH_INTEGER,
791
+ RRDF_FIELD_VISUAL_BAR,
792
+ RRDF_FIELD_TRANSFORM_NUMBER,
793
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
794
+ RRDF_FIELD_SUMMARY_SUM,
795
+ RRDF_FIELD_FILTER_RANGE,
796
+ RRDF_FIELD_OPTS_VISIBLE, NULL
797
+);
798
+
799
+// Add row coloring
800
+buffer_json_add_array_item_object(wb);
801
+buffer_json_member_add_object(wb, "rowOptions");
802
+buffer_json_member_add_string(wb, "severity", "error");
803
+buffer_json_object_close(wb);
804
+buffer_json_object_close(wb);
805
+```
806
+
807
+### Using the Facets Library (Log Explorers)
808
+
809
+```c
810
+// Initialize facets for log exploration
811
+FACETS *facets = facets_create(...);
812
+
813
+// Set up query search
814
+facets_set_query(facets, query_string);
815
+
816
+// Add a faceted field
817
+facets_register_facet_id(facets, "level",
818
+ FACET_KEY_OPTION_FACET | FACET_KEY_OPTION_FTS | FACET_KEY_OPTION_REORDER);
819
+
820
+// Generate the response
821
+facets_table_config(facets, wb);
822
+```
823
+
824
+## Error Handling
825
+
826
+Always handle the info request first:
827
+
828
+```json
829
+{
830
+ "v": 3, // Enable POST requests
831
+ "status": 200,
832
+ "type": "table",
833
+ "has_history": false, // or true for log explorers
834
+ "help": "Function description",
835
+ "accepted_params": [...],
836
+ "required_params": [...]
837
+}
838
+```
839
+
840
+### Complete Info Response Example
841
+
842
+```json
843
+{
844
+ "v": 3,
845
+ "status": 200,
846
+ "type": "table",
847
+ "has_history": true,
848
+ "help": "System log explorer with faceted search",
849
+ "accepted_params": [
850
+ "info", "after", "before", "direction", "last", "anchor",
851
+ "query", "facets", "histogram", "if_modified_since",
852
+ "data_only", "delta", "tail", "sampling"
853
+ ],
854
+ "required_params": [
855
+ {
856
+ "id": "unit",
857
+ "name": "System Unit",
858
+ "type": "select",
859
+ "options": [
860
+ {"id": "nginx.service", "name": "Nginx"},
861
+ {"id": "mysql.service", "name": "MySQL"}
862
+ ]
863
+ }
864
+ ]
865
+}
866
+```
867
+
868
+**Critical Fields:**
869
+- `"v": 3` enables POST requests with JSON payloads
870
+- `accepted_params` determines which parameters the function accepts
871
+- `required_params` generates filter UI and validates execution
872
+
873
+For errors, return:
874
+
875
+```json
876
+{
877
+ "status": 400,
878
+ "error_message": "Descriptive error message"
879
+}
880
+```
881
+
882
+### Performance Optimization
883
+
884
+**For Large Datasets:**
885
+
886
+1. **Enable Sampling**:
887
+```json
888
+{
889
+ "accepted_params": ["sampling"],
890
+ "sampling": 10 // 1 in 10 sampling
891
+}
892
+```
893
+
894
+2. **Use Delta Updates**:
895
+```json
896
+{
897
+ "if_modified_since": 1697644320000000,
898
+ "delta": true,
899
+ "data_only": true
900
+}
901
+```
902
+
903
+3. **Implement Tail Limiting**:
904
+```c
905
+// Limit tail data to prevent memory issues
906
+if (tail_mode) {
907
+ limit_results_to(500);
908
+}
909
+```
910
+
911
+4. **Support Anchor Pagination**:
912
+```json
913
+{
914
+ "pagination": {
915
+ "enabled": true,
916
+ "column": "timestamp",
917
+ "key": "anchor",
918
+ "units": "timestamp_usec"
919
+ }
920
+}
921
+```
922
+
923
+## Best Practices Summary
924
+
925
+### For Simple Tables
926
+1. Always include one `unique_key` column
927
+2. Use `bar-with-integer` for metrics with `max` values
928
+3. Add `filter: "range"` for numbers, `"multiselect"` for categories
929
+4. Use `rowOptions` for status indication
930
+5. Set meaningful `default_sort_column`
931
+6. Define column `summary` types for grouping (SUM for metrics, COUNT for processes)
932
+7. Add charts for key metrics with appropriate `groupBy` and `aggregation`
933
+8. Include `group_by` options for common analysis patterns
934
+
935
+### For Log Explorers
936
+1. Set `has_history: true`
937
+2. Use microsecond timestamps with `datetime_usec`
938
+3. Mark important fields with `"fts"` for search
939
+4. Use `filter: "facet"` instead of `"multiselect"`
940
+5. Include proper facets with counts
941
+6. Implement anchor-based pagination
942
+
943
+### General
944
+1. Test with `?info` requests first
945
+2. Include helpful `help` text
946
+3. Handle errors gracefully
947
+4. Use appropriate `units` for clarity
948
+5. Follow existing function patterns in your codebase
949
+
950
+
951
+---
952
+
953
+This guide covers everything you need to build both simple monitoring functions and advanced log explorers. For implementation details and edge cases, see [FUNCTIONS_REFERENCE.md](FUNCTIONS_REFERENCE.md).
src/crates/netdata-plugin/docs/FUNCTION_UI_REFERENCE.md
new
+1677
@@ -0,0 +1,1677 @@
1
+# Netdata Functions v3 Protocol - Technical Reference
2
+
3
+> **Note**: This is the technical specification. For a practical guide to implementing functions, see [FUNCTIONS_DEVELOPER_GUIDE.md](FUNCTIONS_DEVELOPER_GUIDE.md).
4
+
5
+## Overview
6
+
7
+This document provides the complete technical reference for Netdata Functions protocol v3, combining all information about simple tables (has_history=false) and log explorers (has_history=true). This is the authoritative internal documentation for maintaining and extending Netdata functions.
8
+
9
+## Table of Contents
10
+
11
+1. [Protocol Overview](#protocol-overview)
12
+2. [Request Flow](#request-flow)
13
+3. [Simple Table Format](#simple-table-format)
14
+4. [Log Explorer Format](#log-explorer-format)
15
+5. [Field Types and Enumerations](#field-types-and-enumerations)
16
+6. [UI Implementation](#ui-implementation)
17
+7. [Backend Implementation](#backend-implementation)
18
+8. [Best Practices](#best-practices)
19
+9. [Known Functions](#known-functions)
20
+10. [Development Checklist](#development-checklist)
21
+
22
+## Protocol Overview
23
+
24
+Netdata Functions allow collectors/plugins to expose interactive data through a streaming protocol. The protocol has evolved from GET-based CLI parameters (legacy) to POST-based JSON payloads (modern v3).
25
+
26
+### Function Types
27
+
28
+1. **Simple Table View** (`has_history: false`)
29
+ - Basic tabular data display with frontend-side filtering and search
30
+ - Examples: `processes`, `network-connections`, `block-devices`
31
+
32
+2. **Log Explorer Format** (`has_history: true`)
33
+ - Advanced table with backend-powered faceted search, histograms, and infinite scroll
34
+ - Examples: `systemd-journal`, `windows-events`
35
+
36
+### Critical Implementation Differences
37
+
38
+| Aspect | Simple Tables | Log Explorers |
39
+|--------|---------------|---------------|
40
+| **Data Processing** | All data sent to frontend | Backend filters before sending |
41
+| **Facet Counts** | Frontend counts occurrences in received data | Backend computes counts during query execution |
42
+| **Full-Text Search** | Frontend substring search across visible data | Backend pattern matching with facets library |
43
+| **Histograms** | Not supported - no time-based visualization | Optional - backend generates Netdata chart format |
44
+| **Performance** | Limited by browser memory and processing | Scales to millions of records with sampling |
45
+| **Query Parameter** | Ignored by backend (frontend only) | Processed by backend using simple patterns |
46
+
47
+## Request Flow
48
+
49
+### Modern Flow (v:3)
50
+
51
+1. **Info Request** (Always GET)
52
+ ```
53
+ GET /api/v3/function?function=systemd-journal info after:1234567890 before:1234567890
54
+ ```
55
+
56
+2. **Info Response**
57
+ ```json
58
+ {
59
+ "v": 3, // Indicates POST should be used for data requests
60
+ "accepted_params": [...],
61
+ "required_params": [...],
62
+ "status": 200,
63
+ "type": "table",
64
+ "has_history": false
65
+ }
66
+ ```
67
+
68
+3. **Data Request** (POST when v=3)
69
+ ```json
70
+ {
71
+ "query": "*error* !*debug*",
72
+ "selections": {
73
+ "priority": ["error", "warning"],
74
+ "unit": ["nginx.service"]
75
+ },
76
+ "after": 1234567890,
77
+ "before": 1234567890,
78
+ "last": 100
79
+ }
80
+ ```
81
+
82
+### Preflight Info Request Details
83
+
84
+**Request Format:**
85
+```
86
+GET /api/v3/function?function=systemd-journal info after:1234567890 before:1234567890
87
+```
88
+
89
+**Required Response Fields:**
90
+```json
91
+{
92
+ "v": 3,
93
+ "status": 200,
94
+ "type": "table",
95
+ "has_history": false,
96
+ "accepted_params": ["info", "after", "before", "direction", "last"],
97
+ "required_params": [
98
+ {
99
+ "id": "priority",
100
+ "name": "Log Level",
101
+ "type": "select",
102
+ "options": [
103
+ {"id": "error", "name": "Error", "defaultSelected": true},
104
+ {"id": "warn", "name": "Warning"}
105
+ ]
106
+ }
107
+ ],
108
+ "help": "Function description"
109
+}
110
+```
111
+
112
+**Frontend Processing:**
113
+- `accepted_params`: Validates which parameters can be sent to function
114
+- `required_params`: Generates filter UI, prevents execution if missing
115
+- `v: 3`: Enables POST requests with JSON payloads
116
+- Missing required parameters show user-friendly error messages
117
+
118
+### Backend Implementation
119
+
120
+```c
121
+// Simplified from logs_query_status.h
122
+if(payload) {
123
+ // POST request - parse JSON payload
124
+ facets_use_hashes_for_ids(facets, false); // Use plain field names
125
+ rq->fields_are_ids = false;
126
+} else {
127
+ // GET request - parse CLI parameters (legacy)
128
+ facets_use_hashes_for_ids(facets, true); // Use hash IDs
129
+ rq->fields_are_ids = true;
130
+}
131
+```
132
+
133
+### Key Differences
134
+
135
+| Aspect | GET (Legacy) | POST (Modern v3) |
136
+|--------|--------------|------------------|
137
+| Version | v \< 3 | v = 3 |
138
+| Field IDs | 11-char hashes | Plain names |
139
+| Parameters | URL encoded | JSON body |
140
+| Facet filters | `field_hash:value1,value2` | `{"field": ["value1", "value2"]}` |
141
+| Full-text search | `query:search terms` | `{"query": "search terms"}` |
142
+
143
+### Standard Accepted Parameters
144
+
145
+**Core Parameters (All Functions):**
146
+- `"info"` - Function info requests (always supported)
147
+- `"after"` - Time range start (seconds epoch)
148
+- `"before"` - Time range end (seconds epoch)
149
+
150
+**Log Explorer Parameters (has_history=true):**
151
+- `"direction"` - Query direction: `"backward"` | `"forward"`
152
+- `"last"` - Result limit (default: 200)
153
+- `"anchor"` - Pagination cursor (timestamp or row identifier)
154
+- `"query"` - Full-text search using Netdata simple patterns
155
+- `"facets"` - Facet selection: `"field1,field2"`
156
+- `"histogram"` - Histogram field selection
157
+- `"if_modified_since"` - Conditional updates (microseconds epoch)
158
+- `"data_only"` - Skip metadata in response (boolean)
159
+- `"delta"` - Incremental responses (boolean)
160
+- `"tail"` - Streaming mode (boolean)
161
+- `"sampling"` - Data sampling control
162
+
163
+**UI Feature Parameters:**
164
+- `"slice"` - Enables "Full data queries" toggle in UI
165
+
166
+**Frontend Behavior:**
167
+```javascript
168
+// Only accepted parameters are sent to functions
169
+const allowedFilterIds = [...selectedFacets, ...requiredParamIds, ...acceptedParams]
170
+filtersToSend = allowedFilterIds.reduce((acc, filterId) => {
171
+ if (filterId in filters) acc[filterId] = filters[filterId]
172
+ return acc
173
+}, {})
174
+```
175
+
176
+### Query Parameter (Netdata Simple Patterns)
177
+
178
+The `query` parameter uses Netdata's simple pattern matching for full-text search across all fields:
179
+
180
+**Pattern Syntax:**
181
+- `|` - Pattern separator (OR logic between patterns)
182
+- `*` - Wildcard matching any number of characters
183
+- `!` - Negates the pattern (exclude matches)
184
+- Spaces are matched literally (not pattern separators)
185
+- Case-insensitive matching
186
+- Default behavior is substring matching (no wildcards needed)
187
+
188
+**Matching Modes:**
189
+- `pattern` - Substring match (default) - finds "pattern" anywhere
190
+- `*pattern` - Suffix match - finds strings ending with "pattern"
191
+- `pattern*` - Prefix match - finds strings starting with "pattern"
192
+- `*pattern*` - Substring match (explicit) - same as default
193
+
194
+**Examples:**
195
+```json
196
+{
197
+ "query": "error" // Finds "error" anywhere (substring)
198
+ "query": "error|warning" // Finds "error" OR "warning"
199
+ "query": "error|warning|critical" // Multiple OR patterns
200
+ "query": "!debug" // Exclude ALL rows containing "debug"
201
+ "query": "!*debugging*|*debug*" // Include "debug" but exclude "debugging"
202
+ "query": "connection failed" // Find exact phrase (spaces included)
203
+ "query": "*error" // Find strings ending with "error"
204
+ "query": "nginx*" // Find strings starting with "nginx"
205
+}
206
+```
207
+
208
+**Pattern Evaluation Rules:**
209
+
210
+1. **Within a field**: Left-to-right, first match wins
211
+ - `"!*debugging*|*debug*"` - If text contains "debugging", it's negative match. Otherwise, if contains "debug", it's positive match.
212
+
213
+2. **Across all fields**: ALL fields are evaluated (no short-circuit)
214
+ - Every field with FTS enabled is checked against the pattern
215
+ - Positive and negative matches are counted separately
216
+ - Field evaluation order doesn't affect the outcome
217
+
218
+3. **Row decision (after all fields evaluated)**:
219
+ - **Excluded if**: ANY field has a negative match (regardless of positive matches)
220
+ - **Included if**: At least one positive match AND zero negative matches
221
+ - **Excluded if**: No positive matches found
222
+
223
+**Example**: Query `"!*debugging*|*debug*"`
224
+```
225
+Row 1: message="debug info", category="debugging tips"
226
+ → message: positive match (debug)
227
+ → category: negative match (debugging)
228
+ → Result: EXCLUDED (has negative match)
229
+
230
+Row 2: message="debug info", category="testing"
231
+ → message: positive match (debug)
232
+ → category: no match
233
+ → Result: INCLUDED (positive match, no negative)
234
+
235
+Row 3: message="error info", category="testing"
236
+ → message: no match
237
+ → category: no match
238
+ → Result: EXCLUDED (no positive matches)
239
+```
240
+
241
+**Key Point**: The order fields are evaluated doesn't matter - the same counters are updated and the same decision is made regardless of whether positive or negative matches are found first.
242
+
243
+**Common Use Cases:**
244
+- `"timeout|failed|refused"` - Find various connection issues
245
+- `"!trace|!debug|nginx|apache"` - Find web server logs, but exclude debug/trace
246
+- `"error code 500"` - Find exact phrase with spaces
247
+- `"critical|fatal|emergency"` - Find severe log levels
248
+- `"!*test*|!*debug*|*"` - Include everything except test and debug content
249
+
250
+**Implementation Details:**
251
+- Uses `simple_pattern_create(query, "|", SIMPLE_PATTERN_SUBSTRING, false)`
252
+- Searches all fields marked with `FACET_KEY_OPTION_FTS` or when `FACETS_OPTION_ALL_KEYS_FTS` is set
253
+- No support for `?` single-character wildcards
254
+- Escaping with `\` is supported for literal matches
255
+
256
+**Important**: Hash IDs (like `priority_hash`) are obsolete and only exist for backward compatibility with GET requests. All new functions should use v:3 with plain field names.
257
+
258
+## Simple Table Format
259
+
260
+### Complete Structure
261
+
262
+```json
263
+{
264
+ // Required fields
265
+ "status": 200,
266
+ "type": "table",
267
+ "has_history": false,
268
+ "data": [
269
+ [value1, value2, value3, ...], // Row 1
270
+ [value1, value2, value3, ...], // Row 2
271
+ // Special rowOptions as last element
272
+ [..., {"rowOptions": {"severity": "warning|error|notice|normal"}}]
273
+ ],
274
+ "columns": {
275
+ "column_name": {
276
+ // Required
277
+ "index": 0, // Position in data array
278
+ "name": "Display Name", // Column header
279
+ "type": "string", // Field type
280
+
281
+ // Optional
282
+ "unique_key": false, // Row identifier (exactly one required)
283
+ "visible": true, // Default visibility
284
+ "sticky": false, // Pin when scrolling
285
+ "visualization": "value", // How to render
286
+ "transform": "none", // Value transformation
287
+ "decimal_points": 2, // For numbers
288
+ "units": "bytes", // Display units
289
+ "max": 100, // For bar types
290
+ "sort": "descending", // Default sort
291
+ "sortable": true, // User can sort
292
+ "filter": "multiselect", // Filter type
293
+ "full_width": false, // Expand to fill
294
+ "wrap": false, // Text wrapping
295
+ "summary": "sum" // Aggregation (backend only)
296
+ }
297
+ },
298
+
299
+ // Optional extensions
300
+ "help": "Function description",
301
+ "update_every": 1,
302
+ "expires": 1234567890,
303
+ "default_sort_column": "column_name",
304
+ "group_by": {
305
+ "aggregated": [{
306
+ "id": "group_id",
307
+ "name": "Group Name",
308
+ "column": "column_to_group_by"
309
+ }]
310
+ },
311
+ "charts": {
312
+ "chart_id": {
313
+ "name": "Chart Name",
314
+ "type": "stacked|bar",
315
+ "columns": ["col1", "col2"]
316
+ }
317
+ },
318
+ "accepted_params": [{
319
+ "id": "param_id",
320
+ "name": "Parameter Name",
321
+ "type": "string|integer|boolean",
322
+ "default": "default_value"
323
+ }],
324
+ "required_params": ["param1", "param2"]
325
+}
326
+```
327
+
328
+### Row Options
329
+
330
+Special last element in data array for row styling:
331
+
332
+```json
333
+[..., {"rowOptions": {"severity": "error"}}] // Red background
334
+[..., {"rowOptions": {"severity": "warning"}}] // Yellow background
335
+[..., {"rowOptions": {"severity": "notice"}}] // Blue background
336
+[..., {"rowOptions": {"severity": "normal"}}] // Default appearance
337
+```
338
+
339
+## Log Explorer Format
340
+
341
+### Complete Structure
342
+
343
+```json
344
+{
345
+ // Basic fields (same as simple table)
346
+ "status": 200,
347
+ "type": "table",
348
+ "has_history": true, // REQUIRED: Enables log explorer UI
349
+ "help": "System log explorer",
350
+ "update_every": 1,
351
+
352
+ // Table metadata
353
+ "table": {
354
+ "id": "logs",
355
+ "has_history": true,
356
+ "pin_alert": false
357
+ },
358
+
359
+ // Faceted filters (dynamic with counts)
360
+ "facets": [
361
+ {
362
+ "id": "priority", // Plain field name (not hash)
363
+ "name": "Priority",
364
+ "order": 1,
365
+ "defaultExpanded": true,
366
+ "options": [
367
+ {
368
+ "id": "ERROR",
369
+ "name": "ERROR",
370
+ "count": 45, // Real-time count
371
+ "order": 1
372
+ }
373
+ ]
374
+ }
375
+ ],
376
+
377
+ // Enhanced columns
378
+ "columns": {
379
+ "timestamp": {
380
+ "index": 0,
381
+ "id": "timestamp",
382
+ "name": "Time",
383
+ "type": "timestamp",
384
+ "transform": "datetime_usec",
385
+ "sort": "descending|fixed",
386
+ "sortable": false,
387
+ "sticky": true
388
+ },
389
+ "level": {
390
+ "index": 1,
391
+ "id": "priority", // Links to facet
392
+ "name": "Level",
393
+ "type": "string",
394
+ "visualization": "pill",
395
+ "filter": "facet", // Not multiselect!
396
+ "options": ["facet", "visible", "sticky"]
397
+ },
398
+ "message": {
399
+ "index": 3,
400
+ "id": "message",
401
+ "name": "Message",
402
+ "type": "string",
403
+ "full_width": true,
404
+ "options": [
405
+ "full_width",
406
+ "wrap",
407
+ "visible",
408
+ "main_text", // Primary content
409
+ "fts", // Full-text searchable
410
+ "rich_text" // May contain formatting
411
+ ]
412
+ }
413
+ },
414
+
415
+ // Data with microsecond timestamps
416
+ "data": [
417
+ [
418
+ 1697644320000000, // Microseconds
419
+ {"severity": "error"}, // rowOptions
420
+ "ERROR", // level
421
+ "nginx", // source
422
+ "Connection failed" // message
423
+ ]
424
+ ],
425
+
426
+ // Histogram configuration
427
+ "available_histograms": [
428
+ {"id": "priority", "name": "Priority", "order": 1},
429
+ {"id": "source", "name": "Source", "order": 2}
430
+ ],
431
+ "histogram": {
432
+ "id": "priority",
433
+ "name": "Priority",
434
+ "chart": {
435
+ "summary": {/* Netdata chart metadata */},
436
+ "result": {
437
+ "labels": ["time", "ERROR", "WARN", "INFO"],
438
+ "data": [
439
+ [1697644200, 5, 12, 234],
440
+ [1697644260, 3, 8, 198]
441
+ ]
442
+ }
443
+ }
444
+ },
445
+
446
+ // Pagination metadata
447
+ "items": {
448
+ "evaluated": 50000, // Total scanned
449
+ "matched": 2520, // Match filters
450
+ "unsampled": 100, // Skipped (sampling)
451
+ "estimated": 0, // Statistical estimate
452
+ "returned": 100, // In this response
453
+ "max_to_return": 100,
454
+ "before": 0,
455
+ "after": 2420
456
+ },
457
+
458
+ // Navigation anchor
459
+ "anchor": {
460
+ "last_modified": 1697644320000000,
461
+ "direction": "backward" // or "forward"
462
+ },
463
+
464
+ // Request echo (optional)
465
+ "request": {
466
+ "query": "*error* *warning*",
467
+ "filters": ["priority:error,warning"],
468
+ "histogram": "priority"
469
+ },
470
+
471
+ // Additional metadata
472
+ "expires": 1697644920000,
473
+ "sampling": 10 // 1 in N sampling
474
+}
475
+```
476
+
477
+### Key Differences from Simple Tables
478
+
479
+| Feature | Simple Table | Log Explorer |
480
+|---------|--------------|--------------|
481
+| **Facet counts** | Frontend computes from data | Backend computes in facets library |
482
+| **Full-text search** | Frontend substring matching | Backend pattern matching |
483
+| **Histograms** | Not supported | Optional (backend generated) |
484
+| Filtering | Static multiselect | Dynamic facets with counts |
485
+| Pagination | All data at once | Anchor-based infinite scroll |
486
+| Time visualization | None | Histogram chart |
487
+| Navigation | None | Bi-directional with timestamps |
488
+| Performance | All data loaded | Sampling for large datasets |
489
+
490
+### Log-Specific Column Options
491
+
492
+| Option | UI Effect |
493
+|--------|-----------|
494
+| `"facet"` | Field is filterable via facets |
495
+| `"fts"` | Full-text searchable |
496
+| `"main_text"` | Primary content field |
497
+| `"rich_text"` | May contain formatting |
498
+| `"pretty_xml"` | Format as XML |
499
+| `"hidden"` | Hide by default |
500
+
501
+## Facet Value Pills and Aggregated Counts
502
+
503
+### Overview
504
+
505
+Facet values in the sidebar display pills with counts that change based on the function's aggregation mode. The UI uses a smart component that displays different formats:
506
+
507
+1. **Simple Counts**: Just the number of matching rows
508
+2. **Aggregated Counts**: Shows both aggregated count and original count with a union symbol
509
+
510
+### UI Implementation
511
+
512
+The pills are rendered using this logic:
513
+
514
+```javascript
515
+{!!actualCount && <TextSmall>{actualCount} ⊃ </TextSmall>}
516
+<TextSmall>{(pill || count).toString()}</TextSmall>
517
+```
518
+
519
+**Symbol**: `⊃` renders as `⊃` (superset symbol, looks like rotated 'u')
520
+
521
+**Display Formats**:
522
+- Simple: `"42"` (just the count)
523
+- Aggregated: `"15 ⊃ 42"` (15 aggregated items containing 42 total)
524
+
525
+### Backend Configuration
526
+
527
+Functions enable aggregated counts by including an `aggregated_view` object in their response:
528
+
529
+```json
530
+{
531
+ "aggregated_view": {
532
+ "column": "Count",
533
+ "results_label": "unique combinations",
534
+ "aggregated_label": "sockets"
535
+ }
536
+}
537
+```
538
+
539
+**Example from network-connections function**:
540
+```c
541
+// In network-viewer.c when aggregated=true
542
+buffer_json_member_add_object(wb, "aggregated_view");
543
+{
544
+ buffer_json_member_add_string(wb, "column", "Count");
545
+ buffer_json_member_add_string(wb, "results_label", "unique combinations");
546
+ buffer_json_member_add_string(wb, "aggregated_label", "sockets");
547
+}
548
+buffer_json_object_close(wb);
549
+```
550
+
551
+## Charts Configuration
552
+
553
+Functions can provide both standard charts (computed by frontend) and custom visualizations.
554
+
555
+### Standard Charts
556
+
557
+**Configuration:**
558
+```json
559
+{
560
+ "charts": {
561
+ "cpu_usage": {
562
+ "name": "CPU Usage by Service",
563
+ "type": "stacked-bar",
564
+ "columns": ["user_cpu", "system_cpu"],
565
+ "groupBy": "column",
566
+ "aggregation": "sum"
567
+ }
568
+ },
569
+ "default_charts": [
570
+ ["cpu_usage", "service_type"]
571
+ ]
572
+}
573
+```
574
+
575
+**Supported Types:**
576
+- `"bar"` - Basic bar chart
577
+- `"stacked-bar"` - Multi-column stacked bars
578
+- `"doughnut"` - Pie/doughnut chart
579
+- `"value"` - Simple numeric display
580
+
581
+**GroupBy Options:**
582
+- `"column"` (default) - Group by selected filter column
583
+- `"all"` - Aggregate all data together
584
+
585
+### Custom Charts
586
+
587
+For specialized visualizations beyond standard chart types, some functions may use predefined custom chart types:
588
+
589
+```json
590
+{
591
+ "customCharts": {
592
+ "network_topology": {
593
+ "type": "network-viewer",
594
+ "config": {
595
+ "layout": "force",
596
+ "showLabels": true
597
+ }
598
+ }
599
+ }
600
+}
601
+```
602
+
603
+**Available Custom Types:**
604
+- `"network-viewer"` - Interactive network topology (for network-connections function)
605
+
606
+### Frontend Processing
607
+
608
+Charts are computed from table data:
609
+1. Frontend groups data by selected column
610
+2. Applies aggregation function (sum, count, avg, etc.)
611
+3. Renders chart with grouped results
612
+
613
+### Table Row Grouping
614
+
615
+Simple tables support grouping rows with backend-defined aggregation rules.
616
+
617
+**Backend Summary Types:**
618
+```c
619
+typedef enum {
620
+ RRDF_FIELD_SUMMARY_COUNT, // Count rows in group
621
+ RRDF_FIELD_SUMMARY_UNIQUECOUNT, // Count unique values
622
+ RRDF_FIELD_SUMMARY_SUM, // Sum numeric values
623
+ RRDF_FIELD_SUMMARY_MIN, // Minimum value
624
+ RRDF_FIELD_SUMMARY_MAX, // Maximum value
625
+ RRDF_FIELD_SUMMARY_MEAN, // Average value
626
+ RRDF_FIELD_SUMMARY_MEDIAN, // Median value
627
+} RRDF_FIELD_SUMMARY;
628
+```
629
+
630
+**Column Summary Configuration:**
631
+```c
632
+buffer_rrdf_table_add_field(
633
+ wb, field_id++, "cpu", "CPU Usage",
634
+ RRDF_FIELD_TYPE_INTEGER,
635
+ RRDF_FIELD_VISUAL_VALUE,
636
+ RRDF_FIELD_TRANSFORM_NUMBER,
637
+ 2, "%", NAN, RRDF_FIELD_SORT_DESCENDING, NULL,
638
+ RRDF_FIELD_SUMMARY_SUM, // How to aggregate when grouping
639
+ RRDF_FIELD_FILTER_RANGE,
640
+ RRDF_FIELD_OPTS_VISIBLE, NULL
641
+);
642
+```
643
+
644
+**Group By Support:**
645
+```json
646
+{
647
+ "group_by": {
648
+ "aggregated": [{
649
+ "id": "by_status",
650
+ "name": "By Status",
651
+ "column": "status"
652
+ }]
653
+ }
654
+}
655
+```
656
+
657
+### Frontend Processing
658
+
659
+The frontend processes table data to generate facet counts:
660
+
661
+```javascript
662
+const getFilterTableOptions = (data, { param, columns, aggregatedView } = {}) =>
663
+ Object.entries(
664
+ data.reduce((h, fn) => {
665
+ h[fn[param]] = {
666
+ count: (h[fn[param]]?.count || 0) + (fn.hidden ? 0 : 1),
667
+ ...(aggregatedView && {
668
+ actualCount: (h[fn[param]]?.actualCount || 0) +
669
+ (fn.hidden ? 0 : fn[aggregatedView.column] || 1),
670
+ actualCountLabel: aggregatedView.aggregatedLabel,
671
+ countLabel: aggregatedView.resultsLabel,
672
+ }),
673
+ }
674
+ return h
675
+ }, {})
676
+ ).map(([id, values]) => ({ id, ...values }))
677
+```
678
+
679
+### How It Works
680
+
681
+The aggregated count system uses an existing data column to track how many items were aggregated into each row:
682
+
683
+1. **Backend declares** which column contains the aggregation count:
684
+ ```json
685
+ "aggregated_view": {
686
+ "column": "Count" // Use the "Count" column from data rows
687
+ }
688
+ ```
689
+
690
+2. **Frontend calculates** two values for each facet value:
691
+ - **`count`**: How many rows have this facet value
692
+ - **`actualCount`**: Sum of the aggregation column for all rows with this facet value
693
+
694
+3. **Example**: Network connections aggregated by protocol
695
+
696
+ **Data returned by backend:**
697
+ ```
698
+ Direction | Protocol | LocalPort | RemotePort | Count
699
+ ---------|----------|-----------|------------|-------
700
+ Inbound | TCP | * | * | 5
701
+ Inbound | UDP | * | * | 3
702
+ Outbound | TCP | * | * | 7
703
+ Outbound | UDP | * | * | 2
704
+ ```
705
+
706
+ **Facet pills displayed in UI:**
707
+ - Direction facet:
708
+ - Inbound: `8 ⊃ 2` (8 connections aggregated into 2 rows)
709
+ - Outbound: `9 ⊃ 2` (9 connections aggregated into 2 rows)
710
+ - Protocol facet:
711
+ - TCP: `12 ⊃ 2` (12 connections aggregated into 2 rows)
712
+ - UDP: `5 ⊃ 2` (5 connections aggregated into 2 rows)
713
+
714
+ **Reading the pills**: `12 ⊃ 2` means "12 original items shown in 2 table rows"
715
+
716
+### Tooltip Content
717
+
718
+The tooltip provides human-readable context:
719
+- Simple mode: `"42 results"`
720
+- Aggregated mode: `"15 sockets aggregated in 42 unique combinations"`
721
+
722
+### Current Implementations
723
+
724
+| Function | Aggregated Mode | Trigger | Count Meaning |
725
+|----------|----------------|---------|---------------|
726
+| `network-connections` | `sockets:aggregated` | Parameter | Sockets → unique combinations |
727
+| Other functions | N/A | None currently | Single count only |
728
+
729
+### Adding Aggregated Counts
730
+
731
+To add aggregated count support to a function:
732
+
733
+1. **Backend**: Add `aggregated_view` object to response when in aggregated mode
734
+2. **Data**: Include aggregation column with numeric values
735
+3. **Frontend**: No changes needed - automatically processes based on `aggregated_view` presence
736
+
737
+## Field Types and Enumerations
738
+
739
+### Field Types (RRDF_FIELD_TYPE_*)
740
+
741
+#### Implemented in UI
742
+
743
+| Type | UI Component | Use Case | Notes |
744
+|------|--------------|----------|-------|
745
+| `string` | ValueCell | Text, names, categories | Left-aligned |
746
+| `integer` | ValueCell | Numbers, counts, IDs | Right-aligned |
747
+| `bar-with-integer` | BarCell | Percentages, metrics | Requires `max` |
748
+| `duration` | BarCell | Time intervals | Auto-formats seconds |
749
+| `timestamp` | DatetimeCell* | Date/time points | *UI maps to datetime |
750
+| `feedTemplate` | FeedTemplateCell | Rich content | Auto full_width |
751
+
752
+#### Fallback Implementation
753
+
754
+| Type | Behavior | Notes |
755
+|------|----------|-------|
756
+| `boolean` | ValueCell | No special boolean UI |
757
+| `detail-string` | ValueCell | No expandable functionality |
758
+| `array` | ValueCell | Works with `pill` visualization |
759
+| `none` | ValueCell | Avoid using |
760
+
761
+### Visual Types (RRDF_FIELD_VISUAL_*)
762
+
763
+| Type | UI Component | Use Case |
764
+|------|--------------|----------|
765
+| `value` | ValueCell | Standard text display (default) |
766
+| `bar` | BarCell | Progress bar without text |
767
+| `pill` | PillCell | Badge/tag display |
768
+| `richValue` | RichValueCell | Enhanced value display |
769
+| `feedTemplate` | FeedTemplateCell | Full-width template |
770
+| `rowOptions` | null | Special row configuration |
771
+
772
+**Fallback Behavior**: `gauge` is not a recognized visualization. If used, it is ignored, and the renderer falls back to using the field's `type` to select a component.
773
+
774
+### Transform Types (RRDF_FIELD_TRANSFORM_*)
775
+
776
+| Type | Input | Output | Notes |
777
+|------|-------|--------|-------|
778
+| `none` | Any | Unchanged | Default |
779
+| `number` | Number | Formatted with decimals | Uses `decimal_points` |
780
+| `duration` | Seconds | "Xd Yh Zm" | Human-readable |
781
+| `datetime` | Epoch ms | Localized date/time | |
782
+| `datetime_usec` | Epoch μs | Localized date/time | For logs |
783
+| `xml` | XML string | Formatted XML | No specialized UI |
784
+
785
+### Conditional Patterns and Dependencies
786
+
787
+#### Type and Transform Compatibility
788
+
789
+Not all `transform` values are compatible with all `type` values. The backend enforces the following compatibility rules:
790
+
791
+| Field Type (`type`) | Compatible Transforms (`transform`) |
792
+|---|---|
793
+| `timestamp` | `datetime_ms`, `datetime_usec` |
794
+| `duration` | `duration_s` |
795
+| `integer`, `bar-with-integer` | `number` |
796
+| `string`, `boolean`, `array` | `none`, `xml` |
797
+
798
+Using an incompatible transform will result in unexpected behavior or errors.
799
+
800
+#### `rowOptions` Dummy Column
801
+
802
+To add `rowOptions` for row-level severity styling, a special "dummy" column must be added to the `columns` definition. This column is not displayed in the UI but provides the necessary metadata. It must be created with this specific combination of values:
803
+
804
+* **type**: `none` (`RRDF_FIELD_TYPE_NONE`)
805
+* **visualization**: `rowOptions` (`RRDF_FIELD_VISUAL_ROW_OPTIONS`)
806
+* **options flag**: `dummy` (`RRDF_FIELD_OPTS_DUMMY`)
807
+
808
+**Example C code:**
809
+```c
810
+buffer_rrdf_table_add_field(wb, field_id++, "row_options", "Row Options",
811
+ RRDF_FIELD_TYPE_NONE, RRDF_FIELD_VISUAL_ROW_OPTIONS, RRDF_FIELD_TRANSFORM_NONE,
812
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
813
+ RRDF_FIELD_SUMMARY_COUNT, RRDF_FIELD_FILTER_NONE, RRDF_FIELD_OPTS_DUMMY, NULL);
814
+```
815
+
816
+### Filter Types (RRDF_FIELD_FILTER_*)
817
+
818
+| Type | UI Component | Use Case | Location |
819
+|------|--------------|----------|----------|
820
+| `multiselect` | Checkboxes | Column filtering (default) | Dynamic filters |
821
+| `range` | RangeFilter | Numeric min/max | Dynamic filters |
822
+| `facet` | Facets component | With counts (logs) | Sidebar |
823
+
824
+### Field Options (RRDF_FIELD_OPTS_*)
825
+
826
+| Option | Bit | UI Effect |
827
+|--------|-----|-----------|
828
+| `unique_key` | 0x01 | Row identifier (one required) |
829
+| `visible` | 0x02 | Show by default |
830
+| `sticky` | 0x04 | Pin column when scrolling |
831
+| `full_width` | 0x08 | Expand to fill space |
832
+| `wrap` | 0x10 | Enable text wrapping |
833
+| `dummy` | 0x20 | Internal use only |
834
+| `expanded_filter` | 0x40 | Expand filter by default |
835
+
836
+### Sort Options (RRDF_FIELD_SORT_*)
837
+
838
+- `ascending` - Sort low to high
839
+- `descending` - Sort high to low
840
+- Fixed sort (0x80) - Prevent user sorting
841
+
842
+### Summary Types (RRDF_FIELD_SUMMARY_*)
843
+
844
+Backend calculates but UI doesn't display directly:
845
+- `count`, `sum`, `min`, `max`, `mean`, `median`
846
+- `uniqueCount` - Number of unique values
847
+- `extent` - Range [min, max] (UI supported)
848
+- `unique` - List of unique values (UI supported)
849
+
850
+## UI Implementation
851
+
852
+### Column Width System
853
+
854
+The UI automatically sizes columns based on metadata:
855
+
856
+| Size | Pixels | Applied To |
857
+|------|--------|------------|
858
+| xxxs | 90px | unique_key fields, bar types |
859
+| xxs | 110px | Default for most types |
860
+| xs | 130px | Available but rarely used |
861
+| sm | 160px | timestamp, datetime types |
862
+| md-xl | 190-290px | Available but rarely used |
863
+| xxl | 1000px | feedTemplate with full_width |
864
+
865
+**Algorithm**:
866
+1. If `full_width`: → xxl with expansion
867
+2. If `unique_key`: → xxxs
868
+3. By visualization: bar → xxxs
869
+4. By type: feedTemplate → xxl, timestamp → sm
870
+5. Default → xxs
871
+
872
+### Component Mapping
873
+
874
+```javascript
875
+// Field type → Component
876
+componentByType = {
877
+ "bar": BarCell,
878
+ "bar-with-integer": BarCell,
879
+ "duration": BarCell,
880
+ "pill": PillCell,
881
+ "feedTemplate": FeedTemplateCell,
882
+ "datetime": DatetimeCell,
883
+ // Others → ValueCell
884
+}
885
+
886
+// Visualization → Component
887
+componentByVisualization = {
888
+ "bar": BarCell,
889
+ "pill": PillCell,
890
+ "richValue": RichValueCell,
891
+ "feedTemplate": FeedTemplateCell,
892
+ "rowOptions": null, // Skip rendering
893
+ // Others → ValueCell
894
+}
895
+```
896
+
897
+### UI Component Architecture
898
+
899
+The frontend uses a modular architecture with:
900
+- **Value Components**: Handle different field type rendering
901
+- **Table Normalizer**: Processes function responses into UI-ready format
902
+- **Filter Components**: Implement multiselect, range, and facet filtering
903
+- **Chart Components**: Standard chart rendering (bar, stacked-bar, doughnut)
904
+- **Custom Visualizations**: Extensible system for specialized charts
905
+
906
+## Backend Implementation
907
+
908
+### Key Functions
909
+
910
+```c
911
+// Add a field to the table
912
+buffer_rrdf_table_add_field(
913
+ BUFFER *wb,
914
+ size_t field_id,
915
+ const char *key,
916
+ const char *name,
917
+ RRDF_FIELD_TYPE type,
918
+ RRDF_FIELD_VISUAL visual,
919
+ RRDF_FIELD_TRANSFORM transform,
920
+ size_t decimal_points,
921
+ const char *units,
922
+ NETDATA_DOUBLE max,
923
+ RRDF_FIELD_SORT sort,
924
+ const char *pointer_to_dim_in_rrdr,
925
+ RRDF_FIELD_SUMMARY summary,
926
+ RRDF_FIELD_FILTER filter,
927
+ RRDF_FIELD_OPTS options,
928
+ const char *default_value
929
+);
930
+```
931
+
932
+### Real Examples
933
+
934
+```c
935
+// CPU usage with progress bar
936
+buffer_rrdf_table_add_field(
937
+ wb, field_id++, "CPU", "CPU %",
938
+ RRDF_FIELD_TYPE_BAR_WITH_INTEGER,
939
+ RRDF_FIELD_VISUAL_BAR,
940
+ RRDF_FIELD_TRANSFORM_NUMBER,
941
+ 2, "%", 100.0, RRDF_FIELD_SORT_DESCENDING, NULL,
942
+ RRDF_FIELD_SUMMARY_SUM,
943
+ RRDF_FIELD_FILTER_RANGE,
944
+ RRDF_FIELD_OPTS_VISIBLE, NULL
945
+);
946
+
947
+// Process name (unique key)
948
+buffer_rrdf_table_add_field(
949
+ wb, field_id++, "Name", "Name",
950
+ RRDF_FIELD_TYPE_STRING,
951
+ RRDF_FIELD_VISUAL_VALUE,
952
+ RRDF_FIELD_TRANSFORM_NONE,
953
+ 0, NULL, NAN, RRDF_FIELD_SORT_ASCENDING, NULL,
954
+ RRDF_FIELD_SUMMARY_COUNT,
955
+ RRDF_FIELD_FILTER_MULTISELECT,
956
+ RRDF_FIELD_OPTS_VISIBLE | RRDF_FIELD_OPTS_STICKY | RRDF_FIELD_OPTS_UNIQUE_KEY,
957
+ NULL
958
+);
959
+
960
+// Row options in data
961
+buffer_json_add_array_item_object(wb);
962
+buffer_json_member_add_object(wb, "rowOptions");
963
+buffer_json_member_add_string(wb, "severity", "error");
964
+buffer_json_object_close(wb);
965
+buffer_json_object_close(wb);
966
+```
967
+
968
+### Key Backend Files
969
+
970
+- **Enums**: `netdata/src/libnetdata/buffer/functions_fields.h`
971
+- **Implementation**: `netdata/src/libnetdata/buffer/functions_fields.c`
972
+- **Facets Library**: `netdata/src/libnetdata/facets/`
973
+- **Example Functions**:
974
+ - `apps.plugin/apps_functions.c`
975
+ - `network-viewer.plugin/network-viewer.c`
976
+ - `systemd-journal.plugin/systemd-journal.c`
977
+
978
+## Best Practices
979
+
980
+### Always Include
981
+- One field with `unique_key` option
982
+- Meaningful `name` for display
983
+- Appropriate `type` for the data
984
+- Info response with `"v": 3`
985
+
986
+### For Numeric Data
987
+- Use `bar-with-integer` for percentages
988
+- Set appropriate `max` value
989
+- Include `units` for clarity
990
+- Use `range` filter
991
+
992
+### For Time Data
993
+- `duration` type for intervals
994
+- `timestamp` type for points in time
995
+- Use appropriate transform
996
+
997
+### For Status/Severity
998
+- Include `rowOptions` as last array element
999
+- Use standard severity levels: error, warning, notice, normal
1000
+
1001
+### For Filtering
1002
+- `multiselect` (default) for categories
1003
+- `range` for numeric data
1004
+- `facet` only for has_history=true
1005
+
1006
+### Common Patterns
1007
+- Numeric metrics → `bar-with-integer` + `range` filter
1008
+- Categories → `string` + `multiselect` filter
1009
+- Time intervals → `duration` + `duration` transform
1010
+- Status → `string` + `pill` visualization
1011
+- Row coloring → `rowOptions` with severity
1012
+
1013
+## Known Functions
1014
+
1015
+### Simple Table Functions (has_history=false)
1016
+
1017
+| Function | Plugin | Key Features |
1018
+|----------|--------|--------------|
1019
+| processes | apps.plugin | CPU/memory bars, grouping |
1020
+| socket | ebpf.plugin | Complex filters, charts |
1021
+| network-connections | network-viewer.plugin | Aggregated views, severity |
1022
+| systemd-list-units | systemd-units.plugin | Unit status, severity |
1023
+| ipmi-sensors | freeipmi.plugin | Hardware monitoring |
1024
+| block-devices | proc.plugin | I/O statistics, charts |
1025
+| network-interfaces | proc.plugin | Network stats, severity |
1026
+| mount-points | diskspace.plugin | Filesystem usage |
1027
+| cgroup-top | cgroups.plugin | Container metrics |
1028
+| systemd-top | cgroups.plugin | Service metrics |
1029
+| metrics-cardinality | web api | Dynamic columns |
1030
+| streaming | web api | Replication status |
1031
+| all-queries | web api | Monitors the progress of in-flight queries |
1032
+
1033
+### Log Explorer Functions (has_history=true)
1034
+
1035
+| Function | Plugin | Key Features |
1036
+|----------|--------|--------------|
1037
+| systemd-journal | systemd-journal.plugin | System logs, faceted search |
1038
+| windows-events | windows-events.plugin | Windows logs, faceted search |
1039
+
1040
+
1041
+## Development Checklist
1042
+
1043
+### Creating a New Function
1044
+
1045
+- [ ] Choose function type (simple table or log explorer)
1046
+- [ ] Implement info response with `"v": 3`
1047
+- [ ] Define columns with appropriate types and options
1048
+- [ ] Include one `unique_key` field
1049
+- [ ] Add proper error handling
1050
+- [ ] Test with POST requests
1051
+- [ ] Document accepted/required parameters
1052
+
1053
+### Format Validation
1054
+
1055
+- [ ] Verify JSON structure matches specification
1056
+- [ ] Check all required fields are present
1057
+- [ ] Test empty result sets
1058
+- [ ] Test large datasets
1059
+- [ ] Verify error responses
1060
+- [ ] Test special characters in data
1061
+- [ ] Check numeric precision
1062
+- [ ] Verify date/time formatting
1063
+- [ ] Test sorting and filtering
1064
+
1065
+### UI Integration
1066
+
1067
+- [ ] Confirm field types map to UI components
1068
+- [ ] Verify filters work correctly
1069
+- [ ] Check column widths display properly
1070
+- [ ] Test row coloring with severity
1071
+- [ ] Verify transforms apply correctly
1072
+- [ ] Check responsive behavior
1073
+
1074
+### Performance
1075
+
1076
+- [ ] Handle large datasets efficiently
1077
+- [ ] Implement sampling for logs if needed
1078
+- [ ] Set appropriate `update_every`
1079
+- [ ] Consider pagination/anchoring
1080
+- [ ] Test with concurrent requests
1081
+
1082
+## Corner Cases and Edge Handling
1083
+
1084
+### Empty Result Sets
1085
+
1086
+The protocol handles empty results gracefully:
1087
+
1088
+```json
1089
+{
1090
+ "status": 200,
1091
+ "type": "table",
1092
+ "has_history": false,
1093
+ "columns": {...}, // Full column definitions
1094
+ "data": [] // Empty array is valid
1095
+}
1096
+```
1097
+
1098
+- Minimum valid response: `{"status": 200, "type": "table", "columns": {}, "data": []}`
1099
+- UI displays "No data available" message
1100
+- Column headers still render for context
1101
+
1102
+### Null and Missing Values
1103
+
1104
+- `null` values in data arrays are rendered as empty cells
1105
+- Missing array elements default to `null`
1106
+- `NaN` for numeric fields: For fields with `transform: "number"`, `NaN` values will be displayed as the string "NaN". For `timestamp` fields with `datetime` or `datetime_usec` transforms, `NaN` epoch values will display as empty cells.
1107
+- Empty strings render as empty cells
1108
+- Backend uses `NAN` constant for missing numeric values
1109
+
1110
+### Special Characters
1111
+
1112
+The protocol properly escapes:
1113
+- JSON special characters (`"`, `\`, control chars)
1114
+- HTML entities (escaped by React components): React automatically escapes HTML content to prevent XSS attacks. Any HTML tags or entities in the data will be displayed as literal text, not rendered as HTML.
1115
+- Unicode characters (UTF-8 support throughout)
1116
+- SQL injection protection in queries
1117
+
1118
+### Numeric Precision
1119
+
1120
+- `decimal_points` field controls display precision, using JavaScript's `toFixed()` method.
1121
+- Backend uses `NETDATA_DOUBLE` type for floating-point numbers.
1122
+- Frontend's number formatting:
1123
+ - Some components use `Intl.NumberFormat` with a fixed locale (e.g., "en-US").
1124
+ - Others use `toLocaleString()` which respects the browser's locale.
1125
+ - Therefore, locale-specific formatting is applied in some, but not all, cases.
1126
+- Very large numbers: Due to the use of `toFixed()`, very large numbers will be displayed as a long string with the specified decimal places, not automatically in scientific notation.
1127
+- Infinity/NaN handling:
1128
+ - `NaN` values in numeric fields will be displayed as the string "NaN".
1129
+ - `Infinity` and `-Infinity` values will be displayed as the strings "Infinity" and "-Infinity" respectively.
1130
+ - For `timestamp` fields with `datetime` or `datetime_usec` transforms, `NaN` epoch values will display as empty cells.
1131
+
1132
+### Timestamp Format Requirements
1133
+
1134
+**Simple Tables**: Use milliseconds with `datetime` transform
1135
+```json
1136
+{
1137
+ "type": "timestamp",
1138
+ "transform": "datetime", // Expects milliseconds
1139
+ "data": [1697644320000] // JavaScript Date format
1140
+}
1141
+```
1142
+
1143
+**Log Explorers**: Use microseconds with `datetime_usec` transform
1144
+```json
1145
+{
1146
+ "type": "timestamp",
1147
+ "transform": "datetime_usec", // Expects microseconds
1148
+ "data": [1697644320000000] // Microsecond precision
1149
+}
1150
+```
1151
+
1152
+**Frontend Conversion:**
1153
+```javascript
1154
+// datetime_usec automatically converts to milliseconds
1155
+if (usec) {
1156
+ epoch = epoch ? Math.floor(epoch / 1000) : epoch
1157
+}
1158
+```
1159
+
1160
+**API Parameters**: `after` and `before` are automatically converted from milliseconds to seconds when sent to functions.
1161
+
1162
+**Display Format:**
1163
+- Timezone: Determined by URL parameter (`utc`) or system default
1164
+- Format: Uses `Intl.DateTimeFormat` with browser's locale
1165
+- Both types render as localized date/time strings with seconds precision
1166
+
1167
+### Sorting Capabilities
1168
+
1169
+- **Backend Control**: Each column defines default sort with `RRDF_FIELD_SORT_*`
1170
+- **User Control**: `sortable: true` enables UI sorting (default)
1171
+- **Fixed Sort**: Bit flag 0x80 prevents user sorting
1172
+- **Initial Sort**: `default_sort_column` specifies startup sort
1173
+- **Multi-Column**: UI supports sorting by any sortable column
1174
+- **Performance**: Client-side sorting for simple tables
1175
+
1176
+### Filtering Capabilities
1177
+
1178
+- **Multiselect**: Default filter type with checkboxes
1179
+ - Shows all unique values from the column
1180
+ - Multiple selections allowed
1181
+ - OR logic between selections
1182
+- **Range**: Numeric filters with min/max sliders
1183
+ - Requires numeric field type
1184
+ - Auto-detects min/max from data
1185
+ - Inclusive filtering
1186
+- **Facet**: Advanced filtering for log explorer
1187
+ - Shows counts next to each option
1188
+ - Dynamic updates with other filters
1189
+ - Indexed for performance
1190
+
1191
+## Anchor-Based Pagination
1192
+
1193
+Log explorer functions use anchor-based pagination for efficient navigation through large datasets.
1194
+
1195
+### Configuration
1196
+
1197
+```json
1198
+{
1199
+ "pagination": {
1200
+ "enabled": true,
1201
+ "column": "timestamp",
1202
+ "key": "anchor",
1203
+ "units": "timestamp_usec"
1204
+ }
1205
+}
1206
+```
1207
+
1208
+### Frontend Implementation
1209
+
1210
+**Anchor Management:**
1211
+```javascript
1212
+// Frontend calculates anchors from data boundaries
1213
+anchorBefore: latestData[latestData.length - 1][pagination.column],
1214
+anchorAfter: latestData[0][pagination.column],
1215
+anchorUnits: pagination.units
1216
+```
1217
+
1218
+**Infinite Scroll Navigation:**
1219
+- **Backward**: Scroll down loads older data using `anchorBefore`
1220
+- **Forward**: Scroll up loads newer data using `anchorAfter`
1221
+- **State Tracking**: `hasNextPage`, `hasPrevPage` control load triggers
1222
+
1223
+**Required Parameters:**
1224
+- `anchor: {VALUE}` - Pagination cursor value
1225
+- `direction: "backward"|"forward"` - Navigation direction
1226
+- `last: NUMBER` - Page size (default: 200)
1227
+
1228
+### PLAY Mode Integration
1229
+
1230
+When in PLAY mode (`after < 0`), pagination automatically coordinates with real-time updates:
1231
+
1232
+```javascript
1233
+{
1234
+ direction: "forward",
1235
+ merge: true,
1236
+ tail: true,
1237
+ delta: true,
1238
+ anchor: anchorAfter
1239
+}
1240
+```
1241
+
1242
+### Large Data Sets
1243
+
1244
+Simple tables load all data at once, but handle large sets efficiently:
1245
+- Virtual scrolling for thousands of rows
1246
+- Client-side filtering/sorting
1247
+- No built-in pagination (all data in response)
1248
+
1249
+Log explorer uses anchor-based pagination:
1250
+- Efficient navigation through millions of records
1251
+- Configurable page size (`last` parameter)
1252
+- Bi-directional infinite scrolling
1253
+- Sampling for very large sets
1254
+
1255
+## Incremental Updates (Delta Mode)
1256
+
1257
+Delta mode enables efficient real-time updates by sending only changes since the last request.
1258
+
1259
+### When Delta is Enabled
1260
+
1261
+```javascript
1262
+{
1263
+ if_modified_since: 1697644320000000, // Previous modification timestamp
1264
+ direction: "forward",
1265
+ merge: true,
1266
+ tail: true,
1267
+ delta: true,
1268
+ data_only: true,
1269
+ anchor: anchorAfter
1270
+}
1271
+```
1272
+
1273
+### Delta Response Types
1274
+
1275
+**Facets Delta:**
1276
+```json
1277
+{
1278
+ "facetsDelta": [
1279
+ {
1280
+ "id": "priority",
1281
+ "options": [
1282
+ {"id": "ERROR", "count": 5}, // Incremental counts
1283
+ {"id": "WARN", "count": 12}
1284
+ ]
1285
+ }
1286
+ ]
1287
+}
1288
+```
1289
+
1290
+**Histogram Delta:**
1291
+```json
1292
+{
1293
+ "histogramDelta": {
1294
+ "chart": {
1295
+ "result": {
1296
+ "labels": ["time", "ERROR", "WARN"],
1297
+ "data": [
1298
+ [1697644320, 3, 8] // New data points only
1299
+ ]
1300
+ }
1301
+ }
1302
+ }
1303
+}
1304
+```
1305
+
1306
+### Data Merging
1307
+
1308
+Frontend merges delta responses with existing data:
1309
+- **Facet counts**: Accumulated using `count = (existing || 0) + (delta || 0)`
1310
+- **Table data**: Appended/prepended based on `direction`
1311
+- **Histogram data**: New data points added to existing chart
1312
+
1313
+## Real-Time Updates (PLAY Mode)
1314
+
1315
+PLAY mode enables live data streaming with efficient polling and conditional updates.
1316
+
1317
+### PLAY Mode Detection
1318
+
1319
+- **PLAY Mode**: `after < 0` (relative time from now)
1320
+- **PAUSE Mode**: `after > 0` (absolute timestamp)
1321
+
1322
+### Parameter Coordination
1323
+
1324
+When `if_modified_since` is present, the system automatically includes:
1325
+
1326
+```json
1327
+{
1328
+ "if_modified_since": 1697644320000000,
1329
+ "direction": "forward",
1330
+ "merge": true,
1331
+ "tail": true,
1332
+ "delta": true,
1333
+ "data_only": true,
1334
+ "anchor": "anchorAfter"
1335
+}
1336
+```
1337
+
1338
+### Error Handling
1339
+
1340
+**304 Not Modified**: Indicates no new data available
1341
+```json
1342
+{
1343
+ "status": 304
1344
+}
1345
+```
1346
+
1347
+Frontend handles 304 responses gracefully without showing errors to users.
1348
+
1349
+### Polling Behavior
1350
+
1351
+- **Polling Interval**: Based on function's `update_every` value
1352
+- **Auto-Pause**: When window loses focus or user hovers over data
1353
+- **Conditional Requests**: Uses `if_modified_since` to avoid unnecessary data transfers
1354
+
1355
+### Required vs Optional Fields
1356
+
1357
+**Minimum Required Fields:**
1358
+```json
1359
+{
1360
+ "status": 200, // Required
1361
+ "type": "table", // Required
1362
+ "columns": {}, // Required (can be empty)
1363
+ "data": [] // Required (can be empty)
1364
+}
1365
+```
1366
+
1367
+**Common Optional Fields:**
1368
+- `has_history`: Default false
1369
+- `help`: Documentation text
1370
+- `update_every`: Default 1
1371
+- `expires`: Cache control
1372
+- `default_sort_column`: Initial sort
1373
+- All column options except `index`, `name`, `type`
1374
+
1375
+## Error Handling
1376
+
1377
+### Error Response Format
1378
+
1379
+When a function encounters an error, the backend returns a JSON object. The primary error generation function (`rrd_call_function_error`) produces the following minimal format:
1380
+
1381
+```json
1382
+{
1383
+ "status": 400, // The HTTP status code (e.g., 400, 404, 500)
1384
+ "error_message": "A descriptive error message" // A human-readable message explaining the error
1385
+}
1386
+```
1387
+
1388
+**Frontend Consumption and Interpretation:**
1389
+The frontend is designed to handle a more comprehensive error structure, allowing for richer error display and localization. When an error occurs, the frontend will attempt to extract information from the received error object using the following hierarchy:
1390
+
1391
+* **`status`**: The HTTP status code, used for general error classification (e.g., 400 for bad request, 404 for not found).
1392
+* **`error_message`**: The primary detailed message from the backend. This is the most consistently populated field by current backend functions.
1393
+* **`error`**: A short, machine-readable error identifier (e.g., "MissingParameter"). While not consistently generated by `rrd_call_function_error`, other parts of the system or future backend implementations might provide this.
1394
+* **`message`**: A user-friendly message. The frontend often maps `error_message` or an internal `errorMsgKey` to this for display.
1395
+* **`help`**: Optional additional guidance for resolving the error. This field is not currently generated by `rrd_call_function_error`.
1396
+
1397
+**Example of Frontend Interpretation (Conceptual):**
1398
+The frontend might internally map specific `error_message` strings to predefined `errorMsgKey` values to provide localized or more context-specific messages to the user. For instance, a backend `error_message` like "The 'time_range' parameter is required" might be mapped to an `errorMsgKey` of "ErrMissingTimeRange" in the frontend, which then displays a user-friendly message like "Please specify a time range for this function."
1399
+
1400
+Therefore, while the backend currently provides `status` and `error_message`, developers should be aware that the frontend's error handling is capable of utilizing the more detailed fields (`error`, `message`, `help`) if they are provided by the backend in the future or by other API endpoints.
1401
+
1402
+**Common Error Codes:**
1403
+
1404
+
1405
+### Common Error Codes
1406
+
1407
+| Code | Use Case | Example |
1408
+|------|----------|---------|
1409
+| 400 | Bad input | Missing/invalid parameters |
1410
+| 401 | Auth required | User not logged in |
1411
+| 403 | Forbidden | Insufficient permissions |
1412
+| 404 | Not found | No data matches query |
1413
+| 500 | Server error | Internal failures |
1414
+| 503 | Unavailable | Service overloaded |
1415
+
1416
+## Protocol Integration
1417
+
1418
+### PLUGINSD Commands
1419
+
1420
+Functions integrate with Netdata through the PLUGINSD protocol:
1421
+
1422
+**Registration:**
1423
+```
1424
+FUNCTION "function_name" timeout "help text" "tags" "http_access" priority version
1425
+```
1426
+
1427
+**Execution Flow:**
1428
+1. **Collector → Agent**: `FUNCTION` registers the function
1429
+2. **Agent → Collector**: `FUNCTION_CALL` with transaction ID
1430
+3. **Collector → Agent**: `FUNCTION_RESULT_BEGIN` status format expires
1431
+4. **Collector → Agent**: Response payload
1432
+5. **Collector → Agent**: `FUNCTION_RESULT_END`
1433
+
1434
+**With Payload:**
1435
+```
1436
+FUNCTION_PAYLOAD_BEGIN transaction timeout function access source content_type
1437
+<payload data>
1438
+FUNCTION_PAYLOAD_END
1439
+```
1440
+
1441
+**Cancellation:**
1442
+```
1443
+FUNCTION_CANCEL transaction_id
1444
+```
1445
+
1446
+**Progress Updates:**
1447
+```
1448
+FUNCTION_PROGRESS transaction_id done total
1449
+```
1450
+
1451
+### Streaming Protocol
1452
+
1453
+Functions support streaming for real-time updates:
1454
+
1455
+```c
1456
+// Transaction management
1457
+dictionary_set(parser->inflight.functions, transaction_str, &function_data);
1458
+
1459
+// Timeout handling
1460
+if (*pf->stop_monotonic_ut + RRDFUNCTIONS_TIMEOUT_EXTENSION_UT < now_ut) {
1461
+ // Function timed out
1462
+}
1463
+
1464
+// Progress callback
1465
+if(stream_has_capability(s, STREAM_CAP_PROGRESS)) {
1466
+ // Enable progress updates
1467
+}
1468
+```
1469
+
1470
+### Key Files
1471
+- `pluginsd_functions.c`: Function execution and management
1472
+- `stream-sender-execute.c`: Streaming function calls
1473
+- `plugins.d/README.md`: Protocol documentation
1474
+- `plugins.d/functions-table.md`: Table format specification
1475
+
1476
+## Migration Guide
1477
+
1478
+### Upgrading from GET to POST (v:3)
1479
+
1480
+1. Update info response to include `"v": 3`
1481
+2. Change field IDs from hashes to plain names
1482
+3. Test with JSON POST payloads
1483
+4. Remove hash generation code
1484
+5. Update documentation
1485
+
1486
+### Adding Log Explorer Features
1487
+
1488
+1. Set `has_history: true`
1489
+2. Implement facets with counts
1490
+3. Add timestamp column with microseconds
1491
+4. Support anchor-based pagination
1492
+5. Add histogram data
1493
+6. Implement full-text search
1494
+
1495
+## Status and Next Steps
1496
+
1497
+This document represents the complete v3 protocol specification. Key areas for future development:
1498
+
1499
+1. **Protocol Extensions**
1500
+ - Streaming updates for real-time data
1501
+ - Aggregation pipelines
1502
+ - Custom visualization types
1503
+
1504
+2. **UI Enhancements**
1505
+ - Additional field types (gauge, sparkline)
1506
+ - Custom column renderers
1507
+ - Advanced filtering options
1508
+
1509
+3. **Performance Optimizations**
1510
+ - Server-side pagination for simple tables
1511
+ - Incremental updates
1512
+ - Result caching
1513
+
1514
+---
1515
+
1516
+## Appendix A: Validation Checklist and Progress
1517
+
1518
+*This section tracks the validation work completed and remaining tasks*
1519
+
1520
+### Format Discovery
1521
+- [x] Analyze `processes` function implementation in apps.plugin
1522
+- [x] Analyze `network-connections` function in network-viewer.plugin
1523
+- [x] Identify common patterns between implementations
1524
+- [x] Extract all enum definitions used in responses
1525
+- [x] Document each field type and possible values
1526
+- [x] Identify optional vs required fields
1527
+
1528
+### Enumeration Completeness
1529
+- [x] Find all enum definitions in netdata C code
1530
+- [x] Map enum values to their string representations
1531
+- [x] Document the purpose of each enum value
1532
+- [ ] Check for any conditional enum values
1533
+
1534
+### Function Coverage
1535
+- [x] Scan all collectors/plugins for function implementations
1536
+- [x] List all simple table functions (has_history=false)
1537
+- [x] Verify format consistency across all functions
1538
+- [x] Document any function-specific extensions
1539
+
1540
+### UI Mapping
1541
+- [x] Analyze cloud-frontend code for function rendering
1542
+- [x] Map each format field to UI component
1543
+- [x] Document how enum values affect UI behavior
1544
+- [x] Identify any frontend-specific transformations
1545
+
1546
+### Corner Cases
1547
+- [x] Empty result sets
1548
+- [x] Large data sets (pagination?)
1549
+- [x] Error responses
1550
+- [x] Null/missing values
1551
+- [x] Special characters in data
1552
+- [x] Numeric precision/formatting
1553
+- [x] Date/time formatting
1554
+- [x] Sorting capabilities
1555
+- [x] Filtering capabilities
1556
+
1557
+### Protocol Validation
1558
+- [x] Request format documentation
1559
+- [x] Response format documentation
1560
+- [x] Error handling patterns
1561
+- [x] Streaming protocol integration
1562
+
1563
+### Cross-Reference Checks
1564
+- [x] Compare documented format with actual implementations
1565
+- [x] Verify all functions conform to documented format
1566
+- [x] Check for undocumented features in UI
1567
+- [x] Validate against any existing documentation
1568
+
1569
+---
1570
+
1571
+## Appendix B: Investigation History
1572
+
1573
+### Analysis Log
1574
+
1575
+*This section documents the investigation process to avoid repeating work*
1576
+
1577
+#### Session 1 - Initial Setup and Analysis
1578
+- Created FUNCTIONS.md structure
1579
+- Established checklist for comprehensive validation
1580
+- Analyzed `processes` function in apps.plugin
1581
+ - Location: `/netdata/src/collectors/apps.plugin/apps_functions.c`
1582
+ - Registration: `apps_plugin.c:752`
1583
+ - Uses standard table format with array-based data rows
1584
+- Analyzed `network-connections` function in network-viewer.plugin
1585
+ - Location: `/netdata/src/collectors/network-viewer.plugin/network-viewer.c`
1586
+ - Registration via PLUGINSD protocol
1587
+ - Supports aggregated and detailed views
1588
+- Extracted all RRDF enum definitions from:
1589
+ - `/netdata/src/libnetdata/buffer/functions_fields.h`
1590
+ - `/netdata/src/libnetdata/buffer/functions_fields.c`
1591
+- Documented complete enum value mappings for field types, visualizations, transforms, etc.
1592
+
1593
+#### Session 2 - Protocol Integration and Corner Cases
1594
+- Investigated PLUGINSD protocol integration
1595
+ - Found in `pluginsd_functions.c` and `stream-sender-execute.c`
1596
+ - Commands: FUNCTION, FUNCTION_CALL, FUNCTION_RESULT_BEGIN/END
1597
+ - Support for payloads, cancellation, and progress updates
1598
+- Analyzed corner case handling:
1599
+ - Empty results: `{"status": 200, "type": "table", "columns": {}, "data": []}`
1600
+ - Null values: Rendered as empty cells in UI
1601
+ - Special characters: Proper JSON escaping throughout
1602
+ - Numeric precision: Controlled by `decimal_points` field
1603
+ - Date/time: Millisecond epochs for tables, microsecond for logs
1604
+- Identified required vs optional fields:
1605
+ - Required: status, type, columns, data
1606
+ - Optional: Everything else (has_history, help, etc.)
1607
+- Documented sorting and filtering capabilities:
1608
+ - Sorting: Backend-controlled defaults, user-sortable columns
1609
+ - Filtering: Multiselect (default), range (numeric), facet (logs)
1610
+ - Large datasets: Virtual scrolling for tables, pagination for logs
1611
+
1612
+### Key Findings
1613
+1. **Data Format**: Simple tables use array-of-arrays for data rows
1614
+2. **Column Definition**: Each column has extensive metadata controlling display and behavior
1615
+3. **Common Functions**: Both implement `buffer_rrdf_table_add_field()` for column definitions
1616
+4. **Response Builder**: Uses `buffer_json_*` functions to build JSON responses
1617
+5. **Field Options**: Bit flags allow combining multiple options per field
1618
+6. **Protocol Evolution**: GET with hashes → POST with plain field names (v:3)
1619
+7. **UI Mapping**: Comprehensive component mapping based on type/visualization
1620
+8. **Error Handling**: Standardized error response format with HTTP codes
1621
+9. **Performance**: Client-side operations for tables, server-side for logs
1622
+10. **Extensibility**: Optional fields allow function-specific features
1623
+
1624
+### Format Compliance Summary
1625
+- **All 12 simple table functions are fully compliant** with the documented format
1626
+- **Common extensions** found:
1627
+ - `rowOptions` field for row severity/status (5/12 functions)
1628
+ - `charts` and `default_charts` definitions (most functions)
1629
+ - `group_by` aggregation options (most functions)
1630
+ - `accepted_params` and `required_params` (metrics-cardinality)
1631
+ - `default_sort_column` for initial sorting
1632
+- **No breaking deviations** - all extensions are additive and optional
1633
+
1634
+---
1635
+
1636
+## Appendix C: Log Explorer UI Features Detail
1637
+
1638
+*Detailed UI behaviors when has_history=true*
1639
+
1640
+### UI Features Enabled
1641
+
1642
+When properly formatted, the log explorer UI provides:
1643
+
1644
+1. **Sidebar with Faceted Filters**
1645
+ - Shows facets with real-time counts
1646
+ - Multi-select filtering
1647
+ - Collapsible sections
1648
+ - Search within facets
1649
+
1650
+2. **Time-based Histogram**
1651
+ - Visual log distribution over time
1652
+ - Click and drag to select time ranges
1653
+ - Switch between different histogram fields
1654
+ - Auto-updates with filters
1655
+
1656
+3. **Advanced Table Features**
1657
+ - Infinite scroll using anchor navigation
1658
+ - No manual pagination controls
1659
+ - Automatic row coloring by severity
1660
+ - Full-width message display
1661
+ - Column pinning and resizing
1662
+
1663
+4. **Search and Navigation**
1664
+ - Full-text search box
1665
+ - Bi-directional navigation (forward/backward)
1666
+ - Jump to specific time
1667
+ - Export filtered results
1668
+
1669
+5. **Live Features**
1670
+ - Tail mode for real-time updates
1671
+ - Auto-refresh based on `update_every`
1672
+ - Delta updates for efficiency
1673
+ - Notification of new entries
1674
+
1675
+---
1676
+
1677
+*Last Updated: Based on analysis of Netdata codebase and cloud-frontend implementation*
src/crates/netdata-plugin/docs/README.md
new
+868
@@ -0,0 +1,868 @@
1
+# External plugins
2
+
3
+`plugins.d` is the Netdata internal plugin that collects metrics
4
+from external processes, thus allowing Netdata to use **external plugins**.
5
+
6
+## Provided External Plugins
7
+
8
+| plugin | language | O/S | description |
9
+|:------------------------------------------------------------------------------------------------------:|:--------:|:--------------:|:----------------------------------------------------------------------------------------------------------------------------------------|
10
+| [apps.plugin](/src/collectors/apps.plugin/README.md) | `C` | linux, freebsd | monitors the whole process tree on Linux and FreeBSD and breaks down system resource usage by **process**, **user** and **user group**. |
11
+| [charts.d.plugin](/src/collectors/charts.d.plugin/README.md) | `BASH` | all | a **plugin orchestrator** for data collection modules written in `BASH` v4+. |
12
+| [cups.plugin](/src/collectors/cups.plugin/README.md) | `C` | all | monitors **CUPS** |
13
+| [ebpf.plugin](/src/collectors/ebpf.plugin/README.md) | `C` | linux | monitors different metrics on environments using kernel internal functions. |
14
+| [go.d.plugin](/src/go/plugin/go.d/README.md) | `GO` | all | collects metrics from the system, applications, or third-party APIs. |
15
+| [ioping.plugin](/src/collectors/ioping.plugin/README.md) | `C` | all | measures disk latency. |
16
+| [freeipmi.plugin](/src/collectors/freeipmi.plugin/README.md) | `C` | linux | collects metrics from enterprise hardware sensors, on Linux servers. |
17
+| [nfacct.plugin](/src/collectors/nfacct.plugin/README.md) | `C` | linux | collects netfilter firewall, connection tracker and accounting metrics using `libmnl` and `libnetfilter_acct`. |
18
+| [xenstat.plugin](/src/collectors/xenstat.plugin/README.md) | `C` | linux | collects XenServer and XCP-ng metrics using `lxenstat`. |
19
+| [perf.plugin](/src/collectors/perf.plugin/README.md) | `C` | linux | collects CPU performance metrics using performance monitoring units (PMU). |
20
+| [python.d.plugin](/src/collectors/python.d.plugin/README.md) | `python` | all | a **plugin orchestrator** for data collection modules written in `python` v2 or v3 (both are supported). |
21
+| [slabinfo.plugin](/src/collectors/slabinfo.plugin/README.md) | `C` | linux | collects kernel internal cache objects (SLAB) metrics. |
22
+
23
+Plugin orchestrators may also be described as **modular plugins**. They are modular since they accept custom made modules to be included. Writing modules for these plugins is easier than accessing the native Netdata API directly. You will find modules already available for each orchestrator under the directory of the particular modular plugin (e.g. under python.d.plugin for the python orchestrator).
24
+Each of these modular plugins has each own methods for defining modules. Please check the examples and their documentation.
25
+
26
+## Motivation
27
+
28
+This plugin allows Netdata to use **external plugins** for data collection:
29
+
30
+1. external data collection plugins may be written in any computer language.
31
+
32
+2. external data collection plugins may use O/S capabilities or `setuid` to
33
+ run with escalated privileges (compared to the `netdata` daemon).
34
+ The communication between the external plugin and Netdata is unidirectional
35
+ (from the plugin to Netdata), so that Netdata cannot manipulate an external
36
+ plugin running with escalated privileges.
37
+
38
+## Operation
39
+
40
+Each of the external plugins is expected to run forever.
41
+Netdata will start it when it starts and stop it when it exits.
42
+
43
+If the external plugin exits or crashes, Netdata will log an error.
44
+If the external plugin exits or crashes without pushing metrics to Netdata, Netdata will not start it again.
45
+
46
+- Plugins that exit with any value other than zero, will be disabled. Plugins that exit with zero, will be restarted after some time.
47
+- Plugins may also be disabled by Netdata if they output things that Netdata does not understand.
48
+
49
+The `stdout` of external plugins is connected to Netdata to receive metrics,
50
+with the API defined below.
51
+
52
+The `stderr` of external plugins is connected to Netdata's `error.log`.
53
+
54
+Plugins can create any number of charts with any number of dimensions each. Each chart can have its own characteristics independently of the others generated by the same plugin. For example, one chart may have an update frequency of 1 second, another may have 5 seconds and a third may have 10 seconds.
55
+
56
+## Configuration
57
+
58
+Netdata will supply the environment variables `NETDATA_USER_CONFIG_DIR` (for user supplied) and `NETDATA_STOCK_CONFIG_DIR` (for Netdata supplied) configuration files to identify the directory where configuration files are stored. It is up to the plugin to read the configuration it needs.
59
+
60
+The `netdata.conf` section `[plugins]` section contains a list of all the plugins found at the system where Netdata runs, with a boolean setting to enable them or not.
61
+
62
+Example:
63
+
64
+```
65
+[plugins]
66
+ # enable running new plugins = yes
67
+ # check for new plugins every = 60
68
+
69
+ # charts.d = yes
70
+ # ioping = yes
71
+ # python.d = yes
72
+```
73
+
74
+The setting `enable running new plugins` sets the default behavior for all external plugins. It can be
75
+overridden for distinct plugins by modifying the appropriate plugin value configuration to either `yes` or `no`.
76
+
77
+The setting `check for new plugins every` sets the interval between scans of the directory
78
+`/usr/libexec/netdata/plugins.d`. New plugins can be added any time, and Netdata will detect them in a timely manner.
79
+
80
+For each of the external plugins enabled, another `netdata.conf` section
81
+is created, in the form of `[plugin:NAME]`, where `NAME` is the name of the external plugin.
82
+This section allows controlling the update frequency of the plugin and provide
83
+additional command line arguments to it.
84
+
85
+For example, for `apps.plugin` the following section is available:
86
+
87
+```
88
+[plugin:apps]
89
+ # update every = 1
90
+ # command options =
91
+```
92
+
93
+- `update every` controls the granularity of the external plugin.
94
+- `command options` allows giving additional command line options to the plugin.
95
+
96
+Netdata will provide to the external plugins the environment variable `NETDATA_UPDATE_EVERY`, in seconds (the default is 1). This is the **minimum update frequency** for all charts. A plugin that is updating values more frequently than this, is just wasting resources.
97
+
98
+Netdata will call the plugin with just one command line parameter: the number of seconds the user requested this plugin to update its data (by default is also 1).
99
+
100
+Other than the above, the plugin configuration is up to the plugin.
101
+
102
+Keep in mind, that the user may use Netdata configuration to overwrite chart and dimension parameters. This is transparent to the plugin.
103
+
104
+### Autoconfiguration
105
+
106
+Plugins should attempt to autoconfigure themselves when possible.
107
+
108
+For example, if your plugin wants to monitor `squid`, you can search for it on port `3128` or `8080`. If any succeeds, you can proceed. If it fails you can output an error (on stderr) saying that you cannot find `squid` running and giving instructions about the plugin configuration. Then you can stop (exit with non-zero value), so that Netdata will not attempt to start the plugin again.
109
+
110
+## External Plugins API
111
+
112
+Any program that can print a few values to its standard output can become a Netdata external plugin.
113
+
114
+Netdata parses lines starting with:
115
+
116
+- `CHART` - create or update a chart
117
+- `DIMENSION` - add or update a dimension to the chart just created
118
+- `VARIABLE` - define a variable (to be used in health calculations)
119
+- `CLABEL` - add a label to a chart
120
+- `CLABEL_COMMIT` - commit added labels to the chart
121
+- `FUNCTION` - define a function that can be called later to execute it
122
+- `BEGIN` - initialize data collection for a chart
123
+- `SET` - set the value of a dimension for the initialized chart
124
+- `END` - complete data collection for the initialized chart
125
+- `FLUSH` - ignore the last collected values
126
+- `DISABLE` - disable this plugin
127
+- `FUNCTION` - define functions
128
+- `FUNCTION_PROGRESS` - report the progress of a function execution
129
+- `FUNCTION_RESULT_BEGIN` - to initiate the transmission of function results
130
+- `FUNCTION_RESULT_END` - to end the transmission of function result
131
+- `CONFIG` - to define dynamic configuration entities
132
+
133
+a single program can produce any number of charts with any number of dimensions each.
134
+
135
+Charts can be added any time (not just the beginning).
136
+
137
+Netdata may send the following commands to the plugin's `stdin`:
138
+
139
+- `FUNCTION` - to call a specific function, with all parameters inline
140
+- `FUNCTION_PAYLOAD` - to call a specific function, with a payload of parameters
141
+- `FUNCTION_PAYLOAD_END` - to end the payload of parameters
142
+- `FUNCTION_CANCEL` - to cancel a running function transaction - no response is required
143
+- `FUNCTION_PROGRESS` - to report that a user asked the progress of running function call - no response is required
144
+
145
+### Command line parameters
146
+
147
+The plugin **MUST** accept just **one** parameter: **the number of seconds it is
148
+expected to update the values for its charts**. The value passed by Netdata
149
+to the plugin is controlled via its configuration file (so there is no need
150
+for the plugin to handle this configuration option).
151
+
152
+The external plugin can overwrite the update frequency. For example, the server may
153
+request per second updates, but the plugin may ignore it and update its charts
154
+every 5 seconds.
155
+
156
+### Environment variables
157
+
158
+There are a few environment variables that are set by `netdata` and are
159
+available for the plugin to use.
160
+
161
+| variable | description |
162
+|:--------------------------------:|:-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
163
+| `NETDATA_USER_CONFIG_DIR` | The directory where all Netdata-related user configuration should be stored. If the plugin requires custom user configuration, this is the place the user has saved it (normally under `/etc/netdata`). |
164
+| `NETDATA_STOCK_CONFIG_DIR` | The directory where all Netdata -related stock configuration should be stored. If the plugin is shipped with configuration files, this is the place they can be found (normally under `/usr/lib/netdata/conf.d`). |
165
+| `NETDATA_PLUGINS_DIR` | The directory where all Netdata plugins are stored. |
166
+| `NETDATA_USER_PLUGINS_DIRS` | The list of directories where custom plugins are stored. |
167
+| `NETDATA_WEB_DIR` | The directory where the web files of Netdata are saved. |
168
+| `NETDATA_CACHE_DIR` | The directory where the cache files of Netdata are stored. Use this directory if the plugin requires a place to store data. A new directory should be created for the plugin for this purpose, inside this directory. |
169
+| `NETDATA_LOG_DIR` | The directory where the log files are stored. By default the `stderr` output of the plugin will be saved in the `error.log` file of Netdata. |
170
+| `NETDATA_HOST_PREFIX` | This is used in environments where system directories like `/sys` and `/proc` have to be accessed at a different path. |
171
+| `NETDATA_DEBUG_FLAGS` | This is a number (probably in hex starting with `0x`), that enables certain Netdata debugging features. Check **\[[Tracing Options]]** for more information. |
172
+| `NETDATA_UPDATE_EVERY` | The minimum number of seconds between chart refreshes. This is like the **internal clock** of Netdata (it is user configurable, defaulting to `1`). There is no meaning for a plugin to update its values more frequently than this number of seconds. |
173
+| `NETDATA_INVOCATION_ID` | A random UUID in compact form, representing the unique invocation identifier of Netdata. When running under systemd, Netdata uses the `INVOCATION_ID` set by systemd. |
174
+| `NETDATA_LOG_METHOD` | One of `syslog`, `journal`, `stderr` or `none`, indicating the preferred log method of external plugins. |
175
+| `NETDATA_LOG_FORMAT` | One of `journal`, `logfmt` or `json`, indicating the format of the logs. Plugins can use the Netdata `systemd-cat-native` command to log always in `journal` format, and have it automatically converted to the format expected by netdata. |
176
+| `NETDATA_LOG_LEVEL` | One of `emergency`, `alert`, `critical`, `error`, `warning`, `notice`, `info`, `debug`. Plugins are expected to log events with the given priority and the more important ones. |
177
+| `NETDATA_SYSLOG_FACILITY` | Set only when the `NETDATA_LOG_METHOD` is `syslog`. Possible values are `auth`, `authpriv`, `cron`, `daemon`, `ftp`, `kern`, `lpr`, `mail`, `news`, `syslog`, `user`, `uucp` and `local0` to `local7` |
178
+| `NETDATA_ERRORS_THROTTLE_PERIOD` | The log throttling period in seconds. |
179
+| `NETDATA_ERRORS_PER_PERIOD` | The allowed number of log events per period. |
180
+| `NETDATA_SYSTEMD_JOURNAL_PATH` | When `NETDATA_LOG_METHOD` is set to `journal`, this is the systemd-journald socket path to use. |
181
+
182
+### The output of the plugin
183
+
184
+The plugin should output instructions for Netdata to its output (`stdout`). Since this uses pipes, please make sure you flush stdout after every iteration.
185
+
186
+#### DISABLE
187
+
188
+`DISABLE` will disable this plugin. This will prevent Netdata from restarting the plugin. You can also exit with the value `1` to have the same effect.
189
+
190
+#### HOST_DEFINE
191
+
192
+`HOST_DEFINE` defines a new (or updates an existing) virtual host.
193
+
194
+The template is:
195
+
196
+> HOST_DEFINE machine_guid hostname
197
+
198
+where:
199
+
200
+- `machine_guid`
201
+
202
+ uniquely identifies the host, this is what will be needed to add charts to the host.
203
+
204
+- `hostname`
205
+
206
+ is the hostname of the virtual host
207
+
208
+#### HOST_LABEL
209
+
210
+`HOST_LABEL` adds a key-value pair to the virtual host labels. It has to be given between `HOST_DEFINE` and `HOST_DEFINE_END`.
211
+
212
+The template is:
213
+
214
+> HOST_LABEL key value
215
+
216
+where:
217
+
218
+- `key`
219
+
220
+ uniquely identifies the key of the label
221
+
222
+- `value`
223
+
224
+ is the value associated with this key
225
+
226
+There are a few special keys that are used to define the system information of the monitored system:
227
+
228
+- `_cloud_provider_type`
229
+- `_cloud_instance_type`
230
+- `_cloud_instance_region`
231
+- `_os_name`
232
+- `_os_version`
233
+- `_kernel_version`
234
+- `_system_cores`
235
+- `_system_cpu_freq`
236
+- `_system_ram_total`
237
+- `_system_disk_space`
238
+- `_architecture`
239
+- `_virtualization`
240
+- `_container`
241
+- `_container_detection`
242
+- `_virt_detection`
243
+- `_is_k8s_node`
244
+- `_install_type`
245
+- `_prebuilt_arch`
246
+- `_prebuilt_dist`
247
+
248
+#### HOST_DEFINE_END
249
+
250
+`HOST_DEFINE_END` commits the host information, creating a new host entity, or updating an existing one with the same `machine_guid`.
251
+
252
+#### HOST
253
+
254
+`HOST` switches data collection between hosts.
255
+
256
+The template is:
257
+
258
+> HOST machine_guid
259
+
260
+where:
261
+
262
+- `machine_guid`
263
+
264
+ is the UUID of the host to switch to. After this command, every other command following it is assumed to be associated with this host.
265
+ Setting machine_guid to `localhost` switches data collection to the local host.
266
+
267
+#### CHART
268
+
269
+`CHART` defines a new chart.
270
+
271
+the template is:
272
+
273
+> CHART type.id name title units \[family \[context \[charttype \[priority \[update_every \[options \[plugin [module]]]]]]]]
274
+
275
+ where:
276
+
277
+- `type.id`
278
+
279
+ uniquely identifies the chart,
280
+ this is what will be needed to add values to the chart
281
+
282
+ the `type` part controls the menu the charts will appear in
283
+
284
+- `name`
285
+
286
+ is the name that will be presented to the user instead of `id` in `type.id`. This means that only the `id` part of
287
+ `type.id` is changed. When a name has been given, the chart is indexed (and can be referred) as both `type.id` and
288
+ `type.name`. You can set name to `''`, or `null`, or `(null)` to disable it. If a chart with the same name already
289
+ exists, a serial number is automatically attached to the name to avoid naming collisions.
290
+
291
+- `title`
292
+
293
+ the text above the chart
294
+
295
+- `units`
296
+
297
+ the label of the vertical axis of the chart,
298
+ all dimensions added to a chart should have the same units
299
+ of measurement
300
+
301
+- `family`
302
+
303
+ is used to group charts together
304
+ (for example all eth0 charts should say: eth0),
305
+ if empty or missing, the `id` part of `type.id` will be used
306
+
307
+ this controls the sub-menu on the dashboard
308
+
309
+- `context`
310
+
311
+ the context is giving the template of the chart. For example, if multiple charts present the same information for a different family, they should have the same `context`
312
+
313
+ this is used for looking up rendering information for the chart (colors, sizes, informational texts) and also apply alerts to it
314
+
315
+- `charttype`
316
+
317
+ one of `line`, `area`, `stacked` or `heatmap`,
318
+ if empty or missing, the `line` will be used
319
+
320
+- `priority`
321
+
322
+ is the relative priority of the charts as rendered on the web page,
323
+ lower numbers make the charts appear before the ones with higher numbers,
324
+ if empty or missing, `1000` will be used
325
+
326
+- `update_every`
327
+
328
+ overwrite the update frequency set by the server,
329
+ if empty or missing, the user configured value will be used
330
+
331
+- `options`
332
+
333
+ a space separated list of options, enclosed in quotes. The following options are currently supported: `obsolete` to mark a chart as obsolete (Netdata will hide it and delete it after some time), `store_first` to make Netdata store the first collected value, assuming there was an invisible previous value set to zero (this is used by statsd charts - if the first data collected value of incremental dimensions is not zero based, unrealistic spikes will appear with this option set) and `hidden` to perform all operations on a chart, but do not offer it on dashboards (the chart will be send to external databases). `CHART` options have been added in Netdata v1.7 and the `hidden` option was added in 1.10.
334
+
335
+- `plugin` and `module`
336
+
337
+ both are just names that are used to let the user identify the plugin and the module that generated the chart. If `plugin` is unset or empty, Netdata will automatically set the filename of the plugin that generated the chart. `module` has not default.
338
+
339
+#### DIMENSION
340
+
341
+`DIMENSION` defines a new dimension for the chart
342
+
343
+the template is:
344
+
345
+> DIMENSION id \[name \[algorithm \[multiplier \[divisor [options]]]]]
346
+
347
+ where:
348
+
349
+- `id`
350
+
351
+ the `id` of this dimension (it is a text value, not numeric),
352
+ this will be needed later to add values to the dimension
353
+
354
+ We suggest to avoid using `.` in dimension ids. External databases expect metrics to be `.` separated and people will get confused if a dimension id contains a dot.
355
+
356
+- `name`
357
+
358
+ the name of the dimension as it will appear at the legend of the chart,
359
+ if empty or missing the `id` will be used
360
+
361
+- `algorithm`
362
+
363
+ one of:
364
+
365
+ - `absolute`
366
+
367
+ the value is to drawn as-is (interpolated to second boundary),
368
+ if `algorithm` is empty, invalid or missing, `absolute` is used
369
+
370
+ - `incremental`
371
+
372
+ the value increases over time,
373
+ the difference from the last value is presented in the chart,
374
+ the server interpolates the value and calculates a per second figure
375
+
376
+ - `percentage-of-absolute-row`
377
+
378
+ the % of this value compared to the total of all dimensions
379
+
380
+ - `percentage-of-incremental-row`
381
+
382
+ the % of this value compared to the incremental total of
383
+ all dimensions
384
+
385
+- `multiplier`
386
+
387
+ an integer value to multiply the collected value,
388
+ if empty or missing, `1` is used
389
+
390
+- `divisor`
391
+
392
+ an integer value to divide the collected value,
393
+ if empty or missing, `1` is used
394
+
395
+- `options`
396
+
397
+ a space separated list of options, enclosed in quotes. Options supported: `obsolete` to mark a dimension as obsolete (Netdata will delete it after some time) and `hidden` to make this dimension hidden, it will take part in the calculations but will not be presented in the chart.
398
+
399
+#### VARIABLE
400
+
401
+> VARIABLE [SCOPE] name = value
402
+
403
+`VARIABLE` defines a variable that can be used in alerts. This is to used for setting constants (like the max connections a server may accept).
404
+
405
+Variables support 2 scopes:
406
+
407
+- `GLOBAL` or `HOST` to define the variable at the host level.
408
+- `LOCAL` or `CHART` to define the variable at the chart level. Use chart-local variables when the same variable may exist for different charts (i.e. Netdata monitors 2 mysql servers, and you need to set the `max_connections` each server accepts). Using chart-local variables is the ideal to build alert templates.
409
+
410
+The position of the `VARIABLE` line, sets its default scope (in case you do not specify a scope). So, defining a `VARIABLE` before any `CHART`, or between `END` and `BEGIN` (outside any chart), sets `GLOBAL` scope, while defining a `VARIABLE` just after a `CHART` or a `DIMENSION`, or within the `BEGIN` - `END` block of a chart, sets `LOCAL` scope.
411
+
412
+These variables can be set and updated at any point.
413
+
414
+Variable names should use alphanumeric characters, the `.` and the `_`.
415
+
416
+The `value` is floating point (Netdata used `long double`).
417
+
418
+Variables are transferred to upstream Netdata servers (streaming and database replication).
419
+
420
+#### CLABEL
421
+
422
+> CLABEL name value source
423
+
424
+`CLABEL` defines a label used to organize and identify a chart.
425
+
426
+Name and value accept characters according to the following table:
427
+
428
+| Character | Symbol | Label Name | Label Value |
429
+|---------------------|:------:|:----------:|:-----------:|
430
+| UTF-8 character | UTF-8 | _ | keep |
431
+| Lower case letter | [a-z] | keep | keep |
432
+| Upper case letter | [A-Z] | keep | [a-z] |
433
+| Digit | [0-9] | keep | keep |
434
+| Underscore | _ | keep | keep |
435
+| Minus | - | keep | keep |
436
+| Plus | + | _ | keep |
437
+| Colon | : | _ | keep |
438
+| Semicolon | ; | _ | : |
439
+| Equal | = | _ | : |
440
+| Period | . | keep | keep |
441
+| Comma | , | . | . |
442
+| Slash | / | keep | keep |
443
+| Backslash | \ | / | / |
444
+| At | @ | _ | keep |
445
+| Space | ' ' | _ | keep |
446
+| Opening parenthesis | ( | _ | keep |
447
+| Closing parenthesis | ) | _ | keep |
448
+| Anything else | | _ | _ |
449
+
450
+The `source` is an integer field that can have the following values:
451
+- `1`: The value was set automatically.
452
+- `2`: The value was set manually.
453
+- `4`: This is a K8 label.
454
+- `8`: This is a label defined using `netdata` Agent-Cloud link.
455
+
456
+#### CLABEL_COMMIT
457
+
458
+`CLABEL_COMMIT` indicates that all labels were defined and the chart can be updated.
459
+
460
+#### FUNCTION
461
+
462
+The plugin can register functions to Netdata, like this:
463
+
464
+> FUNCTION [GLOBAL] "name and parameters of the function" timeout "help string for users" "tags" "access" priority version
465
+
466
+- Tags currently recognized are either `top` or `logs` (or both, space separated).
467
+- Access is one of `any`, `member`, or `admin`:
468
+ - `any` to offer the function to all users of Netdata, even if they are not authenticated.
469
+ - `member` to offer the function to all authenticated members of Netdata.
470
+ - `admin` to offer the function only to authenticated administrators.
471
+- Priority defines the position of the function relative to the other functions (default is 100).
472
+- Version defines the version of the function (default is 0).
473
+
474
+Users can use a function to ask for more information from the collector. Netdata maintains a registry of functions in 2 levels:
475
+
476
+- per node
477
+- per chart
478
+
479
+Both node and chart functions are exactly the same, but chart functions allow Netdata to relate functions with charts and therefore present a context-sensitive menu of functions related to the chart the user is using.
480
+
481
+Users can get a list of all the registered functions using the `/api/v1/functions` endpoint of Netdata and call functions using the `/api/v1/function` API call of Netdata.
482
+
483
+Once a function is called, the plugin will receive at its standard input a command that looks like this:
484
+
485
+```
486
+FUNCTION transaction_id timeout "name and parameters of the function as one quoted parameter" "user permissions value" "source of request"
487
+```
488
+
489
+When the function to be called is to receive a payload of parameters, the call looks like this:
490
+
491
+```
492
+FUNCTION_PAYLOAD transaction_id timeout "name and parameters of the function as one quoted parameter" "user permissions value" "source of request" "content/type"
493
+body of the payload, formatted according to content/type
494
+FUNCTION PAYLOAD END
495
+```
496
+
497
+In this case, Netdata will send:
498
+
499
+- A line starting with `FUNCTION_PAYLOAD` together with the required metadata for the function, like the transaction id, the function name and its parameters, the timeout and the content type. This line ends with a newline.
500
+- Then, the payload itself (which may or may not have newlines in it). The payload should be parsed according to the content type parameter.
501
+- Finally, a line starting with `FUNCTION_PAYLOAD_END`, so it is expected like `\nFUNCTION_PAYLOAD_END\n`.
502
+
503
+Note 1: The plugins.d protocol allows parameters without single or double quotes if they don't contain spaces. However, the plugin should be able to parse parameters even if they are enclosed in single or double quotes. If the first character of a parameter is a single quote, its last character should also be a single quote too, and similarly for double quotes.
504
+
505
+Note 2: Netdata always sends the function and its parameters enclosed in double quotes. If the function command and its parameters contain quotes, they are converted to single quotes.
506
+
507
+The plugin is expected to parse and validate `name and parameters of the function as one quotes parameter`. Netdata allows the user interface to manipulate this string by appending more parameters.
508
+
509
+If the plugin rejects the request, it should respond with this:
510
+
511
+```
512
+FUNCTION_RESULT_BEGIN transaction_id 400 application/json
513
+{
514
+ "status": 400,
515
+ "error_message": "description of the rejection reasons"
516
+}
517
+FUNCTION_RESULT_END
518
+```
519
+
520
+If the plugin prepares a response, it should send (via its standard output, together with the collected data, but not interleaved with them):
521
+
522
+```
523
+FUNCTION_RESULT_BEGIN transaction_id http_response_code content_type expiration
524
+```
525
+
526
+Where:
527
+
528
+ - `transaction_id` is the transaction id that Netdata sent for this function execution
529
+ - `http_response_code` is the http error code Netdata should respond with, 200 is the "ok" response
530
+ - `content_type` is the content type of the response
531
+ - `expiration` is the absolute timestamp (number, unix epoch) this response expires
532
+
533
+Immediately after this, all text is assumed to be the response content.
534
+The content is text and line oriented. The maximum line length accepted is 15kb. Longer lines will be truncated.
535
+The type of the context itself depends on the plugin and the UI.
536
+
537
+To terminate the message, Netdata seeks a line with just this:
538
+
539
+```
540
+FUNCTION_RESULT_END
541
+```
542
+
543
+This defines the end of the message. `FUNCTION_RESULT_END` should appear in a line alone, without any other text, so it is wise to add `\n` before and after it.
544
+
545
+After this line, Netdata resumes processing collected metrics from the plugin.
546
+
547
+The maximum uncompressed payload size Netdata will accept is 100MB.
548
+
549
+##### Functions cancellation
550
+
551
+Netdata is able to detect when a user made an API request, but abandoned it before it was completed. If this happens to an API called for a function served by the plugin, Netdata will generate a `FUNCTION_CANCEL` request to let the plugin know that it can stop processing the query.
552
+
553
+After receiving such a command, the plugin **must still send a response for the original function request**, to wake up any waiting threads before they timeout. The http response code is not important, since the response will be discarded, however for auditing reasons we suggest to send back a 499 http response code. This is not a standard response code according to the HTTP protocol, but web servers like `nginx` are using it to indicate that a request was abandoned by a user.
554
+
555
+##### Functions progress
556
+
557
+When a request takes too long to be processed, Netdata allows the plugin to report progress to Netdata, which in turn will report progress to the caller.
558
+
559
+The plugin can send `FUNCTION_PROGRESS` like this:
560
+
561
+```
562
+FUNCTION_PROGRESS transaction_id done all
563
+```
564
+
565
+Where:
566
+
567
+- `transaction_id` is the transaction id of the function request
568
+- `done` is an integer value indicating the amount of work done
569
+- `all` is an integer value indicating the total amount of work to be done
570
+
571
+Netdata supports two kinds of progress:
572
+- progress as a percentage, which is calculated as `done * 100 / all`
573
+- progress without knowing the total amount of work to be done, which is enabled when the plugin reports `all` as zero.
574
+
575
+##### Functions timeout
576
+
577
+All functions calls specify a timeout, at which all the intermediate routing nodes (parents, web server threads) will time out and abort the call.
578
+
579
+However, all intermediate routing nodes are configured to extend the timeout when the caller asks for progress. This works like this:
580
+
581
+When a progress request is received, if the expected timeout of the request is less than or equal to 10 seconds, the expected timeout is extended by 10 seconds.
582
+
583
+Usually, the user interface asks for a progress every second. So, during the last 10 seconds of the timeout, every progress request made shifts the timeout 10 seconds to the future.
584
+
585
+To accomplish this, when Netdata receives a progress request by a user, it generates progress requests to the plugin, updating all the intermediate nodes to extend their timeout if necessary.
586
+
587
+The plugin will receive progress requests like this:
588
+
589
+```
590
+FUNCTION_PROGRESS transaction_id
591
+```
592
+
593
+There is no need to respond to this command. It is only there to let the plugin know that a user is still waiting for the query to finish.
594
+
595
+#### CONFIG
596
+
597
+`CONFIG` commands sent from the plugin to Netdata define dynamic configuration entities. These configurable entities are exposed to the user interface, allowing users to change configuration at runtime.
598
+
599
+Dynamically configurations made this way are saved to disk by Netdata and are replayed automatically when Netdata or the plugin restarts.
600
+
601
+`CONFIG` commands look like this:
602
+
603
+```
604
+CONFIG id action ...
605
+```
606
+
607
+Where:
608
+
609
+- `id` is a unique identifier for the configurable entity. This should by design be unique across Netdata. It should be something like `plugin:module:jobs`, e.g. `go.d:postgresql:jobs:masterdb`. This is assumed to be colon-separated with the last part (`masterdb` in our example), being the one displayed to users when there ano conflicts under the same configuration path.
610
+- `action` can be:
611
+ - `create`, to declare the dynamic configuration entity
612
+ - `delete`, to delete the dynamic configuration entity - this does not delete user configuration, we if an entity with the same id is created in the future, the saved configuration will be given to it.
613
+ - `status`, to update the dynamic configuration entity status
614
+
615
+> IMPORTANT:<br/>
616
+> The plugin should blindly create, delete and update the status of its dynamic configuration entities, without any special logic applied to it. Netdata needs to be updated of what is actually happening at the plugin. Keep in mind that creating dynamic configuration entities triggers responses from Netdata, depending on its type and status. Re-creating a job, triggers the same responses every time, so make sure you create jobs only when you add jobs.
617
+
618
+When the `action` is `create`, the following additional parameters are expected:
619
+
620
+```
621
+CONFIG id action status type "path" source_type "source" "supported commands" "view permissions" "edit permissions"
622
+```
623
+
624
+Where:
625
+
626
+- `action` should be `create`
627
+- `status` can be:
628
+ - `accepted`, the plugin accepted the configuration, but it is not running yet.
629
+ - `running`, the plugin accepted and runs the configuration.
630
+ - `failed`, the plugin tries to run the configuration but it fails.
631
+ - `incomplete`, the plugin needs additional settings to run this configuration. This is usually used for the cases the plugin discovered a job, but important information is missing for it to work.
632
+ - `disabled`, the configuration has been disabled by a user.
633
+ - `orphan`, the configuration is not claimed by any plugin. This is used internally by Netdata to mark the configuration nodes available, for which there is no plugin related to them. Do not use in plugins directly.
634
+- `type` can be `single`, `template` or `job`:
635
+ - `single` is used when the configurable entity is fixed and users should never be able to add or delete it.
636
+ - `template` is used to define a template based on which users can add multiple configurations, like adding data collection jobs. So, the plugin defines the template of the jobs and users are presented with a `[+]` button to add such configuration jobs. The plugin can define multiple templates by giving different `id`s to them.
637
+ - `job` is used to define a job of a template. The plugin should always add all its jobs, independently of the way they have been discovered. It is important to note the relation between `template` and `job` when it comes it the `id`: The `id` of the template should be the prefix of the `job`'s `id`. For example, if the template is `go.d:postgresql:jobs`, then all its jobs be like `go.d:postgresql:jobs:jobname`.
638
+- `path` is the absolute path of the configurable entity inside the tree of Netdata configurations. Usually, this is should be `/collectors`.
639
+- `source` can be `internal`, `stock`, `user`, `discovered` or `dyncfg`:
640
+ - `internal` is used for configurations that are based on internal code settings
641
+ - `stock` is used for default configurations
642
+ - `discovered` is used for dynamic configurations the plugin discovers by its own
643
+ - `user` is used for user configurations, usually via a configuration file
644
+ - `dyncfg` is used for configuration received via this dynamic configuration mechanism
645
+- `source` should provide more details about the exact source of the configuration, like `line@file`, or `user@ip`, etc.
646
+- `supported_commands` is a space separated list of the following keywords, enclosed in single or double quotes. These commands are used by the user interface to determine the actions the users can take:
647
+ - `schema`, to expose the JSON schema for the user interface. This is mandatory for all configurable entities. When `schema` requests are received, Netdata will first attempt to load the schema from `/etc/netdata/schema.d/` and `/var/lib/netdata/conf.d/schema.d`. For jobs, it will serve the schema of their template. If no schema is found for the required `id`, the `schema` request will be forwarded to the plugin, which is expected to send back the relevant schema.
648
+ - `get`, to expose the current configuration values, according the schema defined. `templates` cannot support `get`, since they don't maintain any data.
649
+ - `update`, to receive configuration updates for this entity. `templates` cannot support `update`, since they don't maintain any data.
650
+ - `test`, like `update` but only test the configuration and report success or failure.
651
+ - `add`, to receive job creation commands for templates. Only `templates` should support this command.
652
+ - `remove`, to remove a configuration. Only `jobs` should support this command.
653
+ - `enable` and `disable`, to receive user requests to enable and disable this entity. Adding only one of `enable` or `disable` to the supported commands, Netdata will add both of them. The plugin should expose these commands on `templates` only when it wants to receive `enable` and `disable` commands for all the `jobs` of this `template`.
654
+ - `restart`, to restart a job.
655
+- `view permissions` and `edit permissions` are bitmaps of the Netdata permission system to control access to the configuration. If set to zero, Netdata will require a signed in user with view and edit permissions to the Netdata's configuration system.
656
+
657
+The plugin receives commands as if it had exposed a `FUNCTION` named `config`. Netdata formats all these calls like this:
658
+
659
+```
660
+config id command
661
+```
662
+
663
+Where `id` is the unique id of the configurable entity and `command` is one of the supported commands the plugin sent to Netdata.
664
+
665
+The plugin will receive (for commands: `schema`, `get`, `remove`, `enable`, `disable` and `restart`):
666
+
667
+```
668
+FUNCTION transaction_id timeout "config id command" "user permissions value" "source string"
669
+```
670
+
671
+or (for commands: `update`, `add` and `test`):
672
+
673
+```
674
+FUNCTION_PAYLOAD transaction_id timeout "config id command" "user permissions value" "source string" "content/type"
675
+body of the payload formatted according to content/type
676
+FUNCTION_PAYLOAD_END
677
+```
678
+
679
+Once received, the plugin should process it and respond accordingly.
680
+
681
+Immediately after the plugin adds a configuration entity, if the commands `enable` and `disable` are supported by it, Netdata will send either `enable` or `disable` for it, based on the last user action, which has been persisted to disk.
682
+
683
+Plugin responses follow the same format `FUNCTIONS` do:
684
+
685
+```
686
+FUNCTION_RESULT_BEGIN transaction_id http_response_code content/type expiration
687
+body of the response formatted according to content/type
688
+FUNCTION_RESULT_END
689
+```
690
+
691
+Successful responses (HTTP response code 200) to `schema` and `get` should send back the relevant JSON object.
692
+All other responses should have the following response body:
693
+
694
+```json
695
+{
696
+ "status" : 404,
697
+ "message" : "some text"
698
+}
699
+```
700
+
701
+The user interface presents the message to users, even when the response is successful (HTTP code 200).
702
+
703
+When responding to additions and updates, Netdata uses the following success response codes to derive additional information:
704
+
705
+- `200`, responding with 200, means the configuration has been accepted and it is running.
706
+- `202`, responding with 202, means the configuration has been accepted but it is not yet running. A subsequent `status` action will update it.
707
+- `298`, responding with 298, means the configuration has been accepted but it is disabled for some reason (probably because it matches nothing or the contents are not useful - use the `message` to provide additional information).
708
+- `299`, responding with 299, means the configuration has been accepted but a restart is required to apply it.
709
+
710
+## Data collection
711
+
712
+data collection is defined as a series of `BEGIN` -> `SET` -> `END` lines
713
+
714
+> BEGIN type.id [microseconds]
715
+
716
+- `type.id`
717
+
718
+ is the unique identification of the chart (as given in `CHART`)
719
+
720
+- `microseconds`
721
+
722
+ is the number of microseconds since the last update of the chart. It is optional.
723
+
724
+ Under heavy system load, the system may have some latency transferring
725
+ data from the plugins to Netdata via the pipe. This number improves
726
+ accuracy significantly, since the plugin is able to calculate the
727
+ duration between its iterations better than Netdata.
728
+
729
+ The first time the plugin is started, no microseconds should be given
730
+ to Netdata.
731
+
732
+> SET id = value
733
+
734
+- `id`
735
+
736
+ is the unique identification of the dimension (of the chart just began)
737
+
738
+- `value`
739
+
740
+ is the collected value, only integer values are collected. If you want to push fractional values, multiply this value by 100 or 1000 and set the `DIMENSION` divider to 1000.
741
+
742
+> END
743
+
744
+ END does not take any parameters, it commits the collected values for all dimensions to the chart. If a dimensions was not `SET`, its value will be empty for this commit.
745
+
746
+More `SET` lines may appear to update all the dimensions of the chart.
747
+All of them in one `BEGIN` -> `END` block.
748
+
749
+All `SET` lines within a single `BEGIN` -> `END` block have to refer to the
750
+same chart.
751
+
752
+If more charts need to be updated, each chart should have its own
753
+`BEGIN` -> `SET` -> `END` block.
754
+
755
+If, for any reason, a plugin has issued a `BEGIN` but wants to cancel it,
756
+it can issue a `FLUSH`. The `FLUSH` command will instruct Netdata to ignore
757
+all the values collected since the last `BEGIN` command.
758
+
759
+If a plugin does not behave properly (outputs invalid lines, or does not
760
+follow these guidelines), will be disabled by Netdata.
761
+
762
+### collected values
763
+
764
+Netdata will collect any **signed** value in the 64bit range:
765
+`-9.223.372.036.854.775.808` to `+9.223.372.036.854.775.807`
766
+
767
+If a value is not collected, leave it empty, like this:
768
+
769
+`SET id =`
770
+
771
+or do not output the line at all.
772
+
773
+## Modular Plugins
774
+
775
+1. **python**, use `python.d.plugin`, there are many examples in the [python.d
776
+ directory](/src/collectors/python.d.plugin/README.md)
777
+
778
+ python is ideal for Netdata plugins. It is a simple, yet powerful way to collect data, it has a very small memory footprint, although it is not the most CPU efficient way to do it.
779
+
780
+2. **BASH**, use `charts.d.plugin`, there are many examples in the [charts.d
781
+ directory](/src/collectors/charts.d.plugin/README.md)
782
+
783
+ BASH is the simplest scripting language for collecting values. It is the less efficient though in terms of CPU resources. You can use it to collect data quickly, but extensive use of it might use a lot of system resources.
784
+
785
+3. **C**
786
+
787
+ Of course, C is the most efficient way of collecting data. This is why Netdata itself is written in C.
788
+
789
+## Writing Plugins Properly
790
+
791
+There are a few rules for writing plugins properly:
792
+
793
+1. Respect system resources
794
+
795
+ Pay special attention to efficiency:
796
+
797
+ - Initialize everything once, at the beginning. Initialization is not an expensive operation. Your plugin will most probably be started once and run forever. So, do whatever heavy operation is needed at the beginning, just once.
798
+ - Do the absolutely minimum while iterating to collect values repeatedly.
799
+ - If you need to connect to another server to collect values, avoid re-connects if possible. Connect just once, with keep-alive (for HTTP) enabled and collect values using the same connection.
800
+ - Avoid any CPU or memory heavy operation while collecting data. If you control memory allocation, avoid any memory allocation while iterating to collect values.
801
+ - Avoid running external commands when possible. If you are writing shell scripts avoid especially pipes (each pipe is another fork, a very expensive operation).
802
+
803
+2. The best way to iterate at a constant pace is this pseudo code:
804
+
805
+```js
806
+ var update_every = argv[1] * 1000; /* seconds * 1000 = milliseconds */
807
+
808
+ readConfiguration();
809
+
810
+ if(!verifyWeCanCollectValues()) {
811
+ print("DISABLE");
812
+ exit(1);
813
+ }
814
+
815
+ createCharts(); /* print CHART and DIMENSION statements */
816
+
817
+ var loops = 0;
818
+ var last_run = 0;
819
+ var next_run = 0;
820
+ var dt_since_last_run = 0;
821
+ var now = 0;
822
+
823
+ while(true) {
824
+ /* find the current time in milliseconds */
825
+ now = currentTimeStampInMilliseconds();
826
+
827
+ /*
828
+ * find the time of the next loop
829
+ * this makes sure we are always aligned
830
+ * with the Netdata daemon
831
+ */
832
+ next_run = now - (now % update_every) + update_every;
833
+
834
+ /*
835
+ * wait until it is time
836
+ * it is important to do it in a loop
837
+ * since many wait functions can be interrupted
838
+ */
839
+ while( now < next_run ) {
840
+ sleepMilliseconds(next_run - now);
841
+ now = currentTimeStampInMilliseconds();
842
+ }
843
+
844
+ /* calculate the time passed since the last run */
845
+ if ( loops > 0 )
846
+ dt_since_last_run = (now - last_run) * 1000; /* in microseconds */
847
+
848
+ /* prepare for the next loop */
849
+ last_run = now;
850
+ loops++;
851
+
852
+ /* do your magic here to collect values */
853
+ collectValues();
854
+
855
+ /* send the collected data to Netdata */
856
+ printValues(dt_since_last_run); /* print BEGIN, SET, END statements */
857
+ }
858
+```
859
+
860
+ Using the above procedure, your plugin will be synchronized to start data collection on steps of `update_every`. There will be no need to keep track of latencies in data collection.
861
+
862
+ Netdata interpolates values to second boundaries, so even if your plugin is not perfectly aligned it does not matter. Netdata will find out. When your plugin works in increments of `update_every`, there will be no gaps in the charts due to the possible cumulative micro-delays in data collection. Gaps will only appear if the data collection is really delayed.
863
+
864
+3. If you are not sure of memory leaks, exit every one hour. Netdata will re-start your process.
865
+
866
+4. If possible, try to autodetect if your plugin should be enabled, without any configuration.
867
+
868
+
src/crates/netdata-plugin/error/Cargo.toml
new
+11
@@ -0,0 +1,11 @@
1
+[package]
2
+name = "netdata-plugin-error"
3
+version.workspace = true
4
+edition.workspace = true
5
+description = "Common error types for Netdata plugin crates"
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dependencies]
11
+thiserror = { workspace = true }
\ No newline at end of file
src/crates/netdata-plugin/error/src/lib.rs
new
+40
@@ -0,0 +1,40 @@
1
+use thiserror::Error;
2
+
3
+/// Result type for netdata plugin operations
4
+pub type Result<T> = std::result::Result<T, NetdataPluginError>;
5
+
6
+/// Error types that can occur in netdata plugin operations
7
+#[derive(Error, Debug)]
8
+pub enum NetdataPluginError {
9
+ /// Transport layer error (I/O, network)
10
+ #[error("transport error: {0}")]
11
+ Transport(#[from] std::io::Error),
12
+
13
+ /// Protocol parsing or communication error
14
+ #[error("protocol error: {message}")]
15
+ Protocol { message: String },
16
+
17
+ /// Runtime error during plugin execution
18
+ #[error("runtime error: {message}")]
19
+ Runtime { message: String },
20
+
21
+ /// Configuration error
22
+ #[error("configuration error: {message}")]
23
+ Config { message: String },
24
+
25
+ /// Function handler error
26
+ #[error("function handler error: {message}")]
27
+ FunctionHandler { message: String },
28
+
29
+ /// Schema validation error
30
+ #[error("schema validation error: {message}")]
31
+ Schema { message: String },
32
+
33
+ /// Transport is closed
34
+ #[error("transport is closed")]
35
+ Closed,
36
+
37
+ /// Generic error with custom message
38
+ #[error("{message}")]
39
+ Other { message: String },
40
+}
\ No newline at end of file
src/crates/netdata-plugin/foundation/Cargo.toml
new
+11
@@ -0,0 +1,11 @@
1
+[package]
2
+name = "foundation"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dev-dependencies]
11
+tokio = { workspace = true, features = ["rt", "time", "macros"] }
src/crates/netdata-plugin/foundation/src/lib.rs
new
+8
@@ -0,0 +1,8 @@
1
+//! Foundational utilities for Netdata plugins.
2
+//!
3
+//! This crate provides low-level primitives and utilities that other plugin
4
+//! crates build upon, including async operation management and control flow.
5
+
6
+// Timeout management
7
+pub mod timeout;
8
+pub use timeout::Timeout;
src/crates/netdata-plugin/foundation/src/timeout.rs
new
+161
@@ -0,0 +1,161 @@
1
+//! Timeout management for async operations with thread-safe deadline modification.
2
+
3
+use std::sync::Arc;
4
+use std::sync::atomic::{AtomicU64, Ordering};
5
+use std::time::{Duration, Instant};
6
+
7
+/// A thread-safe timeout that can be checked and extended from multiple threads.
8
+///
9
+/// This is useful for operations that need:
10
+/// - Dynamic timeout extension (e.g., on progress updates)
11
+/// - Parallel checks across multiple async tasks
12
+/// - Future support for cancellation signals
13
+///
14
+/// Extensions are bounded: the remaining time will never exceed the initial budget.
15
+#[derive(Debug, Clone)]
16
+pub struct Timeout {
17
+ start: Instant,
18
+ budget_us: u64,
19
+ deadline_us: Arc<AtomicU64>,
20
+}
21
+
22
+impl Timeout {
23
+ /// Create a new timeout with the given budget.
24
+ pub fn new(budget: Duration) -> Self {
25
+ let start = Instant::now();
26
+ let budget_us = budget.as_micros() as u64;
27
+
28
+ Self {
29
+ start,
30
+ budget_us,
31
+ deadline_us: Arc::new(AtomicU64::new(budget_us)),
32
+ }
33
+ }
34
+
35
+ /// Check if the timeout has expired.
36
+ pub fn is_expired(&self) -> bool {
37
+ self.remaining().is_zero()
38
+ }
39
+
40
+ /// Get remaining time. Returns Duration::ZERO if expired.
41
+ pub fn remaining(&self) -> Duration {
42
+ let deadline_us = self.deadline_us.load(Ordering::Relaxed);
43
+ let elapsed_us = self.start.elapsed().as_micros() as u64;
44
+
45
+ if elapsed_us >= deadline_us {
46
+ Duration::ZERO
47
+ } else {
48
+ Duration::from_micros(deadline_us - elapsed_us)
49
+ }
50
+ }
51
+
52
+ /// Reset the timeout to the initial budget from the current time.
53
+ ///
54
+ /// This is useful for operations that report progress and should
55
+ /// get their full timeout budget again.
56
+ ///
57
+ /// For example, if the initial timeout was 10 seconds and we're at t=5s with 5s remaining,
58
+ /// calling reset() will give the operation another 10s (deadline becomes t=15s).
59
+ pub fn reset(&self) {
60
+ let elapsed_us = self.start.elapsed().as_micros() as u64;
61
+ let deadline_us = elapsed_us + self.budget_us;
62
+ self.deadline_us.store(deadline_us, Ordering::Relaxed);
63
+ }
64
+}
65
+
66
+#[cfg(test)]
67
+mod tests {
68
+ use super::*;
69
+ use std::thread;
70
+
71
+ #[test]
72
+ fn test_timeout_not_expired() {
73
+ let timeout = Timeout::new(Duration::from_secs(10));
74
+ assert!(!timeout.is_expired());
75
+ assert!(timeout.remaining() > Duration::ZERO);
76
+ }
77
+
78
+ #[test]
79
+ fn test_timeout_expired() {
80
+ let timeout = Timeout::new(Duration::from_micros(1));
81
+ thread::sleep(Duration::from_millis(10));
82
+ assert!(timeout.is_expired());
83
+ assert_eq!(timeout.remaining(), Duration::ZERO);
84
+ }
85
+
86
+ #[test]
87
+ fn test_timeout_reset() {
88
+ let timeout = Timeout::new(Duration::from_millis(100));
89
+ thread::sleep(Duration::from_millis(60));
90
+
91
+ // Should have ~40ms remaining
92
+ let remaining_before = timeout.remaining();
93
+ assert!(remaining_before < Duration::from_millis(50));
94
+
95
+ // Reset the timeout
96
+ timeout.reset();
97
+
98
+ // Should now have the full initial budget (~100ms) remaining
99
+ let remaining_after = timeout.remaining();
100
+ assert!(remaining_after >= Duration::from_millis(90));
101
+ assert!(remaining_after <= Duration::from_millis(100));
102
+ }
103
+
104
+ #[test]
105
+ fn test_timeout_clone_shared_deadline() {
106
+ let timeout1 = Timeout::new(Duration::from_secs(10));
107
+ let timeout2 = timeout1.clone();
108
+
109
+ thread::sleep(Duration::from_millis(100));
110
+
111
+ // Reset from one clone
112
+ timeout1.reset();
113
+
114
+ // Both should see the reset
115
+ let remaining1 = timeout1.remaining();
116
+ let remaining2 = timeout2.remaining();
117
+
118
+ assert!((remaining1.as_secs() as i64 - remaining2.as_secs() as i64).abs() < 1);
119
+ assert!(remaining1.as_secs() >= 9); // ~10 seconds (reset to initial budget)
120
+ }
121
+
122
+ #[tokio::test]
123
+ async fn test_tokio_timeout_with_zero_duration() {
124
+ use tokio::time::timeout;
125
+
126
+ // Create an expired timeout
127
+ let expired_timeout = Timeout::new(Duration::from_micros(1));
128
+ thread::sleep(Duration::from_millis(10));
129
+ assert_eq!(expired_timeout.remaining(), Duration::ZERO);
130
+
131
+ // Verify tokio::time::timeout with ZERO duration times out immediately
132
+ let result = timeout(expired_timeout.remaining(), async {
133
+ tokio::time::sleep(Duration::from_millis(100)).await;
134
+ "should_not_complete"
135
+ })
136
+ .await;
137
+
138
+ // Should timeout immediately
139
+ assert!(result.is_err(), "Expected timeout with Duration::ZERO");
140
+ }
141
+
142
+ #[tokio::test]
143
+ async fn test_tokio_timeout_with_remaining_time() {
144
+ use tokio::time::timeout;
145
+
146
+ // Create a timeout with plenty of time
147
+ let valid_timeout = Timeout::new(Duration::from_secs(10));
148
+ assert!(valid_timeout.remaining() > Duration::ZERO);
149
+
150
+ // Verify tokio::time::timeout with remaining time completes
151
+ let result = timeout(valid_timeout.remaining(), async {
152
+ tokio::time::sleep(Duration::from_millis(10)).await;
153
+ "completed"
154
+ })
155
+ .await;
156
+
157
+ // Should complete successfully
158
+ assert!(result.is_ok(), "Expected completion with sufficient time");
159
+ assert_eq!(result.unwrap(), "completed");
160
+ }
161
+}
src/crates/netdata-plugin/protocol/Cargo.toml
new
+28
@@ -0,0 +1,28 @@
1
+[package]
2
+name = "netdata-plugin-protocol"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[lints]
7
+workspace = true
8
+
9
+[dependencies]
10
+netdata-plugin-error = { path = "../error" }
11
+netdata-plugin-types = { path = "../types" }
12
+
13
+phf = { workspace = true }
14
+atoi = { workspace = true }
15
+tokio-util = { workspace = true }
16
+bytes = { workspace = true }
17
+futures = { workspace = true }
18
+tokio = { workspace = true, features = ["io-std", "io-util"] }
19
+serde = { workspace = true }
20
+serde_json = { workspace = true }
21
+tracing = { workspace = true }
22
+
23
+[build-dependencies]
24
+phf_codegen = { workspace = true }
25
+
26
+[dev-dependencies]
27
+tokio = { version = "1.0", features = ["full"] }
28
+futures-util = { version = "0.3", features = ["sink"] }
src/crates/netdata-plugin/protocol/build.rs
new
+66
@@ -0,0 +1,66 @@
1
+use std::env;
2
+use std::fs::File;
3
+use std::io::{BufWriter, Write};
4
+use std::path::Path;
5
+
6
+fn main() {
7
+ let path = Path::new(&env::var("OUT_DIR").unwrap()).join("tokens.rs");
8
+ let mut file = BufWriter::new(File::create(&path).unwrap());
9
+
10
+ writeln!(
11
+ &mut file,
12
+ "static COMMAND_MAP: phf::Map<&'static [u8], Token> = {};",
13
+ phf_codegen::Map::<&[u8]>::new()
14
+ .entry(b"CHART", "Token::Chart")
15
+ .entry(b"CHART_DEFINITION_END", "Token::ChartDefinitionEnd")
16
+ .entry(b"DIMENSION", "Token::Dimension")
17
+ .entry(b"BEGIN", "Token::Begin")
18
+ .entry(b"END", "Token::End")
19
+ .entry(b"SET", "Token::Set")
20
+ .entry(b"FLUSH", "Token::Flush")
21
+ .entry(b"DISABLE", "Token::Disable")
22
+ .entry(b"VARIABLE", "Token::Variable")
23
+ .entry(b"LABEL", "Token::Label")
24
+ .entry(b"OVERWRITE", "Token::Overwrite")
25
+ .entry(b"CLABEL", "Token::Clabel")
26
+ .entry(b"CLABEL_COMMIT", "Token::ClabelCommit")
27
+ .entry(b"EXIT", "Token::Exit")
28
+ .entry(b"BEGIN2", "Token::Begin2")
29
+ .entry(b"SET2", "Token::Set2")
30
+ .entry(b"END2", "Token::End2")
31
+ .entry(b"HOST_DEFINE", "Token::HostDefine")
32
+ .entry(b"HOST_DEFINE_END", "Token::HostDefineEnd")
33
+ .entry(b"HOST_LABEL", "Token::HostLabel")
34
+ .entry(b"HOST", "Token::Host")
35
+ .entry(b"REPLAY_CHART", "Token::ReplayChart")
36
+ .entry(b"RBEGIN", "Token::Rbegin")
37
+ .entry(b"RSET", "Token::Rset")
38
+ .entry(b"RDSTATE", "Token::RdState")
39
+ .entry(b"RSSTATE", "Token::RsState")
40
+ .entry(b"REND", "Token::Rend")
41
+ .entry(b"FUNCTION", "Token::Function")
42
+ .entry(b"FUNCTION_RESULT_BEGIN", "Token::FunctionResultBegin")
43
+ .entry(b"FUNCTION_RESULT_END", "Token::FunctionResultEnd")
44
+ .entry(b"FUNCTION_PAYLOAD", "Token::FunctionPayloadBegin")
45
+ .entry(b"FUNCTION_PAYLOAD_END", "Token::FunctionPayloadEnd")
46
+ .entry(b"FUNCTION_CANCEL", "Token::FunctionCancel")
47
+ .entry(b"FUNCTION_PROGRESS", "Token::FunctionProgress")
48
+ .entry(b"QUIT", "Token::Quit")
49
+ .entry(b"CONFIG", "Token::Config")
50
+ .entry(b"NODE_ID", "Token::NodeId")
51
+ .entry(b"CLAIMED_ID", "Token::ClaimedId")
52
+ .entry(b"JSON", "Token::Json")
53
+ .entry(b"JSON_PAYLOAD_END", "Token::JsonPayloadEnd")
54
+ .entry(b"STREAM_PATH", "Token::StreamPath")
55
+ .entry(b"ML_MODEL", "Token::MlModel")
56
+ .entry(b"TRUST_DURATIONS", "Token::TrustDurations")
57
+ .entry(b"DYNCFG_ENABLE", "Token::DynCfg")
58
+ .entry(b"DYNCFG_REGISTER_MODULE", "Token::DynCfgRegisterModule")
59
+ .entry(b"DYNCFG_REGISTER_JOB", "Token::DynCfgRegisterJob")
60
+ .entry(b"DYNCFG_RESET", "Token::DynCfgReset")
61
+ .entry(b"REPORT_JOB_STATUS", "Token::ReportJobStatus")
62
+ .entry(b"DELETE_JOB", "Token::DeleteJob")
63
+ .build()
64
+ )
65
+ .unwrap();
66
+}
src/crates/netdata-plugin/protocol/examples/config_declaration_encode.rs
new
+37
@@ -0,0 +1,37 @@
1
+use netdata_plugin_protocol::{
2
+ ConfigDeclaration, DynCfgCmds, DynCfgSourceType, DynCfgStatus, DynCfgType, HttpAccess,
3
+};
4
+use netdata_plugin_protocol::{Message, MessageWriter};
5
+
6
+#[tokio::main]
7
+async fn main() {
8
+ // Create a sample ConfigDeclaration
9
+ let config_declaration = ConfigDeclaration {
10
+ id: "go.d:nginx".to_string(),
11
+ status: DynCfgStatus::Accepted,
12
+ type_: DynCfgType::Template,
13
+ path: "/collectors".to_string(),
14
+ source_type: DynCfgSourceType::Internal,
15
+ source: "whatever internal source".to_string(),
16
+ cmds: DynCfgCmds::SCHEMA | DynCfgCmds::ADD | DynCfgCmds::ENABLE | DynCfgCmds::DISABLE,
17
+ view_access: HttpAccess::empty(),
18
+ edit_access: HttpAccess::empty(),
19
+ };
20
+
21
+ // Create message from the config declaration
22
+ let message = Message::ConfigDeclaration(Box::new(config_declaration));
23
+
24
+ // Create message writer for stdout
25
+ let stdout = tokio::io::stdout();
26
+ let mut writer = MessageWriter::new(stdout);
27
+
28
+ // Send the message to stdout
29
+ match writer.send(message).await {
30
+ Ok(()) => {
31
+ // Message sent successfully
32
+ }
33
+ Err(e) => {
34
+ eprintln!("Error sending message: {}", e);
35
+ }
36
+ }
37
+}
src/crates/netdata-plugin/protocol/src/http_content.rs
new
+366
@@ -0,0 +1,366 @@
1
+#![allow(dead_code)]
2
+
3
+use std::fmt;
4
+
5
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
6
+pub enum HttpContent {
7
+ ApplicationJson,
8
+ TextPlain,
9
+ TextHtml,
10
+ TextCss,
11
+ TextYaml,
12
+ ApplicationYaml,
13
+ TextXml,
14
+ TextXsl,
15
+ ApplicationXml,
16
+ ApplicationJavascript,
17
+ ApplicationOctetStream,
18
+ ImageSvgXml,
19
+ ApplicationXFontTruetype,
20
+ ApplicationXFontOpentype,
21
+ ApplicationFontWoff,
22
+ ApplicationFontWoff2,
23
+ ApplicationVndMsFontobject,
24
+ ImagePng,
25
+ ImageJpeg,
26
+ ImageGif,
27
+ ImageXIcon,
28
+ ImageBmp,
29
+ ImageIcns,
30
+ AudioMpeg,
31
+ AudioOgg,
32
+ VideoMp4,
33
+ ApplicationPdf,
34
+ ApplicationZip,
35
+ Prometheus,
36
+}
37
+
38
+#[derive(Debug, Clone)]
39
+pub struct HttpContentInfo {
40
+ pub format: &'static str,
41
+ pub content_type: HttpContent,
42
+ pub needs_charset: bool,
43
+ pub options: Option<&'static str>,
44
+}
45
+
46
+impl HttpContent {
47
+ /// Get the primary format string for this content type
48
+ pub fn as_str(&self) -> &'static str {
49
+ self.info().format
50
+ }
51
+
52
+ /// Get full information about this content type
53
+ pub fn info(&self) -> HttpContentInfo {
54
+ use HttpContent::*;
55
+ match self {
56
+ ApplicationJson => HttpContentInfo {
57
+ format: "application/json",
58
+ content_type: ApplicationJson,
59
+ needs_charset: true,
60
+ options: None,
61
+ },
62
+ TextPlain => HttpContentInfo {
63
+ format: "text/plain",
64
+ content_type: TextPlain,
65
+ needs_charset: true,
66
+ options: None,
67
+ },
68
+ TextHtml => HttpContentInfo {
69
+ format: "text/html",
70
+ content_type: TextHtml,
71
+ needs_charset: true,
72
+ options: None,
73
+ },
74
+ TextCss => HttpContentInfo {
75
+ format: "text/css",
76
+ content_type: TextCss,
77
+ needs_charset: true,
78
+ options: None,
79
+ },
80
+ TextYaml => HttpContentInfo {
81
+ format: "text/yaml",
82
+ content_type: TextYaml,
83
+ needs_charset: true,
84
+ options: None,
85
+ },
86
+ ApplicationYaml => HttpContentInfo {
87
+ format: "application/yaml",
88
+ content_type: ApplicationYaml,
89
+ needs_charset: true,
90
+ options: None,
91
+ },
92
+ TextXml => HttpContentInfo {
93
+ format: "text/xml",
94
+ content_type: TextXml,
95
+ needs_charset: true,
96
+ options: None,
97
+ },
98
+ TextXsl => HttpContentInfo {
99
+ format: "text/xsl",
100
+ content_type: TextXsl,
101
+ needs_charset: true,
102
+ options: None,
103
+ },
104
+ ApplicationXml => HttpContentInfo {
105
+ format: "application/xml",
106
+ content_type: ApplicationXml,
107
+ needs_charset: true,
108
+ options: None,
109
+ },
110
+ ApplicationJavascript => HttpContentInfo {
111
+ format: "application/javascript",
112
+ content_type: ApplicationJavascript,
113
+ needs_charset: true,
114
+ options: None,
115
+ },
116
+ ApplicationOctetStream => HttpContentInfo {
117
+ format: "application/octet-stream",
118
+ content_type: ApplicationOctetStream,
119
+ needs_charset: false,
120
+ options: None,
121
+ },
122
+ ImageSvgXml => HttpContentInfo {
123
+ format: "image/svg+xml",
124
+ content_type: ImageSvgXml,
125
+ needs_charset: false,
126
+ options: None,
127
+ },
128
+ ApplicationXFontTruetype => HttpContentInfo {
129
+ format: "application/x-font-truetype",
130
+ content_type: ApplicationXFontTruetype,
131
+ needs_charset: false,
132
+ options: None,
133
+ },
134
+ ApplicationXFontOpentype => HttpContentInfo {
135
+ format: "application/x-font-opentype",
136
+ content_type: ApplicationXFontOpentype,
137
+ needs_charset: false,
138
+ options: None,
139
+ },
140
+ ApplicationFontWoff => HttpContentInfo {
141
+ format: "application/font-woff",
142
+ content_type: ApplicationFontWoff,
143
+ needs_charset: false,
144
+ options: None,
145
+ },
146
+ ApplicationFontWoff2 => HttpContentInfo {
147
+ format: "application/font-woff2",
148
+ content_type: ApplicationFontWoff2,
149
+ needs_charset: false,
150
+ options: None,
151
+ },
152
+ ApplicationVndMsFontobject => HttpContentInfo {
153
+ format: "application/vnd.ms-fontobject",
154
+ content_type: ApplicationVndMsFontobject,
155
+ needs_charset: false,
156
+ options: None,
157
+ },
158
+ ImagePng => HttpContentInfo {
159
+ format: "image/png",
160
+ content_type: ImagePng,
161
+ needs_charset: false,
162
+ options: None,
163
+ },
164
+ ImageJpeg => HttpContentInfo {
165
+ format: "image/jpeg",
166
+ content_type: ImageJpeg,
167
+ needs_charset: false,
168
+ options: None,
169
+ },
170
+ ImageGif => HttpContentInfo {
171
+ format: "image/gif",
172
+ content_type: ImageGif,
173
+ needs_charset: false,
174
+ options: None,
175
+ },
176
+ ImageXIcon => HttpContentInfo {
177
+ format: "image/x-icon",
178
+ content_type: ImageXIcon,
179
+ needs_charset: false,
180
+ options: None,
181
+ },
182
+ ImageBmp => HttpContentInfo {
183
+ format: "image/bmp",
184
+ content_type: ImageBmp,
185
+ needs_charset: false,
186
+ options: None,
187
+ },
188
+ ImageIcns => HttpContentInfo {
189
+ format: "image/icns",
190
+ content_type: ImageIcns,
191
+ needs_charset: false,
192
+ options: None,
193
+ },
194
+ AudioMpeg => HttpContentInfo {
195
+ format: "audio/mpeg",
196
+ content_type: AudioMpeg,
197
+ needs_charset: false,
198
+ options: None,
199
+ },
200
+ AudioOgg => HttpContentInfo {
201
+ format: "audio/ogg",
202
+ content_type: AudioOgg,
203
+ needs_charset: false,
204
+ options: None,
205
+ },
206
+ VideoMp4 => HttpContentInfo {
207
+ format: "video/mp4",
208
+ content_type: VideoMp4,
209
+ needs_charset: false,
210
+ options: None,
211
+ },
212
+ ApplicationPdf => HttpContentInfo {
213
+ format: "application/pdf",
214
+ content_type: ApplicationPdf,
215
+ needs_charset: false,
216
+ options: None,
217
+ },
218
+ ApplicationZip => HttpContentInfo {
219
+ format: "application/zip",
220
+ content_type: ApplicationZip,
221
+ needs_charset: false,
222
+ options: None,
223
+ },
224
+ Prometheus => HttpContentInfo {
225
+ format: "text/plain",
226
+ content_type: Prometheus,
227
+ needs_charset: true,
228
+ options: Some("version=0.0.4"),
229
+ },
230
+ }
231
+ }
232
+
233
+ /// Parse a content type from a string
234
+ #[allow(clippy::should_implement_trait)]
235
+ pub fn from_str(format: &str) -> Option<Self> {
236
+ use HttpContent::*;
237
+
238
+ // Create a static lookup table for all formats
239
+ match format {
240
+ // Primary formats
241
+ "application/json" => Some(ApplicationJson),
242
+ "text/plain" => Some(TextPlain),
243
+ "text/html" => Some(TextHtml),
244
+ "text/css" => Some(TextCss),
245
+ "text/yaml" => Some(TextYaml),
246
+ "application/yaml" => Some(ApplicationYaml),
247
+ "text/xml" => Some(TextXml),
248
+ "text/xsl" => Some(TextXsl),
249
+ "application/xml" => Some(ApplicationXml),
250
+ "application/javascript" => Some(ApplicationJavascript),
251
+ "application/octet-stream" => Some(ApplicationOctetStream),
252
+ "image/svg+xml" => Some(ImageSvgXml),
253
+ "application/x-font-truetype" => Some(ApplicationXFontTruetype),
254
+ "application/x-font-opentype" => Some(ApplicationXFontOpentype),
255
+ "application/font-woff" => Some(ApplicationFontWoff),
256
+ "application/font-woff2" => Some(ApplicationFontWoff2),
257
+ "application/vnd.ms-fontobject" => Some(ApplicationVndMsFontobject),
258
+ "image/png" => Some(ImagePng),
259
+ "image/jpeg" => Some(ImageJpeg),
260
+ "image/gif" => Some(ImageGif),
261
+ "image/x-icon" => Some(ImageXIcon),
262
+ "image/bmp" => Some(ImageBmp),
263
+ "image/icns" => Some(ImageIcns),
264
+ "audio/mpeg" => Some(AudioMpeg),
265
+ "audio/ogg" => Some(AudioOgg),
266
+ "video/mp4" => Some(VideoMp4),
267
+ "application/pdf" => Some(ApplicationPdf),
268
+ "application/zip" => Some(ApplicationZip),
269
+
270
+ // Secondary formats (aliases)
271
+ "prometheus" => Some(Prometheus),
272
+ "text" | "txt" => Some(TextPlain),
273
+ "json" => Some(ApplicationJson),
274
+ "html" => Some(TextHtml),
275
+ "xml" => Some(ApplicationXml),
276
+
277
+ _ => None,
278
+ }
279
+ }
280
+
281
+ /// Parse with a default fallback (matching the C function behavior)
282
+ pub fn from_str_or_default(format: &str) -> Self {
283
+ Self::from_str(format).unwrap_or(HttpContent::TextPlain)
284
+ }
285
+
286
+ /// Check if this content type needs a charset parameter
287
+ pub fn needs_charset(&self) -> bool {
288
+ self.info().needs_charset
289
+ }
290
+
291
+ /// Get the full content type header value (with charset if needed)
292
+ pub fn to_header_value(self, charset: Option<&str>) -> String {
293
+ let info = self.info();
294
+ let mut result = String::from(info.format);
295
+
296
+ if let Some(options) = info.options {
297
+ result.push_str("; ");
298
+ result.push_str(options);
299
+ }
300
+
301
+ if info.needs_charset {
302
+ if let Some(charset) = charset {
303
+ result.push_str("; charset=");
304
+ result.push_str(charset);
305
+ }
306
+ }
307
+
308
+ result
309
+ }
310
+}
311
+
312
+impl fmt::Display for HttpContent {
313
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
314
+ write!(f, "{}", self.as_str())
315
+ }
316
+}
317
+
318
+#[cfg(test)]
319
+mod tests {
320
+ use super::*;
321
+
322
+ #[test]
323
+ fn test_from_str() {
324
+ assert_eq!(
325
+ HttpContent::from_str("application/json"),
326
+ Some(HttpContent::ApplicationJson)
327
+ );
328
+ assert_eq!(
329
+ HttpContent::from_str("json"),
330
+ Some(HttpContent::ApplicationJson)
331
+ );
332
+ assert_eq!(
333
+ HttpContent::from_str("text/plain"),
334
+ Some(HttpContent::TextPlain)
335
+ );
336
+ assert_eq!(HttpContent::from_str("unknown"), None);
337
+ }
338
+
339
+ #[test]
340
+ fn test_from_str_with_default() {
341
+ assert_eq!(
342
+ HttpContent::from_str_or_default("application/json"),
343
+ HttpContent::ApplicationJson
344
+ );
345
+ assert_eq!(
346
+ HttpContent::from_str_or_default("unknown"),
347
+ HttpContent::TextPlain
348
+ );
349
+ }
350
+
351
+ #[test]
352
+ fn test_to_header_value() {
353
+ assert_eq!(
354
+ HttpContent::ApplicationJson.to_header_value(Some("utf-8")),
355
+ "application/json; charset=utf-8"
356
+ );
357
+ assert_eq!(
358
+ HttpContent::ImagePng.to_header_value(Some("utf-8")),
359
+ "image/png"
360
+ );
361
+ assert_eq!(
362
+ HttpContent::Prometheus.to_header_value(Some("utf-8")),
363
+ "text/plain; version=0.0.4; charset=utf-8"
364
+ );
365
+ }
366
+}
src/crates/netdata-plugin/protocol/src/lib.rs
new
+17
@@ -0,0 +1,17 @@
1
+mod line_parser;
2
+mod word_iterator;
3
+
4
+mod http_content;
5
+mod message_parser;
6
+mod tokio_codec;
7
+mod transport;
8
+
9
+// Re-export types from netdata-plugin-types
10
+pub use netdata_plugin_types::{
11
+ ConfigDeclaration, DynCfgCmds, DynCfgSourceType, DynCfgStatus, DynCfgType, FunctionCall,
12
+ FunctionCancel, FunctionDeclaration, FunctionProgress, FunctionResult, HttpAccess,
13
+};
14
+
15
+pub use message_parser::Message;
16
+pub use netdata_plugin_error::{NetdataPluginError, Result};
17
+pub use transport::{MessageReader, MessageWriter, Transport, TransportError};
src/crates/netdata-plugin/protocol/src/line_parser.rs
new
+405
@@ -0,0 +1,405 @@
1
+#![allow(dead_code)]
2
+
3
+//! Low-level line parser for Netdata's plugin protocol.
4
+
5
+use crate::word_iterator::WordIterator;
6
+
7
+/// Tokens for pluginsd protocol commands
8
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
9
+enum Token {
10
+ Chart,
11
+ ChartDefinitionEnd,
12
+ Dimension,
13
+ Begin,
14
+ End,
15
+ Set,
16
+ Flush,
17
+ Disable,
18
+ Variable,
19
+ Label,
20
+ Overwrite,
21
+ Clabel,
22
+ ClabelCommit,
23
+ Exit,
24
+ Begin2,
25
+ Set2,
26
+ End2,
27
+ HostDefine,
28
+ HostDefineEnd,
29
+ HostLabel,
30
+ Host,
31
+ ReplayChart,
32
+ Rbegin,
33
+ Rset,
34
+ RdState,
35
+ RsState,
36
+ Rend,
37
+ Function,
38
+ FunctionResultBegin,
39
+ FunctionResultEnd,
40
+ FunctionPayloadBegin,
41
+ FunctionPayloadEnd,
42
+ FunctionCancel,
43
+ FunctionProgress,
44
+ Quit,
45
+ Config,
46
+ NodeId,
47
+ ClaimedId,
48
+ Json,
49
+ JsonPayloadEnd,
50
+ StreamPath,
51
+ MlModel,
52
+ TrustDurations,
53
+ DynCfg,
54
+ DynCfgRegisterModule,
55
+ DynCfgRegisterJob,
56
+ DynCfgReset,
57
+ ReportJobStatus,
58
+ DeleteJob,
59
+}
60
+
61
+// Include the generated phf map
62
+include!(concat!(env!("OUT_DIR"), "/tokens.rs"));
63
+
64
+/// Result type for parser operations
65
+pub type Result<T> = core::result::Result<T, Error>;
66
+
67
+/// Parser errors
68
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
69
+pub enum Error {
70
+ /// Need more data to complete parsing
71
+ IncompleteLine,
72
+ /// Line that is not recognized
73
+ MalformedLine,
74
+ /// I/O error
75
+ Io,
76
+}
77
+
78
+// Implement conversion from std::io::Error for tokio_util compatibility
79
+impl From<std::io::Error> for Error {
80
+ fn from(_: std::io::Error) -> Self {
81
+ Error::Io
82
+ }
83
+}
84
+
85
+/// Commands supported by Netdata's protocol
86
+pub enum Command<'a> {
87
+ // Chart definition commands
88
+ Chart { args: &'a [u8] },
89
+ Dimension { args: &'a [u8] },
90
+ Variable { args: &'a [u8] },
91
+ ChartDefinitionEnd,
92
+
93
+ // Data update commands
94
+ Begin { args: &'a [u8] },
95
+ Set { args: &'a [u8] },
96
+ End { args: &'a [u8] },
97
+
98
+ // Function commands
99
+ Function { args: &'a [u8] },
100
+ FunctionPayloadBegin { args: &'a [u8] },
101
+ FunctionPayload { args: &'a [u8] },
102
+ FunctionResultBegin { args: &'a [u8] },
103
+ FunctionResultEnd { args: &'a [u8] },
104
+ FunctionCancel { args: &'a [u8] },
105
+ FunctionProgress { args: &'a [u8] },
106
+
107
+ // Label commands
108
+ Clabel { args: &'a [u8] },
109
+ ClabelCommit,
110
+
111
+ // Multi-line commands (these would trigger state changes)
112
+ Json { args: &'a [u8] },
113
+
114
+ // Payload data (when in multi-line mode)
115
+ FunctionResultPayload { data: &'a [u8] },
116
+ FunctionPayloadData { data: &'a [u8] },
117
+
118
+ // Unknown command (fallback)
119
+ Unknown,
120
+
121
+ EmptyLine,
122
+}
123
+
124
+impl core::fmt::Debug for Command<'_> {
125
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
126
+ match self {
127
+ Command::Chart { args } => f
128
+ .debug_struct("Chart")
129
+ .field("args", &ByteStr(args))
130
+ .finish(),
131
+ Command::Dimension { args } => f
132
+ .debug_struct("Dimension")
133
+ .field("args", &ByteStr(args))
134
+ .finish(),
135
+ Command::Variable { args } => f
136
+ .debug_struct("Variable")
137
+ .field("args", &ByteStr(args))
138
+ .finish(),
139
+ Command::ChartDefinitionEnd => f.write_str("ChartDefinitionEnd"),
140
+ Command::Begin { args } => f
141
+ .debug_struct("Begin")
142
+ .field("args", &ByteStr(args))
143
+ .finish(),
144
+ Command::Set { args } => f.debug_struct("Set").field("args", &ByteStr(args)).finish(),
145
+ Command::End { args } => f.debug_struct("End").field("args", &ByteStr(args)).finish(),
146
+ Command::Function { args } => f
147
+ .debug_struct("Function")
148
+ .field("args", &ByteStr(args))
149
+ .finish(),
150
+ Command::FunctionPayloadBegin { args } => f
151
+ .debug_struct("FunctionPayloadBegin")
152
+ .field("args", &ByteStr(args))
153
+ .finish(),
154
+ Command::FunctionPayload { args } => f
155
+ .debug_struct("FunctionPayload")
156
+ .field("args", &ByteStr(args))
157
+ .finish(),
158
+ Command::Clabel { args } => f
159
+ .debug_struct("Clabel")
160
+ .field("args", &ByteStr(args))
161
+ .finish(),
162
+ Command::ClabelCommit => f.write_str("ClabelCommit"),
163
+ Command::Json { args } => f
164
+ .debug_struct("Json")
165
+ .field("args", &ByteStr(args))
166
+ .finish(),
167
+ Command::FunctionResultPayload { data } => f
168
+ .debug_struct("FunctionResultPayload")
169
+ .field("data", &ByteStr(data))
170
+ .finish(),
171
+ Command::FunctionPayloadData { data } => f
172
+ .debug_struct("FunctionPayloadData")
173
+ .field("data", &ByteStr(data))
174
+ .finish(),
175
+ Command::Unknown => f.debug_struct("Unknown").finish(),
176
+ Command::EmptyLine => f.write_str("EmptyLine"),
177
+ Command::FunctionResultBegin { args } => f
178
+ .debug_struct("FunctionResultBegin")
179
+ .field("data", &ByteStr(args))
180
+ .finish(),
181
+ Command::FunctionResultEnd { args } => f
182
+ .debug_struct("FunctionResultEnd")
183
+ .field("data", &ByteStr(args))
184
+ .finish(),
185
+ Command::FunctionCancel { args } => f
186
+ .debug_struct("FunctionCancel")
187
+ .field("args", &ByteStr(args))
188
+ .finish(),
189
+ Command::FunctionProgress { args } => f
190
+ .debug_struct("FunctionProgress")
191
+ .field("args", &ByteStr(args))
192
+ .finish(),
193
+ }
194
+ }
195
+}
196
+
197
+// Helper wrapper for displaying byte slices as strings
198
+struct ByteStr<'a>(&'a [u8]);
199
+
200
+impl core::fmt::Debug for ByteStr<'_> {
201
+ fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
202
+ write!(f, "\"")?;
203
+ for &byte in self.0 {
204
+ match byte {
205
+ b'\n' => write!(f, "\\n")?,
206
+ b'\r' => write!(f, "\\r")?,
207
+ b'\t' => write!(f, "\\t")?,
208
+ b'\\' => write!(f, "\\\\")?,
209
+ b'"' => write!(f, "\\\"")?,
210
+ 0x20..=0x7E => write!(f, "{}", byte as char)?,
211
+ _ => write!(f, "\\x{:02x}", byte)?,
212
+ }
213
+ }
214
+ write!(f, "\"")?;
215
+ Ok(())
216
+ }
217
+}
218
+
219
+/// A parsed line with references to the original buffer
220
+#[derive(Debug)]
221
+pub struct ParsedLine<'a> {
222
+ /// The parsed command
223
+ pub command: Option<Command<'a>>,
224
+ /// Byte offset where this line ends in the original buffer (after line terminator)
225
+ pub consumed: usize,
226
+}
227
+
228
+/// Parser state for handling multi-line commands
229
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
230
+pub enum ParserState {
231
+ /// Normal line-by-line parsing mode
232
+ Normal,
233
+ /// Inside a FUNCTION_RESULT block, looking for FUNCTION_RESULT_END
234
+ FunctionResult,
235
+ /// Inside a FUNCTION_PAYLOAD block, looking for FUNCTION_PAYLOAD_END
236
+ FunctionPayload,
237
+ /// Inside a JSON block, looking for JSON_PAYLOAD_END
238
+ JsonPayload,
239
+}
240
+
241
+impl Default for ParserState {
242
+ fn default() -> Self {
243
+ Self::Normal
244
+ }
245
+}
246
+
247
+/// Low-level line parser
248
+#[derive(Default, Debug)]
249
+pub struct LineParser {
250
+ state: ParserState,
251
+ /// Position where the current payload block started
252
+ payload_start: Option<usize>,
253
+}
254
+
255
+impl LineParser {
256
+ /// Parse the next line or payload block from the buffer
257
+ pub fn parse<'a>(&mut self, buffer: &'a [u8]) -> Result<ParsedLine<'a>> {
258
+ match self.state {
259
+ ParserState::Normal => self.parse_normal_line(buffer),
260
+ ParserState::FunctionResult => self.parse_payload_block(buffer, b"FUNCTION_RESULT_END"),
261
+ ParserState::FunctionPayload => {
262
+ self.parse_payload_block(buffer, b"FUNCTION_PAYLOAD_END")
263
+ }
264
+ ParserState::JsonPayload => self.parse_payload_block(buffer, b"JSON_PAYLOAD_END"),
265
+ }
266
+ }
267
+
268
+ /// Parse a normal line (terminated by \n)
269
+ fn parse_normal_line<'a>(&mut self, buffer: &'a [u8]) -> Result<ParsedLine<'a>> {
270
+ let Some(newline_pos) = find_byte(buffer, b'\n') else {
271
+ return Err(Error::IncompleteLine);
272
+ };
273
+
274
+ let mut words = WordIterator::new(&buffer[..newline_pos]);
275
+ let consumed = newline_pos + 1;
276
+
277
+ let Some(token) = words.next() else {
278
+ return Ok(ParsedLine {
279
+ command: None,
280
+ consumed,
281
+ });
282
+ };
283
+
284
+ let args = words.remainder();
285
+
286
+ // Parse the command word using perfect hash lookup
287
+ let command = match COMMAND_MAP.get(token) {
288
+ Some(Token::Chart) => Command::Chart { args },
289
+ Some(Token::ChartDefinitionEnd) => Command::ChartDefinitionEnd,
290
+ Some(Token::Dimension) => Command::Dimension { args },
291
+ Some(Token::Clabel) => Command::Clabel { args },
292
+ Some(Token::ClabelCommit) => Command::ClabelCommit,
293
+
294
+ Some(Token::Begin) => Command::Begin { args },
295
+ Some(Token::Set) => Command::Set { args },
296
+ Some(Token::End) => Command::End { args },
297
+ Some(Token::Function) => Command::Function { args },
298
+ Some(Token::FunctionResultBegin) => {
299
+ self.state = ParserState::FunctionResult;
300
+ self.payload_start = Some(0);
301
+ Command::FunctionResultBegin { args }
302
+ }
303
+ Some(Token::FunctionResultEnd) => Command::FunctionResultEnd { args },
304
+ Some(Token::FunctionPayloadBegin) => {
305
+ self.state = ParserState::FunctionPayload;
306
+ self.payload_start = Some(0);
307
+ Command::FunctionPayloadBegin { args }
308
+ }
309
+ Some(Token::FunctionPayloadEnd) => Command::FunctionPayload { args },
310
+ Some(Token::FunctionProgress) => Command::FunctionProgress { args },
311
+ Some(Token::FunctionCancel) => Command::FunctionCancel { args },
312
+ Some(Token::Json) => {
313
+ self.state = ParserState::JsonPayload;
314
+ self.payload_start = Some(0);
315
+ Command::Json { args }
316
+ }
317
+
318
+ Some(token) => {
319
+ panic!("Unhandled line parser token: {:#?}", token)
320
+ }
321
+ None => Command::Unknown,
322
+ };
323
+
324
+ Ok(ParsedLine {
325
+ command: Some(command),
326
+ consumed,
327
+ })
328
+ }
329
+
330
+ /// Parse a payload block (data between start and end markers)
331
+ fn parse_payload_block<'a>(
332
+ &mut self,
333
+ buffer: &'a [u8],
334
+ end_marker: &[u8],
335
+ ) -> Result<ParsedLine<'a>> {
336
+ let start_offset = self.payload_start.unwrap_or(0);
337
+
338
+ // If start_offset is beyond buffer, we need to adjust
339
+ if start_offset >= buffer.len() {
340
+ return Err(Error::IncompleteLine);
341
+ }
342
+
343
+ // Look for the end marker on its own line, starting from the payload start
344
+ let search_buffer = &buffer[start_offset..];
345
+ let mut pos = 0;
346
+
347
+ while pos < search_buffer.len() {
348
+ if let Some(newline_pos) = find_byte(&search_buffer[pos..], b'\n') {
349
+ let line_start = pos;
350
+ let line_end = pos + newline_pos;
351
+ let line = &search_buffer[line_start..line_end];
352
+
353
+ // Check if this line is the end marker
354
+ if line == end_marker {
355
+ // Found the end marker
356
+ let payload_end = line_start; // End of payload (before the end marker line)
357
+ let consumed = start_offset + line_end + 1; // Total consumed from original buffer
358
+
359
+ // The payload is everything from start to just before the end marker
360
+ let payload = if payload_end > 0 {
361
+ &search_buffer[..payload_end - 1] // Exclude the newline before end marker
362
+ } else {
363
+ b"" // Empty payload
364
+ };
365
+
366
+ let command = match self.state {
367
+ ParserState::FunctionResult => {
368
+ Command::FunctionResultPayload { data: payload }
369
+ }
370
+ ParserState::FunctionPayload => {
371
+ Command::FunctionPayloadData { data: payload }
372
+ }
373
+ ParserState::JsonPayload => {
374
+ Command::FunctionResultPayload { data: payload }
375
+ } // JSON uses same as function result
376
+ ParserState::Normal => {
377
+ unreachable!("Should not be parsing payload in Normal state")
378
+ }
379
+ };
380
+
381
+ self.state = ParserState::Normal;
382
+ self.payload_start = None;
383
+
384
+ return Ok(ParsedLine {
385
+ command: Some(command),
386
+ consumed,
387
+ });
388
+ }
389
+
390
+ // Move to the next line
391
+ pos = line_end + 1;
392
+ } else {
393
+ // No complete line found, need more data
394
+ return Err(Error::IncompleteLine);
395
+ }
396
+ }
397
+
398
+ Err(Error::IncompleteLine)
399
+ }
400
+}
401
+
402
+/// Helper function to find a byte in a slice
403
+fn find_byte(haystack: &[u8], needle: u8) -> Option<usize> {
404
+ haystack.iter().position(|&b| b == needle)
405
+}
src/crates/netdata-plugin/protocol/src/message_parser.rs
new
+276
@@ -0,0 +1,276 @@
1
+use crate::http_content::HttpContent;
2
+use crate::line_parser::{Command, LineParser};
3
+use crate::word_iterator::WordIterator;
4
+use netdata_plugin_types::*;
5
+
6
+/// Parser direction configuration
7
+#[derive(Debug, Clone, Copy, PartialEq)]
8
+enum ParserDirection {
9
+ /// Input parser - (parsing commands from the agent to plugins)
10
+ Input,
11
+ /// Output parser (parsing commands from plugins to the agent)
12
+ Output,
13
+}
14
+
15
+/// High-level message parser that groups related commands
16
+#[derive(Debug)]
17
+pub struct MessageParser {
18
+ pub(crate) line_parser: LineParser,
19
+ current_message: Option<Message>,
20
+ direction: ParserDirection,
21
+}
22
+
23
+impl Default for MessageParser {
24
+ fn default() -> Self {
25
+ Self::new(ParserDirection::Output)
26
+ }
27
+}
28
+
29
+#[derive(Debug)]
30
+pub enum Message {
31
+ FunctionDeclaration(Box<FunctionDeclaration>),
32
+ FunctionCall(Box<FunctionCall>),
33
+ FunctionResult(Box<FunctionResult>),
34
+ FunctionCancel(Box<FunctionCancel>),
35
+ FunctionProgress(Box<FunctionProgress>),
36
+ ConfigDeclaration(Box<ConfigDeclaration>),
37
+}
38
+
39
+pub fn name_and_args(buffer: &[u8]) -> Option<(String, Vec<String>)> {
40
+ let mut words = WordIterator::new(buffer);
41
+ let name = words.next_string()?;
42
+
43
+ let mut args = Vec::new();
44
+ for word in words {
45
+ args.push(String::from_utf8_lossy(word).into_owned());
46
+ }
47
+
48
+ Some((name, args))
49
+}
50
+
51
+impl MessageParser {
52
+ /// Create a new message parser with the specified direction
53
+ fn new(direction: ParserDirection) -> Self {
54
+ Self {
55
+ line_parser: LineParser::default(),
56
+ current_message: None,
57
+ direction,
58
+ }
59
+ }
60
+
61
+ /// Create a new input parser (parsing commands sent TO plugins)
62
+ pub fn input() -> Self {
63
+ Self::new(ParserDirection::Input)
64
+ }
65
+
66
+ /// Create a new output parser (parsing commands FROM plugins)
67
+ pub fn output() -> Self {
68
+ Self::new(ParserDirection::Output)
69
+ }
70
+
71
+ /// Process a single command and optionally return a completed message
72
+ pub(crate) fn process_command(&mut self, command: Command) -> Option<Message> {
73
+ match command {
74
+ Command::Function { args } => {
75
+ self.current_message = self.parse_function(args);
76
+ self.current_message.take()
77
+ }
78
+
79
+ Command::FunctionResultBegin { args } => {
80
+ let prev_message = self.current_message.take();
81
+ self.current_message = self.parse_function_result_begin(args);
82
+ prev_message
83
+ }
84
+
85
+ Command::FunctionPayloadBegin { args } => {
86
+ let prev_message = self.current_message.take();
87
+ self.current_message = self.parse_function_payload_begin(args);
88
+ prev_message
89
+ }
90
+
91
+ Command::FunctionPayloadData { data } => {
92
+ if let Some(Message::FunctionCall(func_call)) = &mut self.current_message {
93
+ if let Some(ref mut payload) = func_call.payload {
94
+ payload.extend_from_slice(data);
95
+ } else {
96
+ func_call.payload = Some(data.to_vec());
97
+ }
98
+ }
99
+ self.current_message.take()
100
+ }
101
+
102
+ Command::FunctionPayload { args: _ } => None,
103
+
104
+ Command::FunctionResultPayload { data } => {
105
+ if let Some(Message::FunctionResult(func_result)) = &mut self.current_message {
106
+ func_result.payload.extend_from_slice(data);
107
+ }
108
+ None
109
+ }
110
+
111
+ Command::FunctionResultEnd { args: _ } => self.current_message.take(),
112
+
113
+ Command::FunctionCancel { args } => {
114
+ self.current_message = self.parse_function_cancel(args);
115
+ self.current_message.take()
116
+ }
117
+
118
+ Command::FunctionProgress { args } => {
119
+ self.current_message = self.parse_function_progress(args);
120
+ self.current_message.take()
121
+ }
122
+
123
+ Command::Begin { args: _ } | Command::Set { args: _ } | Command::End { args: _ } => {
124
+ None
125
+ }
126
+
127
+ cmd => {
128
+ eprintln!("Got cmd: {:#?}", cmd);
129
+ None
130
+ }
131
+ }
132
+ }
133
+
134
+ /// Parse FUNCTION command arguments - behavior depends on parser direction
135
+ fn parse_function(&self, args: &[u8]) -> Option<Message> {
136
+ match self.direction {
137
+ ParserDirection::Input => self.parse_function_call(args),
138
+ ParserDirection::Output => self.parse_function_declaration(args),
139
+ }
140
+ }
141
+
142
+ /// Parse FUNCTION command arguments for output context (function definition)
143
+ fn parse_function_declaration(&self, args: &[u8]) -> Option<Message> {
144
+ let mut words = WordIterator::new(args);
145
+
146
+ let (global, name) = {
147
+ let first_word = words.next_string()?;
148
+
149
+ if first_word == "GLOBAL" {
150
+ // GLOBAL flag is present, next word is the function name
151
+ (true, words.next_string()?)
152
+ } else {
153
+ // No GLOBAL flag, first word is the function name
154
+ (false, first_word)
155
+ }
156
+ };
157
+
158
+ let timeout = words.next_u32()?;
159
+ let help = words.next_string()?;
160
+ let tags = words.next_string();
161
+ let access = words.next().map(HttpAccess::from_slice);
162
+ let priority = words.next_u32();
163
+ let version = words.next_u32();
164
+
165
+ let function_declaration = Box::new(FunctionDeclaration {
166
+ global,
167
+ name,
168
+ timeout,
169
+ help,
170
+ tags,
171
+ access,
172
+ priority,
173
+ version,
174
+ });
175
+
176
+ Some(Message::FunctionDeclaration(function_declaration))
177
+ }
178
+
179
+ /// Parse FUNCTION command arguments for input context (function call)
180
+ /// Expected format: FUNCTION transaction timeout function access source
181
+ fn parse_function_call(&self, args: &[u8]) -> Option<Message> {
182
+ let mut words = WordIterator::new(args);
183
+
184
+ let transaction = words.next_string()?;
185
+ let timeout = words.next_u32()?;
186
+ let (name, args) = {
187
+ let buffer = words.next_str()?.as_bytes();
188
+ name_and_args(buffer)?
189
+ };
190
+ let access = words.next().map(HttpAccess::from_slice);
191
+ let source = words.next_string();
192
+ let payload = None;
193
+
194
+ let function_call = Box::new(FunctionCall {
195
+ transaction,
196
+ timeout,
197
+ name,
198
+ args,
199
+ access,
200
+ source,
201
+ payload,
202
+ });
203
+
204
+ Some(Message::FunctionCall(function_call))
205
+ }
206
+
207
+ /// Parse FUNCTION_RESULT_BEGIN command and create FunctionResult with payload
208
+ /// Expected format: FUNCTION_RESULT_BEGIN transaction status format expires
209
+ fn parse_function_result_begin(&self, args: &[u8]) -> Option<Message> {
210
+ let mut words = WordIterator::new(args);
211
+
212
+ let transaction = words.next_string()?;
213
+ let status = words.next_u32()?;
214
+ let format = HttpContent::from_str_or_default(words.next_str()?).to_string();
215
+ let expires = words.next_u64()?;
216
+
217
+ let function_result = Box::new(FunctionResult {
218
+ transaction,
219
+ status,
220
+ format,
221
+ expires,
222
+ payload: Vec::new(),
223
+ });
224
+
225
+ Some(Message::FunctionResult(function_result))
226
+ }
227
+
228
+ /// Parse FUNCTION_PAYLOAD command and create FunctionCall with empty payload
229
+ /// Expected format: FUNCTION_PAYLOAD transaction timeout function access source
230
+ fn parse_function_payload_begin(&self, args: &[u8]) -> Option<Message> {
231
+ let mut words = WordIterator::new(args);
232
+
233
+ let transaction = words.next_string()?;
234
+ let timeout = words.next_u32()?;
235
+ let (name, args) = {
236
+ let buffer = words.next_str()?.as_bytes();
237
+ name_and_args(buffer)?
238
+ };
239
+ let access = words.next().map(HttpAccess::from_slice);
240
+ let source = words.next_string();
241
+
242
+ let function_call = Box::new(FunctionCall {
243
+ transaction,
244
+ timeout,
245
+ name,
246
+ args,
247
+ access,
248
+ source,
249
+ payload: Some(Vec::new()),
250
+ });
251
+
252
+ Some(Message::FunctionCall(function_call))
253
+ }
254
+
255
+ /// Parse FUNCTION_CANCEL command
256
+ /// Expected format: FUNCTION_CANCEL transaction
257
+ fn parse_function_cancel(&self, args: &[u8]) -> Option<Message> {
258
+ let mut words = WordIterator::new(args);
259
+
260
+ let transaction = words.next_string()?;
261
+ let function_cancel = Box::new(FunctionCancel { transaction });
262
+
263
+ Some(Message::FunctionCancel(function_cancel))
264
+ }
265
+
266
+ /// Parse FUNCTION_PROGRESS command
267
+ /// Expected format: FUNCTION_PROGRESS transaction
268
+ fn parse_function_progress(&self, args: &[u8]) -> Option<Message> {
269
+ let mut words = WordIterator::new(args);
270
+
271
+ let transaction = words.next_string()?;
272
+ let function_progress = Box::new(FunctionProgress { transaction });
273
+
274
+ Some(Message::FunctionProgress(function_progress))
275
+ }
276
+}
src/crates/netdata-plugin/protocol/src/tokio_codec.rs
new
+209
@@ -0,0 +1,209 @@
1
+//! tokio_util::codec implementations for the Netdata protocol
2
+
3
+use crate::message_parser::{Message, MessageParser};
4
+use bytes::{Buf, BytesMut};
5
+use netdata_plugin_types::FunctionCall;
6
+use tokio_util::codec::{Decoder, Encoder};
7
+
8
+/// Helper function to quote strings if needed
9
+fn quote_if_needed(s: &str) -> String {
10
+ if s.is_empty() {
11
+ "''".to_string()
12
+ } else if s.contains([' ', '\t', '\'', '"']) {
13
+ // Use single quotes and escape any single quotes within
14
+ format!("'{}'", s.replace('\'', "\\'"))
15
+ } else {
16
+ s.to_string()
17
+ }
18
+}
19
+
20
+/// Helper function to build function command parts
21
+fn build_function_parts(func_call: &FunctionCall, command: &str) -> Vec<String> {
22
+ let mut parts = vec![
23
+ command.to_string(),
24
+ quote_if_needed(&func_call.transaction),
25
+ func_call.timeout.to_string(),
26
+ quote_if_needed(&func_call.name),
27
+ ];
28
+
29
+ if let Some(access) = func_call.access {
30
+ parts.push(access.to_string());
31
+ }
32
+
33
+ if let Some(source) = func_call.source.as_ref() {
34
+ parts.push(quote_if_needed(source));
35
+ }
36
+
37
+ parts
38
+}
39
+
40
+/// Decoder implementation for MessageParser
41
+impl Decoder for MessageParser {
42
+ type Item = Message;
43
+ type Error = crate::line_parser::Error;
44
+
45
+ fn decode(&mut self, src: &mut BytesMut) -> crate::line_parser::Result<Option<Self::Item>> {
46
+ let mut total_consumed = 0;
47
+ let buffer = src.as_ref();
48
+
49
+ loop {
50
+ let remaining = &buffer[total_consumed..];
51
+ if remaining.is_empty() {
52
+ break;
53
+ }
54
+
55
+ let parsed_line = match self.line_parser.parse(remaining) {
56
+ Ok(parsed_line) => parsed_line,
57
+ Err(crate::line_parser::Error::IncompleteLine) => {
58
+ // Need more data - advance buffer by what we consumed so far
59
+ src.advance(total_consumed);
60
+ return Ok(None);
61
+ }
62
+ Err(e) => {
63
+ // Advance buffer and return error
64
+ src.advance(total_consumed);
65
+ return Err(e);
66
+ }
67
+ };
68
+
69
+ total_consumed += parsed_line.consumed;
70
+
71
+ let Some(command) = parsed_line.command else {
72
+ continue;
73
+ };
74
+
75
+ if let Some(message) = self.process_command(command) {
76
+ // Found a complete message - advance buffer and return it
77
+ src.advance(total_consumed);
78
+ return Ok(Some(message));
79
+ }
80
+ }
81
+
82
+ // Processed all available data but no complete message yet
83
+ src.advance(total_consumed);
84
+ Ok(None)
85
+ }
86
+}
87
+
88
+/// Encoder implementation for MessageParser to serialize Messages back to protocol format
89
+impl Encoder<Message> for MessageParser {
90
+ type Error = std::io::Error;
91
+
92
+ fn encode(
93
+ &mut self,
94
+ item: Message,
95
+ dst: &mut BytesMut,
96
+ ) -> std::result::Result<(), Self::Error> {
97
+ match item {
98
+ Message::ConfigDeclaration(cfg_decl) => {
99
+ // CONFIG <id> CREATE <status> <type> <path> <source_type> <source> <cmds> <view_access> <edit_access>
100
+ let parts = vec![
101
+ "CONFIG".to_string(),
102
+ quote_if_needed(&cfg_decl.id),
103
+ "CREATE".to_string(),
104
+ cfg_decl.status.to_string(),
105
+ cfg_decl.type_.to_string(),
106
+ quote_if_needed(&cfg_decl.path),
107
+ cfg_decl.source_type.to_string(),
108
+ quote_if_needed(&cfg_decl.source),
109
+ quote_if_needed(&cfg_decl.cmds.to_string()),
110
+ cfg_decl.view_access.to_string(),
111
+ cfg_decl.edit_access.to_string(),
112
+ ];
113
+
114
+ dst.extend_from_slice(format!("{}\n", parts.join(" ")).as_bytes());
115
+ }
116
+ Message::FunctionDeclaration(func_decl) => {
117
+ // FUNCTION [GLOBAL] name timeout help [tags [access [priority [version]]]
118
+ let mut parts = Vec::with_capacity(8);
119
+ parts.push("FUNCTION".to_string());
120
+
121
+ if func_decl.global {
122
+ parts.push("GLOBAL".to_string());
123
+ }
124
+
125
+ parts.push(quote_if_needed(&func_decl.name));
126
+ parts.push(func_decl.timeout.to_string());
127
+ parts.push(quote_if_needed(&func_decl.help));
128
+
129
+ if let Some(tags) = func_decl.tags.as_ref() {
130
+ parts.push(quote_if_needed(tags));
131
+
132
+ if let Some(access) = func_decl.access {
133
+ parts.push(access.to_string());
134
+
135
+ if let Some(priority) = func_decl.priority {
136
+ parts.push(priority.to_string());
137
+
138
+ if let Some(version) = func_decl.version {
139
+ parts.push(version.to_string());
140
+ }
141
+ }
142
+ }
143
+ }
144
+
145
+ dst.extend_from_slice(format!("{}\n", parts.join(" ")).as_bytes());
146
+ }
147
+
148
+ Message::FunctionCall(func_call) => match &func_call.payload {
149
+ Some(payload) => {
150
+ let parts = build_function_parts(&func_call, "FUNCTION_PAYLOAD");
151
+ dst.extend_from_slice(format!("{}\n", parts.join(" ")).as_bytes());
152
+
153
+ if !payload.is_empty() {
154
+ dst.extend_from_slice(payload.as_slice());
155
+ if !payload.ends_with(b"\n") {
156
+ dst.extend_from_slice(b"\n");
157
+ }
158
+ }
159
+ dst.extend_from_slice(b"FUNCTION_PAYLOAD_END\n");
160
+ }
161
+ None => {
162
+ let parts = build_function_parts(&func_call, "FUNCTION");
163
+ dst.extend_from_slice(format!("{}\n", parts.join(" ")).as_bytes());
164
+ }
165
+ },
166
+
167
+ Message::FunctionResult(func_result) => {
168
+ // FUNCTION_RESULT_BEGIN [transaction_id] [status_code] [content_type] [expires]
169
+ dst.extend_from_slice(
170
+ format!(
171
+ "FUNCTION_RESULT_BEGIN {} {} {} {}\n",
172
+ func_result.transaction,
173
+ func_result.status,
174
+ func_result.format,
175
+ func_result.expires
176
+ )
177
+ .as_bytes(),
178
+ );
179
+
180
+ // Function payload data
181
+ if !func_result.payload.is_empty() {
182
+ dst.extend_from_slice(func_result.payload.as_slice());
183
+ if !func_result.payload.ends_with(b"\n") {
184
+ dst.extend_from_slice(b"\n");
185
+ }
186
+ }
187
+
188
+ // FUNCTION_RESULT_END
189
+ dst.extend_from_slice(b"FUNCTION_RESULT_END\n");
190
+ }
191
+
192
+ Message::FunctionCancel(func_cancel) => {
193
+ // FUNCTION_CANCEL transaction
194
+ dst.extend_from_slice(
195
+ format!(
196
+ "FUNCTION_CANCEL {}\n",
197
+ quote_if_needed(&func_cancel.transaction)
198
+ )
199
+ .as_bytes(),
200
+ );
201
+ }
202
+ Message::FunctionProgress(_) => {
203
+ unimplemented!()
204
+ }
205
+ }
206
+
207
+ Ok(())
208
+ }
209
+}
src/crates/netdata-plugin/protocol/src/transport.rs
new
+168
@@ -0,0 +1,168 @@
1
+use crate::message_parser::{Message, MessageParser};
2
+use futures::{SinkExt, Stream, StreamExt};
3
+use netdata_plugin_error::{NetdataPluginError, Result};
4
+use std::pin::Pin;
5
+use std::task::{Context, Poll};
6
+use tokio::io::{AsyncRead, AsyncWrite};
7
+use tokio_util::codec::{FramedRead, FramedWrite};
8
+
9
+// TransportError is now replaced by NetdataPluginError
10
+pub type TransportError = NetdataPluginError;
11
+
12
+/// Reader for receiving Netdata protocol messages
13
+#[derive(Debug)]
14
+pub struct MessageReader<R>
15
+where
16
+ R: AsyncRead + Unpin,
17
+{
18
+ reader: FramedRead<R, MessageParser>,
19
+}
20
+
21
+impl<R> MessageReader<R>
22
+where
23
+ R: AsyncRead + Unpin,
24
+{
25
+ /// Create a new message reader
26
+ pub fn new(reader: R) -> Self {
27
+ Self {
28
+ reader: FramedRead::new(reader, MessageParser::input()),
29
+ }
30
+ }
31
+
32
+ /// Receive the next message
33
+ pub async fn recv(&mut self) -> Option<Result<Message>> {
34
+ self.reader.next().await.map(|result| {
35
+ result.map_err(|e| NetdataPluginError::Protocol {
36
+ message: format!("{:?}", e),
37
+ })
38
+ })
39
+ }
40
+}
41
+
42
+impl<R> Stream for MessageReader<R>
43
+where
44
+ R: AsyncRead + Unpin,
45
+{
46
+ type Item = Result<Message>;
47
+
48
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
49
+ self.reader.poll_next_unpin(cx).map(|opt| {
50
+ opt.map(|result| {
51
+ result.map_err(|e| NetdataPluginError::Protocol {
52
+ message: format!("{:?}", e),
53
+ })
54
+ })
55
+ })
56
+ }
57
+}
58
+
59
+/// Writer for sending Netdata protocol messages
60
+#[derive(Debug)]
61
+pub struct MessageWriter<W>
62
+where
63
+ W: AsyncWrite + Unpin,
64
+{
65
+ writer: FramedWrite<W, MessageParser>,
66
+}
67
+
68
+impl<W> MessageWriter<W>
69
+where
70
+ W: AsyncWrite + Unpin,
71
+{
72
+ /// Create a new message writer
73
+ pub fn new(writer: W) -> Self {
74
+ Self {
75
+ writer: FramedWrite::new(writer, MessageParser::output()),
76
+ }
77
+ }
78
+
79
+ /// Send a message
80
+ pub async fn send(&mut self, message: Message) -> Result<()> {
81
+ use tokio::io::AsyncWriteExt;
82
+ self.writer.send(message).await?;
83
+ self.writer.flush().await?;
84
+ self.writer.get_mut().flush().await?;
85
+ Ok(())
86
+ }
87
+
88
+ /// Force flush the underlying writer
89
+ pub async fn flush(&mut self) -> Result<()> {
90
+ use tokio::io::AsyncWriteExt;
91
+ self.writer.get_mut().flush().await?;
92
+ Ok(())
93
+ }
94
+
95
+ /// Write raw bytes directly to the underlying writer (for chart protocol)
96
+ ///
97
+ /// This bypasses the message framing and writes raw bytes directly.
98
+ /// Used for emitting chart protocol data (CHART, BEGIN, SET, END commands).
99
+ pub async fn write_raw(&mut self, data: &[u8]) -> Result<()> {
100
+ use tokio::io::AsyncWriteExt;
101
+ self.writer.get_mut().write_all(data).await?;
102
+ self.writer.get_mut().flush().await?;
103
+ Ok(())
104
+ }
105
+}
106
+
107
+/// Legacy Transport type for backward compatibility
108
+/// Consider using MessageReader and MessageWriter directly for new code
109
+pub struct Transport<R, W>
110
+where
111
+ R: AsyncRead + Unpin,
112
+ W: AsyncWrite + Unpin,
113
+{
114
+ reader: MessageReader<R>,
115
+ writer: MessageWriter<W>,
116
+}
117
+
118
+impl<R, W> Transport<R, W>
119
+where
120
+ R: AsyncRead + Unpin,
121
+ W: AsyncWrite + Unpin,
122
+{
123
+ /// Create a new transport with separate reader and writer streams
124
+ pub fn new_with_streams(reader: R, writer: W) -> Self {
125
+ Self {
126
+ reader: MessageReader::new(reader),
127
+ writer: MessageWriter::new(writer),
128
+ }
129
+ }
130
+
131
+ /// Send a message through the transport
132
+ pub async fn send(&mut self, message: Message) -> Result<()> {
133
+ self.writer.send(message).await
134
+ }
135
+
136
+ /// Force flush the underlying writer
137
+ pub async fn flush(&mut self) -> Result<()> {
138
+ self.writer.flush().await
139
+ }
140
+
141
+ /// Receive the next message from the transport
142
+ pub async fn recv(&mut self) -> Option<Result<Message>> {
143
+ self.reader.recv().await
144
+ }
145
+
146
+ /// Send a message and receive the next response
147
+ pub async fn request(&mut self, message: Message) -> Result<Option<Message>> {
148
+ self.send(message).await?;
149
+ match self.recv().await {
150
+ Some(Ok(response)) => Ok(Some(response)),
151
+ Some(Err(e)) => Err(e),
152
+ None => Ok(None),
153
+ }
154
+ }
155
+}
156
+
157
+impl Transport<tokio::io::Stdin, tokio::io::Stdout> {
158
+ /// Create a new transport using stdin/stdout
159
+ pub fn new() -> Self {
160
+ Self::new_with_streams(tokio::io::stdin(), tokio::io::stdout())
161
+ }
162
+}
163
+
164
+impl Default for Transport<tokio::io::Stdin, tokio::io::Stdout> {
165
+ fn default() -> Self {
166
+ Self::new()
167
+ }
168
+}
src/crates/netdata-plugin/protocol/src/word_iterator.rs
new
+103
@@ -0,0 +1,103 @@
1
+use atoi::atoi;
2
+
3
+/// Check if a byte is whitespace (space or tab)
4
+fn is_whitespace(b: u8) -> bool {
5
+ b == b' ' || b == b'\t'
6
+}
7
+
8
+/// Word iterator for parsing space-separated arguments
9
+pub(crate) struct WordIterator<'a> {
10
+ buffer: &'a [u8],
11
+ pos: usize,
12
+}
13
+
14
+impl<'a> WordIterator<'a> {
15
+ /// Create a new word iterator
16
+ pub(crate) fn new(buffer: &'a [u8]) -> Self {
17
+ Self { buffer, pos: 0 }
18
+ }
19
+
20
+ pub(crate) fn remainder(&self) -> &'a [u8] {
21
+ let mut pos = self.pos;
22
+
23
+ // Skip leading whitespace
24
+ while pos < self.buffer.len() && is_whitespace(self.buffer[pos]) {
25
+ pos += 1;
26
+ }
27
+
28
+ if pos < self.buffer.len() {
29
+ &self.buffer[pos..]
30
+ } else {
31
+ b""
32
+ }
33
+ }
34
+
35
+ pub(crate) fn next_string(&mut self) -> Option<String> {
36
+ let s = self.next()?;
37
+
38
+ Some(String::from_utf8_lossy(s).into_owned())
39
+ }
40
+
41
+ pub(crate) fn next_u32(&mut self) -> Option<u32> {
42
+ let s = self.next()?;
43
+
44
+ atoi(s)
45
+ }
46
+
47
+ pub(crate) fn next_u64(&mut self) -> Option<u64> {
48
+ let s = self.next()?;
49
+
50
+ atoi(s)
51
+ }
52
+
53
+ pub(crate) fn next_str(&mut self) -> Option<&str> {
54
+ let s = self.next()?;
55
+ std::str::from_utf8(s).ok()
56
+ }
57
+}
58
+
59
+impl<'a> Iterator for WordIterator<'a> {
60
+ type Item = &'a [u8];
61
+
62
+ fn next(&mut self) -> Option<Self::Item> {
63
+ // Skip whitespace
64
+ while self.pos < self.buffer.len() && is_whitespace(self.buffer[self.pos]) {
65
+ self.pos += 1;
66
+ }
67
+
68
+ if self.pos >= self.buffer.len() {
69
+ return None;
70
+ }
71
+
72
+ let start = self.pos;
73
+
74
+ // Check for quoted string
75
+ if self.buffer[self.pos] == b'"' || self.buffer[self.pos] == b'\'' {
76
+ let quote = self.buffer[self.pos];
77
+ self.pos += 1; // Skip opening quote
78
+
79
+ // Find closing quote (with basic escape support)
80
+ while self.pos < self.buffer.len() {
81
+ if self.buffer[self.pos] == b'\\' && self.pos + 1 < self.buffer.len() {
82
+ self.pos += 2; // Skip escape sequence
83
+ } else if self.buffer[self.pos] == quote {
84
+ let word = &self.buffer[start + 1..self.pos]; // Exclude quotes
85
+ self.pos += 1; // Skip closing quote
86
+ return Some(word);
87
+ } else {
88
+ self.pos += 1;
89
+ }
90
+ }
91
+
92
+ // Unclosed quote - return rest of buffer
93
+ Some(&self.buffer[start + 1..])
94
+ } else {
95
+ // Non-quoted word
96
+ while self.pos < self.buffer.len() && !is_whitespace(self.buffer[self.pos]) {
97
+ self.pos += 1;
98
+ }
99
+
100
+ Some(&self.buffer[start..self.pos])
101
+ }
102
+ }
103
+}
src/crates/netdata-plugin/rt/Cargo.toml
new
+36
@@ -0,0 +1,36 @@
1
+[package]
2
+name = "rt"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[dependencies]
11
+# Local crates
12
+foundation = { workspace = true }
13
+netdata-plugin-error = { path = "../error" }
14
+netdata-plugin-protocol = { path = "../protocol" }
15
+netdata-plugin-schema = { path = "../schema" }
16
+netdata-plugin-charts-derive = { path = "../charts-derive" }
17
+
18
+# Async runtime and tokio utilities
19
+tokio = { version = "1.0", features = ["full"] }
20
+tokio-util = { version = "0.7", features = ["codec"] }
21
+
22
+async-trait = { workspace = true }
23
+futures = { workspace = true }
24
+tracing = { workspace = true }
25
+schemars = { workspace = true }
26
+serde_json = { workspace = true }
27
+serde = { workspace = true }
28
+tracing-futures = { workspace = true }
29
+parking_lot = { workspace = true }
30
+bytes = { workspace = true }
31
+itoa = { workspace = true }
32
+tracing-subscriber = { workspace = true, features = ["env-filter", "json"] }
33
+tracing-journald = { workspace = true }
34
+
35
+[dev-dependencies]
36
+console-subscriber = "0.4"
src/crates/netdata-plugin/rt/src/charts/chart_trait.rs
new
+239
@@ -0,0 +1,239 @@
1
+//! NetdataChart trait for declarative chart definition.
2
+
3
+use super::metadata::{ChartMetadata, ChartType, DimensionAlgorithm, DimensionMetadata};
4
+use super::writer::ChartWriter;
5
+use schemars::{schema_for, JsonSchema};
6
+use serde_json::Value;
7
+
8
+/// Trait for writing chart dimensions efficiently.
9
+///
10
+/// This trait is automatically implemented by the `#[derive(NetdataChart)]` macro.
11
+/// It generates code that directly writes dimension values without JSON serialization.
12
+pub trait ChartDimensions {
13
+ /// Write all dimension values to the chart writer
14
+ fn write_dimensions(&self, writer: &mut ChartWriter);
15
+}
16
+
17
+/// Trait for types that can be used as Netdata charts.
18
+///
19
+/// Use `schemars` attributes to annotate your struct with chart metadata,
20
+/// and derive `NetdataChart` to implement the efficient dimension writing.
21
+///
22
+/// # Example
23
+///
24
+/// ```ignore
25
+/// use schemars::JsonSchema;
26
+/// use netdata_plugin_charts::NetdataChart;
27
+///
28
+/// #[derive(JsonSchema, NetdataChart, Default, Clone, PartialEq)]
29
+/// #[schemars(
30
+/// extend("x-chart-id" = "system.cpu"),
31
+/// extend("x-chart-title" = "CPU Usage"),
32
+/// extend("x-chart-units" = "percentage"),
33
+/// )]
34
+/// struct CpuMetrics {
35
+/// user: u64,
36
+/// system: u64,
37
+/// idle: u64,
38
+/// }
39
+/// ```
40
+pub trait NetdataChart: JsonSchema + ChartDimensions {
41
+ /// Extract chart metadata from the JSON schema annotations
42
+ fn chart_metadata() -> ChartMetadata {
43
+ let root_schema = schema_for!(Self);
44
+ extract_chart_metadata(&root_schema)
45
+ }
46
+}
47
+
48
+/// Blanket implementation - all types that implement JsonSchema + ChartDimensions automatically implement NetdataChart
49
+impl<T: JsonSchema + ChartDimensions> NetdataChart for T {}
50
+
51
+/// Trait for charts that represent multiple instances (e.g., per-CPU, per-disk).
52
+///
53
+/// When a chart has an instance field, each instance gets its own chart with
54
+/// a unique ID. The chart ID should contain `{instance}` as a template variable.
55
+pub trait InstancedChart: NetdataChart + Clone {
56
+ /// Get the instance identifier from this struct
57
+ fn instance_id(&self) -> &str;
58
+
59
+ /// Set the instance identifier (called when creating new instances)
60
+ fn set_instance_id(&mut self, id: &str);
61
+}
62
+
63
+/// Extract chart metadata from a JSON schema
64
+fn extract_chart_metadata<T: serde::Serialize>(schema: &T) -> ChartMetadata {
65
+ // Convert schema to JSON for easier processing
66
+ let schema_value = serde_json::to_value(schema).unwrap_or(Value::Null);
67
+ let Some(obj) = schema_value.as_object() else {
68
+ return ChartMetadata::new("unknown");
69
+ };
70
+
71
+ // Extract chart-level metadata
72
+ let mut metadata = ChartMetadata::new(
73
+ extract_string_from_json(obj, "x-chart-id").unwrap_or_else(|| "unknown".to_string()),
74
+ );
75
+
76
+ if let Some(name) = extract_string_from_json(obj, "x-chart-name") {
77
+ metadata.name = name;
78
+ }
79
+
80
+ if let Some(title) = extract_string_from_json(obj, "x-chart-title") {
81
+ metadata.title = title;
82
+ }
83
+
84
+ if let Some(units) = extract_string_from_json(obj, "x-chart-units") {
85
+ metadata.units = units;
86
+ }
87
+
88
+ if let Some(family) = extract_string_from_json(obj, "x-chart-family") {
89
+ metadata.family = family;
90
+ }
91
+
92
+ if let Some(context) = extract_string_from_json(obj, "x-chart-context") {
93
+ metadata.context = context;
94
+ }
95
+
96
+ if let Some(chart_type) = extract_string_from_json(obj, "x-chart-type") {
97
+ metadata.chart_type = match chart_type.as_str() {
98
+ "line" => ChartType::Line,
99
+ "area" => ChartType::Area,
100
+ "stacked" => ChartType::Stacked,
101
+ _ => ChartType::Line,
102
+ };
103
+ }
104
+
105
+ if let Some(priority) = extract_i64_from_json(obj, "x-chart-priority") {
106
+ metadata.priority = priority;
107
+ }
108
+
109
+ if let Some(update_every) = extract_u64_from_json(obj, "x-chart-update-every") {
110
+ metadata.update_every = update_every;
111
+ }
112
+
113
+ // Extract dimensions from properties
114
+ if let Some(properties) = obj.get("properties").and_then(|v| v.as_object()) {
115
+ for (field_name, field_schema) in properties {
116
+ if let Some(field_obj) = field_schema.as_object() {
117
+ // Check if this field is the instance identifier
118
+ if extract_bool_from_json(field_obj, "x-chart-instance").unwrap_or(false) {
119
+ metadata.instance_field = Some(field_name.clone());
120
+
121
+ // Instance fields can be hidden dimensions
122
+ if !extract_bool_from_json(field_obj, "x-dimension-hidden").unwrap_or(true) {
123
+ let dim = extract_dimension_metadata(field_name, field_obj);
124
+ metadata.dimensions.insert(field_name.clone(), dim);
125
+ }
126
+ continue;
127
+ }
128
+
129
+ // Skip if explicitly marked as not a dimension
130
+ if extract_bool_from_json(field_obj, "x-dimension-hidden").unwrap_or(false) {
131
+ continue;
132
+ }
133
+
134
+ // Extract dimension metadata
135
+ let dim = extract_dimension_metadata(field_name, field_obj);
136
+ metadata.dimensions.insert(field_name.clone(), dim);
137
+ }
138
+ }
139
+ }
140
+
141
+ metadata
142
+}
143
+
144
+/// Extract dimension metadata from a field schema
145
+fn extract_dimension_metadata(field_name: &str, field_obj: &serde_json::Map<String, Value>) -> DimensionMetadata {
146
+ let mut dim = DimensionMetadata::new(field_name);
147
+
148
+ if let Some(name) = extract_string_from_json(field_obj, "x-dimension-name") {
149
+ dim.name = name;
150
+ }
151
+
152
+ if let Some(algorithm) = extract_string_from_json(field_obj, "x-dimension-algorithm") {
153
+ dim.algorithm = match algorithm.as_str() {
154
+ "absolute" => DimensionAlgorithm::Absolute,
155
+ "incremental" => DimensionAlgorithm::Incremental,
156
+ "percentage-of-absolute-row" => DimensionAlgorithm::PercentageOfAbsoluteRow,
157
+ "percentage-of-incremental-row" => DimensionAlgorithm::PercentageOfIncrementalRow,
158
+ _ => DimensionAlgorithm::Absolute,
159
+ };
160
+ }
161
+
162
+ if let Some(multiplier) = extract_i64_from_json(field_obj, "x-dimension-multiplier") {
163
+ dim.multiplier = multiplier;
164
+ }
165
+
166
+ if let Some(divisor) = extract_i64_from_json(field_obj, "x-dimension-divisor") {
167
+ dim.divisor = divisor;
168
+ }
169
+
170
+ if extract_bool_from_json(field_obj, "x-dimension-hidden").unwrap_or(false) {
171
+ dim.hidden = true;
172
+ }
173
+
174
+ dim
175
+}
176
+
177
+/// Helper to extract string from JSON object
178
+fn extract_string_from_json(obj: &serde_json::Map<String, Value>, key: &str) -> Option<String> {
179
+ obj.get(key)
180
+ .and_then(|v| v.as_str())
181
+ .map(|s| s.to_string())
182
+}
183
+
184
+/// Helper to extract i64 from JSON object
185
+fn extract_i64_from_json(obj: &serde_json::Map<String, Value>, key: &str) -> Option<i64> {
186
+ obj.get(key).and_then(|v| match v {
187
+ Value::Number(n) => n.as_i64(),
188
+ _ => None,
189
+ })
190
+}
191
+
192
+/// Helper to extract u64 from JSON object
193
+fn extract_u64_from_json(obj: &serde_json::Map<String, Value>, key: &str) -> Option<u64> {
194
+ obj.get(key).and_then(|v| match v {
195
+ Value::Number(n) => n.as_u64(),
196
+ _ => None,
197
+ })
198
+}
199
+
200
+/// Helper to extract bool from JSON object
201
+fn extract_bool_from_json(obj: &serde_json::Map<String, Value>, key: &str) -> Option<bool> {
202
+ obj.get(key).and_then(|v| v.as_bool())
203
+}
204
+
205
+#[cfg(test)]
206
+mod tests {
207
+ use super::*;
208
+
209
+ #[derive(JsonSchema, Default, Clone, PartialEq)]
210
+ #[schemars(
211
+ extend("x-chart-id" = "test.chart"),
212
+ extend("x-chart-title" = "Test Chart"),
213
+ extend("x-chart-units" = "widgets"),
214
+ extend("x-chart-type" = "stacked")
215
+ )]
216
+ struct TestMetrics {
217
+ value1: u64,
218
+ value2: u64,
219
+ }
220
+
221
+ impl ChartDimensions for TestMetrics {
222
+ fn write_dimensions(&self, writer: &mut crate::charts::writer::ChartWriter) {
223
+ writer.write_dimension("value1", self.value1 as i64);
224
+ writer.write_dimension("value2", self.value2 as i64);
225
+ }
226
+ }
227
+
228
+ #[test]
229
+ fn test_extract_metadata() {
230
+ let metadata = TestMetrics::chart_metadata();
231
+ assert_eq!(metadata.id, "test.chart");
232
+ assert_eq!(metadata.title, "Test Chart");
233
+ assert_eq!(metadata.units, "widgets");
234
+ assert_eq!(metadata.chart_type, ChartType::Stacked);
235
+ assert_eq!(metadata.dimensions.len(), 2);
236
+ assert!(metadata.dimensions.contains_key("value1"));
237
+ assert!(metadata.dimensions.contains_key("value2"));
238
+ }
239
+}
src/crates/netdata-plugin/rt/src/charts/handle.rs
new
+71
@@ -0,0 +1,71 @@
1
+//! Chart handle for updating metric values.
2
+
3
+use parking_lot::RwLock;
4
+use std::sync::Arc;
5
+
6
+/// A handle to a chart that allows updating its values.
7
+///
8
+/// Multiple handles can exist for the same chart (they share the underlying data via Arc).
9
+/// This allows different parts of your code to update the same chart independently.
10
+#[derive(Clone)]
11
+pub struct ChartHandle<T> {
12
+ pub(crate) data: Arc<RwLock<T>>,
13
+}
14
+
15
+impl<T> ChartHandle<T> {
16
+ /// Create a new chart handle with the given initial value
17
+ pub(crate) fn new(initial: T) -> Self {
18
+ Self {
19
+ data: Arc::new(RwLock::new(initial)),
20
+ }
21
+ }
22
+
23
+ /// Update the chart data using a closure
24
+ ///
25
+ /// # Example
26
+ ///
27
+ /// ```ignore
28
+ /// handle.update(|metrics| {
29
+ /// metrics.user = 42;
30
+ /// metrics.system = 13;
31
+ /// });
32
+ /// ```
33
+ pub fn update<F>(&self, f: F)
34
+ where
35
+ F: FnOnce(&mut T),
36
+ {
37
+ let mut guard = self.data.write();
38
+ f(&mut *guard);
39
+ }
40
+
41
+ /// Get a write lock to the chart data
42
+ ///
43
+ /// This allows direct mutable access to the data. Remember to drop
44
+ /// the guard when you're done to release the lock.
45
+ ///
46
+ /// # Example
47
+ ///
48
+ /// ```ignore
49
+ /// {
50
+ /// let mut guard = handle.write();
51
+ /// guard.user = 42;
52
+ /// guard.system = 13;
53
+ /// } // Lock released here
54
+ /// ```
55
+ pub fn write(&self) -> parking_lot::RwLockWriteGuard<'_, T> {
56
+ self.data.write()
57
+ }
58
+
59
+ /// Get a read lock to the chart data
60
+ pub fn read(&self) -> parking_lot::RwLockReadGuard<'_, T> {
61
+ self.data.read()
62
+ }
63
+}
64
+
65
+impl<T: std::fmt::Debug> std::fmt::Debug for ChartHandle<T> {
66
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
67
+ f.debug_struct("ChartHandle")
68
+ .field("data", &*self.data.read())
69
+ .finish()
70
+ }
71
+}
src/crates/netdata-plugin/rt/src/charts/metadata.rs
new
+198
@@ -0,0 +1,198 @@
1
+//! Chart and dimension metadata types.
2
+
3
+use std::collections::HashMap;
4
+
5
+/// Chart type supported by Netdata
6
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
7
+pub enum ChartType {
8
+ #[default]
9
+ Line,
10
+ Area,
11
+ Stacked,
12
+}
13
+
14
+impl ChartType {
15
+ pub fn as_str(&self) -> &'static str {
16
+ match self {
17
+ ChartType::Line => "line",
18
+ ChartType::Area => "area",
19
+ ChartType::Stacked => "stacked",
20
+ }
21
+ }
22
+}
23
+
24
+/// Dimension algorithm for value processing
25
+#[derive(Debug, Clone, PartialEq, Eq, Default)]
26
+pub enum DimensionAlgorithm {
27
+ /// Store the value as-is
28
+ #[default]
29
+ Absolute,
30
+ /// Calculate difference from previous value (for counters)
31
+ Incremental,
32
+ /// Calculate percentage of dimension relative to row total
33
+ PercentageOfAbsoluteRow,
34
+ /// Calculate percentage of dimension relative to incremental row
35
+ PercentageOfIncrementalRow,
36
+}
37
+
38
+impl DimensionAlgorithm {
39
+ pub fn as_str(&self) -> &'static str {
40
+ match self {
41
+ DimensionAlgorithm::Absolute => "absolute",
42
+ DimensionAlgorithm::Incremental => "incremental",
43
+ DimensionAlgorithm::PercentageOfAbsoluteRow => "percentage-of-absolute-row",
44
+ DimensionAlgorithm::PercentageOfIncrementalRow => "percentage-of-incremental-row",
45
+ }
46
+ }
47
+}
48
+
49
+/// Metadata for a single dimension
50
+#[derive(Debug, Clone)]
51
+pub struct DimensionMetadata {
52
+ /// Dimension ID (used in SET commands)
53
+ pub id: String,
54
+ /// Display name (shown in UI)
55
+ pub name: String,
56
+ /// Algorithm for processing values
57
+ pub algorithm: DimensionAlgorithm,
58
+ /// Multiplier for values (default 1)
59
+ pub multiplier: i64,
60
+ /// Divisor for values (default 1)
61
+ pub divisor: i64,
62
+ /// Whether this dimension is hidden
63
+ pub hidden: bool,
64
+}
65
+
66
+impl DimensionMetadata {
67
+ pub fn new(id: impl Into<String>) -> Self {
68
+ let id = id.into();
69
+ Self {
70
+ name: id.clone(),
71
+ id,
72
+ algorithm: DimensionAlgorithm::default(),
73
+ multiplier: 1,
74
+ divisor: 1,
75
+ hidden: false,
76
+ }
77
+ }
78
+
79
+ /// Emit the DIMENSION command
80
+ pub fn emit(&self) -> String {
81
+ let flags = if self.hidden { "hidden" } else { "" };
82
+ format!(
83
+ "DIMENSION {} '{}' {} {} {} {}\n",
84
+ self.id,
85
+ self.name,
86
+ self.algorithm.as_str(),
87
+ self.multiplier,
88
+ self.divisor,
89
+ flags
90
+ )
91
+ }
92
+}
93
+
94
+/// Metadata for a chart
95
+#[derive(Debug, Clone)]
96
+pub struct ChartMetadata {
97
+ /// Chart ID (template: "cpu.{instance}" or concrete: "cpu.cpu0")
98
+ pub id: String,
99
+ /// Chart name (optional, usually empty)
100
+ pub name: String,
101
+ /// Chart title (shown in UI)
102
+ pub title: String,
103
+ /// Units for the chart
104
+ pub units: String,
105
+ /// Family grouping
106
+ pub family: String,
107
+ /// Context for alerts and API
108
+ pub context: String,
109
+ /// Chart type (line, area, stacked)
110
+ pub chart_type: ChartType,
111
+ /// Priority for ordering (lower = higher priority)
112
+ pub priority: i64,
113
+ /// Update interval in seconds
114
+ pub update_every: u64,
115
+ /// Dimensions in this chart
116
+ pub dimensions: HashMap<String, DimensionMetadata>,
117
+ /// The field name that serves as the instance identifier (if this is a template)
118
+ pub instance_field: Option<String>,
119
+}
120
+
121
+impl ChartMetadata {
122
+ pub fn new(id: impl Into<String>) -> Self {
123
+ let id = id.into();
124
+ Self {
125
+ title: id.clone(),
126
+ context: id.clone(),
127
+ id,
128
+ name: String::new(),
129
+ units: String::from("value"),
130
+ family: String::new(),
131
+ chart_type: ChartType::default(),
132
+ priority: 1000,
133
+ update_every: 1,
134
+ dimensions: HashMap::new(),
135
+ instance_field: None,
136
+ }
137
+ }
138
+
139
+ /// Check if this is a template chart (contains {instance})
140
+ pub fn is_template(&self) -> bool {
141
+ self.instance_field.is_some() || self.id.contains("{instance}")
142
+ }
143
+
144
+ /// Instantiate a template with a concrete instance ID
145
+ pub fn instantiate(&self, instance_id: &str) -> Self {
146
+ ChartMetadata {
147
+ id: self.id.replace("{instance}", instance_id),
148
+ name: self.name.replace("{instance}", instance_id),
149
+ title: self.title.replace("{instance}", instance_id),
150
+ units: self.units.clone(),
151
+ family: self.family.replace("{instance}", instance_id),
152
+ context: self.context.replace("{instance}", instance_id),
153
+ chart_type: self.chart_type.clone(),
154
+ priority: self.priority,
155
+ update_every: self.update_every,
156
+ dimensions: self.dimensions.clone(),
157
+ instance_field: None, // No longer a template after instantiation
158
+ }
159
+ }
160
+
161
+ /// Emit the CHART command
162
+ pub fn emit_definition(&self) -> String {
163
+ let mut output = format!(
164
+ "CHART {} '{}' '{}' '{}' '{}' '{}' {} {} {}\n",
165
+ self.id,
166
+ self.name,
167
+ self.title,
168
+ self.units,
169
+ self.family,
170
+ self.context,
171
+ self.chart_type.as_str(),
172
+ self.priority,
173
+ self.update_every
174
+ );
175
+
176
+ // Emit dimensions
177
+ for dim in self.dimensions.values() {
178
+ output.push_str(&dim.emit());
179
+ }
180
+
181
+ output
182
+ }
183
+
184
+ /// Emit BEGIN command
185
+ pub fn emit_begin(&self) -> String {
186
+ format!("BEGIN {}\n", self.id)
187
+ }
188
+
189
+ /// Emit SET command for a dimension
190
+ pub fn emit_set(&self, dimension_id: &str, value: i64) -> String {
191
+ format!("SET {} = {}\n", dimension_id, value)
192
+ }
193
+
194
+ /// Emit END command
195
+ pub fn emit_end(&self) -> String {
196
+ "END\n".to_string()
197
+ }
198
+}
src/crates/netdata-plugin/rt/src/charts/mod.rs
new
+20
@@ -0,0 +1,20 @@
1
+//! Declarative chart creation and management for Netdata plugins.
2
+//!
3
+//! This module provides declarative chart definitions using Rust structs
4
+//! annotated with `schemars` attributes. A derive macro generates efficient dimension
5
+//! writing code, and the registry handles automatic sampling and batched emission.
6
+
7
+mod chart_trait;
8
+mod handle;
9
+mod metadata;
10
+mod registry;
11
+mod tracker;
12
+mod writer;
13
+
14
+// Re-export public API
15
+pub use chart_trait::{ChartDimensions, InstancedChart, NetdataChart};
16
+pub use handle::ChartHandle;
17
+pub use metadata::{ChartMetadata, ChartType, DimensionAlgorithm, DimensionMetadata};
18
+pub use registry::ChartRegistry;
19
+pub use tracker::TrackedChart;
20
+pub use writer::ChartWriter;
src/crates/netdata-plugin/rt/src/charts/registry.rs
new
+228
@@ -0,0 +1,228 @@
1
+//! Chart registry for managing and scheduling chart updates.
2
+
3
+use super::chart_trait::{InstancedChart, NetdataChart};
4
+use super::handle::ChartHandle;
5
+use super::tracker::TrackedChart;
6
+use super::writer::ChartWriter;
7
+use async_trait::async_trait;
8
+use bytes::BytesMut;
9
+use netdata_plugin_protocol::MessageWriter;
10
+use parking_lot::RwLock;
11
+use std::sync::Arc;
12
+use std::time::Duration;
13
+use tokio::io::AsyncWrite;
14
+use tokio::sync::Mutex;
15
+use tokio::task::JoinSet;
16
+use tokio_util::sync::CancellationToken;
17
+
18
+/// Registry for managing charts and their sampling schedules.
19
+///
20
+/// The registry maintains a collection of charts and spawns background tasks
21
+/// to sample them at their configured intervals. Chart data is written through
22
+/// the provided MessageWriter, coordinating with other plugin output.
23
+pub struct ChartRegistry<W>
24
+where
25
+ W: AsyncWrite + Unpin + Send + 'static,
26
+{
27
+ samplers: Vec<Box<dyn ChartSampler>>,
28
+ cancellation: CancellationToken,
29
+ writer: Arc<Mutex<MessageWriter<W>>>,
30
+}
31
+
32
+impl<W> ChartRegistry<W>
33
+where
34
+ W: AsyncWrite + Unpin + Send + 'static,
35
+{
36
+ /// Create a new chart registry with a shared message writer
37
+ pub fn new(writer: Arc<Mutex<MessageWriter<W>>>) -> Self {
38
+ Self {
39
+ samplers: Vec::new(),
40
+ cancellation: CancellationToken::new(),
41
+ writer,
42
+ }
43
+ }
44
+
45
+ /// Register a chart and get a handle to update it.
46
+ ///
47
+ /// The chart will be sampled at the given interval and updates will be
48
+ /// emitted through the message writer using the Netdata chart protocol.
49
+ ///
50
+ /// # Example
51
+ ///
52
+ /// ```ignore
53
+ /// let handle = registry.register_chart(
54
+ /// CpuMetrics::default(),
55
+ /// Duration::from_secs(1),
56
+ /// );
57
+ ///
58
+ /// // Update the chart from anywhere
59
+ /// handle.update(|m| {
60
+ /// m.user = 42;
61
+ /// m.system = 13;
62
+ /// });
63
+ /// ```
64
+ pub fn register_chart<T>(&mut self, initial: T, interval: Duration) -> ChartHandle<T>
65
+ where
66
+ T: NetdataChart + Default + PartialEq + Clone + Send + Sync + 'static,
67
+ {
68
+ let handle = ChartHandle::new(initial.clone());
69
+
70
+ let sampler = SingletonChartSampler {
71
+ data: handle.clone(),
72
+ tracker: TrackedChart::new(initial, interval),
73
+ writer: ChartWriter::new(),
74
+ };
75
+
76
+ self.samplers.push(Box::new(sampler));
77
+ handle
78
+ }
79
+
80
+ /// Register an instanced chart (for per-instance charts like per-CPU metrics).
81
+ ///
82
+ /// This is a specialized version of register_chart that properly handles
83
+ /// template instantiation for charts that have instance identifiers.
84
+ pub fn register_instanced_chart<T>(&mut self, initial: T, interval: Duration) -> ChartHandle<T>
85
+ where
86
+ T: InstancedChart + Default + PartialEq + Send + Sync + 'static,
87
+ {
88
+ let handle = ChartHandle::new(initial.clone());
89
+
90
+ let sampler = SingletonChartSampler {
91
+ data: handle.clone(),
92
+ tracker: TrackedChart::new_instanced(initial, interval),
93
+ writer: ChartWriter::new(),
94
+ };
95
+
96
+ self.samplers.push(Box::new(sampler));
97
+ handle
98
+ }
99
+
100
+ /// Get a cancellation token that can be used to stop the registry
101
+ pub fn cancellation_token(&self) -> CancellationToken {
102
+ self.cancellation.clone()
103
+ }
104
+
105
+ /// Run the registry, sampling all charts at their configured intervals.
106
+ ///
107
+ /// This method consumes the registry and runs until cancelled.
108
+ pub async fn run(mut self) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
109
+ const BATCH_SIZE: usize = 1000;
110
+ let mut tasks = JoinSet::new();
111
+
112
+ // Group samplers into batches to reduce writer lock contention
113
+ let mut batch_samplers = Vec::new();
114
+ let mut current_batch = Vec::new();
115
+
116
+ for sampler in self.samplers.drain(..) {
117
+ current_batch.push(sampler);
118
+ if current_batch.len() >= BATCH_SIZE {
119
+ batch_samplers.push(std::mem::take(&mut current_batch));
120
+ }
121
+ }
122
+
123
+ // Don't forget the last partial batch
124
+ if !current_batch.is_empty() {
125
+ batch_samplers.push(current_batch);
126
+ }
127
+
128
+ for mut batch in batch_samplers {
129
+ let token = self.cancellation.child_token();
130
+ let writer = Arc::clone(&self.writer);
131
+
132
+ // All samplers in a batch should have the same interval (typically 1 second)
133
+ let interval = batch
134
+ .first()
135
+ .map(|s| s.interval())
136
+ .unwrap_or(Duration::from_secs(1));
137
+
138
+ tasks.spawn(async move {
139
+ // Reusable buffer for entire batch (512KB should be enough for 1000 charts)
140
+ let mut batch_buffer = BytesMut::with_capacity(512 * 1024);
141
+ let mut interval_timer = tokio::time::interval(interval);
142
+ // Use Burst to catch up if a tick is delayed, ensuring consistent emission rate
143
+ interval_timer.set_missed_tick_behavior(tokio::time::MissedTickBehavior::Burst);
144
+
145
+ loop {
146
+ tokio::select! {
147
+ _ = token.cancelled() => break,
148
+ _ = interval_timer.tick() => {
149
+ batch_buffer.clear();
150
+
151
+ // Capture collection time at the end of the interval
152
+ let collection_time = std::time::SystemTime::now();
153
+
154
+ // Sample all charts in batch to the shared buffer
155
+ for sampler in &mut batch {
156
+ sampler.sample_to_buffer(&mut batch_buffer, collection_time).await;
157
+ }
158
+
159
+ // Single writer lock acquisition for entire batch
160
+ if !batch_buffer.is_empty() {
161
+ let mut w = writer.lock().await;
162
+ let _ = w.write_raw(&batch_buffer).await;
163
+ }
164
+ }
165
+ }
166
+ }
167
+ });
168
+ }
169
+
170
+ // Wait for all batch tasks to finish
171
+ while let Some(result) = tasks.join_next().await {
172
+ result?;
173
+ }
174
+
175
+ Ok(())
176
+ }
177
+}
178
+
179
+/// Internal trait for chart samplers
180
+#[async_trait]
181
+trait ChartSampler: Send + Sync {
182
+ /// Sample the chart and write to the provided buffer (without flushing to stdout)
183
+ ///
184
+ /// # Parameters
185
+ /// - `buffer`: Buffer to write the chart data to
186
+ /// - `collection_time`: When the data was collected
187
+ async fn sample_to_buffer(&mut self, buffer: &mut bytes::BytesMut, collection_time: std::time::SystemTime);
188
+ fn interval(&self) -> Duration;
189
+}
190
+
191
+/// Sampler for singleton charts
192
+struct SingletonChartSampler<T> {
193
+ data: ChartHandle<T>,
194
+ tracker: TrackedChart<T>,
195
+ writer: ChartWriter,
196
+}
197
+
198
+#[async_trait]
199
+impl<T> ChartSampler for SingletonChartSampler<T>
200
+where
201
+ T: NetdataChart + Default + PartialEq + Clone + Send + Sync,
202
+{
203
+ async fn sample_to_buffer(&mut self, buffer: &mut BytesMut, collection_time: std::time::SystemTime) {
204
+ // Sample the current value
205
+ let current = {
206
+ let guard = self.data.read();
207
+ (*guard).clone()
208
+ };
209
+
210
+ // Update tracker
211
+ self.tracker.update(current);
212
+
213
+ // Emit definition if first time
214
+ if !self.tracker.defined {
215
+ self.tracker.emit_definition(&mut self.writer);
216
+ }
217
+
218
+ // Always emit update - Netdata requires regular updates even if values don't change
219
+ self.tracker.emit_update(&mut self.writer, collection_time);
220
+
221
+ // Append writer's buffer to the batch buffer and clear writer
222
+ self.writer.append_to(buffer);
223
+ }
224
+
225
+ fn interval(&self) -> Duration {
226
+ self.tracker.interval
227
+ }
228
+}
src/crates/netdata-plugin/rt/src/charts/tracker.rs
new
+148
@@ -0,0 +1,148 @@
1
+//! Tracked chart with change detection and emission.
2
+
3
+use super::chart_trait::{InstancedChart, NetdataChart};
4
+use super::metadata::ChartMetadata;
5
+use super::writer::ChartWriter;
6
+use std::time::{Duration, SystemTime};
7
+
8
+/// A tracked chart that detects changes and emits Netdata protocol commands.
9
+pub struct TrackedChart<T> {
10
+ current: T,
11
+ previous: T,
12
+ pub(crate) metadata: ChartMetadata,
13
+ pub(crate) interval: Duration,
14
+ pub(crate) defined: bool,
15
+}
16
+
17
+impl<T: NetdataChart + Default + PartialEq + Clone> TrackedChart<T> {
18
+ /// Create a new tracked chart with the given initial value and interval
19
+ pub fn new(initial: T, interval: Duration) -> Self {
20
+ let metadata = T::chart_metadata();
21
+ Self::new_with_metadata(initial, interval, metadata)
22
+ }
23
+
24
+ /// Create a new tracked chart with explicit metadata (used for instantiated templates)
25
+ pub(crate) fn new_with_metadata(initial: T, interval: Duration, metadata: ChartMetadata) -> Self {
26
+ Self {
27
+ previous: initial.clone(),
28
+ current: initial,
29
+ metadata,
30
+ interval,
31
+ defined: false,
32
+ }
33
+ }
34
+
35
+ /// Update the current values
36
+ pub fn update(&mut self, new_values: T) {
37
+ self.previous = std::mem::replace(&mut self.current, new_values);
38
+ }
39
+
40
+ /// Check if values changed since last update
41
+ ///
42
+ /// Note: This is provided for informational purposes (e.g., logging, debugging).
43
+ /// The registry always emits updates regardless of changes, as required by Netdata's protocol.
44
+ pub fn has_changed(&self) -> bool {
45
+ self.current != self.previous
46
+ }
47
+
48
+ /// Emit the chart definition (CHART + DIMENSION commands) to the writer
49
+ ///
50
+ /// This should be called once before emitting any updates.
51
+ pub fn emit_definition(&mut self, writer: &mut ChartWriter) {
52
+ if self.defined {
53
+ return;
54
+ }
55
+ self.defined = true;
56
+ writer.write_chart_definition(&self.metadata);
57
+ }
58
+
59
+ /// Emit a chart update (BEGIN + SET + END commands) to the writer
60
+ ///
61
+ /// This uses the ChartDimensions trait for efficient zero-allocation dimension writing.
62
+ ///
63
+ /// # Parameters
64
+ /// - `writer`: The writer to emit the update to
65
+ /// - `collection_time`: When the data was collected
66
+ pub fn emit_update(&self, writer: &mut ChartWriter, collection_time: SystemTime)
67
+ where
68
+ T: super::chart_trait::ChartDimensions,
69
+ {
70
+ writer.begin_chart(&self.metadata.id, self.interval);
71
+ self.current.write_dimensions(writer);
72
+ writer.end_chart(collection_time);
73
+ }
74
+}
75
+
76
+// For instanced charts, we need special handling
77
+impl<T: InstancedChart + Default + PartialEq> TrackedChart<T> {
78
+ /// Create a tracked chart for an instanced chart
79
+ pub fn new_instanced(initial: T, interval: Duration) -> Self {
80
+ let template_metadata = T::chart_metadata();
81
+ let instance_id = initial.instance_id();
82
+ let metadata = template_metadata.instantiate(instance_id);
83
+
84
+ Self {
85
+ previous: initial.clone(),
86
+ current: initial,
87
+ metadata,
88
+ interval,
89
+ defined: false,
90
+ }
91
+ }
92
+}
93
+
94
+#[cfg(test)]
95
+mod tests {
96
+ use super::*;
97
+ use schemars::JsonSchema;
98
+ use serde::{Deserialize, Serialize};
99
+
100
+ #[derive(JsonSchema, Default, Clone, PartialEq, Serialize, Deserialize)]
101
+ #[schemars(
102
+ extend("x-chart-id" = "test.metrics"),
103
+ extend("x-chart-title" = "Test Metrics"),
104
+ )]
105
+ struct TestMetrics {
106
+ value1: u64,
107
+ value2: u64,
108
+ }
109
+
110
+ impl super::super::ChartDimensions for TestMetrics {
111
+ fn write_dimensions(&self, writer: &mut ChartWriter) {
112
+ writer.write_dimension("value1", self.value1 as i64);
113
+ writer.write_dimension("value2", self.value2 as i64);
114
+ }
115
+ }
116
+
117
+ #[test]
118
+ fn test_change_detection() {
119
+ let initial = TestMetrics { value1: 10, value2: 20 };
120
+ let mut tracker = TrackedChart::new(initial.clone(), Duration::from_secs(1));
121
+
122
+ assert!(!tracker.has_changed());
123
+
124
+ tracker.update(TestMetrics { value1: 15, value2: 20 });
125
+ assert!(tracker.has_changed());
126
+
127
+ tracker.update(TestMetrics { value1: 15, value2: 20 });
128
+ assert!(!tracker.has_changed());
129
+ }
130
+
131
+ #[test]
132
+ fn test_emit_definition() {
133
+ let initial = TestMetrics::default();
134
+ let mut tracker = TrackedChart::new(initial, Duration::from_secs(1));
135
+ let mut writer = ChartWriter::new();
136
+
137
+ tracker.emit_definition(&mut writer);
138
+ let def = String::from_utf8_lossy(writer.buffer());
139
+ assert!(def.contains("CHART test.metrics"));
140
+ assert!(def.contains("DIMENSION value1"));
141
+ assert!(def.contains("DIMENSION value2"));
142
+
143
+ // Second call should not write anything
144
+ writer.clear();
145
+ tracker.emit_definition(&mut writer);
146
+ assert_eq!(writer.buffer_len(), 0);
147
+ }
148
+}
src/crates/netdata-plugin/rt/src/charts/writer.rs
new
+227
@@ -0,0 +1,227 @@
1
+//! High-performance chart writer with minimal allocations.
2
+
3
+use super::metadata::{ChartMetadata, DimensionMetadata};
4
+use bytes::{BufMut, BytesMut};
5
+use std::io::{self, Write};
6
+use std::time::{Duration, SystemTime, UNIX_EPOCH};
7
+
8
+/// High-performance writer for Netdata chart protocol.
9
+///
10
+/// Uses a reusable buffer to minimize allocations. The buffer is reused across
11
+/// multiple chart updates, only growing when necessary.
12
+pub struct ChartWriter {
13
+ buffer: BytesMut,
14
+}
15
+
16
+impl ChartWriter {
17
+ /// Create a new chart writer with default capacity (4KB)
18
+ pub fn new() -> Self {
19
+ Self::with_capacity(4096)
20
+ }
21
+
22
+ /// Create a new chart writer with specified capacity
23
+ pub fn with_capacity(capacity: usize) -> Self {
24
+ Self {
25
+ buffer: BytesMut::with_capacity(capacity),
26
+ }
27
+ }
28
+
29
+ /// Write a chart definition (CHART + DIMENSION commands)
30
+ pub fn write_chart_definition(&mut self, metadata: &ChartMetadata) {
31
+ // CHART command
32
+ self.buffer.put_slice(b"CHART ");
33
+ self.buffer.put_slice(metadata.id.as_bytes());
34
+ self.buffer.put_slice(b" '");
35
+ self.buffer.put_slice(metadata.name.as_bytes());
36
+ self.buffer.put_slice(b"' '");
37
+ self.buffer.put_slice(metadata.title.as_bytes());
38
+ self.buffer.put_slice(b"' '");
39
+ self.buffer.put_slice(metadata.units.as_bytes());
40
+ self.buffer.put_slice(b"' '");
41
+ self.buffer.put_slice(metadata.family.as_bytes());
42
+ self.buffer.put_slice(b"' '");
43
+ self.buffer.put_slice(metadata.context.as_bytes());
44
+ self.buffer.put_slice(b"' ");
45
+ self.buffer.put_slice(metadata.chart_type.as_str().as_bytes());
46
+ self.buffer.put_slice(b" ");
47
+ self.write_i64(metadata.priority);
48
+ self.buffer.put_slice(b" ");
49
+ self.write_u64(metadata.update_every);
50
+ self.buffer.put_u8(b'\n');
51
+
52
+ // DIMENSION commands
53
+ for dim in metadata.dimensions.values() {
54
+ self.write_dimension_definition(dim);
55
+ }
56
+ }
57
+
58
+ /// Write a dimension definition (DIMENSION command)
59
+ fn write_dimension_definition(&mut self, dim: &DimensionMetadata) {
60
+ self.buffer.put_slice(b"DIMENSION ");
61
+ self.buffer.put_slice(dim.id.as_bytes());
62
+ self.buffer.put_slice(b" '");
63
+ self.buffer.put_slice(dim.name.as_bytes());
64
+ self.buffer.put_slice(b"' ");
65
+ self.buffer.put_slice(dim.algorithm.as_str().as_bytes());
66
+ self.buffer.put_slice(b" ");
67
+ self.write_i64(dim.multiplier);
68
+ self.buffer.put_slice(b" ");
69
+ self.write_i64(dim.divisor);
70
+
71
+ if dim.hidden {
72
+ self.buffer.put_slice(b" hidden");
73
+ }
74
+
75
+ self.buffer.put_u8(b'\n');
76
+ }
77
+
78
+ /// Begin a chart update (BEGIN command)
79
+ ///
80
+ /// The `update_every` specifies the collection interval.
81
+ /// This helps Netdata perform accurate interpolation.
82
+ pub fn begin_chart(&mut self, chart_id: &str, update_every: Duration) {
83
+ self.buffer.put_slice(b"BEGIN ");
84
+ self.buffer.put_slice(chart_id.as_bytes());
85
+ self.buffer.put_u8(b' ');
86
+ // Netdata expects microseconds
87
+ self.write_u64(update_every.as_micros() as u64);
88
+ self.buffer.put_u8(b'\n');
89
+ }
90
+
91
+ /// Write a dimension value (SET command)
92
+ pub fn write_dimension(&mut self, dimension_id: &str, value: i64) {
93
+ self.buffer.put_slice(b"SET ");
94
+ self.buffer.put_slice(dimension_id.as_bytes());
95
+ self.buffer.put_slice(b" = ");
96
+ self.write_i64(value);
97
+ self.buffer.put_u8(b'\n');
98
+ }
99
+
100
+ /// End a chart update (END command)
101
+ ///
102
+ /// The `collection_time` specifies when the data was collected.
103
+ /// This allows Netdata to accurately align data points and perform proper interpolation.
104
+ pub fn end_chart(&mut self, collection_time: SystemTime) {
105
+ self.buffer.put_slice(b"END ");
106
+ // Netdata expects Unix timestamp in seconds
107
+ let secs = collection_time
108
+ .duration_since(UNIX_EPOCH)
109
+ .unwrap_or(Duration::ZERO)
110
+ .as_secs();
111
+ self.write_u64(secs);
112
+ self.buffer.put_u8(b'\n');
113
+ }
114
+
115
+ /// Write an i64 value using itoa (zero-allocation integer formatting)
116
+ #[inline]
117
+ fn write_i64(&mut self, value: i64) {
118
+ let mut buf = itoa::Buffer::new();
119
+ let s = buf.format(value);
120
+ self.buffer.put_slice(s.as_bytes());
121
+ }
122
+
123
+ /// Write a u64 value using itoa (zero-allocation integer formatting)
124
+ #[inline]
125
+ fn write_u64(&mut self, value: u64) {
126
+ let mut buf = itoa::Buffer::new();
127
+ let s = buf.format(value);
128
+ self.buffer.put_slice(s.as_bytes());
129
+ }
130
+
131
+ /// Flush the buffer to stdout
132
+ pub fn flush(&mut self) -> io::Result<()> {
133
+ let stdout = io::stdout();
134
+ let mut handle = stdout.lock();
135
+ handle.write_all(&self.buffer)?;
136
+ handle.flush()?;
137
+ self.buffer.clear();
138
+ Ok(())
139
+ }
140
+
141
+ /// Get the current buffer size (for monitoring/debugging)
142
+ pub fn buffer_len(&self) -> usize {
143
+ self.buffer.len()
144
+ }
145
+
146
+ /// Get a reference to the buffer (for testing)
147
+ pub fn buffer(&self) -> &[u8] {
148
+ &self.buffer
149
+ }
150
+
151
+ /// Clear the buffer without flushing
152
+ pub fn clear(&mut self) {
153
+ self.buffer.clear();
154
+ }
155
+
156
+ /// Append this writer's buffer to another buffer and clear this writer
157
+ pub fn append_to(&mut self, target: &mut BytesMut) {
158
+ target.put(&self.buffer[..]);
159
+ self.buffer.clear();
160
+ }
161
+}
162
+
163
+impl Default for ChartWriter {
164
+ fn default() -> Self {
165
+ Self::new()
166
+ }
167
+}
168
+
169
+#[cfg(test)]
170
+mod tests {
171
+ use super::*;
172
+ use crate::{ChartMetadata, ChartType, DimensionAlgorithm, DimensionMetadata};
173
+
174
+ #[test]
175
+ fn test_write_chart_definition() {
176
+ let mut writer = ChartWriter::new();
177
+ let mut metadata = ChartMetadata::new("test.chart");
178
+ metadata.title = "Test Chart".to_string();
179
+ metadata.units = "widgets".to_string();
180
+ metadata.chart_type = ChartType::Line;
181
+
182
+ let mut dim = DimensionMetadata::new("value1");
183
+ dim.algorithm = DimensionAlgorithm::Absolute;
184
+ metadata.dimensions.insert("value1".to_string(), dim);
185
+
186
+ writer.write_chart_definition(&metadata);
187
+
188
+ let output = String::from_utf8_lossy(&writer.buffer);
189
+ assert!(output.contains("CHART test.chart"));
190
+ assert!(output.contains("Test Chart"));
191
+ assert!(output.contains("DIMENSION value1"));
192
+ }
193
+
194
+ #[test]
195
+ fn test_write_chart_update() {
196
+ let mut writer = ChartWriter::new();
197
+
198
+ writer.begin_chart("test.chart", Duration::from_secs(1));
199
+ writer.write_dimension("value1", 42);
200
+ writer.write_dimension("value2", 13);
201
+ writer.end_chart(UNIX_EPOCH + Duration::from_secs(1609459200)); // 2021-01-01 00:00:00 UTC
202
+
203
+ let output = String::from_utf8_lossy(&writer.buffer);
204
+ assert_eq!(output, "BEGIN test.chart 1000000\nSET value1 = 42\nSET value2 = 13\nEND 1609459200\n");
205
+ }
206
+
207
+ #[test]
208
+ fn test_reusable_buffer() {
209
+ let mut writer = ChartWriter::new();
210
+
211
+ // First update
212
+ writer.begin_chart("test.chart", Duration::from_secs(1));
213
+ writer.write_dimension("value", 1);
214
+ writer.end_chart(UNIX_EPOCH + Duration::from_secs(1609459200));
215
+ let len1 = writer.buffer_len();
216
+ writer.clear();
217
+
218
+ // Second update - buffer should be reused
219
+ writer.begin_chart("test.chart", Duration::from_secs(1));
220
+ writer.write_dimension("value", 2);
221
+ writer.end_chart(UNIX_EPOCH + Duration::from_secs(1609459201));
222
+ let len2 = writer.buffer_len();
223
+
224
+ // Buffer capacity should be same (reused)
225
+ assert_eq!(len1, len2);
226
+ }
227
+}
src/crates/netdata-plugin/rt/src/lib.rs
new
+1032
@@ -0,0 +1,1032 @@
1
+//! A runtime framework for building Netdata plugins with asynchronous function handlers.
2
+//!
3
+//! This crate provides a complete runtime system for creating Netdata plugins that can expose
4
+//! custom functions to the Netdata monitoring system. It handles all the communication protocol,
5
+//! serialization, concurrent execution, and lifecycle management.
6
+//!
7
+//! # Overview
8
+//!
9
+//! The framework is built around the [`FunctionHandler`] trait, which developers implement to
10
+//! create custom functions that Netdata can call. The [`PluginRuntime`] manages these handlers
11
+//! and provides:
12
+//!
13
+//! - Automatic JSON serialization/deserialization
14
+//! - Concurrent function execution
15
+//! - Graceful cancellation support
16
+//! - Progress reporting capabilities
17
+//! - Transaction management
18
+//! - Clean shutdown handling
19
+//!
20
+//! # Example
21
+//!
22
+//! ```no_run
23
+//! use async_trait::async_trait;
24
+//! use netdata_plugin_error::Result;
25
+//! use netdata_plugin_protocol::FunctionDeclaration;
26
+//! use rt::{FunctionHandler, PluginRuntime};
27
+//! use serde::{Deserialize, Serialize};
28
+//!
29
+//! #[derive(Deserialize)]
30
+//! struct MyRequest {
31
+//! name: String,
32
+//! }
33
+//!
34
+//! #[derive(Serialize)]
35
+//! struct MyResponse {
36
+//! greeting: String,
37
+//! }
38
+//!
39
+//! struct MyHandler;
40
+//!
41
+//! #[async_trait]
42
+//! impl FunctionHandler for MyHandler {
43
+//! type Request = MyRequest;
44
+//! type Response = MyResponse;
45
+//!
46
+//! async fn on_call(&self, request: Self::Request) -> Result<Self::Response> {
47
+//! Ok(MyResponse {
48
+//! greeting: format!("Hello, {}!", request.name),
49
+//! })
50
+//! }
51
+//!
52
+//! async fn on_cancellation(&self) -> Result<Self::Response> {
53
+//! Err(netdata_plugin_error::NetdataPluginError::Other {
54
+//! message: "Operation cancelled".to_string(),
55
+//! })
56
+//! }
57
+//!
58
+//! async fn on_progress(&self) {
59
+//! // Report progress if needed
60
+//! }
61
+//!
62
+//! fn declaration(&self) -> FunctionDeclaration {
63
+//! FunctionDeclaration::new("greet", "A greeting function")
64
+//! }
65
+//! }
66
+//!
67
+//! #[tokio::main]
68
+//! async fn main() -> std::result::Result<(), Box<dyn std::error::Error>> {
69
+//! let mut runtime = PluginRuntime::new("my_plugin");
70
+//! runtime.register_handler(MyHandler);
71
+//! runtime.run().await?;
72
+//! Ok(())
73
+//! }
74
+//! ```
75
+//!
76
+//! # Architecture
77
+//!
78
+//! ## Communication Flow
79
+//!
80
+//! 1. Plugin declares available functions to Netdata via stdout
81
+//! 2. Netdata sends function calls via stdin
82
+//! 3. Runtime dispatches calls to registered handlers
83
+//! 4. Handlers execute asynchronously with cancellation/progress support
84
+//! 5. Results are sent back to Netdata via stdout
85
+//!
86
+//! ## Concurrency Model
87
+//!
88
+//! The runtime uses Tokio for asynchronous execution, allowing multiple function calls to be
89
+//! processed concurrently. Each function call is tracked as a transaction with its own
90
+//! cancellation token and control channel.
91
+
92
+#![allow(unused_imports)]
93
+
94
+use async_trait::async_trait;
95
+use futures::StreamExt;
96
+use futures::future::BoxFuture;
97
+use futures::stream::FuturesUnordered;
98
+use netdata_plugin_error::Result;
99
+use netdata_plugin_protocol::{
100
+ FunctionCall, FunctionCancel, FunctionDeclaration, FunctionProgress, FunctionResult, Message,
101
+ MessageReader, MessageWriter,
102
+};
103
+use serde::Serialize;
104
+use serde::de::DeserializeOwned;
105
+use serde_json::json;
106
+use std::collections::HashMap;
107
+use std::sync::Arc;
108
+use std::time::Duration;
109
+use tokio::io::{AsyncRead, AsyncWrite};
110
+use tokio::sync::{Mutex, mpsc};
111
+use tokio_util::sync::CancellationToken;
112
+use tracing::{debug, error, info, instrument, warn};
113
+
114
+// Charts module and re-exports
115
+pub mod charts;
116
+pub use charts::{
117
+ ChartDimensions, ChartHandle, ChartMetadata, ChartRegistry, ChartType, DimensionAlgorithm,
118
+ DimensionMetadata, InstancedChart, TrackedChart,
119
+};
120
+
121
+// Re-export the trait and derive macro
122
+// Note: In Rust, derive macros and traits can have the same name because they're in different namespaces
123
+pub use charts::NetdataChart;
124
+pub use netdata_plugin_charts_derive::NetdataChart;
125
+
126
+// Netdata environment utilities
127
+pub mod netdata_env;
128
+pub use netdata_env::{LogFormat, LogLevel, LogMethod, NetdataEnv, SyslogFacility};
129
+
130
+// Tracing initialization
131
+mod tracing_setup;
132
+pub use tracing_setup::init_tracing;
133
+
134
+// Re-export foundational utilities
135
+pub use foundation::Timeout;
136
+
137
+/// Internal control signals sent to running functions.
138
+enum RuntimeSignal {
139
+ /// Signal to request progress update from a running function.
140
+ Progress,
141
+}
142
+
143
+/// Represents an active function call transaction.
144
+///
145
+/// Each transaction tracks a single function invocation, including its
146
+/// unique identifier, control channel for signals, and cancellation token.
147
+struct Transaction {
148
+ /// Unique identifier for this transaction.
149
+ id: String,
150
+ /// Channel for sending control signals to the running function.
151
+ control_tx: mpsc::Sender<RuntimeSignal>,
152
+ /// Token for cancelling this specific function execution.
153
+ cancellation_token: CancellationToken,
154
+}
155
+
156
+/// Execution context provided to function handlers.
157
+///
158
+/// Contains all the information and control mechanisms needed for
159
+/// a function to execute, handle cancellation, and report progress.
160
+struct FunctionContext {
161
+ /// The original function call request from Netdata.
162
+ function_call: Box<FunctionCall>,
163
+ /// Token for detecting cancellation requests.
164
+ cancellation_token: CancellationToken,
165
+ /// Receiver for runtime control signals (e.g., progress requests).
166
+ signal_rx: Mutex<mpsc::Receiver<RuntimeSignal>>,
167
+}
168
+
169
+/// Type alias for a future that produces a function result.
170
+type FunctionFuture = BoxFuture<'static, (String, FunctionResult)>;
171
+
172
+/// Trait for implementing Netdata function handlers.
173
+///
174
+/// This is the main trait that developers implement to create custom functions
175
+/// that can be called by Netdata. The trait provides automatic serialization,
176
+/// cancellation handling, and progress reporting.
177
+///
178
+/// # Type Parameters
179
+///
180
+/// * `Request` - The type of the incoming request payload (must be deserializable from JSON)
181
+/// * `Response` - The type of the response payload (must be serializable to JSON)
182
+///
183
+/// # Example
184
+///
185
+/// ```
186
+/// use async_trait::async_trait;
187
+/// use netdata_plugin_error::Result;
188
+/// use serde::{Deserialize, Serialize};
189
+///
190
+/// #[derive(Deserialize)]
191
+/// struct AddRequest {
192
+/// a: i32,
193
+/// b: i32,
194
+/// }
195
+///
196
+/// #[derive(Serialize)]
197
+/// struct AddResponse {
198
+/// sum: i32,
199
+/// }
200
+///
201
+/// struct AddHandler;
202
+///
203
+/// #[async_trait]
204
+/// impl FunctionHandler for AddHandler {
205
+/// type Request = AddRequest;
206
+/// type Response = AddResponse;
207
+///
208
+/// async fn on_call(&self, request: Self::Request) -> Result<Self::Response> {
209
+/// Ok(AddResponse {
210
+/// sum: request.a + request.b,
211
+/// })
212
+/// }
213
+///
214
+/// async fn on_cancellation(&self) -> Result<Self::Response> {
215
+/// Err(netdata_plugin_error::NetdataPluginError::Other {
216
+/// message: "Addition cancelled".to_string(),
217
+/// })
218
+/// }
219
+///
220
+/// async fn on_progress(&self) {
221
+/// // Not needed for quick operations
222
+/// }
223
+///
224
+/// fn declaration(&self) -> FunctionDeclaration {
225
+/// FunctionDeclaration::new("add", "Adds two numbers")
226
+/// }
227
+/// }
228
+/// ```
229
+#[async_trait]
230
+pub trait FunctionHandler: Send + Sync + 'static {
231
+ /// The request payload type that will be deserialized from JSON.
232
+ ///
233
+ /// This type must implement `DeserializeOwned` to be deserializable from
234
+ /// the JSON payload sent by Netdata.
235
+ type Request: DeserializeOwned + Send;
236
+
237
+ /// The response type that will be serialized to JSON.
238
+ ///
239
+ /// This type must implement `Serialize` to be serializable to JSON
240
+ /// for sending back to Netdata.
241
+ type Response: Serialize + Send;
242
+
243
+ /// Main function logic executed when the function is called.
244
+ ///
245
+ /// This method contains the primary computation or operation that the
246
+ /// function performs. It receives the deserialized request and should
247
+ /// return either a successful response or an error.
248
+ ///
249
+ /// # Arguments
250
+ ///
251
+ /// * `request` - The deserialized request payload
252
+ ///
253
+ /// # Returns
254
+ ///
255
+ /// A `Result` containing either the response payload or an error
256
+ ///
257
+ /// # Cancellation
258
+ ///
259
+ /// This method may be interrupted if a cancellation is requested.
260
+ /// When cancelled, the runtime will call [`on_cancellation`](Self::on_cancellation) instead.
261
+ async fn on_call(&self, transaction: String, request: Self::Request) -> Result<Self::Response>;
262
+
263
+ /// Handle cancellation requests while the function is running.
264
+ ///
265
+ /// Called when Netdata requests cancellation of a running function.
266
+ /// This method should quickly return an appropriate error or partial result.
267
+ ///
268
+ /// # Returns
269
+ ///
270
+ /// Typically returns an error indicating the operation was cancelled,
271
+ /// but may return a partial result if appropriate.
272
+ async fn on_cancellation(&self, transaction: String) -> Result<Self::Response>;
273
+
274
+ /// Handle progress report requests while the function is running.
275
+ ///
276
+ /// Called when Netdata requests a progress update from a long-running function.
277
+ /// This method should log or report the current progress but doesn't need
278
+ /// to return a value (progress is typically reported through logging).
279
+ ///
280
+ /// # Note
281
+ ///
282
+ /// This is called asynchronously while `on_call` is still running,
283
+ /// so any shared state must be properly synchronized.
284
+ async fn on_progress(&self, transaction: String);
285
+
286
+ /// Provide the function's declaration metadata.
287
+ ///
288
+ /// Returns a [`FunctionDeclaration`] that describes this function to Netdata,
289
+ /// including its name and description.
290
+ ///
291
+ /// # Returns
292
+ ///
293
+ /// A function declaration with the function's name and description.
294
+ fn declaration(&self) -> FunctionDeclaration;
295
+}
296
+
297
+/// Internal trait for handling raw function calls with serialization.
298
+///
299
+/// This trait is used internally to bridge between the typed [`FunctionHandler`]
300
+/// trait and the raw message protocol used by Netdata.
301
+#[async_trait]
302
+trait RawFunctionHandler: Send + Sync {
303
+ /// Handle a raw function call with the given context.
304
+ ///
305
+ /// # Arguments
306
+ ///
307
+ /// * `ctx` - The execution context containing the function call details
308
+ ///
309
+ /// # Returns
310
+ ///
311
+ /// A [`FunctionResult`] to be sent back to Netdata
312
+ async fn handle_raw(&self, ctx: Arc<FunctionContext>) -> FunctionResult;
313
+
314
+ /// Get the function declaration for this handler.
315
+ fn declaration(&self) -> FunctionDeclaration;
316
+}
317
+
318
+/// Adapter that bridges typed handlers with the raw protocol.
319
+///
320
+/// This struct wraps a [`FunctionHandler`] implementation and provides
321
+/// automatic JSON serialization/deserialization for the request and response.
322
+struct HandlerAdapter<H: FunctionHandler> {
323
+ handler: Arc<H>,
324
+}
325
+
326
+#[async_trait]
327
+impl<H: FunctionHandler> RawFunctionHandler for HandlerAdapter<H> {
328
+ async fn handle_raw(&self, ctx: Arc<FunctionContext>) -> FunctionResult {
329
+ let transaction = ctx.function_call.transaction.clone();
330
+
331
+ // Deserialize the request payload
332
+ let payload: H::Request = match &ctx.function_call.payload {
333
+ Some(bytes) => match serde_json::from_slice(bytes) {
334
+ Ok(p) => p,
335
+ Err(e) => {
336
+ error!("failed to deserialize request payload: {}", e);
337
+ return FunctionResult {
338
+ transaction,
339
+ status: 400,
340
+ expires: 0,
341
+ format: "text/plain".to_string(),
342
+ payload: format!("Invalid request: {}", e).as_bytes().to_vec(),
343
+ };
344
+ }
345
+ },
346
+ None => match serde_json::from_slice(b"{}") {
347
+ Ok(p) => p,
348
+ Err(e) => {
349
+ let payload =
350
+ serde_json::to_vec(&json!({ "error": "Request payload is empty", }))
351
+ .expect("serializing a json value to work");
352
+
353
+ error!("failed to deserialize empty payload: {}", e);
354
+ return FunctionResult {
355
+ transaction,
356
+ status: 400,
357
+ expires: 0,
358
+ format: "text/plain".to_string(),
359
+ payload,
360
+ };
361
+ }
362
+ },
363
+ };
364
+
365
+ // Drive the handler with cancellation and progress handling
366
+ let handler = self.handler.clone();
367
+
368
+ let mut call_future = Box::pin(handler.on_call(transaction.clone(), payload));
369
+ let mut signal_rx = ctx.signal_rx.lock().await;
370
+
371
+ let result = loop {
372
+ tokio::select! {
373
+ // Poll the main computation
374
+ result = &mut call_future => {
375
+ break result;
376
+ }
377
+ // Handle progress requests
378
+ Some(msg) = signal_rx.recv() => {
379
+ match msg {
380
+ RuntimeSignal::Progress => {
381
+ handler.on_progress(transaction.clone()).await;
382
+ }
383
+ }
384
+ }
385
+ // Handle cancellation
386
+ _ = ctx.cancellation_token.cancelled() => {
387
+ break handler.on_cancellation(transaction.clone()).await;
388
+ }
389
+ }
390
+ };
391
+
392
+ let current_timestamp = std::time::SystemTime::now()
393
+ .duration_since(std::time::UNIX_EPOCH)
394
+ .expect("Time went backwards")
395
+ .as_secs();
396
+
397
+ let expires: u64 = current_timestamp + 2;
398
+
399
+ // Process the result
400
+ match result {
401
+ Ok(response) => {
402
+ // Serialize the response
403
+ match serde_json::to_vec_pretty(&response) {
404
+ Ok(payload) => FunctionResult {
405
+ transaction,
406
+ status: 200,
407
+ expires,
408
+ format: "application/json".to_string(),
409
+ payload,
410
+ },
411
+ Err(e) => {
412
+ error!("failed to serialize response: {}", e);
413
+ FunctionResult {
414
+ transaction,
415
+ status: 500,
416
+ expires: 0,
417
+ format: "text/plain".to_string(),
418
+ payload: format!("Serialization error: {}", e).as_bytes().to_vec(),
419
+ }
420
+ }
421
+ }
422
+ }
423
+ Err(e) => {
424
+ error!("function handler error: {}", e);
425
+ let error_json = json!({
426
+ "error": format!("{}", e),
427
+ "status": 500
428
+ });
429
+ FunctionResult {
430
+ transaction,
431
+ status: 500,
432
+ expires: 0,
433
+ format: "application/json".to_string(),
434
+ payload: serde_json::to_vec_pretty(&error_json).unwrap_or_else(|_| {
435
+ format!(r#"{{"error": "Failed to serialize error response"}}"#)
436
+ .as_bytes()
437
+ .to_vec()
438
+ }),
439
+ }
440
+ }
441
+ }
442
+ }
443
+
444
+ fn declaration(&self) -> FunctionDeclaration {
445
+ self.handler.declaration()
446
+ }
447
+}
448
+
449
+/// Main runtime for managing Netdata plugin execution.
450
+///
451
+/// The `PluginRuntime` orchestrates all aspects of a Netdata plugin's lifecycle:
452
+/// - Registering function handlers
453
+/// - Communicating with Netdata via streams (stdio, TCP, etc.)
454
+/// - Managing concurrent function executions
455
+/// - Handling cancellation and progress requests
456
+/// - Graceful shutdown on signals
457
+///
458
+/// # Type Parameters
459
+///
460
+/// * `R` - The reader type (must implement `AsyncRead + Unpin`)
461
+/// * `W` - The writer type (must implement `AsyncWrite + Unpin`)
462
+///
463
+/// # Example
464
+///
465
+/// ```no_run
466
+/// use rt::PluginRuntime;
467
+///
468
+/// #[tokio::main]
469
+/// async fn main() -> Result<(), Box<dyn std::error::Error>> {
470
+/// let mut runtime = PluginRuntime::new("my_plugin");
471
+/// // Register handlers here
472
+/// runtime.run().await?;
473
+/// Ok(())
474
+/// }
475
+/// ```
476
+///
477
+/// Type alias for the standard plugin runtime using stdin/stdout.
478
+/// This is the typical configuration for production Netdata plugins.
479
+pub type StdPluginRuntime = PluginRuntime<tokio::io::Stdin, tokio::io::Stdout>;
480
+
481
+pub struct PluginRuntime<R, W>
482
+where
483
+ R: AsyncRead + Unpin + Send + 'static,
484
+ W: AsyncWrite + Unpin + Send + 'static,
485
+{
486
+ /// Name of this plugin (used for identification).
487
+ plugin_name: String,
488
+ /// Reader for incoming messages from Netdata.
489
+ reader: MessageReader<R>,
490
+ /// Writer for outgoing messages to Netdata.
491
+ writer: Arc<Mutex<MessageWriter<W>>>,
492
+
493
+ /// Registry of all available function handlers.
494
+ function_handlers: HashMap<String, Arc<dyn RawFunctionHandler>>,
495
+
496
+ /// Active transactions (ongoing function calls).
497
+ transaction_registry: HashMap<String, Arc<Transaction>>,
498
+ /// Futures representing running function executions.
499
+ futures: FuturesUnordered<FunctionFuture>,
500
+
501
+ /// Token for initiating graceful shutdown.
502
+ shutdown_token: CancellationToken,
503
+
504
+ /// Optional chart registry for managing metrics emission.
505
+ chart_registry: Option<ChartRegistry<W>>,
506
+ /// Handle to chart registry background task.
507
+ chart_registry_handle: Option<
508
+ tokio::task::JoinHandle<std::result::Result<(), Box<dyn std::error::Error + Send + Sync>>>,
509
+ >,
510
+}
511
+
512
+impl PluginRuntime<tokio::io::Stdin, tokio::io::Stdout> {
513
+ /// Create a new plugin runtime with the given name using stdin/stdout.
514
+ ///
515
+ /// This is the default constructor that creates a runtime communicating
516
+ /// via standard input and output streams.
517
+ ///
518
+ /// # Arguments
519
+ ///
520
+ /// * `name` - The name of the plugin (used for identification in logs)
521
+ ///
522
+ /// # Returns
523
+ ///
524
+ /// A new `PluginRuntime` instance ready to accept handler registrations.
525
+ pub fn new(name: &str) -> Self {
526
+ Self::with_streams(name, tokio::io::stdin(), tokio::io::stdout())
527
+ }
528
+}
529
+
530
+impl<R, W> PluginRuntime<R, W>
531
+where
532
+ R: AsyncRead + Unpin + Send + 'static,
533
+ W: AsyncWrite + Unpin + Send + 'static,
534
+{
535
+ /// Create a new plugin runtime with custom reader and writer streams.
536
+ ///
537
+ /// This allows the runtime to work with any streams that implement
538
+ /// `AsyncRead` and `AsyncWrite`, such as TCP connections, Unix sockets,
539
+ /// or in-memory buffers.
540
+ ///
541
+ /// # Arguments
542
+ ///
543
+ /// * `name` - The name of the plugin (used for identification in logs)
544
+ /// * `reader` - The input stream to read messages from
545
+ /// * `writer` - The output stream to write messages to
546
+ ///
547
+ /// # Returns
548
+ ///
549
+ /// A new `PluginRuntime` instance ready to accept handler registrations.
550
+ ///
551
+ /// # Example
552
+ ///
553
+ /// ```no_run
554
+ /// use rt::PluginRuntime;
555
+ /// use tokio::net::TcpStream;
556
+ ///
557
+ /// #[tokio::main]
558
+ /// async fn main() -> Result<(), Box<dyn std::error::Error>> {
559
+ /// let stream = TcpStream::connect("127.0.0.1:8080").await?;
560
+ /// let (reader, writer) = stream.into_split();
561
+ ///
562
+ /// let mut runtime = PluginRuntime::with_streams("my_plugin", reader, writer);
563
+ /// // Register handlers here
564
+ /// runtime.run().await?;
565
+ /// Ok(())
566
+ /// }
567
+ /// ```
568
+ pub fn with_streams(name: &str, reader: R, writer: W) -> Self {
569
+ Self {
570
+ plugin_name: String::from(name),
571
+ reader: MessageReader::new(reader),
572
+ writer: Arc::new(Mutex::new(MessageWriter::new(writer))),
573
+
574
+ function_handlers: HashMap::new(),
575
+ transaction_registry: HashMap::new(),
576
+ futures: FuturesUnordered::new(),
577
+
578
+ shutdown_token: CancellationToken::default(),
579
+ chart_registry: None,
580
+ chart_registry_handle: None,
581
+ }
582
+ }
583
+
584
+ /// Get a clone of the shared message writer.
585
+ ///
586
+ /// This allows external code to write protocol messages while coordinating
587
+ /// with the runtime's own writes (e.g., for function results, chart data).
588
+ ///
589
+ /// # Returns
590
+ ///
591
+ /// An Arc-wrapped, mutex-protected MessageWriter that coordinates with
592
+ /// the runtime's stdin/stdout handling.
593
+ ///
594
+ /// # Example
595
+ ///
596
+ /// ```ignore
597
+ /// let writer = runtime.writer();
598
+ ///
599
+ /// // Write raw protocol messages
600
+ /// let mut w = writer.lock().await;
601
+ /// w.write_raw(b"CHART ...\n").await?;
602
+ /// ```
603
+ pub fn writer(&self) -> Arc<Mutex<MessageWriter<W>>> {
604
+ Arc::clone(&self.writer)
605
+ }
606
+
607
+ /// Register a function handler with the runtime.
608
+ ///
609
+ /// The handler will be available for Netdata to call once the runtime starts.
610
+ /// Multiple handlers can be registered, each with a unique function name.
611
+ ///
612
+ /// # Arguments
613
+ ///
614
+ /// * `handler` - The function handler implementation
615
+ ///
616
+ /// # Panics
617
+ ///
618
+ /// May panic if two handlers with the same function name are registered.
619
+ pub fn register_handler<H: FunctionHandler + 'static>(&mut self, handler: H) {
620
+ let adapter = HandlerAdapter {
621
+ handler: Arc::new(handler),
622
+ };
623
+ let name = adapter.declaration().name.clone();
624
+ self.function_handlers.insert(name, Arc::new(adapter));
625
+ }
626
+
627
+ /// Register a chart for metrics emission.
628
+ ///
629
+ /// Charts are automatically sampled at the given interval and emitted through
630
+ /// the shared message writer. Returns a handle that can be used to update
631
+ /// chart values from anywhere in your code.
632
+ ///
633
+ /// # Arguments
634
+ ///
635
+ /// * `initial` - The initial chart value
636
+ /// * `interval` - How often to sample and emit the chart
637
+ ///
638
+ /// # Returns
639
+ ///
640
+ /// A `ChartHandle` for updating the chart values
641
+ ///
642
+ /// # Example
643
+ ///
644
+ /// ```ignore
645
+ /// let metrics = runtime.register_chart(
646
+ /// MyMetrics::default(),
647
+ /// Duration::from_secs(1),
648
+ /// );
649
+ ///
650
+ /// // Later, update from anywhere:
651
+ /// metrics.update(|m| {
652
+ /// m.counter += 1;
653
+ /// });
654
+ /// ```
655
+ pub fn register_chart<T>(&mut self, initial: T, interval: Duration) -> ChartHandle<T>
656
+ where
657
+ T: NetdataChart + Default + PartialEq + Clone + Send + Sync + 'static,
658
+ {
659
+ let registry = self
660
+ .chart_registry
661
+ .get_or_insert_with(|| ChartRegistry::new(Arc::clone(&self.writer)));
662
+ registry.register_chart(initial, interval)
663
+ }
664
+
665
+ /// Register an instanced chart for per-instance metrics.
666
+ ///
667
+ /// Similar to `register_chart`, but for charts that have multiple instances
668
+ /// (e.g., per-CPU core metrics, per-disk I/O stats).
669
+ ///
670
+ /// # Arguments
671
+ ///
672
+ /// * `initial` - The initial chart value with instance ID set
673
+ /// * `interval` - How often to sample and emit the chart
674
+ ///
675
+ /// # Returns
676
+ ///
677
+ /// A `ChartHandle` for updating the chart values
678
+ pub fn register_instanced_chart<T>(&mut self, initial: T, interval: Duration) -> ChartHandle<T>
679
+ where
680
+ T: InstancedChart + Default + PartialEq + Send + Sync + 'static,
681
+ {
682
+ let registry = self
683
+ .chart_registry
684
+ .get_or_insert_with(|| ChartRegistry::new(Arc::clone(&self.writer)));
685
+ registry.register_instanced_chart(initial, interval)
686
+ }
687
+
688
+ /// Start the plugin runtime and begin processing messages.
689
+ ///
690
+ /// This method:
691
+ /// 1. Sets up signal handlers for graceful shutdown
692
+ /// 2. Declares all registered functions to Netdata
693
+ /// 3. Enters the main message processing loop
694
+ /// 4. Handles shutdown when requested
695
+ ///
696
+ /// # Returns
697
+ ///
698
+ /// Returns `Ok(())` on successful shutdown, or an error if a critical failure occurs.
699
+ ///
700
+ /// # Note
701
+ ///
702
+ /// This method runs indefinitely until shutdown is requested (via Ctrl-C or stdin closing).
703
+ pub async fn run(mut self) -> Result<()> {
704
+ info!("starting plugin runtime: {}", self.plugin_name);
705
+
706
+ self.handle_ctr_c();
707
+
708
+ // Start chart registry if charts were registered
709
+ if let Some(registry) = self.chart_registry.take() {
710
+ let registry_token = registry.cancellation_token();
711
+ let shutdown_token = self.shutdown_token.clone();
712
+
713
+ let handle = tokio::spawn(async move {
714
+ tokio::select! {
715
+ result = registry.run() => {
716
+ if let Err(e) = &result {
717
+ error!("chart registry error: {}", e);
718
+ }
719
+ result
720
+ }
721
+ _ = shutdown_token.cancelled() => {
722
+ registry_token.cancel();
723
+ Ok(())
724
+ }
725
+ }
726
+ });
727
+
728
+ self.chart_registry_handle = Some(handle);
729
+ info!("chart registry started");
730
+ }
731
+
732
+ self.declare_functions().await?;
733
+ self.process_messages().await?;
734
+ self.shutdown().await?;
735
+
736
+ Ok(())
737
+ }
738
+
739
+ /// Setup Ctrl-C signal handler for graceful shutdown.
740
+ fn handle_ctr_c(&self) {
741
+ let shutdown_token = self.shutdown_token.clone();
742
+
743
+ tokio::spawn(async move {
744
+ match tokio::signal::ctrl_c().await {
745
+ Ok(()) => {
746
+ info!("received ctrl-c signal, initiating graceful shutdown");
747
+ shutdown_token.cancel();
748
+ }
749
+ Err(e) => {
750
+ error!("failed to listen for Ctrl-C signal: {}", e);
751
+ }
752
+ }
753
+ });
754
+ }
755
+
756
+ /// Declare all registered functions to Netdata.
757
+ ///
758
+ /// Sends a [`FunctionDeclaration`] message for each registered handler,
759
+ /// informing Netdata about available functions and their metadata.
760
+ async fn declare_functions(&self) -> Result<()> {
761
+ let mut writer = self.writer.lock().await;
762
+
763
+ for (name, handler) in self.function_handlers.iter() {
764
+ info!("declaring function: {}", name);
765
+
766
+ let message = Message::FunctionDeclaration(Box::new(handler.declaration()));
767
+ if let Err(e) = writer.send(message).await {
768
+ error!("failed to declare function {}: {}", name, e);
769
+ return Err(e);
770
+ }
771
+ }
772
+
773
+ writer.flush().await?;
774
+
775
+ Ok(())
776
+ }
777
+
778
+ /// Main message processing loop.
779
+ ///
780
+ /// Continuously processes incoming messages from Netdata and completed function futures.
781
+ /// Handles function calls, cancellations, progress requests, and completions.
782
+ async fn process_messages(&mut self) -> Result<()> {
783
+ info!("starting message processing loop");
784
+
785
+ loop {
786
+ tokio::select! {
787
+ // Make the shutdown signal higher priority by putting it first
788
+ _ = self.shutdown_token.cancelled() => {
789
+ info!("shutdown requested... Stop processing messages from stdin");
790
+ // Reader will be dropped here, closing stdin
791
+ break;
792
+ }
793
+ message = self.reader.next() => {
794
+ if self.handle_message(message).await? {
795
+ break;
796
+ }
797
+ }
798
+ Some((transaction, result)) = self.futures.next() => {
799
+ self.handle_completed(transaction, result).await?;
800
+ }
801
+ }
802
+ }
803
+
804
+ Ok(())
805
+ }
806
+
807
+ /// Handle a single incoming message from Netdata.
808
+ ///
809
+ /// # Returns
810
+ ///
811
+ /// Returns `true` if the message loop should terminate, `false` otherwise.
812
+ async fn handle_message(&mut self, message: Option<Result<Message>>) -> Result<bool> {
813
+ match message {
814
+ Some(Ok(Message::FunctionCall(function_call))) => {
815
+ self.handle_function_call(function_call);
816
+ }
817
+ Some(Ok(Message::FunctionCancel(function_cancel))) => {
818
+ self.handle_function_cancel(function_cancel.as_ref());
819
+ }
820
+ Some(Ok(Message::FunctionProgress(function_progress))) => {
821
+ self.handle_function_progress(&function_progress).await;
822
+ }
823
+ Some(Ok(msg)) => {
824
+ debug!("received message: {:?}", msg);
825
+ }
826
+ Some(Err(e)) => {
827
+ error!("error parsing message: {:?}", e);
828
+ }
829
+ None => {
830
+ info!("input stream ended");
831
+ self.shutdown_token.cancel();
832
+ return Ok(true); // Signal to break the loop
833
+ }
834
+ }
835
+ Ok(false)
836
+ }
837
+
838
+ /// Handle an incoming function call request.
839
+ ///
840
+ /// Creates a new transaction, sets up the execution context, and spawns
841
+ /// the function handler to process the request asynchronously.
842
+ fn handle_function_call(&mut self, function_call: Box<FunctionCall>) {
843
+ if self
844
+ .transaction_registry
845
+ .contains_key(&function_call.transaction)
846
+ {
847
+ warn!(
848
+ "Ignoring existing transaction {:#?} for function {:#?}",
849
+ function_call.transaction, function_call.name
850
+ );
851
+ return;
852
+ }
853
+
854
+ // patch function-call for systemd-journal. will remove this once
855
+ // we convert the frontend request from a GET to POST.
856
+ let mut function_call = function_call;
857
+ {
858
+ if function_call.name == "journal-viewer" {
859
+ if !function_call.args.is_empty() {
860
+ let mut map = serde_json::Map::new();
861
+ map.insert("info".to_string(), serde_json::json!(true));
862
+
863
+ for arg in &function_call.args {
864
+ if let Some(after_str) = arg.strip_prefix("after:") {
865
+ if let Ok(after_val) = after_str.parse::<u64>() {
866
+ map.insert("after".to_string(), serde_json::json!(after_val));
867
+ }
868
+ } else if let Some(before_str) = arg.strip_prefix("before:") {
869
+ if let Ok(before_val) = before_str.parse::<u64>() {
870
+ map.insert("before".to_string(), serde_json::json!(before_val));
871
+ }
872
+ }
873
+ }
874
+
875
+ let json = serde_json::Value::Object(map);
876
+ let payload = serde_json::to_vec(&json).unwrap();
877
+ function_call.payload = Some(payload);
878
+ }
879
+ }
880
+ }
881
+
882
+ // Get handler
883
+ let Some(handler) = self.function_handlers.get(&function_call.name).cloned() else {
884
+ error!("could not find function {:#?}", function_call.name);
885
+ return;
886
+ };
887
+
888
+ // Create a new function context
889
+ let (control_tx, control_rx) = mpsc::channel(4);
890
+ let cancellation_token = CancellationToken::new();
891
+
892
+ let function_context = Arc::new(FunctionContext {
893
+ function_call,
894
+ cancellation_token: cancellation_token.clone(),
895
+ signal_rx: Mutex::new(control_rx),
896
+ });
897
+
898
+ // Create new transaction
899
+ let id = function_context.function_call.transaction.clone();
900
+ let transaction = Arc::new(Transaction {
901
+ id,
902
+ cancellation_token,
903
+ control_tx,
904
+ });
905
+ self.transaction_registry
906
+ .insert(transaction.id.clone(), transaction.clone());
907
+
908
+ // Create future
909
+ let future = Box::pin(async move {
910
+ let result = handler.handle_raw(function_context).await;
911
+ (transaction.id.clone(), result)
912
+ });
913
+ self.futures.push(future);
914
+ }
915
+
916
+ /// Handle a function cancellation request.
917
+ ///
918
+ /// Signals the corresponding transaction to cancel its execution.
919
+ fn handle_function_cancel(&mut self, function_cancel: &FunctionCancel) {
920
+ let Some(transaction) = self.transaction_registry.get(&function_cancel.transaction) else {
921
+ warn!(
922
+ "Can not cancel non-existing transaction {}",
923
+ function_cancel.transaction
924
+ );
925
+ return;
926
+ };
927
+
928
+ info!("cancelling transaction {}", function_cancel.transaction);
929
+ transaction.cancellation_token.cancel();
930
+ }
931
+
932
+ /// Handle a progress report request.
933
+ ///
934
+ /// Sends a progress signal to the corresponding running function.
935
+ async fn handle_function_progress(&mut self, function_progress: &FunctionProgress) {
936
+ let Some(transaction) = self
937
+ .transaction_registry
938
+ .get(&function_progress.transaction)
939
+ else {
940
+ warn!(
941
+ "can not get progress of non-existing transaction {}",
942
+ function_progress.transaction
943
+ );
944
+ return;
945
+ };
946
+
947
+ info!(
948
+ "requesting progress of transaction {}",
949
+ function_progress.transaction
950
+ );
951
+ let _ = transaction.control_tx.send(RuntimeSignal::Progress).await;
952
+ }
953
+
954
+ /// Handle a completed function execution.
955
+ ///
956
+ /// Removes the transaction from the registry and sends the result back to Netdata.
957
+ async fn handle_completed(
958
+ &mut self,
959
+ transaction: String,
960
+ result: FunctionResult,
961
+ ) -> Result<()> {
962
+ self.transaction_registry.remove(&transaction);
963
+ self.writer
964
+ .lock()
965
+ .await
966
+ .send(Message::FunctionResult(Box::new(result)))
967
+ .await?;
968
+ Ok(())
969
+ }
970
+
971
+ /// Perform graceful shutdown of the runtime.
972
+ ///
973
+ /// Cancels all active transactions and waits for them to complete
974
+ /// (up to a timeout of 10 seconds). Any functions that don't complete
975
+ /// within the timeout are forcefully aborted.
976
+ ///
977
+ /// # Returns
978
+ ///
979
+ /// Returns `Ok(())` after shutdown completes (either cleanly or after timeout).
980
+ async fn shutdown(&mut self) -> Result<()> {
981
+ let in_flight = self.transaction_registry.len();
982
+
983
+ if in_flight == 0 {
984
+ info!("clean shutdown - no in-flight functions");
985
+ } else {
986
+ info!("shutting down with {} in-flight functions...", in_flight);
987
+
988
+ // Send cancel to all active transactions
989
+ for transaction in self.transaction_registry.values() {
990
+ transaction.cancellation_token.cancel();
991
+ }
992
+
993
+ // Wait for functions to complete with a timeout
994
+ let timeout = Duration::from_secs(10);
995
+ let mut completed = 0;
996
+
997
+ match tokio::time::timeout(timeout, async {
998
+ while let Some((transaction, result)) = self.futures.next().await {
999
+ if let Err(e) = self.handle_completed(transaction, result).await {
1000
+ error!("error handling completed function during shutdown: {}", e);
1001
+ }
1002
+ completed += 1;
1003
+ }
1004
+ })
1005
+ .await
1006
+ {
1007
+ Ok(()) => {
1008
+ info!("clean shutdown - all {} functions completed", completed);
1009
+ }
1010
+ Err(_) => {
1011
+ let aborted = in_flight - completed;
1012
+ warn!(
1013
+ "shutdown timeout - {} functions completed, {} aborted",
1014
+ completed, aborted
1015
+ );
1016
+ }
1017
+ }
1018
+ }
1019
+
1020
+ // Wait for chart registry to finish
1021
+ if let Some(handle) = self.chart_registry_handle.take() {
1022
+ info!("waiting for chart registry to finish...");
1023
+ match handle.await {
1024
+ Ok(Ok(())) => info!("chart registry shut down cleanly"),
1025
+ Ok(Err(e)) => warn!("Chart registry error during shutdown: {}", e),
1026
+ Err(e) => warn!("Chart registry task panicked: {}", e),
1027
+ }
1028
+ }
1029
+
1030
+ Ok(())
1031
+ }
1032
+}
src/crates/netdata-plugin/rt/src/netdata_env.rs
renamed
src/crates/netdata-plugin/rt/src/tracing_setup.rs
new
+102
@@ -0,0 +1,102 @@
1
+//! Tracing configuration for Netdata plugins
2
+//!
3
+//! This module provides automatic tracing initialization with environment detection:
4
+//!
5
+//! ## Log Output
6
+//! - **Systemd journal**: When `NETDATA_SYSTEMD_JOURNAL_PATH` is set
7
+//! - **Stderr**: Otherwise (formatted text with thread IDs and line numbers)
8
+//!
9
+//! ## Log Level
10
+//! - Uses `NETDATA_LOG_LEVEL` environment variable if set
11
+//! - Defaults to `info` level if not set
12
+//! - Supported levels: emergency, alert, critical, error, warning, notice, info, debug
13
+
14
+use tracing_subscriber::{EnvFilter, prelude::*};
15
+
16
+use crate::netdata_env::{LogLevel, LogMethod, NetdataEnv};
17
+
18
+/// Detect the appropriate log method based on NetdataEnv
19
+fn detect_log_method(netdata_env: &NetdataEnv) -> LogMethod {
20
+ // If log_method is explicitly set, use it
21
+ if let Some(ref method) = netdata_env.log_method {
22
+ return method.clone();
23
+ }
24
+
25
+ // Otherwise, auto-detect based on systemd_journal_path
26
+ if netdata_env.systemd_journal_path.is_some() {
27
+ LogMethod::Journal
28
+ } else {
29
+ LogMethod::Stderr
30
+ }
31
+}
32
+
33
+/// Get a human-readable description of the log method
34
+fn log_method_description(method: &LogMethod) -> &'static str {
35
+ match method {
36
+ LogMethod::Syslog => "syslog",
37
+ LogMethod::Journal => "systemd journal",
38
+ LogMethod::Stderr => "stderr",
39
+ LogMethod::None => "disabled",
40
+ }
41
+}
42
+
43
+/// Convert Netdata LogLevel to tracing filter string
44
+fn log_level_to_filter(level: &LogLevel) -> &'static str {
45
+ match level {
46
+ LogLevel::Emergency | LogLevel::Alert | LogLevel::Critical => "error",
47
+ LogLevel::Error => "error",
48
+ LogLevel::Warning => "warn",
49
+ LogLevel::Notice | LogLevel::Info => "info",
50
+ LogLevel::Debug => "debug",
51
+ }
52
+}
53
+
54
+/// Initialize tracing with automatic environment detection.
55
+///
56
+/// Uses NETDATA_LOG_LEVEL environment variable if set, otherwise defaults to "info".
57
+pub fn init_tracing() {
58
+ // Read Netdata environment configuration
59
+ let netdata_env = NetdataEnv::from_environment();
60
+
61
+ // Determine output destination
62
+ let log_method = detect_log_method(&netdata_env);
63
+
64
+ // Determine log level from environment
65
+ let filter_str = netdata_env
66
+ .log_level
67
+ .as_ref()
68
+ .map(log_level_to_filter)
69
+ .unwrap_or("info");
70
+
71
+ // Create environment filter
72
+ let env_filter = EnvFilter::new(filter_str);
73
+
74
+ // Build the registry with base layers
75
+ let registry = tracing_subscriber::registry().with(env_filter);
76
+
77
+ // Add output layer based on log method
78
+ match log_method {
79
+ LogMethod::Journal => {
80
+ let journald_layer = tracing_journald::layer().expect("failed to connect to journald");
81
+ registry.with(journald_layer).init();
82
+ }
83
+ LogMethod::Stderr | LogMethod::Syslog | LogMethod::None => {
84
+ // For now, stderr is used for Stderr, Syslog (not implemented), and None (fallback)
85
+ let fmt_layer = tracing_subscriber::fmt::layer()
86
+ .with_writer(std::io::stderr)
87
+ .with_target(true)
88
+ .with_thread_ids(true)
89
+ .with_line_number(true)
90
+ .with_ansi(false);
91
+ registry.with(fmt_layer).init();
92
+ }
93
+ }
94
+
95
+ tracing::info!(
96
+ method = ?log_method,
97
+ level = filter_str,
98
+ "tracing initialized, logging to {} with filter '{}'",
99
+ log_method_description(&log_method),
100
+ filter_str,
101
+ );
102
+}
src/crates/netdata-plugin/schema/Cargo.toml
new
+20
@@ -0,0 +1,20 @@
1
+[package]
2
+name = "netdata-plugin-schema"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+description = "Library for generating Netdata-compatible JSON Schema with UI annotations"
7
+license = "MIT OR Apache-2.0"
8
+
9
+[lib]
10
+doctest = false
11
+
12
+[lints]
13
+workspace = true
14
+
15
+[dependencies]
16
+netdata-plugin-error = { path = "../error" }
17
+netdata-plugin-types = { path = "../types" }
18
+schemars = { workspace = true }
19
+serde = { workspace = true }
20
+serde_json = { workspace = true }
src/crates/netdata-plugin/schema/examples/simple_usage.rs
new
+56
@@ -0,0 +1,56 @@
1
+use netdata_plugin_schema::NetdataSchema;
2
+use schemars::JsonSchema;
3
+use serde::{Deserialize, Serialize};
4
+
5
+#[derive(Clone, Debug, JsonSchema, Serialize, Deserialize)]
6
+#[schemars(
7
+ title = "Web Server Configuration",
8
+ description = "Configuration for a simple web server",
9
+ extend("x-ui-flavour" = "tabs"),
10
+ extend("x-ui-options" = {
11
+ "tabs": [
12
+ {
13
+ "title": "Server Settings",
14
+ "fields": ["host", "port", "workers"]
15
+ },
16
+ {
17
+ "title": "Security",
18
+ "fields": ["enable_tls", "tls_cert_path", "api_key"]
19
+ }
20
+ ]
21
+ }),
22
+ extend("x-config-id" = "demo_plugin:my_config"),
23
+ extend("x-config-path" = "/collectors"),
24
+ extend("x-config-type" = "single"),
25
+ extend("x-config-status" = "running"),
26
+ extend("x-config-source-type" = "stock"),
27
+ extend("x-config-source" = "Plugin-generated configuration"),
28
+ extend("x-config-cmds" = "schema|get|update"),
29
+ extend("x-config-view-access" = 0),
30
+ extend("x-config-edit-access" = 0),
31
+)]
32
+struct WebServerConfig {
33
+ #[schemars(
34
+ title = "Host Address",
35
+ description = "The IP address to bind the server to",
36
+ example = "0.0.0.0",
37
+ extend("x-ui-help" = "Use 0.0.0.0 to bind to all interfaces"),
38
+ extend("x-ui-placeholder" = "127.0.0.1")
39
+ )]
40
+ host: String,
41
+
42
+ #[schemars(
43
+ title = "Port",
44
+ description = "TCP port number",
45
+ range(min = 1, max = 65535),
46
+ example = 8080,
47
+ extend("x-ui-help" = "Choose an available port number"),
48
+ extend("x-ui-placeholder" = "8080")
49
+ )]
50
+ port: u16,
51
+}
52
+
53
+fn main() {
54
+ let schema = WebServerConfig::netdata_schema();
55
+ println!("{}", serde_json::to_string_pretty(&schema).unwrap());
56
+}
src/crates/netdata-plugin/schema/src/lib.rs
new
+299
@@ -0,0 +1,299 @@
1
+//! # Netdata Schema Generation Library
2
+//!
3
+//! This library provides functionality to generate Netdata-compatible JSON schemas
4
+//! with UI annotations and configuration declarations from Rust types annotated with schemars attributes.
5
+//!
6
+//! ## Basic Usage
7
+//!
8
+//! ```rust
9
+//! use schemars::JsonSchema;
10
+//! use netdata_plugin_schema::NetdataSchema;
11
+//!
12
+//! #[derive(Clone, Debug, JsonSchema)]
13
+//! #[schemars(
14
+//! extend("x-ui-flavour" = "tabs"),
15
+//! extend("x-config-id" = "my_plugin:my_config"),
16
+//! extend("x-config-path" = "/collectors")
17
+//! )]
18
+//! struct MyConfig {
19
+//! #[schemars(
20
+//! title = "Server URL",
21
+//! extend("x-ui-help" = "Enter the server URL"),
22
+//! extend("x-ui-placeholder" = "https://example.com")
23
+//! )]
24
+//! url: String,
25
+//! }
26
+//!
27
+//! // NetdataSchema is automatically implemented for all JsonSchema types
28
+//!
29
+//! let netdata_schema = MyConfig::netdata_schema();
30
+//! println!("{}", serde_json::to_string_pretty(&netdata_schema).unwrap());
31
+//!
32
+//! // Config declaration is included in the schema if x-config-* metadata is present
33
+//! if let Some(config_decl) = netdata_schema.get("configDeclaration") {
34
+//! println!("Config ID: {}", config_decl["id"]);
35
+//! }
36
+//! ```
37
+
38
+use schemars::transform::{Transform, transform_subschemas};
39
+use schemars::{JsonSchema, Schema, SchemaGenerator, generate::SchemaSettings};
40
+use serde_json::{Map, Value};
41
+
42
+// Re-export types for convenience
43
+pub use netdata_plugin_types::{
44
+ ConfigDeclaration, DynCfgCmds, DynCfgSourceType, DynCfgStatus, DynCfgType, HttpAccess,
45
+};
46
+
47
+/// Transform that collects UI schema information from x-ui-* extensions
48
+/// and removes them from the JSON schema, collecting them separately
49
+#[derive(Default)]
50
+struct CollectUISchema {
51
+ ui_schema: Map<String, Value>,
52
+ current_path: Vec<String>,
53
+}
54
+
55
+impl Transform for CollectUISchema {
56
+ fn transform(&mut self, schema: &mut Schema) {
57
+ let Some(obj) = schema.as_object_mut() else {
58
+ return;
59
+ };
60
+
61
+ // Collect UI extensions from current schema
62
+ let mut ui_props = Map::new();
63
+ let mut keys_to_remove = Vec::new();
64
+
65
+ for (key, value) in obj.iter() {
66
+ if let Some(ui_key) = key.strip_prefix("x-ui-") {
67
+ ui_props.insert(format!("ui:{}", ui_key), value.clone());
68
+ keys_to_remove.push(key.clone());
69
+ } else if key == "x-sensitive" && value == &Value::Bool(true) {
70
+ ui_props.insert(
71
+ "ui:widget".to_string(),
72
+ Value::String("password".to_string()),
73
+ );
74
+ keys_to_remove.push(key.clone());
75
+ }
76
+ }
77
+
78
+ // Remove the x-ui-* extensions from the JSON schema
79
+ for key in keys_to_remove {
80
+ obj.remove(&key);
81
+ }
82
+
83
+ // If we have UI properties, add them to the UI schema at the current path
84
+ if !ui_props.is_empty() {
85
+ let ui_path = if self.current_path.is_empty() {
86
+ ".".to_string()
87
+ } else {
88
+ self.current_path.join(".")
89
+ };
90
+
91
+ if ui_path == "." {
92
+ // Root level - merge into root UI schema
93
+ for (key, value) in ui_props {
94
+ self.ui_schema.insert(key, value);
95
+ }
96
+ } else {
97
+ self.ui_schema.insert(ui_path, Value::Object(ui_props));
98
+ }
99
+ }
100
+
101
+ // Handle properties recursively
102
+ if let Some(properties) = obj.get_mut("properties").and_then(|v| v.as_object_mut()) {
103
+ for (prop_name, prop_schema) in properties.iter_mut() {
104
+ if let Ok(schema_ref) = prop_schema.try_into() {
105
+ self.current_path.push(prop_name.clone());
106
+ self.transform(schema_ref);
107
+ self.current_path.pop();
108
+ }
109
+ }
110
+ }
111
+
112
+ // Handle definitions recursively
113
+ if let Some(definitions) = obj.get_mut("definitions").and_then(|v| v.as_object_mut()) {
114
+ for (def_name, def_schema) in definitions.iter_mut() {
115
+ if let Ok(schema_ref) = def_schema.try_into() {
116
+ self.current_path.push(def_name.clone());
117
+ self.transform(schema_ref);
118
+ self.current_path.pop();
119
+ }
120
+ }
121
+ }
122
+
123
+ // Handle other subschemas
124
+ transform_subschemas(self, schema);
125
+ }
126
+}
127
+
128
+/// Transform that collects config declaration information from x-config-* extensions
129
+#[derive(Default)]
130
+struct CollectConfigDeclaration {
131
+ config_declaration: Option<ConfigDeclaration>,
132
+}
133
+
134
+impl Transform for CollectConfigDeclaration {
135
+ fn transform(&mut self, schema: &mut Schema) {
136
+ let Some(obj) = schema.as_object_mut() else {
137
+ return;
138
+ };
139
+
140
+ // Only process root-level schema (where config declarations should be)
141
+ let mut config_props = ConfigDeclarationBuilder::default();
142
+ let mut keys_to_remove = Vec::new();
143
+
144
+ for (key, value) in obj.iter() {
145
+ if let Some(config_key) = key.strip_prefix("x-config-") {
146
+ if let Some(str_value) = value.as_str() {
147
+ match config_key {
148
+ "id" => config_props.id = Some(str_value.to_string()),
149
+ "path" => config_props.path = Some(str_value.to_string()),
150
+ "source" => config_props.source = Some(str_value.to_string()),
151
+ "type" => config_props.type_ = DynCfgType::from_name(str_value),
152
+ "status" => config_props.status = DynCfgStatus::from_name(str_value),
153
+ "source-type" => {
154
+ config_props.source_type = DynCfgSourceType::from_name(str_value)
155
+ }
156
+ "cmds" => config_props.cmds = Some(parse_cmds_string(str_value)),
157
+ unknown => {
158
+ panic!("Unknown config declaration attribute: {}", unknown);
159
+ }
160
+ }
161
+ } else if let Some(int_value) = value.as_u64() {
162
+ match config_key {
163
+ "view-access" => {
164
+ config_props.view_access = Some(HttpAccess::from_u32(int_value as u32))
165
+ }
166
+ "edit-access" => {
167
+ config_props.edit_access = Some(HttpAccess::from_u32(int_value as u32))
168
+ }
169
+ unknown => {
170
+ panic!("Unknown config declaration attribute: {}", unknown);
171
+ }
172
+ }
173
+ }
174
+ keys_to_remove.push(key.clone());
175
+ }
176
+ }
177
+
178
+ // Remove the x-config-* extensions from the JSON schema
179
+ for key in keys_to_remove {
180
+ obj.remove(&key);
181
+ }
182
+
183
+ // Build config declaration
184
+ self.config_declaration = Some(config_props.build());
185
+ }
186
+}
187
+
188
+#[derive(Debug, Default)]
189
+struct ConfigDeclarationBuilder {
190
+ id: Option<String>,
191
+ status: Option<DynCfgStatus>,
192
+ type_: Option<DynCfgType>,
193
+ path: Option<String>,
194
+ source_type: Option<DynCfgSourceType>,
195
+ source: Option<String>,
196
+ cmds: Option<DynCfgCmds>,
197
+ view_access: Option<HttpAccess>,
198
+ edit_access: Option<HttpAccess>,
199
+}
200
+
201
+impl ConfigDeclarationBuilder {
202
+ fn build(self) -> ConfigDeclaration {
203
+ ConfigDeclaration {
204
+ id: self.id.unwrap(),
205
+ status: self.status.unwrap(),
206
+ type_: self.type_.unwrap(),
207
+ path: self.path.unwrap(),
208
+ source_type: self.source_type.unwrap(),
209
+ source: self.source.unwrap(),
210
+ cmds: self.cmds.unwrap(),
211
+ view_access: self.view_access.unwrap(),
212
+ edit_access: self.edit_access.unwrap(),
213
+ }
214
+ }
215
+}
216
+
217
+/// Parse command string like "schema|get|update" into DynCfgCmds flags
218
+fn parse_cmds_string(cmds_str: &str) -> DynCfgCmds {
219
+ // Use the existing parsing functionality from DynCfgCmds
220
+ DynCfgCmds::from_str_multi(cmds_str).unwrap_or_else(DynCfgCmds::empty)
221
+}
222
+
223
+/// Configuration for Netdata schema generation
224
+#[derive(Debug, Clone)]
225
+struct NetdataSchemaConfig {
226
+ /// Whether to include the full page UI option
227
+ full_page: bool,
228
+ /// JSON Schema settings to use
229
+ schema_settings: SchemaSettings,
230
+}
231
+
232
+impl Default for NetdataSchemaConfig {
233
+ fn default() -> Self {
234
+ Self {
235
+ full_page: true,
236
+ schema_settings: SchemaSettings::draft07(),
237
+ }
238
+ }
239
+}
240
+
241
+/// Trait for types that can generate Netdata-compatible schemas with UI and config declarations
242
+pub trait NetdataSchema: JsonSchema {
243
+ /// Generate a comprehensive Netdata-compatible schema with jsonSchema, uiSchema, and configDeclaration
244
+ fn netdata_schema() -> serde_json::Value
245
+ where
246
+ Self: Sized,
247
+ {
248
+ let config = NetdataSchemaConfig::default();
249
+ let generator = SchemaGenerator::new(config.schema_settings.clone());
250
+ let mut json_schema = generator.into_root_schema_for::<Self>();
251
+
252
+ // Apply our UI schema collector transform
253
+ let mut ui_collector = CollectUISchema::default();
254
+ ui_collector.transform(&mut json_schema);
255
+
256
+ // Apply our config declaration collector
257
+ let mut config_collector = CollectConfigDeclaration::default();
258
+ config_collector.transform(&mut json_schema);
259
+
260
+ // Create the UI schema from collected information
261
+ let mut ui_schema = ui_collector.ui_schema;
262
+
263
+ // Add default UI options
264
+ if config.full_page {
265
+ ui_schema.insert(
266
+ "uiOptions".to_string(),
267
+ serde_json::json!({
268
+ "fullPage": true
269
+ }),
270
+ );
271
+ }
272
+
273
+ // Build the result object
274
+ let mut result = serde_json::json!({
275
+ "jsonSchema": json_schema,
276
+ "uiSchema": ui_schema
277
+ });
278
+
279
+ // Add config declaration if present
280
+ if let Some(config_decl) = config_collector.config_declaration {
281
+ result["configDeclaration"] = serde_json::json!({
282
+ "id": config_decl.id,
283
+ "status": config_decl.status.name(),
284
+ "type": config_decl.type_.name(),
285
+ "path": config_decl.path,
286
+ "sourceType": config_decl.source_type.name(),
287
+ "source": config_decl.source,
288
+ "cmds": config_decl.cmds.to_pipe_separated(),
289
+ "viewAccess": u32::from(config_decl.view_access),
290
+ "editAccess": u32::from(config_decl.edit_access)
291
+ });
292
+ }
293
+
294
+ result
295
+ }
296
+}
297
+
298
+/// Blanket implementation for all JsonSchema types
299
+impl<T> NetdataSchema for T where T: JsonSchema {}
src/crates/netdata-plugin/types/Cargo.toml
new
+12
@@ -0,0 +1,12 @@
1
+[package]
2
+name = "netdata-plugin-types"
3
+version.workspace = true
4
+edition.workspace = true
5
+
6
+[lints]
7
+workspace = true
8
+
9
+[dependencies]
10
+netdata-plugin-error = { path = "../error" }
11
+bitflags = { workspace = true }
12
+serde_json = { workspace = true }
src/crates/netdata-plugin/types/src/config.rs
new
+112
@@ -0,0 +1,112 @@
1
+use crate::{DynCfgCmds, DynCfgSourceType, DynCfgStatus, DynCfgType, HttpAccess};
2
+use netdata_plugin_error::{NetdataPluginError, Result};
3
+use std::convert::TryFrom;
4
+
5
+#[derive(Debug, Clone)]
6
+pub struct ConfigDeclaration {
7
+ pub id: String,
8
+ pub status: DynCfgStatus,
9
+ pub type_: DynCfgType,
10
+ pub path: String,
11
+ pub source_type: DynCfgSourceType,
12
+ pub source: String,
13
+ pub cmds: DynCfgCmds,
14
+ pub view_access: HttpAccess,
15
+ pub edit_access: HttpAccess,
16
+}
17
+
18
+impl TryFrom<&serde_json::Value> for ConfigDeclaration {
19
+ type Error = NetdataPluginError;
20
+
21
+ fn try_from(schema: &serde_json::Value) -> Result<Self> {
22
+ let config_decl = schema
23
+ .get("configDeclaration")
24
+ .ok_or(NetdataPluginError::Schema {
25
+ message: String::from("Missing configDeclaration key"),
26
+ })?;
27
+
28
+ let id = config_decl
29
+ .get("id")
30
+ .and_then(|v| v.as_str())
31
+ .ok_or(NetdataPluginError::Schema {
32
+ message: String::from("Missing id key in configDeclaration"),
33
+ })?
34
+ .to_string();
35
+
36
+ let status = config_decl
37
+ .get("status")
38
+ .and_then(|v| v.as_str())
39
+ .and_then(DynCfgStatus::from_name)
40
+ .ok_or(NetdataPluginError::Schema {
41
+ message: String::from("Missing status key in configDeclaration"),
42
+ })?;
43
+
44
+ let type_ = config_decl
45
+ .get("type")
46
+ .and_then(|v| v.as_str())
47
+ .and_then(DynCfgType::from_name)
48
+ .ok_or(NetdataPluginError::Schema {
49
+ message: String::from("Missing type key in configDeclaration"),
50
+ })?;
51
+
52
+ let path = config_decl
53
+ .get("path")
54
+ .and_then(|v| v.as_str())
55
+ .ok_or(NetdataPluginError::Schema {
56
+ message: String::from("Missing path key in configDeclaration"),
57
+ })?
58
+ .to_string();
59
+
60
+ let source_type = config_decl
61
+ .get("sourceType")
62
+ .and_then(|v| v.as_str())
63
+ .and_then(DynCfgSourceType::from_name)
64
+ .ok_or(NetdataPluginError::Schema {
65
+ message: String::from("Missing sourceType key in configDeclaration"),
66
+ })?;
67
+
68
+ let source = config_decl
69
+ .get("source")
70
+ .and_then(|v| v.as_str())
71
+ .ok_or(NetdataPluginError::Schema {
72
+ message: String::from("Missing source key in configDeclaration"),
73
+ })?
74
+ .to_string();
75
+
76
+ let cmds = config_decl
77
+ .get("cmds")
78
+ .and_then(|v| v.as_str())
79
+ .and_then(DynCfgCmds::from_str_multi)
80
+ .ok_or(NetdataPluginError::Schema {
81
+ message: String::from("Missing cmds key in configDeclaration"),
82
+ })?;
83
+
84
+ let view_access = config_decl
85
+ .get("viewAccess")
86
+ .and_then(|v| v.as_u64())
87
+ .map(|v| HttpAccess::from_u32(v as u32))
88
+ .ok_or(NetdataPluginError::Schema {
89
+ message: String::from("Missing viewAccess key in configDeclaration"),
90
+ })?;
91
+
92
+ let edit_access = config_decl
93
+ .get("editAccess")
94
+ .and_then(|v| v.as_u64())
95
+ .map(|v| HttpAccess::from_u32(v as u32))
96
+ .ok_or(NetdataPluginError::Schema {
97
+ message: String::from("Missing editAccess key in configDeclaration"),
98
+ })?;
99
+
100
+ Ok(ConfigDeclaration {
101
+ id,
102
+ status,
103
+ type_,
104
+ path,
105
+ source_type,
106
+ source,
107
+ cmds,
108
+ view_access,
109
+ edit_access,
110
+ })
111
+ }
112
+}
src/crates/netdata-plugin/types/src/dyncfg_cmds.rs
new
+336
@@ -0,0 +1,336 @@
1
+#![allow(dead_code)]
2
+
3
+use bitflags::bitflags;
4
+use std::fmt;
5
+use std::str::FromStr;
6
+
7
+bitflags! {
8
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
9
+ pub struct DynCfgCmds: u32 {
10
+ const GET = 1 << 0;
11
+ const SCHEMA = 1 << 1;
12
+ const UPDATE = 1 << 2;
13
+ const ADD = 1 << 3;
14
+ const TEST = 1 << 4;
15
+ const REMOVE = 1 << 5;
16
+ const ENABLE = 1 << 6;
17
+ const DISABLE = 1 << 7;
18
+ const RESTART = 1 << 8;
19
+ const USERCONFIG = 1 << 9;
20
+ }
21
+}
22
+
23
+impl DynCfgCmds {
24
+ /// Get the string name for a single command flag
25
+ pub fn flag_name(flag: Self) -> Option<&'static str> {
26
+ match flag {
27
+ Self::GET => Some("get"),
28
+ Self::SCHEMA => Some("schema"),
29
+ Self::UPDATE => Some("update"),
30
+ Self::ADD => Some("add"),
31
+ Self::TEST => Some("test"),
32
+ Self::REMOVE => Some("remove"),
33
+ Self::ENABLE => Some("enable"),
34
+ Self::DISABLE => Some("disable"),
35
+ Self::RESTART => Some("restart"),
36
+ Self::USERCONFIG => Some("userconfig"),
37
+ _ => None,
38
+ }
39
+ }
40
+
41
+ /// Parse a single command name to its flag
42
+ pub fn from_cmd_name(name: &str) -> Option<Self> {
43
+ match name.trim() {
44
+ "get" => Some(Self::GET),
45
+ "schema" => Some(Self::SCHEMA),
46
+ "update" => Some(Self::UPDATE),
47
+ "add" => Some(Self::ADD),
48
+ "test" => Some(Self::TEST),
49
+ "remove" => Some(Self::REMOVE),
50
+ "enable" => Some(Self::ENABLE),
51
+ "disable" => Some(Self::DISABLE),
52
+ "restart" => Some(Self::RESTART),
53
+ "userconfig" => Some(Self::USERCONFIG),
54
+ _ => None,
55
+ }
56
+ }
57
+
58
+ /// Parse from string with space or pipe-separated command names
59
+ /// Examples: "get | schema | restart", "get schema restart", "get|schema|restart"
60
+ pub fn from_str_multi(s: &str) -> Option<Self> {
61
+ let s = s.trim();
62
+ if s.is_empty() {
63
+ return Some(Self::empty());
64
+ }
65
+
66
+ let mut result = Self::empty();
67
+
68
+ // Split by both spaces and pipes, then filter out empty parts
69
+ for part in s.split([' ', '|']) {
70
+ let part = part.trim();
71
+ if part.is_empty() {
72
+ continue;
73
+ }
74
+
75
+ match Self::from_cmd_name(part) {
76
+ Some(flag) => result |= flag,
77
+ None => return None, // Invalid command name
78
+ }
79
+ }
80
+
81
+ Some(result)
82
+ }
83
+
84
+ /// Parse from byte slice
85
+ pub fn from_slice(bytes: &[u8]) -> Option<Self> {
86
+ let s = std::str::from_utf8(bytes).ok()?;
87
+ Self::from_str_multi(s)
88
+ }
89
+
90
+ /// Convert to a space-separated string representation
91
+ pub fn to_space_separated(&self) -> String {
92
+ if self.is_empty() {
93
+ return String::new();
94
+ }
95
+
96
+ let mut parts = Vec::new();
97
+
98
+ // Check each flag in order
99
+ for &flag in &[
100
+ Self::GET,
101
+ Self::SCHEMA,
102
+ Self::UPDATE,
103
+ Self::ADD,
104
+ Self::TEST,
105
+ Self::REMOVE,
106
+ Self::ENABLE,
107
+ Self::DISABLE,
108
+ Self::RESTART,
109
+ Self::USERCONFIG,
110
+ ] {
111
+ if self.contains(flag) {
112
+ if let Some(name) = Self::flag_name(flag) {
113
+ parts.push(name);
114
+ }
115
+ }
116
+ }
117
+
118
+ parts.join(" ")
119
+ }
120
+
121
+ /// Convert to a pipe-separated string representation
122
+ pub fn to_pipe_separated(&self) -> String {
123
+ if self.is_empty() {
124
+ return String::new();
125
+ }
126
+
127
+ let mut parts = Vec::new();
128
+
129
+ // Check each flag in order
130
+ for &flag in &[
131
+ Self::GET,
132
+ Self::SCHEMA,
133
+ Self::UPDATE,
134
+ Self::ADD,
135
+ Self::TEST,
136
+ Self::REMOVE,
137
+ Self::ENABLE,
138
+ Self::DISABLE,
139
+ Self::RESTART,
140
+ Self::USERCONFIG,
141
+ ] {
142
+ if self.contains(flag) {
143
+ if let Some(name) = Self::flag_name(flag) {
144
+ parts.push(name);
145
+ }
146
+ }
147
+ }
148
+
149
+ parts.join(" | ")
150
+ }
151
+}
152
+
153
+impl fmt::Display for DynCfgCmds {
154
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
155
+ write!(f, "{}", self.to_pipe_separated())
156
+ }
157
+}
158
+
159
+impl FromStr for DynCfgCmds {
160
+ type Err = ();
161
+
162
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
163
+ Self::from_str_multi(s).ok_or(())
164
+ }
165
+}
166
+
167
+impl From<u32> for DynCfgCmds {
168
+ fn from(value: u32) -> Self {
169
+ Self::from_bits_truncate(value)
170
+ }
171
+}
172
+
173
+impl From<DynCfgCmds> for u32 {
174
+ fn from(cmds: DynCfgCmds) -> Self {
175
+ cmds.bits()
176
+ }
177
+}
178
+
179
+#[cfg(test)]
180
+mod tests {
181
+ use super::*;
182
+
183
+ #[test]
184
+ fn test_flag_name() {
185
+ assert_eq!(DynCfgCmds::flag_name(DynCfgCmds::GET), Some("get"));
186
+ assert_eq!(DynCfgCmds::flag_name(DynCfgCmds::SCHEMA), Some("schema"));
187
+ assert_eq!(
188
+ DynCfgCmds::flag_name(DynCfgCmds::USERCONFIG),
189
+ Some("userconfig")
190
+ );
191
+ assert_eq!(
192
+ DynCfgCmds::flag_name(DynCfgCmds::GET | DynCfgCmds::SCHEMA),
193
+ None
194
+ );
195
+ }
196
+
197
+ #[test]
198
+ fn test_from_cmd_name() {
199
+ assert_eq!(DynCfgCmds::from_cmd_name("get"), Some(DynCfgCmds::GET));
200
+ assert_eq!(
201
+ DynCfgCmds::from_cmd_name("schema"),
202
+ Some(DynCfgCmds::SCHEMA)
203
+ );
204
+ assert_eq!(
205
+ DynCfgCmds::from_cmd_name("userconfig"),
206
+ Some(DynCfgCmds::USERCONFIG)
207
+ );
208
+ assert_eq!(
209
+ DynCfgCmds::from_cmd_name(" restart "),
210
+ Some(DynCfgCmds::RESTART)
211
+ );
212
+ assert_eq!(DynCfgCmds::from_cmd_name("invalid"), None);
213
+ }
214
+
215
+ #[test]
216
+ fn test_from_str_multi_pipe_separated() {
217
+ let result = DynCfgCmds::from_str_multi("get | schema | restart").unwrap();
218
+ assert_eq!(
219
+ result,
220
+ DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART
221
+ );
222
+
223
+ let result = DynCfgCmds::from_str_multi("get|schema|restart").unwrap();
224
+ assert_eq!(
225
+ result,
226
+ DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART
227
+ );
228
+ }
229
+
230
+ #[test]
231
+ fn test_from_str_multi_space_separated() {
232
+ let result = DynCfgCmds::from_str_multi("get schema restart").unwrap();
233
+ assert_eq!(
234
+ result,
235
+ DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART
236
+ );
237
+
238
+ let result = DynCfgCmds::from_str_multi(" get schema restart ").unwrap();
239
+ assert_eq!(
240
+ result,
241
+ DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART
242
+ );
243
+ }
244
+
245
+ #[test]
246
+ fn test_from_str_multi_mixed_separators() {
247
+ let result = DynCfgCmds::from_str_multi("get | schema restart").unwrap();
248
+ assert_eq!(
249
+ result,
250
+ DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART
251
+ );
252
+ }
253
+
254
+ #[test]
255
+ fn test_from_str_multi_single_command() {
256
+ let result = DynCfgCmds::from_str_multi("get").unwrap();
257
+ assert_eq!(result, DynCfgCmds::GET);
258
+ }
259
+
260
+ #[test]
261
+ fn test_from_str_multi_empty() {
262
+ let result = DynCfgCmds::from_str_multi("").unwrap();
263
+ assert_eq!(result, DynCfgCmds::empty());
264
+
265
+ let result = DynCfgCmds::from_str_multi(" ").unwrap();
266
+ assert_eq!(result, DynCfgCmds::empty());
267
+ }
268
+
269
+ #[test]
270
+ fn test_from_str_multi_invalid() {
271
+ assert_eq!(DynCfgCmds::from_str_multi("get | invalid | schema"), None);
272
+ assert_eq!(DynCfgCmds::from_str_multi("invalid"), None);
273
+ }
274
+
275
+ #[test]
276
+ fn test_from_slice() {
277
+ let result = DynCfgCmds::from_slice(b"get | schema").unwrap();
278
+ assert_eq!(result, DynCfgCmds::GET | DynCfgCmds::SCHEMA);
279
+
280
+ assert_eq!(DynCfgCmds::from_slice(&[0xFF, 0xFE]), None); // Invalid UTF-8
281
+ }
282
+
283
+ #[test]
284
+ fn test_to_space_separated() {
285
+ let cmds = DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART;
286
+ assert_eq!(cmds.to_space_separated(), "get schema restart");
287
+
288
+ assert_eq!(DynCfgCmds::empty().to_space_separated(), "");
289
+ assert_eq!(DynCfgCmds::GET.to_space_separated(), "get");
290
+ }
291
+
292
+ #[test]
293
+ fn test_to_pipe_separated() {
294
+ let cmds = DynCfgCmds::GET | DynCfgCmds::SCHEMA | DynCfgCmds::RESTART;
295
+ assert_eq!(cmds.to_pipe_separated(), "get | schema | restart");
296
+
297
+ assert_eq!(DynCfgCmds::empty().to_pipe_separated(), "");
298
+ assert_eq!(DynCfgCmds::GET.to_pipe_separated(), "get");
299
+ }
300
+
301
+ #[test]
302
+ fn test_display() {
303
+ let cmds = DynCfgCmds::GET | DynCfgCmds::SCHEMA;
304
+ assert_eq!(format!("{}", cmds), "get | schema");
305
+ }
306
+
307
+ #[test]
308
+ fn test_from_str_trait() {
309
+ let cmds: DynCfgCmds = "get | schema".parse().unwrap();
310
+ assert_eq!(cmds, DynCfgCmds::GET | DynCfgCmds::SCHEMA);
311
+
312
+ assert!("invalid".parse::<DynCfgCmds>().is_err());
313
+ }
314
+
315
+ #[test]
316
+ fn test_u32_conversion() {
317
+ let cmds = DynCfgCmds::GET | DynCfgCmds::SCHEMA; // bits 0 and 1 = 3
318
+ let value: u32 = cmds.into();
319
+ assert_eq!(value, 3);
320
+
321
+ let cmds_from_u32: DynCfgCmds = 3u32.into();
322
+ assert_eq!(cmds_from_u32, cmds);
323
+ }
324
+
325
+ #[test]
326
+ fn test_bitflags_operations() {
327
+ let cmds = DynCfgCmds::GET | DynCfgCmds::SCHEMA;
328
+
329
+ assert!(cmds.contains(DynCfgCmds::GET));
330
+ assert!(cmds.contains(DynCfgCmds::SCHEMA));
331
+ assert!(!cmds.contains(DynCfgCmds::UPDATE));
332
+
333
+ assert!(cmds.intersects(DynCfgCmds::GET | DynCfgCmds::UPDATE));
334
+ assert!(!cmds.intersects(DynCfgCmds::UPDATE | DynCfgCmds::ADD));
335
+ }
336
+}
src/crates/netdata-plugin/types/src/dyncfg_source_type.rs
new
+123
@@ -0,0 +1,123 @@
1
+#![allow(dead_code)]
2
+
3
+use std::fmt;
4
+use std::str::FromStr;
5
+
6
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7
+pub enum DynCfgSourceType {
8
+ Internal,
9
+ Stock,
10
+ User,
11
+ Dyncfg,
12
+ Discovered,
13
+}
14
+
15
+impl DynCfgSourceType {
16
+ /// Get the string name for this source type
17
+ pub fn name(&self) -> &'static str {
18
+ match self {
19
+ Self::Internal => "internal",
20
+ Self::Stock => "stock",
21
+ Self::User => "user",
22
+ Self::Dyncfg => "dyncfg",
23
+ Self::Discovered => "discovered",
24
+ }
25
+ }
26
+
27
+ /// Parse from string name
28
+ pub fn from_name(name: &str) -> Option<Self> {
29
+ match name {
30
+ "internal" => Some(Self::Internal),
31
+ "stock" => Some(Self::Stock),
32
+ "user" => Some(Self::User),
33
+ "dyncfg" => Some(Self::Dyncfg),
34
+ "discovered" => Some(Self::Discovered),
35
+ _ => None,
36
+ }
37
+ }
38
+
39
+ /// Parse from byte slice
40
+ pub fn from_slice(bytes: &[u8]) -> Option<Self> {
41
+ let s = std::str::from_utf8(bytes).ok()?.trim();
42
+ Self::from_name(s)
43
+ }
44
+}
45
+
46
+impl Default for DynCfgSourceType {
47
+ fn default() -> Self {
48
+ Self::Internal
49
+ }
50
+}
51
+
52
+impl fmt::Display for DynCfgSourceType {
53
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
54
+ write!(f, "{}", self.name())
55
+ }
56
+}
57
+
58
+impl FromStr for DynCfgSourceType {
59
+ type Err = ();
60
+
61
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
62
+ Self::from_name(s).ok_or(())
63
+ }
64
+}
65
+
66
+#[cfg(test)]
67
+mod tests {
68
+ use super::*;
69
+
70
+ #[test]
71
+ fn test_name() {
72
+ assert_eq!(DynCfgSourceType::Internal.name(), "internal");
73
+ assert_eq!(DynCfgSourceType::Stock.name(), "stock");
74
+ assert_eq!(DynCfgSourceType::User.name(), "user");
75
+ assert_eq!(DynCfgSourceType::Dyncfg.name(), "dyncfg");
76
+ assert_eq!(DynCfgSourceType::Discovered.name(), "discovered");
77
+ }
78
+
79
+ #[test]
80
+ fn test_from_name() {
81
+ assert_eq!(DynCfgSourceType::from_name("internal"), Some(DynCfgSourceType::Internal));
82
+ assert_eq!(DynCfgSourceType::from_name("stock"), Some(DynCfgSourceType::Stock));
83
+ assert_eq!(DynCfgSourceType::from_name("user"), Some(DynCfgSourceType::User));
84
+ assert_eq!(DynCfgSourceType::from_name("dyncfg"), Some(DynCfgSourceType::Dyncfg));
85
+ assert_eq!(DynCfgSourceType::from_name("discovered"), Some(DynCfgSourceType::Discovered));
86
+ assert_eq!(DynCfgSourceType::from_name("invalid"), None);
87
+ }
88
+
89
+ #[test]
90
+ fn test_from_slice() {
91
+ assert_eq!(DynCfgSourceType::from_slice(b"internal"), Some(DynCfgSourceType::Internal));
92
+ assert_eq!(DynCfgSourceType::from_slice(b" stock "), Some(DynCfgSourceType::Stock));
93
+ assert_eq!(DynCfgSourceType::from_slice(b"user"), Some(DynCfgSourceType::User));
94
+ assert_eq!(DynCfgSourceType::from_slice(b"dyncfg"), Some(DynCfgSourceType::Dyncfg));
95
+ assert_eq!(DynCfgSourceType::from_slice(b"discovered"), Some(DynCfgSourceType::Discovered));
96
+ assert_eq!(DynCfgSourceType::from_slice(b"invalid"), None);
97
+ assert_eq!(DynCfgSourceType::from_slice(&[0xFF, 0xFE]), None); // Invalid UTF-8
98
+ }
99
+
100
+ #[test]
101
+ fn test_display() {
102
+ assert_eq!(format!("{}", DynCfgSourceType::Internal), "internal");
103
+ assert_eq!(format!("{}", DynCfgSourceType::Stock), "stock");
104
+ assert_eq!(format!("{}", DynCfgSourceType::User), "user");
105
+ assert_eq!(format!("{}", DynCfgSourceType::Dyncfg), "dyncfg");
106
+ assert_eq!(format!("{}", DynCfgSourceType::Discovered), "discovered");
107
+ }
108
+
109
+ #[test]
110
+ fn test_from_str() {
111
+ assert_eq!("internal".parse::<DynCfgSourceType>(), Ok(DynCfgSourceType::Internal));
112
+ assert_eq!("stock".parse::<DynCfgSourceType>(), Ok(DynCfgSourceType::Stock));
113
+ assert_eq!("user".parse::<DynCfgSourceType>(), Ok(DynCfgSourceType::User));
114
+ assert_eq!("dyncfg".parse::<DynCfgSourceType>(), Ok(DynCfgSourceType::Dyncfg));
115
+ assert_eq!("discovered".parse::<DynCfgSourceType>(), Ok(DynCfgSourceType::Discovered));
116
+ assert_eq!("invalid".parse::<DynCfgSourceType>(), Err(()));
117
+ }
118
+
119
+ #[test]
120
+ fn test_default() {
121
+ assert_eq!(DynCfgSourceType::default(), DynCfgSourceType::Internal);
122
+ }
123
+}
\ No newline at end of file
src/crates/netdata-plugin/types/src/dyncfg_status.rs
new
+125
@@ -0,0 +1,125 @@
1
+#![allow(dead_code)]
2
+
3
+use std::fmt;
4
+use std::str::FromStr;
5
+
6
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7
+pub enum DynCfgStatus {
8
+ None,
9
+ Accepted,
10
+ Running,
11
+ Failed,
12
+ Disabled,
13
+ Orphan,
14
+ Incomplete,
15
+}
16
+
17
+impl DynCfgStatus {
18
+ /// Get the string name for this status
19
+ pub fn name(&self) -> &'static str {
20
+ match self {
21
+ Self::None => "none",
22
+ Self::Accepted => "accepted",
23
+ Self::Running => "running",
24
+ Self::Failed => "failed",
25
+ Self::Disabled => "disabled",
26
+ Self::Orphan => "orphan",
27
+ Self::Incomplete => "incomplete",
28
+ }
29
+ }
30
+
31
+ /// Parse from string name
32
+ pub fn from_name(name: &str) -> Option<Self> {
33
+ match name {
34
+ "none" => Some(Self::None),
35
+ "accepted" => Some(Self::Accepted),
36
+ "running" => Some(Self::Running),
37
+ "failed" => Some(Self::Failed),
38
+ "disabled" => Some(Self::Disabled),
39
+ "orphan" => Some(Self::Orphan),
40
+ "incomplete" => Some(Self::Incomplete),
41
+ _ => None,
42
+ }
43
+ }
44
+
45
+ /// Parse from byte slice
46
+ pub fn from_slice(bytes: &[u8]) -> Option<Self> {
47
+ let s = std::str::from_utf8(bytes).ok()?.trim();
48
+ Self::from_name(s)
49
+ }
50
+}
51
+
52
+impl Default for DynCfgStatus {
53
+ fn default() -> Self {
54
+ Self::None
55
+ }
56
+}
57
+
58
+impl fmt::Display for DynCfgStatus {
59
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
60
+ write!(f, "{}", self.name())
61
+ }
62
+}
63
+
64
+impl FromStr for DynCfgStatus {
65
+ type Err = ();
66
+
67
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
68
+ Self::from_name(s).ok_or(())
69
+ }
70
+}
71
+
72
+#[cfg(test)]
73
+mod tests {
74
+ use super::*;
75
+
76
+ #[test]
77
+ fn test_name() {
78
+ assert_eq!(DynCfgStatus::None.name(), "none");
79
+ assert_eq!(DynCfgStatus::Accepted.name(), "accepted");
80
+ assert_eq!(DynCfgStatus::Running.name(), "running");
81
+ assert_eq!(DynCfgStatus::Failed.name(), "failed");
82
+ assert_eq!(DynCfgStatus::Disabled.name(), "disabled");
83
+ assert_eq!(DynCfgStatus::Orphan.name(), "orphan");
84
+ assert_eq!(DynCfgStatus::Incomplete.name(), "incomplete");
85
+ }
86
+
87
+ #[test]
88
+ fn test_from_name() {
89
+ assert_eq!(DynCfgStatus::from_name("none"), Some(DynCfgStatus::None));
90
+ assert_eq!(DynCfgStatus::from_name("accepted"), Some(DynCfgStatus::Accepted));
91
+ assert_eq!(DynCfgStatus::from_name("running"), Some(DynCfgStatus::Running));
92
+ assert_eq!(DynCfgStatus::from_name("failed"), Some(DynCfgStatus::Failed));
93
+ assert_eq!(DynCfgStatus::from_name("disabled"), Some(DynCfgStatus::Disabled));
94
+ assert_eq!(DynCfgStatus::from_name("orphan"), Some(DynCfgStatus::Orphan));
95
+ assert_eq!(DynCfgStatus::from_name("incomplete"), Some(DynCfgStatus::Incomplete));
96
+ assert_eq!(DynCfgStatus::from_name("invalid"), None);
97
+ }
98
+
99
+ #[test]
100
+ fn test_from_slice() {
101
+ assert_eq!(DynCfgStatus::from_slice(b"running"), Some(DynCfgStatus::Running));
102
+ assert_eq!(DynCfgStatus::from_slice(b" failed "), Some(DynCfgStatus::Failed));
103
+ assert_eq!(DynCfgStatus::from_slice(b"invalid"), None);
104
+ assert_eq!(DynCfgStatus::from_slice(&[0xFF, 0xFE]), None); // Invalid UTF-8
105
+ }
106
+
107
+ #[test]
108
+ fn test_display() {
109
+ assert_eq!(format!("{}", DynCfgStatus::None), "none");
110
+ assert_eq!(format!("{}", DynCfgStatus::Accepted), "accepted");
111
+ assert_eq!(format!("{}", DynCfgStatus::Running), "running");
112
+ }
113
+
114
+ #[test]
115
+ fn test_from_str() {
116
+ assert_eq!("none".parse::<DynCfgStatus>(), Ok(DynCfgStatus::None));
117
+ assert_eq!("running".parse::<DynCfgStatus>(), Ok(DynCfgStatus::Running));
118
+ assert_eq!("invalid".parse::<DynCfgStatus>(), Err(()));
119
+ }
120
+
121
+ #[test]
122
+ fn test_default() {
123
+ assert_eq!(DynCfgStatus::default(), DynCfgStatus::None);
124
+ }
125
+}
\ No newline at end of file
src/crates/netdata-plugin/types/src/dyncfg_type.rs
new
+113
@@ -0,0 +1,113 @@
1
+#![allow(dead_code)]
2
+
3
+use std::fmt;
4
+use std::str::FromStr;
5
+
6
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
7
+pub enum DynCfgType {
8
+ Single,
9
+ Template,
10
+ Job,
11
+}
12
+
13
+impl DynCfgType {
14
+ /// Get the string name for this type
15
+ pub fn name(&self) -> &'static str {
16
+ match self {
17
+ Self::Single => "single",
18
+ Self::Template => "template",
19
+ Self::Job => "job",
20
+ }
21
+ }
22
+
23
+ /// Parse from string name
24
+ pub fn from_name(name: &str) -> Option<Self> {
25
+ match name {
26
+ "single" => Some(Self::Single),
27
+ "template" => Some(Self::Template),
28
+ "job" => Some(Self::Job),
29
+ _ => None,
30
+ }
31
+ }
32
+
33
+ /// Parse from byte slice
34
+ pub fn from_slice(bytes: &[u8]) -> Option<Self> {
35
+ let s = std::str::from_utf8(bytes).ok()?.trim();
36
+ Self::from_name(s)
37
+ }
38
+}
39
+
40
+impl Default for DynCfgType {
41
+ fn default() -> Self {
42
+ Self::Single
43
+ }
44
+}
45
+
46
+impl fmt::Display for DynCfgType {
47
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
48
+ write!(f, "{}", self.name())
49
+ }
50
+}
51
+
52
+impl FromStr for DynCfgType {
53
+ type Err = ();
54
+
55
+ fn from_str(s: &str) -> Result<Self, Self::Err> {
56
+ Self::from_name(s).ok_or(())
57
+ }
58
+}
59
+
60
+#[cfg(test)]
61
+mod tests {
62
+ use super::*;
63
+
64
+ #[test]
65
+ fn test_name() {
66
+ assert_eq!(DynCfgType::Single.name(), "single");
67
+ assert_eq!(DynCfgType::Template.name(), "template");
68
+ assert_eq!(DynCfgType::Job.name(), "job");
69
+ }
70
+
71
+ #[test]
72
+ fn test_from_name() {
73
+ assert_eq!(DynCfgType::from_name("single"), Some(DynCfgType::Single));
74
+ assert_eq!(
75
+ DynCfgType::from_name("template"),
76
+ Some(DynCfgType::Template)
77
+ );
78
+ assert_eq!(DynCfgType::from_name("job"), Some(DynCfgType::Job));
79
+ assert_eq!(DynCfgType::from_name("invalid"), None);
80
+ }
81
+
82
+ #[test]
83
+ fn test_from_slice() {
84
+ assert_eq!(DynCfgType::from_slice(b"single"), Some(DynCfgType::Single));
85
+ assert_eq!(
86
+ DynCfgType::from_slice(b" template "),
87
+ Some(DynCfgType::Template)
88
+ );
89
+ assert_eq!(DynCfgType::from_slice(b"job"), Some(DynCfgType::Job));
90
+ assert_eq!(DynCfgType::from_slice(b"invalid"), None);
91
+ assert_eq!(DynCfgType::from_slice(&[0xFF, 0xFE]), None); // Invalid UTF-8
92
+ }
93
+
94
+ #[test]
95
+ fn test_display() {
96
+ assert_eq!(format!("{}", DynCfgType::Single), "single");
97
+ assert_eq!(format!("{}", DynCfgType::Template), "template");
98
+ assert_eq!(format!("{}", DynCfgType::Job), "job");
99
+ }
100
+
101
+ #[test]
102
+ fn test_from_str() {
103
+ assert_eq!("single".parse::<DynCfgType>(), Ok(DynCfgType::Single));
104
+ assert_eq!("template".parse::<DynCfgType>(), Ok(DynCfgType::Template));
105
+ assert_eq!("job".parse::<DynCfgType>(), Ok(DynCfgType::Job));
106
+ assert_eq!("invalid".parse::<DynCfgType>(), Err(()));
107
+ }
108
+
109
+ #[test]
110
+ fn test_default() {
111
+ assert_eq!(DynCfgType::default(), DynCfgType::Single);
112
+ }
113
+}
src/crates/netdata-plugin/types/src/functions.rs
new
+85
@@ -0,0 +1,85 @@
1
+use crate::HttpAccess;
2
+
3
+/// A function declaration message for Netdata's external plugin protocol
4
+#[derive(Debug, Clone)]
5
+pub struct FunctionDeclaration {
6
+ /// True if the function is global
7
+ pub global: bool,
8
+ /// The name of the function
9
+ pub name: String,
10
+ /// Timeout in seconds for function execution
11
+ pub timeout: u32,
12
+ /// Help text describing what the function does
13
+ pub help: String,
14
+ /// Tags of the function
15
+ pub tags: Option<String>,
16
+ /// Access control flags for the function
17
+ pub access: Option<HttpAccess>,
18
+ /// Priority level for function execution
19
+ pub priority: Option<u32>,
20
+ /// Version of the function
21
+ pub version: Option<u32>,
22
+}
23
+
24
+impl FunctionDeclaration {
25
+ pub fn new(name: &str, help: &str) -> Self {
26
+ Self {
27
+ name: String::from(name),
28
+ help: String::from(help),
29
+ timeout: 10,
30
+ tags: Some("logs".to_string()),
31
+ access: Some(HttpAccess::from_u32(0)),
32
+ priority: Some(200),
33
+ version: Some(1),
34
+ global: false,
35
+ }
36
+ }
37
+}
38
+
39
+/// A function call message for invoking functions
40
+#[derive(Debug, Clone)]
41
+pub struct FunctionCall {
42
+ /// Transaction ID for this function call
43
+ pub transaction: String,
44
+ /// Timeout in seconds for function execution
45
+ pub timeout: u32,
46
+ /// Function name to call
47
+ pub name: String,
48
+ /// Function arguments
49
+ pub args: Vec<String>,
50
+ /// Access control flags for the function
51
+ pub access: Option<HttpAccess>,
52
+ /// Source information containing caller details
53
+ pub source: Option<String>,
54
+ /// Payload data for the function call (optional)
55
+ pub payload: Option<Vec<u8>>,
56
+}
57
+
58
+/// A function result message containing the response payload
59
+#[derive(Debug, Clone)]
60
+pub struct FunctionResult {
61
+ /// Transaction ID or unique identifier for this function call
62
+ pub transaction: String,
63
+ /// Status of the function call
64
+ pub status: u32,
65
+ /// Content type of the result (e.g., "application/json", "text/plain")
66
+ pub format: String,
67
+ /// Expires timestamp
68
+ pub expires: u64,
69
+ /// Result payload data
70
+ pub payload: Vec<u8>,
71
+}
72
+
73
+/// A function cancel message for terminating function execution
74
+#[derive(Debug, Clone)]
75
+pub struct FunctionCancel {
76
+ /// Transaction ID of the function call to cancel
77
+ pub transaction: String,
78
+}
79
+
80
+/// A message for reporting function call progress
81
+#[derive(Debug, Clone)]
82
+pub struct FunctionProgress {
83
+ /// Transaction ID of the function call that should report the progress
84
+ pub transaction: String,
85
+}
src/crates/netdata-plugin/types/src/http_access.rs
new
+155
@@ -0,0 +1,155 @@
1
+#![allow(dead_code)]
2
+
3
+use bitflags::bitflags;
4
+use std::fmt;
5
+
6
+bitflags! {
7
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
8
+ pub struct HttpAccess: u32 {
9
+ const SIGNED_ID = 1 << 0;
10
+ const SAME_SPACE = 1 << 1;
11
+ const COMMERCIAL_SPACE = 1 << 2;
12
+ const ANONYMOUS_DATA = 1 << 3;
13
+ const SENSITIVE_DATA = 1 << 4;
14
+ const VIEW_AGENT_CONFIG = 1 << 5;
15
+ const EDIT_AGENT_CONFIG = 1 << 6;
16
+ const VIEW_NOTIFICATIONS_CONFIG = 1 << 7;
17
+ const EDIT_NOTIFICATIONS_CONFIG = 1 << 8;
18
+ const VIEW_ALERTS_SILENCING = 1 << 9;
19
+ const EDIT_ALERTS_SILENCING = 1 << 10;
20
+ }
21
+}
22
+
23
+impl HttpAccess {
24
+ pub const ALL: Self = Self::from_bits_truncate(0x7FF);
25
+
26
+ // Old role mappings
27
+ pub const MAP_OLD_ANY: Self = Self::ANONYMOUS_DATA;
28
+
29
+ pub const MAP_OLD_MEMBER: Self = Self::SIGNED_ID
30
+ .union(Self::SAME_SPACE)
31
+ .union(Self::ANONYMOUS_DATA)
32
+ .union(Self::SENSITIVE_DATA);
33
+
34
+ pub const MAP_OLD_ADMIN: Self = Self::SIGNED_ID
35
+ .union(Self::SAME_SPACE)
36
+ .union(Self::ANONYMOUS_DATA)
37
+ .union(Self::SENSITIVE_DATA)
38
+ .union(Self::VIEW_AGENT_CONFIG)
39
+ .union(Self::EDIT_AGENT_CONFIG);
40
+
41
+ pub fn from_hex(s: &str) -> Option<Self> {
42
+ let s = s.trim();
43
+ if s.is_empty() {
44
+ return Some(Self::empty());
45
+ }
46
+
47
+ let s = s.strip_prefix("0x").unwrap_or(s);
48
+ u32::from_str_radix(s, 16)
49
+ .ok()
50
+ .map(|v| Self::from_bits_truncate(v & Self::ALL.bits()))
51
+ }
52
+
53
+ pub fn from_slice(bytes: &[u8]) -> Self {
54
+ let s = std::str::from_utf8(bytes).unwrap_or("").trim();
55
+ if s.is_empty() {
56
+ return Self::empty();
57
+ }
58
+
59
+ match s {
60
+ "any" | "all" => Self::MAP_OLD_ANY,
61
+ "member" | "members" => Self::MAP_OLD_MEMBER,
62
+ "admin" | "admins" => Self::MAP_OLD_ADMIN,
63
+ _ => Self::from_hex(s).unwrap_or_else(Self::empty),
64
+ }
65
+ }
66
+
67
+ pub fn has(&self, other: Self) -> bool {
68
+ self.contains(other)
69
+ }
70
+
71
+ pub fn as_u32(&self) -> u32 {
72
+ self.bits()
73
+ }
74
+
75
+ pub fn from_u32(value: u32) -> Self {
76
+ Self::from_bits_truncate(value & Self::ALL.bits())
77
+ }
78
+}
79
+
80
+impl From<u32> for HttpAccess {
81
+ fn from(value: u32) -> Self {
82
+ Self::from_u32(value)
83
+ }
84
+}
85
+
86
+impl From<HttpAccess> for u32 {
87
+ fn from(access: HttpAccess) -> Self {
88
+ access.bits()
89
+ }
90
+}
91
+
92
+impl fmt::Display for HttpAccess {
93
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
94
+ write!(f, "0x{:x}", self.bits())
95
+ }
96
+}
97
+
98
+#[cfg(test)]
99
+mod tests {
100
+ use super::*;
101
+
102
+ #[test]
103
+ fn test_from_hex() {
104
+ assert_eq!(
105
+ HttpAccess::from_hex("0x9"),
106
+ Some(HttpAccess::from_bits_truncate(0x9))
107
+ );
108
+ assert_eq!(HttpAccess::from_hex("7ff"), Some(HttpAccess::ALL));
109
+ assert_eq!(HttpAccess::from_hex(""), Some(HttpAccess::empty()));
110
+ }
111
+
112
+ #[test]
113
+ fn test_from_hex_mapping_old_roles() {
114
+ assert_eq!(HttpAccess::from_slice(b"any"), HttpAccess::MAP_OLD_ANY);
115
+ assert_eq!(HttpAccess::from_slice(b"all"), HttpAccess::MAP_OLD_ANY);
116
+ assert_eq!(
117
+ HttpAccess::from_slice(b"member"),
118
+ HttpAccess::MAP_OLD_MEMBER
119
+ );
120
+ assert_eq!(
121
+ HttpAccess::from_slice(b"members"),
122
+ HttpAccess::MAP_OLD_MEMBER
123
+ );
124
+ assert_eq!(HttpAccess::from_slice(b"admin"), HttpAccess::MAP_OLD_ADMIN);
125
+ assert_eq!(HttpAccess::from_slice(b"admins"), HttpAccess::MAP_OLD_ADMIN);
126
+ assert_eq!(HttpAccess::from_slice(b"0x7ff"), HttpAccess::ALL);
127
+ assert_eq!(HttpAccess::from_slice(b""), HttpAccess::empty());
128
+ }
129
+
130
+ #[test]
131
+ fn test_has() {
132
+ let access = HttpAccess::from_hex("0x9").unwrap();
133
+ assert!(access.has(HttpAccess::SIGNED_ID));
134
+ assert!(access.has(HttpAccess::ANONYMOUS_DATA));
135
+ assert!(!access.has(HttpAccess::SENSITIVE_DATA));
136
+ }
137
+
138
+ #[test]
139
+ fn test_u32_conversion() {
140
+ // Test from_u32 and as_u32
141
+ let access = HttpAccess::from_u32(0x9);
142
+ assert_eq!(access.as_u32(), 0x9);
143
+
144
+ // Test From traits
145
+ let access: HttpAccess = 0x9u32.into();
146
+ assert_eq!(access, HttpAccess::from_bits_truncate(0x9));
147
+
148
+ let value: u32 = HttpAccess::SIGNED_ID.into();
149
+ assert_eq!(value, 1);
150
+
151
+ // Test that values beyond ALL are masked
152
+ let access = HttpAccess::from_u32(0xFFFF);
153
+ assert_eq!(access.as_u32(), 0x7FF);
154
+ }
155
+}
src/crates/netdata-plugin/types/src/lib.rs
new
+23
@@ -0,0 +1,23 @@
1
+//! Common types for Netdata plugins
2
+//!
3
+//! This crate provides standalone types used across Netdata's plugin ecosystem,
4
+//! including HTTP access control, dynamic configuration types, and command flags.
5
+
6
+mod config;
7
+mod dyncfg_cmds;
8
+mod dyncfg_source_type;
9
+mod dyncfg_status;
10
+mod dyncfg_type;
11
+mod functions;
12
+mod http_access;
13
+
14
+pub use config::ConfigDeclaration;
15
+pub use dyncfg_cmds::DynCfgCmds;
16
+pub use dyncfg_source_type::DynCfgSourceType;
17
+pub use dyncfg_status::DynCfgStatus;
18
+pub use dyncfg_type::DynCfgType;
19
+
20
+pub use functions::{
21
+ FunctionCall, FunctionCancel, FunctionDeclaration, FunctionProgress, FunctionResult,
22
+};
23
+pub use http_access::HttpAccess;
src/crates/rdp/.gitignore
new
+1
@@ -0,0 +1 @@
1
+/target
src/crates/rdp/Cargo.toml
new
+19
@@ -0,0 +1,19 @@
1
+[package]
2
+name = "rdp"
3
+version.workspace = true
4
+edition.workspace = true
5
+rust-version.workspace = true
6
+
7
+[lints]
8
+workspace = true
9
+
10
+[lib]
11
+name = "rdp"
12
+path = "src/lib.rs"
13
+
14
+[[bin]]
15
+name = "rdp"
16
+path = "src/main.rs"
17
+
18
+[dependencies]
19
+md5 = { workspace = true }
src/crates/rdp/src/lib.rs
new
+829
@@ -0,0 +1,829 @@
1
+// String tokenizer and parser for field names with compact encoding
2
+//
3
+// Tokenizes strings into words (lowercase, UPPERCASE, Capitalized) and separators (. _ -)
4
+// Parses tokens into fields: Lowercase, Uppercase, LowerCamel, UpperCamel, Empty
5
+// Encodes token stream into compact lossless representation
6
+//
7
+// Example: "log.body.HostName" → encoded: "C3aao" (5 chars: 2-char checksum + 3-char structure)
8
+
9
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
10
+enum CharType {
11
+ Lowercase,
12
+ Uppercase,
13
+ Dot,
14
+ Underscore,
15
+ Hyphen,
16
+}
17
+
18
+fn char_type(c: u8) -> Option<CharType> {
19
+ if c.is_ascii_lowercase() {
20
+ Some(CharType::Lowercase)
21
+ } else if c.is_ascii_uppercase() || c.is_ascii_digit() {
22
+ Some(CharType::Uppercase)
23
+ } else if c == b'.' {
24
+ Some(CharType::Dot)
25
+ } else if c == b'_' {
26
+ Some(CharType::Underscore)
27
+ } else if c == b'-' {
28
+ Some(CharType::Hyphen)
29
+ } else {
30
+ None
31
+ }
32
+}
33
+
34
+/// Prefix for field names that were remapped due to containing invalid characters.
35
+const REMAPPED_PREFIX: &str = "ND_";
36
+
37
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
38
+enum TokenType {
39
+ Lowercase,
40
+ Uppercase,
41
+ Capitalized,
42
+}
43
+
44
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
45
+enum Separator {
46
+ Dot,
47
+ Hyphen,
48
+ Underscore,
49
+}
50
+
51
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
52
+enum Token {
53
+ Word {
54
+ kind: TokenType,
55
+ start: usize,
56
+ end: usize,
57
+ },
58
+ Separator(Separator),
59
+}
60
+
61
+fn create_token(
62
+ first: CharType,
63
+ has_lowercase: bool,
64
+ has_uppercase: bool,
65
+ start: usize,
66
+ end: usize,
67
+) -> Token {
68
+ match first {
69
+ CharType::Lowercase => {
70
+ assert!(!has_uppercase);
71
+ Token::Word {
72
+ kind: TokenType::Lowercase,
73
+ start,
74
+ end,
75
+ }
76
+ }
77
+ CharType::Uppercase => {
78
+ assert!(!(has_lowercase && has_uppercase));
79
+
80
+ if has_lowercase {
81
+ // Rest are lowercase
82
+ Token::Word {
83
+ kind: TokenType::Capitalized,
84
+ start,
85
+ end,
86
+ }
87
+ } else {
88
+ // All uppercase
89
+ Token::Word {
90
+ kind: TokenType::Uppercase,
91
+ start,
92
+ end,
93
+ }
94
+ }
95
+ }
96
+ CharType::Dot => Token::Separator(Separator::Dot),
97
+ CharType::Hyphen => Token::Separator(Separator::Hyphen),
98
+ CharType::Underscore => Token::Separator(Separator::Underscore),
99
+ }
100
+}
101
+
102
+fn tokenize(s: &[u8]) -> Option<Vec<Token>> {
103
+ let mut tokens = Vec::new();
104
+
105
+ if s.is_empty() {
106
+ return Some(tokens);
107
+ }
108
+
109
+ let mut start = 0;
110
+ let mut chars = s.iter().enumerate().peekable();
111
+ let mut prev_type: Option<CharType> = None;
112
+
113
+ // track the current word characteristics
114
+ let mut first_type: Option<CharType> = None;
115
+ let mut has_lowercase = false;
116
+ let mut has_uppercase = false;
117
+
118
+ while let Some((i, &ch)) = chars.next() {
119
+ let curr_type = char_type(ch)?;
120
+
121
+ let Some(prev) = prev_type else {
122
+ // first character of the string
123
+ first_type = Some(curr_type);
124
+ prev_type = Some(curr_type);
125
+ continue;
126
+ };
127
+
128
+ let should_split = match (prev, curr_type) {
129
+ // special characters are always single tokens
130
+ (CharType::Dot, _) | (CharType::Underscore, _) | (CharType::Hyphen, _) => true,
131
+ (_, CharType::Dot) | (_, CharType::Underscore) | (_, CharType::Hyphen) => true,
132
+
133
+ // same type - check for special cases
134
+ (CharType::Uppercase, CharType::Uppercase) => {
135
+ // Check if next char is lowercase: "HTTPResponse" -> split between 'P' and 'R'
136
+ if let Some(&(_, &next_ch)) = chars.peek() {
137
+ matches!(char_type(next_ch)?, CharType::Lowercase)
138
+ } else {
139
+ false
140
+ }
141
+ }
142
+ (CharType::Lowercase, CharType::Lowercase) => false,
143
+
144
+ // Uppercase to Lowercase can be Capitalized - don't split yet
145
+ (CharType::Uppercase, CharType::Lowercase) => {
146
+ // only continue if we're at the first transition (potential Capitalized word)
147
+ has_uppercase && has_lowercase
148
+ }
149
+
150
+ // different types - split
151
+ _ => true,
152
+ };
153
+
154
+ if should_split {
155
+ // create word based on what we've seen
156
+ let token = create_token(first_type?, has_lowercase, has_uppercase, start, i);
157
+ tokens.push(token);
158
+
159
+ // reset tracking for new word
160
+ start = i;
161
+ first_type = Some(curr_type);
162
+ has_lowercase = false;
163
+ has_uppercase = false;
164
+ } else {
165
+ // continue current word, update tracking
166
+ if curr_type == CharType::Lowercase {
167
+ has_lowercase = true;
168
+ } else if curr_type == CharType::Uppercase {
169
+ has_uppercase = true;
170
+ }
171
+ }
172
+
173
+ prev_type = Some(curr_type);
174
+ }
175
+
176
+ // add the last word
177
+ if start < s.len() {
178
+ // make sure the rest of the word contains valid chars
179
+ if !s[start..].iter().all(|ch| char_type(*ch).is_some()) {
180
+ return None;
181
+ };
182
+
183
+ let token = create_token(first_type?, has_lowercase, has_uppercase, start, s.len());
184
+ tokens.push(token);
185
+ }
186
+
187
+ Some(tokens)
188
+}
189
+
190
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
191
+enum Field<'a> {
192
+ Lowercase(&'a [u8]),
193
+ Uppercase(&'a [u8]),
194
+ LowerCamel(&'a [u8]),
195
+ UpperCamel(&'a [u8]),
196
+ Empty,
197
+}
198
+
199
+#[derive(Debug, Clone, PartialEq, Eq)]
200
+enum Node<'a> {
201
+ Field(Field<'a>),
202
+ Separator(Separator),
203
+}
204
+
205
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
206
+enum FieldType {
207
+ Lowercase,
208
+ Uppercase,
209
+ LowerCamel,
210
+ UpperCamel,
211
+}
212
+
213
+#[derive(Debug, Clone, Copy)]
214
+struct FieldBuilder {
215
+ field_type: FieldType,
216
+ start: usize,
217
+ end: usize,
218
+ extended: bool,
219
+}
220
+
221
+impl FieldBuilder {
222
+ fn new(field_type: FieldType, start: usize, end: usize) -> Self {
223
+ Self {
224
+ field_type,
225
+ start,
226
+ end,
227
+ extended: false,
228
+ }
229
+ }
230
+
231
+ fn can_add(&self, word_type: TokenType) -> bool {
232
+ matches!(
233
+ (self.field_type, word_type),
234
+ (FieldType::Lowercase, TokenType::Lowercase)
235
+ | (FieldType::Uppercase, TokenType::Uppercase)
236
+ | (FieldType::LowerCamel, TokenType::Capitalized)
237
+ | (FieldType::UpperCamel, TokenType::Capitalized)
238
+ )
239
+ }
240
+
241
+ fn extend_to(&mut self, end: usize) {
242
+ self.end = end;
243
+ self.extended = true;
244
+ }
245
+
246
+ fn transition_to_lower_camel(&mut self) {
247
+ self.field_type = FieldType::LowerCamel;
248
+ }
249
+
250
+ fn is_single_lowercase(&self) -> bool {
251
+ self.field_type == FieldType::Lowercase && !self.extended
252
+ }
253
+
254
+ fn into_field<'a>(self, source: &'a [u8]) -> Field<'a> {
255
+ let slice = &source[self.start..self.end];
256
+
257
+ match self.field_type {
258
+ FieldType::Lowercase => Field::Lowercase(slice),
259
+ FieldType::Uppercase => Field::Uppercase(slice),
260
+ FieldType::LowerCamel => Field::LowerCamel(slice),
261
+ FieldType::UpperCamel => Field::UpperCamel(slice),
262
+ }
263
+ }
264
+}
265
+
266
+#[allow(dead_code)]
267
+fn token_type(word: &Token) -> TokenType {
268
+ match word {
269
+ Token::Word { kind, .. } => *kind,
270
+ _ => unreachable!(),
271
+ }
272
+}
273
+
274
+fn parse<'a>(source: &'a [u8], tokens: &[Token]) -> Vec<Node<'a>> {
275
+ let mut nodes = Vec::new();
276
+
277
+ // handle leading empty field
278
+ if matches!(tokens.first(), Some(Token::Separator(_))) {
279
+ nodes.push(Node::Field(Field::Empty));
280
+ }
281
+
282
+ let mut field_builder: Option<FieldBuilder> = None;
283
+
284
+ for i in 0..tokens.len() {
285
+ match tokens[i] {
286
+ Token::Separator(sep) => {
287
+ // finish current field if any
288
+ if let Some(field) = field_builder.take() {
289
+ nodes.push(Node::Field(field.into_field(source)));
290
+ }
291
+
292
+ nodes.push(Node::Separator(sep));
293
+
294
+ // check for empty field (consecutive separators or trailing separator)
295
+ if i + 1 >= tokens.len() || matches!(tokens[i + 1], Token::Separator(_)) {
296
+ nodes.push(Node::Field(Field::Empty));
297
+ }
298
+ }
299
+ Token::Word {
300
+ kind: wtype,
301
+ start,
302
+ end,
303
+ } => {
304
+ if let Some(ref mut field) = field_builder {
305
+ if field.can_add(wtype) {
306
+ // we can extend the field with this word type
307
+ field.extend_to(end);
308
+ } else if field.is_single_lowercase() && wtype == TokenType::Capitalized {
309
+ // transition from single lowercase to LowerCamel
310
+ field.transition_to_lower_camel();
311
+ field.extend_to(end);
312
+ } else {
313
+ // finish current field and start new one
314
+ let finished_field = field_builder.take().unwrap();
315
+ nodes.push(Node::Field(finished_field.into_field(source)));
316
+
317
+ let field_type = match wtype {
318
+ TokenType::Lowercase => FieldType::Lowercase,
319
+ TokenType::Uppercase => FieldType::Uppercase,
320
+ TokenType::Capitalized => FieldType::UpperCamel,
321
+ };
322
+ field_builder = Some(FieldBuilder::new(field_type, start, end));
323
+ }
324
+ } else {
325
+ // no current field builder, start a new one
326
+ let field_type = match wtype {
327
+ TokenType::Lowercase => FieldType::Lowercase,
328
+ TokenType::Uppercase => FieldType::Uppercase,
329
+ TokenType::Capitalized => FieldType::UpperCamel,
330
+ };
331
+ field_builder = Some(FieldBuilder::new(field_type, start, end));
332
+ }
333
+ }
334
+ }
335
+ }
336
+
337
+ // finish any remaining field
338
+ if let Some(field) = field_builder {
339
+ nodes.push(Node::Field(field.into_field(source)));
340
+ }
341
+
342
+ nodes
343
+}
344
+
345
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
346
+enum FieldSeparatorPair {
347
+ // Lowercase field followed by...
348
+ LowercaseDot,
349
+ LowercaseUnderscore,
350
+ LowercaseHyphen,
351
+ LowercaseNoSep,
352
+ LowercaseEnd,
353
+
354
+ // LowerCamel field followed by...
355
+ LowerCamelDot,
356
+ LowerCamelUnderscore,
357
+ LowerCamelHyphen,
358
+ LowerCamelNoSep,
359
+ LowerCamelEnd,
360
+
361
+ // UpperCamel field followed by...
362
+ UpperCamelDot,
363
+ UpperCamelUnderscore,
364
+ UpperCamelHyphen,
365
+ UpperCamelNoSep,
366
+ UpperCamelEnd,
367
+
368
+ // Uppercase field followed by...
369
+ UppercaseDot,
370
+ UppercaseUnderscore,
371
+ UppercaseHyphen,
372
+ UppercaseNoSep,
373
+ UppercaseEnd,
374
+
375
+ // Empty field followed by...
376
+ EmptyDot,
377
+ EmptyUnderscore,
378
+ EmptyHyphen,
379
+ EmptyEnd,
380
+}
381
+
382
+impl FieldSeparatorPair {
383
+ fn to_char(self) -> char {
384
+ match self {
385
+ // Lowercase (a-e)
386
+ FieldSeparatorPair::LowercaseDot => 'a',
387
+ FieldSeparatorPair::LowercaseUnderscore => 'b',
388
+ FieldSeparatorPair::LowercaseHyphen => 'c',
389
+ FieldSeparatorPair::LowercaseNoSep => 'd',
390
+ FieldSeparatorPair::LowercaseEnd => 'e',
391
+
392
+ // LowerCamel (f-j)
393
+ FieldSeparatorPair::LowerCamelDot => 'f',
394
+ FieldSeparatorPair::LowerCamelUnderscore => 'g',
395
+ FieldSeparatorPair::LowerCamelHyphen => 'h',
396
+ FieldSeparatorPair::LowerCamelNoSep => 'i',
397
+ FieldSeparatorPair::LowerCamelEnd => 'j',
398
+
399
+ // UpperCamel (k-o)
400
+ FieldSeparatorPair::UpperCamelDot => 'k',
401
+ FieldSeparatorPair::UpperCamelUnderscore => 'l',
402
+ FieldSeparatorPair::UpperCamelHyphen => 'm',
403
+ FieldSeparatorPair::UpperCamelNoSep => 'n',
404
+ FieldSeparatorPair::UpperCamelEnd => 'o',
405
+
406
+ // Uppercase (p-t)
407
+ FieldSeparatorPair::UppercaseDot => 'p',
408
+ FieldSeparatorPair::UppercaseUnderscore => 'q',
409
+ FieldSeparatorPair::UppercaseHyphen => 'r',
410
+ FieldSeparatorPair::UppercaseNoSep => 's',
411
+ FieldSeparatorPair::UppercaseEnd => 't',
412
+
413
+ // Empty (u-x)
414
+ FieldSeparatorPair::EmptyDot => 'u',
415
+ FieldSeparatorPair::EmptyUnderscore => 'v',
416
+ FieldSeparatorPair::EmptyHyphen => 'w',
417
+ FieldSeparatorPair::EmptyEnd => 'x',
418
+ }
419
+ }
420
+
421
+ #[allow(dead_code)]
422
+ fn from_char(c: char) -> Option<Self> {
423
+ match c {
424
+ 'a' => Some(FieldSeparatorPair::LowercaseDot),
425
+ 'b' => Some(FieldSeparatorPair::LowercaseUnderscore),
426
+ 'c' => Some(FieldSeparatorPair::LowercaseHyphen),
427
+ 'd' => Some(FieldSeparatorPair::LowercaseNoSep),
428
+ 'e' => Some(FieldSeparatorPair::LowercaseEnd),
429
+
430
+ 'f' => Some(FieldSeparatorPair::LowerCamelDot),
431
+ 'g' => Some(FieldSeparatorPair::LowerCamelUnderscore),
432
+ 'h' => Some(FieldSeparatorPair::LowerCamelHyphen),
433
+ 'i' => Some(FieldSeparatorPair::LowerCamelNoSep),
434
+ 'j' => Some(FieldSeparatorPair::LowerCamelEnd),
435
+
436
+ 'k' => Some(FieldSeparatorPair::UpperCamelDot),
437
+ 'l' => Some(FieldSeparatorPair::UpperCamelUnderscore),
438
+ 'm' => Some(FieldSeparatorPair::UpperCamelHyphen),
439
+ 'n' => Some(FieldSeparatorPair::UpperCamelNoSep),
440
+ 'o' => Some(FieldSeparatorPair::UpperCamelEnd),
441
+
442
+ 'p' => Some(FieldSeparatorPair::UppercaseDot),
443
+ 'q' => Some(FieldSeparatorPair::UppercaseUnderscore),
444
+ 'r' => Some(FieldSeparatorPair::UppercaseHyphen),
445
+ 's' => Some(FieldSeparatorPair::UppercaseNoSep),
446
+ 't' => Some(FieldSeparatorPair::UppercaseEnd),
447
+
448
+ 'u' => Some(FieldSeparatorPair::EmptyDot),
449
+ 'v' => Some(FieldSeparatorPair::EmptyUnderscore),
450
+ 'w' => Some(FieldSeparatorPair::EmptyHyphen),
451
+ 'x' => Some(FieldSeparatorPair::EmptyEnd),
452
+
453
+ _ => None,
454
+ }
455
+ }
456
+}
457
+
458
+fn compute_checksum(s: &str) -> String {
459
+ use std::hash::{DefaultHasher, Hash, Hasher};
460
+
461
+ let mut hasher = DefaultHasher::new();
462
+ s.hash(&mut hasher);
463
+ let hash = hasher.finish();
464
+
465
+ // Two character checksum for 36^2 = 1,296 possible values
466
+ let first_idx = ((hash / 36) % 36) as usize;
467
+ let second_idx = (hash % 36) as usize;
468
+
469
+ let first_char = if first_idx < 26 {
470
+ (b'A' + first_idx as u8) as char
471
+ } else {
472
+ (b'0' + (first_idx - 26) as u8) as char
473
+ };
474
+
475
+ let second_char = if second_idx < 26 {
476
+ (b'A' + second_idx as u8) as char
477
+ } else {
478
+ (b'0' + (second_idx - 26) as u8) as char
479
+ };
480
+
481
+ format!("{}{}", first_char, second_char)
482
+}
483
+
484
+/// Returns true if the encoded string contains a checksum prefix.
485
+/// Checksums are added for strings containing camel case fields.
486
+fn has_checksum(encoded: &str) -> bool {
487
+ if let Some(first_char) = encoded.chars().next() {
488
+ // Checksum uses A-Z and 0-9, structure encoding uses a-x
489
+ first_char.is_ascii_uppercase() || first_char.is_ascii_digit()
490
+ } else {
491
+ false
492
+ }
493
+}
494
+
495
+fn encode_nodes(source: &str, nodes: &[Node]) -> String {
496
+ // Check if any field is camel case
497
+ let has_camel_case = nodes.iter().any(|node| {
498
+ matches!(
499
+ node,
500
+ Node::Field(Field::LowerCamel(_)) | Node::Field(Field::UpperCamel(_))
501
+ )
502
+ });
503
+
504
+ let mut result = String::new();
505
+
506
+ // Add checksum at the beginning if there's any camel case
507
+ if has_camel_case {
508
+ result.push_str(&compute_checksum(source));
509
+ }
510
+
511
+ let mut i = 0;
512
+ while i < nodes.len() {
513
+ if let Node::Field(field) = &nodes[i] {
514
+ // Look ahead to see what follows this field
515
+ let next_is_separator =
516
+ i + 1 < nodes.len() && matches!(nodes[i + 1], Node::Separator(_));
517
+ let next_is_field = i + 1 < nodes.len() && matches!(nodes[i + 1], Node::Field(_));
518
+
519
+ let pair = match field {
520
+ Field::Lowercase(_) => {
521
+ if next_is_separator {
522
+ match nodes[i + 1] {
523
+ Node::Separator(Separator::Dot) => FieldSeparatorPair::LowercaseDot,
524
+ Node::Separator(Separator::Underscore) => {
525
+ FieldSeparatorPair::LowercaseUnderscore
526
+ }
527
+ Node::Separator(Separator::Hyphen) => {
528
+ FieldSeparatorPair::LowercaseHyphen
529
+ }
530
+ _ => unreachable!(),
531
+ }
532
+ } else if next_is_field {
533
+ FieldSeparatorPair::LowercaseNoSep
534
+ } else {
535
+ FieldSeparatorPair::LowercaseEnd
536
+ }
537
+ }
538
+ Field::LowerCamel(_) => {
539
+ if next_is_separator {
540
+ match nodes[i + 1] {
541
+ Node::Separator(Separator::Dot) => FieldSeparatorPair::LowerCamelDot,
542
+ Node::Separator(Separator::Underscore) => {
543
+ FieldSeparatorPair::LowerCamelUnderscore
544
+ }
545
+ Node::Separator(Separator::Hyphen) => {
546
+ FieldSeparatorPair::LowerCamelHyphen
547
+ }
548
+ _ => unreachable!(),
549
+ }
550
+ } else if next_is_field {
551
+ FieldSeparatorPair::LowerCamelNoSep
552
+ } else {
553
+ FieldSeparatorPair::LowerCamelEnd
554
+ }
555
+ }
556
+ Field::UpperCamel(_) => {
557
+ if next_is_separator {
558
+ match nodes[i + 1] {
559
+ Node::Separator(Separator::Dot) => FieldSeparatorPair::UpperCamelDot,
560
+ Node::Separator(Separator::Underscore) => {
561
+ FieldSeparatorPair::UpperCamelUnderscore
562
+ }
563
+ Node::Separator(Separator::Hyphen) => {
564
+ FieldSeparatorPair::UpperCamelHyphen
565
+ }
566
+ _ => unreachable!(),
567
+ }
568
+ } else if next_is_field {
569
+ FieldSeparatorPair::UpperCamelNoSep
570
+ } else {
571
+ FieldSeparatorPair::UpperCamelEnd
572
+ }
573
+ }
574
+ Field::Uppercase(_) => {
575
+ if next_is_separator {
576
+ match nodes[i + 1] {
577
+ Node::Separator(Separator::Dot) => FieldSeparatorPair::UppercaseDot,
578
+ Node::Separator(Separator::Underscore) => {
579
+ FieldSeparatorPair::UppercaseUnderscore
580
+ }
581
+ Node::Separator(Separator::Hyphen) => {
582
+ FieldSeparatorPair::UppercaseHyphen
583
+ }
584
+ _ => unreachable!(),
585
+ }
586
+ } else if next_is_field {
587
+ FieldSeparatorPair::UppercaseNoSep
588
+ } else {
589
+ FieldSeparatorPair::UppercaseEnd
590
+ }
591
+ }
592
+ Field::Empty => {
593
+ if next_is_separator {
594
+ match nodes[i + 1] {
595
+ Node::Separator(Separator::Dot) => FieldSeparatorPair::EmptyDot,
596
+ Node::Separator(Separator::Underscore) => {
597
+ FieldSeparatorPair::EmptyUnderscore
598
+ }
599
+ Node::Separator(Separator::Hyphen) => FieldSeparatorPair::EmptyHyphen,
600
+ _ => unreachable!(),
601
+ }
602
+ } else {
603
+ FieldSeparatorPair::EmptyEnd
604
+ }
605
+ }
606
+ };
607
+
608
+ result.push(pair.to_char());
609
+
610
+ // Skip the separator if we just encoded it
611
+ if next_is_separator {
612
+ i += 2; // Skip field + separator
613
+ } else {
614
+ i += 1; // Skip just the field
615
+ }
616
+ } else {
617
+ // This shouldn't happen if nodes are well-formed
618
+ // (fields and separators should alternate)
619
+ i += 1;
620
+ }
621
+ }
622
+
623
+ result
624
+}
625
+
626
+/// Encodes a key string into a compact representation.
627
+///
628
+/// The encoding is lossless for all practical naming conventions and includes:
629
+/// - Structure encoding: captures field types (lowercase, UPPERCASE, camelCase) and separators (. _ -)
630
+/// - Checksum: 2-character prefix (A-Z, 0-9) added for strings with camel case fields
631
+///
632
+/// # Examples
633
+///
634
+/// ```
635
+/// use rdp::encode;
636
+///
637
+/// // Simple lowercase field - no checksum
638
+/// assert_eq!(encode("hello"), "e");
639
+///
640
+/// // Camel case - includes 2-char checksum
641
+/// let encoded = encode("helloWorld");
642
+/// assert_eq!(encoded.len(), 3); // 2-char checksum + 1-char structure
643
+///
644
+/// // Complex field with camel case
645
+/// let encoded = encode("log.body.HostName");
646
+/// assert_eq!(encoded.len(), 5); // 2-char checksum + 3-char structure
647
+/// ```
648
+fn encode(b: &[u8]) -> String {
649
+ let Some(tokens) = tokenize(b) else {
650
+ let digest = md5::compute(b);
651
+ return format!("{}{:X}", REMAPPED_PREFIX, digest);
652
+ };
653
+
654
+ // SAFETY: We'll only get tokens when we have valid ASCII characters in
655
+ // the input byte slice.
656
+ let s = unsafe { str::from_utf8_unchecked(b) };
657
+
658
+ let nodes = parse(b, &tokens);
659
+ encode_nodes(s, &nodes)
660
+}
661
+
662
+/// Compresses runs of 3 or more consecutive identical characters using run-length encoding.
663
+///
664
+/// Runs are encoded as count + character, with a maximum count of 9 per segment.
665
+/// For runs longer than 9, multiple segments are created.
666
+///
667
+/// # Examples
668
+///
669
+/// ```
670
+/// # use rdp::compress_runs;
671
+/// assert_eq!(compress_runs("aaa"), "3a");
672
+/// assert_eq!(compress_runs("aaaaaaaaaa"), "9aa"); // 10 a's → 9a + a
673
+/// assert_eq!(compress_runs("aaaaaaaaaaaa"), "9a3a"); // 12 a's → 9a + 3a
674
+/// assert_eq!(compress_runs("aabbbcc"), "aa3bcc");
675
+/// ```
676
+fn compress_runs(s: &str) -> String {
677
+ if s.is_empty() {
678
+ return String::new();
679
+ }
680
+
681
+ let mut result = String::new();
682
+ let chars: Vec<char> = s.chars().collect();
683
+ let mut i = 0;
684
+
685
+ while i < chars.len() {
686
+ let ch = chars[i];
687
+ let mut count = 1;
688
+
689
+ // Count consecutive identical characters
690
+ while i + count < chars.len() && chars[i + count] == ch {
691
+ count += 1;
692
+ }
693
+
694
+ // Process the run
695
+ if count <= 2 {
696
+ // Output as-is for runs of 1 or 2
697
+ for _ in 0..count {
698
+ result.push(ch);
699
+ }
700
+ } else {
701
+ // Compress runs of 3+
702
+ let mut remaining = count;
703
+ while remaining > 0 {
704
+ if remaining > 9 {
705
+ result.push('9');
706
+ result.push(ch);
707
+ remaining -= 9;
708
+ } else if remaining > 2 {
709
+ result.push(char::from_digit(remaining as u32, 10).unwrap());
710
+ result.push(ch);
711
+ remaining = 0;
712
+ } else {
713
+ // Output remaining 1 or 2 characters as-is
714
+ for _ in 0..remaining {
715
+ result.push(ch);
716
+ }
717
+ remaining = 0;
718
+ }
719
+ }
720
+ }
721
+
722
+ i += count;
723
+ }
724
+
725
+ result
726
+}
727
+
728
+/// Returns the fully encoded key: ND prefix + compressed uppercase structure encoding + normalized key.
729
+///
730
+/// The result combines:
731
+/// - "ND" prefix (Netdata namespace identifier)
732
+/// - The compact structure encoding with run-length compression and uppercased
733
+/// - An underscore separator
734
+/// - The original key converted to uppercase with dots and hyphens replaced by underscores,
735
+/// and common prefixes shortened:
736
+/// - "RESOURCE_ATTRIBUTES_" → "RA_"
737
+/// - "LOG_ATTRIBUTES_" → "LA_"
738
+/// - "LOG_BODY_" → "LB_"
739
+///
740
+/// Run-length compression replaces 3+ consecutive identical characters with count + character.
741
+/// The checksum (first 2 characters if present) is never compressed.
742
+///
743
+/// # MD5 Fallback
744
+///
745
+/// Falls back to `ND_<32-hex-chars>` format when:
746
+/// - Input is not valid UTF-8
747
+/// - Input contains invalid characters (anything except a-z, A-Z, '.', '-', '_')
748
+/// - Encoded result would exceed 64 bytes (systemd's field name limit)
749
+///
750
+/// The result is guaranteed to be systemd-compatible and ≤ 64 bytes.
751
+///
752
+/// # Examples
753
+///
754
+/// ```
755
+/// use rdp::encode_full;
756
+///
757
+/// // Simple lowercase field
758
+/// assert_eq!(encode_full(b"hello"), "HELLO_NDE");
759
+///
760
+/// // With dot separators - no compression (only 2 consecutive a's)
761
+/// assert_eq!(encode_full(b"log.body.hostname"), "LB_HOSTNAME_NDAAE");
762
+///
763
+/// // Many nested levels - structure compression (10 a's → 9a + a)
764
+/// assert_eq!(encode_full(b"my.very.deeply.nested.field.that.ends.in.the.abyss"), "MY_VERY_DEEPLY_NESTED_FIELD_THAT_ENDS_IN_THE_ABYSS_ND9AE");
765
+///
766
+/// // With camel case (includes checksum - not compressed)
767
+/// let full = encode_full(b"log.body.HostName");
768
+/// assert!(full.starts_with("LB_HOSTNAME_ND")); // prefix + normalized + ND + checksum
769
+/// assert!(full.ends_with("83AAO")); // 2-char checksum + structure (2 a's not compressed)
770
+///
771
+/// // With hyphens
772
+/// assert_eq!(encode_full(b"hello-world"), "HELLO_WORLD_NDCE");
773
+///
774
+/// // With resource.attributes prefix - compression (3 a's → 3a)
775
+/// assert_eq!(encode_full(b"resource.attributes.host.name"), "RA_HOST_NAME_ND3AE");
776
+///
777
+/// // With invalid characters (space) - falls back to MD5
778
+/// let md5_result = encode_full(b"field name");
779
+/// assert!(md5_result.starts_with("ND_"));
780
+/// assert_eq!(md5_result.len(), 35); // ND_ + 32 hex chars
781
+///
782
+/// // Non-UTF8 - falls back to MD5
783
+/// let non_utf8 = b"\xFF\xFE invalid";
784
+/// let result = encode_full(non_utf8);
785
+/// assert!(result.starts_with("ND_"));
786
+/// assert_eq!(result.len(), 35);
787
+///
788
+/// // Long names that would exceed 64 bytes - falls back to MD5
789
+/// let long_name = b"very.long.deeply.nested.field.name.that.would.definitely.exceed.the.systemd.limit";
790
+/// let result = encode_full(long_name);
791
+/// assert!(result.starts_with("ND_"));
792
+/// assert!(result.len() <= 64);
793
+/// ```
794
+pub fn encode_full(field_name: &[u8]) -> String {
795
+ let encoded = encode(field_name);
796
+
797
+ // Compress runs in the structure encoding (but not the checksum)
798
+ let compressed = if has_checksum(&encoded) {
799
+ // Keep checksum as-is, compress the rest
800
+ let checksum = &encoded[..2];
801
+ let structure = &encoded[2..];
802
+ format!("{}{}", checksum, compress_runs(structure))
803
+ } else {
804
+ // No checksum, compress everything
805
+ compress_runs(&encoded)
806
+ };
807
+
808
+ let s = unsafe { String::from_utf8_unchecked(field_name.to_vec()) };
809
+ let mut normalized = s.to_uppercase().replace(['.', '-'], "_");
810
+
811
+ // Replace common prefixes with shorter versions
812
+ if let Some(suffix) = normalized.strip_prefix("RESOURCE_ATTRIBUTES_") {
813
+ normalized = format!("RA_{}", suffix);
814
+ } else if let Some(suffix) = normalized.strip_prefix("LOG_ATTRIBUTES_") {
815
+ normalized = format!("LA_{}", suffix);
816
+ } else if let Some(suffix) = normalized.strip_prefix("LOG_BODY_") {
817
+ normalized = format!("LB_{}", suffix);
818
+ }
819
+
820
+ let result = format!("ND{}_{}", compressed.to_uppercase(), normalized);
821
+
822
+ // If the result exceeds systemd's 64-byte limit, fall back to MD5
823
+ if result.len() > 64 {
824
+ let digest = md5::compute(field_name);
825
+ return format!("{}{:X}", REMAPPED_PREFIX, digest);
826
+ }
827
+
828
+ result
829
+}
src/crates/rdp/src/main.rs
new
+241
@@ -0,0 +1,241 @@
1
+use rdp::encode_full;
2
+
3
+fn main() {
4
+ let mut aws_keys = vec![
5
+ "log.attributes.log.file.path",
6
+ "log.attributes.log.iostream",
7
+ "log.attributes.logtag",
8
+ "log.dropped_attributes_count",
9
+ "log.event_name",
10
+ "log.flags",
11
+ "log.observed_time_unix_nano",
12
+ "log.severity_number",
13
+ "log.severity_text",
14
+ "log.time_unix_nano",
15
+ "resource.attributes.host.name",
16
+ "resource.attributes.k8s.container.name",
17
+ "resource.attributes.k8s.container.restart_count",
18
+ "resource.attributes.k8s.namespace.name",
19
+ "resource.attributes.k8s.node.name",
20
+ "resource.attributes.k8s.pod.name",
21
+ "resource.attributes.k8s.pod.start_time",
22
+ "resource.attributes.k8s.pod.uid",
23
+ "resource.attributes.os.type",
24
+ "resource.schema_url",
25
+ "resource.attributes.k8s.deployment.name",
26
+ "log.body.level",
27
+ "log.body.message",
28
+ "log.body.time",
29
+ "log.body.severity",
30
+ "log.body.timestamp",
31
+ "log.body.local_addr.IP",
32
+ "log.body.local_addr.Port",
33
+ "log.body.local_addr.Zone",
34
+ "log.body.remote_addr.ForceQuery",
35
+ "log.body.remote_addr.Fragment",
36
+ "log.body.remote_addr.Host",
37
+ "log.body.remote_addr.OmitHost",
38
+ "log.body.remote_addr.Opaque",
39
+ "log.body.remote_addr.Path",
40
+ "log.body.remote_addr.RawFragment",
41
+ "log.body.remote_addr.RawPath",
42
+ "log.body.remote_addr.RawQuery",
43
+ "log.body.remote_addr.Scheme",
44
+ "log.body.remote_addr.User",
45
+ "log.body.file",
46
+ "log.body.id",
47
+ "log.body.line",
48
+ "log.body.TR",
49
+ "log.body.Ta",
50
+ "log.body.Tc",
51
+ "log.body.Th",
52
+ "log.body.Ti",
53
+ "log.body.Tr",
54
+ "log.body.Tw",
55
+ "log.body.actconn",
56
+ "log.body.backend_name",
57
+ "log.body.backend_queue",
58
+ "log.body.beconn",
59
+ "log.body.bytes_read",
60
+ "log.body.cf_conn_ip",
61
+ "log.body.client_ip",
62
+ "log.body.client_port",
63
+ "log.body.date_time",
64
+ "log.body.feconn",
65
+ "log.body.frontend_name",
66
+ "log.body.http_method",
67
+ "log.body.http_query",
68
+ "log.body.http_uri",
69
+ "log.body.http_version",
70
+ "log.body.retries",
71
+ "log.body.server_name",
72
+ "log.body.srv_queue",
73
+ "log.body.srvconn",
74
+ "log.body.ssl_ciphers",
75
+ "log.body.ssl_version",
76
+ "log.body.status_code",
77
+ "log.body.termination_state",
78
+ "log.body.unique_id",
79
+ "log.body.size",
80
+ "log.body.status",
81
+ "resource.attributes.k8s.daemonset.name",
82
+ "resource.attributes.component",
83
+ "log.body.fields.otelcol.component.id",
84
+ "log.body.fields.otelcol.component.kind",
85
+ "log.body.fields.otelcol.signal",
86
+ "log.body.fields.resource.service.instance.id",
87
+ "log.body.fields.resource.service.name",
88
+ "log.body.fields.resource.service.version",
89
+ "log.body.fields.data",
90
+ "log.body.fields.metrics",
91
+ "log.body.fields.resource",
92
+ "log.body.topic",
93
+ "log.body.msg",
94
+ "log.body.method",
95
+ "log.body.path",
96
+ "log.body.agent",
97
+ "log.body.host",
98
+ "log.body.protocol",
99
+ "log.body.referrer",
100
+ "log.body.user_id",
101
+ "log.body.duration",
102
+ "log.body.ts",
103
+ "log.body.logger",
104
+ "log.body.request.host",
105
+ "log.body.request.method",
106
+ "log.body.request.proto",
107
+ "log.body.request.remote_ip",
108
+ "log.body.request.remote_port",
109
+ "log.body.request.uri",
110
+ "log.body.serviceURL.ForceQuery",
111
+ "log.body.serviceURL.Fragment",
112
+ "log.body.serviceURL.Host",
113
+ "log.body.serviceURL.OmitHost",
114
+ "log.body.serviceURL.Opaque",
115
+ "log.body.serviceURL.Path",
116
+ "log.body.serviceURL.RawFragment",
117
+ "log.body.serviceURL.RawPath",
118
+ "log.body.serviceURL.RawQuery",
119
+ "log.body.serviceURL.Scheme",
120
+ "log.body.serviceURL.User",
121
+ "log.body.ClientAddr",
122
+ "log.body.ClientHost",
123
+ "log.body.ClientPort",
124
+ "log.body.ClientUsername",
125
+ "log.body.DownstreamContentSize",
126
+ "log.body.DownstreamStatus",
127
+ "log.body.Duration",
128
+ "log.body.GzipRatio",
129
+ "log.body.OriginContentSize",
130
+ "log.body.OriginDuration",
131
+ "log.body.OriginStatus",
132
+ "log.body.Overhead",
133
+ "log.body.RequestAddr",
134
+ "log.body.RequestContentSize",
135
+ "log.body.RequestCount",
136
+ "log.body.RequestHost",
137
+ "log.body.RequestMethod",
138
+ "log.body.RequestPath",
139
+ "log.body.RequestPort",
140
+ "log.body.RequestProtocol",
141
+ "log.body.RequestScheme",
142
+ "log.body.RetryAttempts",
143
+ "log.body.RouterName",
144
+ "log.body.ServiceAddr",
145
+ "log.body.ServiceName",
146
+ "log.body.ServiceURL",
147
+ "log.body.StartLocal",
148
+ "log.body.StartUTC",
149
+ "log.body.entryPointName",
150
+ "log.body.request_User-Agent",
151
+ "resource.attributes.app",
152
+ "log.body.apiVersion",
153
+ "log.body.eventTime",
154
+ "log.body.firstTimestamp",
155
+ "log.body.involvedObject.apiVersion",
156
+ "log.body.involvedObject.kind",
157
+ "log.body.involvedObject.name",
158
+ "log.body.involvedObject.resourceVersion",
159
+ "log.body.involvedObject.uid",
160
+ "log.body.kind",
161
+ "log.body.lastTimestamp",
162
+ "log.body.metadata.creationTimestamp",
163
+ "log.body.metadata.name",
164
+ "log.body.metadata.namespace",
165
+ "log.body.metadata.resourceVersion",
166
+ "log.body.metadata.uid",
167
+ "log.body.reason",
168
+ "log.body.reportingComponent",
169
+ "log.body.reportingInstance",
170
+ "log.body.type",
171
+ "log.body.involvedObject.namespace",
172
+ "log.body.count",
173
+ "log.body.source.component",
174
+ "log.body.record-id",
175
+ "log.body.record-type",
176
+ "log.body.service",
177
+ "log.body.bootstrap",
178
+ "log.body.config_file_source",
179
+ "log.body.session_id",
180
+ "log.body.source.host",
181
+ "log.body.involvedObject.fieldPath",
182
+ "log.body.authority",
183
+ "log.body.forwarded-for",
184
+ "log.body.referer",
185
+ "log.body.request-id",
186
+ "log.body.response-code",
187
+ "log.body.response-code-details",
188
+ "log.body.upstream-cluster",
189
+ "log.body.user-agent",
190
+ "log.body.svc",
191
+ "resource.attributes.k8s.cronjob.name",
192
+ "resource.attributes.k8s.job.name",
193
+ "log.body.account",
194
+ "log.body.action",
195
+ "log.body.arn",
196
+ "log.body.caller",
197
+ "log.body.err",
198
+ "log.body.job_type",
199
+ "log.body.region",
200
+ "log.body.version",
201
+ "log.body.fields.component",
202
+ "log.body.fields.path",
203
+ "log.body.error",
204
+ "log.body.agents_missing_from_emqx",
205
+ "log.body.agents_unreachable_postgres_disconnected",
206
+ "log.body.emqx_client_id_count",
207
+ "log.body.failed",
208
+ "log.body.fields.error",
209
+ "log.body.fields.interval",
210
+ "log.body.fields_raw",
211
+ "log.body.progress",
212
+ "log.body.successful",
213
+ "log.body.unreachable_agents_cleaned",
214
+ "my.deeply.nested.key.checks.run.length.encoding.works.well",
215
+ "log.body.HOSTNAME",
216
+ "HTTPSConnection",
217
+ "OAuth2Token",
218
+ "foo.bar",
219
+ "foo_bar",
220
+ "fooBar",
221
+ "resource.attributes.service.instance.environment.region.zone",
222
+ // "resource.cαλημέρα",
223
+ ];
224
+
225
+ // Sort keys to ensure consistent ordering
226
+ aws_keys.sort();
227
+
228
+ // Collect all output lines for checksum calculation
229
+ let mut all_output = String::new();
230
+
231
+ for otel_key in &aws_keys {
232
+ let systemd_key = encode_full(otel_key.as_bytes());
233
+ let line = format!("{:<40} {}\n", otel_key, systemd_key);
234
+ print!("{}", line);
235
+ all_output.push_str(&line);
236
+ }
237
+
238
+ // Calculate and print MD5 checksum of all output
239
+ let digest = md5::compute(all_output.as_bytes());
240
+ println!("\nMD5 Checksum: {:x}", digest);
241
+}
src/daemon/dyncfg/README.md
+5
-9
@@ -417,17 +417,13 @@ The health module uses the high-level API with IDs like:
417
418
It supports multiple configuration objects, validation of alert definitions, and conversion between different configuration formats (JSON and traditional Netdata health configuration syntax).
419
420
-### systemd-journal.plugin (External Plugin)
420
+### journal-viewer-plugin (External Plugin)
421
422
-The systemd-journal.plugin is an external plugin written in C that uses DynCfg to manage journal directory configurations:
422
+The journal-viewer-plugin is an external plugin written in Rust that provides systemd journal log viewing and analysis:
423
424
-- `src/collectors/systemd-journal.plugin/systemd-journal-dyncfg.c`: Implements DynCfg integration
425
-- Registers as `systemd-journal:monitored-directories` ID
426
-- Uses a SINGLE configuration type for its directory list
427
-- Provides validation of directory paths for security
428
-- Implements GET and UPDATE commands
429
-
430
-This is a good example of an external plugin using the DynCfg system for a single configuration object.
424
+- **Location**: `src/crates/netdata-log-viewer/journal-viewer-plugin/`
425
+- Provides systemd journal log querying and visualization capabilities
426
+- Implements the Netdata plugin protocol for communication with the agent
427
428
### go.d.plugin (External Plugin)
429
src/libnetdata/facets/README.md
+1
-1
@@ -263,7 +263,7 @@ Log databases often contain out-of-order entries:
263
## Integration with Netdata Architecture
264
265
### Plugin Communication
266
-- The facets library runs within plugins (systemd-journal.plugin, windows-events.plugin)
266
+- The facets library runs within plugins
267
- Plugins can run anywhere in the Netdata ecosystem (parent nodes, child nodes, etc.)
268
- Communication with plugins happens via **rrdfunctions** - Netdata's function execution framework
269
- rrdfunctions provide the transport layer to send requests to plugins and receive responses
src/libnetdata/log/README.md
+1
-1
@@ -284,7 +284,7 @@ Netdata assigns unique UUIDs to specific event types for easy filtering and corr
284
| `d1f59606-dd4d-41e3-b217-a0cfcae8e632` | Extreme Cardinality | Metric cardinality exceeds safe limits |
285
| `4fdf4081-6c12-4623-a032-b7fe73beacb8` | User Configuration | Dynamic configuration changed by user |
286
287
-You can view these events using the Netdata systemd-journal.plugin at the `MESSAGE_ID` filter,
287
+You can view these events using the Netdata `log-viewer` plugin at the `MESSAGE_ID` filter,
288
or using `journalctl` like this:
289
290
```bash
src/libnetdata/log/nd_log-internals.c
+2
@@ -778,6 +778,8 @@ int nd_log_collectors_fd(void) {
778
if(nd_log.sources[NDLS_COLLECTORS].method == NDLM_FILE && nd_log.sources[NDLS_COLLECTORS].fd != -1)
779
return nd_log.sources[NDLS_COLLECTORS].fd;
780
781
+ // For NDLM_JOURNAL, return STDERR_FILENO so the log-forwarder
782
+ // can read from the pipe and properly format messages for journal
783
return STDERR_FILENO;
784
}
785
src/plugins.d/plugins_d.c
+6
@@ -328,6 +328,12 @@ void *pluginsd_main(void *ptr) {
328
continue;
329
}
330
331
+ // Blacklist obsolete plugins that have been replaced
332
+ if (strcmp(pluginname, "systemd-journal") == 0) {
333
+ netdata_log_info("plugin '%s' has been replaced by 'journal-viewer' and is no longer supported", file->d_name);
334
+ continue;
335
+ }
336
+
337
int enabled = inicfg_get_boolean(&netdata_config, CONFIG_SECTION_PLUGINS, pluginname, automatic_run);
338
if (unlikely(!enabled)) {
339
netdata_log_debug(D_PLUGINSD, "plugin '%s' is not enabled", file->d_name);