@cryptotaxi247 / netdata / commits / 4eed47099

Test netflow enrichment paths (#22453)

Costa Tsaousis committed May 8, 2026 at 22:38 UTC 4eed4709942b72aad20dcaa162e5d9347e63ca08
52 files changed +6294 -282
.agents/sow/done/SOW-0015-20260508-netflow-enrichment-verification.md new
+1093
@@ -0,0 +1,1093 @@
1 +# SOW-0015 - NetFlow Enrichment Verification
2 +
3 +## Status
4 +
5 +Status: completed
6 +
7 +Sub-state: End-user downloader documentation mismatch on supported providers corrected.
8 +
9 +## Requirements
10 +
11 +### Purpose
12 +
13 +Ensure the Netdata Agent ships network-flow enrichment methods that are fit for DevOps/SRE use: tested in code, validated in practice where possible, documented with working instructions, and proven to produce the fields users expect.
14 +
15 +### User Request
16 +
17 +The user wants the NetFlow enrichment methods fully tested, including unit tests, integration tests, practical validation on the locally running Netdata Agent, and documentation corrections when instructions are wrong. The user asked the assistant to work autonomously and stop only when progress requires user involvement.
18 +
19 +Correction from the user: the scope includes all enrichment paths, plus possible NetFlow and sFlow validation gaps. The user is already testing IPFIX on this workstation, and the default geolocation database is known to work. The priority is to test everything else that is not already covered by unit tests; CI-job integration can come later and is not the immediate gate for this SOW.
20 +
21 +The private live flow exporter endpoint is intentionally not recorded here. Runtime evidence may refer to it as `[PRIVATE_FLOW_EXPORTER]`.
22 +
23 +Scope:
24 +
25 +1. Flow protocols with missing proof: NetFlow and sFlow. IPFIX is treated as a locally working baseline because the user is actively testing it on this workstation.
26 +2. Default geolocation database: treated as a working baseline, but existing unit/runtime proof will be recorded.
27 +3. DB-IP IP Intelligence.
28 +4. IPtoASN.
29 +5. Custom MMDB Database.
30 +6. MaxMind GeoIP / GeoLite2.
31 +7. Static metadata.
32 +8. Sampling overrides.
33 +9. Static networks.
34 +10. Classifiers.
35 +11. ASN and network provider chains.
36 +12. Decapsulation enrichment.
37 +13. AWS IP Ranges.
38 +14. Azure IP Ranges.
39 +15. GCP IP Ranges.
40 +16. Generic JSON-over-HTTP IPAM.
41 +17. NetBox.
42 +18. bio-rd / RIPE RIS.
43 +19. BMP.
44 +
45 +### Clarified Beta Validation Target
46 +
47 +The target is **not** proving that every external routing stack is production-stable. The target is proving Netdata's side is sane:
48 +
49 +- It connects when given valid config.
50 +- It rejects or reports bad config clearly.
51 +- It consumes the expected payload/schema.
52 +- It maps fields correctly into enrichment state.
53 +- It respects configured options.
54 +- It does not panic, hang, leak tasks, or silently ignore data.
55 +- The docs describe commands/configs that actually work.
56 +- The beta can ship without obvious "this was never exercised" failures.
57 +
58 +This SOW must not block on the user having a router that supports BMP or a production bio-rd/RIS deployment.
59 +
60 +For BMP, GoBGP/FRR plus deterministic BMP frames are sufficient to validate Netdata's listener and route ingestion contract.
61 +
62 +For BioRIS, a deterministic bio-rd-compatible gRPC service is sufficient to validate Netdata's client behavior. Running upstream bio-rd is useful as a smoke test, but it is not a hard shipping blocker if upstream itself is unstable in a local lab.
63 +
64 +End-user documentation remains product documentation. It must describe supported commands, configuration, payload/schema expectations, limitations, and troubleshooting. It must not discuss internal test gaps, untested status, or why a test could not be performed.
65 +
66 +### Assistant Understanding
67 +
68 +Facts:
69 +
70 +- The current `netdata` instance on this workstation is available for practical validation and already receives flow traffic from `[PRIVATE_FLOW_EXPORTER]`.
71 +- `build-install-netflow-plugin.sh` builds and installs the Rust `netflow-plugin`, then restarts `netdata` unless called with `--no-restart`.
72 +- The backend stores flow records in journals, so local validation can inspect journal-backed output through plugin queries and, where useful, `journalctl`.
73 +- Prior NetFlow documentation work recorded that BMP, BioRIS, and Network Sources had parser/transform unit tests but lacked runtime I/O integration tests.
74 +- User-facing documentation is in scope when validation proves instructions are wrong or incomplete.
75 +- The user reports IPFIX and the default geolocation database already work on this workstation.
76 +
77 +Inferences:
78 +
79 +- The core gap is not absence of all tests; it is missing proof across the real boundaries for methods not already covered: documented config, MMDB files, HTTP fetchers, gRPC streams, BMP TCP sessions, runtime state publication, flow enrichment, journal-backed query output, and generated integration documentation.
80 +- BMP and bio-rd validation can progress without the user by using local deterministic speakers, public/open-source fixtures, FRR/GoBGP, and a bio-rd-compatible gRPC test service. Production-router proof is not an acceptance criterion for this beta validation pass.
81 +
82 +Unknowns:
83 +
84 +- Whether all required fixture formats can be generated in Rust using existing dependencies, or whether narrow dev-dependencies are required.
85 +- Whether a practical local bio-rd setup can be made reliable enough for automated validation without pulling in a large external runtime.
86 +- Whether the installed local `netdata` configuration can be safely changed in-place for all live validation cases, or whether some cases should use temporary config and `--no-restart` style isolated test runs.
87 +
88 +### Acceptance Criteria
89 +
90 +- Each scoped enrichment method has automated unit or integration-test coverage proving the configured source is parsed/fetched/accepted and produces expected enrichment attributes, or the SOW records existing coverage and why no new test was needed.
91 +- NetFlow and sFlow have automated validation for protocol decode/enrichment interaction when existing tests do not already prove it.
92 +- Runtime I/O paths are covered where applicable:
93 + - HTTP fetch and refresh for cloud/IPAM/NetBox network sources.
94 + - gRPC `GetRouters`, `DumpRIB`, and `ObserveRIB` behavior for BioRIS or an equivalent local fake.
95 + - BMP TCP listener behavior with real BMP-framed bytes, route upsert, withdrawal, and session cleanup.
96 +- At least one test path proves enrichment reaches flow records and the fields exposed by journal-backed query output.
97 +- Documentation examples and setup instructions for the scoped methods are executed or mechanically tested where possible; incorrect instructions are fixed from source files and generated artifacts are regenerated.
98 +- Practical local validation is attempted on the installed Netdata Agent using the build/install script and journal-backed output.
99 +- Any remaining blockers are listed with evidence and the exact user involvement required.
100 +
101 +## Analysis
102 +
103 +Sources checked:
104 +
105 +- `AGENTS.md`
106 +- `.agents/sow/SOW.template.md`
107 +- `.agents/sow/done/SOW-0014-20260506-netflow-sflow-ipfix-documentation-guide.md`
108 +- `.agents/sow/specs/sensitive-data-discipline.md`
109 +- `.agents/skills/project-writing-collectors/SKILL.md`
110 +- `.agents/skills/integrations-lifecycle/SKILL.md`
111 +- `src/crates/netflow-plugin/Cargo.toml`
112 +- `src/crates/Cargo.toml`
113 +- `src/crates/netflow-plugin/src/`
114 +- `src/crates/netflow-plugin/metadata.yaml`
115 +- `src/crates/netflow-plugin/configs/netflow.yaml`
116 +- `docs/network-flows/`
117 +- `build-install-netflow-plugin.sh`
118 +- Local mirrored upstream repositories for BMP/BioRIS/network-source fixture patterns.
119 +
120 +Current state:
121 +
122 +- `src/crates/netflow-plugin/src/main.rs` starts BMP, BioRIS, and network-source tasks when configured.
123 +- `src/crates/netflow-plugin/src/decoder/state/runtime/decode.rs` applies the enricher to decoded flow records.
124 +- Existing tests cover many pure parsing and transform helpers, but not all documented real-source workflows.
125 +- `src/crates/netflow-plugin/Cargo.toml` has no HTTP mock or MMDB writer dev-dependency at SOW start.
126 +
127 +Risks:
128 +
129 +- Tests that rely on live Internet endpoints can become flaky; automated tests should use frozen, attributed fixtures and local fake services so they are suitable for future CI even though CI-job wiring is not the immediate target.
130 +- Real cloud/IPAM/BGP payloads may contain sensitive network information; durable artifacts must use sanitized fixtures only.
131 +- Installing and restarting the local plugin affects the user's running Netdata instance; use the provided script deliberately and record the command/result.
132 +- Documentation is generated from metadata for integration cards; generated files must not be hand-edited.
133 +
134 +## Pre-Implementation Gate
135 +
136 +Status: ready
137 +
138 +Problem / root-cause model:
139 +
140 +- NetFlow enrichment has broad configuration and documentation coverage, but several enrichment methods lack tests at the boundary where real failures happen. A method can have parser tests and still fail because the documented transform is stale, the HTTP fetcher does not publish runtime state, the MMDB schema is not actually decoded, the BMP listener never accepts a real frame, NetFlow/sFlow decode does not interact correctly with enrichment, or the final enriched attributes never reach journal-backed query output.
141 +
142 +Evidence reviewed:
143 +
144 +- Prior SOW evidence: `.agents/sow/done/SOW-0014-20260506-netflow-sflow-ipfix-documentation-guide.md` records runtime I/O gaps for BMP, BioRIS, and Network Sources.
145 +- Local code evidence:
146 + - `src/crates/netflow-plugin/src/main.rs` task startup for dynamic enrichment.
147 + - `src/crates/netflow-plugin/src/network_sources/fetch.rs` HTTP fetch path.
148 + - `src/crates/netflow-plugin/src/network_sources/service.rs` refresh/publish path.
149 + - `src/crates/netflow-plugin/src/routing/bmp/listener.rs` BMP TCP listener path.
150 + - `src/crates/netflow-plugin/src/routing/bioris/runtime/refresh/instance.rs` BioRIS router/RIB refresh path.
151 + - `src/crates/netflow-plugin/src/enrichment/resolve.rs` provider-chain and merge behavior.
152 + - `src/crates/netflow-plugin/src/flow/record/journal/network.rs` journal field output.
153 +- Open-source reference evidence:
154 + - `akvorado/akvorado @ 646eb033a57692f5c2918bb65768e15c921b766e`, `outlet/routing/provider/bmp/root_test.go`, `outlet/routing/provider/bmp/tests.go`, `outlet/routing/provider/bioris/root_test.go`, `cmd/akvorado/testdata/configurations/clickhouse-network-sources/in.yaml`.
155 + - `pmacct/pmacct @ b06e6829bdc523200cbed2942b7045d64146cb71`, `tests/208-BMP-mem-leak-test/`, `tests/400-IPFIXv10-BMP-CISCO-SRv6-multiple-sources/`.
156 +
157 +Affected contracts and surfaces:
158 +
159 +- Rust code under `src/crates/netflow-plugin/src/`.
160 +- Test fixtures under `src/crates/netflow-plugin/testdata/`, with attribution.
161 +- `src/crates/netflow-plugin/Cargo.toml` and possibly `src/crates/Cargo.toml` for narrow test dependencies.
162 +- User-facing source docs under `docs/network-flows/`.
163 +- Integration source metadata `src/crates/netflow-plugin/metadata.yaml`.
164 +- Generated integration cards under `src/crates/netflow-plugin/integrations/`.
165 +- Installed local `netdata` runtime during practical validation.
166 +- SOW/spec/skill artifacts if durable workflow knowledge changes.
167 +
168 +Existing patterns to reuse:
169 +
170 +- Existing Rust module-local `tests.rs` files.
171 +- Existing flow pcap fixtures and `src/crates/netflow-plugin/testdata/ATTRIBUTION.md`.
172 +- Existing `ingest_test_support` and query tests for journal-backed records.
173 +- Existing integration generator workflow from `integrations-lifecycle`.
174 +- Akvorado-style BMP fixture tests and network-source transform tests, adapted to Netdata's codebase and licensing constraints.
175 +
176 +Risk and blast radius:
177 +
178 +- Most implementation should be test-only, but validation may expose runtime bugs requiring production code fixes.
179 +- BMP and BioRIS involve async network services; tests must use deterministic localhost ports, explicit shutdown, and timeouts.
180 +- Live validation on the installed Agent can restart `netdata`; this is acceptable per user-provided workflow but must use targeted process discipline and avoid broad kills.
181 +- Docs changes affect generated public integration pages and Learn ingestion; regeneration and link/content checks are required when docs change.
182 +
183 +Sensitive data handling plan:
184 +
185 +- Do not write literal private IPs, endpoints, node IDs, tokens, real customer data, or personal names to SOWs, specs, docs, skills, code comments, or fixtures.
186 +- Use `[PRIVATE_FLOW_EXPORTER]`, RFC documentation addresses, loopback addresses, or synthetic fixture data.
187 +- Store any raw runtime outputs that may contain private data only under `.local/`, which is gitignored; summarize redacted evidence in this SOW.
188 +- If a live output contains sensitive data needed for debugging, redact before recording and do not commit the raw output.
189 +
190 +Implementation plan:
191 +
192 +1. Build a coverage matrix from code, docs, config, metadata, and tests for all enrichment methods plus NetFlow/sFlow protocol validation.
193 +2. Add deterministic fixtures and unit/contract tests for enrichment methods that are not already proven.
194 +3. Add integration-style tests that prove runtime state feeds flow enrichment and journal/query-visible fields.
195 +4. Correct source documentation and metadata examples that fail validation; regenerate generated integration artifacts.
196 +5. Run narrow Rust tests, docs/generator validation, and any broader CI-equivalent commands needed by touched surfaces.
197 +6. Build/install the plugin with `build-install-netflow-plugin.sh`, restart `netdata`, and validate practical behavior with journal-backed output.
198 +7. Record exact remaining blockers that require user-provided external systems or credentials.
199 +
200 +Validation plan:
201 +
202 +- Run targeted `cargo test -p netflow-plugin ...` commands from `src/crates`.
203 +- Run full `cargo test -p netflow-plugin` if feasible after targeted tests pass.
204 +- Run integration generator commands when metadata or generated cards change.
205 +- Validate docs snippets/transforms against frozen or live-official payload shapes where feasible.
206 +- Run the local build/install script and verify the installed plugin produces enriched flow output.
207 +- Inspect `journalctl` and/or plugin query paths for runtime errors and expected enriched fields.
208 +- Search for same-failure patterns across provider docs/tests before closing.
209 +
210 +Artifact impact plan:
211 +
212 +- AGENTS.md: no update expected unless workflow rules are found wrong.
213 +- Runtime project skills: update only if this work discovers durable collector/integration workflow knowledge not already captured.
214 +- Specs: create or update a NetFlow enrichment behavior spec if tests establish durable contracts not captured elsewhere.
215 +- End-user/operator docs: update docs and metadata when instructions are wrong or incomplete.
216 +- End-user/operator skills: update only if public query skills are affected by changed flow-query instructions.
217 +- SOW lifecycle: this SOW is active in `current/`; close only after validation, artifact gates, and follow-up mapping.
218 +
219 +Open-source reference evidence:
220 +
221 +- `akvorado/akvorado @ 646eb033a57692f5c2918bb65768e15c921b766e`
222 + - `outlet/routing/provider/bmp/root_test.go`
223 + - `outlet/routing/provider/bmp/tests.go`
224 + - `outlet/routing/provider/bioris/root_test.go`
225 + - `cmd/akvorado/testdata/configurations/clickhouse-network-sources/in.yaml`
226 +- `pmacct/pmacct @ b06e6829bdc523200cbed2942b7045d64146cb71`
227 + - `tests/208-BMP-mem-leak-test/`
228 + - `tests/400-IPFIXv10-BMP-CISCO-SRv6-multiple-sources/`
229 +
230 +Open decisions:
231 +
232 +- Resolved by user: pursue full testing, practical validation, unit/integration coverage, and documentation correction.
233 +- Resolved by user: assistant may proceed autonomously and stop only when user involvement is strictly required.
234 +- Resolved by user: immediate scope is tests and practical validation, not CI-job integration.
235 +- Resolved by user: IPFIX and default geolocation database are working local baselines; prioritize the remaining unproven methods.
236 +- No implementation-blocking user decision is open at SOW creation.
237 +
238 +## Implications And Decisions
239 +
240 +1. Decision: confidence level.
241 + - Selected: full verification using unit tests, CI-safe integration/contract tests, practical local validation, and docs correction.
242 + - Rationale: user explicitly rejected partial confidence and wants proof that shipped instructions work.
243 +
244 +2. Decision: autonomy.
245 + - Selected: proceed without user involvement until an external dependency, credential, or product/risk decision blocks progress.
246 + - Rationale: user explicitly granted room to work autonomously.
247 +
248 +3. Decision: corrected scope.
249 + - Selected: all enrichment paths plus NetFlow and sFlow validation gaps; IPFIX and default geolocation database are baselines rather than the primary gap.
250 + - Rationale: user corrected the scope after SOW creation and clarified that the immediate ask is testing missing unit/integration coverage, not CI-job setup.
251 +
252 +4. Decision: downloader/provider coverage.
253 + - Selected: enhance the topology IP intelligence downloader so every supported free Geo/ASN provider path that can feed Netdata is exercised through generated MMDB output, update user documentation accordingly, test against actual provider data when licensing and public access permit it, then prove the generated output is usable by the netflow plugin.
254 + - Rationale: the netflow plugin reads MMDBs, while several provider sources start as CSV/TSV/tar/zip files. The missing proof is the conversion boundary from provider data into the MMDB files the plugin consumes.
255 + - Implication: this SOW now includes `src/go/tools/topology-ip-intel-downloader/`, its tests/docs/config examples, netflow integration metadata/examples that reference generated files, and runtime validation using generated MMDBs.
256 + - Risk: live provider downloads can be flaky, rate-limited, unavailable, or license-gated; CI-safe automated tests must remain fixture/local-data based, while actual-data validation is recorded as local evidence.
257 +
258 +5. Decision: local live-service validation.
259 + - Selected: after downloader/provider proof, test local NetBox, generic IPAM, bio-rd, and GoBGP/FRR/BMP paths as far as possible without user involvement.
260 + - Rationale: these are installable or emulatable local systems on this workstation and are better proof than only mocked unit tests.
261 + - Implication: use temporary/local runtime services, store raw outputs under `.local/`, and only record sanitized evidence in durable artifacts.
262 + - Risk: full router-equivalent proof may still require a real router or production routing topology; local GoBGP/FRR/BMP proof is protocol-level, not proof against a vendor router.
263 +
264 +6. Decision: beta external-routing validation bar.
265 + - Selected: prove Netdata's BMP and BioRIS contracts with deterministic payloads, local protocol speakers, and bio-rd-compatible gRPC services; do not block on production-router or production-bio-rd stability.
266 + - Rationale: the user does not have a router supporting these protocols, and the purpose is to catch Netdata-side issues before beta: connection/config failures, wrong fields, wrong payload schemas, ignored options, panics, hangs, leaked tasks, silent drops, and incorrect docs.
267 + - Implication: FRR/GoBGP plus deterministic BMP frames are sufficient for BMP. A deterministic bio-rd-compatible gRPC service is sufficient for BioRIS client validation. Upstream bio-rd remains useful as a smoke test, but upstream instability does not by itself block the beta.
268 + - Risk: this does not prove behavior against every vendor router or every production bio-rd deployment. That broader interoperability proof is expected from beta user feedback and later targeted fixes, while this SOW removes the obvious Netdata-side failures before shipping.
269 +
270 +## Plan
271 +
272 +1. Coverage matrix: map every enrichment method and NetFlow/sFlow from config/docs to runtime code and tests.
273 +2. Test harness: add deterministic local fixtures and fake services.
274 +3. Runtime proof: connect source-specific data to `FlowEnricher` and journal/query-visible fields.
275 +4. Documentation proof: execute or mechanically validate examples; fix source docs/metadata and regenerate.
276 +5. Local installed proof: build/install, restart `netdata`, inspect logs/journals/query output.
277 +6. Closeout: record validation, sensitive-data gate, artifact gates, and blockers requiring user action.
278 +
279 +## Execution Log
280 +
281 +### 2026-05-08
282 +
283 +- Created SOW after user approved autonomous full verification scope.
284 +- Recorded private flow exporter as `[PRIVATE_FLOW_EXPORTER]` instead of writing the literal endpoint.
285 +- Added CI-safe automated coverage for previously weak boundaries:
286 + - MaxMind/GeoLite2 ASN and City MMDB decoding with attributed public test databases.
287 + - Custom MMDB-style `netdata.ip_class` metadata from both ASN and GeoIP databases.
288 + - Static/runtime network-source merge behavior.
289 + - Documented AWS, Azure, GCP, generic JSON IPAM, and NetBox transform shapes.
290 + - HTTP network-source fetch, HTTP error handling, refresh loop publication, and cancellation.
291 + - NetFlow v9 and sFlow decode paths applying enrichment during decode.
292 + - Journal-backed ingest/query proof that enriched fields are persisted and query-visible.
293 + - BMP TCP listener accepting real BMP-framed Initiation and Termination messages.
294 + - BioRIS gRPC listener consuming `GetRouters`, `DumpRIB`, and `ObserveRIB` from a local fake service.
295 + - Stock `netflow.yaml` parse/validate coverage.
296 +- Added attributed MaxMind test MMDB fixtures under `src/crates/netflow-plugin/testdata/mmdb/`.
297 +- Found and fixed a production-code mismatch: `netdata.ip_class` was decoded from ASN MMDB records but not from GeoIP MMDB records, even though the MMDB downloader/docs can stamp the `netdata` metadata section into both outputs.
298 +- Found and fixed a Function request compatibility issue: public flow-query examples use `after: -3600`, `before: 0`, and numeric `top_n`; the plugin only accepted unsigned absolute timestamps and string enum `top_n` values. The request parser now accepts documented relative time bounds and numeric/string `top_n`.
299 +- Found and fixed a full-suite E2E reliability issue: five E2E tests passed individually but timed out in the default parallel full-suite run because the shared ingest wait was only 10 seconds. The shared wait is now 30 seconds.
300 +- Removed an existing unused import warning in the startup memory test module.
301 +- Corrected user-facing integration metadata and regenerated integration pages:
302 + - NetBox v2 tokens are documented as NetBox 4.5+; NetBox 4.0-4.4 use legacy tokens.
303 + - NetBox `?limit=0` is documented as dependent on server `MAX_PAGE_SIZE` policy.
304 + - BMP docs now state v3 is processed, v4 frames are decoded but ignored, and draft v1/v2 frames are decode errors.
305 +- Fixed the public `query-netdata-agents` helper after using it during runtime validation:
306 + - Bash-only nameref use no longer breaks in the workstation's default `zsh`.
307 + - Internal `path` variables no longer clear zsh `PATH`.
308 + - Masked curl logging now redacts node id, machine GUID, claim id, space id, and room id, not only tokens.
309 + - Added/updated no-token-leak self-tests for Bash and zsh.
310 +- Added `docs/netdata-ai/skills/query-netdata-cloud/how-tos/validate-local-netflow-function.md` because the Cloud/direct-agent validation workflow required multi-step analysis not present in the how-to catalog.
311 +- Ran official/live schema checks for provider payloads:
312 + - AWS `ip-ranges.json`: verified `ip_prefix`, `ipv6_prefix`, `region`, `service`, `network_border_group`, `syncToken`, and `createDate`.
313 + - GCP `cloud.json` and `goog.json`: verified `syncToken`, `creationTime`, `ipv4Prefix`/`ipv6Prefix`, and optional `service`/`scope` behavior.
314 + - Azure Service Tags current public JSON: verified `changeNumber`, `cloud`, `values[].name`, and `properties.addressPrefixes`, `region`, `platform`, `systemService`, `networkFeatures`.
315 + - NetBox current docs: verified v2 tokens are NetBox 4.5+.
316 +- Built and installed the plugin with `./build-install-netflow-plugin.sh`; `netdata` restarted and remained active.
317 +- Found a local runtime configuration drift before the final install validation:
318 + - `/etc/netdata/netflow.yaml` used a stale journal schema and failed startup.
319 + - Backed it up to `/etc/netdata/netflow.yaml.sow-0015-backup-20260508034532`.
320 + - Installed the current stock config from `src/crates/netflow-plugin/configs/netflow.yaml`.
321 + - Added stock-config parse/validate coverage so this drift is caught by tests.
322 +- Replayed pcap-derived UDP payloads into the installed collector on `127.0.0.1:2055`:
323 + - NetFlow v5 fixture.
324 + - NetFlow v9 template/options/data fixtures.
325 + - IPFIX template/data fixtures.
326 + - sFlow expanded-sample fixture.
327 +- Verified installed runtime output through journal-backed evidence:
328 + - `journalctl --directory=/var/cache/netdata/flows/raw --since [REPLAY_START] -o export` reported rows for `FLOW_VERSION=ipfix`, `FLOW_VERSION=v5`, `FLOW_VERSION=v9`, and `FLOW_VERSION=sflow`.
329 + - The same row set contained ASN and country fields for source/destination enrichment.
330 +- Verified Cloud-proxied Function output after install:
331 + - `agents_call_function --via cloud --function flows:netflow --body '{"info":true}'` returned `status: 200`, `type: flows`, and history metadata.
332 + - A documented `table-sankey` query body with `after: -3600`, `before: 0`, and numeric `top_n: 100` returned `status: 200`, flow rows, stats, and zero journal/template/parse errors.
333 +- Reopened SOW after user decision to:
334 + - enhance the topology IP intelligence downloader for all supported Geo/ASN providers that can feed generated MMDBs;
335 + - update related docs;
336 + - test providers with actual data where public/licensed access permits;
337 + - validate NetBox, generic IPAM, bio-rd, and GoBGP/FRR/BMP locally.
338 +- Enhanced `src/go/tools/topology-ip-intel-downloader/` provider support:
339 + - ASN: DB-IP MMDB/CSV, IPtoASN, CAIDA prefix2as, MaxMind GeoLite2 ASN.
340 + - GEO: DB-IP country/city MMDB/CSV, IPtoASN country, MaxMind GeoLite2 Country CSV, IP2Location Lite country, IPDeny country zones, IPIP country.
341 + - Added source-specific archive decoding for MaxMind tar.gz/zip, IP2Location zip, IPDeny tar.gz, IPIP zip, CAIDA latest-routeviews resolution, and redacted MaxMind query metadata.
342 + - Added parser tests for CAIDA, MaxMind Country CSV zip, IP2Location, IPDeny, IPIP, MaxMind env expansion/redaction, and all new provider tokens.
343 +- Added netflow plugin contract tests for generated Netdata topology MMDBs:
344 + - committed tiny ASN/GEO MMDB fixtures generated by the downloader;
345 + - `netdata_topology_mmdb_enrichment_populates_asn_and_geo_fields`;
346 + - opt-in actual-data test `actual_provider_mmdb_outputs_are_readable_when_root_is_set`.
347 +- Ran actual public provider downloads and generated MMDBs under `.local/audits/topology-ip-intel/providers`:
348 + - `dbip:asn-lite@mmdb`: 469,774 ASN ranges.
349 + - `dbip:asn-lite@csv`: 469,777 ASN ranges.
350 + - `iptoasn:combined` as ASN: 618,851 ASN ranges.
351 + - `caida:prefix2as`: 388,967 ASN ranges from `routeviews-rv2-20260506-1200.pfx2as.gz`.
352 + - `dbip:country-lite@mmdb`: 702,373 geo ranges.
353 + - `dbip:country-lite@csv`: 702,409 geo ranges.
354 + - `dbip:city-lite@mmdb`: 8,061,643 geo ranges.
355 + - `dbip:city-lite@csv`: 8,061,679 geo ranges.
356 + - `iptoasn:combined` as GEO: 286,373 geo ranges.
357 + - `ip2location:country-lite`: 273,642 geo ranges.
358 + - `ipdeny:country-zones`: 108,309 geo ranges.
359 + - `ipip:country`: 507,524 geo ranges.
360 +- After the user pointed to a local update-ipsets environment source, live MaxMind downloads were also run without printing the key:
361 + - `maxmind:geolite2-asn`: 501,939 ASN ranges.
362 + - `maxmind:geolite2-country`: 599,247 geo ranges.
363 +- Validated local live services:
364 + - NetBox: cloned `netbox-community/netbox-docker @ 55edb986b22c12b69ec89c79c8ccc7fdea88c5b5`, started NetBox v4.6 under Docker Compose, inserted a prefix, fetched `/api/ipam/prefixes/?limit=1000`, and applied the documented transform to produce `{prefix,name,tenant,role,site}`.
365 + - Generic JSON-over-HTTP IPAM: served a local JSON endpoint and validated the documented transform shape.
366 + - GoBGP/BMP: built `osrg/gobgp @ 5f191066a78e2c1e929c54b5b75fe2c683c166e4`; added an opt-in test that starts a Netdata BMP listener, starts `gobgpd`, adds a route with `gobgp`, and verifies the dynamic routing runtime receives ASN and AS-path.
367 + - FRR/BMP: pulled `frrouting/frr:latest`, verified documented BMP config with `bgpd -M bmp -C`, then ran a container that established a BMP connection and emitted BMP bytes to a local listener.
368 + - bio-rd: built `bio-routing/bio-rd @ 14a8de966e8b3f488a207aa0d02454a447ed5c99`; its RIS gRPC path is covered by Netdata's in-process test, but a practical GoBGP-to-bio-rd no-neighbor local-RIB setup crashed upstream bio-rd on a BMP Peer Up. This is recorded as an external blocker, not a Netdata client failure.
369 +- Corrected user-facing docs:
370 + - downloader README and sample config now list all supported provider tokens/formats and MaxMind license-key behavior;
371 + - MaxMind integration docs now describe the topology downloader path for GeoLite2 ASN/Country;
372 + - NetBox docs now show the NetBox 4.5+ v2 bearer header as `Bearer nbt_<12-char-key>.<40-char-token>` and explain that sending a v2 token with the legacy `Token` prefix produces an invalid-v1-token error.
373 +
374 +## Validation
375 +
376 +Acceptance criteria evidence:
377 +
378 +- Scoped enrichment coverage:
379 + - Custom MMDB / MaxMind / GeoLite2: `src/crates/netflow-plugin/src/enrichment/tests.rs` covers public ASN and City MMDB fixtures, ASN org, country/city/state/lat/lon, and `netdata.ip_class`.
380 + - DB-IP / IPtoASN baseline: existing default geolocation runtime was treated as user-confirmed baseline; new MMDB tests exercise the same resolver path with deterministic fixtures.
381 + - AWS / Azure / GCP / generic IPAM / NetBox: `src/crates/netflow-plugin/src/network_sources/tests.rs` covers documented transform payloads and the HTTP fetch/publish path.
382 + - Static metadata, sampling overrides, static networks, classifiers, ASN/network provider chains, and decapsulation: existing tests remain in `src/crates/netflow-plugin/src/enrichment/tests.rs` and decoder fixture tests; new runtime merge and decode/enrichment tests cover the boundary.
383 + - BioRIS: `src/crates/netflow-plugin/src/routing/bioris/tests.rs` covers a real in-process gRPC service and runtime route lookup after `DumpRIB`.
384 + - BMP: `src/crates/netflow-plugin/src/routing/bmp/tests.rs` covers a real TCP listener and BMP-framed messages.
385 + - Downloader provider conversion: `src/go/tools/topology-ip-intel-downloader/*_test.go` covers every supported built-in provider token/format and the provider-specific parser/decode paths.
386 + - Generated Netdata topology MMDB consumption: `src/crates/netflow-plugin/src/enrichment/tests.rs` reads downloader-generated MMDB fixtures and proves the Rust enricher sees ASN, ASN name, country, city, state, and coordinates.
387 + - Actual downloaded provider outputs: opt-in test `actual_provider_mmdb_outputs_are_readable_when_root_is_set` was run against `.local/audits/topology-ip-intel/providers` and read every public provider output with the Rust resolver, including live MaxMind GeoLite2 ASN and Country outputs.
388 + - Real GoBGP BMP speaker: opt-in test `bmp_listener_accepts_gobgp_route_when_binaries_are_set` was run against locally built `gobgpd`/`gobgp` and proved a route added through GoBGP reaches the Netdata dynamic-routing runtime.
389 + - NetFlow/sFlow: `src/crates/netflow-plugin/src/decoder/tests.rs` proves NetFlow v9 and sFlow records receive enrichment during decode; installed runtime replay produced v5/v9/sFlow journal rows.
390 + - Journal/query visibility: `src/crates/netflow-plugin/src/main_tests.rs` proves enriched fields are written to the raw journal and are query-visible by `SRC_NET_NAME`.
391 +
392 +Tests or equivalent validation:
393 +
394 +- `cargo fmt -p netflow-plugin` passed.
395 +- Targeted tests passed:
396 + - `cargo test -p netflow-plugin apply_geo_record_accepts_netdata_ip_class_metadata -- --nocapture`
397 + - `cargo test -p netflow-plugin maxmind_geolite2_mmdb_enrichment_populates_asn_and_geo_fields -- --nocapture`
398 + - `cargo test -p netflow-plugin request_deserialization_accepts -- --nocapture`
399 + - `cargo test -p netflow-plugin resolve_time_bounds_treats_negative_after_as_relative_to_before -- --nocapture`
400 +- Full Rust plugin suite passed after fixes:
401 + - `cargo test -p netflow-plugin --manifest-path src/crates/Cargo.toml`
402 + - Result: 447 passed, 18 ignored, 0 failed.
403 + - `tests/grpc_build.rs`: 1 passed, 0 failed.
404 +- Downloader tests passed:
405 + - `go test ./tools/topology-ip-intel-downloader` from `src/go`.
406 +- Actual provider output tests passed:
407 + - `NETDATA_TOPOLOGY_IP_INTEL_PROVIDER_ROOT="$(pwd)/.local/audits/topology-ip-intel/providers" cargo test -p netflow-plugin actual_provider_mmdb_outputs_are_readable_when_root_is_set --manifest-path src/crates/Cargo.toml`
408 + - `cargo test -p netflow-plugin netdata_topology_mmdb_enrichment_populates_asn_and_geo_fields --manifest-path src/crates/Cargo.toml`
409 +- Real GoBGP BMP test passed:
410 + - `NETDATA_GOBGPD="$(pwd)/.local/audits/netflow-live/bin/gobgpd" NETDATA_GOBGP="$(pwd)/.local/audits/netflow-live/bin/gobgp" cargo test -p netflow-plugin bmp_listener_accepts_gobgp_route_when_binaries_are_set --manifest-path src/crates/Cargo.toml`
411 +- Network-source and routing targeted tests passed:
412 + - `cargo test -p netflow-plugin network_sources --manifest-path src/crates/Cargo.toml`
413 + - `cargo test -p netflow-plugin routing::bioris --manifest-path src/crates/Cargo.toml`
414 + - `cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml`
415 +- Integration metadata/docs validation passed:
416 + - `python3 integrations/gen_integrations.py`
417 + - `python3 integrations/gen_docs_integrations.py`
418 + - `python3` YAML parse for `src/crates/netflow-plugin/metadata.yaml`
419 + - `git diff --check`
420 +- SOW audit was run after moving this file to `done/`. Status/directory checks passed; the command exited nonzero only on a pre-existing false positive in an unmodified mirror-skill Git SSH URL that the audit pattern classifies as an email-like string.
421 +- Public helper validation passed:
422 + - `bash -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'`
423 + - `zsh -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'`
424 + - `shellcheck --external-sources docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`
425 +
426 +Real-use evidence:
427 +
428 +- `./build-install-netflow-plugin.sh` completed:
429 + - Release build finished.
430 + - Plugin installed to `/usr/libexec/netdata/plugins.d/netflow-plugin`.
431 + - `systemctl is-active netdata` returned `active`.
432 +- Installed runtime:
433 + - `journalctl --namespace=netdata SYSLOG_IDENTIFIER=netflow-plugin --since '2 minutes ago'` showed startup, Function declaration, journal scanning, and no startup failure.
434 + - `ss -lunp` showed `netflow-plugin` listening on UDP `0.0.0.0:2055`.
435 +- Runtime pcap replay:
436 + - Eight UDP payloads were sent to `127.0.0.1:2055` from deterministic local pcap fixtures.
437 + - Raw journal inspection since replay start reported:
438 + - rows: 513
439 + - `FLOW_VERSION[ipfix]`: 479
440 + - `FLOW_VERSION[v5]`: 29
441 + - `FLOW_VERSION[v9]`: 4
442 + - `FLOW_VERSION[sflow]`: 1
443 + - source ASN fields: 369
444 + - destination ASN fields: 502
445 + - source country fields: 370
446 + - destination country fields: 503
447 +- Cloud-proxied Function validation:
448 + - Local `/api/v3/info` showed one local agent, Cloud status online, and node/machine/claim identifiers present; raw identifiers were stored only under `.local/`.
449 + - Cloud discovery found the local node reachable in visible rooms by exact local node/machine identifiers.
450 + - `flows:netflow` info call returned `status: 200` and `type: flows`.
451 + - A documented flow query returned `status: 200`, 101 flow rows, expected group-by fields, and stats including decoded NetFlow v5, NetFlow v9, IPFIX, and sFlow counters with zero parse/template/journal-write errors.
452 +- Direct-agent bearer validation:
453 + - A Cloud bearer mint using the local `/api/v3/info` node id, machine GUID, and claim id tuple returned HTTP 200 with a token present.
454 + - `agents_call_function --via agent --function flows:netflow --body '{"info":true}'` against `127.0.0.1:19999` returned `status: 200`, `type: flows`, and `has_history: true`.
455 +- Actual provider downloader validation:
456 + - Every live provider listed in the execution log downloaded, parsed, and wrote a non-empty MMDB output.
457 + - The Rust resolver read all generated outputs successfully in the opt-in actual-data test, including live MaxMind GeoLite2 ASN and Country outputs.
458 +- NetBox live validation:
459 + - Local NetBox v4.6 reached healthy status under Docker Compose.
460 + - `/api/ipam/prefixes/?limit=1000` returned the inserted `198.51.100.0/24` prefix.
461 + - The documented NetBox transform produced `{"prefix":"198.51.100.0/24","tenant":"","role":"","site":"","name":"netdata edge subnet"}`.
462 + - NetBox v4.6 rejected a v2 token sent as legacy `Token` with "Invalid v1 token"; docs were corrected to show `Bearer nbt_<12-char-key>.<40-char-token>`.
463 +- Generic IPAM live validation:
464 + - A local HTTP JSON endpoint returned a prefix list and the documented transform produced `{"prefix":"203.0.113.0/24","name":"corp edge","tenant":"prod"}`.
465 +- BioRIS / BMP live validation:
466 + - GoBGP -> Netdata BMP listener route publication passed with real GoBGP binaries.
467 + - FRR `bgpd -M bmp -C` accepted the documented BMP config, and a running `frrouting/frr` container established a BMP connection and emitted bytes to a local listener.
468 + - GoBGP -> upstream bio-rd RIS crashed inside bio-rd on BMP Peer Up in the no-neighbor local-RIB setup; Netdata's BioRIS client path remains covered by the in-process gRPC service test.
469 +
470 +Reviewer findings:
471 +
472 +- No external AI reviewer was requested by the user.
473 +- Self-review findings handled during execution:
474 + - GeoIP MMDB did not apply `netdata.ip_class`; fixed and tested.
475 + - Public flow-query request shape did not accept documented negative `after` and numeric `top_n`; fixed and tested.
476 + - Public direct-agent helper failed under zsh and leaked identifiers in masked logs; fixed and tested.
477 + - E2E full-suite ingest wait was too short under parallel load; fixed and full suite passed.
478 +
479 +Same-failure scan:
480 +
481 +- Searched for provider-transform examples and `top_n` examples in public flow-query docs/skills.
482 +- The public examples using numeric `top_n` now work because the plugin accepts both numeric and string values.
483 +- The public examples using negative relative `after` now work because request parsing and time-bound resolution support it.
484 +- Searched generated integration side effects after docs generation; reverted unrelated SNMP generated-page churn and verified no SNMP diff remained.
485 +
486 +Sensitive data gate:
487 +
488 +- Raw `/api/v3/info`, Cloud discovery payloads, bearer endpoint responses, and flow Function responses are stored only under `.local/audits/query-netdata-agents/`, which is gitignored.
489 +- This SOW records sanitized summaries only:
490 + - private flow exporter is `[PRIVATE_FLOW_EXPORTER]`;
491 + - local hostname is `[LOCAL_HOST]`;
492 + - no node ids, machine GUIDs, claim ids, tokens, bearer values, raw private endpoints, or raw flow rows are recorded here.
493 +- Public helper logging now masks Cloud tokens, agent bearers, node ids, machine GUIDs, claim ids, space ids, and room ids in stderr.
494 +
495 +Artifact maintenance gate:
496 +
497 +- AGENTS.md: no update needed; workflow rules were sufficient.
498 +- Runtime project skills: no project-local collector/integration skill update needed; existing `project-writing-collectors` and `integrations-lifecycle` rules were sufficient.
499 +- Specs: no separate spec update needed; this SOW did not change a durable product contract beyond making implementation match already published Function-query examples and MMDB metadata behavior.
500 +- End-user/operator docs: updated `src/crates/netflow-plugin/metadata.yaml` and regenerated affected integration pages.
501 +- End-user/operator skills: updated `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`; added Cloud and direct-agent how-tos for local flow Function validation.
502 +- SOW lifecycle: SOW is marked `completed` and moved to `.agents/sow/done/` with the implementation and validation commit. Remaining items require user-supplied credentials or real external systems and are recorded below.
503 +
504 +Specs update:
505 +
506 +- No spec file was updated. Reason: the durable behavior changes are already captured in code/tests and public operator docs/skills:
507 + - `after: -N`, `before: 0`, and numeric `top_n` are public Function request behavior now covered by tests.
508 + - MMDB `netdata.ip_class` handling now matches existing downloader/documentation intent.
509 +
510 +Project skills update:
511 +
512 +- No runtime project skill update needed.
513 +- Public operator skill helper was updated because its direct-agent transport wrapper failed under zsh and did not mask all identity fields.
514 +
515 +End-user/operator docs update:
516 +
517 +- Updated:
518 + - `src/crates/netflow-plugin/metadata.yaml`
519 + - `src/crates/netflow-plugin/integrations/netbox.md`
520 + - `src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md`
521 + - `src/crates/netflow-plugin/integrations/bmp_bgp_monitoring_protocol.md`
522 + - `src/go/tools/topology-ip-intel-downloader/README.md`
523 + - `src/go/tools/topology-ip-intel-downloader/configs/topology-ip-intel.yaml`
524 + - `docs/netdata-ai/skills/query-netdata-cloud/how-tos/validate-local-netflow-function.md`
525 + - `docs/netdata-ai/skills/query-netdata-cloud/how-tos/INDEX.md`
526 + - `docs/netdata-ai/skills/query-netdata-agents/how-tos/validate-direct-local-flow-function.md`
527 + - `docs/netdata-ai/skills/query-netdata-agents/how-tos/INDEX.md`
528 +
529 +End-user/operator skills update:
530 +
531 +- Updated `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`.
532 +- Added `docs/netdata-ai/skills/query-netdata-agents/how-tos/validate-direct-local-flow-function.md`.
533 +- Validation: Bash self-test, zsh self-test, and shellcheck all passed.
534 +
535 +Lessons:
536 +
537 +- Testing the public instructions against the installed agent exposed real API-compatibility bugs that pure unit tests had missed.
538 +- Direct-agent bearer validation works when the wrapper uses the exact local `/api/v3/info` node id, machine GUID, and claim id tuple. Mixed node/machine/claim tuples should be treated as identity-resolution bugs before treating direct bearer as blocked.
539 +- Masked command logging must redact identifiers as well as tokens, because node ids, machine GUIDs, claim ids, space ids, and room ids are sensitive enough to keep out of durable artifacts.
540 +
541 +Follow-up mapping:
542 +
543 +- Requires user involvement if further proof is required:
544 + - Real BMP from a production router: user does not run BGP/BMP locally. Current proof covers deterministic BMP frames, real GoBGP-to-Netdata BMP route publication, and FRR BMP config/connection. A real router session still requires a router or lab BGP topology.
545 + - Real bio-rd / RIPE RIS with live routes: Netdata's BioRIS client path is covered by an in-process gRPC service, but the local GoBGP-to-upstream-bio-rd setup crashed inside bio-rd on BMP Peer Up. Further proof needs either a stable bio-rd/RIS endpoint or time to debug/report/fix upstream bio-rd.
546 + - User-owned NetBox / custom IPAM endpoints: local NetBox and generic IPAM were validated; proof against the user's own systems requires their endpoint and token if desired.
547 + - Custom operator-provided MMDB: current proof uses public MaxMind test MMDB fixtures and a synthetic `netdata.ip_class` record; no user custom MMDB was supplied.
548 +- Explicitly out of this SOW:
549 + - CI job wiring. The user corrected that the immediate scope is unit/integration testing and practical validation, not CI-job integration.
550 +
551 +## Outcome
552 +
553 +Autonomous work completed as far as possible without external systems or user credentials. Automated coverage, downloader provider support, actual public-provider data validation, local NetBox/IPAM validation, GoBGP/FRR BMP validation, docs/source corrections, installed runtime validation, journal proof, Cloud-proxied Function validation, and direct-agent bearer Function validation are complete.
554 +
555 +## Lessons Extracted
556 +
557 +- Run public examples against the installed plugin, not only through Rust unit tests. This caught `after`/`top_n` request-shape bugs.
558 +- Include shell compatibility in public helper tests when examples are likely to be run from users' default shells.
559 +- Use the exact local `/api/v3/info` node id, machine GUID, and claim id tuple for direct-agent bearer minting; mixed identity tuples are the common failure mode.
560 +- Treat identity fields as sensitive in wrapper logging; masking only bearer/token values is insufficient.
561 +
562 +## Followup
563 +
564 +User-required items are listed in Follow-up mapping above. No autonomous follow-up remains unattempted in this SOW.
565 +
566 +## PR Review Iteration - 2026-05-08
567 +
568 +Trigger:
569 +
570 +- User asked to solve PR comments/reviews on PR 22453 before continuing.
571 +
572 +Finding sources fetched:
573 +
574 +- `bash .agents/skills/pr-reviews/scripts/fetch-all.sh 22453`
575 + - 8 open review threads.
576 + - All open review threads are from `cubic-dev-ai[bot]`.
577 + - No human review comments need a user decision before implementation.
578 +- `bash .agents/skills/pr-reviews/scripts/fetch-sonar-findings.sh 22453`
579 + - 0 SonarCloud issues.
580 + - 0 SonarCloud hotspots.
581 +- `bash .agents/skills/pr-reviews/scripts/ci-status.sh 22453`
582 + - 0 failing checks.
583 + - Most checks are still pending after the latest push.
584 +
585 +Open review threads to verify:
586 +
587 +- `src/crates/netflow-plugin/src/routing/bioris/tests.rs:242`: local gRPC test reserves and releases an ephemeral port before server bind.
588 +- `src/crates/netflow-plugin/src/routing/bmp/tests.rs:274`: BMP listener test reserves and releases an ephemeral port before listener bind.
589 +- `src/crates/netflow-plugin/src/query/planner/request.rs:20`: timestamp clamp may collapse `after` and `before`.
590 +- `docs/netdata-ai/skills/query-netdata-agents/how-tos/validate-direct-local-flow-function.md:25`: guide persists raw `/api/v3/info` payload containing sensitive identifiers.
591 +- `src/crates/netflow-plugin/src/network_sources/tests.rs:171`: NetBox transform test maps `site` but does not assert it.
592 +- `src/go/tools/topology-ip-intel-downloader/config.go:280`: IPDeny built-in source uses HTTP.
593 +- `src/crates/netflow-plugin/src/routing/bmp/tests.rs:431`: async test uses blocking `std::process::Command::status()`.
594 +- `src/go/tools/topology-ip-intel-downloader/parse.go:775`: IPDeny `.ZONE` extension handling is case-sensitive after case-insensitive filter.
595 +
596 +Plan:
597 +
598 +- Verify each bot finding against current code.
599 +- Fix valid findings and search touched PR files for the same failure class.
600 +- Run the narrow tests for modified areas plus formatting and diff checks.
601 +- Re-fetch findings before push.
602 +- Push one commit for the review fixes, reply per thread, resolve threads, and retrigger AI reviewers.
603 +
604 +Verified fixes:
605 +
606 +- BioRIS gRPC test now binds the Tokio listener first and serves tonic from that bound listener via `TcpListenerStream`, removing the reserve/release race.
607 +- BMP listener tests now pass a pre-bound Tokio listener into a shared listener helper, removing the reserve/release race for the Netdata listener side.
608 +- The remaining GoBGP API port is for an external daemon that requires a concrete address; the test records that limitation and keeps the path opt-in.
609 +- Query time-bound resolution now clamps first and enforces `after < before` after clamping.
610 +- Direct-agent and Cloud flow-validation how-tos no longer persist raw `/api/v3/info`; they keep the raw identity payload in shell memory and print only presence checks.
611 +- NetBox transform tests now assert the documented `site` mapping.
612 +- IPDeny built-in URL now uses HTTPS; live HEAD check returned HTTP 200 for the HTTPS URL.
613 +- GoBGP helper commands in the async BMP test now use `tokio::process::Command`.
614 +- IPDeny `.zone` parsing now handles uppercase `.ZONE` suffixes and has order-independent test coverage.
615 +
616 +Validation after fixes:
617 +
618 +- `go test ./tools/topology-ip-intel-downloader`
619 +- `cargo test -p netflow-plugin documented_cloud_and_ipam_transforms_decode_provider_payloads --manifest-path src/crates/Cargo.toml`
620 +- `cargo test -p netflow-plugin resolve_time_bounds --manifest-path src/crates/Cargo.toml`
621 +- `cargo test -p netflow-plugin routing::bioris --manifest-path src/crates/Cargo.toml`
622 +- `cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml`
623 +- `NETDATA_GOBGPD="$(pwd)/.local/audits/netflow-live/bin/gobgpd" NETDATA_GOBGP="$(pwd)/.local/audits/netflow-live/bin/gobgp" cargo test -p netflow-plugin bmp_listener_accepts_gobgp_route_when_binaries_are_set --manifest-path src/crates/Cargo.toml`
624 +- `cargo test -p netflow-plugin --manifest-path src/crates/Cargo.toml`
625 + - Result: 448 passed, 18 ignored, 0 failed.
626 + - `tests/grpc_build.rs`: 1 passed, 0 failed.
627 +- How-to jq snippets were smoke-tested with a synthetic `/api/v3/info` payload without writing raw identity JSON.
628 +- `git diff --check`
629 +
630 +Final pre-push sync:
631 +
632 +- `bash .agents/skills/pr-reviews/scripts/fetch-all.sh 22453`
633 + - 8 open review threads, unchanged from the initial set.
634 + - No new comments.
635 +- `bash .agents/skills/pr-reviews/scripts/fetch-sonar-findings.sh 22453`
636 + - 0 issues.
637 + - 0 hotspots.
638 +- `bash .agents/skills/pr-reviews/scripts/ci-status.sh 22453`
639 + - 0 failing checks.
640 + - 90 checks still running on the previous commit.
641 +
642 +Artifact maintenance for review pass:
643 +
644 +- AGENTS.md: no update needed; existing PR-review and sensitive-data rules were sufficient.
645 +- Runtime project skills: no update needed.
646 +- Specs: no update needed; fixes align implementation/tests/docs to existing intended behavior.
647 +- End-user/operator docs: updated direct-agent and Cloud validation how-tos to avoid durable raw identity payloads.
648 +- End-user/operator skills: no script behavior changed in this review pass.
649 +- SOW lifecycle: SOW returned to `completed` and is moved back to `.agents/sow/done/` with the review-fix commit.
650 +
651 +## Regression Log
652 +
653 +### Regression - 2026-05-08 - Missing Provider Integration Modules
654 +
655 +What broke:
656 +
657 +- The previous completion enhanced and validated the topology IP intelligence downloader for CAIDA, IP2Location, IPDeny, and IPIP, but the user-facing NetFlow enrichment metadata still exposed only DB-IP, MaxMind, IPtoASN, and Custom MMDB as IP intelligence integrations.
658 +- This made the generated integrations catalog incomplete: users could discover downloader tokens in the downloader README, but not from the NetFlow enrichment-method module list.
659 +
660 +Evidence:
661 +
662 +- `src/crates/netflow-plugin/metadata.yaml` had modules for `dbip`, `maxmind`, `iptoasn`, and `custom-mmdb`.
663 +- Generated pages under `src/crates/netflow-plugin/integrations/` existed only for `db-ip_ip_intelligence.md`, `maxmind_geoip_-_geolite2.md`, `iptoasn.md`, and `custom_mmdb_database.md`.
664 +- Downloader code supports `caida:prefix2as`, `ip2location:country-lite`, `ipdeny:country-zones`, and `ipip:country` in `src/go/tools/topology-ip-intel-downloader/config.go`.
665 +
666 +Why previous validation missed it:
667 +
668 +- Validation proved downloader parsing, live provider downloads, generated MMDB output, and Rust resolver consumption, but did not compare the complete downloader provider matrix against generated integration module coverage.
669 +
670 +Repair plan:
671 +
672 +- Add first-class `metadata.yaml` modules for CAIDA Routeviews Prefix-to-AS, IP2Location LITE IP-Country, IPDeny Country Zones, and IPIP Country.
673 +- Cross-link these modules with the existing DB-IP, MaxMind, IPtoASN, and Custom MMDB modules.
674 +- Regenerate generated integration pages from `metadata.yaml`.
675 +- Validate metadata generation and downloader tests.
676 +
677 +Sensitive data handling:
678 +
679 +- No secrets, private endpoints, node identifiers, or raw provider payloads are needed in durable artifacts. Provider descriptions use public source names, public URLs, and downloader tokens only.
680 +
681 +Implementation:
682 +
683 +- Added first-class NetFlow enrichment metadata modules for:
684 + - `caida-prefix2as` - CAIDA Routeviews Prefix-to-AS.
685 + - `ip2location` - IP2Location LITE IP-Country.
686 + - `ipdeny` - IPDeny Country Zones.
687 + - `ipip` - IPIP Country Database.
688 +- Updated related-resource links from DB-IP, MaxMind, IPtoASN, and Custom MMDB so the generated integration catalog surfaces the full supported IP intelligence provider set.
689 +- Regenerated the new NetFlow integration pages from `src/crates/netflow-plugin/metadata.yaml`.
690 +
691 +Validation:
692 +
693 +- `go test ./tools/topology-ip-intel-downloader` passed from `src/go`.
694 +- `python3 integrations/gen_integrations.py` passed.
695 +- `python3 integrations/gen_docs_integrations.py` generated the new NetFlow provider pages. It also reproduced a pre-existing unrelated SNMP generated-page diff; that unrelated SNMP churn was removed from this changeset.
696 +- A metadata coverage check verified all expected IP intelligence modules are present: `dbip`, `maxmind`, `iptoasn`, `custom-mmdb`, `caida-prefix2as`, `ip2location`, `ipdeny`, and `ipip`.
697 +- Generated page existence check verified:
698 + - `src/crates/netflow-plugin/integrations/caida_routeviews_prefix-to-as.md`
699 + - `src/crates/netflow-plugin/integrations/ip2location_lite_ip-country.md`
700 + - `src/crates/netflow-plugin/integrations/ipdeny_country_zones.md`
701 + - `src/crates/netflow-plugin/integrations/ipip_country_database.md`
702 +- `git diff --check` passed after removing unrelated generated SNMP churn.
703 +
704 +Artifact maintenance:
705 +
706 +- AGENTS.md: no update needed; existing integration-generation and SOW regression rules were sufficient.
707 +- Runtime project skills: no update needed; `integrations-lifecycle` already documents the source/generator contract.
708 +- Specs: no update needed; this is a documentation/catalog completeness repair for already-implemented provider behavior.
709 +- End-user/operator docs: updated `src/crates/netflow-plugin/metadata.yaml` and generated four new provider integration pages.
710 +- End-user/operator skills: no update needed; no public skill workflow changed.
711 +- SOW lifecycle: regression recorded here, then SOW returned to `completed` and moved back to `.agents/sow/done/` with the metadata/doc commit.
712 +
713 +### PR Review Follow-up - 2026-05-08 - Copilot Findings
714 +
715 +Trigger:
716 +
717 +- Copilot opened PR review threads on:
718 + - `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh:171`
719 + - `src/go/tools/topology-ip-intel-downloader/fetch.go:295`
720 +
721 +Findings:
722 +
723 +- `_agents_set_outvar` used `eval "${name}=\${value}"` to write external command output into a caller-provided variable.
724 +- The variable name was validated, and the previous assignment form avoided common word-splitting behavior, but the pattern was still fragile and hard to audit because token and claim values originate from `curl` / `jq` output.
725 +- `decodeMaxMindASNPayload` accepted every `extractMMDBFromTar` error as "not a tarred MMDB" and returned the ungzipped content. That preserved plain gzipped MMDB support, but it also hid corrupt tar-like payloads behind a later, less-specific MMDB parse error.
726 +
727 +Implementation:
728 +
729 +- Replaced the eval assignment with `printf -v "${_agents_out_name}" '%s' "${_agents_out_value}"`.
730 +- Kept variable-name validation before assignment.
731 +- Added a self-test payload containing whitespace, glob characters, shell metacharacters, command-substitution text, backticks, quotes, and a newline, verifying the helper preserves it as data and does not interpret it.
732 +- Updated the public `query-netdata-agents` skill text from "bash nameref" to "validated caller-local output variable", matching the zsh-compatible implementation.
733 +- Updated MaxMind ASN payload decoding so plain gzipped MMDB payloads still pass through, while tar-like payloads with invalid tar structure now return a clear extraction error.
734 +- Added downloader test coverage for corrupt tar-like MaxMind ASN payloads.
735 +
736 +Validation:
737 +
738 +- `bash -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
739 +- `zsh -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
740 +- `shellcheck --external-sources docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh` passed.
741 +- `git diff --check` passed.
742 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on the existing unmodified `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern that the audit classifies as email-like sensitive data.
743 +- `git diff --check` passed.
744 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on the existing unmodified `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern that the audit classifies as email-like sensitive data.
745 +- `go test ./tools/topology-ip-intel-downloader` passed from `src/go`.
746 +- Same-pattern search found no remaining eval-based `_agents_set_outvar` assignment in the public skill path.
747 +
748 +Artifact maintenance:
749 +
750 +- AGENTS.md: no update needed; existing sensitive-data and public-skill rules were sufficient.
751 +- Runtime project skills: no update needed.
752 +- Specs: no update needed; this repaired implementation safety/error handling for existing helper/provider contracts.
753 +- End-user/operator docs: updated `docs/netdata-ai/skills/query-netdata-agents/SKILL.md` to describe the actual output-variable contract.
754 +- End-user/operator skills: updated `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`.
755 +- SOW lifecycle: reopened from `done`, recorded this PR review follow-up, then returned to `completed` and moved back to `.agents/sow/done/` with the fix commit.
756 +
757 +### External App Validation Follow-up - 2026-05-08
758 +
759 +Trigger:
760 +
761 +- User asked to proceed with the remaining external application validation after PR review cleanup.
762 +
763 +Scope exercised:
764 +
765 +- Generic JSON-over-HTTP IPAM.
766 +- NetBox.
767 +- FRR BMP.
768 +- GoBGP BMP.
769 +- bio-rd RIS / BioRIS.
770 +
771 +Evidence and results:
772 +
773 +- Generic JSON-over-HTTP IPAM:
774 + - Started a temporary localhost HTTP server serving sanitized RFC documentation prefixes.
775 + - Fetched `networks.json` over HTTP and applied the documented transform `.[] | {prefix: .cidr, name: .label, tenant: .tenant}`.
776 + - Result: `{"prefix":"203.0.113.0/24","name":"corp edge","tenant":"prod"}`.
777 + - Temporary HTTP process was verified stopped.
778 +- NetBox:
779 + - Started local `netbox-docker` under a dedicated Compose project.
780 + - Initial health check exceeded the Docker health timeout while migrations were still finishing, but the NetBox login endpoint returned HTTP 200 and the API was usable after startup.
781 + - Bootstrapped a local prefix and queried `/api/ipam/prefixes/?limit=1000` with a generated local token.
782 + - Applied the documented transform and verified the expected result:
783 + `{"prefix":"198.51.100.0/24","tenant":"","role":"","site":"","name":"netdata edge subnet"}`.
784 + - Compose project was shut down with volumes removed.
785 +- FRR BMP:
786 + - Used `quay.io/frrouting/frr:10.6.0`.
787 + - `bgpd -M bmp -C -f /etc/frr/bgpd.conf` accepted the BMP config. FRR logged warnings about missing synthetic OPEN messages for the static announcement, but did not reject the config.
788 + - A running FRR container connected to a local BMP listener and emitted 297 bytes. The captured stream began with BMP v3 and an FRRouting initiation payload.
789 + - Temporary listener process and container were verified stopped.
790 +- GoBGP BMP:
791 + - `NETDATA_GOBGPD=.local/audits/netflow-live/bin/gobgpd NETDATA_GOBGP=.local/audits/netflow-live/bin/gobgp cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml`
792 + - Result: 13 passed, 0 failed, including `bmp_listener_accepts_gobgp_route_when_binaries_are_set`.
793 +- BioRIS / bio-rd:
794 + - Upstream source evidence shows `bio-rd` `cmd/ris` is BMP-backed:
795 + - `bio-routing/bio-rd @ 14a8de966e8b3f488a207aa0d02454a447ed5c99`, `cmd/ris/main.go:52-83` creates a BMP receiver, listens for BMP, adds configured BMP servers, then exposes the RIS gRPC server.
796 + - `bio-routing/bio-rd @ 14a8de966e8b3f488a207aa0d02454a447ed5c99`, `cmd/ris/config/config.go:10-20` defines `bmp_servers`.
797 + - Official RIPE RIS documentation checked:
798 + - `https://ris-live.ripe.net/` describes RIS Live as WebSocket JSON.
799 + - `https://ris.ripe.net/docs/` lists route collectors, raw MRT files, RIS Live, RISwhois, and routing beacons.
800 + - No official public RIPE RIS endpoint implementing `bio.ris.RoutingInformationService` gRPC was found.
801 + - GoBGP-to-upstream-bio-rd reached BMP Peer Up, then upstream bio-rd crashed in its BGP/BMP handling. This is an upstream/runtime issue, not a Netdata BioRIS client failure.
802 + - FRR-to-upstream-bio-rd did not crash; `riscli routers` saw a router from `[LOCAL_HOST]`, but the local loopback lab did not produce a stable non-empty RIB. The logs showed peer up/down behavior and duplicate-neighbor handling.
803 + - Netdata's BioRIS client path remains covered by the in-process gRPC test that exercises `GetRouters`, `DumpRIB`, and `ObserveRIB`.
804 +
805 +Documentation correction:
806 +
807 +- Corrected `src/crates/netflow-plugin/metadata.yaml` so the BioRIS module says Netdata consumes only a bio-rd-compatible `RoutingInformationService` gRPC endpoint.
808 +- Corrected the generated `src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md` from metadata.
809 +- Corrected `docs/network-flows/enrichment.md` and `docs/network-flows/configuration.md` link text / routing-overlay language.
810 +- Removed misleading instructions that implied users can point `grpc_addr` at RIPE RIS Live, RIPEstat, MRT dumps, route collector sessions, or looking-glass sources directly.
811 +
812 +Validation after documentation correction:
813 +
814 +- `python3 integrations/gen_integrations.py` passed.
815 +- `python3 integrations/gen_docs_integrations.py` passed.
816 +- `python3 -c` YAML parse of `src/crates/netflow-plugin/metadata.yaml` passed.
817 +- `cargo test -p netflow-plugin network_sources --manifest-path src/crates/Cargo.toml`
818 + - Result: 17 passed, 0 failed.
819 +- `cargo test -p netflow-plugin routing::bioris --manifest-path src/crates/Cargo.toml`
820 + - Result: 7 passed, 0 failed.
821 +- `NETDATA_GOBGPD="$(pwd)/.local/audits/netflow-live/bin/gobgpd" NETDATA_GOBGP="$(pwd)/.local/audits/netflow-live/bin/gobgp" cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml`
822 + - Result: 13 passed, 0 failed.
823 +- `git diff --check` passed.
824 +- Generated-doc noise from unrelated SNMP and trailing blank-line changes was removed from the changeset.
825 +- A process/container cleanup check found no remaining temporary live-validation processes or containers.
826 +
827 +Artifact maintenance:
828 +
829 +- AGENTS.md: no update needed; existing SOW, sensitive-data, and generated-doc rules were sufficient.
830 +- Runtime project skills: no update needed; `project-writing-collectors`, `integrations-lifecycle`, and `mirrored-repos` already cover the workflow.
831 +- Specs: no update needed; this corrected public/operator documentation to match existing implementation and upstream protocol reality.
832 +- End-user/operator docs: updated BioRIS metadata, generated integration docs, and hand-authored network-flow docs.
833 +- End-user/operator skills: no update needed; no public skill workflow changed.
834 +- SOW lifecycle: reopened from `done`, recorded external validation, then returned to `completed` and moved back to `.agents/sow/done/` with the validation/doc commit.
835 +
836 +Additional proof outside the clarified beta validation bar:
837 +
838 +- A real stable bio-rd-compatible `RoutingInformationService` endpoint with a non-empty RIB would expand proof beyond Netdata's client contract into production-style BioRIS interoperability.
839 +- A real router/BMP source would expand proof beyond deterministic BMP frames, GoBGP route publication, and FRR config/connection validation into vendor-router interoperability.
840 +- Debugging, reporting, or fixing upstream bio-rd behavior would be useful only if this beta work is expanded into upstream bio-rd stabilization.
841 +- User-owned NetBox or custom IPAM endpoints would expand proof beyond local NetBox and local generic IPAM validation into user-specific environment validation.
842 +
843 +These are not blockers under the clarified beta validation target. The final autonomous work in this pass strengthened Netdata-side automated coverage for configuration, payload/schema consumption, field mapping, option handling, and stability.
844 +
845 +### Netdata-side Contract Hardening - 2026-05-08
846 +
847 +Trigger:
848 +
849 +- The user clarified that this beta validation does not need to prove every external routing stack is production-stable.
850 +- The required bar is to prove Netdata's side is sane: valid configs connect, bad configs are rejected or reported clearly, expected payloads are consumed, fields map correctly into enrichment state, options are respected, no obvious panic/hang/task leak/silent ignore exists, and user docs describe working commands/configs.
851 +
852 +Implementation:
853 +
854 +- BioRIS:
855 + - Added endpoint URI contract coverage for explicit schemes and `grpc_secure`.
856 + - Added invalid-endpoint coverage proving malformed gRPC URIs produce a clear `invalid BioRIS endpoint URI` error before network dialing.
857 + - Added router IP parsing coverage for plain IPs and socket-address forms.
858 + - Extended the in-process `RoutingInformationService` fixture so tests record `DumpRIB` and `ObserveRIB` requests.
859 + - Added coverage proving configured `vrf_id` and `vrf` are sent to `DumpRIB` and `ObserveRIB`.
860 + - Added coverage proving `DumpRIB` and `ObserveRIB` advertisements publish AS path, communities, and large communities into the runtime trie.
861 + - Added coverage proving `ObserveRIB` withdrawals remove the more-specific route and fall back to the broader route.
862 +- BMP:
863 + - Added deterministic `apply_update` tests proving `collect_asns`, `collect_as_paths`, and `collect_communities` are respected.
864 + - Added deterministic update/withdrawal tests proving AS number, AS path, communities, large communities, next hop, and route deletion map correctly into the runtime trie.
865 +- Config validation:
866 + - Added coverage rejecting enabled BioRIS instances with `grpc_addr` that has neither `host:port` nor an explicit URI scheme.
867 + - Added coverage rejecting zero `timeout`, `refresh`, and `refresh_timeout` values for enabled BioRIS.
868 +
869 +Validation:
870 +
871 +- `cargo test -p netflow-plugin routing::bioris --manifest-path src/crates/Cargo.toml`
872 + - Result: 11 passed, 0 failed.
873 +- `cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml`
874 + - Result: 15 passed, 0 failed, including the opt-in GoBGP route publication test because local GoBGP binaries are present.
875 +- `cargo test -p netflow-plugin plugin_config --manifest-path src/crates/Cargo.toml`
876 + - Result: 30 passed, 0 failed.
877 +- `cargo test -p netflow-plugin --manifest-path src/crates/Cargo.toml`
878 + - Result: 456 passed, 18 ignored, 0 failed.
879 + - `tests/grpc_build.rs`: 1 passed, 0 failed.
880 +- `git diff --check` passed.
881 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on the existing unmodified `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern that the audit classifies as email-like sensitive data.
882 +
883 +End-user documentation check:
884 +
885 +- No end-user docs were changed in this hardening pass.
886 +- Previous BioRIS product docs already describe BioRIS as a bio-rd-compatible `RoutingInformationService` gRPC endpoint and do not describe internal test gaps.
887 +- A docs search was run for internal test-gap phrases across `docs/network-flows`, `src/crates/netflow-plugin/metadata.yaml`, and generated NetFlow integration pages. No product-doc statement was found saying BioRIS or BMP are untested or explaining why internal testing could not be performed.
888 +
889 +Artifact maintenance:
890 +
891 +- AGENTS.md: no update needed; existing SOW, sensitive-data, and validation rules were sufficient.
892 +- Runtime project skills: no update needed; `project-writing-collectors` and `integrations-lifecycle` already cover this workflow.
893 +- Specs: no update needed; this hardening adds tests for existing intended contracts, not a new public contract.
894 +- End-user/operator docs: no update needed in this pass; previous product-doc correction remains valid.
895 +- End-user/operator skills: no update needed; no public skill workflow changed.
896 +- SOW lifecycle: reopened from `done`, clarified the beta validation bar, recorded the contract-hardening work, then returned to `completed` and moved back to `.agents/sow/done/` with the hardening commit.
897 +
898 +Outcome:
899 +
900 +- Under the clarified beta validation target, no autonomous Netdata-side blocker remains for BMP or BioRIS contract proof.
901 +- Broader production-router or production-bio-rd interoperability is explicitly outside this beta blocker and can be handled later as expanded validation or beta feedback.
902 +
903 +### PR Review Follow-up - 2026-05-08 - Direct-agent Output Variable Failure Propagation
904 +
905 +Trigger:
906 +
907 +- Copilot opened one unresolved PR review thread on `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh:192`.
908 +
909 +Finding:
910 +
911 +- `_agents_get_claim_id` called `_agents_set_outvar` without checking its exit status.
912 +- Same-pattern search found additional `_agents_set_outvar` and helper-boundary call sites in `_agents_resolve_bearer` and `agents_query_agent`.
913 +- If a caller supplied an invalid output variable name, the helper could continue after an assignment failure and make a public wrapper proceed with an unset or stale claim/bearer value.
914 +
915 +Implementation:
916 +
917 +- Propagated `_agents_set_outvar` failures with `|| return 1` at every call site.
918 +- Propagated `_agents_get_claim_id` failure from `_agents_resolve_bearer`.
919 +- Propagated `_agents_resolve_bearer` failure from `agents_query_agent`.
920 +- Extended `agents_selftest_no_token_leak` with an invalid-output-variable path proving `_agents_get_claim_id` returns failure when `_agents_set_outvar` rejects the caller-provided variable name.
921 +
922 +Validation:
923 +
924 +- `bash -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
925 +- `zsh -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
926 +- `shellcheck --external-sources docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh` passed.
927 +- Same-pattern search confirmed all `_agents_set_outvar` call sites now propagate failure.
928 +- `git diff --check` passed.
929 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on the existing unmodified `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern that the audit classifies as email-like sensitive data.
930 +
931 +Artifact maintenance:
932 +
933 +- AGENTS.md: no update needed; existing public-skill and sensitive-data rules were sufficient.
934 +- Runtime project skills: no update needed.
935 +- Specs: no update needed; this repaired helper failure propagation for an existing public-skill contract.
936 +- End-user/operator docs: no separate docs update needed; the script behavior now matches the existing token-safe helper contract.
937 +- End-user/operator skills: updated `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`.
938 +- SOW lifecycle: reopened from `done`, recorded this PR review follow-up, then returned to `completed` and moved back to `.agents/sow/done/` with the fix commit.
939 +
940 +### PR Review Follow-up - 2026-05-08 - Deterministic CAIDA Prefix2AS Resolution
941 +
942 +Trigger:
943 +
944 +- Copilot's refreshed review generated no new inline comments, but its review body listed a low-confidence note about `resolveCAIDAPrefix2ASURL` choosing the last `.pfx2as.gz` log entry.
945 +
946 +Findings:
947 +
948 +- The `parse.go` low-confidence note about missing `netipx` import was false; `src/go/tools/topology-ip-intel-downloader/parse.go` already imports `go4.org/netipx`.
949 +- The CAIDA note was valid. The resolver walked the creation log and retained the last matching candidate, so an out-of-order log could select an older dataset.
950 +- The live CAIDA creation log uses tab-separated rows with a numeric timestamp before the `.pfx2as.gz` path, so the resolver can select deterministically by timestamp and tie-break by path.
951 +
952 +Implementation:
953 +
954 +- Changed `resolveCAIDAPrefix2ASURL` to collect all `.pfx2as.gz` candidates.
955 +- Sorted candidates by parsed numeric timestamp when available, with path as a deterministic fallback/tie-breaker.
956 +- Updated `TestResolveCAIDAPrefix2ASURL` so an older candidate appears after the newer candidate, proving the resolver no longer depends on log order.
957 +
958 +Validation:
959 +
960 +- `curl -fsSL https://data.caida.org/datasets/routing/routeviews-prefix2as/pfx2as-creation.log | tail -n 20` confirmed the current CAIDA log row shape includes a numeric timestamp and path ending in `.pfx2as.gz`.
961 +- `go test ./tools/topology-ip-intel-downloader` from `src/go` passed.
962 +- `git diff --check` passed.
963 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on the existing unmodified `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern that the audit classifies as email-like sensitive data.
964 +
965 +Artifact maintenance:
966 +
967 +- AGENTS.md: no update needed; existing PR review and SOW lifecycle rules were sufficient.
968 +- Runtime project skills: no update needed; this did not change how future agents should work.
969 +- Specs: no update needed; this hardens an existing downloader behavior without changing the public contract.
970 +- End-user/operator docs: no update needed; no user-facing command, option, or provider contract changed.
971 +- End-user/operator skills: no update needed.
972 +- SOW lifecycle: reopened from `done`, recorded this PR review follow-up, then returned to `completed` for the deterministic CAIDA resolver commit.
973 +
974 +### PR Review Follow-up - 2026-05-08 - Helper Comment And Parser Preallocation
975 +
976 +Trigger:
977 +
978 +- Copilot opened a new unresolved PR review thread on `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh:172`.
979 +- The same review body listed low-confidence notes about fixed `1<<20` and `1<<18` parser preallocations in `src/go/tools/topology-ip-intel-downloader/parse.go`.
980 +
981 +Findings:
982 +
983 +- The shell helper logic works under both bash and zsh in local smoke tests, but the source comment over-explained caller-local dynamic scoping and made the zsh support claim look broader than necessary.
984 +- Same-pattern search found no other `Bash and zsh`, `dynamic scope`, or `dynamic scoping` wording in the direct-agent skill helper.
985 +- Same-pattern search found all remaining fixed large range-slice preallocations in `parse.go`.
986 +- The parser preallocation concern was valid: small payloads and tests paid immediate allocation cost for capacities intended for full datasets.
987 +
988 +Implementation:
989 +
990 +- Tightened the `_agents_set_outvar` comment to document only the `eval` avoidance reason.
991 +- Added `estimatedRangeCapacity` for bounded initial capacity based on payload or zip-entry size.
992 +- Replaced fixed `1<<20` and `1<<18` parser preallocations with size-based estimates where useful, or natural growth where no reliable row-count proxy exists.
993 +- Added unit coverage for the capacity-estimation helper.
994 +
995 +Validation:
996 +
997 +- Same-pattern search found no remaining fixed `make([]asnRange|[]geoRange, 0, 1<<...)` allocations in `parse.go`.
998 +- Same-pattern search found no remaining direct-agent helper comments claiming bash/zsh dynamic scoping.
999 +- `go test ./tools/topology-ip-intel-downloader` from `src/go` passed.
1000 +- `bash -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
1001 +- `zsh -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
1002 +- `shellcheck --external-sources docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh` passed.
1003 +
1004 +Artifact maintenance:
1005 +
1006 +- AGENTS.md: no update needed; existing PR review and SOW lifecycle rules were sufficient.
1007 +- Runtime project skills: no update needed.
1008 +- Specs: no update needed; this is implementation hardening and comment clarification for existing behavior.
1009 +- End-user/operator docs: no update needed; no user-facing command, option, or provider contract changed.
1010 +- End-user/operator skills: updated `docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh`.
1011 +- SOW lifecycle: reopened from `done`, recorded this PR review follow-up, then returned to `completed` for the helper-comment and parser-preallocation commit.
1012 +
1013 +### PR Review Follow-up - 2026-05-08 - Downloader URL, Archive, ZIP, And GoBGP Live-Test Hardening
1014 +
1015 +Trigger:
1016 +
1017 +- Copilot's refreshed review generated no new inline comments, but its review body listed low-confidence notes about:
1018 + - MaxMind ASN tar detection relying on the fixed `ustar` marker.
1019 + - CAIDA prefix2as URL construction using fixed suffix trimming and string concatenation.
1020 + - ZIP entry lookup using OS-specific `filepath.Base`.
1021 + - The optional GoBGP BMP live test reserving and then releasing an API port before starting `gobgpd`.
1022 +
1023 +Findings:
1024 +
1025 +- The downloader notes were valid hardening opportunities:
1026 + - `extractMMDBFromTar` did reject tar payloads before attempting tar iteration unless the `ustar` marker was present.
1027 + - `resolveCAIDAPrefix2ASURL` did build the resolved URL by trimming `pfx2as-creation.log` from the full URL string.
1028 + - ZIP members use slash-separated paths, so `path.Base` is the correct package for archive member names.
1029 +- The GoBGP note was a real race in theory. The test is opt-in and local-only, but a retry loop keeps the validation from becoming flaky on shared machines.
1030 +
1031 +Implementation:
1032 +
1033 +- Changed CAIDA prefix2as URL resolution to parse `logURL`, remove query/fragment, normalize the directory path, and resolve the selected candidate with `url.ResolveReference`.
1034 +- Changed MaxMind ASN tar extraction to attempt tar iteration first, while still treating corrupt tar-like payloads with a `ustar` marker as extraction errors instead of silently accepting them as raw MMDB bytes.
1035 +- Added test coverage for CAIDA log URLs with query strings and nested paths.
1036 +- Added test coverage for gzipped legacy tar payloads without the `ustar` marker.
1037 +- Switched archive member base-name handling from `filepath.Base` to `path.Base`.
1038 +- Added retry behavior around the optional GoBGP API port reservation in the BMP live test.
1039 +
1040 +Validation:
1041 +
1042 +- `go test ./tools/topology-ip-intel-downloader` from `src/go` passed.
1043 +- `NETDATA_GOBGPD="$(pwd)/.local/audits/netflow-live/bin/gobgpd" NETDATA_GOBGP="$(pwd)/.local/audits/netflow-live/bin/gobgp" cargo test -p netflow-plugin routing::bmp --manifest-path src/crates/Cargo.toml` passed with 15 tests.
1044 +- `bash -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
1045 +- `zsh -c 'source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh; agents_selftest_no_token_leak'` passed.
1046 +- `shellcheck --external-sources docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh` passed.
1047 +
1048 +Artifact maintenance:
1049 +
1050 +- AGENTS.md: no update needed; existing PR review and SOW lifecycle rules were sufficient.
1051 +- Runtime project skills: no update needed.
1052 +- Specs: no update needed; this is implementation hardening for existing downloader and test behavior.
1053 +- End-user/operator docs: no update needed; no user-facing command, option, or provider contract changed.
1054 +- End-user/operator skills: no additional update needed beyond the direct-agent helper comment from the previous follow-up.
1055 +- SOW lifecycle: reopened from `done`, recorded this PR review follow-up, then returned to `completed` for the downloader and BMP live-test hardening commit.
1056 +
1057 +### PR Review Follow-up - 2026-05-08 - Intel Downloader Provider Documentation
1058 +
1059 +Trigger:
1060 +
1061 +- Copilot's refreshed review body listed a low-confidence note about `netipx` import status and a note about `maxmind:geolite2-country@csv` being a ZIP bundle rather than a single raw CSV file.
1062 +- The user asked whether the PR was ready to merge and approved fixing the remaining documentation mismatch first.
1063 +
1064 +Findings:
1065 +
1066 +- The `netipx` import note was false. `src/go/tools/topology-ip-intel-downloader/parse.go` imports `go4.org/netipx`, and the downloader package test passes.
1067 +- The MaxMind `format: csv` note exposed a real user-documentation issue. `docs/network-flows/intel-downloader.md` still listed only DB-IP and IPtoASN, and still said MaxMind was not supported by the downloader.
1068 +- The current code supports DB-IP, IPtoASN, CAIDA prefix2as, MaxMind GeoLite2 ASN, MaxMind GeoLite2 Country, IP2Location, IPDeny, and IPIP through the downloader.
1069 +- MaxMind GeoLite2 Country and IP2Location country-lite use provider CSV ZIP bundles. For MaxMind Country, `format: csv` means the official GeoLite2 Country CSV ZIP bundle with locations plus IPv4/IPv6 block CSVs; it does not mean a single raw CSV file.
1070 +
1071 +Implementation:
1072 +
1073 +- Updated `docs/network-flows/intel-downloader.md` to list all supported provider/artifact combinations.
1074 +- Removed the stale statement saying MaxMind is unsupported by the downloader.
1075 +- Added explicit MaxMind `MAXMIND_LICENSE_KEY` and CSV ZIP bundle wording.
1076 +- Added notes for CAIDA, IP2Location, IPDeny, and IPIP payload shapes.
1077 +- Updated the refresh-cadence wording and per-provider links.
1078 +
1079 +Validation:
1080 +
1081 +- `rg` confirmed the stale "MaxMind unsupported" wording was removed and the new provider entries / MaxMind CSV ZIP clarification are present.
1082 +- `go test ./tools/topology-ip-intel-downloader` from `src/go` passed.
1083 +- `git diff --check` passed.
1084 +- `bash .agents/sow/audit.sh` verified SOW status/directory consistency. It exited nonzero on existing unrelated audit findings, including `.agents/skills/mirror-netdata-repos/SKILL.md:112` Git SSH URL pattern classified as email-like sensitive data.
1085 +
1086 +Artifact maintenance:
1087 +
1088 +- AGENTS.md: no update needed; this work followed the existing PR review and SOW lifecycle rules.
1089 +- Runtime project skills: no update needed.
1090 +- Specs: no update needed; source code and provider metadata already define the current downloader contract.
1091 +- End-user/operator docs: updated `docs/network-flows/intel-downloader.md`.
1092 +- End-user/operator skills: no update needed; no public skill workflow changed.
1093 +- SOW lifecycle: reopened from `done`, recorded this end-user documentation correction, then returned to `completed` for the docs commit.
AGENTS.md
+46 -55
@@ -1,22 +1,39 @@
1 # AGENTS.md
2
3 -This file provides guidance to AI coding agents (Claude Code, Codex CLI, Gemini CLI, Opencode, Qwen-code, Crush, and others) working with code in this repository. The repo-root `CLAUDE.md` and `GEMINI.md` are relative symlinks to this file so every tool reads the same instructions.
3 +## Goals
4
5 -THE MOST IMPORTANT RULES ARE:
5 +This repository is the Netdata Agent codebase. It is a large, multi-language, multi-platform monolith that serves production monitoring, troubleshooting, data collection, alerting, storage, streaming, cloud integration, packaging, and documentation workflows.
6
7 -1. You MUST ALWAYS find the root cause of a problem, before giving a solution.
8 -2. Patching without understanding the problem IS NOT ALLOWED.
9 -3. Before patching code, we MUST understand the code base and the potential implications of our changes.
10 -4. We do not duplicate code. We first check if similar code already exists and to reuse it.
7 +Work in this repository must prioritize root-cause understanding, correctness, performance, maintainability, portability, security, and consistency with existing project conventions.
8
12 -## Goals
9 +CRITICAL RULES:
10
14 -This repository is the Netdata Agent codebase. It is a large, multi-language, multi-platform monolith that serves production monitoring, troubleshooting, data collection, alerting, storage, streaming, cloud integration, packaging, and documentation workflows.
11 +1. You MUST ALWAYS find the root cause of a problem, before offering/giving a solution.
12 + Patching without understanding the problem IS NOT ALLOWED.
13 +
14 +2. Before patching code, you MUST understand the codebase and the potential implications of the changes.
15 + What else is affected? What else is using this part of the code?
16 +
17 +3. Do not duplicate code.
18 + First check if similar code already exists and reuse it.
19 +
20 +USER COMMUNICATION:
21 +
22 +1. ALWAYS DO YOUR HOMEWORK BEFORE ASKING QUESTIONS OR REQUESTING USER DECISIONS.
23 + PROACTIVELY CHECK ALL RELATED ASPECTS AND ALL POSSIBILITIES SO THAT YOUR QUESTIONS AND REQUESTS ARE WELL INFORMED AND TO THE POINT.
24 +
25 +2. NEVER WRITE WALLS OF TEXT TO THE USER, UNLESS THEY ASKED FOR IT.
26 + YOUR COMMUNICATION MUST BE SIMPLE, DIRECT, LEAN, ORDERED BY IMPORTANCE.
27 + PROVIDE THE FULL PICTURE AT THE BEGINNING, START FROM THE HIGH LEVEL, AND LET THE USER ASK FOR DETAILS.
28
16 -Work in this repository must prioritize root-cause understanding, correctness, performance, maintainability, portability, security, and consistency with existing project conventions. Because the project is too broad for one bootstrap pass, SOW coverage grows incrementally by the area being worked.
29 +3. NEVER AGREE TO THE USER WHEN THE FACTS CONTRADICT THEIR UNDERSTANDING.
30 + YOU MUST ALWAYS PROVIDE CLEAR DESCRIPTIONS OF THE RISKS AND IMPLICATIONS OF THEIR DECISIONS.
31 + YOU ARE HELPFUL WHEN YOU ACCURATELY REVEAL THE TRUTH, NOT WHEN YOU AGREE.
32
33 ## SOW System
34
35 +Project SOW status: initialized
36 +
37 This project uses a local Statement of Work system.
38
39 The SOW system is self-contained in this repository. Normal SOW work must not depend on `~/.agents`, `~/.AGENTS.md`, global skills, global templates, or global scripts. Use this `AGENTS.md`, project-local SOW files, project-local specs, project-local skills, and the active SOW.
@@ -32,7 +49,7 @@ Before non-trivial work:
49
50 1. Read pending/current SOWs for overlap, contradictions, and existing decisions.
51 2. Read relevant specs under `.agents/sow/specs/`.
35 -3. Inspect `.agents/skills/project-*/SKILL.md` if any exist, and load every runtime project skill whose trigger matches the work.
52 +3. Inspect `.agents/skills/*/SKILL.md` if any exist, and load every runtime project skill whose trigger matches the work.
53 4. Inspect legacy runtime skills listed below when the user request matches their frontmatter trigger.
54 5. Inspect code, docs, tests, and existing project instructions as ground truth.
55 6. Ask the user only for irreducible product/design/risk decisions.
@@ -58,7 +75,7 @@ If sensitive data is required to continue, stop and ask the user for a secure ha
75
76 ### Open-Source Reference Evidence
77
61 -When SOW evidence comes from local mirrored open-source repositories under `/opt/baddisk/monitoring/repos/`, cite the upstream repository and checked commit instead of the workstation absolute path.
78 +When SOW evidence comes from other open-source repositories, cite the upstream repository and checked commit instead of the workstation absolute path.
79
80 Use:
81
@@ -67,7 +84,7 @@ owner/repo @ commit
84 relative/path/inside/repo:line
85 ```
86
70 -Resolve `owner/repo` from the repository remote, record the checked commit, and keep paths relative to the upstream repository root. Never write `/opt/baddisk/monitoring/repos/...` paths into SOW evidence.
87 +Resolve `owner/repo` from the repository remote, record the checked commit, and keep paths relative to the upstream repository root. Never write absolute paths into SOW evidence.
88
89 ### Pre-Implementation Gate
90
@@ -255,11 +272,7 @@ Specs describe current reality, not aspiration. If specs and code disagree, reco
272
273 Project skills are memory of HOW to work here.
274
258 -Runtime input project skills should live under `.agents/skills/project-*/SKILL.md`. The `project-` prefix is the generic hook meaning "agents working in this repo must consider this skill." Before non-trivial work, inspect those skill descriptions and load every matching runtime skill.
259 -
260 -Do not create generic `project-*` skills only to make the framework look complete. The user requested that project skills for this repository grow incrementally.
261 -
262 -Existing non-`project-*` skills under `.agents/skills/` are preserved as legacy runtime skills. Use them when the request matches their frontmatter trigger. Do not rename them or add wrappers during this bootstrap pass.
275 +Runtime input project skills should live under `.agents/skills/*/SKILL.md`. Before non-trivial work, inspect those skill descriptions and load every matching runtime skill.
276
277 Output/reference skills may also exist under product documentation or generated skill directories. Do not rename, shorten, or change their descriptions only to satisfy runtime discovery. Update them when their related public/operator workflow changes.
278
@@ -269,7 +282,7 @@ End-user-facing AI skills under `docs/netdata-ai/skills/` follow the directory s
282
283 Each public skill is reachable from `.agents/skills/<skill-name>` via a relative symlink (`.agents/skills/<name>` → `../../docs/netdata-ai/skills/<name>`) so local AI assistants reading from `.agents/skills/` see the same skill as end-users. Create the symlink with `ln -srfn`. Verify with `readlink -f .agents/skills/<name>`.
284
272 -Public-skill scripts must follow the same `_lib.sh` shape as the legacy private skills (`set -euo pipefail`, ANSI colors with real ESC bytes via `$'\033[...]'`, `<prefix>_repo_root` via `git rev-parse --show-toplevel`, `<prefix>_load_env` that sources `<repo>/.env` with `: "${VAR:?}"` validation, `<prefix>_audit_dir` that creates `<repo>/.local/audits/<topic>/`, masked-token `<prefix>_run`/`<prefix>_run_read` wrappers).
285 +Public-skill scripts must follow the same `_lib.sh` shape as existing skills (`set -euo pipefail`, ANSI colors with real ESC bytes via `$'\033[...]'`, `<prefix>_repo_root` via `git rev-parse --show-toplevel`, `<prefix>_load_env` that sources `<repo>/.env` with `: "${VAR:?}"` validation, `<prefix>_audit_dir` that creates `<repo>/.local/audits/<topic>/`, masked-token `<prefix>_run`/`<prefix>_run_read` wrappers).
286
287 Public-skill scripts that touch credentials (cloud tokens, per-agent bearers, claim ids, session cookies) MUST be **token-safe** -- helpers that handle credential bytes are named with a leading underscore (`_skill_*`, internal-only) and return them via bash namerefs into the caller's local variables, NEVER to stdout. Public wrappers (no leading underscore) read credentials from `.env` internally and emit ONLY the response body. Each token-handling lib must ship a `<prefix>_selftest_no_token_leak` function that drives every public wrapper with a sentinel token and asserts the sentinel never appears on captured stdout.
288
@@ -277,7 +290,7 @@ Public-skill scripts that touch credentials (cloud tokens, per-agent bearers, cl
290
291 Each public skill ships a `how-tos/` subdirectory with `INDEX.md`. The catalog is **live**: every time an AI assistant is asked a concrete question that requires analysis (multiple wrapper calls, jq pipelines, or cross-referencing more than one per-domain guide) and the answer isn't already documented under `how-tos/`, the assistant MUST author a new how-to and add it to `INDEX.md` BEFORE completing the task. This rule is repeated in each skill's `SKILL.md` so future assistants honor it. Skipping it means the next assistant repeats the same analysis from scratch -- an explicit framework violation.
292
280 -The legacy private skills (`coverity-audit`, `sonarqube-audit`, `graphql-audit`, `pr-reviews`) keep their `.agents/skills/<name>/` location -- they are intentionally private and have no `docs/netdata-ai/skills/` counterpart.
293 +The existing private skills (`coverity-audit`, `sonarqube-audit`, `graphql-audit`, `pr-reviews`) keep their `.agents/skills/<name>/` location -- they are intentionally private and have no `docs/netdata-ai/skills/` counterpart.
294
295 ### Project Skills Index
296
@@ -286,12 +299,19 @@ Runtime input skills:
299 - `.agents/skills/project-snmp-profiles-authoring/`
300 Trigger: editing SNMP profile YAMLs, topology SNMP profiles, ddsnmp profile parsing, or SNMP profile-format documentation.
301 Purpose: require MIB `MAX-ACCESS` checks and index-derived extraction for `not-accessible` INDEX objects.
302 +
303 - `.agents/skills/project-writing-collectors/`
304 Trigger: authoring or modifying any Netdata data-collection plugin or module (Go go.d / ibm.d, Rust crates, internal C plugins, external plugins via PLUGINSD). Read before adding a new collector, modifying an existing one, working on NetFlow/sFlow/IPFIX, OTEL ingestion, topology, SNMP profiles, or interactive Functions.
305 Status: live. Updates that close gaps or fix outdated pointers must ship in the same PR that exposed the issue.
306 +
307 +- `.agents/skills/project-writing-go-modules-framework-v2/`
308 + Trigger: creating or migrating a Go go.d collector to framework V2; touching `CollectorV2`, `metrix.CollectorStore`, `ChartTemplateYAML` / `charts.yaml`, `charttpl`, `chartengine`, V2 host scopes, or V2 collector tests.
309 + Purpose: mirror maintainer-preferred framework V2 patterns from accepted collectors so new or migrated modules blend with repository style.
310 +
311 - `.agents/skills/integrations-lifecycle/`
312 Trigger: editing any `metadata.yaml`; modifying `integrations/` generators, schemas, or templates; working with `integrations.js` / `integrations.json` / per-integration `.md` files / `COLLECTORS.md` / `SECRETS.md` / `SERVICE-DISCOVERY.md`; ibm.d module generation (`contexts.yaml` -> `metadata.yaml`); CI workflows `generate-integrations.yml` and `check-markdown.yml`; the five-file collector-consistency rule.
313 Status: live. SKILL.md plus per-domain guides (`pipeline.md`, `schema-reference.md`, `per-type-matrix.md`, `artifacts-and-banners.md`, `ibm-d.md`, `consistency.md`, `in-app-contract.md`, `gotchas.md`) and `recipes/`, `how-tos/` directories.
314 +
315 - `.agents/skills/learn-site-structure/`
316 Trigger: adding/moving/renaming/deleting any docs page that should appear on `learn.netdata.cloud`; editing `<repo>/docs/.map/map.yaml`; investigating why a Learn page looks the way it does; reading the live `ingest/ingest.py` orchestrator or the legacy `ingest.js` / `ingest.md` (which are stale); MDX escape rules; redirects; the Netlify deploy contract.
317 Status: live. SKILL.md plus per-domain guides (`mapping.md`, `pipeline.md`, `sidebars.md`, `mdx-rules.md`, `redirects.md`, `pitfalls-and-gotchas.md`, `authoring-boundary.md`) and `recipes/`, `how-tos/` directories.
@@ -301,24 +321,27 @@ Runtime input skills:
321 - `.agents/skills/query-agent-events/`
322 Trigger: investigating crashes, panics, or fatals across the Netdata fleet; downloading events from the agent-events ingestion namespace; analyzing AE_* fields and their enums; understanding the 23h client-side dedup or the after-the-fact event timing; using the systemd-journal Function multi-value `selections` filter for index-friendly queries.
323 Status: live. SKILL.md plus per-domain guides (`AE_FIELDS.md`, `transports.md`, `update-cadence.md`, `query-discipline.md`, `finding-crashes.md`, `finding-fatals.md`), scripts (`scripts/_lib.sh`, `get-events.sh`, `analyze-events.sh`, `redact-events.sh`) and `recipes/`, `how-tos/` directories. Bug-investigation tool, NOT a generic logs query skill -- consumes `query-netdata-{cloud,agents}` for transport.
324 +
325 - `.agents/skills/mirror-netdata-repos/`
326 Trigger: setting up or updating a local mirror of Netdata-org source repositories at `${NETDATA_REPOS_DIR}` for cross-repo grep / code review without GitHub API calls; running the vendored sync script; questions about the reset-to-default-branch safety mechanism or the `--repo NAME` scoping flag.
327 Status: live. SKILL.md (single-file overview) plus the vendored `scripts/sync-netdata-repos.sh` (env-driven, sanitized, `--repo` scoping, `gh` optional for Phase 2) and `how-tos/` catalog. Independent from any other repo mirrors this workstation may have.
328
308 -Legacy runtime skills:
309 -
329 - `.agents/skills/coverity-audit/`
330 Trigger: Coverity Scan defect triage for this repository.
331 Status: preserved under legacy name; project-skill alignment is deferred and tracked by `.agents/sow/pending/SOW-0003-20260501-legacy-runtime-skill-alignment.md`.
332 +
333 - `.agents/skills/sonarqube-audit/`
334 Trigger: SonarCloud findings triage for this repository.
335 Status: preserved under legacy name; project-skill alignment is deferred and tracked by `.agents/sow/pending/SOW-0003-20260501-legacy-runtime-skill-alignment.md`.
336 +
337 - `.agents/skills/graphql-audit/`
338 Trigger: GitHub Code Scanning/CodeQL triage for this repository.
339 Status: preserved under legacy name; project-skill alignment is deferred and tracked by `.agents/sow/pending/SOW-0003-20260501-legacy-runtime-skill-alignment.md`.
340 +
341 - `.agents/skills/pr-reviews/`
342 Trigger: PR comment and review iteration work for this repository.
343 Status: preserved under legacy name; project-skill alignment is deferred and tracked by `.agents/sow/pending/SOW-0003-20260501-legacy-runtime-skill-alignment.md`.
344 +
345 - `.agents/skills/codacy-audit/`
346 Trigger: Codacy Cloud workflow for this repository -- pre-push local analysis (`codacy-analysis-cli` via docker or local binary) and read-only PR-issue fetching via the v3 API.
347 Status: live. SKILL.md plus `scripts/_lib.sh` (token-safe wrappers + sentinel no-leak self-test), `scripts/analyze-local.sh`, `scripts/pr-issues.sh`, and a live `how-tos/INDEX.md` catalog. Read-only by design; write actions deferred to a future SOW.
@@ -329,6 +352,7 @@ Public skills (canonical under `docs/netdata-ai/skills/<name>/`; relative symlin
352 Trigger: querying Netdata Cloud REST API -- metrics, logs (systemd-journal), alerts, generic Function calls on a node.
353 Symlink: `.agents/skills/query-netdata-cloud` -> `../../docs/netdata-ai/skills/query-netdata-cloud`.
354 Status: live. SKILL.md plus per-domain guides (`query-metrics.md`, `query-logs.md`, `query-alerts.md`, `query-functions.md`).
355 +
356 - `docs/netdata-ai/skills/query-netdata-agents/`
357 Trigger: querying Netdata Agents directly on port 19999, including auto-mint of per-agent bearer tokens from a Cloud token.
358 Symlink: `.agents/skills/query-netdata-agents` -> `../../docs/netdata-ai/skills/query-netdata-agents`.
@@ -339,6 +363,7 @@ Output/reference skills:
363 - `docs/netdata-ai/skills/`
364 Consumer: downstream assistants and users of Netdata AI skill artifacts.
365 Update when: public/operator AI skill docs, examples, commands, schemas, or workflows change.
366 +
367 - `src/ai-skills/`
368 Consumer: downstream assistants and users of generated or source AI skill artifacts when this tree is present in the working copy.
369 Update when: generated/source AI skill behavior, tests, examples, commands, schemas, or workflows change.
@@ -402,38 +427,6 @@ These files MUST be consistent with each other. For example:
427 - "`netdata`" (lowercase, code-formatted) when referring to the process
428 - See DICTIONARY.md for precise terminology
429
405 -## AI agent skills
406 -
407 -Repo-scoped skills for AI agents live under `.agents/skills/<skill-name>/`.
408 -Each skill is self-contained: a `SKILL.md` with frontmatter (`name`,
409 -`description`) plus its own `scripts/` directory. Skills carry the operational
410 -knowledge for tasks that recur across sessions (Coverity triage, SonarCloud
411 -triage, GitHub Code Scanning triage, etc.).
412 -
413 -These existing skills are legacy runtime skills, not SOW-generated `project-*`
414 -skills. They are preserved under their original names during the incremental
415 -SOW bootstrap.
416 -
417 -When an agent learns something new while running a skill (a new gotcha, a
418 -working API call, a corrected workflow) it MUST update the skill's
419 -`SKILL.md` and commit it before proceeding. Knowledge that isn't committed
420 -is lost.
421 -
422 -Currently available skills:
423 -- `.agents/skills/coverity-audit/` - Coverity Scan defect triage
424 -- `.agents/skills/sonarqube-audit/` - SonarCloud findings triage
425 -- `.agents/skills/graphql-audit/` - GitHub Code Scanning (CodeQL) triage
426 -- `.agents/skills/pr-reviews/` - PR comment / review iteration loop
427 -- `.agents/skills/codacy-audit/` - Codacy Cloud pre-push analysis + PR-issue triage
428 -
429 -### Preservation Notes
430 -
431 -- The pre-SOW `AGENTS.md` was copied to `AGENTS.md.pre-sow.bak` before this merge.
432 -- Existing top-level rules, collector consistency requirements, C code notes, naming conventions, local-only directory rules, and `.env` secret rules were preserved.
433 -- Existing non-`project-*` operational skills were preserved under their current paths.
434 -- No `project-*` skills were created during bootstrap by user request.
435 -- Existing root `TODO*.md` files were preserved in place and are tracked by a pending SOW for future classification.
436 -
430 ## Local-only working directory
431
432 `/.local/` at the repo root is gitignored and reserved for per-user runtime
@@ -471,5 +464,3 @@ the keys you need.
464 covering every key -- what it is, where to find the value, sample
465 format, common mistakes, and which skills require it. When a script
466 errors with `<KEY> is empty`, check `.agents/ENV.md` for that key.
474 -
475 -Project SOW status: initialized
docs/netdata-ai/skills/query-netdata-agents/SKILL.md
+5 -4
@@ -71,8 +71,8 @@ implementation.
71 5. **Bearer values stay in `.env` and `.local/`.** The bearer
72 cache file at `<repo>/.local/audits/query-netdata-agents/
73 bearers/<machine_guid>.json` is mode 0600 and gitignored. The
74 - internal helper `_agents_resolve_bearer` returns it via bash
75 - nameref, never to stdout.
74 + internal helper `_agents_resolve_bearer` returns it through a
75 + validated caller-local output variable, never to stdout.
76 6. **For bearer-protected agents, default to the Cloud-token
77 flow** in this skill (it auto-mints + caches the bearer).
78
@@ -256,8 +256,9 @@ agents_call_function \
256 ### Internal helpers (do NOT call directly)
257
258 These start with `_` and operate on token bytes inside their own
259 -scope. They return token data via bash namerefs (so the assistant
260 -never sees them on stdout). Don't shell-out to them.
259 +scope. They return token data through validated caller-local output
260 +variables (so the assistant never sees them on stdout). Don't
261 +shell-out to them.
262
263 | Internal | Purpose |
264 |---|---|
docs/netdata-ai/skills/query-netdata-agents/how-tos/INDEX.md
+1
@@ -80,6 +80,7 @@ how-to is written.)
80
81 - `topology-summary-direct.md` (stub -- not yet authored)
82 - `flows-top-talkers-direct.md` (stub -- not yet authored)
83 +- [validate-direct-local-flow-function.md](./validate-direct-local-flow-function.md) -- prove a local Cloud-connected `flows:netflow` Function works through a Cloud-minted direct-agent bearer.
84
85 ### Metrics
86
docs/netdata-ai/skills/query-netdata-agents/how-tos/validate-direct-local-flow-function.md new
+107
@@ -0,0 +1,107 @@
1 +# Validate a local flow Function through direct-agent bearer auth
2 +
3 +## Question
4 +
5 +How can an assistant prove that a local Cloud-connected Netdata Agent
6 +accepts a Cloud-minted per-agent bearer and serves `flows:netflow`
7 +directly, without exposing Cloud tokens, agent bearers, node ids,
8 +machine GUIDs, claim ids, or raw flow rows?
9 +
10 +## Inputs
11 +
12 +- Local agent URL, usually `http://127.0.0.1:19999`.
13 +- `NETDATA_CLOUD_TOKEN` and `NETDATA_CLOUD_HOSTNAME` in `<repo>/.env`.
14 +- The local agent must be connected to Cloud and expose `flows:netflow`.
15 +
16 +## Steps
17 +
18 +1. Capture the local identity tuple in memory and print only presence
19 + checks:
20 +
21 + ```bash
22 + INFO_JSON="$(curl -sS --max-time 10 http://127.0.0.1:19999/api/v3/info)"
23 +
24 + jq '{
25 + agent_count: (.agents | length),
26 + node_id_present: ((.agents[0].nd // "") | length > 0),
27 + machine_guid_present: ((.agents[0].mg // "") | length > 0),
28 + claim_id_present: ((.agents[0].cloud.claim_id // "") | length > 0),
29 + cloud_status: .agents[0].cloud.status
30 + }' <<<"$INFO_JSON"
31 + ```
32 +
33 +2. Load the token-safe direct-agent wrappers:
34 +
35 + ```bash
36 + source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh
37 + agents_load_env
38 + ```
39 +
40 +3. Call the flow Function through the direct-agent path:
41 +
42 + ```bash
43 + NODE_UUID="$(jq -r '.agents[0].nd' \
44 + <<<"$INFO_JSON")"
45 + MACHINE_GUID="$(jq -r '.agents[0].mg' \
46 + <<<"$INFO_JSON")"
47 +
48 + mkdir -p .local/audits/query-netdata-agents
49 +
50 + agents_call_function \
51 + --via agent \
52 + --node "$NODE_UUID" \
53 + --host 127.0.0.1:19999 \
54 + --machine-guid "$MACHINE_GUID" \
55 + --function flows:netflow \
56 + --body '{"info":true}' \
57 + > .local/audits/query-netdata-agents/flows-netflow-info-agent.json
58 + ```
59 +
60 +4. Print a sanitized result:
61 +
62 + ```bash
63 + jq '{
64 + status,
65 + type,
66 + has_history,
67 + response_keys: keys
68 + }' .local/audits/query-netdata-agents/flows-netflow-info-agent.json
69 + ```
70 +
71 +## Output
72 +
73 +Expected success shape:
74 +
75 +```json
76 +{
77 + "status": 200,
78 + "type": "flows",
79 + "has_history": true
80 +}
81 +```
82 +
83 +The wrapper logs masked curl commands on stderr. The Cloud token,
84 +per-agent bearer, node id, machine GUID, and claim id must not appear
85 +in stdout or durable artifacts.
86 +
87 +## Notes / gotchas
88 +
89 +- Use the exact `nd`, `mg`, and `cloud.claim_id` tuple from the same
90 + local `/api/v3/info` response. A mixed tuple from a different node,
91 + parent, child, room, or stale cache can produce Cloud or agent
92 + rejection even when the Cloud-proxied Function path works.
93 +- The direct agent uses `X-Netdata-Auth: Bearer <agent-bearer>`, not
94 + `Authorization: Bearer <cloud-token>`.
95 +- The helper caches the raw bearer under
96 + `.local/audits/query-netdata-agents/bearers/`; that directory is
97 + gitignored and should stay mode `0700`, with bearer files mode
98 + `0600`.
99 +- For content validation of flow rows, prefer a grouped query and print
100 + only counts/statistics. Do not paste raw flow rows into durable files.
101 +
102 +## Source guides
103 +
104 +- [Direct-agent skill](../SKILL.md)
105 +- [Direct Function calls](../query-functions.md)
106 +- [Network-flow Functions](../query-flows.md)
107 +- [Cloud flow validation sibling how-to](../../query-netdata-cloud/how-tos/validate-local-netflow-function.md)
docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh
+112 -23
@@ -10,8 +10,8 @@
10 # claim_id to stdout.
11 # * Internal helpers (named `_agents_*`, leading underscore) may
12 # handle token bytes inside their own scope but must return
13 -# them only via `local -n` namerefs into the caller's local
14 -# variables -- never to stdout.
13 +# them only through validated caller-local variable names --
14 +# never to stdout.
15 # * `_agents_log_masked` redacts token / bearer bytes in stderr
16 # argv echoes.
17 # * The unit test `agents_selftest_no_token_leak` drives every
@@ -143,6 +143,14 @@ _agents_log_masked() {
143 # name; the rejoined arg is harmless even if the
144 # leading text is the bare header. Tests cover this.
145 fi
146 + arg="$(printf '%s' "${arg}" | sed -E \
147 + -e 's/(node_id=)[^&]+/\1<NODE_ID>/g' \
148 + -e 's/(machine_guid=)[^&]+/\1<MACHINE_GUID>/g' \
149 + -e 's/(claim_id=)[^&]+/\1<CLAIM_ID>/g' \
150 + -e 's#(/api/v2/nodes/)[0-9a-fA-F-]{36}#\1<NODE_ID>#g' \
151 + -e 's#(/api/v[0-9]+/spaces/)[0-9a-fA-F-]{36}#\1<SPACE_ID>#g' \
152 + -e 's#(/rooms/)[0-9a-fA-F-]{36}#\1<ROOM_ID>#g' \
153 + -e 's#(/host/)[0-9a-fA-F-]{36}#\1<NODE_ID>#g')"
154 printf >&2 '%q ' "${arg}"
155 done
156 printf >&2 '%s\n' "${AGENTS_NC}"
@@ -152,21 +160,35 @@ _agents_log_masked() {
160 # Internal: claim_id / bearer mint / cache
161 # ---------------------------------------------------------------------------
162
163 +_agents_set_outvar() {
164 + local _agents_out_name="${1:?output variable name required}"
165 + local _agents_out_value="${2-}"
166 + if [[ ! "${_agents_out_name}" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then
167 + echo -e "${AGENTS_RED}[ERROR]${AGENTS_NC} Invalid output variable name: ${_agents_out_name}" >&2
168 + return 1
169 + fi
170 + # Avoid eval here because values come from curl/jq output.
171 + if ! printf -v "${_agents_out_name}" '%s' "${_agents_out_value}"; then
172 + echo -e "${AGENTS_RED}[ERROR]${AGENTS_NC} Failed to set output variable: ${_agents_out_name}" >&2
173 + return 1
174 + fi
175 +}
176 +
177 # Resolve claim_id from a node's /api/v3/info. The /info endpoint
156 -# is unauthenticated. INTERNAL: returns via nameref into a caller
157 -# local; never prints to stdout.
178 +# is unauthenticated. INTERNAL: writes to a caller-local variable;
179 +# never prints to stdout.
180 #
181 # Args:
182 # $1 = OUTVAR -- caller-local variable name to receive the claim_id
183 # $2 = HOST -- host:port (e.g. "agent-events:19999")
184 _agents_get_claim_id() {
163 - local -n _out="$1"; shift
185 + local _out_var="${1:?usage: _agents_get_claim_id OUTVAR <host:port>}"; shift
186 local host="${1:?usage: _agents_get_claim_id OUTVAR <host:port>}"
165 - local resp claim
187 + local resp resolved_claim
188 if resp="$(curl -sS --max-time 10 "http://${host}/api/v3/info" 2>/dev/null)"; then
167 - claim="$(jq -r '.agents[0].cloud.claim_id // empty' <<< "${resp}" 2>/dev/null)"
168 - if [[ -n "${claim}" && "${claim}" != "null" ]]; then
169 - _out="${claim}"
189 + resolved_claim="$(jq -r '.agents[0].cloud.claim_id // empty' <<< "${resp}" 2>/dev/null)"
190 + if [[ -n "${resolved_claim}" && "${resolved_claim}" != "null" ]]; then
191 + _agents_set_outvar "${_out_var}" "${resolved_claim}" || return 1
192 return 0
193 fi
194 fi
@@ -207,8 +229,8 @@ _agents_exp_to_seconds() {
229 fi
230 }
231
210 -# Cache-aware bearer resolution. INTERNAL: returns via nameref;
211 -# never prints the bearer to stdout.
232 +# Cache-aware bearer resolution. INTERNAL: writes to a caller-local
233 +# variable; never prints the bearer to stdout.
234 #
235 # Args:
236 # $1 = OUTVAR -- caller-local variable to receive the bearer
@@ -220,7 +242,7 @@ _agents_exp_to_seconds() {
242 # Mode 0600. Stamps `_cached_at` (unix-seconds) so the cache window
243 # survives Cloud responses with expiration=0.
244 _agents_resolve_bearer() {
223 - local -n _out="$1"; shift
245 + local _out_var="${1:?usage: _agents_resolve_bearer OUTVAR <node_id> <machine_guid> <host:port>}"; shift
246 local node_id="${1:?usage: _agents_resolve_bearer OUTVAR <node_id> <machine_guid> <host:port>}"
247 local mg="${2:?machine_guid required}"
248 local host="${3:?host required}"
@@ -248,11 +270,11 @@ _agents_resolve_bearer() {
270 # ~3h-TTL bearers, so 2h leaves a 1h safety margin.
271 if (( exp_s > 0 )); then
272 if (( exp_s - now > 3600 )); then
251 - _out="${cached_token}"
273 + _agents_set_outvar "${_out_var}" "${cached_token}" || return 1
274 return 0
275 fi
276 elif (( cached_at > 0 )) && (( now - cached_at < 7200 )); then
255 - _out="${cached_token}"
277 + _agents_set_outvar "${_out_var}" "${cached_token}" || return 1
278 return 0
279 fi
280 fi
@@ -260,7 +282,7 @@ _agents_resolve_bearer() {
282
283 # Need to mint -- resolve claim_id first.
284 local claim
263 - _agents_get_claim_id claim "${host}"
285 + _agents_get_claim_id claim "${host}" || return 1
286
287 local resp
288 resp="$(_agents_mint_bearer_json "${node_id}" "${mg}" "${claim}")"
@@ -273,7 +295,7 @@ _agents_resolve_bearer() {
295 # Stamp cache and persist.
296 jq --argjson t "${now}" '. + {_cached_at: $t}' <<< "${resp}" > "${cache_file}"
297 chmod 0600 "${cache_file}"
276 - _out="$(jq -r '.token' "${cache_file}")"
298 + _agents_set_outvar "${_out_var}" "$(jq -r '.token' "${cache_file}")" || return 1
299 }
300
301 # ---------------------------------------------------------------------------
@@ -294,13 +316,13 @@ _agents_resolve_bearer() {
316 # agents_query_cloud POST /api/v2/nodes/$NODE/function?function=systemd-journal '{"info":true}'
317 agents_query_cloud() {
318 local method="${1:?usage: agents_query_cloud METHOD PATH [BODY]}"
297 - local path="${2:?path required}"
319 + local api_path="${2:?path required}"
320 local body="${3:-}"
321
322 local args=(curl --fail --silent --show-error --max-time 120 -X "${method}" \
323 -H "Authorization: Bearer ${NETDATA_CLOUD_TOKEN}" \
324 -H 'Content-Type: application/json' \
303 - "https://${NETDATA_CLOUD_HOSTNAME}${path}")
325 + "https://${NETDATA_CLOUD_HOSTNAME}${api_path}")
326 if [[ -n "${body}" ]]; then
327 args+=(-d "${body}")
328 fi
@@ -321,7 +343,7 @@ agents_query_cloud() {
343 # agents_query_agent --node $NODE --host $HOST --machine-guid $MG \
344 # POST /api/v3/function?function=systemd-journal '{"info":true}'
345 agents_query_agent() {
324 - local node="" host="" mg="" method="" path="" body=""
346 + local node="" host="" mg="" method="" api_path="" body=""
347 while (( $# > 0 )); do
348 local arg="$1"
349 case "$arg" in
@@ -337,7 +359,7 @@ agents_query_agent() {
359 esac
360 done
361 method="${1:?usage: agents_query_agent --node N --host H --machine-guid M METHOD PATH [BODY]}"
340 - path="${2:?path required}"
362 + api_path="${2:?path required}"
363 body="${3:-}"
364
365 : "${node:?--node required}"
@@ -345,12 +367,12 @@ agents_query_agent() {
367 : "${mg:?--machine-guid required}"
368
369 local bearer
348 - _agents_resolve_bearer bearer "${node}" "${mg}" "${host}"
370 + _agents_resolve_bearer bearer "${node}" "${mg}" "${host}" || return 1
371
372 local args=(curl --fail --silent --show-error --max-time 120 -X "${method}" \
373 -H "X-Netdata-Auth: Bearer ${bearer}" \
374 -H 'Content-Type: application/json' \
353 - "http://${host}/host/${node}${path}")
375 + "http://${host}/host/${node}${api_path}")
376 if [[ -n "${body}" ]]; then
377 args+=(-d "${body}")
378 fi
@@ -412,6 +434,9 @@ agents_call_function() {
434 agents_selftest_no_token_leak() {
435 local sentinel='UNIQUE_SENTINEL_TOKEN_xK4mP7qR9sT2vW8y'
436 local fake_bearer='deadbeef-1234-5678-9abc-def012345678'
437 + local fake_claim='11111111-2222-3333-4444-555555555555'
438 + local fake_node='22222222-3333-4444-5555-666666666666'
439 + local fake_mg='33333333-4444-5555-6666-777777777777'
440
441 # Save real values, swap in sentinels, run wrappers in dry-run,
442 # capture stdout, restore.
@@ -434,6 +459,9 @@ agents_selftest_no_token_leak() {
459 # 2. _agents_log_masked must mask Bearer <uuid> patterns.
460 out="$(_agents_log_masked curl -H "Authorization: Bearer ${sentinel}" \
461 -H "X-Netdata-Auth: Bearer ${fake_bearer}" \
462 + "https://app.netdata.cloud/api/v2/bearer_get_token?node_id=${fake_node}&machine_guid=${fake_mg}&claim_id=${fake_claim}" \
463 + "https://app.netdata.cloud/api/v3/spaces/${fake_node}/rooms/${fake_mg}/nodes" \
464 + "http://agent.test:19999/host/${fake_node}/api/v3/function?function=flows:netflow" \
465 https://example.invalid 2>&1 1>/dev/null)"
466 if [[ "${out}" == *"${sentinel}"* ]]; then
467 echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_log_masked leaked NETDATA_CLOUD_TOKEN to stderr" >&2
@@ -445,8 +473,69 @@ agents_selftest_no_token_leak() {
473 NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
474 return 1
475 fi
476 + if [[ "${out}" == *"${fake_claim}"* || "${out}" == *"${fake_node}"* || "${out}" == *"${fake_mg}"* ]]; then
477 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_log_masked leaked node identity to stderr" >&2
478 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
479 + return 1
480 + fi
481 +
482 + # 3. _agents_get_claim_id must write through the caller-provided
483 + # output variable even when the caller names the output variable `claim`.
484 + # This mirrors _agents_resolve_bearer and catches local-variable
485 + # shadowing regressions before direct-agent calls need credentials.
486 + local fake_bin_dir old_path claim
487 + fake_bin_dir="$(mktemp -d)"
488 + old_path="${PATH}"
489 + cat > "${fake_bin_dir}/curl" <<EOF
490 +#!/usr/bin/env bash
491 +printf '%s\n' '{"agents":[{"cloud":{"claim_id":"${fake_claim}"}}]}'
492 +EOF
493 + chmod +x "${fake_bin_dir}/curl"
494 + PATH="${fake_bin_dir}:${PATH}"
495 + claim=""
496 + if ! _agents_get_claim_id claim "agent.test:19999"; then
497 + PATH="${old_path}"
498 + rm -rf "${fake_bin_dir}"
499 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_get_claim_id failed with fake agent info" >&2
500 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
501 + return 1
502 + fi
503 + if [[ "${claim}" != "${fake_claim}" ]]; then
504 + PATH="${old_path}"
505 + rm -rf "${fake_bin_dir}"
506 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_get_claim_id did not populate caller output variable" >&2
507 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
508 + return 1
509 + fi
510 + if _agents_get_claim_id "not-a-valid-name" "agent.test:19999" 2>/dev/null; then
511 + PATH="${old_path}"
512 + rm -rf "${fake_bin_dir}"
513 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_get_claim_id ignored invalid output variable failure" >&2
514 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
515 + return 1
516 + fi
517 + PATH="${old_path}"
518 + rm -rf "${fake_bin_dir}"
519 +
520 + # 4. _agents_set_outvar must preserve shell metacharacters as data.
521 + # This protects bearer/claim assignment from accidental eval-style
522 + # interpretation of external command output.
523 + local assigned marker weird_value
524 + marker="unchanged"
525 + weird_value=$'space * ? ; marker=changed $(echo bad) `bad`\nline2 "quote"'
526 + assigned=""
527 + if ! _agents_set_outvar assigned "${weird_value}"; then
528 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_set_outvar failed on metacharacter payload" >&2
529 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
530 + return 1
531 + fi
532 + if [[ "${assigned}" != "${weird_value}" || "${marker}" != "unchanged" ]]; then
533 + echo -e "${AGENTS_RED}[FAIL]${AGENTS_NC} _agents_set_outvar interpreted metacharacters instead of assigning data" >&2
534 + NETDATA_CLOUD_TOKEN="${real_token}"; unset AGENTS_DRY_RUN
535 + return 1
536 + fi
537
449 - # 3. The unit test passes if both checks above passed.
538 + # 5. The unit test passes if all checks above passed.
539 NETDATA_CLOUD_TOKEN="${real_token}"
540 unset AGENTS_DRY_RUN
541 echo -e "${AGENTS_GREEN}[PASS]${AGENTS_NC} no-token-leak self-test" >&2
docs/netdata-ai/skills/query-netdata-cloud/how-tos/INDEX.md
+1
@@ -86,6 +86,7 @@ how-to is written.)
86 ### Topology / flows
87
88 - `local-l2-topology-summary.md` (stub -- not yet authored)
89 +- [`validate-local-netflow-function.md`](./validate-local-netflow-function.md)
90 - `top-flow-talkers-last-hour.md` (stub -- not yet authored)
91
92 ### Members / rooms / feed
docs/netdata-ai/skills/query-netdata-cloud/how-tos/validate-local-netflow-function.md new
+118
@@ -0,0 +1,118 @@
1 +# Validate a local Cloud-connected flow Function
2 +
3 +## Question
4 +
5 +How can an assistant validate `flows:netflow` on a local Netdata Agent
6 +that is connected to Netdata Cloud, without exposing Cloud tokens,
7 +agent bearers, node ids, or raw flow rows?
8 +
9 +## Inputs
10 +
11 +- Local agent URL, usually `http://127.0.0.1:19999`.
12 +- `NETDATA_CLOUD_TOKEN` and `NETDATA_CLOUD_HOSTNAME` in `<repo>/.env`.
13 +- The agent must have `flows:netflow` registered.
14 +
15 +## Steps
16 +
17 +1. Capture local agent identity in memory without printing identifiers:
18 +
19 + ```bash
20 + INFO_JSON="$(curl -sS --max-time 10 http://127.0.0.1:19999/api/v3/info)"
21 +
22 + jq -r '.agents[0] | {
23 + cloud_status: .cloud.status,
24 + node_id_present: ((.nd // "") | length > 0),
25 + machine_guid_present: ((.mg // "") | length > 0),
26 + claim_id_present: ((.cloud.claim_id // "") | length > 0)
27 + }' <<<"$INFO_JSON"
28 + ```
29 +
30 +2. Load the token-safe wrappers:
31 +
32 + ```bash
33 + source docs/netdata-ai/skills/query-netdata-agents/scripts/_lib.sh
34 + agents_load_env
35 + ```
36 +
37 +3. Verify the Function info envelope via Cloud:
38 +
39 + ```bash
40 + NODE_UUID="$(jq -r '.agents[0].nd' \
41 + <<<"$INFO_JSON")"
42 +
43 + mkdir -p .local/audits/query-netdata-agents
44 +
45 + agents_call_function \
46 + --via cloud \
47 + --node "$NODE_UUID" \
48 + --function flows:netflow \
49 + --body '{"info":true}' \
50 + > .local/audits/query-netdata-agents/flows-netflow-info-cloud.json
51 +
52 + jq '{status, type, has_history,
53 + accepted_params_count: (.accepted_params | length),
54 + required_params_count: (.required_params | length)}' \
55 + .local/audits/query-netdata-agents/flows-netflow-info-cloud.json
56 + ```
57 +
58 +4. Run a real flow query using the documented request shape:
59 +
60 + ```bash
61 + read -r -d '' BODY <<'JSON'
62 + {
63 + "mode": "flows",
64 + "view": "table-sankey",
65 + "after": -3600,
66 + "before": 0,
67 + "group_by": ["SRC_AS_NAME", "PROTOCOL", "DST_AS_NAME"],
68 + "sort_by": "bytes",
69 + "top_n": 100
70 + }
71 + JSON
72 +
73 + agents_call_function \
74 + --via cloud \
75 + --node "$NODE_UUID" \
76 + --function flows:netflow \
77 + --body "$BODY" \
78 + > .local/audits/query-netdata-agents/flows-netflow-last-hour-cloud.json
79 +
80 + jq '{status, type, view: .data.view,
81 + flows_count: (.data.flows | length),
82 + group_by: .data.group_by,
83 + stats: .data.stats}' \
84 + .local/audits/query-netdata-agents/flows-netflow-last-hour-cloud.json
85 + ```
86 +
87 +## Output
88 +
89 +Return only a sanitized summary:
90 +
91 +- Function info `status` and `type`.
92 +- Flow query row count.
93 +- Group-by fields.
94 +- Selected aggregate counters from `.data.stats`, such as
95 + `decoded_netflow_v5`, `decoded_netflow_v9`, `decoded_ipfix`,
96 + `decoded_sflow`, `journal_entries_written`, and
97 + `journal_write_errors`.
98 +
99 +Do not paste node ids, machine GUIDs, claim ids, Cloud tokens, agent
100 +bearers, raw IP addresses, or raw flow rows into durable artifacts.
101 +
102 +## Notes / gotchas
103 +
104 +- Prefer the Cloud transport for validation. It needs only the Cloud
105 + token and does not require a direct agent bearer.
106 +- Direct-agent validation is also possible. Use the sibling
107 + direct-agent how-to when the test must prove the bearer mint/cache
108 + path and the `X-Netdata-Auth` call path.
109 +- Negative `after` values are relative to `before`; `before: 0` means
110 + now. `top_n` accepts the documented values `25`, `50`, `100`,
111 + `200`, or `500`.
112 +
113 +## Source guides
114 +
115 +- [Network-flow Functions](../query-flows.md)
116 +- [Generic Function invocation](../query-functions.md)
117 +- [Direct-agent sibling skill](../../query-netdata-agents/SKILL.md)
118 +- [Direct local flow Function validation](../../query-netdata-agents/how-tos/validate-direct-local-flow-function.md)
docs/network-flows/configuration.md
+1 -1
@@ -229,7 +229,7 @@ A note on `default_sampling_rate` vs. `override_sampling_rate`: both keys accept
229 For the cross-cutting picture — order of evaluation, the `asn_providers` and `net_providers` chains, the MMDB shared mechanism, the static-vs-dynamic composition rules — see the [Enrichment](/docs/network-flows/enrichment.md) page. Per-method configuration details (URLs, refresh cadence, license, vendor commands) live on the integration cards under flows.enrichment-methods:
230
231 - IP intelligence (MMDB): [DB-IP](/src/crates/netflow-plugin/integrations/db-ip_ip_intelligence.md), [MaxMind GeoIP / GeoLite2](/src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md), [IPtoASN](/src/crates/netflow-plugin/integrations/iptoasn.md), [Custom MMDB](/src/crates/netflow-plugin/integrations/custom_mmdb_database.md).
232 -- BGP routing: [BMP](/src/crates/netflow-plugin/integrations/bmp_bgp_monitoring_protocol.md), [bio-rd / RIPE RIS](/src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md).
232 +- BGP routing: [BMP](/src/crates/netflow-plugin/integrations/bmp_bgp_monitoring_protocol.md), [bio-rd RIS](/src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md).
233 - Network sources: [AWS IP Ranges](/src/crates/netflow-plugin/integrations/aws_ip_ranges.md), [Azure IP Ranges](/src/crates/netflow-plugin/integrations/azure_ip_ranges.md), [GCP IP Ranges](/src/crates/netflow-plugin/integrations/gcp_ip_ranges.md), [NetBox](/src/crates/netflow-plugin/integrations/netbox.md), [Generic JSON-over-HTTP IPAM](/src/crates/netflow-plugin/integrations/generic_json-over-http_ipam.md).
234 - YAML-defined: [Static Metadata](/src/crates/netflow-plugin/integrations/static_metadata.md), [Classifiers](/src/crates/netflow-plugin/integrations/classifiers.md), [Decapsulation](/src/crates/netflow-plugin/integrations/decapsulation.md).
235 - Operational: [Enrichment Intel Downloader](/docs/network-flows/intel-downloader.md) — the bundled refresh tool for MMDB providers.
docs/network-flows/enrichment.md
+5 -3
@@ -283,7 +283,9 @@ See the [Decapsulation integration card](/src/crates/netflow-plugin/integrations
283
284 ## Routing overlay (BMP and BioRIS share the trie)
285
286 -BMP and BioRIS are separate transports that feed the **same** in-memory routing trie. A deployment running BMP from internal routers and BioRIS for an external view (RIPE RIS) gets unified enrichment without duplicate trie entries.
286 +BMP and BioRIS are separate transports that feed the **same** in-memory routing trie. A deployment running BMP from internal routers and BioRIS from a separate bio-rd-compatible RIS endpoint gets unified enrichment without duplicate trie entries.
287 +
288 +BioRIS means Netdata consumes the `bio.ris.RoutingInformationService` gRPC API. Netdata does not connect directly to RIPE RIS Live, RIPEstat, RIS MRT dumps, or RIPE route collector sessions. If you need a RIPE-derived external view, put a converter or bio-rd-compatible service in front of that data and point Netdata at the resulting gRPC endpoint.
289
290 Each prefix entry holds a list of routes keyed by `(peer, route_key)` so multipath BGP and multiple peers contributing the same prefix coexist. Lookups walk the trie longest-prefix-first, then refine within candidates by:
291
@@ -312,7 +314,7 @@ AS *names* still come from the ASN MMDB, not from BGP. BGP gives accurate AS num
314 ### Restart behaviour
315
316 - **GeoIP databases** are reloaded automatically — no special handling needed.
315 -- **The routing trie is not persisted.** A restart wipes BGP-derived data; it is re-learned as routers re-send Initiation + Update messages (BMP) or as bio-rd's next refresh cycle dumps the RIB (BioRIS). Convergence ranges from seconds (FRR) to minutes (Cisco IOS-XR, RIPE RIS full feed). Schedule restarts off-peak when BGP attribution matters.
317 +- **The routing trie is not persisted.** A restart wipes BGP-derived data; it is re-learned as routers re-send Initiation + Update messages (BMP) or as bio-rd's next refresh cycle dumps the RIB (BioRIS). Convergence ranges from seconds (FRR) to minutes for full-table BioRIS feeds. Schedule restarts off-peak when BGP attribution matters.
318 - **Network-source records** are re-fetched on the next interval tick. There is no persistence between restarts.
319 - **Classifier caches** are wiped — they refill on first hit per target.
320
@@ -392,7 +394,7 @@ These cards carry the per-method specifics — installation steps, refresh caden
394
395 **BGP routing**
396 - [BMP (BGP Monitoring Protocol)](/src/crates/netflow-plugin/integrations/bmp_bgp_monitoring_protocol.md) — routers push BGP updates over TCP
395 -- [bio-rd / RIPE RIS](/src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md) — pull BGP data from a bio-rd RIS gRPC daemon
397 +- [bio-rd RIS](/src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md) — pull BGP data from a bio-rd-compatible RIS gRPC daemon
398
399 **Network identity (cloud IP ranges, IPAM)**
400 - [AWS IP Ranges](/src/crates/netflow-plugin/integrations/aws_ip_ranges.md) — public AWS prefix list
docs/network-flows/intel-downloader.md
+18 -4
@@ -45,14 +45,24 @@ The tool only knows how to talk to a fixed set of providers — anything else is
45 | `dbip:country-lite` | Geo | `mmdb` (default) or `csv` | DB-IP free monthly download page |
46 | `dbip:city-lite` | Geo | `mmdb` (default) or `csv` | DB-IP free monthly download page |
47 | `iptoasn:combined` | ASN or Geo | `tsv` | `https://iptoasn.com/data/ip2asn-combined.tsv.gz` (direct URL) |
48 +| `caida:prefix2as` | ASN | `tsv` | CAIDA RouteViews prefix-to-AS creation log |
49 +| `maxmind:geolite2-asn` | ASN | `mmdb` | MaxMind authenticated GeoLite2 download |
50 +| `maxmind:geolite2-country` | Geo | `csv` | MaxMind authenticated GeoLite2 Country CSV ZIP download |
51 +| `ip2location:country-lite` | Geo | `csv` | IP2Location Lite country CSV ZIP download |
52 +| `ipdeny:country-zones` | Geo | `cidr` | IPDeny country zone archive |
53 +| `ipip:country` | Geo | `txt` | IPIP country text ZIP download |
54
55 DB-IP artifacts are resolved from the current monthly URL on the DB-IP landing page (`https://db-ip.com/db/download/<artifact>`). The downloaded URL uses the DB-IP free database pattern `https://download.db-ip.com/free/dbip-<artifact>-YYYY-MM.<ext>.gz`.
56
57 The IPtoASN TSV feed is converted into the same Netdata MMDB layout as the DB-IP feeds, so consumers don't care which source produced the file.
58
53 -> **MaxMind GeoIP / GeoLite2 is not supported by this tool.** The downloader has no `license_key` field, no `MAXMIND_LICENSE_KEY` env var, and no MaxMind URL builder. If you want to use MaxMind, run MaxMind's own [`geoipupdate`](/src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md) and point `enrichment.geoip.asn_database` / `enrichment.geoip.geo_database` at the files it produces.
59 +CAIDA prefix2as is ASN-only and has no AS organization names. The downloader resolves the latest `.pfx2as.gz` entry from CAIDA's creation log before fetching it.
60
55 -You can still pull *any* MMDB build (including a custom one) into the resolver by configuring `enrichment.geoip.asn_database` / `geo_database` directly — the downloader is one of several producers; the plugin doesn't care who wrote the MMDB. See the [Custom MMDB Database](/src/crates/netflow-plugin/integrations/custom_mmdb_database.md) card.
61 +MaxMind built-in sources require `MAXMIND_LICENSE_KEY` in the downloader environment. `maxmind:geolite2-asn@mmdb` downloads the official GeoLite2 ASN tarball and extracts the MMDB member. `maxmind:geolite2-country@csv` downloads the official GeoLite2 Country CSV **ZIP bundle** and needs the locations file plus the IPv4/IPv6 block CSVs inside that bundle; `csv` here does not mean a single raw CSV file.
62 +
63 +IP2Location `country-lite@csv` is also the provider's official CSV ZIP bundle. IPDeny `country-zones@cidr` is the `all-zones.tar.gz` archive, and IPIP `country@txt` is the country text ZIP.
64 +
65 +You can still pull *any* MMDB build (including a custom one) into the resolver by configuring `enrichment.geoip.asn_database` / `geo_database` directly — the downloader is one of several producers; the plugin doesn't care who wrote the MMDB. See the [Custom MMDB Database](/src/crates/netflow-plugin/integrations/custom_mmdb_database.md) card. If you prefer MaxMind's own updater, run [`geoipupdate`](/src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md) and point `enrichment.geoip.asn_database` / `enrichment.geoip.geo_database` at the MMDB files it produces.
66
67 ## Configuration file
68
@@ -161,7 +171,7 @@ sudo systemctl daemon-reload
171 sudo systemctl enable --now netdata-topology-ip-intel.timer
172 ```
173
164 -DB-IP refreshes its free Lite databases monthly; weekly is a safe over-poll that picks up every release within a few days while staying polite to the upstream. IPtoASN refreshes hourly, but downstream consumers rarely need that resolution — daily is plenty if you switch to it.
174 +Refresh cadence depends on the sources you enable. DB-IP refreshes its free Lite databases monthly; weekly is a safe over-poll that picks up every release within a few days while staying polite to the upstream. IPtoASN refreshes hourly, but downstream consumers rarely need that resolution — daily is plenty if you switch to it. CAIDA prefix2as, MaxMind, IP2Location, IPDeny, and IPIP have their own publication schedules and terms; choose a timer cadence that is polite to the upstream and fast enough for your environment.
175
176 Run the packaged binary as the `netdata` user (or root) so it can write to `/var/cache/netdata/topology-ip-intel/`.
177
@@ -242,7 +252,11 @@ When the plugin auto-detects MMDBs this way it forces `optional: true` on the ge
252 - Per-provider details (refresh cadence, license, schema, attribution requirements):
253 - [DB-IP IP Intelligence](/src/crates/netflow-plugin/integrations/db-ip_ip_intelligence.md) — the default the downloader fetches.
254 - [IPtoASN](/src/crates/netflow-plugin/integrations/iptoasn.md) — public-domain TSV feed; converted to MMDB by this tool.
245 - - [MaxMind GeoIP / GeoLite2](/src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md) — *not* fetched by this tool; use `geoipupdate`.
255 + - [CAIDA RouteViews Prefix-to-AS](/src/crates/netflow-plugin/integrations/caida_routeviews_prefix-to-as.md) — prefix-to-AS TSV feed; ASN-only.
256 + - [MaxMind GeoIP / GeoLite2](/src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md) — authenticated MaxMind downloads or MMDB files managed by `geoipupdate`.
257 + - [IP2Location LITE IP-Country](/src/crates/netflow-plugin/integrations/ip2location_lite_ip-country.md) — public country-only CSV ZIP feed.
258 + - [IPDeny Country Zones](/src/crates/netflow-plugin/integrations/ipdeny_country_zones.md) — country CIDR archive.
259 + - [IPIP Country Database](/src/crates/netflow-plugin/integrations/ipip_country_database.md) — country text ZIP feed.
260 - [Custom MMDB Database](/src/crates/netflow-plugin/integrations/custom_mmdb_database.md) — your own MMDB build.
261 - The enrichment mechanism that consumes these files: [Enrichment](/docs/network-flows/enrichment.md) (the MMDB shared mechanism section).
262 - The plugin knobs that point at the files: [Configuration › `enrichment.geoip`](/docs/network-flows/configuration.md#enrichment).
src/crates/Cargo.lock
+1
@@ -2454,6 +2454,7 @@ dependencies = [
2454 "async-trait",
2455 "bitflags 2.11.0",
2456 "bitvec",
2457 + "bytes",
2458 "bytesize",
2459 "chrono",
2460 "clap",
src/crates/netflow-plugin/Cargo.toml
+2 -1
@@ -49,7 +49,7 @@ serde_json = { workspace = true }
49 serde_yaml = { workspace = true }
50 sflow-parser = { workspace = true }
51 tokio = { workspace = true }
52 -tokio-stream = { workspace = true }
52 +tokio-stream = { workspace = true, features = ["net"] }
53 tokio-util = { workspace = true }
54 tracing = { workspace = true }
55 regex = { workspace = true }
@@ -70,6 +70,7 @@ netdata-plugin-error = { workspace = true }
70 netdata-plugin-protocol = { workspace = true }
71
72 [dev-dependencies]
73 +bytes = { workspace = true }
74 etherparse = { workspace = true }
75 pcap-file = { workspace = true }
76 proptest = { workspace = true }
src/crates/netflow-plugin/integrations/bio-rd_-_ripe_ris.md
+40 -26
@@ -23,20 +23,21 @@ Module: bioris
23
24 ## Overview
25
26 -Enrich network flows with BGP routing context received from bio-rd / RIPE RIS
27 -over gRPC. [bio-rd](https://github.com/bio-routing/bio-rd)
28 -is a Go-based BGP/BMP daemon that you run yourself. You point it at one or more
29 -BGP / BMP sources -- your own routers, a [RIPE RIS](https://www.ripe.net/analyse/internet-measurements/routing-information-service-ris)
30 -Route Collector you peer with, or any other reachable BGP source -- and it
31 -exposes the resulting RIB through a gRPC interface (`RoutingInformationService`).
32 -Netdata is a **client** of that interface, not of RIPE NCC directly.
33 -
34 -Pick this when you want a third-party / external BGP view (e.g. RIPE RIS's view of the
35 -public DFZ) without exposing a BMP listener on your agent or running BMP across your
36 -network. If your routers can speak BMP straight to Netdata, the `bmp` integration is
37 -simpler -- BioRIS makes sense when bio-rd is already part of your routing toolbox or
38 -when you want to peer with an external party (RIPE RIS, looking-glass) and re-export
39 -to Netdata.
26 +Enrich network flows with BGP routing context received from a bio-rd-compatible
27 +`RoutingInformationService` gRPC endpoint. [bio-rd](https://github.com/bio-routing/bio-rd)
28 +is a Go-based BGP/BMP daemon that you run yourself. Its `ris` daemon receives BMP
29 +sessions from routers or collectors and exposes the resulting RIB through gRPC.
30 +Netdata is a **client** of that gRPC interface only.
31 +
32 +Netdata does **not** connect directly to RIPE NCC RIS Live, RIPEstat, RIS MRT dumps,
33 +or RIPE route collector sessions. RIPE RIS public access is WebSocket JSON, HTTP API,
34 +and MRT dump based. To use a RIPE-derived external view, run a bridge or service that
35 +imports that data into a bio-rd-compatible `RoutingInformationService`, then point
36 +Netdata at that service.
37 +
38 +Pick this when bio-rd is already part of your routing toolbox, or when you want Netdata
39 +to consume a routing view that is already exposed through the bio-rd RIS gRPC API. If
40 +your routers can speak BMP straight to Netdata, the `bmp` integration is simpler.
41
42 BioRIS populates the same flow-record fields as BMP -- both feed a single shared
43 in-memory routing trie. See the BMP integration card for the side-by-side comparison
@@ -128,10 +129,13 @@ cd bio-rd/cmd/ris
129 go build -o /usr/local/bin/ris .
130 ```
131
131 -Configure `ris` to peer with one or more BGP / BMP sources (your own routers,
132 -RIPE RIS Route Collectors, looking-glass servers, etc.). Refer to the bio-rd
133 -documentation for the peering setup -- this is bio-rd's configuration, not
134 -Netdata's.
132 +Configure `ris` with one or more BMP sources using bio-rd's `bmp_addr` /
133 +`bmp_servers` configuration. Refer to the bio-rd documentation for the BMP setup
134 +-- this is bio-rd's configuration, not Netdata's.
135 +
136 +Do not set Netdata's `grpc_addr` to a RIPE RIS Live URL, RIPEstat API URL, MRT dump
137 +URL, or route collector session address. Those endpoints are not the
138 +`RoutingInformationService` gRPC API that Netdata consumes.
139
140 Run the daemon with a gRPC port:
141 `/usr/local/bin/ris --grpc_port 50051 --config.file /etc/bio-rd.yml`
@@ -248,9 +252,10 @@ enrichment:
252
253 ###### Combined with BMP into a single trie
254
251 -Run BMP from internal routers and BioRIS for an external (RIPE RIS) view.
252 -Both populate the same shared trie -- lookups pick the best-matching route
253 -across both sources at query time.
255 +Run BMP from internal routers and BioRIS from a separate bio-rd-compatible RIS
256 +service, for example one populated from an external routing view. Both populate
257 +the same shared trie -- lookups pick the best-matching route across both sources
258 +at query time.
259
260
261 <details open><summary>Config</summary>
@@ -282,20 +287,29 @@ message stream reads (default 10s).
287
288 ### Initial dump takes minutes for full feeds
289
285 -A full IPv4+IPv6 RIB from a route collector is millions of prefixes. The
286 -first `DumpRIB` after enabling (or after a plugin restart) takes time -- BGP
287 -attribution is incomplete until it finishes. Subsequent `ObserveRIB` streams
288 -are incremental.
290 +A full IPv4+IPv6 RIB from a full-table bio-rd source can contain millions of
291 +prefixes. The first `DumpRIB` after enabling (or after a plugin restart) takes
292 +time -- BGP attribution is incomplete until it finishes. Subsequent
293 +`ObserveRIB` streams are incremental.
294
295
296 ### Plugin restart wipes the trie
297
298 The trie is in-memory only -- restarting the netflow plugin loses every
299 learned BGP route. Convergence over BioRIS depends on the upstream feed; a
295 -full DumpRIB from a RIPE RIS Route Collector can take minutes. Schedule
300 +full DumpRIB from a full-table bio-rd source can take minutes. Schedule
301 restarts off-peak if BGP attribution matters for your workflow.
302
303
304 +### Pointing grpc_addr at RIPE RIS does not work
305 +
306 +`grpc_addr` must point to a bio-rd-compatible `RoutingInformationService`
307 +endpoint. RIPE RIS Live, RIPEstat, RIS MRT dumps, and route collector BGP
308 +sessions use different protocols, so they cannot be used directly as
309 +`ris_instances[].grpc_addr`. Put a converter or bio-rd-compatible service in
310 +front of RIPE-derived data if you need that external view.
311 +
312 +
313 ### Memory growth without bound
314
315 The trie has no time-based eviction. Routes are removed only when the
src/crates/netflow-plugin/integrations/bmp_bgp_monitoring_protocol.md
+6 -4
@@ -54,8 +54,9 @@ convention -- RFC 7854 does not register a port, and IANA does not assign one
54 for BMP. Each connecting router must first send an Initiation message; the plugin
55 then processes RouteMonitoring (carrying BGP UPDATE), PeerDownNotification, and
56 Termination frames. PeerUp, StatisticsReport, and RouteMirroring frames are
57 -accepted but not acted on. Only BMP **version 3** is processed; v1 and v2 frames
58 -are silently dropped.
57 +accepted but not acted on. Only BMP **version 3** is processed. Version 4 frames
58 +are decoded but ignored; draft v1/v2 frames are decode errors and count toward
59 +the consecutive decode-error threshold.
60
61 NLRI families parsed: IPv4/IPv6 unicast, IPv4/IPv6 MPLS-labelled, VPNv4, VPNv6,
62 and EVPN IP-prefix routes.
@@ -112,8 +113,9 @@ Common vendor configuration patterns:
113 module: `bgpd` must be started with `-M bmp` or every BMP command silently
114 fails.
115
115 -The plugin parses RFC 7854 BMP **version 3** only. Older draft versions (v1,
116 -v2) are silently dropped.
116 +The plugin parses RFC 7854 BMP **version 3** only. Version 4 frames are decoded
117 +but ignored. Older draft versions (v1, v2) are decode errors and count toward
118 +the consecutive decode-error threshold.
119
120
121 #### TCP reachability between routers and the agent
src/crates/netflow-plugin/integrations/caida_routeviews_prefix-to-as.md new
+190
@@ -0,0 +1,190 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/integrations/caida_routeviews_prefix-to-as.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/metadata.yaml"
4 +sidebar_label: "CAIDA Routeviews Prefix-to-AS"
5 +learn_status: "Published"
6 +learn_rel_path: "Network Flows/Enrichment Methods"
7 +keywords: ['caida', 'routeviews', 'prefix2as', 'pfx2as', 'asn', 'bgp', 'ip intelligence', 'flow enrichment']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE FLOWS' metadata.yaml FILE"
9 +endmeta-->
10 +
11 +<!-- markdownlint-disable-file -->
12 +
13 +# CAIDA Routeviews Prefix-to-AS
14 +
15 +
16 +<img src="https://netdata.cloud/img/network-wired.svg" width="150"/>
17 +
18 +
19 +Plugin: netflow-plugin
20 +Module: caida-prefix2as
21 +
22 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
23 +
24 +## Overview
25 +
26 +Enrich network flows with AS numbers from CAIDA's Routeviews Prefix-to-AS
27 +mapping dataset. CAIDA derives this public dataset from RouteViews BGP data and
28 +publishes one prefix-to-origin-AS mapping per line. The official format is
29 +tab-separated `IP prefix`, `prefix length`, and `AS number`; multi-origin AS
30 +entries can contain multiple ASNs, and the Netdata downloader keeps the first
31 +listed origin AS as the simplified per-prefix value.
32 +
33 +CAIDA Prefix-to-AS is ASN-only. It does not provide AS organization names or
34 +geographic fields. Pair it with DB-IP, MaxMind, IP2Location, IPDeny, IPIP, or
35 +another geo provider when you also need `*_COUNTRY`, city, state, or
36 +coordinates.
37 +
38 +For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
39 +provider chains), see
40 +[IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
41 +
42 +
43 +The netflow plugin reads MMDB files only. CAIDA publishes compressed text data,
44 +so use Netdata's `topology-ip-intel-downloader` to fetch the latest Routeviews
45 +Prefix-to-AS file, convert it to `topology-ip-asn.mmdb`, and place it in the
46 +standard auto-detected cache directory. The plugin reloads the generated MMDB
47 +automatically when the file changes.
48 +
49 +
50 +This integration is only supported on the following platforms:
51 +
52 +- Linux
53 +
54 +This integration runs as a single instance per Netdata Agent.
55 +
56 +
57 +### Default Behavior
58 +
59 +#### Auto-Detection
60 +
61 +Not used by the default install. After the downloader writes `topology-ip-asn.mmdb` into the standard cache path, the plugin auto-detects the generated file.
62 +
63 +#### Limits
64 +
65 +ASN number only. No AS organization name, country, state, city, or coordinates are available from this source.
66 +
67 +#### Performance Impact
68 +
69 +Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot.
70 +
71 +## Setup
72 +
73 +
74 +### Prerequisites
75 +
76 +#### Downloader with CAIDA Prefix-to-AS support
77 +
78 +Run the bundled downloader when available. Packaged 32-bit installs do not
79 +include this binary:
80 +
81 +```bash
82 +sudo /usr/sbin/topology-ip-intel-downloader \
83 + --asn caida:prefix2as \
84 + --geo dbip:city-lite
85 +```
86 +
87 +`caida:prefix2as` is valid only for `--asn`. The example keeps DB-IP
88 +city-lite for geo fields; substitute any supported `--geo` provider if
89 +country-only enrichment is sufficient.
90 +
91 +
92 +#### CAIDA acceptable use terms
93 +
94 +CAIDA documents acceptable-use and attribution requirements on the dataset
95 +page. Review the terms before redistributing generated datasets or derived
96 +reports.
97 +
98 +
99 +
100 +### Configuration
101 +
102 +#### Options
103 +
104 +Once the downloader has produced a CAIDA-derived ASN MMDB in the cache
105 +directory, the plugin auto-detects it. To pin paths explicitly, set
106 +`enrichment.geoip.asn_database` and keep `geo_database` pointed at your chosen
107 +geographic provider.
108 +
109 +
110 +<details open><summary>Config options</summary>
111 +
112 +
113 +
114 +| Option | Description | Default | Required |
115 +|:-----|:------------|:--------|:---------:|
116 +| enrichment.geoip.asn_database | Path to the CAIDA-derived ASN MMDB. Empty = auto-detect from the cache directory. | [] | no |
117 +| enrichment.geoip.geo_database | Optional geographic MMDB paths supplied by another provider. | [] | no |
118 +| enrichment.geoip.optional | When true, missing or unreadable MMDBs are warnings, not fatal at startup. | false (true when auto-detected) | no |
119 +
120 +
121 +</details>
122 +
123 +
124 +
125 +#### via File
126 +
127 +The configuration file name for this integration is `netflow.yaml`.
128 +
129 +
130 +You can edit the configuration file using the [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#edit-configuration-files) script from the
131 +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#locate-your-config-directory).
132 +
133 +```bash
134 +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata
135 +sudo ./edit-config netflow.yaml
136 +```
137 +
138 +##### Examples
139 +
140 +###### After running the downloader with CAIDA ASN and DB-IP geo
141 +
142 +Auto-detection picks up both generated cache files.
143 +
144 +```yaml
145 +enrichment:
146 + geoip:
147 + asn_database: []
148 + geo_database: []
149 + optional: true
150 +
151 +```
152 +###### Explicit CAIDA ASN path
153 +
154 +Pin CAIDA for AS numbers and keep a separate geo source for country/city fields.
155 +
156 +<details open><summary>Config</summary>
157 +
158 +```yaml
159 +enrichment:
160 + geoip:
161 + asn_database:
162 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
163 + geo_database:
164 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
165 + optional: false
166 +
167 +```
168 +</details>
169 +
170 +
171 +
172 +### AS names are empty
173 +
174 +CAIDA Prefix-to-AS contains AS numbers, not organization names. Layer a
175 +provider with AS organization data, such as MaxMind GeoLite2 ASN or DB-IP
176 +ASN Lite, after the CAIDA MMDB in `asn_database` if AS names matter.
177 +
178 +
179 +### Country fields are empty
180 +
181 +This source is ASN-only. Configure a `--geo` provider in the downloader or
182 +point `enrichment.geoip.geo_database` at a separate geo MMDB.
183 +
184 +
185 +### Multi-origin AS simplification
186 +
187 +CAIDA can represent multi-origin prefixes. The downloader uses the first
188 +listed AS for the generated single-value MMDB record. If you need full MOAS
189 +semantics, use BMP or BioRIS routing enrichment instead of an offline ASN
190 +lookup database.
src/crates/netflow-plugin/integrations/ip2location_lite_ip-country.md new
+185
@@ -0,0 +1,185 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/integrations/ip2location_lite_ip-country.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/metadata.yaml"
4 +sidebar_label: "IP2Location LITE IP-Country"
5 +learn_status: "Published"
6 +learn_rel_path: "Network Flows/Enrichment Methods"
7 +keywords: ['ip2location', 'ip2location lite', 'geoip', 'country', 'csv', 'ip intelligence', 'flow enrichment']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE FLOWS' metadata.yaml FILE"
9 +endmeta-->
10 +
11 +<!-- markdownlint-disable-file -->
12 +
13 +# IP2Location LITE IP-Country
14 +
15 +
16 +<img src="https://netdata.cloud/img/network-wired.svg" width="150"/>
17 +
18 +
19 +Plugin: netflow-plugin
20 +Module: ip2location
21 +
22 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
23 +
24 +## Overview
25 +
26 +Enrich network flows with country codes from the IP2Location LITE
27 +IP-Country database. The supported downloader source is the DB1 LITE CSV ZIP
28 +(`IP2LOCATION-LITE-DB1.CSV.ZIP`), whose documented fields are
29 +`ip_from`, `ip_to`, `country_code`, and `country_name`.
30 +
31 +IP2Location DB1 is country-only. It does not provide ASN, AS organization,
32 +state, city, latitude, or longitude. Pair it with CAIDA, IPtoASN, DB-IP, or
33 +MaxMind ASN data when you also need AS fields.
34 +
35 +For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
36 +provider chains), see
37 +[IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
38 +
39 +
40 +The netflow plugin reads MMDB files only. Use Netdata's
41 +`topology-ip-intel-downloader` to fetch the IP2Location LITE CSV ZIP, convert
42 +the ranges to `topology-ip-geo.mmdb`, and write the file into the standard
43 +auto-detected cache directory. The plugin reloads the generated MMDB
44 +automatically when the file changes.
45 +
46 +
47 +This integration is only supported on the following platforms:
48 +
49 +- Linux
50 +
51 +This integration runs as a single instance per Netdata Agent.
52 +
53 +
54 +### Default Behavior
55 +
56 +#### Auto-Detection
57 +
58 +Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file.
59 +
60 +#### Limits
61 +
62 +Country only. No ASN, AS name, state, city, latitude, or longitude are available from this source.
63 +
64 +#### Performance Impact
65 +
66 +Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot.
67 +
68 +## Setup
69 +
70 +
71 +### Prerequisites
72 +
73 +#### Downloader with IP2Location LITE support
74 +
75 +Run the bundled downloader when available. Packaged 32-bit installs do not
76 +include this binary:
77 +
78 +```bash
79 +sudo /usr/sbin/topology-ip-intel-downloader \
80 + --asn iptoasn:combined \
81 + --geo ip2location:country-lite
82 +```
83 +
84 +`ip2location:country-lite` is valid only for `--geo`. The example pairs it
85 +with IPtoASN so the resulting cache contains both AS and country fields.
86 +
87 +
88 +#### IP2Location LITE license
89 +
90 +Review the IP2Location LITE terms before redistributing the data or
91 +generated derivative files. The plugin consumes only the generated MMDB and
92 +does not contact IP2Location during flow decoding.
93 +
94 +
95 +
96 +### Configuration
97 +
98 +#### Options
99 +
100 +Once the downloader has produced an IP2Location-derived geo MMDB in the cache
101 +directory, the plugin auto-detects it. To pin paths explicitly, set
102 +`enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
103 +ASN provider.
104 +
105 +
106 +<details open><summary>Config options</summary>
107 +
108 +
109 +
110 +| Option | Description | Default | Required |
111 +|:-----|:------------|:--------|:---------:|
112 +| enrichment.geoip.asn_database | Optional ASN MMDB paths supplied by another provider. | [] | no |
113 +| enrichment.geoip.geo_database | Path to the IP2Location-derived geo MMDB. Empty = auto-detect from the cache directory. | [] | no |
114 +| enrichment.geoip.optional | When true, missing or unreadable MMDBs are warnings, not fatal at startup. | false (true when auto-detected) | no |
115 +
116 +
117 +</details>
118 +
119 +
120 +
121 +#### via File
122 +
123 +The configuration file name for this integration is `netflow.yaml`.
124 +
125 +
126 +You can edit the configuration file using the [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#edit-configuration-files) script from the
127 +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#locate-your-config-directory).
128 +
129 +```bash
130 +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata
131 +sudo ./edit-config netflow.yaml
132 +```
133 +
134 +##### Examples
135 +
136 +###### After running the downloader with IPtoASN ASN and IP2Location country
137 +
138 +Auto-detection picks up both generated cache files.
139 +
140 +```yaml
141 +enrichment:
142 + geoip:
143 + asn_database: []
144 + geo_database: []
145 + optional: true
146 +
147 +```
148 +###### Explicit IP2Location country path
149 +
150 +Pin IP2Location for country and keep a separate ASN source.
151 +
152 +<details open><summary>Config</summary>
153 +
154 +```yaml
155 +enrichment:
156 + geoip:
157 + asn_database:
158 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
159 + geo_database:
160 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
161 + optional: false
162 +
163 +```
164 +</details>
165 +
166 +
167 +
168 +### ASN fields are empty
169 +
170 +IP2Location DB1 is country-only. Configure `--asn iptoasn:combined`,
171 +`--asn caida:prefix2as`, or another ASN provider in the downloader.
172 +
173 +
174 +### City or coordinates are empty
175 +
176 +The DB1 LITE source does not include city, state, latitude, or longitude.
177 +Use DB-IP city-lite, MaxMind GeoLite2 City, GeoIP2 City, or a custom MMDB
178 +when those fields are required.
179 +
180 +
181 +### Stale country data
182 +
183 +Check the upstream IP2Location LITE page for the current version and next
184 +update date, then schedule the downloader accordingly. The plugin reloads
185 +the generated MMDB in place when the file signature changes.
src/crates/netflow-plugin/integrations/ipdeny_country_zones.md new
+186
@@ -0,0 +1,186 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/integrations/ipdeny_country_zones.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/metadata.yaml"
4 +sidebar_label: "IPDeny Country Zones"
5 +learn_status: "Published"
6 +learn_rel_path: "Network Flows/Enrichment Methods"
7 +keywords: ['ipdeny', 'country zones', 'geoip', 'country', 'cidr', 'ip intelligence', 'flow enrichment']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE FLOWS' metadata.yaml FILE"
9 +endmeta-->
10 +
11 +<!-- markdownlint-disable-file -->
12 +
13 +# IPDeny Country Zones
14 +
15 +
16 +<img src="https://netdata.cloud/img/network-wired.svg" width="150"/>
17 +
18 +
19 +Plugin: netflow-plugin
20 +Module: ipdeny
21 +
22 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
23 +
24 +## Overview
25 +
26 +Enrich network flows with country codes from IPDeny country zone files.
27 +IPDeny publishes country-specific CIDR lists and an `all-zones.tar.gz`
28 +archive. The Netdata downloader reads the IPv4 country-zone archive and maps
29 +each `<country>.zone` member to that ISO country code.
30 +
31 +IPDeny Country Zones are country-only and the current built-in source is IPv4
32 +only. They do not provide ASN, AS organization, state, city, latitude, or
33 +longitude. Pair IPDeny with another ASN source when you also need AS fields,
34 +and use DB-IP, MaxMind, IP2Location, or a custom MMDB when IPv6 country
35 +coverage is required.
36 +
37 +For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
38 +provider chains), see
39 +[IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
40 +
41 +
42 +The netflow plugin reads MMDB files only. Use Netdata's
43 +`topology-ip-intel-downloader` to fetch IPDeny's HTTPS country-zone archive,
44 +convert the CIDR lists to `topology-ip-geo.mmdb`, and write the file into the
45 +standard auto-detected cache directory. The plugin reloads the generated MMDB
46 +automatically when the file changes.
47 +
48 +
49 +This integration is only supported on the following platforms:
50 +
51 +- Linux
52 +
53 +This integration runs as a single instance per Netdata Agent.
54 +
55 +
56 +### Default Behavior
57 +
58 +#### Auto-Detection
59 +
60 +Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file.
61 +
62 +#### Limits
63 +
64 +Country-only IPv4 source. No ASN, AS name, IPv6 coverage, state, city, latitude, or longitude are available from this built-in source.
65 +
66 +#### Performance Impact
67 +
68 +Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot.
69 +
70 +## Setup
71 +
72 +
73 +### Prerequisites
74 +
75 +#### Downloader with IPDeny support
76 +
77 +Run the bundled downloader when available. Packaged 32-bit installs do not
78 +include this binary:
79 +
80 +```bash
81 +sudo /usr/sbin/topology-ip-intel-downloader \
82 + --asn iptoasn:combined \
83 + --geo ipdeny:country-zones
84 +```
85 +
86 +`ipdeny:country-zones` is valid only for `--geo`. The example pairs it
87 +with IPtoASN so the resulting cache contains both AS and country fields.
88 +
89 +
90 +#### IPDeny usage policy
91 +
92 +IPDeny publishes usage limits and copyright policy on its site. Review
93 +those terms before mirroring, redistributing, or downloading more often
94 +than needed.
95 +
96 +
97 +
98 +### Configuration
99 +
100 +#### Options
101 +
102 +Once the downloader has produced an IPDeny-derived geo MMDB in the cache
103 +directory, the plugin auto-detects it. To pin paths explicitly, set
104 +`enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
105 +ASN provider.
106 +
107 +
108 +<details open><summary>Config options</summary>
109 +
110 +
111 +
112 +| Option | Description | Default | Required |
113 +|:-----|:------------|:--------|:---------:|
114 +| enrichment.geoip.asn_database | Optional ASN MMDB paths supplied by another provider. | [] | no |
115 +| enrichment.geoip.geo_database | Path to the IPDeny-derived geo MMDB. Empty = auto-detect from the cache directory. | [] | no |
116 +| enrichment.geoip.optional | When true, missing or unreadable MMDBs are warnings, not fatal at startup. | false (true when auto-detected) | no |
117 +
118 +
119 +</details>
120 +
121 +
122 +
123 +#### via File
124 +
125 +The configuration file name for this integration is `netflow.yaml`.
126 +
127 +
128 +You can edit the configuration file using the [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#edit-configuration-files) script from the
129 +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#locate-your-config-directory).
130 +
131 +```bash
132 +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata
133 +sudo ./edit-config netflow.yaml
134 +```
135 +
136 +##### Examples
137 +
138 +###### After running the downloader with IPtoASN ASN and IPDeny country
139 +
140 +Auto-detection picks up both generated cache files.
141 +
142 +```yaml
143 +enrichment:
144 + geoip:
145 + asn_database: []
146 + geo_database: []
147 + optional: true
148 +
149 +```
150 +###### Explicit IPDeny country path
151 +
152 +Pin IPDeny for country and keep a separate ASN source.
153 +
154 +<details open><summary>Config</summary>
155 +
156 +```yaml
157 +enrichment:
158 + geoip:
159 + asn_database:
160 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
161 + geo_database:
162 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
163 + optional: false
164 +
165 +```
166 +</details>
167 +
168 +
169 +
170 +### IPv6 country fields are empty
171 +
172 +The current built-in IPDeny downloader source uses the IPv4
173 +`all-zones.tar.gz` archive. Use DB-IP, MaxMind, IP2Location, or a custom
174 +MMDB if IPv6 country enrichment is required.
175 +
176 +
177 +### ASN fields are empty
178 +
179 +IPDeny zone files carry country CIDRs only. Configure a separate ASN
180 +provider in the downloader.
181 +
182 +
183 +### Download blocked or rate-limited
184 +
185 +Check IPDeny's usage limits and retry later. The plugin continues using the
186 +last generated MMDB file until the downloader publishes a replacement.
src/crates/netflow-plugin/integrations/ipip_country_database.md new
+185
@@ -0,0 +1,185 @@
1 +<!--startmeta
2 +custom_edit_url: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/integrations/ipip_country_database.md"
3 +meta_yaml: "https://github.com/netdata/netdata/edit/master/src/crates/netflow-plugin/metadata.yaml"
4 +sidebar_label: "IPIP Country Database"
5 +learn_status: "Published"
6 +learn_rel_path: "Network Flows/Enrichment Methods"
7 +keywords: ['ipip', '17mon', 'geoip', 'country', 'ip intelligence', 'flow enrichment']
8 +message: "DO NOT EDIT THIS FILE DIRECTLY, IT IS GENERATED BY THE FLOWS' metadata.yaml FILE"
9 +endmeta-->
10 +
11 +<!-- markdownlint-disable-file -->
12 +
13 +# IPIP Country Database
14 +
15 +
16 +<img src="https://netdata.cloud/img/network-wired.svg" width="150"/>
17 +
18 +
19 +Plugin: netflow-plugin
20 +Module: ipip
21 +
22 +<img src="https://img.shields.io/badge/maintained%20by-Netdata-%2300ab44" />
23 +
24 +## Overview
25 +
26 +Enrich network flows with country codes from IPIP's country database. The
27 +supported downloader source fetches IPIP's `country.zip` archive and reads the
28 +`country.txt` file, where each line maps a prefix to a country token.
29 +
30 +IPIP Country is country-only. The current `country.zip` source validated during
31 +this work contained IPv4 prefixes only. It does not provide ASN, AS
32 +organization, state, city, latitude, or longitude. Pair IPIP with another ASN
33 +source when you also need AS fields, and use DB-IP, MaxMind, IP2Location, or a
34 +custom MMDB when IPv6 country coverage is required.
35 +
36 +For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
37 +provider chains), see
38 +[IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
39 +
40 +
41 +The netflow plugin reads MMDB files only. Use Netdata's
42 +`topology-ip-intel-downloader` to fetch the IPIP country ZIP, convert the
43 +prefix list to `topology-ip-geo.mmdb`, and write the file into the standard
44 +auto-detected cache directory. The plugin reloads the generated MMDB
45 +automatically when the file changes.
46 +
47 +
48 +This integration is only supported on the following platforms:
49 +
50 +- Linux
51 +
52 +This integration runs as a single instance per Netdata Agent.
53 +
54 +
55 +### Default Behavior
56 +
57 +#### Auto-Detection
58 +
59 +Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file.
60 +
61 +#### Limits
62 +
63 +Country-only source. The currently validated `country.zip` payload is IPv4-only and carries no ASN, AS name, state, city, latitude, or longitude.
64 +
65 +#### Performance Impact
66 +
67 +Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot.
68 +
69 +## Setup
70 +
71 +
72 +### Prerequisites
73 +
74 +#### Downloader with IPIP support
75 +
76 +Run the bundled downloader when available. Packaged 32-bit installs do not
77 +include this binary:
78 +
79 +```bash
80 +sudo /usr/sbin/topology-ip-intel-downloader \
81 + --asn iptoasn:combined \
82 + --geo ipip:country
83 +```
84 +
85 +`ipip:country` is valid only for `--geo`. The example pairs it with
86 +IPtoASN so the resulting cache contains both AS and country fields.
87 +
88 +
89 +#### IPIP data terms
90 +
91 +Review IPIP's current terms before redistributing the data or generated
92 +derivative files. The plugin consumes only the generated MMDB and does not
93 +contact IPIP during flow decoding.
94 +
95 +
96 +
97 +### Configuration
98 +
99 +#### Options
100 +
101 +Once the downloader has produced an IPIP-derived geo MMDB in the cache
102 +directory, the plugin auto-detects it. To pin paths explicitly, set
103 +`enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
104 +ASN provider.
105 +
106 +
107 +<details open><summary>Config options</summary>
108 +
109 +
110 +
111 +| Option | Description | Default | Required |
112 +|:-----|:------------|:--------|:---------:|
113 +| enrichment.geoip.asn_database | Optional ASN MMDB paths supplied by another provider. | [] | no |
114 +| enrichment.geoip.geo_database | Path to the IPIP-derived geo MMDB. Empty = auto-detect from the cache directory. | [] | no |
115 +| enrichment.geoip.optional | When true, missing or unreadable MMDBs are warnings, not fatal at startup. | false (true when auto-detected) | no |
116 +
117 +
118 +</details>
119 +
120 +
121 +
122 +#### via File
123 +
124 +The configuration file name for this integration is `netflow.yaml`.
125 +
126 +
127 +You can edit the configuration file using the [`edit-config`](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#edit-configuration-files) script from the
128 +Netdata [config directory](https://github.com/netdata/netdata/blob/master/docs/netdata-agent/configuration/README.md#locate-your-config-directory).
129 +
130 +```bash
131 +cd /etc/netdata 2>/dev/null || cd /opt/netdata/etc/netdata
132 +sudo ./edit-config netflow.yaml
133 +```
134 +
135 +##### Examples
136 +
137 +###### After running the downloader with IPtoASN ASN and IPIP country
138 +
139 +Auto-detection picks up both generated cache files.
140 +
141 +```yaml
142 +enrichment:
143 + geoip:
144 + asn_database: []
145 + geo_database: []
146 + optional: true
147 +
148 +```
149 +###### Explicit IPIP country path
150 +
151 +Pin IPIP for country and keep a separate ASN source.
152 +
153 +<details open><summary>Config</summary>
154 +
155 +```yaml
156 +enrichment:
157 + geoip:
158 + asn_database:
159 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
160 + geo_database:
161 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
162 + optional: false
163 +
164 +```
165 +</details>
166 +
167 +
168 +
169 +### IPv6 country fields are empty
170 +
171 +The currently validated IPIP `country.zip` source contains IPv4 prefixes.
172 +Use DB-IP, MaxMind, IP2Location, or a custom MMDB if IPv6 country
173 +enrichment is required.
174 +
175 +
176 +### ASN fields are empty
177 +
178 +IPIP country data carries country prefixes only. Configure a separate ASN
179 +provider in the downloader.
180 +
181 +
182 +### City or coordinates are empty
183 +
184 +The supported IPIP source is country-only. Use DB-IP city-lite, MaxMind
185 +GeoLite2 City, GeoIP2 City, or a custom MMDB for richer geo fields.
src/crates/netflow-plugin/integrations/maxmind_geoip_-_geolite2.md
+33 -9
@@ -50,13 +50,19 @@ ASN provider chain, failure-modes table), see
50 [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
51
52
53 -You download the MaxMind MMDB files yourself using `geoipupdate` (MaxMind's
54 -official downloader, configured with your account ID + license key) or via direct
55 -HTTPS download with the same credentials. MaxMind requires authentication for all
56 -downloads -- there is no anonymous access. The bundled
57 -`topology-ip-intel-downloader` shipped by Netdata does **not** know how to fetch
58 -MaxMind databases (it supports DB-IP and IPtoASN only); use `geoipupdate` and
59 -schedule it via cron or systemd-timer.
53 +You can use MaxMind in two ways:
54 +
55 +- Use `geoipupdate` (MaxMind's official downloader, configured with your account
56 + ID + license key) or direct HTTPS download, then point `asn_database` and
57 + `geo_database` at the resulting MaxMind MMDB files.
58 +- Use Netdata's `topology-ip-intel-downloader` with `MAXMIND_LICENSE_KEY` in
59 + the environment to fetch GeoLite2 ASN and GeoLite2 Country CSV sources and
60 + generate the standard Netdata topology MMDB files.
61 +
62 +MaxMind requires authentication for all downloads -- there is no anonymous
63 +access. Commercial GeoIP2 databases and richer GeoLite2 City files should still
64 +be maintained with `geoipupdate` or direct paths when you want city and
65 +coordinate enrichment.
66
67 Once the files are on disk, point `asn_database` and `geo_database` at their
68 paths in `netflow.yaml`. The plugin reloads on file change automatically -- a
@@ -75,11 +81,11 @@ This integration runs as a single instance per Netdata Agent.
81
82 #### Auto-Detection
83
78 -Not auto-detected. The plugin auto-detects the bundled DB-IP files; to use MaxMind you must configure the database paths explicitly. Set `enrichment.geoip.asn_database` and `enrichment.geoip.geo_database` to the MaxMind file paths.
84 +Arbitrary MaxMind paths are not auto-detected; configure `enrichment.geoip.asn_database` and `enrichment.geoip.geo_database` for those. MaxMind-derived topology MMDB files generated by `topology-ip-intel-downloader` are auto-detected because they use the standard Netdata cache paths.
85
86 #### Limits
87
82 -Lookup coverage, fields, license terms, and refresh cadence depend on the MaxMind database edition you configure. Netdata does not fetch MaxMind databases automatically.
88 +Lookup coverage, fields, license terms, and refresh cadence depend on the MaxMind database edition you configure. The plugin itself does no network fetching; the downloader is an operator-run refresh step.
89
90 #### Performance Impact
91
@@ -131,6 +137,24 @@ upstream publish frequency:
137 Source: <https://support.maxmind.com/hc/en-us/articles/4408216129947>.
138
139
140 +#### Optional Netdata topology downloader path
141 +
142 +To generate Netdata's auto-detected topology MMDB files from GeoLite2 ASN
143 +and GeoLite2 Country, run:
144 +
145 +```bash
146 +sudo MAXMIND_LICENSE_KEY="YOUR_LICENSE_KEY" \
147 + /usr/sbin/topology-ip-intel-downloader \
148 + --asn maxmind:geolite2-asn \
149 + --geo maxmind:geolite2-country
150 +```
151 +
152 +This produces `/var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb`
153 +and `/var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb`. The
154 +generated metadata redacts URL query strings; do not put the license key
155 +directly in committed config files.
156 +
157 +
158 #### Single MMDB per database type (dual-stack)
159
160 A single MaxMind MMDB file covers both IPv4 and IPv6 in one binary tree --
src/crates/netflow-plugin/integrations/netbox.md
+22 -16
@@ -52,15 +52,16 @@ formats:
52
53 - **Legacy v1 tokens** (`Authorization: Token <token>`) -- accepted by all NetBox
54 versions, simplest to wire up.
55 -- **v2 tokens** (NetBox 4.x, `Authorization: Bearer nbt_<key>.<token>`) -- the
56 - prefix `nbt_` and the random key are concatenated with the token via a dot.
55 +- **v2 tokens** (NetBox 4.5+, `Authorization: Bearer nbt_<12-char-key>.<40-char-token>`) --
56 + the prefix `nbt_`, the token key, a dot, and the token secret are all part of
57 + the header value.
58
59 The plugin transports either format -- the value is whatever NetBox issued for the
60 service account.
61
62 The plugin runs the configured `transform` (jaq -- a jq-equivalent) over the
63 parsed JSON body and produces per-prefix objects. NetBox's response is paginated;
63 -the plugin does **not** follow `next` links. Pass `?limit=0` (NetBox 4.x default
64 +the plugin does **not** follow `next` links. Pass `?limit=0` (NetBox default
65 `MAX_PAGE_SIZE` is 1000; setting `0` removes the cap when the server config
66 allows) or an explicit `?limit=N` greater than your prefix count, or expose a
67 server-side aggregator that returns the full list at one URL.
@@ -96,15 +97,16 @@ One HTTP request per refresh interval plus a jq transform over the response. Run
97
98 In NetBox, create or reuse a service-account user, then generate an API token
99 under "Admin > Users > Tokens". Restrict the token to read-only and (for
99 -v4.x) limit the scope to `ipam.view_prefix`. The token value goes in the
100 +NetBox 4.x) limit the scope to `ipam.view_prefix`. The token value goes in the
101 `Authorization` header.
102
103 The plugin only reads -- never writes -- so a read-only token is sufficient
104 and recommended. Token format depends on the NetBox version:
105
106 - NetBox 3.x or earlier: `Token <40-char-hex>` (legacy).
106 -- NetBox 4.x: either legacy `Token <hex>` or new `Bearer nbt_<key>.<token>`
107 - (v2 tokens, opt-in).
107 +- NetBox 4.0 through 4.4: legacy `Token <hex>`.
108 +- NetBox 4.5+: legacy `Token <hex>` or new
109 + `Bearer nbt_<12-char-key>.<40-char-token>` (v2 tokens).
110
111
112 #### Bulk endpoint (`?limit=0` or aggregator)
@@ -143,8 +145,8 @@ auth helper.
145
146 | Option | Description | Default | Required |
147 |:-----|:------------|:--------|:---------:|
146 -| url | NetBox prefixes API endpoint, including `?limit=` (recommend `?limit=0` on 4.x for full inventory in one shot). | | yes |
147 -| headers.Authorization | NetBox API token. Use `Token <hex>` for legacy v1 or `Bearer nbt_<key>.<token>` for v4.x v2 tokens. | | yes |
148 +| url | NetBox prefixes API endpoint, including `?limit=` (recommend `?limit=0` for full inventory in one shot when the server's `MAX_PAGE_SIZE` allows it). | | yes |
149 +| headers.Authorization | NetBox API token. Use `Token <hex>` for legacy v1 or `Bearer nbt_<12-char-key>.<40-char-token>` for v4.5+ v2 tokens. | | yes |
150 | interval | How often to refresh. NetBox is your source of truth; 5 minutes is typical for IPAMs that change frequently, 1 hour is fine for static inventories. | 60s | no |
151 | timeout | HTTP request timeout. Bump to 30-60s if your NetBox returns thousands of prefixes in one shot. | 10s | no |
152 | transform | jq expression mapping NetBox's `.results[]` to per-prefix objects with `prefix` and any of `name`, `role`, `site`, `region`, `country`, `state`, `city`, `tenant`, `asn`, `asn_name`. | . | yes |
@@ -170,12 +172,12 @@ sudo ./edit-config netflow.yaml
172
173 ##### Examples
174
173 -###### NetBox 4.x with v2 token, scope-aware mapping
175 +###### NetBox 4.5+ with v2 token, scope-aware mapping
176
177 Standard NetBox 4.2+ wiring. Maps tenant, role, the new `scope` field
178 (covers site / region / site-group / location), and the human-readable
179 description. `?limit=0` returns all prefixes in one call when
178 -`MAX_PAGE_SIZE` is unset on the NetBox server.
180 +`MAX_PAGE_SIZE` is set to `0` or `None` on the NetBox server.
181
182
183 ```yaml
@@ -184,7 +186,7 @@ enrichment:
186 netbox:
187 url: "https://netbox.example.internal/api/ipam/prefixes/?limit=0"
188 headers:
187 - Authorization: "Bearer nbt_AbCdEf0123456789.GhIjKl0123456789"
189 + Authorization: "Bearer nbt_<12-char-key>.<40-char-token>"
190 interval: 5m
191 timeout: 30s
192 transform: |
@@ -294,7 +296,7 @@ enrichment:
296 ### Only first 50 prefixes loaded
297
298 NetBox's default `PAGINATE_COUNT` is 50 and the plugin does not follow `next`
297 -links. Pass `?limit=0` (NetBox 4.x removes the cap when `MAX_PAGE_SIZE` is
299 +links. Pass `?limit=0` (NetBox removes the cap when `MAX_PAGE_SIZE` is
300 `0` server-side) or `?limit=N` larger than your inventory. For inventories
301 above the server's `MAX_PAGE_SIZE` (default 1000), expose a server-side
302 aggregator endpoint.
@@ -302,10 +304,14 @@ aggregator endpoint.
304
305 ### 401 / 403 from NetBox
306
305 -Token missing, expired, or wrong format. Verify with:
306 -`curl -H "Authorization: Token <tok>" https://netbox/api/ipam/prefixes/`.
307 -On NetBox 4.x check whether the token is v1 (`Token <hex>`) or v2
308 -(`Bearer nbt_<key>.<token>`) and use the matching header. Watch the journal
307 +Token missing, expired, or wrong format. Verify with one of:
308 +`curl -H "Authorization: Token <legacy-token>" https://netbox/api/ipam/prefixes/`
309 +or
310 +`curl -H "Authorization: Bearer nbt_<12-char-key>.<40-char-token>" https://netbox/api/ipam/prefixes/`.
311 +On NetBox 4.5+ check whether the token is v1 (`Token <hex>`) or v2
312 +(`Bearer nbt_<12-char-key>.<40-char-token>`) and use the matching header.
313 +A v2 token sent with the legacy `Token` prefix is rejected as an invalid v1
314 +token. Watch the journal
315 for `network-sources` warnings -- HTTP errors are logged there as
316 refresh-failed warnings.
317
src/crates/netflow-plugin/metadata.yaml
+795 -55
@@ -465,6 +465,14 @@ modules:
465 module_name: maxmind
466 - plugin_name: netflow-plugin
467 module_name: iptoasn
468 + - plugin_name: netflow-plugin
469 + module_name: caida-prefix2as
470 + - plugin_name: netflow-plugin
471 + module_name: ip2location
472 + - plugin_name: netflow-plugin
473 + module_name: ipdeny
474 + - plugin_name: netflow-plugin
475 + module_name: ipip
476 - plugin_name: netflow-plugin
477 module_name: custom-mmdb
478 info_provided_to_referring_integrations:
@@ -660,6 +668,14 @@ modules:
668 module_name: dbip
669 - plugin_name: netflow-plugin
670 module_name: iptoasn
671 + - plugin_name: netflow-plugin
672 + module_name: caida-prefix2as
673 + - plugin_name: netflow-plugin
674 + module_name: ip2location
675 + - plugin_name: netflow-plugin
676 + module_name: ipdeny
677 + - plugin_name: netflow-plugin
678 + module_name: ipip
679 - plugin_name: netflow-plugin
680 module_name: custom-mmdb
681 info_provided_to_referring_integrations:
@@ -693,13 +709,19 @@ modules:
709 ASN provider chain, failure-modes table), see
710 [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
711 method_description: |
696 - You download the MaxMind MMDB files yourself using `geoipupdate` (MaxMind's
697 - official downloader, configured with your account ID + license key) or via direct
698 - HTTPS download with the same credentials. MaxMind requires authentication for all
699 - downloads -- there is no anonymous access. The bundled
700 - `topology-ip-intel-downloader` shipped by Netdata does **not** know how to fetch
701 - MaxMind databases (it supports DB-IP and IPtoASN only); use `geoipupdate` and
702 - schedule it via cron or systemd-timer.
712 + You can use MaxMind in two ways:
713 +
714 + - Use `geoipupdate` (MaxMind's official downloader, configured with your account
715 + ID + license key) or direct HTTPS download, then point `asn_database` and
716 + `geo_database` at the resulting MaxMind MMDB files.
717 + - Use Netdata's `topology-ip-intel-downloader` with `MAXMIND_LICENSE_KEY` in
718 + the environment to fetch GeoLite2 ASN and GeoLite2 Country CSV sources and
719 + generate the standard Netdata topology MMDB files.
720 +
721 + MaxMind requires authentication for all downloads -- there is no anonymous
722 + access. Commercial GeoIP2 databases and richer GeoLite2 City files should still
723 + be maintained with `geoipupdate` or direct paths when you want city and
724 + coordinate enrichment.
725
726 Once the files are on disk, point `asn_database` and `geo_database` at their
727 paths in `netflow.yaml`. The plugin reloads on file change automatically -- a
@@ -714,9 +736,9 @@ modules:
736 description: ""
737 default_behavior:
738 auto_detection:
717 - description: "Not auto-detected. The plugin auto-detects the bundled DB-IP files; to use MaxMind you must configure the database paths explicitly. Set `enrichment.geoip.asn_database` and `enrichment.geoip.geo_database` to the MaxMind file paths."
739 + description: "Arbitrary MaxMind paths are not auto-detected; configure `enrichment.geoip.asn_database` and `enrichment.geoip.geo_database` for those. MaxMind-derived topology MMDB files generated by `topology-ip-intel-downloader` are auto-detected because they use the standard Netdata cache paths."
740 limits:
719 - description: "Lookup coverage, fields, license terms, and refresh cadence depend on the MaxMind database edition you configure. Netdata does not fetch MaxMind databases automatically."
741 + description: "Lookup coverage, fields, license terms, and refresh cadence depend on the MaxMind database edition you configure. The plugin itself does no network fetching; the downloader is an operator-run refresh step."
742 performance_impact:
743 description: "Lookups are local MMDB reads with no per-flow network call. Memory use is mostly the mapped database files and the kernel page cache needed to keep active pages hot."
744 setup:
@@ -759,6 +781,22 @@ modules:
781 Friday" per <https://www.maxmind.com/en/geoip2-databases>.
782
783 Source: <https://support.maxmind.com/hc/en-us/articles/4408216129947>.
784 + - title: Optional Netdata topology downloader path
785 + description: |
786 + To generate Netdata's auto-detected topology MMDB files from GeoLite2 ASN
787 + and GeoLite2 Country, run:
788 +
789 + ```bash
790 + sudo MAXMIND_LICENSE_KEY="YOUR_LICENSE_KEY" \
791 + /usr/sbin/topology-ip-intel-downloader \
792 + --asn maxmind:geolite2-asn \
793 + --geo maxmind:geolite2-country
794 + ```
795 +
796 + This produces `/var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb`
797 + and `/var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb`. The
798 + generated metadata redacts URL query strings; do not put the license key
799 + directly in committed config files.
800 - title: Single MMDB per database type (dual-stack)
801 description: |
802 A single MaxMind MMDB file covers both IPv4 and IPv6 in one binary tree --
@@ -899,6 +937,14 @@ modules:
937 module_name: dbip
938 - plugin_name: netflow-plugin
939 module_name: maxmind
940 + - plugin_name: netflow-plugin
941 + module_name: caida-prefix2as
942 + - plugin_name: netflow-plugin
943 + module_name: ip2location
944 + - plugin_name: netflow-plugin
945 + module_name: ipdeny
946 + - plugin_name: netflow-plugin
947 + module_name: ipip
948 - plugin_name: netflow-plugin
949 module_name: custom-mmdb
950 info_provided_to_referring_integrations:
@@ -1100,6 +1146,672 @@ modules:
1146 `*_COUNTRY` columns.
1147 availability: []
1148 scopes: []
1149 + - meta:
1150 + plugin_name: netflow-plugin
1151 + module_name: caida-prefix2as
1152 + monitored_instance:
1153 + name: CAIDA Routeviews Prefix-to-AS
1154 + link: https://www.caida.org/catalog/datasets/routeviews-prefix2as/
1155 + categories:
1156 + - flows.enrichment-methods
1157 + icon_filename: network-wired.svg
1158 + keywords:
1159 + - caida
1160 + - routeviews
1161 + - prefix2as
1162 + - pfx2as
1163 + - asn
1164 + - bgp
1165 + - ip intelligence
1166 + - flow enrichment
1167 + related_resources:
1168 + integrations:
1169 + list:
1170 + - plugin_name: netflow-plugin
1171 + module_name: dbip
1172 + - plugin_name: netflow-plugin
1173 + module_name: maxmind
1174 + - plugin_name: netflow-plugin
1175 + module_name: iptoasn
1176 + - plugin_name: netflow-plugin
1177 + module_name: custom-mmdb
1178 + info_provided_to_referring_integrations:
1179 + description: ""
1180 + overview:
1181 + data_collection:
1182 + metrics_description: |
1183 + Enrich network flows with AS numbers from CAIDA's Routeviews Prefix-to-AS
1184 + mapping dataset. CAIDA derives this public dataset from RouteViews BGP data and
1185 + publishes one prefix-to-origin-AS mapping per line. The official format is
1186 + tab-separated `IP prefix`, `prefix length`, and `AS number`; multi-origin AS
1187 + entries can contain multiple ASNs, and the Netdata downloader keeps the first
1188 + listed origin AS as the simplified per-prefix value.
1189 +
1190 + CAIDA Prefix-to-AS is ASN-only. It does not provide AS organization names or
1191 + geographic fields. Pair it with DB-IP, MaxMind, IP2Location, IPDeny, IPIP, or
1192 + another geo provider when you also need `*_COUNTRY`, city, state, or
1193 + coordinates.
1194 +
1195 + For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
1196 + provider chains), see
1197 + [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
1198 + method_description: |
1199 + The netflow plugin reads MMDB files only. CAIDA publishes compressed text data,
1200 + so use Netdata's `topology-ip-intel-downloader` to fetch the latest Routeviews
1201 + Prefix-to-AS file, convert it to `topology-ip-asn.mmdb`, and place it in the
1202 + standard auto-detected cache directory. The plugin reloads the generated MMDB
1203 + automatically when the file changes.
1204 + supported_platforms:
1205 + include:
1206 + - Linux
1207 + exclude: []
1208 + multi_instance: false
1209 + additional_permissions:
1210 + description: ""
1211 + default_behavior:
1212 + auto_detection:
1213 + description: "Not used by the default install. After the downloader writes `topology-ip-asn.mmdb` into the standard cache path, the plugin auto-detects the generated file."
1214 + limits:
1215 + description: "ASN number only. No AS organization name, country, state, city, or coordinates are available from this source."
1216 + performance_impact:
1217 + description: "Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot."
1218 + setup:
1219 + prerequisites:
1220 + list:
1221 + - title: Downloader with CAIDA Prefix-to-AS support
1222 + description: |
1223 + Run the bundled downloader when available. Packaged 32-bit installs do not
1224 + include this binary:
1225 +
1226 + ```bash
1227 + sudo /usr/sbin/topology-ip-intel-downloader \
1228 + --asn caida:prefix2as \
1229 + --geo dbip:city-lite
1230 + ```
1231 +
1232 + `caida:prefix2as` is valid only for `--asn`. The example keeps DB-IP
1233 + city-lite for geo fields; substitute any supported `--geo` provider if
1234 + country-only enrichment is sufficient.
1235 + - title: CAIDA acceptable use terms
1236 + description: |
1237 + CAIDA documents acceptable-use and attribution requirements on the dataset
1238 + page. Review the terms before redistributing generated datasets or derived
1239 + reports.
1240 + configuration:
1241 + file:
1242 + name: netflow.yaml
1243 + options:
1244 + description: |
1245 + Once the downloader has produced a CAIDA-derived ASN MMDB in the cache
1246 + directory, the plugin auto-detects it. To pin paths explicitly, set
1247 + `enrichment.geoip.asn_database` and keep `geo_database` pointed at your chosen
1248 + geographic provider.
1249 + folding:
1250 + title: Config options
1251 + enabled: true
1252 + list:
1253 + - name: enrichment.geoip.asn_database
1254 + description: Path to the CAIDA-derived ASN MMDB. Empty = auto-detect from the cache directory.
1255 + default_value: "[]"
1256 + required: false
1257 + - name: enrichment.geoip.geo_database
1258 + description: Optional geographic MMDB paths supplied by another provider.
1259 + default_value: "[]"
1260 + required: false
1261 + - name: enrichment.geoip.optional
1262 + description: When true, missing or unreadable MMDBs are warnings, not fatal at startup.
1263 + default_value: "false (true when auto-detected)"
1264 + required: false
1265 + examples:
1266 + folding:
1267 + title: Config
1268 + enabled: true
1269 + list:
1270 + - name: After running the downloader with CAIDA ASN and DB-IP geo
1271 + folding:
1272 + enabled: false
1273 + description: Auto-detection picks up both generated cache files.
1274 + config: |
1275 + enrichment:
1276 + geoip:
1277 + asn_database: []
1278 + geo_database: []
1279 + optional: true
1280 + - name: Explicit CAIDA ASN path
1281 + folding:
1282 + enabled: true
1283 + description: Pin CAIDA for AS numbers and keep a separate geo source for country/city fields.
1284 + config: |
1285 + enrichment:
1286 + geoip:
1287 + asn_database:
1288 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
1289 + geo_database:
1290 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
1291 + optional: false
1292 + troubleshooting:
1293 + problems:
1294 + list:
1295 + - name: AS names are empty
1296 + description: |
1297 + CAIDA Prefix-to-AS contains AS numbers, not organization names. Layer a
1298 + provider with AS organization data, such as MaxMind GeoLite2 ASN or DB-IP
1299 + ASN Lite, after the CAIDA MMDB in `asn_database` if AS names matter.
1300 + - name: Country fields are empty
1301 + description: |
1302 + This source is ASN-only. Configure a `--geo` provider in the downloader or
1303 + point `enrichment.geoip.geo_database` at a separate geo MMDB.
1304 + - name: Multi-origin AS simplification
1305 + description: |
1306 + CAIDA can represent multi-origin prefixes. The downloader uses the first
1307 + listed AS for the generated single-value MMDB record. If you need full MOAS
1308 + semantics, use BMP or BioRIS routing enrichment instead of an offline ASN
1309 + lookup database.
1310 + alerts: []
1311 + metrics:
1312 + folding:
1313 + title: Metrics
1314 + enabled: false
1315 + description: |
1316 + Enriches flow records with AS numbers; produces no metrics of its own.
1317 + Verify on the Network Flows view via `SRC_AS` and `DST_AS`.
1318 + availability: []
1319 + scopes: []
1320 + - meta:
1321 + plugin_name: netflow-plugin
1322 + module_name: ip2location
1323 + monitored_instance:
1324 + name: IP2Location LITE IP-Country
1325 + link: https://lite.ip2location.com/database/db1-ip-country?lang=en_US
1326 + categories:
1327 + - flows.enrichment-methods
1328 + icon_filename: network-wired.svg
1329 + keywords:
1330 + - ip2location
1331 + - ip2location lite
1332 + - geoip
1333 + - country
1334 + - csv
1335 + - ip intelligence
1336 + - flow enrichment
1337 + related_resources:
1338 + integrations:
1339 + list:
1340 + - plugin_name: netflow-plugin
1341 + module_name: dbip
1342 + - plugin_name: netflow-plugin
1343 + module_name: maxmind
1344 + - plugin_name: netflow-plugin
1345 + module_name: iptoasn
1346 + - plugin_name: netflow-plugin
1347 + module_name: custom-mmdb
1348 + info_provided_to_referring_integrations:
1349 + description: ""
1350 + overview:
1351 + data_collection:
1352 + metrics_description: |
1353 + Enrich network flows with country codes from the IP2Location LITE
1354 + IP-Country database. The supported downloader source is the DB1 LITE CSV ZIP
1355 + (`IP2LOCATION-LITE-DB1.CSV.ZIP`), whose documented fields are
1356 + `ip_from`, `ip_to`, `country_code`, and `country_name`.
1357 +
1358 + IP2Location DB1 is country-only. It does not provide ASN, AS organization,
1359 + state, city, latitude, or longitude. Pair it with CAIDA, IPtoASN, DB-IP, or
1360 + MaxMind ASN data when you also need AS fields.
1361 +
1362 + For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
1363 + provider chains), see
1364 + [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
1365 + method_description: |
1366 + The netflow plugin reads MMDB files only. Use Netdata's
1367 + `topology-ip-intel-downloader` to fetch the IP2Location LITE CSV ZIP, convert
1368 + the ranges to `topology-ip-geo.mmdb`, and write the file into the standard
1369 + auto-detected cache directory. The plugin reloads the generated MMDB
1370 + automatically when the file changes.
1371 + supported_platforms:
1372 + include:
1373 + - Linux
1374 + exclude: []
1375 + multi_instance: false
1376 + additional_permissions:
1377 + description: ""
1378 + default_behavior:
1379 + auto_detection:
1380 + description: "Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file."
1381 + limits:
1382 + description: "Country only. No ASN, AS name, state, city, latitude, or longitude are available from this source."
1383 + performance_impact:
1384 + description: "Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot."
1385 + setup:
1386 + prerequisites:
1387 + list:
1388 + - title: Downloader with IP2Location LITE support
1389 + description: |
1390 + Run the bundled downloader when available. Packaged 32-bit installs do not
1391 + include this binary:
1392 +
1393 + ```bash
1394 + sudo /usr/sbin/topology-ip-intel-downloader \
1395 + --asn iptoasn:combined \
1396 + --geo ip2location:country-lite
1397 + ```
1398 +
1399 + `ip2location:country-lite` is valid only for `--geo`. The example pairs it
1400 + with IPtoASN so the resulting cache contains both AS and country fields.
1401 + - title: IP2Location LITE license
1402 + description: |
1403 + Review the IP2Location LITE terms before redistributing the data or
1404 + generated derivative files. The plugin consumes only the generated MMDB and
1405 + does not contact IP2Location during flow decoding.
1406 + configuration:
1407 + file:
1408 + name: netflow.yaml
1409 + options:
1410 + description: |
1411 + Once the downloader has produced an IP2Location-derived geo MMDB in the cache
1412 + directory, the plugin auto-detects it. To pin paths explicitly, set
1413 + `enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
1414 + ASN provider.
1415 + folding:
1416 + title: Config options
1417 + enabled: true
1418 + list:
1419 + - name: enrichment.geoip.asn_database
1420 + description: Optional ASN MMDB paths supplied by another provider.
1421 + default_value: "[]"
1422 + required: false
1423 + - name: enrichment.geoip.geo_database
1424 + description: Path to the IP2Location-derived geo MMDB. Empty = auto-detect from the cache directory.
1425 + default_value: "[]"
1426 + required: false
1427 + - name: enrichment.geoip.optional
1428 + description: When true, missing or unreadable MMDBs are warnings, not fatal at startup.
1429 + default_value: "false (true when auto-detected)"
1430 + required: false
1431 + examples:
1432 + folding:
1433 + title: Config
1434 + enabled: true
1435 + list:
1436 + - name: After running the downloader with IPtoASN ASN and IP2Location country
1437 + folding:
1438 + enabled: false
1439 + description: Auto-detection picks up both generated cache files.
1440 + config: |
1441 + enrichment:
1442 + geoip:
1443 + asn_database: []
1444 + geo_database: []
1445 + optional: true
1446 + - name: Explicit IP2Location country path
1447 + folding:
1448 + enabled: true
1449 + description: Pin IP2Location for country and keep a separate ASN source.
1450 + config: |
1451 + enrichment:
1452 + geoip:
1453 + asn_database:
1454 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
1455 + geo_database:
1456 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
1457 + optional: false
1458 + troubleshooting:
1459 + problems:
1460 + list:
1461 + - name: ASN fields are empty
1462 + description: |
1463 + IP2Location DB1 is country-only. Configure `--asn iptoasn:combined`,
1464 + `--asn caida:prefix2as`, or another ASN provider in the downloader.
1465 + - name: City or coordinates are empty
1466 + description: |
1467 + The DB1 LITE source does not include city, state, latitude, or longitude.
1468 + Use DB-IP city-lite, MaxMind GeoLite2 City, GeoIP2 City, or a custom MMDB
1469 + when those fields are required.
1470 + - name: Stale country data
1471 + description: |
1472 + Check the upstream IP2Location LITE page for the current version and next
1473 + update date, then schedule the downloader accordingly. The plugin reloads
1474 + the generated MMDB in place when the file signature changes.
1475 + alerts: []
1476 + metrics:
1477 + folding:
1478 + title: Metrics
1479 + enabled: false
1480 + description: |
1481 + Enriches flow records with country codes; produces no metrics of its own.
1482 + Verify on the Network Flows view via `SRC_COUNTRY` and `DST_COUNTRY`.
1483 + availability: []
1484 + scopes: []
1485 + - meta:
1486 + plugin_name: netflow-plugin
1487 + module_name: ipdeny
1488 + monitored_instance:
1489 + name: IPDeny Country Zones
1490 + link: https://www.ipdeny.com/ipblocks/
1491 + categories:
1492 + - flows.enrichment-methods
1493 + icon_filename: network-wired.svg
1494 + keywords:
1495 + - ipdeny
1496 + - country zones
1497 + - geoip
1498 + - country
1499 + - cidr
1500 + - ip intelligence
1501 + - flow enrichment
1502 + related_resources:
1503 + integrations:
1504 + list:
1505 + - plugin_name: netflow-plugin
1506 + module_name: dbip
1507 + - plugin_name: netflow-plugin
1508 + module_name: maxmind
1509 + - plugin_name: netflow-plugin
1510 + module_name: iptoasn
1511 + - plugin_name: netflow-plugin
1512 + module_name: custom-mmdb
1513 + info_provided_to_referring_integrations:
1514 + description: ""
1515 + overview:
1516 + data_collection:
1517 + metrics_description: |
1518 + Enrich network flows with country codes from IPDeny country zone files.
1519 + IPDeny publishes country-specific CIDR lists and an `all-zones.tar.gz`
1520 + archive. The Netdata downloader reads the IPv4 country-zone archive and maps
1521 + each `<country>.zone` member to that ISO country code.
1522 +
1523 + IPDeny Country Zones are country-only and the current built-in source is IPv4
1524 + only. They do not provide ASN, AS organization, state, city, latitude, or
1525 + longitude. Pair IPDeny with another ASN source when you also need AS fields,
1526 + and use DB-IP, MaxMind, IP2Location, or a custom MMDB when IPv6 country
1527 + coverage is required.
1528 +
1529 + For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
1530 + provider chains), see
1531 + [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
1532 + method_description: |
1533 + The netflow plugin reads MMDB files only. Use Netdata's
1534 + `topology-ip-intel-downloader` to fetch IPDeny's HTTPS country-zone archive,
1535 + convert the CIDR lists to `topology-ip-geo.mmdb`, and write the file into the
1536 + standard auto-detected cache directory. The plugin reloads the generated MMDB
1537 + automatically when the file changes.
1538 + supported_platforms:
1539 + include:
1540 + - Linux
1541 + exclude: []
1542 + multi_instance: false
1543 + additional_permissions:
1544 + description: ""
1545 + default_behavior:
1546 + auto_detection:
1547 + description: "Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file."
1548 + limits:
1549 + description: "Country-only IPv4 source. No ASN, AS name, IPv6 coverage, state, city, latitude, or longitude are available from this built-in source."
1550 + performance_impact:
1551 + description: "Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot."
1552 + setup:
1553 + prerequisites:
1554 + list:
1555 + - title: Downloader with IPDeny support
1556 + description: |
1557 + Run the bundled downloader when available. Packaged 32-bit installs do not
1558 + include this binary:
1559 +
1560 + ```bash
1561 + sudo /usr/sbin/topology-ip-intel-downloader \
1562 + --asn iptoasn:combined \
1563 + --geo ipdeny:country-zones
1564 + ```
1565 +
1566 + `ipdeny:country-zones` is valid only for `--geo`. The example pairs it
1567 + with IPtoASN so the resulting cache contains both AS and country fields.
1568 + - title: IPDeny usage policy
1569 + description: |
1570 + IPDeny publishes usage limits and copyright policy on its site. Review
1571 + those terms before mirroring, redistributing, or downloading more often
1572 + than needed.
1573 + configuration:
1574 + file:
1575 + name: netflow.yaml
1576 + options:
1577 + description: |
1578 + Once the downloader has produced an IPDeny-derived geo MMDB in the cache
1579 + directory, the plugin auto-detects it. To pin paths explicitly, set
1580 + `enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
1581 + ASN provider.
1582 + folding:
1583 + title: Config options
1584 + enabled: true
1585 + list:
1586 + - name: enrichment.geoip.asn_database
1587 + description: Optional ASN MMDB paths supplied by another provider.
1588 + default_value: "[]"
1589 + required: false
1590 + - name: enrichment.geoip.geo_database
1591 + description: Path to the IPDeny-derived geo MMDB. Empty = auto-detect from the cache directory.
1592 + default_value: "[]"
1593 + required: false
1594 + - name: enrichment.geoip.optional
1595 + description: When true, missing or unreadable MMDBs are warnings, not fatal at startup.
1596 + default_value: "false (true when auto-detected)"
1597 + required: false
1598 + examples:
1599 + folding:
1600 + title: Config
1601 + enabled: true
1602 + list:
1603 + - name: After running the downloader with IPtoASN ASN and IPDeny country
1604 + folding:
1605 + enabled: false
1606 + description: Auto-detection picks up both generated cache files.
1607 + config: |
1608 + enrichment:
1609 + geoip:
1610 + asn_database: []
1611 + geo_database: []
1612 + optional: true
1613 + - name: Explicit IPDeny country path
1614 + folding:
1615 + enabled: true
1616 + description: Pin IPDeny for country and keep a separate ASN source.
1617 + config: |
1618 + enrichment:
1619 + geoip:
1620 + asn_database:
1621 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
1622 + geo_database:
1623 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
1624 + optional: false
1625 + troubleshooting:
1626 + problems:
1627 + list:
1628 + - name: IPv6 country fields are empty
1629 + description: |
1630 + The current built-in IPDeny downloader source uses the IPv4
1631 + `all-zones.tar.gz` archive. Use DB-IP, MaxMind, IP2Location, or a custom
1632 + MMDB if IPv6 country enrichment is required.
1633 + - name: ASN fields are empty
1634 + description: |
1635 + IPDeny zone files carry country CIDRs only. Configure a separate ASN
1636 + provider in the downloader.
1637 + - name: Download blocked or rate-limited
1638 + description: |
1639 + Check IPDeny's usage limits and retry later. The plugin continues using the
1640 + last generated MMDB file until the downloader publishes a replacement.
1641 + alerts: []
1642 + metrics:
1643 + folding:
1644 + title: Metrics
1645 + enabled: false
1646 + description: |
1647 + Enriches IPv4 flow records with country codes; produces no metrics of its own.
1648 + Verify on the Network Flows view via `SRC_COUNTRY` and `DST_COUNTRY`.
1649 + availability: []
1650 + scopes: []
1651 + - meta:
1652 + plugin_name: netflow-plugin
1653 + module_name: ipip
1654 + monitored_instance:
1655 + name: IPIP Country Database
1656 + link: https://en.ipip.net/about.html
1657 + categories:
1658 + - flows.enrichment-methods
1659 + icon_filename: network-wired.svg
1660 + keywords:
1661 + - ipip
1662 + - 17mon
1663 + - geoip
1664 + - country
1665 + - ip intelligence
1666 + - flow enrichment
1667 + related_resources:
1668 + integrations:
1669 + list:
1670 + - plugin_name: netflow-plugin
1671 + module_name: dbip
1672 + - plugin_name: netflow-plugin
1673 + module_name: maxmind
1674 + - plugin_name: netflow-plugin
1675 + module_name: iptoasn
1676 + - plugin_name: netflow-plugin
1677 + module_name: custom-mmdb
1678 + info_provided_to_referring_integrations:
1679 + description: ""
1680 + overview:
1681 + data_collection:
1682 + metrics_description: |
1683 + Enrich network flows with country codes from IPIP's country database. The
1684 + supported downloader source fetches IPIP's `country.zip` archive and reads the
1685 + `country.txt` file, where each line maps a prefix to a country token.
1686 +
1687 + IPIP Country is country-only. The current `country.zip` source validated during
1688 + this work contained IPv4 prefixes only. It does not provide ASN, AS
1689 + organization, state, city, latitude, or longitude. Pair IPIP with another ASN
1690 + source when you also need AS fields, and use DB-IP, MaxMind, IP2Location, or a
1691 + custom MMDB when IPv6 country coverage is required.
1692 +
1693 + For the IP-intelligence mechanism (lookup order, MMDB composition, refresh,
1694 + provider chains), see
1695 + [IP Intelligence](https://learn.netdata.cloud/docs/network-flows/enrichment).
1696 + method_description: |
1697 + The netflow plugin reads MMDB files only. Use Netdata's
1698 + `topology-ip-intel-downloader` to fetch the IPIP country ZIP, convert the
1699 + prefix list to `topology-ip-geo.mmdb`, and write the file into the standard
1700 + auto-detected cache directory. The plugin reloads the generated MMDB
1701 + automatically when the file changes.
1702 + supported_platforms:
1703 + include:
1704 + - Linux
1705 + exclude: []
1706 + multi_instance: false
1707 + additional_permissions:
1708 + description: ""
1709 + default_behavior:
1710 + auto_detection:
1711 + description: "Not used by the default install. After the downloader writes `topology-ip-geo.mmdb` into the standard cache path, the plugin auto-detects the generated file."
1712 + limits:
1713 + description: "Country-only source. The currently validated `country.zip` payload is IPv4-only and carries no ASN, AS name, state, city, latitude, or longitude."
1714 + performance_impact:
1715 + description: "Lookups are local MMDB reads after conversion. Memory use is mostly the mapped database file and the kernel page cache needed to keep active pages hot."
1716 + setup:
1717 + prerequisites:
1718 + list:
1719 + - title: Downloader with IPIP support
1720 + description: |
1721 + Run the bundled downloader when available. Packaged 32-bit installs do not
1722 + include this binary:
1723 +
1724 + ```bash
1725 + sudo /usr/sbin/topology-ip-intel-downloader \
1726 + --asn iptoasn:combined \
1727 + --geo ipip:country
1728 + ```
1729 +
1730 + `ipip:country` is valid only for `--geo`. The example pairs it with
1731 + IPtoASN so the resulting cache contains both AS and country fields.
1732 + - title: IPIP data terms
1733 + description: |
1734 + Review IPIP's current terms before redistributing the data or generated
1735 + derivative files. The plugin consumes only the generated MMDB and does not
1736 + contact IPIP during flow decoding.
1737 + configuration:
1738 + file:
1739 + name: netflow.yaml
1740 + options:
1741 + description: |
1742 + Once the downloader has produced an IPIP-derived geo MMDB in the cache
1743 + directory, the plugin auto-detects it. To pin paths explicitly, set
1744 + `enrichment.geoip.geo_database` and keep `asn_database` pointed at your chosen
1745 + ASN provider.
1746 + folding:
1747 + title: Config options
1748 + enabled: true
1749 + list:
1750 + - name: enrichment.geoip.asn_database
1751 + description: Optional ASN MMDB paths supplied by another provider.
1752 + default_value: "[]"
1753 + required: false
1754 + - name: enrichment.geoip.geo_database
1755 + description: Path to the IPIP-derived geo MMDB. Empty = auto-detect from the cache directory.
1756 + default_value: "[]"
1757 + required: false
1758 + - name: enrichment.geoip.optional
1759 + description: When true, missing or unreadable MMDBs are warnings, not fatal at startup.
1760 + default_value: "false (true when auto-detected)"
1761 + required: false
1762 + examples:
1763 + folding:
1764 + title: Config
1765 + enabled: true
1766 + list:
1767 + - name: After running the downloader with IPtoASN ASN and IPIP country
1768 + folding:
1769 + enabled: false
1770 + description: Auto-detection picks up both generated cache files.
1771 + config: |
1772 + enrichment:
1773 + geoip:
1774 + asn_database: []
1775 + geo_database: []
1776 + optional: true
1777 + - name: Explicit IPIP country path
1778 + folding:
1779 + enabled: true
1780 + description: Pin IPIP for country and keep a separate ASN source.
1781 + config: |
1782 + enrichment:
1783 + geoip:
1784 + asn_database:
1785 + - /var/cache/netdata/topology-ip-intel/topology-ip-asn.mmdb
1786 + geo_database:
1787 + - /var/cache/netdata/topology-ip-intel/topology-ip-geo.mmdb
1788 + optional: false
1789 + troubleshooting:
1790 + problems:
1791 + list:
1792 + - name: IPv6 country fields are empty
1793 + description: |
1794 + The currently validated IPIP `country.zip` source contains IPv4 prefixes.
1795 + Use DB-IP, MaxMind, IP2Location, or a custom MMDB if IPv6 country
1796 + enrichment is required.
1797 + - name: ASN fields are empty
1798 + description: |
1799 + IPIP country data carries country prefixes only. Configure a separate ASN
1800 + provider in the downloader.
1801 + - name: City or coordinates are empty
1802 + description: |
1803 + The supported IPIP source is country-only. Use DB-IP city-lite, MaxMind
1804 + GeoLite2 City, GeoIP2 City, or a custom MMDB for richer geo fields.
1805 + alerts: []
1806 + metrics:
1807 + folding:
1808 + title: Metrics
1809 + enabled: false
1810 + description: |
1811 + Enriches flow records with country codes; produces no metrics of its own.
1812 + Verify on the Network Flows view via `SRC_COUNTRY` and `DST_COUNTRY`.
1813 + availability: []
1814 + scopes: []
1815 - meta:
1816 plugin_name: netflow-plugin
1817 module_name: custom-mmdb
@@ -1126,6 +1838,14 @@ modules:
1838 module_name: maxmind
1839 - plugin_name: netflow-plugin
1840 module_name: iptoasn
1841 + - plugin_name: netflow-plugin
1842 + module_name: caida-prefix2as
1843 + - plugin_name: netflow-plugin
1844 + module_name: ip2location
1845 + - plugin_name: netflow-plugin
1846 + module_name: ipdeny
1847 + - plugin_name: netflow-plugin
1848 + module_name: ipip
1849 info_provided_to_referring_integrations:
1850 description: ""
1851 overview:
@@ -1362,8 +2082,9 @@ modules:
2082 for BMP. Each connecting router must first send an Initiation message; the plugin
2083 then processes RouteMonitoring (carrying BGP UPDATE), PeerDownNotification, and
2084 Termination frames. PeerUp, StatisticsReport, and RouteMirroring frames are
1365 - accepted but not acted on. Only BMP **version 3** is processed; v1 and v2 frames
1366 - are silently dropped.
2085 + accepted but not acted on. Only BMP **version 3** is processed. Version 4 frames
2086 + are decoded but ignored; draft v1/v2 frames are decode errors and count toward
2087 + the consecutive decode-error threshold.
2088
2089 NLRI families parsed: IPv4/IPv6 unicast, IPv4/IPv6 MPLS-labelled, VPNv4, VPNv6,
2090 and EVPN IP-prefix routes.
@@ -1409,8 +2130,9 @@ modules:
2130 module: `bgpd` must be started with `-M bmp` or every BMP command silently
2131 fails.
2132
1412 - The plugin parses RFC 7854 BMP **version 3** only. Older draft versions (v1,
1413 - v2) are silently dropped.
2133 + The plugin parses RFC 7854 BMP **version 3** only. Version 4 frames are decoded
2134 + but ignored. Older draft versions (v1, v2) are decode errors and count toward
2135 + the consecutive decode-error threshold.
2136 - title: TCP reachability between routers and the agent
2137 description: |
2138 Routers initiate the connection -- the plugin is a passive listener. Allow
@@ -1700,20 +2422,21 @@ modules:
2422 overview:
2423 data_collection:
2424 metrics_description: |
1703 - Enrich network flows with BGP routing context received from bio-rd / RIPE RIS
1704 - over gRPC. [bio-rd](https://github.com/bio-routing/bio-rd)
1705 - is a Go-based BGP/BMP daemon that you run yourself. You point it at one or more
1706 - BGP / BMP sources -- your own routers, a [RIPE RIS](https://www.ripe.net/analyse/internet-measurements/routing-information-service-ris)
1707 - Route Collector you peer with, or any other reachable BGP source -- and it
1708 - exposes the resulting RIB through a gRPC interface (`RoutingInformationService`).
1709 - Netdata is a **client** of that interface, not of RIPE NCC directly.
1710 -
1711 - Pick this when you want a third-party / external BGP view (e.g. RIPE RIS's view of the
1712 - public DFZ) without exposing a BMP listener on your agent or running BMP across your
1713 - network. If your routers can speak BMP straight to Netdata, the `bmp` integration is
1714 - simpler -- BioRIS makes sense when bio-rd is already part of your routing toolbox or
1715 - when you want to peer with an external party (RIPE RIS, looking-glass) and re-export
1716 - to Netdata.
2425 + Enrich network flows with BGP routing context received from a bio-rd-compatible
2426 + `RoutingInformationService` gRPC endpoint. [bio-rd](https://github.com/bio-routing/bio-rd)
2427 + is a Go-based BGP/BMP daemon that you run yourself. Its `ris` daemon receives BMP
2428 + sessions from routers or collectors and exposes the resulting RIB through gRPC.
2429 + Netdata is a **client** of that gRPC interface only.
2430 +
2431 + Netdata does **not** connect directly to RIPE NCC RIS Live, RIPEstat, RIS MRT dumps,
2432 + or RIPE route collector sessions. RIPE RIS public access is WebSocket JSON, HTTP API,
2433 + and MRT dump based. To use a RIPE-derived external view, run a bridge or service that
2434 + imports that data into a bio-rd-compatible `RoutingInformationService`, then point
2435 + Netdata at that service.
2436 +
2437 + Pick this when bio-rd is already part of your routing toolbox, or when you want Netdata
2438 + to consume a routing view that is already exposed through the bio-rd RIS gRPC API. If
2439 + your routers can speak BMP straight to Netdata, the `bmp` integration is simpler.
2440
2441 BioRIS populates the same flow-record fields as BMP -- both feed a single shared
2442 in-memory routing trie. See the BMP integration card for the side-by-side comparison
@@ -1793,10 +2516,13 @@ modules:
2516 go build -o /usr/local/bin/ris .
2517 ```
2518
1796 - Configure `ris` to peer with one or more BGP / BMP sources (your own routers,
1797 - RIPE RIS Route Collectors, looking-glass servers, etc.). Refer to the bio-rd
1798 - documentation for the peering setup -- this is bio-rd's configuration, not
1799 - Netdata's.
2519 + Configure `ris` with one or more BMP sources using bio-rd's `bmp_addr` /
2520 + `bmp_servers` configuration. Refer to the bio-rd documentation for the BMP setup
2521 + -- this is bio-rd's configuration, not Netdata's.
2522 +
2523 + Do not set Netdata's `grpc_addr` to a RIPE RIS Live URL, RIPEstat API URL, MRT dump
2524 + URL, or route collector session address. Those endpoints are not the
2525 + `RoutingInformationService` gRPC API that Netdata consumes.
2526
2527 Run the daemon with a gRPC port:
2528 `/usr/local/bin/ris --grpc_port 50051 --config.file /etc/bio-rd.yml`
@@ -1897,9 +2623,10 @@ modules:
2623 folding:
2624 enabled: true
2625 description: |
1900 - Run BMP from internal routers and BioRIS for an external (RIPE RIS) view.
1901 - Both populate the same shared trie -- lookups pick the best-matching route
1902 - across both sources at query time.
2626 + Run BMP from internal routers and BioRIS from a separate bio-rd-compatible RIS
2627 + service, for example one populated from an external routing view. Both populate
2628 + the same shared trie -- lookups pick the best-matching route across both sources
2629 + at query time.
2630 config: |
2631 enrichment:
2632 routing_dynamic:
@@ -1922,16 +2649,23 @@ modules:
2649 message stream reads (default 10s).
2650 - name: Initial dump takes minutes for full feeds
2651 description: |
1925 - A full IPv4+IPv6 RIB from a route collector is millions of prefixes. The
1926 - first `DumpRIB` after enabling (or after a plugin restart) takes time -- BGP
1927 - attribution is incomplete until it finishes. Subsequent `ObserveRIB` streams
1928 - are incremental.
2652 + A full IPv4+IPv6 RIB from a full-table bio-rd source can contain millions of
2653 + prefixes. The first `DumpRIB` after enabling (or after a plugin restart) takes
2654 + time -- BGP attribution is incomplete until it finishes. Subsequent
2655 + `ObserveRIB` streams are incremental.
2656 - name: Plugin restart wipes the trie
2657 description: |
2658 The trie is in-memory only -- restarting the netflow plugin loses every
2659 learned BGP route. Convergence over BioRIS depends on the upstream feed; a
1933 - full DumpRIB from a RIPE RIS Route Collector can take minutes. Schedule
2660 + full DumpRIB from a full-table bio-rd source can take minutes. Schedule
2661 restarts off-peak if BGP attribution matters for your workflow.
2662 + - name: Pointing grpc_addr at RIPE RIS does not work
2663 + description: |
2664 + `grpc_addr` must point to a bio-rd-compatible `RoutingInformationService`
2665 + endpoint. RIPE RIS Live, RIPEstat, RIS MRT dumps, and route collector BGP
2666 + sessions use different protocols, so they cannot be used directly as
2667 + `ris_instances[].grpc_addr`. Put a converter or bio-rd-compatible service in
2668 + front of RIPE-derived data if you need that external view.
2669 - name: Memory growth without bound
2670 description: |
2671 The trie has no time-based eviction. Routes are removed only when the
@@ -2885,15 +3619,16 @@ modules:
3619
3620 - **Legacy v1 tokens** (`Authorization: Token <token>`) -- accepted by all NetBox
3621 versions, simplest to wire up.
2888 - - **v2 tokens** (NetBox 4.x, `Authorization: Bearer nbt_<key>.<token>`) -- the
2889 - prefix `nbt_` and the random key are concatenated with the token via a dot.
3622 + - **v2 tokens** (NetBox 4.5+, `Authorization: Bearer nbt_<12-char-key>.<40-char-token>`) --
3623 + the prefix `nbt_`, the token key, a dot, and the token secret are all part of
3624 + the header value.
3625
3626 The plugin transports either format -- the value is whatever NetBox issued for the
3627 service account.
3628
3629 The plugin runs the configured `transform` (jaq -- a jq-equivalent) over the
3630 parsed JSON body and produces per-prefix objects. NetBox's response is paginated;
2896 - the plugin does **not** follow `next` links. Pass `?limit=0` (NetBox 4.x default
3631 + the plugin does **not** follow `next` links. Pass `?limit=0` (NetBox default
3632 `MAX_PAGE_SIZE` is 1000; setting `0` removes the cap when the server config
3633 allows) or an explicit `?limit=N` greater than your prefix count, or expose a
3634 server-side aggregator that returns the full list at one URL.
@@ -2918,15 +3653,16 @@ modules:
3653 description: |
3654 In NetBox, create or reuse a service-account user, then generate an API token
3655 under "Admin > Users > Tokens". Restrict the token to read-only and (for
2921 - v4.x) limit the scope to `ipam.view_prefix`. The token value goes in the
3656 + NetBox 4.x) limit the scope to `ipam.view_prefix`. The token value goes in the
3657 `Authorization` header.
3658
3659 The plugin only reads -- never writes -- so a read-only token is sufficient
3660 and recommended. Token format depends on the NetBox version:
3661
3662 - NetBox 3.x or earlier: `Token <40-char-hex>` (legacy).
2928 - - NetBox 4.x: either legacy `Token <hex>` or new `Bearer nbt_<key>.<token>`
2929 - (v2 tokens, opt-in).
3663 + - NetBox 4.0 through 4.4: legacy `Token <hex>`.
3664 + - NetBox 4.5+: legacy `Token <hex>` or new
3665 + `Bearer nbt_<12-char-key>.<40-char-token>` (v2 tokens).
3666 - title: Bulk endpoint (`?limit=0` or aggregator)
3667 description: |
3668 The plugin fetches a single page. NetBox's default `PAGINATE_COUNT` is 50
@@ -2956,11 +3692,11 @@ modules:
3692 enabled: true
3693 list:
3694 - name: url
2959 - description: NetBox prefixes API endpoint, including `?limit=` (recommend `?limit=0` on 4.x for full inventory in one shot).
3695 + description: NetBox prefixes API endpoint, including `?limit=` (recommend `?limit=0` for full inventory in one shot when the server's `MAX_PAGE_SIZE` allows it).
3696 default_value: ""
3697 required: true
3698 - name: headers.Authorization
2963 - description: 'NetBox API token. Use `Token <hex>` for legacy v1 or `Bearer nbt_<key>.<token>` for v4.x v2 tokens.'
3699 + description: 'NetBox API token. Use `Token <hex>` for legacy v1 or `Bearer nbt_<12-char-key>.<40-char-token>` for v4.5+ v2 tokens.'
3700 default_value: ""
3701 required: true
3702 - name: interval
@@ -2984,21 +3720,21 @@ modules:
3720 title: Config
3721 enabled: true
3722 list:
2987 - - name: NetBox 4.x with v2 token, scope-aware mapping
3723 + - name: NetBox 4.5+ with v2 token, scope-aware mapping
3724 folding:
3725 enabled: false
3726 description: |
3727 Standard NetBox 4.2+ wiring. Maps tenant, role, the new `scope` field
3728 (covers site / region / site-group / location), and the human-readable
3729 description. `?limit=0` returns all prefixes in one call when
2994 - `MAX_PAGE_SIZE` is unset on the NetBox server.
3730 + `MAX_PAGE_SIZE` is set to `0` or `None` on the NetBox server.
3731 config: |
3732 enrichment:
3733 network_sources:
3734 netbox:
3735 url: "https://netbox.example.internal/api/ipam/prefixes/?limit=0"
3736 headers:
3001 - Authorization: "Bearer nbt_AbCdEf0123456789.GhIjKl0123456789"
3737 + Authorization: "Bearer nbt_<12-char-key>.<40-char-token>"
3738 interval: 5m
3739 timeout: 30s
3740 transform: |
@@ -3089,16 +3825,20 @@ modules:
3825 - name: Only first 50 prefixes loaded
3826 description: |
3827 NetBox's default `PAGINATE_COUNT` is 50 and the plugin does not follow `next`
3092 - links. Pass `?limit=0` (NetBox 4.x removes the cap when `MAX_PAGE_SIZE` is
3828 + links. Pass `?limit=0` (NetBox removes the cap when `MAX_PAGE_SIZE` is
3829 `0` server-side) or `?limit=N` larger than your inventory. For inventories
3830 above the server's `MAX_PAGE_SIZE` (default 1000), expose a server-side
3831 aggregator endpoint.
3832 - name: 401 / 403 from NetBox
3833 description: |
3098 - Token missing, expired, or wrong format. Verify with:
3099 - `curl -H "Authorization: Token <tok>" https://netbox/api/ipam/prefixes/`.
3100 - On NetBox 4.x check whether the token is v1 (`Token <hex>`) or v2
3101 - (`Bearer nbt_<key>.<token>`) and use the matching header. Watch the journal
3834 + Token missing, expired, or wrong format. Verify with one of:
3835 + `curl -H "Authorization: Token <legacy-token>" https://netbox/api/ipam/prefixes/`
3836 + or
3837 + `curl -H "Authorization: Bearer nbt_<12-char-key>.<40-char-token>" https://netbox/api/ipam/prefixes/`.
3838 + On NetBox 4.5+ check whether the token is v1 (`Token <hex>`) or v2
3839 + (`Bearer nbt_<12-char-key>.<40-char-token>`) and use the matching header.
3840 + A v2 token sent with the legacy `Token` prefix is rejected as an invalid v1
3841 + token. Watch the journal
3842 for `network-sources` warnings -- HTTP errors are logged there as
3843 refresh-failed warnings.
3844 - name: site is empty after upgrade to NetBox 4.2
src/crates/netflow-plugin/src/api/flows/handler.rs
+1 -1
@@ -211,7 +211,7 @@ fn args_to_value(function_call: &FunctionCall) -> Value {
211 for arg in &function_call.args {
212 if let Some((key, value)) = arg.split_once(':') {
213 let json_value = if numeric_fields.contains(&key) {
214 - value.parse::<u64>().map_or_else(
214 + value.parse::<i64>().map_or_else(
215 |_| serde_json::json!(value),
216 |number| serde_json::json!(number),
217 )
src/crates/netflow-plugin/src/decoder/tests.rs
+106
@@ -8,6 +8,8 @@ use super::{
8 field_tracks_presence, finalize_canonical_flow_fields, normalize_direction_value,
9 observe_v9_templates_from_raw_payload, to_field_token, xxhash64,
10 };
11 +use crate::enrichment::FlowEnricher;
12 +use crate::plugin_config::{EnrichmentConfig, NetworkAttributesConfig, NetworkAttributesValue};
13 use etherparse::{NetSlice, SlicedPacket, TransportSlice};
14 use netflow_parser::variable_versions::v9_lookup::V9Field;
15 use pcap_file::pcap::PcapReader;
@@ -389,6 +391,51 @@ fn akvorado_sflow_expanded_sample_fixture_matches_expected_projection() {
391 );
392 }
393
394 +#[test]
395 +fn sflow_fixture_applies_enrichment_during_decode() {
396 + let cfg = EnrichmentConfig {
397 + networks: BTreeMap::from([
398 + (
399 + "52.52.52.0/24".to_string(),
400 + NetworkAttributesValue::Attributes(NetworkAttributesConfig {
401 + name: "sflow-src".to_string(),
402 + tenant: "fixture".to_string(),
403 + asn: 64_502,
404 + ..Default::default()
405 + }),
406 + ),
407 + (
408 + "53.53.53.0/24".to_string(),
409 + NetworkAttributesValue::Attributes(NetworkAttributesConfig {
410 + name: "sflow-dst".to_string(),
411 + tenant: "fixture".to_string(),
412 + asn: 64_503,
413 + ..Default::default()
414 + }),
415 + ),
416 + ]),
417 + ..Default::default()
418 + };
419 + let flows = decode_fixture_sequence_with_enrichment(&["data-sflow-expanded-sample.pcap"], &cfg);
420 + assert_eq!(flows.len(), 1);
421 + let flow = &flows[0].record.to_fields();
422 +
423 + assert_fields(
424 + flow,
425 + &[
426 + ("FLOW_VERSION", "sflow"),
427 + ("SRC_NET_NAME", "sflow-src"),
428 + ("DST_NET_NAME", "sflow-dst"),
429 + ("SRC_NET_TENANT", "fixture"),
430 + ("DST_NET_TENANT", "fixture"),
431 + ("SRC_AS", "64502"),
432 + ("DST_AS", "64503"),
433 + ("SRC_AS_NAME", "AS64502"),
434 + ("DST_AS_NAME", "AS64503"),
435 + ],
436 + );
437 +}
438 +
439 #[test]
440 fn emits_flow_records_for_v5_fixture() {
441 let mut decoders = FlowDecoders::new();
@@ -566,6 +613,46 @@ fn akvorado_v9_data_fixture_core_fields_match_including_sampling_rate() {
613 assert_eq!(first.get("TCP_FLAGS").map(String::as_str), Some("16"));
614 }
615
616 +#[test]
617 +fn netflow_v9_fixture_applies_enrichment_during_decode() {
618 + let cfg = EnrichmentConfig {
619 + networks: BTreeMap::from([(
620 + "198.38.121.0/24".to_string(),
621 + NetworkAttributesValue::Attributes(NetworkAttributesConfig {
622 + name: "netflow-src".to_string(),
623 + role: "fixture".to_string(),
624 + tenant: "decode-test".to_string(),
625 + asn: 64_501,
626 + ..Default::default()
627 + }),
628 + )]),
629 + ..Default::default()
630 + };
631 + let flows = decode_fixture_sequence_with_enrichment(
632 + &[
633 + "options-template.pcap",
634 + "options-data.pcap",
635 + "template.pcap",
636 + "data.pcap",
637 + ],
638 + &cfg,
639 + );
640 + let flow = find_flow(&flows, &[("SRC_ADDR", "198.38.121.178")]);
641 +
642 + assert_fields(
643 + &flow,
644 + &[
645 + ("FLOW_VERSION", "v9"),
646 + ("SRC_NET_NAME", "netflow-src"),
647 + ("SRC_NET_ROLE", "fixture"),
648 + ("SRC_NET_TENANT", "decode-test"),
649 + ("SRC_AS", "64501"),
650 + ("SRC_AS_NAME", "AS64501"),
651 + ("DST_NET_NAME", ""),
652 + ],
653 + );
654 +}
655 +
656 #[test]
657 fn akvorado_v9_data_fixture_all_flows_match_expected_projection() {
658 let base = fixture_dir();
@@ -3066,6 +3153,25 @@ fn decode_fixture_sequence_with_options(
3153 out
3154 }
3155
3156 +fn decode_fixture_sequence_with_enrichment(
3157 + fixtures: &[&str],
3158 + cfg: &EnrichmentConfig,
3159 +) -> Vec<DecodedFlow> {
3160 + let base = fixture_dir();
3161 + let mut decoders = FlowDecoders::new();
3162 + decoders
3163 + .set_enricher(FlowEnricher::from_config(cfg).expect("build enricher for decoder fixture"));
3164 + let mut out = Vec::new();
3165 + for fixture in fixtures {
3166 + out.extend(decode_pcap_flows_at(
3167 + &base.join(fixture),
3168 + &mut decoders,
3169 + TEST_INPUT_REALTIME_USEC,
3170 + ));
3171 + }
3172 + out
3173 +}
3174 +
3175 fn project_flows(flows: &[DecodedFlow], keys: &[&'static str]) -> Vec<FlowFields> {
3176 flows
3177 .iter()
src/crates/netflow-plugin/src/enrichment/data/geoip/decode.rs
+5 -2
@@ -22,8 +22,8 @@ pub(crate) fn decode_asn_name(record: &AsnLookupRecord) -> Option<String> {
22 .map(str::to_string)
23 }
24
25 -pub(crate) fn decode_ip_class(record: &AsnLookupRecord) -> Option<String> {
26 - let ip_class = record.netdata.ip_class.trim();
25 +pub(crate) fn decode_ip_class(netdata: &NetdataLookupRecord) -> Option<String> {
26 + let ip_class = netdata.ip_class.trim();
27 (!ip_class.is_empty()).then(|| ip_class.to_string())
28 }
29
@@ -38,6 +38,9 @@ pub(crate) fn parse_asn_text(value: &str) -> Option<u32> {
38 }
39
40 pub(crate) fn apply_geo_record(out: &mut NetworkAttributes, record: &GeoLookupRecord) {
41 + if let Some(ip_class) = decode_ip_class(&record.netdata) {
42 + out.ip_class = ip_class;
43 + }
44 if let Some(country) = &record.country
45 && let Some(code) = country_code(country)
46 && !code.is_empty()
src/crates/netflow-plugin/src/enrichment/data/geoip/resolver.rs
+1 -1
@@ -134,7 +134,7 @@ impl GeoIpResolver {
134 if let Some(asn_name) = decode_asn_name(&record) {
135 out.asn_name = asn_name;
136 }
137 - if let Some(ip_class) = decode_ip_class(&record) {
137 + if let Some(ip_class) = decode_ip_class(&record.netdata) {
138 out.ip_class = ip_class;
139 }
140 }
src/crates/netflow-plugin/src/enrichment/data/geoip/types.rs
+2
@@ -45,6 +45,8 @@ pub(crate) struct NetdataLookupRecord {
45
46 #[derive(Debug, Deserialize)]
47 pub(crate) struct GeoLookupRecord {
48 + #[serde(default)]
49 + pub(crate) netdata: NetdataLookupRecord,
50 #[serde(default)]
51 pub(crate) country: Option<CountryValue>,
52 #[serde(default)]
src/crates/netflow-plugin/src/enrichment/tests.rs
+267 -4
@@ -1,11 +1,14 @@
1 use super::*;
2 +use crate::network_sources::NetworkSourceRecord;
3 use crate::plugin_config::{
4 AsnProviderConfig, GeoIpConfig, NetProviderConfig, NetworkAttributesConfig,
4 - NetworkAttributesValue, RoutingDynamicBmpConfig, RoutingDynamicConfig, StaticExporterConfig,
5 - StaticInterfaceConfig, StaticMetadataConfig, StaticRoutingConfig, StaticRoutingEntryConfig,
6 - StaticRoutingLargeCommunityConfig,
5 + NetworkAttributesValue, RemoteNetworkSourceConfig, RoutingDynamicBmpConfig,
6 + RoutingDynamicConfig, StaticExporterConfig, StaticInterfaceConfig, StaticMetadataConfig,
7 + StaticRoutingConfig, StaticRoutingEntryConfig, StaticRoutingLargeCommunityConfig,
8 };
9 use std::io::Write;
10 +use std::net::Ipv4Addr;
11 +use std::path::PathBuf;
12 use tempfile::tempdir;
13
14 #[test]
@@ -1227,6 +1230,152 @@ fn geoip_only_enrichment_keeps_flow_without_static_metadata() {
1230 assert_eq!(fields.get("SAMPLING_RATE").map(String::as_str), Some("0"));
1231 }
1232
1233 +#[test]
1234 +fn maxmind_geolite2_mmdb_enrichment_populates_asn_and_geo_fields() {
1235 + let cfg = EnrichmentConfig {
1236 + geoip: GeoIpConfig {
1237 + asn_database: vec![mmdb_fixture_path("GeoLite2-ASN-Test.mmdb")],
1238 + geo_database: vec![mmdb_fixture_path("GeoLite2-City-Test.mmdb")],
1239 + optional: false,
1240 + },
1241 + ..Default::default()
1242 + };
1243 + let mut enricher = FlowEnricher::from_config(&cfg)
1244 + .expect("build enricher")
1245 + .expect("enricher must be enabled");
1246 +
1247 + let mut fields = base_fields("192.0.2.10", 10, 20, 1000, 10, 20);
1248 + fields.insert("SRC_ADDR", Ipv4Addr::new(1, 128, 0, 123).to_string());
1249 + fields.insert("DST_ADDR", Ipv4Addr::new(81, 2, 69, 142).to_string());
1250 + fields.insert("SRC_AS", "0".to_string());
1251 + fields.insert("DST_AS", "0".to_string());
1252 + fields.insert("SRC_MASK", "24".to_string());
1253 + fields.insert("DST_MASK", "24".to_string());
1254 +
1255 + assert!(enricher.enrich_fields(&mut fields));
1256 + assert_eq!(fields.get("SRC_AS").map(String::as_str), Some("1221"));
1257 + assert_eq!(
1258 + fields.get("SRC_AS_NAME").map(String::as_str),
1259 + Some("AS1221 Telstra Pty Ltd")
1260 + );
1261 + assert_eq!(fields.get("DST_COUNTRY").map(String::as_str), Some("GB"));
1262 + assert_eq!(
1263 + fields.get("DST_GEO_CITY").map(String::as_str),
1264 + Some("London")
1265 + );
1266 + assert_eq!(fields.get("DST_GEO_STATE").map(String::as_str), Some("ENG"));
1267 + assert_eq!(
1268 + fields.get("DST_GEO_LATITUDE").map(String::as_str),
1269 + Some("51.514200")
1270 + );
1271 + assert_eq!(
1272 + fields.get("DST_GEO_LONGITUDE").map(String::as_str),
1273 + Some("-0.093100")
1274 + );
1275 +}
1276 +
1277 +#[test]
1278 +fn netdata_topology_mmdb_enrichment_populates_asn_and_geo_fields() {
1279 + let cfg = EnrichmentConfig {
1280 + geoip: GeoIpConfig {
1281 + asn_database: vec![mmdb_fixture_path("Netdata-Topology-ASN-Test.mmdb")],
1282 + geo_database: vec![mmdb_fixture_path("Netdata-Topology-GEO-Test.mmdb")],
1283 + optional: false,
1284 + },
1285 + ..Default::default()
1286 + };
1287 + let mut enricher = FlowEnricher::from_config(&cfg)
1288 + .expect("build enricher")
1289 + .expect("enricher must be enabled");
1290 +
1291 + let mut fields = base_fields("192.0.2.10", 10, 20, 1000, 10, 20);
1292 + fields.insert("SRC_ADDR", Ipv4Addr::new(8, 8, 8, 8).to_string());
1293 + fields.insert("DST_ADDR", Ipv4Addr::new(8, 8, 8, 9).to_string());
1294 + fields.insert("SRC_AS", "0".to_string());
1295 + fields.insert("DST_AS", "0".to_string());
1296 + fields.insert("SRC_MASK", "24".to_string());
1297 + fields.insert("DST_MASK", "24".to_string());
1298 +
1299 + assert!(enricher.enrich_fields(&mut fields));
1300 + assert_eq!(fields.get("SRC_AS").map(String::as_str), Some("15169"));
1301 + assert_eq!(
1302 + fields.get("SRC_AS_NAME").map(String::as_str),
1303 + Some("AS15169 Google LLC")
1304 + );
1305 + assert_eq!(fields.get("DST_COUNTRY").map(String::as_str), Some("US"));
1306 + assert_eq!(
1307 + fields.get("DST_GEO_CITY").map(String::as_str),
1308 + Some("Mountain View")
1309 + );
1310 + assert_eq!(
1311 + fields.get("DST_GEO_STATE").map(String::as_str),
1312 + Some("California")
1313 + );
1314 + assert_eq!(
1315 + fields.get("DST_GEO_LATITUDE").map(String::as_str),
1316 + Some("37.405600")
1317 + );
1318 + assert_eq!(
1319 + fields.get("DST_GEO_LONGITUDE").map(String::as_str),
1320 + Some("-122.077500")
1321 + );
1322 +}
1323 +
1324 +#[test]
1325 +fn actual_provider_mmdb_outputs_are_readable_when_root_is_set() {
1326 + let Some(root) = std::env::var_os("NETDATA_TOPOLOGY_IP_INTEL_PROVIDER_ROOT") else {
1327 + return;
1328 + };
1329 + let root = PathBuf::from(root);
1330 + let public_ip = "8.8.9.9".parse().expect("public test address");
1331 +
1332 + for case in [
1333 + "dbip_asn_mmdb",
1334 + "dbip_asn_csv",
1335 + "iptoasn_asn",
1336 + "caida_asn",
1337 + "maxmind_asn",
1338 + ] {
1339 + let path = root.join(case).join("topology-ip-asn.mmdb");
1340 + let resolver = GeoIpResolver::from_config(&GeoIpConfig {
1341 + asn_database: vec![path.to_string_lossy().into_owned()],
1342 + geo_database: Vec::new(),
1343 + optional: false,
1344 + })
1345 + .unwrap_or_else(|err| panic!("{case}: failed to load ASN MMDB: {err}"))
1346 + .unwrap_or_else(|| panic!("{case}: ASN resolver disabled"));
1347 + let attrs = resolver
1348 + .lookup(public_ip)
1349 + .unwrap_or_else(|| panic!("{case}: ASN lookup returned no attributes"));
1350 + assert_ne!(attrs.asn, 0, "{case}: expected non-zero ASN");
1351 + }
1352 +
1353 + for case in [
1354 + "dbip_geo_country_mmdb",
1355 + "dbip_geo_country_csv",
1356 + "dbip_geo_city_mmdb",
1357 + "dbip_geo_city_csv",
1358 + "iptoasn_geo",
1359 + "maxmind_geo",
1360 + "ip2location_geo",
1361 + "ipdeny_geo",
1362 + "ipip_geo",
1363 + ] {
1364 + let path = root.join(case).join("topology-ip-geo.mmdb");
1365 + let resolver = GeoIpResolver::from_config(&GeoIpConfig {
1366 + asn_database: Vec::new(),
1367 + geo_database: vec![path.to_string_lossy().into_owned()],
1368 + optional: false,
1369 + })
1370 + .unwrap_or_else(|err| panic!("{case}: failed to load GEO MMDB: {err}"))
1371 + .unwrap_or_else(|| panic!("{case}: GEO resolver disabled"));
1372 + let attrs = resolver
1373 + .lookup(public_ip)
1374 + .unwrap_or_else(|| panic!("{case}: GEO lookup returned no attributes"));
1375 + assert!(!attrs.country.is_empty(), "{case}: expected country");
1376 + }
1377 +}
1378 +
1379 #[test]
1380 fn static_metadata_without_sampling_keeps_flow() {
1381 let cfg = EnrichmentConfig {
@@ -1812,6 +1961,91 @@ fn network_enrichment_merges_supernet_and_subnet_attributes() {
1961 assert_eq!(fields.get("DST_AS").map(String::as_str), Some("64501"));
1962 }
1963
1964 +#[test]
1965 +fn network_sources_runtime_enrichment_merges_with_static_networks() {
1966 + let cfg = EnrichmentConfig {
1967 + metadata_static: metadata_config_for_192(),
1968 + default_sampling_rate: Some(SamplingRateSetting::Single(1000)),
1969 + network_sources: BTreeMap::from([(
1970 + "ipam".to_string(),
1971 + RemoteNetworkSourceConfig {
1972 + url: "http://127.0.0.1/ipam.json".to_string(),
1973 + ..Default::default()
1974 + },
1975 + )]),
1976 + networks: BTreeMap::from([(
1977 + "198.51.100.0/24".to_string(),
1978 + NetworkAttributesValue::Attributes(NetworkAttributesConfig {
1979 + name: "static-edge".to_string(),
1980 + region: "static-region".to_string(),
1981 + country: "FR".to_string(),
1982 + ..Default::default()
1983 + }),
1984 + )]),
1985 + ..Default::default()
1986 + };
1987 +
1988 + let mut enricher = FlowEnricher::from_config(&cfg)
1989 + .expect("build enricher")
1990 + .expect("enricher must be enabled");
1991 + let runtime = enricher
1992 + .network_sources_runtime()
1993 + .expect("network source runtime");
1994 + runtime.replace_records(vec![NetworkSourceRecord {
1995 + prefix: "198.51.100.0/24"
1996 + .parse()
1997 + .expect("parse network source prefix"),
1998 + attrs: NetworkAttributes {
1999 + name: "runtime-edge".to_string(),
2000 + role: "cloud".to_string(),
2001 + tenant: "tenant-a".to_string(),
2002 + asn: 64_500,
2003 + asn_name: "Example Transit".to_string(),
2004 + ..Default::default()
2005 + },
2006 + }]);
2007 +
2008 + let mut fields = base_fields("192.0.2.10", 10, 20, 1000, 10, 20);
2009 + fields.insert("SRC_ADDR", "198.51.100.10".to_string());
2010 + fields.insert("DST_ADDR", "198.51.100.20".to_string());
2011 + fields.insert("SRC_AS", "0".to_string());
2012 + fields.insert("DST_AS", "0".to_string());
2013 + fields.insert("SRC_MASK", "24".to_string());
2014 + fields.insert("DST_MASK", "24".to_string());
2015 +
2016 + assert!(enricher.enrich_fields(&mut fields));
2017 + assert_eq!(fields.get("SRC_AS").map(String::as_str), Some("64500"));
2018 + assert_eq!(
2019 + fields.get("SRC_AS_NAME").map(String::as_str),
2020 + Some("AS64500 Example Transit")
2021 + );
2022 + assert_eq!(
2023 + fields.get("SRC_NET_NAME").map(String::as_str),
2024 + Some("static-edge")
2025 + );
2026 + assert_eq!(
2027 + fields.get("SRC_NET_ROLE").map(String::as_str),
2028 + Some("cloud")
2029 + );
2030 + assert_eq!(
2031 + fields.get("SRC_NET_REGION").map(String::as_str),
2032 + Some("static-region")
2033 + );
2034 + assert_eq!(
2035 + fields.get("SRC_NET_TENANT").map(String::as_str),
2036 + Some("tenant-a")
2037 + );
2038 + assert_eq!(fields.get("SRC_COUNTRY").map(String::as_str), Some("FR"));
2039 + assert_eq!(
2040 + fields.get("DST_NET_NAME").map(String::as_str),
2041 + Some("static-edge")
2042 + );
2043 + assert_eq!(
2044 + fields.get("DST_NET_ROLE").map(String::as_str),
2045 + Some("cloud")
2046 + );
2047 +}
2048 +
2049 #[test]
2050 fn network_attributes_asn_override_clears_stale_asn_name() {
2051 let mut base = NetworkAttributes {
@@ -1909,13 +2143,14 @@ fn decode_ip_class_returns_non_empty_value() {
2143 },
2144 };
2145
1912 - assert_eq!(decode_ip_class(&record).as_deref(), Some("private"));
2146 + assert_eq!(decode_ip_class(&record.netdata).as_deref(), Some("private"));
2147 }
2148
2149 #[test]
2150 fn apply_geo_record_normalizes_coordinates_and_rejects_invalid_values() {
2151 let mut attrs = NetworkAttributes::default();
2152 let record = GeoLookupRecord {
2153 + netdata: NetdataLookupRecord::default(),
2154 country: None,
2155 city: None,
2156 location: Some(LocationValue {
@@ -1932,6 +2167,7 @@ fn apply_geo_record_normalizes_coordinates_and_rejects_invalid_values() {
2167 assert_eq!(attrs.longitude, "2.352200");
2168
2169 let invalid = GeoLookupRecord {
2170 + netdata: NetdataLookupRecord::default(),
2171 country: None,
2172 city: None,
2173 location: Some(LocationValue {
@@ -1947,6 +2183,25 @@ fn apply_geo_record_normalizes_coordinates_and_rejects_invalid_values() {
2183 assert_eq!(attrs.longitude, "2.352200");
2184 }
2185
2186 +#[test]
2187 +fn apply_geo_record_accepts_netdata_ip_class_metadata() {
2188 + let mut attrs = NetworkAttributes::default();
2189 + let record = GeoLookupRecord {
2190 + netdata: NetdataLookupRecord {
2191 + ip_class: "private".to_string(),
2192 + },
2193 + country: None,
2194 + city: None,
2195 + location: None,
2196 + subdivisions: Vec::new(),
2197 + region: None,
2198 + };
2199 +
2200 + apply_geo_record(&mut attrs, &record);
2201 +
2202 + assert_eq!(attrs.ip_class, "private");
2203 +}
2204 +
2205 #[test]
2206 fn effective_as_name_uses_private_space_label_only_for_private_zero_asn() {
2207 let attrs = NetworkAttributes {
@@ -2141,6 +2396,14 @@ fn base_fields(
2396 ])
2397 }
2398
2399 +fn mmdb_fixture_path(name: &str) -> String {
2400 + PathBuf::from(env!("CARGO_MANIFEST_DIR"))
2401 + .join("testdata/mmdb")
2402 + .join(name)
2403 + .to_string_lossy()
2404 + .into_owned()
2405 +}
2406 +
2407 fn test_enricher_for_provider_order() -> FlowEnricher {
2408 FlowEnricher {
2409 default_sampling_rate: PrefixMap::default(),
src/crates/netflow-plugin/src/main_tests.rs
+91 -19
@@ -9,7 +9,7 @@ use journal_core::repository::File as RepoFile;
9 use journal_core::{Direction, JournalFile, JournalReader, Location};
10 use pcap_file::pcap::PcapReader;
11 use rt::ProgressState;
12 -use std::collections::HashMap;
12 +use std::collections::{BTreeMap, HashMap};
13 use std::fs;
14 use std::net::UdpSocket as StdUdpSocket;
15 use std::num::NonZeroU64;
@@ -21,6 +21,8 @@ use tempfile::TempDir;
21 use tokio::net::UdpSocket;
22 use tokio_util::sync::CancellationToken;
23
24 +const E2E_INGEST_WAIT_TIMEOUT: Duration = Duration::from_secs(30);
25 +
26 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
27 async fn e2e_ingest_writes_journals_and_query_reads_flows() {
28 let (cfg, metrics, _open_tiers, _tier_flow_indexes, _tmp) = ingest_fixture("nfv5.pcap").await;
@@ -33,7 +35,7 @@ async fn e2e_ingest_writes_journals_and_query_reads_flows() {
35 let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
36 .await
37 .expect("create query service");
36 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
38 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
39 let request = query::FlowsRequest {
40 view: query::ViewMode::TableSankey,
41 after: Some(1),
@@ -73,6 +75,61 @@ async fn e2e_ingest_writes_journals_and_query_reads_flows() {
75 );
76 }
77
78 +#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
79 +async fn e2e_ingest_persists_enriched_fields_and_query_reads_them() {
80 + let (cfg, _metrics, _open_tiers, _tier_flow_indexes, _tmp) =
81 + ingest_fixture_with_config("nfv5.pcap", plugin_config::TimestampSource::Input, |cfg| {
82 + cfg.enrichment.networks = BTreeMap::from([(
83 + "161.202.212.0/24".to_string(),
84 + plugin_config::NetworkAttributesValue::Attributes(
85 + plugin_config::NetworkAttributesConfig {
86 + name: "journal-src".to_string(),
87 + tenant: "fixture".to_string(),
88 + asn: 64_504,
89 + ..Default::default()
90 + },
91 + ),
92 + )]);
93 + })
94 + .await;
95 +
96 + let fields = first_raw_journal_fields(&cfg.journal.raw_tier_dir());
97 + assert_eq!(
98 + fields.get("SRC_NET_NAME").map(String::as_str),
99 + Some("journal-src")
100 + );
101 + assert_eq!(
102 + fields.get("SRC_NET_TENANT").map(String::as_str),
103 + Some("fixture")
104 + );
105 + assert_eq!(fields.get("SRC_AS").map(String::as_str), Some("64504"));
106 +
107 + let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
108 + .await
109 + .expect("create query service");
110 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
111 + let output = query_service
112 + .query_flows(&query::FlowsRequest {
113 + view: query::ViewMode::TableSankey,
114 + after: Some(1),
115 + before: Some(before),
116 + group_by: vec!["SRC_NET_NAME".to_string()],
117 + top_n: query::TopN::N100,
118 + ..Default::default()
119 + })
120 + .await
121 + .expect("query enriched fields");
122 +
123 + assert!(
124 + output.flows.iter().any(|row| {
125 + row["key"]["SRC_NET_NAME"]
126 + .as_str()
127 + .is_some_and(|value| value == "journal-src")
128 + }),
129 + "expected query output to include enriched SRC_NET_NAME"
130 + );
131 +}
132 +
133 #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
134 async fn e2e_timestamp_source_first_switched_is_persisted_as_source_timestamp() {
135 let (cfg, _metrics, open_tiers, _tier_flow_indexes, _tmp) =
@@ -155,7 +212,7 @@ async fn e2e_query_service_timeseries_path_returns_chart_data() {
212 let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
213 .await
214 .expect("create query service");
158 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
215 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
216 let after = before.saturating_sub(3600);
217
218 let output = query_service
@@ -217,7 +274,7 @@ async fn e2e_flows_function_returns_expected_response_sections() {
274 .await
275 .expect("create query service");
276 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
220 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
277 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
278
279 let response = handler
280 .handle_request(query::FlowsRequest {
@@ -341,7 +398,7 @@ async fn e2e_flows_function_marks_progress_complete_with_execution_context() {
398 .await
399 .expect("create query service");
400 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
344 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
401 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
402 let progress = ProgressState::default();
403 let execution = query::QueryExecutionContext::new(progress.clone(), CancellationToken::new());
404
@@ -394,7 +451,7 @@ async fn e2e_flows_function_marks_progress_complete_for_empty_projected_query()
451 Arc::new(ingest::IngestMetrics::default()),
452 Arc::new(query_service),
453 );
397 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
454 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
455 let progress = ProgressState::default();
456 let execution = query::QueryExecutionContext::new(progress.clone(), CancellationToken::new());
457
@@ -443,7 +500,7 @@ async fn e2e_flows_function_honors_cancelled_execution_context() {
500 .await
501 .expect("create query service");
502 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
446 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
503 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
504 let cancellation = CancellationToken::new();
505 cancellation.cancel();
506 let execution = query::QueryExecutionContext::new(ProgressState::default(), cancellation);
@@ -521,7 +578,7 @@ async fn e2e_flows_metrics_function_returns_top_n_chart_with_on_disk_tier_fallba
578 .await
579 .expect("create query service");
580 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
524 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
581 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
582 let after = before.saturating_sub(3600);
583 let materialized_tier_files = tier_file_count(&cfg.journal.hour_1_tier_dir())
584 + tier_file_count(&cfg.journal.minute_5_tier_dir())
@@ -614,7 +671,7 @@ async fn e2e_aggregated_safe_group_by_falls_back_to_on_disk_lower_tiers() {
671 let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
672 .await
673 .expect("create query service");
617 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
674 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
675 let request = query::FlowsRequest {
676 view: query::ViewMode::TableSankey,
677 after: Some(1),
@@ -682,7 +739,7 @@ async fn e2e_country_map_reuses_tuple_table_shape_with_country_keys() {
739 .await
740 .expect("create query service");
741 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
685 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
742 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
743
744 let response = handler
745 .handle_request(query::FlowsRequest {
@@ -757,7 +814,7 @@ async fn e2e_state_map_reuses_tuple_table_shape_with_state_keys() {
814 .await
815 .expect("create query service");
816 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
760 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
817 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
818
819 let response = handler
820 .handle_request(query::FlowsRequest {
@@ -808,7 +865,7 @@ async fn e2e_city_map_reuses_tuple_table_shape_with_city_and_coordinate_keys() {
865 .await
866 .expect("create query service");
867 let handler = NetflowFlowsHandler::new(Arc::clone(&metrics), Arc::new(query_service));
811 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
868 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
869
870 let response = handler
871 .handle_request(query::FlowsRequest {
@@ -900,7 +957,7 @@ async fn e2e_selection_filter_uses_streaming_reader_path() {
957 let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
958 .await
959 .expect("create query service");
903 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
960 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
961
962 let request_base = query::FlowsRequest {
963 view: query::ViewMode::TableSankey,
@@ -1019,7 +1076,7 @@ async fn e2e_post_style_nested_required_controls_still_filter_correctly() {
1076 let (query_service, _notify_rx) = query::FlowQueryService::new(&cfg)
1077 .await
1078 .expect("create query service");
1022 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
1079 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
1080 let payload = format!(
1081 r#"{{
1082 "after":1,
@@ -1083,7 +1140,7 @@ async fn profile_live_day_query_against_local_journals() {
1140 .await
1141 .expect("create query service for live journals");
1142
1086 - let before = Utc::now().timestamp().max(1) as u32;
1143 + let before = Utc::now().timestamp().max(1);
1144 let after = before.saturating_sub(24 * 60 * 60);
1145 let request = query::FlowsRequest {
1146 view: query::ViewMode::TableSankey,
@@ -1234,7 +1291,7 @@ async fn profile_fixed_raw_plugin_scan_only_against_local_journals() {
1291 .await
1292 .expect("create query service for fixed raw journals");
1293
1237 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
1294 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
1295 let request = query::FlowsRequest {
1296 view: query::ViewMode::TableSankey,
1297 after: Some(1),
@@ -1317,7 +1374,7 @@ async fn profile_fixed_raw_plugin_stage_breakdown_against_local_journals() {
1374 .await
1375 .expect("create query service for fixed raw journals");
1376
1320 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
1377 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
1378 let request = query::FlowsRequest {
1379 view: query::ViewMode::TableSankey,
1380 after: Some(1),
@@ -1470,7 +1527,7 @@ async fn profile_fixed_raw_query_processing_against_local_journals() {
1527 .await
1528 .expect("create query service for fixed raw journals");
1529
1473 - let before = (Utc::now().timestamp().max(1) as u32).saturating_add(3600);
1530 + let before = Utc::now().timestamp().max(1).saturating_add(3600);
1531 let request = query::FlowsRequest {
1532 view: query::ViewMode::TableSankey,
1533 after: Some(1),
@@ -1556,6 +1613,20 @@ async fn ingest_fixture_with_timestamp_source(
1613 Arc<RwLock<tiering::OpenTierState>>,
1614 Arc<RwLock<tiering::TierFlowIndexStore>>,
1615 TempDir,
1616 +) {
1617 + ingest_fixture_with_config(fixture_name, timestamp_source, |_| {}).await
1618 +}
1619 +
1620 +async fn ingest_fixture_with_config(
1621 + fixture_name: &str,
1622 + timestamp_source: plugin_config::TimestampSource,
1623 + configure: impl FnOnce(&mut plugin_config::PluginConfig),
1624 +) -> (
1625 + plugin_config::PluginConfig,
1626 + Arc<ingest::IngestMetrics>,
1627 + Arc<RwLock<tiering::OpenTierState>>,
1628 + Arc<RwLock<tiering::TierFlowIndexStore>>,
1629 + TempDir,
1630 ) {
1631 let tmp = tempfile::tempdir().expect("create temp dir");
1632 let listen = reserve_udp_listen_addr();
@@ -1565,6 +1636,7 @@ async fn ingest_fixture_with_timestamp_source(
1636 cfg.listener.sync_interval = Duration::from_millis(50);
1637 cfg.listener.sync_every_entries = 1;
1638 cfg.protocols.timestamp_source = timestamp_source;
1639 + configure(&mut cfg);
1640
1641 let metrics = Arc::new(ingest::IngestMetrics::default());
1642 let open_tiers = Arc::new(RwLock::new(tiering::OpenTierState::default()));
@@ -1596,7 +1668,7 @@ async fn ingest_fixture_with_timestamp_source(
1668 }
1669
1670 async fn wait_for_ingest_progress(metrics: &Arc<ingest::IngestMetrics>) {
1599 - tokio::time::timeout(Duration::from_secs(10), async {
1671 + tokio::time::timeout(E2E_INGEST_WAIT_TIMEOUT, async {
1672 loop {
1673 if metrics.journal_entries_written.load(Ordering::Relaxed) > 0 {
1674 break;
src/crates/netflow-plugin/src/network_sources/tests.rs
+281 -1
@@ -1,7 +1,11 @@
1 use super::decode::decode_remote_records;
2 -use super::fetch::{build_client, parse_source_method};
2 +use super::fetch::{build_client, fetch_source_once, parse_source_method};
3 use super::transform::{compile_transform, run_transform};
4 use super::*;
5 +use std::net::IpAddr;
6 +use tokio::io::{AsyncReadExt, AsyncWriteExt};
7 +use tokio::net::TcpListener;
8 +use tokio::task::JoinHandle;
9
10 #[test]
11 fn decode_remote_records_supports_results_transform() {
@@ -54,6 +58,145 @@ fn decode_remote_records_supports_akvorado_style_transform() {
58 assert_eq!(rows[1].attrs.role, "route53_healthchecks");
59 }
60
61 +#[test]
62 +fn documented_cloud_and_ipam_transforms_decode_provider_payloads() {
63 + let cases = [
64 + (
65 + "aws",
66 + r#"(.prefixes + .ipv6_prefixes)[] | {
67 + prefix: (.ip_prefix // .ipv6_prefix),
68 + tenant: "amazon",
69 + region: .region,
70 + role: (.service | ascii_downcase)
71 +}"#,
72 + serde_json::json!({
73 + "prefixes": [
74 + {"ip_prefix": "198.51.100.0/24", "region": "us-east-1", "service": "EC2"}
75 + ],
76 + "ipv6_prefixes": [
77 + {"ipv6_prefix": "2001:db8:1::/48", "region": "eu-west-1", "service": "AMAZON"}
78 + ]
79 + }),
80 + vec![
81 + ("198.51.100.0/24", "amazon", "us-east-1", "ec2", "", ""),
82 + ("2001:db8:1::/48", "amazon", "eu-west-1", "amazon", "", ""),
83 + ],
84 + ),
85 + (
86 + "azure",
87 + r#".values[]
88 +| .properties as $p
89 +| $p.addressPrefixes[]
90 +| {
91 + prefix: .,
92 + tenant: "azure",
93 + region: ($p.region // ""),
94 + role: (($p.systemService // "") | ascii_downcase)
95 + }"#,
96 + serde_json::json!({
97 + "values": [
98 + {
99 + "name": "AzureCloud.eastus",
100 + "properties": {
101 + "region": "eastus",
102 + "systemService": "AzureStorage",
103 + "addressPrefixes": ["203.0.113.0/24"]
104 + }
105 + }
106 + ]
107 + }),
108 + vec![("203.0.113.0/24", "azure", "eastus", "azurestorage", "", "")],
109 + ),
110 + (
111 + "gcp",
112 + r#".prefixes[] | {
113 + prefix: (.ipv4Prefix // .ipv6Prefix),
114 + tenant: "gcp",
115 + role: "google-cloud",
116 + region: .scope
117 +}"#,
118 + serde_json::json!({
119 + "prefixes": [
120 + {"ipv4Prefix": "192.0.2.0/24", "scope": "us-central1"},
121 + {"ipv6Prefix": "2001:db8:2::/48", "scope": "europe-west1"}
122 + ]
123 + }),
124 + vec![
125 + ("192.0.2.0/24", "gcp", "us-central1", "google-cloud", "", ""),
126 + (
127 + "2001:db8:2::/48",
128 + "gcp",
129 + "europe-west1",
130 + "google-cloud",
131 + "",
132 + "",
133 + ),
134 + ],
135 + ),
136 + (
137 + "netbox",
138 + r#".results[] | {
139 + prefix: .prefix,
140 + tenant: (.tenant.name // ""),
141 + role: (.role.name // ""),
142 + site: (.scope.name // ""),
143 + name: (.description // "")
144 +}"#,
145 + serde_json::json!({
146 + "results": [
147 + {
148 + "prefix": "198.51.100.0/25",
149 + "tenant": {"name": "tenant-a"},
150 + "role": {"name": "edge"},
151 + "scope": {"name": "dc1"},
152 + "description": "edge subnet"
153 + }
154 + ]
155 + }),
156 + vec![(
157 + "198.51.100.0/25",
158 + "tenant-a",
159 + "",
160 + "edge",
161 + "dc1",
162 + "edge subnet",
163 + )],
164 + ),
165 + (
166 + "generic",
167 + r#".[] | {
168 + prefix: .prefix,
169 + name: .name,
170 + tenant: .env
171 +}"#,
172 + serde_json::json!([
173 + {"prefix": "203.0.113.0/25", "name": "app subnet", "env": "prod"}
174 + ]),
175 + vec![("203.0.113.0/25", "prod", "", "", "", "app subnet")],
176 + ),
177 + ];
178 +
179 + for (name, expression, payload, expected) in cases {
180 + let transform = compile_transform(expression)
181 + .unwrap_or_else(|err| panic!("{name} transform should compile: {err}"));
182 + let rows = decode_remote_records(payload, &transform)
183 + .unwrap_or_else(|err| panic!("{name} transform should decode: {err}"));
184 + assert_eq!(rows.len(), expected.len(), "{name} row count mismatch");
185 + for (idx, (prefix, tenant, region, role, site, net_name)) in expected.iter().enumerate() {
186 + assert_eq!(
187 + rows[idx].prefix,
188 + prefix.parse::<IpNet>().expect("parse expected prefix"),
189 + "{name} prefix mismatch at row {idx}"
190 + );
191 + assert_eq!(rows[idx].attrs.tenant, *tenant, "{name} tenant mismatch");
192 + assert_eq!(rows[idx].attrs.region, *region, "{name} region mismatch");
193 + assert_eq!(rows[idx].attrs.role, *role, "{name} role mismatch");
194 + assert_eq!(rows[idx].attrs.site, *site, "{name} site mismatch");
195 + assert_eq!(rows[idx].attrs.name, *net_name, "{name} name mismatch");
196 + }
197 + }
198 +}
199 +
200 #[test]
201 fn decode_remote_records_preserves_optional_asn_name() {
202 let transform = compile_transform(".results[]").expect("compile transform");
@@ -184,3 +327,140 @@ fn build_client_rejects_explicit_insecure_tls_opt_out() {
327 .contains("certificate verification cannot be disabled")
328 );
329 }
330 +
331 +#[tokio::test]
332 +async fn fetch_source_once_performs_http_get_and_decodes_records() {
333 + let (url, request_task) = serve_json_once(
334 + "200 OK",
335 + r#"{"results":[{"prefix":"198.51.100.0/24","tenant":"runtime","asn":"AS64512"}]}"#,
336 + )
337 + .await;
338 + let source = RemoteNetworkSourceConfig {
339 + url,
340 + headers: BTreeMap::from([("x-netdata-test".to_string(), "present".to_string())]),
341 + proxy: false,
342 + transform: ".results[]".to_string(),
343 + ..Default::default()
344 + };
345 + let client = build_client(false, &source.tls).expect("build client");
346 + let transform = compile_transform(&source.transform).expect("compile transform");
347 +
348 + let records = fetch_source_once(&client, &source, &transform)
349 + .await
350 + .expect("fetch source");
351 +
352 + assert_eq!(records.len(), 1);
353 + assert_eq!(
354 + records[0].prefix,
355 + "198.51.100.0/24"
356 + .parse::<IpNet>()
357 + .expect("parse expected prefix")
358 + );
359 + assert_eq!(records[0].attrs.tenant, "runtime");
360 + assert_eq!(records[0].attrs.asn, 64_512);
361 +
362 + let request = request_task.await.expect("join HTTP server");
363 + assert!(request.starts_with("GET / HTTP/1.1"), "{request}");
364 + assert!(
365 + request
366 + .to_ascii_lowercase()
367 + .contains("x-netdata-test: present"),
368 + "{request}"
369 + );
370 +}
371 +
372 +#[tokio::test]
373 +async fn fetch_source_once_rejects_http_error_status() {
374 + let (url, request_task) = serve_json_once("503 Service Unavailable", r#"{}"#).await;
375 + let source = RemoteNetworkSourceConfig {
376 + url,
377 + proxy: false,
378 + transform: ".".to_string(),
379 + ..Default::default()
380 + };
381 + let client = build_client(false, &source.tls).expect("build client");
382 + let transform = compile_transform(&source.transform).expect("compile transform");
383 +
384 + let err = fetch_source_once(&client, &source, &transform)
385 + .await
386 + .expect_err("HTTP error should fail");
387 +
388 + assert!(err.to_string().contains("unexpected HTTP status 503"));
389 + let _ = request_task.await.expect("join HTTP server");
390 +}
391 +
392 +#[tokio::test]
393 +async fn network_sources_refresher_publishes_first_successful_fetch_to_runtime() {
394 + let (url, request_task) = serve_json_once(
395 + "200 OK",
396 + r#"[{"prefix":"198.51.100.0/24","name":"runtime-net","tenant":"runtime"}]"#,
397 + )
398 + .await;
399 + let runtime = NetworkSourcesRuntime::default();
400 + let shutdown = CancellationToken::new();
401 + let source = RemoteNetworkSourceConfig {
402 + url,
403 + proxy: false,
404 + transform: ".[]".to_string(),
405 + ..Default::default()
406 + };
407 + let refresher_runtime = runtime.clone();
408 + let refresher_shutdown = shutdown.clone();
409 + let task = tokio::spawn(async move {
410 + run_network_sources_refresher(
411 + BTreeMap::from([("ipam".to_string(), source)]),
412 + refresher_runtime,
413 + refresher_shutdown,
414 + )
415 + .await
416 + });
417 +
418 + tokio::time::timeout(Duration::from_secs(3), async {
419 + let address: IpAddr = "198.51.100.42".parse().expect("parse address");
420 + loop {
421 + let matches = runtime.matching_attributes_ascending(address);
422 + if let Some((prefix_len, attrs)) = matches.first() {
423 + assert_eq!(*prefix_len, 24);
424 + assert_eq!(attrs.name, "runtime-net");
425 + assert_eq!(attrs.tenant, "runtime");
426 + break;
427 + }
428 + tokio::time::sleep(Duration::from_millis(25)).await;
429 + }
430 + })
431 + .await
432 + .expect("runtime did not receive network-source records");
433 +
434 + let _ = request_task.await.expect("join HTTP server");
435 + shutdown.cancel();
436 + task.await
437 + .expect("join refresher")
438 + .expect("refresher should stop cleanly");
439 +}
440 +
441 +async fn serve_json_once(status: &'static str, body: &'static str) -> (String, JoinHandle<String>) {
442 + let listener = TcpListener::bind("127.0.0.1:0")
443 + .await
444 + .expect("bind HTTP fixture server");
445 + let url = format!(
446 + "http://{}",
447 + listener.local_addr().expect("fixture server addr")
448 + );
449 + let task = tokio::spawn(async move {
450 + let (mut stream, _) = listener.accept().await.expect("accept HTTP request");
451 + let mut buffer = vec![0_u8; 4096];
452 + let n = stream.read(&mut buffer).await.expect("read HTTP request");
453 + let request = String::from_utf8_lossy(&buffer[..n]).into_owned();
454 + let response = format!(
455 + "HTTP/1.1 {status}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{body}",
456 + body.len()
457 + );
458 + stream
459 + .write_all(response.as_bytes())
460 + .await
461 + .expect("write HTTP response");
462 + request
463 + });
464 +
465 + (url, task)
466 +}
src/crates/netflow-plugin/src/plugin_config_tests.rs
+60
@@ -58,6 +58,58 @@ fn validate_accepts_enabled_bioris_with_instance() {
58 cfg.validate().expect("configuration should be valid");
59 }
60
61 +#[test]
62 +fn validate_rejects_bioris_instance_without_port_or_scheme() {
63 + let mut cfg = PluginConfig::default();
64 + cfg.enrichment.routing_dynamic.bioris.enabled = true;
65 + cfg.enrichment.routing_dynamic.bioris.ris_instances.push(
66 + RoutingDynamicBiorisRisInstanceConfig {
67 + grpc_addr: "ris.example.internal".to_string(),
68 + grpc_secure: false,
69 + vrf_id: 0,
70 + vrf: String::new(),
71 + },
72 + );
73 +
74 + let err = cfg.validate().expect_err("expected validation error");
75 + assert!(
76 + err.to_string()
77 + .contains("grpc_addr must include host:port or an explicit URI scheme")
78 + );
79 +}
80 +
81 +#[test]
82 +fn validate_rejects_zero_bioris_timeouts_when_enabled() {
83 + for field in ["timeout", "refresh", "refresh_timeout"] {
84 + let mut cfg = PluginConfig::default();
85 + cfg.enrichment.routing_dynamic.bioris.enabled = true;
86 + cfg.enrichment.routing_dynamic.bioris.ris_instances.push(
87 + RoutingDynamicBiorisRisInstanceConfig {
88 + grpc_addr: "127.0.0.1:50051".to_string(),
89 + grpc_secure: false,
90 + vrf_id: 0,
91 + vrf: String::new(),
92 + },
93 + );
94 + match field {
95 + "timeout" => cfg.enrichment.routing_dynamic.bioris.timeout = Duration::ZERO,
96 + "refresh" => cfg.enrichment.routing_dynamic.bioris.refresh = Duration::ZERO,
97 + "refresh_timeout" => {
98 + cfg.enrichment.routing_dynamic.bioris.refresh_timeout = Duration::ZERO;
99 + }
100 + _ => unreachable!(),
101 + }
102 +
103 + let err = cfg.validate().expect_err("expected validation error");
104 + assert!(
105 + err.to_string().contains(&format!(
106 + "enrichment.routing_dynamic.bioris.{field} must be greater than 0"
107 + )),
108 + "unexpected error for {field}: {err:#}"
109 + );
110 + }
111 +}
112 +
113 #[test]
114 fn validate_rejects_network_source_verify_false() {
115 let mut cfg = PluginConfig::default();
@@ -238,6 +290,14 @@ journal:
290 assert!(!cfg.enabled);
291 }
292
293 +#[test]
294 +fn stock_netflow_yaml_parses_and_validates() {
295 + let yaml = include_str!("../configs/netflow.yaml");
296 + let cfg: PluginConfig = serde_yaml::from_str(yaml).expect("stock netflow.yaml should parse");
297 +
298 + cfg.validate().expect("stock netflow.yaml should validate");
299 +}
300 +
301 #[test]
302 fn journal_tier_retention_uses_built_in_tier_defaults() {
303 let cfg = PluginConfig::default();
src/crates/netflow-plugin/src/query/planner/request.rs
+19 -4
@@ -1,20 +1,35 @@
1 use super::*;
2
3 pub(crate) fn resolve_time_bounds(request: &FlowsRequest) -> (u32, u32) {
4 - let now = Utc::now().timestamp().max(1) as u32;
4 + let now = Utc::now().timestamp().max(1);
5 let mut before = request.before.unwrap_or(now);
6 if before == 0 {
7 before = now;
8 + } else if before < 0 {
9 + before = now.saturating_add(before).max(1);
10 }
9 - let mut after = request
10 - .after
11 - .unwrap_or_else(|| before.saturating_sub(DEFAULT_QUERY_WINDOW_SECONDS));
11 + let after = match request.after {
12 + Some(after) if after < 0 => before.saturating_add(after),
13 + Some(after) => after,
14 + None => before.saturating_sub(i64::from(DEFAULT_QUERY_WINDOW_SECONDS)),
15 + }
16 + .max(0);
17 + let before = clamp_query_timestamp(before);
18 + let mut after = clamp_query_timestamp(after);
19 if after >= before {
20 after = before.saturating_sub(1);
21 }
22 (after, before)
23 }
24
25 +fn clamp_query_timestamp(timestamp: i64) -> u32 {
26 + if timestamp <= 0 {
27 + 0
28 + } else {
29 + timestamp.min(i64::from(u32::MAX)) as u32
30 + }
31 +}
32 +
33 pub(crate) fn sanitize_limit(top_n: TopN) -> usize {
34 top_n.as_usize().clamp(DEFAULT_QUERY_LIMIT, MAX_QUERY_LIMIT)
35 }
src/crates/netflow-plugin/src/query/request/model/types.rs
+39 -10
@@ -4,8 +4,8 @@ use super::super::*;
4 pub(crate) struct FlowsRequest {
5 pub(crate) mode: RequestMode,
6 pub(crate) view: ViewMode,
7 - pub(crate) after: Option<u32>,
8 - pub(crate) before: Option<u32>,
7 + pub(crate) after: Option<i64>,
8 + pub(crate) before: Option<i64>,
9 pub(crate) query: String,
10 pub(crate) selections: HashMap<String, Vec<String>>,
11 pub(crate) facets: Option<Vec<String>>,
@@ -23,9 +23,9 @@ pub(crate) struct RawFlowsRequest {
23 #[serde(default)]
24 pub(crate) view: Option<ViewMode>,
25 #[serde(default)]
26 - pub(crate) after: Option<u32>,
26 + pub(crate) after: Option<i64>,
27 #[serde(default)]
28 - pub(crate) before: Option<u32>,
28 + pub(crate) before: Option<i64>,
29 #[serde(default)]
30 pub(crate) query: String,
31 #[serde(default, deserialize_with = "deserialize_selections")]
@@ -67,18 +67,13 @@ pub(crate) enum ViewMode {
67 CityMap,
68 }
69
70 -#[derive(Debug, Deserialize, Clone, Copy, PartialEq, Eq, Default)]
70 +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
71 pub(crate) enum TopN {
72 #[default]
73 - #[serde(rename = "25")]
73 N25,
75 - #[serde(rename = "50")]
74 N50,
77 - #[serde(rename = "100")]
75 N100,
79 - #[serde(rename = "200")]
76 N200,
81 - #[serde(rename = "500")]
77 N500,
78 }
79
@@ -113,6 +108,21 @@ impl Default for FlowsRequest {
108 }
109
110 impl TopN {
111 + fn from_u64(value: u64) -> Option<Self> {
112 + match value {
113 + 25 => Some(Self::N25),
114 + 50 => Some(Self::N50),
115 + 100 => Some(Self::N100),
116 + 200 => Some(Self::N200),
117 + 500 => Some(Self::N500),
118 + _ => None,
119 + }
120 + }
121 +
122 + fn parse(value: &str) -> Option<Self> {
123 + value.trim().parse::<u64>().ok().and_then(Self::from_u64)
124 + }
125 +
126 pub(crate) fn as_usize(self) -> usize {
127 match self {
128 Self::N25 => 25,
@@ -124,6 +134,25 @@ impl TopN {
134 }
135 }
136
137 +impl<'de> Deserialize<'de> for TopN {
138 + fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
139 + where
140 + D: Deserializer<'de>,
141 + {
142 + match Value::deserialize(deserializer)? {
143 + Value::String(value) => Self::parse(&value)
144 + .ok_or_else(|| D::Error::custom(format!("unsupported top_n `{value}`"))),
145 + Value::Number(value) => value
146 + .as_u64()
147 + .and_then(Self::from_u64)
148 + .ok_or_else(|| D::Error::custom(format!("unsupported top_n `{value}`"))),
149 + value => Err(D::Error::custom(format!(
150 + "top_n must be one of 25, 50, 100, 200, 500, got {value}"
151 + ))),
152 + }
153 + }
154 +}
155 +
156 impl SortBy {
157 pub(crate) fn as_str(self) -> &'static str {
158 match self {
src/crates/netflow-plugin/src/query/tests.rs
+57 -1
@@ -359,11 +359,67 @@ fn request_deserialization_defaults_missing_view_group_by_sort_by_and_top_n() {
359 )
360 .expect_err("invalid top_n should fail");
361 assert!(
362 - invalid_top_n.to_string().contains("unknown variant `42`"),
362 + invalid_top_n.to_string().contains("unsupported top_n `42`"),
363 "unexpected error: {invalid_top_n}"
364 );
365 }
366
367 +#[test]
368 +fn request_deserialization_accepts_numeric_top_n_values() {
369 + let request = serde_json::from_str::<FlowsRequest>(
370 + r#"{"view":"table-sankey","group_by":["PROTOCOL"],"sort_by":"bytes","top_n":100}"#,
371 + )
372 + .expect("numeric top_n should match documented JSON examples");
373 +
374 + assert_eq!(request.top_n, super::TopN::N100);
375 +
376 + let invalid_top_n = serde_json::from_str::<FlowsRequest>(
377 + r#"{"view":"table-sankey","group_by":["PROTOCOL"],"sort_by":"bytes","top_n":42}"#,
378 + )
379 + .expect_err("unsupported numeric top_n should fail");
380 + assert!(
381 + invalid_top_n.to_string().contains("unsupported top_n `42`"),
382 + "unexpected error: {invalid_top_n}"
383 + );
384 +}
385 +
386 +#[test]
387 +fn request_deserialization_accepts_relative_time_bounds() {
388 + let request = serde_json::from_str::<FlowsRequest>(
389 + r#"{"view":"table-sankey","after":-3600,"before":0,"group_by":["PROTOCOL"],"sort_by":"bytes","top_n":100}"#,
390 + )
391 + .expect("documented relative time bounds should parse");
392 +
393 + assert_eq!(request.after, Some(-3600));
394 + assert_eq!(request.before, Some(0));
395 + assert_eq!(request.top_n, super::TopN::N100);
396 +}
397 +
398 +#[test]
399 +fn resolve_time_bounds_treats_negative_after_as_relative_to_before() {
400 + let request = FlowsRequest {
401 + after: Some(-300),
402 + before: Some(1_000),
403 + ..Default::default()
404 + };
405 +
406 + assert_eq!(super::resolve_time_bounds(&request), (700, 1_000));
407 +}
408 +
409 +#[test]
410 +fn resolve_time_bounds_preserves_order_after_clamping() {
411 + let request = FlowsRequest {
412 + after: Some(i64::MAX - 10),
413 + before: Some(i64::MAX),
414 + ..Default::default()
415 + };
416 +
417 + assert_eq!(
418 + super::resolve_time_bounds(&request),
419 + (u32::MAX - 1, u32::MAX)
420 + );
421 +}
422 +
423 #[test]
424 fn request_deserialization_accepts_state_and_city_map_views() {
425 let state = serde_json::from_str::<FlowsRequest>(
src/crates/netflow-plugin/src/routing/bioris/client.rs
+1 -1
@@ -20,7 +20,7 @@ pub(super) async fn connect_bioris_client(
20 Ok(RoutingInformationServiceClient::new(channel))
21 }
22
23 -fn build_endpoint_uri(instance: &RoutingDynamicBiorisRisInstanceConfig) -> String {
23 +pub(super) fn build_endpoint_uri(instance: &RoutingDynamicBiorisRisInstanceConfig) -> String {
24 if instance.grpc_addr.contains("://") {
25 instance.grpc_addr.clone()
26 } else if instance.grpc_secure {
src/crates/netflow-plugin/src/routing/bioris/tests.rs
+544 -1
@@ -1,7 +1,16 @@
1 -use super::client::bioris_peer_key;
1 +use super::client::{bioris_peer_key, build_endpoint_uri, connect_bioris_client, parse_router_ip};
2 use super::route::{proto_ip_to_ip_addr, route_to_update, route_withdraw_keys};
3 use super::runtime::AfiSafi;
4 use super::*;
5 +use crate::ingest::IngestMetrics;
6 +use crate::plugin_config::RoutingDynamicBiorisConfig;
7 +use std::sync::Arc;
8 +use std::sync::Mutex;
9 +use tokio::net::TcpListener;
10 +use tokio::sync::mpsc;
11 +use tokio_stream::wrappers::{ReceiverStream, TcpListenerStream};
12 +use tokio_util::sync::CancellationToken;
13 +use tonic::transport::Server;
14
15 #[test]
16 fn proto_ip_conversion_handles_ipv4_and_ipv6() {
@@ -36,6 +45,69 @@ fn proto_ip_conversion_rejects_unspecified_version() {
45 assert_eq!(proto_ip_to_ip_addr(ip), None);
46 }
47
48 +#[test]
49 +fn endpoint_uri_respects_explicit_scheme_and_grpc_secure() {
50 + let plain = RoutingDynamicBiorisRisInstanceConfig {
51 + grpc_addr: "127.0.0.1:50051".to_string(),
52 + grpc_secure: false,
53 + vrf_id: 0,
54 + vrf: String::new(),
55 + };
56 + assert_eq!(build_endpoint_uri(&plain), "http://127.0.0.1:50051");
57 +
58 + let secure = RoutingDynamicBiorisRisInstanceConfig {
59 + grpc_secure: true,
60 + ..plain.clone()
61 + };
62 + assert_eq!(build_endpoint_uri(&secure), "https://127.0.0.1:50051");
63 +
64 + let explicit = RoutingDynamicBiorisRisInstanceConfig {
65 + grpc_addr: "http://ris.example.test:50051".to_string(),
66 + grpc_secure: true,
67 + vrf_id: 0,
68 + vrf: String::new(),
69 + };
70 + assert_eq!(
71 + build_endpoint_uri(&explicit),
72 + "http://ris.example.test:50051"
73 + );
74 +}
75 +
76 +#[test]
77 +fn parse_router_ip_accepts_plain_ips_and_socket_addresses() {
78 + assert_eq!(
79 + parse_router_ip(" 203.0.113.10 "),
80 + Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 10)))
81 + );
82 + assert_eq!(
83 + parse_router_ip("203.0.113.10:179"),
84 + Some(IpAddr::V4(Ipv4Addr::new(203, 0, 113, 10)))
85 + );
86 + assert_eq!(
87 + parse_router_ip("[2001:db8::1]:179"),
88 + Some(IpAddr::V6("2001:db8::1".parse().expect("parse v6")))
89 + );
90 + assert_eq!(parse_router_ip("not an address"), None);
91 +}
92 +
93 +#[tokio::test]
94 +async fn connect_bioris_client_reports_invalid_endpoint_uri() {
95 + let instance = RoutingDynamicBiorisRisInstanceConfig {
96 + grpc_addr: "http://[::1".to_string(),
97 + grpc_secure: false,
98 + vrf_id: 0,
99 + vrf: String::new(),
100 + };
101 +
102 + let err = connect_bioris_client(&instance, Duration::from_millis(50))
103 + .await
104 + .expect_err("invalid URI should fail before any network connection");
105 + assert!(
106 + format!("{err:#}").contains("invalid BioRIS endpoint URI"),
107 + "unexpected error: {err:#}"
108 + );
109 +}
110 +
111 #[test]
112 fn route_to_update_flattens_set_path_segments_like_akvorado() {
113 let route = Route {
@@ -231,3 +303,474 @@ fn peer_key_stability_is_deterministic() {
303 );
304 assert_eq!(peer_a, peer_b);
305 }
306 +
307 +#[tokio::test]
308 +async fn bioris_listener_fetches_dump_rib_from_in_process_grpc_server() {
309 + let server_listener = TcpListener::bind("127.0.0.1:0")
310 + .await
311 + .expect("bind BioRIS fixture server");
312 + let server_addr = server_listener
313 + .local_addr()
314 + .expect("read BioRIS fixture server address");
315 + let server_shutdown = CancellationToken::new();
316 + let server_shutdown_task = server_shutdown.clone();
317 + let service = TestRisService::new(vec![test_bioris_route()], None);
318 + let server = tokio::spawn(async move {
319 + Server::builder()
320 + .add_service(
321 + proto::bio::ris::routing_information_service_server::RoutingInformationServiceServer::new(
322 + service,
323 + ),
324 + )
325 + .serve_with_incoming_shutdown(
326 + TcpListenerStream::new(server_listener),
327 + server_shutdown_task.cancelled(),
328 + )
329 + .await
330 + });
331 +
332 + let runtime = DynamicRoutingRuntime::default();
333 + let metrics = Arc::new(IngestMetrics::default());
334 + let shutdown = CancellationToken::new();
335 + let listener_runtime = runtime.clone();
336 + let listener_metrics = Arc::clone(&metrics);
337 + let listener_shutdown = shutdown.clone();
338 + let listener = tokio::spawn(async move {
339 + run_bioris_listener(
340 + RoutingDynamicBiorisConfig {
341 + enabled: true,
342 + ris_instances: vec![RoutingDynamicBiorisRisInstanceConfig {
343 + grpc_addr: server_addr.to_string(),
344 + grpc_secure: false,
345 + vrf_id: 10,
346 + vrf: "default".to_string(),
347 + }],
348 + timeout: Duration::from_secs(1),
349 + refresh: Duration::from_secs(10),
350 + refresh_timeout: Duration::from_secs(1),
351 + },
352 + listener_runtime,
353 + listener_metrics,
354 + listener_shutdown,
355 + )
356 + .await
357 + });
358 +
359 + tokio::time::timeout(Duration::from_secs(3), async {
360 + loop {
361 + if let Some(route) = runtime.lookup(
362 + "203.0.113.42".parse().expect("parse route lookup address"),
363 + Some("198.51.100.1".parse().expect("parse next-hop")),
364 + None,
365 + ) {
366 + assert_eq!(route.asn, 64_501);
367 + assert_eq!(route.as_path, vec![64_500, 64_501]);
368 + assert_eq!(route.communities, vec![100, 200]);
369 + assert_eq!(route.large_communities.len(), 1);
370 + assert_eq!(route.large_communities[0].asn, 64_500);
371 + assert_eq!(route.large_communities[0].local_data1, 10);
372 + assert_eq!(route.large_communities[0].local_data2, 20);
373 + break;
374 + }
375 + tokio::time::sleep(Duration::from_millis(25)).await;
376 + }
377 + })
378 + .await
379 + .expect("BioRIS listener did not publish DumpRIB route");
380 +
381 + shutdown.cancel();
382 + listener
383 + .await
384 + .expect("join BioRIS listener")
385 + .expect("BioRIS listener should stop cleanly");
386 + server_shutdown.cancel();
387 + server
388 + .await
389 + .expect("join BioRIS fixture server")
390 + .expect("BioRIS fixture server should stop cleanly");
391 +}
392 +
393 +#[tokio::test]
394 +async fn bioris_listener_sends_configured_vrf_and_applies_observe_updates() {
395 + let server_listener = TcpListener::bind("127.0.0.1:0")
396 + .await
397 + .expect("bind BioRIS fixture server");
398 + let server_addr = server_listener
399 + .local_addr()
400 + .expect("read BioRIS fixture server address");
401 + let (observe_tx, observe_rx) = mpsc::channel(8);
402 + let service = TestRisService::new(vec![test_bioris_route()], Some(observe_rx));
403 + let dump_requests = service.dump_requests();
404 + let observe_requests = service.observe_requests();
405 + let server_shutdown = CancellationToken::new();
406 + let server_shutdown_task = server_shutdown.clone();
407 + let server = tokio::spawn(async move {
408 + Server::builder()
409 + .add_service(
410 + proto::bio::ris::routing_information_service_server::RoutingInformationServiceServer::new(
411 + service,
412 + ),
413 + )
414 + .serve_with_incoming_shutdown(
415 + TcpListenerStream::new(server_listener),
416 + server_shutdown_task.cancelled(),
417 + )
418 + .await
419 + });
420 +
421 + let runtime = DynamicRoutingRuntime::default();
422 + let metrics = Arc::new(IngestMetrics::default());
423 + let shutdown = CancellationToken::new();
424 + let listener_runtime = runtime.clone();
425 + let listener_metrics = Arc::clone(&metrics);
426 + let listener_shutdown = shutdown.clone();
427 + let listener = tokio::spawn(async move {
428 + run_bioris_listener(
429 + RoutingDynamicBiorisConfig {
430 + enabled: true,
431 + ris_instances: vec![RoutingDynamicBiorisRisInstanceConfig {
432 + grpc_addr: server_addr.to_string(),
433 + grpc_secure: false,
434 + vrf_id: 42,
435 + vrf: "blue".to_string(),
436 + }],
437 + timeout: Duration::from_secs(1),
438 + refresh: Duration::from_secs(10),
439 + refresh_timeout: Duration::from_secs(1),
440 + },
441 + listener_runtime,
442 + listener_metrics,
443 + listener_shutdown,
444 + )
445 + .await
446 + });
447 +
448 + wait_for_route_asn(&runtime, "203.0.113.42", 64_501)
449 + .await
450 + .expect("initial DumpRIB route should be published");
451 +
452 + let observe_route = test_bioris_observe_route();
453 + observe_tx
454 + .send(Ok(RibUpdate {
455 + advertisement: true,
456 + is_initial_dump: false,
457 + end_of_rib: false,
458 + route: Some(observe_route.clone()),
459 + }))
460 + .await
461 + .expect("send observe advertisement");
462 + let observed = wait_for_route_asn(&runtime, "203.0.113.200", 64_511)
463 + .await
464 + .expect("ObserveRIB advertisement should update runtime");
465 + assert_eq!(observed.as_path, vec![64_510, 64_511]);
466 + assert_eq!(observed.communities, vec![300]);
467 + assert_eq!(observed.large_communities.len(), 1);
468 + assert_eq!(observed.large_communities[0].asn, 64_511);
469 + assert_eq!(observed.large_communities[0].local_data1, 30);
470 + assert_eq!(observed.large_communities[0].local_data2, 40);
471 +
472 + observe_tx
473 + .send(Ok(RibUpdate {
474 + advertisement: false,
475 + is_initial_dump: false,
476 + end_of_rib: false,
477 + route: Some(observe_route),
478 + }))
479 + .await
480 + .expect("send observe withdrawal");
481 + wait_for_route_asn(&runtime, "203.0.113.200", 64_501)
482 + .await
483 + .expect("ObserveRIB withdrawal should remove more-specific route");
484 +
485 + {
486 + let requests = dump_requests.lock().expect("lock dump requests");
487 + assert!(
488 + requests.iter().any(|request| {
489 + request.router == "203.0.113.10"
490 + && request.vrf_id == 42
491 + && request.vrf == "blue"
492 + && request.afisafi == AfiSafi::Ipv4Unicast.as_proto()
493 + }),
494 + "DumpRIB IPv4 request did not include configured router/VRF: {requests:?}"
495 + );
496 + assert!(
497 + requests
498 + .iter()
499 + .any(|request| request.afisafi == AfiSafi::Ipv6Unicast.as_proto()),
500 + "DumpRIB IPv6 request was not attempted: {requests:?}"
501 + );
502 + }
503 + {
504 + let requests = observe_requests.lock().expect("lock observe requests");
505 + assert!(
506 + requests.iter().any(|request| {
507 + request.router == "203.0.113.10"
508 + && request.vrf_id == 42
509 + && request.vrf == "blue"
510 + && request.afisafi == AfiSafi::Ipv4Unicast.as_observe_proto()
511 + && request.allow_unready_rib
512 + }),
513 + "ObserveRIB IPv4 request did not include configured router/VRF: {requests:?}"
514 + );
515 + }
516 +
517 + shutdown.cancel();
518 + listener
519 + .await
520 + .expect("join BioRIS listener")
521 + .expect("BioRIS listener should stop cleanly");
522 + server_shutdown.cancel();
523 + server
524 + .await
525 + .expect("join BioRIS fixture server")
526 + .expect("BioRIS fixture server should stop cleanly");
527 +}
528 +
529 +async fn wait_for_route_asn(
530 + runtime: &DynamicRoutingRuntime,
531 + lookup: &str,
532 + asn: u32,
533 +) -> Option<crate::enrichment::StaticRoutingEntry> {
534 + let lookup = lookup.parse().expect("parse lookup address");
535 + tokio::time::timeout(Duration::from_secs(3), async {
536 + loop {
537 + if let Some(route) = runtime.lookup(lookup, None, None)
538 + && route.asn == asn
539 + {
540 + return route;
541 + }
542 + tokio::time::sleep(Duration::from_millis(25)).await;
543 + }
544 + })
545 + .await
546 + .ok()
547 +}
548 +
549 +#[derive(Debug)]
550 +struct TestRisService {
551 + dump_routes: Vec<Route>,
552 + observe_rx: Arc<Mutex<Option<mpsc::Receiver<std::result::Result<RibUpdate, tonic::Status>>>>>,
553 + dump_requests: Arc<Mutex<Vec<DumpRibRequest>>>,
554 + observe_requests: Arc<Mutex<Vec<ObserveRibRequest>>>,
555 +}
556 +
557 +impl TestRisService {
558 + fn new(
559 + dump_routes: Vec<Route>,
560 + observe_rx: Option<mpsc::Receiver<std::result::Result<RibUpdate, tonic::Status>>>,
561 + ) -> Self {
562 + Self {
563 + dump_routes,
564 + observe_rx: Arc::new(Mutex::new(observe_rx)),
565 + dump_requests: Arc::new(Mutex::new(Vec::new())),
566 + observe_requests: Arc::new(Mutex::new(Vec::new())),
567 + }
568 + }
569 +
570 + fn dump_requests(&self) -> Arc<Mutex<Vec<DumpRibRequest>>> {
571 + Arc::clone(&self.dump_requests)
572 + }
573 +
574 + fn observe_requests(&self) -> Arc<Mutex<Vec<ObserveRibRequest>>> {
575 + Arc::clone(&self.observe_requests)
576 + }
577 +}
578 +
579 +#[tonic::async_trait]
580 +impl proto::bio::ris::routing_information_service_server::RoutingInformationService
581 + for TestRisService
582 +{
583 + async fn lpm(
584 + &self,
585 + _request: tonic::Request<proto::bio::ris::LpmRequest>,
586 + ) -> std::result::Result<tonic::Response<proto::bio::ris::LpmResponse>, tonic::Status> {
587 + Ok(tonic::Response::new(proto::bio::ris::LpmResponse {
588 + routes: Vec::new(),
589 + }))
590 + }
591 +
592 + async fn get(
593 + &self,
594 + _request: tonic::Request<proto::bio::ris::GetRequest>,
595 + ) -> std::result::Result<tonic::Response<proto::bio::ris::GetResponse>, tonic::Status> {
596 + Ok(tonic::Response::new(proto::bio::ris::GetResponse {
597 + routes: Vec::new(),
598 + }))
599 + }
600 +
601 + async fn get_routers(
602 + &self,
603 + _request: tonic::Request<proto::bio::ris::GetRoutersRequest>,
604 + ) -> std::result::Result<tonic::Response<proto::bio::ris::GetRoutersResponse>, tonic::Status>
605 + {
606 + Ok(tonic::Response::new(proto::bio::ris::GetRoutersResponse {
607 + routers: vec![proto::bio::ris::Router {
608 + sys_name: "fixture-router".to_string(),
609 + vrf_ids: vec![10],
610 + address: "203.0.113.10".to_string(),
611 + }],
612 + }))
613 + }
614 +
615 + async fn get_longer(
616 + &self,
617 + _request: tonic::Request<proto::bio::ris::GetLongerRequest>,
618 + ) -> std::result::Result<tonic::Response<proto::bio::ris::GetLongerResponse>, tonic::Status>
619 + {
620 + Ok(tonic::Response::new(proto::bio::ris::GetLongerResponse {
621 + routes: Vec::new(),
622 + }))
623 + }
624 +
625 + type ObserveRIBStream = ReceiverStream<std::result::Result<RibUpdate, tonic::Status>>;
626 +
627 + async fn observe_rib(
628 + &self,
629 + request: tonic::Request<proto::bio::ris::ObserveRibRequest>,
630 + ) -> std::result::Result<tonic::Response<Self::ObserveRIBStream>, tonic::Status> {
631 + let request = request.into_inner();
632 + let is_ipv4 = request.afisafi == AfiSafi::Ipv4Unicast.as_observe_proto();
633 + self.observe_requests
634 + .lock()
635 + .expect("lock observe request log")
636 + .push(request);
637 + let rx = if is_ipv4 {
638 + self.observe_rx
639 + .lock()
640 + .expect("lock observe receiver")
641 + .take()
642 + .unwrap_or_else(empty_observe_receiver)
643 + } else {
644 + empty_observe_receiver()
645 + };
646 + Ok(tonic::Response::new(ReceiverStream::new(rx)))
647 + }
648 +
649 + type DumpRIBStream = tokio_stream::Iter<
650 + std::vec::IntoIter<std::result::Result<proto::bio::ris::DumpRibReply, tonic::Status>>,
651 + >;
652 +
653 + async fn dump_rib(
654 + &self,
655 + request: tonic::Request<proto::bio::ris::DumpRibRequest>,
656 + ) -> std::result::Result<tonic::Response<Self::DumpRIBStream>, tonic::Status> {
657 + let request = request.into_inner();
658 + self.dump_requests
659 + .lock()
660 + .expect("lock dump request log")
661 + .push(request.clone());
662 + let routes = if request.afisafi == AfiSafi::Ipv4Unicast.as_proto() {
663 + self.dump_routes
664 + .iter()
665 + .cloned()
666 + .map(|route| Ok(proto::bio::ris::DumpRibReply { route: Some(route) }))
667 + .collect()
668 + } else {
669 + Vec::new()
670 + };
671 + Ok(tonic::Response::new(tokio_stream::iter(routes)))
672 + }
673 +}
674 +
675 +fn test_bioris_route() -> Route {
676 + Route {
677 + pfx: Some(ProtoPrefix {
678 + address: Some(ProtoIp {
679 + higher: 0,
680 + lower: u64::from(u32::from(Ipv4Addr::new(203, 0, 113, 0))),
681 + version: ProtoIpVersion::IPv4 as i32,
682 + }),
683 + length: 24,
684 + }),
685 + paths: vec![proto::bio::route::Path {
686 + r#type: 0,
687 + static_path: None,
688 + bgp_path: Some(BgpPath {
689 + path_identifier: 1,
690 + next_hop: Some(ProtoIp {
691 + higher: 0,
692 + lower: u64::from(u32::from(Ipv4Addr::new(198, 51, 100, 1))),
693 + version: ProtoIpVersion::IPv4 as i32,
694 + }),
695 + local_pref: 0,
696 + as_path: vec![ProtoAsPathSegment {
697 + as_sequence: true,
698 + asns: vec![64_500, 64_501],
699 + }],
700 + origin: 0,
701 + med: 0,
702 + ebgp: false,
703 + bgp_identifier: 0,
704 + source: None,
705 + communities: vec![100, 200],
706 + large_communities: vec![proto::bio::route::LargeCommunity {
707 + global_administrator: 64_500,
708 + data_part1: 10,
709 + data_part2: 20,
710 + }],
711 + originator_id: 0,
712 + cluster_list: vec![],
713 + unknown_attributes: vec![],
714 + bmp_post_policy: false,
715 + only_to_customer: 0,
716 + }),
717 + hidden_reason: 0,
718 + time_learned: 0,
719 + grp_path: None,
720 + }],
721 + }
722 +}
723 +
724 +fn test_bioris_observe_route() -> Route {
725 + Route {
726 + pfx: Some(ProtoPrefix {
727 + address: Some(ProtoIp {
728 + higher: 0,
729 + lower: u64::from(u32::from(Ipv4Addr::new(203, 0, 113, 128))),
730 + version: ProtoIpVersion::IPv4 as i32,
731 + }),
732 + length: 25,
733 + }),
734 + paths: vec![proto::bio::route::Path {
735 + r#type: 0,
736 + static_path: None,
737 + bgp_path: Some(BgpPath {
738 + path_identifier: 2,
739 + next_hop: Some(ProtoIp {
740 + higher: 0,
741 + lower: u64::from(u32::from(Ipv4Addr::new(198, 51, 100, 2))),
742 + version: ProtoIpVersion::IPv4 as i32,
743 + }),
744 + local_pref: 0,
745 + as_path: vec![ProtoAsPathSegment {
746 + as_sequence: true,
747 + asns: vec![64_510, 64_511],
748 + }],
749 + origin: 0,
750 + med: 0,
751 + ebgp: false,
752 + bgp_identifier: 0,
753 + source: None,
754 + communities: vec![300],
755 + large_communities: vec![proto::bio::route::LargeCommunity {
756 + global_administrator: 64_511,
757 + data_part1: 30,
758 + data_part2: 40,
759 + }],
760 + originator_id: 0,
761 + cluster_list: vec![],
762 + unknown_attributes: vec![],
763 + bmp_post_policy: false,
764 + only_to_customer: 0,
765 + }),
766 + hidden_reason: 0,
767 + time_learned: 0,
768 + grp_path: None,
769 + }],
770 + }
771 +}
772 +
773 +fn empty_observe_receiver() -> mpsc::Receiver<std::result::Result<RibUpdate, tonic::Status>> {
774 + let (_tx, rx) = mpsc::channel(1);
775 + rx
776 +}
src/crates/netflow-plugin/src/routing/bmp/listener.rs
+12
@@ -20,6 +20,18 @@ pub(crate) async fn run_bmp_listener(
20 let listener = TcpListener::bind(listen_addr)
21 .await
22 .with_context(|| format!("failed to bind BMP listener on {}", listen_addr))?;
23 + run_bmp_listener_with_bound_listener(listener, config, runtime, shutdown).await
24 +}
25 +
26 +pub(crate) async fn run_bmp_listener_with_bound_listener(
27 + listener: TcpListener,
28 + config: RoutingDynamicBmpConfig,
29 + runtime: DynamicRoutingRuntime,
30 + shutdown: CancellationToken,
31 +) -> Result<()> {
32 + let listen_addr = listener
33 + .local_addr()
34 + .with_context(|| "failed to read BMP listener address")?;
35 let accepted_rds = Arc::new(
36 parse_configured_rds(&config.rds)
37 .with_context(|| "invalid enrichment.routing_dynamic.bmp.rds entries")?,
src/crates/netflow-plugin/src/routing/bmp/tests.rs
+468 -3
@@ -1,17 +1,27 @@
1 use super::rd::{parse_configured_rds, parse_rd_text};
2 use super::routes::{
3 - flatten_as_path, flatten_as4_path, ipv4_mpls_label_routes, ipv4_mpls_vpn_routes,
3 + apply_update, flatten_as_path, flatten_as4_path, ipv4_mpls_label_routes, ipv4_mpls_vpn_routes,
4 l2_evpn_routes, mp_reach_to_routes, mp_unreach_to_routes, path_id_component,
5 route_distinguisher_to_u64,
6 };
7 use super::session::{BmpSessionDecision, bmp_session_decision};
8 use super::*;
9 use ipnet::{Ipv4Net, Ipv6Net};
10 +use netgauze_bgp_pkt::community::{Community, LargeCommunity};
11 use netgauze_bgp_pkt::nlri::{
12 EthernetSegmentIdentifier, EthernetTag, Ipv4MplsVpnUnicastAddress, Ipv4NlriMplsLabelsAddress,
12 - Ipv4Unicast, L2EvpnIpv4PrefixRoute, L2EvpnIpv6PrefixRoute, MplsLabel,
13 + Ipv4Unicast, Ipv4UnicastAddress, L2EvpnIpv4PrefixRoute, L2EvpnIpv6PrefixRoute, MplsLabel,
14 };
14 -use netgauze_bgp_pkt::path_attribute::{As2PathSegment, As4PathSegment};
15 +use netgauze_bgp_pkt::path_attribute::{
16 + As2PathSegment, As4PathSegment, Communities, LargeCommunities, NextHop, PathAttribute,
17 +};
18 +use std::collections::HashSet;
19 +use std::process::{Child, Command, Stdio};
20 +use std::time::Duration;
21 +use tokio::io::AsyncWriteExt;
22 +use tokio::net::{TcpListener, TcpStream};
23 +use tokio::process::Command as TokioCommand;
24 +use tokio_util::codec::Encoder;
25
26 #[test]
27 fn parse_rd_text_accepts_akvorado_formats() {
@@ -230,6 +240,113 @@ fn flatten_as4_path_keeps_first_from_set_segments() {
240 assert_eq!(flatten_as4_path(&path), vec![100, 200, 300]);
241 }
242
243 +#[test]
244 +fn bmp_apply_update_respects_collection_flags() {
245 + let runtime = DynamicRoutingRuntime::default();
246 + let peer = DynamicRoutingPeerKey {
247 + exporter: "192.0.2.10:10179".parse().expect("parse exporter"),
248 + session_id: 7,
249 + peer_id: "peer".to_string(),
250 + };
251 + let config = RoutingDynamicBmpConfig {
252 + collect_asns: false,
253 + collect_as_paths: false,
254 + collect_communities: false,
255 + ..Default::default()
256 + };
257 +
258 + apply_update(
259 + &peer,
260 + 64_512,
261 + BmpPeerType::GlobalInstancePeer {
262 + ipv6: false,
263 + post_policy: false,
264 + asn2: false,
265 + adj_rib_out: false,
266 + },
267 + 0,
268 + &test_bmp_update(),
269 + &config,
270 + &HashSet::new(),
271 + &runtime,
272 + );
273 +
274 + let route = runtime
275 + .lookup(
276 + "203.0.113.42".parse().expect("parse lookup address"),
277 + Some("198.51.100.1".parse().expect("parse next-hop")),
278 + Some("192.0.2.10".parse().expect("parse exporter")),
279 + )
280 + .expect("BMP route should be published");
281 + assert_eq!(route.asn, 0);
282 + assert!(route.as_path.is_empty());
283 + assert!(route.communities.is_empty());
284 + assert!(route.large_communities.is_empty());
285 + assert_eq!(
286 + route.next_hop,
287 + Some(IpAddr::V4(Ipv4Addr::new(198, 51, 100, 1)))
288 + );
289 +}
290 +
291 +#[test]
292 +fn bmp_apply_update_maps_as_path_communities_and_withdraws() {
293 + let runtime = DynamicRoutingRuntime::default();
294 + let peer = DynamicRoutingPeerKey {
295 + exporter: "192.0.2.10:10179".parse().expect("parse exporter"),
296 + session_id: 7,
297 + peer_id: "peer".to_string(),
298 + };
299 + let config = RoutingDynamicBmpConfig::default();
300 +
301 + apply_update(
302 + &peer,
303 + 64_512,
304 + BmpPeerType::GlobalInstancePeer {
305 + ipv6: false,
306 + post_policy: false,
307 + asn2: false,
308 + adj_rib_out: false,
309 + },
310 + 0,
311 + &test_bmp_update(),
312 + &config,
313 + &HashSet::new(),
314 + &runtime,
315 + );
316 +
317 + let route = runtime
318 + .lookup(
319 + "203.0.113.42".parse().expect("parse lookup address"),
320 + None,
321 + None,
322 + )
323 + .expect("BMP route should be published");
324 + assert_eq!(route.asn, 64_501);
325 + assert_eq!(route.as_path, vec![64_500, 64_501]);
326 + assert_eq!(route.communities, vec![100, 200]);
327 + assert_eq!(route.large_communities.len(), 1);
328 + assert_eq!(route.large_communities[0].asn, 64_500);
329 + assert_eq!(route.large_communities[0].local_data1, 10);
330 + assert_eq!(route.large_communities[0].local_data2, 20);
331 +
332 + apply_update(
333 + &peer,
334 + 64_512,
335 + BmpPeerType::GlobalInstancePeer {
336 + ipv6: false,
337 + post_policy: false,
338 + asn2: false,
339 + adj_rib_out: false,
340 + },
341 + 0,
342 + &test_bmp_withdraw_update(),
343 + &config,
344 + &HashSet::new(),
345 + &runtime,
346 + );
347 + assert_eq!(runtime.route_count(), 0);
348 +}
349 +
350 #[test]
351 fn bmp_session_requires_initiation_first() {
352 let mut initialized = false;
@@ -263,3 +380,351 @@ fn bmp_session_accepts_initiation_and_then_processes_messages() {
380 BmpSessionDecision::CloseTermination
381 );
382 }
383 +
384 +#[tokio::test]
385 +async fn bmp_listener_accepts_encoded_initiation_and_termination_messages() {
386 + let listener = bind_loopback_listener().await;
387 + let listen = listener.local_addr().expect("read BMP listen address");
388 + let shutdown = CancellationToken::new();
389 + let runtime = DynamicRoutingRuntime::default();
390 + let config = RoutingDynamicBmpConfig {
391 + enabled: true,
392 + listen: listen.to_string(),
393 + keep: Duration::from_millis(10),
394 + ..Default::default()
395 + };
396 + let listener_shutdown = shutdown.clone();
397 + let listener = tokio::spawn(async move {
398 + super::listener::run_bmp_listener_with_bound_listener(
399 + listener,
400 + config,
401 + runtime,
402 + listener_shutdown,
403 + )
404 + .await
405 + });
406 +
407 + let mut stream = connect_with_retry(listen).await;
408 + stream
409 + .write_all(&encode_bmp(BmpMessage::V3(BmpMessageValue::Initiation(
410 + netgauze_bmp_pkt::v3::InitiationMessage::new(Vec::new()),
411 + ))))
412 + .await
413 + .expect("write BMP initiation");
414 + stream
415 + .write_all(&encode_bmp(BmpMessage::V3(BmpMessageValue::Termination(
416 + netgauze_bmp_pkt::v3::TerminationMessage::new(Vec::new()),
417 + ))))
418 + .await
419 + .expect("write BMP termination");
420 + drop(stream);
421 +
422 + shutdown.cancel();
423 + listener
424 + .await
425 + .expect("join BMP listener")
426 + .expect("BMP listener should stop cleanly");
427 +}
428 +
429 +#[tokio::test]
430 +async fn bmp_listener_accepts_gobgp_route_when_binaries_are_set() {
431 + let Some(gobgpd) = std::env::var_os("NETDATA_GOBGPD") else {
432 + return;
433 + };
434 + let Some(gobgp) = std::env::var_os("NETDATA_GOBGP") else {
435 + return;
436 + };
437 +
438 + let bmp_listener = bind_loopback_listener().await;
439 + let listen = bmp_listener.local_addr().expect("read BMP listen address");
440 + // GoBGP's gRPC API is owned by the external daemon, so this opt-in
441 + // live test can only pass a concrete address and let gobgpd bind it.
442 + let dir = tempfile::tempdir().expect("create GoBGP fixture dir");
443 + let config = dir.path().join("gobgpd.toml");
444 + std::fs::write(
445 + &config,
446 + format!(
447 + r#"[global.config]
448 + as = 64512
449 + router-id = "192.0.2.1"
450 + port = -1
451 +
452 +[[bmp-servers]]
453 + [bmp-servers.config]
454 + address = "{}"
455 + port = {}
456 + route-monitoring-policy = "all"
457 +"#,
458 + listen.ip(),
459 + listen.port()
460 + ),
461 + )
462 + .expect("write GoBGP fixture config");
463 +
464 + let shutdown = CancellationToken::new();
465 + let runtime = DynamicRoutingRuntime::default();
466 + let config_bmp = RoutingDynamicBmpConfig {
467 + enabled: true,
468 + listen: listen.to_string(),
469 + keep: Duration::from_millis(10),
470 + ..Default::default()
471 + };
472 + let listener_runtime = runtime.clone();
473 + let listener_shutdown = shutdown.clone();
474 + let listener = tokio::spawn(async move {
475 + super::listener::run_bmp_listener_with_bound_listener(
476 + bmp_listener,
477 + config_bmp,
478 + listener_runtime,
479 + listener_shutdown,
480 + )
481 + .await
482 + });
483 +
484 + let mut last_retry_error = None;
485 + for _ in 0..5 {
486 + let gobgp_api = reserve_loopback_addr();
487 + let mut daemon = ChildGuard {
488 + child: Command::new(&gobgpd)
489 + .arg("--api-hosts")
490 + .arg(gobgp_api.to_string())
491 + .arg("-f")
492 + .arg(&config)
493 + .arg("-l")
494 + .arg("info")
495 + .arg("-p")
496 + .stdout(Stdio::null())
497 + .stderr(Stdio::null())
498 + .spawn()
499 + .expect("start gobgpd"),
500 + };
501 +
502 + match run_gobgp_route_fixture(
503 + &gobgp,
504 + gobgp_api,
505 + &runtime,
506 + "203.0.113.0/24",
507 + "203.0.113.42",
508 + )
509 + .await
510 + {
511 + Ok(()) => {
512 + stop_child(&mut daemon.child);
513 + shutdown.cancel();
514 + listener
515 + .await
516 + .expect("join BMP listener")
517 + .expect("BMP listener should stop cleanly");
518 + return;
519 + }
520 + Err(err) => {
521 + let daemon_exited = matches!(daemon.child.try_wait(), Ok(Some(_)));
522 + stop_child(&mut daemon.child);
523 + if daemon_exited {
524 + last_retry_error = Some(err);
525 + continue;
526 + }
527 + panic!("{err}");
528 + }
529 + }
530 + }
531 +
532 + shutdown.cancel();
533 + listener
534 + .await
535 + .expect("join BMP listener")
536 + .expect("BMP listener should stop cleanly");
537 + panic!(
538 + "GoBGP BMP fixture failed after retrying reserved API ports: {}",
539 + last_retry_error.unwrap_or_else(|| "unknown startup failure".to_string())
540 + );
541 +}
542 +
543 +fn reserve_loopback_addr() -> SocketAddr {
544 + let socket = std::net::TcpListener::bind("127.0.0.1:0").expect("reserve loopback socket");
545 + let addr = socket.local_addr().expect("read loopback address");
546 + drop(socket);
547 + addr
548 +}
549 +
550 +async fn bind_loopback_listener() -> TcpListener {
551 + TcpListener::bind("127.0.0.1:0")
552 + .await
553 + .expect("bind loopback listener")
554 +}
555 +
556 +async fn connect_with_retry(addr: SocketAddr) -> TcpStream {
557 + tokio::time::timeout(Duration::from_secs(3), async {
558 + loop {
559 + match TcpStream::connect(addr).await {
560 + Ok(stream) => return stream,
561 + Err(_) => tokio::time::sleep(Duration::from_millis(10)).await,
562 + }
563 + }
564 + })
565 + .await
566 + .expect("connect to BMP listener")
567 +}
568 +
569 +fn encode_bmp(message: BmpMessage) -> Vec<u8> {
570 + let mut codec = BmpCodec::default();
571 + let mut out = bytes::BytesMut::new();
572 + codec.encode(message, &mut out).expect("encode BMP message");
573 + out.to_vec()
574 +}
575 +
576 +fn test_bmp_update() -> BgpUpdateMessage {
577 + BgpUpdateMessage::new(
578 + vec![],
579 + vec![
580 + PathAttribute::from(
581 + false,
582 + true,
583 + false,
584 + true,
585 + PathAttributeValue::AsPath(AsPath::As4PathSegments(vec![As4PathSegment::new(
586 + AsPathSegmentType::AsSequence,
587 + vec![64_500, 64_501],
588 + )])),
589 + )
590 + .expect("build AS path attribute"),
591 + PathAttribute::from(
592 + false,
593 + true,
594 + false,
595 + false,
596 + PathAttributeValue::NextHop(NextHop::new(Ipv4Addr::new(198, 51, 100, 1))),
597 + )
598 + .expect("build next-hop attribute"),
599 + PathAttribute::from(
600 + true,
601 + true,
602 + false,
603 + false,
604 + PathAttributeValue::Communities(Communities::new(vec![
605 + Community::new(100),
606 + Community::new(200),
607 + ])),
608 + )
609 + .expect("build communities attribute"),
610 + PathAttribute::from(
611 + true,
612 + true,
613 + false,
614 + false,
615 + PathAttributeValue::LargeCommunities(LargeCommunities::new(vec![
616 + LargeCommunity::new(64_500, 10, 20),
617 + ])),
618 + )
619 + .expect("build large communities attribute"),
620 + ],
621 + vec![Ipv4UnicastAddress::new_no_path_id(
622 + Ipv4Unicast::from_net(
623 + Ipv4Net::new(Ipv4Addr::new(203, 0, 113, 0), 24).expect("v4 prefix"),
624 + )
625 + .expect("build IPv4 unicast NLRI"),
626 + )],
627 + )
628 +}
629 +
630 +fn test_bmp_withdraw_update() -> BgpUpdateMessage {
631 + BgpUpdateMessage::new(
632 + vec![Ipv4UnicastAddress::new_no_path_id(
633 + Ipv4Unicast::from_net(
634 + Ipv4Net::new(Ipv4Addr::new(203, 0, 113, 0), 24).expect("v4 prefix"),
635 + )
636 + .expect("build IPv4 unicast withdrawal"),
637 + )],
638 + vec![],
639 + vec![],
640 + )
641 +}
642 +
643 +async fn run_gobgp_route_fixture(
644 + gobgp: &std::ffi::OsStr,
645 + api: SocketAddr,
646 + runtime: &DynamicRoutingRuntime,
647 + prefix: &str,
648 + lookup: &str,
649 +) -> Result<(), String> {
650 + let mut ready = false;
651 + for _ in 0..100 {
652 + if TokioCommand::new(gobgp)
653 + .arg("-u")
654 + .arg(api.ip().to_string())
655 + .arg("-p")
656 + .arg(api.port().to_string())
657 + .arg("global")
658 + .stdout(Stdio::null())
659 + .stderr(Stdio::null())
660 + .status()
661 + .await
662 + .map(|status| status.success())
663 + .unwrap_or(false)
664 + {
665 + ready = true;
666 + break;
667 + }
668 + tokio::time::sleep(Duration::from_millis(25)).await;
669 + }
670 + if !ready {
671 + return Err(format!("gobgp API did not become ready on {api}"));
672 + }
673 +
674 + let status = TokioCommand::new(gobgp)
675 + .arg("-u")
676 + .arg(api.ip().to_string())
677 + .arg("-p")
678 + .arg(api.port().to_string())
679 + .args([
680 + "global",
681 + "rib",
682 + "add",
683 + "-a",
684 + "ipv4",
685 + prefix,
686 + "nexthop",
687 + "192.0.2.254",
688 + "aspath",
689 + "64500,64501",
690 + ])
691 + .status()
692 + .await
693 + .map_err(|err| format!("run gobgp route add: {err}"))?;
694 + if !status.success() {
695 + return Err(format!("gobgp route add failed with status {status}"));
696 + }
697 +
698 + let lookup_addr = lookup.parse().expect("parse route lookup address");
699 + tokio::time::timeout(Duration::from_secs(5), async {
700 + loop {
701 + if let Some(route) = runtime.lookup(lookup_addr, None, None) {
702 + assert_eq!(route.asn, 64_501);
703 + assert_eq!(route.as_path, vec![64_500, 64_501]);
704 + return;
705 + }
706 + tokio::time::sleep(Duration::from_millis(25)).await;
707 + }
708 + })
709 + .await
710 + .map_err(|_| "GoBGP BMP route was not published to dynamic routing runtime".to_string())?;
711 + Ok(())
712 +}
713 +
714 +fn stop_child(child: &mut Child) {
715 + if matches!(child.try_wait(), Ok(Some(_))) {
716 + return;
717 + }
718 + let _ = child.kill();
719 + let _ = child.wait();
720 +}
721 +
722 +struct ChildGuard {
723 + child: Child,
724 +}
725 +
726 +impl Drop for ChildGuard {
727 + fn drop(&mut self) {
728 + stop_child(&mut self.child);
729 + }
730 +}
src/crates/netflow-plugin/src/startup_memory_tests.rs
-1
@@ -1,5 +1,4 @@
1 use super::{charts, facet_runtime, ingest, plugin_config, query, tiering};
2 -use bytesize::ByteSize;
2 use rt::PluginRuntime;
3 use std::env;
4 use std::fs;
src/crates/netflow-plugin/testdata/ATTRIBUTION.md
+9
@@ -45,6 +45,15 @@ Files (verbatim copies from Akvorado testdata):
45
46 These files are used by unit/integration-style tests to validate decoding of real NetFlow/IPFIX/sFlow records.
47
48 +Source: https://github.com/maxmind/MaxMind-DB
49 +License: Apache-2.0 or MIT
50 +
51 +Files (verbatim copies of MaxMind-DB test databases):
52 +- test-data/GeoLite2-ASN-Test.mmdb
53 +- test-data/GeoLite2-City-Test.mmdb
54 +
55 +These files are used by unit tests to validate real MMDB lookups for ASN and geolocation enrichment.
56 +
57 Additional stress-test captures downloaded at test time (license per tcpreplay capture page):
58 - https://s3.amazonaws.com/tcpreplay-pcap-files/smallFlows.pcap
59 - https://s3.amazonaws.com/tcpreplay-pcap-files/bigFlows.pcap
src/crates/netflow-plugin/testdata/mmdb/GeoLite2-ASN-Test.mmdb
Binary files /dev/null and b/src/crates/netflow-plugin/testdata/mmdb/GeoLite2-ASN-Test.mmdb differ
src/crates/netflow-plugin/testdata/mmdb/GeoLite2-City-Test.mmdb
Binary files /dev/null and b/src/crates/netflow-plugin/testdata/mmdb/GeoLite2-City-Test.mmdb differ
src/crates/netflow-plugin/testdata/mmdb/Netdata-Topology-ASN-Test.mmdb
Binary files /dev/null and b/src/crates/netflow-plugin/testdata/mmdb/Netdata-Topology-ASN-Test.mmdb differ
src/crates/netflow-plugin/testdata/mmdb/Netdata-Topology-GEO-Test.mmdb
Binary files /dev/null and b/src/crates/netflow-plugin/testdata/mmdb/Netdata-Topology-GEO-Test.mmdb differ
src/go/tools/topology-ip-intel-downloader/README.md
+48 -5
@@ -92,6 +92,31 @@ topology-ip-intel-downloader \
92 --geo dbip:city-lite
93 ```
94
95 +Use public ASN and country providers:
96 +
97 +```bash
98 +topology-ip-intel-downloader \
99 + --asn iptoasn:combined \
100 + --geo ip2location:country-lite
101 +```
102 +
103 +Use MaxMind GeoLite2 sources when `MAXMIND_LICENSE_KEY` is available in the
104 +environment:
105 +
106 +```bash
107 +MAXMIND_LICENSE_KEY="..." topology-ip-intel-downloader \
108 + --asn maxmind:geolite2-asn \
109 + --geo maxmind:geolite2-country
110 +```
111 +
112 +Use CAIDA RouteViews prefix2as for ASN-only attribution:
113 +
114 +```bash
115 +topology-ip-intel-downloader \
116 + --asn caida:prefix2as \
117 + --geo dbip:country-lite
118 +```
119 +
120 Prefer a custom GEO source, then fall back to DB-IP:
121
122 ```bash
@@ -126,6 +151,10 @@ The config file uses ordered `sources[]` entries. Each entry is explicit about:
151 Built-in DB-IP sources can omit `url` and `path`. The downloader resolves the
152 current monthly download URL from the official DB-IP landing page.
153
154 +Built-in MaxMind sources require `MAXMIND_LICENSE_KEY` in the environment.
155 +The generated metadata redacts URL query strings so license keys are not written
156 +to `topology-ip-intel.json`.
157 +
158 User-provided databases can point directly to:
159
160 - a local file with `path`
@@ -170,16 +199,30 @@ Important:
199
200 ## Supported built-in sources
201
173 -- `dbip:asn-lite`
174 -- `dbip:country-lite`
175 -- `dbip:city-lite`
176 -- `iptoasn:combined`
202 +ASN sources:
203 +
204 +- `dbip:asn-lite` (`mmdb`, `csv`)
205 +- `iptoasn:combined` (`tsv`)
206 +- `caida:prefix2as` (`tsv`)
207 +- `maxmind:geolite2-asn` (`mmdb`, requires `MAXMIND_LICENSE_KEY`)
208 +
209 +GEO sources:
210 +
211 +- `dbip:country-lite` (`mmdb`, `csv`)
212 +- `dbip:city-lite` (`mmdb`, `csv`)
213 +- `iptoasn:combined` (`tsv`, country only)
214 +- `maxmind:geolite2-country` (`csv`, requires `MAXMIND_LICENSE_KEY`, country only)
215 +- `ip2location:country-lite` (`csv`, country only)
216 +- `ipdeny:country-zones` (`cidr`, country only)
217 +- `ipip:country` (`txt`, country only)
218
219 Supported formats:
220
221 - `mmdb`
222 - `csv`
182 -- `tsv` for `iptoasn:combined`
223 +- `tsv` for `iptoasn:combined` and `caida:prefix2as`
224 +- `cidr` for `ipdeny:country-zones`
225 +- `txt` for `ipip:country`
226
227 The downloader accepts direct file/URL config for advanced cases, but the CLI
228 source tokens are intentionally focused on built-in source families.
src/go/tools/topology-ip-intel-downloader/config.go
+85 -6
@@ -16,8 +16,13 @@ import (
16 )
17
18 const (
19 - providerIPToASN = "iptoasn"
20 - providerDBIP = "dbip"
19 + providerIPToASN = "iptoasn"
20 + providerDBIP = "dbip"
21 + providerCAIDA = "caida"
22 + providerMaxMind = "maxmind"
23 + providerIP2Location = "ip2location"
24 + providerIPDeny = "ipdeny"
25 + providerIPIP = "ipip"
26 )
27
28 const (
@@ -26,10 +31,16 @@ const (
31 )
32
33 const (
29 - artifactIPToASNCombined = "combined"
30 - artifactDBIPASNLite = "asn-lite"
31 - artifactDBIPCountryLite = "country-lite"
32 - artifactDBIPCityLite = "city-lite"
34 + artifactIPToASNCombined = "combined"
35 + artifactDBIPASNLite = "asn-lite"
36 + artifactDBIPCountryLite = "country-lite"
37 + artifactDBIPCityLite = "city-lite"
38 + artifactCAIDAPrefix2AS = "prefix2as"
39 + artifactMaxMindGeoLite2ASN = "geolite2-asn"
40 + artifactMaxMindGeoLite2Country = "geolite2-country"
41 + artifactIP2LocationCountryLite = "country-lite"
42 + artifactIPDenyCountryZones = "country-zones"
43 + artifactIPIPCountry = "country"
44 )
45
46 const (
@@ -42,6 +53,8 @@ const (
53 formatMMDB = "mmdb"
54 formatCSV = "csv"
55 formatTSV = "tsv"
56 + formatCIDR = "cidr"
57 + formatTXT = "txt"
58 )
59
60 type config struct {
@@ -218,6 +231,72 @@ func builtInSource(provider, artifact string) (builtInSourceSpec, bool) {
231 formatCSV: {},
232 },
233 }, true
234 + case provider == providerCAIDA && artifact == artifactCAIDAPrefix2AS:
235 + return builtInSourceSpec{
236 + pageURL: "https://data.caida.org/datasets/routing/routeviews-prefix2as/pfx2as-creation.log",
237 + defaultFormat: formatTSV,
238 + allowedFamily: map[string]struct{}{
239 + sourceFamilyASN: {},
240 + },
241 + allowedFormat: map[string]struct{}{
242 + formatTSV: {},
243 + },
244 + }, true
245 + case provider == providerMaxMind && artifact == artifactMaxMindGeoLite2ASN:
246 + return builtInSourceSpec{
247 + directURL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-ASN&license_key=${MAXMIND_LICENSE_KEY}&suffix=tar.gz",
248 + defaultFormat: formatMMDB,
249 + allowedFamily: map[string]struct{}{
250 + sourceFamilyASN: {},
251 + },
252 + allowedFormat: map[string]struct{}{
253 + formatMMDB: {},
254 + },
255 + }, true
256 + case provider == providerMaxMind && artifact == artifactMaxMindGeoLite2Country:
257 + return builtInSourceSpec{
258 + directURL: "https://download.maxmind.com/app/geoip_download?edition_id=GeoLite2-Country-CSV&license_key=${MAXMIND_LICENSE_KEY}&suffix=zip",
259 + defaultFormat: formatCSV,
260 + allowedFamily: map[string]struct{}{
261 + sourceFamilyGeo: {},
262 + },
263 + allowedFormat: map[string]struct{}{
264 + formatCSV: {},
265 + },
266 + }, true
267 + case provider == providerIP2Location && artifact == artifactIP2LocationCountryLite:
268 + return builtInSourceSpec{
269 + directURL: "https://download.ip2location.com/lite/IP2LOCATION-LITE-DB1.CSV.ZIP",
270 + defaultFormat: formatCSV,
271 + allowedFamily: map[string]struct{}{
272 + sourceFamilyGeo: {},
273 + },
274 + allowedFormat: map[string]struct{}{
275 + formatCSV: {},
276 + },
277 + }, true
278 + case provider == providerIPDeny && artifact == artifactIPDenyCountryZones:
279 + return builtInSourceSpec{
280 + directURL: "https://www.ipdeny.com/ipblocks/data/countries/all-zones.tar.gz",
281 + defaultFormat: formatCIDR,
282 + allowedFamily: map[string]struct{}{
283 + sourceFamilyGeo: {},
284 + },
285 + allowedFormat: map[string]struct{}{
286 + formatCIDR: {},
287 + },
288 + }, true
289 + case provider == providerIPIP && artifact == artifactIPIPCountry:
290 + return builtInSourceSpec{
291 + directURL: "https://cdn.ipip.net/17mon/country.zip",
292 + defaultFormat: formatTXT,
293 + allowedFamily: map[string]struct{}{
294 + sourceFamilyGeo: {},
295 + },
296 + allowedFormat: map[string]struct{}{
297 + formatTXT: {},
298 + },
299 + }, true
300 default:
301 return builtInSourceSpec{}, false
302 }
src/go/tools/topology-ip-intel-downloader/configs/topology-ip-intel.yaml
+33
@@ -21,6 +21,39 @@ sources:
21 artifact: city-lite
22 format: mmdb
23
24 + # Other supported ASN providers:
25 + # - provider: iptoasn
26 + # artifact: combined
27 + # format: tsv
28 + # - provider: caida
29 + # artifact: prefix2as
30 + # format: tsv
31 + # - provider: maxmind
32 + # artifact: geolite2-asn
33 + # format: mmdb
34 + #
35 + # Other supported GEO providers:
36 + # - provider: dbip
37 + # artifact: country-lite
38 + # format: mmdb
39 + # - provider: iptoasn
40 + # artifact: combined
41 + # format: tsv
42 + # - provider: maxmind
43 + # artifact: geolite2-country
44 + # format: csv
45 + # - provider: ip2location
46 + # artifact: country-lite
47 + # format: csv
48 + # - provider: ipdeny
49 + # artifact: country-zones
50 + # format: cidr
51 + # - provider: ipip
52 + # artifact: country
53 + # format: txt
54 + #
55 + # MaxMind entries require MAXMIND_LICENSE_KEY in the downloader environment.
56 +
57 output:
58 # Destination directory for generated databases.
59 directory: /var/cache/netdata/topology-ip-intel
src/go/tools/topology-ip-intel-downloader/fetch.go
+200 -14
@@ -3,18 +3,26 @@
3 package main
4
5 import (
6 + "archive/tar"
7 "archive/zip"
8 "bytes"
9 "compress/gzip"
10 + "errors"
11 "fmt"
12 "io"
13 "net/http"
14 + "net/url"
15 "os"
16 + "path"
17 "regexp"
18 + "sort"
19 + "strconv"
20 "strings"
21 "time"
22 )
23
24 +var errTarMMDBNotFound = errors.New("tar payload has no mmdb member")
25 +
26 type downloader struct {
27 client *http.Client
28 userAgent string
@@ -37,7 +45,7 @@ func (d *downloader) readDataset(source sourceEntry) (generationDatasetRef, []by
45 if err != nil {
46 return generationDatasetRef{}, nil, err
47 }
40 - payload, err := decodePayload(raw)
48 + payload, err := decodePayloadForSource(source, raw)
49 if err != nil {
50 return generationDatasetRef{}, nil, err
51 }
@@ -70,34 +78,54 @@ func (d *downloader) resolveSource(source sourceEntry) (resolvedSource, error) {
78
79 switch {
80 case source.path != "":
81 + path, err := expandEnvPlaceholders(source.path)
82 + if err != nil {
83 + return resolvedSource{}, err
84 + }
85 ref.Source = "path"
74 - ref.Path = source.path
86 + ref.Path = path
87 return resolvedSource{
88 ref: ref,
77 - fetchPath: source.path,
89 + fetchPath: path,
90 }, nil
91 case source.url != "":
92 + fetchURL, err := expandEnvPlaceholders(source.url)
93 + if err != nil {
94 + return resolvedSource{}, err
95 + }
96 ref.Source = "url"
81 - ref.URL = source.url
97 + ref.URL = sanitizeURLForMetadata(source.url)
98 return resolvedSource{
99 ref: ref,
84 - fetchURL: source.url,
100 + fetchURL: fetchURL,
101 }, nil
102 case spec.directURL != "":
103 + fetchURL, err := expandEnvPlaceholders(spec.directURL)
104 + if err != nil {
105 + return resolvedSource{}, err
106 + }
107 ref.Source = "builtin"
88 - ref.URL = spec.directURL
108 + ref.URL = sanitizeURLForMetadata(spec.directURL)
109 return resolvedSource{
110 ref: ref,
91 - fetchURL: spec.directURL,
111 + fetchURL: fetchURL,
112 }, nil
113 case spec.pageURL != "":
94 - resolvedURL, err := d.resolveDBIPArtifactURL(spec.pageURL, source.artifact, source.format)
114 + var (
115 + resolvedURL string
116 + err error
117 + )
118 + if source.provider == providerCAIDA && source.artifact == artifactCAIDAPrefix2AS {
119 + resolvedURL, err = d.resolveCAIDAPrefix2ASURL(spec.pageURL)
120 + } else {
121 + resolvedURL, err = d.resolveDBIPArtifactURL(spec.pageURL, source.artifact, source.format)
122 + }
123 if err != nil {
124 return resolvedSource{}, err
125 }
126 ref.Source = "builtin"
99 - ref.DownloadPage = spec.pageURL
100 - ref.ResolvedURL = resolvedURL
127 + ref.DownloadPage = sanitizeURLForMetadata(spec.pageURL)
128 + ref.ResolvedURL = sanitizeURLForMetadata(resolvedURL)
129 return resolvedSource{
130 ref: ref,
131 fetchURL: resolvedURL,
@@ -139,6 +167,100 @@ func (d *downloader) resolveDBIPArtifactURL(pageURL, artifact, format string) (s
167 return string(match), nil
168 }
169
170 +func (d *downloader) resolveCAIDAPrefix2ASURL(logURL string) (string, error) {
171 + page, err := d.readHTTP(logURL)
172 + if err != nil {
173 + return "", fmt.Errorf("failed to fetch CAIDA prefix2as creation log %s: %w", redactURLForDisplay(logURL), err)
174 + }
175 +
176 + type caidaCandidate struct {
177 + path string
178 + timestamp int64
179 + hasTime bool
180 + }
181 +
182 + candidates := make([]caidaCandidate, 0)
183 + for _, line := range strings.Split(string(page), "\n") {
184 + fields := strings.Fields(strings.TrimSpace(line))
185 + if len(fields) == 0 {
186 + continue
187 + }
188 + candidate := fields[len(fields)-1]
189 + if strings.HasSuffix(candidate, ".pfx2as.gz") {
190 + entry := caidaCandidate{path: candidate}
191 + if len(fields) >= 2 {
192 + if timestamp, err := strconv.ParseInt(fields[len(fields)-2], 10, 64); err == nil {
193 + entry.timestamp = timestamp
194 + entry.hasTime = true
195 + }
196 + }
197 + candidates = append(candidates, entry)
198 + }
199 + }
200 + if len(candidates) == 0 {
201 + return "", fmt.Errorf("failed to resolve latest CAIDA prefix2as download from %s", redactURLForDisplay(logURL))
202 + }
203 + sort.Slice(candidates, func(i, j int) bool {
204 + if candidates[i].hasTime != candidates[j].hasTime {
205 + return !candidates[i].hasTime
206 + }
207 + if candidates[i].timestamp != candidates[j].timestamp {
208 + return candidates[i].timestamp < candidates[j].timestamp
209 + }
210 + return candidates[i].path < candidates[j].path
211 + })
212 + latest := candidates[len(candidates)-1].path
213 +
214 + base, err := url.Parse(logURL)
215 + if err != nil {
216 + return "", fmt.Errorf("failed to parse CAIDA prefix2as creation log URL %s: %w", redactURLForDisplay(logURL), err)
217 + }
218 + base.RawQuery = ""
219 + base.Fragment = ""
220 + if !strings.HasSuffix(base.Path, "/") {
221 + base.Path = path.Dir(base.Path) + "/"
222 + }
223 + ref, err := url.Parse(latest)
224 + if err != nil {
225 + return "", fmt.Errorf("failed to parse CAIDA prefix2as candidate %q: %w", latest, err)
226 + }
227 + return base.ResolveReference(ref).String(), nil
228 +}
229 +
230 +func expandEnvPlaceholders(raw string) (string, error) {
231 + missing := map[string]struct{}{}
232 + expanded := os.Expand(raw, func(name string) string {
233 + value, ok := os.LookupEnv(name)
234 + if !ok || strings.TrimSpace(value) == "" {
235 + missing[name] = struct{}{}
236 + return ""
237 + }
238 + return value
239 + })
240 + if len(missing) == 0 {
241 + return expanded, nil
242 + }
243 + names := make([]string, 0, len(missing))
244 + for name := range missing {
245 + names = append(names, name)
246 + }
247 + sort.Strings(names)
248 + return "", fmt.Errorf("missing environment variable(s): %s", strings.Join(names, ", "))
249 +}
250 +
251 +func sanitizeURLForMetadata(raw string) string {
252 + parsed, err := url.Parse(raw)
253 + if err != nil || parsed.Host == "" {
254 + return "<redacted>"
255 + }
256 + parsed.User = nil
257 + if parsed.RawQuery != "" {
258 + parsed.RawQuery = "redacted"
259 + }
260 + parsed.Fragment = ""
261 + return parsed.String()
262 +}
263 +
264 func (d *downloader) readRaw(path, rawURL string) ([]byte, error) {
265 if path != "" {
266 content, err := os.ReadFile(path)
@@ -151,31 +273,67 @@ func (d *downloader) readRaw(path, rawURL string) ([]byte, error) {
273 }
274
275 func (d *downloader) readHTTP(rawURL string) ([]byte, error) {
276 + displayURL := redactURLForDisplay(rawURL)
277 req, err := http.NewRequest(http.MethodGet, rawURL, nil)
278 if err != nil {
156 - return nil, fmt.Errorf("failed to build request %s: %w", rawURL, err)
279 + return nil, fmt.Errorf("failed to build request %s: %w", displayURL, err)
280 }
281 req.Header.Set("User-Agent", d.userAgent)
282
283 start := time.Now()
284 resp, err := d.client.Do(req)
285 if err != nil {
163 - return nil, fmt.Errorf("failed to fetch %s: %w", rawURL, err)
286 + return nil, fmt.Errorf("failed to fetch %s: %w", displayURL, err)
287 }
288 defer resp.Body.Close()
289
290 if resp.StatusCode != http.StatusOK {
168 - return nil, fmt.Errorf("failed to fetch %s: unexpected status %d", rawURL, resp.StatusCode)
291 + return nil, fmt.Errorf("failed to fetch %s: unexpected status %d", displayURL, resp.StatusCode)
292 }
293
294 content, err := io.ReadAll(resp.Body)
295 if err != nil {
173 - return nil, fmt.Errorf("failed to read %s: %w", rawURL, err)
296 + return nil, fmt.Errorf("failed to read %s: %w", displayURL, err)
297 }
298 _ = start
299 return content, nil
300 }
301
302 +func decodePayloadForSource(source sourceEntry, raw []byte) ([]byte, error) {
303 + switch {
304 + case source.provider == providerMaxMind && source.artifact == artifactMaxMindGeoLite2ASN:
305 + return decodeMaxMindASNPayload(raw)
306 + case source.provider == providerMaxMind && source.artifact == artifactMaxMindGeoLite2Country:
307 + return raw, nil
308 + case source.provider == providerIP2Location && source.artifact == artifactIP2LocationCountryLite:
309 + return raw, nil
310 + case source.provider == providerIPDeny && source.artifact == artifactIPDenyCountryZones:
311 + return raw, nil
312 + case source.provider == providerIPIP && source.artifact == artifactIPIPCountry:
313 + return raw, nil
314 + default:
315 + return decodePayload(raw)
316 + }
317 +}
318 +
319 +func decodeMaxMindASNPayload(raw []byte) ([]byte, error) {
320 + if len(raw) < 2 || raw[0] != 0x1f || raw[1] != 0x8b {
321 + return raw, nil
322 + }
323 + content, err := decodeGzip(raw)
324 + if err != nil {
325 + return nil, err
326 + }
327 + mmdb, err := extractMMDBFromTar(content)
328 + if err == nil {
329 + return mmdb, nil
330 + }
331 + if !errors.Is(err, errTarMMDBNotFound) {
332 + return nil, fmt.Errorf("failed to extract MaxMind ASN MMDB from tar payload: %w", err)
333 + }
334 + return content, nil
335 +}
336 +
337 func decodePayload(raw []byte) ([]byte, error) {
338 if len(raw) >= 2 && raw[0] == 0x1f && raw[1] == 0x8b {
339 return decodeGzip(raw)
@@ -222,3 +380,31 @@ func decodeZip(raw []byte) ([]byte, error) {
380 }
381 return nil, fmt.Errorf("zip payload has no regular files")
382 }
383 +
384 +func extractMMDBFromTar(raw []byte) ([]byte, error) {
385 + tr := tar.NewReader(bytes.NewReader(raw))
386 + for {
387 + header, err := tr.Next()
388 + if err == io.EOF {
389 + return nil, errTarMMDBNotFound
390 + }
391 + if err != nil {
392 + if !looksLikeTarPayload(raw) {
393 + return nil, errTarMMDBNotFound
394 + }
395 + return nil, err
396 + }
397 + if header.Typeflag != tar.TypeReg || !strings.HasSuffix(strings.ToLower(header.Name), ".mmdb") {
398 + continue
399 + }
400 + content, err := io.ReadAll(tr)
401 + if err != nil {
402 + return nil, fmt.Errorf("failed to read tar member %s: %w", header.Name, err)
403 + }
404 + return content, nil
405 + }
406 +}
407 +
408 +func looksLikeTarPayload(raw []byte) bool {
409 + return len(raw) >= 512 && bytes.Equal(raw[257:262], []byte("ustar"))
410 +}
src/go/tools/topology-ip-intel-downloader/fetch_test.go
+132
@@ -6,6 +6,7 @@ import (
6 "archive/zip"
7 "bytes"
8 "compress/gzip"
9 + "fmt"
10 "net/http"
11 "net/http/httptest"
12 "testing"
@@ -60,6 +61,120 @@ func TestResolveDBIPArtifactURL(t *testing.T) {
61 require.Equal(t, "https://download.db-ip.com/free/dbip-city-lite-2026-03.mmdb.gz", resolved)
62 }
63
64 +func TestResolveCAIDAPrefix2ASURL(t *testing.T) {
65 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
66 + require.Equal(t, "/routing/pfx2as-creation.log", r.URL.Path)
67 + _, _ = w.Write([]byte(
68 + "1\t1778084001\t2026/05/routeviews-rv2-20260505-1200.pfx2as.gz\n" +
69 + "2\t1778170446\t2026/05/routeviews-rv2-20260506-1200.pfx2as.gz\n" +
70 + "3\t1777997562\t2026/05/routeviews-rv2-20260504-1200.pfx2as.gz\n",
71 + ))
72 + }))
73 + defer server.Close()
74 +
75 + dl := newDownloader(httpConfig{timeout: time.Second, userAgent: "test"})
76 + resolved, err := dl.resolveCAIDAPrefix2ASURL(server.URL + "/routing/pfx2as-creation.log?mirror=local")
77 + require.NoError(t, err)
78 + require.Equal(t, server.URL+"/routing/2026/05/routeviews-rv2-20260506-1200.pfx2as.gz", resolved)
79 +}
80 +
81 +func TestResolveMaxMindSourceExpandsEnvForFetchAndRedactsMetadata(t *testing.T) {
82 + t.Setenv("MAXMIND_LICENSE_KEY", "secret-license-key")
83 +
84 + dl := newDownloader(httpConfig{timeout: time.Second, userAgent: "test"})
85 + resolved, err := dl.resolveSource(sourceEntry{
86 + name: "maxmind-asn",
87 + family: sourceFamilyASN,
88 + provider: providerMaxMind,
89 + artifact: artifactMaxMindGeoLite2ASN,
90 + format: formatMMDB,
91 + })
92 + require.NoError(t, err)
93 + require.Contains(t, resolved.fetchURL, "secret-license-key")
94 + require.NotContains(t, resolved.ref.URL, "secret-license-key")
95 + require.Contains(t, resolved.ref.URL, "redacted")
96 +}
97 +
98 +func TestResolveMaxMindSourceReportsMissingEnvWithoutSecretURL(t *testing.T) {
99 + t.Setenv("MAXMIND_LICENSE_KEY", "")
100 +
101 + dl := newDownloader(httpConfig{timeout: time.Second, userAgent: "test"})
102 + _, err := dl.resolveSource(sourceEntry{
103 + name: "maxmind-asn",
104 + family: sourceFamilyASN,
105 + provider: providerMaxMind,
106 + artifact: artifactMaxMindGeoLite2ASN,
107 + format: formatMMDB,
108 + })
109 + require.Error(t, err)
110 + require.Contains(t, err.Error(), "MAXMIND_LICENSE_KEY")
111 +}
112 +
113 +func TestDecodeMaxMindASNPayloadExtractsTarredMMDB(t *testing.T) {
114 + payload := "fake mmdb"
115 + tarball := buildTarGZ(t, map[string]string{
116 + "GeoLite2-ASN_20260508/GeoLite2-ASN.mmdb": payload,
117 + })
118 +
119 + decoded, err := decodeMaxMindASNPayload(tarball)
120 + require.NoError(t, err)
121 + require.Equal(t, []byte(payload), decoded)
122 +}
123 +
124 +func TestDecodeMaxMindASNPayloadExtractsTarWithoutUSTARMagic(t *testing.T) {
125 + payload := "legacy tar mmdb"
126 + rawTar := buildTar(t, map[string]string{
127 + "GeoLite2-ASN_20260508/GeoLite2-ASN.mmdb": payload,
128 + })
129 + clearTarMagicAndRecomputeChecksum(t, rawTar[:512])
130 +
131 + var compressed bytes.Buffer
132 + zw := gzip.NewWriter(&compressed)
133 + _, err := zw.Write(rawTar)
134 + require.NoError(t, err)
135 + require.NoError(t, zw.Close())
136 +
137 + decoded, err := decodeMaxMindASNPayload(compressed.Bytes())
138 + require.NoError(t, err)
139 + require.Equal(t, []byte(payload), decoded)
140 +}
141 +
142 +func TestDecodeMaxMindASNPayloadAcceptsGzippedMMDB(t *testing.T) {
143 + payload := []byte("fake mmdb")
144 + var compressed bytes.Buffer
145 + zw := gzip.NewWriter(&compressed)
146 + _, err := zw.Write(payload)
147 + require.NoError(t, err)
148 + require.NoError(t, zw.Close())
149 +
150 + decoded, err := decodeMaxMindASNPayload(compressed.Bytes())
151 + require.NoError(t, err)
152 + require.Equal(t, payload, decoded)
153 +}
154 +
155 +func TestDecodeMaxMindASNPayloadRejectsCorruptTarPayload(t *testing.T) {
156 + tarLikeCorruptPayload := make([]byte, 512)
157 + copy(tarLikeCorruptPayload[257:], []byte("ustar"))
158 +
159 + var compressed bytes.Buffer
160 + zw := gzip.NewWriter(&compressed)
161 + _, err := zw.Write(tarLikeCorruptPayload)
162 + require.NoError(t, err)
163 + require.NoError(t, zw.Close())
164 +
165 + _, err = decodeMaxMindASNPayload(compressed.Bytes())
166 + require.Error(t, err)
167 + require.Contains(t, err.Error(), "failed to extract MaxMind ASN MMDB from tar payload")
168 +}
169 +
170 +func TestExpandEnvPlaceholdersRequiresMissingVariable(t *testing.T) {
171 + t.Setenv("TOPOLOGY_TEST_PRESENT", "value")
172 + _, err := expandEnvPlaceholders("https://example.test/${TOPOLOGY_TEST_MISSING}/${TOPOLOGY_TEST_PRESENT}")
173 + require.Error(t, err)
174 + require.Contains(t, err.Error(), "TOPOLOGY_TEST_MISSING")
175 + require.NotContains(t, err.Error(), "value")
176 +}
177 +
178 func TestResolveSourceUsesLocalPathWithoutNetwork(t *testing.T) {
179 dl := newDownloader(httpConfig{timeout: time.Second, userAgent: "test"})
180 resolved, err := dl.resolveSource(sourceEntry{
@@ -75,3 +190,20 @@ func TestResolveSourceUsesLocalPathWithoutNetwork(t *testing.T) {
190 require.Equal(t, "path", resolved.ref.Source)
191 require.Equal(t, "/tmp/custom-city.mmdb", resolved.ref.Path)
192 }
193 +
194 +func clearTarMagicAndRecomputeChecksum(t *testing.T, header []byte) {
195 + t.Helper()
196 +
197 + require.Len(t, header, 512)
198 + for i := 257; i < 265; i++ {
199 + header[i] = 0
200 + }
201 + for i := 148; i < 156; i++ {
202 + header[i] = ' '
203 + }
204 + sum := 0
205 + for _, b := range header {
206 + sum += int(b)
207 + }
208 + copy(header[148:156], []byte(fmt.Sprintf("%06o\x00 ", sum)))
209 +}
src/go/tools/topology-ip-intel-downloader/flags_test.go
+63
@@ -24,6 +24,69 @@ func TestParseSourceTokenUsesBuiltInDefaultFormat(t *testing.T) {
24 require.Equal(t, formatMMDB, source.format)
25 }
26
27 +func TestParseSourceTokenSupportsAllBuiltInProviderFamilies(t *testing.T) {
28 + tests := map[string]struct {
29 + family string
30 + token string
31 + provider string
32 + artifact string
33 + format string
34 + }{
35 + "caida asn": {
36 + family: sourceFamilyASN,
37 + token: "caida:prefix2as",
38 + provider: providerCAIDA,
39 + artifact: artifactCAIDAPrefix2AS,
40 + format: formatTSV,
41 + },
42 + "maxmind asn": {
43 + family: sourceFamilyASN,
44 + token: "maxmind:geolite2-asn",
45 + provider: providerMaxMind,
46 + artifact: artifactMaxMindGeoLite2ASN,
47 + format: formatMMDB,
48 + },
49 + "maxmind country": {
50 + family: sourceFamilyGeo,
51 + token: "maxmind:geolite2-country",
52 + provider: providerMaxMind,
53 + artifact: artifactMaxMindGeoLite2Country,
54 + format: formatCSV,
55 + },
56 + "ip2location country": {
57 + family: sourceFamilyGeo,
58 + token: "ip2location:country-lite",
59 + provider: providerIP2Location,
60 + artifact: artifactIP2LocationCountryLite,
61 + format: formatCSV,
62 + },
63 + "ipdeny country": {
64 + family: sourceFamilyGeo,
65 + token: "ipdeny:country-zones",
66 + provider: providerIPDeny,
67 + artifact: artifactIPDenyCountryZones,
68 + format: formatCIDR,
69 + },
70 + "ipip country": {
71 + family: sourceFamilyGeo,
72 + token: "ipip:country",
73 + provider: providerIPIP,
74 + artifact: artifactIPIPCountry,
75 + format: formatTXT,
76 + },
77 + }
78 +
79 + for name, tc := range tests {
80 + t.Run(name, func(t *testing.T) {
81 + source, err := parseSourceToken(tc.family, tc.token)
82 + require.NoError(t, err)
83 + require.Equal(t, tc.provider, source.provider)
84 + require.Equal(t, tc.artifact, source.artifact)
85 + require.Equal(t, tc.format, source.format)
86 + })
87 + }
88 +}
89 +
90 func TestParseSourceTokenRejectsInvalidFamilyArtifactCombination(t *testing.T) {
91 _, err := parseSourceToken(sourceFamilyGeo, "dbip:asn-lite")
92 require.Error(t, err)
src/go/tools/topology-ip-intel-downloader/parse.go
+479 -7
@@ -3,14 +3,18 @@
3 package main
4
5 import (
6 + "archive/tar"
7 + "archive/zip"
8 "bufio"
9 "bytes"
10 + "compress/gzip"
11 "encoding/csv"
12 "fmt"
13 "io"
14 "math/big"
15 "net"
16 "net/netip"
17 + "path"
18 "strconv"
19 "strings"
20
@@ -122,6 +126,16 @@ func parseASNSource(source sourceEntry, payload []byte) ([]asnRange, error) {
126 default:
127 return nil, fmt.Errorf("unsupported dbip ASN format %q", source.format)
128 }
129 + case source.provider == providerCAIDA && source.artifact == artifactCAIDAPrefix2AS:
130 + if source.format != formatTSV {
131 + return nil, fmt.Errorf("caida prefix2as requires tsv format, got %q", source.format)
132 + }
133 + return parseCAIDAPrefix2AS(payload)
134 + case source.provider == providerMaxMind && source.artifact == artifactMaxMindGeoLite2ASN:
135 + if source.format != formatMMDB {
136 + return nil, fmt.Errorf("maxmind geolite2 ASN requires mmdb format, got %q", source.format)
137 + }
138 + return parseDBIPAsnMMDB(payload)
139 default:
140 return nil, fmt.Errorf(
141 "unsupported ASN source %q/%q",
@@ -156,6 +170,26 @@ func parseGeoSource(source sourceEntry, payload []byte) ([]geoRange, error) {
170 default:
171 return nil, fmt.Errorf("unsupported dbip GEO format %q", source.format)
172 }
173 + case source.provider == providerMaxMind && source.artifact == artifactMaxMindGeoLite2Country:
174 + if source.format != formatCSV {
175 + return nil, fmt.Errorf("maxmind geolite2 country requires csv format, got %q", source.format)
176 + }
177 + return parseMaxMindCountryCSVZip(payload)
178 + case source.provider == providerIP2Location && source.artifact == artifactIP2LocationCountryLite:
179 + if source.format != formatCSV {
180 + return nil, fmt.Errorf("ip2location country-lite requires csv format, got %q", source.format)
181 + }
182 + return parseIP2LocationCountryZip(payload)
183 + case source.provider == providerIPDeny && source.artifact == artifactIPDenyCountryZones:
184 + if source.format != formatCIDR {
185 + return nil, fmt.Errorf("ipdeny country-zones requires cidr format, got %q", source.format)
186 + }
187 + return parseIPDenyCountryTarGZ(payload)
188 + case source.provider == providerIPIP && source.artifact == artifactIPIPCountry:
189 + if source.format != formatTXT {
190 + return nil, fmt.Errorf("ipip country requires txt format, got %q", source.format)
191 + }
192 + return parseIPIPCountryZip(payload)
193 default:
194 return nil, fmt.Errorf(
195 "unsupported GEO source %q/%q",
@@ -165,8 +199,19 @@ func parseGeoSource(source sourceEntry, payload []byte) ([]geoRange, error) {
199 }
200 }
201
202 +func estimatedRangeCapacity(size uint64, averageLineBytes, maxCapacity int) int {
203 + if size == 0 || averageLineBytes <= 0 || maxCapacity <= 0 {
204 + return 0
205 + }
206 + capacity := size / uint64(averageLineBytes)
207 + if capacity > uint64(maxCapacity) {
208 + return maxCapacity
209 + }
210 + return int(capacity)
211 +}
212 +
213 func parseIPToASNCombinedTSVAsn(payload []byte) ([]asnRange, error) {
169 - asnRanges := make([]asnRange, 0, 1<<20)
214 + asnRanges := make([]asnRange, 0, estimatedRangeCapacity(uint64(len(payload)), 64, 1<<20))
215
216 scanner := bufio.NewScanner(bytes.NewReader(payload))
217 lineNo := 0
@@ -203,7 +248,7 @@ func parseIPToASNCombinedTSVAsn(payload []byte) ([]asnRange, error) {
248 }
249
250 func parseIPToASNCombinedTSVGeo(payload []byte) ([]geoRange, error) {
206 - geoRanges := make([]geoRange, 0, 1<<20)
251 + geoRanges := make([]geoRange, 0, estimatedRangeCapacity(uint64(len(payload)), 64, 1<<20))
252
253 scanner := bufio.NewScanner(bytes.NewReader(payload))
254 lineNo := 0
@@ -242,7 +287,7 @@ func parseDBIPAsnCSV(payload []byte) ([]asnRange, error) {
287 reader.FieldsPerRecord = -1
288 reader.TrimLeadingSpace = true
289
245 - out := make([]asnRange, 0, 1<<18)
290 + out := make([]asnRange, 0, estimatedRangeCapacity(uint64(len(payload)), 80, 1<<18))
291 lineNo := 0
292 for {
293 row, err := reader.Read()
@@ -283,12 +328,60 @@ func parseDBIPAsnCSV(payload []byte) ([]asnRange, error) {
328 return out, nil
329 }
330
331 +func parseCAIDAPrefix2AS(payload []byte) ([]asnRange, error) {
332 + out := make([]asnRange, 0, estimatedRangeCapacity(uint64(len(payload)), 32, 1<<20))
333 + scanner := bufio.NewScanner(bytes.NewReader(payload))
334 + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
335 +
336 + lineNo := 0
337 + for scanner.Scan() {
338 + lineNo++
339 + line := strings.TrimSpace(scanner.Text())
340 + if line == "" || strings.HasPrefix(line, "#") {
341 + continue
342 + }
343 +
344 + fields := strings.Fields(line)
345 + if len(fields) < 3 {
346 + return nil, fmt.Errorf("caida prefix2as line %d: expected >= 3 columns", lineNo)
347 + }
348 +
349 + prefixLength, err := strconv.Atoi(strings.TrimSpace(fields[1]))
350 + if err != nil {
351 + return nil, fmt.Errorf("caida prefix2as line %d: invalid prefix length %q: %w", lineNo, fields[1], err)
352 + }
353 + prefix, err := netip.ParsePrefix(fmt.Sprintf("%s/%d", strings.TrimSpace(fields[0]), prefixLength))
354 + if err != nil {
355 + return nil, fmt.Errorf("caida prefix2as line %d: invalid prefix: %w", lineNo, err)
356 + }
357 + start, end := rangeFromPrefix(prefix.Masked())
358 +
359 + asn, err := parsePrimaryASN(fields[2])
360 + if err != nil {
361 + return nil, fmt.Errorf("caida prefix2as line %d: %w", lineNo, err)
362 + }
363 + if asn == 0 {
364 + continue
365 + }
366 +
367 + rec := asnRange{start: start, end: end, asn: asn}
368 + if err := rec.validate(); err != nil {
369 + return nil, fmt.Errorf("caida prefix2as line %d: %w", lineNo, err)
370 + }
371 + out = append(out, rec)
372 + }
373 + if err := scanner.Err(); err != nil {
374 + return nil, fmt.Errorf("failed to scan caida prefix2as payload: %w", err)
375 + }
376 + return out, nil
377 +}
378 +
379 func parseDBIPCountryCSV(payload []byte) ([]geoRange, error) {
380 reader := csv.NewReader(strings.NewReader(string(payload)))
381 reader.FieldsPerRecord = -1
382 reader.TrimLeadingSpace = true
383
291 - out := make([]geoRange, 0, 1<<18)
384 + out := make([]geoRange, 0, estimatedRangeCapacity(uint64(len(payload)), 64, 1<<18))
385 lineNo := 0
386 for {
387 row, err := reader.Read()
@@ -330,7 +423,7 @@ func parseDBIPCityCSV(payload []byte) ([]geoRange, error) {
423 reader.FieldsPerRecord = -1
424 reader.TrimLeadingSpace = true
425
333 - out := make([]geoRange, 0, 1<<18)
426 + out := make([]geoRange, 0, estimatedRangeCapacity(uint64(len(payload)), 128, 1<<18))
427 lineNo := 0
428 for {
429 row, err := reader.Read()
@@ -399,7 +492,7 @@ func parseDBIPAsnMMDB(payload []byte) ([]asnRange, error) {
492 return nil, fmt.Errorf("failed to open dbip ASN mmdb: %w", err)
493 }
494
402 - out := make([]asnRange, 0, 1<<18)
495 + out := make([]asnRange, 0)
496 networks := reader.Networks(maxminddb.SkipAliasedNetworks)
497 for networks.Next() {
498 var record dbipAsnMMDBRecord
@@ -441,7 +534,7 @@ func parseDBIPGeoMMDB(payload []byte) ([]geoRange, error) {
534 return nil, fmt.Errorf("failed to open dbip GEO mmdb: %w", err)
535 }
536
444 - out := make([]geoRange, 0, 1<<18)
537 + out := make([]geoRange, 0)
538 networks := reader.Networks(maxminddb.SkipAliasedNetworks)
539 for networks.Next() {
540 var record dbipGeoMMDBRecord
@@ -480,6 +573,272 @@ func parseDBIPGeoMMDB(payload []byte) ([]geoRange, error) {
573 return out, nil
574 }
575
576 +func parseMaxMindCountryCSVZip(payload []byte) ([]geoRange, error) {
577 + archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
578 + if err != nil {
579 + return nil, fmt.Errorf("failed to open maxmind country csv zip: %w", err)
580 + }
581 +
582 + countryByID, err := parseMaxMindCountryLocations(archive)
583 + if err != nil {
584 + return nil, err
585 + }
586 +
587 + out := make([]geoRange, 0)
588 + for _, suffix := range []string{
589 + "GeoLite2-Country-Blocks-IPv4.csv",
590 + "GeoLite2-Country-Blocks-IPv6.csv",
591 + } {
592 + ranges, err := parseMaxMindCountryBlocks(archive, suffix, countryByID)
593 + if err != nil {
594 + return nil, err
595 + }
596 + out = append(out, ranges...)
597 + }
598 + return out, nil
599 +}
600 +
601 +func parseMaxMindCountryLocations(archive *zip.Reader) (map[string]string, error) {
602 + file, err := openZipEntrySuffix(archive, "GeoLite2-Country-Locations-en.csv")
603 + if err != nil {
604 + return nil, err
605 + }
606 + rc, err := file.Open()
607 + if err != nil {
608 + return nil, err
609 + }
610 + defer rc.Close()
611 +
612 + csvr := csv.NewReader(rc)
613 + csvr.FieldsPerRecord = -1
614 + header, err := csvr.Read()
615 + if err != nil {
616 + return nil, fmt.Errorf("failed to read maxmind locations header: %w", err)
617 + }
618 + idx := csvHeaderIndex(header)
619 + idIdx, ok := idx["geoname_id"]
620 + if !ok {
621 + return nil, fmt.Errorf("maxmind locations missing geoname_id column")
622 + }
623 + countryIdx, ok := idx["country_iso_code"]
624 + if !ok {
625 + return nil, fmt.Errorf("maxmind locations missing country_iso_code column")
626 + }
627 +
628 + out := map[string]string{}
629 + lineNo := 1
630 + for {
631 + row, err := csvr.Read()
632 + if err == io.EOF {
633 + break
634 + }
635 + if err != nil {
636 + return nil, fmt.Errorf("maxmind locations line %d: %w", lineNo+1, err)
637 + }
638 + lineNo++
639 + if len(row) <= idIdx || len(row) <= countryIdx {
640 + continue
641 + }
642 + id := strings.TrimSpace(row[idIdx])
643 + country := normalizeCountry(row[countryIdx])
644 + if id != "" && country != "" {
645 + out[id] = country
646 + }
647 + }
648 + return out, nil
649 +}
650 +
651 +func parseMaxMindCountryBlocks(
652 + archive *zip.Reader,
653 + suffix string,
654 + countryByID map[string]string,
655 +) ([]geoRange, error) {
656 + file, err := openZipEntrySuffix(archive, suffix)
657 + if err != nil {
658 + return nil, err
659 + }
660 + rc, err := file.Open()
661 + if err != nil {
662 + return nil, err
663 + }
664 + defer rc.Close()
665 +
666 + csvr := csv.NewReader(rc)
667 + csvr.FieldsPerRecord = -1
668 + header, err := csvr.Read()
669 + if err != nil {
670 + return nil, fmt.Errorf("failed to read maxmind blocks header %s: %w", suffix, err)
671 + }
672 + idx := csvHeaderIndex(header)
673 + networkIdx, ok := idx["network"]
674 + if !ok {
675 + return nil, fmt.Errorf("maxmind blocks %s missing network column", suffix)
676 + }
677 + idColumns := []string{"geoname_id", "registered_country_geoname_id", "represented_country_geoname_id"}
678 +
679 + out := make([]geoRange, 0, estimatedRangeCapacity(file.UncompressedSize64, 96, 1<<18))
680 + lineNo := 1
681 + for {
682 + row, err := csvr.Read()
683 + if err == io.EOF {
684 + break
685 + }
686 + if err != nil {
687 + return nil, fmt.Errorf("maxmind blocks %s line %d: %w", suffix, lineNo+1, err)
688 + }
689 + lineNo++
690 + if len(row) <= networkIdx {
691 + continue
692 + }
693 +
694 + country := ""
695 + for _, column := range idColumns {
696 + columnIdx, ok := idx[column]
697 + if !ok || len(row) <= columnIdx {
698 + continue
699 + }
700 + if c := countryByID[strings.TrimSpace(row[columnIdx])]; c != "" {
701 + country = c
702 + break
703 + }
704 + }
705 + if country == "" {
706 + continue
707 + }
708 +
709 + rec, err := geoRangeFromToken(row[networkIdx], country)
710 + if err != nil {
711 + return nil, fmt.Errorf("maxmind blocks %s line %d: %w", suffix, lineNo, err)
712 + }
713 + out = append(out, rec)
714 + }
715 + return out, nil
716 +}
717 +
718 +func parseIP2LocationCountryZip(payload []byte) ([]geoRange, error) {
719 + archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
720 + if err != nil {
721 + return nil, fmt.Errorf("failed to open ip2location country zip: %w", err)
722 + }
723 + file, err := openZipEntryBase(archive, "IP2LOCATION-LITE-DB1.CSV")
724 + if err != nil {
725 + return nil, err
726 + }
727 + rc, err := file.Open()
728 + if err != nil {
729 + return nil, err
730 + }
731 + defer rc.Close()
732 +
733 + csvr := csv.NewReader(rc)
734 + csvr.FieldsPerRecord = -1
735 + out := make([]geoRange, 0, estimatedRangeCapacity(file.UncompressedSize64, 64, 1<<18))
736 + lineNo := 0
737 + for {
738 + row, err := csvr.Read()
739 + if err == io.EOF {
740 + break
741 + }
742 + if err != nil {
743 + return nil, fmt.Errorf("ip2location line %d: %w", lineNo+1, err)
744 + }
745 + lineNo++
746 + if len(row) < 3 {
747 + continue
748 + }
749 + country := normalizeCountry(row[2])
750 + if country == "" {
751 + continue
752 + }
753 + start, end, err := parseRangeEndpoints(row[0], row[1])
754 + if err != nil {
755 + return nil, fmt.Errorf("ip2location line %d: %w", lineNo, err)
756 + }
757 + rec := geoRange{start: start, end: end, country: country}
758 + if err := rec.validate(); err != nil {
759 + return nil, fmt.Errorf("ip2location line %d: %w", lineNo, err)
760 + }
761 + out = append(out, rec)
762 + }
763 + return out, nil
764 +}
765 +
766 +func parseIPDenyCountryTarGZ(payload []byte) ([]geoRange, error) {
767 + gz, err := gzip.NewReader(bytes.NewReader(payload))
768 + if err != nil {
769 + return nil, fmt.Errorf("failed to open ipdeny tar.gz: %w", err)
770 + }
771 + defer gz.Close()
772 +
773 + out := make([]geoRange, 0)
774 + tr := tar.NewReader(gz)
775 + for {
776 + header, err := tr.Next()
777 + if err == io.EOF {
778 + break
779 + }
780 + if err != nil {
781 + return nil, fmt.Errorf("failed to read ipdeny tar: %w", err)
782 + }
783 + name := path.Base(header.Name)
784 + zoneName, ok := strings.CutSuffix(strings.ToLower(name), ".zone")
785 + if header.Typeflag != tar.TypeReg || !ok {
786 + continue
787 + }
788 + country := normalizeCountry(zoneName)
789 + if country == "" {
790 + continue
791 + }
792 + ranges, err := parseCountryTokenLines(tr, country, "ipdeny "+header.Name)
793 + if err != nil {
794 + return nil, err
795 + }
796 + out = append(out, ranges...)
797 + }
798 + return out, nil
799 +}
800 +
801 +func parseIPIPCountryZip(payload []byte) ([]geoRange, error) {
802 + archive, err := zip.NewReader(bytes.NewReader(payload), int64(len(payload)))
803 + if err != nil {
804 + return nil, fmt.Errorf("failed to open ipip country zip: %w", err)
805 + }
806 + file, err := openZipEntryBase(archive, "country.txt")
807 + if err != nil {
808 + return nil, err
809 + }
810 + rc, err := file.Open()
811 + if err != nil {
812 + return nil, err
813 + }
814 + defer rc.Close()
815 +
816 + out := make([]geoRange, 0, estimatedRangeCapacity(file.UncompressedSize64, 48, 1<<18))
817 + scanner := bufio.NewScanner(rc)
818 + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
819 + lineNo := 0
820 + for scanner.Scan() {
821 + lineNo++
822 + fields := strings.Fields(strings.ReplaceAll(scanner.Text(), "\r", ""))
823 + if len(fields) < 2 {
824 + continue
825 + }
826 + country := normalizeCountry(fields[len(fields)-1])
827 + if country == "" {
828 + continue
829 + }
830 + rec, err := geoRangeFromToken(fields[0], country)
831 + if err != nil {
832 + return nil, fmt.Errorf("ipip line %d: %w", lineNo, err)
833 + }
834 + out = append(out, rec)
835 + }
836 + if err := scanner.Err(); err != nil {
837 + return nil, fmt.Errorf("failed to scan ipip country payload: %w", err)
838 + }
839 + return out, nil
840 +}
841 +
842 func dbipCountryCode(value *dbipCountryMMDBValue) string {
843 if value == nil {
844 return ""
@@ -594,3 +953,116 @@ func parseASN(raw string) (uint32, error) {
953 }
954 return uint32(n), nil
955 }
956 +
957 +func parsePrimaryASN(raw string) (uint32, error) {
958 + value := strings.TrimSpace(strings.Trim(raw, "\"{}"))
959 + if value == "" {
960 + return 0, nil
961 + }
962 + for _, sep := range []string{"_", ","} {
963 + if idx := strings.Index(value, sep); idx >= 0 {
964 + value = strings.TrimSpace(value[:idx])
965 + }
966 + }
967 + return parseASN(value)
968 +}
969 +
970 +func csvHeaderIndex(header []string) map[string]int {
971 + out := make(map[string]int, len(header))
972 + for i, name := range header {
973 + out[strings.TrimSpace(name)] = i
974 + }
975 + return out
976 +}
977 +
978 +func openZipEntryBase(archive *zip.Reader, name string) (*zip.File, error) {
979 + for _, file := range archive.File {
980 + if path.Base(file.Name) == name {
981 + return file, nil
982 + }
983 + }
984 + return nil, fmt.Errorf("zip entry %q not found", name)
985 +}
986 +
987 +func openZipEntrySuffix(archive *zip.Reader, suffix string) (*zip.File, error) {
988 + for _, file := range archive.File {
989 + if strings.HasSuffix(file.Name, suffix) {
990 + return file, nil
991 + }
992 + }
993 + return nil, fmt.Errorf("zip entry with suffix %q not found", suffix)
994 +}
995 +
996 +func parseCountryTokenLines(r io.Reader, country, label string) ([]geoRange, error) {
997 + out := make([]geoRange, 0)
998 + scanner := bufio.NewScanner(r)
999 + scanner.Buffer(make([]byte, 0, 64*1024), 1024*1024)
1000 + lineNo := 0
1001 + for scanner.Scan() {
1002 + lineNo++
1003 + token := strings.TrimSpace(scanner.Text())
1004 + if token == "" || strings.HasPrefix(token, "#") {
1005 + continue
1006 + }
1007 + fields := strings.Fields(token)
1008 + if len(fields) == 0 {
1009 + continue
1010 + }
1011 + rec, err := geoRangeFromToken(fields[0], country)
1012 + if err != nil {
1013 + return nil, fmt.Errorf("%s line %d: %w", label, lineNo, err)
1014 + }
1015 + out = append(out, rec)
1016 + }
1017 + if err := scanner.Err(); err != nil {
1018 + return nil, fmt.Errorf("failed to scan %s: %w", label, err)
1019 + }
1020 + return out, nil
1021 +}
1022 +
1023 +func geoRangeFromToken(raw, country string) (geoRange, error) {
1024 + start, end, err := rangeFromToken(raw)
1025 + if err != nil {
1026 + return geoRange{}, err
1027 + }
1028 + rec := geoRange{start: start, end: end, country: country}
1029 + if err := rec.validate(); err != nil {
1030 + return geoRange{}, err
1031 + }
1032 + return rec, nil
1033 +}
1034 +
1035 +func rangeFromToken(raw string) (netip.Addr, netip.Addr, error) {
1036 + token := strings.TrimSpace(strings.Trim(raw, "\""))
1037 + if token == "" {
1038 + return netip.Addr{}, netip.Addr{}, fmt.Errorf("empty range token")
1039 + }
1040 +
1041 + if strings.Contains(token, "/") {
1042 + prefix, err := netip.ParsePrefix(token)
1043 + if err != nil {
1044 + return netip.Addr{}, netip.Addr{}, err
1045 + }
1046 + start, end := rangeFromPrefix(prefix.Masked())
1047 + return start, end, nil
1048 + }
1049 +
1050 + if strings.Contains(token, "-") {
1051 + left, right, ok := strings.Cut(strings.ReplaceAll(token, " ", ""), "-")
1052 + if !ok {
1053 + return netip.Addr{}, netip.Addr{}, fmt.Errorf("invalid range token %q", raw)
1054 + }
1055 + return parseRangeEndpoints(left, right)
1056 + }
1057 +
1058 + addr, err := parseIP(token)
1059 + if err != nil {
1060 + return netip.Addr{}, netip.Addr{}, err
1061 + }
1062 + return addr, addr, nil
1063 +}
1064 +
1065 +func rangeFromPrefix(prefix netip.Prefix) (netip.Addr, netip.Addr) {
1066 + rng := netipx.RangeOfPrefix(prefix.Masked())
1067 + return rng.From(), rng.To()
1068 +}
src/go/tools/topology-ip-intel-downloader/parse_test.go
+139
@@ -3,6 +3,11 @@
3 package main
4
5 import (
6 + "archive/tar"
7 + "archive/zip"
8 + "bytes"
9 + "compress/gzip"
10 + "strings"
11 "testing"
12
13 "github.com/stretchr/testify/require"
@@ -63,6 +68,94 @@ func TestParseDBIPCityCSV(t *testing.T) {
68 require.InDelta(t, -122.0839, ranges[0].longitude, 0.0001)
69 }
70
71 +func TestParseCAIDAPrefix2AS(t *testing.T) {
72 + payload := []byte(
73 + "1.0.0.0\t24\t13335_38803\n" +
74 + "2001:db8::\t48\t64512\n",
75 + )
76 + ranges, err := parseCAIDAPrefix2AS(payload)
77 + require.NoError(t, err)
78 + require.Len(t, ranges, 2)
79 + require.Equal(t, "1.0.0.0", ranges[0].start.String())
80 + require.Equal(t, "1.0.0.255", ranges[0].end.String())
81 + require.EqualValues(t, 13335, ranges[0].asn)
82 + require.Equal(t, "2001:db8::", ranges[1].start.String())
83 + require.EqualValues(t, 64512, ranges[1].asn)
84 +}
85 +
86 +func TestParseMaxMindCountryCSVZip(t *testing.T) {
87 + payload := buildZip(t, map[string]string{
88 + "GeoLite2-Country-CSV_20260501/GeoLite2-Country-Blocks-IPv4.csv": strings.Join([]string{
89 + "network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,is_anycast",
90 + "1.0.0.0/24,6252001,,,,0,0",
91 + }, "\n"),
92 + "GeoLite2-Country-CSV_20260501/GeoLite2-Country-Blocks-IPv6.csv": strings.Join([]string{
93 + "network,geoname_id,registered_country_geoname_id,represented_country_geoname_id,is_anonymous_proxy,is_satellite_provider,is_anycast",
94 + "2001:db8::/48,2921044,,,,0,0",
95 + }, "\n"),
96 + "GeoLite2-Country-CSV_20260501/GeoLite2-Country-Locations-en.csv": strings.Join([]string{
97 + "geoname_id,locale_code,continent_code,continent_name,country_iso_code,country_name,is_in_european_union",
98 + "6252001,en,NA,North America,US,United States,0",
99 + "2921044,en,EU,Europe,DE,Germany,1",
100 + }, "\n"),
101 + })
102 +
103 + ranges, err := parseMaxMindCountryCSVZip(payload)
104 + require.NoError(t, err)
105 + require.Len(t, ranges, 2)
106 + require.Equal(t, "US", ranges[0].country)
107 + require.Equal(t, "1.0.0.0", ranges[0].start.String())
108 + require.Equal(t, "DE", ranges[1].country)
109 + require.Equal(t, "2001:db8::", ranges[1].start.String())
110 +}
111 +
112 +func TestParseIP2LocationCountryZip(t *testing.T) {
113 + payload := buildZip(t, map[string]string{
114 + "IP2LOCATION-LITE-DB1.CSV": "\"16777216\",\"16777471\",\"AU\",\"Australia\"\n",
115 + })
116 + ranges, err := parseIP2LocationCountryZip(payload)
117 + require.NoError(t, err)
118 + require.Len(t, ranges, 1)
119 + require.Equal(t, "1.0.0.0", ranges[0].start.String())
120 + require.Equal(t, "1.0.0.255", ranges[0].end.String())
121 + require.Equal(t, "AU", ranges[0].country)
122 +}
123 +
124 +func TestParseIPDenyCountryTarGZ(t *testing.T) {
125 + payload := buildTarGZ(t, map[string]string{
126 + "./us.zone": "1.0.0.0/24\n",
127 + "./GR.ZONE": "1.0.1.0/24\n",
128 + })
129 + ranges, err := parseIPDenyCountryTarGZ(payload)
130 + require.NoError(t, err)
131 + require.Len(t, ranges, 2)
132 + byCountry := make(map[string]geoRange, len(ranges))
133 + for _, rec := range ranges {
134 + byCountry[rec.country] = rec
135 + }
136 + require.Equal(t, "1.0.0.255", byCountry["US"].end.String())
137 + require.Equal(t, "1.0.1.255", byCountry["GR"].end.String())
138 +}
139 +
140 +func TestParseIPIPCountryZip(t *testing.T) {
141 + payload := buildZip(t, map[string]string{
142 + "country.txt": "1.0.0.0/24\tANYCAST\n1.0.1.0/24\tCN\n",
143 + })
144 + ranges, err := parseIPIPCountryZip(payload)
145 + require.NoError(t, err)
146 + require.Len(t, ranges, 1)
147 + require.Equal(t, "CN", ranges[0].country)
148 + require.Equal(t, "1.0.1.0", ranges[0].start.String())
149 + require.Equal(t, "1.0.1.255", ranges[0].end.String())
150 +}
151 +
152 +func TestEstimatedRangeCapacity(t *testing.T) {
153 + require.Equal(t, 0, estimatedRangeCapacity(0, 64, 1<<20))
154 + require.Equal(t, 0, estimatedRangeCapacity(128, 0, 1<<20))
155 + require.Equal(t, 2, estimatedRangeCapacity(128, 64, 1<<20))
156 + require.Equal(t, 4, estimatedRangeCapacity(1024, 64, 4))
157 +}
158 +
159 func TestParseIPDecimalIPv4(t *testing.T) {
160 addr, err := parseIP("16777216") // 1.0.0.0
161 require.NoError(t, err)
@@ -73,3 +166,49 @@ func TestParseRangeRejectsFamilyMix(t *testing.T) {
166 _, _, err := parseRangeEndpoints("1.0.0.0", "2001:db8::1")
167 require.Error(t, err)
168 }
169 +
170 +func buildZip(t *testing.T, files map[string]string) []byte {
171 + t.Helper()
172 +
173 + var buf bytes.Buffer
174 + zw := zip.NewWriter(&buf)
175 + for name, content := range files {
176 + w, err := zw.Create(name)
177 + require.NoError(t, err)
178 + _, err = w.Write([]byte(content))
179 + require.NoError(t, err)
180 + }
181 + require.NoError(t, zw.Close())
182 + return buf.Bytes()
183 +}
184 +
185 +func buildTarGZ(t *testing.T, files map[string]string) []byte {
186 + t.Helper()
187 +
188 + rawTar := buildTar(t, files)
189 + var buf bytes.Buffer
190 + gz := gzip.NewWriter(&buf)
191 + _, err := gz.Write(rawTar)
192 + require.NoError(t, err)
193 + require.NoError(t, gz.Close())
194 + return buf.Bytes()
195 +}
196 +
197 +func buildTar(t *testing.T, files map[string]string) []byte {
198 + t.Helper()
199 +
200 + var buf bytes.Buffer
201 + tw := tar.NewWriter(&buf)
202 + for name, content := range files {
203 + err := tw.WriteHeader(&tar.Header{
204 + Name: name,
205 + Mode: 0o644,
206 + Size: int64(len(content)),
207 + })
208 + require.NoError(t, err)
209 + _, err = tw.Write([]byte(content))
210 + require.NoError(t, err)
211 + }
212 + require.NoError(t, tw.Close())
213 + return buf.Bytes()
214 +}