| 1 | // SPDX-License-Identifier: GPL-3.0-or-later |
| 2 | |
| 3 | package main |
| 4 | |
| 5 | import ( |
| 6 | "encoding/json" |
| 7 | "fmt" |
| 8 | "net" |
| 9 | "net/netip" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/maxmind/mmdbwriter" |
| 15 | "github.com/maxmind/mmdbwriter/mmdbtype" |
| 16 | "go4.org/netipx" |
| 17 | ) |
| 18 | |
| 19 | type generationMetadata struct { |
| 20 | GeneratedAt string `json:"generated_at"` |
| 21 | GeneratedBy string `json:"generated_by"` |
| 22 | Sources []generationDatasetRef `json:"sources"` |
| 23 | Counts struct { |
| 24 | AsnRanges int `json:"asn_ranges"` |
| 25 | GeoRanges int `json:"geo_ranges"` |
| 26 | } `json:"counts"` |
| 27 | Policy struct { |
| 28 | LocalhostCIDRs []string `json:"localhost_cidrs"` |
| 29 | PrivateCIDRs []string `json:"private_cidrs"` |
| 30 | InterestingCIDRs []string `json:"interesting_cidrs"` |
| 31 | } `json:"policy"` |
| 32 | Output struct { |
| 33 | AsnFile string `json:"asn_file,omitempty"` |
| 34 | GeoFile string `json:"geo_file,omitempty"` |
| 35 | MetadataFile string `json:"metadata_file"` |
| 36 | } `json:"output"` |
| 37 | } |
| 38 | |
| 39 | type generationDatasetRef struct { |
| 40 | Name string `json:"name,omitempty"` |
| 41 | Family string `json:"family"` |
| 42 | Provider string `json:"provider,omitempty"` |
| 43 | Artifact string `json:"artifact,omitempty"` |
| 44 | Source string `json:"source,omitempty"` |
| 45 | Format string `json:"format,omitempty"` |
| 46 | URL string `json:"url,omitempty"` |
| 47 | Path string `json:"path,omitempty"` |
| 48 | DownloadPage string `json:"download_page,omitempty"` |
| 49 | ResolvedURL string `json:"resolved_url,omitempty"` |
| 50 | } |
| 51 | |
| 52 | func writeOutputs( |
| 53 | cfg config, |
| 54 | asnRanges []asnRange, |
| 55 | geoRanges []geoRange, |
| 56 | sources []generationDatasetRef, |
| 57 | ) error { |
| 58 | if err := os.MkdirAll(cfg.output.directory, 0o755); err != nil { |
| 59 | return fmt.Errorf("failed to create output dir %s: %w", cfg.output.directory, err) |
| 60 | } |
| 61 | |
| 62 | classes, err := classifyRanges(cfg.policy) |
| 63 | if err != nil { |
| 64 | return err |
| 65 | } |
| 66 | |
| 67 | asnPath := filepath.Join(cfg.output.directory, cfg.output.asnFile) |
| 68 | geoPath := filepath.Join(cfg.output.directory, cfg.output.geoFile) |
| 69 | metadataPath := filepath.Join(cfg.output.directory, cfg.output.metadataFile) |
| 70 | |
| 71 | stageDir, err := os.MkdirTemp(cfg.output.directory, ".tmp-topology-ip-intel-stage-*") |
| 72 | if err != nil { |
| 73 | return fmt.Errorf("failed to create staging dir in %s: %w", cfg.output.directory, err) |
| 74 | } |
| 75 | defer os.RemoveAll(stageDir) |
| 76 | |
| 77 | stagedASNPath := filepath.Join(stageDir, cfg.output.asnFile) |
| 78 | stagedGeoPath := filepath.Join(stageDir, cfg.output.geoFile) |
| 79 | stagedMetadataPath := filepath.Join(stageDir, cfg.output.metadataFile) |
| 80 | |
| 81 | asnEnabled := cfg.hasFamily(sourceFamilyASN) |
| 82 | geoEnabled := cfg.hasFamily(sourceFamilyGeo) |
| 83 | |
| 84 | if asnEnabled { |
| 85 | if err := writeAsnDatabase(stagedASNPath, asnRanges, classes); err != nil { |
| 86 | return fmt.Errorf("asn database generation failed: %w", err) |
| 87 | } |
| 88 | } |
| 89 | if geoEnabled { |
| 90 | if err := writeGeoDatabase(stagedGeoPath, geoRanges, classes); err != nil { |
| 91 | return fmt.Errorf("geo database generation failed: %w", err) |
| 92 | } |
| 93 | } |
| 94 | |
| 95 | md := generationMetadata{} |
| 96 | md.GeneratedAt = time.Now().UTC().Format(time.RFC3339) |
| 97 | md.GeneratedBy = "topology-ip-intel-downloader" |
| 98 | md.Sources = append([]generationDatasetRef(nil), sources...) |
| 99 | md.Counts.AsnRanges = len(asnRanges) |
| 100 | md.Counts.GeoRanges = len(geoRanges) |
| 101 | md.Policy.LocalhostCIDRs = append([]string{}, cfg.policy.localhostCIDRs...) |
| 102 | md.Policy.PrivateCIDRs = append([]string{}, cfg.policy.privateCIDRs...) |
| 103 | md.Policy.InterestingCIDRs = append([]string{}, cfg.policy.interestingCIDRs...) |
| 104 | if asnEnabled { |
| 105 | md.Output.AsnFile = cfg.output.asnFile |
| 106 | } |
| 107 | if geoEnabled { |
| 108 | md.Output.GeoFile = cfg.output.geoFile |
| 109 | } |
| 110 | md.Output.MetadataFile = cfg.output.metadataFile |
| 111 | |
| 112 | blob, err := json.MarshalIndent(md, "", " ") |
| 113 | if err != nil { |
| 114 | return fmt.Errorf("failed to encode metadata json: %w", err) |
| 115 | } |
| 116 | if err := os.WriteFile(stagedMetadataPath, blob, 0o644); err != nil { |
| 117 | return fmt.Errorf("failed to write staged metadata %s: %w", stagedMetadataPath, err) |
| 118 | } |
| 119 | |
| 120 | if asnEnabled { |
| 121 | if err := renameFileAtomic(stagedASNPath, asnPath); err != nil { |
| 122 | return err |
| 123 | } |
| 124 | } else if err := removeIfExists(asnPath); err != nil { |
| 125 | return err |
| 126 | } |
| 127 | |
| 128 | if geoEnabled { |
| 129 | if err := renameFileAtomic(stagedGeoPath, geoPath); err != nil { |
| 130 | return err |
| 131 | } |
| 132 | } else if err := removeIfExists(geoPath); err != nil { |
| 133 | return err |
| 134 | } |
| 135 | |
| 136 | if err := renameFileAtomic(stagedMetadataPath, metadataPath); err != nil { |
| 137 | return err |
| 138 | } |
| 139 | return nil |
| 140 | } |
| 141 | |
| 142 | func writeAsnDatabase(path string, ranges []asnRange, classes []classification) error { |
| 143 | writer, err := mmdbwriter.New(mmdbwriter.Options{ |
| 144 | DatabaseType: "Netdata-Topology-ASN", |
| 145 | Description: map[string]string{"en": "Netdata topology ASN mapping"}, |
| 146 | IPVersion: 6, |
| 147 | RecordSize: 28, |
| 148 | DisableIPv4Aliasing: true, |
| 149 | IncludeReservedNetworks: true, |
| 150 | }) |
| 151 | if err != nil { |
| 152 | return fmt.Errorf("failed to create ASN MMDB writer: %w", err) |
| 153 | } |
| 154 | |
| 155 | for idx, rec := range ranges { |
| 156 | record := mmdbtype.Map{} |
| 157 | if rec.asn != 0 { |
| 158 | record["autonomous_system_number"] = mmdbtype.Uint32(rec.asn) |
| 159 | } |
| 160 | if rec.org != "" { |
| 161 | record["autonomous_system_organization"] = mmdbtype.String(rec.org) |
| 162 | } |
| 163 | if err := insertRange(writer, rec.start, rec.end, record); err != nil { |
| 164 | return fmt.Errorf("asn range %d (%s-%s): %w", idx, rec.start, rec.end, err) |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | if err := applyClassifications(writer, classes); err != nil { |
| 169 | return err |
| 170 | } |
| 171 | return writeMMDBAtomic(path, writer) |
| 172 | } |
| 173 | |
| 174 | func writeGeoDatabase(path string, ranges []geoRange, classes []classification) error { |
| 175 | writer, err := mmdbwriter.New(mmdbwriter.Options{ |
| 176 | DatabaseType: "Netdata-Topology-GEO", |
| 177 | Description: map[string]string{"en": "Netdata topology geographic mapping"}, |
| 178 | IPVersion: 6, |
| 179 | RecordSize: 28, |
| 180 | DisableIPv4Aliasing: true, |
| 181 | IncludeReservedNetworks: true, |
| 182 | }) |
| 183 | if err != nil { |
| 184 | return fmt.Errorf("failed to create GEO MMDB writer: %w", err) |
| 185 | } |
| 186 | |
| 187 | for idx, rec := range ranges { |
| 188 | record := mmdbtype.Map{} |
| 189 | if rec.country != "" { |
| 190 | record["country"] = mmdbtype.Map{ |
| 191 | "iso_code": mmdbtype.String(rec.country), |
| 192 | } |
| 193 | } |
| 194 | if rec.city != "" { |
| 195 | record["city"] = mmdbtype.Map{ |
| 196 | "names": mmdbtype.Map{ |
| 197 | "en": mmdbtype.String(rec.city), |
| 198 | }, |
| 199 | } |
| 200 | } |
| 201 | if rec.state != "" { |
| 202 | record["region"] = mmdbtype.String(rec.state) |
| 203 | record["subdivisions"] = mmdbtype.Slice{ |
| 204 | mmdbtype.Map{ |
| 205 | "names": mmdbtype.Map{ |
| 206 | "en": mmdbtype.String(rec.state), |
| 207 | }, |
| 208 | }, |
| 209 | } |
| 210 | } |
| 211 | if rec.hasLocation { |
| 212 | record["location"] = mmdbtype.Map{ |
| 213 | "latitude": mmdbtype.Float64(rec.latitude), |
| 214 | "longitude": mmdbtype.Float64(rec.longitude), |
| 215 | } |
| 216 | } |
| 217 | if err := insertRange(writer, rec.start, rec.end, record); err != nil { |
| 218 | return fmt.Errorf("geo range %d (%s-%s): %w", idx, rec.start, rec.end, err) |
| 219 | } |
| 220 | } |
| 221 | |
| 222 | if err := applyClassifications(writer, classes); err != nil { |
| 223 | return err |
| 224 | } |
| 225 | return writeMMDBAtomic(path, writer) |
| 226 | } |
| 227 | |
| 228 | func applyClassifications(writer *mmdbwriter.Tree, classes []classification) error { |
| 229 | for _, classSet := range classes { |
| 230 | record := mmdbtype.Map{ |
| 231 | "netdata": mmdbtype.Map{ |
| 232 | "ip_class": mmdbtype.String(classSet.class), |
| 233 | "track_individual": mmdbtype.Bool(trackIndividual(classSet.class)), |
| 234 | }, |
| 235 | } |
| 236 | for _, prefix := range classSet.prefixes { |
| 237 | ipRange := netipx.RangeOfPrefix(prefix) |
| 238 | start := addrToNetIP(ipRange.From()) |
| 239 | end := addrToNetIP(ipRange.To()) |
| 240 | if err := writer.InsertRange(start, end, record); err != nil { |
| 241 | return fmt.Errorf("failed to apply class %s for %s: %w", classSet.class, prefix, err) |
| 242 | } |
| 243 | } |
| 244 | } |
| 245 | return nil |
| 246 | } |
| 247 | |
| 248 | func insertRange(writer *mmdbwriter.Tree, start, end netip.Addr, value mmdbtype.DataType) error { |
| 249 | if value == nil { |
| 250 | value = mmdbtype.Map{} |
| 251 | } |
| 252 | if err := writer.InsertRange(addrToNetIP(start), addrToNetIP(end), value); err != nil { |
| 253 | return err |
| 254 | } |
| 255 | return nil |
| 256 | } |
| 257 | |
| 258 | func addrToNetIP(addr netip.Addr) net.IP { |
| 259 | if addr.Is4() { |
| 260 | a := addr.As4() |
| 261 | return net.IPv4(a[0], a[1], a[2], a[3]) |
| 262 | } |
| 263 | a := addr.As16() |
| 264 | return net.IP(a[:]) |
| 265 | } |
| 266 | |
| 267 | func writeMMDBAtomic(path string, writer *mmdbwriter.Tree) error { |
| 268 | dir := filepath.Dir(path) |
| 269 | tmp, err := os.CreateTemp(dir, ".tmp-topology-ip-intel-*.mmdb") |
| 270 | if err != nil { |
| 271 | return fmt.Errorf("failed to create temporary file in %s: %w", dir, err) |
| 272 | } |
| 273 | tmpPath := tmp.Name() |
| 274 | |
| 275 | cleanup := func() { |
| 276 | _ = tmp.Close() |
| 277 | _ = os.Remove(tmpPath) |
| 278 | } |
| 279 | |
| 280 | if _, err := writer.WriteTo(tmp); err != nil { |
| 281 | cleanup() |
| 282 | return fmt.Errorf("failed to write temporary MMDB %s: %w", tmpPath, err) |
| 283 | } |
| 284 | if err := tmp.Chmod(0o644); err != nil { |
| 285 | cleanup() |
| 286 | return fmt.Errorf("failed to chmod temporary MMDB %s: %w", tmpPath, err) |
| 287 | } |
| 288 | if err := tmp.Close(); err != nil { |
| 289 | cleanup() |
| 290 | return fmt.Errorf("failed to close temporary MMDB %s: %w", tmpPath, err) |
| 291 | } |
| 292 | if err := os.Rename(tmpPath, path); err != nil { |
| 293 | cleanup() |
| 294 | return fmt.Errorf("failed to atomically replace %s: %w", path, err) |
| 295 | } |
| 296 | return nil |
| 297 | } |
| 298 | |
| 299 | func removeIfExists(path string) error { |
| 300 | if err := os.Remove(path); err != nil && !os.IsNotExist(err) { |
| 301 | return fmt.Errorf("failed to remove stale output %s: %w", path, err) |
| 302 | } |
| 303 | return nil |
| 304 | } |
| 305 | |
| 306 | func renameFileAtomic(from, to string) error { |
| 307 | if err := os.Rename(from, to); err != nil { |
| 308 | return fmt.Errorf("failed to atomically replace %s: %w", to, err) |
| 309 | } |
| 310 | return nil |
| 311 | } |