| 1 | package config |
| 2 | |
| 3 | import ( |
| 4 | "fmt" |
| 5 | "strings" |
| 6 | "time" |
| 7 | |
| 8 | "github.com/libp2p/go-libp2p-kad-dht/amino" |
| 9 | ) |
| 10 | |
| 11 | const ( |
| 12 | DefaultProvideEnabled = true |
| 13 | DefaultProvideStrategy = "all" |
| 14 | |
| 15 | // DefaultProvideBloomFPRate is the target false positive rate for the |
| 16 | // bloom filter used by +unique and +entities reprovide cycles and |
| 17 | // fast-provide-dag walks. Expressed as 1/N (one false positive per N |
| 18 | // lookups). At ~1 in 4.75M (~0.00002%) each CID costs ~4 bytes before |
| 19 | // ipfs/bbloom's power-of-two rounding. |
| 20 | // |
| 21 | // Kubo owns this default independently of boxo/dag/walker; the two |
| 22 | // values may diverge over time without coordination. |
| 23 | DefaultProvideBloomFPRate = 4_750_000 |
| 24 | |
| 25 | // MinProvideBloomFPRate is the smallest accepted Provide.BloomFPRate. |
| 26 | // Below 1 in 1M the bloom filter becomes lossy enough to drop a |
| 27 | // meaningful fraction of CIDs from each reprovide cycle (e.g. at |
| 28 | // rate=10_000 a 100M-CID repo skips ~10K CIDs per cycle). |
| 29 | MinProvideBloomFPRate = 1_000_000 |
| 30 | |
| 31 | // DHT provider defaults |
| 32 | DefaultProvideDHTInterval = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326 |
| 33 | DefaultProvideDHTMaxWorkers = 16 // Unified default for both sweep and legacy providers |
| 34 | DefaultProvideDHTSweepEnabled = true |
| 35 | DefaultProvideDHTResumeEnabled = true |
| 36 | DefaultProvideDHTDedicatedPeriodicWorkers = 2 |
| 37 | DefaultProvideDHTDedicatedBurstWorkers = 1 |
| 38 | DefaultProvideDHTMaxProvideConnsPerWorker = 20 |
| 39 | DefaultProvideDHTKeystoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes) |
| 40 | DefaultProvideDHTOfflineDelay = 2 * time.Hour |
| 41 | DefaultProvideDHTSendProviderRecordTimeout = 10 * time.Second |
| 42 | |
| 43 | // DefaultFastProvideTimeout is the maximum time allowed for fast-provide operations. |
| 44 | // Prevents hanging on network issues when providing root CID. |
| 45 | // 10 seconds is sufficient for DHT operations with sweep provider or accelerated client. |
| 46 | DefaultFastProvideTimeout = 10 * time.Second |
| 47 | ) |
| 48 | |
| 49 | type ProvideStrategy int |
| 50 | |
| 51 | const ( |
| 52 | ProvideStrategyAll ProvideStrategy = 1 << iota |
| 53 | ProvideStrategyPinned |
| 54 | ProvideStrategyRoots |
| 55 | ProvideStrategyMFS |
| 56 | ProvideStrategyUnique // bloom filter cross-DAG deduplication |
| 57 | ProvideStrategyEntities // entity-aware traversal (implies Unique) |
| 58 | ) |
| 59 | |
| 60 | // Provide configures both immediate CID announcements (provide operations) for new content |
| 61 | // and periodic re-announcements of existing CIDs (reprovide operations). |
| 62 | // This section combines the functionality previously split between Provider and Reprovider. |
| 63 | type Provide struct { |
| 64 | // Enabled controls whether both provide and reprovide systems are enabled. |
| 65 | // When disabled, the node will not announce any content to the routing system. |
| 66 | Enabled Flag `json:",omitempty"` |
| 67 | |
| 68 | // Strategy determines which CIDs are announced to the routing system. |
| 69 | // Default: DefaultProvideStrategy |
| 70 | Strategy *OptionalString `json:",omitempty"` |
| 71 | |
| 72 | // BloomFPRate sets the target false positive rate of the bloom filter |
| 73 | // used by Provide.Strategy modifiers +unique and +entities (and the |
| 74 | // matching fast-provide-dag walk). Expressed as 1/N (one false |
| 75 | // positive per N lookups), so higher N means lower FP rate but more |
| 76 | // memory per CID. Only takes effect when Provide.Strategy includes |
| 77 | // +unique or +entities. |
| 78 | // |
| 79 | // Default: DefaultProvideBloomFPRate |
| 80 | BloomFPRate *OptionalInteger `json:",omitempty"` |
| 81 | |
| 82 | // DHT configures DHT-specific provide and reprovide settings. |
| 83 | DHT ProvideDHT |
| 84 | } |
| 85 | |
| 86 | // ProvideDHT configures DHT provider settings for both immediate announcements |
| 87 | // and periodic reprovides. |
| 88 | type ProvideDHT struct { |
| 89 | // Interval sets the time between rounds of reproviding local content |
| 90 | // to the routing system. Set to "0" to disable content reproviding. |
| 91 | // Default: DefaultProvideDHTInterval |
| 92 | Interval *OptionalDuration `json:",omitempty"` |
| 93 | |
| 94 | // MaxWorkers sets the maximum number of concurrent workers for provide operations. |
| 95 | // When SweepEnabled is false: controls NEW CID announcements only. |
| 96 | // When SweepEnabled is true: controls total worker pool for all operations. |
| 97 | // Default: DefaultProvideDHTMaxWorkers |
| 98 | MaxWorkers *OptionalInteger `json:",omitempty"` |
| 99 | |
| 100 | // SweepEnabled activates the sweeping reprovider system which spreads |
| 101 | // reprovide operations over time. |
| 102 | // Default: DefaultProvideDHTSweepEnabled |
| 103 | SweepEnabled Flag `json:",omitempty"` |
| 104 | |
| 105 | // DedicatedPeriodicWorkers sets workers dedicated to periodic reprovides (sweep mode only). |
| 106 | // Default: DefaultProvideDHTDedicatedPeriodicWorkers |
| 107 | DedicatedPeriodicWorkers *OptionalInteger `json:",omitempty"` |
| 108 | |
| 109 | // DedicatedBurstWorkers sets workers dedicated to burst provides (sweep mode only). |
| 110 | // Default: DefaultProvideDHTDedicatedBurstWorkers |
| 111 | DedicatedBurstWorkers *OptionalInteger `json:",omitempty"` |
| 112 | |
| 113 | // MaxProvideConnsPerWorker sets concurrent connections per worker for sending provider records (sweep mode only). |
| 114 | // Default: DefaultProvideDHTMaxProvideConnsPerWorker |
| 115 | MaxProvideConnsPerWorker *OptionalInteger `json:",omitempty"` |
| 116 | |
| 117 | // KeystoreBatchSize sets the batch size for keystore operations during reprovide refresh (sweep mode only). |
| 118 | // Default: DefaultProvideDHTKeystoreBatchSize |
| 119 | KeystoreBatchSize *OptionalInteger `json:",omitempty"` |
| 120 | |
| 121 | // OfflineDelay sets the delay after which the provider switches from Disconnected to Offline state (sweep mode only). |
| 122 | // Default: DefaultProvideDHTOfflineDelay |
| 123 | OfflineDelay *OptionalDuration `json:",omitempty"` |
| 124 | |
| 125 | // SendProviderRecordTimeout sets the per-peer timeout applied to a single |
| 126 | // ADD_PROVIDER RPC. A peer that accepts the libp2p stream but never reads |
| 127 | // the request must not pin a provide worker goroutine indefinitely; this |
| 128 | // timeout bounds the wait (sweep mode only). |
| 129 | // Default: DefaultProvideDHTSendProviderRecordTimeout |
| 130 | SendProviderRecordTimeout *OptionalDuration `json:",omitempty"` |
| 131 | |
| 132 | // ResumeEnabled controls whether the provider resumes from its previous state on restart. |
| 133 | // When enabled, the provider persists its reprovide cycle state and provide queue to the datastore, |
| 134 | // and restores them on restart. When disabled, the provider starts fresh on each restart. |
| 135 | // Default: true |
| 136 | ResumeEnabled Flag `json:",omitempty"` |
| 137 | } |
| 138 | |
| 139 | func ParseProvideStrategy(s string) (ProvideStrategy, error) { |
| 140 | var strategy ProvideStrategy |
| 141 | for part := range strings.SplitSeq(s, "+") { |
| 142 | switch part { |
| 143 | case "all", "flat": |
| 144 | strategy |= ProvideStrategyAll |
| 145 | case "": |
| 146 | // empty string (default config) maps to "all", |
| 147 | // but empty tokens from splitting (e.g. "pinned+") are invalid |
| 148 | if s == "" { |
| 149 | strategy |= ProvideStrategyAll |
| 150 | } else { |
| 151 | return 0, fmt.Errorf("invalid provide strategy: empty token in %q", s) |
| 152 | } |
| 153 | case "pinned": |
| 154 | strategy |= ProvideStrategyPinned |
| 155 | case "roots": |
| 156 | strategy |= ProvideStrategyRoots |
| 157 | case "mfs": |
| 158 | strategy |= ProvideStrategyMFS |
| 159 | case "unique": |
| 160 | strategy |= ProvideStrategyUnique |
| 161 | case "entities": |
| 162 | strategy |= ProvideStrategyEntities | ProvideStrategyUnique |
| 163 | default: |
| 164 | return 0, fmt.Errorf("unknown provide strategy token: %q in %q", part, s) |
| 165 | } |
| 166 | } |
| 167 | // "all" provides every block and cannot be combined with selective strategies |
| 168 | if strategy&ProvideStrategyAll != 0 && strategy != ProvideStrategyAll { |
| 169 | return 0, fmt.Errorf("\"all\" strategy cannot be combined with other strategies in %q", s) |
| 170 | } |
| 171 | // +unique/+entities require a base strategy that walks DAGs (pinned and/or mfs) |
| 172 | wantsDedup := strategy&(ProvideStrategyUnique|ProvideStrategyEntities) != 0 |
| 173 | if wantsDedup { |
| 174 | walksDAGs := strategy&(ProvideStrategyPinned|ProvideStrategyMFS) != 0 |
| 175 | if !walksDAGs { |
| 176 | return 0, fmt.Errorf("+unique/+entities must combine with pinned and/or mfs in %q", s) |
| 177 | } |
| 178 | if strategy&ProvideStrategyRoots != 0 { |
| 179 | return 0, fmt.Errorf("+unique/+entities is incompatible with roots in %q", s) |
| 180 | } |
| 181 | } |
| 182 | return strategy, nil |
| 183 | } |
| 184 | |
| 185 | // MustParseProvideStrategy is like ParseProvideStrategy but panics on error. |
| 186 | // Use with strategy strings that have already been validated at startup. |
| 187 | func MustParseProvideStrategy(s string) ProvideStrategy { |
| 188 | strategy, err := ParseProvideStrategy(s) |
| 189 | if err != nil { |
| 190 | panic(err) |
| 191 | } |
| 192 | return strategy |
| 193 | } |
| 194 | |
| 195 | // ValidateProvideConfig validates the Provide configuration according to DHT requirements. |
| 196 | func ValidateProvideConfig(cfg *Provide) error { |
| 197 | // Validate Provide.Strategy |
| 198 | strategy := cfg.Strategy.WithDefault(DefaultProvideStrategy) |
| 199 | if _, err := ParseProvideStrategy(strategy); err != nil { |
| 200 | return fmt.Errorf("Provide.Strategy: %w", err) |
| 201 | } |
| 202 | |
| 203 | // Validate Provide.BloomFPRate |
| 204 | if !cfg.BloomFPRate.IsDefault() { |
| 205 | rate := cfg.BloomFPRate.WithDefault(DefaultProvideBloomFPRate) |
| 206 | if rate < MinProvideBloomFPRate { |
| 207 | return fmt.Errorf("Provide.BloomFPRate must be >= %d (1 in 1M), got %d", MinProvideBloomFPRate, rate) |
| 208 | } |
| 209 | } |
| 210 | |
| 211 | // Validate Provide.DHT.Interval |
| 212 | if !cfg.DHT.Interval.IsDefault() { |
| 213 | interval := cfg.DHT.Interval.WithDefault(DefaultProvideDHTInterval) |
| 214 | if interval > amino.DefaultProvideValidity { |
| 215 | return fmt.Errorf("Provide.DHT.Interval (%v) must be less than or equal to DHT provider record validity (%v)", interval, amino.DefaultProvideValidity) |
| 216 | } |
| 217 | if interval < 0 { |
| 218 | return fmt.Errorf("Provide.DHT.Interval must be non-negative, got %v", interval) |
| 219 | } |
| 220 | // Provide.DHT.Interval=0 used to disable the entire provide system as a |
| 221 | // side effect. It now disables only the periodic reprovide schedule: |
| 222 | // new CIDs still announce via fast-provide-root and 'ipfs provide once'. |
| 223 | // Operators upgrading from earlier kubo versions must opt in to one of |
| 224 | // the two semantics by setting Provide.Enabled explicitly: |
| 225 | // - Provide.Enabled=false fully disables providing (the old behaviour). |
| 226 | // - Provide.Enabled=true keeps ad-hoc providing while disabling the |
| 227 | // periodic reprovide schedule. |
| 228 | if interval == 0 && cfg.Enabled == Default { |
| 229 | return fmt.Errorf("Provide.DHT.Interval=0 no longer disables the provide system on its own; set Provide.Enabled explicitly: " + |
| 230 | "Provide.Enabled=false to fully disable providing, or Provide.Enabled=true to keep ad-hoc 'ipfs provide once' " + |
| 231 | "and fast-provide-root working while skipping the periodic reprovide schedule") |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // Validate MaxWorkers |
| 236 | if !cfg.DHT.MaxWorkers.IsDefault() { |
| 237 | maxWorkers := cfg.DHT.MaxWorkers.WithDefault(DefaultProvideDHTMaxWorkers) |
| 238 | if maxWorkers <= 0 { |
| 239 | return fmt.Errorf("Provide.DHT.MaxWorkers must be positive, got %d", maxWorkers) |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | // Validate DedicatedPeriodicWorkers |
| 244 | if !cfg.DHT.DedicatedPeriodicWorkers.IsDefault() { |
| 245 | workers := cfg.DHT.DedicatedPeriodicWorkers.WithDefault(DefaultProvideDHTDedicatedPeriodicWorkers) |
| 246 | if workers < 0 { |
| 247 | return fmt.Errorf("Provide.DHT.DedicatedPeriodicWorkers must be non-negative, got %d", workers) |
| 248 | } |
| 249 | } |
| 250 | |
| 251 | // Validate DedicatedBurstWorkers |
| 252 | if !cfg.DHT.DedicatedBurstWorkers.IsDefault() { |
| 253 | workers := cfg.DHT.DedicatedBurstWorkers.WithDefault(DefaultProvideDHTDedicatedBurstWorkers) |
| 254 | if workers < 0 { |
| 255 | return fmt.Errorf("Provide.DHT.DedicatedBurstWorkers must be non-negative, got %d", workers) |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | // Validate MaxProvideConnsPerWorker |
| 260 | if !cfg.DHT.MaxProvideConnsPerWorker.IsDefault() { |
| 261 | conns := cfg.DHT.MaxProvideConnsPerWorker.WithDefault(DefaultProvideDHTMaxProvideConnsPerWorker) |
| 262 | if conns <= 0 { |
| 263 | return fmt.Errorf("Provide.DHT.MaxProvideConnsPerWorker must be positive, got %d", conns) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | // Validate KeystoreBatchSize |
| 268 | if !cfg.DHT.KeystoreBatchSize.IsDefault() { |
| 269 | batchSize := cfg.DHT.KeystoreBatchSize.WithDefault(DefaultProvideDHTKeystoreBatchSize) |
| 270 | if batchSize <= 0 { |
| 271 | return fmt.Errorf("Provide.DHT.KeystoreBatchSize must be positive, got %d", batchSize) |
| 272 | } |
| 273 | } |
| 274 | |
| 275 | // Validate OfflineDelay |
| 276 | if !cfg.DHT.OfflineDelay.IsDefault() { |
| 277 | delay := cfg.DHT.OfflineDelay.WithDefault(DefaultProvideDHTOfflineDelay) |
| 278 | if delay < 0 { |
| 279 | return fmt.Errorf("Provide.DHT.OfflineDelay must be non-negative, got %v", delay) |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // Validate SendProviderRecordTimeout |
| 284 | if !cfg.DHT.SendProviderRecordTimeout.IsDefault() { |
| 285 | timeout := cfg.DHT.SendProviderRecordTimeout.WithDefault(DefaultProvideDHTSendProviderRecordTimeout) |
| 286 | if timeout <= 0 { |
| 287 | return fmt.Errorf("Provide.DHT.SendProviderRecordTimeout must be positive, got %v", timeout) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | return nil |
| 292 | } |
| 293 | |
| 294 | // ShouldProvideForStrategy determines if content should be provided based on the provide strategy |
| 295 | // and content characteristics (pinned status, root status, MFS status). |
| 296 | func ShouldProvideForStrategy(strategy ProvideStrategy, isPinned bool, isPinnedRoot bool, isMFS bool) bool { |
| 297 | if strategy&ProvideStrategyAll != 0 { |
| 298 | // 'all' strategy: always provide |
| 299 | return true |
| 300 | } |
| 301 | |
| 302 | // For combined strategies, check each component |
| 303 | if strategy&ProvideStrategyPinned != 0 && isPinned { |
| 304 | return true |
| 305 | } |
| 306 | if strategy&ProvideStrategyRoots != 0 && isPinnedRoot { |
| 307 | return true |
| 308 | } |
| 309 | if strategy&ProvideStrategyMFS != 0 && isMFS { |
| 310 | return true |
| 311 | } |
| 312 | |
| 313 | return false |
| 314 | } |