refactor(config): migration 17-to-18 to unify Provider/Reprovider into Provide.DHT (#10951)
* refactor: consolidate Provider/Reprovider into unified Provide config - merge Provider and Reprovider configs into single Provide section - add fs-repo-17-to-18 migration for config consolidation - improve migration ergonomics with common package utilities - convert deprecated "flat" strategy to "all" during migration - improve Provide docs * docs: add total_provide_count metric guidance - document how to monitor provide success rates via prometheus metrics - add performance comparison section to changelog - explain how to evaluate sweep vs legacy provider effectiveness * fix: add OpenTelemetry meter provider for metrics - set up meter provider with Prometheus exporter in daemon - enables metrics from external libs like go-libp2p-kad-dht - fixes missing total_provide_count_total when SweepEnabled=true - update docs to reflect actual metric names --------- Co-authored-by: gammazero <11790789+gammazero@users.noreply.github.com> Co-authored-by: guillaumemichel <guillaume@michel.id> Co-authored-by: Daniel Norman <1992255+2color@users.noreply.github.com> Co-authored-by: Hector Sanjuan <code@hector.link>
Marcin Rataj committed
Sep 18, 2025 at 22:17 UTC
71e883440ee499d3a95c53021a2d6601d52b5168
49 files changed
+2676
-969
client/rpc/api_test.go
+1
-1
@@ -47,7 +47,7 @@ func (np NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdent
47
c.Experimental.FilestoreEnabled = true
48
// only provide things we pin. Allows to test
49
// provide operations.
50
- c.Reprovider.Strategy = config.NewOptionalString("roots")
50
+ c.Provide.Strategy = config.NewOptionalString("roots")
51
n.WriteConfig(c)
52
n.StartDaemon("--enable-pubsub-experiment", "--offline="+strconv.FormatBool(!online))
53
cmd/ipfs/kubo/daemon.go
+44
-18
@@ -43,6 +43,9 @@ import (
43
manet "github.com/multiformats/go-multiaddr/net"
44
prometheus "github.com/prometheus/client_golang/prometheus"
45
promauto "github.com/prometheus/client_golang/prometheus/promauto"
46
+ "go.opentelemetry.io/otel"
47
+ promexporter "go.opentelemetry.io/otel/exporters/prometheus"
48
+ sdkmetric "go.opentelemetry.io/otel/sdk/metric"
49
)
50
51
const (
@@ -211,6 +214,21 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
214
log.Errorf("Injecting prometheus handler for metrics failed with message: %s\n", err.Error())
215
}
216
217
+ // Set up OpenTelemetry meter provider to enable metrics from external libraries
218
+ // like go-libp2p-kad-dht. Without this, metrics registered via otel.Meter()
219
+ // (such as total_provide_count from sweep provider) won't be exposed at the
220
+ // /debug/metrics/prometheus endpoint.
221
+ if exporter, err := promexporter.New(
222
+ promexporter.WithRegisterer(prometheus.DefaultRegisterer),
223
+ ); err != nil {
224
+ log.Errorf("Creating prometheus exporter for OpenTelemetry failed: %s (some metrics will be missing from /debug/metrics/prometheus)\n", err.Error())
225
+ } else {
226
+ meterProvider := sdkmetric.NewMeterProvider(
227
+ sdkmetric.WithReader(exporter),
228
+ )
229
+ otel.SetMeterProvider(meterProvider)
230
+ }
231
+
232
// let the user know we're going.
233
fmt.Printf("Initializing daemon...\n")
234
@@ -486,25 +504,33 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
504
// This should never happen, but better safe than sorry
505
log.Fatal("Private network does not work with Routing.Type=auto. Update your config to Routing.Type=dht (or none, and do manual peering)")
506
}
489
- if cfg.Provider.Strategy.WithDefault("") != "" && cfg.Reprovider.Strategy.IsDefault() {
490
- log.Fatal("Invalid config. Remove unused Provider.Strategy and set Reprovider.Strategy instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy")
507
+ // Check for deprecated Provider/Reprovider configuration after migration
508
+ // This should never happen for regular users, but is useful error for people who have Docker orchestration
509
+ // that blindly sets config keys (overriding automatic Kubo migration).
510
+ //nolint:staticcheck // intentionally checking deprecated fields
511
+ if cfg.Provider.Enabled != config.Default || !cfg.Provider.Strategy.IsDefault() || !cfg.Provider.WorkerCount.IsDefault() {
512
+ log.Fatal("Deprecated configuration detected. Manually migrate 'Provider' fields to 'Provide' and remove 'Provider' from your config. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide")
513
}
492
- // Check for deprecated "flat" strategy
493
- if cfg.Reprovider.Strategy.WithDefault("") == "flat" {
494
- log.Error("Reprovider.Strategy='flat' is deprecated and will be removed in the next release. Please update your config to use 'all' instead.")
514
+ //nolint:staticcheck // intentionally checking deprecated fields
515
+ if !cfg.Reprovider.Interval.IsDefault() || !cfg.Reprovider.Strategy.IsDefault() {
516
+ log.Fatal("Deprecated configuration detected. Manually migrate 'Reprovider' fields to 'Provide': Reprovider.Strategy -> Provide.Strategy, Reprovider.Interval -> Provide.Interval. Remove 'Reprovider' from your config. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide")
517
+ }
518
+ // Check for deprecated "flat" strategy (should have been migrated to "all")
519
+ if cfg.Provide.Strategy.WithDefault("") == "flat" {
520
+ log.Fatal("Provide.Strategy='flat' is no longer supported. Use 'all' instead. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy")
521
}
522
if cfg.Experimental.StrategicProviding {
497
- log.Error("Experimental.StrategicProviding was removed. Remove it from your config and set Provider.Enabled=false to remove this message. Documentation: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#strategic-providing")
498
- cfg.Experimental.StrategicProviding = false
499
- cfg.Provider.Enabled = config.False
523
+ log.Fatal("Experimental.StrategicProviding was removed. Remove it from your config. Documentation: https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#strategic-providing")
524
+ }
525
+ // Check for invalid MaxWorkers=0 with SweepEnabled
526
+ if cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled) &&
527
+ cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers) == 0 {
528
+ log.Fatal("Invalid configuration: Provide.DHT.MaxWorkers cannot be 0 when Provide.DHT.SweepEnabled=true. Set Provide.DHT.MaxWorkers to a positive value (e.g., 16) to control resource usage. Documentation: https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtmaxworkers")
529
}
530
if routingOption == routingOptionDelegatedKwd {
531
// Delegated routing is read-only mode - content providing must be disabled
503
- if cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
504
- log.Fatal("Routing.Type=delegated does not support content providing. Set Provider.Enabled=false in your config.")
505
- }
506
- if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) != 0 {
507
- log.Fatal("Routing.Type=delegated does not support content providing. Set Reprovider.Interval='0' in your config.")
532
+ if cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) {
533
+ log.Fatal("Routing.Type=delegated does not support content providing. Set Provide.Enabled=false in your config.")
534
}
535
}
536
@@ -659,7 +685,7 @@ take effect.
685
686
if !offline {
687
// Warn users when provide systems are disabled
662
- if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
688
+ if !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) {
689
fmt.Print(`
690
691
⚠️ Provide and Reprovide systems are disabled due to 'Provide.Enabled=false'
@@ -667,12 +693,12 @@ take effect.
693
⚠️ If this is not intentional, call 'ipfs config profile apply announce-on' or set Provide.Enabled=true'
694
695
`)
670
- } else if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
696
+ } else if cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0 {
697
fmt.Print(`
698
673
-⚠️ Provide and Reprovide systems are disabled due to 'Reprovider.Interval=0'
674
-⚠️ Local CIDs will not be announced to Amino DHT, making them impossible to retrieve without manual peering
675
-⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Reprovider.Interval=22h'
699
+⚠️ Providing to the DHT is disabled due to 'Provide.DHT.Interval=0'
700
+⚠️ Local CIDs will not be provided to Amino DHT, making them impossible to retrieve without manual peering
701
+⚠️ If this is not intentional, call 'ipfs config profile apply announce-on', or set 'Provide.DHT.Interval=22h'
702
703
`)
704
}
config/config.go
+3
-2
@@ -35,8 +35,9 @@ type Config struct {
35
Migration Migration
36
AutoConf AutoConf
37
38
- Provider Provider
39
- Reprovider Reprovider
38
+ Provide Provide // Merged Provider and Reprovider configuration
39
+ Provider Provider // Deprecated: use Provide. Will be removed in a future release.
40
+ Reprovider Reprovider // Deprecated: use Provide. Will be removed in a future release.
41
HTTPRetrieval HTTPRetrieval
42
Experimental Experiments
43
Plugins Plugins
config/config_test.go
+14
-4
@@ -134,14 +134,24 @@ func TestCheckKey(t *testing.T) {
134
t.Fatal("Foo.Bar isn't a valid key in the config")
135
}
136
137
- err = CheckKey("Reprovider.Strategy")
137
+ err = CheckKey("Provide.Strategy")
138
if err != nil {
139
- t.Fatalf("%s: %s", err, "Reprovider.Strategy is a valid key in the config")
139
+ t.Fatalf("%s: %s", err, "Provide.Strategy is a valid key in the config")
140
}
141
142
- err = CheckKey("Provider.Foo")
142
+ err = CheckKey("Provide.DHT.MaxWorkers")
143
+ if err != nil {
144
+ t.Fatalf("%s: %s", err, "Provide.DHT.MaxWorkers is a valid key in the config")
145
+ }
146
+
147
+ err = CheckKey("Provide.DHT.Interval")
148
+ if err != nil {
149
+ t.Fatalf("%s: %s", err, "Provide.DHT.Interval is a valid key in the config")
150
+ }
151
+
152
+ err = CheckKey("Provide.Foo")
153
if err == nil {
144
- t.Fatal("Provider.Foo isn't a valid key in the config")
154
+ t.Fatal("Provide.Foo isn't a valid key in the config")
155
}
156
157
err = CheckKey("Gateway.PublicGateways.Foo.Paths")
config/init.go
-4
@@ -60,10 +60,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
60
NoFetch: false,
61
HTTPHeaders: map[string][]string{},
62
},
63
- Reprovider: Reprovider{
64
- Interval: nil,
65
- Strategy: nil,
66
- },
63
Pinning: Pinning{
64
RemoteServices: map[string]RemotePinningService{},
65
},
config/profile.go
+6
-6
@@ -275,7 +275,7 @@ fetching may be degraded.
275
},
276
},
277
"announce-off": {
278
- Description: `Disables Provide and Reprovide systems (announcing to Amino DHT).
278
+ Description: `Disables Provide system (announcing to Amino DHT).
279
280
USE WITH CAUTION:
281
The main use case for this is setups with manual Peering.Peers config.
@@ -284,16 +284,16 @@ fetching may be degraded.
284
one hosting it, and other peers are not already connected to it.
285
`,
286
Transform: func(c *Config) error {
287
- c.Provider.Enabled = False
288
- c.Reprovider.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide
287
+ c.Provide.Enabled = False
288
+ c.Provide.DHT.Interval = NewOptionalDuration(0) // 0 disables periodic reprovide
289
return nil
290
},
291
},
292
"announce-on": {
293
- Description: `Re-enables Provide and Reprovide systems (reverts announce-off profile).`,
293
+ Description: `Re-enables Provide system (reverts announce-off profile).`,
294
Transform: func(c *Config) error {
295
- c.Provider.Enabled = True
296
- c.Reprovider.Interval = NewOptionalDuration(DefaultReproviderInterval) // have to apply explicit default because nil would be ignored
295
+ c.Provide.Enabled = True
296
+ c.Provide.DHT.Interval = NewOptionalDuration(DefaultProvideDHTInterval) // have to apply explicit default because nil would be ignored
297
return nil
298
},
299
},
config/provide.go
new
+103
@@ -0,0 +1,103 @@
1
+package config
2
+
3
+import (
4
+ "strings"
5
+ "time"
6
+)
7
+
8
+const (
9
+ DefaultProvideEnabled = true
10
+ DefaultProvideStrategy = "all"
11
+
12
+ // DHT provider defaults
13
+ DefaultProvideDHTInterval = 22 * time.Hour // https://github.com/ipfs/kubo/pull/9326
14
+ DefaultProvideDHTMaxWorkers = 16 // Unified default for both sweep and legacy providers
15
+ DefaultProvideDHTSweepEnabled = false
16
+ DefaultProvideDHTDedicatedPeriodicWorkers = 2
17
+ DefaultProvideDHTDedicatedBurstWorkers = 1
18
+ DefaultProvideDHTMaxProvideConnsPerWorker = 16
19
+ DefaultProvideDHTKeyStoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes)
20
+ DefaultProvideDHTOfflineDelay = 2 * time.Hour
21
+)
22
+
23
+type ProvideStrategy int
24
+
25
+const (
26
+ ProvideStrategyAll ProvideStrategy = 1 << iota
27
+ ProvideStrategyPinned
28
+ ProvideStrategyRoots
29
+ ProvideStrategyMFS
30
+)
31
+
32
+// Provide configures both immediate CID announcements (provide operations) for new content
33
+// and periodic re-announcements of existing CIDs (reprovide operations).
34
+// This section combines the functionality previously split between Provider and Reprovider.
35
+type Provide struct {
36
+ // Enabled controls whether both provide and reprovide systems are enabled.
37
+ // When disabled, the node will not announce any content to the routing system.
38
+ Enabled Flag `json:",omitempty"`
39
+
40
+ // Strategy determines which CIDs are announced to the routing system.
41
+ // Default: DefaultProvideStrategy
42
+ Strategy *OptionalString `json:",omitempty"`
43
+
44
+ // DHT configures DHT-specific provide and reprovide settings.
45
+ DHT ProvideDHT
46
+}
47
+
48
+// ProvideDHT configures DHT provider settings for both immediate announcements
49
+// and periodic reprovides.
50
+type ProvideDHT struct {
51
+ // Interval sets the time between rounds of reproviding local content
52
+ // to the routing system. Set to "0" to disable content reproviding.
53
+ // Default: DefaultProvideDHTInterval
54
+ Interval *OptionalDuration `json:",omitempty"`
55
+
56
+ // MaxWorkers sets the maximum number of concurrent workers for provide operations.
57
+ // When SweepEnabled is false: controls NEW CID announcements only.
58
+ // When SweepEnabled is true: controls total worker pool for all operations.
59
+ // Default: DefaultProvideDHTMaxWorkers
60
+ MaxWorkers *OptionalInteger `json:",omitempty"`
61
+
62
+ // SweepEnabled activates the sweeping reprovider system which spreads
63
+ // reprovide operations over time. This will become the default in a future release.
64
+ // Default: DefaultProvideDHTSweepEnabled
65
+ SweepEnabled Flag `json:",omitempty"`
66
+
67
+ // DedicatedPeriodicWorkers sets workers dedicated to periodic reprovides (sweep mode only).
68
+ // Default: DefaultProvideDHTDedicatedPeriodicWorkers
69
+ DedicatedPeriodicWorkers *OptionalInteger `json:",omitempty"`
70
+
71
+ // DedicatedBurstWorkers sets workers dedicated to burst provides (sweep mode only).
72
+ // Default: DefaultProvideDHTDedicatedBurstWorkers
73
+ DedicatedBurstWorkers *OptionalInteger `json:",omitempty"`
74
+
75
+ // MaxProvideConnsPerWorker sets concurrent connections per worker for sending provider records (sweep mode only).
76
+ // Default: DefaultProvideDHTMaxProvideConnsPerWorker
77
+ MaxProvideConnsPerWorker *OptionalInteger `json:",omitempty"`
78
+
79
+ // KeyStoreBatchSize sets the batch size for keystore operations during reprovide refresh (sweep mode only).
80
+ // Default: DefaultProvideDHTKeyStoreBatchSize
81
+ KeyStoreBatchSize *OptionalInteger `json:",omitempty"`
82
+
83
+ // OfflineDelay sets the delay after which the provider switches from Disconnected to Offline state (sweep mode only).
84
+ // Default: DefaultProvideDHTOfflineDelay
85
+ OfflineDelay *OptionalDuration `json:",omitempty"`
86
+}
87
+
88
+func ParseProvideStrategy(s string) ProvideStrategy {
89
+ var strategy ProvideStrategy
90
+ for _, part := range strings.Split(s, "+") {
91
+ switch part {
92
+ case "all", "flat", "": // special case, does not mix with others ("flat" is deprecated, maps to "all")
93
+ return ProvideStrategyAll
94
+ case "pinned":
95
+ strategy |= ProvideStrategyPinned
96
+ case "roots":
97
+ strategy |= ProvideStrategyRoots
98
+ case "mfs":
99
+ strategy |= ProvideStrategyMFS
100
+ }
101
+ }
102
+ return strategy
103
+}
config/provide_test.go
new
+27
@@ -0,0 +1,27 @@
1
+package config
2
+
3
+import "testing"
4
+
5
+func TestParseProvideStrategy(t *testing.T) {
6
+ tests := []struct {
7
+ input string
8
+ expect ProvideStrategy
9
+ }{
10
+ {"all", ProvideStrategyAll},
11
+ {"pinned", ProvideStrategyPinned},
12
+ {"mfs", ProvideStrategyMFS},
13
+ {"pinned+mfs", ProvideStrategyPinned | ProvideStrategyMFS},
14
+ {"invalid", 0},
15
+ {"all+invalid", ProvideStrategyAll},
16
+ {"", ProvideStrategyAll},
17
+ {"flat", ProvideStrategyAll}, // deprecated, maps to "all"
18
+ {"flat+all", ProvideStrategyAll},
19
+ }
20
+
21
+ for _, tt := range tests {
22
+ result := ParseProvideStrategy(tt.input)
23
+ if result != tt.expect {
24
+ t.Errorf("ParseProvideStrategy(%q) = %d, want %d", tt.input, result, tt.expect)
25
+ }
26
+ }
27
+}
config/provider.go
+11
-9
@@ -1,14 +1,16 @@
1
package config
2
3
-const (
4
- DefaultProviderEnabled = true
5
- DefaultProviderWorkerCount = 16
6
-)
7
-
3
// Provider configuration describes how NEW CIDs are announced the moment they are created.
9
-// For periodical reprovide configuration, see Reprovider.*
4
+// For periodical reprovide configuration, see Provide.*
5
+//
6
+// Deprecated: use Provide instead. This will be removed in a future release.
7
type Provider struct {
11
- Enabled Flag `json:",omitempty"`
12
- Strategy *OptionalString `json:",omitempty"` // Unused, you are likely looking for Reprovider.Strategy instead
13
- WorkerCount *OptionalInteger `json:",omitempty"` // Number of concurrent provides allowed, 0 means unlimited
8
+ // Deprecated: use Provide.Enabled instead. This will be removed in a future release.
9
+ Enabled Flag `json:",omitempty"`
10
+
11
+ // Deprecated: unused, you are likely looking for Provide.Strategy instead. This will be removed in a future release.
12
+ Strategy *OptionalString `json:",omitempty"`
13
+
14
+ // Deprecated: use Provide.DHT.MaxWorkers instead. This will be removed in a future release.
15
+ WorkerCount *OptionalInteger `json:",omitempty"`
16
}
config/reprovider.go
+7
-63
@@ -1,69 +1,13 @@
1
package config
2
3
-import (
4
- "strings"
5
- "time"
6
-)
7
-
8
-const (
9
- DefaultReproviderInterval = time.Hour * 22 // https://github.com/ipfs/kubo/pull/9326
10
- DefaultReproviderStrategy = "all"
11
-
12
- DefaultReproviderSweepEnabled = false
13
- DefaultReproviderSweepMaxWorkers = 4
14
- DefaultReproviderSweepDedicatedPeriodicWorkers = 2
15
- DefaultReproviderSweepDedicatedBurstWorkers = 1
16
- DefaultReproviderSweepMaxProvideConnsPerWorker = 16
17
- DefaultReproviderSweepKeyStoreBatchSize = 1 << 14 // ~544 KiB per batch (1 multihash = 34 bytes)
18
- DefaultReproviderSweepOfflineDelay = 2 * time.Hour
19
-)
20
-
21
-type ReproviderStrategy int
22
-
23
-const (
24
- ReproviderStrategyAll ReproviderStrategy = 1 << iota
25
- ReproviderStrategyPinned
26
- ReproviderStrategyRoots
27
- ReproviderStrategyMFS
28
-)
29
-
3
// Reprovider configuration describes how CID from local datastore are periodically re-announced to routing systems.
31
-// For provide behavior of ad-hoc or newly created CIDs and their first-time announcement, see Provider.*
4
+// For provide behavior of ad-hoc or newly created CIDs and their first-time announcement, see Provide.*
5
+//
6
+// Deprecated: use Provide instead. This will be removed in a future release.
7
type Reprovider struct {
33
- Interval *OptionalDuration `json:",omitempty"` // Time period to reprovide locally stored objects to the network
34
- Strategy *OptionalString `json:",omitempty"` // Which keys to announce
35
-
36
- Sweep Sweep
37
-}
38
-
39
-// Sweep configuration describes how the Sweeping Reprovider is configured if enabled.
40
-type Sweep struct {
41
- Enabled Flag `json:",omitempty"`
42
-
43
- MaxWorkers *OptionalInteger // Max number of concurrent workers performing a provide operation.
44
- DedicatedPeriodicWorkers *OptionalInteger // Number of workers dedicated to periodic reprovides.
45
- DedicatedBurstWorkers *OptionalInteger // Number of workers dedicated to initial provides or burst reproviding keyspace regions after a period of inactivity.
46
- MaxProvideConnsPerWorker *OptionalInteger // Number of connections that a worker is able to open to send provider records during a (re)provide operation.
47
-
48
- KeyStoreGCInterval *OptionalDuration // Interval for garbage collection in KeyStore.
49
- KeyStoreBatchSize *OptionalInteger // Number of multihashes to keep in memory when gc'ing the KeyStore.
50
-
51
- OfflineDelay *OptionalDuration // Delay after which the provides changes state from Disconnected to Offline.
52
-}
8
+ // Deprecated: use Provide.DHT.Interval instead. This will be removed in a future release.
9
+ Interval *OptionalDuration `json:",omitempty"`
10
54
-func ParseReproviderStrategy(s string) ReproviderStrategy {
55
- var strategy ReproviderStrategy
56
- for _, part := range strings.Split(s, "+") {
57
- switch part {
58
- case "all", "flat", "": // special case, does not mix with others ("flat" is deprecated, maps to "all")
59
- return ReproviderStrategyAll
60
- case "pinned":
61
- strategy |= ReproviderStrategyPinned
62
- case "roots":
63
- strategy |= ReproviderStrategyRoots
64
- case "mfs":
65
- strategy |= ReproviderStrategyMFS
66
- }
67
- }
68
- return strategy
11
+ // Deprecated: use Provide.Strategy instead. This will be removed in a future release.
12
+ Strategy *OptionalString `json:",omitempty"`
13
}
config/reprovider_test.go
deleted
-27
@@ -1,27 +0,0 @@
1
-package config
2
-
3
-import "testing"
4
-
5
-func TestParseReproviderStrategy(t *testing.T) {
6
- tests := []struct {
7
- input string
8
- expect ReproviderStrategy
9
- }{
10
- {"all", ReproviderStrategyAll},
11
- {"pinned", ReproviderStrategyPinned},
12
- {"mfs", ReproviderStrategyMFS},
13
- {"pinned+mfs", ReproviderStrategyPinned | ReproviderStrategyMFS},
14
- {"invalid", 0},
15
- {"all+invalid", ReproviderStrategyAll},
16
- {"", ReproviderStrategyAll},
17
- {"flat", ReproviderStrategyAll}, // deprecated, maps to "all"
18
- {"flat+all", ReproviderStrategyAll},
19
- }
20
-
21
- for _, tt := range tests {
22
- result := ParseReproviderStrategy(tt.input)
23
- if result != tt.expect {
24
- t.Errorf("ParseReproviderStrategy(%q) = %d, want %d", tt.input, result, tt.expect)
25
- }
26
- }
27
-}
core/commands/add.go
+1
-1
@@ -82,7 +82,7 @@ to form the IPFS MerkleDAG. Learn more: https://docs.ipfs.tech/concepts/merkle-d
82
83
If the daemon is not running, it will just add locally to the repo at $IPFS_PATH.
84
If the daemon is started later, it will be advertised after a few
85
-seconds when the reprovider runs.
85
+seconds when the provide system runs.
86
87
BASIC EXAMPLES:
88
core/commands/provide.go
+6
-6
@@ -45,12 +45,12 @@ var provideClearCmd = &cmds.Command{
45
Helptext: cmds.HelpText{
46
Tagline: "Clear all CIDs from the provide queue.",
47
ShortDescription: `
48
-Clear all CIDs from the reprovide queue.
48
+Clear all CIDs pending to be provided for the first time.
49
50
Note: Kubo will automatically clear the queue when it detects a change of
51
-Reprovider.Strategy upon a restart. For more information about reprovider
51
+Provide.Strategy upon a restart. For more information about provide
52
strategies, see:
53
-https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy
53
+https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy
54
`,
55
},
56
Options: []cmds.Option{
@@ -100,8 +100,8 @@ var provideStatCmd = &cmds.Command{
100
Tagline: "Returns statistics about the node's provider system.",
101
ShortDescription: `
102
Returns statistics about the content the node is reproviding every
103
-Reprovider.Interval according to Reprovider.Strategy:
104
-https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovider
103
+Provide.DHT.Interval according to Provide.Strategy:
104
+https://github.com/ipfs/kubo/blob/master/docs/config.md#provide
105
106
This interface is not stable and may change from release to release.
107
@@ -121,7 +121,7 @@ This interface is not stable and may change from release to release.
121
122
provideSys, ok := nd.Provider.(provider.System)
123
if !ok {
124
- return errors.New("stats not available with experimental sweeping provider (Reprovider.Sweep.Enabled=true)")
124
+ return errors.New("stats not available with experimental sweeping provider (Provide.DHT.SweepEnabled=true)")
125
}
126
127
stats, err := provideSys.Stat()
core/commands/routing.go
+7
-7
@@ -166,8 +166,8 @@ var provideRefRoutingCmd = &cmds.Command{
166
if err != nil {
167
return err
168
}
169
- if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
170
- return errors.New("invalid configuration: Provider.Enabled is set to 'false'")
169
+ if !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) {
170
+ return errors.New("invalid configuration: Provide.Enabled is set to 'false'")
171
}
172
173
if len(nd.PeerHost.Network().Conns()) == 0 {
@@ -270,15 +270,15 @@ Trigger reprovider to announce our data to network.
270
if err != nil {
271
return err
272
}
273
- if !cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) {
274
- return errors.New("invalid configuration: Provider.Enabled is set to 'false'")
273
+ if !cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) {
274
+ return errors.New("invalid configuration: Provide.Enabled is set to 'false'")
275
}
276
- if cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) == 0 {
277
- return errors.New("invalid configuration: Reprovider.Interval is set to '0'")
276
+ if cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) == 0 {
277
+ return errors.New("invalid configuration: Provide.DHT.Interval is set to '0'")
278
}
279
provideSys, ok := nd.Provider.(*node.LegacyProvider)
280
if !ok {
281
- return errors.New("manual reprovide not available with experimental sweeping provider (Reprovider.Sweep.Enabled=true)")
281
+ return errors.New("manual reprovide not available with experimental sweeping provider (Provide.DHT.SweepEnabled=true)")
282
}
283
284
err = provideSys.Reprovide(req.Context)
core/core.go
+17
-17
@@ -92,23 +92,23 @@ type IpfsNode struct {
92
RecordValidator record.Validator
93
94
// Online
95
- PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
96
- Peering *peering.PeeringService `optional:"true"`
97
- Filters *ma.Filters `optional:"true"`
98
- Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
99
- ContentDiscovery routing.ContentDiscovery `optional:"true"` // the discovery part of the routing system
100
- DNSResolver *madns.Resolver // the DNS resolver
101
- IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
102
- UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
103
- OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
104
- OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
105
- Exchange exchange.Interface // the block exchange + strategy
106
- Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance
107
- Namesys namesys.NameSystem // the name system, resolves paths to hashes
108
- ProvidingStrategy config.ReproviderStrategy `optional:"true"`
109
- ProvidingKeyChanFunc provider.KeyChanFunc `optional:"true"`
110
- IpnsRepub *ipnsrp.Republisher `optional:"true"`
111
- ResourceManager network.ResourceManager `optional:"true"`
95
+ PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
96
+ Peering *peering.PeeringService `optional:"true"`
97
+ Filters *ma.Filters `optional:"true"`
98
+ Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
99
+ ContentDiscovery routing.ContentDiscovery `optional:"true"` // the discovery part of the routing system
100
+ DNSResolver *madns.Resolver // the DNS resolver
101
+ IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
102
+ UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
103
+ OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
104
+ OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
105
+ Exchange exchange.Interface // the block exchange + strategy
106
+ Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance
107
+ Namesys namesys.NameSystem // the name system, resolves paths to hashes
108
+ ProvidingStrategy config.ProvideStrategy `optional:"true"`
109
+ ProvidingKeyChanFunc provider.KeyChanFunc `optional:"true"`
110
+ IpnsRepub *ipnsrp.Republisher `optional:"true"`
111
+ ResourceManager network.ResourceManager `optional:"true"`
112
113
PubSub *pubsub.PubSub `optional:"true"`
114
PSRouter *psrouter.PubsubValueStore `optional:"true"`
core/coreapi/coreapi.go
+1
-1
@@ -70,7 +70,7 @@ type CoreAPI struct {
70
unixFSPathResolver pathresolver.Resolver
71
72
provider node.DHTProvider
73
- providingStrategy config.ReproviderStrategy
73
+ providingStrategy config.ProvideStrategy
74
75
pubSub *pubsub.PubSub
76
core/coreapi/test/api_test.go
+1
-1
@@ -72,7 +72,7 @@ func (NodeProvider) MakeAPISwarm(t *testing.T, ctx context.Context, fullIdentity
72
c.AutoTLS.Enabled = config.False // disable so no /ws listener is added
73
// For provider tests, avoid that content gets
74
// auto-provided without calling "provide" (unless pinned).
75
- c.Reprovider.Strategy = config.NewOptionalString("roots")
75
+ c.Provide.Strategy = config.NewOptionalString("roots")
76
77
ds := syncds.MutexWrap(datastore.NewMapDatastore())
78
r := &repo.Mock{
core/coreapi/unixfs.go
+1
-1
@@ -120,7 +120,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
120
// nor by the pinner (the pinner doesn't traverse the pinned DAG itself, it only
121
// handles roots). This wrapping ensures all blocks of pinned content get provided.
122
if settings.Pin && !settings.OnlyHash &&
123
- (api.providingStrategy&config.ReproviderStrategyPinned) != 0 {
123
+ (api.providingStrategy&config.ProvideStrategyPinned) != 0 {
124
dserv = &providingDagService{dserv, api.provider}
125
}
126
core/node/core.go
+5
-5
@@ -52,7 +52,7 @@ func Pinning(strategy string) func(bstore blockstore.Blockstore, ds format.DAGSe
52
// Parse strategy at function creation time (not inside the returned function)
53
// This happens before the provider is created, which is why we pass the strategy
54
// string and parse it here, rather than using fx-provided ProvidingStrategy.
55
- strategyFlag := config.ParseReproviderStrategy(strategy)
55
+ strategyFlag := config.ParseProvideStrategy(strategy)
56
57
return func(bstore blockstore.Blockstore,
58
ds format.DAGService,
@@ -72,8 +72,8 @@ func Pinning(strategy string) func(bstore blockstore.Blockstore, ds format.DAGSe
72
ctx := context.TODO()
73
74
var opts []dspinner.Option
75
- roots := (strategyFlag & config.ReproviderStrategyRoots) != 0
76
- pinned := (strategyFlag & config.ReproviderStrategyPinned) != 0
75
+ roots := (strategyFlag & config.ProvideStrategyRoots) != 0
76
+ pinned := (strategyFlag & config.ProvideStrategyPinned) != 0
77
78
// Important: Only one of WithPinnedProvider or WithRootsProvider should be active.
79
// Having both would cause duplicate root advertisements since "pinned" includes all
@@ -236,8 +236,8 @@ func Files(strategy string) func(mctx helpers.MetricsCtx, lc fx.Lifecycle, repo
236
// strategy - it ensures all MFS content gets announced as it's added or
237
// modified. For non-mfs strategies, we set provider to nil to avoid
238
// unnecessary providing.
239
- strategyFlag := config.ParseReproviderStrategy(strategy)
240
- if strategyFlag&config.ReproviderStrategyMFS == 0 {
239
+ strategyFlag := config.ParseProvideStrategy(strategy)
240
+ if strategyFlag&config.ProvideStrategyMFS == 0 {
241
prov = nil
242
}
243
core/node/groups.go
+5
-5
@@ -254,7 +254,7 @@ func Storage(bcfg *BuildCfg, cfg *config.Config) fx.Option {
254
cacheOpts,
255
cfg.Datastore.HashOnRead,
256
cfg.Datastore.WriteThrough.WithDefault(config.DefaultWriteThrough),
257
- cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy),
257
+ cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy),
258
)),
259
finalBstore,
260
)
@@ -347,9 +347,9 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
347
isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
348
isHTTPRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
349
350
- // Right now Provider and Reprovider systems are tied together - disabling Reprovider by setting interval to 0 disables Provider
351
- // and vice versa: Provider.Enabled=false will disable both Provider of new CIDs and the Reprovider of old ones.
352
- isProviderEnabled := cfg.Provider.Enabled.WithDefault(config.DefaultProviderEnabled) && cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval) != 0
350
+ // The Provide system handles both new CID announcements and periodic re-announcements.
351
+ // Disabling is controlled by Provide.Enabled=false or setting Interval to 0.
352
+ isProviderEnabled := cfg.Provide.Enabled.WithDefault(config.DefaultProvideEnabled) && cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval) != 0
353
354
return fx.Options(
355
fx.Provide(BitswapOptions(cfg)),
@@ -442,7 +442,7 @@ func IPFS(ctx context.Context, bcfg *BuildCfg) fx.Option {
442
uio.HAMTShardingSize = int(shardSingThresholdInt)
443
uio.DefaultShardWidth = int(shardMaxFanout)
444
445
- providerStrategy := cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy)
445
+ providerStrategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
446
447
return fx.Options(
448
bcfgOpts,
core/node/provider.go
+62
-69
@@ -24,7 +24,6 @@ import (
24
"github.com/libp2p/go-libp2p-kad-dht/fullrt"
25
dht_pb "github.com/libp2p/go-libp2p-kad-dht/pb"
26
dhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider"
27
- "github.com/libp2p/go-libp2p-kad-dht/provider/buffered"
27
ddhtprovider "github.com/libp2p/go-libp2p-kad-dht/provider/dual"
28
"github.com/libp2p/go-libp2p-kad-dht/provider/keystore"
29
routinghelpers "github.com/libp2p/go-libp2p-routing-helpers"
@@ -85,7 +84,7 @@ type DHTProvider interface {
84
// The keys are not deleted from the keystore, so they will continue to be
85
// reprovided as scheduled.
86
Clear() int
88
- // RefreshSchedule scans the Keystore for any keys that are not currently
87
+ // RefreshSchedule scans the KeyStore for any keys that are not currently
88
// scheduled for reproviding. If such keys are found, it schedules their
89
// associated keyspace region to be reprovided.
90
//
@@ -107,6 +106,9 @@ var (
106
_ DHTProvider = &LegacyProvider{}
107
)
108
109
+// NoopProvider is a no-operation provider implementation that does nothing.
110
+// It is used when providing is disabled or when no DHT is available.
111
+// All methods return successfully without performing any actual operations.
112
type NoopProvider struct{}
113
114
func (r *NoopProvider) StartProviding(bool, ...mh.Multihash) error { return nil }
@@ -114,9 +116,14 @@ func (r *NoopProvider) ProvideOnce(...mh.Multihash) error { return nil
116
func (r *NoopProvider) Clear() int { return 0 }
117
func (r *NoopProvider) RefreshSchedule() error { return nil }
118
117
-// LegacyProvider is a wrapper around the boxo/provider.System. This DHT
118
-// provide system manages reprovides by bursts where it sequentially reprovides
119
-// all keys.
119
+// LegacyProvider is a wrapper around the boxo/provider.System that implements
120
+// the DHTProvider interface. This provider manages reprovides using a burst
121
+// strategy where it sequentially reprovides all keys at once during each
122
+// reprovide interval, rather than spreading the load over time.
123
+//
124
+// This is the legacy provider implementation that can cause resource spikes
125
+// during reprovide operations. For more efficient providing, consider using
126
+// the SweepingProvider which spreads the load over the reprovide interval.
127
type LegacyProvider struct {
128
provider.System
129
}
@@ -297,7 +304,7 @@ type addrsFilter interface {
304
}
305
306
func SweepingProviderOpt(cfg *config.Config) fx.Option {
300
- reprovideInterval := cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval)
307
+ reprovideInterval := cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval)
308
type providerInput struct {
309
fx.In
310
DHT routing.Routing `name:"dhtc"`
@@ -305,21 +312,14 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
312
}
313
sweepingReprovider := fx.Provide(func(in providerInput) (DHTProvider, *keystore.ResettableKeystore, error) {
314
ds := in.Repo.Datastore()
308
- ks, err := keystore.NewResettableKeystore(ds,
315
+ keyStore, err := keystore.NewResettableKeystore(ds,
316
keystore.WithPrefixBits(16),
317
keystore.WithDatastorePath("/provider/keystore"),
311
- keystore.WithBatchSize(int(cfg.Reprovider.Sweep.KeyStoreBatchSize.WithDefault(config.DefaultReproviderSweepKeyStoreBatchSize))),
318
+ keystore.WithBatchSize(int(cfg.Provide.DHT.KeyStoreBatchSize.WithDefault(config.DefaultProvideDHTKeyStoreBatchSize))),
319
)
320
if err != nil {
321
return &NoopProvider{}, nil, err
322
}
316
-
317
- bufferedProviderOpts := []buffered.Option{
318
- buffered.WithBatchSize(1 << 10),
319
- buffered.WithDsName("bprov"),
320
- buffered.WithIdleWriteTime(time.Minute),
321
- }
322
-
323
var impl dhtImpl
324
switch inDht := in.DHT.(type) {
325
case *dht.IpfsDHT:
@@ -329,22 +329,23 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
329
case *dual.DHT:
330
if inDht != nil {
331
prov, err := ddhtprovider.New(inDht,
332
- ddhtprovider.WithKeystore(ks),
332
+ ddhtprovider.WithKeystore(keyStore),
333
334
ddhtprovider.WithReprovideInterval(reprovideInterval),
335
ddhtprovider.WithMaxReprovideDelay(time.Hour),
336
- ddhtprovider.WithOfflineDelay(cfg.Reprovider.Sweep.OfflineDelay.WithDefault(config.DefaultReproviderSweepOfflineDelay)),
336
+ ddhtprovider.WithOfflineDelay(cfg.Provide.DHT.OfflineDelay.WithDefault(config.DefaultProvideDHTOfflineDelay)),
337
ddhtprovider.WithConnectivityCheckOnlineInterval(1*time.Minute),
338
339
- ddhtprovider.WithMaxWorkers(int(cfg.Reprovider.Sweep.MaxWorkers.WithDefault(config.DefaultReproviderSweepMaxWorkers))),
340
- ddhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Reprovider.Sweep.DedicatedPeriodicWorkers.WithDefault(config.DefaultReproviderSweepDedicatedPeriodicWorkers))),
341
- ddhtprovider.WithDedicatedBurstWorkers(int(cfg.Reprovider.Sweep.DedicatedBurstWorkers.WithDefault(config.DefaultReproviderSweepDedicatedBurstWorkers))),
342
- ddhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Reprovider.Sweep.MaxProvideConnsPerWorker.WithDefault(config.DefaultReproviderSweepMaxProvideConnsPerWorker))),
339
+ ddhtprovider.WithMaxWorkers(int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers))),
340
+ ddhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Provide.DHT.DedicatedPeriodicWorkers.WithDefault(config.DefaultProvideDHTDedicatedPeriodicWorkers))),
341
+ ddhtprovider.WithDedicatedBurstWorkers(int(cfg.Provide.DHT.DedicatedBurstWorkers.WithDefault(config.DefaultProvideDHTDedicatedBurstWorkers))),
342
+ ddhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Provide.DHT.MaxProvideConnsPerWorker.WithDefault(config.DefaultProvideDHTMaxProvideConnsPerWorker))),
343
)
344
if err != nil {
345
return nil, nil, err
346
}
347
- return buffered.New(prov, ds, bufferedProviderOpts...), ks, nil
347
+ _ = prov
348
+ return prov, keyStore, nil
349
}
350
case *fullrt.FullRT:
351
if inDht != nil {
@@ -352,7 +353,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
353
}
354
}
355
if impl == nil {
355
- return &NoopProvider{}, nil, nil
356
+ return &NoopProvider{}, nil, errors.New("provider: no valid DHT available for providing")
357
}
358
359
var selfAddrsFunc func() []ma.Multiaddr
@@ -362,7 +363,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
363
selfAddrsFunc = func() []ma.Multiaddr { return impl.Host().Addrs() }
364
}
365
opts := []dhtprovider.Option{
365
- dhtprovider.WithKeystore(ks),
366
+ dhtprovider.WithKeystore(keyStore),
367
dhtprovider.WithPeerID(impl.Host().ID()),
368
dhtprovider.WithRouter(impl),
369
dhtprovider.WithMessageSender(impl.MessageSender()),
@@ -374,40 +375,37 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
375
dhtprovider.WithReplicationFactor(amino.DefaultBucketSize),
376
dhtprovider.WithReprovideInterval(reprovideInterval),
377
dhtprovider.WithMaxReprovideDelay(time.Hour),
377
- dhtprovider.WithOfflineDelay(cfg.Reprovider.Sweep.OfflineDelay.WithDefault(config.DefaultReproviderSweepOfflineDelay)),
378
+ dhtprovider.WithOfflineDelay(cfg.Provide.DHT.OfflineDelay.WithDefault(config.DefaultProvideDHTOfflineDelay)),
379
dhtprovider.WithConnectivityCheckOnlineInterval(1 * time.Minute),
380
380
- dhtprovider.WithMaxWorkers(int(cfg.Reprovider.Sweep.MaxWorkers.WithDefault(config.DefaultReproviderSweepMaxWorkers))),
381
- dhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Reprovider.Sweep.DedicatedPeriodicWorkers.WithDefault(config.DefaultReproviderSweepDedicatedPeriodicWorkers))),
382
- dhtprovider.WithDedicatedBurstWorkers(int(cfg.Reprovider.Sweep.DedicatedBurstWorkers.WithDefault(config.DefaultReproviderSweepDedicatedBurstWorkers))),
383
- dhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Reprovider.Sweep.MaxProvideConnsPerWorker.WithDefault(config.DefaultReproviderSweepMaxProvideConnsPerWorker))),
381
+ dhtprovider.WithMaxWorkers(int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers))),
382
+ dhtprovider.WithDedicatedPeriodicWorkers(int(cfg.Provide.DHT.DedicatedPeriodicWorkers.WithDefault(config.DefaultProvideDHTDedicatedPeriodicWorkers))),
383
+ dhtprovider.WithDedicatedBurstWorkers(int(cfg.Provide.DHT.DedicatedBurstWorkers.WithDefault(config.DefaultProvideDHTDedicatedBurstWorkers))),
384
+ dhtprovider.WithMaxProvideConnsPerWorker(int(cfg.Provide.DHT.MaxProvideConnsPerWorker.WithDefault(config.DefaultProvideDHTMaxProvideConnsPerWorker))),
385
}
386
387
prov, err := dhtprovider.New(opts...)
387
- if err != nil {
388
- return &NoopProvider{}, nil, err
389
- }
390
- return buffered.New(prov, ds, bufferedProviderOpts...), ks, nil
388
+ return prov, keyStore, err
389
})
390
391
type keystoreInput struct {
392
fx.In
393
Provider DHTProvider
396
- Keystore *keystore.ResettableKeystore
394
+ KeyStore *keystore.ResettableKeystore
395
KeyProvider provider.KeyChanFunc
396
}
399
- initKeystore := fx.Invoke(func(lc fx.Lifecycle, in keystoreInput) {
397
+ initKeyStore := fx.Invoke(func(lc fx.Lifecycle, in keystoreInput) {
398
var (
399
cancel context.CancelFunc
400
done = make(chan struct{})
401
)
402
405
- syncKeystore := func(ctx context.Context) error {
403
+ syncKeyStore := func(ctx context.Context) error {
404
kcf, err := in.KeyProvider(ctx)
405
if err != nil {
406
return err
407
}
410
- if err := in.Keystore.ResetCids(ctx, kcf); err != nil {
408
+ if err := in.KeyStore.ResetCids(ctx, kcf); err != nil {
409
return err
410
}
411
if err := in.Provider.RefreshSchedule(); err != nil {
@@ -418,15 +416,12 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
416
417
lc.Append(fx.Hook{
418
OnStart: func(ctx context.Context) error {
421
- if in.Provider == nil || in.Keystore == nil {
422
- return nil
423
- }
419
// Set the KeyProvider as a garbage collection function for the
425
- // keystore. Periodically purge the Keystore from all its keys and
420
+ // keystore. Periodically purge the KeyStore from all its keys and
421
// replace them with the keys that needs to be reprovided, coming from
422
// the KeyChanFunc. So far, this is the less worse way to remove CIDs
423
// that shouldn't be reprovided from the provider's state.
429
- if err := syncKeystore(ctx); err != nil {
424
+ if err := syncKeyStore(ctx); err != nil {
425
return err
426
}
427
@@ -443,7 +438,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
438
case <-gcCtx.Done():
439
return
440
case <-ticker.C:
446
- if err := syncKeystore(gcCtx); err != nil {
441
+ if err := syncKeyStore(gcCtx); err != nil {
442
logger.Errorw("provider keystore sync", "err", err)
443
}
444
}
@@ -452,11 +447,7 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
447
return nil
448
},
449
OnStop: func(ctx context.Context) error {
455
- if in.Provider == nil || in.Keystore == nil {
456
- return nil
457
- }
450
if cancel != nil {
459
- // Cancel Keystore garbage collection loop
451
cancel()
452
}
453
select {
@@ -464,43 +455,45 @@ func SweepingProviderOpt(cfg *config.Config) fx.Option {
455
case <-ctx.Done():
456
return ctx.Err()
457
}
467
-
468
- // Keystore state isn't be persisted across restarts.
469
- return in.Keystore.Empty(ctx)
458
+ // KeyStore state isn't be persisted across restarts.
459
+ if err := in.KeyStore.Empty(ctx); err != nil {
460
+ return err
461
+ }
462
+ return in.KeyStore.Close()
463
},
464
})
465
})
466
467
return fx.Options(
468
sweepingReprovider,
476
- initKeystore,
469
+ initKeyStore,
470
)
471
}
472
473
// ONLINE/OFFLINE
474
482
-// OnlineProviders groups units managing provider routing records online
475
+// OnlineProviders groups units managing provide routing records online
476
func OnlineProviders(provide bool, cfg *config.Config) fx.Option {
477
if !provide {
478
return OfflineProviders()
479
}
480
488
- providerStrategy := cfg.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy)
481
+ providerStrategy := cfg.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
482
490
- strategyFlag := config.ParseReproviderStrategy(providerStrategy)
483
+ strategyFlag := config.ParseProvideStrategy(providerStrategy)
484
if strategyFlag == 0 {
492
- return fx.Error(fmt.Errorf("unknown reprovider strategy %q", providerStrategy))
485
+ return fx.Error(fmt.Errorf("provider: unknown strategy %q", providerStrategy))
486
}
487
488
opts := []fx.Option{
489
fx.Provide(setReproviderKeyProvider(providerStrategy)),
490
}
498
- if cfg.Reprovider.Sweep.Enabled.WithDefault(config.DefaultReproviderSweepEnabled) {
491
+ if cfg.Provide.DHT.SweepEnabled.WithDefault(config.DefaultProvideDHTSweepEnabled) {
492
opts = append(opts, SweepingProviderOpt(cfg))
493
} else {
501
- reprovideInterval := cfg.Reprovider.Interval.WithDefault(config.DefaultReproviderInterval)
494
+ reprovideInterval := cfg.Provide.DHT.Interval.WithDefault(config.DefaultProvideDHTInterval)
495
acceleratedDHTClient := cfg.Routing.AcceleratedDHTClient.WithDefault(config.DefaultAcceleratedDHTClient)
503
- provideWorkerCount := int(cfg.Provider.WorkerCount.WithDefault(config.DefaultProviderWorkerCount))
496
+ provideWorkerCount := int(cfg.Provide.DHT.MaxWorkers.WithDefault(config.DefaultProvideDHTMaxWorkers))
497
498
opts = append(opts, LegacyProviderOpt(reprovideInterval, providerStrategy, acceleratedDHTClient, provideWorkerCount))
499
}
@@ -508,7 +501,7 @@ func OnlineProviders(provide bool, cfg *config.Config) fx.Option {
501
return fx.Options(opts...)
502
}
503
511
-// OfflineProviders groups units managing provider routing records offline
504
+// OfflineProviders groups units managing provide routing records offline
505
func OfflineProviders() fx.Option {
506
return fx.Provide(func() DHTProvider {
507
return &NoopProvider{}
@@ -519,11 +512,11 @@ func mfsProvider(mfsRoot *mfs.Root, fetcher fetcher.Factory) provider.KeyChanFun
512
return func(ctx context.Context) (<-chan cid.Cid, error) {
513
err := mfsRoot.FlushMemFree(ctx)
514
if err != nil {
522
- return nil, fmt.Errorf("error flushing mfs, cannot provide MFS: %w", err)
515
+ return nil, fmt.Errorf("provider: error flushing MFS, cannot provide MFS: %w", err)
516
}
517
rootNode, err := mfsRoot.GetDirectory().GetNode()
518
if err != nil {
526
- return nil, fmt.Errorf("error loading mfs root, cannot provide MFS: %w", err)
519
+ return nil, fmt.Errorf("provider: error loading MFS root, cannot provide MFS: %w", err)
520
}
521
522
kcf := provider.NewDAGProvider(rootNode.Cid(), fetcher)
@@ -543,7 +536,7 @@ type provStrategyIn struct {
536
537
type provStrategyOut struct {
538
fx.Out
546
- ProvidingStrategy config.ReproviderStrategy
539
+ ProvidingStrategy config.ProvideStrategy
540
ProvidingKeyChanFunc provider.KeyChanFunc
541
}
542
@@ -553,18 +546,18 @@ type provStrategyOut struct {
546
// - "pinned": All pinned content (roots + children)
547
// - "mfs": Only MFS content
548
// - "all": all blocks
556
-func createKeyProvider(strategyFlag config.ReproviderStrategy, in provStrategyIn) provider.KeyChanFunc {
549
+func createKeyProvider(strategyFlag config.ProvideStrategy, in provStrategyIn) provider.KeyChanFunc {
550
switch strategyFlag {
558
- case config.ReproviderStrategyRoots:
551
+ case config.ProvideStrategyRoots:
552
return provider.NewBufferedProvider(dspinner.NewPinnedProvider(true, in.Pinner, in.OfflineIPLDFetcher))
560
- case config.ReproviderStrategyPinned:
553
+ case config.ProvideStrategyPinned:
554
return provider.NewBufferedProvider(dspinner.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher))
562
- case config.ReproviderStrategyPinned | config.ReproviderStrategyMFS:
555
+ case config.ProvideStrategyPinned | config.ProvideStrategyMFS:
556
return provider.NewPrioritizedProvider(
557
provider.NewBufferedProvider(dspinner.NewPinnedProvider(false, in.Pinner, in.OfflineIPLDFetcher)),
558
mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher),
559
)
567
- case config.ReproviderStrategyMFS:
560
+ case config.ProvideStrategyMFS:
561
return mfsProvider(in.MFSRoot, in.OfflineUnixFSFetcher)
562
default: // "all", "", "flat" (compat)
563
return in.Blockstore.AllKeysChan
@@ -616,7 +609,7 @@ func handleStrategyChange(strategy string, provider DHTProvider, ds datastore.Da
609
return
610
}
611
619
- logger.Infow("Reprovider.Strategy changed, clearing provide queue", "previous", previous, "current", strategy)
612
+ logger.Infow("Provide.Strategy changed, clearing provide queue", "previous", previous, "current", strategy)
613
provider.Clear()
614
615
if err := persistStrategy(ctx, strategy, ds); err != nil {
@@ -625,7 +618,7 @@ func handleStrategyChange(strategy string, provider DHTProvider, ds datastore.Da
618
}
619
620
func setReproviderKeyProvider(strategy string) func(in provStrategyIn) provStrategyOut {
628
- strategyFlag := config.ParseReproviderStrategy(strategy)
621
+ strategyFlag := config.ParseProvideStrategy(strategy)
622
623
return func(in provStrategyIn) provStrategyOut {
624
// Create the appropriate key provider based on strategy
core/node/storage.go
+2
-2
@@ -41,8 +41,8 @@ func BaseBlockstoreCtor(
41
// Important: Provide calls from blockstore are intentionally BLOCKING.
42
// The Provider implementation (not the blockstore) should handle concurrency/queuing.
43
// This avoids spawning unbounded goroutines for concurrent block additions.
44
- strategyFlag := config.ParseReproviderStrategy(providingStrategy)
45
- if strategyFlag&config.ReproviderStrategyAll != 0 {
44
+ strategyFlag := config.ParseProvideStrategy(providingStrategy)
45
+ if strategyFlag&config.ProvideStrategyAll != 0 {
46
opts = append(opts, blockstore.Provider(prov))
47
}
48
docs/changelogs/v0.38.md
+29
-4
@@ -10,24 +10,49 @@ This release was brought to you by the [Shipyard](https://ipshipyard.com/) team.
10
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
+ - [🚀 Repository migration: simplified provide configuration](#-repository-migration-simplified-provide-configuration)
14
- [🧹 Experimental Sweeping DHT Provider](#-experimental-sweeping-dht-provider)
15
+ - [📊 Exposed DHT metrics](#-exposed-dht-metrics)
16
+ - [🚨 Improved gateway error pages with diagnostic tools](#-improved-gateway-error-pages-with-diagnostic-tools)
17
+ - [🛠️ Identity CID size enforcement and `ipfs files write` fixes](#-identity-cid-size-enforcement-and-ipfs-files-write-fixes)
18
- [📦️ Important dependency updates](#-important-dependency-updates)
19
- [📝 Changelog](#-changelog)
20
- [👨👩👧👦 Contributors](#-contributors)
21
22
### Overview
23
24
+Kubo 0.38.0 simplifies content announcement configuration, introduces an experimental sweeping DHT provider for efficient large-scale operations, and includes various performance improvements.
25
+
26
### 🔦 Highlights
27
28
+#### 🚀 Repository migration: simplified provide configuration
29
+
30
+This release migrates the repository from version 17 to version 18, simplifying how you configure content announcements.
31
+
32
+The old `Provider` and `Reprovider` sections are now combined into a single [`Provide`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provide) section. Your existing settings are automatically migrated - no manual changes needed.
33
+
34
+**Migration happens automatically** when you run `ipfs daemon --migrate`. For manual migration: `ipfs repo migrate --to=18`.
35
+
36
+Read more about the new system below.
37
+
38
#### 🧹 Experimental Sweeping DHT Provider
39
24
-An experimental alternative to both the default DHT provider and the resource-intensive [accelerated DHT client](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient) is now available. When enabled via [`Reprovider.Sweep.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovidersweep), this sweeping DHT provider explores keyspace regions instead of providing keys one-by-one.
40
+A new experimental DHT provider is available as an alternative to both the default provider and the resource-intensive [accelerated DHT client](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingaccelerateddhtclient). Enable it via [`Provide.DHT.SweepEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedhtssweepenabled).
41
+
42
+> [!NOTE]
43
+> This feature is experimental and opt-in. In the future, it will become the default and replace the legacy system. Some commands like `ipfs stats provide` and `ipfs routing provide` are not yet available with sweep mode. Run `ipfs provide --help` for alternatives.
44
+
45
+**How it works:** Instead of providing keys one-by-one, the sweep provider systematically explores DHT keyspace regions in batches.
46
+
47
+**Benefits for large-scale operations:** Handles hundreds of thousands of CIDs with reduced memory and network connections, spreads operations evenly to eliminate resource spikes, maintains state across restarts through persistent keystore, and provides better metrics visibility.
48
+
49
+**Monitoring and debugging:** Legacy mode (`SweepEnabled=false`) tracks `provider_reprovider_provide_count` and `provider_reprovider_reprovide_count`, while sweep mode (`SweepEnabled=true`) tracks `total_provide_count_total`. Enable debug logging with `GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug` to see detailed logs from either system.
50
26
-This aims to help both large and small storage providers efficiently advertise hundreds of thousands of CIDs by batching operations and spreading work evenly over time.
51
+For configuration details, see [`Provide.DHT`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providedht). For metrics documentation, see [Provide metrics](https://github.com/ipfs/kubo/blob/master/docs/metrics.md#provide).
52
28
-**Note:** While this feature is experimental, some commands like `ipfs stats provide` and manual reprovide (`ipfs routing provide`) are not available. Run `ipfs provide --help` for alternative commands.
53
+#### 📊 Exposed DHT metrics
54
30
-For configuration options and more details, see [`Reprovider.Sweep`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reprovidersweep) in the config documentation.
55
+Kubo now exposes DHT metrics from go-libp2p-kad-dht, including `total_provide_count_total` for sweep provider operations and RPC metrics prefixed with `rpc_inbound_` and `rpc_outbound_` for DHT message traffic. See [Kubo metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md) for details.
56
57
#### 🚨 Improved gateway error pages with diagnostic tools
58
docs/config.md
+269
-226
@@ -125,6 +125,18 @@ config file at runtime.
125
- [`Pinning.RemoteServices: Policies.MFS.Enabled`](#pinningremoteservices-policiesmfsenabled)
126
- [`Pinning.RemoteServices: Policies.MFS.PinName`](#pinningremoteservices-policiesmfspinname)
127
- [`Pinning.RemoteServices: Policies.MFS.RepinInterval`](#pinningremoteservices-policiesmfsrepininterval)
128
+ - [`Provide`](#provide)
129
+ - [`Provide.Enabled`](#provideenabled)
130
+ - [`Provide.Strategy`](#providestrategy)
131
+ - [`Provide.DHT`](#providedht)
132
+ - [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers)
133
+ - [`Provide.DHT.Interval`](#providedhtinterval)
134
+ - [`Provide.DHT.SweepEnabled`](#providedhtssweepenabled)
135
+ - [`Provide.DHT.DedicatedPeriodicWorkers`](#providedhtdedicatedperiodicworkers)
136
+ - [`Provide.DHT.DedicatedBurstWorkers`](#providedhtdedicatedburstworkers)
137
+ - [`Provide.DHT.MaxProvideConnsPerWorker`](#providedhtmaxprovideconnsperworker)
138
+ - [`Provide.DHT.KeyStoreBatchSize`](#providedhtkeystorebatchsize)
139
+ - [`Provide.DHT.OfflineDelay`](#providedhtofflinedelay)
140
- [`Provider`](#provider)
141
- [`Provider.Enabled`](#providerenabled)
142
- [`Provider.Strategy`](#providerstrategy)
@@ -139,8 +151,7 @@ config file at runtime.
151
- [`Peering.Peers`](#peeringpeers)
152
- [`Reprovider`](#reprovider)
153
- [`Reprovider.Interval`](#reproviderinterval)
142
- - [`Reprovider.Strategy`](#reproviderstrategy)
143
- - [`Reprovider.Sweep`](#reprovidersweep)
154
+ - [`Reprovider.Strategy`](#providestrategy)
155
- [`Routing`](#routing)
156
- [`Routing.Type`](#routingtype)
157
- [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient)
@@ -1368,7 +1379,7 @@ Below is a list of the most common gateway setups.
1379
}
1380
}'
1381
```
1371
- - **Performance:** consider running with `Routing.AcceleratedDHTClient=true` and either `Provider.Enabled=false` (avoid providing newly retrieved blocks) or `Provider.WorkerCount=0` (provide as fast as possible, at the cost of increased load)
1382
+ - **Performance:** Consider enabling `Routing.AcceleratedDHTClient=true` to improve content routing lookups. Separately, gateway operators should decide if the gateway node should also co-host and provide (announce) fetched content to the DHT. If providing content, enable `Provide.DHT.SweepEnabled=true` for efficient announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`. For a read-only gateway that doesn't announce content, use `Provide.Enabled=false`.
1383
- **Backward-compatible:** this feature enables automatic redirects from content paths to subdomains:
1384
1385
`http://dweb.link/ipfs/{cid}` → `http://{cid}.ipfs.dweb.link`
@@ -1393,7 +1404,7 @@ Below is a list of the most common gateway setups.
1404
}
1405
}'
1406
```
1396
- - **Performance:** when running an open, recursive gateway consider running with `Routing.AcceleratedDHTClient=true` and either `Provider.Enabled=false` (avoid providing newly retrieved blocks) or `Provider.WorkerCount=0` (provide as fast as possible, at the cost of increased load)
1407
+ - **Performance:** Consider enabling `Routing.AcceleratedDHTClient=true` to improve content routing lookups. When running an open, recursive gateway, decide if the gateway should also co-host and provide (announce) fetched content to the DHT. If providing content, enable `Provide.DHT.SweepEnabled=true` for efficient announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`. For a read-only gateway that doesn't announce content, use `Provide.Enabled=false`.
1408
1409
* Public [DNSLink](https://dnslink.io/) gateway resolving every hostname passed in `Host` header.
1410
```console
@@ -1833,36 +1844,113 @@ Default: `"5m"`
1844
1845
Type: `duration`
1846
1836
-## `Provider`
1837
-
1838
-Configuration applied to the initial one-time announcement of fresh CIDs
1839
-created with `ipfs add`, `ipfs files`, `ipfs dag import`, `ipfs block|dag put`
1840
-commands.
1847
+## `Provide`
1848
1842
-For periodical DHT reprovide settings, see [`Reprovide.*`](#reprovider).
1849
+Configures CID announcements to the routing system, including both immediate
1850
+announcements for new content (provide) and periodic re-announcements
1851
+(reprovide) on systems that require it, like Amino DHT. While designed to support
1852
+multiple routing systems in the future, the current default configuration only supports providing to the Amino DHT.
1853
1844
-### `Provider.Enabled`
1854
+### `Provide.Enabled`
1855
1846
-Controls whether Kubo provider and reprovide systems are enabled.
1856
+Controls whether Kubo provide and reprovide systems are enabled.
1857
1858
> [!CAUTION]
1849
-> Disabling this, will disable BOTH `Provider` system for new CIDs
1850
-> and the periodical reprovide ([`Reprovider.Interval`](#reprovider)) of old CIDs.
1859
+> Disabling this will prevent other nodes from discovering your content.
1860
+> Your node will stop announcing data to the routing system, making it
1861
+> inaccessible unless peers connect to you directly.
1862
1863
Default: `true`
1864
1865
Type: `flag`
1866
1856
-### `Provider.Strategy`
1867
+### `Provide.Strategy`
1868
1858
-Legacy, not used at the moment, see [`Reprovider.Strategy`](#reproviderstrategy) instead.
1869
+Tells the provide system what should be announced. Valid strategies are:
1870
1860
-### `Provider.WorkerCount`
1871
+- `"all"` - announce all CIDs of stored blocks
1872
+- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
1873
+ - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
1874
+- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
1875
+ - **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks.
1876
+ It makes sense only for use cases where the entire DAG is fetched in full,
1877
+ and a graceful resume does not have to be guaranteed: the lack of child
1878
+ announcements means an interrupted retrieval won't be able to find
1879
+ providers for the missing block in the middle of a file, unless the peer
1880
+ happens to already be connected to a provider and asks for child CID over
1881
+ bitswap.
1882
+- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`)
1883
+ - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced.
1884
+- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
1885
+ - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
1886
+ - Order: first `pinned` and then the locally available part of `mfs`.
1887
+
1888
+**Strategy changes automatically clear the provide queue.** When you change `Provide.Strategy` and restart Kubo, the provide queue is automatically cleared to ensure only content matching your new strategy is announced. You can also manually clear the queue using `ipfs provide clear`.
1889
+
1890
+**Memory requirements:**
1891
+
1892
+- Reproviding larger pinsets using the `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million CIDs for reproviding to the Amino DHT.
1893
+- This is due to the use of a buffered provider, which loads all CIDs into memory to avoid holding a lock on the entire pinset during the reprovide cycle.
1894
+
1895
+Default: `"all"`
1896
+
1897
+Type: `optionalString` (unset for the default)
1898
+
1899
+### `Provide.DHT`
1900
+
1901
+Configuration for providing data to Amino DHT peers.
1902
+
1903
+#### Monitoring Provide Operations
1904
+
1905
+You can monitor the effectiveness of your provide configuration through metrics exposed at the Prometheus endpoint: `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`).
1906
+
1907
+Different metrics are available depending on whether you use legacy mode (`SweepEnabled=false`) or sweep mode (`SweepEnabled=true`). See [Provide metrics documentation](https://github.com/ipfs/kubo/blob/master/docs/metrics.md#provide) for details.
1908
+
1909
+To enable detailed debug logging for both providers, set:
1910
+```sh
1911
+GOLOG_LOG_LEVEL=error,provider=debug,dht/provider=debug
1912
+```
1913
+- `provider=debug` enables generic logging (legacy provider and any non-dht operations)
1914
+- `dht/provider=debug` enables logging for the sweep provider
1915
+
1916
+#### `Provide.DHT.Interval`
1917
+
1918
+Sets how often to re-announce content to the DHT. Provider records on Amino DHT
1919
+expire after [`amino.DefaultProvideValidity`](https://github.com/libp2p/go-libp2p-kad-dht/blob/v0.34.0/amino/defaults.go#L40-L43),
1920
+also known as Provider Record Expiration Interval.
1921
1862
-Sets the maximum number of _concurrent_ DHT provide operations (announcement of new CIDs).
1922
+An interval of about half the expiration window ensures provider records
1923
+are refreshed well before they expire. This keeps your content continuously
1924
+discoverable accounting for network churn without overwhelming the network with too frequent announcements.
1925
1864
-[`Reprovider`](#reprovider) operations do **not** count against this limit.
1865
-A value of `0` allows an unlimited number of provide workers.
1926
+- If unset, it uses the implicit safe default.
1927
+- If set to the value `"0"` it will disable content reproviding to DHT.
1928
+
1929
+> [!CAUTION]
1930
+> Disabling this will prevent other nodes from discovering your content via the DHT.
1931
+> Your node will stop announcing data to the DHT, making it
1932
+> inaccessible unless peers connect to you directly. Since provider
1933
+> records expire after `amino.DefaultProvideValidity`, your content will become undiscoverable
1934
+> after this period.
1935
+
1936
+Default: `22h`
1937
+
1938
+Type: `optionalDuration` (unset for the default)
1939
+
1940
+#### `Provide.DHT.MaxWorkers`
1941
+
1942
+Sets the maximum number of _concurrent_ DHT provide operations.
1943
+
1944
+**When `Provide.DHT.SweepEnabled` is false (legacy mode):**
1945
+- Controls NEW CID announcements only
1946
+- Reprovide operations do **not** count against this limit
1947
+- A value of `0` allows unlimited provide workers
1948
+
1949
+**When `Provide.DHT.SweepEnabled` is true:**
1950
+- Controls the total worker pool for both provide and reprovide operations
1951
+- Workers are split between periodic reprovides and burst provides
1952
+- Use a positive value to control resource usage
1953
+- See [`DedicatedPeriodicWorkers`](#providedhtdedicatedperiodicworkers) and [`DedicatedBurstWorkers`](#providedhtdedicatedburstworkers) for task allocation
1954
1955
If the [accelerated DHT client](#routingaccelerateddhtclient) is enabled, each
1956
provide operation opens ~20 connections in parallel. With the standard DHT
@@ -1873,8 +1961,9 @@ connections this setting can generate.
1961
1962
> [!CAUTION]
1963
> For nodes without strict connection limits that need to provide large volumes
1876
-> of content immediately, we recommend enabling the `Routing.AcceleratedDHTClient` and
1877
-> setting `Provider.WorkerCount` to `0` (unlimited).
1964
+> of content, we recommend first trying `Provide.DHT.SweepEnabled=true` for efficient
1965
+> announcements. If announcements are still not fast enough, adjust `Provide.DHT.MaxWorkers`.
1966
+> As a last resort, consider enabling `Routing.AcceleratedDHTClient=true` but be aware that it is very resource hungry.
1967
>
1968
> At the same time, mind that raising this value too high may lead to increased load.
1969
> Proceed with caution, ensure proper hardware and networking are in place.
@@ -1883,6 +1972,154 @@ Default: `16`
1972
1973
Type: `optionalInteger` (non-negative; `0` means unlimited number of workers)
1974
1975
+#### `Provide.DHT.SweepEnabled`
1976
+
1977
+Whether Provide Sweep is enabled. If not enabled, the legacy
1978
+[`boxo/provider`](https://github.com/ipfs/boxo/tree/main/provider) is used for
1979
+both provides and reprovides.
1980
+
1981
+Provide Sweep is a resource efficient technique for advertising content to
1982
+the Amino DHT swarm. The Provide Sweep module tracks the keys that should be periodically reprovided in
1983
+the `KeyStore`. It splits the keys into DHT keyspace regions by proximity (XOR
1984
+distance), and schedules when reprovides should happen in order to spread the
1985
+reprovide operation over time to avoid a spike in resource utilization. It
1986
+basically sweeps the keyspace _from left to right_ over the
1987
+[`Provide.DHT.Interval`](#providedhtinterval) time period, and reprovides keys
1988
+matching to the visited keyspace region.
1989
+
1990
+Provide Sweep aims at replacing the inefficient legacy `boxo/provider`
1991
+module, and is currently opt-in. You can compare the effectiveness of sweep mode vs legacy mode by monitoring the appropriate metrics (see [Monitoring Provide Operations](#monitoring-provide-operations) above).
1992
+
1993
+Whenever new keys should be advertised to the Amino DHT, `kubo` calls
1994
+`StartProviding()`, triggering an initial `provide` operation for the given
1995
+keys. The keys will be added to the `KeyStore` tracking which keys should be
1996
+reprovided and when they should be reprovided. Calling `StopProviding()`
1997
+removes the keys from the `KeyStore`. However, it is currently tricky for
1998
+`kubo` to detect when a key should stop being advertised. Hence, `kubo` will
1999
+periodically refresh the `KeyStore` at each [`Provide.DHT.Interval`](#providedhtinterval)
2000
+by providing it a channel of all the keys it is expected to contain according
2001
+to the [`Provide.Strategy`](#providestrategy). During this operation,
2002
+all keys in the `Keystore` are purged, and only the given ones remain scheduled.
2003
+
2004
+> [!NOTE]
2005
+> This feature is opt-in for now, but will become the default in a future release.
2006
+> Eventually, this configuration flag will be removed once the feature is stable.
2007
+
2008
+Default: `false`
2009
+
2010
+Type: `flag`
2011
+
2012
+
2013
+#### `Provide.DHT.DedicatedPeriodicWorkers`
2014
+
2015
+Number of workers dedicated to periodic keyspace region reprovides. Only applies when `Provide.DHT.SweepEnabled` is true.
2016
+
2017
+Among the [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers), this
2018
+number of workers will be dedicated to the periodic region reprovide only. The sum of
2019
+`DedicatedPeriodicWorkers` and `DedicatedBurstWorkers` should not exceed `MaxWorkers`.
2020
+Any remaining workers (MaxWorkers - DedicatedPeriodicWorkers - DedicatedBurstWorkers)
2021
+form a shared pool that can be used for either type of work as needed.
2022
+
2023
+Default: `2`
2024
+
2025
+Type: `optionalInteger` (`0` means there are no dedicated workers, but the
2026
+operation can be performed by free non-dedicated workers)
2027
+
2028
+#### `Provide.DHT.DedicatedBurstWorkers`
2029
+
2030
+Number of workers dedicated to burst provides. Only applies when `Provide.DHT.SweepEnabled` is true.
2031
+
2032
+Burst provides are triggered by:
2033
+- Manual provide commands (`ipfs routing provide`)
2034
+- New content matching your `Provide.Strategy` (blocks from `ipfs add`, bitswap, or trustless gateway requests)
2035
+- Catch-up reprovides after being disconnected/offline for a while
2036
+
2037
+Having dedicated burst workers ensures that bulk operations (like adding many CIDs
2038
+or reconnecting to the network) don't delay regular periodic reprovides, and vice versa.
2039
+
2040
+Among the [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers), this
2041
+number of workers will be dedicated to burst provides only. In addition to
2042
+these, if there are available workers in the pool, they can also be used for
2043
+burst provides.
2044
+
2045
+Default: `1`
2046
+
2047
+Type: `optionalInteger` (`0` means there are no dedicated workers, but the
2048
+operation can be performed by free non-dedicated workers)
2049
+
2050
+#### `Provide.DHT.MaxProvideConnsPerWorker`
2051
+
2052
+Maximum number of connections that a single worker can use to send provider
2053
+records over the network.
2054
+
2055
+When reproviding CIDs corresponding to a keyspace region, the reprovider must
2056
+send a provider record to the 20 closest peers to the CID (in XOR distance) for
2057
+each CID belonging to this keyspace region.
2058
+
2059
+The reprovider opens a connection to a peer from that region, sends it all its
2060
+allocated provider records. Once done, it opens a connection to the next peer
2061
+from that keyspace region until all provider records are assigned.
2062
+
2063
+This option defines how many such connections can be open concurrently by a
2064
+single worker.
2065
+
2066
+Default: `16`
2067
+
2068
+Type: `optionalInteger` (non-negative)
2069
+
2070
+#### `Provide.DHT.KeyStoreBatchSize`
2071
+
2072
+During the garbage collection, all keys stored in the KeyStore are removed, and
2073
+the keys are streamed from a channel to fill the KeyStore again with up-to-date
2074
+keys. Since a high number of CIDs to reprovide can easily fill up the memory,
2075
+keys are read and written in batches to optimize for memory usage.
2076
+
2077
+This option defines how many multihashes should be contained within a batch. A
2078
+multihash is usually represented by 34 bytes.
2079
+
2080
+Default: `16384` (~544 KiB per batch)
2081
+
2082
+Type: `optionalInteger` (non-negative)
2083
+
2084
+#### `Provide.DHT.OfflineDelay`
2085
+
2086
+The `SweepingProvider` has 3 states: `ONLINE`, `DISCONNECTED` and `OFFLINE`. It
2087
+starts `OFFLINE`, and as the node bootstraps, it changes its state to `ONLINE`.
2088
+
2089
+When the provider loses connection to all DHT peers, it switches to the
2090
+`DISCONNECTED` state. In this state, new provides will be added to the provide
2091
+queue, and provided as soon as the node comes back online.
2092
+
2093
+After a node has been `DISCONNECTED` for `OfflineDelay`, it goes to `OFFLINE`
2094
+state. When `OFFLINE`, the provider drops the provide queue, and returns errors
2095
+to new provide requests. However, when `OFFLINE` the provider still adds the
2096
+keys to its state, so keys will eventually be provided in the
2097
+[`Provide.DHT.Interval`](#providedhtinterval) after the provider comes back
2098
+`ONLINE`.
2099
+
2100
+Default: `2h`
2101
+
2102
+Type: `optionalDuration`
2103
+
2104
+## `Provider`
2105
+
2106
+### `Provider.Enabled`
2107
+
2108
+**REMOVED**
2109
+
2110
+Replaced with [`Provide.Enabled`](#provideenabled).
2111
+
2112
+### `Provider.Strategy`
2113
+
2114
+**REMOVED**
2115
+
2116
+This field was unused. Use [`Provide.Strategy`](#providestrategy) instead.
2117
+
2118
+### `Provider.WorkerCount`
2119
+
2120
+**REMOVED**
2121
+
2122
+Replaced with [`Provide.DHT.MaxWorkers`](#providedhtmaxworkers).
2123
## `Pubsub`
2124
2125
**DEPRECATED**: See [#9717](https://github.com/ipfs/kubo/issues/9717)
@@ -2050,212 +2287,15 @@ Type: `array[peering]`
2287
2288
### `Reprovider.Interval`
2289
2053
-Sets the time between rounds of reproviding local content to the routing
2054
-system.
2055
-
2056
-- If unset, it uses the implicit safe default.
2057
-- If set to the value `"0"` it will disable content reproviding.
2058
-
2059
-Note: disabling content reproviding will result in other nodes on the network
2060
-not being able to discover that you have the objects that you have. If you want
2061
-to have this disabled and keep the network aware of what you have, you must
2062
-manually announce your content periodically or run your own routing system
2063
-and convince users to add it to [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
2064
-
2065
-> [!CAUTION]
2066
-> To maintain backward-compatibility, setting `Reprovider.Interval=0` will also disable Provider system (equivalent of `Provider.Enabled=false`)
2067
-
2068
-Default: `22h` (`DefaultReproviderInterval`)
2290
+**REMOVED**
2291
2070
-Type: `optionalDuration` (unset for the default)
2292
+Replaced with [`Provide.DHT.Interval`](#providedhtinterval).
2293
2294
### `Reprovider.Strategy`
2295
2074
-Tells reprovider what should be announced. Valid strategies are:
2075
-
2076
-- `"all"` - announce all CIDs of stored blocks
2077
-- `"pinned"` - only announce recursively pinned CIDs (`ipfs pin add -r`, both roots and child blocks)
2078
- - Order: root blocks of direct and recursive pins are announced first, then the child blocks of recursive pins
2079
-- `"roots"` - only announce the root block of explicitly pinned CIDs (`ipfs pin add`)
2080
- - **⚠️ BE CAREFUL:** node with `roots` strategy will not announce child blocks.
2081
- It makes sense only for use cases where the entire DAG is fetched in full,
2082
- and a graceful resume does not have to be guaranteed: the lack of child
2083
- announcements means an interrupted retrieval won't be able to find
2084
- providers for the missing block in the middle of a file, unless the peer
2085
- happens to already be connected to a provider and ask for child CID over
2086
- bitswap.
2087
-- `"mfs"` - announce only the local CIDs that are part of the MFS (`ipfs files`)
2088
- - Note: MFS is lazy-loaded. Only the MFS blocks present in local datastore are announced.
2089
-- `"pinned+mfs"` - a combination of the `pinned` and `mfs` strategies.
2090
- - **ℹ️ NOTE:** This is the suggested strategy for users who run without GC and don't want to provide everything in cache.
2091
- - Order: first `pinned` and then the locally available part of `mfs`.
2092
-
2093
-**Strategy changes automatically clear the provide queue.** When you change `Reprovider.Strategy` and restart Kubo, the provide queue is automatically cleared to ensure only content matching your new strategy is announced. You can also manually clear the queue using `ipfs provide clear`.
2094
-
2095
-**Memory requirements:**
2096
-
2097
-- Reproviding larger pinsets using the `mfs`, `pinned`, `pinned+mfs` or `roots` strategies requires additional memory, with an estimated ~1 GiB of RAM per 20 million items for reproviding to the Amino DHT.
2098
-- This is due to the use of a buffered provider, which avoids holding a lock on the entire pinset during the reprovide cycle.
2099
-
2100
-Default: `"all"`
2101
-
2102
-Type: `optionalString` (unset for the default)
2103
-
2104
-### Reprovider.Sweep
2105
-
2106
-Reprovider Sweep is a resource efficient technique for advertising content to
2107
-the Amino DHT swarm.
2108
-
2109
-The Reprovider module tracks the keys that should be periodically reprovided in
2110
-the `KeyStore`. It splits the keys into DHT keyspace regions by proximity (XOR
2111
-distance), and schedules when reprovides should happen in order to spread the
2112
-reprovide operation over time to avoid a spike in resource utilization. It
2113
-basically sweeps the keyspace _from left to right_ over the
2114
-[`Reprovider.Interval`](#reproviderinterval) time period, and reprovides keys
2115
-matching to the visited keyspace region.
2116
-
2117
-Reprovider Sweep aims at replacing the inefficient legacy `boxo/provider`
2118
-module, and is currently opt-in.
2119
-
2120
-Whenever new keys should be advertised to the Amino DHT, `kubo` calls
2121
-`StartProviding()`, triggering an initial `provide` operation for the given
2122
-keys. The keys will be added to the `KeyStore` tracking which keys should be
2123
-reprovided and when they should be reprovided. Calling `StopProviding()`
2124
-removes the keys from the `KeyStore`. However, it is currently tricky for
2125
-`kubo` to detect when a key should stop being advertised. Hence, `kubo` will
2126
-periodically refresh the `KeyStore` at each [`Reprovider.Interval`](#reproviderinterval)
2127
-by providing it a channel of all the keys it is expected to contain according
2128
-to the [`Reprovider.Strategy`](#reproviderstrategy). During this operation,
2129
-all keys in the `Keystore` are purged, and only the given ones remain scheduled.
2130
-
2131
-#### Reprovider.Sweep.Enabled
2132
-
2133
-Whether Reprovider Sweep is enabled. If not enabled, the
2134
-[`boxo/provider`](https://github.com/ipfs/boxo/tree/main/provider) is used for
2135
-both provides and reprovides.
2136
-
2137
-Default: `false`
2138
-
2139
-Type: `flag`
2140
-
2141
-#### Reprovider.Sweep.MaxWorkers
2142
-
2143
-The maximum number of workers used by the `SweepingReprovider` to provide and
2144
-reprovide CIDs to the DHT swarm.
2145
-
2146
-A worker performs Kademlia `GetClosestPeers` operations (max 1 at a time) to
2147
-explore a region of the DHT keyspace, and then sends provider records to the
2148
-nodes from that keyspace region. `GetClosestPeers` is capped to `10` concurrent
2149
-connections [`amino` DHT
2150
-defaults](https://github.com/libp2p/go-libp2p-kad-dht/blob/master/amino/defaults.go).
2151
-The number of simultaneous connections used to send provider records is defined
2152
-by
2153
-[`Reprovider.Sweep.MaxProvideConnsPerWorker`](#reprovidersweepmaxprovideconnsperworker).
2154
-
2155
-The workers are split between two tasks categories:
2156
-
2157
-1. Periodic reprovides (see
2158
- [`Reprovider.Sweep.DedicatedPeriodicWorkers`](#reprovidersweepdedicatedperiodicworkers))
2159
-2. Burst provides (see
2160
- [`Reprovider.Sweep.DedicatedBurstWorkers`](#reprovidersweepdedicatedburstworkers))
2161
-
2162
-[`Reprovider.Sweep.DedicatedPeriodicWorkers`](#reprovidersweepdedicatedperiodicworkers)
2163
-workers are allocated to the periodic reprovides only,
2164
-[`Reprovider.Sweep.DedicatedBurstWorkers`](#reprovidersweepdedicatedburstworkers)
2165
-workers are allocated to burst provides only, and the rest of
2166
-[`Reprovider.Sweep.MaxWorkers`](#reprovidersweepmaxworkers) can be used for
2167
-either task (first come, first served).
2168
-
2169
-Default: `4`
2170
-
2171
-Type: `optionalInteger` (non-negative)
2172
-
2173
-#### Reprovider.Sweep.DedicatedPeriodicWorkers
2174
-
2175
-Number of workers dedicated to periodic keyspace region reprovides.
2176
-
2177
-Among the [`Reprovider.Sweep.MaxWorkers`](#reprovidersweepmaxworkers), this
2178
-number of workers will be dedicated to the periodic region reprovide only. In
2179
-addition to these, if there are available workers in the pool, they can also be
2180
-used for periodic reprovides.
2181
-
2182
-Default: `2`
2183
-
2184
-Type: `optionalInteger` (`0` means there are no dedicated workers, but the
2185
-operation can be performed by free non-dedicated workers)
2186
-
2187
-#### Reprovider.Sweep.DedicatedBurstWorkers
2188
-
2189
-Number of workers dedicated to burst provides.
2190
-
2191
-Burst provides are triggered when a new keys must be advertised to the DHT
2192
-immediately, or when a node comes back online and must catch up the reprovides
2193
-that should have happened while it was offline.
2194
-
2195
-Among the [`Reprovider.Sweep.MaxWorkers`](#reprovidersweepmaxworkers), this
2196
-number of workers will be dedicated to burst provides only. In addition to
2197
-these, if there are available workers in the pool, they can also be used for
2198
-burst provides.
2199
-
2200
-Default: `1`
2201
-
2202
-Type: `optionalInteger` (`0` means there are no dedicated workers, but the
2203
-operation can be performed by free non-dedicated workers)
2204
-
2205
-#### Reprovider.Sweep.MaxProvideConnsPerWorker
2206
-
2207
-Maximum number of connections that a single worker can use to send provider
2208
-records over the network.
2209
-
2210
-When reproviding CIDs corresponding to a keyspace region, the reprovider must
2211
-send a provider record to the 20 closest peers to the CID (in XOR distance) for
2212
-each CID belonging to this keyspace region.
2213
-
2214
-The reprovider opens a connection to a peer from that region, send it all its
2215
-allocated provider records. Once done, it opens a connection to the next peer
2216
-from that keyspace region until all provider records are assigned.
2217
-
2218
-This option defines how many such connections can be open concurrently by a
2219
-single worker.
2220
-
2221
-Default: `16`
2222
-
2223
-Type: `optionalInteger` (non-negative)
2224
-
2225
-#### Reprovider.Sweep.KeyStoreBatchSize
2226
-
2227
-During the garbage collection, all keys stored in the KeyStore are removed, and
2228
-the keys are streamed from a channel to fill the KeyStore again with up-to-date
2229
-keys. Since a high number of CIDs to reprovide can easily fill up the memory,
2230
-keys are read and written in batches to optimize for memory usage.
2231
-
2232
-This option defines how many multihashes should be contained within a batch. A
2233
-multihash is usually represented by 34 bytes.
2234
-
2235
-Default: `16384` (~544 KiB per batch)
2236
-
2237
-Type: `optionalInteger` (non-negative)
2238
-
2239
-#### Reprovider.Sweep.OfflineDelay
2240
-
2241
-The `SweepingProvider` has 3 states: `ONLINE`, `DISCONNECTED` and `OFFLINE`. It
2242
-starts `OFFLINE`, and as the node bootstraps, it changes its state to `ONLINE`.
2243
-
2244
-When the provider loses connection to all DHT peers, it switches to the
2245
-`DISCONNECTED` state. In this state, new provides will be added to the provide
2246
-queue, and provided as soon as the node comes back online.
2247
-
2248
-After a node has been `DISCONNECTED` for `OfflineDelay`, it goes to `OFFLINE`
2249
-state. When `OFFLINE`, the provider drops the provide queue, and returns errors
2250
-to new provide requests. However, when `OFFLINE` the provide still adds the
2251
-keys to its state, so keys will eventually be provided in the
2252
-[`Reprovider.Interval`](#reproviderinterval) after the provider comes back
2253
-`ONLINE`.
2254
-
2255
-Default: `2h`
2256
-
2257
-Type: `optionalDuration`
2296
+**REMOVED**
2297
2298
+Replaced with [`Provide.Strategy`](#providestrategy).
2299
## `Routing`
2300
2301
Contains options for content, peer, and IPNS routing mechanisms.
@@ -2334,6 +2374,9 @@ When it is enabled:
2374
- Client DHT operations (reads and writes) should complete much faster
2375
- The provider will now use a keyspace sweeping mode allowing to keep alive
2376
CID sets that are multiple orders of magnitude larger.
2377
+ - **Note:** For improved provide/reprovide operations specifically, consider using
2378
+ [`Provide.DHT.SweepEnabled`](#providedhtssweepenabled) instead, which offers similar
2379
+ benefits with lower resource consumption.
2380
- The standard Bucket-Routing-Table DHT will still run for the DHT server (if
2381
the DHT server is enabled). This means the classical routing table will
2382
still be used to answer other nodes.
@@ -2346,7 +2389,7 @@ When it is enabled:
2389
- The resource usage is not smooth as the client crawls the network in rounds and reproviding is similarly done in rounds
2390
- Users who previously had a lot of content but were unable to advertise it on the network will see an increase in
2391
egress bandwidth as their nodes start to advertise all of their CIDs into the network. If you have lots of data
2349
- entering your node that you don't want to advertise, then consider using [Reprovider Strategies](#reproviderstrategy)
2392
+ entering your node that you don't want to advertise, then consider using [Provide Strategies](#providestrategy)
2393
to reduce the number of CIDs that you are reproviding. Similarly, if you are running a node that deals mostly with
2394
short-lived temporary data (e.g. you use a separate node for ingesting data then for storing and serving it) then
2395
you may benefit from using [Strategic Providing](experimental-features.md#strategic-providing) to prevent advertising
@@ -3618,7 +3661,7 @@ Reduces daemon overhead on the system by disabling optional swarm services.
3661
3662
### `announce-off` profile
3663
3621
-Disables [Reprovider](#reprovider) system (and announcing to Amino DHT).
3664
+Disables [Provide](#provide) system (and announcing to Amino DHT).
3665
3666
> [!CAUTION]
3667
> The main use case for this is setups with manual Peering.Peers config.
@@ -3628,7 +3671,7 @@ Disables [Reprovider](#reprovider) system (and announcing to Amino DHT).
3671
3672
### `announce-on` profile
3673
3631
-(Re-)enables [Reprovider](#reprovider) system (reverts [`announce-off` profile](#announce-off-profile)).
3674
+(Re-)enables [Provide](#provide) system (reverts [`announce-off` profile](#announce-off-profile)).
3675
3676
### `legacy-cid-v0` profile
3677
docs/experimental-features.md
+1
-1
@@ -539,7 +539,7 @@ ipfs config --json Swarm.RelayClient.Enabled true
539
540
`Experimental.StrategicProviding` was removed in Kubo v0.35.
541
542
-Replaced by [`Provide.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providerenabled) and [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy).
542
+Replaced by [`Provide.Enabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#provideenabled) and [`Provide.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#providestrategy).
543
544
## GraphSync
545
docs/metrics.md
new
+118
@@ -0,0 +1,118 @@
1
+## Kubo metrics
2
+
3
+By default, a Prometheus endpoint is exposed by Kubo at `http://127.0.0.1:5001/debug/metrics/prometheus`.
4
+
5
+It includes default [Prometheus Go client metrics](https://prometheus.io/docs/guides/go-application/) + Kubo-specific metrics listed below.
6
+
7
+### Table of Contents
8
+
9
+- [DHT RPC](#dht-rpc)
10
+ - [Inbound RPC metrics](#inbound-rpc-metrics)
11
+ - [Outbound RPC metrics](#outbound-rpc-metrics)
12
+- [Provide](#provide)
13
+ - [Legacy Provider](#legacy-provider)
14
+ - [DHT Provider](#dht-provider)
15
+- [Gateway (`boxo/gateway`)](#gateway-boxogateway)
16
+ - [HTTP metrics](#http-metrics)
17
+ - [Blockstore cache metrics](#blockstore-cache-metrics)
18
+ - [Backend metrics](#backend-metrics)
19
+- [Generic HTTP Servers](#generic-http-servers)
20
+ - [Core HTTP metrics](#core-http-metrics-ipfs_http_)
21
+ - [HTTP Server metrics](#http-server-metrics-http_server_)
22
+- [OpenTelemetry Metadata](#opentelemetry-metadata)
23
+
24
+> [!WARNING]
25
+> This documentation is incomplete. For an up-to-date list of metrics available at daemon startup, see [test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile](https://github.com/ipfs/kubo/blob/master/test/sharness/t0119-prometheus-data/prometheus_metrics_added_by_measure_profile).
26
+>
27
+> Additional metrics may appear during runtime as some components (like boxo/gateway) register metrics only after their first event occurs (e.g., HTTP request/response).
28
+
29
+## DHT RPC
30
+
31
+Metrics from `go-libp2p-kad-dht` for DHT RPC operations:
32
+
33
+### Inbound RPC metrics
34
+
35
+- `rpc_inbound_messages_total` - Counter: total messages received per RPC
36
+- `rpc_inbound_message_errors_total` - Counter: total errors for received messages
37
+- `rpc_inbound_bytes_[bucket|sum|count]` - Histogram: distribution of received bytes per RPC
38
+- `rpc_inbound_request_latency_[bucket|sum|count]` - Histogram: latency distribution for inbound RPCs
39
+
40
+### Outbound RPC metrics
41
+
42
+- `rpc_outbound_messages_total` - Counter: total messages sent per RPC
43
+- `rpc_outbound_message_errors_total` - Counter: total errors for sent messages
44
+- `rpc_outbound_requests_total` - Counter: total requests sent
45
+- `rpc_outbound_request_errors_total` - Counter: total errors for sent requests
46
+- `rpc_outbound_bytes_[bucket|sum|count]` - Histogram: distribution of sent bytes per RPC
47
+- `rpc_outbound_request_latency_[bucket|sum|count]` - Histogram: latency distribution for outbound RPCs
48
+
49
+## Provide
50
+
51
+### Legacy Provider
52
+
53
+Metrics for the legacy provider system when `Provide.DHT.SweepEnabled=false`:
54
+
55
+- `provider_reprovider_provide_count` - Counter: total successful provide operations since node startup
56
+- `provider_reprovider_reprovide_count` - Counter: total reprovide sweep operations since node startup
57
+
58
+### DHT Provider
59
+
60
+Metrics for the DHT provider system when `Provide.DHT.SweepEnabled=true`:
61
+
62
+- `total_provide_count_total` - Counter: total successful provide operations since node startup (includes both one-time provides and periodic provides done on `Provide.DHT.Interval`)
63
+
64
+> [!NOTE]
65
+> These metrics are exposed by [go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht/). You can enable debug logging for DHT provider activity with `GOLOG_LOG_LEVEL=dht/provider=debug`.
66
+
67
+## Gateway (`boxo/gateway`)
68
+
69
+> [!TIP]
70
+> These metrics are limited to [IPFS Gateway](https://specs.ipfs.tech/http-gateways/) endpoints. For general HTTP metrics across all endpoints, consider using a reverse proxy.
71
+
72
+Gateway metrics appear after the first HTTP request is processed:
73
+
74
+### HTTP metrics
75
+
76
+- `ipfs_http_gw_responses_total{code}` - Counter: total HTTP responses by status code
77
+- `ipfs_http_gw_retrieval_timeouts_total{code,truncated}` - Counter: requests that timed out during content retrieval
78
+- `ipfs_http_gw_concurrent_requests` - Gauge: number of requests currently being processed
79
+
80
+### Blockstore cache metrics
81
+
82
+- `ipfs_http_blockstore_cache_hit` - Counter: global block cache hits
83
+- `ipfs_http_blockstore_cache_requests` - Counter: global block cache requests
84
+
85
+### Backend metrics
86
+
87
+- `ipfs_gw_backend_api_call_duration_seconds_[bucket|sum|count]{backend_method}` - Histogram: time spent in IPFSBackend API calls
88
+
89
+## Generic HTTP Servers
90
+
91
+> [!TIP]
92
+> The metrics below are not very useful and exist mostly for historical reasons. If you need non-gateway HTTP metrics, it's better to put a reverse proxy in front of Kubo and use its metrics.
93
+
94
+### Core HTTP metrics (`ipfs_http_*`)
95
+
96
+Prometheus metrics for the HTTP API exposed at port 5001:
97
+
98
+- `ipfs_http_requests_total{method,code,handler}` - Counter: total HTTP requests (Legacy - new metrics are provided by boxo/gateway for gateway traffic)
99
+- `ipfs_http_request_duration_seconds[_sum|_count]{handler}` - Summary: request processing duration
100
+- `ipfs_http_request_size_bytes[_sum|_count]{handler}` - Summary: request body sizes
101
+- `ipfs_http_response_size_bytes[_sum|_count]{handler}` - Summary: response body sizes
102
+
103
+### HTTP Server metrics (`http_server_*`)
104
+
105
+Additional HTTP instrumentation for all handlers (Gateway, API commands, etc.):
106
+
107
+- `http_server_request_body_size_bytes_[bucket|count|sum]` - Histogram: distribution of request body sizes
108
+- `http_server_request_duration_seconds_[bucket|count|sum]` - Histogram: distribution of request processing times
109
+- `http_server_response_body_size_bytes_[bucket|count|sum]` - Histogram: distribution of response body sizes
110
+
111
+These metrics are automatically added to Gateway handlers, Hostname Gateway, Libp2p Gateway, and API command handlers.
112
+
113
+## OpenTelemetry Metadata
114
+
115
+Kubo uses Prometheus for metrics collection for historical reasons, but OpenTelemetry metrics are automatically exposed through the same Prometheus endpoint. These metadata metrics provide context about the instrumentation:
116
+
117
+- `otel_scope_info` - Information about instrumentation libraries producing metrics
118
+- `target_info` - Service metadata including version and instance information
\ No newline at end of file
go.mod
+2
@@ -80,7 +80,9 @@ require (
80
go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.63.0
81
go.opentelemetry.io/contrib/propagators/autoprop v0.46.1
82
go.opentelemetry.io/otel v1.38.0
83
+ go.opentelemetry.io/otel/exporters/prometheus v0.56.0
84
go.opentelemetry.io/otel/sdk v1.38.0
85
+ go.opentelemetry.io/otel/sdk/metric v1.38.0
86
go.opentelemetry.io/otel/trace v1.38.0
87
go.uber.org/dig v1.19.0
88
go.uber.org/fx v1.24.0
go.sum
+2
@@ -950,6 +950,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0 h1:lwI4D
950
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0/go.mod h1:Kz/oCE7z5wuyhPxsXDuaPteSWqjSBD5YaSdbxZYGbGk=
951
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0 h1:aTL7F04bJHUlztTsNGJ2l+6he8c+y/b//eR0jjjemT4=
952
go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.38.0/go.mod h1:kldtb7jDTeol0l3ewcmd8SDvx3EmIE7lyvqbasU3QC4=
953
+go.opentelemetry.io/otel/exporters/prometheus v0.56.0 h1:GnCIi0QyG0yy2MrJLzVrIM7laaJstj//flf1zEJCG+E=
954
+go.opentelemetry.io/otel/exporters/prometheus v0.56.0/go.mod h1:JQcVZtbIIPM+7SWBB+T6FK+xunlyidwLp++fN0sUaOk=
955
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0 h1:kJxSDN4SgWWTjG/hPp3O7LCGLcHXFlvS2/FFOrwL+SE=
956
go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.38.0/go.mod h1:mgIOzS7iZeKJdeB8/NYHrJ48fdGc71Llo5bJ1J4DWUE=
957
go.opentelemetry.io/otel/exporters/zipkin v1.38.0 h1:0rJ2TmzpHDG+Ib9gPmu3J3cE0zXirumQcKS4wCoZUa0=
plugin/plugins/telemetry/telemetry.go
+1
-1
@@ -407,7 +407,7 @@ func (p *telemetryPlugin) collectBasicInfo() {
407
}
408
p.event.UptimeBucket = uptimeBucket
409
410
- p.event.ReproviderStrategy = p.config.Reprovider.Strategy.WithDefault(config.DefaultReproviderStrategy)
410
+ p.event.ReproviderStrategy = p.config.Provide.Strategy.WithDefault(config.DefaultProvideStrategy)
411
}
412
413
func (p *telemetryPlugin) collectRoutingInfo() {
repo/fsrepo/migrations/common/base.go
new
+97
@@ -0,0 +1,97 @@
1
+package common
2
+
3
+import (
4
+ "fmt"
5
+ "io"
6
+ "path/filepath"
7
+)
8
+
9
+// BaseMigration provides common functionality for migrations
10
+type BaseMigration struct {
11
+ FromVersion string
12
+ ToVersion string
13
+ Description string
14
+ Convert func(in io.ReadSeeker, out io.Writer) error
15
+}
16
+
17
+// Versions returns the version string for this migration
18
+func (m *BaseMigration) Versions() string {
19
+ return fmt.Sprintf("%s-to-%s", m.FromVersion, m.ToVersion)
20
+}
21
+
22
+// configBackupSuffix returns the backup suffix for the config file
23
+// e.g. ".16-to-17.bak" results in "config.16-to-17.bak"
24
+func (m *BaseMigration) configBackupSuffix() string {
25
+ return fmt.Sprintf(".%s-to-%s.bak", m.FromVersion, m.ToVersion)
26
+}
27
+
28
+// Reversible returns true as we keep backups
29
+func (m *BaseMigration) Reversible() bool {
30
+ return true
31
+}
32
+
33
+// Apply performs the migration
34
+func (m *BaseMigration) Apply(opts Options) error {
35
+ if opts.Verbose {
36
+ fmt.Printf("applying %s repo migration\n", m.Versions())
37
+ if m.Description != "" {
38
+ fmt.Printf("> %s\n", m.Description)
39
+ }
40
+ }
41
+
42
+ // Check version
43
+ if err := CheckVersion(opts.Path, m.FromVersion); err != nil {
44
+ return err
45
+ }
46
+
47
+ configPath := filepath.Join(opts.Path, "config")
48
+
49
+ // Perform migration with backup
50
+ if err := WithBackup(configPath, m.configBackupSuffix(), m.Convert); err != nil {
51
+ return err
52
+ }
53
+
54
+ // Update version
55
+ if err := WriteVersion(opts.Path, m.ToVersion); err != nil {
56
+ if opts.Verbose {
57
+ fmt.Printf("failed to update version file to %s\n", m.ToVersion)
58
+ }
59
+ return err
60
+ }
61
+
62
+ if opts.Verbose {
63
+ fmt.Println("updated version file")
64
+ fmt.Printf("Migration %s succeeded\n", m.Versions())
65
+ }
66
+
67
+ return nil
68
+}
69
+
70
+// Revert reverts the migration
71
+func (m *BaseMigration) Revert(opts Options) error {
72
+ if opts.Verbose {
73
+ fmt.Println("reverting migration")
74
+ }
75
+
76
+ // Check we're at the expected version
77
+ if err := CheckVersion(opts.Path, m.ToVersion); err != nil {
78
+ return err
79
+ }
80
+
81
+ // Restore backup
82
+ configPath := filepath.Join(opts.Path, "config")
83
+ if err := RevertBackup(configPath, m.configBackupSuffix()); err != nil {
84
+ return err
85
+ }
86
+
87
+ // Revert version
88
+ if err := WriteVersion(opts.Path, m.FromVersion); err != nil {
89
+ return err
90
+ }
91
+
92
+ if opts.Verbose {
93
+ fmt.Printf("lowered version number to %s\n", m.FromVersion)
94
+ }
95
+
96
+ return nil
97
+}
repo/fsrepo/migrations/common/config_helpers.go
new
+353
@@ -0,0 +1,353 @@
1
+package common
2
+
3
+import (
4
+ "fmt"
5
+ "maps"
6
+ "slices"
7
+ "strings"
8
+)
9
+
10
+// GetField retrieves a field from a nested config structure using a dot-separated path
11
+// Example: GetField(config, "DNS.Resolvers") returns config["DNS"]["Resolvers"]
12
+func GetField(config map[string]any, path string) (any, bool) {
13
+ parts := strings.Split(path, ".")
14
+ current := config
15
+
16
+ for i, part := range parts {
17
+ // Last part - return the value
18
+ if i == len(parts)-1 {
19
+ val, exists := current[part]
20
+ return val, exists
21
+ }
22
+
23
+ // Navigate deeper
24
+ next, exists := current[part]
25
+ if !exists {
26
+ return nil, false
27
+ }
28
+
29
+ // Ensure it's a map
30
+ nextMap, ok := next.(map[string]any)
31
+ if !ok {
32
+ return nil, false
33
+ }
34
+ current = nextMap
35
+ }
36
+
37
+ return nil, false
38
+}
39
+
40
+// SetField sets a field in a nested config structure using a dot-separated path
41
+// It creates intermediate maps as needed
42
+func SetField(config map[string]any, path string, value any) {
43
+ parts := strings.Split(path, ".")
44
+ current := config
45
+
46
+ for i, part := range parts {
47
+ // Last part - set the value
48
+ if i == len(parts)-1 {
49
+ current[part] = value
50
+ return
51
+ }
52
+
53
+ // Navigate or create intermediate maps
54
+ next, exists := current[part]
55
+ if !exists {
56
+ // Create new intermediate map
57
+ newMap := make(map[string]any)
58
+ current[part] = newMap
59
+ current = newMap
60
+ } else {
61
+ // Ensure it's a map
62
+ nextMap, ok := next.(map[string]any)
63
+ if !ok {
64
+ // Can't navigate further, replace with new map
65
+ newMap := make(map[string]any)
66
+ current[part] = newMap
67
+ current = newMap
68
+ } else {
69
+ current = nextMap
70
+ }
71
+ }
72
+ }
73
+}
74
+
75
+// DeleteField removes a field from a nested config structure
76
+func DeleteField(config map[string]any, path string) bool {
77
+ parts := strings.Split(path, ".")
78
+
79
+ // Handle simple case
80
+ if len(parts) == 1 {
81
+ _, exists := config[parts[0]]
82
+ delete(config, parts[0])
83
+ return exists
84
+ }
85
+
86
+ // Navigate to parent
87
+ parentPath := strings.Join(parts[:len(parts)-1], ".")
88
+ parent, exists := GetField(config, parentPath)
89
+ if !exists {
90
+ return false
91
+ }
92
+
93
+ parentMap, ok := parent.(map[string]any)
94
+ if !ok {
95
+ return false
96
+ }
97
+
98
+ fieldName := parts[len(parts)-1]
99
+ _, exists = parentMap[fieldName]
100
+ delete(parentMap, fieldName)
101
+ return exists
102
+}
103
+
104
+// MoveField moves a field from one location to another
105
+func MoveField(config map[string]any, from, to string) error {
106
+ value, exists := GetField(config, from)
107
+ if !exists {
108
+ return fmt.Errorf("source field %s does not exist", from)
109
+ }
110
+
111
+ SetField(config, to, value)
112
+ DeleteField(config, from)
113
+ return nil
114
+}
115
+
116
+// RenameField renames a field within the same parent
117
+func RenameField(config map[string]any, path, oldName, newName string) error {
118
+ var parent map[string]any
119
+ if path == "" {
120
+ parent = config
121
+ } else {
122
+ p, exists := GetField(config, path)
123
+ if !exists {
124
+ return fmt.Errorf("parent path %s does not exist", path)
125
+ }
126
+ var ok bool
127
+ parent, ok = p.(map[string]any)
128
+ if !ok {
129
+ return fmt.Errorf("parent path %s is not a map", path)
130
+ }
131
+ }
132
+
133
+ value, exists := parent[oldName]
134
+ if !exists {
135
+ return fmt.Errorf("field %s does not exist", oldName)
136
+ }
137
+
138
+ parent[newName] = value
139
+ delete(parent, oldName)
140
+ return nil
141
+}
142
+
143
+// SetDefault sets a field value only if it doesn't already exist
144
+func SetDefault(config map[string]any, path string, value any) {
145
+ if _, exists := GetField(config, path); !exists {
146
+ SetField(config, path, value)
147
+ }
148
+}
149
+
150
+// TransformField applies a transformation function to a field value
151
+func TransformField(config map[string]any, path string, transformer func(any) any) error {
152
+ value, exists := GetField(config, path)
153
+ if !exists {
154
+ return fmt.Errorf("field %s does not exist", path)
155
+ }
156
+
157
+ newValue := transformer(value)
158
+ SetField(config, path, newValue)
159
+ return nil
160
+}
161
+
162
+// EnsureFieldIs checks if a field equals expected value, sets it if missing
163
+func EnsureFieldIs(config map[string]any, path string, expected any) {
164
+ current, exists := GetField(config, path)
165
+ if !exists || current != expected {
166
+ SetField(config, path, expected)
167
+ }
168
+}
169
+
170
+// MergeInto merges multiple source fields into a destination map
171
+func MergeInto(config map[string]any, destination string, sources ...string) {
172
+ var destMap map[string]any
173
+
174
+ // Get existing destination if it exists
175
+ if existing, exists := GetField(config, destination); exists {
176
+ if m, ok := existing.(map[string]any); ok {
177
+ destMap = m
178
+ }
179
+ }
180
+
181
+ // Merge each source
182
+ for _, source := range sources {
183
+ if value, exists := GetField(config, source); exists {
184
+ if sourceMap, ok := value.(map[string]any); ok {
185
+ if destMap == nil {
186
+ destMap = make(map[string]any)
187
+ }
188
+ maps.Copy(destMap, sourceMap)
189
+ }
190
+ }
191
+ }
192
+
193
+ if destMap != nil {
194
+ SetField(config, destination, destMap)
195
+ }
196
+}
197
+
198
+// CopyField copies a field value to a new location (keeps original)
199
+func CopyField(config map[string]any, from, to string) error {
200
+ value, exists := GetField(config, from)
201
+ if !exists {
202
+ return fmt.Errorf("source field %s does not exist", from)
203
+ }
204
+
205
+ SetField(config, to, value)
206
+ return nil
207
+}
208
+
209
+// ConvertInterfaceSlice converts []interface{} to []string
210
+func ConvertInterfaceSlice(slice []interface{}) []string {
211
+ result := make([]string, 0, len(slice))
212
+ for _, item := range slice {
213
+ if str, ok := item.(string); ok {
214
+ result = append(result, str)
215
+ }
216
+ }
217
+ return result
218
+}
219
+
220
+// GetOrCreateSection gets or creates a map section in config
221
+func GetOrCreateSection(config map[string]any, path string) map[string]any {
222
+ existing, exists := GetField(config, path)
223
+ if exists {
224
+ if section, ok := existing.(map[string]any); ok {
225
+ return section
226
+ }
227
+ }
228
+
229
+ // Create new section
230
+ section := make(map[string]any)
231
+ SetField(config, path, section)
232
+ return section
233
+}
234
+
235
+// SafeCastMap safely casts to map[string]any with fallback to empty map
236
+func SafeCastMap(value any) map[string]any {
237
+ if m, ok := value.(map[string]any); ok {
238
+ return m
239
+ }
240
+ return make(map[string]any)
241
+}
242
+
243
+// SafeCastSlice safely casts to []interface{} with fallback to empty slice
244
+func SafeCastSlice(value any) []interface{} {
245
+ if s, ok := value.([]interface{}); ok {
246
+ return s
247
+ }
248
+ return []interface{}{}
249
+}
250
+
251
+// ReplaceDefaultsWithAuto replaces default values with "auto" in a map
252
+func ReplaceDefaultsWithAuto(values map[string]any, defaults map[string]string) map[string]string {
253
+ result := make(map[string]string)
254
+ for k, v := range values {
255
+ if vStr, ok := v.(string); ok {
256
+ if replacement, isDefault := defaults[vStr]; isDefault {
257
+ result[k] = replacement
258
+ } else {
259
+ result[k] = vStr
260
+ }
261
+ }
262
+ }
263
+ return result
264
+}
265
+
266
+// EnsureSliceContains ensures a slice field contains a value
267
+func EnsureSliceContains(config map[string]any, path string, value string) {
268
+ existing, exists := GetField(config, path)
269
+ if !exists {
270
+ SetField(config, path, []string{value})
271
+ return
272
+ }
273
+
274
+ if slice, ok := existing.([]interface{}); ok {
275
+ // Check if value already exists
276
+ for _, item := range slice {
277
+ if str, ok := item.(string); ok && str == value {
278
+ return // Already contains value
279
+ }
280
+ }
281
+ // Add value
282
+ SetField(config, path, append(slice, value))
283
+ } else if strSlice, ok := existing.([]string); ok {
284
+ if !slices.Contains(strSlice, value) {
285
+ SetField(config, path, append(strSlice, value))
286
+ }
287
+ } else {
288
+ // Replace with new slice containing value
289
+ SetField(config, path, []string{value})
290
+ }
291
+}
292
+
293
+// ReplaceInSlice replaces old values with new in a slice field
294
+func ReplaceInSlice(config map[string]any, path string, oldValue, newValue string) {
295
+ existing, exists := GetField(config, path)
296
+ if !exists {
297
+ return
298
+ }
299
+
300
+ if slice, ok := existing.([]interface{}); ok {
301
+ result := make([]string, 0, len(slice))
302
+ for _, item := range slice {
303
+ if str, ok := item.(string); ok {
304
+ if str == oldValue {
305
+ result = append(result, newValue)
306
+ } else {
307
+ result = append(result, str)
308
+ }
309
+ }
310
+ }
311
+ SetField(config, path, result)
312
+ }
313
+}
314
+
315
+// GetMapSection gets a map section with error handling
316
+func GetMapSection(config map[string]any, path string) (map[string]any, error) {
317
+ value, exists := GetField(config, path)
318
+ if !exists {
319
+ return nil, fmt.Errorf("section %s does not exist", path)
320
+ }
321
+
322
+ section, ok := value.(map[string]any)
323
+ if !ok {
324
+ return nil, fmt.Errorf("section %s is not a map", path)
325
+ }
326
+
327
+ return section, nil
328
+}
329
+
330
+// CloneStringMap clones a map[string]any to map[string]string
331
+func CloneStringMap(m map[string]any) map[string]string {
332
+ result := make(map[string]string, len(m))
333
+ for k, v := range m {
334
+ if str, ok := v.(string); ok {
335
+ result[k] = str
336
+ }
337
+ }
338
+ return result
339
+}
340
+
341
+// IsEmptySlice checks if a value is an empty slice
342
+func IsEmptySlice(value any) bool {
343
+ if value == nil {
344
+ return true
345
+ }
346
+ if slice, ok := value.([]interface{}); ok {
347
+ return len(slice) == 0
348
+ }
349
+ if slice, ok := value.([]string); ok {
350
+ return len(slice) == 0
351
+ }
352
+ return false
353
+}
repo/fsrepo/migrations/common/migration.go
new
+16
@@ -0,0 +1,16 @@
1
+// Package common contains common types and interfaces for file system repository migrations
2
+package common
3
+
4
+// Options contains migration options for embedded migrations
5
+type Options struct {
6
+ Path string
7
+ Verbose bool
8
+}
9
+
10
+// Migration is the interface that all migrations must implement
11
+type Migration interface {
12
+ Versions() string
13
+ Apply(opts Options) error
14
+ Revert(opts Options) error
15
+ Reversible() bool
16
+}
repo/fsrepo/migrations/common/testing_helpers.go
new
+290
@@ -0,0 +1,290 @@
1
+package common
2
+
3
+import (
4
+ "bytes"
5
+ "encoding/json"
6
+ "fmt"
7
+ "maps"
8
+ "os"
9
+ "path/filepath"
10
+ "reflect"
11
+ "testing"
12
+)
13
+
14
+// TestCase represents a single migration test case
15
+type TestCase struct {
16
+ Name string
17
+ InputConfig map[string]any
18
+ Assertions []ConfigAssertion
19
+}
20
+
21
+// ConfigAssertion represents an assertion about the migrated config
22
+type ConfigAssertion struct {
23
+ Path string
24
+ Expected any
25
+}
26
+
27
+// RunMigrationTest runs a migration test with the given test case
28
+func RunMigrationTest(t *testing.T, migration Migration, tc TestCase) {
29
+ t.Helper()
30
+
31
+ // Convert input to JSON
32
+ inputJSON, err := json.MarshalIndent(tc.InputConfig, "", " ")
33
+ if err != nil {
34
+ t.Fatalf("failed to marshal input config: %v", err)
35
+ }
36
+
37
+ // Run the migration's convert function
38
+ var output bytes.Buffer
39
+ if baseMig, ok := migration.(*BaseMigration); ok {
40
+ err = baseMig.Convert(bytes.NewReader(inputJSON), &output)
41
+ if err != nil {
42
+ t.Fatalf("migration failed: %v", err)
43
+ }
44
+ } else {
45
+ t.Skip("migration is not a BaseMigration")
46
+ }
47
+
48
+ // Parse output
49
+ var result map[string]any
50
+ err = json.Unmarshal(output.Bytes(), &result)
51
+ if err != nil {
52
+ t.Fatalf("failed to unmarshal output: %v", err)
53
+ }
54
+
55
+ // Run assertions
56
+ for _, assertion := range tc.Assertions {
57
+ AssertConfigField(t, result, assertion.Path, assertion.Expected)
58
+ }
59
+}
60
+
61
+// AssertConfigField asserts that a field in the config has the expected value
62
+func AssertConfigField(t *testing.T, config map[string]any, path string, expected any) {
63
+ t.Helper()
64
+
65
+ actual, exists := GetField(config, path)
66
+ if expected == nil {
67
+ if exists {
68
+ t.Errorf("expected field %s to not exist, but it has value: %v", path, actual)
69
+ }
70
+ return
71
+ }
72
+
73
+ if !exists {
74
+ t.Errorf("expected field %s to exist with value %v, but it doesn't exist", path, expected)
75
+ return
76
+ }
77
+
78
+ // Handle different types of comparisons
79
+ switch exp := expected.(type) {
80
+ case []string:
81
+ actualSlice, ok := actual.([]interface{})
82
+ if !ok {
83
+ t.Errorf("field %s: expected []string, got %T", path, actual)
84
+ return
85
+ }
86
+ if len(exp) != len(actualSlice) {
87
+ t.Errorf("field %s: expected slice of length %d, got %d", path, len(exp), len(actualSlice))
88
+ return
89
+ }
90
+ for i, expVal := range exp {
91
+ if actualSlice[i] != expVal {
92
+ t.Errorf("field %s[%d]: expected %v, got %v", path, i, expVal, actualSlice[i])
93
+ }
94
+ }
95
+ case map[string]string:
96
+ actualMap, ok := actual.(map[string]any)
97
+ if !ok {
98
+ t.Errorf("field %s: expected map, got %T", path, actual)
99
+ return
100
+ }
101
+ for k, v := range exp {
102
+ if actualMap[k] != v {
103
+ t.Errorf("field %s[%s]: expected %v, got %v", path, k, v, actualMap[k])
104
+ }
105
+ }
106
+ default:
107
+ if actual != expected {
108
+ t.Errorf("field %s: expected %v, got %v", path, expected, actual)
109
+ }
110
+ }
111
+}
112
+
113
+// GenerateTestConfig creates a basic test config with the given fields
114
+func GenerateTestConfig(fields map[string]any) map[string]any {
115
+ // Start with a minimal valid config
116
+ config := map[string]any{
117
+ "Identity": map[string]any{
118
+ "PeerID": "QmTest",
119
+ },
120
+ }
121
+
122
+ // Merge in the provided fields
123
+ maps.Copy(config, fields)
124
+
125
+ return config
126
+}
127
+
128
+// CreateTestRepo creates a temporary test repository with the given version and config
129
+func CreateTestRepo(t *testing.T, version int, config map[string]any) string {
130
+ t.Helper()
131
+
132
+ tempDir := t.TempDir()
133
+
134
+ // Write version file
135
+ versionPath := filepath.Join(tempDir, "version")
136
+ err := os.WriteFile(versionPath, []byte(fmt.Sprintf("%d", version)), 0644)
137
+ if err != nil {
138
+ t.Fatalf("failed to write version file: %v", err)
139
+ }
140
+
141
+ // Write config file
142
+ configPath := filepath.Join(tempDir, "config")
143
+ configData, err := json.MarshalIndent(config, "", " ")
144
+ if err != nil {
145
+ t.Fatalf("failed to marshal config: %v", err)
146
+ }
147
+ err = os.WriteFile(configPath, configData, 0644)
148
+ if err != nil {
149
+ t.Fatalf("failed to write config file: %v", err)
150
+ }
151
+
152
+ return tempDir
153
+}
154
+
155
+// AssertMigrationSuccess runs a full migration and checks that it succeeds
156
+func AssertMigrationSuccess(t *testing.T, migration Migration, fromVersion, toVersion int, inputConfig map[string]any) map[string]any {
157
+ t.Helper()
158
+
159
+ // Create test repo
160
+ repoPath := CreateTestRepo(t, fromVersion, inputConfig)
161
+
162
+ // Run migration
163
+ opts := Options{
164
+ Path: repoPath,
165
+ Verbose: false,
166
+ }
167
+
168
+ err := migration.Apply(opts)
169
+ if err != nil {
170
+ t.Fatalf("migration failed: %v", err)
171
+ }
172
+
173
+ // Check version was updated
174
+ versionBytes, err := os.ReadFile(filepath.Join(repoPath, "version"))
175
+ if err != nil {
176
+ t.Fatalf("failed to read version file: %v", err)
177
+ }
178
+ actualVersion := string(versionBytes)
179
+ if actualVersion != fmt.Sprintf("%d", toVersion) {
180
+ t.Errorf("expected version %d, got %s", toVersion, actualVersion)
181
+ }
182
+
183
+ // Read and return the migrated config
184
+ configBytes, err := os.ReadFile(filepath.Join(repoPath, "config"))
185
+ if err != nil {
186
+ t.Fatalf("failed to read config file: %v", err)
187
+ }
188
+
189
+ var result map[string]any
190
+ err = json.Unmarshal(configBytes, &result)
191
+ if err != nil {
192
+ t.Fatalf("failed to unmarshal config: %v", err)
193
+ }
194
+
195
+ return result
196
+}
197
+
198
+// AssertMigrationReversible checks that a migration can be reverted
199
+func AssertMigrationReversible(t *testing.T, migration Migration, fromVersion, toVersion int, inputConfig map[string]any) {
200
+ t.Helper()
201
+
202
+ // Create test repo at target version
203
+ repoPath := CreateTestRepo(t, toVersion, inputConfig)
204
+
205
+ // Create backup file (simulating a previous migration)
206
+ backupPath := filepath.Join(repoPath, fmt.Sprintf("config.%d-to-%d.bak", fromVersion, toVersion))
207
+ originalConfig, err := json.MarshalIndent(inputConfig, "", " ")
208
+ if err != nil {
209
+ t.Fatalf("failed to marshal original config: %v", err)
210
+ }
211
+
212
+ if err := os.WriteFile(backupPath, originalConfig, 0644); err != nil {
213
+ t.Fatalf("failed to write backup file: %v", err)
214
+ }
215
+
216
+ // Run revert
217
+ if err := migration.Revert(Options{Path: repoPath}); err != nil {
218
+ t.Fatalf("revert failed: %v", err)
219
+ }
220
+
221
+ // Verify version was reverted
222
+ versionBytes, err := os.ReadFile(filepath.Join(repoPath, "version"))
223
+ if err != nil {
224
+ t.Fatalf("failed to read version file: %v", err)
225
+ }
226
+
227
+ if actualVersion := string(versionBytes); actualVersion != fmt.Sprintf("%d", fromVersion) {
228
+ t.Errorf("expected version %d after revert, got %s", fromVersion, actualVersion)
229
+ }
230
+
231
+ // Verify config was reverted
232
+ configBytes, err := os.ReadFile(filepath.Join(repoPath, "config"))
233
+ if err != nil {
234
+ t.Fatalf("failed to read reverted config file: %v", err)
235
+ }
236
+
237
+ var revertedConfig map[string]any
238
+ if err := json.Unmarshal(configBytes, &revertedConfig); err != nil {
239
+ t.Fatalf("failed to unmarshal reverted config: %v", err)
240
+ }
241
+
242
+ // Compare reverted config with original
243
+ compareConfigs(t, inputConfig, revertedConfig, "")
244
+}
245
+
246
+// compareConfigs recursively compares two config maps and reports differences
247
+func compareConfigs(t *testing.T, expected, actual map[string]any, path string) {
248
+ t.Helper()
249
+
250
+ // Build current path helper
251
+ buildPath := func(key string) string {
252
+ if path == "" {
253
+ return key
254
+ }
255
+ return path + "." + key
256
+ }
257
+
258
+ // Check all expected fields exist and match
259
+ for key, expectedValue := range expected {
260
+ currentPath := buildPath(key)
261
+
262
+ actualValue, exists := actual[key]
263
+ if !exists {
264
+ t.Errorf("reverted config missing field %s", currentPath)
265
+ continue
266
+ }
267
+
268
+ switch exp := expectedValue.(type) {
269
+ case map[string]any:
270
+ act, ok := actualValue.(map[string]any)
271
+ if !ok {
272
+ t.Errorf("field %s: expected map, got %T", currentPath, actualValue)
273
+ continue
274
+ }
275
+ compareConfigs(t, exp, act, currentPath)
276
+ default:
277
+ if !reflect.DeepEqual(expectedValue, actualValue) {
278
+ t.Errorf("field %s: expected %v, got %v after revert",
279
+ currentPath, expectedValue, actualValue)
280
+ }
281
+ }
282
+ }
283
+
284
+ // Check for unexpected fields using maps.Keys (Go 1.23+)
285
+ for key := range actual {
286
+ if _, exists := expected[key]; !exists {
287
+ t.Errorf("reverted config has unexpected field %s", buildPath(key))
288
+ }
289
+ }
290
+}
repo/fsrepo/migrations/common/utils.go
new
+107
@@ -0,0 +1,107 @@
1
+package common
2
+
3
+import (
4
+ "encoding/json"
5
+ "fmt"
6
+ "io"
7
+ "os"
8
+ "path/filepath"
9
+ "strings"
10
+
11
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/atomicfile"
12
+)
13
+
14
+// CheckVersion verifies the repo is at the expected version
15
+func CheckVersion(repoPath string, expectedVersion string) error {
16
+ versionPath := filepath.Join(repoPath, "version")
17
+ versionBytes, err := os.ReadFile(versionPath)
18
+ if err != nil {
19
+ return fmt.Errorf("could not read version file: %w", err)
20
+ }
21
+ version := strings.TrimSpace(string(versionBytes))
22
+ if version != expectedVersion {
23
+ return fmt.Errorf("expected version %s, got %s", expectedVersion, version)
24
+ }
25
+ return nil
26
+}
27
+
28
+// WriteVersion writes the version to the repo
29
+func WriteVersion(repoPath string, version string) error {
30
+ versionPath := filepath.Join(repoPath, "version")
31
+ return os.WriteFile(versionPath, []byte(version), 0644)
32
+}
33
+
34
+// Must panics if the error is not nil. Use only for errors that cannot be handled gracefully.
35
+func Must(err error) {
36
+ if err != nil {
37
+ panic(fmt.Errorf("error can't be dealt with transactionally: %w", err))
38
+ }
39
+}
40
+
41
+// WithBackup performs a config file operation with automatic backup and rollback on error
42
+func WithBackup(configPath string, backupSuffix string, fn func(in io.ReadSeeker, out io.Writer) error) error {
43
+ in, err := os.Open(configPath)
44
+ if err != nil {
45
+ return err
46
+ }
47
+ defer in.Close()
48
+
49
+ // Create backup
50
+ backup, err := atomicfile.New(configPath+backupSuffix, 0600)
51
+ if err != nil {
52
+ return err
53
+ }
54
+
55
+ // Copy to backup
56
+ if _, err := backup.ReadFrom(in); err != nil {
57
+ Must(backup.Abort())
58
+ return err
59
+ }
60
+
61
+ // Reset input for reading
62
+ if _, err := in.Seek(0, io.SeekStart); err != nil {
63
+ Must(backup.Abort())
64
+ return err
65
+ }
66
+
67
+ // Create output file
68
+ out, err := atomicfile.New(configPath, 0600)
69
+ if err != nil {
70
+ Must(backup.Abort())
71
+ return err
72
+ }
73
+
74
+ // Run the conversion function
75
+ if err := fn(in, out); err != nil {
76
+ Must(out.Abort())
77
+ Must(backup.Abort())
78
+ return err
79
+ }
80
+
81
+ // Close everything on success
82
+ Must(out.Close())
83
+ Must(backup.Close())
84
+
85
+ return nil
86
+}
87
+
88
+// RevertBackup restores a backup file
89
+func RevertBackup(configPath string, backupSuffix string) error {
90
+ return os.Rename(configPath+backupSuffix, configPath)
91
+}
92
+
93
+// ReadConfig reads and unmarshals a JSON config file into a map
94
+func ReadConfig(r io.Reader) (map[string]any, error) {
95
+ confMap := make(map[string]any)
96
+ if err := json.NewDecoder(r).Decode(&confMap); err != nil {
97
+ return nil, err
98
+ }
99
+ return confMap, nil
100
+}
101
+
102
+// WriteConfig marshals and writes a config map as indented JSON
103
+func WriteConfig(w io.Writer, config map[string]any) error {
104
+ enc := json.NewEncoder(w)
105
+ enc.SetIndent("", " ")
106
+ return enc.Encode(config)
107
+}
repo/fsrepo/migrations/embedded.go
+17
-12
@@ -6,25 +6,30 @@ import (
6
"log"
7
"os"
8
9
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
10
mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration"
11
+ mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration"
12
)
13
12
-// EmbeddedMigration represents an embedded migration that can be run directly
13
-type EmbeddedMigration interface {
14
- Versions() string
15
- Apply(opts mg16.Options) error
16
- Revert(opts mg16.Options) error
17
- Reversible() bool
14
+// embeddedMigrations contains all embedded migrations
15
+// Using a slice to maintain order and allow for future range-based operations
16
+var embeddedMigrations = []common.Migration{
17
+ mg16.Migration,
18
+ mg17.Migration,
19
}
20
20
-// embeddedMigrations contains all embedded migrations
21
-var embeddedMigrations = map[string]EmbeddedMigration{
22
- "fs-repo-16-to-17": &mg16.Migration{},
21
+// migrationsByName provides quick lookup by name
22
+var migrationsByName = make(map[string]common.Migration)
23
+
24
+func init() {
25
+ for _, m := range embeddedMigrations {
26
+ migrationsByName["fs-repo-"+m.Versions()] = m
27
+ }
28
}
29
30
// RunEmbeddedMigration runs an embedded migration if available
31
func RunEmbeddedMigration(ctx context.Context, migrationName string, ipfsDir string, revert bool) error {
27
- migration, exists := embeddedMigrations[migrationName]
32
+ migration, exists := migrationsByName[migrationName]
33
if !exists {
34
return fmt.Errorf("embedded migration %s not found", migrationName)
35
}
@@ -36,7 +41,7 @@ func RunEmbeddedMigration(ctx context.Context, migrationName string, ipfsDir str
41
logger := log.New(os.Stdout, "", 0)
42
logger.Printf("Running embedded migration %s...", migrationName)
43
39
- opts := mg16.Options{
44
+ opts := common.Options{
45
Path: ipfsDir,
46
Verbose: true,
47
}
@@ -58,7 +63,7 @@ func RunEmbeddedMigration(ctx context.Context, migrationName string, ipfsDir str
63
64
// HasEmbeddedMigration checks if a migration is available as embedded
65
func HasEmbeddedMigration(migrationName string) bool {
61
- _, exists := embeddedMigrations[migrationName]
66
+ _, exists := migrationsByName[migrationName]
67
return exists
68
}
69
repo/fsrepo/migrations/fs-repo-16-to-17/main.go
+4
-4
@@ -28,6 +28,7 @@ import (
28
"fmt"
29
"os"
30
31
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
32
mg16 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-16-to-17/migration"
33
)
34
@@ -43,17 +44,16 @@ func main() {
44
os.Exit(1)
45
}
46
46
- m := mg16.Migration{}
47
- opts := mg16.Options{
47
+ opts := common.Options{
48
Path: *path,
49
Verbose: *verbose,
50
}
51
52
var err error
53
if *revert {
54
- err = m.Revert(opts)
54
+ err = mg16.Migration.Revert(opts)
55
} else {
56
- err = m.Apply(opts)
56
+ err = mg16.Migration.Apply(opts)
57
}
58
59
if err != nil {
repo/fsrepo/migrations/fs-repo-16-to-17/migration/migration.go
+74
-345
@@ -7,27 +7,13 @@
7
package mg16
8
9
import (
10
- "encoding/json"
11
- "fmt"
10
"io"
13
- "os"
14
- "path/filepath"
15
- "reflect"
11
"slices"
17
- "strings"
12
13
"github.com/ipfs/kubo/config"
20
- "github.com/ipfs/kubo/repo/fsrepo/migrations/atomicfile"
14
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
15
)
16
23
-// Options contains migration options for embedded migrations
24
-type Options struct {
25
- Path string
26
- Verbose bool
27
-}
28
-
29
-const backupSuffix = ".16-to-17.bak"
30
-
17
// DefaultBootstrapAddresses are the hardcoded bootstrap addresses from Kubo 0.36
18
// for IPFS. they are nodes run by the IPFS team. docs on these later.
19
// As with all p2p networks, bootstrap is an important security concern.
@@ -42,148 +28,23 @@ var DefaultBootstrapAddresses = []string{
28
"/ip4/104.131.131.82/udp/4001/quic-v1/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ", // mars.i.ipfs.io
29
}
30
45
-// Migration implements the migration described above.
46
-type Migration struct{}
47
-
48
-// Versions returns the current version string for this migration.
49
-func (m Migration) Versions() string {
50
- return "16-to-17"
51
-}
52
-
53
-// Reversible returns true, as we keep old config around
54
-func (m Migration) Reversible() bool {
55
- return true
56
-}
57
-
58
-// Apply update the config.
59
-func (m Migration) Apply(opts Options) error {
60
- if opts.Verbose {
61
- fmt.Printf("applying %s repo migration\n", m.Versions())
62
- }
63
-
64
- // Check version
65
- if err := checkVersion(opts.Path, "16"); err != nil {
66
- return err
67
- }
68
-
69
- if opts.Verbose {
70
- fmt.Println("> Upgrading config to use AutoConf system")
71
- }
72
-
73
- path := filepath.Join(opts.Path, "config")
74
- in, err := os.Open(path)
75
- if err != nil {
76
- return err
77
- }
78
-
79
- // make backup
80
- backup, err := atomicfile.New(path+backupSuffix, 0600)
81
- if err != nil {
82
- return err
83
- }
84
- if _, err := backup.ReadFrom(in); err != nil {
85
- panicOnError(backup.Abort())
86
- return err
87
- }
88
- if _, err := in.Seek(0, io.SeekStart); err != nil {
89
- panicOnError(backup.Abort())
90
- return err
91
- }
92
-
93
- // Create a temp file to write the output to on success
94
- out, err := atomicfile.New(path, 0600)
95
- if err != nil {
96
- panicOnError(backup.Abort())
97
- panicOnError(in.Close())
98
- return err
99
- }
100
-
101
- if err := convert(in, out, opts.Path); err != nil {
102
- panicOnError(out.Abort())
103
- panicOnError(backup.Abort())
104
- panicOnError(in.Close())
105
- return err
106
- }
107
-
108
- if err := in.Close(); err != nil {
109
- panicOnError(out.Abort())
110
- panicOnError(backup.Abort())
111
- }
112
-
113
- if err := writeVersion(opts.Path, "17"); err != nil {
114
- fmt.Println("failed to update version file to 17")
115
- // There was an error so abort writing the output and clean up temp file
116
- panicOnError(out.Abort())
117
- panicOnError(backup.Abort())
118
- return err
119
- } else {
120
- // Write the output and clean up temp file
121
- panicOnError(out.Close())
122
- panicOnError(backup.Close())
123
- }
124
-
125
- if opts.Verbose {
126
- fmt.Println("updated version file")
127
- fmt.Println("Migration 16 to 17 succeeded")
128
- }
129
- return nil
130
-}
131
-
132
-// panicOnError is reserved for checks we can't solve transactionally if an error occurs
133
-func panicOnError(e error) {
134
- if e != nil {
135
- panic(fmt.Errorf("error can't be dealt with transactionally: %w", e))
136
- }
137
-}
138
-
139
-func (m Migration) Revert(opts Options) error {
140
- if opts.Verbose {
141
- fmt.Println("reverting migration")
142
- }
143
-
144
- if err := checkVersion(opts.Path, "17"); err != nil {
145
- return err
146
- }
147
-
148
- cfg := filepath.Join(opts.Path, "config")
149
- if err := os.Rename(cfg+backupSuffix, cfg); err != nil {
150
- return err
151
- }
152
-
153
- if err := writeVersion(opts.Path, "16"); err != nil {
154
- return err
155
- }
156
- if opts.Verbose {
157
- fmt.Println("lowered version number to 16")
158
- }
159
-
160
- return nil
161
-}
162
-
163
-// checkVersion verifies the repo is at the expected version
164
-func checkVersion(repoPath string, expectedVersion string) error {
165
- versionPath := filepath.Join(repoPath, "version")
166
- versionBytes, err := os.ReadFile(versionPath)
167
- if err != nil {
168
- return fmt.Errorf("could not read version file: %w", err)
169
- }
170
- version := strings.TrimSpace(string(versionBytes))
171
- if version != expectedVersion {
172
- return fmt.Errorf("expected version %s, got %s", expectedVersion, version)
173
- }
174
- return nil
31
+// Migration is the main exported migration for 16-to-17
32
+var Migration = &common.BaseMigration{
33
+ FromVersion: "16",
34
+ ToVersion: "17",
35
+ Description: "Upgrading config to use AutoConf system",
36
+ Convert: convert,
37
}
38
177
-// writeVersion writes the version to the repo
178
-func writeVersion(repoPath string, version string) error {
179
- versionPath := filepath.Join(repoPath, "version")
180
- return os.WriteFile(versionPath, []byte(version), 0644)
39
+// NewMigration creates a new migration instance (for compatibility)
40
+func NewMigration() common.Migration {
41
+ return Migration
42
}
43
44
// convert converts the config from version 16 to 17
184
-func convert(in io.Reader, out io.Writer, repoPath string) error {
185
- confMap := make(map[string]any)
186
- if err := json.NewDecoder(in).Decode(&confMap); err != nil {
45
+func convert(in io.ReadSeeker, out io.Writer) error {
46
+ confMap, err := common.ReadConfig(in)
47
+ if err != nil {
48
return err
49
}
50
@@ -193,7 +54,7 @@ func convert(in io.Reader, out io.Writer, repoPath string) error {
54
}
55
56
// Migrate Bootstrap peers
196
- if err := migrateBootstrap(confMap, repoPath); err != nil {
57
+ if err := migrateBootstrap(confMap); err != nil {
58
return err
59
}
60
@@ -213,88 +74,62 @@ func convert(in io.Reader, out io.Writer, repoPath string) error {
74
}
75
76
// Save new config
216
- fixed, err := json.MarshalIndent(confMap, "", " ")
217
- if err != nil {
218
- return err
219
- }
220
-
221
- if _, err := out.Write(fixed); err != nil {
222
- return err
223
- }
224
- _, err = out.Write([]byte("\n"))
225
- return err
77
+ return common.WriteConfig(out, confMap)
78
}
79
80
// enableAutoConf adds AutoConf section to config
81
func enableAutoConf(confMap map[string]any) error {
230
- // Check if AutoConf already exists
231
- if _, exists := confMap["AutoConf"]; exists {
232
- return nil
233
- }
234
-
235
- // Add empty AutoConf section - all fields will use implicit defaults:
82
+ // Add empty AutoConf section if it doesn't exist - all fields will use implicit defaults:
83
// - Enabled defaults to true (via DefaultAutoConfEnabled)
84
// - URL defaults to mainnet URL (via DefaultAutoConfURL)
85
// - RefreshInterval defaults to 24h (via DefaultAutoConfRefreshInterval)
86
// - TLSInsecureSkipVerify defaults to false (no WithDefault, but false is zero value)
240
- confMap["AutoConf"] = map[string]any{}
241
-
87
+ common.SetDefault(confMap, "AutoConf", map[string]any{})
88
return nil
89
}
90
91
// migrateBootstrap migrates bootstrap peers to use "auto"
246
-func migrateBootstrap(confMap map[string]any, repoPath string) error {
92
+func migrateBootstrap(confMap map[string]any) error {
93
bootstrap, exists := confMap["Bootstrap"]
94
if !exists {
95
// No bootstrap section, add "auto"
250
- confMap["Bootstrap"] = []string{"auto"}
96
+ confMap["Bootstrap"] = []string{config.AutoPlaceholder}
97
return nil
98
}
99
254
- bootstrapSlice, ok := bootstrap.([]interface{})
255
- if !ok {
100
+ // Convert to string slice using helper
101
+ bootstrapPeers := common.ConvertInterfaceSlice(common.SafeCastSlice(bootstrap))
102
+ if len(bootstrapPeers) == 0 && bootstrap != nil {
103
// Invalid bootstrap format, replace with "auto"
257
- confMap["Bootstrap"] = []string{"auto"}
104
+ confMap["Bootstrap"] = []string{config.AutoPlaceholder}
105
return nil
106
}
107
261
- // Convert to string slice
262
- var bootstrapPeers []string
263
- for _, peer := range bootstrapSlice {
264
- if peerStr, ok := peer.(string); ok {
265
- bootstrapPeers = append(bootstrapPeers, peerStr)
266
- }
267
- }
268
-
269
- // Check if we should replace with "auto"
270
- newBootstrap := processBootstrapPeers(bootstrapPeers, repoPath)
108
+ // Process bootstrap peers according to migration rules
109
+ newBootstrap := processBootstrapPeers(bootstrapPeers)
110
confMap["Bootstrap"] = newBootstrap
111
112
return nil
113
}
114
115
// processBootstrapPeers processes bootstrap peers according to migration rules
277
-func processBootstrapPeers(peers []string, repoPath string) []string {
116
+func processBootstrapPeers(peers []string) []string {
117
// If empty, use "auto"
118
if len(peers) == 0 {
280
- return []string{"auto"}
119
+ return []string{config.AutoPlaceholder}
120
}
121
283
- // Separate default peers from custom ones
284
- var customPeers []string
285
- var hasDefaultPeers bool
122
+ // Filter out default peers to get only custom ones
123
+ customPeers := slices.DeleteFunc(slices.Clone(peers), func(peer string) bool {
124
+ return slices.Contains(DefaultBootstrapAddresses, peer)
125
+ })
126
287
- for _, peer := range peers {
288
- if slices.Contains(DefaultBootstrapAddresses, peer) {
289
- hasDefaultPeers = true
290
- } else {
291
- customPeers = append(customPeers, peer)
292
- }
293
- }
127
+ // Check if any default peers were removed
128
+ hasDefaultPeers := len(customPeers) < len(peers)
129
130
// If we have default peers, replace them with "auto"
131
if hasDefaultPeers {
297
- return append([]string{"auto"}, customPeers...)
132
+ return append([]string{config.AutoPlaceholder}, customPeers...)
133
}
134
135
// No default peers found, keep as is
@@ -303,68 +138,25 @@ func processBootstrapPeers(peers []string, repoPath string) []string {
138
139
// migrateDNSResolvers migrates DNS resolvers to use "auto" for "." eTLD
140
func migrateDNSResolvers(confMap map[string]any) error {
306
- dnsSection, exists := confMap["DNS"]
307
- if !exists {
308
- // No DNS section, create it with "auto"
309
- confMap["DNS"] = map[string]any{
310
- "Resolvers": map[string]string{
311
- ".": config.AutoPlaceholder,
312
- },
313
- }
314
- return nil
315
- }
316
-
317
- dns, ok := dnsSection.(map[string]any)
318
- if !ok {
319
- // Invalid DNS format, replace with "auto"
320
- confMap["DNS"] = map[string]any{
321
- "Resolvers": map[string]string{
322
- ".": config.AutoPlaceholder,
323
- },
324
- }
325
- return nil
326
- }
141
+ // Get or create DNS section
142
+ dns := common.GetOrCreateSection(confMap, "DNS")
143
328
- resolvers, exists := dns["Resolvers"]
329
- if !exists {
330
- // No resolvers, add "auto"
331
- dns["Resolvers"] = map[string]string{
332
- ".": config.AutoPlaceholder,
333
- }
334
- return nil
335
- }
144
+ // Get existing resolvers or create empty map
145
+ resolvers := common.SafeCastMap(dns["Resolvers"])
146
337
- resolversMap, ok := resolvers.(map[string]any)
338
- if !ok {
339
- // Invalid resolvers format, replace with "auto"
340
- dns["Resolvers"] = map[string]string{
341
- ".": config.AutoPlaceholder,
342
- }
343
- return nil
344
- }
345
-
346
- // Convert to string map and replace default resolvers with "auto"
347
- stringResolvers := make(map[string]string)
147
+ // Define default resolvers that should be replaced with "auto"
148
defaultResolvers := map[string]string{
349
- "https://dns.eth.limo/dns-query": "auto",
350
- "https://dns.eth.link/dns-query": "auto",
351
- "https://resolver.cloudflare-eth.com/dns-query": "auto",
149
+ "https://dns.eth.limo/dns-query": config.AutoPlaceholder,
150
+ "https://dns.eth.link/dns-query": config.AutoPlaceholder,
151
+ "https://resolver.cloudflare-eth.com/dns-query": config.AutoPlaceholder,
152
}
153
354
- for k, v := range resolversMap {
355
- if vStr, ok := v.(string); ok {
356
- // Check if this is a default resolver that should be replaced
357
- if replacement, isDefault := defaultResolvers[vStr]; isDefault {
358
- stringResolvers[k] = replacement
359
- } else {
360
- stringResolvers[k] = vStr
361
- }
362
- }
363
- }
154
+ // Replace default resolvers with "auto"
155
+ stringResolvers := common.ReplaceDefaultsWithAuto(resolvers, defaultResolvers)
156
365
- // If "." is not set or empty, set it to "auto"
157
+ // Ensure "." is set to "auto" if not already set
158
if _, exists := stringResolvers["."]; !exists {
367
- stringResolvers["."] = "auto"
159
+ stringResolvers["."] = config.AutoPlaceholder
160
}
161
162
dns["Resolvers"] = stringResolvers
@@ -373,120 +165,57 @@ func migrateDNSResolvers(confMap map[string]any) error {
165
166
// migrateDelegatedRouters migrates DelegatedRouters to use "auto"
167
func migrateDelegatedRouters(confMap map[string]any) error {
376
- routing, exists := confMap["Routing"]
377
- if !exists {
378
- // No routing section, create it with "auto"
379
- confMap["Routing"] = map[string]any{
380
- "DelegatedRouters": []string{"auto"},
381
- }
382
- return nil
383
- }
168
+ // Get or create Routing section
169
+ routing := common.GetOrCreateSection(confMap, "Routing")
170
385
- routingMap, ok := routing.(map[string]any)
386
- if !ok {
387
- // Invalid routing format, replace with "auto"
388
- confMap["Routing"] = map[string]any{
389
- "DelegatedRouters": []string{"auto"},
390
- }
391
- return nil
392
- }
393
-
394
- delegatedRouters, exists := routingMap["DelegatedRouters"]
395
- if !exists {
396
- // No delegated routers, add "auto"
397
- routingMap["DelegatedRouters"] = []string{"auto"}
398
- return nil
399
- }
171
+ // Get existing delegated routers
172
+ delegatedRouters, exists := routing["DelegatedRouters"]
173
174
// Check if it's empty or nil
402
- if shouldReplaceWithAuto(delegatedRouters) {
403
- routingMap["DelegatedRouters"] = []string{"auto"}
175
+ if !exists || common.IsEmptySlice(delegatedRouters) {
176
+ routing["DelegatedRouters"] = []string{config.AutoPlaceholder}
177
return nil
178
}
179
180
// Process the list to replace cid.contact with "auto" and preserve others
408
- if slice, ok := delegatedRouters.([]interface{}); ok {
409
- var newRouters []string
410
- hasAuto := false
411
-
412
- for _, router := range slice {
413
- if routerStr, ok := router.(string); ok {
414
- if routerStr == "https://cid.contact" {
415
- if !hasAuto {
416
- newRouters = append(newRouters, "auto")
417
- hasAuto = true
418
- }
419
- } else {
420
- newRouters = append(newRouters, routerStr)
421
- }
181
+ routers := common.ConvertInterfaceSlice(common.SafeCastSlice(delegatedRouters))
182
+ var newRouters []string
183
+ hasAuto := false
184
+
185
+ for _, router := range routers {
186
+ if router == "https://cid.contact" {
187
+ if !hasAuto {
188
+ newRouters = append(newRouters, config.AutoPlaceholder)
189
+ hasAuto = true
190
}
191
+ } else {
192
+ newRouters = append(newRouters, router)
193
}
194
+ }
195
425
- // If empty after processing, add "auto"
426
- if len(newRouters) == 0 {
427
- newRouters = []string{"auto"}
428
- }
429
-
430
- routingMap["DelegatedRouters"] = newRouters
196
+ // If empty after processing, add "auto"
197
+ if len(newRouters) == 0 {
198
+ newRouters = []string{config.AutoPlaceholder}
199
}
200
201
+ routing["DelegatedRouters"] = newRouters
202
return nil
203
}
204
205
// migrateDelegatedPublishers migrates DelegatedPublishers to use "auto"
206
func migrateDelegatedPublishers(confMap map[string]any) error {
438
- ipns, exists := confMap["Ipns"]
439
- if !exists {
440
- // No IPNS section, create it with "auto"
441
- confMap["Ipns"] = map[string]any{
442
- "DelegatedPublishers": []string{"auto"},
443
- }
444
- return nil
445
- }
446
-
447
- ipnsMap, ok := ipns.(map[string]any)
448
- if !ok {
449
- // Invalid IPNS format, replace with "auto"
450
- confMap["Ipns"] = map[string]any{
451
- "DelegatedPublishers": []string{"auto"},
452
- }
453
- return nil
454
- }
207
+ // Get or create Ipns section
208
+ ipns := common.GetOrCreateSection(confMap, "Ipns")
209
456
- delegatedPublishers, exists := ipnsMap["DelegatedPublishers"]
457
- if !exists {
458
- // No delegated publishers, add "auto"
459
- ipnsMap["DelegatedPublishers"] = []string{"auto"}
460
- return nil
461
- }
210
+ // Get existing delegated publishers
211
+ delegatedPublishers, exists := ipns["DelegatedPublishers"]
212
213
// Check if it's empty or nil - only then replace with "auto"
214
// Otherwise preserve custom publishers
465
- if shouldReplaceWithAuto(delegatedPublishers) {
466
- ipnsMap["DelegatedPublishers"] = []string{"auto"}
215
+ if !exists || common.IsEmptySlice(delegatedPublishers) {
216
+ ipns["DelegatedPublishers"] = []string{config.AutoPlaceholder}
217
}
218
// If there are custom publishers, leave them as is
219
220
return nil
221
}
472
-
473
-// shouldReplaceWithAuto checks if a field should be replaced with "auto"
474
-func shouldReplaceWithAuto(field any) bool {
475
- // If it's nil, replace with "auto"
476
- if field == nil {
477
- return true
478
- }
479
-
480
- // If it's an empty slice, replace with "auto"
481
- if slice, ok := field.([]interface{}); ok {
482
- return len(slice) == 0
483
- }
484
-
485
- // If it's an empty array, replace with "auto"
486
- if reflect.TypeOf(field).Kind() == reflect.Slice {
487
- v := reflect.ValueOf(field)
488
- return v.Len() == 0
489
- }
490
-
491
- return false
492
-}
repo/fsrepo/migrations/fs-repo-16-to-17/migration/migration_test.go
+6
-8
@@ -7,6 +7,7 @@ import (
7
"path/filepath"
8
"testing"
9
10
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
11
"github.com/stretchr/testify/assert"
12
"github.com/stretchr/testify/require"
13
)
@@ -15,9 +16,7 @@ import (
16
func runMigrationOnJSON(t *testing.T, input string) map[string]interface{} {
17
t.Helper()
18
var output bytes.Buffer
18
- // Use t.TempDir() for test isolation and parallel execution support
19
- tempDir := t.TempDir()
20
- err := convert(bytes.NewReader([]byte(input)), &output, tempDir)
19
+ err := convert(bytes.NewReader([]byte(input)), &output)
20
require.NoError(t, err)
21
22
var result map[string]interface{}
@@ -137,13 +136,12 @@ func TestMigration(t *testing.T) {
136
require.NoError(t, err)
137
138
// Run migration
140
- migration := &Migration{}
141
- opts := Options{
139
+ opts := common.Options{
140
Path: tempDir,
141
Verbose: true,
142
}
143
146
- err = migration.Apply(opts)
144
+ err = Migration.Apply(opts)
145
require.NoError(t, err)
146
147
// Verify version was updated
@@ -191,7 +189,7 @@ func TestMigration(t *testing.T) {
189
assert.Equal(t, "auto", delegatedPublishers[0], "Expected DelegatedPublishers to be ['auto']")
190
191
// Test revert
194
- err = migration.Revert(opts)
192
+ err = Migration.Revert(opts)
193
require.NoError(t, err)
194
195
// Verify version was reverted
@@ -273,7 +271,7 @@ func TestBootstrapMigration(t *testing.T) {
271
for _, tt := range tests {
272
t.Run(tt.name, func(t *testing.T) {
273
t.Parallel()
276
- result := processBootstrapPeers(tt.peers, "")
274
+ result := processBootstrapPeers(tt.peers)
275
require.Equal(t, len(tt.expected), len(result), "Expected %d peers, got %d", len(tt.expected), len(result))
276
for i, expected := range tt.expected {
277
assert.Equal(t, expected, result[i], "Expected peer %d to be %s", i, expected)
repo/fsrepo/migrations/fs-repo-17-to-18/main.go
new
+60
@@ -0,0 +1,60 @@
1
+// Package main implements fs-repo-17-to-18 migration for IPFS repositories.
2
+//
3
+// This migration consolidates the Provider and Reprovider configurations into
4
+// a unified Provide configuration section.
5
+//
6
+// Changes made:
7
+// - Migrates Provider.Enabled to Provide.Enabled
8
+// - Migrates Provider.WorkerCount to Provide.DHT.MaxWorkers
9
+// - Migrates Reprovider.Strategy to Provide.Strategy (converts "flat" to "all")
10
+// - Migrates Reprovider.Interval to Provide.DHT.Interval
11
+// - Removes deprecated Provider and Reprovider sections
12
+//
13
+// The migration is reversible and creates config.17-to-18.bak for rollback.
14
+//
15
+// Usage:
16
+//
17
+// fs-repo-17-to-18 -path /path/to/ipfs/repo [-verbose] [-revert]
18
+//
19
+// This migration is embedded in Kubo and runs automatically during daemon startup.
20
+// This standalone binary is provided for manual migration scenarios.
21
+package main
22
+
23
+import (
24
+ "flag"
25
+ "fmt"
26
+ "os"
27
+
28
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
29
+ mg17 "github.com/ipfs/kubo/repo/fsrepo/migrations/fs-repo-17-to-18/migration"
30
+)
31
+
32
+func main() {
33
+ var path = flag.String("path", "", "Path to IPFS repository")
34
+ var verbose = flag.Bool("verbose", false, "Enable verbose output")
35
+ var revert = flag.Bool("revert", false, "Revert migration")
36
+ flag.Parse()
37
+
38
+ if *path == "" {
39
+ fmt.Fprintf(os.Stderr, "Error: -path flag is required\n")
40
+ flag.Usage()
41
+ os.Exit(1)
42
+ }
43
+
44
+ opts := common.Options{
45
+ Path: *path,
46
+ Verbose: *verbose,
47
+ }
48
+
49
+ var err error
50
+ if *revert {
51
+ err = mg17.Migration.Revert(opts)
52
+ } else {
53
+ err = mg17.Migration.Apply(opts)
54
+ }
55
+
56
+ if err != nil {
57
+ fmt.Fprintf(os.Stderr, "Migration failed: %v\n", err)
58
+ os.Exit(1)
59
+ }
60
+}
repo/fsrepo/migrations/fs-repo-17-to-18/migration/migration.go
new
+121
@@ -0,0 +1,121 @@
1
+// package mg17 contains the code to perform 17-18 repository migration in Kubo.
2
+// This handles the following:
3
+// - Migrate Provider and Reprovider configs to unified Provide config
4
+// - Clear deprecated Provider and Reprovider fields
5
+// - Increment repo version to 18
6
+package mg17
7
+
8
+import (
9
+ "fmt"
10
+ "io"
11
+
12
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
13
+)
14
+
15
+// Migration is the main exported migration for 17-to-18
16
+var Migration = &common.BaseMigration{
17
+ FromVersion: "17",
18
+ ToVersion: "18",
19
+ Description: "Migrating Provider and Reprovider configuration to unified Provide configuration",
20
+ Convert: convert,
21
+}
22
+
23
+// NewMigration creates a new migration instance (for compatibility)
24
+func NewMigration() common.Migration {
25
+ return Migration
26
+}
27
+
28
+// convert performs the actual configuration transformation
29
+func convert(in io.ReadSeeker, out io.Writer) error {
30
+ // Read the configuration
31
+ confMap, err := common.ReadConfig(in)
32
+ if err != nil {
33
+ return err
34
+ }
35
+
36
+ // Create new Provide section with DHT subsection from Provider and Reprovider
37
+ provide := make(map[string]any)
38
+ dht := make(map[string]any)
39
+ hasNonDefaultValues := false
40
+
41
+ // Migrate Provider fields if they exist
42
+ provider := common.SafeCastMap(confMap["Provider"])
43
+ if enabled, exists := provider["Enabled"]; exists {
44
+ provide["Enabled"] = enabled
45
+ // Log migration for non-default values
46
+ if enabledBool, ok := enabled.(bool); ok && !enabledBool {
47
+ fmt.Printf(" Migrated Provider.Enabled=%v to Provide.Enabled=%v\n", enabledBool, enabledBool)
48
+ hasNonDefaultValues = true
49
+ }
50
+ }
51
+ if workerCount, exists := provider["WorkerCount"]; exists {
52
+ dht["MaxWorkers"] = workerCount
53
+ // Log migration for all worker count values
54
+ if count, ok := workerCount.(float64); ok {
55
+ fmt.Printf(" Migrated Provider.WorkerCount=%v to Provide.DHT.MaxWorkers=%v\n", int(count), int(count))
56
+ hasNonDefaultValues = true
57
+
58
+ // Additional guidance for high WorkerCount
59
+ if count > 5 {
60
+ fmt.Printf(" ⚠️ For better resource utilization, consider enabling Provide.DHT.SweepEnabled=true\n")
61
+ fmt.Printf(" and adjusting Provide.DHT.DedicatedBurstWorkers if announcement of new CIDs\n")
62
+ fmt.Printf(" should take priority over periodic reprovide interval.\n")
63
+ }
64
+ }
65
+ }
66
+ // Note: Skip Provider.Strategy as it was unused
67
+
68
+ // Migrate Reprovider fields if they exist
69
+ reprovider := common.SafeCastMap(confMap["Reprovider"])
70
+ if strategy, exists := reprovider["Strategy"]; exists {
71
+ if strategyStr, ok := strategy.(string); ok {
72
+ // Convert deprecated "flat" strategy to "all"
73
+ if strategyStr == "flat" {
74
+ provide["Strategy"] = "all"
75
+ fmt.Printf(" Migrated deprecated Reprovider.Strategy=\"flat\" to Provide.Strategy=\"all\"\n")
76
+ } else {
77
+ // Migrate any other strategy value as-is
78
+ provide["Strategy"] = strategyStr
79
+ fmt.Printf(" Migrated Reprovider.Strategy=\"%s\" to Provide.Strategy=\"%s\"\n", strategyStr, strategyStr)
80
+ }
81
+ hasNonDefaultValues = true
82
+ } else {
83
+ // Not a string, set to default "all" to ensure valid config
84
+ provide["Strategy"] = "all"
85
+ fmt.Printf(" Warning: Reprovider.Strategy was not a string, setting Provide.Strategy=\"all\"\n")
86
+ hasNonDefaultValues = true
87
+ }
88
+ }
89
+ if interval, exists := reprovider["Interval"]; exists {
90
+ dht["Interval"] = interval
91
+ // Log migration for non-default intervals
92
+ if intervalStr, ok := interval.(string); ok && intervalStr != "22h" && intervalStr != "" {
93
+ fmt.Printf(" Migrated Reprovider.Interval=\"%s\" to Provide.DHT.Interval=\"%s\"\n", intervalStr, intervalStr)
94
+ hasNonDefaultValues = true
95
+ }
96
+ }
97
+ // Note: Sweep is a new field introduced in v0.38, not present in v0.37
98
+ // So we don't need to migrate it from Reprovider
99
+
100
+ // Set the DHT section if we have any DHT fields to migrate
101
+ if len(dht) > 0 {
102
+ provide["DHT"] = dht
103
+ }
104
+
105
+ // Set the new Provide section if we have any fields to migrate
106
+ if len(provide) > 0 {
107
+ confMap["Provide"] = provide
108
+ }
109
+
110
+ // Clear old Provider and Reprovider sections
111
+ delete(confMap, "Provider")
112
+ delete(confMap, "Reprovider")
113
+
114
+ // Print documentation link if we migrated any non-default values
115
+ if hasNonDefaultValues {
116
+ fmt.Printf(" See: https://github.com/ipfs/kubo/blob/master/docs/config.md#provide\n")
117
+ }
118
+
119
+ // Write the updated config
120
+ return common.WriteConfig(out, confMap)
121
+}
repo/fsrepo/migrations/fs-repo-17-to-18/migration/migration_test.go
new
+176
@@ -0,0 +1,176 @@
1
+package mg17
2
+
3
+import (
4
+ "testing"
5
+
6
+ "github.com/ipfs/kubo/repo/fsrepo/migrations/common"
7
+)
8
+
9
+func TestMigration17to18(t *testing.T) {
10
+ migration := NewMigration()
11
+
12
+ testCases := []common.TestCase{
13
+ {
14
+ Name: "Migrate Provider and Reprovider to Provide",
15
+ InputConfig: common.GenerateTestConfig(map[string]any{
16
+ "Provider": map[string]any{
17
+ "Enabled": true,
18
+ "WorkerCount": 8,
19
+ "Strategy": "unused", // This field was unused and should be ignored
20
+ },
21
+ "Reprovider": map[string]any{
22
+ "Strategy": "pinned",
23
+ "Interval": "12h",
24
+ },
25
+ }),
26
+ Assertions: []common.ConfigAssertion{
27
+ {Path: "Provide.Enabled", Expected: true},
28
+ {Path: "Provide.DHT.MaxWorkers", Expected: float64(8)}, // JSON unmarshals to float64
29
+ {Path: "Provide.Strategy", Expected: "pinned"},
30
+ {Path: "Provide.DHT.Interval", Expected: "12h"},
31
+ {Path: "Provider", Expected: nil}, // Should be deleted
32
+ {Path: "Reprovider", Expected: nil}, // Should be deleted
33
+ },
34
+ },
35
+ {
36
+ Name: "Convert flat strategy to all",
37
+ InputConfig: common.GenerateTestConfig(map[string]any{
38
+ "Provider": map[string]any{
39
+ "Enabled": false,
40
+ },
41
+ "Reprovider": map[string]any{
42
+ "Strategy": "flat", // Deprecated, should be converted to "all"
43
+ "Interval": "24h",
44
+ },
45
+ }),
46
+ Assertions: []common.ConfigAssertion{
47
+ {Path: "Provide.Enabled", Expected: false},
48
+ {Path: "Provide.Strategy", Expected: "all"}, // "flat" converted to "all"
49
+ {Path: "Provide.DHT.Interval", Expected: "24h"},
50
+ {Path: "Provider", Expected: nil},
51
+ {Path: "Reprovider", Expected: nil},
52
+ },
53
+ },
54
+ {
55
+ Name: "Handle missing Provider section",
56
+ InputConfig: common.GenerateTestConfig(map[string]any{
57
+ "Reprovider": map[string]any{
58
+ "Strategy": "roots",
59
+ "Interval": "6h",
60
+ },
61
+ }),
62
+ Assertions: []common.ConfigAssertion{
63
+ {Path: "Provide.Strategy", Expected: "roots"},
64
+ {Path: "Provide.DHT.Interval", Expected: "6h"},
65
+ {Path: "Provider", Expected: nil},
66
+ {Path: "Reprovider", Expected: nil},
67
+ },
68
+ },
69
+ {
70
+ Name: "Handle missing Reprovider section",
71
+ InputConfig: common.GenerateTestConfig(map[string]any{
72
+ "Provider": map[string]any{
73
+ "Enabled": true,
74
+ "WorkerCount": 16,
75
+ },
76
+ }),
77
+ Assertions: []common.ConfigAssertion{
78
+ {Path: "Provide.Enabled", Expected: true},
79
+ {Path: "Provide.DHT.MaxWorkers", Expected: float64(16)},
80
+ {Path: "Provider", Expected: nil},
81
+ {Path: "Reprovider", Expected: nil},
82
+ },
83
+ },
84
+ {
85
+ Name: "Handle empty Provider and Reprovider sections",
86
+ InputConfig: common.GenerateTestConfig(map[string]any{
87
+ "Provider": map[string]any{},
88
+ "Reprovider": map[string]any{},
89
+ }),
90
+ Assertions: []common.ConfigAssertion{
91
+ {Path: "Provide", Expected: nil}, // No fields to migrate
92
+ {Path: "Provider", Expected: nil},
93
+ {Path: "Reprovider", Expected: nil},
94
+ },
95
+ },
96
+ {
97
+ Name: "Handle missing both sections",
98
+ InputConfig: common.GenerateTestConfig(map[string]any{
99
+ "Datastore": map[string]any{
100
+ "StorageMax": "10GB",
101
+ },
102
+ }),
103
+ Assertions: []common.ConfigAssertion{
104
+ {Path: "Provide", Expected: nil}, // No Provider/Reprovider to migrate
105
+ {Path: "Provider", Expected: nil},
106
+ {Path: "Reprovider", Expected: nil},
107
+ {Path: "Datastore.StorageMax", Expected: "10GB"}, // Other config preserved
108
+ },
109
+ },
110
+ {
111
+ Name: "Preserve other config sections",
112
+ InputConfig: common.GenerateTestConfig(map[string]any{
113
+ "Provider": map[string]any{
114
+ "Enabled": true,
115
+ },
116
+ "Reprovider": map[string]any{
117
+ "Strategy": "all",
118
+ },
119
+ "Swarm": map[string]any{
120
+ "ConnMgr": map[string]any{
121
+ "Type": "basic",
122
+ },
123
+ },
124
+ }),
125
+ Assertions: []common.ConfigAssertion{
126
+ {Path: "Provide.Enabled", Expected: true},
127
+ {Path: "Provide.Strategy", Expected: "all"},
128
+ {Path: "Swarm.ConnMgr.Type", Expected: "basic"}, // Other config preserved
129
+ {Path: "Provider", Expected: nil},
130
+ {Path: "Reprovider", Expected: nil},
131
+ },
132
+ },
133
+ }
134
+
135
+ for _, tc := range testCases {
136
+ t.Run(tc.Name, func(t *testing.T) {
137
+ common.RunMigrationTest(t, migration, tc)
138
+ })
139
+ }
140
+}
141
+
142
+func TestMigration17to18Reversible(t *testing.T) {
143
+ migration := NewMigration()
144
+
145
+ // Test that migration is reversible
146
+ inputConfig := common.GenerateTestConfig(map[string]any{
147
+ "Provide": map[string]any{
148
+ "Enabled": true,
149
+ "WorkerCount": 8,
150
+ "Strategy": "pinned",
151
+ "Interval": "12h",
152
+ },
153
+ })
154
+
155
+ // Test full migration and revert
156
+ migratedConfig := common.AssertMigrationSuccess(t, migration, 17, 18, inputConfig)
157
+
158
+ // Check that Provide section exists after migration
159
+ common.AssertConfigField(t, migratedConfig, "Provide.Enabled", true)
160
+
161
+ // Test revert
162
+ common.AssertMigrationReversible(t, migration, 17, 18, migratedConfig)
163
+}
164
+
165
+func TestMigration17to18Integration(t *testing.T) {
166
+ migration := NewMigration()
167
+
168
+ // Test that the migration properly integrates with the common framework
169
+ if migration.Versions() != "17-to-18" {
170
+ t.Errorf("expected versions '17-to-18', got '%s'", migration.Versions())
171
+ }
172
+
173
+ if !migration.Reversible() {
174
+ t.Error("migration should be reversible")
175
+ }
176
+}
test/cli/autoconf/expand_test.go
+4
-4
@@ -337,8 +337,8 @@ func testExpandAutoFiltersUnsupportedPathsDelegated(t *testing.T) {
337
node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"})
338
node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"})
339
// Disable content providing when using delegated routing
340
- node.SetIPFSConfig("Provider.Enabled", false)
341
- node.SetIPFSConfig("Reprovider.Interval", "0")
340
+ node.SetIPFSConfig("Provide.Enabled", false)
341
+ node.SetIPFSConfig("Provide.DHT.Interval", "0")
342
343
// Load test autoconf data with unsupported paths
344
autoConfData := loadTestDataExpand(t, "autoconf_with_unsupported_paths.json")
@@ -421,8 +421,8 @@ func testExpandAutoWithoutCacheDelegated(t *testing.T) {
421
node.SetIPFSConfig("Routing.DelegatedRouters", []string{"auto"})
422
node.SetIPFSConfig("Ipns.DelegatedPublishers", []string{"auto"})
423
// Disable content providing when using delegated routing
424
- node.SetIPFSConfig("Provider.Enabled", false)
425
- node.SetIPFSConfig("Reprovider.Interval", "0")
424
+ node.SetIPFSConfig("Provide.Enabled", false)
425
+ node.SetIPFSConfig("Provide.DHT.Interval", "0")
426
427
// Load test autoconf data with unsupported paths (this won't be used since no daemon)
428
autoConfData := loadTestDataExpand(t, "autoconf_with_unsupported_paths.json")
test/cli/autoconf/ipns_test.go
+2
-2
@@ -200,8 +200,8 @@ func setupNodeWithAutoconf(t *testing.T, publisherURL string, routingType string
200
201
// Additional config for delegated routing mode
202
if routingType == "delegated" {
203
- node.SetIPFSConfig("Provider.Enabled", false)
204
- node.SetIPFSConfig("Reprovider.Interval", "0s")
203
+ node.SetIPFSConfig("Provide.Enabled", false)
204
+ node.SetIPFSConfig("Provide.DHT.Interval", "0s")
205
}
206
207
// Add bootstrap peers for connectivity
test/cli/migrations/migration_16_to_latest_test.go
renamed
+103
-32
@@ -1,8 +1,6 @@
1
package migrations
2
3
// NOTE: These migration tests require the local Kubo binary (built with 'make build') to be in PATH.
4
-// The tests migrate from repo version 16 to 17, which requires Kubo version 0.37.0+ (expects repo v17).
5
-// If using system ipfs binary v0.36.0 or older (expects repo v16), no migration will be triggered.
4
//
5
// To run these tests successfully:
6
// export PATH="$(pwd)/cmd/ipfs:$PATH"
@@ -12,6 +10,7 @@ import (
10
"bufio"
11
"context"
12
"encoding/json"
13
+ "fmt"
14
"io"
15
"os"
16
"os/exec"
@@ -20,11 +19,28 @@ import (
19
"testing"
20
"time"
21
22
+ ipfs "github.com/ipfs/kubo"
23
"github.com/ipfs/kubo/test/cli/harness"
24
"github.com/stretchr/testify/require"
25
)
26
27
-func TestMigration16To17(t *testing.T) {
27
+// TestMigration16ToLatest tests migration from repo version 16 to the latest version.
28
+//
29
+// This test uses a real IPFS repository snapshot from Kubo v0.36.0 (the last version that used repo v16).
30
+// The intention is to confirm that users can upgrade from Kubo v0.36.0 to the latest version by applying
31
+// all intermediate migrations successfully.
32
+//
33
+// NOTE: This test comprehensively tests all migration methods (daemon --migrate, repo migrate,
34
+// and reverse migration) because 16-to-17 was the first embedded migration that did not fetch
35
+// external files. It serves as a reference implementation for migration testing.
36
+//
37
+// Future migrations can have simplified tests (like 17-to-18 in migration_17_to_latest_test.go)
38
+// that focus on specific migration logic rather than testing all migration methods.
39
+//
40
+// If you need to test migration of configuration keys that appeared in later repo versions,
41
+// create a new test file migration_N_to_latest_test.go with a separate IPFS repository test vector
42
+// from the appropriate Kubo version.
43
+func TestMigration16ToLatest(t *testing.T) {
44
t.Parallel()
45
46
// Primary tests using 'ipfs daemon --migrate' command (default in Docker)
@@ -71,12 +87,13 @@ func testDaemonMigrationWithAuto(t *testing.T) {
87
// Verify migration was successful based on monitoring
88
require.True(t, migrationSuccess, "Migration should have been successful")
89
require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered")
74
- require.Contains(t, stdoutOutput, "Migration 16 to 17 succeeded", "Migration should have completed successfully")
90
+ require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully")
91
76
- // Verify version was updated to 17
92
+ // Verify version was updated to latest
93
versionData, err := os.ReadFile(versionPath)
94
require.NoError(t, err)
79
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Version should be updated to 17")
95
+ expectedVersion := fmt.Sprint(ipfs.RepoVersion)
96
+ require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion)
97
98
// Verify migration results using DRY helper
99
helper := NewMigrationTestHelper(t, configPath)
@@ -131,7 +148,7 @@ func testDaemonMigrationWithoutAuto(t *testing.T) {
148
// Verify migration was successful based on monitoring
149
require.True(t, migrationSuccess, "Migration should have been successful")
150
require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered")
134
- require.Contains(t, stdoutOutput, "Migration 16 to 17 succeeded", "Migration should have completed successfully")
151
+ require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully")
152
153
// Verify migration results: custom values preserved alongside "auto"
154
helper := NewMigrationTestHelper(t, configPath)
@@ -487,12 +504,13 @@ func testDaemonMissingFieldsHandling(t *testing.T) {
504
// Verify migration was successful
505
require.True(t, migrationSuccess, "Migration should have been successful")
506
require.Contains(t, stdoutOutput, "applying 16-to-17 repo migration", "Migration should have been triggered")
490
- require.Contains(t, stdoutOutput, "Migration 16 to 17 succeeded", "Migration should have completed successfully")
507
+ require.Contains(t, stdoutOutput, "Migration 16-to-17 succeeded", "Migration should have completed successfully")
508
492
- // Verify version was updated
509
+ // Verify version was updated to latest
510
versionData, err := os.ReadFile(versionPath)
511
require.NoError(t, err)
495
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Version should be updated to 17")
512
+ expectedVersion := fmt.Sprint(ipfs.RepoVersion)
513
+ require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion)
514
515
// Verify migration adds all required fields to minimal config
516
NewMigrationTestHelper(t, configPath).
@@ -543,10 +561,11 @@ func testRepoBackwardMigration(t *testing.T) {
561
result := node.RunIPFS("repo", "migrate")
562
require.Empty(t, result.Stderr.String(), "Forward migration should succeed")
563
546
- // Verify we're at v17
564
+ // Verify we're at the latest version
565
versionData, err := os.ReadFile(versionPath)
566
require.NoError(t, err)
549
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Should be at version 17 after forward migration")
567
+ expectedVersion := fmt.Sprint(ipfs.RepoVersion)
568
+ require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Should be at version %s (latest) after forward migration", expectedVersion)
569
570
// Now run reverse migration back to v16
571
result = node.RunIPFS("repo", "migrate", "--to=16", "--allow-downgrade")
@@ -565,18 +584,40 @@ func testRepoBackwardMigration(t *testing.T) {
584
585
// runDaemonMigrationWithMonitoring starts daemon --migrate, monitors output until "Daemon is ready",
586
// then gracefully shuts down the daemon and returns the captured output and success status.
568
-// This is a generic helper that can monitor for any migration patterns.
587
+// This monitors for all expected migrations from version 16 to latest.
588
func runDaemonMigrationWithMonitoring(t *testing.T, node *harness.Node) (string, bool) {
570
- // Use specific patterns for 16-to-17 migration
571
- return runDaemonWithMigrationMonitoring(t, node, "applying 16-to-17 repo migration", "Migration 16 to 17 succeeded")
589
+ // Monitor migrations from repo v16 to latest
590
+ return runDaemonWithExpectedMigrations(t, node, 16, ipfs.RepoVersion)
591
+}
592
+
593
+// runDaemonWithExpectedMigrations monitors daemon startup for a sequence of migrations from startVersion to endVersion
594
+func runDaemonWithExpectedMigrations(t *testing.T, node *harness.Node, startVersion, endVersion int) (string, bool) {
595
+ // Build list of expected migrations
596
+ var expectedMigrations []struct {
597
+ pattern string
598
+ success string
599
+ }
600
+
601
+ for v := startVersion; v < endVersion; v++ {
602
+ from := v
603
+ to := v + 1
604
+ expectedMigrations = append(expectedMigrations, struct {
605
+ pattern string
606
+ success string
607
+ }{
608
+ pattern: fmt.Sprintf("applying %d-to-%d repo migration", from, to),
609
+ success: fmt.Sprintf("Migration %d-to-%d succeeded", from, to),
610
+ })
611
+ }
612
+
613
+ return runDaemonWithMultipleMigrationMonitoring(t, node, expectedMigrations)
614
}
615
574
-// runDaemonWithMigrationMonitoring is a generic helper for running daemon --migrate and monitoring output.
575
-// It waits for the daemon to be ready, then shuts it down gracefully.
576
-// migrationPattern: pattern to detect migration started (e.g., "applying X-to-Y repo migration")
577
-// successPattern: pattern to detect migration succeeded (e.g., "Migration X to Y succeeded")
578
-// Returns the stdout output and whether both patterns were detected.
579
-func runDaemonWithMigrationMonitoring(t *testing.T, node *harness.Node, migrationPattern, successPattern string) (string, bool) {
616
+// runDaemonWithMultipleMigrationMonitoring monitors daemon startup for multiple sequential migrations
617
+func runDaemonWithMultipleMigrationMonitoring(t *testing.T, node *harness.Node, expectedMigrations []struct {
618
+ pattern string
619
+ success string
620
+}) (string, bool) {
621
// Create context with timeout as safety net
622
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
623
defer cancel()
@@ -601,7 +642,11 @@ func runDaemonWithMigrationMonitoring(t *testing.T, node *harness.Node, migratio
642
require.NoError(t, err)
643
644
var allOutput strings.Builder
604
- var migrationDetected, migrationSucceeded, daemonReady bool
645
+ var daemonReady bool
646
+
647
+ // Track which migrations have been detected
648
+ migrationsDetected := make([]bool, len(expectedMigrations))
649
+ migrationsSucceeded := make([]bool, len(expectedMigrations))
650
651
// Monitor stdout for completion signals
652
scanner := bufio.NewScanner(stdout)
@@ -611,11 +656,13 @@ func runDaemonWithMigrationMonitoring(t *testing.T, node *harness.Node, migratio
656
allOutput.WriteString(line + "\n")
657
658
// Check for migration messages
614
- if migrationPattern != "" && strings.Contains(line, migrationPattern) {
615
- migrationDetected = true
616
- }
617
- if successPattern != "" && strings.Contains(line, successPattern) {
618
- migrationSucceeded = true
659
+ for i, migration := range expectedMigrations {
660
+ if strings.Contains(line, migration.pattern) {
661
+ migrationsDetected[i] = true
662
+ }
663
+ if strings.Contains(line, migration.success) {
664
+ migrationsSucceeded[i] = true
665
+ }
666
}
667
if strings.Contains(line, "Daemon is ready") {
668
daemonReady = true
@@ -667,17 +714,41 @@ func runDaemonWithMigrationMonitoring(t *testing.T, node *harness.Node, migratio
714
// Wait for process to exit
715
_ = cmd.Wait()
716
670
- // Return success if we detected migration
671
- success := migrationDetected && migrationSucceeded
672
- return allOutput.String(), success
717
+ // Check all migrations were detected and succeeded
718
+ allDetected := true
719
+ allSucceeded := true
720
+ for i := range expectedMigrations {
721
+ if !migrationsDetected[i] {
722
+ allDetected = false
723
+ t.Logf("Migration %s was not detected", expectedMigrations[i].pattern)
724
+ }
725
+ if !migrationsSucceeded[i] {
726
+ allSucceeded = false
727
+ t.Logf("Migration %s did not succeed", expectedMigrations[i].success)
728
+ }
729
+ }
730
+
731
+ return allOutput.String(), allDetected && allSucceeded
732
}
733
734
// Check if process has exited (e.g., due to startup failure after migration)
735
if cmd.ProcessState != nil && cmd.ProcessState.Exited() {
736
// Process exited - migration may have completed but daemon failed to start
737
// This is expected for corrupted config tests
679
- success := migrationDetected && migrationSucceeded
680
- return allOutput.String(), success
738
+
739
+ // Check all migrations status
740
+ allDetected := true
741
+ allSucceeded := true
742
+ for i := range expectedMigrations {
743
+ if !migrationsDetected[i] {
744
+ allDetected = false
745
+ }
746
+ if !migrationsSucceeded[i] {
747
+ allSucceeded = false
748
+ }
749
+ }
750
+
751
+ return allOutput.String(), allDetected && allSucceeded
752
}
753
}
754
}
test/cli/migrations/migration_17_to_latest_test.go
new
+360
@@ -0,0 +1,360 @@
1
+package migrations
2
+
3
+// NOTE: These migration tests require the local Kubo binary (built with 'make build') to be in PATH.
4
+//
5
+// To run these tests successfully:
6
+// export PATH="$(pwd)/cmd/ipfs:$PATH"
7
+// go test ./test/cli/migrations/
8
+
9
+import (
10
+ "context"
11
+ "encoding/json"
12
+ "fmt"
13
+ "os"
14
+ "os/exec"
15
+ "path/filepath"
16
+ "strings"
17
+ "testing"
18
+ "time"
19
+
20
+ ipfs "github.com/ipfs/kubo"
21
+ "github.com/ipfs/kubo/test/cli/harness"
22
+ "github.com/stretchr/testify/require"
23
+)
24
+
25
+// TestMigration17ToLatest tests migration from repo version 17 to the latest version.
26
+//
27
+// Since we don't have a v17 repo fixture, we start with v16 and migrate it to v17 first,
28
+// then test the 17-to-18 migration specifically.
29
+//
30
+// This test focuses on the Provider/Reprovider to Provide consolidation that happens in 17-to-18.
31
+func TestMigration17ToLatest(t *testing.T) {
32
+ t.Parallel()
33
+
34
+ // Tests for Provider/Reprovider to Provide migration (17-to-18)
35
+ t.Run("daemon migrate: Provider/Reprovider to Provide consolidation", testProviderReproviderMigration)
36
+ t.Run("daemon migrate: flat strategy conversion", testFlatStrategyConversion)
37
+ t.Run("daemon migrate: empty Provider/Reprovider sections", testEmptyProviderReproviderMigration)
38
+ t.Run("daemon migrate: partial configuration (Provider only)", testProviderOnlyMigration)
39
+ t.Run("daemon migrate: partial configuration (Reprovider only)", testReproviderOnlyMigration)
40
+ t.Run("repo migrate: invalid strategy values preserved", testInvalidStrategyMigration)
41
+ t.Run("repo migrate: Provider/Reprovider to Provide consolidation", testRepoProviderReproviderMigration)
42
+}
43
+
44
+// =============================================================================
45
+// MIGRATION 17-to-18 SPECIFIC TESTS: Provider/Reprovider to Provide consolidation
46
+// =============================================================================
47
+
48
+func testProviderReproviderMigration(t *testing.T) {
49
+ // TEST: 17-to-18 migration with explicit Provider/Reprovider configuration
50
+ node := setupV17RepoWithProviderConfig(t)
51
+
52
+ configPath := filepath.Join(node.Dir, "config")
53
+ versionPath := filepath.Join(node.Dir, "version")
54
+
55
+ // Run migration using daemon --migrate command
56
+ stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node)
57
+
58
+ // Debug: Print the actual output
59
+ t.Logf("Daemon output:\n%s", stdoutOutput)
60
+
61
+ // Verify migration was successful
62
+ require.True(t, migrationSuccess, "Migration should have been successful")
63
+ require.Contains(t, stdoutOutput, "applying 17-to-18 repo migration", "Migration 17-to-18 should have been triggered")
64
+ require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded", "Migration 17-to-18 should have completed successfully")
65
+
66
+ // Verify version was updated to latest
67
+ versionData, err := os.ReadFile(versionPath)
68
+ require.NoError(t, err)
69
+ expectedVersion := fmt.Sprint(ipfs.RepoVersion)
70
+ require.Equal(t, expectedVersion, strings.TrimSpace(string(versionData)), "Version should be updated to %s (latest)", expectedVersion)
71
+
72
+ // =============================================================================
73
+ // MIGRATION 17-to-18 ASSERTIONS: Provider/Reprovider to Provide consolidation
74
+ // =============================================================================
75
+ helper := NewMigrationTestHelper(t, configPath)
76
+
77
+ // Verify Provider/Reprovider migration to Provide
78
+ helper.RequireProviderMigration().
79
+ RequireFieldEquals("Provide.Enabled", true). // Migrated from Provider.Enabled
80
+ RequireFieldEquals("Provide.DHT.MaxWorkers", float64(8)). // Migrated from Provider.WorkerCount
81
+ RequireFieldEquals("Provide.Strategy", "roots"). // Migrated from Reprovider.Strategy
82
+ RequireFieldEquals("Provide.DHT.Interval", "24h") // Migrated from Reprovider.Interval
83
+
84
+ // Verify old sections are removed
85
+ helper.RequireFieldAbsent("Provider").
86
+ RequireFieldAbsent("Reprovider")
87
+}
88
+
89
+func testFlatStrategyConversion(t *testing.T) {
90
+ // TEST: 17-to-18 migration with "flat" strategy that should convert to "all"
91
+ node := setupV17RepoWithFlatStrategy(t)
92
+
93
+ configPath := filepath.Join(node.Dir, "config")
94
+
95
+ // Run migration using daemon --migrate command
96
+ stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node)
97
+
98
+ // Verify migration was successful
99
+ require.True(t, migrationSuccess, "Migration should have been successful")
100
+ require.Contains(t, stdoutOutput, "applying 17-to-18 repo migration", "Migration 17-to-18 should have been triggered")
101
+ require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded", "Migration 17-to-18 should have completed successfully")
102
+
103
+ // =============================================================================
104
+ // MIGRATION 17-to-18 ASSERTIONS: "flat" to "all" strategy conversion
105
+ // =============================================================================
106
+ helper := NewMigrationTestHelper(t, configPath)
107
+
108
+ // Verify "flat" was converted to "all"
109
+ helper.RequireProviderMigration().
110
+ RequireFieldEquals("Provide.Strategy", "all"). // "flat" converted to "all"
111
+ RequireFieldEquals("Provide.DHT.Interval", "12h")
112
+}
113
+
114
+func testEmptyProviderReproviderMigration(t *testing.T) {
115
+ // TEST: 17-to-18 migration with empty Provider and Reprovider sections
116
+ node := setupV17RepoWithEmptySections(t)
117
+
118
+ configPath := filepath.Join(node.Dir, "config")
119
+
120
+ // Run migration
121
+ stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node)
122
+
123
+ // Verify migration was successful
124
+ require.True(t, migrationSuccess, "Migration should have been successful")
125
+ require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded")
126
+
127
+ // Verify empty sections are removed and no Provide section is created
128
+ helper := NewMigrationTestHelper(t, configPath)
129
+ helper.RequireFieldAbsent("Provider").
130
+ RequireFieldAbsent("Reprovider").
131
+ RequireFieldAbsent("Provide") // No Provide section should be created for empty configs
132
+}
133
+
134
+func testProviderOnlyMigration(t *testing.T) {
135
+ // TEST: 17-to-18 migration with only Provider configuration
136
+ node := setupV17RepoWithProviderOnly(t)
137
+
138
+ configPath := filepath.Join(node.Dir, "config")
139
+
140
+ // Run migration
141
+ stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node)
142
+
143
+ // Verify migration was successful
144
+ require.True(t, migrationSuccess, "Migration should have been successful")
145
+ require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded")
146
+
147
+ // Verify only Provider fields are migrated
148
+ helper := NewMigrationTestHelper(t, configPath)
149
+ helper.RequireProviderMigration().
150
+ RequireFieldEquals("Provide.Enabled", false).
151
+ RequireFieldEquals("Provide.DHT.MaxWorkers", float64(32)).
152
+ RequireFieldAbsent("Provide.Strategy"). // No Reprovider.Strategy to migrate
153
+ RequireFieldAbsent("Provide.DHT.Interval") // No Reprovider.Interval to migrate
154
+}
155
+
156
+func testReproviderOnlyMigration(t *testing.T) {
157
+ // TEST: 17-to-18 migration with only Reprovider configuration
158
+ node := setupV17RepoWithReproviderOnly(t)
159
+
160
+ configPath := filepath.Join(node.Dir, "config")
161
+
162
+ // Run migration
163
+ stdoutOutput, migrationSuccess := runDaemonMigrationFromV17(t, node)
164
+
165
+ // Verify migration was successful
166
+ require.True(t, migrationSuccess, "Migration should have been successful")
167
+ require.Contains(t, stdoutOutput, "Migration 17-to-18 succeeded")
168
+
169
+ // Verify only Reprovider fields are migrated
170
+ helper := NewMigrationTestHelper(t, configPath)
171
+ helper.RequireProviderMigration().
172
+ RequireFieldEquals("Provide.Strategy", "pinned").
173
+ RequireFieldEquals("Provide.DHT.Interval", "48h").
174
+ RequireFieldAbsent("Provide.Enabled"). // No Provider.Enabled to migrate
175
+ RequireFieldAbsent("Provide.DHT.MaxWorkers") // No Provider.WorkerCount to migrate
176
+}
177
+
178
+func testInvalidStrategyMigration(t *testing.T) {
179
+ // TEST: 17-to-18 migration with invalid strategy values (should be preserved as-is)
180
+ // The migration itself should succeed, but daemon start will fail due to invalid strategy
181
+ node := setupV17RepoWithInvalidStrategy(t)
182
+
183
+ configPath := filepath.Join(node.Dir, "config")
184
+
185
+ // Run the migration using 'ipfs repo migrate' (not daemon --migrate)
186
+ // because daemon would fail to start with invalid strategy after migration
187
+ result := node.RunIPFS("repo", "migrate")
188
+ require.Empty(t, result.Stderr.String(), "Migration should succeed without errors")
189
+
190
+ // Verify invalid strategy is preserved as-is (not validated during migration)
191
+ helper := NewMigrationTestHelper(t, configPath)
192
+ helper.RequireProviderMigration().
193
+ RequireFieldEquals("Provide.Strategy", "invalid-strategy") // Should be preserved
194
+
195
+ // Now verify that daemon fails to start with invalid strategy
196
+ // Note: We cannot use --offline as it skips provider validation
197
+ // Use a context with timeout to avoid hanging
198
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
199
+ defer cancel()
200
+
201
+ cmd := exec.CommandContext(ctx, node.IPFSBin, "daemon")
202
+ cmd.Dir = node.Dir
203
+ for k, v := range node.Runner.Env {
204
+ cmd.Env = append(cmd.Env, k+"="+v)
205
+ }
206
+
207
+ output, err := cmd.CombinedOutput()
208
+
209
+ // The daemon should fail (either with error or timeout if it's hanging)
210
+ require.Error(t, err, "Daemon should fail to start with invalid strategy")
211
+
212
+ // Check if we got the expected error message
213
+ outputStr := string(output)
214
+ t.Logf("Daemon output with invalid strategy: %s", outputStr)
215
+
216
+ // The error should mention unknown strategy
217
+ require.Contains(t, outputStr, "unknown strategy", "Should report unknown strategy error")
218
+}
219
+
220
+func testRepoProviderReproviderMigration(t *testing.T) {
221
+ // TEST: 17-to-18 migration using 'ipfs repo migrate' command
222
+ node := setupV17RepoWithProviderConfig(t)
223
+
224
+ configPath := filepath.Join(node.Dir, "config")
225
+
226
+ // Run migration using 'ipfs repo migrate' command
227
+ result := node.RunIPFS("repo", "migrate")
228
+ require.Empty(t, result.Stderr.String(), "Migration should succeed without errors")
229
+
230
+ // Verify same results as daemon migrate
231
+ helper := NewMigrationTestHelper(t, configPath)
232
+ helper.RequireProviderMigration().
233
+ RequireFieldEquals("Provide.Enabled", true).
234
+ RequireFieldEquals("Provide.DHT.MaxWorkers", float64(8)).
235
+ RequireFieldEquals("Provide.Strategy", "roots").
236
+ RequireFieldEquals("Provide.DHT.Interval", "24h")
237
+}
238
+
239
+// =============================================================================
240
+// HELPER FUNCTIONS
241
+// =============================================================================
242
+
243
+// setupV17RepoWithProviderConfig creates a v17 repo with Provider/Reprovider configuration
244
+func setupV17RepoWithProviderConfig(t *testing.T) *harness.Node {
245
+ return setupV17RepoWithConfig(t,
246
+ map[string]interface{}{
247
+ "Enabled": true,
248
+ "WorkerCount": 8,
249
+ },
250
+ map[string]interface{}{
251
+ "Strategy": "roots",
252
+ "Interval": "24h",
253
+ })
254
+}
255
+
256
+// setupV17RepoWithFlatStrategy creates a v17 repo with "flat" strategy for testing conversion
257
+func setupV17RepoWithFlatStrategy(t *testing.T) *harness.Node {
258
+ return setupV17RepoWithConfig(t,
259
+ map[string]interface{}{
260
+ "Enabled": false,
261
+ },
262
+ map[string]interface{}{
263
+ "Strategy": "flat", // This should be converted to "all"
264
+ "Interval": "12h",
265
+ })
266
+}
267
+
268
+// setupV17RepoWithConfig is a helper that creates a v17 repo with specified Provider/Reprovider config
269
+func setupV17RepoWithConfig(t *testing.T, providerConfig, reproviderConfig map[string]interface{}) *harness.Node {
270
+ node := setupStaticV16Repo(t)
271
+
272
+ // First migrate to v17
273
+ result := node.RunIPFS("repo", "migrate", "--to=17")
274
+ require.Empty(t, result.Stderr.String(), "Migration to v17 should succeed")
275
+
276
+ // Update config with specified Provider and Reprovider settings
277
+ configPath := filepath.Join(node.Dir, "config")
278
+ var config map[string]interface{}
279
+ configData, err := os.ReadFile(configPath)
280
+ require.NoError(t, err)
281
+ require.NoError(t, json.Unmarshal(configData, &config))
282
+
283
+ if providerConfig != nil {
284
+ config["Provider"] = providerConfig
285
+ } else {
286
+ config["Provider"] = map[string]interface{}{}
287
+ }
288
+
289
+ if reproviderConfig != nil {
290
+ config["Reprovider"] = reproviderConfig
291
+ } else {
292
+ config["Reprovider"] = map[string]interface{}{}
293
+ }
294
+
295
+ modifiedConfigData, err := json.MarshalIndent(config, "", " ")
296
+ require.NoError(t, err)
297
+ require.NoError(t, os.WriteFile(configPath, modifiedConfigData, 0644))
298
+
299
+ return node
300
+}
301
+
302
+// setupV17RepoWithEmptySections creates a v17 repo with empty Provider/Reprovider sections
303
+func setupV17RepoWithEmptySections(t *testing.T) *harness.Node {
304
+ return setupV17RepoWithConfig(t,
305
+ map[string]interface{}{},
306
+ map[string]interface{}{})
307
+}
308
+
309
+// setupV17RepoWithProviderOnly creates a v17 repo with only Provider configuration
310
+func setupV17RepoWithProviderOnly(t *testing.T) *harness.Node {
311
+ return setupV17RepoWithConfig(t,
312
+ map[string]interface{}{
313
+ "Enabled": false,
314
+ "WorkerCount": 32,
315
+ },
316
+ map[string]interface{}{})
317
+}
318
+
319
+// setupV17RepoWithReproviderOnly creates a v17 repo with only Reprovider configuration
320
+func setupV17RepoWithReproviderOnly(t *testing.T) *harness.Node {
321
+ return setupV17RepoWithConfig(t,
322
+ map[string]interface{}{},
323
+ map[string]interface{}{
324
+ "Strategy": "pinned",
325
+ "Interval": "48h",
326
+ })
327
+}
328
+
329
+// setupV17RepoWithInvalidStrategy creates a v17 repo with an invalid strategy value
330
+func setupV17RepoWithInvalidStrategy(t *testing.T) *harness.Node {
331
+ return setupV17RepoWithConfig(t,
332
+ map[string]interface{}{},
333
+ map[string]interface{}{
334
+ "Strategy": "invalid-strategy", // This is not a valid strategy
335
+ "Interval": "24h",
336
+ })
337
+}
338
+
339
+// runDaemonMigrationFromV17 monitors daemon startup for 17-to-18 migration only
340
+func runDaemonMigrationFromV17(t *testing.T, node *harness.Node) (string, bool) {
341
+ // Monitor only the 17-to-18 migration
342
+ expectedMigrations := []struct {
343
+ pattern string
344
+ success string
345
+ }{
346
+ {
347
+ pattern: "applying 17-to-18 repo migration",
348
+ success: "Migration 17-to-18 succeeded",
349
+ },
350
+ }
351
+
352
+ return runDaemonWithMultipleMigrationMonitoring(t, node, expectedMigrations)
353
+}
354
+
355
+// RequireProviderMigration verifies that Provider/Reprovider have been migrated to Provide section
356
+func (h *MigrationTestHelper) RequireProviderMigration() *MigrationTestHelper {
357
+ return h.RequireFieldExists("Provide").
358
+ RequireFieldAbsent("Provider").
359
+ RequireFieldAbsent("Reprovider")
360
+}
test/cli/migrations/migration_mixed_15_to_latest_test.go
renamed
+83
-35
@@ -1,8 +1,14 @@
1
package migrations
2
3
-// NOTE: These legacy migration tests require the local Kubo binary (built with 'make build') to be in PATH.
4
-// The tests migrate from repo version 15 to 17, which requires both external (15→16) and embedded (16→17) migrations.
5
-// This validates the transition from legacy external binaries to modern embedded migrations.
3
+// NOTE: These mixed migration tests validate the transition from old Kubo versions that used external
4
+// migration binaries to the latest version with embedded migrations. This ensures users can upgrade
5
+// from very old installations (v15) to the latest version seamlessly.
6
+//
7
+// The tests verify hybrid migration paths:
8
+// - Forward: external binary (15→16) + embedded migrations (16→latest)
9
+// - Backward: embedded migrations (latest→16) + external binary (16→15)
10
+//
11
+// This confirms compatibility between the old external migration system and the new embedded system.
12
//
13
// To run these tests successfully:
14
// export PATH="$(pwd)/cmd/ipfs:$PATH"
@@ -22,30 +28,36 @@ import (
28
"testing"
29
"time"
30
31
+ ipfs "github.com/ipfs/kubo"
32
"github.com/ipfs/kubo/test/cli/harness"
33
"github.com/stretchr/testify/require"
34
)
35
29
-func TestMigration15To17(t *testing.T) {
36
+// TestMixedMigration15ToLatest tests migration from old Kubo (v15 with external migrations)
37
+// to the latest version using a hybrid approach: external binary for 15→16, then embedded
38
+// migrations for 16→latest. This ensures backward compatibility for users upgrading from
39
+// very old Kubo installations.
40
+func TestMixedMigration15ToLatest(t *testing.T) {
41
t.Parallel()
42
32
- // Test legacy migration from v15 to v17 (combines external 15→16 + embedded 16→17)
33
- t.Run("daemon migrate: legacy 15 to 17", testDaemonMigration15To17)
34
- t.Run("repo migrate: legacy 15 to 17", testRepoMigration15To17)
43
+ // Test mixed migration from v15 to latest (combines external 15→16 + embedded 16→latest)
44
+ t.Run("daemon migrate: mixed 15 to latest", testDaemonMigration15ToLatest)
45
+ t.Run("repo migrate: mixed 15 to latest", testRepoMigration15ToLatest)
46
}
47
37
-func TestMigration17To15Downgrade(t *testing.T) {
48
+// TestMixedMigrationLatestTo15Downgrade tests downgrading from the latest version back to v15
49
+// using a hybrid approach: embedded migrations for latest→16, then external binary for 16→15.
50
+// This ensures the migration system works bidirectionally for recovery scenarios.
51
+func TestMixedMigrationLatestTo15Downgrade(t *testing.T) {
52
t.Parallel()
53
40
- // Test reverse hybrid migration from v17 to v15 (embedded 17→16 + external 16→15)
41
- t.Run("repo migrate: reverse hybrid 17 to 15", testRepoReverseHybridMigration17To15)
54
+ // Test reverse hybrid migration from latest to v15 (embedded latest→16 + external 16→15)
55
+ t.Run("repo migrate: reverse hybrid latest to 15", testRepoReverseHybridMigrationLatestTo15)
56
}
57
44
-func testDaemonMigration15To17(t *testing.T) {
45
- // TEST: Migration from v15 to v17 using 'ipfs daemon --migrate'
46
- // This tests the dual migration path: external binary (15→16) + embedded (16→17)
47
- // NOTE: This test may need to be revised/updated once repo version 18 is released,
48
- // at that point only keep tests that use 'ipfs repo migrate'
58
+func testDaemonMigration15ToLatest(t *testing.T) {
59
+ // TEST: Migration from v15 to latest using 'ipfs daemon --migrate'
60
+ // This tests the mixed migration path: external binary (15→16) + embedded (16→latest)
61
node := setupStaticV15Repo(t)
62
63
// Create mock migration binary for 15→16 (16→17 will use embedded migration)
@@ -76,13 +88,16 @@ func testDaemonMigration15To17(t *testing.T) {
88
// Verify hybrid migration was successful
89
require.True(t, migrationSuccess, "Hybrid migration should have been successful")
90
require.Contains(t, stdoutOutput, "Phase 1: External migration from v15 to v16", "Should detect external migration phase")
79
- require.Contains(t, stdoutOutput, "Phase 2: Embedded migration from v16 to v17", "Should detect embedded migration phase")
91
+ // Verify each embedded migration step from 16 to latest
92
+ verifyMigrationSteps(t, stdoutOutput, 16, ipfs.RepoVersion, true)
93
+ require.Contains(t, stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion), "Should detect embedded migration phase")
94
require.Contains(t, stdoutOutput, "Hybrid migration completed successfully", "Should confirm hybrid migration completion")
95
82
- // Verify final version is 17
96
+ // Verify final version is latest
97
versionData, err = os.ReadFile(versionPath)
98
require.NoError(t, err)
85
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Version should be updated to 17")
99
+ latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
100
+ require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest")
101
102
// Verify config is still valid JSON and key fields preserved
103
var finalConfig map[string]interface{}
@@ -103,8 +118,8 @@ func testDaemonMigration15To17(t *testing.T) {
118
require.NotNil(t, autoConf, "AutoConf should be added by 16→17 migration")
119
}
120
106
-func testRepoMigration15To17(t *testing.T) {
107
- // TEST: Migration from v15 to v17 using 'ipfs repo migrate'
121
+func testRepoMigration15ToLatest(t *testing.T) {
122
+ // TEST: Migration from v15 to latest using 'ipfs repo migrate'
123
// Comparison test to verify repo migrate produces same results as daemon migrate
124
node := setupStaticV15Repo(t)
125
@@ -132,10 +147,11 @@ func testRepoMigration15To17(t *testing.T) {
147
})
148
require.Empty(t, result.Stderr.String(), "Migration should succeed without errors")
149
135
- // Verify final version is 17
150
+ // Verify final version is latest
151
versionData, err = os.ReadFile(versionPath)
152
require.NoError(t, err)
138
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Version should be updated to 17")
153
+ latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
154
+ require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Version should be updated to latest")
155
156
// Verify config is valid JSON
157
var finalConfig map[string]interface{}
@@ -177,7 +193,7 @@ func runDaemonWithLegacyMigrationMonitoring(t *testing.T, node *harness.Node) (s
193
// Check for hybrid migration patterns in output
194
hasHybridStart := strings.Contains(stdoutOutput, "Using hybrid migration strategy")
195
hasPhase1 := strings.Contains(stdoutOutput, "Phase 1: External migration from v15 to v16")
180
- hasPhase2 := strings.Contains(stdoutOutput, "Phase 2: Embedded migration from v16 to v17")
196
+ hasPhase2 := strings.Contains(stdoutOutput, fmt.Sprintf("Phase 2: Embedded migration from v16 to v%d", ipfs.RepoVersion))
197
hasHybridSuccess := strings.Contains(stdoutOutput, "Hybrid migration completed successfully")
198
199
// Success requires daemon to start and hybrid migration patterns to be detected
@@ -342,6 +358,37 @@ func main() {
358
require.NoError(t, err, "Mock binary should exist")
359
}
360
361
+// expectedMigrationSteps generates the expected migration step strings for a version range.
362
+// For forward migrations (from < to), it returns strings like "Running embedded migration fs-repo-16-to-17"
363
+// For reverse migrations (from > to), it returns strings for the reverse path.
364
+func expectedMigrationSteps(from, to int, forward bool) []string {
365
+ var steps []string
366
+
367
+ if forward {
368
+ // Forward migration: increment by 1 each step
369
+ for v := from; v < to; v++ {
370
+ migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v+1)
371
+ steps = append(steps, fmt.Sprintf("Running embedded migration %s", migrationName))
372
+ }
373
+ } else {
374
+ // Reverse migration: decrement by 1 each step
375
+ for v := from; v > to; v-- {
376
+ migrationName := fmt.Sprintf("fs-repo-%d-to-%d", v, v-1)
377
+ steps = append(steps, fmt.Sprintf("Running reverse migration %s", migrationName))
378
+ }
379
+ }
380
+
381
+ return steps
382
+}
383
+
384
+// verifyMigrationSteps checks that all expected migration steps appear in the output
385
+func verifyMigrationSteps(t *testing.T, output string, from, to int, forward bool) {
386
+ steps := expectedMigrationSteps(from, to, forward)
387
+ for _, step := range steps {
388
+ require.Contains(t, output, step, "Migration output should contain: %s", step)
389
+ }
390
+}
391
+
392
// getNestedValue retrieves a nested value from a config map using dot notation
393
func getNestedValue(config map[string]interface{}, path string) interface{} {
394
parts := strings.Split(path, ".")
@@ -362,11 +409,11 @@ func getNestedValue(config map[string]interface{}, path string) interface{} {
409
return current
410
}
411
365
-func testRepoReverseHybridMigration17To15(t *testing.T) {
366
- // TEST: Reverse hybrid migration from v17 to v15 using 'ipfs repo migrate --to=15 --allow-downgrade'
412
+func testRepoReverseHybridMigrationLatestTo15(t *testing.T) {
413
+ // TEST: Reverse hybrid migration from latest to v15 using 'ipfs repo migrate --to=15 --allow-downgrade'
414
// This tests reverse hybrid migration: embedded (17→16) + external (16→15)
415
369
- // Start with v15 fixture and migrate forward to v17 to create proper backup files
416
+ // Start with v15 fixture and migrate forward to latest to create proper backup files
417
node := setupStaticV15Repo(t)
418
419
// Create mock migration binary for 15→16 (needed for forward migration)
@@ -377,8 +424,8 @@ func testRepoReverseHybridMigration17To15(t *testing.T) {
424
configPath := filepath.Join(node.Dir, "config")
425
versionPath := filepath.Join(node.Dir, "version")
426
380
- // Step 1: Forward migration from v15 to v17 to create backup files
381
- t.Log("Step 1: Forward migration v15 → v17")
427
+ // Step 1: Forward migration from v15 to latest to create backup files
428
+ t.Logf("Step 1: Forward migration v15 → v%d", ipfs.RepoVersion)
429
result := node.Runner.Run(harness.RunRequest{
430
Path: node.IPFSBin,
431
Args: []string{"repo", "migrate"},
@@ -396,21 +443,22 @@ func testRepoReverseHybridMigration17To15(t *testing.T) {
443
444
require.Empty(t, result.Stderr.String(), "Forward migration should succeed without errors")
445
399
- // Verify we're at v17 after forward migration
446
+ // Verify we're at latest version after forward migration
447
versionData, err := os.ReadFile(versionPath)
448
require.NoError(t, err)
402
- require.Equal(t, "17", strings.TrimSpace(string(versionData)), "Should be at version 17 after forward migration")
449
+ latestVersion := fmt.Sprintf("%d", ipfs.RepoVersion)
450
+ require.Equal(t, latestVersion, strings.TrimSpace(string(versionData)), "Should be at latest version after forward migration")
451
452
// Read config after forward migration to use as baseline for downgrade
405
- var v17Config map[string]interface{}
453
+ var latestConfig map[string]interface{}
454
configData, err := os.ReadFile(configPath)
455
require.NoError(t, err)
408
- require.NoError(t, json.Unmarshal(configData, &v17Config))
456
+ require.NoError(t, json.Unmarshal(configData, &latestConfig))
457
410
- originalPeerID := getNestedValue(v17Config, "Identity.PeerID")
458
+ originalPeerID := getNestedValue(latestConfig, "Identity.PeerID")
459
412
- // Step 2: Reverse hybrid migration from v17 to v15
413
- t.Log("Step 2: Reverse hybrid migration v17 → v15")
460
+ // Step 2: Reverse hybrid migration from latest to v15
461
+ t.Logf("Step 2: Reverse hybrid migration v%d → v15", ipfs.RepoVersion)
462
result = node.Runner.Run(harness.RunRequest{
463
Path: node.IPFSBin,
464
Args: []string{"repo", "migrate", "--to=15", "--allow-downgrade"},
test/cli/provider_test.go
+45
-45
@@ -58,11 +58,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
58
}
59
}
60
61
- t.Run("Provider.Enabled=true announces new CIDs created by ipfs add", func(t *testing.T) {
61
+ t.Run("Provide.Enabled=true announces new CIDs created by ipfs add", func(t *testing.T) {
62
t.Parallel()
63
64
nodes := initNodes(t, 2, func(n *harness.Node) {
65
- n.SetIPFSConfig("Provider.Enabled", true)
65
+ n.SetIPFSConfig("Provide.Enabled", true)
66
})
67
defer nodes.StopDaemons()
68
@@ -70,11 +70,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
70
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
71
})
72
73
- t.Run("Provider.Enabled=true announces new CIDs created by ipfs add --pin=false with default strategy", func(t *testing.T) {
73
+ t.Run("Provide.Enabled=true announces new CIDs created by ipfs add --pin=false with default strategy", func(t *testing.T) {
74
t.Parallel()
75
76
nodes := initNodes(t, 2, func(n *harness.Node) {
77
- n.SetIPFSConfig("Provider.Enabled", true)
77
+ n.SetIPFSConfig("Provide.Enabled", true)
78
// Default strategy is "all" which should provide even unpinned content
79
})
80
defer nodes.StopDaemons()
@@ -83,11 +83,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
83
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
84
})
85
86
- t.Run("Provider.Enabled=true announces new CIDs created by ipfs block put --pin=false with default strategy", func(t *testing.T) {
86
+ t.Run("Provide.Enabled=true announces new CIDs created by ipfs block put --pin=false with default strategy", func(t *testing.T) {
87
t.Parallel()
88
89
nodes := initNodes(t, 2, func(n *harness.Node) {
90
- n.SetIPFSConfig("Provider.Enabled", true)
90
+ n.SetIPFSConfig("Provide.Enabled", true)
91
// Default strategy is "all" which should provide unpinned content from block put
92
})
93
defer nodes.StopDaemons()
@@ -97,11 +97,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
97
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
98
})
99
100
- t.Run("Provider.Enabled=true announces new CIDs created by ipfs dag put --pin=false with default strategy", func(t *testing.T) {
100
+ t.Run("Provide.Enabled=true announces new CIDs created by ipfs dag put --pin=false with default strategy", func(t *testing.T) {
101
t.Parallel()
102
103
nodes := initNodes(t, 2, func(n *harness.Node) {
104
- n.SetIPFSConfig("Provider.Enabled", true)
104
+ n.SetIPFSConfig("Provide.Enabled", true)
105
// Default strategy is "all" which should provide unpinned content from dag put
106
})
107
defer nodes.StopDaemons()
@@ -111,11 +111,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
111
expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
112
})
113
114
- t.Run("Provider.Enabled=false disables announcement of new CID from ipfs add", func(t *testing.T) {
114
+ t.Run("Provide.Enabled=false disables announcement of new CID from ipfs add", func(t *testing.T) {
115
t.Parallel()
116
117
nodes := initNodes(t, 2, func(n *harness.Node) {
118
- n.SetIPFSConfig("Provider.Enabled", false)
118
+ n.SetIPFSConfig("Provide.Enabled", false)
119
})
120
defer nodes.StopDaemons()
121
@@ -123,17 +123,17 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
123
expectNoProviders(t, cid, nodes[1:]...)
124
})
125
126
- t.Run("Provider.Enabled=false disables manual announcement via RPC command", func(t *testing.T) {
126
+ t.Run("Provide.Enabled=false disables manual announcement via RPC command", func(t *testing.T) {
127
t.Parallel()
128
129
nodes := initNodes(t, 2, func(n *harness.Node) {
130
- n.SetIPFSConfig("Provider.Enabled", false)
130
+ n.SetIPFSConfig("Provide.Enabled", false)
131
})
132
defer nodes.StopDaemons()
133
134
cid := nodes[0].IPFSAddStr(time.Now().String())
135
res := nodes[0].RunIPFS("routing", "provide", cid)
136
- assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provider.Enabled is set to 'false'")
136
+ assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.Enabled is set to 'false'")
137
assert.Equal(t, 1, res.ExitCode())
138
139
expectNoProviders(t, cid, nodes[1:]...)
@@ -144,7 +144,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
144
t.Parallel()
145
146
nodes := initNodes(t, 2, func(n *harness.Node) {
147
- n.SetIPFSConfig("Reprovider.Interval", "0")
147
+ n.SetIPFSConfig("Provide.DHT.Interval", "0")
148
})
149
defer nodes.StopDaemons()
150
@@ -153,11 +153,11 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
153
})
154
155
// It is a lesser evil - forces users to fix their config and have some sort of interval
156
- t.Run("Manual Reprovider trigger does not work when periodic Reprovider is disabled", func(t *testing.T) {
156
+ t.Run("Manual Reprovide trigger does not work when periodic reprovide is disabled", func(t *testing.T) {
157
t.Parallel()
158
159
nodes := initNodes(t, 2, func(n *harness.Node) {
160
- n.SetIPFSConfig("Reprovider.Interval", "0")
160
+ n.SetIPFSConfig("Provide.DHT.Interval", "0")
161
})
162
defer nodes.StopDaemons()
163
@@ -166,18 +166,18 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
166
expectNoProviders(t, cid, nodes[1:]...)
167
168
res := nodes[0].RunIPFS("routing", "reprovide")
169
- assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Reprovider.Interval is set to '0'")
169
+ assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.DHT.Interval is set to '0'")
170
assert.Equal(t, 1, res.ExitCode())
171
172
expectNoProviders(t, cid, nodes[1:]...)
173
})
174
175
// It is a lesser evil - forces users to fix their config and have some sort of interval
176
- t.Run("Manual Reprovider trigger does not work when Provider system is disabled", func(t *testing.T) {
176
+ t.Run("Manual Reprovide trigger does not work when Provide system is disabled", func(t *testing.T) {
177
t.Parallel()
178
179
nodes := initNodes(t, 2, func(n *harness.Node) {
180
- n.SetIPFSConfig("Provider.Enabled", false)
180
+ n.SetIPFSConfig("Provide.Enabled", false)
181
})
182
defer nodes.StopDaemons()
183
@@ -186,7 +186,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
186
expectNoProviders(t, cid, nodes[1:]...)
187
188
res := nodes[0].RunIPFS("routing", "reprovide")
189
- assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provider.Enabled is set to 'false'")
189
+ assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Provide.Enabled is set to 'false'")
190
assert.Equal(t, 1, res.ExitCode())
191
192
expectNoProviders(t, cid, nodes[1:]...)
@@ -196,7 +196,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
196
t.Parallel()
197
198
nodes := initNodes(t, 2, func(n *harness.Node) {
199
- n.SetIPFSConfig("Reprovider.Strategy", "all")
199
+ n.SetIPFSConfig("Provide.Strategy", "all")
200
})
201
defer nodes.StopDaemons()
202
@@ -208,7 +208,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
208
t.Parallel()
209
210
nodes := initNodes(t, 2, func(n *harness.Node) {
211
- n.SetIPFSConfig("Reprovider.Strategy", "pinned")
211
+ n.SetIPFSConfig("Provide.Strategy", "pinned")
212
})
213
defer nodes.StopDaemons()
214
@@ -225,7 +225,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
225
t.Parallel()
226
227
nodes := initNodes(t, 2, func(n *harness.Node) {
228
- n.SetIPFSConfig("Reprovider.Strategy", "pinned+mfs")
228
+ n.SetIPFSConfig("Provide.Strategy", "pinned+mfs")
229
})
230
defer nodes.StopDaemons()
231
@@ -245,7 +245,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
245
t.Parallel()
246
247
nodes := initNodes(t, 2, func(n *harness.Node) {
248
- n.SetIPFSConfig("Reprovider.Strategy", "roots")
248
+ n.SetIPFSConfig("Provide.Strategy", "roots")
249
})
250
defer nodes.StopDaemons()
251
@@ -262,7 +262,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
262
t.Parallel()
263
264
nodes := initNodes(t, 2, func(n *harness.Node) {
265
- n.SetIPFSConfig("Reprovider.Strategy", "mfs")
265
+ n.SetIPFSConfig("Provide.Strategy", "mfs")
266
})
267
defer nodes.StopDaemons()
268
@@ -283,7 +283,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
283
t.Parallel()
284
285
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
286
- n.SetIPFSConfig("Reprovider.Strategy", "")
286
+ n.SetIPFSConfig("Provide.Strategy", "")
287
})
288
289
cid := nodes[0].IPFSAddStr(time.Now().String())
@@ -301,7 +301,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
301
t.Parallel()
302
303
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
304
- n.SetIPFSConfig("Reprovider.Strategy", "all")
304
+ n.SetIPFSConfig("Provide.Strategy", "all")
305
})
306
307
cid := nodes[0].IPFSAddStr(time.Now().String())
@@ -322,7 +322,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
322
bar := random.Bytes(1000)
323
324
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
325
- n.SetIPFSConfig("Reprovider.Strategy", "pinned")
325
+ n.SetIPFSConfig("Provide.Strategy", "pinned")
326
})
327
328
// Add a pin while offline so it cannot be provided
@@ -357,7 +357,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
357
bar := random.Bytes(1000)
358
359
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
360
- n.SetIPFSConfig("Reprovider.Strategy", "roots")
360
+ n.SetIPFSConfig("Provide.Strategy", "roots")
361
})
362
n0pid := nodes[0].PeerID().String()
363
@@ -388,7 +388,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
388
bar := random.Bytes(1000)
389
390
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
391
- n.SetIPFSConfig("Reprovider.Strategy", "mfs")
391
+ n.SetIPFSConfig("Provide.Strategy", "mfs")
392
})
393
n0pid := nodes[0].PeerID().String()
394
@@ -412,7 +412,7 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
412
t.Parallel()
413
414
nodes := initNodesWithoutStart(t, 2, func(n *harness.Node) {
415
- n.SetIPFSConfig("Reprovider.Strategy", "pinned+mfs")
415
+ n.SetIPFSConfig("Provide.Strategy", "pinned+mfs")
416
})
417
n0pid := nodes[0].PeerID().String()
418
@@ -444,9 +444,9 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
444
445
nodes := harness.NewT(t).NewNodes(1).Init()
446
nodes.ForEachPar(func(n *harness.Node) {
447
- n.SetIPFSConfig("Provider.Enabled", true)
448
- n.SetIPFSConfig("Reprovider.Interval", "22h")
449
- n.SetIPFSConfig("Reprovider.Strategy", "all")
447
+ n.SetIPFSConfig("Provide.Enabled", true)
448
+ n.SetIPFSConfig("Provide.DHT.Interval", "22h")
449
+ n.SetIPFSConfig("Provide.Strategy", "all")
450
})
451
nodes.StartDaemons()
452
defer nodes.StopDaemons()
@@ -472,9 +472,9 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
472
473
nodes := harness.NewT(t).NewNodes(1).Init()
474
nodes.ForEachPar(func(n *harness.Node) {
475
- n.SetIPFSConfig("Provider.Enabled", true)
476
- n.SetIPFSConfig("Reprovider.Interval", "22h")
477
- n.SetIPFSConfig("Reprovider.Strategy", "all")
475
+ n.SetIPFSConfig("Provide.Enabled", true)
476
+ n.SetIPFSConfig("Provide.DHT.Interval", "22h")
477
+ n.SetIPFSConfig("Provide.Strategy", "all")
478
})
479
nodes.StartDaemons()
480
defer nodes.StopDaemons()
@@ -492,9 +492,9 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
492
493
nodes := harness.NewT(t).NewNodes(1).Init()
494
nodes.ForEachPar(func(n *harness.Node) {
495
- n.SetIPFSConfig("Provider.Enabled", false)
496
- n.SetIPFSConfig("Reprovider.Interval", "22h")
497
- n.SetIPFSConfig("Reprovider.Strategy", "all")
495
+ n.SetIPFSConfig("Provide.Enabled", false)
496
+ n.SetIPFSConfig("Provide.DHT.Interval", "22h")
497
+ n.SetIPFSConfig("Provide.Strategy", "all")
498
})
499
nodes.StartDaemons()
500
defer nodes.StopDaemons()
@@ -509,9 +509,9 @@ func runProviderSuite(t *testing.T, reprovide bool, apply cfgApplier) {
509
510
nodes := harness.NewT(t).NewNodes(1).Init()
511
nodes.ForEachPar(func(n *harness.Node) {
512
- n.SetIPFSConfig("Provider.Enabled", true)
513
- n.SetIPFSConfig("Reprovider.Interval", "22h")
514
- n.SetIPFSConfig("Reprovider.Strategy", "all")
512
+ n.SetIPFSConfig("Provide.Enabled", true)
513
+ n.SetIPFSConfig("Provide.DHT.Interval", "22h")
514
+ n.SetIPFSConfig("Provide.Strategy", "all")
515
})
516
nodes.StartDaemons()
517
defer nodes.StopDaemons()
@@ -546,14 +546,14 @@ func TestProvider(t *testing.T) {
546
name: "LegacyProvider",
547
reprovide: true,
548
apply: func(n *harness.Node) {
549
- n.SetIPFSConfig("Reprovider.Sweep.Enabled", false)
549
+ n.SetIPFSConfig("Provide.DHT.SweepEnabled", false)
550
},
551
},
552
{
553
name: "SweepingProvider",
554
reprovide: false,
555
apply: func(n *harness.Node) {
556
- n.SetIPFSConfig("Reprovider.Sweep.Enabled", true)
556
+ n.SetIPFSConfig("Provide.DHT.SweepEnabled", true)
557
},
558
},
559
}
test/sharness/t0119-prometheus-data/prometheus_metrics
+11
@@ -54,6 +54,15 @@ go_memstats_stack_sys_bytes
54
go_memstats_sys_bytes
55
go_sched_gomaxprocs_threads
56
go_threads
57
+http_server_request_body_size_bytes_bucket
58
+http_server_request_body_size_bytes_count
59
+http_server_request_body_size_bytes_sum
60
+http_server_request_duration_seconds_bucket
61
+http_server_request_duration_seconds_count
62
+http_server_request_duration_seconds_sum
63
+http_server_response_body_size_bytes_bucket
64
+http_server_response_body_size_bytes_count
65
+http_server_response_body_size_bytes_sum
66
ipfs_bitswap_active_block_tasks
67
ipfs_bitswap_active_tasks
68
ipfs_bitswap_bcast_skips_total
@@ -231,6 +240,7 @@ libp2p_relaysvc_status
240
libp2p_swarm_dial_ranking_delay_seconds_bucket
241
libp2p_swarm_dial_ranking_delay_seconds_count
242
libp2p_swarm_dial_ranking_delay_seconds_sum
243
+otel_scope_info
244
process_cpu_seconds_total
245
process_max_fds
246
process_network_receive_bytes_total
@@ -242,3 +252,4 @@ process_virtual_memory_bytes
252
process_virtual_memory_max_bytes
253
provider_reprovider_provide_count
254
provider_reprovider_reprovide_count
255
+target_info
version.go
+1
-1
@@ -14,7 +14,7 @@ const CurrentVersionNumber = "0.38.0-dev"
14
const ApiVersion = "/kubo/" + CurrentVersionNumber + "/" //nolint
15
16
// RepoVersion is the version number that we are currently expecting to see.
17
-const RepoVersion = 17
17
+const RepoVersion = 18
18
19
// GetUserAgentVersion is the libp2p user agent used by go-ipfs.
20
//