@cryptotaxi247 / kubo / commits / 37c506074

Update to boxo with refactored providerQueryManager. (#10595)

Hector Sanjuan committed Nov 26, 2024 at 12:34 UTC 37c5060742368cfc37a0da53c6c2ad500109365e
19 files changed +93 -47
core/commands/bitswap.go
+3 -18
@@ -5,7 +5,6 @@ import (
5 "io"
6
7 cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"
8 - e "github.com/ipfs/kubo/core/commands/e"
8
9 humanize "github.com/dustin/go-humanize"
10 bitswap "github.com/ipfs/boxo/bitswap"
@@ -53,10 +52,7 @@ Print out all blocks currently on the bitswap wantlist for the local peer.`,
52 return ErrNotOnline
53 }
54
56 - bs, ok := nd.Exchange.(*bitswap.Bitswap)
57 - if !ok {
58 - return e.TypeErr(bs, nd.Exchange)
59 - }
55 + bs := nd.Bitswap
56
57 pstr, found := req.Options[peerOptionName].(string)
58 if found {
@@ -112,12 +108,7 @@ var bitswapStatCmd = &cmds.Command{
108 return cmds.Errorf(cmds.ErrClient, "unable to run offline: %s", ErrNotOnline)
109 }
110
115 - bs, ok := nd.Exchange.(*bitswap.Bitswap)
116 - if !ok {
117 - return e.TypeErr(bs, nd.Exchange)
118 - }
119 -
120 - st, err := bs.Stat()
111 + st, err := nd.Bitswap.Stat()
112 if err != nil {
113 return err
114 }
@@ -134,7 +125,6 @@ var bitswapStatCmd = &cmds.Command{
125 human, _ := req.Options[bitswapHumanOptionName].(bool)
126
127 fmt.Fprintln(w, "bitswap status")
137 - fmt.Fprintf(w, "\tprovides buffer: %d / %d\n", s.ProvideBufLen, bitswap.HasBlockBufferSize)
128 fmt.Fprintf(w, "\tblocks received: %d\n", s.BlocksReceived)
129 fmt.Fprintf(w, "\tblocks sent: %d\n", s.BlocksSent)
130 if human {
@@ -190,17 +180,12 @@ prints the ledger associated with a given peer.
180 return ErrNotOnline
181 }
182
193 - bs, ok := nd.Exchange.(*bitswap.Bitswap)
194 - if !ok {
195 - return e.TypeErr(bs, nd.Exchange)
196 - }
197 -
183 partner, err := peer.Decode(req.Arguments[0])
184 if err != nil {
185 return err
186 }
187
203 - return cmds.EmitOnce(res, bs.LedgerForPeer(partner))
188 + return cmds.EmitOnce(res, nd.Bitswap.LedgerForPeer(partner))
189 },
190 Encoders: cmds.EncoderMap{
191 cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, out *server.Receipt) error {
core/core.go
+3 -1
@@ -19,6 +19,7 @@ import (
19 pin "github.com/ipfs/boxo/pinning/pinner"
20 "github.com/ipfs/go-datastore"
21
22 + bitswap "github.com/ipfs/boxo/bitswap"
23 bserv "github.com/ipfs/boxo/blockservice"
24 bstore "github.com/ipfs/boxo/blockstore"
25 exchange "github.com/ipfs/boxo/exchange"
@@ -102,7 +103,8 @@ type IpfsNode struct {
103 UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
104 OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
105 OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
105 - Exchange exchange.Interface // the block exchange + strategy (bitswap)
106 + Exchange exchange.Interface // the block exchange + strategy
107 + Bitswap *bitswap.Bitswap `optional:"true"` // The Bitswap instance
108 Namesys namesys.NameSystem // the name system, resolves paths to hashes
109 Provider provider.System // the value provider system
110 IpnsRepub *ipnsrp.Republisher `optional:"true"`
core/coreapi/pin.go
+1 -1
@@ -44,7 +44,7 @@ func (api *PinAPI) Add(ctx context.Context, p path.Path, opts ...caopts.PinAddOp
44 return fmt.Errorf("pin: %s", err)
45 }
46
47 - if err := api.provider.Provide(dagNode.Cid()); err != nil {
47 + if err := api.provider.Provide(ctx, dagNode.Cid(), true); err != nil {
48 return err
49 }
50
core/coreapi/unixfs.go
+1 -1
@@ -173,7 +173,7 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.Node, opts ...options
173 }
174
175 if !settings.OnlyHash {
176 - if err := api.provider.Provide(nd.Cid()); err != nil {
176 + if err := api.provider.Provide(ctx, nd.Cid(), true); err != nil {
177 return path.ImmutablePath{}, err
178 }
179 }
core/node/bitswap.go
+54 -9
@@ -5,9 +5,12 @@ import (
5 "time"
6
7 "github.com/ipfs/boxo/bitswap"
8 + "github.com/ipfs/boxo/bitswap/client"
9 "github.com/ipfs/boxo/bitswap/network"
10 blockstore "github.com/ipfs/boxo/blockstore"
11 exchange "github.com/ipfs/boxo/exchange"
12 + "github.com/ipfs/boxo/exchange/providing"
13 + provider "github.com/ipfs/boxo/provider"
14 "github.com/ipfs/kubo/config"
15 irouting "github.com/ipfs/kubo/routing"
16 "github.com/libp2p/go-libp2p/core/host"
@@ -34,7 +37,7 @@ type bitswapOptionsOut struct {
37
38 // BitswapOptions creates configuration options for Bitswap from the config file
39 // and whether to provide data.
37 -func BitswapOptions(cfg *config.Config, provide bool) interface{} {
40 +func BitswapOptions(cfg *config.Config) interface{} {
41 return func() bitswapOptionsOut {
42 var internalBsCfg config.InternalBitswap
43 if cfg.Internal.Bitswap != nil {
@@ -42,7 +45,6 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} {
45 }
46
47 opts := []bitswap.Option{
45 - bitswap.ProvideEnabled(provide),
48 bitswap.ProviderSearchDelay(internalBsCfg.ProviderSearchDelay.WithDefault(DefaultProviderSearchDelay)), // See https://github.com/ipfs/go-ipfs/issues/8807 for rationale
49 bitswap.EngineBlockstoreWorkerCount(int(internalBsCfg.EngineBlockstoreWorkerCount.WithDefault(DefaultEngineBlockstoreWorkerCount))),
50 bitswap.TaskWorkerCount(int(internalBsCfg.TaskWorkerCount.WithDefault(DefaultTaskWorkerCount))),
@@ -55,7 +57,7 @@ func BitswapOptions(cfg *config.Config, provide bool) interface{} {
57 }
58 }
59
58 -type onlineExchangeIn struct {
60 +type bitswapIn struct {
61 fx.In
62
63 Mctx helpers.MetricsCtx
@@ -65,19 +67,62 @@ type onlineExchangeIn struct {
67 BitswapOpts []bitswap.Option `group:"bitswap-options"`
68 }
69
68 -// OnlineExchange creates new LibP2P backed block exchange (BitSwap).
70 +// Bitswap creates the BitSwap server/client instance.
71 // Additional options to bitswap.New can be provided via the "bitswap-options"
72 // group.
71 -func OnlineExchange() interface{} {
72 - return func(in onlineExchangeIn, lc fx.Lifecycle) exchange.Interface {
73 - bitswapNetwork := network.NewFromIpfsHost(in.Host, in.Rt)
73 +func Bitswap(provide bool) interface{} {
74 + return func(in bitswapIn, lc fx.Lifecycle) *bitswap.Bitswap {
75 + bitswapNetwork := network.NewFromIpfsHost(in.Host)
76 +
77 + var provider client.ProviderFinder
78 + if provide {
79 + provider = in.Rt
80 + }
81 + bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, in.Bs, in.BitswapOpts...)
82
75 - exch := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, in.Bs, in.BitswapOpts...)
83 lc.Append(fx.Hook{
84 OnStop: func(ctx context.Context) error {
78 - return exch.Close()
85 + return bs.Close()
86 },
87 })
88 + return bs
89 + }
90 +}
91 +
92 +// OnlineExchange creates new LibP2P backed block exchange.
93 +func OnlineExchange() interface{} {
94 + return func(in *bitswap.Bitswap, lc fx.Lifecycle) exchange.Interface {
95 + lc.Append(fx.Hook{
96 + OnStop: func(ctx context.Context) error {
97 + return in.Close()
98 + },
99 + })
100 + return in
101 + }
102 +}
103 +
104 +type providingExchangeIn struct {
105 + fx.In
106 +
107 + BaseExch exchange.Interface
108 + Provider provider.System
109 +}
110 +
111 +// ProvidingExchange creates a providing.Exchange with the existing exchange
112 +// and the provider.System.
113 +// We cannot do this in OnlineExchange because it causes cycles so this is for
114 +// a decorator.
115 +func ProvidingExchange(provide bool) interface{} {
116 + return func(in providingExchangeIn, lc fx.Lifecycle) exchange.Interface {
117 + exch := in.BaseExch
118 + if provide {
119 + exch = providing.New(in.BaseExch, in.Provider)
120 + lc.Append(fx.Hook{
121 + OnStop: func(ctx context.Context) error {
122 + return exch.Close()
123 + },
124 + })
125 + }
126 return exch
127 }
128 }
core/node/groups.go
+4 -1
@@ -293,8 +293,11 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
293 shouldBitswapProvide := !cfg.Experimental.StrategicProviding
294
295 return fx.Options(
296 - fx.Provide(BitswapOptions(cfg, shouldBitswapProvide)),
296 + fx.Provide(BitswapOptions(cfg)),
297 + fx.Provide(Bitswap(shouldBitswapProvide)),
298 fx.Provide(OnlineExchange()),
299 + // Replace our Exchange with a Providing exchange!
300 + fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
301 fx.Provide(DNSResolver),
302 fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
303 fx.Provide(Peering),
docs/changelogs/v0.32.md
+1 -1
@@ -1,6 +1,6 @@
1 # Kubo changelog v0.32
2
3 -- [v0.32.0](#v0310)
3 +- [v0.32.0](#v0320)
4
5 ## v0.32.0
6
docs/changelogs/v0.33.md
+2
@@ -16,6 +16,8 @@
16
17 #### 📦️ Dependency updates
18
19 +- update `boxo` to [v0.24.TODO](https://github.com/ipfs/boxo/releases/tag/v0.24.TODO)
20 +
21 ### 📝 Changelog
22
23 ### 👨‍👩‍👧‍👦 Contributors
docs/examples/kubo-as-a-library/go.mod
+1 -1
@@ -7,7 +7,7 @@ go 1.23
7 replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
10 + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1
11 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12 github.com/libp2p/go-libp2p v0.37.0
13 github.com/multiformats/go-multiaddr v0.13.0
docs/examples/kubo-as-a-library/go.sum
+2 -2
@@ -298,8 +298,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
298 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
299 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
300 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
301 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
302 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
301 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
302 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
303 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
304 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
305 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
go.mod
+1 -1
@@ -22,7 +22,7 @@ require (
22 github.com/hashicorp/go-version v1.7.0
23 github.com/ipfs-shipyard/nopfs v0.0.12
24 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
25 - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f
25 + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1
26 github.com/ipfs/go-block-format v0.2.0
27 github.com/ipfs/go-cid v0.4.1
28 github.com/ipfs/go-cidutil v0.1.0
go.sum
+2 -2
@@ -362,8 +362,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7Uy
362 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
363 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
364 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
365 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
366 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
365 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
366 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
367 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
368 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
369 github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
test/cli/delegated_routing_v1_http_proxy_test.go
+4 -2
@@ -60,8 +60,10 @@ func TestRoutingV1Proxy(t *testing.T) {
60 nodes := setupNodes(t)
61
62 cidStr := nodes[0].IPFSAddStr(testutils.RandomStr(1000))
63 -
64 - res := nodes[1].IPFS("routing", "findprovs", cidStr)
63 + // Reprovide as initialProviderDelay still ongoing
64 + res := nodes[0].IPFS("bitswap", "reprovide")
65 + require.NoError(t, res.Err)
66 + res = nodes[1].IPFS("routing", "findprovs", cidStr)
67 assert.Equal(t, nodes[0].PeerID().String(), res.Stdout.Trimmed())
68 })
69
test/cli/delegated_routing_v1_http_server_test.go
+4
@@ -14,6 +14,7 @@ import (
14 "github.com/ipfs/kubo/test/cli/harness"
15 "github.com/libp2p/go-libp2p/core/peer"
16 "github.com/stretchr/testify/assert"
17 + "github.com/stretchr/testify/require"
18 )
19
20 func TestRoutingV1Server(t *testing.T) {
@@ -38,6 +39,9 @@ func TestRoutingV1Server(t *testing.T) {
39 text := "hello world " + uuid.New().String()
40 cidStr := nodes[2].IPFSAddStr(text)
41 _ = nodes[3].IPFSAddStr(text)
42 + // Reprovide as initialProviderDelay still ongoing
43 + res := nodes[3].IPFS("bitswap", "reprovide")
44 + require.NoError(t, res.Err)
45
46 cid, err := cid.Decode(cidStr)
47 assert.NoError(t, err)
test/cli/provider_test.go
+3
@@ -42,6 +42,9 @@ func TestProvider(t *testing.T) {
42 defer nodes.StopDaemons()
43
44 cid := nodes[0].IPFSAddStr(time.Now().String())
45 + // Reprovide as initialProviderDelay still ongoing
46 + res := nodes[0].IPFS("bitswap", "reprovide")
47 + require.NoError(t, res.Err)
48 expectProviders(t, cid, nodes[0].PeerID().String(), nodes[1:]...)
49 })
50
test/cli/routing_dht_test.go
+4 -1
@@ -84,7 +84,10 @@ func testRoutingDHT(t *testing.T, enablePubsub bool) {
84 t.Run("ipfs routing findprovs", func(t *testing.T) {
85 t.Parallel()
86 hash := nodes[3].IPFSAddStr("some stuff")
87 - res := nodes[4].IPFS("routing", "findprovs", hash)
87 + // Reprovide as initialProviderDelay still ongoing
88 + res := nodes[3].IPFS("bitswap", "reprovide")
89 + require.NoError(t, res.Err)
90 + res = nodes[4].IPFS("routing", "findprovs", hash)
91 assert.Equal(t, nodes[3].PeerID().String(), res.Stdout.Trimmed())
92 })
93
test/dependencies/go.mod
+1 -1
@@ -119,7 +119,7 @@ require (
119 github.com/huin/goupnp v1.3.0 // indirect
120 github.com/inconshreveable/mousetrap v1.1.0 // indirect
121 github.com/ipfs/bbloom v0.0.4 // indirect
122 - github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f // indirect
122 + github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 // indirect
123 github.com/ipfs/go-block-format v0.2.0 // indirect
124 github.com/ipfs/go-cid v0.4.1 // indirect
125 github.com/ipfs/go-datastore v0.6.0 // indirect
test/dependencies/go.sum
+2 -2
@@ -318,8 +318,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
318 github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
319 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
320 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
321 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f h1:3qgBUQ6BYfEAPaoSYoH90PKwVT1/iFLX7fDGGkvXZ8Y=
322 -github.com/ipfs/boxo v0.24.4-0.20241119153247-5929aca3037f/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
321 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1 h1:Ox1qTlON8qG46rUL7dDEwnIt7W9MhaidtvR/97RywWw=
322 +github.com/ipfs/boxo v0.24.4-0.20241125210908-37756ce2eeb1/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4=
323 github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
324 github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM=
325 github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s=
test/sharness/t0220-bitswap.sh
-3
@@ -18,7 +18,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
18 test_expect_success "'ipfs bitswap stat' output looks good" '
19 cat <<EOF | unexpand -t2 >expected &&
20 bitswap status
21 - provides buffer: 0 / 256
21 blocks received: 0
22 blocks sent: 0
23 data received: 0
@@ -56,7 +55,6 @@ test_expect_success "'ipfs bitswap stat' succeeds" '
55 test_expect_success "'ipfs bitswap stat' output looks good" '
56 cat <<EOF | unexpand -t2 >expected &&
57 bitswap status
59 - provides buffer: 0 / 256
58 blocks received: 0
59 blocks sent: 0
60 data received: 0
@@ -85,7 +83,6 @@ test_expect_success "'ipfs bitswap stat --human' succeeds" '
83 test_expect_success "'ipfs bitswap stat --human' output looks good" '
84 cat <<EOF | unexpand -t2 >expected &&
85 bitswap status
88 - provides buffer: 0 / 256
86 blocks received: 0
87 blocks sent: 0
88 data received: 0 B