feat: opt-in http retrieval client (#10772)
* Feat: http retrieval as experimental feature This introduces the http-retrieval capability as an experimental feature. It can be enabled in the configuration `Experimental.HTTPRetrieval.Enabled = true`. Documentation and changelog to be added later. * refactor: HTTPRetrieval.Enabled as Flag * docs(config): HTTPRetrieval section * refactor: reusable MockHTTPContentRouter * feat: HTTPRetrieval.TLSInsecureSkipVerify allows self-signed certificates in tests * feat(config): HTTPRetrieval.MaxBlockSize * test: end-to-end HTTPRetrieval.Enabled this spawns two http services on localhost: 1. HTTP router that returns HTTP provider when /routing/v1/providers/cid i queried 2. HTTP provider that returns a block when /ipfs/cid is queried 3. Configures Kubo to use (1) instead of cid.contact this seems to work (running test with DEBUG=true shows (1) was queried for the test CID and returned multiaddr of (2), but Kubo never requested test CID block from (2) – needs investigation * fix: enable /routing/v1/peers for non-cid.contact we artificially limited every delegated routing endpoint because of cid.contact being limited to one endpoint * feat: Routing.DelegatedRouters make it easy to override the hardcoded implicit HTTP routeur URL without having to set the entire custom Router.Routers and Router.Methods (http_retrieval_client_test.go still needs to be fixed in future commit) * test: flag remaining work * docs: review feedback * refactor: providerQueryMgr with bitswapNetworks this fixes two regressions: (1) introduced in https://github.com/ipfs/kubo/issues/10717 where we only used bitswapLib2p query manager (this is why E2E did not act on http provider) (2) introduced in https://github.com/ipfs/kubo/pull/10765 where it was not possible to set binary peerID in IgnoreProviders (we changed to []string) * refactor: Bitswap.Libp2pEnabled replaces Bitswap.Enabled with Bitswap.Libp2pEnabled adds tests that confirm it is possible to disable libp2p bitswap fully and only keep http in client mode also, removes the need for passing empty blockstore in client-only mode * docs: changelog --------- Co-authored-by: Marcin Rataj <lidel@lidel.org>
Hector Sanjuan committed
May 6, 2025 at 19:06 UTC
b5d73695ba0fdbd71147b5c0e874d65a7bea915a
17 files changed
+751
-203
cmd/ipfs/kubo/daemon.go
+1
@@ -458,6 +458,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
458
cfg.Identity.PeerID,
459
cfg.Addresses,
460
cfg.Identity.PrivKey,
461
+ cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled),
462
)
463
default:
464
return fmt.Errorf("unrecognized routing option: %s", routingOption)
config/bitswap.go
+5
-4
@@ -2,13 +2,14 @@ package config
2
3
// Bitswap holds Bitswap configuration options
4
type Bitswap struct {
5
- // Enabled controls both client and server (enabled by default)
6
- Enabled Flag `json:",omitempty"`
7
- // ServerEnabled controls if the node responds to WANTs (depends on Enabled, enabled by default)
5
+ // Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default)
6
+ // (This can be disabled if HTTPRetrieval.Enabled is set to true)
7
+ Libp2pEnabled Flag `json:",omitempty"`
8
+ // ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default)
9
ServerEnabled Flag `json:",omitempty"`
10
}
11
12
const (
12
- DefaultBitswapEnabled = true
13
+ DefaultBitswapLibp2pEnabled = true
14
DefaultBitswapServerEnabled = true
15
)
config/config.go
+8
-7
@@ -33,13 +33,14 @@ type Config struct {
33
DNS DNS
34
Migration Migration
35
36
- Provider Provider
37
- Reprovider Reprovider
38
- Experimental Experiments
39
- Plugins Plugins
40
- Pinning Pinning
41
- Import Import
42
- Version Version
36
+ Provider Provider
37
+ Reprovider Reprovider
38
+ HTTPRetrieval HTTPRetrieval
39
+ Experimental Experiments
40
+ Plugins Plugins
41
+ Pinning Pinning
42
+ Import Import
43
+ Version Version
44
45
Internal Internal // experimental/unstable options
46
config/http_retrieval.go
new
+19
@@ -0,0 +1,19 @@
1
+package config
2
+
3
+// HTTPRetrieval is the configuration object for HTTP Retrieval settings.
4
+// Implicit defaults can be found in core/node/bitswap.go
5
+type HTTPRetrieval struct {
6
+ Enabled Flag `json:",omitempty"`
7
+ Allowlist []string `json:",omitempty"`
8
+ Denylist []string `json:",omitempty"`
9
+ NumWorkers *OptionalInteger `json:",omitempty"`
10
+ MaxBlockSize *OptionalString `json:",omitempty"`
11
+ TLSInsecureSkipVerify Flag `json:",omitempty"`
12
+}
13
+
14
+const (
15
+ DefaultHTTPRetrievalEnabled = false // opt-in for now, until we figure out https://github.com/ipfs/specs/issues/496
16
+ DefaultHTTPRetrievalNumWorkers = 16
17
+ DefaultHTTPRetrievalTLSInsecureSkipVerify = false // only for testing with self-signed HTTPS certs
18
+ DefaultHTTPRetrievalMaxBlockSize = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes
19
+)
config/init.go
-7
@@ -48,13 +48,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
48
},
49
},
50
51
- Routing: Routing{
52
- Type: nil,
53
- Methods: nil,
54
- Routers: nil,
55
- IgnoreProviders: []peer.ID{},
56
- },
57
-
51
// setup the node mount points.
52
Mounts: Mounts{
53
IPFS: "/ipfs",
config/routing.go
+14
-9
@@ -6,27 +6,30 @@ import (
6
"os"
7
"runtime"
8
"strings"
9
-
10
- peer "github.com/libp2p/go-libp2p/core/peer"
9
)
10
11
const (
12
DefaultAcceleratedDHTClient = false
13
DefaultLoopbackAddressesOnLanDHT = false
14
+ CidContactRoutingURL = "https://cid.contact"
15
+ PublicGoodDelegatedRoutingURL = "https://delegated-ipfs.dev" // cid.contact + amino dht (incl. IPNS PUTs)
16
+ EnvHTTPRouters = "IPFS_HTTP_ROUTERS"
17
+ EnvHTTPRoutersFilterProtocols = "IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS"
18
)
19
20
var (
21
// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
20
- DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{
21
- "https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
22
+ DefaultHTTPRouters = getEnvOrDefault(EnvHTTPRouters, []string{
23
+ CidContactRoutingURL, // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
24
})
25
26
// Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484)
27
// and also filter out locally
26
- DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{
28
+ DefaultHTTPRoutersFilterProtocols = getEnvOrDefault(EnvHTTPRoutersFilterProtocols, []string{
29
"unknown", // allow results without protocol list, we can do libp2p identify to test them
30
"transport-bitswap",
29
- // TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed
31
+ // http is added dynamically in routing/delegated.go.
32
+ // 'transport-ipfs-gateway-http'
33
})
34
)
35
@@ -43,11 +46,13 @@ type Routing struct {
46
47
LoopbackAddressesOnLanDHT Flag `json:",omitempty"`
48
46
- IgnoreProviders []peer.ID
49
+ IgnoreProviders []string `json:",omitempty"`
50
+
51
+ DelegatedRouters []string `json:",omitempty"`
52
48
- Routers Routers
53
+ Routers Routers `json:",omitempty"`
54
50
- Methods Methods
55
+ Methods Methods `json:",omitempty"`
56
}
57
58
type Router struct {
core/node/bitswap.go
+53
-24
@@ -2,24 +2,28 @@ package node
2
3
import (
4
"context"
5
+ "errors"
6
"io"
7
"time"
8
9
+ "github.com/dustin/go-humanize"
10
"github.com/ipfs/boxo/bitswap"
11
"github.com/ipfs/boxo/bitswap/client"
12
+ "github.com/ipfs/boxo/bitswap/network"
13
bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
14
+ "github.com/ipfs/boxo/bitswap/network/httpnet"
15
blockstore "github.com/ipfs/boxo/blockstore"
16
exchange "github.com/ipfs/boxo/exchange"
17
"github.com/ipfs/boxo/exchange/providing"
18
provider "github.com/ipfs/boxo/provider"
19
rpqm "github.com/ipfs/boxo/routing/providerquerymanager"
20
"github.com/ipfs/go-cid"
17
- "github.com/ipfs/go-datastore"
21
ipld "github.com/ipfs/go-ipld-format"
22
+ version "github.com/ipfs/kubo"
23
"github.com/ipfs/kubo/config"
24
irouting "github.com/ipfs/kubo/routing"
25
"github.com/libp2p/go-libp2p/core/host"
22
- "github.com/libp2p/go-libp2p/core/routing"
26
+ peer "github.com/libp2p/go-libp2p/core/peer"
27
"go.uber.org/fx"
28
29
blocks "github.com/ipfs/go-block-format"
@@ -79,38 +83,63 @@ type bitswapIn struct {
83
// Bitswap creates the BitSwap server/client instance.
84
// If Bitswap.ServerEnabled is false, the node will act only as a client
85
// using an empty blockstore to prevent serving blocks to other peers.
82
-func Bitswap(provide bool) interface{} {
86
+func Bitswap(serverEnabled bool) interface{} {
87
return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
84
- bitswapNetwork := bsnet.NewFromIpfsHost(in.Host)
85
- var blockstoree blockstore.Blockstore = in.Bs
86
- var provider routing.ContentDiscovery
88
+ var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
89
+ var bitswapBlockstore blockstore.Blockstore = in.Bs
90
88
- if provide {
91
+ libp2pEnabled := in.Cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
92
+ if libp2pEnabled {
93
+ bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
94
+ }
95
90
- var maxProviders int = DefaultMaxProviders
91
- if in.Cfg.Internal.Bitswap != nil {
92
- maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
96
+ if httpCfg := in.Cfg.HTTPRetrieval; httpCfg.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) {
97
+ maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
98
+ if err != nil {
99
+ return nil, err
100
}
94
-
95
- pqm, err := rpqm.New(bitswapNetwork,
96
- in.Rt,
97
- rpqm.WithMaxProviders(maxProviders),
98
- rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...),
101
+ bitswapHTTP := httpnet.New(in.Host,
102
+ httpnet.WithHTTPWorkers(int(httpCfg.NumWorkers.WithDefault(config.DefaultHTTPRetrievalNumWorkers))),
103
+ httpnet.WithAllowlist(httpCfg.Allowlist),
104
+ httpnet.WithDenylist(httpCfg.Denylist),
105
+ httpnet.WithInsecureSkipVerify(httpCfg.TLSInsecureSkipVerify.WithDefault(config.DefaultHTTPRetrievalTLSInsecureSkipVerify)),
106
+ httpnet.WithMaxBlockSize(int64(maxBlockSize)),
107
+ httpnet.WithUserAgent(version.GetUserAgentVersion()),
108
)
109
+ bitswapNetworks = network.New(in.Host.Peerstore(), bitswapLibp2p, bitswapHTTP)
110
+ } else if libp2pEnabled {
111
+ bitswapNetworks = bitswapLibp2p
112
+ } else {
113
+ return nil, errors.New("invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
114
+ }
115
+
116
+ // Kubo uses own, customized ProviderQueryManager
117
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
118
+ var maxProviders int = DefaultMaxProviders
119
+ if in.Cfg.Internal.Bitswap != nil {
120
+ maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
121
+ }
122
+ ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders))
123
+ for _, str := range in.Cfg.Routing.IgnoreProviders {
124
+ pid, err := peer.Decode(str)
125
if err != nil {
126
return nil, err
127
}
103
- in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
104
- in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(true))
105
- provider = pqm
106
- } else {
107
- provider = nil
108
- // When server is disabled, use an empty blockstore to prevent serving blocks
109
- blockstoree = blockstore.NewBlockstore(datastore.NewMapDatastore())
110
- in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(false))
128
+ ignoredPeerIDs = append(ignoredPeerIDs, pid)
129
+ }
130
+ providerQueryMgr, err := rpqm.New(bitswapNetworks,
131
+ in.Rt,
132
+ rpqm.WithMaxProviders(maxProviders),
133
+ rpqm.WithIgnoreProviders(ignoredPeerIDs...),
134
+ )
135
+ if err != nil {
136
+ return nil, err
137
}
138
113
- bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, blockstoree, in.BitswapOpts...)
139
+ // Explicitly enable/disable server to ensure desired provide mode
140
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
141
+
142
+ bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
143
144
lc.Append(fx.Hook{
145
OnStop: func(ctx context.Context) error {
core/node/groups.go
+7
-6
@@ -335,17 +335,18 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
335
recordLifetime = d
336
}
337
338
- isBitswapEnabled := cfg.Bitswap.Enabled.WithDefault(config.DefaultBitswapEnabled)
338
+ isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
339
isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
340
- // Don't provide from bitswap when the strategic provider service is active
341
- shouldBitswapProvide := isBitswapEnabled && isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
340
+
341
+ // Don't provide from bitswap when the legacy noop experiment "strategic provider service" is active
342
+ isBitswapServerEnabled = isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
343
344
return fx.Options(
345
fx.Provide(BitswapOptions(cfg)),
345
- fx.Provide(Bitswap(shouldBitswapProvide)),
346
- fx.Provide(OnlineExchange(isBitswapEnabled)),
346
+ fx.Provide(Bitswap(isBitswapServerEnabled)),
347
+ fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
348
// Replace our Exchange with a Providing exchange!
348
- fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
349
+ fx.Decorate(ProvidingExchange(isBitswapServerEnabled)),
350
fx.Provide(DNSResolver),
351
fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
352
fx.Provide(Peering),
core/node/libp2p/routingopt.go
+37
-12
@@ -2,6 +2,7 @@ package libp2p
2
3
import (
4
"context"
5
+ "os"
6
"time"
7
8
"github.com/ipfs/go-datastore"
@@ -29,21 +30,44 @@ type RoutingOptionArgs struct {
30
31
type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error)
32
33
+var noopRouter = routinghelpers.Null{}
34
+
35
func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) {
36
var routers []*routinghelpers.ParallelRouter
37
+ httpRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
38
+
39
+ // Use config.DefaultHTTPRouters if custom override was sent via config.EnvHTTPRouters
40
+ // or if user did not set any preference in cfg.Routing.DelegatedRouters
41
+ var httpRouterEndpoints []string
42
+ if os.Getenv(config.EnvHTTPRouters) != "" || len(cfg.Routing.DelegatedRouters) == 0 {
43
+ httpRouterEndpoints = config.DefaultHTTPRouters
44
+ } else {
45
+ httpRouterEndpoints = cfg.Routing.DelegatedRouters
46
+ }
47
+
48
// Append HTTP routers for additional speed
35
- for _, endpoint := range config.DefaultHTTPRouters {
36
- httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
49
+ for _, endpoint := range httpRouterEndpoints {
50
+ httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey, httpRetrievalEnabled)
51
if err != nil {
52
return nil, err
53
}
40
-
54
+ // Mapping router to /routing/v1/* endpoints
55
+ // https://specs.ipfs.tech/routing/http-routing-v1/
56
r := &irouting.Composer{
42
- GetValueRouter: routinghelpers.Null{},
43
- PutValueRouter: routinghelpers.Null{},
44
- ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide
45
- FindPeersRouter: routinghelpers.Null{},
46
- FindProvidersRouter: httpRouter,
57
+ GetValueRouter: httpRouter, // GET /routing/v1/ipns
58
+ PutValueRouter: httpRouter, // PUT /routing/v1/ipns
59
+ ProvideRouter: noopRouter, // we don't have spec for sending provides to /routing/v1 (revisit once https://github.com/ipfs/specs/pull/378 or similar is ratified)
60
+ FindPeersRouter: httpRouter, // /routing/v1/peers
61
+ FindProvidersRouter: httpRouter, // /routing/v1/providers
62
+ }
63
+
64
+ if endpoint == config.CidContactRoutingURL {
65
+ // Special-case: cid.contact only supports /routing/v1/providers/cid
66
+ // we disable other endpoints to avoid sending requests that always fail
67
+ r.GetValueRouter = noopRouter
68
+ r.PutValueRouter = noopRouter
69
+ r.ProvideRouter = noopRouter
70
+ r.FindPeersRouter = noopRouter
71
}
72
73
routers = append(routers, &routinghelpers.ParallelRouter{
@@ -119,7 +143,7 @@ func constructDHTRouting(mode dht.ModeOpt) RoutingOption {
143
}
144
145
// ConstructDelegatedRouting is used when Routing.Type = "custom"
122
-func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string) RoutingOption {
146
+func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string, httpRetrieval bool) RoutingOption {
147
return func(args RoutingOptionArgs) (routing.Routing, error) {
148
return irouting.Parse(routers, methods,
149
&irouting.ExtraDHTParams{
@@ -130,9 +154,10 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
154
Context: args.Ctx,
155
},
156
&irouting.ExtraHTTPParams{
133
- PeerID: peerID,
134
- Addrs: httpAddrsFromConfig(addrs),
135
- PrivKeyB64: privKey,
157
+ PeerID: peerID,
158
+ Addrs: httpAddrsFromConfig(addrs),
159
+ PrivKeyB64: privKey,
160
+ HTTPRetrieval: httpRetrieval,
161
},
162
)
163
}
docs/changelogs/v0.35.md
+106
-22
@@ -10,25 +10,50 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
10
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
+ - [Opt-in HTTP Retrieval client](#opt-in-http-retrieval-client)
14
- [Dedicated `Reprovider.Strategy` for MFS](#dedicated-reproviderstrategy-for-mfs)
14
- - [Additional new configuration options](#additional-new-configuration-options)
15
- [Grid view in WebUI](#grid-view-in-webui)
16
- - [Enhanced DAG-Shaping Controls for `ipfs add`](#enhanced-dag-shaping-controls-for-ipfs-add)
17
- - [New `ipfs add` Options](#new-ipfs-add-options)
18
- - [Persistent `Import.*` Configuration](#persistent-import-configuration)
19
- - [Updated `Import` Profiles](#updated-import-profiles)
16
+ - [Enhanced DAG-Shaping Controls](#enhanced-dag-shaping-controls)
17
+ - [New DAG-Shaping `ipfs add` Options](#new-dag-shaping-ipfs-add-options)
18
+ - [Persistent DAG-Shaping `Import.*` Configuration](#persistent-dag-shaping-import-configuration)
19
+ - [Updated DAG-Shaping `Import` Profiles](#updated-dag-shaping-import-profiles)
20
- [`Datastore` Metrics Now Opt-In](#datastore-metrics-now-opt-in)
21
- - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
22
- - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
23
- - [Pebble Database Format Config](#pebble-database-format-config)
21
+ - [Improved performance of data onboarding](#improved-performance-of-data-onboarding)
22
+ - [Fast `ipfs add` in online mode](#fast-ipfs-add-in-online-mode)
23
+ - [Optimized, dedicated queue for providing fresh CIDs](#optimized-dedicated-queue-for-providing-fresh-cids)
24
+ - [Deprecated `ipfs stats provider`](#deprecated-ipfs-stats-provider)
25
+ - [New `Bitswap` configuration options](#new-bitswap-configuration-options)
26
+ - [New `Routing` configuration options](#new-routing-configuration-options)
27
+ - [New Pebble database format config](#new-pebble-database-format-config)
28
- [📦️ Important dependency updates](#-important-dependency-updates)
29
- [📝 Changelog](#-changelog)
30
- [👨👩👧👦 Contributors](#-contributors)
31
32
### Overview
33
34
+This release brings significant UX and performance improvements to data onboarding, provisioning, and retrieval systems.
35
+
36
+New configuration options let you customize the shape of UnixFS DAGs generated during the data import, control the scope of DAGs announced on the Amino DHT, select which delegated routing endpoints are queried, and choose whether to enable HTTP retrieval alongside Bitswap over Libp2p.
37
+
38
+Continue reading for more details.
39
+
40
+
41
### 🔦 Highlights
42
43
+#### Opt-in HTTP Retrieval client
44
+
45
+This release adds experimental support for retrieving blocks directly over HTTPS (HTTP/2), complementing the existing Bitswap over Libp2p.
46
+
47
+The opt-in client enables Kubo to use [delegated routing](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) results with `/tls/http` multiaddrs, connecting to HTTPS servers that support [Trustless HTTP Gateway](https://specs.ipfs.tech/http-gateways/trustless-gateway)'s Block Responses (`?format=raw`, `application/vnd.ipld.raw`). Fetching blocks via HTTPS (HTTP/2) simplifies infrastructure and reduces costs for storage providers by leveraging HTTP caching and CDNs.
48
+
49
+To enable this feature for testing and feedback, set:
50
+
51
+```console
52
+$ ipfs config --json HTTPRetrieval.Enabled true
53
+```
54
+
55
+See [`HTTPRetrieval`](https://github.com/ipfs/kubo/blob/master/docs/config.md#httpretrieval) for more details.
56
+
57
#### Dedicated `Reprovider.Strategy` for MFS
58
59
The [Mutable File System (MFS)](https://docs.ipfs.tech/concepts/glossary/#mfs) in Kubo is a UnixFS filesystem managed with [`ipfs files`](https://docs.ipfs.tech/reference/kubo/cli/#ipfs-files) commands. It supports familiar file operations like cp and mv within a folder-tree structure, automatically updating a MerkleDAG and a "root CID" that reflects the current MFS state. Files in MFS are protected from garbage collection, offering a simpler alternative to `ipfs pin`. This makes it a popular choice for tools like [IPFS Desktop](https://docs.ipfs.tech/install/ipfs-desktop/) and the [WebUI](https://github.com/ipfs/ipfs-webui/#readme).
@@ -39,19 +64,13 @@ Users relying on the `pinned` strategy can switch to `pinned+mfs` and use MFS al
64
65
See [`Reprovider.Strategy`](https://github.com/ipfs/kubo/blob/master/docs/config.md#reproviderstrategy) for more details.
66
42
-#### Additional new configuration options
43
-
44
-- [`Bitswap`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswap) section with `Enabled` and `ServerEnabled` flags determine whether Kubo initializes Bitswap, enabling just the client or both the client and server.
45
-- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones.
46
-- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content.
47
-
67
#### Grid view in WebUI
68
69
The WebUI, accessible at http://127.0.0.1:5001/webui/, now includes support for the grid view on the _Files_ screen:
70
71
> 
72
54
-#### Enhanced DAG-Shaping Controls for `ipfs add`
73
+#### Enhanced DAG-Shaping Controls
74
75
This release advances CIDv1 support by introducing fine-grained control over UnixFS DAG shaping during data ingestion with the `ipfs add` command.
76
@@ -59,7 +78,7 @@ Wider DAG trees (more links per node, higher fanout, larger thresholds) are bene
78
79
Kubo now allows users to act on these tradeoffs and customize the width of the DAG created by `ipfs add` command.
80
62
-##### New `ipfs add` Options
81
+##### New DAG-Shaping `ipfs add` Options
82
83
Three new options allow you to override default settings for specific import operations:
84
@@ -68,7 +87,7 @@ Three new options allow you to override default settings for specific import ope
87
- Note: Directories exceeding this limit or the `Import.UnixFSHAMTDirectorySizeThreshold` are converted to HAMT-based (sharded across multiple blocks) structures.
88
- `--max-hamt-fanout`: Specifies the maximum number of child nodes for HAMT internal structures.
89
71
-##### Persistent `Import.*` Configuration
90
+##### Persistent DAG-Shaping `Import.*` Configuration
91
92
You can set default values for these options using the following configuration settings:
93
- [`Import.UnixFSFileMaxLinks`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfsfilemaxlinks)
@@ -76,7 +95,7 @@ You can set default values for these options using the following configuration s
95
- [`Import.UnixFSHAMTDirectoryMaxFanout`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorymaxfanout)
96
- [`Import.UnixFSHAMTDirectorySizeThreshold`](https://github.com/ipfs/kubo/blob/master/docs/config.md#importunixfshamtdirectorysizethreshold)
97
79
-##### Updated `Import` Profiles
98
+##### Updated DAG-Shaping `Import` Profiles
99
100
The release updated configuration [profiles](https://github.com/ipfs/kubo/blob/master/docs/config.md#profiles) to incorporate these new `Import.*` settings:
101
- Updated Profile: `test-cid-v1` now includes current defaults as explicit `Import.UnixFSFileMaxLinks=174`, `Import.UnixFSDirectoryMaxLinks=0`, `Import.UnixFSHAMTDirectoryMaxFanout=256` and `Import.UnixFSHAMTDirectorySizeThreshold=256KiB`
@@ -95,11 +114,55 @@ Convenience opt-in [profiles](https://github.com/ipfs/kubo/blob/master/docs/conf
114
115
It is also possible to manually add the `measure` wrapper. See examples in [`Datastore.Spec`](https://github.com/ipfs/kubo/blob/master/docs/config.md#datastorespec) documentation.
116
98
-#### Optimized, dedicated queue for providing fresh CIDs
117
+#### Improved performance of data onboarding
118
+
119
+This Kubo release significantly improves both the speed of ingesting data via `ipfs add` and announcing newly produced CIDs to Amino DHT.
120
+
121
+##### Fast `ipfs add` in online mode
122
+
123
+Adding a large directory of data when `ipfs daemon` was running in online mode took a long time. A significant amount of this time was spent writing to and reading from the persisted provider queue. Due to this, many users had to shut down the daemon and perform data import in offline mode. This release fixes this known limitation, significantly improving the speed of `ipfs add`.
124
+
125
+> [!IMPORTANT]
126
+> Performing `ipfs add` of 10GiB file would take about 30 minutes.
127
+> Now it takes close to 30 seconds.
128
+
129
+Kubo v0.34:
130
+
131
+```console
132
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null
133
+ 100.00 MiB / 100.00 MiB [=====================================================================] 100.00%
134
+real 0m6.464s
135
+
136
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null
137
+ 1000.00 MiB / 1000.00 MiB [===================================================================] 100.00%
138
+real 1m10.542s
139
+
140
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null
141
+ 10.00 GiB / 10.00 GiB [=======================================================================] 100.00%
142
+real 24m5.744s
143
+```
144
+
145
+Kubo v0.35:
146
+
147
+```console
148
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-100M > /dev/null
149
+ 100.00 MiB / 100.00 MiB [=====================================================================] 100.00%
150
+real 0m0.326s
151
+
152
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-1G > /dev/null
153
+ 1.00 GiB / 1.00 GiB [=========================================================================] 100.00%
154
+real 0m2.819s
155
+
156
+$ time kubo/cmd/ipfs/ipfs add -r /tmp/testfiles-10G > /dev/null
157
+ 10.00 GiB / 10.00 GiB [=======================================================================] 100.00%
158
+real 0m28.405s
159
+```
160
+
161
+##### Optimized, dedicated queue for providing fresh CIDs
162
163
From `kubo` [`v0.33.0`](https://github.com/ipfs/kubo/releases/tag/v0.33.0),
164
Bitswap stopped advertising newly added and received blocks to the DHT. Since
102
-then `boxo/provider` is responsible for the provide and reprovide logic. Prior
165
+then `boxo/provider` is responsible for the first time provide and the recurring reprovide logic. Prior
166
to `v0.35.0`, provides and reprovides were handled together in batches, leading
167
to delays in initial advertisements (provides).
168
@@ -113,7 +176,7 @@ concurrent provide operations:
176
> [!TIP]
177
> Users who need to provide large volumes of content immediately should consider removing the cap on concurrent provide operations and also set `Routing.AcceleratedDHTClient` to `true`.
178
116
-##### Deprecated `ipfs stats provider`
179
+###### Deprecated `ipfs stats provider`
180
181
Since the `ipfs stats provider` command was displaying statistics for both
182
provides and reprovides, this command isn't relevant anymore after separating
@@ -125,7 +188,28 @@ but for reprovides only.
188
> [!NOTE]
189
> `ipfs stats provider` still works, but is marked as deprecated and will be removed in a future release. Be mindful that the command provides only statistics about reprovides (similar to `ipfs stats reprovide`) and not the new provide queue (this will be fixed as a part of wider refactor planned for a future release).
190
128
-#### Pebble Database Format Config
191
+#### New `Bitswap` configuration options
192
+
193
+- [`Bitswap.Libp2pEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswaplibp2penabled) determines whether Kubo will use Bitswap over libp2p (both client and server).
194
+- [`Bitswap.ServerEnabled`](https://github.com/ipfs/kubo/blob/master/docs/config.md#bitswapserverenabled) controls whether Kubo functions as a Bitswap server to host and respond to block requests.
195
+- [`Internal.Bitswap.ProviderSearchMaxResults`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapprovidersearchmaxresults) for adjusting the maximum number of providers bitswap client should aim at before it stops searching for new ones.
196
+
197
+#### New `Routing` configuration options
198
+
199
+- [`Routing.IgnoreProviders`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingignoreproviders) allows ignoring specific peer IDs when returned by the content routing system as providers of content.
200
+ - Simplifies testing `HTTPRetrieval.Enabled` in setups where Bitswap over Libp2p and HTTP retrieval is served under different PeerIDs.
201
+- [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters) allows customizing HTTP routers used by Kubo when `Routing.Type` is set to `auto` or `autoclient`.
202
+ - Users are now able to adjust the default routing system and directly query custom routers for increased resiliency or when dataset is too big and CIDs are not announced on Amino DHT.
203
+
204
+> [!TIP]
205
+>
206
+> For example, to use Pinata's routing endpoint in addition to IPNI at `cid.contact`:
207
+>
208
+> ```console
209
+> $ ipfs config --json Routing.DelegatedRouters '["https://cid.contact","https://indexer.pinata.cloud"]'
210
+> ```
211
+
212
+#### New Pebble database format config
213
214
This Kubo release provides node operators with more control over [Pebble's `FormatMajorVersion`](https://github.com/cockroachdb/pebble/tree/master?tab=readme-ov-file#format-major-versions). This allows testing a new Kubo release without automatically migrating Pebble datastores, keeping the ability to switch back to older Kubo.
215
docs/config.md
+135
-15
@@ -37,7 +37,7 @@ config file at runtime.
37
- [`AutoTLS.RegistrationDelay`](#autotlsregistrationdelay)
38
- [`AutoTLS.CAEndpoint`](#autotlscaendpoint)
39
- [`Bitswap`](#bitswap)
40
- - [`Bitswap.Enabled`](#bitswapenabled)
40
+ - [`Bitswap.Libp2pEnabled`](#bitswaplibp2penabled)
41
- [`Bitswap.ServerEnabled`](#bitswapserverenabled)
42
- [`Bootstrap`](#bootstrap)
43
- [`Datastore`](#datastore)
@@ -127,6 +127,7 @@ config file at runtime.
127
- [`Routing.AcceleratedDHTClient`](#routingaccelerateddhtclient)
128
- [`Routing.LoopbackAddressesOnLanDHT`](#routingloopbackaddressesonlandht)
129
- [`Routing.IgnoreProviders`](#routingignoreproviders)
130
+ - [`Routing.DelegatedRouters`](#routingdelegatedrouters)
131
- [`Routing.Routers`](#routingrouters)
132
- [`Routing.Routers: Type`](#routingrouters-type)
133
- [`Routing.Routers: Parameters`](#routingrouters-parameters)
@@ -184,6 +185,13 @@ config file at runtime.
185
- [`DNS`](#dns)
186
- [`DNS.Resolvers`](#dnsresolvers)
187
- [`DNS.MaxCacheTTL`](#dnsmaxcachettl)
188
+ - [`HTTPRetrieval`](#httpretrieval)
189
+ - [`HTTPRetrieval.Enabled`](#httpretrievalenabled)
190
+ - [`HTTPRetrieval.Allowlist`](#httpretrievalallowlist)
191
+ - [`HTTPRetrieval.Denylist`](#httpretrievaldenylist)
192
+ - [`HTTPRetrieval.NumWorkers`](#httpretrievalnumworkers)
193
+ - [`HTTPRetrieval.MaxBlockSize`](#httpretrievalmaxblocksize)
194
+ - [`HTTPRetrieval.TLSInsecureSkipVerify`](#httpretrievaltlsinsecureskipverify)
195
- [`Import`](#import)
196
- [`Import.CidVersion`](#importcidversion)
197
- [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)
@@ -624,16 +632,20 @@ Type: `optionalString`
632
633
## `Bitswap`
634
627
-High level client and server configuration of the [Bitswap Protocol](https://specs.ipfs.tech/bitswap-protocol/).
635
+High level client and server configuration of the [Bitswap Protocol](https://specs.ipfs.tech/bitswap-protocol/) over libp2p.
636
637
For internal configuration see [`Internal.Bitswap`](#internalbitswap).
638
631
-### `Bitswap.Enabled`
639
+For HTTP version see [`HTTPRetrieval`](#httpretrieval).
640
633
-Manages both Bitswap client and server functionality. For testing or operating a node without Bitswap requirements.
641
+### `Bitswap.Libp2pEnabled`
642
+
643
+Determines whether Kubo will use Bitswap over libp2p.
644
+
645
+Disabling this, will remove `/ipfs/bitswap/*` protocol support from [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses, effectively shutting down both Bitswap libp2p client and server.
646
647
> [!WARNING]
636
-> Bitswap is a core component of Kubo, and disabling it completely may cause unpredictable outcomes. Treat this as experimental and use it solely for testing purposes.
648
+> Bitswap over libp2p is a core component of Kubo and the oldest way of exchanging blocks. Disabling it completely may cause unpredictable outcomes, such as retrieval failures, if the only providers were libp2p ones. Treat this as experimental and use it solely for testing purposes with `HTTPRetrieval.Enabled`.
649
650
Default: `true`
651
@@ -643,9 +655,9 @@ Type: `flag`
655
656
Determines whether Kubo functions as a Bitswap server to host and respond to block requests.
657
646
-Disabling the server retains client and protocol support in libp2p identify responses but causes Kubo to reply with "don't have" to all block requests.
658
+Disabling the server retains client and protocol support in [libp2p identify](https://github.com/libp2p/specs/blob/master/identify/README.md) responses but causes Kubo to reply with "don't have" to all block requests.
659
648
-Default: `true`
660
+Default: `true` (requires `Bitswap.Libp2pEnabled`)
661
662
Type: `flag`
663
@@ -1725,7 +1737,7 @@ Contains options for content, peer, and IPNS routing mechanisms.
1737
There are multiple routing options: "auto", "autoclient", "none", "dht", "dhtclient", and "custom".
1738
1739
* **DEFAULT:** If unset, or set to "auto", your node will use the public IPFS DHT (aka "Amino")
1728
- and parallel HTTP routers listed below for additional speed.
1740
+ and parallel [`Routing.DelegatedRouters`](#routingdelegatedrouters) for additional speed.
1741
1742
* If set to "autoclient", your node will behave as in "auto" but without running a DHT server.
1743
@@ -1755,15 +1767,13 @@ To force a specific Amino DHT-only mode, client or server, set `Routing.Type` to
1767
unless you're sure your node is reachable from the public network.
1768
1769
When `Routing.Type` is set to `auto` or `autoclient` your node will accelerate some types of routing
1758
-by leveraging HTTP endpoints compatible with [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/)
1770
+by leveraging [`Routing.DelegatedRouters`](#routingdelegatedrouters) HTTP endpoints compatible with [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/)
1771
introduced in [IPIP-337](https://github.com/ipfs/specs/pull/337)
1772
in addition to the Amino DHT.
1761
-By default, an instance of [IPNI](https://github.com/ipni/specs/blob/main/IPNI.md#readme)
1762
-at https://cid.contact is used.
1773
1764
-Alternative routing rules can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`.
1774
+[Advanced routing rules](https://github.com/ipfs/kubo/blob/master/docs/delegated-routing.md) can be configured in `Routing.Routers` after setting `Routing.Type` to `custom`.
1775
1766
-Default: `auto` (DHT + IPNI)
1776
+Default: `auto` (DHT + [`Routing.DelegatedRouters`](#routingdelegatedrouters))
1777
1778
Type: `optionalString` (`null`/missing means the default)
1779
@@ -1828,16 +1838,32 @@ Type: `bool` (missing means `false`)
1838
1839
### `Routing.IgnoreProviders`
1840
1831
-An array of peerIDs. Any provider record associated to one of these peer IDs is ignored.
1841
+An array of [string-encoded PeerIDs](https://github.com/libp2p/specs/blob/master/peer-ids/peer-ids.md#string-representation). Any provider record associated to one of these peer IDs is ignored.
1842
1843
Apart from ignoring specific providers for reasons like misbehaviour etc. this
1844
setting is useful to ignore providers as a way to indicate preference, when the same provider
1845
is found under different peerIDs (i.e. one for HTTP and one for Bitswap retrieval).
1846
1847
+> [!TIP]
1848
+> This denylist operates on PeerIDs.
1849
+> To deny specific HTTP Provider URL, use [`HTTPRetrieval.Denylist`](#httpretrievaldenylist) instead.
1850
+
1851
Default: `[]`
1852
1839
-Type: `array[peerID]`
1853
+Type: `array[string]`
1854
+
1855
+### `Routing.DelegatedRouters`
1856
1857
+This is an array of URL hostnames that support the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) which are used alongside the DHT when [`Routing.Type`](#routingtype) is set to `auto` or `autoclient`.
1858
+
1859
+> [!TIP]
1860
+> Delegated routing allows IPFS implementations to offload tasks like content routing, peer routing, and naming to a separate process or server while also benefiting from HTTP caching.
1861
+>
1862
+> One can run their own delegated router either by implementing the [Delegated Routing V1 HTTP API](https://specs.ipfs.tech/routing/http-routing-v1/) themselves, or by using [Someguy](https://github.com/ipfs/someguy), a turn-key implementation that proxies requests to the Amino DHT and other delegated routing servers, such as the Network Indexer at `cid.contact`. Public utility instance of Someguy is hosted at [`https://delegated-ipfs.dev`](https://docs.ipfs.tech/concepts/public-utilities/#delegated-routing).
1863
+
1864
+Default: `["https://cid.contact"]` (empty or `nil` will also use this default; to disable delegated routing, set `Routing.Type` to `dht` or `dhtclient`)
1865
+
1866
+Type: `array[string]`
1867
1868
### `Routing.Routers`
1869
@@ -2596,6 +2622,100 @@ Default: Respect DNS Response TTL
2622
2623
Type: `optionalDuration`
2624
2625
+## `HTTPRetrieval`
2626
+
2627
+`HTTPRetrieval` is configuration for pure HTTP retrieval based on Trustless HTTP Gateways'
2628
+[Block Responses (`application/vnd.ipld.raw`)](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw)
2629
+which can be used in addition to or instead of retrieving blocks with [Bitswap over Libp2p](#bitswap).
2630
+
2631
+Default: `{}`
2632
+
2633
+Type: `object`
2634
+
2635
+### `HTTPRetrieval.Enabled`
2636
+
2637
+> [!CAUTION]
2638
+> This feature is **EXPERIMENTAL** and may change in future release. Enable with caution, and provide feedback via GitHub issues.
2639
+
2640
+Controls whether HTTP-based block retrieval is enabled.
2641
+
2642
+When enabled, Kubo will be able to act on `/tls/http` (HTTP/2) providers ([Trustless HTTP Gateways](https://specs.ipfs.tech/http-gateways/trustless-gateway/)) returned by the [`Routing.DelegatedRouters`](#routingdelegatedrouters)
2643
+to perform pure HTTP [block retrievals](https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw)
2644
+in addition to [Bitswap over Libp2p](#bitswap).
2645
+
2646
+HTTP requests for `application/vnd.ipld.raw` will be issued instead of Bitswap if a peer has a `/tls/http` multiaddr
2647
+and the HTTPS server returns HTTP 200 for the [probe path](https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths).
2648
+
2649
+> [!IMPORTANT]
2650
+> - Requires TLS and HTTP/2.
2651
+> - This feature works in the same way as Bitswap: connected HTTP-peers receive optimistic block requests even for content that they are not announcing.
2652
+> - HTTP client does not follow redirects. Providers should keep announcements up to date.
2653
+> - IPFS ecosystem is working towards [supporting HTTP providers on Amino DHT](https://github.com/ipfs/specs/issues/496). Currently, HTTP providers are mostly limited to results from [`Routing.DelegatedRouters`](#routingdelegatedrouters) endpoints and requires `Routing.Type=auto|autoclient`.
2654
+
2655
+Default: `false`
2656
+
2657
+Type: `flag`
2658
+
2659
+### `HTTPRetrieval.Allowlist`
2660
+
2661
+Optional list of hostnames for which HTTP retrieval is allowed for.
2662
+If this list is not empty, only hosts matching these entries will be allowed for HTTP retrieval.
2663
+
2664
+> [!TIP]
2665
+> To limit HTTP retrieval to a provider at `/dns4/example.com/tcp/443/tls/http` (which would serve `HEAD|GET https://example.com/ipfs/cid?format=raw`), set this to `["example.com"]`
2666
+
2667
+Default: `[]`
2668
+
2669
+Type: `array[string]`
2670
+
2671
+### `HTTPRetrieval.Denylist`
2672
+
2673
+Optional list of hostnames for which HTTP retrieval is not allowed.
2674
+Denylist entries take precedence over Allowlist entries.
2675
+
2676
+
2677
+> [!TIP]
2678
+> This denylist operates on HTTP endpoint hostnames.
2679
+> To deny specific PeerID, use [`Routing.IgnoreProviders`](#routingignoreproviders) instead.
2680
+
2681
+Default: `[]`
2682
+
2683
+Type: `array[string]`
2684
+
2685
+### `HTTPRetrieval.NumWorkers`
2686
+
2687
+The number of worker goroutines to use for concurrent HTTP retrieval operations.
2688
+This setting controls the level of parallelism for HTTP-based block retrieval operations.
2689
+Higher values can improve performance when retrieving many blocks but may increase resource usage.
2690
+
2691
+Default: `16`
2692
+
2693
+Type: `optionalInteger`
2694
+
2695
+### `HTTPRetrieval.MaxBlockSize`
2696
+
2697
+Sets the maximum size of a block that the HTTP retrieval client will accept.
2698
+
2699
+> [!NOTE]
2700
+> This setting is a security feature designed to protect Kubo from malicious providers who might send excessively large or invalid data.
2701
+> Increasing this value allows Kubo to retrieve larger blocks from compatible HTTP providers, but doing so reduces interoperability with Bitswap, and increases potential security risks.
2702
+>
2703
+> Learn more: [Supporting Large IPLD Blocks: Why block limits?](https://discuss.ipfs.tech/t/supporting-large-ipld-blocks/15093#why-block-limits-5)
2704
+
2705
+Default: `2MiB` (matching [Bitswap size limit](https://specs.ipfs.tech/bitswap-protocol/#block-sizes))
2706
+
2707
+Type: `optionalString`
2708
+
2709
+### `HTTPRetrieval.TLSInsecureSkipVerify`
2710
+
2711
+Disables TLS certificate validation.
2712
+Allows making HTTPS connections to HTTP/2 test servers with self-signed TLS certificates.
2713
+Only for testing, do not use in production.
2714
+
2715
+Default: `false`
2716
+
2717
+Type: `flag`
2718
+
2719
## `Import`
2720
2721
Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.
docs/delegated-routing.md
+8
@@ -16,6 +16,14 @@ which then got changed and standardized as [Routing V1 HTTP API](https://specs.i
16
17
Kubo 0.23.0 release added support for [self-hosting Routing V1 HTTP API server](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.23.md#self-hosting-routingv1-endpoint-for-delegated-routing-needs).
18
19
+
20
+> [!TIP]
21
+> Kubo 0.35 added support for [`Routing.DelegatedRouters`](https://github.com/ipfs/kubo/blob/master/docs/config.md#routingdelegatedrouters).
22
+>
23
+> Most of users are best served by setting delegated HTTP router URLs there and `Routing.Type` to `auto` or `autoclient`, rather than custom routing with complex `Routing.Routers` and `Routing.Methods` directly.
24
+>
25
+> The rest of this documentation should be considered only by advanced users and researchers.
26
+
27
Now we need a better way to add different routers using different protocols
28
like [Routing V1](https://specs.ipfs.tech/routing/http-routing-v1/) or Amino
29
DHT, and be able to configure them (future routing systems to come) to cover different use cases.
routing/delegated.go
+15
-8
@@ -149,12 +149,13 @@ func parse(visited map[string]bool,
149
}
150
151
type ExtraHTTPParams struct {
152
- PeerID string
153
- Addrs []string
154
- PrivKeyB64 string
152
+ PeerID string
153
+ Addrs []string
154
+ PrivKeyB64 string
155
+ HTTPRetrieval bool
156
}
157
157
-func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string) (routing.Routing, error) {
158
+func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey string, httpRetrieval bool) (routing.Routing, error) {
159
return httpRoutingFromConfig(
160
config.Router{
161
Type: "http",
@@ -163,9 +164,10 @@ func ConstructHTTPRouter(endpoint string, peerID string, addrs []string, privKey
164
},
165
},
166
&ExtraHTTPParams{
166
- PeerID: peerID,
167
- Addrs: addrs,
168
- PrivKeyB64: privKey,
167
+ PeerID: peerID,
168
+ Addrs: addrs,
169
+ PrivKeyB64: privKey,
170
+ HTTPRetrieval: httpRetrieval,
171
},
172
)
173
}
@@ -200,13 +202,18 @@ func httpRoutingFromConfig(conf config.Router, extraHTTP *ExtraHTTPParams) (rout
202
return nil, err
203
}
204
205
+ protocols := config.DefaultHTTPRoutersFilterProtocols
206
+ if extraHTTP.HTTPRetrieval {
207
+ protocols = append(protocols, "transport-ipfs-gateway-http")
208
+ }
209
+
210
cli, err := drclient.New(
211
params.Endpoint,
212
drclient.WithHTTPClient(delegateHTTPClient),
213
drclient.WithIdentity(key),
214
drclient.WithProviderInfo(addrInfo.ID, addrInfo.Addrs),
215
drclient.WithUserAgent(version.GetUserAgentVersion()),
209
- drclient.WithProtocolFilter(config.DefaultHTTPRoutersFilterProtocols),
216
+ drclient.WithProtocolFilter(protocols),
217
drclient.WithStreamResultsRequired(), // https://specs.ipfs.tech/routing/http-routing-v1/#streaming
218
drclient.WithDisabledLocalFiltering(false), // force local filtering in case remote server does not support IPIP-484
219
)
test/cli/bitswap_config_test.go
+72
-34
@@ -1,9 +1,11 @@
1
package cli
2
3
import (
4
+ "strings"
5
"testing"
6
"time"
7
8
+ "github.com/ipfs/boxo/bitswap/network/bsnet"
9
"github.com/ipfs/kubo/config"
10
"github.com/ipfs/kubo/test/cli/harness"
11
"github.com/ipfs/kubo/test/cli/testutils"
@@ -61,32 +63,32 @@ func TestBitswapConfig(t *testing.T) {
63
}
64
})
65
64
- t.Run("server disabled and client enabled", func(t *testing.T) {
66
+ t.Run("client still works when server disabled", func(t *testing.T) {
67
t.Parallel()
68
h := harness.NewT(t)
69
68
- provider := h.NewNode().Init()
69
- provider.SetIPFSConfig("Bitswap.ServerEnabled", false)
70
- provider.StartDaemon()
71
-
72
- requester := h.NewNode().Init().StartDaemon()
73
- hash := requester.IPFSAddStr(string(testData))
70
+ requester := h.NewNode().Init()
71
+ requester.SetIPFSConfig("Bitswap.ServerEnabled", false)
72
+ requester.StartDaemon()
73
75
- provider.Connect(requester)
74
+ provider := h.NewNode().Init().StartDaemon()
75
+ hash := provider.IPFSAddStr(string(testData))
76
+ requester.Connect(provider)
77
78
// Even when the server is disabled, the client should be able to retrieve data
78
- res := provider.RunIPFS("cat", hash)
79
+ res := requester.RunIPFS("cat", hash)
80
assert.Equal(t, testData, res.Stdout.Bytes(), "retrieved data should match original")
81
})
82
82
- t.Run("bitswap completely disabled", func(t *testing.T) {
83
+ t.Run("bitswap over libp2p disabled", func(t *testing.T) {
84
t.Parallel()
85
h := harness.NewT(t)
86
87
requester := h.NewNode().Init()
88
requester.UpdateConfig(func(cfg *config.Config) {
88
- cfg.Bitswap.Enabled = config.False
89
+ cfg.Bitswap.Libp2pEnabled = config.False
90
cfg.Bitswap.ServerEnabled = config.False
91
+ cfg.HTTPRetrieval.Enabled = config.True
92
})
93
requester.StartDaemon()
94
@@ -112,27 +114,63 @@ func TestBitswapConfig(t *testing.T) {
114
assert.Equal(t, []byte("random"), res.Stdout.Bytes(), "cat should return the added data")
115
})
116
115
- // TODO: Disabling Bitswap.Enabled should remove /ifps/bitswap* protocols from `ipfs id`
116
- // t.Run("bitswap protocols disabled", func(t *testing.T) {
117
- // t.Parallel()
118
- // harness.EnableDebugLogging()
119
- // h := harness.NewT(t)
120
-
121
- // provider := h.NewNode().Init()
122
- // provider.SetIPFSConfig("Bitswap.ServerEnabled", false)
123
- // provider = provider.StartDaemon()
124
- // requester := h.NewNode().Init().StartDaemon()
125
- // requester.Connect(provider)
126
- // // Parse and check ID output
127
- // res := provider.IPFS("id", "-f", "<protocols>")
128
- // protocols := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n")
129
-
130
- // // No bitswap protocols should be present
131
- // for _, proto := range protocols {
132
- // assert.NotContains(t, proto, bsnet.ProtocolBitswap, "bitswap protocol %s should not be advertised when server is disabled", proto)
133
- // assert.NotContains(t, proto, bsnet.ProtocolBitswapNoVers, "bitswap protocol %s should not be advertised when server is disabled", proto)
134
- // assert.NotContains(t, proto, bsnet.ProtocolBitswapOneOne, "bitswap protocol %s should not be advertised when server is disabled", proto)
135
- // assert.NotContains(t, proto, bsnet.ProtocolBitswapOneZero, "bitswap protocol %s should not be advertised when server is disabled", proto)
136
- // }
137
- // })
117
+ // Disabling Bitswap.Libp2pEnabled should remove /ipfs/bitswap* protocols from `ipfs id`
118
+ t.Run("disabling bitswap over libp2p removes it from identify protocol list", func(t *testing.T) {
119
+ t.Parallel()
120
+ h := harness.NewT(t)
121
+
122
+ provider := h.NewNode().Init()
123
+ provider.UpdateConfig(func(cfg *config.Config) {
124
+ cfg.Bitswap.Libp2pEnabled = config.False
125
+ cfg.Bitswap.ServerEnabled = config.False
126
+ cfg.HTTPRetrieval.Enabled = config.True
127
+ })
128
+ provider = provider.StartDaemon()
129
+ requester := h.NewNode().Init().StartDaemon()
130
+ requester.Connect(provider)
131
+
132
+ // read libp2p identify from remote peer, and print protocols
133
+ res := requester.IPFS("id", "-f", "<protocols>", provider.PeerID().String())
134
+ protocols := strings.Split(strings.TrimSpace(res.Stdout.String()), "\n")
135
+
136
+ // No bitswap protocols should be present
137
+ for _, proto := range protocols {
138
+ assert.NotContains(t, proto, bsnet.ProtocolBitswap, "bitswap protocol %s should not be advertised when server is disabled", proto)
139
+ assert.NotContains(t, proto, bsnet.ProtocolBitswapNoVers, "bitswap protocol %s should not be advertised when server is disabled", proto)
140
+ assert.NotContains(t, proto, bsnet.ProtocolBitswapOneOne, "bitswap protocol %s should not be advertised when server is disabled", proto)
141
+ assert.NotContains(t, proto, bsnet.ProtocolBitswapOneZero, "bitswap protocol %s should not be advertised when server is disabled", proto)
142
+ }
143
+ })
144
+
145
+ // HTTPRetrieval uses bitswap engine, we need it
146
+ t.Run("errors when both HTTP and libp2p are disabled", func(t *testing.T) {
147
+ t.Parallel()
148
+
149
+ // init Kubo repo
150
+ node := harness.NewT(t).NewNode().Init()
151
+ node.UpdateConfig(func(cfg *config.Config) {
152
+ cfg.HTTPRetrieval.Enabled = config.False
153
+ cfg.Bitswap.Libp2pEnabled = config.False
154
+ cfg.Bitswap.ServerEnabled = config.Default
155
+ })
156
+ res := node.RunIPFS("daemon")
157
+ assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
158
+ assert.Equal(t, 1, res.ExitCode())
159
+ })
160
+
161
+ // HTTPRetrieval uses bitswap engine, we need it
162
+ t.Run("errors when user set conflicting HTTP and libp2p flags", func(t *testing.T) {
163
+ t.Parallel()
164
+
165
+ // init Kubo repo
166
+ node := harness.NewT(t).NewNode().Init()
167
+ node.UpdateConfig(func(cfg *config.Config) {
168
+ cfg.HTTPRetrieval.Enabled = config.False
169
+ cfg.Bitswap.Libp2pEnabled = config.False
170
+ cfg.Bitswap.ServerEnabled = config.True // bad user config: cant enable server when libp2p is down
171
+ })
172
+ res := node.RunIPFS("daemon")
173
+ assert.Contains(t, res.Stderr.Trimmed(), "invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
174
+ assert.Equal(t, 1, res.ExitCode())
175
+ })
176
}
test/cli/content_routing_http_test.go
+9
-55
@@ -1,69 +1,20 @@
1
package cli
2
3
import (
4
- "context"
4
"net/http"
5
"net/http/httptest"
6
"os/exec"
8
- "sync"
7
"testing"
8
"time"
9
12
- "github.com/ipfs/boxo/ipns"
10
"github.com/ipfs/boxo/routing/http/server"
14
- "github.com/ipfs/boxo/routing/http/types"
15
- "github.com/ipfs/boxo/routing/http/types/iter"
16
- "github.com/ipfs/go-cid"
11
+ "github.com/ipfs/kubo/config"
12
"github.com/ipfs/kubo/test/cli/harness"
13
"github.com/ipfs/kubo/test/cli/testutils"
19
- "github.com/libp2p/go-libp2p/core/peer"
20
- "github.com/libp2p/go-libp2p/core/routing"
14
+ "github.com/ipfs/kubo/test/cli/testutils/httprouting"
15
"github.com/stretchr/testify/assert"
16
)
17
24
-type fakeHTTPContentRouter struct {
25
- m sync.Mutex
26
- provideBitswapCalls int
27
- findProvidersCalls int
28
- findPeersCalls int
29
-}
30
-
31
-func (r *fakeHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) {
32
- r.m.Lock()
33
- defer r.m.Unlock()
34
- r.findProvidersCalls++
35
- return iter.FromSlice([]iter.Result[types.Record]{}), nil
36
-}
37
-
38
-// nolint deprecated
39
-func (r *fakeHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {
40
- r.m.Lock()
41
- defer r.m.Unlock()
42
- r.provideBitswapCalls++
43
- return 0, nil
44
-}
45
-
46
-func (r *fakeHTTPContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) {
47
- r.m.Lock()
48
- defer r.m.Unlock()
49
- r.findPeersCalls++
50
- return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil
51
-}
52
-
53
-func (r *fakeHTTPContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) {
54
- return nil, routing.ErrNotSupported
55
-}
56
-
57
-func (r *fakeHTTPContentRouter) PutIPNS(ctx context.Context, name ipns.Name, rec *ipns.Record) error {
58
- return routing.ErrNotSupported
59
-}
60
-
61
-func (r *fakeHTTPContentRouter) numFindProvidersCalls() int {
62
- r.m.Lock()
63
- defer r.m.Unlock()
64
- return r.findProvidersCalls
65
-}
66
-
18
// userAgentRecorder records the user agent of every HTTP request
19
type userAgentRecorder struct {
20
delegate http.Handler
@@ -76,16 +27,19 @@ func (r *userAgentRecorder) ServeHTTP(w http.ResponseWriter, req *http.Request)
27
}
28
29
func TestContentRoutingHTTP(t *testing.T) {
79
- cr := &fakeHTTPContentRouter{}
30
+ mockRouter := &httprouting.MockHTTPContentRouter{}
31
32
// run the content routing HTTP server
82
- userAgentRecorder := &userAgentRecorder{delegate: server.Handler(cr)}
33
+ userAgentRecorder := &userAgentRecorder{delegate: server.Handler(mockRouter)}
34
server := httptest.NewServer(userAgentRecorder)
35
t.Cleanup(func() { server.Close() })
36
37
// setup the node
38
node := harness.NewT(t).NewNode().Init()
88
- node.Runner.Env["IPFS_HTTP_ROUTERS"] = server.URL
39
+ node.UpdateConfig(func(cfg *config.Config) {
40
+ // setup Kubo node to use mocked HTTP Router
41
+ cfg.Routing.DelegatedRouters = []string{server.URL}
42
+ })
43
node.StartDaemon()
44
45
// compute a random CID
@@ -107,7 +61,7 @@ func TestContentRoutingHTTP(t *testing.T) {
61
62
// verify the content router was called
63
assert.Eventually(t, func() bool {
110
- return cr.numFindProvidersCalls() > 0
64
+ return mockRouter.NumFindProvidersCalls() > 0
65
}, time.Minute, 10*time.Millisecond)
66
67
assert.NotEmpty(t, userAgentRecorder.userAgents)
test/cli/http_retrieval_client_test.go
new
+145
@@ -0,0 +1,145 @@
1
+package cli
2
+
3
+import (
4
+ "fmt"
5
+ "net"
6
+ "net/http"
7
+ "net/http/httptest"
8
+ "net/url"
9
+ "os"
10
+ "strings"
11
+ "testing"
12
+
13
+ "github.com/ipfs/boxo/routing/http/server"
14
+ "github.com/ipfs/boxo/routing/http/types"
15
+ "github.com/ipfs/go-cid"
16
+ "github.com/ipfs/kubo/config"
17
+ "github.com/ipfs/kubo/test/cli/harness"
18
+ "github.com/ipfs/kubo/test/cli/testutils"
19
+ "github.com/ipfs/kubo/test/cli/testutils/httprouting"
20
+ "github.com/libp2p/go-libp2p/core/peer"
21
+ "github.com/multiformats/go-multiaddr"
22
+ "github.com/stretchr/testify/assert"
23
+)
24
+
25
+func TestHTTPRetrievalClient(t *testing.T) {
26
+ t.Parallel()
27
+
28
+ // many moving pieces here, show more when debug is needed
29
+ debug := os.Getenv("DEBUG") == "true"
30
+
31
+ // usee local /routing/v1/providers/{cid} and
32
+ // /ipfs/{cid} HTTP servers to confirm HTTP-only retrieval works end-to-end.
33
+ t.Run("works end-to-end with an HTTP-only provider", func(t *testing.T) {
34
+ // setup mocked HTTP Router to handle /routing/v1/providers/cid
35
+ mockRouter := &httprouting.MockHTTPContentRouter{Debug: debug}
36
+ delegatedRoutingServer := httptest.NewServer(server.Handler(mockRouter))
37
+ t.Cleanup(func() { delegatedRoutingServer.Close() })
38
+
39
+ // init Kubo repo
40
+ node := harness.NewT(t).NewNode().Init()
41
+
42
+ node.UpdateConfig(func(cfg *config.Config) {
43
+ // explicitly enable http client
44
+ cfg.HTTPRetrieval.Enabled = config.True
45
+ // allow NewMockHTTPProviderServer to use self-signed TLS cert
46
+ cfg.HTTPRetrieval.TLSInsecureSkipVerify = config.True
47
+ // setup client-only routing which asks both HTTP + DHT
48
+ // cfg.Routing.Type = config.NewOptionalString("autoclient")
49
+ // setup Kubo node to use mocked HTTP Router
50
+ cfg.Routing.DelegatedRouters = []string{delegatedRoutingServer.URL}
51
+ })
52
+
53
+ // compute a random CID
54
+ randStr := string(testutils.RandomBytes(100))
55
+ res := node.PipeStrToIPFS(randStr, "add", "-qn", "--cid-version", "1") // -n means dont add to local repo, just produce CID
56
+ wantCIDStr := res.Stdout.Trimmed()
57
+ testCid := cid.MustParse(wantCIDStr)
58
+
59
+ // setup mock HTTP provider
60
+ httpProviderServer := NewMockHTTPProviderServer(testCid, randStr, debug)
61
+ t.Cleanup(func() { httpProviderServer.Close() })
62
+ httpHost, httpPort, err := splitHostPort(httpProviderServer.URL)
63
+ assert.NoError(t, err)
64
+
65
+ // setup /routing/v1/providers/cid result that points at our mocked HTTP provider
66
+ mockHTTPProviderPeerID := "12D3KooWCjfPiojcCUmv78Wd1NJzi4Mraj1moxigp7AfQVQvGLwH" // static, it does not matter, we only care about multiaddr
67
+ mockHTTPMultiaddr, _ := multiaddr.NewMultiaddr(fmt.Sprintf("/ip4/%s/tcp/%s/tls/http", httpHost, httpPort))
68
+ mpid, _ := peer.Decode(mockHTTPProviderPeerID)
69
+ mockRouter.AddProvider(testCid, &types.PeerRecord{
70
+ Schema: types.SchemaPeer,
71
+ ID: &mpid,
72
+ Addrs: []types.Multiaddr{{Multiaddr: mockHTTPMultiaddr}},
73
+ // no explicit Protocols, ensure multiaddr alone is enough
74
+ })
75
+
76
+ // Start Kubo
77
+ node.StartDaemon()
78
+
79
+ if debug {
80
+ fmt.Printf("delegatedRoutingServer.URL: %s\n", delegatedRoutingServer.URL)
81
+ fmt.Printf("httpProviderServer.URL: %s\n", httpProviderServer.URL)
82
+ fmt.Printf("httpProviderServer.Multiaddr: %s\n", mockHTTPMultiaddr)
83
+ fmt.Printf("testCid: %s\n", testCid)
84
+ }
85
+
86
+ // Now, make Kubo to read testCid. it was not added to local blockstore, so it has only one provider -- a HTTP server.
87
+
88
+ // First, confirm delegatedRoutingServer returned HTTP provider
89
+ findprovsRes := node.IPFS("routing", "findprovs", testCid.String())
90
+ assert.Equal(t, mockHTTPProviderPeerID, findprovsRes.Stdout.Trimmed())
91
+
92
+ // Ok, now attempt retrieval.
93
+ // If there was no timeout and returned bytes match expected body, HTTP routing and retrieval worked end-to-end.
94
+ catRes := node.IPFS("cat", testCid.String())
95
+ assert.Equal(t, randStr, catRes.Stdout.Trimmed())
96
+ })
97
+}
98
+
99
+// NewMockHTTPProviderServer pretends to be http provider that supports
100
+// block response https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
101
+func NewMockHTTPProviderServer(c cid.Cid, body string, debug bool) *httptest.Server {
102
+ expectedPathPrefix := "/ipfs/" + c.String()
103
+ handler := http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
104
+ if debug {
105
+ fmt.Printf("NewMockHTTPProviderServer GET %s\n", req.URL.Path)
106
+ }
107
+ if strings.HasPrefix(req.URL.Path, expectedPathPrefix) {
108
+ w.Header().Set("Content-Type", "application/vnd.ipld.raw")
109
+ w.WriteHeader(http.StatusOK)
110
+ if req.Method == "GET" {
111
+ _, err := w.Write([]byte(body))
112
+ if err != nil {
113
+ fmt.Fprintf(os.Stderr, "NewMockHTTPProviderServer GET %s error: %v\n", req.URL.Path, err)
114
+ }
115
+ }
116
+ } else if strings.HasPrefix(req.URL.Path, "/ipfs/bafkqaaa") {
117
+ // This is probe from https://specs.ipfs.tech/http-gateways/trustless-gateway/#dedicated-probe-paths
118
+ w.Header().Set("Content-Type", "application/vnd.ipld.raw")
119
+ w.WriteHeader(http.StatusOK)
120
+ } else {
121
+ http.Error(w, "Not Found", http.StatusNotFound)
122
+ }
123
+ })
124
+
125
+ // Make it HTTP/2 with self-signed TLS cert
126
+ srv := httptest.NewUnstartedServer(handler)
127
+ srv.EnableHTTP2 = true
128
+ srv.StartTLS()
129
+ return srv
130
+}
131
+
132
+func splitHostPort(httpUrl string) (ipAddr string, port string, err error) {
133
+ u, err := url.Parse(httpUrl)
134
+ if err != nil {
135
+ return "", "", err
136
+ }
137
+ if u.Scheme == "" || u.Host == "" {
138
+ return "", "", fmt.Errorf("invalid URL format: missing scheme or host")
139
+ }
140
+ ipAddr, port, err = net.SplitHostPort(u.Host)
141
+ if err != nil {
142
+ return "", "", fmt.Errorf("failed to split host and port from %q: %w", u.Host, err)
143
+ }
144
+ return ipAddr, port, nil
145
+}
test/cli/testutils/httprouting/mock_http_content_router.go
new
+117
@@ -0,0 +1,117 @@
1
+package httprouting
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "sync"
7
+ "time"
8
+
9
+ "github.com/ipfs/boxo/ipns"
10
+ "github.com/ipfs/boxo/routing/http/server"
11
+ "github.com/ipfs/boxo/routing/http/types"
12
+ "github.com/ipfs/boxo/routing/http/types/iter"
13
+ "github.com/ipfs/go-cid"
14
+ "github.com/libp2p/go-libp2p/core/peer"
15
+ "github.com/libp2p/go-libp2p/core/routing"
16
+)
17
+
18
+// MockHTTPContentRouter provides /routing/v1
19
+// (https://specs.ipfs.tech/routing/http-routing-v1/) server implementation
20
+// based on github.com/ipfs/boxo/routing/http/server
21
+type MockHTTPContentRouter struct {
22
+ m sync.Mutex
23
+ provideBitswapCalls int
24
+ findProvidersCalls int
25
+ findPeersCalls int
26
+ providers map[cid.Cid][]types.Record
27
+ peers map[peer.ID][]*types.PeerRecord
28
+ Debug bool
29
+}
30
+
31
+func (r *MockHTTPContentRouter) FindProviders(ctx context.Context, key cid.Cid, limit int) (iter.ResultIter[types.Record], error) {
32
+ if r.Debug {
33
+ fmt.Printf("MockHTTPContentRouter.FindProviders(%s)\n", key.String())
34
+ }
35
+ r.m.Lock()
36
+ defer r.m.Unlock()
37
+ r.findProvidersCalls++
38
+ if r.providers == nil {
39
+ r.providers = make(map[cid.Cid][]types.Record)
40
+ }
41
+ records, found := r.providers[key]
42
+ if !found {
43
+ return iter.FromSlice([]iter.Result[types.Record]{}), nil
44
+ }
45
+ results := make([]iter.Result[types.Record], len(records))
46
+ for i, rec := range records {
47
+ results[i] = iter.Result[types.Record]{Val: rec}
48
+ if r.Debug {
49
+ fmt.Printf("MockHTTPContentRouter.FindProviders(%s) result: %+v\n", key.String(), rec)
50
+ }
51
+ }
52
+ return iter.FromSlice(results), nil
53
+}
54
+
55
+// nolint deprecated
56
+func (r *MockHTTPContentRouter) ProvideBitswap(ctx context.Context, req *server.BitswapWriteProvideRequest) (time.Duration, error) {
57
+ r.m.Lock()
58
+ defer r.m.Unlock()
59
+ r.provideBitswapCalls++
60
+ return 0, nil
61
+}
62
+
63
+func (r *MockHTTPContentRouter) FindPeers(ctx context.Context, pid peer.ID, limit int) (iter.ResultIter[*types.PeerRecord], error) {
64
+ r.m.Lock()
65
+ defer r.m.Unlock()
66
+ r.findPeersCalls++
67
+
68
+ if r.peers == nil {
69
+ r.peers = make(map[peer.ID][]*types.PeerRecord)
70
+ }
71
+ records, found := r.peers[pid]
72
+ if !found {
73
+ return iter.FromSlice([]iter.Result[*types.PeerRecord]{}), nil
74
+ }
75
+
76
+ results := make([]iter.Result[*types.PeerRecord], len(records))
77
+ for i, rec := range records {
78
+ results[i] = iter.Result[*types.PeerRecord]{Val: rec}
79
+ if r.Debug {
80
+ fmt.Printf("MockHTTPContentRouter.FindPeers(%s) result: %+v\n", pid.String(), rec)
81
+ }
82
+ }
83
+ return iter.FromSlice(results), nil
84
+}
85
+
86
+func (r *MockHTTPContentRouter) GetIPNS(ctx context.Context, name ipns.Name) (*ipns.Record, error) {
87
+ return nil, routing.ErrNotSupported
88
+}
89
+
90
+func (r *MockHTTPContentRouter) PutIPNS(ctx context.Context, name ipns.Name, rec *ipns.Record) error {
91
+ return routing.ErrNotSupported
92
+}
93
+
94
+func (r *MockHTTPContentRouter) NumFindProvidersCalls() int {
95
+ r.m.Lock()
96
+ defer r.m.Unlock()
97
+ return r.findProvidersCalls
98
+}
99
+
100
+// AddProvider adds a record for a given CID
101
+func (r *MockHTTPContentRouter) AddProvider(key cid.Cid, record types.Record) {
102
+ r.m.Lock()
103
+ defer r.m.Unlock()
104
+ if r.providers == nil {
105
+ r.providers = make(map[cid.Cid][]types.Record)
106
+ }
107
+ r.providers[key] = append(r.providers[key], record)
108
+
109
+ peerRecord, ok := record.(*types.PeerRecord)
110
+ if ok {
111
+ if r.peers == nil {
112
+ r.peers = make(map[peer.ID][]*types.PeerRecord)
113
+ }
114
+ pid := peerRecord.ID
115
+ r.peers[*pid] = append(r.peers[*pid], peerRecord)
116
+ }
117
+}