fix: disable provide over HTTP with Routing.Type=auto (#9511)
Closes https://github.com/ipfs/kubo/issues/9504
Antonio Navarro Perez committed
Jan 3, 2023 at 21:10 UTC
d6069b93ee8a2f23ca478fbc5938b5e9231a0f22
8 files changed
+111
-13
core/node/libp2p/routingopt.go
+19
-1
@@ -2,6 +2,8 @@ package libp2p
2
3
import (
4
"context"
5
+ "os"
6
+ "strings"
7
"time"
8
9
"github.com/ipfs/go-datastore"
@@ -30,6 +32,13 @@ var defaultHTTPRouters = []string{
32
// TODO: add an independent router from Cloudflare
33
}
34
35
+func init() {
36
+ // Override HTTP routers if custom ones were passed via env
37
+ if routers := os.Getenv("IPFS_HTTP_ROUTERS"); routers != "" {
38
+ defaultHTTPRouters = strings.Split(routers, " ")
39
+ }
40
+}
41
+
42
// ConstructDefaultRouting returns routers used when Routing.Type is unset or set to "auto"
43
func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func(
44
ctx context.Context,
@@ -67,8 +76,17 @@ func ConstructDefaultRouting(peerID string, addrs []string, privKey string) func
76
if err != nil {
77
return nil, err
78
}
79
+
80
+ r := &irouting.Composer{
81
+ GetValueRouter: routinghelpers.Null{},
82
+ PutValueRouter: routinghelpers.Null{},
83
+ ProvideRouter: routinghelpers.Null{}, // modify this when indexers supports provide
84
+ FindPeersRouter: routinghelpers.Null{},
85
+ FindProvidersRouter: httpRouter,
86
+ }
87
+
88
routers = append(routers, &routinghelpers.ParallelRouter{
71
- Router: httpRouter,
89
+ Router: r,
90
IgnoreError: true, // https://github.com/ipfs/kubo/pull/9475#discussion_r1042507387
91
Timeout: 15 * time.Second, // 5x server value from https://github.com/ipfs/kubo/pull/9475#discussion_r1042428529
92
ExecuteAfter: 0,
docs/environment-variables.md
+17
@@ -114,6 +114,23 @@ $ ipfs resolve -r /ipns/dnslink-test2.example.com
114
/ipfs/bafkreicysg23kiwv34eg2d7qweipxwosdo2py4ldv42nbauguluen5v6am
115
```
116
117
+## `IPFS_HTTP_ROUTERS`
118
+
119
+Overrides all implicit HTTP routers enabled when `Routing.Type=auto` with
120
+the space-separated list of URLs provided in this variable.
121
+Useful for testing and debugging in offline contexts.
122
+
123
+Example:
124
+
125
+```console
126
+$ ipfs config Routing.Type auto
127
+$ IPFS_HTTP_ROUTERS="http://127.0.0.1:7423" ipfs daemon
128
+```
129
+
130
+The above will replace implicit HTTP routers with single one, allowing for
131
+inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
132
+or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
133
+
134
## `LIBP2P_TCP_REUSEPORT`
135
136
Kubo tries to reuse the same source port for all connections to improve NAT
docs/examples/kubo-as-a-library/go.mod
+2
-2
@@ -127,7 +127,7 @@ require (
127
github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect
128
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
129
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
130
- github.com/libp2p/go-libp2p-routing-helpers v0.4.1 // indirect
130
+ github.com/libp2p/go-libp2p-routing-helpers v0.5.0 // indirect
131
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
132
github.com/libp2p/go-mplex v0.7.0 // indirect
133
github.com/libp2p/go-msgio v0.2.0 // indirect
@@ -198,7 +198,7 @@ require (
198
go.uber.org/atomic v1.10.0 // indirect
199
go.uber.org/dig v1.15.0 // indirect
200
go.uber.org/fx v1.18.2 // indirect
201
- go.uber.org/multierr v1.8.0 // indirect
201
+ go.uber.org/multierr v1.9.0 // indirect
202
go.uber.org/zap v1.24.0 // indirect
203
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
204
golang.org/x/crypto v0.3.0 // indirect
docs/examples/kubo-as-a-library/go.sum
+4
-4
@@ -793,8 +793,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
793
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
794
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
795
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
796
-github.com/libp2p/go-libp2p-routing-helpers v0.4.1 h1:rOlZiFpUt7SgHm4w62MBvWaQ4UHh7bVJnSnor6RN7j8=
797
-github.com/libp2p/go-libp2p-routing-helpers v0.4.1/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
796
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
797
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
798
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
799
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
800
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
@@ -1377,8 +1377,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
1377
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
1378
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
1379
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
1380
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
1381
-go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
1380
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
1381
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
1382
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
1383
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
1384
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
go.mod
+2
-2
@@ -79,7 +79,7 @@ require (
79
github.com/libp2p/go-libp2p-pubsub v0.8.2
80
github.com/libp2p/go-libp2p-pubsub-router v0.6.0
81
github.com/libp2p/go-libp2p-record v0.2.0
82
- github.com/libp2p/go-libp2p-routing-helpers v0.4.0
82
+ github.com/libp2p/go-libp2p-routing-helpers v0.5.0
83
github.com/libp2p/go-libp2p-testing v0.12.0
84
github.com/libp2p/go-socket-activation v0.1.0
85
github.com/miekg/dns v1.1.50
@@ -231,7 +231,7 @@ require (
231
go.opentelemetry.io/otel/metric v0.30.0 // indirect
232
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
233
go.uber.org/atomic v1.10.0 // indirect
234
- go.uber.org/multierr v1.8.0 // indirect
234
+ go.uber.org/multierr v1.9.0 // indirect
235
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
236
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
237
golang.org/x/net v0.3.0 // indirect
go.sum
+4
-4
@@ -828,8 +828,8 @@ github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqU
828
github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q=
829
github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0=
830
github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk=
831
-github.com/libp2p/go-libp2p-routing-helpers v0.4.0 h1:b7y4aixQ7AwbqYfcOQ6wTw8DQvuRZeTAA0Od3YYN5yc=
832
-github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E=
831
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0 h1:Byujua1X9MeTzbF54i5OwjUNopeg7PYBykuNow/w3p4=
832
+github.com/libp2p/go-libp2p-routing-helpers v0.5.0/go.mod h1:wwK/XSLt6njjO7sRbjhf8w7PGBOfdntMQ2mOQPZ5s/Q=
833
github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8=
834
github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g=
835
github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8=
@@ -1441,8 +1441,8 @@ go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+
1441
go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU=
1442
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
1443
go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
1444
-go.uber.org/multierr v1.8.0 h1:dg6GjLku4EH+249NNmoIciG9N/jURbDG+pFlTkhzIC8=
1445
-go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak=
1444
+go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI=
1445
+go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ=
1446
go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA=
1447
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
1448
go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM=
routing/composer.go
+9
@@ -100,9 +100,18 @@ func (c *Composer) GetValue(ctx context.Context, key string, opts ...routing.Opt
100
func (c *Composer) SearchValue(ctx context.Context, key string, opts ...routing.Option) (<-chan []byte, error) {
101
log.Debug("composer: calling searchValue: ", key)
102
ch, err := c.GetValueRouter.SearchValue(ctx, key, opts...)
103
+
104
+ // avoid nil channels on implementations not supporting SearchValue method.
105
+ if err == routing.ErrNotFound && ch == nil {
106
+ out := make(chan []byte)
107
+ close(out)
108
+ return out, err
109
+ }
110
+
111
if err != nil {
112
log.Debug("composer: calling searchValue error: ", key, err)
113
}
114
+
115
return ch, err
116
}
117
test/sharness/t0172-content-routing-over-http.sh
new
+54
@@ -0,0 +1,54 @@
1
+#!/usr/bin/env bash
2
+
3
+test_description="Test content routing over HTTP"
4
+
5
+. lib/test-lib.sh
6
+
7
+
8
+if ! test_have_prereq SOCAT; then
9
+ skip_all="skipping '$test_description': socat is not available"
10
+ test_done
11
+fi
12
+
13
+test_init_ipfs
14
+
15
+# Run listener on a free port to log HTTP requests sent by Kubo in Routing.Type=auto mode
16
+export ROUTER_PORT=$(comm -23 <(seq 49152 65535 | sort) <(ss -Htan | awk '{print $4}' | cut -d':' -f2) | head -n 1)
17
+export IPFS_HTTP_ROUTERS="http://127.0.0.1:$ROUTER_PORT"
18
+
19
+test_launch_ipfs_daemon
20
+
21
+test_expect_success "start HTTP router proxy" '
22
+ socat TCP-LISTEN:$ROUTER_PORT,reuseaddr,fork,bind=127.0.0.1 STDOUT > http_requests &
23
+ NCPID=$!
24
+'
25
+
26
+## HTTP GETs
27
+
28
+test_expect_success 'create unique CID without adding it to the local datastore' '
29
+ WANT_CID=$(date +"%FT%T.%N%z" | ipfs add -qn)
30
+'
31
+
32
+test_expect_success 'expect HTTP request for unknown CID' '
33
+ ipfs routing findprovs --timeout 3s "$WANT_CID" &&
34
+ test_should_contain "GET /routing/v1/providers/$WANT_CID" http_requests
35
+'
36
+
37
+## HTTP PUTs
38
+
39
+test_expect_success 'add new CID to the local datastore' '
40
+ ADD_CID=$(date +"%FT%T.%N%z" | ipfs add -q)
41
+'
42
+
43
+# cid.contact supports GET-only: https://github.com/ipfs/kubo/issues/9504
44
+# which means no announcements over HTTP should be made.
45
+test_expect_success 'expect no HTTP requests to be sent with locally added CID' '
46
+ test_should_not_contain "$ADD_CID" http_requests
47
+'
48
+
49
+test_expect_success "stop nc" '
50
+ kill "$NCPID" && wait "$NCPID" || true
51
+'
52
+
53
+test_kill_ipfs_daemon
54
+test_done