feat: built-in content blocking based on IPIP-383 (#10161)
Fixes #8492 This introduces "nopfs" as a preloaded plugin into Kubo with support for denylists from https://github.com/ipfs/specs/pull/383 It automatically makes Kubo watch *.deny files found in: - /etc/ipfs/denylists - $XDG_CONFIG_HOME/ipfs/denylists - $IPFS_PATH/denylists * test: Gateway.NoFetch and GatewayOverLibp2p adds missing tests for "no fetch" gateways one can expose, in both cases the offline mode is done by passing custom blockservice/exchange into path resolver, which means global path resolver that has nopfs intercept is not used, and the content blocking does not happen on these gateways. * fix: use offline path resolvers where appropriate this fixes the problem described in https://github.com/ipfs/kubo/pull/10161#issuecomment-1782175955 by adding explicit offline path resolvers that are backed by offline exchange, and using them in NoFetch gateways instead of the default online ones --------- Co-authored-by: Henrique Dias <hacdias@gmail.com> Co-authored-by: Marcin Rataj <lidel@lidel.org>
Hector Sanjuan committed
Oct 28, 2023 at 05:34 UTC
a0f34b16ddc8151fd0ba8ab9674db25354232495
19 files changed
+596
-65
README.md
+1
@@ -30,6 +30,7 @@ Featureset
30
- [HTTP Kubo RPC API](https://docs.ipfs.tech/reference/kubo/rpc/) (`/api/v0`) to access and control the daemon
31
- [Command Line Interface](https://docs.ipfs.tech/reference/kubo/cli/) based on (`/api/v0`) RPC API
32
- [WebUI](https://github.com/ipfs/ipfs-webui/#readme) to manage the Kubo node
33
+- [Content blocking](/docs/content-blocking.md) support for operators of public nodes
34
35
### Other implementations
36
core/commands/dag/export.go
+11
-5
@@ -14,6 +14,7 @@ import (
14
cid "github.com/ipfs/go-cid"
15
ipld "github.com/ipfs/go-ipld-format"
16
"github.com/ipfs/kubo/core/commands/cmdenv"
17
+ "github.com/ipfs/kubo/core/commands/cmdutils"
18
19
cmds "github.com/ipfs/go-ipfs-cmds"
20
gocar "github.com/ipld/go-car"
@@ -21,12 +22,10 @@ import (
22
)
23
24
func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
24
- c, err := cid.Decode(req.Arguments[0])
25
+ // Accept CID or a content path
26
+ p, err := cmdutils.PathOrCidPath(req.Arguments[0])
27
if err != nil {
26
- return fmt.Errorf(
27
- "unable to parse root specification (currently only bare CIDs are supported): %s",
28
- err,
29
- )
28
+ return err
29
}
30
31
api, err := cmdenv.GetApi(env, req)
@@ -34,6 +33,13 @@ func dagExport(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment
33
return err
34
}
35
36
+ // Resolve path and confirm the root block is available, fail fast if not
37
+ b, err := api.Block().Stat(req.Context, p)
38
+ if err != nil {
39
+ return err
40
+ }
41
+ c := b.Path().RootCid()
42
+
43
pipeR, pipeW := io.Pipe()
44
45
errCh := make(chan error, 2) // we only report the 1st error
core/core.go
+31
-27
@@ -76,35 +76,39 @@ type IpfsNode struct {
76
PNetFingerprint libp2p.PNetFingerprint `optional:"true"` // fingerprint of private network
77
78
// Services
79
- Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
80
- Blockstore bstore.GCBlockstore // the block store (lower level)
81
- Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
82
- BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
83
- GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
84
- Blocks bserv.BlockService // the block service, get/add blocks.
85
- DAG ipld.DAGService // the merkle dag service, get/add objects.
86
- IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
87
- UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
88
- Reporter *metrics.BandwidthCounter `optional:"true"`
89
- Discovery mdns.Service `optional:"true"`
90
- FilesRoot *mfs.Root
91
- RecordValidator record.Validator
79
+ Peerstore pstore.Peerstore `optional:"true"` // storage for other Peer instances
80
+ Blockstore bstore.GCBlockstore // the block store (lower level)
81
+ Filestore *filestore.Filestore `optional:"true"` // the filestore blockstore
82
+ BaseBlocks node.BaseBlocks // the raw blockstore, no filestore wrapping
83
+ GCLocker bstore.GCLocker // the locker used to protect the blockstore during gc
84
+ Blocks bserv.BlockService // the block service, get/add blocks.
85
+ DAG ipld.DAGService // the merkle dag service, get/add objects.
86
+ IPLDFetcherFactory fetcher.Factory `name:"ipldFetcher"` // fetcher that paths over the IPLD data model
87
+ UnixFSFetcherFactory fetcher.Factory `name:"unixfsFetcher"` // fetcher that interprets UnixFS data
88
+ OfflineIPLDFetcherFactory fetcher.Factory `name:"offlineIpldFetcher"` // fetcher that paths over the IPLD data model without fetching new blocks
89
+ OfflineUnixFSFetcherFactory fetcher.Factory `name:"offlineUnixfsFetcher"` // fetcher that interprets UnixFS data without fetching new blocks
90
+ Reporter *metrics.BandwidthCounter `optional:"true"`
91
+ Discovery mdns.Service `optional:"true"`
92
+ FilesRoot *mfs.Root
93
+ RecordValidator record.Validator
94
95
// Online
94
- PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
95
- Peering *peering.PeeringService `optional:"true"`
96
- Filters *ma.Filters `optional:"true"`
97
- Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
98
- Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
99
- DNSResolver *madns.Resolver // the DNS resolver
100
- IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
101
- UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
102
- Exchange exchange.Interface // the block exchange + strategy (bitswap)
103
- Namesys namesys.NameSystem // the name system, resolves paths to hashes
104
- Provider provider.System // the value provider system
105
- IpnsRepub *ipnsrp.Republisher `optional:"true"`
106
- GraphExchange graphsync.GraphExchange `optional:"true"`
107
- ResourceManager network.ResourceManager `optional:"true"`
96
+ PeerHost p2phost.Host `optional:"true"` // the network host (server+client)
97
+ Peering *peering.PeeringService `optional:"true"`
98
+ Filters *ma.Filters `optional:"true"`
99
+ Bootstrapper io.Closer `optional:"true"` // the periodic bootstrapper
100
+ Routing irouting.ProvideManyRouter `optional:"true"` // the routing system. recommend ipfs-dht
101
+ DNSResolver *madns.Resolver // the DNS resolver
102
+ IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"` // The IPLD path resolver
103
+ UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"` // The UnixFS path resolver
104
+ OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"` // The IPLD path resolver that uses only locally available blocks
105
+ OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"` // The UnixFS path resolver that uses only locally available blocks
106
+ Exchange exchange.Interface // the block exchange + strategy (bitswap)
107
+ Namesys namesys.NameSystem // the name system, resolves paths to hashes
108
+ Provider provider.System // the value provider system
109
+ IpnsRepub *ipnsrp.Republisher `optional:"true"`
110
+ GraphExchange graphsync.GraphExchange `optional:"true"`
111
+ ResourceManager network.ResourceManager `optional:"true"`
112
113
PubSub *pubsub.PubSub `optional:"true"`
114
PSRouter *psrouter.PubsubValueStore `optional:"true"`
core/corehttp/gateway.go
+16
-2
@@ -81,7 +81,11 @@ func Libp2pGatewayOption() ServeOption {
81
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
82
bserv := blockservice.New(n.Blocks.Blockstore(), offline.Exchange(n.Blocks.Blockstore()))
83
84
- backend, err := gateway.NewBlocksBackend(bserv)
84
+ backend, err := gateway.NewBlocksBackend(bserv,
85
+ // GatewayOverLibp2p only returns things that are in local blockstore
86
+ // (same as Gateway.NoFetch=true), we have to pass offline path resolver
87
+ gateway.WithResolver(n.OfflineUnixFSPathResolver),
88
+ )
89
if err != nil {
90
return nil, err
91
}
@@ -111,6 +115,8 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
115
bserv := n.Blocks
116
var vsRouting routing.ValueStore = n.Routing
117
nsys := n.Namesys
118
+ pathResolver := n.UnixFSPathResolver
119
+
120
if cfg.Gateway.NoFetch {
121
bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
122
@@ -130,9 +136,17 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
136
if err != nil {
137
return nil, fmt.Errorf("error constructing namesys: %w", err)
138
}
139
+
140
+ // Gateway.NoFetch=true requires offline path resolver
141
+ // to avoid fetching missing blocks during path traversal
142
+ pathResolver = n.OfflineUnixFSPathResolver
143
}
144
135
- backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
145
+ backend, err := gateway.NewBlocksBackend(bserv,
146
+ gateway.WithValueStore(vsRouting),
147
+ gateway.WithNameSystem(nsys),
148
+ gateway.WithResolver(pathResolver),
149
+ )
150
if err != nil {
151
return nil, err
152
}
core/node/core.go
+32
-19
@@ -7,6 +7,7 @@ import (
7
"github.com/ipfs/boxo/blockservice"
8
blockstore "github.com/ipfs/boxo/blockstore"
9
exchange "github.com/ipfs/boxo/exchange"
10
+ offline "github.com/ipfs/boxo/exchange/offline"
11
"github.com/ipfs/boxo/fetcher"
12
bsfetcher "github.com/ipfs/boxo/fetcher/impl/blockservice"
13
"github.com/ipfs/boxo/filestore"
@@ -21,9 +22,6 @@ import (
22
format "github.com/ipfs/go-ipld-format"
23
"github.com/ipfs/go-unixfsnode"
24
dagpb "github.com/ipld/go-codec-dagpb"
24
- "github.com/ipld/go-ipld-prime"
25
- basicnode "github.com/ipld/go-ipld-prime/node/basic"
26
- "github.com/ipld/go-ipld-prime/schema"
25
"go.uber.org/fx"
26
27
"github.com/ipfs/kubo/core/node/helpers"
@@ -87,43 +85,58 @@ func (s *syncDagService) Session(ctx context.Context) format.NodeGetter {
85
// FetchersOut allows injection of fetchers.
86
type FetchersOut struct {
87
fx.Out
90
- IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
91
- UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
88
+ IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
89
+ UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
90
+ OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
91
+ OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
92
}
93
94
// FetchersIn allows using fetchers for other dependencies.
95
type FetchersIn struct {
96
fx.In
97
- IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
98
- UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
97
+ IPLDFetcher fetcher.Factory `name:"ipldFetcher"`
98
+ UnixfsFetcher fetcher.Factory `name:"unixfsFetcher"`
99
+ OfflineIPLDFetcher fetcher.Factory `name:"offlineIpldFetcher"`
100
+ OfflineUnixfsFetcher fetcher.Factory `name:"offlineUnixfsFetcher"`
101
}
102
103
// FetcherConfig returns a fetcher config that can build new fetcher instances
104
func FetcherConfig(bs blockservice.BlockService) FetchersOut {
105
ipldFetcher := bsfetcher.NewFetcherConfig(bs)
104
- ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(func(lnk ipld.Link, lnkCtx ipld.LinkContext) (ipld.NodePrototype, error) {
105
- if tlnkNd, ok := lnkCtx.LinkNode.(schema.TypedLinkNode); ok {
106
- return tlnkNd.LinkTargetNodePrototype(), nil
107
- }
108
- return basicnode.Prototype.Any, nil
109
- })
110
-
106
+ ipldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
107
unixFSFetcher := ipldFetcher.WithReifier(unixfsnode.Reify)
112
- return FetchersOut{IPLDFetcher: ipldFetcher, UnixfsFetcher: unixFSFetcher}
108
+
109
+ // Construct offline versions which we can safely use in contexts where
110
+ // path resolution should not fetch new blocks via exchange.
111
+ offlineBs := blockservice.New(bs.Blockstore(), offline.Exchange(bs.Blockstore()))
112
+ offlineIpldFetcher := bsfetcher.NewFetcherConfig(offlineBs)
113
+ offlineIpldFetcher.PrototypeChooser = dagpb.AddSupportToChooser(bsfetcher.DefaultPrototypeChooser)
114
+ offlineUnixFSFetcher := offlineIpldFetcher.WithReifier(unixfsnode.Reify)
115
+
116
+ return FetchersOut{
117
+ IPLDFetcher: ipldFetcher,
118
+ UnixfsFetcher: unixFSFetcher,
119
+ OfflineIPLDFetcher: offlineIpldFetcher,
120
+ OfflineUnixfsFetcher: offlineUnixFSFetcher,
121
+ }
122
}
123
124
// PathResolversOut allows injection of path resolvers
125
type PathResolversOut struct {
126
fx.Out
118
- IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
119
- UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
127
+ IPLDPathResolver pathresolver.Resolver `name:"ipldPathResolver"`
128
+ UnixFSPathResolver pathresolver.Resolver `name:"unixFSPathResolver"`
129
+ OfflineIPLDPathResolver pathresolver.Resolver `name:"offlineIpldPathResolver"`
130
+ OfflineUnixFSPathResolver pathresolver.Resolver `name:"offlineUnixFSPathResolver"`
131
}
132
133
// PathResolverConfig creates path resolvers with the given fetchers.
134
func PathResolverConfig(fetchers FetchersIn) PathResolversOut {
135
return PathResolversOut{
125
- IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
126
- UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
136
+ IPLDPathResolver: pathresolver.NewBasicResolver(fetchers.IPLDFetcher),
137
+ UnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.UnixfsFetcher),
138
+ OfflineIPLDPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineIPLDFetcher),
139
+ OfflineUnixFSPathResolver: pathresolver.NewBasicResolver(fetchers.OfflineUnixfsFetcher),
140
}
141
}
142
docs/changelogs/v0.24.md
+9
@@ -6,6 +6,7 @@
6
7
- [Overview](#overview)
8
- [🔦 Highlights](#-highlights)
9
+ - [Support for content blocking](#support-for-content-blocking)
10
- [Gateway: the root of the CARs are no longer meaningful](#gateway-the-root-of-the-cars-are-no-longer-meaningful)
11
- [IPNS: improved publishing defaults](#ipns-improved-publishing-defaults)
12
- [IPNS: record TTL is used for caching](#ipns-record-ttl-is-used-for-caching)
@@ -16,6 +17,14 @@
17
18
### 🔦 Highlights
19
20
+#### Support for content blocking
21
+
22
+This Kubo release ships with built-in content-blocking subsystem [announced earlier this year](https://blog.ipfs.tech/2023-content-blocking-for-the-ipfs-stack/).
23
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
24
+The official build does not ship with any denylists.
25
+
26
+Learn more at [`/docs/content-blocking.md`](https://github.com/ipfs/kubo/blob/master/docs/content-blocking.md)
27
+
28
#### Gateway: the root of the CARs are no longer meaningful
29
30
When requesting a CAR from the gateway, the root of the CAR might no longer be
docs/content-blocking.md
new
+73
@@ -0,0 +1,73 @@
1
+<h1 align="center">
2
+ <br>
3
+ <a href="#readme"><img src="https://github.com/ipfs-shipyard/nopfs/blob/41484a818e6542314f784da852fc41b76f2d48a6/logo.png?raw=true" alt="content blocking logo" title="content blocking in Kubo" width="200"></a>
4
+ <br>
5
+ Content Blocking in Kubo
6
+ <br>
7
+</h1>
8
+
9
+Kubo ships with built-in support for denylist format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
10
+
11
+## Default behavior
12
+
13
+Official Kubo build does not ship with any denylists enabled by default.
14
+
15
+Content blocking is an opt-in decision made by the operator of `ipfs daemon`.
16
+
17
+## How to enable blocking
18
+
19
+Place a `*.deny` file in one of directories:
20
+
21
+- `$IPFS_PATH/denylists/` (`$HOME/.ipfs/denylists/` if `IPFS_PATH` is not set)
22
+- `$XDG_CONFIG_HOME/ipfs/denylists/` (`$HOME/.config/ipfs/denylists/` if `XDG_CONFIG_HOME` is not set)
23
+- `/etc/ipfs/denylists/` (global)
24
+
25
+Files need to be present before starting the `ipfs daemon` in order to be watched for updates.
26
+
27
+If a new denylist file is added, `ipfs daemon` needs to be restarted.
28
+
29
+CLI and Gateway users will receive errors in response to request impacted by a blocklist:
30
+
31
+```
32
+Error: /ipfs/QmQvjk82hPkSaZsyJ8vNER5cmzKW7HyGX5XVusK7EAenCN is blocked and cannot be provided
33
+```
34
+
35
+End user is not informed about the exact reason, see [How to
36
+debug](#how-to-debug) if you need to find out which line of which denylist
37
+caused the request to be blocked.
38
+
39
+## Denylist file format
40
+
41
+[NOpfs](https://github.com/ipfs-shipyard/nopfs) supports the format from [IPIP-383](https://github.com/ipfs/specs/pull/383).
42
+
43
+Clear-text rules are simple: just put content paths to block, one per line.
44
+Paths with unicode and whitespace need to be percend-encoded:
45
+
46
+```
47
+/ipfs/QmbWqxBEKC3P8tqsKc98xmWNzrzDtRLMiMPL8wBuTGsMnR
48
+/ipfs/bafybeihfg3d7rdltd43u3tfvncx7n5loqofbsobojcadtmokrljfthuc7y/927%20-%20Standards/927%20-%20Standards.png
49
+```
50
+
51
+Sensitive content paths can be double-hashed to block without revealing them.
52
+Double-hashed list example: https://badbits.dwebops.pub/badbits.deny
53
+
54
+See [IPIP-383](https://github.com/ipfs/specs/pull/383) for detailed format specification and more examples.
55
+
56
+## How to suspend blocking without removing denylists
57
+
58
+Set `IPFS_CONTENT_BLOCKING_DISABLE` environment variable to `true` and restart the daemon.
59
+
60
+
61
+## How to debug
62
+
63
+Debug logging of `nopfs` subsystem can be enabled with `GOLOG_LOG_LEVEL="nopfs=debug"`
64
+
65
+All block events are logged as warnings on a separate level named `nopfs-blocks`.
66
+
67
+To only log requests for blocked content set `GOLOG_LOG_LEVEL="nopfs-blocks=warn"`:
68
+
69
+```
70
+WARN (...) QmRFniDxwxoG2n4AcnGhRdjqDjCM5YeUcBE75K8WXmioH3: blocked (test.deny:9)
71
+```
72
+
73
+
docs/environment-variables.md
+6
@@ -131,6 +131,12 @@ The above will replace implicit HTTP routers with single one, allowing for
131
inspection/debug of HTTP requests sent by Kubo via `while true ; do nc -l 7423; done`
132
or more advanced tools like [mitmproxy](https://docs.mitmproxy.org/stable/#mitmproxy).
133
134
+
135
+## `IPFS_CONTENT_BLOCKING_DISABLE`
136
+
137
+Disables the content-blocking subsystem. No denylists will be watched and no
138
+content will be blocked.
139
+
140
## `LIBP2P_TCP_REUSEPORT`
141
142
Kubo tries to reuse the same source port for all connections to improve NAT
docs/examples/kubo-as-a-library/go.mod
+5
-1
@@ -7,7 +7,7 @@ go 1.20
7
replace github.com/ipfs/kubo => ./../../..
8
9
require (
10
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
10
+ github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
11
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12
github.com/libp2p/go-libp2p v0.31.0
13
github.com/multiformats/go-multiaddr v0.11.0
@@ -40,6 +40,7 @@ require (
40
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
41
github.com/flynn/noise v1.0.0 // indirect
42
github.com/francoispqt/gojay v1.2.13 // indirect
43
+ github.com/fsnotify/fsnotify v1.6.0 // indirect
44
github.com/gabriel-vasile/mimetype v1.4.1 // indirect
45
github.com/go-logr/logr v1.2.4 // indirect
46
github.com/go-logr/stdr v1.2.2 // indirect
@@ -60,6 +61,8 @@ require (
61
github.com/hashicorp/golang-lru v0.5.4 // indirect
62
github.com/hashicorp/golang-lru/v2 v2.0.7 // indirect
63
github.com/huin/goupnp v1.2.0 // indirect
64
+ github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c // indirect
65
+ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c // indirect
66
github.com/ipfs/bbloom v0.0.4 // indirect
67
github.com/ipfs/go-bitfield v1.1.0 // indirect
68
github.com/ipfs/go-block-format v0.2.0 // indirect
@@ -194,5 +197,6 @@ require (
197
google.golang.org/grpc v1.55.0 // indirect
198
google.golang.org/protobuf v1.31.0 // indirect
199
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
200
+ gopkg.in/yaml.v3 v3.0.1 // indirect
201
lukechampine.com/blake3 v1.2.1 // indirect
202
)
docs/examples/kubo-as-a-library/go.sum
+8
-2
@@ -163,6 +163,7 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk
163
github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
164
github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
165
github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
166
+github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
167
github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
168
github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
169
github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
@@ -298,10 +299,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
299
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
300
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
301
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
302
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
303
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
304
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
305
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
306
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
307
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
303
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
304
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
308
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
309
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
310
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
311
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
312
github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -1001,6 +1006,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
1006
golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1007
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1008
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1009
+golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1010
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1011
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1012
golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
go.mod
+3
-1
@@ -15,7 +15,9 @@ require (
15
github.com/fsnotify/fsnotify v1.6.0
16
github.com/google/uuid v1.3.1
17
github.com/hashicorp/go-multierror v1.1.1
18
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
18
+ github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c
19
+ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c
20
+ github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
21
github.com/ipfs/go-block-format v0.2.0
22
github.com/ipfs/go-cid v0.4.1
23
github.com/ipfs/go-cidutil v0.1.0
go.sum
+6
-2
@@ -333,10 +333,14 @@ github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY=
333
github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8=
334
github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
335
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
336
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c h1:17FO7HnKiFhO7iadu3zCgII+EblpdRmJt5qg9FqQo8Y=
337
+github.com/ipfs-shipyard/nopfs v0.0.12-0.20231027223058-cde3b5ba964c/go.mod h1:1oj4+g/mN6JRuZiXHt5iFRG02e62wp5AKcB3gdgknbk=
338
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c h1:7UynTbtdlt+w08ggb1UGLGaGjp1mMaZhoTZSctpn5Ak=
339
+github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231027223058-cde3b5ba964c/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
340
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
341
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
338
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
339
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
342
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
343
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
344
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
345
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
346
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
plugin/loader/preload.go
+2
@@ -7,6 +7,7 @@ import (
7
pluginfxtest "github.com/ipfs/kubo/plugin/plugins/fxtest"
8
pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
9
pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
10
+ pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
11
pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
12
)
13
@@ -22,4 +23,5 @@ func init() {
23
Preload(pluginlevelds.Plugins...)
24
Preload(pluginpeerlog.Plugins...)
25
Preload(pluginfxtest.Plugins...)
26
+ Preload(pluginnopfs.Plugins...)
27
}
plugin/loader/preload_list
+1
@@ -11,3 +11,4 @@ flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
11
levelds github.com/ipfs/kubo/plugin/plugins/levelds *
12
peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
13
fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
14
+nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
\ No newline at end of file
plugin/plugins/nopfs/nopfs.go
new
+85
@@ -0,0 +1,85 @@
1
+package nopfs
2
+
3
+import (
4
+ "os"
5
+ "path/filepath"
6
+
7
+ "github.com/ipfs-shipyard/nopfs"
8
+ "github.com/ipfs-shipyard/nopfs/ipfs"
9
+ "github.com/ipfs/kubo/config"
10
+ "github.com/ipfs/kubo/core"
11
+ "github.com/ipfs/kubo/core/node"
12
+ "github.com/ipfs/kubo/plugin"
13
+ "go.uber.org/fx"
14
+)
15
+
16
+// Plugins sets the list of plugins to be loaded.
17
+var Plugins = []plugin.Plugin{
18
+ &nopfsPlugin{},
19
+}
20
+
21
+// fxtestPlugin is used for testing the fx plugin.
22
+// It merely adds an fx option that logs a debug statement, so we can verify that it works in tests.
23
+type nopfsPlugin struct{}
24
+
25
+var _ plugin.PluginFx = (*nopfsPlugin)(nil)
26
+
27
+func (p *nopfsPlugin) Name() string {
28
+ return "nopfs"
29
+}
30
+
31
+func (p *nopfsPlugin) Version() string {
32
+ return "0.0.10"
33
+}
34
+
35
+func (p *nopfsPlugin) Init(env *plugin.Environment) error {
36
+ return nil
37
+}
38
+
39
+// MakeBlocker is a factory for the blocker so that it can be provided with Fx.
40
+func MakeBlocker() (*nopfs.Blocker, error) {
41
+ ipfsPath, err := config.PathRoot()
42
+ if err != nil {
43
+ return nil, err
44
+ }
45
+
46
+ defaultFiles, err := nopfs.GetDenylistFiles()
47
+ if err != nil {
48
+ return nil, err
49
+ }
50
+
51
+ kuboFiles, err := nopfs.GetDenylistFilesInDir(filepath.Join(ipfsPath, "denylists"))
52
+ if err != nil {
53
+ return nil, err
54
+ }
55
+
56
+ files := append(defaultFiles, kuboFiles...)
57
+
58
+ return nopfs.NewBlocker(files)
59
+}
60
+
61
+// PathResolvers returns wrapped PathResolvers for Kubo.
62
+func PathResolvers(fetchers node.FetchersIn, blocker *nopfs.Blocker) node.PathResolversOut {
63
+ res := node.PathResolverConfig(fetchers)
64
+ return node.PathResolversOut{
65
+ IPLDPathResolver: ipfs.WrapResolver(res.IPLDPathResolver, blocker),
66
+ UnixFSPathResolver: ipfs.WrapResolver(res.UnixFSPathResolver, blocker),
67
+ OfflineIPLDPathResolver: ipfs.WrapResolver(res.OfflineIPLDPathResolver, blocker),
68
+ OfflineUnixFSPathResolver: ipfs.WrapResolver(res.OfflineUnixFSPathResolver, blocker),
69
+ }
70
+}
71
+
72
+func (p *nopfsPlugin) Options(info core.FXNodeInfo) ([]fx.Option, error) {
73
+ if os.Getenv("IPFS_CONTENT_BLOCKING_DISABLE") != "" {
74
+ return info.FXOptions, nil
75
+ }
76
+
77
+ opts := append(
78
+ info.FXOptions,
79
+ fx.Provide(MakeBlocker),
80
+ fx.Decorate(ipfs.WrapBlockService),
81
+ fx.Decorate(ipfs.WrapNameSystem),
82
+ fx.Decorate(PathResolvers),
83
+ )
84
+ return opts, nil
85
+}
test/cli/content_blocking_test.go
new
+303
@@ -0,0 +1,303 @@
1
+package cli
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+ "io"
7
+ "log"
8
+ "net/http"
9
+ "net/url"
10
+ "os"
11
+ "path/filepath"
12
+ "strings"
13
+ "testing"
14
+
15
+ "github.com/ipfs/kubo/test/cli/harness"
16
+ "github.com/libp2p/go-libp2p"
17
+ "github.com/libp2p/go-libp2p/core/peer"
18
+ libp2phttp "github.com/libp2p/go-libp2p/p2p/http"
19
+ "github.com/stretchr/testify/assert"
20
+ "github.com/stretchr/testify/require"
21
+)
22
+
23
+func TestContentBlocking(t *testing.T) {
24
+ // NOTE: we can't run this with t.Parallel() because we set IPFS_NS_MAP
25
+ // and running in parallel could impact other tests
26
+
27
+ const blockedMsg = "blocked and cannot be provided"
28
+ const statusExpl = "specific HTTP error code is expected"
29
+ const bodyExpl = "Error message informing about content block is expected"
30
+
31
+ h := harness.NewT(t)
32
+
33
+ // Init IPFS_PATH
34
+ node := h.NewNode().Init("--empty-repo", "--profile=test")
35
+
36
+ // Create CIDs we use in test
37
+ h.WriteFile("blocked-dir/subdir/indirectly-blocked-file.txt", "indirectly blocked file content")
38
+ parentDirCID := node.IPFS("add", "--raw-leaves", "-Q", "-r", filepath.Join(h.Dir, "blocked-dir")).Stdout.Trimmed()
39
+
40
+ h.WriteFile("directly-blocked-file.txt", "directly blocked file content")
41
+ blockedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "directly-blocked-file.txt")).Stdout.Trimmed()
42
+
43
+ h.WriteFile("not-blocked-file.txt", "not blocked file content")
44
+ allowedCID := node.IPFS("add", "--raw-leaves", "-Q", filepath.Join(h.Dir, "not-blocked-file.txt")).Stdout.Trimmed()
45
+
46
+ // Create denylist at $IPFS_PATH/denylists/test.deny
47
+ denylistTmp := h.WriteToTemp("name: test list\n---\n" +
48
+ "//QmX9dhRcQcKUw3Ws8485T5a9dtjrSCQaUAHnG4iK9i4ceM\n" + // Double hash (sha256) CID block: base58btc(sha256-multihash(QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR))
49
+ "//gW813G35CnLsy7gRYYHuf63hrz71U1xoLFDVeV7actx6oX\n" + // Double hash (blake3) Path block under blake3 root CID: base58btc(blake3-multihash(gW7Nhu4HrfDtphEivm3Z9NNE7gpdh5Tga8g6JNZc1S8E47/path))
50
+ "//8526ba05eec55e28f8db5974cc891d0d92c8af69d386fc6464f1e9f372caf549\n" + // Legacy CID double-hash block: sha256(bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq/)
51
+ "//e5b7d2ce2594e2e09901596d8e1f29fa249b74c8c9e32ea01eda5111e4d33f07\n" + // Legacy Path double-hash block: sha256(bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath)
52
+ "/ipfs/" + blockedCID + "\n" + // block specific CID
53
+ "/ipfs/" + parentDirCID + "/subdir*\n" + // block only specific subpath
54
+ "/ipns/blocked-cid.example.com\n" +
55
+ "/ipns/blocked-dnslink.example.com\n")
56
+
57
+ if err := os.MkdirAll(filepath.Join(node.Dir, "denylists"), 0o777); err != nil {
58
+ log.Panicf("failed to create denylists dir: %s", err.Error())
59
+ }
60
+ if err := os.Rename(denylistTmp, filepath.Join(node.Dir, "denylists", "test.deny")); err != nil {
61
+ log.Panicf("failed to create test denylist: %s", err.Error())
62
+ }
63
+
64
+ // Add two entries to namesys resolution cache
65
+ // /ipns/blocked-cid.example.com point at a blocked CID (to confirm blocking impacts /ipns resolution)
66
+ // /ipns/blocked-dnslink.example.com with safe CID (to test blocking of /ipns/ paths)
67
+ os.Setenv("IPFS_NS_MAP", "blocked-cid.example.com:/ipfs/"+blockedCID+",blocked-dnslink.example.com/ipns/QmUNLLsPACCz1vLxQVkXqqLX5R1X345qqfHbsf67hvA3Nn")
68
+ defer os.Unsetenv("IPFS_NS_MAP")
69
+
70
+ // Enable GatewayOverLibp2p as we want to test denylist there too
71
+ node.IPFS("config", "--json", "Experimental.GatewayOverLibp2p", "true")
72
+
73
+ // Start daemon, it should pick up denylist from $IPFS_PATH/denylists/test.deny
74
+ node.StartDaemon() // we need online mode for GatewayOverLibp2p tests
75
+ client := node.GatewayClient()
76
+
77
+ // First, confirm gateway works
78
+ t.Run("Gateway Allows CID that is not blocked", func(t *testing.T) {
79
+ t.Parallel()
80
+ resp := client.Get("/ipfs/" + allowedCID)
81
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
82
+ assert.Equal(t, "not blocked file content", resp.Body)
83
+ })
84
+
85
+ // Then, does the most basic blocking case work?
86
+ t.Run("Gateway Denies directly blocked CID", func(t *testing.T) {
87
+ t.Parallel()
88
+ resp := client.Get("/ipfs/" + blockedCID)
89
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
90
+ assert.NotEqual(t, "directly blocked file content", resp.Body)
91
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
92
+ })
93
+
94
+ // Confirm parent of blocked subpath is not blocked
95
+ t.Run("Gateway Allows parent Path that is not blocked", func(t *testing.T) {
96
+ t.Parallel()
97
+ resp := client.Get("/ipfs/" + parentDirCID)
98
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
99
+ })
100
+
101
+ // Ok, now the full list of test cases we want to cover in both CLI and Gateway
102
+ testCases := []struct {
103
+ name string
104
+ path string
105
+ }{
106
+ {
107
+ name: "directly blocked CID",
108
+ path: "/ipfs/" + blockedCID,
109
+ },
110
+ {
111
+ name: "indirectly blocked file (on a blocked subpath)",
112
+ path: "/ipfs/" + parentDirCID + "/subdir/indirectly-blocked-file.txt",
113
+ },
114
+ {
115
+ name: "/ipns path that resolves to a blocked CID",
116
+ path: "/ipns/blocked-cid.example.com",
117
+ },
118
+ {
119
+ name: "/ipns Path that is blocked by DNSLink name",
120
+ path: "/ipns/blocked-dnslink.example.com",
121
+ },
122
+ {
123
+ name: "double-hash CID block (sha256-multihash)",
124
+ path: "/ipfs/QmVTF1yEejXd9iMgoRTFDxBv7HAz9kuZcQNBzHrceuK9HR",
125
+ },
126
+ {
127
+ name: "double-hash Path block (blake3-multihash)",
128
+ path: "/ipfs/bafyb4ieqht3b2rssdmc7sjv2cy2gfdilxkfh7623nvndziyqnawkmo266a/path",
129
+ },
130
+ {
131
+ name: "legacy CID double-hash block (sha256)",
132
+ path: "/ipfs/bafkqahtcnrxwg23fmqqgi33vmjwgk2dbonuca3dfm5qwg6jamnuwicq",
133
+ },
134
+
135
+ {
136
+ name: "legacy Path double-hash block (sha256)",
137
+ path: "/ipfs/bafyaagyscufaqalqaacauaqiaejao43vmjygc5didacauaqiae/subpath",
138
+ },
139
+ }
140
+
141
+ // Which specific cliCmds we test against testCases
142
+ cliCmds := [][]string{
143
+ {"block", "get"},
144
+ {"block", "stat"},
145
+ {"dag", "get"},
146
+ {"dag", "export"},
147
+ {"dag", "stat"},
148
+ {"cat"},
149
+ {"ls"},
150
+ {"get"},
151
+ {"refs"},
152
+ }
153
+
154
+ expectedMsg := blockedMsg
155
+ for _, testCase := range testCases {
156
+
157
+ // Confirm that denylist is active for every command in 'cliCmds' x 'testCases'
158
+ for _, cmd := range cliCmds {
159
+ cmd := cmd
160
+ cliTestName := fmt.Sprintf("CLI '%s' denies %s", strings.Join(cmd, " "), testCase.name)
161
+ t.Run(cliTestName, func(t *testing.T) {
162
+ t.Parallel()
163
+ args := append(cmd, testCase.path)
164
+ errMsg := node.RunIPFS(args...).Stderr.Trimmed()
165
+ if !strings.Contains(errMsg, expectedMsg) {
166
+ t.Errorf("Expected STDERR error message %q, but got: %q", expectedMsg, errMsg)
167
+ }
168
+ })
169
+ }
170
+
171
+ // Confirm that denylist is active for every content path in 'testCases'
172
+ gwTestName := fmt.Sprintf("Gateway denies %s", testCase.name)
173
+ t.Run(gwTestName, func(t *testing.T) {
174
+ resp := client.Get(testCase.path)
175
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
176
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
177
+ })
178
+
179
+ }
180
+
181
+ // Extra edge cases on subdomain gateway
182
+
183
+ t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain redirect)", func(t *testing.T) {
184
+ t.Parallel()
185
+
186
+ gwURL, _ := url.Parse(node.GatewayURL())
187
+ resp := client.Get("/ipns/blocked-dnslink.example.com", func(r *http.Request) {
188
+ r.Host = "localhost:" + gwURL.Port()
189
+ })
190
+
191
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
192
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
193
+ })
194
+
195
+ t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, no TLS)", func(t *testing.T) {
196
+ t.Parallel()
197
+
198
+ gwURL, _ := url.Parse(node.GatewayURL())
199
+ resp := client.Get("/", func(r *http.Request) {
200
+ r.Host = "blocked-dnslink.example.com.ipns.localhost:" + gwURL.Port()
201
+ })
202
+
203
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
204
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
205
+ })
206
+
207
+ t.Run("Gateway Denies /ipns Path that is blocked by DNSLink name (subdomain, inlined for TLS)", func(t *testing.T) {
208
+ t.Parallel()
209
+
210
+ gwURL, _ := url.Parse(node.GatewayURL())
211
+ resp := client.Get("/", func(r *http.Request) {
212
+ // Inlined DNSLink to fit in single DNS label for TLS interop:
213
+ // https://specs.ipfs.tech/http-gateways/subdomain-gateway/#host-request-header
214
+ r.Host = "blocked--dnslink-example-com.ipns.localhost:" + gwURL.Port()
215
+ })
216
+
217
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
218
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
219
+ })
220
+
221
+ // We need to confirm denylist is active when gateway is run in NoFetch
222
+ // mode (which usually swaps blockservice to a read-only one, and that swap
223
+ // may cause denylists to not be applied, as it is a separate code path)
224
+ t.Run("GatewayNoFetch", func(t *testing.T) {
225
+ // NOTE: we don't run this in parallel, as it requires restart with different config
226
+
227
+ // Switch gateway to NoFetch mode
228
+ node.StopDaemon()
229
+ node.IPFS("config", "--json", "Gateway.NoFetch", "true")
230
+ node.StartDaemon()
231
+
232
+ // update client, as the port of test node might've changed after restart
233
+ client = node.GatewayClient()
234
+
235
+ // First, confirm gateway works
236
+ t.Run("Allows CID that is not blocked", func(t *testing.T) {
237
+ resp := client.Get("/ipfs/" + allowedCID)
238
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
239
+ assert.Equal(t, "not blocked file content", resp.Body)
240
+ })
241
+
242
+ // Then, does the most basic blocking case work?
243
+ t.Run("Denies directly blocked CID", func(t *testing.T) {
244
+ resp := client.Get("/ipfs/" + blockedCID)
245
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
246
+ assert.NotEqual(t, "directly blocked file content", resp.Body)
247
+ assert.Contains(t, resp.Body, blockedMsg, bodyExpl)
248
+ })
249
+
250
+ // Restore default
251
+ node.StopDaemon()
252
+ node.IPFS("config", "--json", "Gateway.NoFetch", "false")
253
+ node.StartDaemon()
254
+ client = node.GatewayClient()
255
+ })
256
+
257
+ // We need to confirm denylist is active on the
258
+ // trustless gateway exposed over libp2p
259
+ // when Experimental.GatewayOverLibp2p=true
260
+ // (https://github.com/ipfs/kubo/blob/master/docs/experimental-features.md#http-gateway-over-libp2p)
261
+ // NOTE: this type fo gateway is hardcoded to be NoFetch: it does not fetch
262
+ // data that is not in local store, so we only need to run it once: a
263
+ // simple smoke-test for allowed CID and blockedCID.
264
+ t.Run("GatewayOverLibp2p", func(t *testing.T) {
265
+ t.Parallel()
266
+
267
+ // Create libp2p client that connects to our node over
268
+ // /http1.1 and then talks gateway semantics over the /ipfs/gateway sub-protocol
269
+ clientHost, err := libp2p.New(libp2p.NoListenAddrs)
270
+ require.NoError(t, err)
271
+ err = clientHost.Connect(context.Background(), peer.AddrInfo{
272
+ ID: node.PeerID(),
273
+ Addrs: node.SwarmAddrs(),
274
+ })
275
+ require.NoError(t, err)
276
+
277
+ libp2pClient, err := (&libp2phttp.Host{StreamHost: clientHost}).NamespacedClient("/ipfs/gateway", peer.AddrInfo{ID: node.PeerID()})
278
+ require.NoError(t, err)
279
+
280
+ t.Run("Serves Allowed CID", func(t *testing.T) {
281
+ t.Parallel()
282
+ resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", allowedCID))
283
+ require.NoError(t, err)
284
+ defer resp.Body.Close()
285
+ assert.Equal(t, http.StatusOK, resp.StatusCode)
286
+ body, err := io.ReadAll(resp.Body)
287
+ require.NoError(t, err)
288
+ require.Equal(t, string(body), "not blocked file content", bodyExpl)
289
+ })
290
+
291
+ t.Run("Denies Blocked CID", func(t *testing.T) {
292
+ t.Parallel()
293
+ resp, err := libp2pClient.Get(fmt.Sprintf("/ipfs/%s?format=raw", blockedCID))
294
+ require.NoError(t, err)
295
+ defer resp.Body.Close()
296
+ assert.Equal(t, http.StatusGone, resp.StatusCode, statusExpl)
297
+ body, err := io.ReadAll(resp.Body)
298
+ require.NoError(t, err)
299
+ assert.NotEqual(t, string(body), "directly blocked file content")
300
+ assert.Contains(t, string(body), blockedMsg, bodyExpl)
301
+ })
302
+ })
303
+}
test/dependencies/go.mod
+1
-1
@@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../
7
require (
8
github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
9
github.com/golangci/golangci-lint v1.54.1
10
- github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd
10
+ github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b
11
github.com/ipfs/go-cid v0.4.1
12
github.com/ipfs/go-cidutil v0.1.0
13
github.com/ipfs/go-datastore v0.6.0
test/dependencies/go.sum
+2
-2
@@ -398,8 +398,8 @@ github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2
398
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
399
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
400
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
401
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd h1:CWz2mhz+cmkLRlKgQYlKXkDtx4oWYkCorSSF4ZWkH3o=
402
-github.com/ipfs/boxo v0.13.2-0.20231018081237-a50f784985dd/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk=
401
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b h1:0Qi1EhB82x3SZJOBieG51BtPvjU3kSy4H8OpMnxKvtk=
402
+github.com/ipfs/boxo v0.13.2-0.20231028021353-182e86f5bb9b/go.mod h1:pu8HsZvuyYeYJsqtLDCoYSvy8rHj6vI3dlh8P0f83Zs=
403
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
404
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
405
github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs=
test/sharness/t0054-dag-car-import-export.sh
+1
-3
@@ -178,13 +178,11 @@ test_expect_success "basic offline export of 'getting started' dag works" '
178
ipfs dag export "$HASH_WELCOME_DOCS" >/dev/null
179
'
180
181
-
182
-echo "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX (currently offline, perhaps retry after attaching to the network)" > offline_fetch_error_expected
181
test_expect_success "basic offline export of nonexistent cid" '
182
! ipfs dag export QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 2> offline_fetch_error_actual >/dev/null
183
'
184
test_expect_success "correct error" '
187
- test_cmp_sorted offline_fetch_error_expected offline_fetch_error_actual
185
+ test_should_contain "Error: block was not found locally (offline): ipld: could not find QmYwAPJXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX" offline_fetch_error_actual
186
'
187
188
cat >multiroot_import_json_stats_expected <<EOE