@cryptotaxi247 / kubo / commits / 353dd49be

refactor: switch gateway code to new API from go-libipfs (#9681)

Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Henrique Dias <hacdias@gmail.com>

Adin Schmahmann committed Mar 30, 2023 at 09:20 UTC 353dd49be239be651650c3ef3dfef83deebac58c
12 files changed +296 -132
core/corehttp/gateway.go
+94 -55
@@ -2,24 +2,26 @@ package corehttp
2
3 import (
4 "context"
5 + "errors"
6 "fmt"
7 "io"
8 "net"
9 "net/http"
10
11 + "github.com/ipfs/boxo/blockservice"
12 iface "github.com/ipfs/boxo/coreiface"
11 - options "github.com/ipfs/boxo/coreiface/options"
12 - nsopts "github.com/ipfs/boxo/coreiface/options/namesys"
13 "github.com/ipfs/boxo/coreiface/path"
14 + "github.com/ipfs/boxo/exchange/offline"
15 "github.com/ipfs/boxo/files"
16 "github.com/ipfs/boxo/gateway"
17 "github.com/ipfs/boxo/namesys"
17 - "github.com/ipfs/go-block-format"
18 + offlineroute "github.com/ipfs/boxo/routing/offline"
19 cid "github.com/ipfs/go-cid"
20 version "github.com/ipfs/kubo"
21 config "github.com/ipfs/kubo/config"
22 core "github.com/ipfs/kubo/core"
22 - coreapi "github.com/ipfs/kubo/core/coreapi"
23 + "github.com/ipfs/kubo/core/node"
24 + "github.com/libp2p/go-libp2p/core/routing"
25 id "github.com/libp2p/go-libp2p/p2p/protocol/identify"
26 "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
27 )
@@ -42,7 +44,7 @@ func GatewayOption(paths ...string) ServeOption {
44 Headers: headers,
45 }
46
45 - gwAPI, err := newGatewayAPI(n)
47 + gwAPI, err := newGatewayBackend(n)
48 if err != nil {
49 return nil, err
50 }
@@ -70,7 +72,7 @@ func HostnameOption() ServeOption {
72 return nil, err
73 }
74
73 - gwAPI, err := newGatewayAPI(n)
75 + gwAPI, err := newGatewayBackend(n)
76 if err != nil {
77 return nil, err
78 }
@@ -93,83 +95,120 @@ func VersionOption() ServeOption {
95 }
96 }
97
96 -type gatewayAPI struct {
97 - ns namesys.NameSystem
98 - api iface.CoreAPI
99 - offlineAPI iface.CoreAPI
100 -}
101 -
102 -func newGatewayAPI(n *core.IpfsNode) (*gatewayAPI, error) {
98 +func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
99 cfg, err := n.Repo.Config()
100 if err != nil {
101 return nil, err
102 }
103
108 - api, err := coreapi.NewCoreAPI(n, options.Api.FetchBlocks(!cfg.Gateway.NoFetch))
109 - if err != nil {
110 - return nil, err
104 + bserv := n.Blocks
105 + var vsRouting routing.ValueStore = n.Routing
106 + nsys := n.Namesys
107 + if cfg.Gateway.NoFetch {
108 + bserv = blockservice.New(bserv.Blockstore(), offline.Exchange(bserv.Blockstore()))
109 +
110 + cs := cfg.Ipns.ResolveCacheSize
111 + if cs == 0 {
112 + cs = node.DefaultIpnsCacheSize
113 + }
114 + if cs < 0 {
115 + return nil, fmt.Errorf("cannot specify negative resolve cache size")
116 + }
117 +
118 + vsRouting = offlineroute.NewOfflineRouter(n.Repo.Datastore(), n.RecordValidator)
119 + nsys, err = namesys.NewNameSystem(vsRouting,
120 + namesys.WithDatastore(n.Repo.Datastore()),
121 + namesys.WithDNSResolver(n.DNSResolver),
122 + namesys.WithCache(cs))
123 + if err != nil {
124 + return nil, fmt.Errorf("error constructing namesys: %w", err)
125 + }
126 }
112 - offlineAPI, err := api.WithOptions(options.Api.Offline(true))
127 +
128 + gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
129 if err != nil {
130 return nil, err
131 }
132 + return &offlineGatewayErrWrapper{gwimpl: gw}, nil
133 +}
134
117 - return &gatewayAPI{
118 - ns: n.Namesys,
119 - api: api,
120 - offlineAPI: offlineAPI,
121 - }, nil
135 +type offlineGatewayErrWrapper struct {
136 + gwimpl gateway.IPFSBackend
137 }
138
124 -func (gw *gatewayAPI) GetUnixFsNode(ctx context.Context, pth path.Resolved) (files.Node, error) {
125 - return gw.api.Unixfs().Get(ctx, pth)
139 +func offlineErrWrap(err error) error {
140 + if errors.Is(err, iface.ErrOffline) {
141 + return fmt.Errorf("%s : %w", err.Error(), gateway.ErrServiceUnavailable)
142 + }
143 + return err
144 }
145
128 -func (gw *gatewayAPI) LsUnixFsDir(ctx context.Context, pth path.Resolved) (<-chan iface.DirEntry, error) {
129 - // Optimization: use Unixfs.Ls without resolving children, but using the
130 - // cumulative DAG size as the file size. This allows for a fast listing
131 - // while keeping a good enough Size field.
132 - return gw.api.Unixfs().Ls(ctx, pth,
133 - options.Unixfs.ResolveChildren(false),
134 - options.Unixfs.UseCumulativeSize(true),
135 - )
146 +func (o *offlineGatewayErrWrapper) Get(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, *gateway.GetResponse, error) {
147 + md, n, err := o.gwimpl.Get(ctx, path)
148 + err = offlineErrWrap(err)
149 + return md, n, err
150 }
151
138 -func (gw *gatewayAPI) GetBlock(ctx context.Context, cid cid.Cid) (blocks.Block, error) {
139 - r, err := gw.api.Block().Get(ctx, path.IpfsPath(cid))
140 - if err != nil {
141 - return nil, err
142 - }
152 +func (o *offlineGatewayErrWrapper) GetRange(ctx context.Context, path gateway.ImmutablePath, ranges ...gateway.GetRange) (gateway.ContentPathMetadata, files.File, error) {
153 + md, n, err := o.gwimpl.GetRange(ctx, path, ranges...)
154 + err = offlineErrWrap(err)
155 + return md, n, err
156 +}
157
144 - data, err := io.ReadAll(r)
145 - if err != nil {
146 - return nil, err
147 - }
158 +func (o *offlineGatewayErrWrapper) GetAll(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
159 + md, n, err := o.gwimpl.GetAll(ctx, path)
160 + err = offlineErrWrap(err)
161 + return md, n, err
162 +}
163
149 - return blocks.NewBlockWithCid(data, cid)
164 +func (o *offlineGatewayErrWrapper) GetBlock(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.File, error) {
165 + md, n, err := o.gwimpl.GetBlock(ctx, path)
166 + err = offlineErrWrap(err)
167 + return md, n, err
168 }
169
152 -func (gw *gatewayAPI) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
153 - return gw.api.Routing().Get(ctx, "/ipns/"+c.String())
170 +func (o *offlineGatewayErrWrapper) Head(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, files.Node, error) {
171 + md, n, err := o.gwimpl.Head(ctx, path)
172 + err = offlineErrWrap(err)
173 + return md, n, err
174 }
175
156 -func (gw *gatewayAPI) GetDNSLinkRecord(ctx context.Context, hostname string) (path.Path, error) {
157 - p, err := gw.ns.Resolve(ctx, "/ipns/"+hostname, nsopts.Depth(1))
158 - if err == namesys.ErrResolveRecursion {
159 - err = nil
160 - }
161 - return path.New(p.String()), err
176 +func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, error) {
177 + md, err := o.gwimpl.ResolvePath(ctx, path)
178 + err = offlineErrWrap(err)
179 + return md, err
180 }
181
164 -func (gw *gatewayAPI) IsCached(ctx context.Context, pth path.Path) bool {
165 - _, err := gw.offlineAPI.Block().Stat(ctx, pth)
166 - return err == nil
182 +func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
183 + md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
184 + err = offlineErrWrap(err)
185 + return md, data, errCh, err
186 }
187
169 -func (gw *gatewayAPI) ResolvePath(ctx context.Context, pth path.Path) (path.Resolved, error) {
170 - return gw.api.ResolvePath(ctx, pth)
188 +func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
189 + return o.gwimpl.IsCached(ctx, path)
190 }
191
192 +func (o *offlineGatewayErrWrapper) GetIPNSRecord(ctx context.Context, c cid.Cid) ([]byte, error) {
193 + rec, err := o.gwimpl.GetIPNSRecord(ctx, c)
194 + err = offlineErrWrap(err)
195 + return rec, err
196 +}
197 +
198 +func (o *offlineGatewayErrWrapper) ResolveMutable(ctx context.Context, path path.Path) (gateway.ImmutablePath, error) {
199 + imPath, err := o.gwimpl.ResolveMutable(ctx, path)
200 + err = offlineErrWrap(err)
201 + return imPath, err
202 +}
203 +
204 +func (o *offlineGatewayErrWrapper) GetDNSLinkRecord(ctx context.Context, s string) (path.Path, error) {
205 + p, err := o.gwimpl.GetDNSLinkRecord(ctx, s)
206 + err = offlineErrWrap(err)
207 + return p, err
208 +}
209 +
210 +var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
211 +
212 var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
213
214 var subdomainGatewaySpec = &gateway.Specification{
core/node/dns.go
+2 -70
@@ -1,88 +1,20 @@
1 package node
2
3 import (
4 - "fmt"
4 "math"
6 - "strings"
5 "time"
6
7 + "github.com/ipfs/boxo/gateway"
8 config "github.com/ipfs/kubo/config"
9 doh "github.com/libp2p/go-doh-resolver"
10 madns "github.com/multiformats/go-multiaddr-dns"
12 -
13 - "github.com/miekg/dns"
11 )
12
16 -var defaultResolvers = map[string]string{
17 - "eth.": "https://resolver.cloudflare-eth.com/dns-query",
18 - "crypto.": "https://resolver.cloudflare-eth.com/dns-query",
19 -}
20 -
21 -func newResolver(url string, opts ...doh.Option) (madns.BasicResolver, error) {
22 - if !strings.HasPrefix(url, "https://") {
23 - return nil, fmt.Errorf("invalid resolver url: %s", url)
24 - }
25 -
26 - return doh.NewResolver(url, opts...)
27 -}
28 -
13 func DNSResolver(cfg *config.Config) (*madns.Resolver, error) {
30 - var opts []madns.Option
31 - var err error
32 -
14 var dohOpts []doh.Option
15 if !cfg.DNS.MaxCacheTTL.IsDefault() {
16 dohOpts = append(dohOpts, doh.WithMaxCacheTTL(cfg.DNS.MaxCacheTTL.WithDefault(time.Duration(math.MaxUint32)*time.Second)))
17 }
18
38 - domains := make(map[string]struct{}) // to track overridden default resolvers
39 - rslvrs := make(map[string]madns.BasicResolver) // to reuse resolvers for the same URL
40 -
41 - for domain, url := range cfg.DNS.Resolvers {
42 - if domain != "." && !dns.IsFqdn(domain) {
43 - return nil, fmt.Errorf("invalid domain %s; must be FQDN", domain)
44 - }
45 -
46 - domains[domain] = struct{}{}
47 - if url == "" {
48 - // allow overriding of implicit defaults with the default resolver
49 - continue
50 - }
51 -
52 - rslv, ok := rslvrs[url]
53 - if !ok {
54 - rslv, err = newResolver(url, dohOpts...)
55 - if err != nil {
56 - return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
57 - }
58 - rslvrs[url] = rslv
59 - }
60 -
61 - if domain != "." {
62 - opts = append(opts, madns.WithDomainResolver(domain, rslv))
63 - } else {
64 - opts = append(opts, madns.WithDefaultResolver(rslv))
65 - }
66 - }
67 -
68 - // fill in defaults if not overridden by the user
69 - for domain, url := range defaultResolvers {
70 - _, ok := domains[domain]
71 - if ok {
72 - continue
73 - }
74 -
75 - rslv, ok := rslvrs[url]
76 - if !ok {
77 - rslv, err = newResolver(url)
78 - if err != nil {
79 - return nil, fmt.Errorf("bad resolver for %s: %w", domain, err)
80 - }
81 - rslvrs[url] = rslv
82 - }
83 -
84 - opts = append(opts, madns.WithDomainResolver(domain, rslv))
85 - }
86 -
87 - return madns.NewResolver(opts...)
19 + return gateway.NewDNSResolver(cfg.DNS.Resolvers, dohOpts...)
20 }
docs/examples/kubo-as-a-library/go.mod
+4 -1
@@ -7,7 +7,7 @@ go 1.18
7 replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 - github.com/ipfs/boxo v0.8.0-rc2
10 + github.com/ipfs/boxo v0.8.0-rc3
11 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12 github.com/libp2p/go-libp2p v0.26.4
13 github.com/multiformats/go-multiaddr v0.8.0
@@ -41,6 +41,7 @@ require (
41 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
42 github.com/flynn/noise v1.0.0 // indirect
43 github.com/francoispqt/gojay v1.2.13 // indirect
44 + github.com/gabriel-vasile/mimetype v1.4.1 // indirect
45 github.com/go-logr/logr v1.2.3 // indirect
46 github.com/go-logr/stdr v1.2.2 // indirect
47 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
@@ -77,6 +78,7 @@ require (
78 github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
79 github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
80 github.com/ipfs/go-ipfs-pq v0.0.3 // indirect
81 + github.com/ipfs/go-ipfs-redirects-file v0.1.1 // indirect
82 github.com/ipfs/go-ipfs-util v0.0.2 // indirect
83 github.com/ipfs/go-ipld-cbor v0.0.6 // indirect
84 github.com/ipfs/go-ipld-format v0.4.0 // indirect
@@ -153,6 +155,7 @@ require (
155 github.com/samber/lo v1.36.0 // indirect
156 github.com/spaolacci/murmur3 v1.1.0 // indirect
157 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
158 + github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
159 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
160 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
161 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
docs/examples/kubo-as-a-library/go.sum
+11 -2
@@ -170,6 +170,8 @@ github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0X
170 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
171 github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ=
172 github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY=
173 +github.com/gabriel-vasile/mimetype v1.4.1 h1:TRWk7se+TOjCYgRth7+1/OYLNiRNIotknkFtf/dnN7Q=
174 +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M=
175 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
176 github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE=
177 github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI=
@@ -335,8 +337,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
337 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
338 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
339 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
338 -github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY=
339 -github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc=
340 +github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc=
341 +github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA=
342 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
343 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
344 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -393,6 +395,8 @@ github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsb
395 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=
396 github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE=
397 github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4=
398 +github.com/ipfs/go-ipfs-redirects-file v0.1.1 h1:Io++k0Vf/wK+tfnhEh63Yte1oQK5VGT2hIEYpD0Rzx8=
399 +github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk=
400 github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc=
401 github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8=
402 github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ=
@@ -800,8 +804,11 @@ github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70
804 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc=
805 github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA=
806 github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M=
807 +github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk=
808 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ=
809 github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM=
810 +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb h1:Ywfo8sUltxogBpFuMOFRrrSifO788kAFxmvVw31PtQQ=
811 +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM=
812 github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo=
813 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
814 github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0=
@@ -997,6 +1004,7 @@ golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qx
1004 golang.org/x/net v0.0.0-20210726213435-c6fcb2dbf985/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
1005 golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
1006 golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk=
1007 +golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
1008 golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g=
1009 golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
1010 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -1088,6 +1096,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc
1096 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1097 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1098 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1099 +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1100 golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1101 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1102 golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ=
go.mod
+2 -2
@@ -16,7 +16,7 @@ require (
16 github.com/gogo/protobuf v1.3.2
17 github.com/google/uuid v1.3.0
18 github.com/hashicorp/go-multierror v1.1.1
19 - github.com/ipfs/boxo v0.8.0-rc2
19 + github.com/ipfs/boxo v0.8.0-rc3
20 github.com/ipfs/go-block-format v0.1.2
21 github.com/ipfs/go-cid v0.4.0
22 github.com/ipfs/go-cidutil v0.1.0
@@ -55,7 +55,6 @@ require (
55 github.com/libp2p/go-libp2p-routing-helpers v0.6.1
56 github.com/libp2p/go-libp2p-testing v0.12.0
57 github.com/libp2p/go-socket-activation v0.1.0
58 - github.com/miekg/dns v1.1.50
58 github.com/mitchellh/go-homedir v1.1.0
59 github.com/multiformats/go-multiaddr v0.8.0
60 github.com/multiformats/go-multiaddr-dns v0.3.1
@@ -173,6 +172,7 @@ require (
172 github.com/mattn/go-runewidth v0.0.4 // indirect
173 github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
174 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
175 + github.com/miekg/dns v1.1.50 // indirect
176 github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect
177 github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect
178 github.com/minio/sha256-simd v1.0.0 // indirect
go.sum
+2 -2
@@ -355,8 +355,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
355 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
356 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
357 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
358 -github.com/ipfs/boxo v0.8.0-rc2 h1:JnSlLKlIURiVsTfPs1BLv3NNCygjV1b07wGva3w4sLY=
359 -github.com/ipfs/boxo v0.8.0-rc2/go.mod h1:EgDiNox/+W/+ySwEotRrHlvdmrhbSAB4p22ELg+ZsCc=
358 +github.com/ipfs/boxo v0.8.0-rc3 h1:rttpGdhLE0zeTec8f2/e5YDgCYzEQf7dI4eRglu2ktc=
359 +github.com/ipfs/boxo v0.8.0-rc3/go.mod h1:RIsi4CnTyQ7AUsNn5gXljJYZlQrHBMnJp94p73liFiA=
360 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
361 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
362 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
test/cli/fixtures/README.md new
+72
@@ -0,0 +1,72 @@
1 +# Dataset Description / Sources
2 +
3 +TestGatewayHAMTDirectory.car generated with:
4 +
5 +```bash
6 +ipfs version
7 +# ipfs version 0.19.0
8 +
9 +export HAMT_DIR=bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm
10 +export IPFS_PATH=$(mktemp -d)
11 +
12 +# Init and start daemon, ensure we have an empty repository.
13 +ipfs init --empty-repo
14 +ipfs daemon &> /dev/null &
15 +export IPFS_PID=$!
16 +
17 +# Retrieve the directory listing, forcing the daemon to download all required DAGs. Kill daemon.
18 +curl -o dir.html http://127.0.0.1:8080/ipfs/$HAMT_DIR/
19 +kill $IPFS_PID
20 +
21 +# Get the list with all the downloaded refs and sanity check.
22 +ipfs refs local > required_refs
23 +cat required_refs | wc -l
24 +# 962
25 +
26 +# Get the list of all the files CIDs inside the directory and sanity check.
27 +cat dir.html| pup '#content tbody .ipfs-hash attr{href}' | sed 's/\/ipfs\///g;s/\?filename=.*//g' > files_refs
28 +cat files_refs | wc -l
29 +# 10100
30 +
31 +# Make and export our fixture.
32 +ipfs files mkdir --cid-version 1 /fixtures
33 +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
34 +cat files_refs | ipfs files write --create /fixtures/files_refs
35 +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
36 +echo $FIXTURE_CID
37 +# bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i
38 +ipfs dag export $FIXTURE_CID > TestGatewayHAMTDirectory.car
39 +```
40 +
41 +TestGatewayMultiRange.car generated with:
42 +
43 +
44 +```sh
45 +ipfs version
46 +# ipfs version 0.19.0
47 +
48 +export FILE_CID=bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau
49 +export IPFS_PATH=$(mktemp -d)
50 +
51 +# Init and start daemon, ensure we have an empty repository.
52 +ipfs init --empty-repo
53 +ipfs daemon &> /dev/null &
54 +export IPFS_PID=$!
55 +
56 +# Get a specific byte range from the file.
57 +curl http://127.0.0.1:8080/ipfs/$FILE_CID -i -H "Range: bytes=1276-1279, 29839070-29839080"
58 +kill $IPFS_PID
59 +
60 +# Get the list with all the downloaded refs and sanity check.
61 +ipfs refs local > required_refs
62 +cat required_refs | wc -l
63 +# 19
64 +
65 +# Make and export our fixture.
66 +ipfs files mkdir --cid-version 1 /fixtures
67 +cat required_refs | xargs -I {} ipfs files cp /ipfs/{} /fixtures/{}
68 +export FIXTURE_CID=$(ipfs files stat --hash /fixtures/)
69 +echo $FIXTURE_CID
70 +# bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y
71 +ipfs dag export $FIXTURE_CID > TestGatewayMultiRange.car
72 +```
test/cli/fixtures/TestGatewayHAMTDirectory.car
Binary files /dev/null and b/test/cli/fixtures/TestGatewayHAMTDirectory.car differ
test/cli/fixtures/TestGatewayMultiRange.car
Binary files /dev/null and b/test/cli/fixtures/TestGatewayMultiRange.car differ
test/cli/gateway_range_test.go new
+75
@@ -0,0 +1,75 @@
1 +package cli
2 +
3 +import (
4 + "fmt"
5 + "net/http"
6 + "os"
7 + "testing"
8 +
9 + "github.com/ipfs/kubo/test/cli/harness"
10 + "github.com/stretchr/testify/assert"
11 +)
12 +
13 +func TestGatewayHAMTDirectory(t *testing.T) {
14 + t.Parallel()
15 +
16 + const (
17 + // The CID of the HAMT-sharded directory that has 10k items
18 + hamtCid = "bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm"
19 +
20 + // fixtureCid is the CID of root of the DAG that is a subset of hamtCid DAG
21 + // representing the minimal set of blocks necessary for directory listing.
22 + // It also includes a "files_refs" file with the list of the references
23 + // we do NOT needs to fetch (files inside the directory)
24 + fixtureCid = "bafybeig3yoibxe56aolixqa4zk55gp5sug3qgaztkakpndzk2b2ynobd4i"
25 + )
26 +
27 + // Start node
28 + h := harness.NewT(t)
29 + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
30 + client := node.GatewayClient()
31 +
32 + // Import fixtures
33 + r, err := os.Open("./fixtures/TestGatewayHAMTDirectory.car")
34 + assert.Nil(t, err)
35 + defer r.Close()
36 + err = node.IPFSDagImport(r, fixtureCid)
37 + assert.Nil(t, err)
38 +
39 + // Fetch HAMT directory succeeds with minimal refs
40 + resp := client.Get(fmt.Sprintf("/ipfs/%s/", hamtCid))
41 + assert.Equal(t, http.StatusOK, resp.StatusCode)
42 +}
43 +
44 +func TestGatewayMultiRange(t *testing.T) {
45 + t.Parallel()
46 +
47 + const (
48 + // fileCid is the CID of the large HAMT-sharded file.
49 + fileCid = "bafybeiae5abzv6j3ucqbzlpnx3pcqbr2otbnpot7d2k5pckmpymin4guau"
50 +
51 + // fixtureCid is the CID of root of the DAG that is a subset of fileCid DAG
52 + // representing the minimal set of blocks necessary for a simple byte range request.
53 + fixtureCid = "bafybeicgsg3lwyn3yl75lw7sn4zhyj5dxtb7wfxwscpq6yzippetmr2w3y"
54 + )
55 +
56 + // Start node
57 + h := harness.NewT(t)
58 + node := h.NewNode().Init("--empty-repo", "--profile=test").StartDaemon("--offline")
59 + client := node.GatewayClient()
60 +
61 + // Import fixtures
62 + r, err := os.Open("./fixtures/TestGatewayMultiRange.car")
63 + assert.Nil(t, err)
64 + defer r.Close()
65 + err = node.IPFSDagImport(r, fixtureCid)
66 + assert.Nil(t, err)
67 +
68 + // Succeeds fetching a range of blocks we have
69 + resp := client.Get(fmt.Sprintf("/ipfs/%s", fileCid), func(r *http.Request) {
70 + r.Header.Set("Range", "bytes=1276-1279, 29839070-29839080")
71 + })
72 + assert.Equal(t, http.StatusPartialContent, resp.StatusCode)
73 + assert.Contains(t, resp.Body, "Content-Range: bytes 1276-1279/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\niana\r\n")
74 + assert.Contains(t, resp.Body, "Content-Range: bytes 29839070-29839080/109266405\r\nContent-Type: text/plain; charset=utf-8\r\n\r\nEXAMPLE.COM\r\n")
75 +}
test/cli/harness/ipfs.go
+19
@@ -78,3 +78,22 @@ func (n *Node) IPFSAdd(content io.Reader, args ...string) string {
78 log.Debugf("add result: %q", out)
79 return out
80 }
81 +
82 +func (n *Node) IPFSDagImport(content io.Reader, cid string, args ...string) error {
83 + log.Debugf("node %d dag import with args: %v", n.ID, args)
84 + fullArgs := []string{"dag", "import", "--pin-roots=false"}
85 + fullArgs = append(fullArgs, args...)
86 + res := n.Runner.MustRun(RunRequest{
87 + Path: n.IPFSBin,
88 + Args: fullArgs,
89 + CmdOpts: []CmdOpt{RunWithStdin(content)},
90 + })
91 + if res.Err != nil {
92 + return res.Err
93 + }
94 + res = n.Runner.MustRun(RunRequest{
95 + Path: n.IPFSBin,
96 + Args: []string{"block", "stat", "--offline", cid},
97 + })
98 + return res.Err
99 +}
test/sharness/t0117-gateway-block.sh
+15
@@ -29,6 +29,21 @@ FILE_CID=bafkreihhpc5y2pqvl5rbe5uuyhqjouybfs3rvlmisccgzue2kkt5zq6upq # ./dir/asc
29 test_cmp expected_block curl_ipfs_dir_block_accept_output
30 '
31
32 + test_expect_success "GET for application/vnd.ipld.raw with single range request includes correct bytes" '
33 + echo -n "application" > expected_file_block_single_range &&
34 + curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_single_range &&
35 + test_cmp expected_file_block_single_range curl_ipfs_file_block_single_range
36 + '
37 +
38 + test_expect_success "GET for application/vnd.ipld.raw with multiple range request includes correct bytes" '
39 + curl -sX GET -H "Accept: application/vnd.ipld.raw" -H "Range: bytes=6-16,0-4" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" -o curl_ipfs_file_block_multiple_range &&
40 + test_should_contain "Content-Range: bytes 6-16/31" curl_ipfs_file_block_multiple_range &&
41 + test_should_contain "Content-Type: application/vnd.ipld.raw" curl_ipfs_file_block_multiple_range &&
42 + test_should_contain "application" curl_ipfs_file_block_multiple_range &&
43 + test_should_contain "Content-Range: bytes 0-4/31" curl_ipfs_file_block_multiple_range &&
44 + test_should_contain "hello" curl_ipfs_file_block_multiple_range
45 + '
46 +
47 # Make sure expected HTTP headers are returned with the block bytes
48
49 test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" '