@cryptotaxi247 / kubo / commits / 2716cd987

feat(gateway): support for IPIP-402 CAR params (#9914)

Henrique Dias committed Jun 8, 2023 at 17:32 UTC 2716cd987fcf9ee3c6c1ffd0d5cc68283c32e557
14 files changed +48 -363
core/corehttp/gateway.go
+17 -20
@@ -28,24 +28,21 @@ import (
28
29 func GatewayOption(paths ...string) ServeOption {
30 return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
31 - gwConfig, err := getGatewayConfig(n)
31 + config, err := getGatewayConfig(n)
32 if err != nil {
33 return nil, err
34 }
35
36 - gwAPI, err := newGatewayBackend(n)
36 + backend, err := newGatewayBackend(n)
37 if err != nil {
38 return nil, err
39 }
40
41 - gw := gateway.NewHandler(gwConfig, gwAPI)
42 - gw = otelhttp.NewHandler(gw, "Gateway")
43 -
44 - // By default, our HTTP handler is the gateway handler.
45 - handler := gw.ServeHTTP
41 + handler := gateway.NewHandler(config, backend)
42 + handler = otelhttp.NewHandler(handler, "Gateway")
43
44 for _, p := range paths {
48 - mux.HandleFunc(p+"/", handler)
45 + mux.HandleFunc(p+"/", handler.ServeHTTP)
46 }
47
48 return mux, nil
@@ -54,18 +51,18 @@ func GatewayOption(paths ...string) ServeOption {
51
52 func HostnameOption() ServeOption {
53 return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
57 - gwConfig, err := getGatewayConfig(n)
54 + config, err := getGatewayConfig(n)
55 if err != nil {
56 return nil, err
57 }
58
62 - gwAPI, err := newGatewayBackend(n)
59 + backend, err := newGatewayBackend(n)
60 if err != nil {
61 return nil, err
62 }
63
64 childMux := http.NewServeMux()
68 - mux.HandleFunc("/", gateway.WithHostname(gwConfig, gwAPI, childMux).ServeHTTP)
65 + mux.HandleFunc("/", gateway.NewHostnameHandler(config, backend, childMux).ServeHTTP)
66 return childMux, nil
67 }
68 }
@@ -111,11 +108,11 @@ func newGatewayBackend(n *core.IpfsNode) (gateway.IPFSBackend, error) {
108 }
109 }
110
114 - gw, err := gateway.NewBlocksGateway(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
111 + backend, err := gateway.NewBlocksBackend(bserv, gateway.WithValueStore(vsRouting), gateway.WithNameSystem(nsys))
112 if err != nil {
113 return nil, err
114 }
118 - return &offlineGatewayErrWrapper{gwimpl: gw}, nil
115 + return &offlineGatewayErrWrapper{gwimpl: backend}, nil
116 }
117
118 type offlineGatewayErrWrapper struct {
@@ -159,10 +156,10 @@ func (o *offlineGatewayErrWrapper) ResolvePath(ctx context.Context, path gateway
156 return md, err
157 }
158
162 -func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath) (gateway.ContentPathMetadata, io.ReadCloser, <-chan error, error) {
163 - md, data, errCh, err := o.gwimpl.GetCAR(ctx, path)
159 +func (o *offlineGatewayErrWrapper) GetCAR(ctx context.Context, path gateway.ImmutablePath, params gateway.CarParams) (gateway.ContentPathMetadata, io.ReadCloser, error) {
160 + md, data, err := o.gwimpl.GetCAR(ctx, path, params)
161 err = offlineErrWrap(err)
165 - return md, data, errCh, err
162 + return md, data, err
163 }
164
165 func (o *offlineGatewayErrWrapper) IsCached(ctx context.Context, path path.Path) bool {
@@ -191,12 +188,12 @@ var _ gateway.IPFSBackend = (*offlineGatewayErrWrapper)(nil)
188
189 var defaultPaths = []string{"/ipfs/", "/ipns/", "/api/", "/p2p/"}
190
194 -var subdomainGatewaySpec = &gateway.Specification{
191 +var subdomainGatewaySpec = &gateway.PublicGateway{
192 Paths: defaultPaths,
193 UseSubdomains: true,
194 }
195
199 -var defaultKnownGateways = map[string]*gateway.Specification{
196 +var defaultKnownGateways = map[string]*gateway.PublicGateway{
197 "localhost": subdomainGatewaySpec,
198 }
199
@@ -218,7 +215,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
215 Headers: headers,
216 DeserializedResponses: cfg.Gateway.DeserializedResponses.WithDefault(config.DefaultDeserializedResponses),
217 NoDNSLink: cfg.Gateway.NoDNSLink,
221 - PublicGateways: map[string]*gateway.Specification{},
218 + PublicGateways: map[string]*gateway.PublicGateway{},
219 }
220
221 // Add default implicit known gateways, such as subdomain gateway on localhost.
@@ -235,7 +232,7 @@ func getGatewayConfig(n *core.IpfsNode) (gateway.Config, error) {
232 continue
233 }
234
238 - gwCfg.PublicGateways[hostname] = &gateway.Specification{
235 + gwCfg.PublicGateways[hostname] = &gateway.PublicGateway{
236 Paths: gw.Paths,
237 NoDNSLink: gw.NoDNSLink,
238 UseSubdomains: gw.UseSubdomains,
docs/changelogs/v0.21.md
+4
@@ -104,6 +104,10 @@ $ curl "https://subdomain-gw.example.net/ipfs/${cid}/"
104
105 Rationale can be found in [kubo#9913](https://github.com/ipfs/kubo/pull/9913).
106
107 +#### Gateway: support for CAR parameters of IPIP-402
108 +
109 +The gateway now supports partial CAR export parameters as indicated in [IPIP-402](https://github.com/ipfs/specs/pull/402).
110 +
111 #### `ipfs dag stat` deduping statistics
112
113 `ipfs dat stat` now accept multiple CIDs and will dump advanced statistics
docs/examples/kubo-as-a-library/go.mod
+4 -2
@@ -7,7 +7,7 @@ go 1.18
7 replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 - github.com/ipfs/boxo v0.9.0
10 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
11 github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
12 github.com/libp2p/go-libp2p v0.27.5
13 github.com/multiformats/go-multiaddr v0.9.0
@@ -87,7 +87,7 @@ require (
87 github.com/ipfs/go-log/v2 v2.5.1 // indirect
88 github.com/ipfs/go-metrics-interface v0.0.1 // indirect
89 github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
90 - github.com/ipfs/go-unixfsnode v1.6.0 // indirect
90 + github.com/ipfs/go-unixfsnode v1.7.1 // indirect
91 github.com/ipld/edelweiss v0.2.0 // indirect
92 github.com/ipld/go-codec-dagpb v1.6.0 // indirect
93 github.com/ipld/go-ipld-prime v0.20.0 // indirect
@@ -139,6 +139,7 @@ require (
139 github.com/opentracing/opentracing-go v1.2.0 // indirect
140 github.com/openzipkin/zipkin-go v0.4.1 // indirect
141 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
142 + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
143 github.com/pkg/errors v0.9.1 // indirect
144 github.com/polydawn/refmt v0.89.0 // indirect
145 github.com/prometheus/client_golang v1.14.0 // indirect
@@ -156,6 +157,7 @@ require (
157 github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
158 github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
159 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
160 + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
161 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
162 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
163 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
docs/examples/kubo-as-a-library/go.sum
+7 -4
@@ -319,8 +319,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
319 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
320 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
321 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
322 -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
323 -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
322 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
323 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
324 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
325 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
326 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -409,8 +409,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
409 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
410 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
411 github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
412 -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
413 -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
412 +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
413 +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
414 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
415 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
416 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
@@ -421,6 +421,7 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt
421 github.com/ipld/go-ipld-prime v0.14.1/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0=
422 github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g=
423 github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M=
424 +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo=
425 github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus=
426 github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc=
427 github.com/jbenet/go-cienv v0.1.0 h1:Vc/s0QbQtoxX8MwwSLWWh+xNNZvM3Lw7NsTcHrvvhMc=
@@ -646,6 +647,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
647 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
648 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
649 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
650 +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
651 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
652 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
653 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -781,6 +783,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
783 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
784 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
785 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
786 +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
787 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
788 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
789 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
go.mod
+4 -2
@@ -16,7 +16,7 @@ require (
16 github.com/gogo/protobuf v1.3.2
17 github.com/google/uuid v1.3.0
18 github.com/hashicorp/go-multierror v1.1.1
19 - github.com/ipfs/boxo v0.9.0
19 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
20 github.com/ipfs/go-block-format v0.1.2
21 github.com/ipfs/go-cid v0.4.1
22 github.com/ipfs/go-cidutil v0.1.0
@@ -37,7 +37,7 @@ require (
37 github.com/ipfs/go-log/v2 v2.5.1
38 github.com/ipfs/go-metrics-interface v0.0.1
39 github.com/ipfs/go-metrics-prometheus v0.0.2
40 - github.com/ipfs/go-unixfsnode v1.6.0
40 + github.com/ipfs/go-unixfsnode v1.7.1
41 github.com/ipld/go-codec-dagpb v1.6.0
42 github.com/ipld/go-ipld-prime v0.20.0
43 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c
@@ -177,6 +177,7 @@ require (
177 github.com/onsi/ginkgo/v2 v2.9.2 // indirect
178 github.com/opencontainers/runtime-spec v1.0.2 // indirect
179 github.com/openzipkin/zipkin-go v0.4.1 // indirect
180 + github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect
181 github.com/pmezard/go-difflib v1.0.0 // indirect
182 github.com/polydawn/refmt v0.89.0 // indirect
183 github.com/prometheus/client_model v0.3.0 // indirect
@@ -197,6 +198,7 @@ require (
198 github.com/tidwall/pretty v1.2.0 // indirect
199 github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb // indirect
200 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
201 + github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect
202 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect
203 github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
204 github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
go.sum
+6 -4
@@ -354,8 +354,8 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:
354 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
355 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
356 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
357 -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
358 -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
357 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
358 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
359 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
360 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
361 github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY=
@@ -448,8 +448,8 @@ github.com/ipfs/go-metrics-prometheus v0.0.2/go.mod h1:ELLU99AQQNi+zX6GCGm2lAgnz
448 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
449 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
450 github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
451 -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
452 -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
451 +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
452 +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
453 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
454 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
455 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
@@ -715,6 +715,7 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D
715 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
716 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
717 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
718 +github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
719 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
720 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
721 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -894,6 +895,7 @@ github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvS
895 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc h1:BCPnHtcboadS0DvysUuJXZ4lWVv5Bh5i7+tbIyi+ck4=
896 github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM=
897 github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0=
898 +github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ=
899 github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI=
900 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o=
901 github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ=
test/dependencies/go.mod
+2 -2
@@ -7,7 +7,7 @@ replace github.com/ipfs/kubo => ../../
7 require (
8 github.com/Kubuxu/gocovmerge v0.0.0-20161216165753-7ecaa51963cd
9 github.com/golangci/golangci-lint v1.49.0
10 - github.com/ipfs/boxo v0.9.0
10 + github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469
11 github.com/ipfs/go-cid v0.4.1
12 github.com/ipfs/go-cidutil v0.1.0
13 github.com/ipfs/go-datastore v0.6.0
@@ -131,7 +131,7 @@ require (
131 github.com/ipfs/go-log/v2 v2.5.1 // indirect
132 github.com/ipfs/go-metrics-interface v0.0.1 // indirect
133 github.com/ipfs/go-peertaskqueue v0.8.1 // indirect
134 - github.com/ipfs/go-unixfsnode v1.6.0 // indirect
134 + github.com/ipfs/go-unixfsnode v1.7.1 // indirect
135 github.com/ipfs/kubo v0.16.0 // indirect
136 github.com/ipld/go-codec-dagpb v1.6.0 // indirect
137 github.com/jackpal/go-nat-pmp v1.0.2 // indirect
test/dependencies/go.sum
+4 -4
@@ -412,8 +412,8 @@ github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NH
412 github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
413 github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
414 github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
415 -github.com/ipfs/boxo v0.9.0 h1:Gb3KGXOZ4J5eCZTsky33tx2oHztrfBo+2IFq6lxmoGM=
416 -github.com/ipfs/boxo v0.9.0/go.mod h1:ic5+bhD5T+A9n0HMkXYHiTzpjjaAZaPeKRQ9dWethTs=
415 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469 h1:pZAeFwl7Ff6L2sWnqM5ekemKH7MOvSPFeyw7473LfAw=
416 +github.com/ipfs/boxo v0.9.1-0.20230608151829-475d57614469/go.mod h1:Fg+BnfxZ0RPzR0nOodzdIq3A7KgoWAOWsEIImrIQdBM=
417 github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
418 github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
419 github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo=
@@ -465,8 +465,8 @@ github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j
465 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg=
466 github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU=
467 github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU=
468 -github.com/ipfs/go-unixfsnode v1.6.0 h1:JOSA02yaLylRNi2rlB4ldPr5VcZhcnaIVj5zNLcOjDo=
469 -github.com/ipfs/go-unixfsnode v1.6.0/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
468 +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s=
469 +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk=
470 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
471 github.com/ipfs/hang-fds v0.1.0 h1:deBiFlWHsVGzJ0ZMaqscEqRM1r2O1rFZ59UiQXb1Xko=
472 github.com/ipfs/hang-fds v0.1.0/go.mod h1:29VLWOn3ftAgNNgXg/al7b11UzuQ+w7AwtCGcTaWkbM=
test/sharness/t0118-gateway-car.sh deleted
-136
@@ -1,136 +0,0 @@
1 -#!/usr/bin/env bash
2 -
3 -test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support"
4 -
5 -. lib/test-lib.sh
6 -
7 -test_init_ipfs
8 -test_launch_ipfs_daemon_without_network
9 -
10 -# CAR stream is not deterministic, as blocks can arrive in random order,
11 -# but if we have a small file that fits into a single block, and export its CID
12 -# we will get a CAR that is a deterministic array of bytes.
13 -
14 -# Import test case
15 -# See the static fixtures in ./t0118-gateway-car/
16 -test_expect_success "Add the dir test directory" '
17 - cp ../t0118-gateway-car/test-dag.car ./test-dag.car &&
18 - cp ../t0118-gateway-car/deterministic.car ./deterministic.car
19 -'
20 -ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
21 -FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
22 -
23 -# GET a reference DAG with dag-cbor+dag-pb+raw blocks as CAR
24 -
25 - # This test uses official CARv1 fixture from https://ipld.io/specs/transport/car/fixture/carv1-basic/
26 - test_expect_success "GET for application/vnd.ipld.car with dag-cbor root returns a CARv1 stream with full DAG" '
27 - ipfs dag import ../t0118-gateway-car/carv1-basic.car &&
28 - DAG_CBOR_CID=bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm &&
29 - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_CID" -o gateway-dag-cbor.car &&
30 - purge_blockstore &&
31 - ipfs dag import gateway-dag-cbor.car &&
32 - ipfs dag stat --offline $DAG_CBOR_CID
33 - '
34 -
35 -# GET unixfs file as CAR
36 -# (by using a single file we ensure deterministic result that can be compared byte-for-byte)
37 -
38 - test_expect_success "GET with format=car param returns a CARv1 stream" '
39 - ipfs dag import test-dag.car &&
40 - curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car &&
41 - test_cmp deterministic.car gateway-param.car
42 - '
43 -
44 - test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" '
45 - ipfs dag import test-dag.car &&
46 - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car &&
47 - test_cmp deterministic.car gateway-header.car
48 - '
49 -
50 - # explicit version=1
51 - test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" '
52 - ipfs dag import test-dag.car &&
53 - curl -sX GET -H "Accept: application/vnd.ipld.car;version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
54 - test_cmp deterministic.car gateway-header-v1.car
55 - '
56 -
57 - # explicit version=1 with whitepace
58 - test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream (with whitespace)" '
59 - ipfs dag import test-dag.car &&
60 - curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
61 - test_cmp deterministic.car gateway-header-v1.car
62 - '
63 -
64 - # explicit version=2
65 - test_expect_success "GET for application/vnd.ipld.raw version=2 returns HTTP 400 Bad Request error" '
66 - curl -svX GET -H "Accept: application/vnd.ipld.car;version=2" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" > curl_output 2>&1 &&
67 - cat curl_output &&
68 - grep "400 Bad Request" curl_output &&
69 - grep "unsupported CAR version" curl_output
70 - '
71 -
72 -# GET unixfs directory as a CAR with DAG and some selector
73 -
74 - # TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769
75 - test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" '
76 - ipfs dag import test-dag.car &&
77 - curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car &&
78 - purge_blockstore &&
79 - ipfs dag import gateway-dir.car &&
80 - ipfs dag stat --offline $ROOT_DIR_CID
81 - '
82 -
83 -# Make sure expected HTTP headers are returned with the CAR bytes
84 -
85 - test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" '
86 - ipfs dag import test-dag.car &&
87 - curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output &&
88 - cat curl_output &&
89 - grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output
90 - '
91 -
92 - # CAR is streamed, gateway may not have the entire thing, unable to calculate total size
93 - test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" '
94 - grep -qv "< Content-Length:" curl_output
95 - '
96 -
97 - test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" '
98 - grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output
99 - '
100 -
101 - test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" '
102 - grep "< X-Content-Type-Options: nosniff" curl_output
103 - '
104 -
105 - # CAR is streamed, gateway may not have the entire thing, unable to support range-requests
106 - # Partial downloads and resumes should be handled using
107 - # IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
108 - test_expect_success "GET response for application/vnd.ipld.car includes Accept-Ranges header" '
109 - grep "< Accept-Ranges: none" curl_output
110 - '
111 -
112 - test_expect_success "GET for application/vnd.ipld.car with query filename includes Content-Disposition with custom filename" '
113 - curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?filename=foobar.car" >/dev/null 2>curl_output_filename &&
114 - cat curl_output_filename &&
115 - grep "< Content-Disposition: attachment\; filename=\"foobar.car\"" curl_output_filename
116 - '
117 -
118 -# Cache control HTTP headers
119 -
120 - test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" '
121 - grep "< Etag: W/\"${FILE_CID}.car\"" curl_output
122 - '
123 -
124 - # (basic checks, detailed behavior for some fields is tested in t0116-gateway-cache.sh)
125 - test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" '
126 - grep "< X-Ipfs-Path" curl_output &&
127 - grep "< X-Ipfs-Roots" curl_output
128 - '
129 -
130 - test_expect_success "GET response for application/vnd.ipld.car includes same Cache-Control as a block or a file" '
131 - grep "< Cache-Control: public, max-age=29030400, immutable" curl_output
132 - '
133 -
134 -test_kill_ipfs_daemon
135 -
136 -test_done
test/sharness/t0118-gateway-car/README.md deleted
-30
@@ -1,30 +0,0 @@
1 -# Dataset description/sources
2 -
3 -- carv1-basic.car
4 - - raw CARv1
5 - - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.car
6 -
7 -- carv1-basic.json
8 - - description of the contents and layout of the raw CAR, encoded in DAG-JSON
9 - - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.json
10 -
11 -- test-dag.car + deterministic.car
12 - - raw CARv1
13 -
14 -generated with:
15 -
16 -```sh
17 -# using ipfs version 0.18.1
18 -mkdir -p subdir &&
19 -echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
20 -ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
21 -FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
22 -ipfs dag export $ROOT_DIR_CID > test-dag.car &&
23 -ipfs dag export $FILE_CID > deterministic.car &&
24 -
25 -echo ROOT_DIR_CID=${ROOT_DIR_CID} # ./
26 -echo FILE_CID=${FILE_CID} # /\subdir/ascii.txt
27 -
28 -# ROOT_DIR_CID=bafybeiefu3d7oytdumk5v7gn6s7whpornueaw7m7u46v2o6omsqcrhhkzi # ./
29 -# FILE_CID=bafkreifkam6ns4aoolg3wedr4uzrs3kvq66p4pecirz6y2vlrngla62mxm # /subdir/ascii.txt
30 -```
test/sharness/t0118-gateway-car/carv1-basic.car
Binary files a/test/sharness/t0118-gateway-car/carv1-basic.car and /dev/null differ
test/sharness/t0118-gateway-car/carv1-basic.json deleted
-159
@@ -1,159 +0,0 @@
1 -{
2 - "blocks": [
3 - {
4 - "blockLength": 55,
5 - "blockOffset": 137,
6 - "cid": {
7 - "/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
8 - },
9 - "content": {
10 - "link": {
11 - "/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
12 - },
13 - "name": "blip"
14 - },
15 - "length": 92,
16 - "offset": 100
17 - },
18 - {
19 - "blockLength": 97,
20 - "blockOffset": 228,
21 - "cid": {
22 - "/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
23 - },
24 - "content": {
25 - "Links": [
26 - {
27 - "Hash": {
28 - "/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
29 - },
30 - "Name": "bear",
31 - "Tsize": 4
32 - },
33 - {
34 - "Hash": {
35 - "/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
36 - },
37 - "Name": "second",
38 - "Tsize": 149
39 - }
40 - ]
41 - },
42 - "length": 133,
43 - "offset": 192
44 - },
45 - {
46 - "blockLength": 4,
47 - "blockOffset": 362,
48 - "cid": {
49 - "/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
50 - },
51 - "content": {
52 - "/": {
53 - "bytes": "Y2NjYw"
54 - }
55 - },
56 - "length": 41,
57 - "offset": 325
58 - },
59 - {
60 - "blockLength": 94,
61 - "blockOffset": 402,
62 - "cid": {
63 - "/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
64 - },
65 - "content": {
66 - "Links": [
67 - {
68 - "Hash": {
69 - "/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
70 - },
71 - "Name": "dog",
72 - "Tsize": 4
73 - },
74 - {
75 - "Hash": {
76 - "/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
77 - },
78 - "Name": "first",
79 - "Tsize": 51
80 - }
81 - ]
82 - },
83 - "length": 130,
84 - "offset": 366
85 - },
86 - {
87 - "blockLength": 4,
88 - "blockOffset": 533,
89 - "cid": {
90 - "/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
91 - },
92 - "content": {
93 - "/": {
94 - "bytes": "YmJiYg"
95 - }
96 - },
97 - "length": 41,
98 - "offset": 496
99 - },
100 - {
101 - "blockLength": 47,
102 - "blockOffset": 572,
103 - "cid": {
104 - "/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
105 - },
106 - "content": {
107 - "Links": [
108 - {
109 - "Hash": {
110 - "/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
111 - },
112 - "Name": "cat",
113 - "Tsize": 4
114 - }
115 - ]
116 - },
117 - "length": 82,
118 - "offset": 537
119 - },
120 - {
121 - "blockLength": 4,
122 - "blockOffset": 656,
123 - "cid": {
124 - "/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
125 - },
126 - "content": {
127 - "/": {
128 - "bytes": "YWFhYQ"
129 - }
130 - },
131 - "length": 41,
132 - "offset": 619
133 - },
134 - {
135 - "blockLength": 18,
136 - "blockOffset": 697,
137 - "cid": {
138 - "/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
139 - },
140 - "content": {
141 - "link": null,
142 - "name": "limbo"
143 - },
144 - "length": 55,
145 - "offset": 660
146 - }
147 - ],
148 - "header": {
149 - "roots": [
150 - {
151 - "/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
152 - },
153 - {
154 - "/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
155 - }
156 - ],
157 - "version": 1
158 - }
159 -}
test/sharness/t0118-gateway-car/deterministic.car
Binary files a/test/sharness/t0118-gateway-car/deterministic.car and /dev/null differ
test/sharness/t0118-gateway-car/test-dag.car
Binary files a/test/sharness/t0118-gateway-car/test-dag.car and /dev/null differ