feat(gateway): TAR response format (#9029)
Implementation of IPIP-288 (https://github.com/ipfs/specs/pull/288) Co-authored-by: Marcin Rataj <lidel@lidel.org>
Henrique Dias committed
Nov 9, 2022 at 19:20 UTC
a210abd74364076404c18df1acbeed8bd6a5d6b7
10 files changed
+222
-12
core/corehttp/gateway_handler.go
+13
-5
@@ -430,6 +430,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
430
carVersion := formatParams["version"]
431
i.serveCAR(r.Context(), w, r, resolvedPath, contentPath, carVersion, begin)
432
return
433
+ case "application/x-tar":
434
+ logger.Debugw("serving tar file", "path", contentPath)
435
+ i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
436
+ return
437
default: // catch-all for unsuported application/vnd.*
438
err := fmt.Errorf("unsupported format %q", responseFormat)
439
webError(w, "failed respond with requested content type", err, http.StatusBadRequest)
@@ -842,9 +846,10 @@ func getEtag(r *http.Request, cid cid.Cid) string {
846
responseFormat, _, err := customResponseFormat(r)
847
if err == nil && responseFormat != "" {
848
// application/vnd.ipld.foo → foo
845
- f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
846
- // Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
847
- suffix = `.` + f + suffix
849
+ // application/x-bar → x-bar
850
+ shortFormat := responseFormat[strings.LastIndexAny(responseFormat, "/.")+1:]
851
+ // Etag: "cid.shortFmt" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
852
+ suffix = `.` + shortFormat + suffix
853
}
854
// TODO: include selector suffix when https://github.com/ipfs/kubo/issues/8769 lands
855
return prefix + cid.String() + suffix
@@ -859,14 +864,17 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
864
return "application/vnd.ipld.raw", nil, nil
865
case "car":
866
return "application/vnd.ipld.car", nil, nil
867
+ case "tar":
868
+ return "application/x-tar", nil, nil
869
}
870
}
871
// Browsers and other user agents will send Accept header with generic types like:
872
// Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
866
- // We only care about explciit, vendor-specific content-types.
873
+ // We only care about explicit, vendor-specific content-types.
874
for _, accept := range r.Header.Values("Accept") {
875
// respond to the very first ipld content type
869
- if strings.HasPrefix(accept, "application/vnd.ipld") {
876
+ if strings.HasPrefix(accept, "application/vnd.ipld") ||
877
+ strings.HasPrefix(accept, "application/x-tar") {
878
mediatype, params, err := mime.ParseMediaType(accept)
879
if err != nil {
880
return "", nil, err
core/corehttp/gateway_handler_tar.go
new
+92
@@ -0,0 +1,92 @@
1
+package corehttp
2
+
3
+import (
4
+ "context"
5
+ "html"
6
+ "net/http"
7
+ "time"
8
+
9
+ files "github.com/ipfs/go-ipfs-files"
10
+ ipath "github.com/ipfs/interface-go-ipfs-core/path"
11
+ "github.com/ipfs/kubo/tracing"
12
+ "go.opentelemetry.io/otel/attribute"
13
+ "go.opentelemetry.io/otel/trace"
14
+ "go.uber.org/zap"
15
+)
16
+
17
+var unixEpochTime = time.Unix(0, 0)
18
+
19
+func (i *gatewayHandler) serveTAR(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
20
+ ctx, span := tracing.Span(ctx, "Gateway", "ServeTAR", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
21
+ defer span.End()
22
+
23
+ ctx, cancel := context.WithCancel(ctx)
24
+ defer cancel()
25
+
26
+ // Get Unixfs file
27
+ file, err := i.api.Unixfs().Get(ctx, resolvedPath)
28
+ if err != nil {
29
+ webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusBadRequest)
30
+ return
31
+ }
32
+ defer file.Close()
33
+
34
+ rootCid := resolvedPath.Cid()
35
+
36
+ // Set Cache-Control and read optional Last-Modified time
37
+ modtime := addCacheControlHeaders(w, r, contentPath, rootCid)
38
+
39
+ // Weak Etag W/ because we can't guarantee byte-for-byte identical
40
+ // responses, but still want to benefit from HTTP Caching. Two TAR
41
+ // responses for the same CID will be logically equivalent,
42
+ // but when TAR is streamed, then in theory, files and directories
43
+ // may arrive in different order (depends on TAR lib and filesystem/inodes).
44
+ etag := `W/` + getEtag(r, rootCid)
45
+ w.Header().Set("Etag", etag)
46
+
47
+ // Finish early if Etag match
48
+ if r.Header.Get("If-None-Match") == etag {
49
+ w.WriteHeader(http.StatusNotModified)
50
+ return
51
+ }
52
+
53
+ // Set Content-Disposition
54
+ var name string
55
+ if urlFilename := r.URL.Query().Get("filename"); urlFilename != "" {
56
+ name = urlFilename
57
+ } else {
58
+ name = rootCid.String() + ".tar"
59
+ }
60
+ setContentDispositionHeader(w, name, "attachment")
61
+
62
+ // Construct the TAR writer
63
+ tarw, err := files.NewTarWriter(w)
64
+ if err != nil {
65
+ webError(w, "could not build tar writer", err, http.StatusInternalServerError)
66
+ return
67
+ }
68
+ defer tarw.Close()
69
+
70
+ // Sets correct Last-Modified header. This code is borrowed from the standard
71
+ // library (net/http/server.go) as we cannot use serveFile without throwing the entire
72
+ // TAR into the memory first.
73
+ if !(modtime.IsZero() || modtime.Equal(unixEpochTime)) {
74
+ w.Header().Set("Last-Modified", modtime.UTC().Format(http.TimeFormat))
75
+ }
76
+
77
+ w.Header().Set("Content-Type", "application/x-tar")
78
+ w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
79
+
80
+ // The TAR has a top-level directory (or file) named by the CID.
81
+ if err := tarw.WriteFile(file, rootCid.String()); err != nil {
82
+ w.Header().Set("X-Stream-Error", err.Error())
83
+ // Trailer headers do not work in web browsers
84
+ // (see https://github.com/mdn/browser-compat-data/issues/14703)
85
+ // and we have limited options around error handling in browser contexts.
86
+ // To improve UX/DX, we finish response stream with error message, allowing client to
87
+ // (1) detect error by having corrupted TAR
88
+ // (2) be able to reason what went wrong by instecting the tail of TAR stream
89
+ _, _ = w.Write([]byte(err.Error()))
90
+ return
91
+ }
92
+}
docs/changelogs/v0.17.md
+20
-1
@@ -9,15 +9,34 @@ Below is an outline of all that is in this release, so you get a sense of all th
9
- [Kubo changelog v0.17](#kubo-changelog-v017)
10
- [v0.17.0](#v0170)
11
- [Overview](#overview)
12
+ - [TOC](#toc)
13
- [🔦 Highlights](#-highlights)
14
+ - [TAR Response Format on Gateways](#tar-response-format-on-gateways)
15
- [Changelog](#changelog)
16
- [Contributors](#contributors)
17
16
-
18
### 🔦 Highlights
19
20
<!-- TODO -->
21
22
+#### TAR Response Format on Gateways
23
+
24
+Implemented [IPIP-288](https://github.com/ipfs/specs/pull/288) which adds
25
+support for requesting deserialized UnixFS directory as a TAR stream.
26
+
27
+HTTP clients can request TAR response by passing the `?format=tar` URL
28
+parameter, or setting `Accept: application/x-tar` HTTP header:
29
+
30
+```console
31
+$ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq
32
+$ curl -H "Accept: application/x-tar" "http://127.0.0.1:8080/ipfs/$DIR_CID" > dir.tar
33
+$ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=tar" | tar xv
34
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq
35
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - alt.txt
36
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1 - transcript.txt
37
+bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq/1 - Barrel - Part 1.png
38
+```
39
+
40
### Changelog
41
42
<!-- TODO -->
docs/examples/kubo-as-a-library/go.mod
+1
-1
@@ -7,7 +7,7 @@ go 1.17
7
replace github.com/ipfs/kubo => ./../../..
8
9
require (
10
- github.com/ipfs/go-ipfs-files v0.1.1
10
+ github.com/ipfs/go-ipfs-files v0.2.0
11
github.com/ipfs/interface-go-ipfs-core v0.7.0
12
github.com/ipfs/kubo v0.14.0-rc1
13
github.com/libp2p/go-libp2p v0.23.2
docs/examples/kubo-as-a-library/go.sum
+2
-2
@@ -559,8 +559,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
559
github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
560
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
561
github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
562
-github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
563
-github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
562
+github.com/ipfs/go-ipfs-files v0.2.0 h1:z6MCYHQSZpDWpUSK59Kf0ajP1fi4gLCf6fIulVsp8A8=
563
+github.com/ipfs/go-ipfs-files v0.2.0/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
564
github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
565
github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
566
github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=
go.mod
+1
-1
@@ -37,7 +37,7 @@ require (
37
github.com/ipfs/go-ipfs-cmds v0.8.1
38
github.com/ipfs/go-ipfs-exchange-interface v0.2.0
39
github.com/ipfs/go-ipfs-exchange-offline v0.3.0
40
- github.com/ipfs/go-ipfs-files v0.1.1
40
+ github.com/ipfs/go-ipfs-files v0.2.0
41
github.com/ipfs/go-ipfs-keystore v0.0.2
42
github.com/ipfs/go-ipfs-pinner v0.2.1
43
github.com/ipfs/go-ipfs-posinfo v0.0.1
go.sum
+2
-2
@@ -552,8 +552,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uY
552
github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s=
553
github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
554
github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs=
555
-github.com/ipfs/go-ipfs-files v0.1.1 h1:/MbEowmpLo9PJTEQk16m9rKzUHjeP4KRU9nWJyJO324=
556
-github.com/ipfs/go-ipfs-files v0.1.1/go.mod h1:8xkIrMWH+Y5P7HvJ4Yc5XWwIW2e52dyXUiC0tZyjDbM=
555
+github.com/ipfs/go-ipfs-files v0.2.0 h1:z6MCYHQSZpDWpUSK59Kf0ajP1fi4gLCf6fIulVsp8A8=
556
+github.com/ipfs/go-ipfs-files v0.2.0/go.mod h1:vT7uaQfIsprKktzbTPLnIsd+NGw9ZbYwSq0g3N74u0M=
557
github.com/ipfs/go-ipfs-keystore v0.0.2 h1:Fa9xg9IFD1VbiZtrNLzsD0GuELVHUFXCWF64kCPfEXU=
558
github.com/ipfs/go-ipfs-keystore v0.0.2/go.mod h1:H49tRmibOEs7gLMgbOsjC4dqh1u5e0R/SWuc2ScfgSo=
559
github.com/ipfs/go-ipfs-pinner v0.2.1 h1:kw9hiqh2p8TatILYZ3WAfQQABby7SQARdrdA+5Z5QfY=
test/sharness/t0122-gateway-tar-data/inside-root.car
Binary files /dev/null and b/test/sharness/t0122-gateway-tar-data/inside-root.car differ
test/sharness/t0122-gateway-tar-data/outside-root.car
Binary files /dev/null and b/test/sharness/t0122-gateway-tar-data/outside-root.car differ
test/sharness/t0122-gateway-tar.sh
new
+91
@@ -0,0 +1,91 @@
1
+#!/usr/bin/env bash
2
+
3
+test_description="Test HTTP Gateway TAR (application/x-tar) Support"
4
+
5
+. lib/test-lib.sh
6
+
7
+test_init_ipfs
8
+test_launch_ipfs_daemon_without_network
9
+
10
+OUTSIDE_ROOT_CID="bafybeicaj7kvxpcv4neaqzwhrqqmdstu4dhrwfpknrgebq6nzcecfucvyu"
11
+INSIDE_ROOT_CID="bafybeibfevfxlvxp5vxobr5oapczpf7resxnleb7tkqmdorc4gl5cdva3y"
12
+
13
+test_expect_success "Add the test directory" '
14
+ mkdir -p rootDir/ipfs &&
15
+ mkdir -p rootDir/ipns &&
16
+ mkdir -p rootDir/api &&
17
+ mkdir -p rootDir/ą/ę &&
18
+ echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
19
+ echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
20
+ echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
21
+ echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
22
+ DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
23
+ FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
24
+ FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
25
+ echo "$FILE_CID / $FILE_SIZE"
26
+'
27
+
28
+test_expect_success "GET TAR with format=tar and extract" '
29
+ curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x
30
+'
31
+
32
+test_expect_success "GET TAR with 'Accept: application/x-tar' and extract" '
33
+ curl -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" | tar -x
34
+'
35
+
36
+test_expect_success "GET TAR with format=tar has expected Content-Type" '
37
+ curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" > curl_output_filename 2>&1 &&
38
+ test_should_contain "Content-Disposition: attachment;" curl_output_filename &&
39
+ test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename &&
40
+ test_should_contain "Content-Type: application/x-tar" curl_output_filename
41
+'
42
+
43
+test_expect_success "GET TAR with 'Accept: application/x-tar' has expected Content-Type" '
44
+ curl -sD - -H "Accept: application/x-tar" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output_filename 2>&1 &&
45
+ test_should_contain "Content-Disposition: attachment;" curl_output_filename &&
46
+ test_should_contain "Etag: W/\"$FILE_CID.x-tar" curl_output_filename &&
47
+ test_should_contain "Content-Type: application/x-tar" curl_output_filename
48
+'
49
+
50
+test_expect_success "GET TAR has expected root file" '
51
+ rm -rf outputDir && mkdir outputDir &&
52
+ curl "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=tar" | tar -x -C outputDir &&
53
+ test -f "outputDir/$FILE_CID" &&
54
+ echo "I am a txt file on path with utf8" > expected &&
55
+ test_cmp expected outputDir/$FILE_CID
56
+'
57
+
58
+test_expect_success "GET TAR has expected root directory" '
59
+ rm -rf outputDir && mkdir outputDir &&
60
+ curl "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=tar" | tar -x -C outputDir &&
61
+ test -d "outputDir/$DIR_CID" &&
62
+ echo "I am a txt file on path with utf8" > expected &&
63
+ test_cmp expected outputDir/$DIR_CID/ą/ę/file-źł.txt
64
+'
65
+
66
+test_expect_success "GET TAR with explicit ?filename= succeeds with modified Content-Disposition header" "
67
+ curl -fo actual -D actual_headers 'http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?filename=testтест.tar&format=tar' &&
68
+ grep -F 'Content-Disposition: attachment; filename=\"test____.tar\"; filename*=UTF-8'\'\''test%D1%82%D0%B5%D1%81%D1%82.tar' actual_headers
69
+"
70
+
71
+test_expect_success "Add CARs with relative paths to test with" '
72
+ ipfs dag import ../t0122-gateway-tar-data/outside-root.car > import_output &&
73
+ test_should_contain $OUTSIDE_ROOT_CID import_output &&
74
+ ipfs dag import ../t0122-gateway-tar-data/inside-root.car > import_output &&
75
+ test_should_contain $INSIDE_ROOT_CID import_output
76
+'
77
+
78
+test_expect_success "GET TAR with relative paths outside root fails" '
79
+ curl -o - "http://127.0.0.1:$GWAY_PORT/ipfs/$OUTSIDE_ROOT_CID?format=tar" > curl_output_filename &&
80
+ test_should_contain "relative UnixFS paths outside the root are now allowed" curl_output_filename
81
+'
82
+
83
+test_expect_success "GET TAR with relative paths inside root works" '
84
+ rm -rf outputDir && mkdir outputDir &&
85
+ curl "http://127.0.0.1:$GWAY_PORT/ipfs/$INSIDE_ROOT_CID?format=tar" | tar -x -C outputDir &&
86
+ test -f outputDir/$INSIDE_ROOT_CID/foobar/file
87
+'
88
+
89
+test_kill_ipfs_daemon
90
+
91
+test_done