fix(gateway): undesired conversions to dag-json and friends (#9566)
* fix(gateway): do not convert unixfs/raw into dag-* unless explicit * fix(gateway): keep only dag-json|dag-cbor handling * fix: allow requesting dag-json as application/json - adds bunch of additional tests including JSON file on UnixFS - fix: dag-json codec (0x0129) can be returned as plain json - fix: json codec (0x0200) cna be retrurned as plain json * fix: using ?format|Accept with CID w/ codec works * docs(changelog): cbor and json on gateway Co-authored-by: Marcin Rataj <lidel@lidel.org>
Henrique Dias committed
Jan 21, 2023 at 04:21 UTC
14703e19e31b78b2170e3a7ee7bc209249c9575f
4 files changed
+241
-190
core/corehttp/gateway_handler.go
+11
-10
@@ -418,9 +418,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
418
419
// Support custom response formats passed via ?format or Accept HTTP header
420
switch responseFormat {
421
- case "":
422
- switch resolvedPath.Cid().Prefix().Codec {
423
- case uint64(mc.Json), uint64(mc.DagJson), uint64(mc.Cbor), uint64(mc.DagCbor):
421
+ case "", "application/json", "application/cbor":
422
+ switch mc.Code(resolvedPath.Cid().Prefix().Codec) {
423
+ case mc.Json, mc.DagJson, mc.Cbor, mc.DagCbor:
424
logger.Debugw("serving codec", "path", contentPath)
425
i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat)
426
default:
@@ -441,14 +441,13 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
441
logger.Debugw("serving tar file", "path", contentPath)
442
i.serveTAR(r.Context(), w, r, resolvedPath, contentPath, begin, logger)
443
return
444
- case "application/json", "application/vnd.ipld.dag-json",
445
- "application/cbor", "application/vnd.ipld.dag-cbor":
444
+ case "application/vnd.ipld.dag-json", "application/vnd.ipld.dag-cbor":
445
logger.Debugw("serving codec", "path", contentPath)
446
i.serveCodec(r.Context(), w, r, resolvedPath, contentPath, begin, responseFormat)
447
return
448
default: // catch-all for unsuported application/vnd.*
449
err := fmt.Errorf("unsupported format %q", responseFormat)
451
- webError(w, "failed respond with requested content type", err, http.StatusBadRequest)
450
+ webError(w, "failed to respond with requested content type", err, http.StatusBadRequest)
451
return
452
}
453
}
@@ -878,14 +877,14 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
877
return "application/vnd.ipld.car", nil, nil
878
case "tar":
879
return "application/x-tar", nil, nil
881
- case "dag-json":
882
- return "application/vnd.ipld.dag-json", nil, nil
880
case "json":
881
return "application/json", nil, nil
885
- case "dag-cbor":
886
- return "application/vnd.ipld.dag-cbor", nil, nil
882
case "cbor":
883
return "application/cbor", nil, nil
884
+ case "dag-json":
885
+ return "application/vnd.ipld.dag-json", nil, nil
886
+ case "dag-cbor":
887
+ return "application/vnd.ipld.dag-cbor", nil, nil
888
}
889
}
890
// Browsers and other user agents will send Accept header with generic types like:
@@ -908,6 +907,8 @@ func customResponseFormat(r *http.Request) (mediaType string, params map[string]
907
}
908
}
909
}
910
+ // If none of special-cased content types is found, return empty string
911
+ // to indicate default, implicit UnixFS response should be prepared
912
return "", nil, nil
913
}
914
core/corehttp/gateway_handler_codec.go
+41
-42
@@ -25,22 +25,25 @@ import (
25
26
// codecToContentType maps the supported IPLD codecs to the HTTP Content
27
// Type they should have.
28
-var codecToContentType = map[uint64]string{
29
- uint64(mc.Json): "application/json",
30
- uint64(mc.Cbor): "application/cbor",
31
- uint64(mc.DagJson): "application/vnd.ipld.dag-json",
32
- uint64(mc.DagCbor): "application/vnd.ipld.dag-cbor",
28
+var codecToContentType = map[mc.Code]string{
29
+ mc.Json: "application/json",
30
+ mc.Cbor: "application/cbor",
31
+ mc.DagJson: "application/vnd.ipld.dag-json",
32
+ mc.DagCbor: "application/vnd.ipld.dag-cbor",
33
}
34
35
-// contentTypeToCodecs maps the HTTP Content Type to the respective
36
-// possible codecs. If the original data is in one of those codecs,
37
-// we stream the raw bytes. Otherwise, we encode in the last codec
38
-// of the list.
39
-var contentTypeToCodecs = map[string][]uint64{
40
- "application/json": {uint64(mc.Json), uint64(mc.DagJson)},
41
- "application/vnd.ipld.dag-json": {uint64(mc.DagJson)},
42
- "application/cbor": {uint64(mc.Cbor), uint64(mc.DagCbor)},
43
- "application/vnd.ipld.dag-cbor": {uint64(mc.DagCbor)},
35
+// contentTypeToRaw maps the HTTP Content Type to the respective codec that
36
+// allows raw response without any conversion.
37
+var contentTypeToRaw = map[string][]mc.Code{
38
+ "application/json": {mc.Json, mc.DagJson},
39
+ "application/cbor": {mc.Cbor, mc.DagCbor},
40
+}
41
+
42
+// contentTypeToCodec maps the HTTP Content Type to the respective codec. We
43
+// only add here the codecs that we want to convert-to-from.
44
+var contentTypeToCodec = map[string]mc.Code{
45
+ "application/vnd.ipld.dag-json": mc.DagJson,
46
+ "application/vnd.ipld.dag-cbor": mc.DagCbor,
47
}
48
49
// contentTypeToExtension maps the HTTP Content Type to the respective file
@@ -56,7 +59,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
59
ctx, span := tracing.Span(ctx, "Gateway", "ServeCodec", trace.WithAttributes(attribute.String("path", resolvedPath.String()), attribute.String("requestedContentType", requestedContentType)))
60
defer span.End()
61
59
- cidCodec := resolvedPath.Cid().Prefix().Codec
62
+ cidCodec := mc.Code(resolvedPath.Cid().Prefix().Codec)
63
responseContentType := requestedContentType
64
65
// If the resolved path still has some remainder, return error for now.
@@ -90,22 +93,36 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
93
// No content type is specified by the user (via Accept, or format=). However,
94
// we support this format. Let's handle it.
95
if requestedContentType == "" {
93
- isDAG := cidCodec == uint64(mc.DagJson) || cidCodec == uint64(mc.DagCbor)
96
+ isDAG := cidCodec == mc.DagJson || cidCodec == mc.DagCbor
97
acceptsHTML := strings.Contains(r.Header.Get("Accept"), "text/html")
98
download := r.URL.Query().Get("download") == "true"
99
100
if isDAG && acceptsHTML && !download {
101
i.serveCodecHTML(ctx, w, r, resolvedPath, contentPath)
102
} else {
103
+ // This covers CIDs with codec 'json' and 'cbor' as those do not have
104
+ // an explicit requested content type.
105
i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime)
106
}
107
108
return
109
}
110
106
- // Otherwise, the user has requested a specific content type. Let's first get
107
- // the codecs that can be used with this content type.
108
- codecs, ok := contentTypeToCodecs[requestedContentType]
111
+ // If DAG-JSON or DAG-CBOR was requested using corresponding plain content type
112
+ // return raw block as-is, without conversion
113
+ skipCodecs, ok := contentTypeToRaw[requestedContentType]
114
+ if ok {
115
+ for _, skipCodec := range skipCodecs {
116
+ if skipCodec == cidCodec {
117
+ i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime)
118
+ return
119
+ }
120
+ }
121
+ }
122
+
123
+ // Otherwise, the user has requested a specific content type (a DAG-* variant).
124
+ // Let's first get the codecs that can be used with this content type.
125
+ toCodec, ok := contentTypeToCodec[requestedContentType]
126
if !ok {
127
// This is never supposed to happen unless function is called with wrong parameters.
128
err := fmt.Errorf("unsupported content type: %s", requestedContentType)
@@ -113,27 +130,7 @@ func (i *gatewayHandler) serveCodec(ctx context.Context, w http.ResponseWriter,
130
return
131
}
132
116
- // If we need to convert, use the last codec (strict dag- variant)
117
- toCodec := codecs[len(codecs)-1]
118
-
119
- // If the requested content type has "dag-", ALWAYS go through the encoding
120
- // process in order to validate the content.
121
- if strings.Contains(requestedContentType, "dag-") {
122
- i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime)
123
- return
124
- }
125
-
126
- // Otherwise, check if the data is encoded with the requested content type.
127
- // If so, we can directly stream the raw data. serveRawBlock cannot be directly
128
- // used here as it sets different headers.
129
- for _, codec := range codecs {
130
- if resolvedPath.Cid().Prefix().Codec == codec {
131
- i.serveCodecRaw(ctx, w, r, resolvedPath, contentPath, name, modtime)
132
- return
133
- }
134
- }
135
-
136
- // Finally, if nothing of the above is true, we have to actually convert the codec.
133
+ // This handles DAG-* conversions and validations.
134
i.serveCodecConverted(ctx, w, r, resolvedPath, contentPath, toCodec, modtime)
135
}
136
@@ -165,6 +162,7 @@ func (i *gatewayHandler) serveCodecHTML(ctx context.Context, w http.ResponseWrit
162
}
163
}
164
165
+// serveCodecRaw returns the raw block without any conversion
166
func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, name string, modtime time.Time) {
167
blockCid := resolvedPath.Cid()
168
blockReader, err := i.api.Block().Get(ctx, resolvedPath)
@@ -184,7 +182,8 @@ func (i *gatewayHandler) serveCodecRaw(ctx context.Context, w http.ResponseWrite
182
_, _, _ = ServeContent(w, r, name, modtime, content)
183
}
184
187
-func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec uint64, modtime time.Time) {
185
+// serveCodecConverted returns payload converted to codec specified in toCodec
186
+func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, toCodec mc.Code, modtime time.Time) {
187
obj, err := i.api.Dag().Get(ctx, resolvedPath.Cid())
188
if err != nil {
189
webError(w, "ipfs dag get "+html.EscapeString(resolvedPath.String()), err, http.StatusInternalServerError)
@@ -199,7 +198,7 @@ func (i *gatewayHandler) serveCodecConverted(ctx context.Context, w http.Respons
198
}
199
finalNode := universal.(ipld.Node)
200
202
- encoder, err := multicodec.LookupEncoder(toCodec)
201
+ encoder, err := multicodec.LookupEncoder(uint64(toCodec))
202
if err != nil {
203
webError(w, err.Error(), err, http.StatusInternalServerError)
204
return
docs/changelogs/v0.18.md
+68
-11
@@ -64,19 +64,74 @@ Learn more in the [`Reprovider` config](https://github.com/ipfs/go-ipfs/blob/mas
64
65
##### (DAG-)JSON and (DAG-)CBOR response formats
66
67
-Implemented [IPIP-328](https://github.com/ipfs/specs/pull/328) which adds support
68
-for DAG-JSON and DAG-CBOR, as well as their non-DAG variants, to the gateway. Now,
69
-CIDs that encode JSON, CBOR, DAG-JSON and DAG-CBOR objects can be retrieved, and
70
-traversed thanks to the [special meaning of CBOR Tag 42](https://github.com/ipld/cid-cbor/).
67
+The IPFS project has reserved the corresponding media types at IANA:
68
+- [`application/vnd.ipld.dag-json`](https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-json)
69
+- [`application/vnd.ipld.dag-cbor`](https://www.iana.org/assignments/media-types/application/vnd.ipld.dag-cbor)
70
72
-HTTP clients can request JSON, CBOR, DAG-JSON, and DAG-CBOR responses by either
73
-passing the query parameter `?format` or setting the `Accept` HTTP header to the
74
-following values:
71
+This release implements them as part of [IPIP-328](https://github.com/ipfs/specs/pull/328)
72
+and adds Gateway support for CIDs with `json` (0x0200), `cbor` (0x51),
73
+[`dag-json`](https://ipld.io/specs/codecs/dag-json/) (0x0129)
74
+and [`dag-cbor`](https://ipld.io/specs/codecs/dag-cbor/spec/) (0x71) codecs.
75
76
-- JSON: `?format=json`, or `Accept: application/json`
77
-- CBOR: `?format=cbor`, or `Accept: application/cbor`
78
-- DAG-JSON: `?format=dag-json`, or `Accept: application/vnd.ipld.dag-json`
79
-- DAG-JSON: `?format=dag-cbor`, or `Accept: application/vnd.ipld.dag-cbor`
76
+To specify the response `Content-Type` explicitly, the HTTP client can override
77
+the codec present in the CID by using the `format` parameter
78
+or setting the `Accept` HTTP header:
79
+
80
+- Plain JSON: `?format=json` or `Accept: application/json`
81
+- Plain CBOR: `?format=cbor` or `Accept: application/cbor`
82
+- DAG-JSON: `?format=dag-json` or `Accept: application/vnd.ipld.dag-json`
83
+- DAG-CBOR: `?format=dag-cbor` or `Accept: application/vnd.ipld.dag-cbor`
84
+
85
+In addition, when DAG-JSON or DAG-CBOR is requested with the `Accept` header
86
+set to `text/html`, the Gateway will return a basic HTML page with download
87
+options, improving the user experience in web browsers.
88
+
89
+###### Example 1: DAG-CBOR and DAG-JSON Conversion on Gateway
90
+
91
+The Gateway supports conversion between DAG-CBOR and DAG-JSON for efficient
92
+end-to-end data structure management: author in CBOR or JSON, store as binary
93
+CBOR and retrieve as JSON via HTTP:
94
+
95
+```console
96
+$ echo '{"test": "json"}' | ipfs dag put # implicit --input-codec dag-json --store-codec dag-cbor
97
+bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4
98
+
99
+$ ipfs block get bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4 | xxd
100
+00000000: a164 7465 7374 646a 736f 6e .dtestdjson
101
+
102
+$ ipfs dag get bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4 # implicit --output-codec dag-json
103
+{"test":"json"}
104
+
105
+$ curl "http://127.0.0.1:8080/ipfs/bafyreico7mjtqtqhvawro3yud5uqn6sc33nzqb7b5j2d7pdmzer5nab4t4?format=dag-json"
106
+{"test":"json"}
107
+```
108
+
109
+###### Example 2: Traversing CBOR DAGs
110
+
111
+Placing a CID in [CBOR Tag 42](https://github.com/ipld/cid-cbor/) enables the
112
+creation of arbitrary DAGs. The equivalent DAG-JSON notation for linking
113
+to different blocks is represented by `{ "/": "cid" }`.
114
+
115
+The Gateway supports traversing these links, enabling access to data
116
+referenced by structures other than regular UnixFS directories:
117
+
118
+```console
119
+$ echo '{"test.jpg": {"/": "bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi"}}' | ipfs dag put
120
+bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4 # dag-cbor document linking to unixfs file
121
+
122
+$ ipfs resolve /ipfs/bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4/test.jpg
123
+/ipfs/bafybeigdyrzt5sfp7udm7hu76uh7y26nf3efuylqabf3oclgtqy55fbzdi
124
+
125
+$ ipfs dag stat bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4
126
+Size: 119827, NumBlocks: 2
127
+
128
+$ curl "http://127.0.0.1:8080/ipfs/bafyreihspwy3zlkzgphmec5d3xb5g5njrqwotd46lyubnelbzktnmsxkq4/test.jpg" > test.jpg
129
+```
130
+
131
+###### Example 3: UnixFS directory listing as JSON
132
+
133
+Finally, Gateway now supports the same [logical format projection](https://ipld.io/specs/codecs/dag-pb/spec/#logical-format) from
134
+DAG-PB to DAG-JSON as the `ipfs dag get` command, enabling the retrieval of directory listings as JSON instead of HTML:
135
136
```console
137
$ export DIR_CID=bafybeigccimv3zqm5g4jt363faybagywkvqbrismoquogimy7kvz2sj7sq
@@ -112,6 +167,8 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq
167
}
168
]
169
}
170
+$ ipfs dag get $DIR_CID
171
+{"Data":{"/":{"bytes":"CAE"}},"Links":[{"Hash":{"/":"Qmc3zqKcwzbbvw3MQm3hXdg8BQoFjGdZiGdAfXAyAGGdLi"},"Name":"1 - Barrel - Part 1 - alt.txt","Tsize":21},{"Hash":{"/":"QmdMxMx29KVYhHnaCc1icWYxQqXwUNCae6t1wS2NqruiHd"},"Name":"1 - Barrel - Part 1 - transcript.txt","Tsize":195},{"Hash":{"/":"QmawceGscqN4o8Y8Fv26UUmB454kn2bnkXV5tEQYc4jBd6"},"Name":"1 - Barrel - Part 1.png","Tsize":24862}]}
172
```
173
174
##### 🐎 Fast directory listings with DAG sizes
test/sharness/t0123-gateway-json-cbor.sh
+121
-127
@@ -12,158 +12,150 @@ test_expect_success "Add the test directory" '
12
mkdir -p rootDir/ipns &&
13
mkdir -p rootDir/api &&
14
mkdir -p rootDir/ą/ę &&
15
+ echo "{ \"test\": \"i am a plain json file\" }" > rootDir/ą/ę/t.json &&
16
echo "I am a txt file on path with utf8" > rootDir/ą/ę/file-źł.txt &&
17
echo "I am a txt file in confusing /api dir" > rootDir/api/file.txt &&
18
echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
19
echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
20
DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
21
+ FILE_JSON_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/t.json | jq -r .Hash) &&
22
FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
23
FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
24
echo "$FILE_CID / $FILE_SIZE"
25
'
26
27
+## Quick regression check for JSON stored on UnixFS:
28
+## it has nothing to do with DAG-JSON and JSON codecs,
29
+## but a lot of JSON data is stored on UnixFS and is requested with or without various hints
30
+## and we want to avoid surprises like https://github.com/protocol/bifrost-infra/issues/2290
31
+test_expect_success "GET UnixFS file with JSON bytes is returned with application/json Content-Type" '
32
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_JSON_CID" > curl_output 2>&1 &&
33
+ curl -sD headers_accept -H "Accept: application/json" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_JSON_CID" > curl_output_accept 2>&1 &&
34
+ ipfs cat $FILE_JSON_CID > ipfs_cat_output 2>&1 &&
35
+ test_should_contain "Content-Type: application/json" headers &&
36
+ test_should_contain "Content-Type: application/json" headers_accept &&
37
+ test_cmp ipfs_cat_output curl_output &&
38
+ test_cmp curl_output curl_output_accept
39
+'
40
+
41
+
42
## Reading UnixFS (data encoded with dag-pb codec) as DAG-CBOR and DAG-JSON
43
+## (returns representation defined in https://ipld.io/specs/codecs/dag-pb/spec/#logical-format)
44
27
-test_dag_pb_headers () {
45
+test_dag_pb_conversion () {
46
name=$1
47
format=$2
48
disposition=$3
49
32
- test_expect_success "GET UnixFS as $name with format=dag-$format has expected Content-Type" '
33
- curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 &&
34
- test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output &&
35
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output &&
36
- test_should_not_contain "Content-Type: application/$format" curl_output
50
+ test_expect_success "GET UnixFS file as $name with format=dag-$format converts to the expected Content-Type" '
51
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 &&
52
+ ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 &&
53
+ test_cmp ipfs_dag_get_output curl_output &&
54
+ test_should_contain "Content-Type: application/vnd.ipld.dag-$format" headers &&
55
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" headers &&
56
+ test_should_not_contain "Content-Type: application/$format" headers
57
'
58
39
- test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' has expected Content-Type" '
59
+ test_expect_success "GET UnixFS directory as $name with format=dag-$format converts to the expected Content-Type" '
60
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 &&
61
+ ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 &&
62
+ test_cmp ipfs_dag_get_output curl_output &&
63
+ test_should_contain "Content-Type: application/vnd.ipld.dag-$format" headers &&
64
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${DIR_CID}.${format}\"" headers &&
65
+ test_should_not_contain "Content-Type: application/$format" headers
66
+ '
67
+
68
+ test_expect_success "GET UnixFS as $name with 'Accept: application/vnd.ipld.dag-$format' converts to the expected Content-Type" '
69
curl -sD - -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 &&
70
test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output &&
71
test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output &&
72
test_should_not_contain "Content-Type: application/$format" curl_output
73
'
74
46
- test_expect_success "GET UnixFS as $name with format=$format has expected Content-Type" '
47
- curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 &&
48
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output &&
49
- test_should_contain "Content-Type: application/$format" curl_output &&
50
- test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output
51
- '
52
-
53
- test_expect_success "GET UnixFS as $name with 'Accept: application/$format' has expected Content-Type" '
54
- curl -sD - -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 &&
55
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${FILE_CID}.${format}\"" curl_output &&
56
- test_should_contain "Content-Type: application/$format" curl_output &&
57
- test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output
58
- '
59
-
60
- test_expect_success "GET UnixFS as $name with 'Accept: foo, application/$format,bar' has expected Content-Type" '
61
- curl -sD - -H "Accept: foo, application/$format,text/plain" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 &&
62
- test_should_contain "Content-Type: application/$format" curl_output
63
- '
64
-}
65
-
66
-test_dag_pb_headers "DAG-JSON" "json" "inline"
67
-test_dag_pb_headers "DAG-CBOR" "cbor" "attachment"
68
-
69
-test_dag_pb () {
70
- name=$1
71
- format=$2
72
-
73
- test_expect_success "GET UnixFS as $name has expected output for file" '
74
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=dag-$format" > curl_output 2>&1 &&
75
- ipfs dag get --output-codec dag-$format $FILE_CID > ipfs_dag_get_output 2>&1 &&
76
- test_cmp ipfs_dag_get_output curl_output
75
+ test_expect_success "GET UnixFS as $name with 'Accept: foo, application/vnd.ipld.dag-$format,bar' converts to the expected Content-Type" '
76
+ curl -sD - -H "Accept: foo, application/vnd.ipld.dag-$format,text/plain" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 &&
77
+ test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output
78
'
79
79
- test_expect_success "GET UnixFS as $name has expected output for directory" '
80
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output 2>&1 &&
81
- ipfs dag get --output-codec dag-$format $DIR_CID > ipfs_dag_get_output 2>&1 &&
82
- test_cmp ipfs_dag_get_output curl_output
80
+ test_expect_success "GET UnixFS with format=$format (not dag-$format) is no-op (no conversion)" '
81
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" > curl_output 2>&1 &&
82
+ ipfs cat $FILE_CID > cat_output &&
83
+ test_cmp cat_output curl_output &&
84
+ test_should_contain "Content-Type: text/plain" headers &&
85
+ test_should_not_contain "Content-Type: application/$format" headers &&
86
+ test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" headers
87
'
88
85
- test_expect_success "GET UnixFS as $name with format=dag-$format and format=$format produce same output" '
86
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=dag-$format" > curl_output_1 2>&1 &&
87
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DIR_CID?format=$format" > curl_output_2 2>&1 &&
88
- test_cmp curl_output_1 curl_output_2
89
+ test_expect_success "GET UnixFS with 'Accept: application/$format' (not dag-$format) is no-op (no conversion)" '
90
+ curl -sD headers -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID" > curl_output 2>&1 &&
91
+ ipfs cat $FILE_CID > cat_output &&
92
+ test_cmp cat_output curl_output &&
93
+ test_should_contain "Content-Type: text/plain" headers &&
94
+ test_should_not_contain "Content-Type: application/$format" headers &&
95
+ test_should_not_contain "Content-Type: application/vnd.ipld.dag-$format" headers
96
'
97
}
98
92
-test_dag_pb "DAG-JSON" "json"
93
-test_dag_pb "DAG-CBOR" "cbor"
99
+test_dag_pb_conversion "DAG-JSON" "json" "inline"
100
+test_dag_pb_conversion "DAG-CBOR" "cbor" "attachment"
101
95
-## Content-Type response based on Accept header and ?format= parameter
102
97
-test_cmp_dag_get () {
103
+# Requesting CID with plain json (0x0200) and cbor (0x51) codecs
104
+# (note these are not UnixFS, not DAG-* variants, just raw block identified by a CID with a special codec)
105
+test_plain_codec () {
106
name=$1
107
format=$2
108
disposition=$3
109
102
- test_expect_success "GET $name without Accept or format= has expected Content-Type" '
103
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
104
- curl -sD - "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 &&
105
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output &&
106
- test_should_contain "Content-Type: application/$format" curl_output
110
+ # no explicit format, just codec in CID
111
+ test_expect_success "GET $name without Accept or format= has expected $format Content-Type and body as-is" '
112
+ CID=$(echo "{ \"test\": \"plain json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
113
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 &&
114
+ ipfs block get $CID > ipfs_block_output 2>&1 &&
115
+ test_cmp ipfs_block_output curl_output &&
116
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" headers &&
117
+ test_should_contain "Content-Type: application/$format" headers
118
'
119
109
- test_expect_success "GET $name without Accept or format= produces correct output" '
110
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
111
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 &&
112
- ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 &&
113
- test_cmp ipfs_dag_get_output curl_output
120
+ # explicit format still gives correct output, just codec in CID
121
+ test_expect_success "GET $name with ?format= has expected $format Content-Type and body as-is" '
122
+ CID=$(echo "{ \"test\": \"plain json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
123
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 &&
124
+ ipfs block get $CID > ipfs_block_output 2>&1 &&
125
+ test_cmp ipfs_block_output curl_output &&
126
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" headers &&
127
+ test_should_contain "Content-Type: application/$format" headers
128
'
129
116
- test_expect_success "GET $name with format=$format produces expected Content-Type" '
117
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
118
- curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 &&
119
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output &&
120
- test_should_contain "Content-Type: application/$format" curl_output
130
+ # explicit format still gives correct output, just codec in CID
131
+ test_expect_success "GET $name with Accept has expected $format Content-Type and body as-is" '
132
+ CID=$(echo "{ \"test\": \"plain json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
133
+ curl -sD headers -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output 2>&1 &&
134
+ ipfs block get $CID > ipfs_block_output 2>&1 &&
135
+ test_cmp ipfs_block_output curl_output &&
136
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" headers &&
137
+ test_should_contain "Content-Type: application/$format" headers
138
'
139
123
- test_expect_success "GET $name with format=$format produces correct output" '
124
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
125
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" > curl_output 2>&1 &&
126
- ipfs dag get --output-codec $format $CID > ipfs_dag_get_output 2>&1 &&
127
- test_cmp ipfs_dag_get_output curl_output
128
- '
129
-
130
- test_expect_success "GET $name with format=dag-$format produces expected Content-Type" '
131
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
132
- curl -sD- "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 &&
133
- test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" curl_output &&
134
- test_should_contain "Content-Type: application/vnd.ipld.dag-$format" curl_output
135
- '
136
-
137
- test_expect_success "GET $name with format=dag-$format produces correct output" '
138
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
139
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output 2>&1 &&
140
+ # explicit dag-* format passed, attempt to parse as dag* variant
141
+ ## Note: this works only for simple JSON that can be upgraded to DAG-JSON.
142
+ test_expect_success "GET $name with format=dag-$format interprets $format as dag-* variant and produces expected Content-Type and body" '
143
+ CID=$(echo "{ \"test\": \"plain-json-that-can-also-be-dag-json\" }" | ipfs dag put --input-codec json --store-codec $format) &&
144
+ curl -sD headers "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" > curl_output_param 2>&1 &&
145
ipfs dag get --output-codec dag-$format $CID > ipfs_dag_get_output 2>&1 &&
141
- test_cmp ipfs_dag_get_output curl_output
146
+ test_cmp ipfs_dag_get_output curl_output_param &&
147
+ test_should_contain "Content-Disposition: ${disposition}\; filename=\"${CID}.${format}\"" headers &&
148
+ test_should_contain "Content-Type: application/vnd.ipld.dag-$format" headers &&
149
+ curl -s -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > curl_output_accept 2>&1 &&
150
+ test_cmp curl_output_param curl_output_accept
151
'
143
-}
144
-
145
-test_cmp_dag_get "JSON" "json" "inline"
146
-test_cmp_dag_get "CBOR" "cbor" "attachment"
147
-
152
149
-## Lossless conversion between JSON and CBOR
150
-
151
-test_expect_success "GET JSON as CBOR produces DAG-CBOR output" '
152
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec json) &&
153
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=cbor" > curl_output 2>&1 &&
154
- ipfs dag get --output-codec dag-cbor $CID > ipfs_dag_get_output 2>&1 &&
155
- test_cmp ipfs_dag_get_output curl_output
156
-'
157
-
158
-test_expect_success "GET CBOR as JSON produces DAG-JSON output" '
159
- CID=$(echo "{ \"test\": \"json\" }" | ipfs dag put --input-codec json --store-codec cbor) &&
160
- curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" > curl_output 2>&1 &&
161
- ipfs dag get --output-codec dag-json $CID > ipfs_dag_get_output 2>&1 &&
162
- test_cmp ipfs_dag_get_output curl_output
163
-'
153
+}
154
155
+test_plain_codec "plain JSON codec" "json" "inline"
156
+test_plain_codec "plain CBOR codec" "cbor" "attachment"
157
166
-## Pathing, traversal
158
+## Pathing, traversal over DAG-JSON and DAG-CBOR
159
160
DAG_CBOR_TRAVERSAL_CID="bafyreibs4utpgbn7uqegmd2goqz4bkyflre2ek2iwv743fhvylwi4zeeim"
161
DAG_JSON_TRAVERSAL_CID="baguqeeram5ujjqrwheyaty3w5gdsmoz6vittchvhk723jjqxk7hakxkd47xq"
@@ -204,17 +196,9 @@ test_expect_success "GET DAG-CBOR traverses multiple links" '
196
test_cmp expected actual
197
'
198
207
-# test_expect_success "GET DAG-PB has expected output" '
208
-# curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_PB_CID?format=dag-json" > curl_output 2>&1 &&
209
-# jq --sort-keys . curl_output > actual &&
210
-# test_cmp ../t0123-gateway-json-cbor/dag-pb.json actual
211
-# '
212
-
213
-
214
-## NATIVE TESTS:
199
+## NATIVE TESTS for DAG-JSON (0x0129) and DAG-CBOR (0x71):
200
## DAG- regression tests for core behaviors when native DAG-(CBOR|JSON) is requested
201
217
-
202
test_native_dag () {
203
name=$1
204
format=$2
@@ -237,10 +221,10 @@ test_native_dag () {
221
test_cmp expected curl_ipfs_dag_param_output
222
'
223
240
- test_expect_success "GET $name from /ipfs with format=$format returns the same payload as format=dag-$format" '
224
+ test_expect_success "GET $name from /ipfs for application/$format returns the same payload as format=dag-$format" '
225
curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o expected &&
242
- curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o curl_ipfs_dag_param_output &&
243
- test_cmp expected curl_ipfs_dag_param_output
226
+ curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" -o plain_output &&
227
+ test_cmp expected plain_output
228
'
229
230
test_expect_success "GET $name from /ipfs with application/vnd.ipld.dag-$format returns the same payload as the raw block" '
@@ -249,6 +233,23 @@ test_native_dag () {
233
test_cmp expected_block curl_ipfs_dag_block_accept_output
234
'
235
236
+ # Make sure DAG-* can be requested as plain JSON or CBOR and response has plain Content-Type for interop purposes
237
+
238
+ test_expect_success "GET $name with format=$format returns same payload as format=dag-$format but with plain Content-Type" '
239
+ curl -s "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-$format" -o expected &&
240
+ curl -sD plain_headers "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=$format" -o plain_output &&
241
+ test_should_contain "Content-Type: application/$format" plain_headers &&
242
+ test_cmp expected plain_output
243
+ '
244
+
245
+ test_expect_success "GET $name with Accept: application/$format returns same payload as application/vnd.ipld.dag-$format but with plain Content-Type" '
246
+ curl -s -H "Accept: application/vnd.ipld.dag-$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > expected &&
247
+ curl -sD plain_headers -H "Accept: application/$format" "http://127.0.0.1:$GWAY_PORT/ipfs/$CID" > plain_output &&
248
+ test_should_contain "Content-Type: application/$format" plain_headers &&
249
+ test_cmp expected plain_output
250
+ '
251
+
252
+
253
# Make sure expected HTTP headers are returned with the dag- block
254
255
test_expect_success "GET response for application/vnd.ipld.dag-$format has expected Content-Type" '
@@ -302,18 +303,11 @@ test_native_dag () {
303
test_should_contain "Content-Type: application/vnd.ipld.dag-$format" output &&
304
test_should_contain "Content-Length: " output
305
'
305
- test_expect_success "HEAD $name with an explicit JSON format returns HTTP 200" '
306
- curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=json" -o output &&
307
- test_should_contain "HTTP/1.1 200 OK" output &&
308
- test_should_contain "Etag: \"$CID.json\"" output &&
309
- test_should_contain "Content-Type: application/json" output &&
310
- test_should_contain "Content-Length: " output
311
- '
312
- test_expect_success "HEAD dag-pb with ?format=$format returns HTTP 200" '
313
- curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$FILE_CID?format=$format" -o output &&
306
+ test_expect_success "HEAD $name with an explicit DAG-JSON format returns HTTP 200" '
307
+ curl -I "http://127.0.0.1:$GWAY_PORT/ipfs/$CID?format=dag-json" -o output &&
308
test_should_contain "HTTP/1.1 200 OK" output &&
315
- test_should_contain "Etag: \"$FILE_CID.$format\"" output &&
316
- test_should_contain "Content-Type: application/$format" output &&
309
+ test_should_contain "Etag: \"$CID.dag-json\"" output &&
310
+ test_should_contain "Content-Type: application/vnd.ipld.dag-json" output &&
311
test_should_contain "Content-Length: " output
312
'
313
test_expect_success "HEAD $name with only-if-cached for missing block returns HTTP 412 Precondition Failed" '