@cryptotaxi247 / kubo / commits / 4cabdfefb

feat(gateway): Block and CAR response formats (#8758)

* feat: serveRawBlock implements ?format=block * feat: serveCar implements ?format=car * feat(gw): ?format= or Accept HTTP header - extracted file-like content type responses to separate .go files - Accept HTTP header with support for application/vnd.ipld.* types * fix: use .bin for raw block content-disposition .raw may be handled by something, depending on OS, and .bin seems to be universally "binary file" across all systems: https://en.wikipedia.org/wiki/List_of_filename_extensions_(A%E2%80%93E) * refactor: gateway_handler_unixfs.go - Moved UnixFS response handling to gateway_handler_unixfs*.go files. - Removed support for X-Ipfs-Gateway-Prefix (Closes #7702) * refactor: prefix cleanup and readable paths - removed dead code after X-Ipfs-Gateway-Prefix is gone (https://github.com/ipfs/go-ipfs/issues/7702) - escaped special characters in content paths returned with http.Error making them both safer and easier to reason about (e.g. when invisible whitespace Unicode is used)

Marcin Rataj committed Mar 17, 2022 at 17:15 UTC 4cabdfefbf9b5d13e5064cedab37b01af18d78b5
14 files changed +992 -404
core/corehttp/gateway_handler.go
+166 -308
@@ -5,7 +5,6 @@ import (
5 "fmt"
6 "html/template"
7 "io"
8 - "mime"
8 "net/http"
9 "net/url"
10 "os"
@@ -16,11 +15,8 @@ import (
15 "strings"
16 "time"
17
19 - humanize "github.com/dustin/go-humanize"
20 - "github.com/gabriel-vasile/mimetype"
21 - "github.com/ipfs/go-cid"
18 + cid "github.com/ipfs/go-cid"
19 files "github.com/ipfs/go-ipfs-files"
23 - assets "github.com/ipfs/go-ipfs/assets"
20 dag "github.com/ipfs/go-merkledag"
21 mfs "github.com/ipfs/go-mfs"
22 path "github.com/ipfs/go-path"
@@ -32,11 +28,13 @@ import (
28 )
29
30 const (
35 - ipfsPathPrefix = "/ipfs/"
36 - ipnsPathPrefix = "/ipns/"
31 + ipfsPathPrefix = "/ipfs/"
32 + ipnsPathPrefix = "/ipns/"
33 + immutableCacheControl = "public, max-age=29030400, immutable"
34 )
35
36 var onlyAscii = regexp.MustCompile("[[:^ascii:]]")
37 +var noModtime = time.Unix(0, 0) // disables Last-Modified header if passed as modtime
38
39 // HTML-based redirect for errors which can be recovered from, but we want
40 // to provide hint to people that they should fix things on their end.
@@ -89,6 +87,7 @@ func (sw *statusResponseWriter) WriteHeader(code int) {
87
88 func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
89 unixfsGetMetric := prometheus.NewSummaryVec(
90 + // TODO: deprecate and switch to content type agnostic metrics: https://github.com/ipfs/go-ipfs/issues/8441
91 prometheus.SummaryOpts{
92 Namespace: "ipfs",
93 Subsystem: "http",
@@ -196,38 +195,17 @@ func (i *gatewayHandler) optionsHandler(w http.ResponseWriter, r *http.Request)
195
196 func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request) {
197 begin := time.Now()
199 - urlPath := r.URL.Path
200 - escapedURLPath := r.URL.EscapedPath()
198
199 logger := log.With("from", r.RequestURI)
200 logger.Debug("http request received")
201
205 - // If the gateway is behind a reverse proxy and mounted at a sub-path,
206 - // the prefix header can be set to signal this sub-path.
207 - // It will be prepended to links in directory listings and the index.html redirect.
208 - // TODO: this feature is deprecated and will be removed (https://github.com/ipfs/go-ipfs/issues/7702)
209 - prefix := ""
210 - if prfx := r.Header.Get("X-Ipfs-Gateway-Prefix"); len(prfx) > 0 {
211 - for _, p := range i.config.PathPrefixes {
212 - if prfx == p || strings.HasPrefix(prfx, p+"/") {
213 - prefix = prfx
214 - break
215 - }
216 - }
217 - logger.Debugw("sub-path (deprecrated)", "prefix", prefix)
218 - }
219 -
220 - // HostnameOption might have constructed an IPNS/IPFS path using the Host header.
221 - // In this case, we need the original path for constructing redirects
222 - // and links that match the requested URL.
223 - // For example, http://example.net would become /ipns/example.net, and
224 - // the redirects and links would end up as http://example.net/ipns/example.net
225 - requestURI, err := url.ParseRequestURI(r.RequestURI)
226 - if err != nil {
227 - webError(w, "failed to parse request path", err, http.StatusInternalServerError)
202 + // X-Ipfs-Gateway-Prefix was removed (https://github.com/ipfs/go-ipfs/issues/7702)
203 + // TODO: remove this after go-ipfs 0.13 ships
204 + if prfx := r.Header.Get("X-Ipfs-Gateway-Prefix"); prfx != "" {
205 + err := fmt.Errorf("X-Ipfs-Gateway-Prefix support was removed: https://github.com/ipfs/go-ipfs/issues/7702")
206 + webError(w, "unsupported HTTP header", err, http.StatusBadRequest)
207 return
208 }
230 - originalUrlPath := prefix + requestURI.Path
209
210 // ?uri query param support for requests produced by web browsers
211 // via navigator.registerProtocolHandler Web API
@@ -248,7 +226,7 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
226 path = path + "?" + u.RawQuery
227 }
228
251 - redirectURL := gopath.Join("/", prefix, u.Scheme, u.Host, path)
229 + redirectURL := gopath.Join("/", u.Scheme, u.Host, path)
230 logger.Debugw("uri param, redirect", "to", redirectURL, "status", http.StatusMovedPermanently)
231 http.Redirect(w, r, redirectURL, http.StatusMovedPermanently)
232 return
@@ -266,9 +244,9 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
244 }
245 }
246
269 - parsedPath := ipath.New(urlPath)
270 - if pathErr := parsedPath.IsValid(); pathErr != nil {
271 - if prefix == "" && fixupSuperfluousNamespace(w, urlPath, r.URL.RawQuery) {
247 + contentPath := ipath.New(r.URL.Path)
248 + if pathErr := contentPath.IsValid(); pathErr != nil {
249 + if fixupSuperfluousNamespace(w, r.URL.Path, r.URL.RawQuery) {
250 // the error was due to redundant namespace, which we were able to fix
251 // by returning error/redirect page, nothing left to do here
252 logger.Debugw("redundant namespace; noop")
@@ -280,304 +258,75 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
258 }
259
260 // Resolve path to the final DAG node for the ETag
283 - resolvedPath, err := i.api.ResolvePath(r.Context(), parsedPath)
261 + resolvedPath, err := i.api.ResolvePath(r.Context(), contentPath)
262 switch err {
263 case nil:
264 case coreiface.ErrOffline:
287 - webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusServiceUnavailable)
265 + webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusServiceUnavailable)
266 return
267 default:
290 - if i.servePretty404IfPresent(w, r, parsedPath) {
268 + // if Accept is text/html, see if ipfs-404.html is present
269 + if i.servePretty404IfPresent(w, r, contentPath) {
270 logger.Debugw("serve pretty 404 if present")
271 return
272 }
273
295 - webError(w, "ipfs resolve -r "+escapedURLPath, err, http.StatusNotFound)
296 - return
297 - }
298 -
299 - dr, err := i.api.Unixfs().Get(r.Context(), resolvedPath)
300 - if err != nil {
301 - webError(w, "ipfs cat "+escapedURLPath, err, http.StatusNotFound)
274 + webError(w, "ipfs resolve -r "+debugStr(contentPath.String()), err, http.StatusNotFound)
275 return
276 }
277
305 - i.unixfsGetMetric.WithLabelValues(parsedPath.Namespace()).Observe(time.Since(begin).Seconds())
306 -
307 - defer dr.Close()
308 -
309 - var responseEtag string
278 + // Detect when explicit Accept header or ?format parameter are present
279 + responseFormat := customResponseFormat(r)
280
311 - // we need to figure out whether this is a directory before doing most of the heavy lifting below
312 - _, ok := dr.(files.Directory)
313 -
314 - if ok && assets.BindataVersionHash != "" {
315 - responseEtag = `"DirIndex-` + assets.BindataVersionHash + `_CID-` + resolvedPath.Cid().String() + `"`
316 - } else {
317 - responseEtag = `"` + resolvedPath.Cid().String() + `"`
281 + // Finish early if client already has matching Etag
282 + if r.Header.Get("If-None-Match") == getEtag(r, resolvedPath.Cid()) {
283 + w.WriteHeader(http.StatusNotModified)
284 + return
285 }
286
320 - // Check etag sent back to us
321 - if r.Header.Get("If-None-Match") == responseEtag || r.Header.Get("If-None-Match") == `W/`+responseEtag {
322 - w.WriteHeader(http.StatusNotModified)
287 + // Update the global metric of the time it takes to read the final root block of the requested resource
288 + // NOTE: for legacy reasons this happens before we go into content-type specific code paths
289 + _, err = i.api.Block().Get(r.Context(), resolvedPath)
290 + if err != nil {
291 + webError(w, "ipfs block get "+resolvedPath.Cid().String(), err, http.StatusInternalServerError)
292 return
293 }
294 + i.unixfsGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
295
296 + // HTTP Headers
297 i.addUserHeaders(w) // ok, _now_ write user's headers.
327 - w.Header().Set("X-IPFS-Path", urlPath)
328 - w.Header().Set("Etag", responseEtag)
298 + w.Header().Set("X-Ipfs-Path", contentPath.String())
299
330 - if rootCids, err := i.buildIpfsRootsHeader(urlPath, r); err == nil {
300 + if rootCids, err := i.buildIpfsRootsHeader(contentPath.String(), r); err == nil {
301 w.Header().Set("X-Ipfs-Roots", rootCids)
332 - } else { // this should never happen, as we resolved the urlPath already
302 + } else { // this should never happen, as we resolved the contentPath already
303 webError(w, "error while resolving X-Ipfs-Roots", err, http.StatusInternalServerError)
304 return
305 }
306
337 - // set these headers _after_ the error, for we may just not have it
338 - // and don't want the client to cache a 500 response...
339 - // and only if it's /ipfs!
340 - // TODO: break this out when we split /ipfs /ipns routes.
341 - modtime := time.Now()
342 -
343 - if f, ok := dr.(files.File); ok {
344 - if strings.HasPrefix(urlPath, ipfsPathPrefix) {
345 - w.Header().Set("Cache-Control", "public, max-age=29030400, immutable")
346 -
347 - // set modtime to a really long time ago, since files are immutable and should stay cached
348 - modtime = time.Unix(1, 0)
349 - }
350 -
351 - urlFilename := r.URL.Query().Get("filename")
352 - var name string
353 - if urlFilename != "" {
354 - disposition := "inline"
355 - if r.URL.Query().Get("download") == "true" {
356 - disposition = "attachment"
357 - }
358 - utf8Name := url.PathEscape(urlFilename)
359 - asciiName := url.PathEscape(onlyAscii.ReplaceAllLiteralString(urlFilename, "_"))
360 - w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, asciiName, utf8Name))
361 - name = urlFilename
362 - } else {
363 - name = getFilename(urlPath)
364 - }
365 -
366 - logger.Debugw("serving file", "name", name)
367 - i.serveFile(w, r, name, modtime, f)
307 + // Support custom response formats passed via ?format or Accept HTTP header
308 + switch responseFormat {
309 + case "": // The implicit response format is UnixFS
310 + logger.Debugw("serving unixfs", "path", contentPath)
311 + i.serveUnixFs(w, r, resolvedPath, contentPath, logger)
312 return
369 - }
370 - dir, ok := dr.(files.Directory)
371 - if !ok {
372 - internalWebError(w, fmt.Errorf("unsupported file type"))
313 + case "application/vnd.ipld.raw":
314 + logger.Debugw("serving raw block", "path", contentPath)
315 + i.serveRawBlock(w, r, resolvedPath.Cid(), contentPath)
316 return
374 - }
375 -
376 - idxPath := ipath.Join(resolvedPath, "index.html")
377 - idx, err := i.api.Unixfs().Get(r.Context(), idxPath)
378 - switch err.(type) {
379 - case nil:
380 - dirwithoutslash := urlPath[len(urlPath)-1] != '/'
381 - goget := r.URL.Query().Get("go-get") == "1"
382 - if dirwithoutslash && !goget {
383 - // See comment above where originalUrlPath is declared.
384 - suffix := "/"
385 - if r.URL.RawQuery != "" {
386 - // preserve query parameters
387 - suffix = suffix + "?" + r.URL.RawQuery
388 - }
389 -
390 - redirectURL := originalUrlPath + suffix
391 - logger.Debugw("serving index.html file", "to", redirectURL, "status", http.StatusFound, "path", idxPath)
392 - http.Redirect(w, r, redirectURL, http.StatusFound)
393 - return
394 - }
395 -
396 - f, ok := idx.(files.File)
397 - if !ok {
398 - internalWebError(w, files.ErrNotReader)
399 - return
400 - }
401 - // static index.html → no need to generate dynamic dir-index-html
402 - // replace mutable DirIndex Etag with immutable dir CID
403 - w.Header().Set("Etag", `"`+resolvedPath.Cid().String()+`"`)
404 -
405 - logger.Debugw("serving index.html file", "path", idxPath)
406 - // write to request
407 - i.serveFile(w, r, "index.html", modtime, f)
408 - return
409 - case resolver.ErrNoLink:
410 - logger.Debugw("no index.html; noop", "path", idxPath)
411 - default:
412 - internalWebError(w, err)
413 - return
414 - }
415 -
416 - // See statusResponseWriter.WriteHeader
417 - // and https://github.com/ipfs/go-ipfs/issues/7164
418 - // Note: this needs to occur before listingTemplate.Execute otherwise we get
419 - // superfluous response.WriteHeader call from prometheus/client_golang
420 - if w.Header().Get("Location") != "" {
421 - logger.Debugw("location moved permanently", "status", http.StatusMovedPermanently)
422 - w.WriteHeader(http.StatusMovedPermanently)
317 + case "application/vnd.ipld.car", "application/vnd.ipld.car; version=1":
318 + logger.Debugw("serving car stream", "path", contentPath)
319 + i.serveCar(w, r, resolvedPath.Cid(), contentPath)
320 return
424 - }
425 -
426 - // A HTML directory index will be presented, be sure to set the correct
427 - // type instead of relying on autodetection (which may fail).
428 - w.Header().Set("Content-Type", "text/html")
429 - if r.Method == http.MethodHead {
430 - logger.Debug("return as request's HTTP method is HEAD")
321 + default: // catch-all for unsuported application/vnd.*
322 + err := fmt.Errorf("unsupported format %q", responseFormat)
323 + webError(w, "failed respond with requested content type", err, http.StatusBadRequest)
324 return
325 }
433 -
434 - // storage for directory listing
435 - var dirListing []directoryItem
436 - dirit := dir.Entries()
437 - for dirit.Next() {
438 - size := "?"
439 - if s, err := dirit.Node().Size(); err == nil {
440 - // Size may not be defined/supported. Continue anyways.
441 - size = humanize.Bytes(uint64(s))
442 - }
443 -
444 - resolved, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name()))
445 - if err != nil {
446 - internalWebError(w, err)
447 - return
448 - }
449 - hash := resolved.Cid().String()
450 -
451 - // See comment above where originalUrlPath is declared.
452 - di := directoryItem{
453 - Size: size,
454 - Name: dirit.Name(),
455 - Path: gopath.Join(originalUrlPath, dirit.Name()),
456 - Hash: hash,
457 - ShortHash: shortHash(hash),
458 - }
459 - dirListing = append(dirListing, di)
460 - }
461 - if dirit.Err() != nil {
462 - internalWebError(w, dirit.Err())
463 - return
464 - }
465 -
466 - // construct the correct back link
467 - // https://github.com/ipfs/go-ipfs/issues/1365
468 - var backLink string = originalUrlPath
469 -
470 - // don't go further up than /ipfs/$hash/
471 - pathSplit := path.SplitList(urlPath)
472 - switch {
473 - // keep backlink
474 - case len(pathSplit) == 3: // url: /ipfs/$hash
475 -
476 - // keep backlink
477 - case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/
478 -
479 - // add the correct link depending on whether the path ends with a slash
480 - default:
481 - if strings.HasSuffix(backLink, "/") {
482 - backLink += "./.."
483 - } else {
484 - backLink += "/.."
485 - }
486 - }
487 -
488 - size := "?"
489 - if s, err := dir.Size(); err == nil {
490 - // Size may not be defined/supported. Continue anyways.
491 - size = humanize.Bytes(uint64(s))
492 - }
493 -
494 - hash := resolvedPath.Cid().String()
495 -
496 - // Gateway root URL to be used when linking to other rootIDs.
497 - // This will be blank unless subdomain or DNSLink resolution is being used
498 - // for this request.
499 - var gwURL string
500 -
501 - // Get gateway hostname and build gateway URL.
502 - if h, ok := r.Context().Value("gw-hostname").(string); ok {
503 - gwURL = "//" + h
504 - } else {
505 - gwURL = ""
506 - }
507 -
508 - dnslink := hasDNSLinkOrigin(gwURL, urlPath)
509 -
510 - // See comment above where originalUrlPath is declared.
511 - tplData := listingTemplateData{
512 - GatewayURL: gwURL,
513 - DNSLink: dnslink,
514 - Listing: dirListing,
515 - Size: size,
516 - Path: urlPath,
517 - Breadcrumbs: breadcrumbs(urlPath, dnslink),
518 - BackLink: backLink,
519 - Hash: hash,
520 - }
521 -
522 - logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash, "duration", time.Since(begin))
523 -
524 - if err := listingTemplate.Execute(w, tplData); err != nil {
525 - internalWebError(w, err)
526 - return
527 - }
528 -}
529 -
530 -func (i *gatewayHandler) serveFile(w http.ResponseWriter, req *http.Request, name string, modtime time.Time, file files.File) {
531 - size, err := file.Size()
532 - if err != nil {
533 - http.Error(w, "cannot serve files with unknown sizes", http.StatusBadGateway)
534 - return
535 - }
536 -
537 - content := &lazySeeker{
538 - size: size,
539 - reader: file,
540 - }
541 -
542 - var ctype string
543 - if _, isSymlink := file.(*files.Symlink); isSymlink {
544 - // We should be smarter about resolving symlinks but this is the
545 - // "most correct" we can be without doing that.
546 - ctype = "inode/symlink"
547 - } else {
548 - ctype = mime.TypeByExtension(gopath.Ext(name))
549 - if ctype == "" {
550 - // uses https://github.com/gabriel-vasile/mimetype library to determine the content type.
551 - // Fixes https://github.com/ipfs/go-ipfs/issues/7252
552 - mimeType, err := mimetype.DetectReader(content)
553 - if err != nil {
554 - http.Error(w, fmt.Sprintf("cannot detect content-type: %s", err.Error()), http.StatusInternalServerError)
555 - return
556 - }
557 -
558 - ctype = mimeType.String()
559 - _, err = content.Seek(0, io.SeekStart)
560 - if err != nil {
561 - http.Error(w, "seeker can't seek", http.StatusInternalServerError)
562 - return
563 - }
564 - }
565 - // Strip the encoding from the HTML Content-Type header and let the
566 - // browser figure it out.
567 - //
568 - // Fixes https://github.com/ipfs/go-ipfs/issues/2203
569 - if strings.HasPrefix(ctype, "text/html;") {
570 - ctype = "text/html"
571 - }
572 - }
573 - w.Header().Set("Content-Type", ctype)
574 -
575 - w = &statusResponseWriter{w}
576 - http.ServeContent(w, req, name, modtime, content)
326 }
327
579 -func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, parsedPath ipath.Path) bool {
580 - resolved404Path, ctype, err := i.searchUpTreeFor404(r, parsedPath)
328 +func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) bool {
329 + resolved404Path, ctype, err := i.searchUpTreeFor404(r, contentPath)
330 if err != nil {
331 return false
332 }
@@ -598,7 +347,7 @@ func (i *gatewayHandler) servePretty404IfPresent(w http.ResponseWriter, r *http.
347 return false
348 }
349
601 - log.Debugw("using pretty 404 file", "path", parsedPath)
350 + log.Debugw("using pretty 404 file", "path", contentPath)
351 w.Header().Set("Content-Type", ctype)
352 w.Header().Set("Content-Length", strconv.FormatInt(size, 10))
353 w.WriteHeader(http.StatusNotFound)
@@ -795,6 +544,67 @@ func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) {
544 }
545 }
546
547 +func addCacheControlHeaders(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid) (modtime time.Time) {
548 + // Set Etag to based on CID (override whatever was set before)
549 + w.Header().Set("Etag", getEtag(r, fileCid))
550 +
551 + // Set Cache-Control and Last-Modified based on contentPath properties
552 + if contentPath.Mutable() {
553 + // mutable namespaces such as /ipns/ can't be cached forever
554 +
555 + /* For now we set Last-Modified to Now() to leverage caching heuristics built into modern browsers:
556 + * https://github.com/ipfs/go-ipfs/pull/8074#pullrequestreview-645196768
557 + * but we should not set it to fake values and use Cache-Control based on TTL instead */
558 + modtime = time.Now()
559 +
560 + // TODO: set Cache-Control based on TTL of IPNS/DNSLink: https://github.com/ipfs/go-ipfs/issues/1818#issuecomment-1015849462
561 + // TODO: set Last-Modified based on /ipns/ publishing timestamp?
562 +
563 + } else {
564 + // immutable! CACHE ALL THE THINGS, FOREVER! wolololol
565 + w.Header().Set("Cache-Control", immutableCacheControl)
566 +
567 + // Set modtime to 'zero time' to disable Last-Modified header (superseded by Cache-Control)
568 + modtime = noModtime
569 +
570 + // TODO: set Last-Modified? - TBD - /ipfs/ modification metadata is present in unixfs 1.5 https://github.com/ipfs/go-ipfs/issues/6920?
571 + }
572 +
573 + return modtime
574 +}
575 +
576 +// Set Content-Disposition if filename URL query param is present, return preferred filename
577 +func addContentDispositionHeader(w http.ResponseWriter, r *http.Request, contentPath ipath.Path) string {
578 + /* This logic enables:
579 + * - creation of HTML links that trigger "Save As.." dialog instead of being rendered by the browser
580 + * - overriding the filename used when saving subresource assets on HTML page
581 + * - providing a default filename for HTTP clients when downloading direct /ipfs/CID without any subpath
582 + */
583 +
584 + // URL param ?filename=cat.jpg triggers Content-Disposition: [..] filename
585 + // which impacts default name used in "Save As.." dialog
586 + name := getFilename(contentPath)
587 + urlFilename := r.URL.Query().Get("filename")
588 + if urlFilename != "" {
589 + disposition := "inline"
590 + // URL param ?download=true triggers Content-Disposition: [..] attachment
591 + // which skips rendering and forces "Save As.." dialog in browsers
592 + if r.URL.Query().Get("download") == "true" {
593 + disposition = "attachment"
594 + }
595 + setContentDispositionHeader(w, urlFilename, disposition)
596 + name = urlFilename
597 + }
598 + return name
599 +}
600 +
601 +// Set Content-Disposition to arbitrary filename and disposition
602 +func setContentDispositionHeader(w http.ResponseWriter, filename string, disposition string) {
603 + utf8Name := url.PathEscape(filename)
604 + asciiName := url.PathEscape(onlyAscii.ReplaceAllLiteralString(filename, "_"))
605 + w.Header().Set("Content-Disposition", fmt.Sprintf("%s; filename=\"%s\"; filename*=UTF-8''%s", disposition, asciiName, utf8Name))
606 +}
607 +
608 // Set X-Ipfs-Roots with logical CID array for efficient HTTP cache invalidation.
609 func (i *gatewayHandler) buildIpfsRootsHeader(contentPath string, r *http.Request) (string, error) {
610 /*
@@ -854,7 +664,7 @@ func webError(w http.ResponseWriter, message string, err error, defaultCode int)
664 func webErrorWithCode(w http.ResponseWriter, message string, err error, code int) {
665 http.Error(w, fmt.Sprintf("%s: %s", message, err), code)
666 if code >= 500 {
857 - log.Warnf("server error: %s: %s", err)
667 + log.Warnf("server error: %s: %s", message, err)
668 }
669 }
670
@@ -863,7 +673,8 @@ func internalWebError(w http.ResponseWriter, err error) {
673 webErrorWithCode(w, "internalWebError", err, http.StatusInternalServerError)
674 }
675
866 -func getFilename(s string) string {
676 +func getFilename(contentPath ipath.Path) string {
677 + s := contentPath.String()
678 if (strings.HasPrefix(s, ipfsPathPrefix) || strings.HasPrefix(s, ipnsPathPrefix)) && strings.Count(gopath.Clean(s), "/") <= 2 {
679 // Don't want to treat ipfs.io in /ipns/ipfs.io as a filename.
680 return ""
@@ -871,13 +682,51 @@ func getFilename(s string) string {
682 return gopath.Base(s)
683 }
684
874 -func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, parsedPath ipath.Path) (ipath.Resolved, string, error) {
685 +// generate Etag value based on HTTP request and CID
686 +func getEtag(r *http.Request, cid cid.Cid) string {
687 + prefix := `"`
688 + suffix := `"`
689 + responseFormat := customResponseFormat(r)
690 + if responseFormat != "" {
691 + // application/vnd.ipld.foo → foo
692 + f := responseFormat[strings.LastIndex(responseFormat, ".")+1:]
693 + // Etag: "cid.foo" (gives us nice compression together with Content-Disposition in block (raw) and car responses)
694 + suffix = `.` + f + suffix
695 + }
696 + // TODO: include selector suffix when https://github.com/ipfs/go-ipfs/issues/8769 lands
697 + return prefix + cid.String() + suffix
698 +}
699 +
700 +// return explicit response format if specified in request as query parameter or via Accept HTTP header
701 +func customResponseFormat(r *http.Request) string {
702 + if formatParam := r.URL.Query().Get("format"); formatParam != "" {
703 + // translate query param to a content type
704 + switch formatParam {
705 + case "raw":
706 + return "application/vnd.ipld.raw"
707 + case "car":
708 + return "application/vnd.ipld.car"
709 + }
710 + }
711 + // Browsers and other user agents will send Accept header with generic types like:
712 + // Accept:text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8
713 + // We only care about explciit, vendor-specific content-types.
714 + for _, accept := range r.Header.Values("Accept") {
715 + // respond to the very first ipld content type
716 + if strings.HasPrefix(accept, "application/vnd.ipld") {
717 + return accept
718 + }
719 + }
720 + return ""
721 +}
722 +
723 +func (i *gatewayHandler) searchUpTreeFor404(r *http.Request, contentPath ipath.Path) (ipath.Resolved, string, error) {
724 filename404, ctype, err := preferred404Filename(r.Header.Values("Accept"))
725 if err != nil {
726 return nil, "", err
727 }
728
880 - pathComponents := strings.Split(parsedPath.String(), "/")
729 + pathComponents := strings.Split(contentPath.String(), "/")
730
731 for idx := len(pathComponents); idx >= 3; idx-- {
732 pretty404 := gopath.Join(append(pathComponents[0:idx], filename404)...)
@@ -913,6 +762,15 @@ func preferred404Filename(acceptHeaders []string) (string, string, error) {
762 return "", "", fmt.Errorf("there is no 404 file for the requested content types")
763 }
764
765 +// returns unquoted path with all special characters revealed as \u codes
766 +func debugStr(path string) string {
767 + q := fmt.Sprintf("%+q", path)
768 + if len(q) >= 3 {
769 + q = q[1 : len(q)-1]
770 + }
771 + return q
772 +}
773 +
774 // Attempt to fix redundant /ipfs/ namespace as long as resulting
775 // 'intended' path is valid. This is in case gremlins were tickled
776 // wrong way and user ended up at /ipfs/ipfs/{cid} or /ipfs/ipns/{id}
core/corehttp/gateway_handler_block.go new
+38
@@ -0,0 +1,38 @@
1 +package corehttp
2 +
3 +import (
4 + "bytes"
5 + "io/ioutil"
6 + "net/http"
7 +
8 + cid "github.com/ipfs/go-cid"
9 + ipath "github.com/ipfs/interface-go-ipfs-core/path"
10 +)
11 +
12 +// serveRawBlock returns bytes behind a raw block
13 +func (i *gatewayHandler) serveRawBlock(w http.ResponseWriter, r *http.Request, blockCid cid.Cid, contentPath ipath.Path) {
14 + blockReader, err := i.api.Block().Get(r.Context(), contentPath)
15 + if err != nil {
16 + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError)
17 + return
18 + }
19 + block, err := ioutil.ReadAll(blockReader)
20 + if err != nil {
21 + webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError)
22 + return
23 + }
24 + content := bytes.NewReader(block)
25 +
26 + // Set Content-Disposition
27 + name := blockCid.String() + ".bin"
28 + setContentDispositionHeader(w, name, "attachment")
29 +
30 + // Set remaining headers
31 + modtime := addCacheControlHeaders(w, r, contentPath, blockCid)
32 + w.Header().Set("Content-Type", "application/vnd.ipld.raw")
33 + w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
34 +
35 + // Done: http.ServeContent will take care of
36 + // If-None-Match+Etag, Content-Length and range requests
37 + http.ServeContent(w, r, name, modtime, content)
38 +}
core/corehttp/gateway_handler_car.go new
+72
@@ -0,0 +1,72 @@
1 +package corehttp
2 +
3 +import (
4 + "context"
5 + "net/http"
6 +
7 + blocks "github.com/ipfs/go-block-format"
8 + cid "github.com/ipfs/go-cid"
9 + coreiface "github.com/ipfs/interface-go-ipfs-core"
10 + ipath "github.com/ipfs/interface-go-ipfs-core/path"
11 + gocar "github.com/ipld/go-car"
12 + selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
13 +)
14 +
15 +// serveCar returns a CAR stream for specific DAG+selector
16 +func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCid cid.Cid, contentPath ipath.Path) {
17 + ctx, cancel := context.WithCancel(r.Context())
18 + defer cancel()
19 +
20 + // Set Content-Disposition
21 + name := rootCid.String() + ".car"
22 + setContentDispositionHeader(w, name, "attachment")
23 +
24 + // Weak Etag W/ because we can't guarantee byte-for-byte identical responses
25 + // (CAR is streamed, and in theory, blocks may arrive from datastore in non-deterministic order)
26 + etag := `W/` + getEtag(r, rootCid)
27 + w.Header().Set("Etag", etag)
28 +
29 + // Finish early if Etag match
30 + if r.Header.Get("If-None-Match") == etag {
31 + w.WriteHeader(http.StatusNotModified)
32 + return
33 + }
34 +
35 + // Make it clear we don't support range-requests over a car stream
36 + // Partial downloads and resumes should be handled using
37 + // IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
38 + w.Header().Set("Accept-Ranges", "none")
39 +
40 + // Explicit Cache-Control to ensure fresh stream on retry.
41 + // CAR stream could be interrupted, and client should be able to resume and get full response, not the truncated one
42 + w.Header().Set("Cache-Control", "no-cache, no-transform")
43 +
44 + w.Header().Set("Content-Type", "application/vnd.ipld.car; version=1")
45 + w.Header().Set("X-Content-Type-Options", "nosniff") // no funny business in the browsers :^)
46 +
47 + // Same go-car settings as dag.export command
48 + store := dagStore{dag: i.api.Dag(), ctx: ctx}
49 +
50 + // TODO: support selectors passed as request param: https://github.com/ipfs/go-ipfs/issues/8769
51 + dag := gocar.Dag{Root: rootCid, Selector: selectorparse.CommonSelector_ExploreAllRecursively}
52 + car := gocar.NewSelectiveCar(ctx, store, []gocar.Dag{dag}, gocar.TraverseLinksOnlyOnce())
53 +
54 + if err := car.Write(w); err != nil {
55 + // We return error as a trailer, however it is not something browsers can access
56 + // (https://github.com/mdn/browser-compat-data/issues/14703)
57 + // Due to this, we suggest client always verify that
58 + // the received CAR stream response is matching requested DAG selector
59 + w.Header().Set("X-Stream-Error", err.Error())
60 + return
61 + }
62 +}
63 +
64 +type dagStore struct {
65 + dag coreiface.APIDagService
66 + ctx context.Context
67 +}
68 +
69 +func (ds dagStore) Get(c cid.Cid) (blocks.Block, error) {
70 + obj, err := ds.dag.Get(ds.ctx, c)
71 + return obj, err
72 +}
core/corehttp/gateway_handler_unixfs.go new
+37
@@ -0,0 +1,37 @@
1 +package corehttp
2 +
3 +import (
4 + "fmt"
5 + "html"
6 + "net/http"
7 +
8 + files "github.com/ipfs/go-ipfs-files"
9 + ipath "github.com/ipfs/interface-go-ipfs-core/path"
10 + "go.uber.org/zap"
11 +)
12 +
13 +func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) {
14 + // Handling UnixFS
15 + dr, err := i.api.Unixfs().Get(r.Context(), resolvedPath)
16 + if err != nil {
17 + webError(w, "ipfs cat "+html.EscapeString(contentPath.String()), err, http.StatusNotFound)
18 + return
19 + }
20 + defer dr.Close()
21 +
22 + // Handling Unixfs file
23 + if f, ok := dr.(files.File); ok {
24 + logger.Debugw("serving unixfs file", "path", contentPath)
25 + i.serveFile(w, r, contentPath, resolvedPath.Cid(), f)
26 + return
27 + }
28 +
29 + // Handling Unixfs directory
30 + dir, ok := dr.(files.Directory)
31 + if !ok {
32 + internalWebError(w, fmt.Errorf("unsupported UnixFs type"))
33 + return
34 + }
35 + logger.Debugw("serving unixfs directory", "path", contentPath)
36 + i.serveDirectory(w, r, resolvedPath, contentPath, dir, logger)
37 +}
core/corehttp/gateway_handler_unixfs_dir.go new
+197
@@ -0,0 +1,197 @@
1 +package corehttp
2 +
3 +import (
4 + "net/http"
5 + "net/url"
6 + gopath "path"
7 + "strings"
8 +
9 + "github.com/dustin/go-humanize"
10 + files "github.com/ipfs/go-ipfs-files"
11 + "github.com/ipfs/go-ipfs/assets"
12 + path "github.com/ipfs/go-path"
13 + "github.com/ipfs/go-path/resolver"
14 + ipath "github.com/ipfs/interface-go-ipfs-core/path"
15 + "go.uber.org/zap"
16 +)
17 +
18 +// serveDirectory returns the best representation of UnixFS directory
19 +//
20 +// It will return index.html if present, or generate directory listing otherwise.
21 +func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, logger *zap.SugaredLogger) {
22 +
23 + // HostnameOption might have constructed an IPNS/IPFS path using the Host header.
24 + // In this case, we need the original path for constructing redirects
25 + // and links that match the requested URL.
26 + // For example, http://example.net would become /ipns/example.net, and
27 + // the redirects and links would end up as http://example.net/ipns/example.net
28 + requestURI, err := url.ParseRequestURI(r.RequestURI)
29 + if err != nil {
30 + webError(w, "failed to parse request path", err, http.StatusInternalServerError)
31 + return
32 + }
33 + originalUrlPath := requestURI.Path
34 +
35 + // Check if directory has index.html, if so, serveFile
36 + idxPath := ipath.Join(resolvedPath, "index.html")
37 + idx, err := i.api.Unixfs().Get(r.Context(), idxPath)
38 + switch err.(type) {
39 + case nil:
40 + cpath := contentPath.String()
41 + dirwithoutslash := cpath[len(cpath)-1] != '/'
42 + goget := r.URL.Query().Get("go-get") == "1"
43 + if dirwithoutslash && !goget {
44 + // See comment above where originalUrlPath is declared.
45 + suffix := "/"
46 + if r.URL.RawQuery != "" {
47 + // preserve query parameters
48 + suffix = suffix + "?" + r.URL.RawQuery
49 + }
50 +
51 + redirectURL := originalUrlPath + suffix
52 + logger.Debugw("serving index.html file", "to", redirectURL, "status", http.StatusFound, "path", idxPath)
53 + http.Redirect(w, r, redirectURL, http.StatusFound)
54 + return
55 + }
56 +
57 + f, ok := idx.(files.File)
58 + if !ok {
59 + internalWebError(w, files.ErrNotReader)
60 + return
61 + }
62 +
63 + logger.Debugw("serving index.html file", "path", idxPath)
64 + // write to request
65 + i.serveFile(w, r, idxPath, resolvedPath.Cid(), f)
66 + return
67 + case resolver.ErrNoLink:
68 + logger.Debugw("no index.html; noop", "path", idxPath)
69 + default:
70 + internalWebError(w, err)
71 + return
72 + }
73 +
74 + // See statusResponseWriter.WriteHeader
75 + // and https://github.com/ipfs/go-ipfs/issues/7164
76 + // Note: this needs to occur before listingTemplate.Execute otherwise we get
77 + // superfluous response.WriteHeader call from prometheus/client_golang
78 + if w.Header().Get("Location") != "" {
79 + logger.Debugw("location moved permanently", "status", http.StatusMovedPermanently)
80 + w.WriteHeader(http.StatusMovedPermanently)
81 + return
82 + }
83 +
84 + // A HTML directory index will be presented, be sure to set the correct
85 + // type instead of relying on autodetection (which may fail).
86 + w.Header().Set("Content-Type", "text/html")
87 +
88 + // Generated dir index requires custom Etag (it may change between go-ipfs versions)
89 + if assets.BindataVersionHash != "" {
90 + dirEtag := `"DirIndex-` + assets.BindataVersionHash + `_CID-` + resolvedPath.Cid().String() + `"`
91 + w.Header().Set("Etag", dirEtag)
92 + if r.Header.Get("If-None-Match") == dirEtag {
93 + w.WriteHeader(http.StatusNotModified)
94 + return
95 + }
96 + }
97 +
98 + if r.Method == http.MethodHead {
99 + logger.Debug("return as request's HTTP method is HEAD")
100 + return
101 + }
102 +
103 + // storage for directory listing
104 + var dirListing []directoryItem
105 + dirit := dir.Entries()
106 + for dirit.Next() {
107 + size := "?"
108 + if s, err := dirit.Node().Size(); err == nil {
109 + // Size may not be defined/supported. Continue anyways.
110 + size = humanize.Bytes(uint64(s))
111 + }
112 +
113 + resolved, err := i.api.ResolvePath(r.Context(), ipath.Join(resolvedPath, dirit.Name()))
114 + if err != nil {
115 + internalWebError(w, err)
116 + return
117 + }
118 + hash := resolved.Cid().String()
119 +
120 + // See comment above where originalUrlPath is declared.
121 + di := directoryItem{
122 + Size: size,
123 + Name: dirit.Name(),
124 + Path: gopath.Join(originalUrlPath, dirit.Name()),
125 + Hash: hash,
126 + ShortHash: shortHash(hash),
127 + }
128 + dirListing = append(dirListing, di)
129 + }
130 + if dirit.Err() != nil {
131 + internalWebError(w, dirit.Err())
132 + return
133 + }
134 +
135 + // construct the correct back link
136 + // https://github.com/ipfs/go-ipfs/issues/1365
137 + var backLink string = originalUrlPath
138 +
139 + // don't go further up than /ipfs/$hash/
140 + pathSplit := path.SplitList(contentPath.String())
141 + switch {
142 + // keep backlink
143 + case len(pathSplit) == 3: // url: /ipfs/$hash
144 +
145 + // keep backlink
146 + case len(pathSplit) == 4 && pathSplit[3] == "": // url: /ipfs/$hash/
147 +
148 + // add the correct link depending on whether the path ends with a slash
149 + default:
150 + if strings.HasSuffix(backLink, "/") {
151 + backLink += "./.."
152 + } else {
153 + backLink += "/.."
154 + }
155 + }
156 +
157 + size := "?"
158 + if s, err := dir.Size(); err == nil {
159 + // Size may not be defined/supported. Continue anyways.
160 + size = humanize.Bytes(uint64(s))
161 + }
162 +
163 + hash := resolvedPath.Cid().String()
164 +
165 + // Gateway root URL to be used when linking to other rootIDs.
166 + // This will be blank unless subdomain or DNSLink resolution is being used
167 + // for this request.
168 + var gwURL string
169 +
170 + // Get gateway hostname and build gateway URL.
171 + if h, ok := r.Context().Value("gw-hostname").(string); ok {
172 + gwURL = "//" + h
173 + } else {
174 + gwURL = ""
175 + }
176 +
177 + dnslink := hasDNSLinkOrigin(gwURL, contentPath.String())
178 +
179 + // See comment above where originalUrlPath is declared.
180 + tplData := listingTemplateData{
181 + GatewayURL: gwURL,
182 + DNSLink: dnslink,
183 + Listing: dirListing,
184 + Size: size,
185 + Path: contentPath.String(),
186 + Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
187 + BackLink: backLink,
188 + Hash: hash,
189 + }
190 +
191 + logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
192 +
193 + if err := listingTemplate.Execute(w, tplData); err != nil {
194 + internalWebError(w, err)
195 + return
196 + }
197 +}
core/corehttp/gateway_handler_unixfs_file.go new
+83
@@ -0,0 +1,83 @@
1 +package corehttp
2 +
3 +import (
4 + "fmt"
5 + "io"
6 + "mime"
7 + "net/http"
8 + gopath "path"
9 + "strings"
10 +
11 + "github.com/gabriel-vasile/mimetype"
12 + cid "github.com/ipfs/go-cid"
13 + files "github.com/ipfs/go-ipfs-files"
14 + ipath "github.com/ipfs/interface-go-ipfs-core/path"
15 +)
16 +
17 +// serveFile returns data behind a file along with HTTP headers based on
18 +// the file itself, its CID and the contentPath used for accessing it.
19 +func (i *gatewayHandler) serveFile(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid, file files.File) {
20 +
21 + // Set Cache-Control and read optional Last-Modified time
22 + modtime := addCacheControlHeaders(w, r, contentPath, fileCid)
23 +
24 + // Set Content-Disposition
25 + name := addContentDispositionHeader(w, r, contentPath)
26 +
27 + // Prepare size value for Content-Length HTTP header (set inside of http.ServeContent)
28 + size, err := file.Size()
29 + if err != nil {
30 + http.Error(w, "cannot serve files with unknown sizes", http.StatusBadGateway)
31 + return
32 + }
33 +
34 + // Lazy seeker enables efficient range-requests and HTTP HEAD responses
35 + content := &lazySeeker{
36 + size: size,
37 + reader: file,
38 + }
39 +
40 + // Calculate deterministic value for Content-Type HTTP header
41 + // (we prefer to do it here, rather than using implicit sniffing in http.ServeContent)
42 + var ctype string
43 + if _, isSymlink := file.(*files.Symlink); isSymlink {
44 + // We should be smarter about resolving symlinks but this is the
45 + // "most correct" we can be without doing that.
46 + ctype = "inode/symlink"
47 + } else {
48 + ctype = mime.TypeByExtension(gopath.Ext(name))
49 + if ctype == "" {
50 + // uses https://github.com/gabriel-vasile/mimetype library to determine the content type.
51 + // Fixes https://github.com/ipfs/go-ipfs/issues/7252
52 + mimeType, err := mimetype.DetectReader(content)
53 + if err != nil {
54 + http.Error(w, fmt.Sprintf("cannot detect content-type: %s", err.Error()), http.StatusInternalServerError)
55 + return
56 + }
57 +
58 + ctype = mimeType.String()
59 + _, err = content.Seek(0, io.SeekStart)
60 + if err != nil {
61 + http.Error(w, "seeker can't seek", http.StatusInternalServerError)
62 + return
63 + }
64 + }
65 + // Strip the encoding from the HTML Content-Type header and let the
66 + // browser figure it out.
67 + //
68 + // Fixes https://github.com/ipfs/go-ipfs/issues/2203
69 + if strings.HasPrefix(ctype, "text/html;") {
70 + ctype = "text/html"
71 + }
72 + }
73 + // Setting explicit Content-Type to avoid mime-type sniffing on the client
74 + // (unifies behavior across gateways and web browsers)
75 + w.Header().Set("Content-Type", ctype)
76 +
77 + // special fixup around redirects
78 + w = &statusResponseWriter{w}
79 +
80 + // Done: http.ServeContent will take care of
81 + // If-None-Match+Etag, Content-Length and range requests
82 + http.ServeContent(w, r, name, modtime, content)
83 +}
core/corehttp/gateway_test.go
+3 -87
@@ -126,12 +126,6 @@ func newTestServerAndNode(t *testing.T, ns mockNamesys) (*httptest.Server, iface
126 t.Fatal(err)
127 }
128
129 - cfg, err := n.Repo.Config()
130 - if err != nil {
131 - t.Fatal(err)
132 - }
133 - cfg.Gateway.PathPrefixes = []string{"/good-prefix"}
134 -
129 // need this variable here since we need to construct handler with
130 // listener, and server with handler. yay cycles.
131 dh := &delegatedHandler{}
@@ -242,7 +236,7 @@ func TestGatewayGet(t *testing.T) {
236 {"127.0.0.1:8080", "/" + k.Cid().String(), http.StatusNotFound, "404 page not found\n"},
237 {"127.0.0.1:8080", k.String(), http.StatusOK, "fnord"},
238 {"127.0.0.1:8080", "/ipns/nxdomain.example.com", http.StatusNotFound, "ipfs resolve -r /ipns/nxdomain.example.com: " + namesys.ErrResolveFailed.Error() + "\n"},
245 - {"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/%0D%0A%0D%0Ahello: " + namesys.ErrResolveFailed.Error() + "\n"},
239 + {"127.0.0.1:8080", "/ipns/%0D%0A%0D%0Ahello", http.StatusNotFound, "ipfs resolve -r /ipns/\\r\\n\\r\\nhello: " + namesys.ErrResolveFailed.Error() + "\n"},
240 {"127.0.0.1:8080", "/ipns/example.com", http.StatusOK, "fnord"},
241 {"example.com", "/", http.StatusOK, "fnord"},
242
@@ -403,7 +397,6 @@ func TestIPNSHostnameRedirect(t *testing.T) {
397 t.Fatal(err)
398 }
399 req.Host = "example.net"
406 - req.Header.Set("X-Ipfs-Gateway-Prefix", "/good-prefix")
400
401 res, err = doWithoutRedirect(req)
402 if err != nil {
@@ -417,8 +410,8 @@ func TestIPNSHostnameRedirect(t *testing.T) {
410 hdr = res.Header["Location"]
411 if len(hdr) < 1 {
412 t.Errorf("location header not present")
420 - } else if hdr[0] != "/good-prefix/foo/" {
421 - t.Errorf("location header is %v, expected /good-prefix/foo/", hdr[0])
413 + } else if hdr[0] != "/foo/" {
414 + t.Errorf("location header is %v, expected /foo/", hdr[0])
415 }
416
417 // make sure /version isn't exposed
@@ -427,7 +420,6 @@ func TestIPNSHostnameRedirect(t *testing.T) {
420 t.Fatal(err)
421 }
422 req.Host = "example.net"
430 - req.Header.Set("X-Ipfs-Gateway-Prefix", "/good-prefix")
423
424 res, err = doWithoutRedirect(req)
425 if err != nil {
@@ -583,82 +575,6 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
575 if !strings.Contains(s, k3.Cid().String()) {
576 t.Fatalf("expected hash in directory listing")
577 }
586 -
587 - // make request to directory listing with prefix
588 - req, err = http.NewRequest(http.MethodGet, ts.URL, nil)
589 - if err != nil {
590 - t.Fatal(err)
591 - }
592 - req.Host = "example.net"
593 - req.Header.Set("X-Ipfs-Gateway-Prefix", "/good-prefix")
594 -
595 - res, err = doWithoutRedirect(req)
596 - if err != nil {
597 - t.Fatal(err)
598 - }
599 -
600 - // expect correct backlinks with prefix
601 - body, err = ioutil.ReadAll(res.Body)
602 - if err != nil {
603 - t.Fatalf("error reading response: %s", err)
604 - }
605 - s = string(body)
606 - t.Logf("body: %s\n", string(body))
607 -
608 - if !matchPathOrBreadcrumbs(s, "/ipns/<a href=\"//example.net/\">example.net</a>") {
609 - t.Fatalf("expected a path in directory listing")
610 - }
611 - if !strings.Contains(s, "<a href=\"/good-prefix/\">") {
612 - t.Fatalf("expected backlink in directory listing")
613 - }
614 - if !strings.Contains(s, "<a href=\"/good-prefix/file.txt\">") {
615 - t.Fatalf("expected file in directory listing")
616 - }
617 - if !strings.Contains(s, k.Cid().String()) {
618 - t.Fatalf("expected hash in directory listing")
619 - }
620 -
621 - // make request to directory listing with illegal prefix
622 - req, err = http.NewRequest(http.MethodGet, ts.URL, nil)
623 - if err != nil {
624 - t.Fatal(err)
625 - }
626 - req.Host = "example.net"
627 - req.Header.Set("X-Ipfs-Gateway-Prefix", "/bad-prefix")
628 -
629 - // make request to directory listing with evil prefix
630 - req, err = http.NewRequest(http.MethodGet, ts.URL, nil)
631 - if err != nil {
632 - t.Fatal(err)
633 - }
634 - req.Host = "example.net"
635 - req.Header.Set("X-Ipfs-Gateway-Prefix", "//good-prefix/foo")
636 -
637 - res, err = doWithoutRedirect(req)
638 - if err != nil {
639 - t.Fatal(err)
640 - }
641 -
642 - // expect correct backlinks without illegal prefix
643 - body, err = ioutil.ReadAll(res.Body)
644 - if err != nil {
645 - t.Fatalf("error reading response: %s", err)
646 - }
647 - s = string(body)
648 - t.Logf("body: %s\n", string(body))
649 -
650 - if !matchPathOrBreadcrumbs(s, "/") {
651 - t.Fatalf("expected a path in directory listing")
652 - }
653 - if !strings.Contains(s, "<a href=\"/\">") {
654 - t.Fatalf("expected backlink in directory listing")
655 - }
656 - if !strings.Contains(s, "<a href=\"/file.txt\">") {
657 - t.Fatalf("expected file in directory listing")
658 - }
659 - if !strings.Contains(s, k.Cid().String()) {
660 - t.Fatalf("expected hash in directory listing")
661 - }
578 }
579
580 func TestCacheControlImmutable(t *testing.T) {
docs/gateway.md
+28 -9
@@ -65,17 +65,36 @@ images, audio, video, PDF) and trigger immediate "save as" dialog by appending
65
66 > https://ipfs.io/ipfs/QmfM2r8seH2GiRaC4esTjeraXEachRt8ZsSeGaWTPLyMoG?filename=hello_world.txt&download=true
67
68 -## MIME-Types
68 +## Response Format
69
70 -TODO
70 +An explicit response format can be requested using `?format=raw|car|..` URL parameter,
71 +or by sending `Accept: application/vnd.ipld.{format}` HTTP header with one of supported content types.
72
72 -## Read-Only API
73 +## Content-Types
74
74 -For convenience, the gateway exposes a read-only API. This read-only API exposes
75 -a read-only, "safe" subset of the normal API.
75 +### `application/vnd.ipld.raw`
76
77 -For example, you use this to download a block:
77 +Returns a byte array for a single `raw` block.
78
79 -```
80 -> curl https://ipfs.io/api/v0/block/get/bafkreifjjcie6lypi6ny7amxnfftagclbuxndqonfipmb64f2km2devei4
81 -```
79 +Sending such requests for `/ipfs/{cid}` allows for efficient fetch of blocks with data
80 +encoded in custom format, without the need for deserialization and traversal on the gateway.
81 +
82 +This is equivalent of `ipfs block get`.
83 +
84 +### `application/vnd.ipld.car`
85 +
86 +Returns a [CAR](https://ipld.io/specs/transport/car/) stream for specific DAG and selector.
87 +
88 +Right now only 'full DAG' implicit selector is implemented.
89 +Support for user-provided IPLD selectors is tracked in https://github.com/ipfs/go-ipfs/issues/8769.
90 +
91 +This is a rough equivalent of `ipfs dag export`.
92 +
93 +## Deprecated Subset of RPC API
94 +
95 +For legacy reasons, the gateway port exposes a small subset of RPC API under `/api/v0/`.
96 +While this read-only API exposes a read-only, "safe" subset of the normal API,
97 +it is deprecated and should not be used for greenfield projects.
98 +
99 +Where possible, leverage `/ipfs/` and `/ipns/` endpoints.
100 +along with `application/vnd.ipld.*` Content-Types instead.
test/sharness/lib/test-lib.sh
+13
@@ -520,3 +520,16 @@ findprovs_expect() {
520 test_cmp findprovsOut expected
521 '
522 }
523 +
524 +purge_blockstore() {
525 + ipfs pin ls --quiet --type=recursive | ipfs pin rm &>/dev/null
526 + ipfs repo gc --silent &>/dev/null
527 +
528 + test_expect_success "pinlist empty" '
529 + [[ -z "$( ipfs pin ls )" ]]
530 + '
531 + test_expect_success "nothing left to gc" '
532 + [[ -z "$( ipfs repo gc )" ]]
533 + '
534 +}
535 +
test/sharness/t0117-gateway-block.sh new
+70
@@ -0,0 +1,70 @@
1 +#!/usr/bin/env bash
2 +
3 +test_description="Test HTTP Gateway Raw Block (application/vnd.ipld.raw) Support"
4 +
5 +. lib/test-lib.sh
6 +
7 +test_init_ipfs
8 +test_launch_ipfs_daemon_without_network
9 +
10 +test_expect_success "Create text fixtures" '
11 + mkdir -p dir &&
12 + echo "hello application/vnd.ipld.raw" > dir/ascii.txt &&
13 + ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 dir) &&
14 + FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/dir/ascii.txt | cut -d "/" -f3)
15 +'
16 +
17 +# GET unixfs dir root block and compare it with `ipfs block get` output
18 +
19 + test_expect_success "GET with format=raw param returns a raw block" '
20 + ipfs block get "/ipfs/$ROOT_DIR_CID/dir" > expected &&
21 + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir?format=raw" -o curl_ipfs_dir_block_param_output &&
22 + test_cmp expected curl_ipfs_dir_block_param_output
23 + '
24 +
25 + test_expect_success "GET for application/vnd.ipld.raw returns a raw block" '
26 + ipfs block get "/ipfs/$ROOT_DIR_CID/dir" > expected_block &&
27 + curl -sX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir" -o curl_ipfs_dir_block_accept_output &&
28 + test_cmp expected_block curl_ipfs_dir_block_accept_output
29 + '
30 +
31 +# Make sure expected HTTP headers are returned with the block bytes
32 +
33 + test_expect_success "GET response for application/vnd.ipld.raw has expected Content-Type" '
34 + curl -svX GET -H "Accept: application/vnd.ipld.raw" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/dir/ascii.txt" >/dev/null 2>curl_output &&
35 + cat curl_output &&
36 + grep "< Content-Type: application/vnd.ipld.raw" curl_output
37 + '
38 +
39 + test_expect_success "GET response for application/vnd.ipld.raw includes Content-Length" '
40 + BYTES=$(ipfs block get $FILE_CID | wc --bytes)
41 + grep "< Content-Length: $BYTES" curl_output
42 + '
43 +
44 + test_expect_success "GET response for application/vnd.ipld.raw includes Content-Disposition" '
45 + grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.bin\"" curl_output
46 + '
47 +
48 + test_expect_success "GET response for application/vnd.ipld.raw includes nosniff hint" '
49 + grep "< X-Content-Type-Options: nosniff" curl_output
50 + '
51 +
52 +# Cache control HTTP headers
53 +# (basic checks, detailed behavior is tested in t0116-gateway-cache.sh)
54 +
55 + test_expect_success "GET response for application/vnd.ipld.raw includes Etag" '
56 + grep "< Etag: \"${FILE_CID}.raw\"" curl_output
57 + '
58 +
59 + test_expect_success "GET response for application/vnd.ipld.raw includes X-Ipfs-Path and X-Ipfs-Roots" '
60 + grep "< X-Ipfs-Path" curl_output &&
61 + grep "< X-Ipfs-Roots" curl_output
62 + '
63 +
64 + test_expect_success "GET response for application/vnd.ipld.raw includes Cache-Control" '
65 + grep "< Cache-Control" curl_output
66 + '
67 +
68 +test_kill_ipfs_daemon
69 +
70 +test_done
test/sharness/t0118-gateway-car.sh new
+116
@@ -0,0 +1,116 @@
1 +#!/usr/bin/env bash
2 +
3 +test_description="Test HTTP Gateway CAR (application/vnd.ipld.car) Support"
4 +
5 +. lib/test-lib.sh
6 +
7 +test_init_ipfs
8 +test_launch_ipfs_daemon_without_network
9 +
10 +# CAR stream is not deterministic, as blocks can arrive in random order,
11 +# but if we have a small file that fits into a single block, and export its CID
12 +# we will get a CAR that is a deterministic array of bytes.
13 +
14 + test_expect_success "Create a deterministic CAR for testing" '
15 + mkdir -p subdir &&
16 + echo "hello application/vnd.ipld.car" > subdir/ascii.txt &&
17 + ROOT_DIR_CID=$(ipfs add -Qrw --cid-version 1 subdir) &&
18 + FILE_CID=$(ipfs resolve -r /ipfs/$ROOT_DIR_CID/subdir/ascii.txt | cut -d "/" -f3) &&
19 + ipfs dag export $ROOT_DIR_CID > test-dag.car &&
20 + ipfs dag export $FILE_CID > deterministic.car &&
21 + purge_blockstore
22 + '
23 +
24 +# GET a reference DAG with dag-cbor+dag-pb+raw blocks as CAR
25 +
26 + # This test uses official CARv1 fixture from https://ipld.io/specs/transport/car/fixture/carv1-basic/
27 + test_expect_success "GET for application/vnd.ipld.car with dag-cbor root returns a CARv1 stream with full DAG" '
28 + ipfs dag import ../t0118-gateway-car/carv1-basic.car &&
29 + DAG_CBOR_CID=bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm &&
30 + curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$DAG_CBOR_CID" -o gateway-dag-cbor.car &&
31 + purge_blockstore &&
32 + ipfs dag import gateway-dag-cbor.car &&
33 + ipfs dag stat --offline $DAG_CBOR_CID
34 + '
35 +
36 +# GET unixfs file as CAR
37 +# (by using a single file we ensure deterministic result that can be compared byte-for-byte)
38 +
39 + test_expect_success "GET with format=car param returns a CARv1 stream" '
40 + ipfs dag import test-dag.car &&
41 + curl -sX GET "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt?format=car" -o gateway-param.car &&
42 + test_cmp deterministic.car gateway-param.car
43 + '
44 +
45 + test_expect_success "GET for application/vnd.ipld.car returns a CARv1 stream" '
46 + ipfs dag import test-dag.car &&
47 + curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header.car &&
48 + test_cmp deterministic.car gateway-header.car
49 + '
50 +
51 + # explicit version=1
52 + test_expect_success "GET for application/vnd.ipld.raw version=1 returns a CARv1 stream" '
53 + ipfs dag import test-dag.car &&
54 + curl -sX GET -H "Accept: application/vnd.ipld.car; version=1" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" -o gateway-header-v1.car &&
55 + test_cmp deterministic.car gateway-header-v1.car
56 + '
57 +
58 +# GET unixfs directory as a CAR with DAG and some selector
59 +
60 + # TODO: this is basic test for "full" selector, we will add support for custom ones in https://github.com/ipfs/go-ipfs/issues/8769
61 + test_expect_success "GET for application/vnd.ipld.car with unixfs dir returns a CARv1 stream with full DAG" '
62 + ipfs dag import test-dag.car &&
63 + curl -sX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID" -o gateway-dir.car &&
64 + purge_blockstore &&
65 + ipfs dag import gateway-dir.car &&
66 + ipfs dag stat --offline $ROOT_DIR_CID
67 + '
68 +
69 +# Make sure expected HTTP headers are returned with the CAR bytes
70 +
71 + test_expect_success "GET response for application/vnd.ipld.car has expected Content-Type" '
72 + ipfs dag import test-dag.car &&
73 + curl -svX GET -H "Accept: application/vnd.ipld.car" "http://127.0.0.1:$GWAY_PORT/ipfs/$ROOT_DIR_CID/subdir/ascii.txt" >/dev/null 2>curl_output &&
74 + cat curl_output &&
75 + grep "< Content-Type: application/vnd.ipld.car; version=1" curl_output
76 + '
77 +
78 + # CAR is streamed, gateway may not have the entire thing, unable to calculate total size
79 + test_expect_success "GET response for application/vnd.ipld.car includes no Content-Length" '
80 + grep -qv "< Content-Length:" curl_output
81 + '
82 +
83 + test_expect_success "GET response for application/vnd.ipld.car includes Content-Disposition" '
84 + grep "< Content-Disposition: attachment\; filename=\"${FILE_CID}.car\"" curl_output
85 + '
86 +
87 + test_expect_success "GET response for application/vnd.ipld.car includes nosniff hint" '
88 + grep "< X-Content-Type-Options: nosniff" curl_output
89 + '
90 +
91 + # CAR is streamed, gateway may not have the entire thing, unable to support range-requests
92 + # Partial downloads and resumes should be handled using
93 + # IPLD selectors: https://github.com/ipfs/go-ipfs/issues/8769
94 + test_expect_success "GET response for application/vnd.ipld.car includes Accept-Ranges header" '
95 + grep "< Accept-Ranges: none" curl_output
96 + '
97 +
98 +# Cache control HTTP headers
99 +
100 + test_expect_success "GET response for application/vnd.ipld.car includes a weak Etag" '
101 + grep "< Etag: W/\"${FILE_CID}.car\"" curl_output
102 + '
103 +
104 + # (basic checks, detailed behavior for some fields is tested in t0116-gateway-cache.sh)
105 + test_expect_success "GET response for application/vnd.ipld.car includes X-Ipfs-Path and X-Ipfs-Roots" '
106 + grep "< X-Ipfs-Path" curl_output &&
107 + grep "< X-Ipfs-Roots" curl_output
108 + '
109 +
110 + test_expect_success "GET response for application/vnd.ipld.car includes expected Cache-Control" '
111 + grep "< Cache-Control: no-cache, no-transform" curl_output
112 + '
113 +
114 +test_kill_ipfs_daemon
115 +
116 +test_done
test/sharness/t0118-gateway-car/README.md new
+10
@@ -0,0 +1,10 @@
1 +# Dataset description/sources
2 +
3 +- carv1-basic.car
4 + - raw CARv1
5 + - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.car
6 +
7 +- carv1-basic.json
8 + - description of the contents and layout of the raw CAR, encoded in DAG-JSON
9 + - Source: https://ipld.io/specs/transport/car/fixture/carv1-basic/carv1-basic.json
10 +
test/sharness/t0118-gateway-car/carv1-basic.car
Binary files /dev/null and b/test/sharness/t0118-gateway-car/carv1-basic.car differ
test/sharness/t0118-gateway-car/carv1-basic.json new
+159
@@ -0,0 +1,159 @@
1 +{
2 + "blocks": [
3 + {
4 + "blockLength": 55,
5 + "blockOffset": 137,
6 + "cid": {
7 + "/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
8 + },
9 + "content": {
10 + "link": {
11 + "/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
12 + },
13 + "name": "blip"
14 + },
15 + "length": 92,
16 + "offset": 100
17 + },
18 + {
19 + "blockLength": 97,
20 + "blockOffset": 228,
21 + "cid": {
22 + "/": "QmNX6Tffavsya4xgBi2VJQnSuqy9GsxongxZZ9uZBqp16d"
23 + },
24 + "content": {
25 + "Links": [
26 + {
27 + "Hash": {
28 + "/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
29 + },
30 + "Name": "bear",
31 + "Tsize": 4
32 + },
33 + {
34 + "Hash": {
35 + "/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
36 + },
37 + "Name": "second",
38 + "Tsize": 149
39 + }
40 + ]
41 + },
42 + "length": 133,
43 + "offset": 192
44 + },
45 + {
46 + "blockLength": 4,
47 + "blockOffset": 362,
48 + "cid": {
49 + "/": "bafkreifw7plhl6mofk6sfvhnfh64qmkq73oeqwl6sloru6rehaoujituke"
50 + },
51 + "content": {
52 + "/": {
53 + "bytes": "Y2NjYw"
54 + }
55 + },
56 + "length": 41,
57 + "offset": 325
58 + },
59 + {
60 + "blockLength": 94,
61 + "blockOffset": 402,
62 + "cid": {
63 + "/": "QmWXZxVQ9yZfhQxLD35eDR8LiMRsYtHxYqTFCBbJoiJVys"
64 + },
65 + "content": {
66 + "Links": [
67 + {
68 + "Hash": {
69 + "/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
70 + },
71 + "Name": "dog",
72 + "Tsize": 4
73 + },
74 + {
75 + "Hash": {
76 + "/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
77 + },
78 + "Name": "first",
79 + "Tsize": 51
80 + }
81 + ]
82 + },
83 + "length": 130,
84 + "offset": 366
85 + },
86 + {
87 + "blockLength": 4,
88 + "blockOffset": 533,
89 + "cid": {
90 + "/": "bafkreiebzrnroamgos2adnbpgw5apo3z4iishhbdx77gldnbk57d4zdio4"
91 + },
92 + "content": {
93 + "/": {
94 + "bytes": "YmJiYg"
95 + }
96 + },
97 + "length": 41,
98 + "offset": 496
99 + },
100 + {
101 + "blockLength": 47,
102 + "blockOffset": 572,
103 + "cid": {
104 + "/": "QmdwjhxpxzcMsR3qUuj7vUL8pbA7MgR3GAxWi2GLHjsKCT"
105 + },
106 + "content": {
107 + "Links": [
108 + {
109 + "Hash": {
110 + "/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
111 + },
112 + "Name": "cat",
113 + "Tsize": 4
114 + }
115 + ]
116 + },
117 + "length": 82,
118 + "offset": 537
119 + },
120 + {
121 + "blockLength": 4,
122 + "blockOffset": 656,
123 + "cid": {
124 + "/": "bafkreidbxzk2ryxwwtqxem4l3xyyjvw35yu4tcct4cqeqxwo47zhxgxqwq"
125 + },
126 + "content": {
127 + "/": {
128 + "bytes": "YWFhYQ"
129 + }
130 + },
131 + "length": 41,
132 + "offset": 619
133 + },
134 + {
135 + "blockLength": 18,
136 + "blockOffset": 697,
137 + "cid": {
138 + "/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
139 + },
140 + "content": {
141 + "link": null,
142 + "name": "limbo"
143 + },
144 + "length": 55,
145 + "offset": 660
146 + }
147 + ],
148 + "header": {
149 + "roots": [
150 + {
151 + "/": "bafyreihyrpefhacm6kkp4ql6j6udakdit7g3dmkzfriqfykhjw6cad5lrm"
152 + },
153 + {
154 + "/": "bafyreidj5idub6mapiupjwjsyyxhyhedxycv4vihfsicm2vt46o7morwlm"
155 + }
156 + ],
157 + "version": 1
158 + }
159 +}