feat(gateway): Gateway.FastDirIndexThreshold (#8853)
* fix(core/gateway): option to limit directory size listing * feat(gw): HTMLDirListingLimit This is alternative take on the way we limit the HTML listing output. Instead of a hard cut-off, we list up to HTMLDirListingLimit. When a directory has more items than HTMLDirListingLimit we show additional header and footer informing user that only $HTMLDirListingLimit items are listed. This is a better UX. * fix: 0 disables Gateway.HTMLDirListingLimit * refactor: Gateway.FastDirIndexThreshold see explainer in docs/config.md * refactor: prealoc slices * docs: Gateway.FastDirIndexThreshold * refactor: core/corehttp/gateway_handler.go https://github.com/ipfs/go-ipfs/pull/8853#discussion_r851437088 * docs: apply suggestions from code review Co-authored-by: Alan Shaw <alan.shaw@protocol.ai> Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Alan Shaw <alan.shaw@protocol.ai>
Lucas Molas committed
Apr 28, 2022 at 14:36 UTC
25cc85fa9359f907f348e0c2139f2b535313c56c
12 files changed
+128
-54
assets/dir-index-html/dir-index.html
+1
-1
@@ -31,7 +31,7 @@
31
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
32
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
33
<div>
34
- <a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
34
+ <a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
35
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
36
</a>
37
</div>
assets/dir-index-html/src/dir-index.html
+1
-1
@@ -30,7 +30,7 @@
30
<div class="menu-item-narrow"><a href="https://ipfs.io" target="_blank" rel="noopener noreferrer">About</a></div>
31
<div class="menu-item-narrow"><a href="https://ipfs.io#install" target="_blank" rel="noopener noreferrer">Install</a></div>
32
<div>
33
- <a href="https://github.com/ipfs/dir-index-html/issues/" target="_blank" rel="noopener noreferrer">
33
+ <a href="https://github.com/ipfs/go-ipfs/issues/new/choose" target="_blank" rel="noopener noreferrer" title="Report a bug">
34
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 18.4 21"><circle cx="7.5" cy="4.8" r="1"/><circle cx="11.1" cy="4.8" r="1"/><path d="M12.7 8.4c-0.5-1.5-1.9-2.5-3.5-2.5 -1.6 0-3 1-3.5 2.5H12.7z"/><path d="M8.5 9.7H5c-0.5 0.8-0.7 1.7-0.7 2.7 0 2.6 1.8 4.8 4.2 5.2V9.7z"/><path d="M13.4 9.7H9.9v7.9c2.4-0.4 4.2-2.5 4.2-5.2C14.1 11.4 13.9 10.5 13.4 9.7z"/><circle cx="15.7" cy="12.9" r="1"/><circle cx="15.1" cy="15.4" r="1"/><circle cx="15.3" cy="10.4" r="1"/><circle cx="2.7" cy="12.9" r="1"/><circle cx="3.3" cy="15.4" r="1"/><circle cx="3.1" cy="10.4" r="1"/></svg>
35
</a>
36
</div>
assets/dir-index-html/test/main.go
+9
-8
@@ -12,14 +12,15 @@ const templateFile = "../dir-index.html"
12
13
// Copied from go-ipfs/core/corehttp/gateway_indexPage.go
14
type listingTemplateData struct {
15
- GatewayURL string
16
- DNSLink bool
17
- Listing []directoryItem
18
- Size string
19
- Path string
20
- Breadcrumbs []breadcrumb
21
- BackLink string
22
- Hash string
15
+ GatewayURL string
16
+ DNSLink bool
17
+ Listing []directoryItem
18
+ Size string
19
+ Path string
20
+ Breadcrumbs []breadcrumb
21
+ BackLink string
22
+ Hash string
23
+ FastDirIndexThreshold int
24
}
25
26
type directoryItem struct {
config/gateway.go
+9
-1
@@ -53,7 +53,15 @@ type Gateway struct {
53
// }
54
PathPrefixes []string
55
56
- // FIXME: Not yet implemented
56
+ // FastDirIndexThreshold is the maximum number of items in a directory
57
+ // before the Gateway switches to a shallow, faster listing which only
58
+ // requires the root node. This allows for listing big directories fast,
59
+ // without the linear slowdown caused by reading size metadata from child
60
+ // nodes.
61
+ // Setting to 0 will enable fast listings for all directories.
62
+ FastDirIndexThreshold *OptionalInteger `json:",omitempty"`
63
+
64
+ // FIXME: Not yet implemented: https://github.com/ipfs/go-ipfs/issues/8059
65
APICommands []string
66
67
// NoFetch configures the gateway to _not_ fetch blocks in response to
core/coreapi/unixfs.go
+1
-1
@@ -302,7 +302,7 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
302
}
303
304
func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.DirEntry, error) {
305
- out := make(chan coreiface.DirEntry)
305
+ out := make(chan coreiface.DirEntry, uio.DefaultShardWidth)
306
307
go func() {
308
defer close(out)
core/corehttp/gateway.go
+8
-6
@@ -16,9 +16,10 @@ import (
16
)
17
18
type GatewayConfig struct {
19
- Headers map[string][]string
20
- Writable bool
21
- PathPrefixes []string
19
+ Headers map[string][]string
20
+ Writable bool
21
+ PathPrefixes []string
22
+ FastDirIndexThreshold int
23
}
24
25
// A helper function to clean up a set of headers:
@@ -89,9 +90,10 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
90
}, headers[ACEHeadersName]...))
91
92
var gateway http.Handler = newGatewayHandler(GatewayConfig{
92
- Headers: headers,
93
- Writable: writable,
94
- PathPrefixes: cfg.Gateway.PathPrefixes,
93
+ Headers: headers,
94
+ Writable: writable,
95
+ PathPrefixes: cfg.Gateway.PathPrefixes,
96
+ FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
97
}, api)
98
99
gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
core/corehttp/gateway_handler_unixfs.go
+2
@@ -18,6 +18,7 @@ import (
18
func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
19
ctx, span := tracing.Span(ctx, "Gateway", "ServeUnixFS", trace.WithAttributes(attribute.String("path", resolvedPath.String())))
20
defer span.End()
21
+
22
// Handling UnixFS
23
dr, err := i.api.Unixfs().Get(ctx, resolvedPath)
24
if err != nil {
@@ -39,6 +40,7 @@ func (i *gatewayHandler) serveUnixFS(ctx context.Context, w http.ResponseWriter,
40
internalWebError(w, fmt.Errorf("unsupported UnixFS type"))
41
return
42
}
43
+
44
logger.Debugw("serving unixfs directory", "path", contentPath)
45
i.serveDirectory(ctx, w, r, resolvedPath, contentPath, dir, begin, logger)
46
}
core/corehttp/gateway_handler_unixfs_dir.go
+39
-27
@@ -15,6 +15,7 @@ import (
15
"github.com/ipfs/go-ipfs/tracing"
16
path "github.com/ipfs/go-path"
17
"github.com/ipfs/go-path/resolver"
18
+ options "github.com/ipfs/interface-go-ipfs-core/options"
19
ipath "github.com/ipfs/interface-go-ipfs-core/path"
20
"go.opentelemetry.io/otel/attribute"
21
"go.opentelemetry.io/otel/trace"
@@ -102,36 +103,46 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
103
return
104
}
105
106
+ // Optimization 1:
107
+ // List children without fetching their root blocks (fast, but no size info)
108
+ results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
109
+ if err != nil {
110
+ internalWebError(w, err)
111
+ return
112
+ }
113
+
114
// storage for directory listing
106
- var dirListing []directoryItem
107
- dirit := dir.Entries()
108
- for dirit.Next() {
109
- size := "?"
110
- if s, err := dirit.Node().Size(); err == nil {
111
- // Size may not be defined/supported. Continue anyways.
112
- size = humanize.Bytes(uint64(s))
113
- }
115
+ dirListing := make([]directoryItem, 0, len(results))
116
115
- resolved, err := i.api.ResolvePath(ctx, ipath.Join(resolvedPath, dirit.Name()))
116
- if err != nil {
117
+ for link := range results {
118
+ if link.Err != nil {
119
internalWebError(w, err)
120
return
121
}
120
- hash := resolved.Cid().String()
121
-
122
- // See comment above where originalUrlPath is declared.
122
+ hash := link.Cid.String()
123
di := directoryItem{
124
- Size: size,
125
- Name: dirit.Name(),
126
- Path: gopath.Join(originalUrlPath, dirit.Name()),
124
+ Size: "", // no size because we did not fetch child nodes
125
+ Name: link.Name,
126
+ Path: gopath.Join(originalUrlPath, link.Name),
127
Hash: hash,
128
ShortHash: shortHash(hash),
129
}
130
dirListing = append(dirListing, di)
131
}
132
- if dirit.Err() != nil {
133
- internalWebError(w, dirit.Err())
134
- return
132
+
133
+ // Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
134
+ if len(dirListing) < i.config.FastDirIndexThreshold {
135
+ dirit := dir.Entries()
136
+ linkNo := 0
137
+ for dirit.Next() {
138
+ size := "?"
139
+ if s, err := dirit.Node().Size(); err == nil {
140
+ // Size may not be defined/supported. Continue anyways.
141
+ size = humanize.Bytes(uint64(s))
142
+ }
143
+ dirListing[linkNo].Size = size
144
+ linkNo++
145
+ }
146
}
147
148
// construct the correct back link
@@ -180,14 +191,15 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
191
192
// See comment above where originalUrlPath is declared.
193
tplData := listingTemplateData{
183
- GatewayURL: gwURL,
184
- DNSLink: dnslink,
185
- Listing: dirListing,
186
- Size: size,
187
- Path: contentPath.String(),
188
- Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
189
- BackLink: backLink,
190
- Hash: hash,
194
+ GatewayURL: gwURL,
195
+ DNSLink: dnslink,
196
+ Listing: dirListing,
197
+ Size: size,
198
+ Path: contentPath.String(),
199
+ Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
200
+ BackLink: backLink,
201
+ Hash: hash,
202
+ FastDirIndexThreshold: i.config.FastDirIndexThreshold,
203
}
204
205
logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
core/corehttp/gateway_indexPage.go
+9
-8
@@ -12,14 +12,15 @@ import (
12
13
// structs for directory listing
14
type listingTemplateData struct {
15
- GatewayURL string
16
- DNSLink bool
17
- Listing []directoryItem
18
- Size string
19
- Path string
20
- Breadcrumbs []breadcrumb
21
- BackLink string
22
- Hash string
15
+ GatewayURL string
16
+ DNSLink bool
17
+ Listing []directoryItem
18
+ Size string
19
+ Path string
20
+ Breadcrumbs []breadcrumb
21
+ BackLink string
22
+ Hash string
23
+ FastDirIndexThreshold int
24
}
25
26
type directoryItem struct {
docs/config.md
+15
@@ -51,6 +51,7 @@ config file at runtime.
51
- [`Gateway.NoDNSLink`](#gatewaynodnslink)
52
- [`Gateway.HTTPHeaders`](#gatewayhttpheaders)
53
- [`Gateway.RootRedirect`](#gatewayrootredirect)
54
+ - [`Gateway.FastDirIndexThreshold`](#gatewayfastdirindexthreshold)
55
- [`Gateway.Writable`](#gatewaywritable)
56
- [`Gateway.PathPrefixes`](#gatewaypathprefixes)
57
- [`Gateway.PublicGateways`](#gatewaypublicgateways)
@@ -646,6 +647,20 @@ Default: `""`
647
648
Type: `string` (url)
649
650
+### `Gateway.FastDirIndexThreshold`
651
+
652
+The maximum number of items in a directory before the Gateway switches
653
+to a shallow, faster listing which only requires the root node.
654
+
655
+This allows for fast listings of big directories, without the linear slowdown caused
656
+by reading size metadata from child nodes.
657
+
658
+Setting to 0 will enable fast listings for all directories.
659
+
660
+Default: `100`
661
+
662
+Type: `optionalInteger`
663
+
664
### `Gateway.Writable`
665
666
A boolean to configure whether the gateway is writeable or not.
test/sharness/lib/test-lib.sh
+9
@@ -388,6 +388,15 @@ test_should_contain() {
388
fi
389
}
390
391
+test_should_not_contain() {
392
+ test "$#" = 2 || error "bug in the test script: not 2 parameters to test_should_not_contain"
393
+ if grep -q "$1" "$2"
394
+ then
395
+ echo "'$2' contains undesired value '$1'"
396
+ return 1
397
+ fi
398
+}
399
+
400
test_str_contains() {
401
find=$1
402
shift
test/sharness/t0115-gateway-dir-listing.sh
+25
-1
@@ -28,7 +28,9 @@ test_expect_success "Add the test directory" '
28
echo "I am a txt file in confusing /ipfs dir" > rootDir/ipfs/file.txt &&
29
echo "I am a txt file in confusing /ipns dir" > rootDir/ipns/file.txt &&
30
DIR_CID=$(ipfs add -Qr --cid-version 1 rootDir) &&
31
- FILE_CID=$(ipfs files stat /ipfs/$DIR_CID/ą/ę/file-źł.txt | head -1)
31
+ FILE_CID=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Hash) &&
32
+ FILE_SIZE=$(ipfs files stat --enc=json /ipfs/$DIR_CID/ą/ę/file-źł.txt | jq -r .Size)
33
+ echo "$FILE_CID / $FILE_SIZE"
34
'
35
36
## ============================================================================
@@ -135,6 +137,28 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.io
137
test_should_contain "<a class=\"ipfs-hash\" translate=\"no\" href=\"https://cid.ipfs.io/#$FILE_CID\" target=\"_blank\" rel=\"noreferrer noopener\">" list_response
138
'
139
140
+## ============================================================================
141
+## Test dir listing of a big directory
142
+## ============================================================================
143
+
144
+test_expect_success "dir listing should resolve child sizes if under Gateway.FastDirIndexThreshold" '
145
+ curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
146
+ test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
147
+ test_should_contain ">${FILE_SIZE} B</td>" list_response
148
+'
149
+
150
+# force fast dir index for all responses
151
+ipfs config --json Gateway.FastDirIndexThreshold 0
152
+# restart daemon to apply config changes
153
+test_kill_ipfs_daemon
154
+test_launch_ipfs_daemon
155
+
156
+test_expect_success "dir listing should not resolve child sizes beyond Gateway.FastDirIndexThreshold" '
157
+ curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
158
+ test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
159
+ test_should_not_contain ">${FILE_SIZE} B</td>" list_response
160
+'
161
+
162
## ============================================================================
163
## End of tests, cleanup
164
## ============================================================================