@cryptotaxi247 / kubo / commits / 7bdb34113

feat: fast directory listings with DAG Size column (#9481)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Henrique Dias committed Dec 12, 2022 at 21:45 UTC 7bdb341132533ef6857849587a8c5cf87e5e6e3c
15 files changed +85 -126
assets/dir-index-html/dir-index.html
+2 -2
@@ -56,7 +56,7 @@
56 </div>
57 {{ if .Size }}
58 <div class="no-linebreak flex-shrink-1 ml-auto">
59 - <strong>&nbsp;{{ .Size }}</strong>
59 + <strong title="Cumulative size of IPFS DAG (data + metadata)">&nbsp;{{ .Size }}</strong>
60 </div>
61 {{ end }}
62 </div>
@@ -89,7 +89,7 @@
89 </a>
90 {{ end }}
91 </td>
92 - <td class="no-linebreak">{{ .Size }}</td>
92 + <td class="no-linebreak" title="Cumulative size of IPFS DAG (data + metadata)">{{ .Size }}</td>
93 </tr>
94 {{ end }}
95 </table>
assets/dir-index-html/src/dir-index.html
+2 -2
@@ -55,7 +55,7 @@
55 </div>
56 {{ if .Size }}
57 <div class="no-linebreak flex-shrink-1 ml-auto">
58 - <strong>&nbsp;{{ .Size }}</strong>
58 + <strong title="Cumulative size of IPFS DAG (data + metadata)">&nbsp;{{ .Size }}</strong>
59 </div>
60 {{ end }}
61 </div>
@@ -88,7 +88,7 @@
88 </a>
89 {{ end }}
90 </td>
91 - <td class="no-linebreak">{{ .Size }}</td>
91 + <td class="no-linebreak" title="Cumulative size of IPFS DAG (data + metadata)">{{ .Size }}</td>
92 </tr>
93 {{ end }}
94 </table>
assets/dir-index-html/test/main.go
+8 -9
@@ -12,15 +12,14 @@ const templateFile = "../dir-index.html"
12
13 // Copied from go-ipfs/core/corehttp/gateway_indexPage.go
14 type listingTemplateData struct {
15 - GatewayURL string
16 - DNSLink bool
17 - Listing []directoryItem
18 - Size string
19 - Path string
20 - Breadcrumbs []breadcrumb
21 - BackLink string
22 - Hash string
23 - FastDirIndexThreshold int
15 + GatewayURL string
16 + DNSLink bool
17 + Listing []directoryItem
18 + Size string
19 + Path string
20 + Breadcrumbs []breadcrumb
21 + BackLink string
22 + Hash string
23 }
24
25 type directoryItem struct {
config/gateway.go
-8
@@ -45,14 +45,6 @@ type Gateway struct {
45 // PathPrefixes was removed: https://github.com/ipfs/go-ipfs/issues/7702
46 PathPrefixes []string
47
48 - // FastDirIndexThreshold is the maximum number of items in a directory
49 - // before the Gateway switches to a shallow, faster listing which only
50 - // requires the root node. This allows for listing big directories fast,
51 - // without the linear slowdown caused by reading size metadata from child
52 - // nodes.
53 - // Setting to 0 will enable fast listings for all directories.
54 - FastDirIndexThreshold *OptionalInteger `json:",omitempty"`
55 -
48 // FIXME: Not yet implemented: https://github.com/ipfs/kubo/issues/8059
49 APICommands []string
50
core/coreapi/unixfs.go
+25 -21
@@ -271,32 +271,36 @@ func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, se
271 lnk.Type = coreiface.TFile
272 lnk.Size = linkres.Link.Size
273 case cid.DagProtobuf:
274 - if !settings.ResolveChildren {
275 - break
276 - }
277 -
278 - linkNode, err := linkres.Link.GetNode(ctx, api.dag)
279 - if err != nil {
280 - lnk.Err = err
281 - break
282 - }
283 -
284 - if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
285 - d, err := ft.FSNodeFromBytes(pn.Data())
274 + if settings.ResolveChildren {
275 + linkNode, err := linkres.Link.GetNode(ctx, api.dag)
276 if err != nil {
277 lnk.Err = err
278 break
279 }
290 - switch d.Type() {
291 - case ft.TFile, ft.TRaw:
292 - lnk.Type = coreiface.TFile
293 - case ft.THAMTShard, ft.TDirectory, ft.TMetadata:
294 - lnk.Type = coreiface.TDirectory
295 - case ft.TSymlink:
296 - lnk.Type = coreiface.TSymlink
297 - lnk.Target = string(d.Data())
280 +
281 + if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
282 + d, err := ft.FSNodeFromBytes(pn.Data())
283 + if err != nil {
284 + lnk.Err = err
285 + break
286 + }
287 + switch d.Type() {
288 + case ft.TFile, ft.TRaw:
289 + lnk.Type = coreiface.TFile
290 + case ft.THAMTShard, ft.TDirectory, ft.TMetadata:
291 + lnk.Type = coreiface.TDirectory
292 + case ft.TSymlink:
293 + lnk.Type = coreiface.TSymlink
294 + lnk.Target = string(d.Data())
295 + }
296 + if !settings.UseCumulativeSize {
297 + lnk.Size = d.FileSize()
298 + }
299 }
299 - lnk.Size = d.FileSize()
300 + }
301 +
302 + if settings.UseCumulativeSize {
303 + lnk.Size = linkres.Link.Size
304 }
305 }
306
core/corehttp/gateway.go
+4 -6
@@ -18,9 +18,8 @@ import (
18 )
19
20 type GatewayConfig struct {
21 - Headers map[string][]string
22 - Writable bool
23 - FastDirIndexThreshold int
21 + Headers map[string][]string
22 + Writable bool
23 }
24
25 // NodeAPI defines the minimal set of API services required by a gateway handler
@@ -83,9 +82,8 @@ func GatewayOption(writable bool, paths ...string) ServeOption {
82 }
83
84 gateway := NewGatewayHandler(GatewayConfig{
86 - Headers: headers,
87 - Writable: writable,
88 - FastDirIndexThreshold: int(cfg.Gateway.FastDirIndexThreshold.WithDefault(100)),
85 + Headers: headers,
86 + Writable: writable,
87 }, api, offlineAPI)
88
89 gateway = otelhttp.NewHandler(gateway, "Gateway.Request")
core/corehttp/gateway_handler_unixfs_dir.go
+18 -30
@@ -105,25 +105,29 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
105 return
106 }
107
108 - // Optimization 1:
109 - // List children without fetching their root blocks (fast, but no size info)
110 - results, err := i.api.Unixfs().Ls(ctx, resolvedPath, options.Unixfs.ResolveChildren(false))
108 + // Optimization: use Unixfs.Ls without resolving children, but using the
109 + // cumulative DAG size as the file size. This allows for a fast listing
110 + // while keeping a good enough Size field.
111 + results, err := i.api.Unixfs().Ls(ctx,
112 + resolvedPath,
113 + options.Unixfs.ResolveChildren(false),
114 + options.Unixfs.UseCumulativeSize(true),
115 + )
116 if err != nil {
117 internalWebError(w, err)
118 return
119 }
120
116 - // storage for directory listing
121 dirListing := make([]directoryItem, 0, len(results))
118 -
122 for link := range results {
123 if link.Err != nil {
124 internalWebError(w, err)
125 return
126 }
127 +
128 hash := link.Cid.String()
129 di := directoryItem{
126 - Size: "", // no size because we did not fetch child nodes
130 + Size: humanize.Bytes(uint64(link.Size)),
131 Name: link.Name,
132 Path: gopath.Join(originalURLPath, link.Name),
133 Hash: hash,
@@ -132,21 +136,6 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
136 dirListing = append(dirListing, di)
137 }
138
135 - // Optimization 2: fetch sizes only for dirs below FastDirIndexThreshold
136 - if len(dirListing) < i.config.FastDirIndexThreshold {
137 - dirit := dir.Entries()
138 - linkNo := 0
139 - for dirit.Next() {
140 - size := "?"
141 - if s, err := dirit.Node().Size(); err == nil {
142 - // Size may not be defined/supported. Continue anyways.
143 - size = humanize.Bytes(uint64(s))
144 - }
145 - dirListing[linkNo].Size = size
146 - linkNo++
147 - }
148 - }
149 -
139 // construct the correct back link
140 // https://github.com/ipfs/kubo/issues/1365
141 backLink := originalURLPath
@@ -195,15 +184,14 @@ func (i *gatewayHandler) serveDirectory(ctx context.Context, w http.ResponseWrit
184
185 // See comment above where originalUrlPath is declared.
186 tplData := listingTemplateData{
198 - GatewayURL: gwURL,
199 - DNSLink: dnslink,
200 - Listing: dirListing,
201 - Size: size,
202 - Path: contentPath.String(),
203 - Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
204 - BackLink: backLink,
205 - Hash: hash,
206 - FastDirIndexThreshold: i.config.FastDirIndexThreshold,
187 + GatewayURL: gwURL,
188 + DNSLink: dnslink,
189 + Listing: dirListing,
190 + Size: size,
191 + Path: contentPath.String(),
192 + Breadcrumbs: breadcrumbs(contentPath.String(), dnslink),
193 + BackLink: backLink,
194 + Hash: hash,
195 }
196
197 logger.Debugw("request processed", "tplDataDNSLink", dnslink, "tplDataSize", size, "tplDataBackLink", backLink, "tplDataHash", hash)
core/corehttp/gateway_indexPage.go
+8 -9
@@ -12,15 +12,14 @@ import (
12
13 // structs for directory listing
14 type listingTemplateData struct {
15 - GatewayURL string
16 - DNSLink bool
17 - Listing []directoryItem
18 - Size string
19 - Path string
20 - Breadcrumbs []breadcrumb
21 - BackLink string
22 - Hash string
23 - FastDirIndexThreshold int
15 + GatewayURL string
16 + DNSLink bool
17 + Listing []directoryItem
18 + Size string
19 + Path string
20 + Breadcrumbs []breadcrumb
21 + BackLink string
22 + Hash string
23 }
24
25 type directoryItem struct {
docs/changelogs/v0.18.md
+11
@@ -11,6 +11,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
11 - [Overview](#overview)
12 - [🔦 Highlights](#-highlights)
13 - [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways)
14 + - [🐎 Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes)
15 - [Content Routing](#content-routing)
16 - [Provider Record Republish and Expiration](#provider-record-republish-and-expiration)
17 - [Lowered `ConnMgr`](#lowered-connmgr)
@@ -71,6 +72,16 @@ $ curl "http://127.0.0.1:8080/ipfs/$DIR_CID?format=dag-json" | jq
72 }
73 ```
74
75 +#### 🐎 Fast directory listings with DAG sizes
76 +
77 +Fast listings are now enabled for _all_ UnixFS directories: big and small.
78 +There is no linear slowdown caused by reading size metadata from child nodes,
79 +and the size of DAG representing child items is always present.
80 +
81 +As an example, the CID
82 +`bafybeiggvykl7skb2ndlmacg2k5modvudocffxjesexlod2pfvg5yhwrqm` represents UnixFS
83 +directory with over 10k (10100) of files. Listing big directories was fast
84 +since Kubo 0.13, but in this release it will also include the size column.
85
86 #### Content Routing
87
docs/config.md
+1 -11
@@ -672,17 +672,7 @@ Type: `string` (url)
672
673 ### `Gateway.FastDirIndexThreshold`
674
675 -The maximum number of items in a directory before the Gateway switches
676 -to a shallow, faster listing which only requires the root node.
677 -
678 -This allows for fast listings of big directories, without the linear slowdown caused
679 -by reading size metadata from child nodes.
680 -
681 -Setting to 0 will enable fast listings for all directories.
682 -
683 -Default: `100`
684 -
685 -Type: `optionalInteger`
675 +**REMOVED**: this option is [no longer necessary](https://github.com/ipfs/kubo/pull/9481). Ignored since [Kubo 0.18](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.18.md).
676
677 ### `Gateway.Writable`
678
docs/examples/kubo-as-a-library/go.mod
+1 -1
@@ -8,7 +8,7 @@ replace github.com/ipfs/kubo => ./../../..
8
9 require (
10 github.com/ipfs/go-ipfs-files v0.2.0
11 - github.com/ipfs/interface-go-ipfs-core v0.8.0
11 + github.com/ipfs/interface-go-ipfs-core v0.8.1
12 github.com/ipfs/kubo v0.14.0-rc1
13 github.com/libp2p/go-libp2p v0.24.1
14 github.com/multiformats/go-multiaddr v0.8.0
docs/examples/kubo-as-a-library/go.sum
+2 -2
@@ -598,8 +598,8 @@ github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuF
598 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
599 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
600 github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
601 -github.com/ipfs/interface-go-ipfs-core v0.8.0 h1:pNs34l947fvNOh+XEjXnHW/GV6HXmEzJNeqZFhX4GoQ=
602 -github.com/ipfs/interface-go-ipfs-core v0.8.0/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
601 +github.com/ipfs/interface-go-ipfs-core v0.8.1 h1:nuFG0YJ429Wd5gtRb3ivlblpknZ5VfDVKZkmOG2TnNQ=
602 +github.com/ipfs/interface-go-ipfs-core v0.8.1/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
603 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
604 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
605 github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=
go.mod
+1 -1
@@ -63,7 +63,7 @@ require (
63 github.com/ipfs/go-unixfs v0.4.1
64 github.com/ipfs/go-unixfsnode v1.4.0
65 github.com/ipfs/go-verifcid v0.0.2
66 - github.com/ipfs/interface-go-ipfs-core v0.8.0
66 + github.com/ipfs/interface-go-ipfs-core v0.8.1
67 github.com/ipld/go-car v0.4.0
68 github.com/ipld/go-car/v2 v2.4.0
69 github.com/ipld/go-codec-dagpb v1.4.1
go.sum
+2 -2
@@ -625,8 +625,8 @@ github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuF
625 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=
626 github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs=
627 github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU=
628 -github.com/ipfs/interface-go-ipfs-core v0.8.0 h1:pNs34l947fvNOh+XEjXnHW/GV6HXmEzJNeqZFhX4GoQ=
629 -github.com/ipfs/interface-go-ipfs-core v0.8.0/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
628 +github.com/ipfs/interface-go-ipfs-core v0.8.1 h1:nuFG0YJ429Wd5gtRb3ivlblpknZ5VfDVKZkmOG2TnNQ=
629 +github.com/ipfs/interface-go-ipfs-core v0.8.1/go.mod h1:WYC2H6Mu7aGqhlupi/CVawcs0X1Me4uRvV0rcTlo3zM=
630 github.com/ipld/edelweiss v0.2.0 h1:KfAZBP8eeJtrLxLhi7r3N0cBCo7JmwSRhOJp3WSpNjk=
631 github.com/ipld/edelweiss v0.2.0/go.mod h1:FJAzJRCep4iI8FOFlRriN9n0b7OuX3T/S9++NpBDmA4=
632 github.com/ipld/go-car v0.4.0 h1:U6W7F1aKF/OJMHovnOVdst2cpQE5GhmHibQkAixgNcQ=
test/sharness/t0115-gateway-dir-listing.sh
-22
@@ -163,28 +163,6 @@ test_expect_success "dnslink gw: hash column should be a CID link to cid.ipfs.te
163 test_should_contain "<a class=\"ipfs-hash\" translate=\"no\" href=\"https://cid.ipfs.tech/#$FILE_CID\" target=\"_blank\" rel=\"noreferrer noopener\">" list_response
164 '
165
166 -## ============================================================================
167 -## Test dir listing of a big directory
168 -## ============================================================================
169 -
170 -test_expect_success "dir listing should resolve child sizes if under Gateway.FastDirIndexThreshold" '
171 - curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
172 - test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
173 - test_should_contain ">${FILE_SIZE} B</td>" list_response
174 -'
175 -
176 -# force fast dir index for all responses
177 -ipfs config --json Gateway.FastDirIndexThreshold 0
178 -# restart daemon to apply config changes
179 -test_kill_ipfs_daemon
180 -test_launch_ipfs_daemon
181 -
182 -test_expect_success "dir listing should not resolve child sizes beyond Gateway.FastDirIndexThreshold" '
183 - curl -sD - http://127.0.0.1:$GWAY_PORT/ipfs/${DIR_CID}/ą/ę/ | tee list_response &&
184 - test_should_contain "/ipfs/${FILE_CID}?filename" list_response &&
185 - test_should_not_contain ">${FILE_SIZE} B</td>" list_response
186 -'
187 -
166 ## ============================================================================
167 ## End of tests, cleanup
168 ## ============================================================================