coreapi: resolve type/size in Unixfs.Ls
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Feb 1, 2019 at 23:07 UTC
628cd7ed07d51fddd7620e153083bb6e5932d741
3 files changed
+98
-14
core/coreapi/interface/options/unixfs.go
+17
@@ -44,6 +44,9 @@ type UnixfsAddSettings struct {
44
45
type UnixfsLsSettings struct {
46
Async bool
47
+
48
+ ResolveType bool
49
+ ResolveSize bool
50
}
51
52
type UnixfsAddOption func(*UnixfsAddSettings) error
@@ -320,3 +323,17 @@ func (unixfsOpts) Async(async bool) UnixfsLsOption {
323
return nil
324
}
325
}
326
+
327
+func (unixfsOpts) ResolveSize(resolve bool) UnixfsLsOption {
328
+ return func(settings *UnixfsLsSettings) error {
329
+ settings.ResolveSize = resolve
330
+ return nil
331
+ }
332
+}
333
+
334
+func (unixfsOpts) ResolveType(resolve bool) UnixfsLsOption {
335
+ return func(settings *UnixfsLsSettings) error {
336
+ settings.ResolveSize = resolve
337
+ return nil
338
+ }
339
+}
\ No newline at end of file
core/coreapi/interface/unixfs.go
+11
-3
@@ -2,10 +2,10 @@ package iface
2
3
import (
4
"context"
5
-
5
"github.com/ipfs/go-ipfs/core/coreapi/interface/options"
6
8
- ft "gx/ipfs/QmQ1JnYpnzkaurjW1yxkQxC2w3K1PorNE1nv1vaP5Le7sq/go-unixfs"
7
+ "gx/ipfs/QmQ1JnYpnzkaurjW1yxkQxC2w3K1PorNE1nv1vaP5Le7sq/go-unixfs/pb"
8
+ ipld "gx/ipfs/QmRL22E4paat7ky7vx9MLpR97JHHbFPrg3ytFQw6qp1y1s/go-ipld-format"
9
"gx/ipfs/QmaXvvAVAQ5ABqM5xtjYmV85xmN5MkWAZsX9H9Fwo4FVXp/go-ipfs-files"
10
)
11
@@ -16,6 +16,14 @@ type AddEvent struct {
16
Size string `json:",omitempty"`
17
}
18
19
+type LsLink struct {
20
+ Link *ipld.Link
21
+ Size uint64
22
+ Type unixfs_pb.Data_DataType
23
+
24
+ Err error
25
+}
26
+
27
// UnixfsAPI is the basic interface to immutable files in IPFS
28
// NOTE: This API is heavily WIP, things are guaranteed to break frequently
29
type UnixfsAPI interface {
@@ -31,5 +39,5 @@ type UnixfsAPI interface {
39
Get(context.Context, Path) (files.Node, error)
40
41
// Ls returns the list of links in a directory
34
- Ls(context.Context, Path, ...options.UnixfsLsOption) (<-chan ft.LinkResult, error)
42
+ Ls(context.Context, Path, ...options.UnixfsLsOption) (<-chan LsLink, error)
43
}
core/coreapi/unixfs.go
+70
-11
@@ -15,11 +15,13 @@ import (
15
unixfile "gx/ipfs/QmQ1JnYpnzkaurjW1yxkQxC2w3K1PorNE1nv1vaP5Le7sq/go-unixfs/file"
16
uio "gx/ipfs/QmQ1JnYpnzkaurjW1yxkQxC2w3K1PorNE1nv1vaP5Le7sq/go-unixfs/io"
17
mfs "gx/ipfs/QmR66iEqVtNMbbZxTHPY3F6W5QLFqZEDbFD7gzbE9HpYXU/go-mfs"
18
+ cid "gx/ipfs/QmR8BauakNcBa3RbE4nbQu76PDiJgoQgz8AJdhJuiU4TAw/go-cid"
19
ipld "gx/ipfs/QmRL22E4paat7ky7vx9MLpR97JHHbFPrg3ytFQw6qp1y1s/go-ipld-format"
20
bstore "gx/ipfs/QmS2aqUZLJp8kF1ihE5rvDGE5LvmKDPnx32w9Z1BW9xLV5/go-ipfs-blockstore"
21
blockservice "gx/ipfs/QmVKQHuzni68SWByzJgBUCwHvvr4TWiXfutNWWwpZpp4rE/go-blockservice"
22
files "gx/ipfs/QmaXvvAVAQ5ABqM5xtjYmV85xmN5MkWAZsX9H9Fwo4FVXp/go-ipfs-files"
23
dag "gx/ipfs/Qmb2UEG2TAeVrEJSjqsZF7Y2he7wRDkrdt6c3bECxwZf4k/go-merkledag"
24
+ merkledag "gx/ipfs/Qmb2UEG2TAeVrEJSjqsZF7Y2he7wRDkrdt6c3bECxwZf4k/go-merkledag"
25
dagtest "gx/ipfs/Qmb2UEG2TAeVrEJSjqsZF7Y2he7wRDkrdt6c3bECxwZf4k/go-merkledag/test"
26
cidutil "gx/ipfs/QmdPQx9fvN5ExVwMhRmh7YpCQJzJrFhd1AjVBwJmRMFJeX/go-cidutil"
27
)
@@ -143,7 +145,7 @@ func (api *UnixfsAPI) Get(ctx context.Context, p coreiface.Path) (files.Node, er
145
146
// Ls returns the contents of an IPFS or IPNS object(s) at path p, with the format:
147
// `<link base58 hash> <link size in bytes> <link name>`
146
-func (api *UnixfsAPI) Ls(ctx context.Context, p coreiface.Path, opts ...options.UnixfsLsOption) (<-chan ft.LinkResult, error) {
148
+func (api *UnixfsAPI) Ls(ctx context.Context, p coreiface.Path, opts ...options.UnixfsLsOption) (<-chan coreiface.LsLink, error) {
149
settings, err := options.UnixfsLsOptions(opts...)
150
if err != nil {
151
return nil, err
@@ -156,36 +158,93 @@ func (api *UnixfsAPI) Ls(ctx context.Context, p coreiface.Path, opts ...options.
158
159
dir, err := uio.NewDirectoryFromNode(api.dag, dagnode)
160
if err == uio.ErrNotADir {
159
- return lsFromLinks(dagnode.Links())
161
+ return api.lsFromLinks(ctx, dagnode.Links(), settings)
162
}
163
if err != nil {
164
return nil, err
165
}
166
167
if !settings.Async {
166
- return lsFromDir(ctx, dir)
168
+ return api.lsFromDir(ctx, dir, settings)
169
}
170
169
- return lsFromLinksAsync(ctx, dir)
171
+ return api.lsFromLinksAsync(ctx, dir, settings)
172
}
173
172
-func lsFromLinksAsync(ctx context.Context, dir uio.Directory) (<-chan ft.LinkResult, error) {
174
+func (api *UnixfsAPI) processLink(ctx context.Context, linkres ft.LinkResult, settings *options.UnixfsLsSettings) coreiface.LsLink {
175
+ lnk := coreiface.LsLink{
176
+ Link: linkres.Link,
177
+ Err: linkres.Err,
178
+ }
179
+ if lnk.Err != nil {
180
+ return lnk
181
+ }
182
+
183
+ switch lnk.Link.Cid.Type() {
184
+ case cid.Raw:
185
+ // No need to check with raw leaves
186
+ lnk.Type = ft.TFile
187
+ lnk.Size = lnk.Link.Size
188
+ case cid.DagProtobuf:
189
+ if !settings.ResolveSize && !settings.ResolveType {
190
+ break
191
+ }
192
174
- return dir.EnumLinksAsync(ctx), nil
193
+ linkNode, err := lnk.Link.GetNode(ctx, api.dag)
194
+ if err != nil {
195
+ lnk.Err = err
196
+ break
197
+ }
198
+
199
+ if pn, ok := linkNode.(*merkledag.ProtoNode); ok {
200
+ d, err := ft.FSNodeFromBytes(pn.Data())
201
+ if err != nil {
202
+ lnk.Err = err
203
+ break
204
+ }
205
+ if settings.ResolveType {
206
+ lnk.Type = d.Type()
207
+ }
208
+ if d.Type() == ft.TFile && settings.ResolveSize {
209
+ lnk.Size = d.FileSize()
210
+ }
211
+ }
212
+ }
213
+
214
+ return lnk
215
}
216
177
-func lsFromDir(ctx context.Context, dir uio.Directory) (<-chan ft.LinkResult, error) {
217
+func (api *UnixfsAPI) lsFromLinksAsync(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.LsLink, error) {
218
+ out := make(chan coreiface.LsLink)
219
+
220
+ go func() {
221
+ defer close(out)
222
+ for l := range dir.EnumLinksAsync(ctx) {
223
+ select {
224
+ case out <- api.processLink(ctx, l, settings): //TODO: perf: processing can be done in background and in parallel
225
+ case <-ctx.Done():
226
+ return
227
+ }
228
+ }
229
+ }()
230
+
231
+ return out, nil
232
+}
233
+
234
+func (api *UnixfsAPI) lsFromDir(ctx context.Context, dir uio.Directory, settings *options.UnixfsLsSettings) (<-chan coreiface.LsLink, error) {
235
l, err := dir.Links(ctx)
236
if err != nil {
237
return nil, err
238
}
182
- return lsFromLinks(l)
239
+ return api.lsFromLinks(ctx, l, settings)
240
}
241
185
-func lsFromLinks(ndlinks []*ipld.Link) (<-chan ft.LinkResult, error) {
186
- links := make(chan ft.LinkResult, len(ndlinks))
242
+func (api *UnixfsAPI) lsFromLinks(ctx context.Context, ndlinks []*ipld.Link, settings *options.UnixfsLsSettings) (<-chan coreiface.LsLink, error) {
243
+ links := make(chan coreiface.LsLink, len(ndlinks))
244
for _, l := range ndlinks {
188
- links <- ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}}
245
+ lr := ft.LinkResult{Link: &ipld.Link{Name: l.Name, Size: l.Size, Cid: l.Cid}}
246
+
247
+ links <- api.processLink(ctx, lr, settings) //TODO: can be parallel if settings.Async
248
}
249
close(links)
250
return links, nil