@cryptotaxi247 / kubo / commits / 7d531c00a

unixfs: add a directory interface

Add a UnixFS `Directory` that hides implementation details and helps to distinguish *what* is a UnixFS directory. Replace the `unixfs.io.Directory` structure that contained the HAMT and basic directory implementations (through inner pointers) with an interface containing the same methods. Implement those methods in two clearly distinct structures for each implementation (`BasicDirectory` and `HAMTDirectory`) avoiding pointer logic and clearly differentiating which implementation does what. The potential basic to HAMT transition was being hidden behind the `AddChild` call at the UnixFS layer (changing one implementation pointer with the other one), it is now being explicitly done at the MFS layer. Rename the `dirbuilder.go` file to `directory.go` and change the `Directory` MFS attribute `dirbuilder` to `unixfsDir` to be consistent. License: MIT Signed-off-by: Lucas Molas <schomatis@gmail.com>

Lucas Molas committed Jun 27, 2018 at 10:29 UTC 7d531c00acb2c0f829ea451c80b19c6abe4ffad3
4 files changed +307 -234
mfs/dir.go
+45 -20
@@ -33,7 +33,9 @@ type Directory struct {
33 lock sync.Mutex
34 ctx context.Context
35
36 - dirbuilder *uio.Directory
36 + // UnixFS directory implementation used for creating,
37 + // reading and editing directories.
38 + unixfsDir uio.Directory
39
40 modTime time.Time
41
@@ -51,25 +53,25 @@ func NewDirectory(ctx context.Context, name string, node ipld.Node, parent child
53 }
54
55 return &Directory{
54 - dserv: dserv,
55 - ctx: ctx,
56 - name: name,
57 - dirbuilder: db,
58 - parent: parent,
59 - childDirs: make(map[string]*Directory),
60 - files: make(map[string]*File),
61 - modTime: time.Now(),
56 + dserv: dserv,
57 + ctx: ctx,
58 + name: name,
59 + unixfsDir: db,
60 + parent: parent,
61 + childDirs: make(map[string]*Directory),
62 + files: make(map[string]*File),
63 + modTime: time.Now(),
64 }, nil
65 }
66
67 // GetPrefix gets the CID prefix of the root node
68 func (d *Directory) GetPrefix() *cid.Prefix {
67 - return d.dirbuilder.GetPrefix()
69 + return d.unixfsDir.GetPrefix()
70 }
71
72 // SetPrefix sets the CID prefix
73 func (d *Directory) SetPrefix(prefix *cid.Prefix) {
72 - d.dirbuilder.SetPrefix(prefix)
74 + d.unixfsDir.SetPrefix(prefix)
75 }
76
77 // closeChild updates the child by the given name to the dag node 'nd'
@@ -103,7 +105,7 @@ func (d *Directory) closeChildUpdate(name string, nd ipld.Node, sync bool) (*dag
105 }
106
107 func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) {
106 - nd, err := d.dirbuilder.GetNode()
108 + nd, err := d.unixfsDir.GetNode()
109 if err != nil {
110 return nil, err
111 }
@@ -122,7 +124,7 @@ func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) {
124 }
125
126 func (d *Directory) updateChild(name string, nd ipld.Node) error {
125 - err := d.dirbuilder.AddChild(d.ctx, name, nd)
127 + err := d.AddUnixFSChild(name, nd)
128 if err != nil {
129 return err
130 }
@@ -206,7 +208,7 @@ func (d *Directory) Uncache(name string) {
208 // childFromDag searches through this directories dag node for a child link
209 // with the given name
210 func (d *Directory) childFromDag(name string) (ipld.Node, error) {
209 - return d.dirbuilder.Find(d.ctx, name)
211 + return d.unixfsDir.Find(d.ctx, name)
212 }
213
214 // childUnsync returns the child under this directory by the given name
@@ -237,7 +239,7 @@ func (d *Directory) ListNames(ctx context.Context) ([]string, error) {
239 defer d.lock.Unlock()
240
241 var out []string
240 - err := d.dirbuilder.ForEachLink(ctx, func(l *ipld.Link) error {
242 + err := d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error {
243 out = append(out, l.Name)
244 return nil
245 })
@@ -262,7 +264,7 @@ func (d *Directory) List(ctx context.Context) ([]NodeListing, error) {
264 func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) error {
265 d.lock.Lock()
266 defer d.lock.Unlock()
265 - return d.dirbuilder.ForEachLink(ctx, func(l *ipld.Link) error {
267 + return d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error {
268 c, err := d.childUnsync(l.Name)
269 if err != nil {
270 return err
@@ -315,7 +317,7 @@ func (d *Directory) Mkdir(name string) (*Directory, error) {
317 return nil, err
318 }
319
318 - err = d.dirbuilder.AddChild(d.ctx, name, ndir)
320 + err = d.AddUnixFSChild(name, ndir)
321 if err != nil {
322 return nil, err
323 }
@@ -336,7 +338,7 @@ func (d *Directory) Unlink(name string) error {
338 delete(d.childDirs, name)
339 delete(d.files, name)
340
339 - return d.dirbuilder.RemoveChild(d.ctx, name)
341 + return d.unixfsDir.RemoveChild(d.ctx, name)
342 }
343
344 func (d *Directory) Flush() error {
@@ -363,7 +365,7 @@ func (d *Directory) AddChild(name string, nd ipld.Node) error {
365 return err
366 }
367
366 - err = d.dirbuilder.AddChild(d.ctx, name, nd)
368 + err = d.AddUnixFSChild(name, nd)
369 if err != nil {
370 return err
371 }
@@ -372,6 +374,29 @@ func (d *Directory) AddChild(name string, nd ipld.Node) error {
374 return nil
375 }
376
377 +// AddUnixFSChild adds a child to the inner UnixFS directory
378 +// and transitions to a HAMT implementation if needed.
379 +func (d *Directory) AddUnixFSChild(name string, node ipld.Node) error {
380 + if uio.UseHAMTSharding {
381 + // If the directory HAMT implementation is being used and this
382 + // directory is actually a basic implementation switch it to HAMT.
383 + if basicDir, ok := d.unixfsDir.(*uio.BasicDirectory); ok {
384 + hamtDir, err := basicDir.SwitchToSharding(d.ctx)
385 + if err != nil {
386 + return err
387 + }
388 + d.unixfsDir = hamtDir
389 + }
390 + }
391 +
392 + err := d.unixfsDir.AddChild(d.ctx, name, node)
393 + if err != nil {
394 + return err
395 + }
396 +
397 + return nil
398 +}
399 +
400 func (d *Directory) sync() error {
401 for name, dir := range d.childDirs {
402 nd, err := dir.GetNode()
@@ -426,7 +451,7 @@ func (d *Directory) GetNode() (ipld.Node, error) {
451 return nil, err
452 }
453
429 - nd, err := d.dirbuilder.GetNode()
454 + nd, err := d.unixfsDir.GetNode()
455 if err != nil {
456 return nil, err
457 }
unixfs/io/dirbuilder.go deleted
-214
@@ -1,214 +0,0 @@
1 -package io
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "os"
7 -
8 - mdag "github.com/ipfs/go-ipfs/merkledag"
9 - format "github.com/ipfs/go-ipfs/unixfs"
10 - hamt "github.com/ipfs/go-ipfs/unixfs/hamt"
11 -
12 - ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
13 - cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
14 -)
15 -
16 -// ShardSplitThreshold specifies how large of an unsharded directory
17 -// the Directory code will generate. Adding entries over this value will
18 -// result in the node being restructured into a sharded object.
19 -var ShardSplitThreshold = 1000
20 -
21 -// UseHAMTSharding is a global flag that signifies whether or not to use the
22 -// HAMT sharding scheme for directory creation
23 -var UseHAMTSharding = false
24 -
25 -// DefaultShardWidth is the default value used for hamt sharding width.
26 -var DefaultShardWidth = 256
27 -
28 -// Directory allows to work with UnixFS directory nodes, adding and removing
29 -// children. It allows to work with different directory schemes,
30 -// like the classic or the HAMT one.
31 -type Directory struct {
32 - dserv ipld.DAGService
33 - dirnode *mdag.ProtoNode
34 -
35 - shard *hamt.Shard
36 -}
37 -
38 -// NewDirectory returns a Directory. It needs a DAGService to add the Children
39 -func NewDirectory(dserv ipld.DAGService) *Directory {
40 - db := new(Directory)
41 - db.dserv = dserv
42 - if UseHAMTSharding {
43 - s, err := hamt.NewShard(dserv, DefaultShardWidth)
44 - if err != nil {
45 - panic(err) // will only panic if DefaultShardWidth is a bad value
46 - }
47 - db.shard = s
48 - } else {
49 - db.dirnode = format.EmptyDirNode()
50 - }
51 - return db
52 -}
53 -
54 -// ErrNotADir implies that the given node was not a unixfs directory
55 -var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
56 -
57 -// NewDirectoryFromNode loads a unixfs directory from the given IPLD node and
58 -// DAGService.
59 -func NewDirectoryFromNode(dserv ipld.DAGService, nd ipld.Node) (*Directory, error) {
60 - pbnd, ok := nd.(*mdag.ProtoNode)
61 - if !ok {
62 - return nil, ErrNotADir
63 - }
64 -
65 - pbd, err := format.FromBytes(pbnd.Data())
66 - if err != nil {
67 - return nil, err
68 - }
69 -
70 - switch pbd.GetType() {
71 - case format.TDirectory:
72 - return &Directory{
73 - dserv: dserv,
74 - dirnode: pbnd.Copy().(*mdag.ProtoNode),
75 - }, nil
76 - case format.THAMTShard:
77 - shard, err := hamt.NewHamtFromDag(dserv, nd)
78 - if err != nil {
79 - return nil, err
80 - }
81 -
82 - return &Directory{
83 - dserv: dserv,
84 - shard: shard,
85 - }, nil
86 - default:
87 - return nil, ErrNotADir
88 - }
89 -}
90 -
91 -// SetPrefix sets the prefix of the root node
92 -func (d *Directory) SetPrefix(prefix *cid.Prefix) {
93 - if d.dirnode != nil {
94 - d.dirnode.SetPrefix(prefix)
95 - }
96 - if d.shard != nil {
97 - d.shard.SetPrefix(prefix)
98 - }
99 -}
100 -
101 -// AddChild adds a (name, key)-pair to the root node.
102 -func (d *Directory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
103 - if d.shard == nil {
104 - if !UseHAMTSharding {
105 - _ = d.dirnode.RemoveNodeLink(name)
106 - return d.dirnode.AddNodeLink(name, nd)
107 - }
108 -
109 - err := d.switchToSharding(ctx)
110 - if err != nil {
111 - return err
112 - }
113 - }
114 -
115 - return d.shard.Set(ctx, name, nd)
116 -}
117 -
118 -func (d *Directory) switchToSharding(ctx context.Context) error {
119 - s, err := hamt.NewShard(d.dserv, DefaultShardWidth)
120 - if err != nil {
121 - return err
122 - }
123 - s.SetPrefix(&d.dirnode.Prefix)
124 -
125 - d.shard = s
126 - for _, lnk := range d.dirnode.Links() {
127 - cnd, err := d.dserv.Get(ctx, lnk.Cid)
128 - if err != nil {
129 - return err
130 - }
131 -
132 - err = d.shard.Set(ctx, lnk.Name, cnd)
133 - if err != nil {
134 - return err
135 - }
136 - }
137 -
138 - d.dirnode = nil
139 - return nil
140 -}
141 -
142 -// ForEachLink applies the given function to Links in the directory.
143 -func (d *Directory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
144 - if d.shard == nil {
145 - for _, l := range d.dirnode.Links() {
146 - if err := f(l); err != nil {
147 - return err
148 - }
149 - }
150 - return nil
151 - }
152 -
153 - return d.shard.ForEachLink(ctx, f)
154 -}
155 -
156 -// Links returns the all the links in the directory node.
157 -func (d *Directory) Links(ctx context.Context) ([]*ipld.Link, error) {
158 - if d.shard == nil {
159 - return d.dirnode.Links(), nil
160 - }
161 -
162 - return d.shard.EnumLinks(ctx)
163 -}
164 -
165 -// Find returns the root node of the file named 'name' within this directory.
166 -// In the case of HAMT-directories, it will traverse the tree.
167 -func (d *Directory) Find(ctx context.Context, name string) (ipld.Node, error) {
168 - if d.shard == nil {
169 - lnk, err := d.dirnode.GetNodeLink(name)
170 - switch err {
171 - case mdag.ErrLinkNotFound:
172 - return nil, os.ErrNotExist
173 - default:
174 - return nil, err
175 - case nil:
176 - }
177 -
178 - return d.dserv.Get(ctx, lnk.Cid)
179 - }
180 -
181 - lnk, err := d.shard.Find(ctx, name)
182 - if err != nil {
183 - return nil, err
184 - }
185 -
186 - return lnk.GetNode(ctx, d.dserv)
187 -}
188 -
189 -// RemoveChild removes the child with the given name.
190 -func (d *Directory) RemoveChild(ctx context.Context, name string) error {
191 - if d.shard == nil {
192 - return d.dirnode.RemoveNodeLink(name)
193 - }
194 -
195 - return d.shard.Remove(ctx, name)
196 -}
197 -
198 -// GetNode returns the root of this Directory
199 -func (d *Directory) GetNode() (ipld.Node, error) {
200 - if d.shard == nil {
201 - return d.dirnode, nil
202 - }
203 -
204 - return d.shard.Node()
205 -}
206 -
207 -// GetPrefix returns the CID Prefix used
208 -func (d *Directory) GetPrefix() *cid.Prefix {
209 - if d.shard == nil {
210 - return &d.dirnode.Prefix
211 - }
212 -
213 - return d.shard.Prefix()
214 -}
unixfs/io/directory.go new
+262
@@ -0,0 +1,262 @@
1 +package io
2 +
3 +import (
4 + "context"
5 + "fmt"
6 + "os"
7 +
8 + mdag "github.com/ipfs/go-ipfs/merkledag"
9 + format "github.com/ipfs/go-ipfs/unixfs"
10 + hamt "github.com/ipfs/go-ipfs/unixfs/hamt"
11 +
12 + ipld "gx/ipfs/QmWi2BYBL5gJ3CiAiQchg6rn1A8iBsrWy51EYxvHVjFvLb/go-ipld-format"
13 + cid "gx/ipfs/QmapdYm1b22Frv3k17fqrBYTFRxwiaVJkB299Mfn33edeB/go-cid"
14 +)
15 +
16 +// ShardSplitThreshold specifies how large of an unsharded directory
17 +// the Directory code will generate. Adding entries over this value will
18 +// result in the node being restructured into a sharded object.
19 +var ShardSplitThreshold = 1000
20 +
21 +// UseHAMTSharding is a global flag that signifies whether or not to use the
22 +// HAMT sharding scheme for directory creation
23 +var UseHAMTSharding = false
24 +
25 +// DefaultShardWidth is the default value used for hamt sharding width.
26 +var DefaultShardWidth = 256
27 +
28 +// Directory defines a UnixFS directory. It is used for creating, reading and
29 +// editing directories. It allows to work with different directory schemes,
30 +// like the basic or the HAMT implementation.
31 +//
32 +// It just allows to perform explicit edits on a single directory, working with
33 +// directory trees is out of its scope, they are managed by the MFS layer
34 +// (which is the main consumer of this interface).
35 +type Directory interface {
36 +
37 + // SetPrefix sets the CID prefix of the root node.
38 + SetPrefix(*cid.Prefix)
39 +
40 + // AddChild adds a (name, key) pair to the root node.
41 + AddChild(context.Context, string, ipld.Node) error
42 +
43 + // ForEachLink applies the given function to Links in the directory.
44 + ForEachLink(context.Context, func(*ipld.Link) error) error
45 +
46 + // Links returns the all the links in the directory node.
47 + Links(context.Context) ([]*ipld.Link, error)
48 +
49 + // Find returns the root node of the file named 'name' within this directory.
50 + // In the case of HAMT-directories, it will traverse the tree.
51 + Find(context.Context, string) (ipld.Node, error)
52 +
53 + // RemoveChild removes the child with the given name.
54 + RemoveChild(context.Context, string) error
55 +
56 + // GetNode returns the root of this directory.
57 + GetNode() (ipld.Node, error)
58 +
59 + // GetPrefix returns the CID Prefix used.
60 + GetPrefix() *cid.Prefix
61 +}
62 +
63 +// TODO: Evaluate removing `dserv` from this layer and providing it in MFS.
64 +// (The functions should in that case add a `DAGService` argument.)
65 +
66 +// BasicDirectory is the basic implementation of `Directory`. All the entries
67 +// are stored in a single node.
68 +type BasicDirectory struct {
69 + node *mdag.ProtoNode
70 + dserv ipld.DAGService
71 +}
72 +
73 +// HAMTDirectory is the HAMT implementation of `Directory`.
74 +// (See package `hamt` for more information.)
75 +type HAMTDirectory struct {
76 + shard *hamt.Shard
77 + dserv ipld.DAGService
78 +}
79 +
80 +// NewDirectory returns a Directory. It needs a `DAGService` to add the children.
81 +func NewDirectory(dserv ipld.DAGService) Directory {
82 + if UseHAMTSharding {
83 + dir := new(HAMTDirectory)
84 + s, err := hamt.NewShard(dserv, DefaultShardWidth)
85 + if err != nil {
86 + panic(err) // will only panic if DefaultShardWidth is a bad value
87 + }
88 + dir.shard = s
89 + dir.dserv = dserv
90 + return dir
91 + }
92 +
93 + dir := new(BasicDirectory)
94 + dir.node = format.EmptyDirNode()
95 + dir.dserv = dserv
96 + return dir
97 +}
98 +
99 +// ErrNotADir implies that the given node was not a unixfs directory
100 +var ErrNotADir = fmt.Errorf("merkledag node was not a directory or shard")
101 +
102 +// NewDirectoryFromNode loads a unixfs directory from the given IPLD node and
103 +// DAGService.
104 +func NewDirectoryFromNode(dserv ipld.DAGService, node ipld.Node) (Directory, error) {
105 + protoBufNode, ok := node.(*mdag.ProtoNode)
106 + if !ok {
107 + return nil, ErrNotADir
108 + }
109 +
110 + fsNode, err := format.FSNodeFromBytes(protoBufNode.Data())
111 + if err != nil {
112 + return nil, err
113 + }
114 +
115 + switch fsNode.GetType() {
116 + case format.TDirectory:
117 + return &BasicDirectory{
118 + dserv: dserv,
119 + node: protoBufNode.Copy().(*mdag.ProtoNode),
120 + }, nil
121 + case format.THAMTShard:
122 + shard, err := hamt.NewHamtFromDag(dserv, node)
123 + if err != nil {
124 + return nil, err
125 + }
126 + return &HAMTDirectory{
127 + dserv: dserv,
128 + shard: shard,
129 + }, nil
130 + }
131 +
132 + return nil, ErrNotADir
133 +}
134 +
135 +// SetPrefix implements the `Directory` interface.
136 +func (d *BasicDirectory) SetPrefix(prefix *cid.Prefix) {
137 + d.node.SetPrefix(prefix)
138 +}
139 +
140 +// AddChild implements the `Directory` interface. It adds (or replaces)
141 +// a link to the given `node` under `name`.
142 +func (d *BasicDirectory) AddChild(ctx context.Context, name string, node ipld.Node) error {
143 + d.node.RemoveNodeLink(name)
144 + // Remove old link (if it existed), don't check a potential `ErrNotFound`.
145 +
146 + return d.node.AddNodeLink(name, node)
147 +}
148 +
149 +// ForEachLink implements the `Directory` interface.
150 +func (d *BasicDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
151 + for _, l := range d.node.Links() {
152 + if err := f(l); err != nil {
153 + return err
154 + }
155 + }
156 + return nil
157 +}
158 +
159 +// Links implements the `Directory` interface.
160 +func (d *BasicDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
161 + return d.node.Links(), nil
162 +}
163 +
164 +// Find implements the `Directory` interface.
165 +func (d *BasicDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
166 + lnk, err := d.node.GetNodeLink(name)
167 + if err == mdag.ErrLinkNotFound {
168 + err = os.ErrNotExist
169 + }
170 + if err != nil {
171 + return nil, err
172 + }
173 +
174 + return d.dserv.Get(ctx, lnk.Cid)
175 +}
176 +
177 +// RemoveChild implements the `Directory` interface.
178 +func (d *BasicDirectory) RemoveChild(ctx context.Context, name string) error {
179 + return d.node.RemoveNodeLink(name)
180 +}
181 +
182 +// GetNode implements the `Directory` interface.
183 +func (d *BasicDirectory) GetNode() (ipld.Node, error) {
184 + return d.node, nil
185 +}
186 +
187 +// GetPrefix implements the `Directory` interface.
188 +func (d *BasicDirectory) GetPrefix() *cid.Prefix {
189 + return &d.node.Prefix
190 +}
191 +
192 +// SwitchToSharding returns a HAMT implementation of this directory.
193 +func (d *BasicDirectory) SwitchToSharding(ctx context.Context) (Directory, error) {
194 + hamtDir := new(HAMTDirectory)
195 + hamtDir.dserv = d.dserv
196 +
197 + shard, err := hamt.NewShard(d.dserv, DefaultShardWidth)
198 + if err != nil {
199 + return nil, err
200 + }
201 + shard.SetPrefix(&d.node.Prefix)
202 + hamtDir.shard = shard
203 +
204 + for _, lnk := range d.node.Links() {
205 + node, err := d.dserv.Get(ctx, lnk.Cid)
206 + if err != nil {
207 + return nil, err
208 + }
209 +
210 + err = hamtDir.shard.Set(ctx, lnk.Name, node)
211 + if err != nil {
212 + return nil, err
213 + }
214 + }
215 +
216 + return hamtDir, nil
217 +}
218 +
219 +// SetPrefix implements the `Directory` interface.
220 +func (d *HAMTDirectory) SetPrefix(prefix *cid.Prefix) {
221 + d.shard.SetPrefix(prefix)
222 +}
223 +
224 +// AddChild implements the `Directory` interface.
225 +func (d *HAMTDirectory) AddChild(ctx context.Context, name string, nd ipld.Node) error {
226 + return d.shard.Set(ctx, name, nd)
227 +}
228 +
229 +// ForEachLink implements the `Directory` interface.
230 +func (d *HAMTDirectory) ForEachLink(ctx context.Context, f func(*ipld.Link) error) error {
231 + return d.shard.ForEachLink(ctx, f)
232 +}
233 +
234 +// Links implements the `Directory` interface.
235 +func (d *HAMTDirectory) Links(ctx context.Context) ([]*ipld.Link, error) {
236 + return d.shard.EnumLinks(ctx)
237 +}
238 +
239 +// Find implements the `Directory` interface. It will traverse the tree.
240 +func (d *HAMTDirectory) Find(ctx context.Context, name string) (ipld.Node, error) {
241 + lnk, err := d.shard.Find(ctx, name)
242 + if err != nil {
243 + return nil, err
244 + }
245 +
246 + return lnk.GetNode(ctx, d.dserv)
247 +}
248 +
249 +// RemoveChild implements the `Directory` interface.
250 +func (d *HAMTDirectory) RemoveChild(ctx context.Context, name string) error {
251 + return d.shard.Remove(ctx, name)
252 +}
253 +
254 +// GetNode implements the `Directory` interface.
255 +func (d *HAMTDirectory) GetNode() (ipld.Node, error) {
256 + return d.shard.Node()
257 +}
258 +
259 +// GetPrefix implements the `Directory` interface.
260 +func (d *HAMTDirectory) GetPrefix() *cid.Prefix {
261 + return d.shard.Prefix()
262 +}
unixfs/io/directory_test.go renamed