@cryptotaxi247 / kubo / commits / 39a87a0fb

Extract /mfs

The /mfs module has been extracted to github.com/ipfs/go-mfs All history has been retained in the new repository. README, LICENSE, Makefiles and CI integration have been added to the new location. License: MIT Signed-off-by: Hector Sanjuan <hector@protocol.ai>

Hector Sanjuan committed Aug 20, 2018 at 15:28 UTC 39a87a0fb01e36d682f9b1bfcd3bf46c2ea2a00d
14 files changed +13 -2541
core/commands/add.go
+1 -1
@@ -10,7 +10,6 @@ import (
10 core "github.com/ipfs/go-ipfs/core"
11 "github.com/ipfs/go-ipfs/core/coreunix"
12 filestore "github.com/ipfs/go-ipfs/filestore"
13 - mfs "github.com/ipfs/go-ipfs/mfs"
13 dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
14 dagtest "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag/test"
15 blockservice "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice"
@@ -20,6 +19,7 @@ import (
19 files "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit/files"
20 mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
21 pb "gx/ipfs/QmPtj12fdwuAqj9sBSTNUxBNu8kCGNp8b3o8yUzMm5GHpq/pb"
22 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
23 cmds "gx/ipfs/QmUQb3xtNzkQCgTj2NjaqcJZNv2nfSSub2QAdy9DtQMRBT/go-ipfs-cmds"
24 offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline"
25 bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore"
core/commands/files.go
+1 -1
@@ -15,7 +15,6 @@ import (
15 lgc "github.com/ipfs/go-ipfs/commands/legacy"
16 core "github.com/ipfs/go-ipfs/core"
17 e "github.com/ipfs/go-ipfs/core/commands/e"
18 - mfs "github.com/ipfs/go-ipfs/mfs"
18 dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
19 bservice "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice"
20 path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path"
@@ -26,6 +25,7 @@ import (
25 humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
26 cmdkit "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit"
27 mh "gx/ipfs/QmPnFwZ2JXKnXgMw8CdBPxn7FWh6LLdjUjxV1fKHuJnkr8/go-multihash"
28 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
29 logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
30 cmds "gx/ipfs/QmUQb3xtNzkQCgTj2NjaqcJZNv2nfSSub2QAdy9DtQMRBT/go-ipfs-cmds"
31 offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline"
core/core.go
+1 -1
@@ -24,7 +24,6 @@ import (
24 rp "github.com/ipfs/go-ipfs/exchange/reprovide"
25 filestore "github.com/ipfs/go-ipfs/filestore"
26 mount "github.com/ipfs/go-ipfs/fuse/mount"
27 - mfs "github.com/ipfs/go-ipfs/mfs"
27 namesys "github.com/ipfs/go-ipfs/namesys"
28 ipnsrp "github.com/ipfs/go-ipfs/namesys/republisher"
29 p2p "github.com/ipfs/go-ipfs/p2p"
@@ -36,6 +35,7 @@ import (
35 ic "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto"
36 p2phost "gx/ipfs/QmQ1hwb95uSSZR8jSPJysnfHxBDQAykSXsmz5TwTzxjq2Z/go-libp2p-host"
37 config "gx/ipfs/QmQSG7YCizeUH2bWatzp6uK9Vm3m7LA5jpxGa9QqgpNKw4/go-ipfs-config"
38 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
39 bitswap "gx/ipfs/QmQk1Rqy5XSBzXykMSsgiXfnhivCSnFpykx4M2j6DD1nBH/go-bitswap"
40 bsnet "gx/ipfs/QmQk1Rqy5XSBzXykMSsgiXfnhivCSnFpykx4M2j6DD1nBH/go-bitswap/network"
41 merkledag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
core/corerepo/gc.go
+1 -1
@@ -7,11 +7,11 @@ import (
7 "time"
8
9 "github.com/ipfs/go-ipfs/core"
10 - mfs "github.com/ipfs/go-ipfs/mfs"
10 gc "github.com/ipfs/go-ipfs/pin/gc"
11 repo "github.com/ipfs/go-ipfs/repo"
12
13 humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
15 logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
16 cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
17 )
core/coreunix/add.go
+1 -1
@@ -11,7 +11,6 @@ import (
11 "strconv"
12
13 core "github.com/ipfs/go-ipfs/core"
14 - mfs "github.com/ipfs/go-ipfs/mfs"
14 "github.com/ipfs/go-ipfs/pin"
15 dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
16 unixfs "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
@@ -20,6 +19,7 @@ import (
19 trickle "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/importer/trickle"
20
21 files "gx/ipfs/QmPVqQHEfLpqK7JLCsUkyam7rhuV3MAeZ9gueQQCrBwCta/go-ipfs-cmdkit/files"
22 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
23 logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
24 chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker"
25 bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore"
fuse/ipns/ipns_unix.go
+1 -1
@@ -12,13 +12,13 @@ import (
12 "os"
13
14 core "github.com/ipfs/go-ipfs/core"
15 - mfs "github.com/ipfs/go-ipfs/mfs"
15 namesys "github.com/ipfs/go-ipfs/namesys"
16 dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
17 path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path"
18 ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
19
20 ci "gx/ipfs/QmPvyPwuCgJ7pDmrKDxRtsScJgBaM5h4EpRL2qQJsmXf4n/go-libp2p-crypto"
21 + mfs "gx/ipfs/QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc/go-mfs"
22 logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
23 fuse "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse"
24 fs "gx/ipfs/QmSJBsmLP1XMjv8hxYg2rUMdPDB7YUpyBo9idjrJ6Cmq6F/fuse/fs"
mfs/dir.go deleted
-462
@@ -1,462 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "context"
5 - "errors"
6 - "fmt"
7 - "os"
8 - "path"
9 - "sync"
10 - "time"
11 -
12 - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
13 - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
14 - uio "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/io"
15 - ufspb "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/pb"
16 -
17 - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
18 - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format"
19 -)
20 -
21 -var ErrNotYetImplemented = errors.New("not yet implemented")
22 -var ErrInvalidChild = errors.New("invalid child node")
23 -var ErrDirExists = errors.New("directory already has entry by that name")
24 -
25 -type Directory struct {
26 - dserv ipld.DAGService
27 - parent childCloser
28 -
29 - childDirs map[string]*Directory
30 - files map[string]*File
31 -
32 - lock sync.Mutex
33 - ctx context.Context
34 -
35 - // UnixFS directory implementation used for creating,
36 - // reading and editing directories.
37 - unixfsDir uio.Directory
38 -
39 - modTime time.Time
40 -
41 - name string
42 -}
43 -
44 -// NewDirectory constructs a new MFS directory.
45 -//
46 -// You probably don't want to call this directly. Instead, construct a new root
47 -// using NewRoot.
48 -func NewDirectory(ctx context.Context, name string, node ipld.Node, parent childCloser, dserv ipld.DAGService) (*Directory, error) {
49 - db, err := uio.NewDirectoryFromNode(dserv, node)
50 - if err != nil {
51 - return nil, err
52 - }
53 -
54 - return &Directory{
55 - dserv: dserv,
56 - ctx: ctx,
57 - name: name,
58 - unixfsDir: db,
59 - parent: parent,
60 - childDirs: make(map[string]*Directory),
61 - files: make(map[string]*File),
62 - modTime: time.Now(),
63 - }, nil
64 -}
65 -
66 -// GetCidBuilder gets the CID builder of the root node
67 -func (d *Directory) GetCidBuilder() cid.Builder {
68 - return d.unixfsDir.GetCidBuilder()
69 -}
70 -
71 -// SetCidBuilder sets the CID builder
72 -func (d *Directory) SetCidBuilder(b cid.Builder) {
73 - d.unixfsDir.SetCidBuilder(b)
74 -}
75 -
76 -// closeChild updates the child by the given name to the dag node 'nd'
77 -// and changes its own dag node
78 -func (d *Directory) closeChild(name string, nd ipld.Node, sync bool) error {
79 - mynd, err := d.closeChildUpdate(name, nd, sync)
80 - if err != nil {
81 - return err
82 - }
83 -
84 - if sync {
85 - return d.parent.closeChild(d.name, mynd, true)
86 - }
87 - return nil
88 -}
89 -
90 -// closeChildUpdate is the portion of closeChild that needs to be locked around
91 -func (d *Directory) closeChildUpdate(name string, nd ipld.Node, sync bool) (*dag.ProtoNode, error) {
92 - d.lock.Lock()
93 - defer d.lock.Unlock()
94 -
95 - err := d.updateChild(name, nd)
96 - if err != nil {
97 - return nil, err
98 - }
99 -
100 - if sync {
101 - return d.flushCurrentNode()
102 - }
103 - return nil, nil
104 -}
105 -
106 -func (d *Directory) flushCurrentNode() (*dag.ProtoNode, error) {
107 - nd, err := d.unixfsDir.GetNode()
108 - if err != nil {
109 - return nil, err
110 - }
111 -
112 - err = d.dserv.Add(d.ctx, nd)
113 - if err != nil {
114 - return nil, err
115 - }
116 -
117 - pbnd, ok := nd.(*dag.ProtoNode)
118 - if !ok {
119 - return nil, dag.ErrNotProtobuf
120 - }
121 -
122 - return pbnd.Copy().(*dag.ProtoNode), nil
123 -}
124 -
125 -func (d *Directory) updateChild(name string, nd ipld.Node) error {
126 - err := d.AddUnixFSChild(name, nd)
127 - if err != nil {
128 - return err
129 - }
130 -
131 - d.modTime = time.Now()
132 -
133 - return nil
134 -}
135 -
136 -func (d *Directory) Type() NodeType {
137 - return TDir
138 -}
139 -
140 -// childNode returns a FSNode under this directory by the given name if it exists.
141 -// it does *not* check the cached dirs and files
142 -func (d *Directory) childNode(name string) (FSNode, error) {
143 - nd, err := d.childFromDag(name)
144 - if err != nil {
145 - return nil, err
146 - }
147 -
148 - return d.cacheNode(name, nd)
149 -}
150 -
151 -// cacheNode caches a node into d.childDirs or d.files and returns the FSNode.
152 -func (d *Directory) cacheNode(name string, nd ipld.Node) (FSNode, error) {
153 - switch nd := nd.(type) {
154 - case *dag.ProtoNode:
155 - i, err := ft.FromBytes(nd.Data())
156 - if err != nil {
157 - return nil, err
158 - }
159 -
160 - switch i.GetType() {
161 - case ufspb.Data_Directory, ufspb.Data_HAMTShard:
162 - ndir, err := NewDirectory(d.ctx, name, nd, d, d.dserv)
163 - if err != nil {
164 - return nil, err
165 - }
166 -
167 - d.childDirs[name] = ndir
168 - return ndir, nil
169 - case ufspb.Data_File, ufspb.Data_Raw, ufspb.Data_Symlink:
170 - nfi, err := NewFile(name, nd, d, d.dserv)
171 - if err != nil {
172 - return nil, err
173 - }
174 - d.files[name] = nfi
175 - return nfi, nil
176 - case ufspb.Data_Metadata:
177 - return nil, ErrNotYetImplemented
178 - default:
179 - return nil, ErrInvalidChild
180 - }
181 - case *dag.RawNode:
182 - nfi, err := NewFile(name, nd, d, d.dserv)
183 - if err != nil {
184 - return nil, err
185 - }
186 - d.files[name] = nfi
187 - return nfi, nil
188 - default:
189 - return nil, fmt.Errorf("unrecognized node type in cache node")
190 - }
191 -}
192 -
193 -// Child returns the child of this directory by the given name
194 -func (d *Directory) Child(name string) (FSNode, error) {
195 - d.lock.Lock()
196 - defer d.lock.Unlock()
197 - return d.childUnsync(name)
198 -}
199 -
200 -func (d *Directory) Uncache(name string) {
201 - d.lock.Lock()
202 - defer d.lock.Unlock()
203 - delete(d.files, name)
204 - delete(d.childDirs, name)
205 -}
206 -
207 -// childFromDag searches through this directories dag node for a child link
208 -// with the given name
209 -func (d *Directory) childFromDag(name string) (ipld.Node, error) {
210 - return d.unixfsDir.Find(d.ctx, name)
211 -}
212 -
213 -// childUnsync returns the child under this directory by the given name
214 -// without locking, useful for operations which already hold a lock
215 -func (d *Directory) childUnsync(name string) (FSNode, error) {
216 - cdir, ok := d.childDirs[name]
217 - if ok {
218 - return cdir, nil
219 - }
220 -
221 - cfile, ok := d.files[name]
222 - if ok {
223 - return cfile, nil
224 - }
225 -
226 - return d.childNode(name)
227 -}
228 -
229 -type NodeListing struct {
230 - Name string
231 - Type int
232 - Size int64
233 - Hash string
234 -}
235 -
236 -func (d *Directory) ListNames(ctx context.Context) ([]string, error) {
237 - d.lock.Lock()
238 - defer d.lock.Unlock()
239 -
240 - var out []string
241 - err := d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error {
242 - out = append(out, l.Name)
243 - return nil
244 - })
245 - if err != nil {
246 - return nil, err
247 - }
248 -
249 - return out, nil
250 -}
251 -
252 -func (d *Directory) List(ctx context.Context) ([]NodeListing, error) {
253 - var out []NodeListing
254 - err := d.ForEachEntry(ctx, func(nl NodeListing) error {
255 - out = append(out, nl)
256 - return nil
257 - })
258 - return out, err
259 -}
260 -
261 -func (d *Directory) ForEachEntry(ctx context.Context, f func(NodeListing) error) error {
262 - d.lock.Lock()
263 - defer d.lock.Unlock()
264 - return d.unixfsDir.ForEachLink(ctx, func(l *ipld.Link) error {
265 - c, err := d.childUnsync(l.Name)
266 - if err != nil {
267 - return err
268 - }
269 -
270 - nd, err := c.GetNode()
271 - if err != nil {
272 - return err
273 - }
274 -
275 - child := NodeListing{
276 - Name: l.Name,
277 - Type: int(c.Type()),
278 - Hash: nd.Cid().String(),
279 - }
280 -
281 - if c, ok := c.(*File); ok {
282 - size, err := c.Size()
283 - if err != nil {
284 - return err
285 - }
286 - child.Size = size
287 - }
288 -
289 - return f(child)
290 - })
291 -}
292 -
293 -func (d *Directory) Mkdir(name string) (*Directory, error) {
294 - d.lock.Lock()
295 - defer d.lock.Unlock()
296 -
297 - fsn, err := d.childUnsync(name)
298 - if err == nil {
299 - switch fsn := fsn.(type) {
300 - case *Directory:
301 - return fsn, os.ErrExist
302 - case *File:
303 - return nil, os.ErrExist
304 - default:
305 - return nil, fmt.Errorf("unrecognized type: %#v", fsn)
306 - }
307 - }
308 -
309 - ndir := ft.EmptyDirNode()
310 - ndir.SetCidBuilder(d.GetCidBuilder())
311 -
312 - err = d.dserv.Add(d.ctx, ndir)
313 - if err != nil {
314 - return nil, err
315 - }
316 -
317 - err = d.AddUnixFSChild(name, ndir)
318 - if err != nil {
319 - return nil, err
320 - }
321 -
322 - dirobj, err := NewDirectory(d.ctx, name, ndir, d, d.dserv)
323 - if err != nil {
324 - return nil, err
325 - }
326 -
327 - d.childDirs[name] = dirobj
328 - return dirobj, nil
329 -}
330 -
331 -func (d *Directory) Unlink(name string) error {
332 - d.lock.Lock()
333 - defer d.lock.Unlock()
334 -
335 - delete(d.childDirs, name)
336 - delete(d.files, name)
337 -
338 - return d.unixfsDir.RemoveChild(d.ctx, name)
339 -}
340 -
341 -func (d *Directory) Flush() error {
342 - nd, err := d.GetNode()
343 - if err != nil {
344 - return err
345 - }
346 -
347 - return d.parent.closeChild(d.name, nd, true)
348 -}
349 -
350 -// AddChild adds the node 'nd' under this directory giving it the name 'name'
351 -func (d *Directory) AddChild(name string, nd ipld.Node) error {
352 - d.lock.Lock()
353 - defer d.lock.Unlock()
354 -
355 - _, err := d.childUnsync(name)
356 - if err == nil {
357 - return ErrDirExists
358 - }
359 -
360 - err = d.dserv.Add(d.ctx, nd)
361 - if err != nil {
362 - return err
363 - }
364 -
365 - err = d.AddUnixFSChild(name, nd)
366 - if err != nil {
367 - return err
368 - }
369 -
370 - d.modTime = time.Now()
371 - return nil
372 -}
373 -
374 -// AddUnixFSChild adds a child to the inner UnixFS directory
375 -// and transitions to a HAMT implementation if needed.
376 -func (d *Directory) AddUnixFSChild(name string, node ipld.Node) error {
377 - if uio.UseHAMTSharding {
378 - // If the directory HAMT implementation is being used and this
379 - // directory is actually a basic implementation switch it to HAMT.
380 - if basicDir, ok := d.unixfsDir.(*uio.BasicDirectory); ok {
381 - hamtDir, err := basicDir.SwitchToSharding(d.ctx)
382 - if err != nil {
383 - return err
384 - }
385 - d.unixfsDir = hamtDir
386 - }
387 - }
388 -
389 - err := d.unixfsDir.AddChild(d.ctx, name, node)
390 - if err != nil {
391 - return err
392 - }
393 -
394 - return nil
395 -}
396 -
397 -func (d *Directory) sync() error {
398 - for name, dir := range d.childDirs {
399 - nd, err := dir.GetNode()
400 - if err != nil {
401 - return err
402 - }
403 -
404 - err = d.updateChild(name, nd)
405 - if err != nil {
406 - return err
407 - }
408 - }
409 -
410 - for name, file := range d.files {
411 - nd, err := file.GetNode()
412 - if err != nil {
413 - return err
414 - }
415 -
416 - err = d.updateChild(name, nd)
417 - if err != nil {
418 - return err
419 - }
420 - }
421 -
422 - return nil
423 -}
424 -
425 -func (d *Directory) Path() string {
426 - cur := d
427 - var out string
428 - for cur != nil {
429 - switch parent := cur.parent.(type) {
430 - case *Directory:
431 - out = path.Join(cur.name, out)
432 - cur = parent
433 - case *Root:
434 - return "/" + out
435 - default:
436 - panic("directory parent neither a directory nor a root")
437 - }
438 - }
439 - return out
440 -}
441 -
442 -func (d *Directory) GetNode() (ipld.Node, error) {
443 - d.lock.Lock()
444 - defer d.lock.Unlock()
445 -
446 - err := d.sync()
447 - if err != nil {
448 - return nil, err
449 - }
450 -
451 - nd, err := d.unixfsDir.GetNode()
452 - if err != nil {
453 - return nil, err
454 - }
455 -
456 - err = d.dserv.Add(d.ctx, nd)
457 - if err != nil {
458 - return nil, err
459 - }
460 -
461 - return nd.Copy(), err
462 -}
mfs/fd.go deleted
-151
@@ -1,151 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "fmt"
5 - "io"
6 -
7 - mod "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/mod"
8 -
9 - context "context"
10 -)
11 -
12 -type FileDescriptor interface {
13 - io.Reader
14 - CtxReadFull(context.Context, []byte) (int, error)
15 -
16 - io.Writer
17 - io.WriterAt
18 -
19 - io.Closer
20 - io.Seeker
21 -
22 - Truncate(int64) error
23 - Size() (int64, error)
24 - Sync() error
25 - Flush() error
26 -}
27 -
28 -type fileDescriptor struct {
29 - inode *File
30 - mod *mod.DagModifier
31 - perms int
32 - sync bool
33 - hasChanges bool
34 -
35 - closed bool
36 -}
37 -
38 -// Size returns the size of the file referred to by this descriptor
39 -func (fi *fileDescriptor) Size() (int64, error) {
40 - return fi.mod.Size()
41 -}
42 -
43 -// Truncate truncates the file to size
44 -func (fi *fileDescriptor) Truncate(size int64) error {
45 - if fi.perms == OpenReadOnly {
46 - return fmt.Errorf("cannot call truncate on readonly file descriptor")
47 - }
48 - fi.hasChanges = true
49 - return fi.mod.Truncate(size)
50 -}
51 -
52 -// Write writes the given data to the file at its current offset
53 -func (fi *fileDescriptor) Write(b []byte) (int, error) {
54 - if fi.perms == OpenReadOnly {
55 - return 0, fmt.Errorf("cannot write on not writeable descriptor")
56 - }
57 - fi.hasChanges = true
58 - return fi.mod.Write(b)
59 -}
60 -
61 -// Read reads into the given buffer from the current offset
62 -func (fi *fileDescriptor) Read(b []byte) (int, error) {
63 - if fi.perms == OpenWriteOnly {
64 - return 0, fmt.Errorf("cannot read on write-only descriptor")
65 - }
66 - return fi.mod.Read(b)
67 -}
68 -
69 -// Read reads into the given buffer from the current offset
70 -func (fi *fileDescriptor) CtxReadFull(ctx context.Context, b []byte) (int, error) {
71 - if fi.perms == OpenWriteOnly {
72 - return 0, fmt.Errorf("cannot read on write-only descriptor")
73 - }
74 - return fi.mod.CtxReadFull(ctx, b)
75 -}
76 -
77 -// Close flushes, then propogates the modified dag node up the directory structure
78 -// and signals a republish to occur
79 -func (fi *fileDescriptor) Close() error {
80 - defer func() {
81 - switch fi.perms {
82 - case OpenReadOnly:
83 - fi.inode.desclock.RUnlock()
84 - case OpenWriteOnly, OpenReadWrite:
85 - fi.inode.desclock.Unlock()
86 - }
87 - }()
88 -
89 - if fi.closed {
90 - panic("attempted to close file descriptor twice!")
91 - }
92 -
93 - if fi.hasChanges {
94 - err := fi.mod.Sync()
95 - if err != nil {
96 - return err
97 - }
98 -
99 - fi.hasChanges = false
100 -
101 - // explicitly stay locked for flushUp call,
102 - // it will manage the lock for us
103 - return fi.flushUp(fi.sync)
104 - }
105 -
106 - return nil
107 -}
108 -
109 -func (fi *fileDescriptor) Sync() error {
110 - return fi.flushUp(false)
111 -}
112 -
113 -func (fi *fileDescriptor) Flush() error {
114 - return fi.flushUp(true)
115 -}
116 -
117 -// flushUp syncs the file and adds it to the dagservice
118 -// it *must* be called with the File's lock taken
119 -func (fi *fileDescriptor) flushUp(fullsync bool) error {
120 - nd, err := fi.mod.GetNode()
121 - if err != nil {
122 - return err
123 - }
124 -
125 - err = fi.inode.dserv.Add(context.TODO(), nd)
126 - if err != nil {
127 - return err
128 - }
129 -
130 - fi.inode.nodelk.Lock()
131 - fi.inode.node = nd
132 - name := fi.inode.name
133 - parent := fi.inode.parent
134 - fi.inode.nodelk.Unlock()
135 -
136 - return parent.closeChild(name, nd, fullsync)
137 -}
138 -
139 -// Seek implements io.Seeker
140 -func (fi *fileDescriptor) Seek(offset int64, whence int) (int64, error) {
141 - return fi.mod.Seek(offset, whence)
142 -}
143 -
144 -// Write At writes the given bytes at the offset 'at'
145 -func (fi *fileDescriptor) WriteAt(b []byte, at int64) (int, error) {
146 - if fi.perms == OpenReadOnly {
147 - return 0, fmt.Errorf("cannot write on not writeable descriptor")
148 - }
149 - fi.hasChanges = true
150 - return fi.mod.WriteAt(b, at)
151 -}
mfs/file.go deleted
-146
@@ -1,146 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "context"
5 - "fmt"
6 - "sync"
7 -
8 - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
9 - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
10 - mod "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/mod"
11 -
12 - chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker"
13 - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format"
14 -)
15 -
16 -type File struct {
17 - parent childCloser
18 -
19 - name string
20 -
21 - desclock sync.RWMutex
22 -
23 - dserv ipld.DAGService
24 - node ipld.Node
25 - nodelk sync.Mutex
26 -
27 - RawLeaves bool
28 -}
29 -
30 -// NewFile returns a NewFile object with the given parameters. If the
31 -// Cid version is non-zero RawLeaves will be enabled.
32 -func NewFile(name string, node ipld.Node, parent childCloser, dserv ipld.DAGService) (*File, error) {
33 - fi := &File{
34 - dserv: dserv,
35 - parent: parent,
36 - name: name,
37 - node: node,
38 - }
39 - if node.Cid().Prefix().Version > 0 {
40 - fi.RawLeaves = true
41 - }
42 - return fi, nil
43 -}
44 -
45 -const (
46 - OpenReadOnly = iota
47 - OpenWriteOnly
48 - OpenReadWrite
49 -)
50 -
51 -func (fi *File) Open(flags int, sync bool) (FileDescriptor, error) {
52 - fi.nodelk.Lock()
53 - node := fi.node
54 - fi.nodelk.Unlock()
55 -
56 - switch node := node.(type) {
57 - case *dag.ProtoNode:
58 - fsn, err := ft.FSNodeFromBytes(node.Data())
59 - if err != nil {
60 - return nil, err
61 - }
62 -
63 - switch fsn.Type() {
64 - default:
65 - return nil, fmt.Errorf("unsupported fsnode type for 'file'")
66 - case ft.TSymlink:
67 - return nil, fmt.Errorf("symlinks not yet supported")
68 - case ft.TFile, ft.TRaw:
69 - // OK case
70 - }
71 - case *dag.RawNode:
72 - // Ok as well.
73 - }
74 -
75 - switch flags {
76 - case OpenReadOnly:
77 - fi.desclock.RLock()
78 - case OpenWriteOnly, OpenReadWrite:
79 - fi.desclock.Lock()
80 - default:
81 - // TODO: support other modes
82 - return nil, fmt.Errorf("mode not supported")
83 - }
84 -
85 - dmod, err := mod.NewDagModifier(context.TODO(), node, fi.dserv, chunker.DefaultSplitter)
86 - if err != nil {
87 - return nil, err
88 - }
89 - dmod.RawLeaves = fi.RawLeaves
90 -
91 - return &fileDescriptor{
92 - inode: fi,
93 - perms: flags,
94 - sync: sync,
95 - mod: dmod,
96 - }, nil
97 -}
98 -
99 -// Size returns the size of this file
100 -func (fi *File) Size() (int64, error) {
101 - fi.nodelk.Lock()
102 - defer fi.nodelk.Unlock()
103 - switch nd := fi.node.(type) {
104 - case *dag.ProtoNode:
105 - pbd, err := ft.FromBytes(nd.Data())
106 - if err != nil {
107 - return 0, err
108 - }
109 - return int64(pbd.GetFilesize()), nil
110 - case *dag.RawNode:
111 - return int64(len(nd.RawData())), nil
112 - default:
113 - return 0, fmt.Errorf("unrecognized node type in mfs/file.Size()")
114 - }
115 -}
116 -
117 -// GetNode returns the dag node associated with this file
118 -func (fi *File) GetNode() (ipld.Node, error) {
119 - fi.nodelk.Lock()
120 - defer fi.nodelk.Unlock()
121 - return fi.node, nil
122 -}
123 -
124 -func (fi *File) Flush() error {
125 - // open the file in fullsync mode
126 - fd, err := fi.Open(OpenWriteOnly, true)
127 - if err != nil {
128 - return err
129 - }
130 -
131 - defer fd.Close()
132 -
133 - return fd.Flush()
134 -}
135 -
136 -func (fi *File) Sync() error {
137 - // just being able to take the writelock means the descriptor is synced
138 - fi.desclock.Lock()
139 - fi.desclock.Unlock()
140 - return nil
141 -}
142 -
143 -// Type returns the type FSNode this is
144 -func (fi *File) Type() NodeType {
145 - return TFile
146 -}
mfs/mfs_test.go deleted
-1184
@@ -1,1184 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "bytes"
5 - "context"
6 - "errors"
7 - "fmt"
8 - "io"
9 - "io/ioutil"
10 - "math/rand"
11 - "os"
12 - "sort"
13 - "sync"
14 - "testing"
15 - "time"
16 -
17 - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
18 - bserv "gx/ipfs/QmTZZrpd9o4vpYr9TEADW2EoJ9fzUtAgpXqjxZHbKR2T15/go-blockservice"
19 - "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path"
20 - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
21 - importer "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/importer"
22 - uio "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs/io"
23 -
24 - u "gx/ipfs/QmPdKqUcHGFdeSpvjVoaTRPPstGif9GBZb5Q56RVw9o69A/go-ipfs-util"
25 - ds "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore"
26 - dssync "gx/ipfs/QmVG5gxteQNEMhrS8prJSmU2C9rebtFuTd3SYZ5kE3YZ5k/go-datastore/sync"
27 - offline "gx/ipfs/QmVozMmsgK2PYyaHQsrcWLBYigb1m6mW8YhCBG2Cb4Uxq9/go-ipfs-exchange-offline"
28 - chunker "gx/ipfs/QmWbCAB5f3LDumj4ncz1UCHSiyXrXxkMxZB6Wv35xi4P8z/go-ipfs-chunker"
29 - bstore "gx/ipfs/QmYBEfMSquSGnuxBthUoBJNs3F6p4VAPPvAgxq6XXGvTPh/go-ipfs-blockstore"
30 - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
31 - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format"
32 -)
33 -
34 -func emptyDirNode() *dag.ProtoNode {
35 - return dag.NodeWithData(ft.FolderPBData())
36 -}
37 -
38 -func getDagserv(t *testing.T) ipld.DAGService {
39 - db := dssync.MutexWrap(ds.NewMapDatastore())
40 - bs := bstore.NewBlockstore(db)
41 - blockserv := bserv.New(bs, offline.Exchange(bs))
42 - return dag.NewDAGService(blockserv)
43 -}
44 -
45 -func getRandFile(t *testing.T, ds ipld.DAGService, size int64) ipld.Node {
46 - r := io.LimitReader(u.NewTimeSeededRand(), size)
47 - return fileNodeFromReader(t, ds, r)
48 -}
49 -
50 -func fileNodeFromReader(t *testing.T, ds ipld.DAGService, r io.Reader) ipld.Node {
51 - nd, err := importer.BuildDagFromReader(ds, chunker.DefaultSplitter(r))
52 - if err != nil {
53 - t.Fatal(err)
54 - }
55 - return nd
56 -}
57 -
58 -func mkdirP(t *testing.T, root *Directory, pth string) *Directory {
59 - dirs := path.SplitList(pth)
60 - cur := root
61 - for _, d := range dirs {
62 - n, err := cur.Mkdir(d)
63 - if err != nil && err != os.ErrExist {
64 - t.Fatal(err)
65 - }
66 - if err == os.ErrExist {
67 - fsn, err := cur.Child(d)
68 - if err != nil {
69 - t.Fatal(err)
70 - }
71 - switch fsn := fsn.(type) {
72 - case *Directory:
73 - n = fsn
74 - case *File:
75 - t.Fatal("tried to make a directory where a file already exists")
76 - }
77 - }
78 -
79 - cur = n
80 - }
81 - return cur
82 -}
83 -
84 -func assertDirAtPath(root *Directory, pth string, children []string) error {
85 - ctx, cancel := context.WithCancel(context.Background())
86 - defer cancel()
87 -
88 - fsn, err := DirLookup(root, pth)
89 - if err != nil {
90 - return err
91 - }
92 -
93 - dir, ok := fsn.(*Directory)
94 - if !ok {
95 - return fmt.Errorf("%s was not a directory", pth)
96 - }
97 -
98 - listing, err := dir.List(ctx)
99 - if err != nil {
100 - return err
101 - }
102 -
103 - var names []string
104 - for _, d := range listing {
105 - names = append(names, d.Name)
106 - }
107 -
108 - sort.Strings(children)
109 - sort.Strings(names)
110 - if !compStrArrs(children, names) {
111 - return errors.New("directories children did not match")
112 - }
113 -
114 - return nil
115 -}
116 -
117 -func compStrArrs(a, b []string) bool {
118 - if len(a) != len(b) {
119 - return false
120 - }
121 -
122 - for i := 0; i < len(a); i++ {
123 - if a[i] != b[i] {
124 - return false
125 - }
126 - }
127 -
128 - return true
129 -}
130 -
131 -func assertFileAtPath(ds ipld.DAGService, root *Directory, expn ipld.Node, pth string) error {
132 - exp, ok := expn.(*dag.ProtoNode)
133 - if !ok {
134 - return dag.ErrNotProtobuf
135 - }
136 -
137 - parts := path.SplitList(pth)
138 - cur := root
139 - for i, d := range parts[:len(parts)-1] {
140 - next, err := cur.Child(d)
141 - if err != nil {
142 - return fmt.Errorf("looking for %s failed: %s", pth, err)
143 - }
144 -
145 - nextDir, ok := next.(*Directory)
146 - if !ok {
147 - return fmt.Errorf("%s points to a non-directory", parts[:i+1])
148 - }
149 -
150 - cur = nextDir
151 - }
152 -
153 - last := parts[len(parts)-1]
154 - finaln, err := cur.Child(last)
155 - if err != nil {
156 - return err
157 - }
158 -
159 - file, ok := finaln.(*File)
160 - if !ok {
161 - return fmt.Errorf("%s was not a file", pth)
162 - }
163 -
164 - rfd, err := file.Open(OpenReadOnly, false)
165 - if err != nil {
166 - return err
167 - }
168 -
169 - out, err := ioutil.ReadAll(rfd)
170 - if err != nil {
171 - return err
172 - }
173 -
174 - expbytes, err := catNode(ds, exp)
175 - if err != nil {
176 - return err
177 - }
178 -
179 - if !bytes.Equal(out, expbytes) {
180 - return fmt.Errorf("incorrect data at path")
181 - }
182 - return nil
183 -}
184 -
185 -func catNode(ds ipld.DAGService, nd *dag.ProtoNode) ([]byte, error) {
186 - r, err := uio.NewDagReader(context.TODO(), nd, ds)
187 - if err != nil {
188 - return nil, err
189 - }
190 - defer r.Close()
191 -
192 - return ioutil.ReadAll(r)
193 -}
194 -
195 -func setupRoot(ctx context.Context, t *testing.T) (ipld.DAGService, *Root) {
196 - ds := getDagserv(t)
197 -
198 - root := emptyDirNode()
199 - rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, c *cid.Cid) error {
200 - fmt.Println("PUBLISHED: ", c)
201 - return nil
202 - })
203 -
204 - if err != nil {
205 - t.Fatal(err)
206 - }
207 -
208 - return ds, rt
209 -}
210 -
211 -func TestBasic(t *testing.T) {
212 - ctx, cancel := context.WithCancel(context.Background())
213 - defer cancel()
214 - ds, rt := setupRoot(ctx, t)
215 -
216 - rootdir := rt.GetDirectory()
217 -
218 - // test making a basic dir
219 - _, err := rootdir.Mkdir("a")
220 - if err != nil {
221 - t.Fatal(err)
222 - }
223 -
224 - path := "a/b/c/d/e/f/g"
225 - d := mkdirP(t, rootdir, path)
226 -
227 - fi := getRandFile(t, ds, 1000)
228 -
229 - // test inserting that file
230 - err = d.AddChild("afile", fi)
231 - if err != nil {
232 - t.Fatal(err)
233 - }
234 -
235 - err = assertFileAtPath(ds, rootdir, fi, "a/b/c/d/e/f/g/afile")
236 - if err != nil {
237 - t.Fatal(err)
238 - }
239 -}
240 -
241 -func TestMkdir(t *testing.T) {
242 - ctx, cancel := context.WithCancel(context.Background())
243 - defer cancel()
244 - _, rt := setupRoot(ctx, t)
245 -
246 - rootdir := rt.GetDirectory()
247 -
248 - dirsToMake := []string{"a", "B", "foo", "bar", "cats", "fish"}
249 - sort.Strings(dirsToMake) // sort for easy comparing later
250 -
251 - for _, d := range dirsToMake {
252 - _, err := rootdir.Mkdir(d)
253 - if err != nil {
254 - t.Fatal(err)
255 - }
256 - }
257 -
258 - err := assertDirAtPath(rootdir, "/", dirsToMake)
259 - if err != nil {
260 - t.Fatal(err)
261 - }
262 -
263 - for _, d := range dirsToMake {
264 - mkdirP(t, rootdir, "a/"+d)
265 - }
266 -
267 - err = assertDirAtPath(rootdir, "/a", dirsToMake)
268 - if err != nil {
269 - t.Fatal(err)
270 - }
271 -
272 - // mkdir over existing dir should fail
273 - _, err = rootdir.Mkdir("a")
274 - if err == nil {
275 - t.Fatal("should have failed!")
276 - }
277 -}
278 -
279 -func TestDirectoryLoadFromDag(t *testing.T) {
280 - ctx, cancel := context.WithCancel(context.Background())
281 - defer cancel()
282 - ds, rt := setupRoot(ctx, t)
283 -
284 - rootdir := rt.GetDirectory()
285 -
286 - nd := getRandFile(t, ds, 1000)
287 - err := ds.Add(ctx, nd)
288 - if err != nil {
289 - t.Fatal(err)
290 - }
291 -
292 - fihash := nd.Cid()
293 -
294 - dir := emptyDirNode()
295 - err = ds.Add(ctx, dir)
296 - if err != nil {
297 - t.Fatal(err)
298 - }
299 -
300 - dirhash := dir.Cid()
301 -
302 - top := emptyDirNode()
303 - top.SetLinks([]*ipld.Link{
304 - {
305 - Name: "a",
306 - Cid: fihash,
307 - },
308 - {
309 - Name: "b",
310 - Cid: dirhash,
311 - },
312 - })
313 -
314 - err = rootdir.AddChild("foo", top)
315 - if err != nil {
316 - t.Fatal(err)
317 - }
318 -
319 - // get this dir
320 - topi, err := rootdir.Child("foo")
321 - if err != nil {
322 - t.Fatal(err)
323 - }
324 -
325 - topd := topi.(*Directory)
326 -
327 - path := topd.Path()
328 - if path != "/foo" {
329 - t.Fatalf("Expected path '/foo', got '%s'", path)
330 - }
331 -
332 - // mkdir over existing but unloaded child file should fail
333 - _, err = topd.Mkdir("a")
334 - if err == nil {
335 - t.Fatal("expected to fail!")
336 - }
337 -
338 - // mkdir over existing but unloaded child dir should fail
339 - _, err = topd.Mkdir("b")
340 - if err == nil {
341 - t.Fatal("expected to fail!")
342 - }
343 -
344 - // adding a child over an existing path fails
345 - err = topd.AddChild("b", nd)
346 - if err == nil {
347 - t.Fatal("expected to fail!")
348 - }
349 -
350 - err = assertFileAtPath(ds, rootdir, nd, "foo/a")
351 - if err != nil {
352 - t.Fatal(err)
353 - }
354 -
355 - err = assertDirAtPath(rootdir, "foo/b", nil)
356 - if err != nil {
357 - t.Fatal(err)
358 - }
359 -
360 - err = rootdir.Unlink("foo")
361 - if err != nil {
362 - t.Fatal(err)
363 - }
364 -
365 - err = assertDirAtPath(rootdir, "", nil)
366 - if err != nil {
367 - t.Fatal(err)
368 - }
369 -}
370 -
371 -func TestMfsFile(t *testing.T) {
372 - ctx, cancel := context.WithCancel(context.Background())
373 - defer cancel()
374 - ds, rt := setupRoot(ctx, t)
375 -
376 - rootdir := rt.GetDirectory()
377 -
378 - fisize := 1000
379 - nd := getRandFile(t, ds, 1000)
380 -
381 - err := rootdir.AddChild("file", nd)
382 - if err != nil {
383 - t.Fatal(err)
384 - }
385 -
386 - fsn, err := rootdir.Child("file")
387 - if err != nil {
388 - t.Fatal(err)
389 - }
390 -
391 - fi := fsn.(*File)
392 -
393 - if fi.Type() != TFile {
394 - t.Fatal("some is seriously wrong here")
395 - }
396 -
397 - wfd, err := fi.Open(OpenReadWrite, true)
398 - if err != nil {
399 - t.Fatal(err)
400 - }
401 -
402 - // assert size is as expected
403 - size, err := fi.Size()
404 - if err != nil {
405 - t.Fatal(err)
406 - }
407 - if size != int64(fisize) {
408 - t.Fatal("size isnt correct")
409 - }
410 -
411 - // write to beginning of file
412 - b := []byte("THIS IS A TEST")
413 - n, err := wfd.Write(b)
414 - if err != nil {
415 - t.Fatal(err)
416 - }
417 -
418 - if n != len(b) {
419 - t.Fatal("didnt write correct number of bytes")
420 - }
421 -
422 - // sync file
423 - err = wfd.Sync()
424 - if err != nil {
425 - t.Fatal(err)
426 - }
427 -
428 - // make sure size hasnt changed
429 - size, err = wfd.Size()
430 - if err != nil {
431 - t.Fatal(err)
432 - }
433 - if size != int64(fisize) {
434 - t.Fatal("size isnt correct")
435 - }
436 -
437 - // seek back to beginning
438 - ns, err := wfd.Seek(0, io.SeekStart)
439 - if err != nil {
440 - t.Fatal(err)
441 - }
442 -
443 - if ns != 0 {
444 - t.Fatal("didnt seek to beginning")
445 - }
446 -
447 - // read back bytes we wrote
448 - buf := make([]byte, len(b))
449 - n, err = wfd.Read(buf)
450 - if err != nil {
451 - t.Fatal(err)
452 - }
453 -
454 - if n != len(buf) {
455 - t.Fatal("didnt read enough")
456 - }
457 -
458 - if !bytes.Equal(buf, b) {
459 - t.Fatal("data read was different than data written")
460 - }
461 -
462 - // truncate file to ten bytes
463 - err = wfd.Truncate(10)
464 - if err != nil {
465 - t.Fatal(err)
466 - }
467 -
468 - size, err = wfd.Size()
469 - if err != nil {
470 - t.Fatal(err)
471 - }
472 -
473 - if size != 10 {
474 - t.Fatal("size was incorrect: ", size)
475 - }
476 -
477 - // 'writeAt' to extend it
478 - data := []byte("this is a test foo foo foo")
479 - nwa, err := wfd.WriteAt(data, 5)
480 - if err != nil {
481 - t.Fatal(err)
482 - }
483 -
484 - if nwa != len(data) {
485 - t.Fatal(err)
486 - }
487 -
488 - // assert size once more
489 - size, err = wfd.Size()
490 - if err != nil {
491 - t.Fatal(err)
492 - }
493 -
494 - if size != int64(5+len(data)) {
495 - t.Fatal("size was incorrect")
496 - }
497 -
498 - // close it out!
499 - err = wfd.Close()
500 - if err != nil {
501 - t.Fatal(err)
502 - }
503 -
504 - // make sure we can get node. TODO: verify it later
505 - _, err = fi.GetNode()
506 - if err != nil {
507 - t.Fatal(err)
508 - }
509 -}
510 -
511 -func randomWalk(d *Directory, n int) (*Directory, error) {
512 - for i := 0; i < n; i++ {
513 - dirents, err := d.List(context.Background())
514 - if err != nil {
515 - return nil, err
516 - }
517 -
518 - var childdirs []NodeListing
519 - for _, child := range dirents {
520 - if child.Type == int(TDir) {
521 - childdirs = append(childdirs, child)
522 - }
523 - }
524 - if len(childdirs) == 0 {
525 - return d, nil
526 - }
527 -
528 - next := childdirs[rand.Intn(len(childdirs))].Name
529 -
530 - nextD, err := d.Child(next)
531 - if err != nil {
532 - return nil, err
533 - }
534 -
535 - d = nextD.(*Directory)
536 - }
537 - return d, nil
538 -}
539 -
540 -func randomName() string {
541 - set := "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ1234567890_"
542 - length := rand.Intn(10) + 2
543 - var out string
544 - for i := 0; i < length; i++ {
545 - j := rand.Intn(len(set))
546 - out += set[j : j+1]
547 - }
548 - return out
549 -}
550 -
551 -func actorMakeFile(d *Directory) error {
552 - d, err := randomWalk(d, rand.Intn(7))
553 - if err != nil {
554 - return err
555 - }
556 -
557 - name := randomName()
558 - f, err := NewFile(name, dag.NodeWithData(ft.FilePBData(nil, 0)), d, d.dserv)
559 - if err != nil {
560 - return err
561 - }
562 -
563 - wfd, err := f.Open(OpenWriteOnly, true)
564 - if err != nil {
565 - return err
566 - }
567 -
568 - rread := rand.New(rand.NewSource(time.Now().UnixNano()))
569 - r := io.LimitReader(rread, int64(77*rand.Intn(123)))
570 - _, err = io.Copy(wfd, r)
571 - if err != nil {
572 - return err
573 - }
574 -
575 - return wfd.Close()
576 -}
577 -
578 -func actorMkdir(d *Directory) error {
579 - d, err := randomWalk(d, rand.Intn(7))
580 - if err != nil {
581 - return err
582 - }
583 -
584 - _, err = d.Mkdir(randomName())
585 -
586 - return err
587 -}
588 -
589 -func randomFile(d *Directory) (*File, error) {
590 - d, err := randomWalk(d, rand.Intn(6))
591 - if err != nil {
592 - return nil, err
593 - }
594 -
595 - ents, err := d.List(context.Background())
596 - if err != nil {
597 - return nil, err
598 - }
599 -
600 - var files []string
601 - for _, e := range ents {
602 - if e.Type == int(TFile) {
603 - files = append(files, e.Name)
604 - }
605 - }
606 -
607 - if len(files) == 0 {
608 - return nil, nil
609 - }
610 -
611 - fname := files[rand.Intn(len(files))]
612 - fsn, err := d.Child(fname)
613 - if err != nil {
614 - return nil, err
615 - }
616 -
617 - fi, ok := fsn.(*File)
618 - if !ok {
619 - return nil, errors.New("file wasn't a file, race?")
620 - }
621 -
622 - return fi, nil
623 -}
624 -
625 -func actorWriteFile(d *Directory) error {
626 - fi, err := randomFile(d)
627 - if err != nil {
628 - return err
629 - }
630 - if fi == nil {
631 - return nil
632 - }
633 -
634 - size := rand.Intn(1024) + 1
635 - buf := make([]byte, size)
636 - rand.Read(buf)
637 -
638 - s, err := fi.Size()
639 - if err != nil {
640 - return err
641 - }
642 -
643 - wfd, err := fi.Open(OpenWriteOnly, true)
644 - if err != nil {
645 - return err
646 - }
647 -
648 - offset := rand.Int63n(s)
649 -
650 - n, err := wfd.WriteAt(buf, offset)
651 - if err != nil {
652 - return err
653 - }
654 - if n != size {
655 - return fmt.Errorf("didnt write enough")
656 - }
657 -
658 - return wfd.Close()
659 -}
660 -
661 -func actorReadFile(d *Directory) error {
662 - fi, err := randomFile(d)
663 - if err != nil {
664 - return err
665 - }
666 - if fi == nil {
667 - return nil
668 - }
669 -
670 - _, err = fi.Size()
671 - if err != nil {
672 - return err
673 - }
674 -
675 - rfd, err := fi.Open(OpenReadOnly, false)
676 - if err != nil {
677 - return err
678 - }
679 -
680 - _, err = ioutil.ReadAll(rfd)
681 - if err != nil {
682 - return err
683 - }
684 -
685 - return rfd.Close()
686 -}
687 -
688 -func testActor(rt *Root, iterations int, errs chan error) {
689 - d := rt.GetDirectory()
690 - for i := 0; i < iterations; i++ {
691 - switch rand.Intn(5) {
692 - case 0:
693 - if err := actorMkdir(d); err != nil {
694 - errs <- err
695 - return
696 - }
697 - case 1, 2:
698 - if err := actorMakeFile(d); err != nil {
699 - errs <- err
700 - return
701 - }
702 - case 3:
703 - if err := actorWriteFile(d); err != nil {
704 - errs <- err
705 - return
706 - }
707 - case 4:
708 - if err := actorReadFile(d); err != nil {
709 - errs <- err
710 - return
711 - }
712 - }
713 - }
714 - errs <- nil
715 -}
716 -
717 -func TestMfsStress(t *testing.T) {
718 - ctx, cancel := context.WithCancel(context.Background())
719 - defer cancel()
720 - _, rt := setupRoot(ctx, t)
721 -
722 - numroutines := 10
723 -
724 - errs := make(chan error)
725 - for i := 0; i < numroutines; i++ {
726 - go testActor(rt, 50, errs)
727 - }
728 -
729 - for i := 0; i < numroutines; i++ {
730 - err := <-errs
731 - if err != nil {
732 - t.Fatal(err)
733 - }
734 - }
735 -}
736 -
737 -func TestMfsHugeDir(t *testing.T) {
738 - ctx, cancel := context.WithCancel(context.Background())
739 - defer cancel()
740 - _, rt := setupRoot(ctx, t)
741 -
742 - for i := 0; i < 10000; i++ {
743 - err := Mkdir(rt, fmt.Sprintf("/dir%d", i), MkdirOpts{Mkparents: false, Flush: false})
744 - if err != nil {
745 - t.Fatal(err)
746 - }
747 - }
748 -}
749 -
750 -func TestMkdirP(t *testing.T) {
751 - ctx, cancel := context.WithCancel(context.Background())
752 - defer cancel()
753 - _, rt := setupRoot(ctx, t)
754 -
755 - err := Mkdir(rt, "/a/b/c/d/e/f", MkdirOpts{Mkparents: true, Flush: true})
756 - if err != nil {
757 - t.Fatal(err)
758 - }
759 -}
760 -
761 -func TestConcurrentWriteAndFlush(t *testing.T) {
762 - ctx, cancel := context.WithCancel(context.Background())
763 - defer cancel()
764 - ds, rt := setupRoot(ctx, t)
765 -
766 - d := mkdirP(t, rt.GetDirectory(), "foo/bar/baz")
767 - fn := fileNodeFromReader(t, ds, bytes.NewBuffer(nil))
768 - err := d.AddChild("file", fn)
769 - if err != nil {
770 - t.Fatal(err)
771 - }
772 -
773 - nloops := 5000
774 -
775 - wg := new(sync.WaitGroup)
776 - wg.Add(1)
777 - go func() {
778 - defer wg.Done()
779 - for i := 0; i < nloops; i++ {
780 - err := writeFile(rt, "/foo/bar/baz/file", []byte("STUFF"))
781 - if err != nil {
782 - t.Error("file write failed: ", err)
783 - return
784 - }
785 - }
786 - }()
787 -
788 - for i := 0; i < nloops; i++ {
789 - _, err := rt.GetDirectory().GetNode()
790 - if err != nil {
791 - t.Fatal(err)
792 - }
793 - }
794 -
795 - wg.Wait()
796 -}
797 -
798 -func TestFlushing(t *testing.T) {
799 - ctx, cancel := context.WithCancel(context.Background())
800 - defer cancel()
801 - _, rt := setupRoot(ctx, t)
802 -
803 - dir := rt.GetDirectory()
804 - c := mkdirP(t, dir, "a/b/c")
805 - d := mkdirP(t, dir, "a/b/d")
806 - e := mkdirP(t, dir, "a/b/e")
807 -
808 - data := []byte("this is a test\n")
809 - nd1 := dag.NodeWithData(ft.FilePBData(data, uint64(len(data))))
810 -
811 - if err := c.AddChild("TEST", nd1); err != nil {
812 - t.Fatal(err)
813 - }
814 - if err := d.AddChild("TEST", nd1); err != nil {
815 - t.Fatal(err)
816 - }
817 - if err := e.AddChild("TEST", nd1); err != nil {
818 - t.Fatal(err)
819 - }
820 - if err := dir.AddChild("FILE", nd1); err != nil {
821 - t.Fatal(err)
822 - }
823 -
824 - if err := FlushPath(rt, "/a/b/c/TEST"); err != nil {
825 - t.Fatal(err)
826 - }
827 -
828 - if err := FlushPath(rt, "/a/b/d/TEST"); err != nil {
829 - t.Fatal(err)
830 - }
831 -
832 - if err := FlushPath(rt, "/a/b/e/TEST"); err != nil {
833 - t.Fatal(err)
834 - }
835 -
836 - if err := FlushPath(rt, "/FILE"); err != nil {
837 - t.Fatal(err)
838 - }
839 -
840 - rnd, err := dir.GetNode()
841 - if err != nil {
842 - t.Fatal(err)
843 - }
844 -
845 - pbrnd, ok := rnd.(*dag.ProtoNode)
846 - if !ok {
847 - t.Fatal(dag.ErrNotProtobuf)
848 - }
849 -
850 - fsnode, err := ft.FSNodeFromBytes(pbrnd.Data())
851 - if err != nil {
852 - t.Fatal(err)
853 - }
854 -
855 - if fsnode.Type() != ft.TDirectory {
856 - t.Fatal("root wasnt a directory")
857 - }
858 -
859 - rnk := rnd.Cid()
860 - exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr"
861 - if rnk.String() != exp {
862 - t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.String())
863 - }
864 -}
865 -
866 -func readFile(rt *Root, path string, offset int64, buf []byte) error {
867 - n, err := Lookup(rt, path)
868 - if err != nil {
869 - return err
870 - }
871 -
872 - fi, ok := n.(*File)
873 - if !ok {
874 - return fmt.Errorf("%s was not a file", path)
875 - }
876 -
877 - fd, err := fi.Open(OpenReadOnly, false)
878 - if err != nil {
879 - return err
880 - }
881 -
882 - _, err = fd.Seek(offset, io.SeekStart)
883 - if err != nil {
884 - return err
885 - }
886 -
887 - nread, err := fd.Read(buf)
888 - if err != nil {
889 - return err
890 - }
891 - if nread != len(buf) {
892 - return fmt.Errorf("didn't read enough")
893 - }
894 -
895 - return fd.Close()
896 -}
897 -
898 -func TestConcurrentReads(t *testing.T) {
899 - ctx, cancel := context.WithCancel(context.Background())
900 - defer cancel()
901 -
902 - ds, rt := setupRoot(ctx, t)
903 -
904 - rootdir := rt.GetDirectory()
905 -
906 - path := "a/b/c"
907 - d := mkdirP(t, rootdir, path)
908 -
909 - buf := make([]byte, 2048)
910 - rand.Read(buf)
911 -
912 - fi := fileNodeFromReader(t, ds, bytes.NewReader(buf))
913 - err := d.AddChild("afile", fi)
914 - if err != nil {
915 - t.Fatal(err)
916 - }
917 -
918 - var wg sync.WaitGroup
919 - nloops := 100
920 - for i := 0; i < 10; i++ {
921 - wg.Add(1)
922 - go func(me int) {
923 - defer wg.Done()
924 - mybuf := make([]byte, len(buf))
925 - for j := 0; j < nloops; j++ {
926 - offset := rand.Intn(len(buf))
927 - length := rand.Intn(len(buf) - offset)
928 -
929 - err := readFile(rt, "/a/b/c/afile", int64(offset), mybuf[:length])
930 - if err != nil {
931 - t.Error("readfile failed: ", err)
932 - return
933 - }
934 -
935 - if !bytes.Equal(mybuf[:length], buf[offset:offset+length]) {
936 - t.Error("incorrect read!")
937 - }
938 - }
939 - }(i)
940 - }
941 - wg.Wait()
942 -}
943 -
944 -func writeFile(rt *Root, path string, data []byte) error {
945 - n, err := Lookup(rt, path)
946 - if err != nil {
947 - return err
948 - }
949 -
950 - fi, ok := n.(*File)
951 - if !ok {
952 - return fmt.Errorf("expected to receive a file, but didnt get one")
953 - }
954 -
955 - fd, err := fi.Open(OpenWriteOnly, true)
956 - if err != nil {
957 - return err
958 - }
959 - defer fd.Close()
960 -
961 - nw, err := fd.Write(data)
962 - if err != nil {
963 - return err
964 - }
965 -
966 - if nw != len(data) {
967 - return fmt.Errorf("wrote incorrect amount: %d != 10", nw)
968 - }
969 -
970 - return nil
971 -}
972 -
973 -func TestConcurrentWrites(t *testing.T) {
974 - ctx, cancel := context.WithCancel(context.Background())
975 - defer cancel()
976 -
977 - ds, rt := setupRoot(ctx, t)
978 -
979 - rootdir := rt.GetDirectory()
980 -
981 - path := "a/b/c"
982 - d := mkdirP(t, rootdir, path)
983 -
984 - fi := fileNodeFromReader(t, ds, bytes.NewReader(make([]byte, 0)))
985 - err := d.AddChild("afile", fi)
986 - if err != nil {
987 - t.Fatal(err)
988 - }
989 -
990 - var wg sync.WaitGroup
991 - nloops := 100
992 - for i := 0; i < 10; i++ {
993 - wg.Add(1)
994 - go func(me int) {
995 - defer wg.Done()
996 - mybuf := bytes.Repeat([]byte{byte(me)}, 10)
997 - for j := 0; j < nloops; j++ {
998 - err := writeFile(rt, "a/b/c/afile", mybuf)
999 - if err != nil {
1000 - t.Error("writefile failed: ", err)
1001 - return
1002 - }
1003 - }
1004 - }(i)
1005 - }
1006 - wg.Wait()
1007 -}
1008 -
1009 -func TestFileDescriptors(t *testing.T) {
1010 - ctx, cancel := context.WithCancel(context.Background())
1011 - defer cancel()
1012 -
1013 - ds, rt := setupRoot(ctx, t)
1014 - dir := rt.GetDirectory()
1015 -
1016 - nd := dag.NodeWithData(ft.FilePBData(nil, 0))
1017 - fi, err := NewFile("test", nd, dir, ds)
1018 - if err != nil {
1019 - t.Fatal(err)
1020 - }
1021 -
1022 - // test read only
1023 - rfd1, err := fi.Open(OpenReadOnly, false)
1024 - if err != nil {
1025 - t.Fatal(err)
1026 - }
1027 -
1028 - err = rfd1.Truncate(0)
1029 - if err == nil {
1030 - t.Fatal("shouldnt be able to truncate readonly fd")
1031 - }
1032 -
1033 - _, err = rfd1.Write([]byte{})
1034 - if err == nil {
1035 - t.Fatal("shouldnt be able to write to readonly fd")
1036 - }
1037 -
1038 - _, err = rfd1.Read([]byte{})
1039 - if err != nil {
1040 - t.Fatalf("expected to be able to read from file: %s", err)
1041 - }
1042 -
1043 - done := make(chan struct{})
1044 - go func() {
1045 - defer close(done)
1046 - // can open second readonly file descriptor
1047 - rfd2, err := fi.Open(OpenReadOnly, false)
1048 - if err != nil {
1049 - t.Error(err)
1050 - return
1051 - }
1052 -
1053 - rfd2.Close()
1054 - }()
1055 -
1056 - select {
1057 - case <-time.After(time.Second):
1058 - t.Fatal("open second file descriptor failed")
1059 - case <-done:
1060 - }
1061 -
1062 - if t.Failed() {
1063 - return
1064 - }
1065 -
1066 - // test not being able to open for write until reader are closed
1067 - done = make(chan struct{})
1068 - go func() {
1069 - defer close(done)
1070 - wfd1, err := fi.Open(OpenWriteOnly, true)
1071 - if err != nil {
1072 - t.Error(err)
1073 - }
1074 -
1075 - wfd1.Close()
1076 - }()
1077 -
1078 - select {
1079 - case <-time.After(time.Millisecond * 200):
1080 - case <-done:
1081 - if t.Failed() {
1082 - return
1083 - }
1084 -
1085 - t.Fatal("shouldnt have been able to open file for writing")
1086 - }
1087 -
1088 - err = rfd1.Close()
1089 - if err != nil {
1090 - t.Fatal(err)
1091 - }
1092 -
1093 - select {
1094 - case <-time.After(time.Second):
1095 - t.Fatal("should have been able to open write fd after closing read fd")
1096 - case <-done:
1097 - }
1098 -
1099 - wfd, err := fi.Open(OpenWriteOnly, true)
1100 - if err != nil {
1101 - t.Fatal(err)
1102 - }
1103 -
1104 - _, err = wfd.Read([]byte{})
1105 - if err == nil {
1106 - t.Fatal("shouldnt have been able to read from write only filedescriptor")
1107 - }
1108 -
1109 - _, err = wfd.Write([]byte{})
1110 - if err != nil {
1111 - t.Fatal(err)
1112 - }
1113 -}
1114 -
1115 -func TestTruncateAtSize(t *testing.T) {
1116 - ctx, cancel := context.WithCancel(context.Background())
1117 - defer cancel()
1118 - ds, rt := setupRoot(ctx, t)
1119 -
1120 - dir := rt.GetDirectory()
1121 -
1122 - nd := dag.NodeWithData(ft.FilePBData(nil, 0))
1123 - fi, err := NewFile("test", nd, dir, ds)
1124 - if err != nil {
1125 - t.Fatal(err)
1126 - }
1127 -
1128 - fd, err := fi.Open(OpenReadWrite, true)
1129 - if err != nil {
1130 - t.Fatal(err)
1131 - }
1132 - defer fd.Close()
1133 - _, err = fd.Write([]byte("test"))
1134 - if err != nil {
1135 - t.Fatal(err)
1136 - }
1137 - fd.Truncate(4)
1138 -}
1139 -
1140 -func TestTruncateAndWrite(t *testing.T) {
1141 - ctx, cancel := context.WithCancel(context.Background())
1142 - defer cancel()
1143 - ds, rt := setupRoot(ctx, t)
1144 -
1145 - dir := rt.GetDirectory()
1146 -
1147 - nd := dag.NodeWithData(ft.FilePBData(nil, 0))
1148 - fi, err := NewFile("test", nd, dir, ds)
1149 - if err != nil {
1150 - t.Fatal(err)
1151 - }
1152 -
1153 - fd, err := fi.Open(OpenReadWrite, true)
1154 - defer fd.Close()
1155 - if err != nil {
1156 - t.Fatal(err)
1157 - }
1158 - for i := 0; i < 200; i++ {
1159 - err = fd.Truncate(0)
1160 - if err != nil {
1161 - t.Fatal(err)
1162 - }
1163 - l, err := fd.Write([]byte("test"))
1164 - if err != nil {
1165 - t.Fatal(err)
1166 - }
1167 - if l != len("test") {
1168 - t.Fatal("incorrect write length")
1169 - }
1170 -
1171 - _, err = fd.Seek(0, io.SeekStart)
1172 - if err != nil {
1173 - t.Fatal(err)
1174 - }
1175 -
1176 - data, err := ioutil.ReadAll(fd)
1177 - if err != nil {
1178 - t.Fatal(err)
1179 - }
1180 - if string(data) != "test" {
1181 - t.Fatalf("read error at read %d, read: %v", i, data)
1182 - }
1183 - }
1184 -}
mfs/ops.go deleted
-223
@@ -1,223 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "fmt"
5 - "os"
6 - gopath "path"
7 - "strings"
8 -
9 - path "gx/ipfs/QmWMcvZbNvk5codeqbm7L89C9kqSwka4KaHnDb8HRnxsSL/go-path"
10 -
11 - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
12 - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format"
13 -)
14 -
15 -// Mv moves the file or directory at 'src' to 'dst'
16 -func Mv(r *Root, src, dst string) error {
17 - srcDir, srcFname := gopath.Split(src)
18 -
19 - var dstDirStr string
20 - var filename string
21 - if dst[len(dst)-1] == '/' {
22 - dstDirStr = dst
23 - filename = srcFname
24 - } else {
25 - dstDirStr, filename = gopath.Split(dst)
26 - }
27 -
28 - // get parent directories of both src and dest first
29 - dstDir, err := lookupDir(r, dstDirStr)
30 - if err != nil {
31 - return err
32 - }
33 -
34 - srcDirObj, err := lookupDir(r, srcDir)
35 - if err != nil {
36 - return err
37 - }
38 -
39 - srcObj, err := srcDirObj.Child(srcFname)
40 - if err != nil {
41 - return err
42 - }
43 -
44 - nd, err := srcObj.GetNode()
45 - if err != nil {
46 - return err
47 - }
48 -
49 - fsn, err := dstDir.Child(filename)
50 - if err == nil {
51 - switch n := fsn.(type) {
52 - case *File:
53 - _ = dstDir.Unlink(filename)
54 - case *Directory:
55 - dstDir = n
56 - default:
57 - return fmt.Errorf("unexpected type at path: %s", dst)
58 - }
59 - } else if err != os.ErrNotExist {
60 - return err
61 - }
62 -
63 - err = dstDir.AddChild(filename, nd)
64 - if err != nil {
65 - return err
66 - }
67 -
68 - return srcDirObj.Unlink(srcFname)
69 -}
70 -
71 -func lookupDir(r *Root, path string) (*Directory, error) {
72 - di, err := Lookup(r, path)
73 - if err != nil {
74 - return nil, err
75 - }
76 -
77 - d, ok := di.(*Directory)
78 - if !ok {
79 - return nil, fmt.Errorf("%s is not a directory", path)
80 - }
81 -
82 - return d, nil
83 -}
84 -
85 -// PutNode inserts 'nd' at 'path' in the given mfs
86 -func PutNode(r *Root, path string, nd ipld.Node) error {
87 - dirp, filename := gopath.Split(path)
88 - if filename == "" {
89 - return fmt.Errorf("cannot create file with empty name")
90 - }
91 -
92 - pdir, err := lookupDir(r, dirp)
93 - if err != nil {
94 - return err
95 - }
96 -
97 - return pdir.AddChild(filename, nd)
98 -}
99 -
100 -// MkdirOpts is used by Mkdir
101 -type MkdirOpts struct {
102 - Mkparents bool
103 - Flush bool
104 - CidBuilder cid.Builder
105 -}
106 -
107 -// Mkdir creates a directory at 'path' under the directory 'd', creating
108 -// intermediary directories as needed if 'mkparents' is set to true
109 -func Mkdir(r *Root, pth string, opts MkdirOpts) error {
110 - if pth == "" {
111 - return fmt.Errorf("no path given to Mkdir")
112 - }
113 - parts := path.SplitList(pth)
114 - if parts[0] == "" {
115 - parts = parts[1:]
116 - }
117 -
118 - // allow 'mkdir /a/b/c/' to create c
119 - if parts[len(parts)-1] == "" {
120 - parts = parts[:len(parts)-1]
121 - }
122 -
123 - if len(parts) == 0 {
124 - // this will only happen on 'mkdir /'
125 - if opts.Mkparents {
126 - return nil
127 - }
128 - return fmt.Errorf("cannot create directory '/': Already exists")
129 - }
130 -
131 - cur := r.GetDirectory()
132 - for i, d := range parts[:len(parts)-1] {
133 - fsn, err := cur.Child(d)
134 - if err == os.ErrNotExist && opts.Mkparents {
135 - mkd, err := cur.Mkdir(d)
136 - if err != nil {
137 - return err
138 - }
139 - if opts.CidBuilder != nil {
140 - mkd.SetCidBuilder(opts.CidBuilder)
141 - }
142 - fsn = mkd
143 - } else if err != nil {
144 - return err
145 - }
146 -
147 - next, ok := fsn.(*Directory)
148 - if !ok {
149 - return fmt.Errorf("%s was not a directory", path.Join(parts[:i]))
150 - }
151 - cur = next
152 - }
153 -
154 - final, err := cur.Mkdir(parts[len(parts)-1])
155 - if err != nil {
156 - if !opts.Mkparents || err != os.ErrExist || final == nil {
157 - return err
158 - }
159 - }
160 - if opts.CidBuilder != nil {
161 - final.SetCidBuilder(opts.CidBuilder)
162 - }
163 -
164 - if opts.Flush {
165 - err := final.Flush()
166 - if err != nil {
167 - return err
168 - }
169 - }
170 -
171 - return nil
172 -}
173 -
174 -// Lookup extracts the root directory and performs a lookup under it.
175 -// TODO: Now that the root is always a directory, can this function
176 -// be collapsed with `DirLookup`? Or at least be made a method of `Root`?
177 -func Lookup(r *Root, path string) (FSNode, error) {
178 - dir := r.GetDirectory()
179 -
180 - return DirLookup(dir, path)
181 -}
182 -
183 -// DirLookup will look up a file or directory at the given path
184 -// under the directory 'd'
185 -func DirLookup(d *Directory, pth string) (FSNode, error) {
186 - pth = strings.Trim(pth, "/")
187 - parts := path.SplitList(pth)
188 - if len(parts) == 1 && parts[0] == "" {
189 - return d, nil
190 - }
191 -
192 - var cur FSNode
193 - cur = d
194 - for i, p := range parts {
195 - chdir, ok := cur.(*Directory)
196 - if !ok {
197 - return nil, fmt.Errorf("cannot access %s: Not a directory", path.Join(parts[:i+1]))
198 - }
199 -
200 - child, err := chdir.Child(p)
201 - if err != nil {
202 - return nil, err
203 - }
204 -
205 - cur = child
206 - }
207 - return cur, nil
208 -}
209 -
210 -func FlushPath(rt *Root, pth string) error {
211 - nd, err := Lookup(rt, pth)
212 - if err != nil {
213 - return err
214 - }
215 -
216 - err = nd.Flush()
217 - if err != nil {
218 - return err
219 - }
220 -
221 - rt.repub.WaitPub()
222 - return nil
223 -}
mfs/repub_test.go deleted
-77
@@ -1,77 +0,0 @@
1 -package mfs
2 -
3 -import (
4 - "context"
5 - "testing"
6 - "time"
7 -
8 - ci "gx/ipfs/QmXG74iiKQnDstVQq9fPFQEB6JTNSWBbAWE1qsq6L4E5sR/go-testutil/ci"
9 - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
10 -)
11 -
12 -func TestRepublisher(t *testing.T) {
13 - if ci.IsRunning() {
14 - t.Skip("dont run timing tests in CI")
15 - }
16 -
17 - ctx := context.TODO()
18 -
19 - pub := make(chan struct{})
20 -
21 - pf := func(ctx context.Context, c *cid.Cid) error {
22 - pub <- struct{}{}
23 - return nil
24 - }
25 -
26 - tshort := time.Millisecond * 50
27 - tlong := time.Second / 2
28 -
29 - rp := NewRepublisher(ctx, pf, tshort, tlong)
30 - go rp.Run()
31 -
32 - rp.Update(nil)
33 -
34 - // should hit short timeout
35 - select {
36 - case <-time.After(tshort * 2):
37 - t.Fatal("publish didnt happen in time")
38 - case <-pub:
39 - }
40 -
41 - cctx, cancel := context.WithCancel(context.Background())
42 -
43 - go func() {
44 - for {
45 - rp.Update(nil)
46 - time.Sleep(time.Millisecond * 10)
47 - select {
48 - case <-cctx.Done():
49 - return
50 - default:
51 - }
52 - }
53 - }()
54 -
55 - select {
56 - case <-pub:
57 - t.Fatal("shouldnt have received publish yet!")
58 - case <-time.After((tlong * 9) / 10):
59 - }
60 - select {
61 - case <-pub:
62 - case <-time.After(tlong / 2):
63 - t.Fatal("waited too long for pub!")
64 - }
65 -
66 - cancel()
67 -
68 - go func() {
69 - err := rp.Close()
70 - if err != nil {
71 - t.Fatal(err)
72 - }
73 - }()
74 -
75 - // final pub from closing
76 - <-pub
77 -}
mfs/system.go deleted
-292
@@ -1,292 +0,0 @@
1 -// package mfs implements an in memory model of a mutable IPFS filesystem.
2 -//
3 -// It consists of four main structs:
4 -// 1) The Filesystem
5 -// The filesystem serves as a container and entry point for various mfs filesystems
6 -// 2) Root
7 -// Root represents an individual filesystem mounted within the mfs system as a whole
8 -// 3) Directories
9 -// 4) Files
10 -package mfs
11 -
12 -import (
13 - "context"
14 - "errors"
15 - "fmt"
16 - "sync"
17 - "time"
18 -
19 - dag "gx/ipfs/QmQzSpSjkdGHW6WFBhUG6P3t9K8yv7iucucT1cQaqJ6tgd/go-merkledag"
20 - ft "gx/ipfs/QmWv8MYwgPK4zXYv1et1snWJ6FWGqaL6xY2y9X1bRSKBxk/go-unixfs"
21 -
22 - logging "gx/ipfs/QmRREK2CAZ5Re2Bd9zZFG6FeYDppUWt5cMgsoUEp3ktgSr/go-log"
23 - cid "gx/ipfs/QmYjnkEL7i731PirfVH1sis89evN7jt4otSHw5D2xXXwUV/go-cid"
24 - ipld "gx/ipfs/QmaA8GkXUYinkkndvg7T6Tx7gYXemhxjaxLisEPes7Rf1P/go-ipld-format"
25 -)
26 -
27 -var ErrNotExist = errors.New("no such rootfs")
28 -
29 -var log = logging.Logger("mfs")
30 -
31 -var ErrIsDirectory = errors.New("error: is a directory")
32 -
33 -type childCloser interface {
34 - closeChild(string, ipld.Node, bool) error
35 -}
36 -
37 -type NodeType int
38 -
39 -const (
40 - TFile NodeType = iota
41 - TDir
42 -)
43 -
44 -// FSNode represents any node (directory, root, or file) in the mfs filesystem.
45 -type FSNode interface {
46 - GetNode() (ipld.Node, error)
47 - Flush() error
48 - Type() NodeType
49 -}
50 -
51 -// Root represents the root of a filesystem tree.
52 -type Root struct {
53 -
54 - // Root directory of the MFS layout.
55 - dir *Directory
56 -
57 - repub *Republisher
58 -}
59 -
60 -// PubFunc is the function used by the `publish()` method.
61 -type PubFunc func(context.Context, *cid.Cid) error
62 -
63 -// NewRoot creates a new Root and starts up a republisher routine for it.
64 -func NewRoot(parent context.Context, ds ipld.DAGService, node *dag.ProtoNode, pf PubFunc) (*Root, error) {
65 -
66 - var repub *Republisher
67 - if pf != nil {
68 - repub = NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3)
69 - repub.setVal(node.Cid())
70 - go repub.Run()
71 - }
72 -
73 - root := &Root{
74 - repub: repub,
75 - }
76 -
77 - pbn, err := ft.FromBytes(node.Data())
78 - if err != nil {
79 - log.Error("IPNS pointer was not unixfs node")
80 - return nil, err
81 - }
82 -
83 - switch pbn.GetType() {
84 - case ft.TDirectory, ft.THAMTShard:
85 - newDir, err := NewDirectory(parent, node.String(), node, root, ds)
86 - if err != nil {
87 - return nil, err
88 - }
89 -
90 - root.dir = newDir
91 - case ft.TFile, ft.TMetadata, ft.TRaw:
92 - return nil, fmt.Errorf("root can't be a file (unixfs type: %s)", pbn.GetType())
93 - default:
94 - return nil, fmt.Errorf("unrecognized unixfs type: %s", pbn.GetType())
95 - }
96 - return root, nil
97 -}
98 -
99 -// GetDirectory returns the root directory.
100 -func (kr *Root) GetDirectory() *Directory {
101 - return kr.dir
102 -}
103 -
104 -// Flush signals that an update has occurred since the last publish,
105 -// and updates the Root republisher.
106 -func (kr *Root) Flush() error {
107 - nd, err := kr.GetDirectory().GetNode()
108 - if err != nil {
109 - return err
110 - }
111 -
112 - if kr.repub != nil {
113 - kr.repub.Update(nd.Cid())
114 - }
115 - return nil
116 -}
117 -
118 -// FlushMemFree flushes the root directory and then uncaches all of its links.
119 -// This has the effect of clearing out potentially stale references and allows
120 -// them to be garbage collected.
121 -// CAUTION: Take care not to ever call this while holding a reference to any
122 -// child directories. Those directories will be bad references and using them
123 -// may have unintended racy side effects.
124 -// A better implemented mfs system (one that does smarter internal caching and
125 -// refcounting) shouldnt need this method.
126 -func (kr *Root) FlushMemFree(ctx context.Context) error {
127 - dir := kr.GetDirectory()
128 -
129 - if err := dir.Flush(); err != nil {
130 - return err
131 - }
132 -
133 - dir.lock.Lock()
134 - defer dir.lock.Unlock()
135 - for name := range dir.files {
136 - delete(dir.files, name)
137 - }
138 - for name := range dir.childDirs {
139 - delete(dir.childDirs, name)
140 - }
141 -
142 - return nil
143 -}
144 -
145 -// closeChild implements the childCloser interface, and signals to the publisher that
146 -// there are changes ready to be published.
147 -func (kr *Root) closeChild(name string, nd ipld.Node, sync bool) error {
148 - err := kr.GetDirectory().dserv.Add(context.TODO(), nd)
149 - if err != nil {
150 - return err
151 - }
152 -
153 - if kr.repub != nil {
154 - kr.repub.Update(nd.Cid())
155 - }
156 - return nil
157 -}
158 -
159 -func (kr *Root) Close() error {
160 - nd, err := kr.GetDirectory().GetNode()
161 - if err != nil {
162 - return err
163 - }
164 -
165 - if kr.repub != nil {
166 - kr.repub.Update(nd.Cid())
167 - return kr.repub.Close()
168 - }
169 -
170 - return nil
171 -}
172 -
173 -// Republisher manages when to publish a given entry.
174 -type Republisher struct {
175 - TimeoutLong time.Duration
176 - TimeoutShort time.Duration
177 - Publish chan struct{}
178 - pubfunc PubFunc
179 - pubnowch chan chan struct{}
180 -
181 - ctx context.Context
182 - cancel func()
183 -
184 - lk sync.Mutex
185 - val *cid.Cid
186 - lastpub *cid.Cid
187 -}
188 -
189 -// NewRepublisher creates a new Republisher object to republish the given root
190 -// using the given short and long time intervals.
191 -func NewRepublisher(ctx context.Context, pf PubFunc, tshort, tlong time.Duration) *Republisher {
192 - ctx, cancel := context.WithCancel(ctx)
193 - return &Republisher{
194 - TimeoutShort: tshort,
195 - TimeoutLong: tlong,
196 - Publish: make(chan struct{}, 1),
197 - pubfunc: pf,
198 - pubnowch: make(chan chan struct{}),
199 - ctx: ctx,
200 - cancel: cancel,
201 - }
202 -}
203 -
204 -func (p *Republisher) setVal(c *cid.Cid) {
205 - p.lk.Lock()
206 - defer p.lk.Unlock()
207 - p.val = c
208 -}
209 -
210 -// WaitPub Returns immediately if `lastpub` value is consistent with the
211 -// current value `val`, else will block until `val` has been published.
212 -func (p *Republisher) WaitPub() {
213 - p.lk.Lock()
214 - consistent := p.lastpub == p.val
215 - p.lk.Unlock()
216 - if consistent {
217 - return
218 - }
219 -
220 - wait := make(chan struct{})
221 - p.pubnowch <- wait
222 - <-wait
223 -}
224 -
225 -func (p *Republisher) Close() error {
226 - err := p.publish(p.ctx)
227 - p.cancel()
228 - return err
229 -}
230 -
231 -// Touch signals that an update has occurred since the last publish.
232 -// Multiple consecutive touches may extend the time period before
233 -// the next Publish occurs in order to more efficiently batch updates.
234 -func (np *Republisher) Update(c *cid.Cid) {
235 - np.setVal(c)
236 - select {
237 - case np.Publish <- struct{}{}:
238 - default:
239 - }
240 -}
241 -
242 -// Run is the main republisher loop.
243 -func (np *Republisher) Run() {
244 - for {
245 - select {
246 - case <-np.Publish:
247 - quick := time.After(np.TimeoutShort)
248 - longer := time.After(np.TimeoutLong)
249 -
250 - wait:
251 - var pubnowresp chan struct{}
252 -
253 - select {
254 - case <-np.ctx.Done():
255 - return
256 - case <-np.Publish:
257 - quick = time.After(np.TimeoutShort)
258 - goto wait
259 - case <-quick:
260 - case <-longer:
261 - case pubnowresp = <-np.pubnowch:
262 - }
263 -
264 - err := np.publish(np.ctx)
265 - if pubnowresp != nil {
266 - pubnowresp <- struct{}{}
267 - }
268 - if err != nil {
269 - log.Errorf("republishRoot error: %s", err)
270 - }
271 -
272 - case <-np.ctx.Done():
273 - return
274 - }
275 - }
276 -}
277 -
278 -// publish calls the `PubFunc`.
279 -func (np *Republisher) publish(ctx context.Context) error {
280 - np.lk.Lock()
281 - topub := np.val
282 - np.lk.Unlock()
283 -
284 - err := np.pubfunc(ctx, topub)
285 - if err != nil {
286 - return err
287 - }
288 - np.lk.Lock()
289 - np.lastpub = topub
290 - np.lk.Unlock()
291 - return nil
292 -}
package.json
+7
@@ -527,6 +527,12 @@
527 "hash": "QmfMirfpEKQFctVpBYTvETxxLoU5q4ZJWsAMrtwSSE2bkn",
528 "name": "go-verifcid",
529 "version": "0.0.3"
530 + },
531 + {
532 + "author": "hsanjuan",
533 + "hash": "QmQeLRo7dHKpmREWkAUXRAri4Bro3BqrDVJJHjHHmKSHMc",
534 + "name": "go-mfs",
535 + "version": "0.0.1"
536 }
537 ],
538 "gxVersion": "0.10.0",
@@ -535,3 +541,4 @@
541 "name": "go-ipfs",
542 "version": "0.4.18-dev"
543 }
544 +