Move parts of `ipfs add` into core/coreunix
License: MIT Signed-off-by: rht <rhtbot@gmail.com>
rht committed
Oct 3, 2015 at 13:59 UTC
86901aff3c309065285250b2007160f6685a5963
3 files changed
+312
-326
core/commands/add.go
+12
-285
@@ -3,34 +3,19 @@ package commands
3
import (
4
"fmt"
5
"io"
6
- "path"
6
7
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/cheggaaa/pb"
9
- ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
- syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
11
- cxt "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
8
+ "github.com/ipfs/go-ipfs/core/coreunix"
9
13
- bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
14
- bserv "github.com/ipfs/go-ipfs/blockservice"
10
cmds "github.com/ipfs/go-ipfs/commands"
11
files "github.com/ipfs/go-ipfs/commands/files"
12
core "github.com/ipfs/go-ipfs/core"
18
- offline "github.com/ipfs/go-ipfs/exchange/offline"
19
- importer "github.com/ipfs/go-ipfs/importer"
20
- "github.com/ipfs/go-ipfs/importer/chunk"
21
- dag "github.com/ipfs/go-ipfs/merkledag"
22
- dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
23
- pin "github.com/ipfs/go-ipfs/pin"
24
- ft "github.com/ipfs/go-ipfs/unixfs"
13
u "github.com/ipfs/go-ipfs/util"
14
)
15
16
// Error indicating the max depth has been exceded.
17
var ErrDepthLimitExceeded = fmt.Errorf("depth limit exceeded")
18
31
-// how many bytes of progress to wait before sending a progress update message
32
-const progressReaderIncrement = 1024 * 256
33
-
19
const (
20
quietOptionName = "quiet"
21
progressOptionName = "progress"
@@ -41,12 +26,6 @@ const (
26
chunkerOptionName = "chunker"
27
)
28
44
-type AddedObject struct {
45
- Name string
46
- Hash string `json:",omitempty"`
47
- Bytes int64 `json:",omitempty"`
48
-}
49
-
29
var AddCmd = &cmds.Command{
30
Helptext: cmds.HelpText{
31
Tagline: "Add an object to ipfs.",
@@ -116,7 +95,6 @@ remains to be implemented.
95
hidden, _, _ := req.Option(hiddenOptionName).Bool()
96
chunker, _, _ := req.Option(chunkerOptionName).String()
97
119
- e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
98
if hash {
99
nilnode, err := core.NewNode(n.Context(), &core.BuildCfg{
100
//TODO: need this to be true or all files
@@ -133,17 +111,12 @@ remains to be implemented.
111
outChan := make(chan interface{}, 8)
112
res.SetOutput((<-chan interface{})(outChan))
113
136
- fileAdder := adder{
137
- ctx: req.Context(),
138
- node: n,
139
- editor: e,
140
- out: outChan,
141
- chunker: chunker,
142
- progress: progress,
143
- hidden: hidden,
144
- trickle: trickle,
145
- wrap: wrap,
146
- }
114
+ fileAdder := coreunix.NewAdder(req.Context(), n, outChan)
115
+ fileAdder.Chunker = chunker
116
+ fileAdder.Progress = progress
117
+ fileAdder.Hidden = hidden
118
+ fileAdder.Trickle = trickle
119
+ fileAdder.Wrap = wrap
120
121
// addAllFiles loops over a convenience slice file to
122
// add each file individually. e.g. 'ipfs add a b c'
@@ -157,22 +130,12 @@ remains to be implemented.
130
return nil // done
131
}
132
160
- if _, err := fileAdder.addFile(file); err != nil {
133
+ if _, err := fileAdder.AddFile(file); err != nil {
134
return err
135
}
136
}
137
}
138
166
- pinRoot := func(rootnd *dag.Node) error {
167
- rnk, err := rootnd.Key()
168
- if err != nil {
169
- return err
170
- }
171
-
172
- n.Pinning.PinWithMode(rnk, pin.Recursive)
173
- return n.Pinning.Flush()
174
- }
175
-
139
addAllAndPin := func(f files.File) error {
140
if err := addAllFiles(f); err != nil {
141
return err
@@ -180,19 +143,14 @@ remains to be implemented.
143
144
if !hash {
145
// copy intermediary nodes from editor to our actual dagservice
183
- err := e.WriteOutputTo(n.DAG)
146
+ err := fileAdder.WriteOutputTo(n.DAG)
147
if err != nil {
148
log.Error("WRITE OUT: ", err)
149
return err
150
}
151
}
152
190
- rootnd, err := fileAdder.RootNode()
191
- if err != nil {
192
- return err
193
- }
194
-
195
- return pinRoot(rootnd)
153
+ return fileAdder.PinRoot()
154
}
155
156
go func() {
@@ -251,7 +209,7 @@ remains to be implemented.
209
var totalProgress, prevFiles, lastBytes int64
210
211
for out := range outChan {
254
- output := out.(*AddedObject)
212
+ output := out.(*coreunix.AddedObject)
213
if len(output.Hash) > 0 {
214
if showProgressBar {
215
// clear progress bar line before we print "added x" output
@@ -287,236 +245,5 @@ remains to be implemented.
245
}
246
}
247
},
290
- Type: AddedObject{},
291
-}
292
-
293
-func NewMemoryDagService() dag.DAGService {
294
- // build mem-datastore for editor's intermediary nodes
295
- bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
296
- bsrv := bserv.New(bs, offline.Exchange(bs))
297
- return dag.NewDAGService(bsrv)
298
-}
299
-
300
-// Internal structure for holding the switches passed to the `add` call
301
-type adder struct {
302
- ctx cxt.Context
303
- node *core.IpfsNode
304
- editor *dagutils.Editor
305
- out chan interface{}
306
- progress bool
307
- hidden bool
308
- trickle bool
309
- wrap bool
310
- chunker string
311
-
312
- nextUntitled int
313
-}
314
-
315
-// Perform the actual add & pin locally, outputting results to reader
316
-func add(n *core.IpfsNode, reader io.Reader, useTrickle bool, chunker string) (*dag.Node, error) {
317
- chnk, err := chunk.FromString(reader, chunker)
318
- if err != nil {
319
- return nil, err
320
- }
321
-
322
- var node *dag.Node
323
- if useTrickle {
324
- node, err = importer.BuildTrickleDagFromReader(
325
- n.DAG,
326
- chnk,
327
- )
328
- } else {
329
- node, err = importer.BuildDagFromReader(
330
- n.DAG,
331
- chnk,
332
- )
333
- }
334
-
335
- if err != nil {
336
- return nil, err
337
- }
338
-
339
- return node, nil
340
-}
341
-
342
-func (params *adder) RootNode() (*dag.Node, error) {
343
- r := params.editor.GetNode()
344
-
345
- // if not wrapping, AND one root file, use that hash as root.
346
- if !params.wrap && len(r.Links) == 1 {
347
- var err error
348
- r, err = r.Links[0].GetNode(params.ctx, params.editor.GetDagService())
349
- // no need to output, as we've already done so.
350
- return r, err
351
- }
352
-
353
- // otherwise need to output, as we have not.
354
- err := outputDagnode(params.out, "", r)
355
- return r, err
356
-}
357
-
358
-func (params *adder) addNode(node *dag.Node, path string) error {
359
- // patch it into the root
360
- if path == "" {
361
- key, err := node.Key()
362
- if err != nil {
363
- return err
364
- }
365
-
366
- path = key.Pretty()
367
- }
368
-
369
- if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil {
370
- return err
371
- }
372
-
373
- return outputDagnode(params.out, path, node)
374
-}
375
-
376
-// Add the given file while respecting the params.
377
-func (params *adder) addFile(file files.File) (*dag.Node, error) {
378
- // Check if file is hidden
379
- if fileIsHidden := files.IsHidden(file); fileIsHidden && !params.hidden {
380
- log.Debugf("%s is hidden, skipping", file.FileName())
381
- return nil, &hiddenFileError{file.FileName()}
382
- }
383
-
384
- // Check if "file" is actually a directory
385
- if file.IsDirectory() {
386
- return params.addDir(file)
387
- }
388
-
389
- if s, ok := file.(*files.Symlink); ok {
390
- sdata, err := ft.SymlinkData(s.Target)
391
- if err != nil {
392
- return nil, err
393
- }
394
-
395
- dagnode := &dag.Node{Data: sdata}
396
- _, err = params.node.DAG.Add(dagnode)
397
- if err != nil {
398
- return nil, err
399
- }
400
-
401
- err = params.addNode(dagnode, s.FileName())
402
- return dagnode, err
403
- }
404
-
405
- // if the progress flag was specified, wrap the file so that we can send
406
- // progress updates to the client (over the output channel)
407
- var reader io.Reader = file
408
- if params.progress {
409
- reader = &progressReader{file: file, out: params.out}
410
- }
411
-
412
- dagnode, err := add(params.node, reader, params.trickle, params.chunker)
413
- if err != nil {
414
- return nil, err
415
- }
416
-
417
- // patch it into the root
418
- log.Infof("adding file: %s", file.FileName())
419
- err = params.addNode(dagnode, file.FileName())
420
- return dagnode, err
421
-}
422
-
423
-func (params *adder) addDir(file files.File) (*dag.Node, error) {
424
- tree := &dag.Node{Data: ft.FolderPBData()}
425
- log.Infof("adding directory: %s", file.FileName())
426
-
427
- for {
428
- file, err := file.NextFile()
429
- if err != nil && err != io.EOF {
430
- return nil, err
431
- }
432
- if file == nil {
433
- break
434
- }
435
-
436
- node, err := params.addFile(file)
437
- if _, ok := err.(*hiddenFileError); ok {
438
- // hidden file error, set the node to nil for below
439
- node = nil
440
- } else if err != nil {
441
- return nil, err
442
- }
443
-
444
- if node != nil {
445
- name := path.Base(file.FileName())
446
-
447
- err = tree.AddNodeLink(name, node)
448
- if err != nil {
449
- return nil, err
450
- }
451
- }
452
- }
453
-
454
- if err := params.addNode(tree, file.FileName()); err != nil {
455
- return nil, err
456
- }
457
-
458
- _, err := params.node.DAG.Add(tree)
459
- if err != nil {
460
- return nil, err
461
- }
462
-
463
- return tree, nil
464
-}
465
-
466
-// outputDagnode sends dagnode info over the output channel
467
-func outputDagnode(out chan interface{}, name string, dn *dag.Node) error {
468
- o, err := getOutput(dn)
469
- if err != nil {
470
- return err
471
- }
472
-
473
- out <- &AddedObject{
474
- Hash: o.Hash,
475
- Name: name,
476
- }
477
-
478
- return nil
479
-}
480
-
481
-type hiddenFileError struct {
482
- fileName string
483
-}
484
-
485
-func (e *hiddenFileError) Error() string {
486
- return fmt.Sprintf("%s is a hidden file", e.fileName)
487
-}
488
-
489
-type ignoreFileError struct {
490
- fileName string
491
-}
492
-
493
-func (e *ignoreFileError) Error() string {
494
- return fmt.Sprintf("%s is an ignored file", e.fileName)
495
-}
496
-
497
-type progressReader struct {
498
- file files.File
499
- out chan interface{}
500
- bytes int64
501
- lastProgress int64
502
-}
503
-
504
-func (i *progressReader) Read(p []byte) (int, error) {
505
- n, err := i.file.Read(p)
506
-
507
- i.bytes += int64(n)
508
- if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
509
- i.lastProgress = i.bytes
510
- i.out <- &AddedObject{
511
- Name: i.file.FileName(),
512
- Bytes: i.bytes,
513
- }
514
- }
515
-
516
- return n, err
517
-}
518
-
519
-// TODO: generalize this to more than unix-fs nodes.
520
-func newDirNode() *dag.Node {
521
- return &dag.Node{Data: ft.FolderPBData()}
248
+ Type: coreunix.AddedObject{},
249
}
core/commands/tar.go
+4
-3
@@ -6,6 +6,7 @@ import (
6
7
cmds "github.com/ipfs/go-ipfs/commands"
8
core "github.com/ipfs/go-ipfs/core"
9
+ "github.com/ipfs/go-ipfs/core/coreunix"
10
path "github.com/ipfs/go-ipfs/path"
11
tar "github.com/ipfs/go-ipfs/tar"
12
)
@@ -58,15 +59,15 @@ var tarAddCmd = &cmds.Command{
59
}
60
61
fi.FileName()
61
- res.SetOutput(&AddedObject{
62
+ res.SetOutput(&coreunix.AddedObject{
63
Name: fi.FileName(),
64
Hash: k.B58String(),
65
})
66
},
66
- Type: AddedObject{},
67
+ Type: coreunix.AddedObject{},
68
Marshalers: cmds.MarshalerMap{
69
cmds.Text: func(res cmds.Response) (io.Reader, error) {
69
- o := res.Output().(*AddedObject)
70
+ o := res.Output().(*coreunix.AddedObject)
71
return strings.NewReader(o.Hash + "\n"), nil
72
},
73
},
core/coreunix/add.go
+296
-38
@@ -1,17 +1,25 @@
1
package coreunix
2
3
import (
4
+ "fmt"
5
"io"
6
"io/ioutil"
7
"os"
8
gopath "path"
9
10
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11
+ syncds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
12
context "github.com/ipfs/go-ipfs/Godeps/_workspace/src/golang.org/x/net/context"
13
+ bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
14
+ bserv "github.com/ipfs/go-ipfs/blockservice"
15
+ "github.com/ipfs/go-ipfs/exchange/offline"
16
+ importer "github.com/ipfs/go-ipfs/importer"
17
+ "github.com/ipfs/go-ipfs/importer/chunk"
18
+ dagutils "github.com/ipfs/go-ipfs/merkledag/utils"
19
+ "github.com/ipfs/go-ipfs/pin"
20
21
"github.com/ipfs/go-ipfs/commands/files"
22
core "github.com/ipfs/go-ipfs/core"
13
- importer "github.com/ipfs/go-ipfs/importer"
14
- chunk "github.com/ipfs/go-ipfs/importer/chunk"
23
merkledag "github.com/ipfs/go-ipfs/merkledag"
24
unixfs "github.com/ipfs/go-ipfs/unixfs"
25
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
@@ -19,22 +27,146 @@ import (
27
28
var log = logging.Logger("coreunix")
29
30
+// how many bytes of progress to wait before sending a progress update message
31
+const progressReaderIncrement = 1024 * 256
32
+
33
+type Link struct {
34
+ Name, Hash string
35
+ Size uint64
36
+}
37
+
38
+type Object struct {
39
+ Hash string
40
+ Links []Link
41
+}
42
+
43
+type hiddenFileError struct {
44
+ fileName string
45
+}
46
+
47
+func (e *hiddenFileError) Error() string {
48
+ return fmt.Sprintf("%s is a hidden file", e.fileName)
49
+}
50
+
51
+type ignoreFileError struct {
52
+ fileName string
53
+}
54
+
55
+func (e *ignoreFileError) Error() string {
56
+ return fmt.Sprintf("%s is an ignored file", e.fileName)
57
+}
58
+
59
+type AddedObject struct {
60
+ Name string
61
+ Hash string `json:",omitempty"`
62
+ Bytes int64 `json:",omitempty"`
63
+}
64
+
65
+func NewAdder(ctx context.Context, n *core.IpfsNode, out chan interface{}) *Adder {
66
+ e := dagutils.NewDagEditor(NewMemoryDagService(), newDirNode())
67
+ return &Adder{
68
+ ctx: ctx,
69
+ node: n,
70
+ editor: e,
71
+ out: out,
72
+ Progress: false,
73
+ Hidden: true,
74
+ Pin: true,
75
+ Trickle: false,
76
+ Wrap: false,
77
+ Chunker: "",
78
+ }
79
+}
80
+
81
+// Internal structure for holding the switches passed to the `add` call
82
+type Adder struct {
83
+ ctx context.Context
84
+ node *core.IpfsNode
85
+ editor *dagutils.Editor
86
+ out chan interface{}
87
+ Progress bool
88
+ Hidden bool
89
+ Pin bool
90
+ Trickle bool
91
+ Wrap bool
92
+ Chunker string
93
+ root *merkledag.Node
94
+}
95
+
96
+// Perform the actual add & pin locally, outputting results to reader
97
+func (params Adder) add(reader io.Reader) (*merkledag.Node, error) {
98
+ chnk, err := chunk.FromString(reader, params.Chunker)
99
+ if err != nil {
100
+ return nil, err
101
+ }
102
+
103
+ if params.Trickle {
104
+ return importer.BuildTrickleDagFromReader(
105
+ params.node.DAG,
106
+ chnk,
107
+ )
108
+ }
109
+ return importer.BuildDagFromReader(
110
+ params.node.DAG,
111
+ chnk,
112
+ )
113
+}
114
+
115
+func (params *Adder) RootNode() (*merkledag.Node, error) {
116
+ // for memoizing
117
+ if params.root != nil {
118
+ return params.root, nil
119
+ }
120
+
121
+ root := params.editor.GetNode()
122
+
123
+ // if not wrapping, AND one root file, use that hash as root.
124
+ if !params.Wrap && len(root.Links) == 1 {
125
+ var err error
126
+ root, err = root.Links[0].GetNode(params.ctx, params.editor.GetDagService())
127
+ params.root = root
128
+ // no need to output, as we've already done so.
129
+ return root, err
130
+ }
131
+
132
+ // otherwise need to output, as we have not.
133
+ err := outputDagnode(params.out, "", root)
134
+ params.root = root
135
+ return root, err
136
+}
137
+
138
+func (params *Adder) PinRoot() error {
139
+ root, err := params.RootNode()
140
+ if err != nil {
141
+ return err
142
+ }
143
+
144
+ rnk, err := root.Key()
145
+ if err != nil {
146
+ return err
147
+ }
148
+
149
+ params.node.Pinning.PinWithMode(rnk, pin.Recursive)
150
+ return params.node.Pinning.Flush()
151
+}
152
+
153
+func (params *Adder) WriteOutputTo(DAG merkledag.DAGService) error {
154
+ return params.editor.WriteOutputTo(DAG)
155
+}
156
+
157
// Add builds a merkledag from the a reader, pinning all objects to the local
158
// datastore. Returns a key representing the root node.
159
func Add(n *core.IpfsNode, r io.Reader) (string, error) {
160
unlock := n.Blockstore.PinLock()
161
defer unlock()
162
28
- // TODO more attractive function signature importer.BuildDagFromReader
163
+ fileAdder := NewAdder(n.Context(), n, nil)
164
30
- dagNode, err := importer.BuildDagFromReader(
31
- n.DAG,
32
- chunk.NewSizeSplitter(r, chunk.DefaultBlockSize),
33
- )
165
+ node, err := fileAdder.add(r)
166
if err != nil {
167
return "", err
168
}
37
- k, err := dagNode.Key()
169
+ k, err := node.Key()
170
if err != nil {
171
return "", err
172
}
@@ -58,7 +190,9 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
190
}
191
defer f.Close()
192
61
- dagnode, err := addFile(n, f)
193
+ fileAdder := NewAdder(n.Context(), n, nil)
194
+
195
+ dagnode, err := fileAdder.AddFile(f)
196
if err != nil {
197
return "", err
198
}
@@ -78,10 +212,11 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
212
func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkledag.Node, error) {
213
file := files.NewReaderFile(filename, filename, ioutil.NopCloser(r), nil)
214
dir := files.NewSliceFile("", "", []files.File{file})
215
+ fileAdder := NewAdder(n.Context(), n, nil)
216
217
unlock := n.Blockstore.PinLock()
218
defer unlock()
84
- dagnode, err := addDir(n, dir)
219
+ dagnode, err := fileAdder.addDir(dir)
220
if err != nil {
221
return "", nil, err
222
}
@@ -92,58 +227,181 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *merkle
227
return gopath.Join(k.String(), filename), dagnode, nil
228
}
229
95
-func add(n *core.IpfsNode, reader io.Reader) (*merkledag.Node, error) {
96
- return importer.BuildDagFromReader(
97
- n.DAG,
98
- chunk.DefaultSplitter(reader),
99
- )
100
-}
230
+func (params *Adder) addNode(node *merkledag.Node, path string) error {
231
+ // patch it into the root
232
+ if path == "" {
233
+ key, err := node.Key()
234
+ if err != nil {
235
+ return err
236
+ }
237
+
238
+ path = key.Pretty()
239
+ }
240
102
-func addNode(n *core.IpfsNode, node *merkledag.Node) error {
103
- if err := n.DAG.AddRecursive(node); err != nil { // add the file to the graph + local storage
241
+ if err := params.editor.InsertNodeAtPath(params.ctx, path, node, newDirNode); err != nil {
242
return err
243
}
106
- ctx, cancel := context.WithCancel(n.Context())
107
- defer cancel()
108
- err := n.Pinning.Pin(ctx, node, true) // ensure we keep it
109
- return err
244
+
245
+ return outputDagnode(params.out, path, node)
246
}
247
112
-func addFile(n *core.IpfsNode, file files.File) (*merkledag.Node, error) {
113
- if file.IsDirectory() {
114
- return addDir(n, file)
248
+// Add the given file while respecting the params.
249
+func (params *Adder) AddFile(file files.File) (*merkledag.Node, error) {
250
+ switch {
251
+ case files.IsHidden(file) && !params.Hidden:
252
+ log.Debugf("%s is hidden, skipping", file.FileName())
253
+ return nil, &hiddenFileError{file.FileName()}
254
+ case file.IsDirectory():
255
+ return params.addDir(file)
256
}
116
- return add(n, file)
117
-}
257
119
-func addDir(n *core.IpfsNode, dir files.File) (*merkledag.Node, error) {
258
+ // case for symlink
259
+ if s, ok := file.(*files.Symlink); ok {
260
+ sdata, err := unixfs.SymlinkData(s.Target)
261
+ if err != nil {
262
+ return nil, err
263
+ }
264
121
- tree := &merkledag.Node{Data: unixfs.FolderPBData()}
265
+ dagnode := &merkledag.Node{Data: sdata}
266
+ _, err = params.node.DAG.Add(dagnode)
267
+ if err != nil {
268
+ return nil, err
269
+ }
270
+
271
+ err = params.addNode(dagnode, s.FileName())
272
+ return dagnode, err
273
+ }
274
+
275
+ // case for regular file
276
+ // if the progress flag was specified, wrap the file so that we can send
277
+ // progress updates to the client (over the output channel)
278
+ var reader io.Reader = file
279
+ if params.Progress {
280
+ reader = &progressReader{file: file, out: params.out}
281
+ }
282
+
283
+ dagnode, err := params.add(reader)
284
+ if err != nil {
285
+ return nil, err
286
+ }
287
+
288
+ // patch it into the root
289
+ log.Infof("adding file: %s", file.FileName())
290
+ err = params.addNode(dagnode, file.FileName())
291
+ return dagnode, err
292
+}
293
+
294
+func (params *Adder) addDir(dir files.File) (*merkledag.Node, error) {
295
+ tree := newDirNode()
296
+ log.Infof("adding directory: %s", dir.FileName())
297
123
-Loop:
298
for {
299
file, err := dir.NextFile()
126
- switch {
127
- case err != nil && err != io.EOF:
300
+ if err != nil && err != io.EOF {
301
return nil, err
129
- case err == io.EOF:
130
- break Loop
302
+ }
303
+ if file == nil {
304
+ break
305
}
306
133
- node, err := addFile(n, file)
134
- if err != nil {
307
+ node, err := params.AddFile(file)
308
+ if _, ok := err.(*hiddenFileError); ok {
309
+ // hidden file error, skip file
310
+ continue
311
+ } else if err != nil {
312
return nil, err
313
}
314
138
- _, name := gopath.Split(file.FileName())
315
+ name := gopath.Base(file.FileName())
316
317
if err := tree.AddNodeLink(name, node); err != nil {
318
return nil, err
319
}
320
}
321
145
- if err := addNode(n, tree); err != nil {
322
+ if err := params.addNode(tree, dir.FileName()); err != nil {
323
+ return nil, err
324
+ }
325
+
326
+ if _, err := params.node.DAG.Add(tree); err != nil {
327
return nil, err
328
}
329
+
330
return tree, nil
331
}
332
+
333
+// outputDagnode sends dagnode info over the output channel
334
+func outputDagnode(out chan interface{}, name string, dn *merkledag.Node) error {
335
+ if out == nil {
336
+ return nil
337
+ }
338
+
339
+ o, err := getOutput(dn)
340
+ if err != nil {
341
+ return err
342
+ }
343
+
344
+ out <- &AddedObject{
345
+ Hash: o.Hash,
346
+ Name: name,
347
+ }
348
+
349
+ return nil
350
+}
351
+
352
+func NewMemoryDagService() merkledag.DAGService {
353
+ // build mem-datastore for editor's intermediary nodes
354
+ bs := bstore.NewBlockstore(syncds.MutexWrap(ds.NewMapDatastore()))
355
+ bsrv := bserv.New(bs, offline.Exchange(bs))
356
+ return merkledag.NewDAGService(bsrv)
357
+}
358
+
359
+// TODO: generalize this to more than unix-fs nodes.
360
+func newDirNode() *merkledag.Node {
361
+ return &merkledag.Node{Data: unixfs.FolderPBData()}
362
+}
363
+
364
+// from core/commands/object.go
365
+func getOutput(dagnode *merkledag.Node) (*Object, error) {
366
+ key, err := dagnode.Key()
367
+ if err != nil {
368
+ return nil, err
369
+ }
370
+
371
+ output := &Object{
372
+ Hash: key.Pretty(),
373
+ Links: make([]Link, len(dagnode.Links)),
374
+ }
375
+
376
+ for i, link := range dagnode.Links {
377
+ output.Links[i] = Link{
378
+ Name: link.Name,
379
+ Hash: link.Hash.B58String(),
380
+ Size: link.Size,
381
+ }
382
+ }
383
+
384
+ return output, nil
385
+}
386
+
387
+type progressReader struct {
388
+ file files.File
389
+ out chan interface{}
390
+ bytes int64
391
+ lastProgress int64
392
+}
393
+
394
+func (i *progressReader) Read(p []byte) (int, error) {
395
+ n, err := i.file.Read(p)
396
+
397
+ i.bytes += int64(n)
398
+ if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
399
+ i.lastProgress = i.bytes
400
+ i.out <- &AddedObject{
401
+ Name: i.file.FileName(),
402
+ Bytes: i.bytes,
403
+ }
404
+ }
405
+
406
+ return n, err
407
+}