coreapi unixfs: progress events
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Oct 4, 2018 at 01:00 UTC
a81ec29baa59d7c62622eec8636ec3e49b34b906
8 files changed
+199
-44
core/commands/add.go
+3
-2
@@ -8,6 +8,7 @@ import (
8
9
core "github.com/ipfs/go-ipfs/core"
10
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
11
+ coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
12
"github.com/ipfs/go-ipfs/core/coreunix"
13
filestore "github.com/ipfs/go-ipfs/filestore"
14
ft "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs"
@@ -371,7 +372,7 @@ You can now check what blocks have been created by:
372
373
break LOOP
374
}
374
- output := out.(*coreunix.AddedObject)
375
+ output := out.(*coreiface.AddEvent)
376
if len(output.Hash) > 0 {
377
lastHash = output.Hash
378
if quieter {
@@ -451,5 +452,5 @@ You can now check what blocks have been created by:
452
}
453
},
454
},
454
- Type: coreunix.AddedObject{},
455
+ Type: coreiface.AddEvent{},
456
}
core/commands/tar.go
+4
-4
@@ -7,7 +7,7 @@ import (
7
cmds "github.com/ipfs/go-ipfs/commands"
8
core "github.com/ipfs/go-ipfs/core"
9
e "github.com/ipfs/go-ipfs/core/commands/e"
10
- "github.com/ipfs/go-ipfs/core/coreunix"
10
+ coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
11
tar "github.com/ipfs/go-ipfs/tar"
12
dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
13
path "gx/ipfs/QmcjwUb36Z16NJkvDX6ccXPqsFswo6AsRXynyXcLLCphV2/go-path"
@@ -60,12 +60,12 @@ represent it.
60
c := node.Cid()
61
62
fi.FileName()
63
- res.SetOutput(&coreunix.AddedObject{
63
+ res.SetOutput(&coreiface.AddEvent{
64
Name: fi.FileName(),
65
Hash: c.String(),
66
})
67
},
68
- Type: coreunix.AddedObject{},
68
+ Type: coreiface.AddEvent{},
69
Marshalers: cmds.MarshalerMap{
70
cmds.Text: func(res cmds.Response) (io.Reader, error) {
71
v, err := unwrapOutput(res.Output())
@@ -73,7 +73,7 @@ represent it.
73
return nil, err
74
}
75
76
- o, ok := v.(*coreunix.AddedObject)
76
+ o, ok := v.(*coreiface.AddEvent)
77
if !ok {
78
return nil, e.TypeErr(o, v)
79
}
core/coreapi/interface/options/unixfs.go
+35
@@ -35,6 +35,10 @@ type UnixfsAddSettings struct {
35
Wrap bool
36
Hidden bool
37
StdinName string
38
+
39
+ Events chan<- interface{}
40
+ Silent bool
41
+ Progress bool
42
}
43
44
type UnixfsAddOption func(*UnixfsAddSettings) error
@@ -59,6 +63,10 @@ func UnixfsAddOptions(opts ...UnixfsAddOption) (*UnixfsAddSettings, cid.Prefix,
63
Wrap: false,
64
Hidden: false,
65
StdinName: "",
66
+
67
+ Events: nil,
68
+ Silent: false,
69
+ Progress: false,
70
}
71
72
for _, opt := range opts {
@@ -236,3 +244,30 @@ func (unixfsOpts) StdinName(name string) UnixfsAddOption {
244
return nil
245
}
246
}
247
+
248
+// Events specifies channel which will be used to report events about ongoing
249
+// Add operation.
250
+//
251
+// Note that if this channel blocks it may slowdown the adder
252
+func (unixfsOpts) Events(sink chan<- interface{}) UnixfsAddOption {
253
+ return func(settings *UnixfsAddSettings) error {
254
+ settings.Events = sink
255
+ return nil
256
+ }
257
+}
258
+
259
+// Silent reduces event output
260
+func (unixfsOpts) Silent(silent bool) UnixfsAddOption {
261
+ return func(settings *UnixfsAddSettings) error {
262
+ settings.Silent = silent
263
+ return nil
264
+ }
265
+}
266
+
267
+// Progress tells the adder whether to enable progress events
268
+func (unixfsOpts) Progress(enable bool) UnixfsAddOption {
269
+ return func(settings *UnixfsAddSettings) error {
270
+ settings.Progress = enable
271
+ return nil
272
+ }
273
+}
core/coreapi/interface/unixfs.go
+9
@@ -9,6 +9,14 @@ import (
9
ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format"
10
)
11
12
+// TODO: ideas on making this more coreapi-ish without breaking the http API?
13
+type AddEvent struct {
14
+ Name string
15
+ Hash string `json:",omitempty"`
16
+ Bytes int64 `json:",omitempty"`
17
+ Size string `json:",omitempty"`
18
+}
19
+
20
// UnixfsAPI is the basic interface to immutable files in IPFS
21
// NOTE: This API is heavily WIP, things are guaranteed to break frequently
22
type UnixfsAPI interface {
@@ -24,6 +32,7 @@ type UnixfsAPI interface {
32
Get(context.Context, Path) (files.File, error)
33
34
// Cat returns a reader for the file
35
+ // TODO: Remove in favour of Get (if we use Get on a file we still have reader directly, so..)
36
Cat(context.Context, Path) (Reader, error)
37
38
// Ls returns the list of links in a directory
core/coreapi/unixfs.go
+5
-2
@@ -64,11 +64,14 @@ func (api *UnixfsAPI) Add(ctx context.Context, files files.File, opts ...options
64
}
65
66
fileAdder.Chunker = settings.Chunker
67
- //fileAdder.Progress = progress
67
+ if settings.Events != nil {
68
+ fileAdder.Out = settings.Events
69
+ fileAdder.Progress = settings.Progress
70
+ }
71
fileAdder.Hidden = settings.Hidden
72
fileAdder.Wrap = settings.Wrap
73
fileAdder.Pin = settings.Pin && !settings.OnlyHash
71
- fileAdder.Silent = true
74
+ fileAdder.Silent = settings.Silent
75
fileAdder.RawLeaves = settings.RawLeaves
76
//fileAdder.NoCopy = nocopy
77
fileAdder.Name = settings.StdinName
core/coreapi/unixfs_test.go
+125
-14
@@ -9,7 +9,9 @@ import (
9
"io/ioutil"
10
"math"
11
"os"
12
+ "strconv"
13
"strings"
14
+ "sync"
15
"testing"
16
17
"github.com/ipfs/go-ipfs/core"
@@ -147,6 +149,13 @@ func twoLevelDir() func() files.File {
149
}
150
}
151
152
+func flatDir() files.File {
153
+ return files.NewSliceFile("t", "t", []files.File{
154
+ files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil),
155
+ files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil),
156
+ })
157
+}
158
+
159
func wrapped(f files.File) files.File {
160
return files.NewSliceFile("", "", []files.File{
161
f,
@@ -170,6 +179,8 @@ func TestAdd(t *testing.T) {
179
180
recursive bool
181
182
+ events []coreiface.AddEvent
183
+
184
opts []options.UnixfsAddOption
185
}{
186
// Simple cases
@@ -263,13 +274,8 @@ func TestAdd(t *testing.T) {
274
},
275
// multi file
276
{
266
- name: "simpleDir",
267
- data: func() files.File {
268
- return files.NewSliceFile("t", "t", []files.File{
269
- files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil),
270
- files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil),
271
- })
272
- },
277
+ name: "simpleDir",
278
+ data: flatDir,
279
recursive: true,
280
path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp",
281
},
@@ -300,7 +306,7 @@ func TestAdd(t *testing.T) {
306
files.NewReaderFile("QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", "QmQy2Dw4Wk7rdJKjThjYXzfFJNaRKRHhHP5gHHXroJMYxk", ioutil.NopCloser(strings.NewReader(helloStr)), nil),
307
})
308
},
303
- opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)},
309
+ opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true)},
310
},
311
{
312
name: "stdinNamed",
@@ -313,7 +319,7 @@ func TestAdd(t *testing.T) {
319
files.NewReaderFile("test", "test", ioutil.NopCloser(strings.NewReader(helloStr)), nil),
320
})
321
},
316
- opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")},
322
+ opts: []options.UnixfsAddOption{options.Unixfs.Wrap(true), options.Unixfs.StdinName("test")},
323
},
324
{
325
name: "twoLevelDirWrapped",
@@ -363,19 +369,71 @@ func TestAdd(t *testing.T) {
369
})
370
},
371
expect: func(files.File) files.File {
366
- return files.NewSliceFile("t", "t", []files.File{
367
- files.NewReaderFile("t/bar", "t/bar", ioutil.NopCloser(strings.NewReader("hello2")), nil),
368
- files.NewReaderFile("t/foo", "t/foo", ioutil.NopCloser(strings.NewReader("hello1")), nil),
369
- })
372
+ return flatDir()
373
},
374
recursive: true,
375
path: "/ipfs/QmRKGpFfR32FVXdvJiHfo4WJ5TDYBsM1P9raAp1p6APWSp",
376
opts: []options.UnixfsAddOption{options.Unixfs.Hidden(false)},
377
},
378
+ // Events / Progress
379
+ {
380
+ name: "simpleAddEvent",
381
+ data: strFile(helloStr),
382
+ path: "/ipfs/zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd",
383
+ events: []coreiface.AddEvent{
384
+ {Name: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Hash: "zb2rhdhmJjJZs9qkhQCpCQ7VREFkqWw3h1r8utjVvQugwHPFd", Size: strconv.Itoa(len(helloStr))},
385
+ },
386
+ opts: []options.UnixfsAddOption{options.Unixfs.RawLeaves(true)},
387
+ },
388
+ {
389
+ name: "silentAddEvent",
390
+ data: twoLevelDir(),
391
+ path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr",
392
+ events: []coreiface.AddEvent{
393
+ {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"},
394
+ {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"},
395
+ },
396
+ recursive: true,
397
+ opts: []options.UnixfsAddOption{options.Unixfs.Silent(true)},
398
+ },
399
+ {
400
+ name: "dirAddEvents",
401
+ data: twoLevelDir(),
402
+ path: "/ipfs/QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr",
403
+ events: []coreiface.AddEvent{
404
+ {Name: "t/abc/def", Hash: "QmNyJpQkU1cEkBwMDhDNFstr42q55mqG5GE5Mgwug4xyGk", Size: "13"},
405
+ {Name: "t/bar", Hash: "QmS21GuXiRMvJKHos4ZkEmQDmRBqRaF5tQS2CQCu2ne9sY", Size: "14"},
406
+ {Name: "t/foo", Hash: "QmfAjGiVpTN56TXi6SBQtstit5BEw3sijKj1Qkxn6EXKzJ", Size: "14"},
407
+ {Name: "t/abc", Hash: "QmU7nuGs2djqK99UNsNgEPGh6GV4662p6WtsgccBNGTDxt", Size: "62"},
408
+ {Name: "t", Hash: "QmVG2ZYCkV1S4TK8URA3a4RupBF17A8yAr4FqsRDXVJASr", Size: "229"},
409
+ },
410
+ recursive: true,
411
+ },
412
+ {
413
+ name: "progress1M",
414
+ data: func() files.File {
415
+ r := bytes.NewReader(bytes.Repeat([]byte{0}, 1000000))
416
+ return files.NewReaderFile("", "", ioutil.NopCloser(r), nil)
417
+ },
418
+ path: "/ipfs/QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD",
419
+ events: []coreiface.AddEvent{
420
+ {Name: "", Bytes: 262144},
421
+ {Name: "", Bytes: 524288},
422
+ {Name: "", Bytes: 786432},
423
+ {Name: "", Bytes: 1000000},
424
+ {Name: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Hash: "QmXXNNbwe4zzpdMg62ZXvnX1oU7MwSrQ3vAEtuwFKCm1oD", Size: "1000256"},
425
+ },
426
+ recursive: true,
427
+ opts: []options.UnixfsAddOption{options.Unixfs.Progress(true)},
428
+ },
429
}
430
431
for _, testCase := range cases {
432
t.Run(testCase.name, func(t *testing.T) {
433
+ ctx, cancel := context.WithCancel(ctx)
434
+ defer cancel()
435
+
436
+ // recursive logic
437
438
data := testCase.data()
439
if testCase.recursive {
@@ -384,7 +442,58 @@ func TestAdd(t *testing.T) {
442
})
443
}
444
387
- p, err := api.Unixfs().Add(ctx, data, testCase.opts...)
445
+ // handle events if relevant to test case
446
+
447
+ opts := testCase.opts
448
+ eventOut := make(chan interface{})
449
+ var evtWg sync.WaitGroup
450
+ if len(testCase.events) > 0 {
451
+ opts = append(opts, options.Unixfs.Events(eventOut))
452
+ evtWg.Add(1)
453
+
454
+ go func() {
455
+ defer evtWg.Done()
456
+ expected := testCase.events
457
+
458
+ for evt := range eventOut {
459
+ event, ok := evt.(*coreiface.AddEvent)
460
+ if !ok {
461
+ t.Fatal("unexpected event type")
462
+ }
463
+
464
+ if len(expected) < 1 {
465
+ t.Fatal("got more events than expected")
466
+ }
467
+
468
+ if expected[0].Size != event.Size {
469
+ t.Errorf("Event.Size didn't match, %s != %s", expected[0].Size, event.Size)
470
+ }
471
+
472
+ if expected[0].Name != event.Name {
473
+ t.Errorf("Event.Name didn't match, %s != %s", expected[0].Name, event.Name)
474
+ }
475
+
476
+ if expected[0].Hash != event.Hash {
477
+ t.Errorf("Event.Hash didn't match, %s != %s", expected[0].Hash, event.Hash)
478
+ }
479
+ if expected[0].Bytes != event.Bytes {
480
+ t.Errorf("Event.Bytes didn't match, %d != %d", expected[0].Bytes, event.Bytes)
481
+ }
482
+
483
+ expected = expected[1:]
484
+ }
485
+
486
+ if len(expected) > 0 {
487
+ t.Fatalf("%d event(s) didn't arrive", len(expected))
488
+ }
489
+ }()
490
+ }
491
+
492
+ // Add!
493
+
494
+ p, err := api.Unixfs().Add(ctx, data, opts...)
495
+ close(eventOut)
496
+ evtWg.Wait()
497
if testCase.err != "" {
498
if err == nil {
499
t.Fatalf("expected an error: %s", testCase.err)
@@ -402,6 +511,8 @@ func TestAdd(t *testing.T) {
511
t.Errorf("expected path %s, got: %s", testCase.path, p)
512
}
513
514
+ // compare file structure with Unixfs().Get
515
+
516
var cmpFile func(orig files.File, got files.File)
517
cmpFile = func(orig files.File, got files.File) {
518
if orig.IsDirectory() != got.IsDirectory() {
core/coreunix/add.go
+11
-17
@@ -11,19 +11,20 @@ import (
11
"strconv"
12
13
core "github.com/ipfs/go-ipfs/core"
14
+ coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
15
"github.com/ipfs/go-ipfs/pin"
15
- unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs"
16
- balanced "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/balanced"
17
- ihelper "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/helpers"
18
- trickle "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/trickle"
19
- dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
16
17
posinfo "gx/ipfs/QmPG32VXR5jmpo9q8R9FNdR4Ae97Ky9CiZE6SctJLUB79H/go-ipfs-posinfo"
18
cid "gx/ipfs/QmPSQnBKM9g7BaUcZCvswUJVscQ1ipjmwxN5PXCjkp9EQ7/go-cid"
19
files "gx/ipfs/QmSP88ryZkHSRn1fnngAaV2Vcn63WUJzAavnRM9CVdU1Ky/go-ipfs-cmdkit/files"
20
+ unixfs "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs"
21
+ balanced "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/balanced"
22
+ ihelper "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/helpers"
23
+ trickle "gx/ipfs/QmU4x3742bvgfxJsByEDpBnifJqjJdV6x528co4hwKCn46/go-unixfs/importer/trickle"
24
chunker "gx/ipfs/QmULKgr55cSWR8Kiwy3cVRcAiGVnR6EVSaB7hJcWS4138p/go-ipfs-chunker"
25
logging "gx/ipfs/QmZChCsSt8DctjceaL56Eibc29CVQq4dGKRXC5JRZ6Ppae/go-log"
26
mfs "gx/ipfs/QmahrY1adY4wvtYEtoGjpZ2GUohTyukrkMkwUR9ytRjTG2/go-mfs"
27
+ dag "gx/ipfs/QmcBoNcAP6qDjgRBew7yjvCqHq7p5jMstE44jPUBWBxzsV/go-merkledag"
28
ipld "gx/ipfs/QmdDXJs4axxefSPgK6Y1QhpJWKuDPnGJiqgq4uncb4rFHL/go-ipld-format"
29
bstore "gx/ipfs/QmdriVJgKx4JADRgh3cYPXqXmsa1A45SvFki1nDWHhQNtC/go-ipfs-blockstore"
30
)
@@ -46,13 +47,6 @@ type Object struct {
47
Size string
48
}
49
49
-type AddedObject struct {
50
- Name string
51
- Hash string `json:",omitempty"`
52
- Bytes int64 `json:",omitempty"`
53
- Size string `json:",omitempty"`
54
-}
55
-
50
// NewAdder Returns a new Adder used for a file add operation.
51
func NewAdder(ctx context.Context, p pin.Pinner, bs bstore.GCBlockstore, ds ipld.DAGService) (*Adder, error) {
52
return &Adder{
@@ -75,7 +69,7 @@ type Adder struct {
69
pinning pin.Pinner
70
blockstore bstore.GCBlockstore
71
dagService ipld.DAGService
78
- Out chan interface{}
72
+ Out chan<- interface{}
73
Progress bool
74
Hidden bool
75
Pin bool
@@ -570,7 +564,7 @@ func (adder *Adder) maybePauseForGC() error {
564
}
565
566
// outputDagnode sends dagnode info over the output channel
573
-func outputDagnode(out chan interface{}, name string, dn ipld.Node) error {
567
+func outputDagnode(out chan<- interface{}, name string, dn ipld.Node) error {
568
if out == nil {
569
return nil
570
}
@@ -580,7 +574,7 @@ func outputDagnode(out chan interface{}, name string, dn ipld.Node) error {
574
return err
575
}
576
583
- out <- &AddedObject{
577
+ out <- &coreiface.AddEvent{
578
Hash: o.Hash,
579
Name: name,
580
Size: o.Size,
@@ -615,7 +609,7 @@ func getOutput(dagnode ipld.Node) (*Object, error) {
609
610
type progressReader struct {
611
file files.File
618
- out chan interface{}
612
+ out chan<- interface{}
613
bytes int64
614
lastProgress int64
615
}
@@ -626,7 +620,7 @@ func (i *progressReader) Read(p []byte) (int, error) {
620
i.bytes += int64(n)
621
if i.bytes-i.lastProgress >= progressReaderIncrement || err == io.EOF {
622
i.lastProgress = i.bytes
629
- i.out <- &AddedObject{
623
+ i.out <- &coreiface.AddEvent{
624
Name: i.file.FileName(),
625
Bytes: i.bytes,
626
}
core/coreunix/add_test.go
+7
-5
@@ -11,6 +11,7 @@ import (
11
"time"
12
13
"github.com/ipfs/go-ipfs/core"
14
+ coreiface "github.com/ipfs/go-ipfs/core/coreapi/interface"
15
"github.com/ipfs/go-ipfs/pin/gc"
16
"github.com/ipfs/go-ipfs/repo"
17
@@ -96,7 +97,7 @@ func TestAddGCLive(t *testing.T) {
97
addedHashes := make(map[string]struct{})
98
select {
99
case o := <-out:
99
- addedHashes[o.(*AddedObject).Hash] = struct{}{}
100
+ addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
101
case <-addDone:
102
t.Fatal("add shouldnt complete yet")
103
}
@@ -124,7 +125,7 @@ func TestAddGCLive(t *testing.T) {
125
126
// receive next object from adder
127
o := <-out
127
- addedHashes[o.(*AddedObject).Hash] = struct{}{}
128
+ addedHashes[o.(*coreiface.AddEvent).Hash] = struct{}{}
129
130
<-gcstarted
131
@@ -140,7 +141,7 @@ func TestAddGCLive(t *testing.T) {
141
var last cid.Cid
142
for a := range out {
143
// wait for it to finish
143
- c, err := cid.Decode(a.(*AddedObject).Hash)
144
+ c, err := cid.Decode(a.(*coreiface.AddEvent).Hash)
145
if err != nil {
146
t.Fatal(err)
147
}
@@ -178,7 +179,8 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) {
179
if err != nil {
180
t.Fatal(err)
181
}
181
- adder.Out = make(chan interface{})
182
+ out := make(chan interface{})
183
+ adder.Out = out
184
adder.Progress = true
185
adder.RawLeaves = rawLeaves
186
adder.NoCopy = true
@@ -196,7 +198,7 @@ func testAddWPosInfo(t *testing.T, rawLeaves bool) {
198
t.Fatal(err)
199
}
200
}()
199
- for range adder.Out {
201
+ for range out {
202
}
203
204
exp := 0