Report progress during 'pin add'.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Feb 17, 2017 at 16:17 UTC
cdd29c24a24c8ee1d60c04b541f07d12cac23cd3
4 files changed
+242
-15
core/commands/pin.go
+82
-9
@@ -4,6 +4,7 @@ import (
4
"bytes"
5
"fmt"
6
"io"
7
+ "time"
8
9
cmds "github.com/ipfs/go-ipfs/commands"
10
core "github.com/ipfs/go-ipfs/core"
@@ -33,6 +34,11 @@ type PinOutput struct {
34
Pins []*cid.Cid
35
}
36
37
+type AddPinOutput struct {
38
+ Pins []*cid.Cid
39
+ Progress int `json:",omitempty"`
40
+}
41
+
42
var addPinCmd = &cmds.Command{
43
Helptext: cmds.HelpText{
44
Tagline: "Pin objects to local storage.",
@@ -44,8 +50,9 @@ var addPinCmd = &cmds.Command{
50
},
51
Options: []cmds.Option{
52
cmds.BoolOption("recursive", "r", "Recursively pin the object linked to by the specified object(s).").Default(true),
53
+ cmds.BoolOption("progress", "Show progress"),
54
},
48
- Type: PinOutput{},
55
+ Type: AddPinOutput{},
56
Run: func(req cmds.Request, res cmds.Response) {
57
n, err := req.InvocContext().GetNode()
58
if err != nil {
@@ -61,22 +68,88 @@ var addPinCmd = &cmds.Command{
68
res.SetError(err, cmds.ErrNormal)
69
return
70
}
71
+ showProgress, _, _ := req.Option("progress").Bool()
72
65
- added, err := corerepo.Pin(n, req.Context(), req.Arguments(), recursive)
66
- if err != nil {
67
- res.SetError(err, cmds.ErrNormal)
73
+ if !showProgress {
74
+ added, err := corerepo.Pin(n, req.Context(), req.Arguments(), recursive)
75
+ if err != nil {
76
+ res.SetError(err, cmds.ErrNormal)
77
+ return
78
+ }
79
+ res.SetOutput(&AddPinOutput{Pins: added})
80
return
81
}
82
71
- res.SetOutput(&PinOutput{added})
83
+ v := new(dag.ProgressTracker)
84
+ ctx := v.DeriveContext(req.Context())
85
+
86
+ ch := make(chan []*cid.Cid)
87
+ go func() {
88
+ defer close(ch)
89
+ added, err := corerepo.Pin(n, ctx, req.Arguments(), recursive)
90
+ if err != nil {
91
+ res.SetError(err, cmds.ErrNormal)
92
+ return
93
+ }
94
+ ch <- added
95
+ }()
96
+ out := make(chan interface{})
97
+ res.SetOutput((<-chan interface{})(out))
98
+ go func() {
99
+ ticker := time.NewTicker(500 * time.Millisecond)
100
+ defer ticker.Stop()
101
+ defer close(out)
102
+ for {
103
+ select {
104
+ case val, ok := <-ch:
105
+ if !ok {
106
+ // error already set just return
107
+ return
108
+ }
109
+ if pv := v.Value(); pv != 0 {
110
+ out <- &AddPinOutput{Progress: v.Value()}
111
+ }
112
+ out <- &AddPinOutput{Pins: val}
113
+ return
114
+ case <-ticker.C:
115
+ out <- &AddPinOutput{Progress: v.Value()}
116
+ case <-ctx.Done():
117
+ res.SetError(ctx.Err(), cmds.ErrNormal)
118
+ return
119
+ }
120
+ }
121
+ }()
122
},
123
Marshalers: cmds.MarshalerMap{
124
cmds.Text: func(res cmds.Response) (io.Reader, error) {
75
- added, ok := res.Output().(*PinOutput)
76
- if !ok {
125
+ var added []*cid.Cid
126
+
127
+ switch out := res.Output().(type) {
128
+ case *AddPinOutput:
129
+ added = out.Pins
130
+ case <-chan interface{}:
131
+ progressLine := false
132
+ for r0 := range out {
133
+ r := r0.(*AddPinOutput)
134
+ if r.Pins != nil {
135
+ added = r.Pins
136
+ } else {
137
+ if progressLine {
138
+ fmt.Fprintf(res.Stderr(), "\r")
139
+ }
140
+ fmt.Fprintf(res.Stderr(), "Fetched/Processed %d nodes", r.Progress)
141
+ progressLine = true
142
+ }
143
+ }
144
+ if progressLine {
145
+ fmt.Fprintf(res.Stderr(), "\n")
146
+ }
147
+ if res.Error() != nil {
148
+ return nil, res.Error()
149
+ }
150
+ default:
151
return nil, u.ErrCast()
152
}
79
-
153
var pintype string
154
rec, found, _ := res.Request().Option("recursive").Bool()
155
if rec || !found {
@@ -86,7 +159,7 @@ var addPinCmd = &cmds.Command{
159
}
160
161
buf := new(bytes.Buffer)
89
- for _, k := range added.Pins {
162
+ for _, k := range added {
163
fmt.Fprintf(buf, "pinned %s %s\n", k, pintype)
164
}
165
return buf, nil
merkledag/merkledag.go
+36
-2
@@ -139,8 +139,21 @@ func (n *dagService) Remove(nd node.Node) error {
139
}
140
141
// FetchGraph fetches all nodes that are children of the given node
142
-func FetchGraph(ctx context.Context, c *cid.Cid, serv DAGService) error {
143
- return EnumerateChildrenAsync(ctx, serv, c, cid.NewSet().Visit)
142
+func FetchGraph(ctx context.Context, root *cid.Cid, serv DAGService) error {
143
+ v, _ := ctx.Value("progress").(*ProgressTracker)
144
+ if v == nil {
145
+ return EnumerateChildrenAsync(ctx, serv, root, cid.NewSet().Visit)
146
+ }
147
+ set := cid.NewSet()
148
+ visit := func(c *cid.Cid) bool {
149
+ if set.Visit(c) {
150
+ v.Increment()
151
+ return true
152
+ } else {
153
+ return false
154
+ }
155
+ }
156
+ return EnumerateChildrenAsync(ctx, serv, root, visit)
157
}
158
159
// FindLinks searches this nodes links for the given key,
@@ -389,6 +402,27 @@ func EnumerateChildren(ctx context.Context, ds LinkService, root *cid.Cid, visit
402
return nil
403
}
404
405
+type ProgressTracker struct {
406
+ Total int
407
+ lk sync.Mutex
408
+}
409
+
410
+func (p *ProgressTracker) DeriveContext(ctx context.Context) context.Context {
411
+ return context.WithValue(ctx, "progress", p)
412
+}
413
+
414
+func (p *ProgressTracker) Increment() {
415
+ p.lk.Lock()
416
+ defer p.lk.Unlock()
417
+ p.Total++
418
+}
419
+
420
+func (p *ProgressTracker) Value() int {
421
+ p.lk.Lock()
422
+ defer p.lk.Unlock()
423
+ return p.Total
424
+}
425
+
426
// FetchGraphConcurrency is total number of concurrent fetches that
427
// 'fetchNodes' will start at a time
428
var FetchGraphConcurrency = 8
merkledag/merkledag_test.go
+78
@@ -7,6 +7,7 @@ import (
7
"fmt"
8
"io"
9
"io/ioutil"
10
+ "math/rand"
11
"strings"
12
"sync"
13
"testing"
@@ -547,3 +548,80 @@ func TestEnumerateAsyncFailsNotFound(t *testing.T) {
548
t.Fatal("this should have failed")
549
}
550
}
551
+
552
+func TestProgressIndicator(t *testing.T) {
553
+ testProgressIndicator(t, 5)
554
+}
555
+
556
+func TestProgressIndicatorNoChildren(t *testing.T) {
557
+ testProgressIndicator(t, 0)
558
+}
559
+
560
+func testProgressIndicator(t *testing.T, depth int) {
561
+ ds := dstest.Mock()
562
+
563
+ top, numChildren := mkDag(ds, depth)
564
+
565
+ v := new(ProgressTracker)
566
+ ctx := v.DeriveContext(context.Background())
567
+
568
+ err := FetchGraph(ctx, top, ds)
569
+ if err != nil {
570
+ t.Fatal(err)
571
+ }
572
+
573
+ if v.Value() != numChildren+1 {
574
+ t.Errorf("wrong number of children reported in progress indicator, expected %d, got %d",
575
+ numChildren+1, v.Value())
576
+ }
577
+}
578
+
579
+func mkDag(ds DAGService, depth int) (*cid.Cid, int) {
580
+ totalChildren := 0
581
+ f := func() *ProtoNode {
582
+ p := new(ProtoNode)
583
+ buf := make([]byte, 16)
584
+ rand.Read(buf)
585
+
586
+ p.SetData(buf)
587
+ _, err := ds.Add(p)
588
+ if err != nil {
589
+ panic(err)
590
+ }
591
+ return p
592
+ }
593
+
594
+ for i := 0; i < depth; i++ {
595
+ thisf := f
596
+ f = func() *ProtoNode {
597
+ pn := mkNodeWithChildren(thisf, 10)
598
+ _, err := ds.Add(pn)
599
+ if err != nil {
600
+ panic(err)
601
+ }
602
+ totalChildren += 10
603
+ return pn
604
+ }
605
+ }
606
+
607
+ nd := f()
608
+ c, err := ds.Add(nd)
609
+ if err != nil {
610
+ panic(err)
611
+ }
612
+
613
+ return c, totalChildren
614
+}
615
+
616
+func mkNodeWithChildren(getChild func() *ProtoNode, width int) *ProtoNode {
617
+ cur := new(ProtoNode)
618
+
619
+ for i := 0; i < width; i++ {
620
+ c := getChild()
621
+ if err := cur.AddNodeLinkClean(fmt.Sprint(i), c); err != nil {
622
+ panic(err)
623
+ }
624
+ }
625
+
626
+ return cur
627
+}
test/sharness/t0085-pins.sh
+46
-4
@@ -10,6 +10,8 @@ test_description="Test ipfs pinning operations"
10
11
12
test_pins() {
13
+ EXTRA_ARGS=$1
14
+
15
test_expect_success "create some hashes" '
16
HASH_A=$(echo "A" | ipfs add -q --pin=false) &&
17
HASH_B=$(echo "B" | ipfs add -q --pin=false) &&
@@ -30,8 +32,8 @@ test_pins() {
32
echo $HASH_G >> hashes
33
'
34
33
- test_expect_success "pin those hashes via stdin" '
34
- cat hashes | ipfs pin add
35
+ test_expect_success "'ipfs pin add $EXTRA_ARGS' via stdin" '
36
+ cat hashes | ipfs pin add $EXTRA_ARGS
37
'
38
39
test_expect_success "unpin those hashes" '
@@ -39,15 +41,30 @@ test_pins() {
41
'
42
}
43
42
-test_pin_dag() {
44
+RANDOM_HASH=Qme8uX5n9hn15pw9p6WcVKoziyyC9LXv4LEgvsmKMULjnV
45
+
46
+test_pins_error_reporting() {
47
+ EXTRA_ARGS=$1
48
+
49
+ test_expect_success "'ipfs pin add $EXTRA_ARGS' on non-existent hash should fail" '
50
+ test_must_fail ipfs pin add $EXTRA_ARGS $RANDOM_HASH 2> err &&
51
+ grep -q "not found" err
52
+ '
53
+}
54
+
55
+test_pin_dag_init() {
56
EXTRA_ARGS=$1
57
58
test_expect_success "'ipfs add $EXTRA_ARGS --pin=false' 1MB file" '
59
random 1048576 56 > afile &&
60
HASH=`ipfs add $EXTRA_ARGS --pin=false -q afile`
61
'
62
+}
63
50
- test_expect_success "'ipfs pin add' file" '
64
+test_pin_dag() {
65
+ test_pin_dag_init $1
66
+
67
+ test_expect_success "'ipfs pin add --progress' file" '
68
ipfs pin add --recursive=true $HASH
69
'
70
@@ -67,20 +84,45 @@ test_pin_dag() {
84
'
85
}
86
87
+test_pin_progress() {
88
+ test_pin_dag_init
89
+
90
+ test_expect_success "'ipfs pin add --progress' file" '
91
+ ipfs pin add --progress $HASH 2> err
92
+ '
93
+
94
+ test_expect_success "pin progress reported correctly" '
95
+ cat err
96
+ grep -q " 5 nodes" err
97
+ '
98
+}
99
+
100
test_init_ipfs
101
102
test_pins
103
+test_pins --progress
104
+
105
+test_pins_error_reporting
106
+test_pins_error_reporting --progress
107
108
test_pin_dag
109
test_pin_dag --raw-leaves
110
111
+test_pin_progress
112
+
113
test_launch_ipfs_daemon --offline
114
115
test_pins
116
+test_pins --progress
117
+
118
+test_pins_error_reporting
119
+test_pins_error_reporting --progress
120
121
test_pin_dag
122
test_pin_dag --raw-leaves
123
124
+test_pin_progress
125
+
126
test_kill_ipfs_daemon
127
128
test_done