Remove chunk channels
License: MIT Signed-off-by: rht <rhtbot@gmail.com>
rht committed
Nov 15, 2015 at 18:50 UTC
b4a3854151a98ff37cecacbcc0132a1a0289152f
5 files changed
+23
-53
importer/balanced/balanced_test.go
+1
-4
@@ -22,15 +22,12 @@ import (
22
// TODO: extract these tests and more as a generic layout test suite
23
24
func buildTestDag(ds dag.DAGService, spl chunk.Splitter) (*dag.Node, error) {
25
- // Start the splitter
26
- blkch, errs := chunk.Chan(spl)
27
-
25
dbp := h.DagBuilderParams{
26
Dagserv: ds,
27
Maxlinks: h.DefaultLinksPerBlock,
28
}
29
33
- return BalancedLayout(dbp.New(blkch, errs))
30
+ return BalancedLayout(dbp.New(spl))
31
}
32
33
func getTestDag(t *testing.T, ds dag.DAGService, size int64, blksize int64) (*dag.Node, []byte) {
importer/helpers/dagbuilder.go
+11
-21
@@ -1,6 +1,7 @@
1
package helpers
2
3
import (
4
+ "github.com/ipfs/go-ipfs/importer/chunk"
5
dag "github.com/ipfs/go-ipfs/merkledag"
6
)
7
@@ -8,8 +9,7 @@ import (
9
// efficiently create unixfs dag trees
10
type DagBuilderHelper struct {
11
dserv dag.DAGService
11
- in <-chan []byte
12
- errs <-chan error
12
+ spl chunk.Splitter
13
recvdErr error
14
nextData []byte // the next item to return.
15
maxlinks int
@@ -24,45 +24,35 @@ type DagBuilderParams struct {
24
Dagserv dag.DAGService
25
}
26
27
-// Generate a new DagBuilderHelper from the given params, using 'in' as a
28
-// data source
29
-func (dbp *DagBuilderParams) New(in <-chan []byte, errs <-chan error) *DagBuilderHelper {
27
+// Generate a new DagBuilderHelper from the given params, which data source comes
28
+// from chunks object
29
+func (dbp *DagBuilderParams) New(spl chunk.Splitter) *DagBuilderHelper {
30
return &DagBuilderHelper{
31
dserv: dbp.Dagserv,
32
- in: in,
33
- errs: errs,
32
+ spl: spl,
33
maxlinks: dbp.Maxlinks,
34
batch: dbp.Dagserv.Batch(),
35
}
36
}
37
39
-// prepareNext consumes the next item from the channel and puts it
38
+// prepareNext consumes the next item from the splitter and puts it
39
// in the nextData field. it is idempotent-- if nextData is full
40
// it will do nothing.
42
-//
43
-// i realized that building the dag becomes _a lot_ easier if we can
44
-// "peek" the "are done yet?" (i.e. not consume it from the channel)
41
func (db *DagBuilderHelper) prepareNext() {
46
- if db.in == nil {
47
- // if our input is nil, there is "nothing to do". we're done.
48
- // as if there was no data at all. (a sort of zero-value)
49
- return
50
- }
51
-
52
- // if we already have data waiting to be consumed, we're ready.
42
+ // if we already have data waiting to be consumed, we're ready
43
if db.nextData != nil {
44
return
45
}
46
57
- // if it's closed, nextData will be correctly set to nil, signaling
58
- // that we're done consuming from the channel.
59
- db.nextData = <-db.in
47
+ // TODO: handle err (which wasn't handled either when the splitter was channeled)
48
+ db.nextData, _ = db.spl.NextBytes()
49
}
50
51
// Done returns whether or not we're done consuming the incoming data.
52
func (db *DagBuilderHelper) Done() bool {
53
// ensure we have an accurate perspective on data
54
// as `done` this may be called before `next`.
55
+ //db.prepareNext() // idempotent
56
db.prepareNext() // idempotent
57
return db.nextData == nil
58
}
importer/importer.go
+2
-8
@@ -39,25 +39,19 @@ func BuildDagFromFile(fpath string, ds dag.DAGService) (*dag.Node, error) {
39
}
40
41
func BuildDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.Node, error) {
42
- // Start the splitter
43
- blkch, errch := chunk.Chan(spl)
44
-
42
dbp := h.DagBuilderParams{
43
Dagserv: ds,
44
Maxlinks: h.DefaultLinksPerBlock,
45
}
46
50
- return bal.BalancedLayout(dbp.New(blkch, errch))
47
+ return bal.BalancedLayout(dbp.New(spl))
48
}
49
50
func BuildTrickleDagFromReader(ds dag.DAGService, spl chunk.Splitter) (*dag.Node, error) {
54
- // Start the splitter
55
- blkch, errch := chunk.Chan(spl)
56
-
51
dbp := h.DagBuilderParams{
52
Dagserv: ds,
53
Maxlinks: h.DefaultLinksPerBlock,
54
}
55
62
- return trickle.TrickleLayout(dbp.New(blkch, errch))
56
+ return trickle.TrickleLayout(dbp.New(spl))
57
}
importer/trickle/trickle_test.go
+5
-14
@@ -21,15 +21,12 @@ import (
21
)
22
23
func buildTestDag(ds merkledag.DAGService, spl chunk.Splitter) (*merkledag.Node, error) {
24
- // Start the splitter
25
- blkch, errs := chunk.Chan(spl)
26
-
24
dbp := h.DagBuilderParams{
25
Dagserv: ds,
26
Maxlinks: h.DefaultLinksPerBlock,
27
}
28
32
- nd, err := TrickleLayout(dbp.New(blkch, errs))
29
+ nd, err := TrickleLayout(dbp.New(spl))
30
if err != nil {
31
return nil, err
32
}
@@ -441,10 +438,9 @@ func TestAppend(t *testing.T) {
438
}
439
440
r := bytes.NewReader(should[nbytes/2:])
444
- blks, errs := chunk.Chan(chunk.NewSizeSplitter(r, 500))
441
442
ctx := context.Background()
447
- nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
443
+ nnode, err := TrickleAppend(ctx, nd, dbp.New(chunk.NewSizeSplitter(r, 500)))
444
if err != nil {
445
t.Fatal(err)
446
}
@@ -494,9 +490,8 @@ func TestMultipleAppends(t *testing.T) {
490
491
ctx := context.Background()
492
for i := 0; i < len(should); i++ {
497
- blks, errs := chunk.Chan(spl(bytes.NewReader(should[i : i+1])))
493
499
- nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
494
+ nnode, err := TrickleAppend(ctx, nd, dbp.New(spl(bytes.NewReader(should[i:i+1]))))
495
if err != nil {
496
t.Fatal(err)
497
}
@@ -538,17 +533,13 @@ func TestAppendSingleBytesToEmpty(t *testing.T) {
533
534
spl := chunk.SizeSplitterGen(500)
535
541
- blks, errs := chunk.Chan(spl(bytes.NewReader(data[:1])))
542
-
536
ctx := context.Background()
544
- nnode, err := TrickleAppend(ctx, nd, dbp.New(blks, errs))
537
+ nnode, err := TrickleAppend(ctx, nd, dbp.New(spl(bytes.NewReader(data[:1]))))
538
if err != nil {
539
t.Fatal(err)
540
}
541
549
- blks, errs = chunk.Chan(spl(bytes.NewReader(data[1:])))
550
-
551
- nnode, err = TrickleAppend(ctx, nnode, dbp.New(blks, errs))
542
+ nnode, err = TrickleAppend(ctx, nnode, dbp.New(spl(bytes.NewReader(data[1:]))))
543
if err != nil {
544
t.Fatal(err)
545
}
unixfs/mod/dagmodifier.go
+4
-6
@@ -103,8 +103,7 @@ func (zr zeroReader) Read(b []byte) (int, error) {
103
func (dm *DagModifier) expandSparse(size int64) error {
104
r := io.LimitReader(zeroReader{}, size)
105
spl := chunk.NewSizeSplitter(r, 4096)
106
- blks, errs := chunk.Chan(spl)
107
- nnode, err := dm.appendData(dm.curNode, blks, errs)
106
+ nnode, err := dm.appendData(dm.curNode, spl)
107
if err != nil {
108
return err
109
}
@@ -191,8 +190,7 @@ func (dm *DagModifier) Sync() error {
190
191
// need to write past end of current dag
192
if !done {
194
- blks, errs := chunk.Chan(dm.splitter(dm.wrBuf))
195
- nd, err = dm.appendData(dm.curNode, blks, errs)
193
+ nd, err = dm.appendData(dm.curNode, dm.splitter(dm.wrBuf))
194
if err != nil {
195
return err
196
}
@@ -286,13 +284,13 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
284
}
285
286
// appendData appends the blocks from the given chan to the end of this dag
289
-func (dm *DagModifier) appendData(node *mdag.Node, blks <-chan []byte, errs <-chan error) (*mdag.Node, error) {
287
+func (dm *DagModifier) appendData(node *mdag.Node, spl chunk.Splitter) (*mdag.Node, error) {
288
dbp := &help.DagBuilderParams{
289
Dagserv: dm.dagserv,
290
Maxlinks: help.DefaultLinksPerBlock,
291
}
292
295
- return trickle.TrickleAppend(dm.ctx, node, dbp.New(blks, errs))
293
+ return trickle.TrickleAppend(dm.ctx, node, dbp.New(spl))
294
}
295
296
// Read data from this dag starting at the current offset