refactor and clean up dagreader
Jeromy committed
Jan 26, 2015 at 01:32 UTC
0023cb3097bc7636af8f3f94500014a9c98a350b
3 files changed
+121
-106
importer/importer_test.go
+44
@@ -263,6 +263,50 @@ func TestSeekToBegin(t *testing.T) {
263
}
264
}
265
266
+func TestSeekToAlmostBegin(t *testing.T) {
267
+ nbytes := int64(10 * 1024)
268
+ should := make([]byte, nbytes)
269
+ u.NewTimeSeededRand().Read(should)
270
+
271
+ read := bytes.NewReader(should)
272
+ dnp := getDagservAndPinner(t)
273
+ nd, err := BuildDagFromReader(read, dnp.ds, dnp.mp, &chunk.SizeSplitter{500})
274
+ if err != nil {
275
+ t.Fatal(err)
276
+ }
277
+
278
+ rs, err := uio.NewDagReader(context.TODO(), nd, dnp.ds)
279
+ if err != nil {
280
+ t.Fatal(err)
281
+ }
282
+
283
+ n, err := io.CopyN(ioutil.Discard, rs, 1024*4)
284
+ if err != nil {
285
+ t.Fatal(err)
286
+ }
287
+ if n != 4096 {
288
+ t.Fatal("Copy didnt copy enough bytes")
289
+ }
290
+
291
+ seeked, err := rs.Seek(1, os.SEEK_SET)
292
+ if err != nil {
293
+ t.Fatal(err)
294
+ }
295
+ if seeked != 1 {
296
+ t.Fatal("Failed to seek to almost beginning")
297
+ }
298
+
299
+ out, err := ioutil.ReadAll(rs)
300
+ if err != nil {
301
+ t.Fatal(err)
302
+ }
303
+
304
+ err = arrComp(out, should[1:])
305
+ if err != nil {
306
+ t.Fatal(err)
307
+ }
308
+}
309
+
310
func TestSeekingConsistency(t *testing.T) {
311
nbytes := int64(128 * 1024)
312
should := make([]byte, nbytes)
merkledag/merkledag.go
+48
-25
@@ -25,8 +25,8 @@ type DAGService interface {
25
26
// GetDAG returns, in order, all the single leve child
27
// nodes of the passed in node.
28
- GetDAG(context.Context, *Node) <-chan *Node
29
- GetNodes(context.Context, []u.Key) <-chan *Node
28
+ GetDAG(context.Context, *Node) []NodeGetter
29
+ GetNodes(context.Context, []u.Key) []NodeGetter
30
}
31
32
func NewDAGService(bs *bserv.BlockService) DAGService {
@@ -168,7 +168,7 @@ func FindLinks(links []u.Key, k u.Key, start int) []int {
168
// GetDAG will fill out all of the links of the given Node.
169
// It returns a channel of nodes, which the caller can receive
170
// all the child nodes of 'root' on, in proper order.
171
-func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
171
+func (ds *dagService) GetDAG(ctx context.Context, root *Node) []NodeGetter {
172
var keys []u.Key
173
for _, lnk := range root.Links {
174
keys = append(keys, u.Key(lnk.Hash))
@@ -177,46 +177,69 @@ func (ds *dagService) GetDAG(ctx context.Context, root *Node) <-chan *Node {
177
return ds.GetNodes(ctx, keys)
178
}
179
180
-func (ds *dagService) GetNodes(ctx context.Context, keys []u.Key) <-chan *Node {
181
- sig := make(chan *Node)
180
+func (ds *dagService) GetNodes(ctx context.Context, keys []u.Key) []NodeGetter {
181
+ promises := make([]NodeGetter, len(keys))
182
+ sendChans := make([]chan<- *Node, len(keys))
183
+ for i, _ := range keys {
184
+ promises[i], sendChans[i] = newNodePromise(ctx)
185
+ }
186
+
187
go func() {
183
- defer close(sig)
188
blkchan := ds.Blocks.GetBlocks(ctx, keys)
189
186
- nodes := make([]*Node, len(keys))
187
- next := 0
190
for {
191
select {
192
case blk, ok := <-blkchan:
193
if !ok {
192
- if next < len(nodes) {
193
- log.Errorf("Did not receive correct number of nodes!")
194
- }
194
return
195
}
196
+
197
nd, err := Decoded(blk.Data)
198
if err != nil {
199
- // NB: can occur in normal situations, with improperly formatted
200
- // input data
199
+ // NB: can happen with improperly formatted input data
200
log.Error("Got back bad block!")
202
- break
201
+ return
202
}
204
- is := FindLinks(keys, blk.Key(), next)
203
+ is := FindLinks(keys, blk.Key(), 0)
204
for _, i := range is {
206
- nodes[i] = nd
207
- }
208
-
209
- for ; next < len(nodes) && nodes[next] != nil; next++ {
210
- select {
211
- case sig <- nodes[next]:
212
- case <-ctx.Done():
213
- return
214
- }
205
+ sendChans[i] <- nd
206
}
207
case <-ctx.Done():
208
return
209
}
210
}
211
}()
221
- return sig
212
+ return promises
213
+}
214
+
215
+func newNodePromise(ctx context.Context) (NodeGetter, chan<- *Node) {
216
+ ch := make(chan *Node, 1)
217
+ return &nodePromise{
218
+ recv: ch,
219
+ ctx: ctx,
220
+ }, ch
221
+}
222
+
223
+type nodePromise struct {
224
+ cache *Node
225
+ recv <-chan *Node
226
+ ctx context.Context
227
+}
228
+
229
+type NodeGetter interface {
230
+ Get() (*Node, error)
231
+}
232
+
233
+func (np *nodePromise) Get() (*Node, error) {
234
+ if np.cache != nil {
235
+ return np.cache, nil
236
+ }
237
+
238
+ select {
239
+ case blk := <-np.recv:
240
+ np.cache = blk
241
+ case <-np.ctx.Done():
242
+ return nil, np.ctx.Err()
243
+ }
244
+ return np.cache, nil
245
}
unixfs/io/dagreader.go
+29
-81
@@ -3,9 +3,7 @@ package io
3
import (
4
"bytes"
5
"errors"
6
- "fmt"
6
"io"
8
- "io/ioutil"
7
"os"
8
9
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
@@ -14,7 +12,6 @@ import (
12
mdag "github.com/jbenet/go-ipfs/merkledag"
13
ft "github.com/jbenet/go-ipfs/unixfs"
14
ftpb "github.com/jbenet/go-ipfs/unixfs/pb"
17
- u "github.com/jbenet/go-ipfs/util"
15
)
16
17
var ErrIsDir = errors.New("this dag node is a directory")
@@ -23,8 +20,9 @@ var ErrIsDir = errors.New("this dag node is a directory")
20
type DagReader struct {
21
serv mdag.DAGService
22
node *mdag.Node
23
+ pbdata *ftpb.Data
24
buf ReadSeekCloser
27
- fetchChan <-chan *mdag.Node
25
+ promises []mdag.NodeGetter
26
linkPosition int
27
offset int64
28
@@ -57,15 +55,15 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (Read
55
return nil, ErrIsDir
56
case ftpb.Data_File:
57
fctx, cancel := context.WithCancel(ctx)
60
- fetchChan := serv.GetDAG(fctx, n)
58
+ promises := serv.GetDAG(fctx, n)
59
return &DagReader{
62
- node: n,
63
- serv: serv,
64
- buf: NewRSNCFromBytes(pb.GetData()),
65
- fetchChan: fetchChan,
66
- ctx: ctx,
67
- fctx: fctx,
68
- cancel: cancel,
60
+ node: n,
61
+ serv: serv,
62
+ buf: NewRSNCFromBytes(pb.GetData()),
63
+ promises: promises,
64
+ ctx: fctx,
65
+ cancel: cancel,
66
+ pbdata: pb,
67
}, nil
68
case ftpb.Data_Raw:
69
// Raw block will just be a single level, return a byte buffer
@@ -78,26 +76,18 @@ func NewDagReader(ctx context.Context, n *mdag.Node, serv mdag.DAGService) (Read
76
// precalcNextBuf follows the next link in line and loads it from the DAGService,
77
// setting the next buffer to read from
78
func (dr *DagReader) precalcNextBuf() error {
81
- var nxt *mdag.Node
82
- var ok bool
83
-
84
- if dr.fetchChan == nil {
85
- // This panic is appropriate because the select statement
86
- // will not panic if you try and read from a nil channel
87
- // it will simply hang.
88
- panic("fetchChan should NOT be nil")
79
+ dr.buf.Close() // Just to make sure
80
+ if dr.linkPosition >= len(dr.promises) {
81
+ return io.EOF
82
}
90
- select {
91
- case nxt, ok = <-dr.fetchChan:
92
- if !ok {
93
- return io.EOF
94
- }
95
- case <-dr.ctx.Done():
96
- return dr.ctx.Err()
83
+ nxt, err := dr.promises[dr.linkPosition].Get()
84
+ if err != nil {
85
+ return err
86
}
87
+ dr.linkPosition++
88
89
pb := new(ftpb.Data)
100
- err := proto.Unmarshal(nxt.Data, pb)
90
+ err = proto.Unmarshal(nxt.Data, pb)
91
if err != nil {
92
return err
93
}
@@ -107,9 +97,7 @@ func (dr *DagReader) precalcNextBuf() error {
97
// A directory should not exist within a file
98
return ft.ErrInvalidDirLocation
99
case ftpb.Data_File:
110
- //TODO: this *should* work, needs testing first
111
- log.Warning("Running untested code for multilayered indirect FS reads.")
112
- subr, err := NewDagReader(dr.fctx, nxt, dr.serv)
100
+ subr, err := NewDagReader(dr.ctx, nxt, dr.serv)
101
if err != nil {
102
return err
103
}
@@ -123,32 +111,9 @@ func (dr *DagReader) precalcNextBuf() error {
111
}
112
}
113
126
-func (dr *DagReader) resetBlockFetch(nlinkpos int) {
127
- dr.cancel()
128
- dr.fetchChan = nil
129
- dr.linkPosition = nlinkpos
130
-
131
- var keys []u.Key
132
- for _, lnk := range dr.node.Links[dr.linkPosition:] {
133
- keys = append(keys, u.Key(lnk.Hash))
134
- }
135
-
136
- fctx, cancel := context.WithCancel(dr.ctx)
137
- dr.cancel = cancel
138
- dr.fctx = fctx
139
- fch := dr.serv.GetNodes(fctx, keys)
140
- dr.fetchChan = fch
141
-}
142
-
114
// Read reads data from the DAG structured file
115
func (dr *DagReader) Read(b []byte) (int, error) {
116
// If no cached buffer, load one
146
- if dr.buf == nil {
147
- err := dr.precalcNextBuf()
148
- if err != nil {
149
- return 0, err
150
- }
151
- }
117
total := 0
118
for {
119
// Attempt to fill bytes from cached buffer
@@ -176,9 +141,7 @@ func (dr *DagReader) Read(b []byte) (int, error) {
141
}
142
143
func (dr *DagReader) Close() error {
179
- if dr.fctx != nil {
180
- dr.cancel()
181
- }
144
+ dr.cancel()
145
return nil
146
}
147
@@ -188,21 +151,11 @@ func (dr *DagReader) Seek(offset int64, whence int) (int64, error) {
151
if offset < 0 {
152
return -1, errors.New("Invalid offset")
153
}
191
- //TODO: this pb should be cached
192
- pb := new(ftpb.Data)
193
- err := proto.Unmarshal(dr.node.Data, pb)
194
- if err != nil {
195
- return -1, err
196
- }
197
-
198
- if offset == 0 {
199
- dr.resetBlockFetch(0)
200
- dr.buf = NewRSNCFromBytes(pb.GetData())
201
- return 0, nil
202
- }
154
155
+ pb := dr.pbdata
156
left := offset
157
if int64(len(pb.Data)) > offset {
158
+ dr.buf.Close()
159
dr.buf = NewRSNCFromBytes(pb.GetData()[offset:])
160
dr.linkPosition = 0
161
dr.offset = offset
@@ -211,23 +164,22 @@ func (dr *DagReader) Seek(offset int64, whence int) (int64, error) {
164
left -= int64(len(pb.Data))
165
}
166
214
- i := 0
215
- for ; i < len(pb.Blocksizes); i++ {
167
+ for i := 0; i < len(pb.Blocksizes); i++ {
168
if pb.Blocksizes[i] > uint64(left) {
169
+ dr.linkPosition = i
170
break
171
} else {
172
left -= int64(pb.Blocksizes[i])
173
}
174
}
222
- dr.resetBlockFetch(i)
223
- err = dr.precalcNextBuf()
175
+
176
+ err := dr.precalcNextBuf()
177
if err != nil {
178
return 0, err
179
}
180
228
- n, err := io.CopyN(ioutil.Discard, dr.buf, left)
181
+ n, err := dr.buf.Seek(left, os.SEEK_SET)
182
if err != nil {
230
- fmt.Printf("the copy failed: %s - [%d]\n", err, n)
183
return -1, err
184
}
185
left -= n
@@ -237,15 +189,11 @@ func (dr *DagReader) Seek(offset int64, whence int) (int64, error) {
189
dr.offset = offset
190
return offset, nil
191
case os.SEEK_CUR:
192
+ // TODO: be smarter here
193
noffset := dr.offset + offset
194
return dr.Seek(noffset, os.SEEK_SET)
195
case os.SEEK_END:
243
- pb := new(ftpb.Data)
244
- err := proto.Unmarshal(dr.node.Data, pb)
245
- if err != nil {
246
- return -1, err
247
- }
248
- noffset := int64(pb.GetFilesize()) - offset
196
+ noffset := int64(dr.pbdata.GetFilesize()) - offset
197
return dr.Seek(noffset, os.SEEK_SET)
198
default:
199
return 0, errors.New("invalid whence")