merkledag traversal
Juan Batiz-Benet committed
Jan 7, 2015 at 11:04 UTC
bcce9ef0880b6320952425700f3f828405ba2d69
2 files changed
+623
merkledag/traverse/traverse.go
new
+226
@@ -0,0 +1,226 @@
1
+// Package traverse provides merkledag traversal functions
2
+package traverse
3
+
4
+import (
5
+ "errors"
6
+
7
+ mdag "github.com/jbenet/go-ipfs/merkledag"
8
+)
9
+
10
+// Order is an identifier for traversal algorithm orders
11
+type Order int
12
+
13
+const (
14
+ DFSPre Order = iota // depth-first pre-order
15
+ DFSPost // depth-first post-order
16
+ BFS // breadth-first
17
+)
18
+
19
+// Options specifies a series of traversal options
20
+type Options struct {
21
+ DAG mdag.DAGService // the dagservice to fetch nodes
22
+ Order Order // what order to traverse in
23
+ Func Func // the function to perform at each step
24
+ ErrFunc ErrFunc // see ErrFunc. Optional
25
+
26
+ SkipDuplicates bool // whether to skip duplicate nodes
27
+}
28
+
29
+// State is a current traversal state
30
+type State struct {
31
+ Node *mdag.Node
32
+ Depth int
33
+}
34
+
35
+type traversal struct {
36
+ opts Options
37
+ seen map[string]struct{}
38
+}
39
+
40
+func (t *traversal) shouldSkip(n *mdag.Node) (bool, error) {
41
+ if t.opts.SkipDuplicates {
42
+ k, err := n.Key()
43
+ if err != nil {
44
+ return true, err
45
+ }
46
+
47
+ if _, found := t.seen[string(k)]; found {
48
+ return true, nil
49
+ }
50
+ t.seen[string(k)] = struct{}{}
51
+ }
52
+
53
+ return false, nil
54
+}
55
+
56
+func (t *traversal) callFunc(next State) error {
57
+ return t.opts.Func(next)
58
+}
59
+
60
+// getNode returns the node for link. If it return an error,
61
+// stop processing. if it returns a nil node, just skip it.
62
+//
63
+// the error handling is a little complicated.
64
+func (t *traversal) getNode(link *mdag.Link) (*mdag.Node, error) {
65
+
66
+ getNode := func(l *mdag.Link) (*mdag.Node, error) {
67
+ next, err := l.GetNode(t.opts.DAG)
68
+ if err != nil {
69
+ return nil, err
70
+ }
71
+
72
+ skip, err := t.shouldSkip(next)
73
+ if skip {
74
+ next = nil
75
+ }
76
+ return next, err
77
+ }
78
+
79
+ next, err := getNode(link)
80
+ if err != nil && t.opts.ErrFunc != nil { // attempt recovery.
81
+ err = t.opts.ErrFunc(err)
82
+ next = nil // skip regardless
83
+ }
84
+ return next, err
85
+}
86
+
87
+// Func is the type of the function called for each dag.Node visited by Traverse.
88
+// The traversal argument contains the current traversal state.
89
+// If an error is returned, processing stops.
90
+type Func func(current State) error
91
+
92
+// If there is a problem walking to the Node, and ErrFunc is provided, Traverse
93
+// will call ErrFunc with the error encountered. ErrFunc can decide how to handle
94
+// that error, and return an error back to Traversal with how to proceed:
95
+// * nil - skip the Node and its children, but continue processing
96
+// * all other errors halt processing immediately.
97
+//
98
+// If ErrFunc is nil, Traversal will stop, as if:
99
+//
100
+// opts.ErrFunc = func(err error) { return err }
101
+//
102
+type ErrFunc func(err error) error
103
+
104
+func Traverse(root *mdag.Node, o Options) error {
105
+ t := traversal{
106
+ opts: o,
107
+ seen: map[string]struct{}{},
108
+ }
109
+
110
+ state := State{
111
+ Node: root,
112
+ Depth: 0,
113
+ }
114
+
115
+ switch o.Order {
116
+ default:
117
+ return dfsPreTraverse(state, &t)
118
+ case DFSPre:
119
+ return dfsPreTraverse(state, &t)
120
+ case DFSPost:
121
+ return dfsPostTraverse(state, &t)
122
+ case BFS:
123
+ return bfsTraverse(state, &t)
124
+ }
125
+}
126
+
127
+type dfsFunc func(state State, t *traversal) error
128
+
129
+func dfsPreTraverse(state State, t *traversal) error {
130
+ if err := t.callFunc(state); err != nil {
131
+ return err
132
+ }
133
+ if err := dfsDescend(dfsPreTraverse, state, t); err != nil {
134
+ return err
135
+ }
136
+ return nil
137
+}
138
+
139
+func dfsPostTraverse(state State, t *traversal) error {
140
+ if err := dfsDescend(dfsPostTraverse, state, t); err != nil {
141
+ return err
142
+ }
143
+ if err := t.callFunc(state); err != nil {
144
+ return err
145
+ }
146
+ return nil
147
+}
148
+
149
+func dfsDescend(df dfsFunc, curr State, t *traversal) error {
150
+ for _, l := range curr.Node.Links {
151
+ node, err := t.getNode(l)
152
+ if err != nil {
153
+ return err
154
+ }
155
+ if node == nil { // skip
156
+ continue
157
+ }
158
+
159
+ next := State{
160
+ Node: node,
161
+ Depth: curr.Depth + 1,
162
+ }
163
+ if err := df(next, t); err != nil {
164
+ return err
165
+ }
166
+ }
167
+ return nil
168
+}
169
+
170
+func bfsTraverse(root State, t *traversal) error {
171
+
172
+ if skip, err := t.shouldSkip(root.Node); skip || err != nil {
173
+ return err
174
+ }
175
+
176
+ var q queue
177
+ q.enq(root)
178
+ for q.len() > 0 {
179
+ curr := q.deq()
180
+ if curr.Node == nil {
181
+ return errors.New("failed to dequeue though queue not empty")
182
+ }
183
+
184
+ // call user's func
185
+ if err := t.callFunc(curr); err != nil {
186
+ return err
187
+ }
188
+
189
+ for _, l := range curr.Node.Links {
190
+ node, err := t.getNode(l)
191
+ if err != nil {
192
+ return err
193
+ }
194
+ if node == nil { // skip
195
+ continue
196
+ }
197
+
198
+ q.enq(State{
199
+ Node: node,
200
+ Depth: curr.Depth + 1,
201
+ })
202
+ }
203
+ }
204
+ return nil
205
+}
206
+
207
+type queue struct {
208
+ s []State
209
+}
210
+
211
+func (q *queue) enq(n State) {
212
+ q.s = append(q.s, n)
213
+}
214
+
215
+func (q *queue) deq() State {
216
+ if len(q.s) < 1 {
217
+ return State{}
218
+ }
219
+ n := q.s[0]
220
+ q.s = q.s[1:]
221
+ return n
222
+}
223
+
224
+func (q *queue) len() int {
225
+ return len(q.s)
226
+}
merkledag/traverse/traverse_test.go
new
+397
@@ -0,0 +1,397 @@
1
+package traverse
2
+
3
+import (
4
+ "bytes"
5
+ "fmt"
6
+ "testing"
7
+
8
+ mdag "github.com/jbenet/go-ipfs/merkledag"
9
+)
10
+
11
+func TestDFSPreNoSkip(t *testing.T) {
12
+ opts := Options{Order: DFSPre}
13
+
14
+ testWalkOutputs(t, newFan(t), opts, []byte(`
15
+0 /a
16
+1 /a/aa
17
+1 /a/ab
18
+1 /a/ac
19
+1 /a/ad
20
+`))
21
+
22
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
23
+0 /a
24
+1 /a/aa
25
+2 /a/aa/aaa
26
+3 /a/aa/aaa/aaaa
27
+4 /a/aa/aaa/aaaa/aaaaa
28
+`))
29
+
30
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
31
+0 /a
32
+1 /a/aa
33
+2 /a/aa/aaa
34
+2 /a/aa/aab
35
+1 /a/ab
36
+2 /a/ab/aba
37
+2 /a/ab/abb
38
+`))
39
+
40
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
41
+0 /a
42
+1 /a/aa
43
+2 /a/aa/aaa
44
+3 /a/aa/aaa/aaaa
45
+4 /a/aa/aaa/aaaa/aaaaa
46
+4 /a/aa/aaa/aaaa/aaaaa
47
+3 /a/aa/aaa/aaaa
48
+4 /a/aa/aaa/aaaa/aaaaa
49
+4 /a/aa/aaa/aaaa/aaaaa
50
+2 /a/aa/aaa
51
+3 /a/aa/aaa/aaaa
52
+4 /a/aa/aaa/aaaa/aaaaa
53
+4 /a/aa/aaa/aaaa/aaaaa
54
+3 /a/aa/aaa/aaaa
55
+4 /a/aa/aaa/aaaa/aaaaa
56
+4 /a/aa/aaa/aaaa/aaaaa
57
+1 /a/aa
58
+2 /a/aa/aaa
59
+3 /a/aa/aaa/aaaa
60
+4 /a/aa/aaa/aaaa/aaaaa
61
+4 /a/aa/aaa/aaaa/aaaaa
62
+3 /a/aa/aaa/aaaa
63
+4 /a/aa/aaa/aaaa/aaaaa
64
+4 /a/aa/aaa/aaaa/aaaaa
65
+2 /a/aa/aaa
66
+3 /a/aa/aaa/aaaa
67
+4 /a/aa/aaa/aaaa/aaaaa
68
+4 /a/aa/aaa/aaaa/aaaaa
69
+3 /a/aa/aaa/aaaa
70
+4 /a/aa/aaa/aaaa/aaaaa
71
+4 /a/aa/aaa/aaaa/aaaaa
72
+`))
73
+}
74
+
75
+func TestDFSPreSkip(t *testing.T) {
76
+ opts := Options{Order: DFSPre, SkipDuplicates: true}
77
+
78
+ testWalkOutputs(t, newFan(t), opts, []byte(`
79
+0 /a
80
+1 /a/aa
81
+1 /a/ab
82
+1 /a/ac
83
+1 /a/ad
84
+`))
85
+
86
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
87
+0 /a
88
+1 /a/aa
89
+2 /a/aa/aaa
90
+3 /a/aa/aaa/aaaa
91
+4 /a/aa/aaa/aaaa/aaaaa
92
+`))
93
+
94
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
95
+0 /a
96
+1 /a/aa
97
+2 /a/aa/aaa
98
+2 /a/aa/aab
99
+1 /a/ab
100
+2 /a/ab/aba
101
+2 /a/ab/abb
102
+`))
103
+
104
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
105
+0 /a
106
+1 /a/aa
107
+2 /a/aa/aaa
108
+3 /a/aa/aaa/aaaa
109
+4 /a/aa/aaa/aaaa/aaaaa
110
+`))
111
+}
112
+
113
+func TestDFSPostNoSkip(t *testing.T) {
114
+ opts := Options{Order: DFSPost}
115
+
116
+ testWalkOutputs(t, newFan(t), opts, []byte(`
117
+1 /a/aa
118
+1 /a/ab
119
+1 /a/ac
120
+1 /a/ad
121
+0 /a
122
+`))
123
+
124
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
125
+4 /a/aa/aaa/aaaa/aaaaa
126
+3 /a/aa/aaa/aaaa
127
+2 /a/aa/aaa
128
+1 /a/aa
129
+0 /a
130
+`))
131
+
132
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
133
+2 /a/aa/aaa
134
+2 /a/aa/aab
135
+1 /a/aa
136
+2 /a/ab/aba
137
+2 /a/ab/abb
138
+1 /a/ab
139
+0 /a
140
+`))
141
+
142
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
143
+4 /a/aa/aaa/aaaa/aaaaa
144
+4 /a/aa/aaa/aaaa/aaaaa
145
+3 /a/aa/aaa/aaaa
146
+4 /a/aa/aaa/aaaa/aaaaa
147
+4 /a/aa/aaa/aaaa/aaaaa
148
+3 /a/aa/aaa/aaaa
149
+2 /a/aa/aaa
150
+4 /a/aa/aaa/aaaa/aaaaa
151
+4 /a/aa/aaa/aaaa/aaaaa
152
+3 /a/aa/aaa/aaaa
153
+4 /a/aa/aaa/aaaa/aaaaa
154
+4 /a/aa/aaa/aaaa/aaaaa
155
+3 /a/aa/aaa/aaaa
156
+2 /a/aa/aaa
157
+1 /a/aa
158
+4 /a/aa/aaa/aaaa/aaaaa
159
+4 /a/aa/aaa/aaaa/aaaaa
160
+3 /a/aa/aaa/aaaa
161
+4 /a/aa/aaa/aaaa/aaaaa
162
+4 /a/aa/aaa/aaaa/aaaaa
163
+3 /a/aa/aaa/aaaa
164
+2 /a/aa/aaa
165
+4 /a/aa/aaa/aaaa/aaaaa
166
+4 /a/aa/aaa/aaaa/aaaaa
167
+3 /a/aa/aaa/aaaa
168
+4 /a/aa/aaa/aaaa/aaaaa
169
+4 /a/aa/aaa/aaaa/aaaaa
170
+3 /a/aa/aaa/aaaa
171
+2 /a/aa/aaa
172
+1 /a/aa
173
+0 /a
174
+`))
175
+}
176
+
177
+func TestDFSPostSkip(t *testing.T) {
178
+ opts := Options{Order: DFSPost, SkipDuplicates: true}
179
+
180
+ testWalkOutputs(t, newFan(t), opts, []byte(`
181
+1 /a/aa
182
+1 /a/ab
183
+1 /a/ac
184
+1 /a/ad
185
+0 /a
186
+`))
187
+
188
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
189
+4 /a/aa/aaa/aaaa/aaaaa
190
+3 /a/aa/aaa/aaaa
191
+2 /a/aa/aaa
192
+1 /a/aa
193
+0 /a
194
+`))
195
+
196
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
197
+2 /a/aa/aaa
198
+2 /a/aa/aab
199
+1 /a/aa
200
+2 /a/ab/aba
201
+2 /a/ab/abb
202
+1 /a/ab
203
+0 /a
204
+`))
205
+
206
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
207
+4 /a/aa/aaa/aaaa/aaaaa
208
+3 /a/aa/aaa/aaaa
209
+2 /a/aa/aaa
210
+1 /a/aa
211
+0 /a
212
+`))
213
+}
214
+
215
+func TestBFSNoSkip(t *testing.T) {
216
+ opts := Options{Order: BFS}
217
+
218
+ testWalkOutputs(t, newFan(t), opts, []byte(`
219
+0 /a
220
+1 /a/aa
221
+1 /a/ab
222
+1 /a/ac
223
+1 /a/ad
224
+`))
225
+
226
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
227
+0 /a
228
+1 /a/aa
229
+2 /a/aa/aaa
230
+3 /a/aa/aaa/aaaa
231
+4 /a/aa/aaa/aaaa/aaaaa
232
+`))
233
+
234
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
235
+0 /a
236
+1 /a/aa
237
+1 /a/ab
238
+2 /a/aa/aaa
239
+2 /a/aa/aab
240
+2 /a/ab/aba
241
+2 /a/ab/abb
242
+`))
243
+
244
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
245
+0 /a
246
+1 /a/aa
247
+1 /a/aa
248
+2 /a/aa/aaa
249
+2 /a/aa/aaa
250
+2 /a/aa/aaa
251
+2 /a/aa/aaa
252
+3 /a/aa/aaa/aaaa
253
+3 /a/aa/aaa/aaaa
254
+3 /a/aa/aaa/aaaa
255
+3 /a/aa/aaa/aaaa
256
+3 /a/aa/aaa/aaaa
257
+3 /a/aa/aaa/aaaa
258
+3 /a/aa/aaa/aaaa
259
+3 /a/aa/aaa/aaaa
260
+4 /a/aa/aaa/aaaa/aaaaa
261
+4 /a/aa/aaa/aaaa/aaaaa
262
+4 /a/aa/aaa/aaaa/aaaaa
263
+4 /a/aa/aaa/aaaa/aaaaa
264
+4 /a/aa/aaa/aaaa/aaaaa
265
+4 /a/aa/aaa/aaaa/aaaaa
266
+4 /a/aa/aaa/aaaa/aaaaa
267
+4 /a/aa/aaa/aaaa/aaaaa
268
+4 /a/aa/aaa/aaaa/aaaaa
269
+4 /a/aa/aaa/aaaa/aaaaa
270
+4 /a/aa/aaa/aaaa/aaaaa
271
+4 /a/aa/aaa/aaaa/aaaaa
272
+4 /a/aa/aaa/aaaa/aaaaa
273
+4 /a/aa/aaa/aaaa/aaaaa
274
+4 /a/aa/aaa/aaaa/aaaaa
275
+4 /a/aa/aaa/aaaa/aaaaa
276
+`))
277
+}
278
+
279
+func TestBFSSkip(t *testing.T) {
280
+ opts := Options{Order: BFS, SkipDuplicates: true}
281
+
282
+ testWalkOutputs(t, newFan(t), opts, []byte(`
283
+0 /a
284
+1 /a/aa
285
+1 /a/ab
286
+1 /a/ac
287
+1 /a/ad
288
+`))
289
+
290
+ testWalkOutputs(t, newLinkedList(t), opts, []byte(`
291
+0 /a
292
+1 /a/aa
293
+2 /a/aa/aaa
294
+3 /a/aa/aaa/aaaa
295
+4 /a/aa/aaa/aaaa/aaaaa
296
+`))
297
+
298
+ testWalkOutputs(t, newBinaryTree(t), opts, []byte(`
299
+0 /a
300
+1 /a/aa
301
+1 /a/ab
302
+2 /a/aa/aaa
303
+2 /a/aa/aab
304
+2 /a/ab/aba
305
+2 /a/ab/abb
306
+`))
307
+
308
+ testWalkOutputs(t, newBinaryDAG(t), opts, []byte(`
309
+0 /a
310
+1 /a/aa
311
+2 /a/aa/aaa
312
+3 /a/aa/aaa/aaaa
313
+4 /a/aa/aaa/aaaa/aaaaa
314
+`))
315
+}
316
+
317
+func testWalkOutputs(t *testing.T, root *mdag.Node, opts Options, expect []byte) {
318
+ expect = bytes.TrimLeft(expect, "\n")
319
+
320
+ var buf bytes.Buffer
321
+ walk := func(current State) error {
322
+ s := fmt.Sprintf("%d %s\n", current.Depth, current.Node.Data)
323
+ t.Logf("walk: %s", s)
324
+ buf.Write([]byte(s))
325
+ return nil
326
+ }
327
+
328
+ opts.Func = walk
329
+ if err := Traverse(root, opts); err != nil {
330
+ t.Error(err)
331
+ return
332
+ }
333
+
334
+ actual := buf.Bytes()
335
+ if !bytes.Equal(actual, expect) {
336
+ t.Error("error: outputs differ")
337
+ t.Logf("expect:\n%s", expect)
338
+ t.Logf("actual:\n%s", actual)
339
+ } else {
340
+ t.Logf("expect matches actual:\n%s", expect)
341
+ }
342
+}
343
+
344
+func newFan(t *testing.T) *mdag.Node {
345
+ a := &mdag.Node{Data: []byte("/a")}
346
+ addChild(t, a, "aa")
347
+ addChild(t, a, "ab")
348
+ addChild(t, a, "ac")
349
+ addChild(t, a, "ad")
350
+ return a
351
+}
352
+
353
+func newLinkedList(t *testing.T) *mdag.Node {
354
+ a := &mdag.Node{Data: []byte("/a")}
355
+ aa := addChild(t, a, "aa")
356
+ aaa := addChild(t, aa, "aaa")
357
+ aaaa := addChild(t, aaa, "aaaa")
358
+ addChild(t, aaaa, "aaaaa")
359
+ return a
360
+}
361
+
362
+func newBinaryTree(t *testing.T) *mdag.Node {
363
+ a := &mdag.Node{Data: []byte("/a")}
364
+ aa := addChild(t, a, "aa")
365
+ ab := addChild(t, a, "ab")
366
+ addChild(t, aa, "aaa")
367
+ addChild(t, aa, "aab")
368
+ addChild(t, ab, "aba")
369
+ addChild(t, ab, "abb")
370
+ return a
371
+}
372
+
373
+func newBinaryDAG(t *testing.T) *mdag.Node {
374
+ a := &mdag.Node{Data: []byte("/a")}
375
+ aa := addChild(t, a, "aa")
376
+ aaa := addChild(t, aa, "aaa")
377
+ aaaa := addChild(t, aaa, "aaaa")
378
+ aaaaa := addChild(t, aaaa, "aaaaa")
379
+ addLink(t, a, aa)
380
+ addLink(t, aa, aaa)
381
+ addLink(t, aaa, aaaa)
382
+ addLink(t, aaaa, aaaaa)
383
+ return a
384
+}
385
+
386
+func addLink(t *testing.T, a, b *mdag.Node) {
387
+ to := string(a.Data) + "2" + string(b.Data)
388
+ if err := a.AddNodeLink(to, b); err != nil {
389
+ t.Error(err)
390
+ }
391
+}
392
+
393
+func addChild(t *testing.T, a *mdag.Node, name string) *mdag.Node {
394
+ c := &mdag.Node{Data: []byte(string(a.Data) + "/" + name)}
395
+ addLink(t, a, c)
396
+ return c
397
+}