parallelize batch flushing
1. Modern storage devices (i.e., SSDs) tend to be highly parallel. 2. Allows us to read and write at the same time (avoids pausing while flushing). fixes https://github.com/ipfs/go-ipfs/issues/898#issuecomment-331849064 License: MIT Signed-off-by: Steven Allen <steven@stebalien.com>
Steven Allen committed
Oct 10, 2017 at 19:21 UTC
09136e931b1748a5c67190af4a2c3f9d162ec25e
2 files changed
+101
-28
merkledag/batch.go
new
+98
@@ -0,0 +1,98 @@
1
+package merkledag
2
+
3
+import (
4
+ "runtime"
5
+
6
+ cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
7
+ node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
8
+ blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format"
9
+)
10
+
11
+// ParallelBatchCommits is the number of batch commits that can be in-flight before blocking.
12
+// TODO: Experiment with multiple datastores, storage devices, and CPUs to find
13
+// the right value/formula.
14
+var ParallelBatchCommits = runtime.NumCPU() * 2
15
+
16
+// Batch is a buffer for batching adds to a dag.
17
+type Batch struct {
18
+ ds *dagService
19
+
20
+ activeCommits int
21
+ commitError error
22
+ commitResults chan error
23
+
24
+ blocks []blocks.Block
25
+ size int
26
+
27
+ MaxSize int
28
+ MaxBlocks int
29
+}
30
+
31
+func (t *Batch) processResults() {
32
+ for t.activeCommits > 0 && t.commitError == nil {
33
+ select {
34
+ case err := <-t.commitResults:
35
+ t.activeCommits--
36
+ if err != nil {
37
+ t.commitError = err
38
+ }
39
+ default:
40
+ return
41
+ }
42
+ }
43
+}
44
+
45
+func (t *Batch) asyncCommit() {
46
+ if len(t.blocks) == 0 || t.commitError != nil {
47
+ return
48
+ }
49
+ if t.activeCommits >= ParallelBatchCommits {
50
+ err := <-t.commitResults
51
+ t.activeCommits--
52
+
53
+ if err != nil {
54
+ t.commitError = err
55
+ return
56
+ }
57
+ }
58
+ go func(b []blocks.Block) {
59
+ _, err := t.ds.Blocks.AddBlocks(b)
60
+ t.commitResults <- err
61
+ }(t.blocks)
62
+
63
+ t.activeCommits++
64
+ t.blocks = nil
65
+ t.size = 0
66
+
67
+ return
68
+}
69
+
70
+// Add adds a node to the batch and commits the batch if necessary.
71
+func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
72
+ // Not strictly necessary but allows us to catch errors early.
73
+ t.processResults()
74
+ if t.commitError != nil {
75
+ return nil, t.commitError
76
+ }
77
+
78
+ t.blocks = append(t.blocks, nd)
79
+ t.size += len(nd.RawData())
80
+ if t.size > t.MaxSize || len(t.blocks) > t.MaxBlocks {
81
+ t.asyncCommit()
82
+ }
83
+ return nd.Cid(), t.commitError
84
+}
85
+
86
+// Commit commits batched nodes.
87
+func (t *Batch) Commit() error {
88
+ t.asyncCommit()
89
+ for t.activeCommits > 0 && t.commitError == nil {
90
+ err := <-t.commitResults
91
+ t.activeCommits--
92
+ if err != nil {
93
+ t.commitError = err
94
+ }
95
+ }
96
+
97
+ return t.commitError
98
+}
merkledag/merkledag.go
+3
-28
@@ -11,7 +11,6 @@ import (
11
12
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
13
node "gx/ipfs/QmPN7cwmpcc4DWXb4KTB9dNAJgjuPY69h3npsMfhRrQL9c/go-ipld-format"
14
- blocks "gx/ipfs/QmSn9Td7xgxm9EV7iEjTckpUWmWApggzPxu7eFGWkkpwin/go-block-format"
14
ipldcbor "gx/ipfs/QmWCs8kMecJwCPK8JThue8TjgM2ieJ2HjTLDu7Cv2NEmZi/go-ipld-cbor"
15
)
16
@@ -75,8 +74,9 @@ func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
74
75
func (n *dagService) Batch() *Batch {
76
return &Batch{
78
- ds: n,
79
- MaxSize: 8 << 20,
77
+ ds: n,
78
+ commitResults: make(chan error, ParallelBatchCommits),
79
+ MaxSize: 8 << 20,
80
81
// By default, only batch up to 128 nodes at a time.
82
// The current implementation of flatfs opens this many file
@@ -389,31 +389,6 @@ func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
389
}
390
}
391
392
-type Batch struct {
393
- ds *dagService
394
-
395
- blocks []blocks.Block
396
- size int
397
- MaxSize int
398
- MaxBlocks int
399
-}
400
-
401
-func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
402
- t.blocks = append(t.blocks, nd)
403
- t.size += len(nd.RawData())
404
- if t.size > t.MaxSize || len(t.blocks) > t.MaxBlocks {
405
- return nd.Cid(), t.Commit()
406
- }
407
- return nd.Cid(), nil
408
-}
409
-
410
-func (t *Batch) Commit() error {
411
- _, err := t.ds.Blocks.AddBlocks(t.blocks)
412
- t.blocks = nil
413
- t.size = 0
414
- return err
415
-}
416
-
392
type GetLinks func(context.Context, *cid.Cid) ([]*node.Link, error)
393
394
// EnumerateChildren will walk the dag below the given root node and add all