merkledag: limit number of objects in a batch to prevent out of fd issues
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com>
Jeromy committed
Mar 6, 2017 at 17:01 UTC
31af335dc045aaa9911abc0dc990a3817ba0dd8a
1 file changed
+14
-5
merkledag/merkledag.go
+14
-5
@@ -68,7 +68,15 @@ func (n *dagService) Add(nd node.Node) (*cid.Cid, error) {
68
}
69
70
func (n *dagService) Batch() *Batch {
71
- return &Batch{ds: n, MaxSize: 8 * 1024 * 1024}
71
+ return &Batch{
72
+ ds: n,
73
+ MaxSize: 8 << 20,
74
+
75
+ // By default, only batch up to 128 nodes at a time.
76
+ // The current implementation of flatfs opens this many file
77
+ // descriptors at the same time for the optimized batch write.
78
+ MaxBlocks: 128,
79
+ }
80
}
81
82
// Get retrieves a node from the dagService, fetching the block in the BlockService
@@ -376,15 +384,16 @@ func (np *nodePromise) Get(ctx context.Context) (node.Node, error) {
384
type Batch struct {
385
ds *dagService
386
379
- blocks []blocks.Block
380
- size int
381
- MaxSize int
387
+ blocks []blocks.Block
388
+ size int
389
+ MaxSize int
390
+ MaxBlocks int
391
}
392
393
func (t *Batch) Add(nd node.Node) (*cid.Cid, error) {
394
t.blocks = append(t.blocks, nd)
395
t.size += len(nd.RawData())
387
- if t.size > t.MaxSize {
396
+ if t.size > t.MaxSize || len(t.blocks) > t.MaxBlocks {
397
return nd.Cid(), t.Commit()
398
}
399
return nd.Cid(), nil