Don't waste 256KiB buffers on small chunks.
License: MIT Signed-off-by: Steven Allen <steven@stebalien.com>
Steven Allen committed
Dec 19, 2017 at 14:23 UTC
aafbe65a1315286cda90c2af59b464a50a9f9379
2 files changed
+22
-10
importer/chunk/splitting.go
+16
-10
@@ -5,6 +5,7 @@ import (
5
"io"
6
7
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
8
+ mpool "gx/ipfs/QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG/go-msgio/mpool"
9
)
10
11
var log = logging.Logger("chunk")
@@ -51,14 +52,14 @@ func Chan(s Splitter) (<-chan []byte, <-chan error) {
52
53
type sizeSplitterv2 struct {
54
r io.Reader
54
- size int64
55
+ size uint32
56
err error
57
}
58
59
func NewSizeSplitter(r io.Reader, size int64) Splitter {
60
return &sizeSplitterv2{
61
r: r,
61
- size: size,
62
+ size: uint32(size),
63
}
64
}
65
@@ -66,17 +67,22 @@ func (ss *sizeSplitterv2) NextBytes() ([]byte, error) {
67
if ss.err != nil {
68
return nil, ss.err
69
}
69
- buf := make([]byte, ss.size)
70
- n, err := io.ReadFull(ss.r, buf)
71
- if err == io.ErrUnexpectedEOF {
70
+
71
+ full := mpool.ByteSlicePool.Get(ss.size).([]byte)[:ss.size]
72
+ n, err := io.ReadFull(ss.r, full)
73
+ switch err {
74
+ case io.ErrUnexpectedEOF:
75
ss.err = io.EOF
73
- err = nil
74
- }
75
- if err != nil {
76
+ small := make([]byte, n)
77
+ copy(small, full)
78
+ mpool.ByteSlicePool.Put(ss.size, full)
79
+ return small, nil
80
+ case nil:
81
+ return full, nil
82
+ default:
83
+ mpool.ByteSlicePool.Put(ss.size, full)
84
return nil, err
85
}
78
-
79
- return buf[:n], nil
86
}
87
88
func (ss *sizeSplitterv2) Reader() io.Reader {
package.json
+6
@@ -509,6 +509,12 @@
509
"hash": "QmYmhgAcvmDGXct1qBvc1kz9BxQSit1XBrTeiGZp2FvRyn",
510
"name": "go-libp2p-blankhost",
511
"version": "0.2.3"
512
+ },
513
+ {
514
+ "author": "jbenet",
515
+ "hash": "QmWBug6eBS7AxRdCDVuSY5CnSit7cS2XnPFYJWqWDumhCG",
516
+ "name": "go-msgio",
517
+ "version": "0.0.3"
518
}
519
],
520
"gxVersion": "0.10.0",