updated leveldb
Juan Batiz-Benet committed
Dec 6, 2014 at 13:37 UTC
3db0da504fd6cc425c01c01834cc493221d9765e
63 files changed
+4084
-1775
Godeps/Godeps.json
+6
-7
@@ -43,11 +43,6 @@
43
"Comment": "go.r60-152",
44
"Rev": "36be16571e14f67e114bb0af619e5de2c1591679"
45
},
46
- {
47
- "ImportPath": "code.google.com/p/snappy-go/snappy",
48
- "Comment": "null-15",
49
- "Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
50
- },
46
{
47
"ImportPath": "github.com/braintree/manners",
48
"Comment": "0.3.1-2-g5280e25",
@@ -99,7 +94,7 @@
94
},
95
{
96
"ImportPath": "github.com/jbenet/go-datastore",
102
- "Rev": "b31aad9b9b22e46d99a270ed5aebb259fab64dcc"
97
+ "Rev": "6a1c83bda2a71a9bdc936749fdb507df958ed949"
98
},
99
{
100
"ImportPath": "github.com/jbenet/go-is-domain",
@@ -150,7 +145,11 @@
145
},
146
{
147
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
153
- "Rev": "99056d50e56252fbe0021d5c893defca5a76baf8"
148
+ "Rev": "871eee0a7546bb7d1b2795142e29c4534abc49b3"
149
+ },
150
+ {
151
+ "ImportPath": "github.com/syndtr/gosnappy/snappy",
152
+ "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
153
},
154
{
155
"ImportPath": "gopkg.in/natefinch/lumberjack.v2",
Godeps/_workspace/src/github.com/jbenet/go-datastore/Godeps/Godeps.json
+6
-7
@@ -1,6 +1,6 @@
1
{
2
"ImportPath": "github.com/jbenet/go-datastore",
3
- "GoVersion": "go1.3.3",
3
+ "GoVersion": "go1.3",
4
"Packages": [
5
"./..."
6
],
@@ -10,11 +10,6 @@
10
"Comment": "null-12",
11
"Rev": "7dda39b2e7d5e265014674c5af696ba4186679e9"
12
},
13
- {
14
- "ImportPath": "code.google.com/p/snappy-go/snappy",
15
- "Comment": "null-15",
16
- "Rev": "12e4b4183793ac4b061921e7980845e750679fd0"
17
- },
13
{
14
"ImportPath": "github.com/codahale/blake2",
15
"Rev": "3fa823583afba430e8fc7cdbcc670dbf90bfacc4"
@@ -33,7 +28,11 @@
28
},
29
{
30
"ImportPath": "github.com/syndtr/goleveldb/leveldb",
36
- "Rev": "cd2b8f743192883ab9fbc5f070ebda1dc90f3732"
31
+ "Rev": "871eee0a7546bb7d1b2795142e29c4534abc49b3"
32
+ },
33
+ {
34
+ "ImportPath": "github.com/syndtr/gosnappy/snappy",
35
+ "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
36
},
37
{
38
"ImportPath": "gopkg.in/check.v1",
Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb/datastore.go
+1
-1
@@ -58,7 +58,7 @@ func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
58
}
59
60
func (d *datastore) Has(key ds.Key) (exists bool, err error) {
61
- return ds.GetBackedHas(d, key)
61
+ return d.DB.Has(key.Bytes(), nil)
62
}
63
64
func (d *datastore) Delete(key ds.Key) (err error) {
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go
+132
-96
@@ -8,65 +8,84 @@ package leveldb
8
9
import (
10
"encoding/binary"
11
- "errors"
11
+ "fmt"
12
13
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15
)
16
16
-var (
17
- errBatchTooShort = errors.New("leveldb: batch is too short")
18
- errBatchBadRecord = errors.New("leveldb: bad record in batch")
19
-)
17
+type ErrBatchCorrupted struct {
18
+ Reason string
19
+}
20
+
21
+func (e *ErrBatchCorrupted) Error() string {
22
+ return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
23
+}
24
+
25
+func newErrBatchCorrupted(reason string) error {
26
+ return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
27
+}
28
21
-const kBatchHdrLen = 8 + 4
29
+const (
30
+ batchHdrLen = 8 + 4
31
+ batchGrowRec = 3000
32
+)
33
23
-type batchReplay interface {
24
- put(key, value []byte, seq uint64)
25
- delete(key []byte, seq uint64)
34
+type BatchReplay interface {
35
+ Put(key, value []byte)
36
+ Delete(key []byte)
37
}
38
39
// Batch is a write batch.
40
type Batch struct {
30
- buf []byte
41
+ data []byte
42
rLen, bLen int
43
seq uint64
44
sync bool
45
}
46
47
func (b *Batch) grow(n int) {
37
- off := len(b.buf)
48
+ off := len(b.data)
49
if off == 0 {
39
- // include headers
40
- off = kBatchHdrLen
41
- n += off
50
+ off = batchHdrLen
51
+ if b.data != nil {
52
+ b.data = b.data[:off]
53
+ }
54
}
43
- if cap(b.buf)-off >= n {
44
- return
55
+ if cap(b.data)-off < n {
56
+ if b.data == nil {
57
+ b.data = make([]byte, off, off+n)
58
+ } else {
59
+ odata := b.data
60
+ div := 1
61
+ if b.rLen > batchGrowRec {
62
+ div = b.rLen / batchGrowRec
63
+ }
64
+ b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
65
+ copy(b.data, odata)
66
+ }
67
}
46
- buf := make([]byte, 2*cap(b.buf)+n)
47
- copy(buf, b.buf)
48
- b.buf = buf[:off]
68
}
69
51
-func (b *Batch) appendRec(t vType, key, value []byte) {
70
+func (b *Batch) appendRec(kt kType, key, value []byte) {
71
n := 1 + binary.MaxVarintLen32 + len(key)
53
- if t == tVal {
72
+ if kt == ktVal {
73
n += binary.MaxVarintLen32 + len(value)
74
}
75
b.grow(n)
57
- off := len(b.buf)
58
- buf := b.buf[:off+n]
59
- buf[off] = byte(t)
76
+ off := len(b.data)
77
+ data := b.data[:off+n]
78
+ data[off] = byte(kt)
79
off += 1
61
- off += binary.PutUvarint(buf[off:], uint64(len(key)))
62
- copy(buf[off:], key)
80
+ off += binary.PutUvarint(data[off:], uint64(len(key)))
81
+ copy(data[off:], key)
82
off += len(key)
64
- if t == tVal {
65
- off += binary.PutUvarint(buf[off:], uint64(len(value)))
66
- copy(buf[off:], value)
83
+ if kt == ktVal {
84
+ off += binary.PutUvarint(data[off:], uint64(len(value)))
85
+ copy(data[off:], value)
86
off += len(value)
87
}
69
- b.buf = buf[:off]
88
+ b.data = data[:off]
89
b.rLen++
90
// Include 8-byte ikey header
91
b.bLen += len(key) + len(value) + 8
@@ -75,18 +94,51 @@ func (b *Batch) appendRec(t vType, key, value []byte) {
94
// Put appends 'put operation' of the given key/value pair to the batch.
95
// It is safe to modify the contents of the argument after Put returns.
96
func (b *Batch) Put(key, value []byte) {
78
- b.appendRec(tVal, key, value)
97
+ b.appendRec(ktVal, key, value)
98
}
99
100
// Delete appends 'delete operation' of the given key to the batch.
101
// It is safe to modify the contents of the argument after Delete returns.
102
func (b *Batch) Delete(key []byte) {
84
- b.appendRec(tDel, key, nil)
103
+ b.appendRec(ktDel, key, nil)
104
+}
105
+
106
+// Dump dumps batch contents. The returned slice can be loaded into the
107
+// batch using Load method.
108
+// The returned slice is not its own copy, so the contents should not be
109
+// modified.
110
+func (b *Batch) Dump() []byte {
111
+ return b.encode()
112
+}
113
+
114
+// Load loads given slice into the batch. Previous contents of the batch
115
+// will be discarded.
116
+// The given slice will not be copied and will be used as batch buffer, so
117
+// it is not safe to modify the contents of the slice.
118
+func (b *Batch) Load(data []byte) error {
119
+ return b.decode(0, data)
120
+}
121
+
122
+// Replay replays batch contents.
123
+func (b *Batch) Replay(r BatchReplay) error {
124
+ return b.decodeRec(func(i int, kt kType, key, value []byte) {
125
+ switch kt {
126
+ case ktVal:
127
+ r.Put(key, value)
128
+ case ktDel:
129
+ r.Delete(key)
130
+ }
131
+ })
132
+}
133
+
134
+// Len returns number of records in the batch.
135
+func (b *Batch) Len() int {
136
+ return b.rLen
137
}
138
139
// Reset resets the batch.
140
func (b *Batch) Reset() {
89
- b.buf = nil
141
+ b.data = b.data[:0]
142
b.seq = 0
143
b.rLen = 0
144
b.bLen = 0
@@ -97,24 +149,10 @@ func (b *Batch) init(sync bool) {
149
b.sync = sync
150
}
151
100
-func (b *Batch) put(key, value []byte, seq uint64) {
101
- if b.rLen == 0 {
102
- b.seq = seq
103
- }
104
- b.Put(key, value)
105
-}
106
-
107
-func (b *Batch) delete(key []byte, seq uint64) {
108
- if b.rLen == 0 {
109
- b.seq = seq
110
- }
111
- b.Delete(key)
112
-}
113
-
152
func (b *Batch) append(p *Batch) {
153
if p.rLen > 0 {
116
- b.grow(len(p.buf) - kBatchHdrLen)
117
- b.buf = append(b.buf, p.buf[kBatchHdrLen:]...)
154
+ b.grow(len(p.data) - batchHdrLen)
155
+ b.data = append(b.data, p.data[batchHdrLen:]...)
156
b.rLen += p.rLen
157
}
158
if p.sync {
@@ -122,95 +160,93 @@ func (b *Batch) append(p *Batch) {
160
}
161
}
162
125
-func (b *Batch) len() int {
126
- return b.rLen
127
-}
128
-
163
+// size returns sums of key/value pair length plus 8-bytes ikey.
164
func (b *Batch) size() int {
165
return b.bLen
166
}
167
168
func (b *Batch) encode() []byte {
169
b.grow(0)
135
- binary.LittleEndian.PutUint64(b.buf, b.seq)
136
- binary.LittleEndian.PutUint32(b.buf[8:], uint32(b.rLen))
170
+ binary.LittleEndian.PutUint64(b.data, b.seq)
171
+ binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
172
138
- return b.buf
173
+ return b.data
174
}
175
141
-func (b *Batch) decode(buf []byte) error {
142
- if len(buf) < kBatchHdrLen {
143
- return errBatchTooShort
176
+func (b *Batch) decode(prevSeq uint64, data []byte) error {
177
+ if len(data) < batchHdrLen {
178
+ return newErrBatchCorrupted("too short")
179
}
180
146
- b.seq = binary.LittleEndian.Uint64(buf)
147
- b.rLen = int(binary.LittleEndian.Uint32(buf[8:]))
181
+ b.seq = binary.LittleEndian.Uint64(data)
182
+ if b.seq < prevSeq {
183
+ return newErrBatchCorrupted("invalid sequence number")
184
+ }
185
+ b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
186
+ if b.rLen < 0 {
187
+ return newErrBatchCorrupted("invalid records length")
188
+ }
189
// No need to be precise at this point, it won't be used anyway
149
- b.bLen = len(buf) - kBatchHdrLen
150
- b.buf = buf
190
+ b.bLen = len(data) - batchHdrLen
191
+ b.data = data
192
193
return nil
194
}
195
155
-func (b *Batch) decodeRec(f func(i int, t vType, key, value []byte)) error {
156
- off := kBatchHdrLen
196
+func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
197
+ off := batchHdrLen
198
for i := 0; i < b.rLen; i++ {
158
- if off >= len(b.buf) {
159
- return errors.New("leveldb: invalid batch record length")
199
+ if off >= len(b.data) {
200
+ return newErrBatchCorrupted("invalid records length")
201
}
202
162
- t := vType(b.buf[off])
163
- if t > tVal {
164
- return errors.New("leveldb: invalid batch record type in batch")
203
+ kt := kType(b.data[off])
204
+ if kt > ktVal {
205
+ return newErrBatchCorrupted("bad record: invalid type")
206
}
207
off += 1
208
168
- x, n := binary.Uvarint(b.buf[off:])
209
+ x, n := binary.Uvarint(b.data[off:])
210
off += n
170
- if n <= 0 || off+int(x) > len(b.buf) {
171
- return errBatchBadRecord
211
+ if n <= 0 || off+int(x) > len(b.data) {
212
+ return newErrBatchCorrupted("bad record: invalid key length")
213
}
173
- key := b.buf[off : off+int(x)]
214
+ key := b.data[off : off+int(x)]
215
off += int(x)
175
-
216
var value []byte
177
- if t == tVal {
178
- x, n := binary.Uvarint(b.buf[off:])
217
+ if kt == ktVal {
218
+ x, n := binary.Uvarint(b.data[off:])
219
off += n
180
- if n <= 0 || off+int(x) > len(b.buf) {
181
- return errBatchBadRecord
220
+ if n <= 0 || off+int(x) > len(b.data) {
221
+ return newErrBatchCorrupted("bad record: invalid value length")
222
}
183
- value = b.buf[off : off+int(x)]
223
+ value = b.data[off : off+int(x)]
224
off += int(x)
225
}
226
187
- f(i, t, key, value)
227
+ f(i, kt, key, value)
228
}
229
230
return nil
231
}
232
193
-func (b *Batch) replay(to batchReplay) error {
194
- return b.decodeRec(func(i int, t vType, key, value []byte) {
195
- switch t {
196
- case tVal:
197
- to.put(key, value, b.seq+uint64(i))
198
- case tDel:
199
- to.delete(key, b.seq+uint64(i))
200
- }
201
- })
202
-}
203
-
233
func (b *Batch) memReplay(to *memdb.DB) error {
205
- return b.decodeRec(func(i int, t vType, key, value []byte) {
206
- ikey := newIKey(key, b.seq+uint64(i), t)
234
+ return b.decodeRec(func(i int, kt kType, key, value []byte) {
235
+ ikey := newIkey(key, b.seq+uint64(i), kt)
236
to.Put(ikey, value)
237
})
238
}
239
240
+func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
241
+ if err := b.decode(prevSeq, data); err != nil {
242
+ return err
243
+ }
244
+ return b.memReplay(to)
245
+}
246
+
247
func (b *Batch) revertMemReplay(to *memdb.DB) error {
212
- return b.decodeRec(func(i int, t vType, key, value []byte) {
213
- ikey := newIKey(key, b.seq+uint64(i), t)
248
+ return b.decodeRec(func(i int, kt kType, key, value []byte) {
249
+ ikey := newIkey(key, b.seq+uint64(i), kt)
250
to.Delete(ikey)
251
})
252
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go
+13
-13
@@ -15,7 +15,7 @@ import (
15
)
16
17
type tbRec struct {
18
- t vType
18
+ kt kType
19
key, value []byte
20
}
21
@@ -23,39 +23,39 @@ type testBatch struct {
23
rec []*tbRec
24
}
25
26
-func (p *testBatch) put(key, value []byte, seq uint64) {
27
- p.rec = append(p.rec, &tbRec{tVal, key, value})
26
+func (p *testBatch) Put(key, value []byte) {
27
+ p.rec = append(p.rec, &tbRec{ktVal, key, value})
28
}
29
30
-func (p *testBatch) delete(key []byte, seq uint64) {
31
- p.rec = append(p.rec, &tbRec{tDel, key, nil})
30
+func (p *testBatch) Delete(key []byte) {
31
+ p.rec = append(p.rec, &tbRec{ktDel, key, nil})
32
}
33
34
func compareBatch(t *testing.T, b1, b2 *Batch) {
35
if b1.seq != b2.seq {
36
t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
37
}
38
- if b1.len() != b2.len() {
39
- t.Fatalf("invalid record length want %d, got %d", b1.len(), b2.len())
38
+ if b1.Len() != b2.Len() {
39
+ t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
40
}
41
p1, p2 := new(testBatch), new(testBatch)
42
- err := b1.replay(p1)
42
+ err := b1.Replay(p1)
43
if err != nil {
44
t.Fatal("error when replaying batch 1: ", err)
45
}
46
- err = b2.replay(p2)
46
+ err = b2.Replay(p2)
47
if err != nil {
48
t.Fatal("error when replaying batch 2: ", err)
49
}
50
for i := range p1.rec {
51
r1, r2 := p1.rec[i], p2.rec[i]
52
- if r1.t != r2.t {
53
- t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.t, r2.t)
52
+ if r1.kt != r2.kt {
53
+ t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
54
}
55
if !bytes.Equal(r1.key, r2.key) {
56
t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
57
}
58
- if r1.t == tVal {
58
+ if r1.kt == ktVal {
59
if !bytes.Equal(r1.value, r2.value) {
60
t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
61
}
@@ -75,7 +75,7 @@ func TestBatch_EncodeDecode(t *testing.T) {
75
b1.Delete([]byte("k"))
76
buf := b1.encode()
77
b2 := new(Batch)
78
- err := b2.decode(buf)
78
+ err := b2.decode(0, buf)
79
if err != nil {
80
t.Error("error when decoding batch: ", err)
81
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go
+2
-1
@@ -128,7 +128,8 @@ const (
128
type nodeState int
129
130
const (
131
- nodeEffective nodeState = iota
131
+ nodeZero nodeState = iota
132
+ nodeEffective
133
nodeEvicted
134
nodeDeleted
135
)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go
+254
-146
@@ -7,10 +7,8 @@
7
package cache
8
9
import (
10
- "fmt"
10
"math/rand"
11
"runtime"
13
- "strings"
12
"sync"
13
"sync/atomic"
14
"testing"
@@ -225,16 +223,16 @@ func TestLRUCache_Purge(t *testing.T) {
223
}
224
225
type testingCacheObjectCounter struct {
228
- created uint32
229
- released uint32
226
+ created uint
227
+ released uint
228
}
229
230
func (c *testingCacheObjectCounter) createOne() {
233
- atomic.AddUint32(&c.created, 1)
231
+ c.created++
232
}
233
234
func (c *testingCacheObjectCounter) releaseOne() {
237
- atomic.AddUint32(&c.released, 1)
235
+ c.released++
236
}
237
238
type testingCacheObject struct {
@@ -243,17 +241,75 @@ type testingCacheObject struct {
241
242
ns, key uint64
243
246
- releaseCalled uint32
244
+ releaseCalled bool
245
}
246
247
func (x *testingCacheObject) Release() {
250
- if atomic.CompareAndSwapUint32(&x.releaseCalled, 0, 1) {
248
+ if !x.releaseCalled {
249
+ x.releaseCalled = true
250
x.cnt.releaseOne()
251
} else {
253
- x.t.Errorf("duplicate setfin NS#%d KEY#%s", x.ns, x.key)
252
+ x.t.Errorf("duplicate setfin NS#%d KEY#%d", x.ns, x.key)
253
}
254
}
255
256
+func TestLRUCache_ConcurrentSetGet(t *testing.T) {
257
+ runtime.GOMAXPROCS(runtime.NumCPU())
258
+
259
+ seed := time.Now().UnixNano()
260
+ t.Logf("seed=%d", seed)
261
+
262
+ const (
263
+ N = 2000000
264
+ M = 4000
265
+ C = 3
266
+ )
267
+
268
+ var set, get uint32
269
+
270
+ wg := &sync.WaitGroup{}
271
+ c := NewLRUCache(M / 4)
272
+ for ni := uint64(0); ni < C; ni++ {
273
+ r0 := rand.New(rand.NewSource(seed + int64(ni)))
274
+ r1 := rand.New(rand.NewSource(seed + int64(ni) + 1))
275
+ ns := c.GetNamespace(ni)
276
+
277
+ wg.Add(2)
278
+ go func(ns Namespace, r *rand.Rand) {
279
+ for i := 0; i < N; i++ {
280
+ x := uint64(r.Int63n(M))
281
+ o := ns.Get(x, func() (int, interface{}) {
282
+ atomic.AddUint32(&set, 1)
283
+ return 1, x
284
+ })
285
+ if v := o.Value().(uint64); v != x {
286
+ t.Errorf("#%d invalid value, got=%d", x, v)
287
+ }
288
+ o.Release()
289
+ }
290
+ wg.Done()
291
+ }(ns, r0)
292
+ go func(ns Namespace, r *rand.Rand) {
293
+ for i := 0; i < N; i++ {
294
+ x := uint64(r.Int63n(M))
295
+ o := ns.Get(x, nil)
296
+ if o != nil {
297
+ atomic.AddUint32(&get, 1)
298
+ if v := o.Value().(uint64); v != x {
299
+ t.Errorf("#%d invalid value, got=%d", x, v)
300
+ }
301
+ o.Release()
302
+ }
303
+ }
304
+ wg.Done()
305
+ }(ns, r1)
306
+ }
307
+
308
+ wg.Wait()
309
+
310
+ t.Logf("set=%d get=%d", set, get)
311
+}
312
+
313
func TestLRUCache_Finalizer(t *testing.T) {
314
const (
315
capacity = 100
@@ -262,10 +318,6 @@ func TestLRUCache_Finalizer(t *testing.T) {
318
keymax = 8000
319
)
320
265
- runtime.GOMAXPROCS(runtime.NumCPU())
266
- defer runtime.GOMAXPROCS(1)
267
-
268
- wg := &sync.WaitGroup{}
321
cnt := &testingCacheObjectCounter{}
322
323
c := NewLRUCache(capacity)
@@ -273,38 +325,40 @@ func TestLRUCache_Finalizer(t *testing.T) {
325
type instance struct {
326
seed int64
327
rnd *rand.Rand
276
- ns uint64
277
- effective int32
328
+ nsid uint64
329
+ ns Namespace
330
+ effective int
331
handles []Handle
332
handlesMap map[uint64]int
333
334
delete bool
335
purge bool
336
zap bool
284
- wantDel int32
285
- delfinCalledAll int32
286
- delfinCalledEff int32
287
- purgefinCalled int32
337
+ wantDel int
338
+ delfinCalled int
339
+ delfinCalledAll int
340
+ delfinCalledEff int
341
+ purgefinCalled int
342
}
343
290
- instanceGet := func(p *instance, ns Namespace, key uint64) {
291
- h := ns.Get(key, func() (charge int, value interface{}) {
344
+ instanceGet := func(p *instance, key uint64) {
345
+ h := p.ns.Get(key, func() (charge int, value interface{}) {
346
to := &testingCacheObject{
347
t: t, cnt: cnt,
294
- ns: p.ns,
348
+ ns: p.nsid,
349
key: key,
350
}
297
- atomic.AddInt32(&p.effective, 1)
351
+ p.effective++
352
cnt.createOne()
353
return 1, releaserFunc{func() {
354
to.Release()
301
- atomic.AddInt32(&p.effective, -1)
355
+ p.effective--
356
}, to}
357
})
358
p.handles = append(p.handles, h)
359
p.handlesMap[key] = p.handlesMap[key] + 1
360
}
307
- instanceRelease := func(p *instance, ns Namespace, i int) {
361
+ instanceRelease := func(p *instance, i int) {
362
h := p.handles[i]
363
key := h.Value().(releaserFunc).value.(*testingCacheObject).key
364
if n := p.handlesMap[key]; n == 0 {
@@ -319,55 +373,71 @@ func TestLRUCache_Finalizer(t *testing.T) {
373
p.handles[len(p.handles) : len(p.handles)+1][0] = nil
374
}
375
322
- seeds := make([]int64, goroutines)
323
- instances := make([]instance, goroutines)
376
+ seed := time.Now().UnixNano()
377
+ t.Logf("seed=%d", seed)
378
+
379
+ instances := make([]*instance, goroutines)
380
for i := range instances {
325
- p := &instances[i]
381
+ p := &instance{}
382
p.handlesMap = make(map[uint64]int)
327
- if seeds[i] == 0 {
328
- seeds[i] = time.Now().UnixNano()
329
- }
330
- p.seed = seeds[i]
383
+ p.seed = seed + int64(i)
384
p.rnd = rand.New(rand.NewSource(p.seed))
332
- p.ns = uint64(i)
385
+ p.nsid = uint64(i)
386
+ p.ns = c.GetNamespace(p.nsid)
387
p.delete = i%6 == 0
388
p.purge = i%8 == 0
389
p.zap = i%12 == 0 || i%3 == 0
390
+ instances[i] = p
391
+ }
392
+
393
+ runr := rand.New(rand.NewSource(seed - 1))
394
+ run := func(rnd *rand.Rand, x []*instance, init func(p *instance) bool, fn func(p *instance, i int) bool) {
395
+ var (
396
+ rx []*instance
397
+ rn []int
398
+ )
399
+ if init == nil {
400
+ rx = append([]*instance{}, x...)
401
+ rn = make([]int, len(x))
402
+ } else {
403
+ for _, p := range x {
404
+ if init(p) {
405
+ rx = append(rx, p)
406
+ rn = append(rn, 0)
407
+ }
408
+ }
409
+ }
410
+ for len(rx) > 0 {
411
+ i := rand.Intn(len(rx))
412
+ if fn(rx[i], rn[i]) {
413
+ rn[i]++
414
+ } else {
415
+ rx = append(rx[:i], rx[i+1:]...)
416
+ rn = append(rn[:i], rn[i+1:]...)
417
+ }
418
+ }
419
}
420
338
- seedsStr := make([]string, len(seeds))
339
- for i, seed := range seeds {
340
- seedsStr[i] = fmt.Sprint(seed)
341
- }
342
- t.Logf("seeds := []int64{%s}", strings.Join(seedsStr, ", "))
343
-
421
// Get and release.
345
- for i := range instances {
346
- p := &instances[i]
347
-
348
- wg.Add(1)
349
- go func(p *instance) {
350
- defer wg.Done()
351
-
352
- ns := c.GetNamespace(p.ns)
353
- for i := 0; i < iterations; i++ {
354
- if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
355
- instanceGet(p, ns, uint64(p.rnd.Intn(keymax)))
356
- } else {
357
- instanceRelease(p, ns, p.rnd.Intn(len(p.handles)))
358
- }
422
+ run(runr, instances, nil, func(p *instance, i int) bool {
423
+ if i < iterations {
424
+ if len(p.handles) == 0 || p.rnd.Int()%2 == 0 {
425
+ instanceGet(p, uint64(p.rnd.Intn(keymax)))
426
+ } else {
427
+ instanceRelease(p, p.rnd.Intn(len(p.handles)))
428
}
360
- }(p)
361
- }
362
- wg.Wait()
429
+ return true
430
+ } else {
431
+ return false
432
+ }
433
+ })
434
435
if used, cap := c.Used(), c.Capacity(); used > cap {
436
t.Errorf("Used > capacity, used=%d cap=%d", used, cap)
437
}
438
439
// Check effective objects.
369
- for i := range instances {
370
- p := &instances[i]
440
+ for i, p := range instances {
441
if int(p.effective) < len(p.handlesMap) {
442
t.Errorf("#%d effective objects < acquired handle, eo=%d ah=%d", i, p.effective, len(p.handlesMap))
443
}
@@ -377,103 +447,93 @@ func TestLRUCache_Finalizer(t *testing.T) {
447
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
448
}
449
380
- // Delete and purge.
381
- for i := range instances {
382
- p := &instances[i]
450
+ // First delete.
451
+ run(runr, instances, func(p *instance) bool {
452
p.wantDel = p.effective
384
-
385
- wg.Add(1)
386
- go func(p *instance) {
387
- defer wg.Done()
388
-
389
- ns := c.GetNamespace(p.ns)
390
-
391
- if p.delete {
392
- for key := uint64(0); key < keymax; key++ {
393
- _, wantExist := p.handlesMap[key]
394
- gotExist := ns.Delete(key, func(exist, pending bool) {
395
- atomic.AddInt32(&p.delfinCalledAll, 1)
396
- if exist {
397
- atomic.AddInt32(&p.delfinCalledEff, 1)
398
- }
399
- })
400
- if !gotExist && wantExist {
401
- t.Errorf("delete on NS#%d KEY#%d not found", p.ns, key)
402
- }
453
+ return p.delete
454
+ }, func(p *instance, i int) bool {
455
+ key := uint64(i)
456
+ if key < keymax {
457
+ _, wantExist := p.handlesMap[key]
458
+ gotExist := p.ns.Delete(key, func(exist, pending bool) {
459
+ p.delfinCalledAll++
460
+ if exist {
461
+ p.delfinCalledEff++
462
}
463
+ })
464
+ if !gotExist && wantExist {
465
+ t.Errorf("delete on NS#%d KEY#%d not found", p.nsid, key)
466
+ }
467
+ return true
468
+ } else {
469
+ return false
470
+ }
471
+ })
472
405
- var delfinCalled int
406
- for key := uint64(0); key < keymax; key++ {
407
- func(key uint64) {
408
- gotExist := ns.Delete(key, func(exist, pending bool) {
409
- if exist && !pending {
410
- t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.ns, key)
411
- }
412
- delfinCalled++
413
- })
414
- if gotExist {
415
- t.Errorf("delete on NS#%d KEY#%d found", p.ns, key)
416
- }
417
- }(key)
418
- }
419
- if delfinCalled != keymax {
420
- t.Errorf("(2) #%d not all delete fin called, diff=%d", p.ns, keymax-delfinCalled)
473
+ // Second delete.
474
+ run(runr, instances, func(p *instance) bool {
475
+ p.delfinCalled = 0
476
+ return p.delete
477
+ }, func(p *instance, i int) bool {
478
+ key := uint64(i)
479
+ if key < keymax {
480
+ gotExist := p.ns.Delete(key, func(exist, pending bool) {
481
+ if exist && !pending {
482
+ t.Errorf("delete fin on NS#%d KEY#%d exist and not pending for deletion", p.nsid, key)
483
}
484
+ p.delfinCalled++
485
+ })
486
+ if gotExist {
487
+ t.Errorf("delete on NS#%d KEY#%d found", p.nsid, key)
488
}
423
-
424
- if p.purge {
425
- ns.Purge(func(ns, key uint64) {
426
- atomic.AddInt32(&p.purgefinCalled, 1)
427
- })
489
+ return true
490
+ } else {
491
+ if p.delfinCalled != keymax {
492
+ t.Errorf("(2) NS#%d not all delete fin called, diff=%d", p.nsid, keymax-p.delfinCalled)
493
}
429
- }(p)
430
- }
431
- wg.Wait()
494
+ return false
495
+ }
496
+ })
497
+
498
+ // Purge.
499
+ run(runr, instances, func(p *instance) bool {
500
+ return p.purge
501
+ }, func(p *instance, i int) bool {
502
+ p.ns.Purge(func(ns, key uint64) {
503
+ p.purgefinCalled++
504
+ })
505
+ return false
506
+ })
507
508
if want := int(cnt.created - cnt.released); c.Size() != want {
509
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
510
}
511
512
// Release.
438
- for i := range instances {
439
- p := &instances[i]
440
-
441
- if !p.zap {
442
- wg.Add(1)
443
- go func(p *instance) {
444
- defer wg.Done()
445
-
446
- ns := c.GetNamespace(p.ns)
447
- for i := len(p.handles) - 1; i >= 0; i-- {
448
- instanceRelease(p, ns, i)
449
- }
450
- }(p)
513
+ run(runr, instances, func(p *instance) bool {
514
+ return !p.zap
515
+ }, func(p *instance, i int) bool {
516
+ if len(p.handles) > 0 {
517
+ instanceRelease(p, len(p.handles)-1)
518
+ return true
519
+ } else {
520
+ return false
521
}
452
- }
453
- wg.Wait()
522
+ })
523
524
if want := int(cnt.created - cnt.released); c.Size() != want {
525
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
526
}
527
528
// Zap.
460
- for i := range instances {
461
- p := &instances[i]
462
-
463
- if p.zap {
464
- wg.Add(1)
465
- go func(p *instance) {
466
- defer wg.Done()
467
-
468
- ns := c.GetNamespace(p.ns)
469
- ns.Zap()
470
-
471
- p.handles = nil
472
- p.handlesMap = nil
473
- }(p)
474
- }
475
- }
476
- wg.Wait()
529
+ run(runr, instances, func(p *instance) bool {
530
+ return p.zap
531
+ }, func(p *instance, i int) bool {
532
+ p.ns.Zap()
533
+ p.handles = nil
534
+ p.handlesMap = nil
535
+ return false
536
+ })
537
538
if want := int(cnt.created - cnt.released); c.Size() != want {
539
t.Errorf("Invalid cache size, want=%d got=%d", want, c.Size())
@@ -485,23 +545,21 @@ func TestLRUCache_Finalizer(t *testing.T) {
545
546
c.Purge(nil)
547
488
- for i := range instances {
489
- p := &instances[i]
490
-
548
+ for _, p := range instances {
549
if p.delete {
550
if p.delfinCalledAll != keymax {
493
- t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.ns, p.purge, p.zap, keymax-p.delfinCalledAll)
551
+ t.Errorf("#%d not all delete fin called, purge=%v zap=%v diff=%d", p.nsid, p.purge, p.zap, keymax-p.delfinCalledAll)
552
}
553
if p.delfinCalledEff != p.wantDel {
496
- t.Errorf("#%d not all effective delete fin called, diff=%d", p.ns, p.wantDel-p.delfinCalledEff)
554
+ t.Errorf("#%d not all effective delete fin called, diff=%d", p.nsid, p.wantDel-p.delfinCalledEff)
555
}
556
if p.purge && p.purgefinCalled > 0 {
499
- t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.ns, p.delete, p.zap, p.purgefinCalled)
557
+ t.Errorf("#%d some purge fin called, delete=%v zap=%v n=%d", p.nsid, p.delete, p.zap, p.purgefinCalled)
558
}
559
} else {
560
if p.purge {
561
if p.purgefinCalled != p.wantDel {
504
- t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.ns, p.delete, p.zap, p.wantDel-p.purgefinCalled)
562
+ t.Errorf("#%d not all purge fin called, delete=%v zap=%v diff=%d", p.nsid, p.delete, p.zap, p.wantDel-p.purgefinCalled)
563
}
564
}
565
}
@@ -512,6 +570,56 @@ func TestLRUCache_Finalizer(t *testing.T) {
570
}
571
}
572
573
+func BenchmarkLRUCache_Set(b *testing.B) {
574
+ c := NewLRUCache(0)
575
+ ns := c.GetNamespace(0)
576
+ b.ResetTimer()
577
+ for i := uint64(0); i < uint64(b.N); i++ {
578
+ set(ns, i, "", 1, nil)
579
+ }
580
+}
581
+
582
+func BenchmarkLRUCache_Get(b *testing.B) {
583
+ c := NewLRUCache(0)
584
+ ns := c.GetNamespace(0)
585
+ b.ResetTimer()
586
+ for i := uint64(0); i < uint64(b.N); i++ {
587
+ set(ns, i, "", 1, nil)
588
+ }
589
+ b.ResetTimer()
590
+ for i := uint64(0); i < uint64(b.N); i++ {
591
+ ns.Get(i, nil)
592
+ }
593
+}
594
+
595
+func BenchmarkLRUCache_Get2(b *testing.B) {
596
+ c := NewLRUCache(0)
597
+ ns := c.GetNamespace(0)
598
+ b.ResetTimer()
599
+ for i := uint64(0); i < uint64(b.N); i++ {
600
+ set(ns, i, "", 1, nil)
601
+ }
602
+ b.ResetTimer()
603
+ for i := uint64(0); i < uint64(b.N); i++ {
604
+ ns.Get(i, func() (charge int, value interface{}) {
605
+ return 0, nil
606
+ })
607
+ }
608
+}
609
+
610
+func BenchmarkLRUCache_Release(b *testing.B) {
611
+ c := NewLRUCache(0)
612
+ ns := c.GetNamespace(0)
613
+ handles := make([]Handle, b.N)
614
+ for i := uint64(0); i < uint64(b.N); i++ {
615
+ handles[i] = set(ns, i, "", 1, nil)
616
+ }
617
+ b.ResetTimer()
618
+ for _, h := range handles {
619
+ h.Release()
620
+ }
621
+}
622
+
623
func BenchmarkLRUCache_SetRelease(b *testing.B) {
624
capacity := b.N / 100
625
if capacity <= 0 {
@@ -521,7 +629,7 @@ func BenchmarkLRUCache_SetRelease(b *testing.B) {
629
ns := c.GetNamespace(0)
630
b.ResetTimer()
631
for i := uint64(0); i < uint64(b.N); i++ {
524
- set(ns, i, nil, 1, nil).Release()
632
+ set(ns, i, "", 1, nil).Release()
633
}
634
}
635
@@ -538,10 +646,10 @@ func BenchmarkLRUCache_SetReleaseTwice(b *testing.B) {
646
nb := b.N - na
647
648
for i := uint64(0); i < uint64(na); i++ {
541
- set(ns, i, nil, 1, nil).Release()
649
+ set(ns, i, "", 1, nil).Release()
650
}
651
652
for i := uint64(0); i < uint64(nb); i++ {
545
- set(ns, i, nil, 1, nil).Release()
653
+ set(ns, i, "", 1, nil).Release()
654
}
655
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru_cache.go
+334
-94
@@ -13,6 +13,13 @@ import (
13
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
14
)
15
16
+// The LLRB implementation were taken from https://github.com/petar/GoLLRB.
17
+// Which contains the following header:
18
+//
19
+// Copyright 2010 Petar Maymounkov. All rights reserved.
20
+// Use of this source code is governed by a BSD-style
21
+// license that can be found in the LICENSE file.
22
+
23
// lruCache represent a LRU cache state.
24
type lruCache struct {
25
mu sync.Mutex
@@ -74,11 +81,7 @@ func (c *lruCache) GetNamespace(id uint64) Namespace {
81
return ns
82
}
83
77
- ns := &lruNs{
78
- lru: c,
79
- id: id,
80
- table: make(map[uint64]*lruNode),
81
- }
84
+ ns := &lruNs{lru: c, id: id}
85
c.table[id] = ns
86
return ns
87
}
@@ -121,6 +124,9 @@ func (c *lruCache) Zap() {
124
func (c *lruCache) evict() {
125
top := &c.recent
126
for n := c.recent.rPrev; c.used > c.capacity && n != top; {
127
+ if n.state != nodeEffective {
128
+ panic("evicting non effective node")
129
+ }
130
n.state = nodeEvicted
131
n.rRemove()
132
n.derefNB()
@@ -130,130 +136,271 @@ func (c *lruCache) evict() {
136
}
137
138
type lruNs struct {
133
- lru *lruCache
134
- id uint64
135
- table map[uint64]*lruNode
136
- state nsState
139
+ lru *lruCache
140
+ id uint64
141
+ rbRoot *lruNode
142
+ state nsState
143
}
144
139
-func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
140
- ns.lru.mu.Lock()
145
+func (ns *lruNs) rbGetOrCreateNode(h *lruNode, key uint64) (hn, n *lruNode) {
146
+ if h == nil {
147
+ n = &lruNode{ns: ns, key: key}
148
+ return n, n
149
+ }
150
142
- if ns.state != nsEffective {
143
- ns.lru.mu.Unlock()
151
+ if key < h.key {
152
+ hn, n = ns.rbGetOrCreateNode(h.rbLeft, key)
153
+ if hn != nil {
154
+ h.rbLeft = hn
155
+ } else {
156
+ return nil, n
157
+ }
158
+ } else if key > h.key {
159
+ hn, n = ns.rbGetOrCreateNode(h.rbRight, key)
160
+ if hn != nil {
161
+ h.rbRight = hn
162
+ } else {
163
+ return nil, n
164
+ }
165
+ } else {
166
+ return nil, h
167
+ }
168
+
169
+ if rbIsRed(h.rbRight) && !rbIsRed(h.rbLeft) {
170
+ h = rbRotLeft(h)
171
+ }
172
+ if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
173
+ h = rbRotRight(h)
174
+ }
175
+ if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
176
+ rbFlip(h)
177
+ }
178
+ return h, n
179
+}
180
+
181
+func (ns *lruNs) getOrCreateNode(key uint64) *lruNode {
182
+ hn, n := ns.rbGetOrCreateNode(ns.rbRoot, key)
183
+ if hn != nil {
184
+ ns.rbRoot = hn
185
+ ns.rbRoot.rbBlack = true
186
+ }
187
+ return n
188
+}
189
+
190
+func (ns *lruNs) rbGetNode(key uint64) *lruNode {
191
+ h := ns.rbRoot
192
+ for h != nil {
193
+ switch {
194
+ case key < h.key:
195
+ h = h.rbLeft
196
+ case key > h.key:
197
+ h = h.rbRight
198
+ default:
199
+ return h
200
+ }
201
+ }
202
+ return nil
203
+}
204
+
205
+func (ns *lruNs) getNode(key uint64) *lruNode {
206
+ return ns.rbGetNode(key)
207
+}
208
+
209
+func (ns *lruNs) rbDeleteNode(h *lruNode, key uint64) *lruNode {
210
+ if h == nil {
211
return nil
212
}
213
147
- node, ok := ns.table[key]
148
- if ok {
149
- switch node.state {
150
- case nodeEvicted:
151
- // Insert to recent list.
152
- node.state = nodeEffective
153
- node.ref++
154
- ns.lru.used += node.charge
155
- ns.lru.evict()
156
- fallthrough
157
- case nodeEffective:
158
- // Bump to front.
159
- node.rRemove()
160
- node.rInsert(&ns.lru.recent)
214
+ if key < h.key {
215
+ if h.rbLeft == nil { // key not present. Nothing to delete
216
+ return h
217
+ }
218
+ if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
219
+ h = rbMoveLeft(h)
220
}
162
- node.ref++
221
+ h.rbLeft = ns.rbDeleteNode(h.rbLeft, key)
222
} else {
164
- if setf == nil {
165
- ns.lru.mu.Unlock()
223
+ if rbIsRed(h.rbLeft) {
224
+ h = rbRotRight(h)
225
+ }
226
+ // If @key equals @h.key and no right children at @h
227
+ if h.key == key && h.rbRight == nil {
228
return nil
229
}
230
+ if h.rbRight != nil && !rbIsRed(h.rbRight) && !rbIsRed(h.rbRight.rbLeft) {
231
+ h = rbMoveRight(h)
232
+ }
233
+ // If @key equals @h.key, and (from above) 'h.Right != nil'
234
+ if h.key == key {
235
+ var x *lruNode
236
+ h.rbRight, x = rbDeleteMin(h.rbRight)
237
+ if x == nil {
238
+ panic("logic")
239
+ }
240
+ x.rbLeft, h.rbLeft = h.rbLeft, nil
241
+ x.rbRight, h.rbRight = h.rbRight, nil
242
+ x.rbBlack = h.rbBlack
243
+ h = x
244
+ } else { // Else, @key is bigger than @h.key
245
+ h.rbRight = ns.rbDeleteNode(h.rbRight, key)
246
+ }
247
+ }
248
+
249
+ return rbFixup(h)
250
+}
251
+
252
+func (ns *lruNs) deleteNode(key uint64) {
253
+ ns.rbRoot = ns.rbDeleteNode(ns.rbRoot, key)
254
+ if ns.rbRoot != nil {
255
+ ns.rbRoot.rbBlack = true
256
+ }
257
+}
258
+
259
+func (ns *lruNs) rbIterateNodes(h *lruNode, pivot uint64, iter func(n *lruNode) bool) bool {
260
+ if h == nil {
261
+ return true
262
+ }
263
+ if h.key >= pivot {
264
+ if !ns.rbIterateNodes(h.rbLeft, pivot, iter) {
265
+ return false
266
+ }
267
+ if !iter(h) {
268
+ return false
269
+ }
270
+ }
271
+ return ns.rbIterateNodes(h.rbRight, pivot, iter)
272
+}
273
+
274
+func (ns *lruNs) iterateNodes(iter func(n *lruNode) bool) {
275
+ ns.rbIterateNodes(ns.rbRoot, 0, iter)
276
+}
277
+
278
+func (ns *lruNs) Get(key uint64, setf SetFunc) Handle {
279
+ ns.lru.mu.Lock()
280
+ defer ns.lru.mu.Unlock()
281
+
282
+ if ns.state != nsEffective {
283
+ return nil
284
+ }
285
286
+ var n *lruNode
287
+ if setf == nil {
288
+ n = ns.getNode(key)
289
+ if n == nil {
290
+ return nil
291
+ }
292
+ } else {
293
+ n = ns.getOrCreateNode(key)
294
+ }
295
+ switch n.state {
296
+ case nodeZero:
297
charge, value := setf()
298
if value == nil {
171
- ns.lru.mu.Unlock()
299
+ ns.deleteNode(key)
300
return nil
301
}
174
-
175
- node = &lruNode{
176
- ns: ns,
177
- key: key,
178
- value: value,
179
- charge: charge,
180
- ref: 1,
302
+ if charge < 0 {
303
+ charge = 0
304
}
182
- ns.table[key] = node
305
+
306
+ n.value = value
307
+ n.charge = charge
308
+ n.state = nodeEvicted
309
310
ns.lru.size += charge
311
ns.lru.alive++
186
- if charge > 0 {
187
- node.ref++
188
- node.rInsert(&ns.lru.recent)
189
- ns.lru.used += charge
190
- ns.lru.evict()
312
+
313
+ fallthrough
314
+ case nodeEvicted:
315
+ if n.charge == 0 {
316
+ break
317
}
318
+
319
+ // Insert to recent list.
320
+ n.state = nodeEffective
321
+ n.ref++
322
+ ns.lru.used += n.charge
323
+ ns.lru.evict()
324
+
325
+ fallthrough
326
+ case nodeEffective:
327
+ // Bump to front.
328
+ n.rRemove()
329
+ n.rInsert(&ns.lru.recent)
330
+ case nodeDeleted:
331
+ // Do nothing.
332
+ default:
333
+ panic("invalid state")
334
}
335
+ n.ref++
336
194
- ns.lru.mu.Unlock()
195
- return &lruHandle{node: node}
337
+ return &lruHandle{node: n}
338
}
339
340
func (ns *lruNs) Delete(key uint64, fin DelFin) bool {
341
ns.lru.mu.Lock()
342
+ defer ns.lru.mu.Unlock()
343
344
if ns.state != nsEffective {
345
if fin != nil {
346
fin(false, false)
347
}
205
- ns.lru.mu.Unlock()
348
return false
349
}
350
209
- node, exist := ns.table[key]
210
- if !exist {
351
+ n := ns.getNode(key)
352
+ if n == nil {
353
if fin != nil {
354
fin(false, false)
355
}
214
- ns.lru.mu.Unlock()
356
return false
357
+
358
}
359
218
- switch node.state {
360
+ switch n.state {
361
+ case nodeEffective:
362
+ ns.lru.used -= n.charge
363
+ n.state = nodeDeleted
364
+ n.delfin = fin
365
+ n.rRemove()
366
+ n.derefNB()
367
+ case nodeEvicted:
368
+ n.state = nodeDeleted
369
+ n.delfin = fin
370
case nodeDeleted:
371
if fin != nil {
372
fin(true, true)
373
}
223
- ns.lru.mu.Unlock()
374
return false
225
- case nodeEffective:
226
- ns.lru.used -= node.charge
227
- node.state = nodeDeleted
228
- node.delfin = fin
229
- node.rRemove()
230
- node.derefNB()
375
default:
232
- node.state = nodeDeleted
233
- node.delfin = fin
376
+ panic("invalid state")
377
}
378
236
- ns.lru.mu.Unlock()
379
return true
380
}
381
382
func (ns *lruNs) purgeNB(fin PurgeFin) {
241
- if ns.state != nsEffective {
242
- return
243
- }
244
-
245
- for _, node := range ns.table {
246
- switch node.state {
247
- case nodeDeleted:
248
- case nodeEffective:
249
- ns.lru.used -= node.charge
250
- node.state = nodeDeleted
251
- node.purgefin = fin
252
- node.rRemove()
253
- node.derefNB()
254
- default:
255
- node.state = nodeDeleted
256
- node.purgefin = fin
383
+ if ns.state == nsEffective {
384
+ var nodes []*lruNode
385
+ ns.iterateNodes(func(n *lruNode) bool {
386
+ nodes = append(nodes, n)
387
+ return true
388
+ })
389
+ for _, n := range nodes {
390
+ switch n.state {
391
+ case nodeEffective:
392
+ ns.lru.used -= n.charge
393
+ n.state = nodeDeleted
394
+ n.purgefin = fin
395
+ n.rRemove()
396
+ n.derefNB()
397
+ case nodeEvicted:
398
+ n.state = nodeDeleted
399
+ n.purgefin = fin
400
+ case nodeDeleted:
401
+ default:
402
+ panic("invalid state")
403
+ }
404
}
405
}
406
}
@@ -265,22 +412,22 @@ func (ns *lruNs) Purge(fin PurgeFin) {
412
}
413
414
func (ns *lruNs) zapNB() {
268
- if ns.state != nsEffective {
269
- return
270
- }
271
-
272
- ns.state = nsZapped
415
+ if ns.state == nsEffective {
416
+ ns.state = nsZapped
417
+
418
+ ns.iterateNodes(func(n *lruNode) bool {
419
+ if n.state == nodeEffective {
420
+ ns.lru.used -= n.charge
421
+ n.rRemove()
422
+ }
423
+ ns.lru.size -= n.charge
424
+ n.state = nodeDeleted
425
+ n.fin()
426
274
- for _, node := range ns.table {
275
- if node.state == nodeEffective {
276
- ns.lru.used -= node.charge
277
- node.rRemove()
278
- }
279
- ns.lru.size -= node.charge
280
- node.state = nodeDeleted
281
- node.fin()
427
+ return true
428
+ })
429
+ ns.rbRoot = nil
430
}
283
- ns.table = nil
431
}
432
433
func (ns *lruNs) Zap() {
@@ -293,7 +440,9 @@ func (ns *lruNs) Zap() {
440
type lruNode struct {
441
ns *lruNs
442
296
- rNext, rPrev *lruNode
443
+ rNext, rPrev *lruNode
444
+ rbLeft, rbRight *lruNode
445
+ rbBlack bool
446
447
key uint64
448
value interface{}
@@ -330,8 +479,10 @@ func (n *lruNode) fin() {
479
r.Release()
480
}
481
if n.purgefin != nil {
482
+ if n.delfin != nil {
483
+ panic("conflicting delete and purge fin")
484
+ }
485
n.purgefin(n.ns.id, n.key)
334
- n.delfin = nil
486
n.purgefin = nil
487
} else if n.delfin != nil {
488
n.delfin(true, false)
@@ -344,7 +495,7 @@ func (n *lruNode) derefNB() {
495
if n.ref == 0 {
496
if n.ns.state == nsEffective {
497
// Remove elemement.
347
- delete(n.ns.table, n.key)
498
+ n.ns.deleteNode(n.key)
499
n.ns.lru.size -= n.charge
500
n.ns.lru.alive--
501
n.fin()
@@ -380,3 +531,92 @@ func (h *lruHandle) Release() {
531
h.node.deref()
532
h.node = nil
533
}
534
+
535
+func rbIsRed(h *lruNode) bool {
536
+ if h == nil {
537
+ return false
538
+ }
539
+ return !h.rbBlack
540
+}
541
+
542
+func rbRotLeft(h *lruNode) *lruNode {
543
+ x := h.rbRight
544
+ if x.rbBlack {
545
+ panic("rotating a black link")
546
+ }
547
+ h.rbRight = x.rbLeft
548
+ x.rbLeft = h
549
+ x.rbBlack = h.rbBlack
550
+ h.rbBlack = false
551
+ return x
552
+}
553
+
554
+func rbRotRight(h *lruNode) *lruNode {
555
+ x := h.rbLeft
556
+ if x.rbBlack {
557
+ panic("rotating a black link")
558
+ }
559
+ h.rbLeft = x.rbRight
560
+ x.rbRight = h
561
+ x.rbBlack = h.rbBlack
562
+ h.rbBlack = false
563
+ return x
564
+}
565
+
566
+func rbFlip(h *lruNode) {
567
+ h.rbBlack = !h.rbBlack
568
+ h.rbLeft.rbBlack = !h.rbLeft.rbBlack
569
+ h.rbRight.rbBlack = !h.rbRight.rbBlack
570
+}
571
+
572
+func rbMoveLeft(h *lruNode) *lruNode {
573
+ rbFlip(h)
574
+ if rbIsRed(h.rbRight.rbLeft) {
575
+ h.rbRight = rbRotRight(h.rbRight)
576
+ h = rbRotLeft(h)
577
+ rbFlip(h)
578
+ }
579
+ return h
580
+}
581
+
582
+func rbMoveRight(h *lruNode) *lruNode {
583
+ rbFlip(h)
584
+ if rbIsRed(h.rbLeft.rbLeft) {
585
+ h = rbRotRight(h)
586
+ rbFlip(h)
587
+ }
588
+ return h
589
+}
590
+
591
+func rbFixup(h *lruNode) *lruNode {
592
+ if rbIsRed(h.rbRight) {
593
+ h = rbRotLeft(h)
594
+ }
595
+
596
+ if rbIsRed(h.rbLeft) && rbIsRed(h.rbLeft.rbLeft) {
597
+ h = rbRotRight(h)
598
+ }
599
+
600
+ if rbIsRed(h.rbLeft) && rbIsRed(h.rbRight) {
601
+ rbFlip(h)
602
+ }
603
+
604
+ return h
605
+}
606
+
607
+func rbDeleteMin(h *lruNode) (hn, n *lruNode) {
608
+ if h == nil {
609
+ return nil, nil
610
+ }
611
+ if h.rbLeft == nil {
612
+ return nil, h
613
+ }
614
+
615
+ if !rbIsRed(h.rbLeft) && !rbIsRed(h.rbLeft.rbLeft) {
616
+ h = rbMoveLeft(h)
617
+ }
618
+
619
+ h.rbLeft, n = rbDeleteMin(h.rbLeft)
620
+
621
+ return rbFixup(h), n
622
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/config.go
deleted
-40
@@ -1,40 +0,0 @@
1
-// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2
-// All rights reserved.
3
-//
4
-// Use of this source code is governed by a BSD-style license that can be
5
-// found in the LICENSE file.
6
-
7
-package leveldb
8
-
9
-const (
10
- kNumLevels = 7
11
-
12
- // Level-0 compaction is started when we hit this many files.
13
- kL0_CompactionTrigger float64 = 4
14
-
15
- // Soft limit on number of level-0 files. We slow down writes at this point.
16
- kL0_SlowdownWritesTrigger = 8
17
-
18
- // Maximum number of level-0 files. We stop writes at this point.
19
- kL0_StopWritesTrigger = 12
20
-
21
- // Maximum level to which a new compacted memdb is pushed if it
22
- // does not create overlap. We try to push to level 2 to avoid the
23
- // relatively expensive level 0=>1 compactions and to avoid some
24
- // expensive manifest file operations. We do not push all the way to
25
- // the largest level since that can generate a lot of wasted disk
26
- // space if the same key space is being repeatedly overwritten.
27
- kMaxMemCompactLevel = 2
28
-
29
- // Maximum size of a table.
30
- kMaxTableSize = 2 * 1048576
31
-
32
- // Maximum bytes of overlaps in grandparent (i.e., level+2) before we
33
- // stop building a single file in a level->level+1 compaction.
34
- kMaxGrandParentOverlapBytes = 10 * kMaxTableSize
35
-
36
- // Maximum number of bytes in all compacted files. We avoid expanding
37
- // the lower level file set of a compaction if it would make the
38
- // total compaction cover more than this many bytes.
39
- kExpCompactionMaxBytes = 25 * kMaxTableSize
40
-)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go
+45
-15
@@ -14,6 +14,7 @@ import (
14
"testing"
15
16
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
17
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
18
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
20
)
@@ -96,21 +97,22 @@ func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
97
}
98
}
99
99
-func (h *dbCorruptHarness) corrupt(ft storage.FileType, offset, n int) {
100
+func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
101
p := &h.dbHarness
102
t := p.t
103
103
- var file storage.File
104
ff, _ := p.stor.GetFiles(ft)
105
- for _, f := range ff {
106
- if file == nil || f.Num() > file.Num() {
107
- file = f
108
- }
105
+ sff := files(ff)
106
+ sff.sort()
107
+ if fi < 0 {
108
+ fi = len(sff) - 1
109
}
110
- if file == nil {
111
- t.Fatalf("no such file with type %q", ft)
110
+ if fi >= len(sff) {
111
+ t.Fatalf("no such file with type %q with index %d", ft, fi)
112
}
113
114
+ file := sff[fi]
115
+
116
r, err := file.Open()
117
if err != nil {
118
t.Fatal("cannot open file: ", err)
@@ -225,8 +227,8 @@ func TestCorruptDB_Journal(t *testing.T) {
227
h.build(100)
228
h.check(100, 100)
229
h.closeDB()
228
- h.corrupt(storage.TypeJournal, 19, 1)
229
- h.corrupt(storage.TypeJournal, 32*1024+1000, 1)
230
+ h.corrupt(storage.TypeJournal, -1, 19, 1)
231
+ h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
232
233
h.openDB()
234
h.check(36, 36)
@@ -242,7 +244,7 @@ func TestCorruptDB_Table(t *testing.T) {
244
h.compactRangeAt(0, "", "")
245
h.compactRangeAt(1, "", "")
246
h.closeDB()
245
- h.corrupt(storage.TypeTable, 100, 1)
247
+ h.corrupt(storage.TypeTable, -1, 100, 1)
248
249
h.openDB()
250
h.check(99, 99)
@@ -256,7 +258,7 @@ func TestCorruptDB_TableIndex(t *testing.T) {
258
h.build(10000)
259
h.compactMem()
260
h.closeDB()
259
- h.corrupt(storage.TypeTable, -2000, 500)
261
+ h.corrupt(storage.TypeTable, -1, -2000, 500)
262
263
h.openDB()
264
h.check(5000, 9999)
@@ -355,7 +357,7 @@ func TestCorruptDB_CorruptedManifest(t *testing.T) {
357
h.compactMem()
358
h.compactRange("", "")
359
h.closeDB()
358
- h.corrupt(storage.TypeManifest, 0, 1000)
360
+ h.corrupt(storage.TypeManifest, -1, 0, 1000)
361
h.openAssert(false)
362
363
h.recover()
@@ -370,7 +372,7 @@ func TestCorruptDB_CompactionInputError(t *testing.T) {
372
h.build(10)
373
h.compactMem()
374
h.closeDB()
373
- h.corrupt(storage.TypeTable, 100, 1)
375
+ h.corrupt(storage.TypeTable, -1, 100, 1)
376
377
h.openDB()
378
h.check(9, 9)
@@ -387,7 +389,7 @@ func TestCorruptDB_UnrelatedKeys(t *testing.T) {
389
h.build(10)
390
h.compactMem()
391
h.closeDB()
390
- h.corrupt(storage.TypeTable, 100, 1)
392
+ h.corrupt(storage.TypeTable, -1, 100, 1)
393
394
h.openDB()
395
h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
@@ -470,3 +472,31 @@ func TestCorruptDB_MissingTableFiles(t *testing.T) {
472
473
h.close()
474
}
475
+
476
+func TestCorruptDB_RecoverTable(t *testing.T) {
477
+ h := newDbCorruptHarnessWopt(t, &opt.Options{
478
+ WriteBuffer: 112 * opt.KiB,
479
+ CompactionTableSize: 90 * opt.KiB,
480
+ Filter: filter.NewBloomFilter(10),
481
+ })
482
+
483
+ h.build(1000)
484
+ h.compactMem()
485
+ h.compactRangeAt(0, "", "")
486
+ h.compactRangeAt(1, "", "")
487
+ seq := h.db.seq
488
+ h.closeDB()
489
+ h.corrupt(storage.TypeTable, 0, 1000, 1)
490
+ h.corrupt(storage.TypeTable, 3, 10000, 1)
491
+ // Corrupted filter shouldn't affect recovery.
492
+ h.corrupt(storage.TypeTable, 3, 113888, 10)
493
+ h.corrupt(storage.TypeTable, -1, 20000, 1)
494
+
495
+ h.recover()
496
+ if h.db.seq != seq {
497
+ t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
498
+ }
499
+ h.check(985, 985)
500
+
501
+ h.close()
502
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go
+184
-75
@@ -7,7 +7,7 @@
7
package leveldb
8
9
import (
10
- "errors"
10
+ "container/list"
11
"fmt"
12
"io"
13
"os"
@@ -17,6 +17,7 @@ import (
17
"sync/atomic"
18
"time"
19
20
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
21
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
22
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
23
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
@@ -46,7 +47,7 @@ type DB struct {
47
48
// Snapshot.
49
snapsMu sync.Mutex
49
- snapsRoot snapshotElement
50
+ snapsList *list.List
51
52
// Stats.
53
aliveSnaps, aliveIters int32
@@ -56,18 +57,19 @@ type DB struct {
57
writeMergedC chan bool
58
writeLockC chan struct{}
59
writeAckC chan error
60
+ writeDelay time.Duration
61
+ writeDelayN int
62
journalC chan *Batch
63
journalAckC chan error
64
65
// Compaction.
63
- tcompCmdC chan cCmd
64
- tcompPauseC chan chan<- struct{}
65
- tcompTriggerC chan struct{}
66
- mcompCmdC chan cCmd
67
- mcompTriggerC chan struct{}
68
- compErrC chan error
69
- compErrSetC chan error
70
- compStats [kNumLevels]cStats
66
+ tcompCmdC chan cCmd
67
+ tcompPauseC chan chan<- struct{}
68
+ mcompCmdC chan cCmd
69
+ compErrC chan error
70
+ compPerErrC chan error
71
+ compErrSetC chan error
72
+ compStats []cStats
73
74
// Close.
75
closeW sync.WaitGroup
@@ -82,9 +84,11 @@ func openDB(s *session) (*DB, error) {
84
db := &DB{
85
s: s,
86
// Initial sequence
85
- seq: s.stSeq,
87
+ seq: s.stSeqNum,
88
// MemDB
89
memPool: make(chan *memdb.DB, 1),
90
+ // Snapshot
91
+ snapsList: list.New(),
92
// Write
93
writeC: make(chan *Batch),
94
writeMergedC: make(chan bool),
@@ -93,17 +97,16 @@ func openDB(s *session) (*DB, error) {
97
journalC: make(chan *Batch),
98
journalAckC: make(chan error),
99
// Compaction
96
- tcompCmdC: make(chan cCmd),
97
- tcompPauseC: make(chan chan<- struct{}),
98
- tcompTriggerC: make(chan struct{}, 1),
99
- mcompCmdC: make(chan cCmd),
100
- mcompTriggerC: make(chan struct{}, 1),
101
- compErrC: make(chan error),
102
- compErrSetC: make(chan error),
100
+ tcompCmdC: make(chan cCmd),
101
+ tcompPauseC: make(chan chan<- struct{}),
102
+ mcompCmdC: make(chan cCmd),
103
+ compErrC: make(chan error),
104
+ compPerErrC: make(chan error),
105
+ compErrSetC: make(chan error),
106
+ compStats: make([]cStats, s.o.GetNumLevel()),
107
// Close
108
closeC: make(chan struct{}),
109
}
106
- db.initSnapshot()
110
111
if err := db.recoverJournal(); err != nil {
112
return nil, err
@@ -119,14 +122,14 @@ func openDB(s *session) (*DB, error) {
122
return nil, err
123
}
124
122
- // Don't include compaction error goroutine into wait group.
125
+ // Doesn't need to be included in the wait group.
126
go db.compactionError()
127
+ go db.mpoolDrain()
128
129
db.closeW.Add(3)
130
go db.tCompaction()
131
go db.mCompaction()
132
go db.jWriter()
129
- go db.mpoolDrain()
133
134
s.logf("db@open done T·%v", time.Since(start))
135
@@ -253,6 +256,10 @@ func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
256
}
257
258
func recoverTable(s *session, o *opt.Options) error {
259
+ o = dupOptions(o)
260
+ // Mask StrictReader, lets StrictRecovery doing its job.
261
+ o.Strict &= ^opt.StrictReader
262
+
263
// Get all tables and sort it by file number.
264
tableFiles_, err := s.getFiles(storage.TypeTable)
265
if err != nil {
@@ -261,10 +268,16 @@ func recoverTable(s *session, o *opt.Options) error {
268
tableFiles := files(tableFiles_)
269
tableFiles.sort()
270
264
- var mSeq uint64
265
- var good, corrupted int
266
- rec := new(sessionRecord)
267
- bpool := util.NewBufferPool(o.GetBlockSize() + 5)
271
+ var (
272
+ maxSeq uint64
273
+ recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
274
+
275
+ // We will drop corrupted table.
276
+ strict = o.GetStrict(opt.StrictRecovery)
277
+
278
+ rec = &sessionRecord{numLevel: o.GetNumLevel()}
279
+ bpool = util.NewBufferPool(o.GetBlockSize() + 5)
280
+ )
281
buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
282
tmp = s.newTemp()
283
writer, err := tmp.Create()
@@ -311,7 +324,12 @@ func recoverTable(s *session, o *opt.Options) error {
324
if err != nil {
325
return err
326
}
314
- defer reader.Close()
327
+ var closed bool
328
+ defer func() {
329
+ if !closed {
330
+ reader.Close()
331
+ }
332
+ }()
333
334
// Get file size.
335
size, err := reader.Seek(0, 2)
@@ -319,25 +337,32 @@ func recoverTable(s *session, o *opt.Options) error {
337
return err
338
}
339
322
- var tSeq uint64
323
- var tgood, tcorrupted, blockerr int
324
- var imin, imax []byte
325
- tr := table.NewReader(reader, size, nil, bpool, o)
340
+ var (
341
+ tSeq uint64
342
+ tgoodKey, tcorruptedKey, tcorruptedBlock int
343
+ imin, imax []byte
344
+ )
345
+ tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
346
+ if err != nil {
347
+ return err
348
+ }
349
iter := tr.NewIterator(nil, nil)
350
iter.(iterator.ErrorCallbackSetter).SetErrorCallback(func(err error) {
328
- s.logf("table@recovery found error @%d %q", file.Num(), err)
329
- blockerr++
351
+ if errors.IsCorrupted(err) {
352
+ s.logf("table@recovery block corruption @%d %q", file.Num(), err)
353
+ tcorruptedBlock++
354
+ }
355
})
356
357
// Scan the table.
358
for iter.Next() {
359
key := iter.Key()
335
- _, seq, _, ok := parseIkey(key)
336
- if !ok {
337
- tcorrupted++
360
+ _, seq, _, kerr := parseIkey(key)
361
+ if kerr != nil {
362
+ tcorruptedKey++
363
continue
364
}
340
- tgood++
365
+ tgoodKey++
366
if seq > tSeq {
367
tSeq = seq
368
}
@@ -352,8 +377,18 @@ func recoverTable(s *session, o *opt.Options) error {
377
}
378
iter.Release()
379
355
- if tgood > 0 {
356
- if tcorrupted > 0 || blockerr > 0 {
380
+ goodKey += tgoodKey
381
+ corruptedKey += tcorruptedKey
382
+ corruptedBlock += tcorruptedBlock
383
+
384
+ if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
385
+ droppedTable++
386
+ s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
387
+ return nil
388
+ }
389
+
390
+ if tgoodKey > 0 {
391
+ if tcorruptedKey > 0 || tcorruptedBlock > 0 {
392
// Rebuild the table.
393
s.logf("table@recovery rebuilding @%d", file.Num())
394
iter := tr.NewIterator(nil, nil)
@@ -362,25 +397,25 @@ func recoverTable(s *session, o *opt.Options) error {
397
if err != nil {
398
return err
399
}
400
+ closed = true
401
reader.Close()
402
if err := file.Replace(tmp); err != nil {
403
return err
404
}
405
size = newSize
406
}
371
- if tSeq > mSeq {
372
- mSeq = tSeq
407
+ if tSeq > maxSeq {
408
+ maxSeq = tSeq
409
}
410
+ recoveredKey += tgoodKey
411
// Add table to level 0.
412
rec.addTable(0, file.Num(), uint64(size), imin, imax)
376
- s.logf("table@recovery recovered @%d N·%d C·%d B·%d S·%d Q·%d", file.Num(), tgood, tcorrupted, blockerr, size, tSeq)
413
+ s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
414
} else {
378
- s.logf("table@recovery unrecoverable @%d C·%d B·%d S·%d", file.Num(), tcorrupted, blockerr, size)
415
+ droppedTable++
416
+ s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
417
}
418
381
- good += tgood
382
- corrupted += tcorrupted
383
-
419
return nil
420
}
421
@@ -397,11 +432,11 @@ func recoverTable(s *session, o *opt.Options) error {
432
}
433
}
434
400
- s.logf("table@recovery recovered F·%d N·%d C·%d Q·%d", len(tableFiles), good, corrupted, mSeq)
435
+ s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
436
}
437
438
// Set sequence number.
404
- rec.setSeq(mSeq + 1)
439
+ rec.setSeqNum(maxSeq)
440
441
// Create new manifest.
442
if err := s.create(); err != nil {
@@ -484,26 +519,30 @@ func (db *DB) recoverJournal() error {
519
if err == io.EOF {
520
break
521
}
487
- return err
522
+ return errors.SetFile(err, file)
523
}
524
525
buf.Reset()
526
if _, err := buf.ReadFrom(r); err != nil {
527
if err == io.ErrUnexpectedEOF {
528
+ // This is error returned due to corruption, with strict == false.
529
continue
530
} else {
495
- return err
531
+ return errors.SetFile(err, file)
532
}
533
}
498
- if err := batch.decode(buf.Bytes()); err != nil {
499
- return err
500
- }
501
- if err := batch.memReplay(mem); err != nil {
502
- return err
534
+ if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
535
+ if strict || !errors.IsCorrupted(err) {
536
+ return errors.SetFile(err, file)
537
+ } else {
538
+ db.s.logf("journal error: %v (skipped)", err)
539
+ // We won't apply sequence number as it might be corrupted.
540
+ continue
541
+ }
542
}
543
544
// Save sequence number.
506
- db.seq = batch.seq + uint64(batch.len())
545
+ db.seq = batch.seq + uint64(batch.Len())
546
547
// Flush it if large enough.
548
if mem.Size() >= writeBuffer {
@@ -564,7 +603,7 @@ func (db *DB) recoverJournal() error {
603
}
604
605
func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
567
- ikey := newIKey(key, seq, tSeek)
606
+ ikey := newIkey(key, seq, ktSeek)
607
608
em, fm := db.getMems()
609
for _, m := range [...]*memDB{em, fm} {
@@ -575,9 +614,13 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
614
615
mk, mv, me := m.mdb.Find(ikey)
616
if me == nil {
578
- ukey, _, t, ok := parseIkey(mk)
579
- if ok && db.s.icmp.uCompare(ukey, key) == 0 {
580
- if t == tDel {
617
+ ukey, _, kt, kerr := parseIkey(mk)
618
+ if kerr != nil {
619
+ // Shouldn't have had happen.
620
+ panic(kerr)
621
+ }
622
+ if db.s.icmp.uCompare(ukey, key) == 0 {
623
+ if kt == ktDel {
624
return nil, ErrNotFound
625
}
626
return append([]byte{}, mv...), nil
@@ -588,17 +631,60 @@ func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, er
631
}
632
633
v := db.s.version()
591
- value, cSched, err := v.get(ikey, ro)
634
+ value, cSched, err := v.get(ikey, ro, false)
635
v.release()
636
if cSched {
637
// Trigger table compaction.
595
- db.compTrigger(db.tcompTriggerC)
638
+ db.compSendTrigger(db.tcompCmdC)
639
+ }
640
+ return
641
+}
642
+
643
+func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
644
+ ikey := newIkey(key, seq, ktSeek)
645
+
646
+ em, fm := db.getMems()
647
+ for _, m := range [...]*memDB{em, fm} {
648
+ if m == nil {
649
+ continue
650
+ }
651
+ defer m.decref()
652
+
653
+ mk, _, me := m.mdb.Find(ikey)
654
+ if me == nil {
655
+ ukey, _, kt, kerr := parseIkey(mk)
656
+ if kerr != nil {
657
+ // Shouldn't have had happen.
658
+ panic(kerr)
659
+ }
660
+ if db.s.icmp.uCompare(ukey, key) == 0 {
661
+ if kt == ktDel {
662
+ return false, nil
663
+ }
664
+ return true, nil
665
+ }
666
+ } else if me != ErrNotFound {
667
+ return false, me
668
+ }
669
+ }
670
+
671
+ v := db.s.version()
672
+ _, cSched, err := v.get(ikey, ro, true)
673
+ v.release()
674
+ if cSched {
675
+ // Trigger table compaction.
676
+ db.compSendTrigger(db.tcompCmdC)
677
+ }
678
+ if err == nil {
679
+ ret = true
680
+ } else if err == ErrNotFound {
681
+ err = nil
682
}
683
return
684
}
685
686
// Get gets the value for the given key. It returns ErrNotFound if the
601
-// DB does not contain the key.
687
+// DB does not contains the key.
688
//
689
// The returned slice is its own copy, it is safe to modify the contents
690
// of the returned slice.
@@ -609,7 +695,23 @@ func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
695
return
696
}
697
612
- return db.get(key, db.getSeq(), ro)
698
+ se := db.acquireSnapshot()
699
+ defer db.releaseSnapshot(se)
700
+ return db.get(key, se.seq, ro)
701
+}
702
+
703
+// Has returns true if the DB does contains the given key.
704
+//
705
+// It is safe to modify the contents of the argument after Get returns.
706
+func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
707
+ err = db.ok()
708
+ if err != nil {
709
+ return
710
+ }
711
+
712
+ se := db.acquireSnapshot()
713
+ defer db.releaseSnapshot(se)
714
+ return db.has(key, se.seq, ro)
715
}
716
717
// NewIterator returns an iterator for the latest snapshot of the
@@ -633,9 +735,11 @@ func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Itera
735
return iterator.NewEmptyIterator(err)
736
}
737
636
- snap := db.newSnapshot()
637
- defer snap.Release()
638
- return snap.NewIterator(slice, ro)
738
+ se := db.acquireSnapshot()
739
+ defer db.releaseSnapshot(se)
740
+ // Iterator holds 'version' lock, 'version' is immutable so snapshot
741
+ // can be released after iterator created.
742
+ return db.newIterator(se.seq, slice, ro)
743
}
744
745
// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
@@ -655,7 +759,7 @@ func (db *DB) GetSnapshot() (*Snapshot, error) {
759
//
760
// Property names:
761
// leveldb.num-files-at-level{n}
658
-// Returns the number of filer at level 'n'.
762
+// Returns the number of files at level 'n'.
763
// leveldb.stats
764
// Returns statistics of the underlying DB.
765
// leveldb.sstables
@@ -685,12 +789,13 @@ func (db *DB) GetProperty(name string) (value string, err error) {
789
v := db.s.version()
790
defer v.release()
791
792
+ numFilesPrefix := "num-files-at-level"
793
switch {
689
- case strings.HasPrefix(p, "num-files-at-level"):
794
+ case strings.HasPrefix(p, numFilesPrefix):
795
var level uint
796
var rest string
692
- n, _ := fmt.Scanf("%d%s", &level, &rest)
693
- if n != 1 || level >= kNumLevels {
797
+ n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
798
+ if n != 1 || int(level) >= db.s.o.GetNumLevel() {
799
err = errors.New("leveldb: GetProperty: invalid property: " + name)
800
} else {
801
value = fmt.Sprint(v.tLen(int(level)))
@@ -752,8 +857,8 @@ func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
857
858
sizes := make(Sizes, 0, len(ranges))
859
for _, r := range ranges {
755
- imin := newIKey(r.Start, kMaxSeq, tSeek)
756
- imax := newIKey(r.Limit, kMaxSeq, tSeek)
860
+ imin := newIkey(r.Start, kMaxSeq, ktSeek)
861
+ imax := newIkey(r.Limit, kMaxSeq, ktSeek)
862
start, err := v.offsetOf(imin)
863
if err != nil {
864
return nil, err
@@ -796,18 +901,23 @@ func (db *DB) Close() error {
901
default:
902
}
903
904
+ // Signal all goroutines.
905
close(db.closeC)
906
801
- // Wait for the close WaitGroup.
907
+ // Wait for all gorotines to exit.
908
db.closeW.Wait()
909
804
- // Close journal.
910
+ // Lock writer and closes journal.
911
db.writeLockC <- struct{}{}
912
if db.journal != nil {
913
db.journal.Close()
914
db.journalWriter.Close()
915
}
916
917
+ if db.writeDelayN > 0 {
918
+ db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
919
+ }
920
+
921
// Close session.
922
db.s.close()
923
db.logf("db@close done T·%v", time.Since(start))
@@ -827,7 +937,6 @@ func (db *DB) Close() error {
937
db.journalWriter = nil
938
db.journalFile = nil
939
db.frozenJournalFile = nil
830
- db.snapsRoot = snapshotElement{}
940
db.closer = nil
941
942
return err
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go
+380
-234
@@ -7,11 +7,12 @@
7
package leveldb
8
9
import (
10
- "errors"
10
"sync"
11
"time"
12
13
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
16
)
17
18
var (
@@ -68,7 +69,7 @@ type cMem struct {
69
}
70
71
func newCMem(s *session) *cMem {
71
- return &cMem{s: s, rec: new(sessionRecord)}
72
+ return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}}
73
}
74
75
func (c *cMem) flush(mem *memdb.DB, level int) error {
@@ -84,7 +85,9 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
85
86
// Pick level.
87
if level < 0 {
87
- level = s.version_NB().pickLevel(t.imin.ukey(), t.imax.ukey())
88
+ v := s.version()
89
+ level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
90
+ v.release()
91
}
92
c.rec.addTableFile(level, t)
93
@@ -95,24 +98,32 @@ func (c *cMem) flush(mem *memdb.DB, level int) error {
98
}
99
100
func (c *cMem) reset() {
98
- c.rec = new(sessionRecord)
101
+ c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()}
102
}
103
104
func (c *cMem) commit(journal, seq uint64) error {
105
c.rec.setJournalNum(journal)
103
- c.rec.setSeq(seq)
106
+ c.rec.setSeqNum(seq)
107
108
// Commit changes.
109
return c.s.commit(c.rec)
110
}
111
112
func (db *DB) compactionError() {
110
- var err error
113
+ var (
114
+ err error
115
+ wlocked bool
116
+ )
117
noerr:
118
+ // No error.
119
for {
120
select {
121
case err = <-db.compErrSetC:
115
- if err != nil {
122
+ switch {
123
+ case err == nil:
124
+ case errors.IsCorrupted(err):
125
+ goto hasperr
126
+ default:
127
goto haserr
128
}
129
case _, _ = <-db.closeC:
@@ -120,17 +131,39 @@ noerr:
131
}
132
}
133
haserr:
134
+ // Transient error.
135
for {
136
select {
137
case db.compErrC <- err:
138
case err = <-db.compErrSetC:
127
- if err == nil {
139
+ switch {
140
+ case err == nil:
141
goto noerr
142
+ case errors.IsCorrupted(err):
143
+ goto hasperr
144
+ default:
145
}
146
case _, _ = <-db.closeC:
147
return
148
}
149
}
150
+hasperr:
151
+ // Persistent error.
152
+ for {
153
+ select {
154
+ case db.compErrC <- err:
155
+ case db.compPerErrC <- err:
156
+ case db.writeLockC <- struct{}{}:
157
+ // Hold write lock, so that write won't pass-through.
158
+ wlocked = true
159
+ case _, _ = <-db.closeC:
160
+ if wlocked {
161
+ // We should release the lock or Close will hang.
162
+ <-db.writeLockC
163
+ }
164
+ return
165
+ }
166
+ }
167
}
168
169
type compactionTransactCounter int
@@ -139,12 +172,17 @@ func (cnt *compactionTransactCounter) incr() {
172
*cnt++
173
}
174
142
-func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactCounter) error, rollback func() error) {
175
+type compactionTransactInterface interface {
176
+ run(cnt *compactionTransactCounter) error
177
+ revert() error
178
+}
179
+
180
+func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
181
defer func() {
182
if x := recover(); x != nil {
145
- if x == errCompactionTransactExiting && rollback != nil {
146
- if err := rollback(); err != nil {
147
- db.logf("%s rollback error %q", name, err)
183
+ if x == errCompactionTransactExiting {
184
+ if err := t.revert(); err != nil {
185
+ db.logf("%s revert error %q", name, err)
186
}
187
}
188
panic(x)
@@ -156,9 +194,13 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
194
backoffMax = 8 * time.Second
195
backoffMul = 2 * time.Second
196
)
159
- backoff := backoffMin
160
- backoffT := time.NewTimer(backoff)
161
- lastCnt := compactionTransactCounter(0)
197
+ var (
198
+ backoff = backoffMin
199
+ backoffT = time.NewTimer(backoff)
200
+ lastCnt = compactionTransactCounter(0)
201
+
202
+ disableBackoff = db.s.o.GetDisableCompactionBackoff()
203
+ )
204
for n := 0; ; n++ {
205
// Check wether the DB is closed.
206
if db.isClosed() {
@@ -170,11 +212,19 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
212
213
// Execute.
214
cnt := compactionTransactCounter(0)
173
- err := exec(&cnt)
215
+ err := t.run(&cnt)
216
+ if err != nil {
217
+ db.logf("%s error I·%d %q", name, cnt, err)
218
+ }
219
220
// Set compaction error status.
221
select {
222
case db.compErrSetC <- err:
223
+ case perr := <-db.compPerErrC:
224
+ if err != nil {
225
+ db.logf("%s exiting (persistent error %q)", name, perr)
226
+ db.compactionExitTransact()
227
+ }
228
case _, _ = <-db.closeC:
229
db.logf("%s exiting", name)
230
db.compactionExitTransact()
@@ -182,31 +232,56 @@ func (db *DB) compactionTransact(name string, exec func(cnt *compactionTransactC
232
if err == nil {
233
return
234
}
185
- db.logf("%s error I·%d %q", name, cnt, err)
186
-
187
- // Reset backoff duration if counter is advancing.
188
- if cnt > lastCnt {
189
- backoff = backoffMin
190
- lastCnt = cnt
235
+ if errors.IsCorrupted(err) {
236
+ db.logf("%s exiting (corruption detected)", name)
237
+ db.compactionExitTransact()
238
}
239
193
- // Backoff.
194
- backoffT.Reset(backoff)
195
- if backoff < backoffMax {
196
- backoff *= backoffMul
197
- if backoff > backoffMax {
198
- backoff = backoffMax
240
+ if !disableBackoff {
241
+ // Reset backoff duration if counter is advancing.
242
+ if cnt > lastCnt {
243
+ backoff = backoffMin
244
+ lastCnt = cnt
245
+ }
246
+
247
+ // Backoff.
248
+ backoffT.Reset(backoff)
249
+ if backoff < backoffMax {
250
+ backoff *= backoffMul
251
+ if backoff > backoffMax {
252
+ backoff = backoffMax
253
+ }
254
+ }
255
+ select {
256
+ case <-backoffT.C:
257
+ case _, _ = <-db.closeC:
258
+ db.logf("%s exiting", name)
259
+ db.compactionExitTransact()
260
}
200
- }
201
- select {
202
- case <-backoffT.C:
203
- case _, _ = <-db.closeC:
204
- db.logf("%s exiting", name)
205
- db.compactionExitTransact()
261
}
262
}
263
}
264
265
+type compactionTransactFunc struct {
266
+ runFunc func(cnt *compactionTransactCounter) error
267
+ revertFunc func() error
268
+}
269
+
270
+func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
271
+ return t.runFunc(cnt)
272
+}
273
+
274
+func (t *compactionTransactFunc) revert() error {
275
+ if t.revertFunc != nil {
276
+ return t.revertFunc()
277
+ }
278
+ return nil
279
+}
280
+
281
+func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
282
+ db.compactionTransact(name, &compactionTransactFunc{run, revert})
283
+}
284
+
285
func (db *DB) compactionExitTransact() {
286
panic(errCompactionTransactExiting)
287
}
@@ -232,20 +307,23 @@ func (db *DB) memCompaction() {
307
}
308
309
// Pause table compaction.
235
- ch := make(chan struct{})
310
+ resumeC := make(chan struct{})
311
select {
237
- case db.tcompPauseC <- (chan<- struct{})(ch):
312
+ case db.tcompPauseC <- (chan<- struct{})(resumeC):
313
+ case <-db.compPerErrC:
314
+ close(resumeC)
315
+ resumeC = nil
316
case _, _ = <-db.closeC:
317
return
318
}
319
242
- db.compactionTransact("mem@flush", func(cnt *compactionTransactCounter) (err error) {
320
+ db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) {
321
stats.startTimer()
322
defer stats.stopTimer()
323
return c.flush(mem.mdb, -1)
324
}, func() error {
325
for _, r := range c.rec.addedTables {
248
- db.logf("mem@flush rollback @%d", r.num)
326
+ db.logf("mem@flush revert @%d", r.num)
327
f := db.s.getTableFile(r.num)
328
if err := f.Remove(); err != nil {
329
return err
@@ -254,13 +332,13 @@ func (db *DB) memCompaction() {
332
return nil
333
})
334
257
- db.compactionTransact("mem@commit", func(cnt *compactionTransactCounter) (err error) {
335
+ db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) {
336
stats.startTimer()
337
defer stats.stopTimer()
338
return c.commit(db.journalFile.Num(), db.frozenSeq)
339
}, nil)
340
263
- db.logf("mem@flush commited F·%d T·%v", len(c.rec.addedTables), stats.duration)
341
+ db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
342
343
for _, r := range c.rec.addedTables {
344
stats.write += r.size
@@ -271,214 +349,261 @@ func (db *DB) memCompaction() {
349
db.dropFrozenMem()
350
351
// Resume table compaction.
274
- select {
275
- case <-ch:
276
- case _, _ = <-db.closeC:
277
- return
352
+ if resumeC != nil {
353
+ select {
354
+ case <-resumeC:
355
+ close(resumeC)
356
+ case _, _ = <-db.closeC:
357
+ return
358
+ }
359
}
360
361
// Trigger table compaction.
281
- db.compTrigger(db.mcompTriggerC)
362
+ db.compSendTrigger(db.tcompCmdC)
363
}
364
284
-func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
285
- rec := new(sessionRecord)
286
- rec.addCompactionPointer(c.level, c.imax)
365
+type tableCompactionBuilder struct {
366
+ db *DB
367
+ s *session
368
+ c *compaction
369
+ rec *sessionRecord
370
+ stat0, stat1 *cStatsStaging
371
288
- if !noTrivial && c.trivial() {
289
- t := c.tables[0][0]
290
- db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
291
- rec.deleteTable(c.level, t.file.Num())
292
- rec.addTableFile(c.level+1, t)
293
- db.compactionTransact("table@move", func(cnt *compactionTransactCounter) (err error) {
294
- return db.s.commit(rec)
295
- }, nil)
296
- return
297
- }
372
+ snapHasLastUkey bool
373
+ snapLastUkey []byte
374
+ snapLastSeq uint64
375
+ snapIter int
376
+ snapKerrCnt int
377
+ snapDropCnt int
378
299
- var stats [2]cStatsStaging
300
- for i, tables := range c.tables {
301
- for _, t := range tables {
302
- stats[i].read += t.size
303
- // Insert deleted tables into record
304
- rec.deleteTable(c.level+i, t.file.Num())
305
- }
306
- }
307
- sourceSize := int(stats[0].read + stats[1].read)
308
- minSeq := db.minSeq()
309
- db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
379
+ kerrCnt int
380
+ dropCnt int
381
311
- var snapUkey []byte
312
- var snapHasUkey bool
313
- var snapSeq uint64
314
- var snapIter int
315
- var snapDropCnt int
316
- var dropCnt int
317
- db.compactionTransact("table@build", func(cnt *compactionTransactCounter) (err error) {
318
- ukey := append([]byte{}, snapUkey...)
319
- hasUkey := snapHasUkey
320
- lseq := snapSeq
321
- dropCnt = snapDropCnt
322
- snapSched := snapIter == 0
323
-
324
- var tw *tWriter
325
- finish := func() error {
326
- t, err := tw.finish()
327
- if err != nil {
328
- return err
382
+ minSeq uint64
383
+ strict bool
384
+ tableSize int
385
+
386
+ tw *tWriter
387
+}
388
+
389
+func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
390
+ // Create new table if not already.
391
+ if b.tw == nil {
392
+ // Check for pause event.
393
+ if b.db != nil {
394
+ select {
395
+ case ch := <-b.db.tcompPauseC:
396
+ b.db.pauseCompaction(ch)
397
+ case _, _ = <-b.db.closeC:
398
+ b.db.compactionExitTransact()
399
+ default:
400
}
330
- rec.addTableFile(c.level+1, t)
331
- stats[1].write += t.size
332
- db.logf("table@build created L%d@%d N·%d S·%s %q:%q", c.level+1, t.file.Num(), tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
333
- return nil
401
}
402
336
- defer func() {
337
- stats[1].stopTimer()
338
- if tw != nil {
339
- tw.drop()
340
- tw = nil
341
- }
342
- }()
403
+ // Create new table.
404
+ var err error
405
+ b.tw, err = b.s.tops.create()
406
+ if err != nil {
407
+ return err
408
+ }
409
+ }
410
344
- stats[1].startTimer()
345
- iter := c.newIterator()
346
- defer iter.Release()
347
- for i := 0; iter.Next(); i++ {
348
- // Incr transact counter.
349
- cnt.incr()
350
-
351
- // Skip until last state.
352
- if i < snapIter {
353
- continue
354
- }
411
+ // Write key/value into table.
412
+ return b.tw.append(key, value)
413
+}
414
356
- ikey := iKey(iter.Key())
415
+func (b *tableCompactionBuilder) needFlush() bool {
416
+ return b.tw.tw.BytesLen() >= b.tableSize
417
+}
418
358
- if c.shouldStopBefore(ikey) && tw != nil {
359
- err = finish()
360
- if err != nil {
361
- return
362
- }
363
- snapSched = true
364
- tw = nil
365
- }
419
+func (b *tableCompactionBuilder) flush() error {
420
+ t, err := b.tw.finish()
421
+ if err != nil {
422
+ return err
423
+ }
424
+ b.rec.addTableFile(b.c.level+1, t)
425
+ b.stat1.write += t.size
426
+ b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
427
+ b.tw = nil
428
+ return nil
429
+}
430
367
- // Scheduled for snapshot, snapshot will used to retry compaction
368
- // if error occured.
369
- if snapSched {
370
- snapUkey = append(snapUkey[:0], ukey...)
371
- snapHasUkey = hasUkey
372
- snapSeq = lseq
373
- snapIter = i
374
- snapDropCnt = dropCnt
375
- snapSched = false
376
- }
431
+func (b *tableCompactionBuilder) cleanup() {
432
+ if b.tw != nil {
433
+ b.tw.drop()
434
+ b.tw = nil
435
+ }
436
+}
437
378
- if seq, vt, ok := ikey.parseNum(); !ok {
379
- // Don't drop error keys
380
- ukey = ukey[:0]
381
- hasUkey = false
382
- lseq = kMaxSeq
383
- } else {
384
- if !hasUkey || db.s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
385
- // First occurrence of this user key
386
- ukey = append(ukey[:0], ikey.ukey()...)
387
- hasUkey = true
388
- lseq = kMaxSeq
389
- }
438
+func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
439
+ snapResumed := b.snapIter > 0
440
+ hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
441
+ lastUkey := append([]byte{}, b.snapLastUkey...)
442
+ lastSeq := b.snapLastSeq
443
+ b.kerrCnt = b.snapKerrCnt
444
+ b.dropCnt = b.snapDropCnt
445
+ // Restore compaction state.
446
+ b.c.restore()
447
391
- drop := false
392
- if lseq <= minSeq {
393
- // Dropped because newer entry for same user key exist
394
- drop = true // (A)
395
- } else if vt == tDel && seq <= minSeq && c.baseLevelForKey(ukey) {
396
- // For this user key:
397
- // (1) there is no data in higher levels
398
- // (2) data in lower levels will have larger seq numbers
399
- // (3) data in layers that are being compacted here and have
400
- // smaller seq numbers will be dropped in the next
401
- // few iterations of this loop (by rule (A) above).
402
- // Therefore this deletion marker is obsolete and can be dropped.
403
- drop = true
404
- }
448
+ defer b.cleanup()
449
406
- lseq = seq
407
- if drop {
408
- dropCnt++
409
- continue
410
- }
411
- }
450
+ b.stat1.startTimer()
451
+ defer b.stat1.stopTimer()
452
413
- // Create new table if not already
414
- if tw == nil {
415
- // Check for pause event.
416
- select {
417
- case ch := <-db.tcompPauseC:
418
- db.pauseCompaction(ch)
419
- case _, _ = <-db.closeC:
420
- db.compactionExitTransact()
421
- default:
422
- }
453
+ iter := b.c.newIterator()
454
+ defer iter.Release()
455
+ for i := 0; iter.Next(); i++ {
456
+ // Incr transact counter.
457
+ cnt.incr()
458
+
459
+ // Skip until last state.
460
+ if i < b.snapIter {
461
+ continue
462
+ }
463
424
- // Create new table.
425
- tw, err = db.s.tops.create()
426
- if err != nil {
427
- return
464
+ resumed := false
465
+ if snapResumed {
466
+ resumed = true
467
+ snapResumed = false
468
+ }
469
+
470
+ ikey := iter.Key()
471
+ ukey, seq, kt, kerr := parseIkey(ikey)
472
+
473
+ if kerr == nil {
474
+ shouldStop := !resumed && b.c.shouldStopBefore(ikey)
475
+
476
+ if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
477
+ // First occurrence of this user key.
478
+
479
+ // Only rotate tables if ukey doesn't hop across.
480
+ if b.tw != nil && (shouldStop || b.needFlush()) {
481
+ if err := b.flush(); err != nil {
482
+ return err
483
+ }
484
+
485
+ // Creates snapshot of the state.
486
+ b.c.save()
487
+ b.snapHasLastUkey = hasLastUkey
488
+ b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
489
+ b.snapLastSeq = lastSeq
490
+ b.snapIter = i
491
+ b.snapKerrCnt = b.kerrCnt
492
+ b.snapDropCnt = b.dropCnt
493
}
429
- }
494
431
- // Write key/value into table
432
- err = tw.append(ikey, iter.Value())
433
- if err != nil {
434
- return
495
+ hasLastUkey = true
496
+ lastUkey = append(lastUkey[:0], ukey...)
497
+ lastSeq = kMaxSeq
498
}
499
437
- // Finish table if it is big enough
438
- if tw.tw.BytesLen() >= kMaxTableSize {
439
- err = finish()
440
- if err != nil {
441
- return
442
- }
443
- snapSched = true
444
- tw = nil
500
+ switch {
501
+ case lastSeq <= b.minSeq:
502
+ // Dropped because newer entry for same user key exist
503
+ fallthrough // (A)
504
+ case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
505
+ // For this user key:
506
+ // (1) there is no data in higher levels
507
+ // (2) data in lower levels will have larger seq numbers
508
+ // (3) data in layers that are being compacted here and have
509
+ // smaller seq numbers will be dropped in the next
510
+ // few iterations of this loop (by rule (A) above).
511
+ // Therefore this deletion marker is obsolete and can be dropped.
512
+ lastSeq = seq
513
+ b.dropCnt++
514
+ continue
515
+ default:
516
+ lastSeq = seq
517
+ }
518
+ } else {
519
+ if b.strict {
520
+ return kerr
521
}
522
+
523
+ // Don't drop corrupted keys.
524
+ hasLastUkey = false
525
+ lastUkey = lastUkey[:0]
526
+ lastSeq = kMaxSeq
527
+ b.kerrCnt++
528
}
529
448
- err = iter.Error()
449
- if err != nil {
450
- return
530
+ if err := b.appendKV(ikey, iter.Value()); err != nil {
531
+ return err
532
}
533
+ }
534
453
- // Finish last table
454
- if tw != nil && !tw.empty() {
455
- err = finish()
456
- if err != nil {
457
- return
458
- }
459
- tw = nil
535
+ if err := iter.Error(); err != nil {
536
+ return err
537
+ }
538
+
539
+ // Finish last table.
540
+ if b.tw != nil && !b.tw.empty() {
541
+ return b.flush()
542
+ }
543
+ return nil
544
+}
545
+
546
+func (b *tableCompactionBuilder) revert() error {
547
+ for _, at := range b.rec.addedTables {
548
+ b.s.logf("table@build revert @%d", at.num)
549
+ f := b.s.getTableFile(at.num)
550
+ if err := f.Remove(); err != nil {
551
+ return err
552
}
553
+ }
554
+ return nil
555
+}
556
+
557
+func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
558
+ defer c.release()
559
+
560
+ rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()}
561
+ rec.addCompPtr(c.level, c.imax)
562
+
563
+ if !noTrivial && c.trivial() {
564
+ t := c.tables[0][0]
565
+ db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
566
+ rec.delTable(c.level, t.file.Num())
567
+ rec.addTableFile(c.level+1, t)
568
+ db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
569
+ return db.s.commit(rec)
570
+ }, nil)
571
return
462
- }, func() error {
463
- for _, r := range rec.addedTables {
464
- db.logf("table@build rollback @%d", r.num)
465
- f := db.s.getTableFile(r.num)
466
- if err := f.Remove(); err != nil {
467
- return err
468
- }
572
+ }
573
+
574
+ var stats [2]cStatsStaging
575
+ for i, tables := range c.tables {
576
+ for _, t := range tables {
577
+ stats[i].read += t.size
578
+ // Insert deleted tables into record
579
+ rec.delTable(c.level+i, t.file.Num())
580
}
470
- return nil
471
- })
581
+ }
582
+ sourceSize := int(stats[0].read + stats[1].read)
583
+ minSeq := db.minSeq()
584
+ db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
585
+
586
+ b := &tableCompactionBuilder{
587
+ db: db,
588
+ s: db.s,
589
+ c: c,
590
+ rec: rec,
591
+ stat1: &stats[1],
592
+ minSeq: minSeq,
593
+ strict: db.s.o.GetStrict(opt.StrictCompaction),
594
+ tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
595
+ }
596
+ db.compactionTransact("table@build", b)
597
598
// Commit changes
474
- db.compactionTransact("table@commit", func(cnt *compactionTransactCounter) (err error) {
599
+ db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
600
stats[1].startTimer()
601
defer stats[1].stopTimer()
602
return db.s.commit(rec)
603
}, nil)
604
605
resultSize := int(stats[1].write)
481
- db.logf("table@compaction commited F%s S%s D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), dropCnt, stats[1].duration)
606
+ db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
607
608
// Save compaction stats
609
for i := range stats {
@@ -494,14 +619,14 @@ func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
619
db.tableCompaction(c, true)
620
}
621
} else {
497
- v := db.s.version_NB()
498
-
622
+ v := db.s.version()
623
m := 1
624
for i, t := range v.tables[1:] {
625
if t.overlaps(db.s.icmp, umin, umax, false) {
626
m = i + 1
627
}
628
}
629
+ v.release()
630
631
for level := 0; level < m; level++ {
632
if c := db.s.getCompactionRange(level, umin, umax); c != nil {
@@ -518,7 +643,9 @@ func (db *DB) tableAutoCompaction() {
643
}
644
645
func (db *DB) tableNeedCompaction() bool {
521
- return db.s.version_NB().needCompaction()
646
+ v := db.s.version()
647
+ defer v.release()
648
+ return v.needCompaction()
649
}
650
651
func (db *DB) pauseCompaction(ch chan<- struct{}) {
@@ -538,7 +665,12 @@ type cIdle struct {
665
}
666
667
func (r cIdle) ack(err error) {
541
- r.ackC <- err
668
+ if r.ackC != nil {
669
+ defer func() {
670
+ recover()
671
+ }()
672
+ r.ackC <- err
673
+ }
674
}
675
676
type cRange struct {
@@ -548,29 +680,45 @@ type cRange struct {
680
}
681
682
func (r cRange) ack(err error) {
551
- defer func() {
552
- recover()
553
- }()
683
if r.ackC != nil {
684
+ defer func() {
685
+ recover()
686
+ }()
687
r.ackC <- err
688
}
689
}
690
559
-func (db *DB) compSendIdle(compC chan<- cCmd) error {
691
+// This will trigger auto compation and/or wait for all compaction to be done.
692
+func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
693
ch := make(chan error)
694
defer close(ch)
695
// Send cmd.
696
select {
697
case compC <- cIdle{ch}:
565
- case err := <-db.compErrC:
566
- return err
698
+ case err = <-db.compErrC:
699
+ return
700
case _, _ = <-db.closeC:
701
return ErrClosed
702
}
703
// Wait cmd.
571
- return <-ch
704
+ select {
705
+ case err = <-ch:
706
+ case err = <-db.compErrC:
707
+ case _, _ = <-db.closeC:
708
+ return ErrClosed
709
+ }
710
+ return err
711
}
712
713
+// This will trigger auto compaction but will not wait for it.
714
+func (db *DB) compSendTrigger(compC chan<- cCmd) {
715
+ select {
716
+ case compC <- cIdle{}:
717
+ default:
718
+ }
719
+}
720
+
721
+// Send range compaction request.
722
func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
723
ch := make(chan error)
724
defer close(ch)
@@ -584,19 +732,14 @@ func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err
732
}
733
// Wait cmd.
734
select {
587
- case err = <-db.compErrC:
735
case err = <-ch:
736
+ case err = <-db.compErrC:
737
+ case _, _ = <-db.closeC:
738
+ return ErrClosed
739
}
740
return err
741
}
742
593
-func (db *DB) compTrigger(compTriggerC chan struct{}) {
594
- select {
595
- case compTriggerC <- struct{}{}:
596
- default:
597
- }
598
-}
599
-
743
func (db *DB) mCompaction() {
744
var x cCmd
745
@@ -615,11 +758,14 @@ func (db *DB) mCompaction() {
758
for {
759
select {
760
case x = <-db.mcompCmdC:
618
- db.memCompaction()
619
- x.ack(nil)
620
- x = nil
621
- case <-db.mcompTriggerC:
622
- db.memCompaction()
761
+ switch x.(type) {
762
+ case cIdle:
763
+ db.memCompaction()
764
+ x.ack(nil)
765
+ x = nil
766
+ default:
767
+ panic("leveldb: unknown command")
768
+ }
769
case _, _ = <-db.closeC:
770
return
771
}
@@ -650,7 +796,6 @@ func (db *DB) tCompaction() {
796
if db.tableNeedCompaction() {
797
select {
798
case x = <-db.tcompCmdC:
653
- case <-db.tcompTriggerC:
799
case ch := <-db.tcompPauseC:
800
db.pauseCompaction(ch)
801
continue
@@ -666,7 +811,6 @@ func (db *DB) tCompaction() {
811
ackQ = ackQ[:0]
812
select {
813
case x = <-db.tcompCmdC:
669
- case <-db.tcompTriggerC:
814
case ch := <-db.tcompPauseC:
815
db.pauseCompaction(ch)
816
continue
@@ -681,6 +825,8 @@ func (db *DB) tCompaction() {
825
case cRange:
826
db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
827
x.ack(nil)
828
+ default:
829
+ panic("leveldb: unknown command")
830
}
831
x = nil
832
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go
+21
-20
@@ -48,7 +48,7 @@ func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.It
48
i = append(i, fmi)
49
}
50
i = append(i, ti...)
51
- strict := db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
51
+ strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
52
mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
53
mi.SetReleaser(&versionReleaser{v: v})
54
return mi
@@ -59,10 +59,10 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
59
if slice != nil {
60
islice = &util.Range{}
61
if slice.Start != nil {
62
- islice.Start = newIKey(slice.Start, kMaxSeq, tSeek)
62
+ islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
63
}
64
if slice.Limit != nil {
65
- islice.Limit = newIKey(slice.Limit, kMaxSeq, tSeek)
65
+ islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
66
}
67
}
68
rawIter := db.newRawIterator(islice, ro)
@@ -71,7 +71,7 @@ func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *d
71
icmp: db.s.icmp,
72
iter: rawIter,
73
seq: seq,
74
- strict: db.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator),
74
+ strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
75
key: make([]byte, 0),
76
value: make([]byte, 0),
77
}
@@ -162,7 +162,7 @@ func (i *dbIter) Seek(key []byte) bool {
162
return false
163
}
164
165
- ikey := newIKey(key, i.seq, tSeek)
165
+ ikey := newIkey(key, i.seq, ktSeek)
166
if i.iter.Seek(ikey) {
167
i.dir = dirSOI
168
return i.next()
@@ -174,15 +174,14 @@ func (i *dbIter) Seek(key []byte) bool {
174
175
func (i *dbIter) next() bool {
176
for {
177
- ukey, seq, t, ok := parseIkey(i.iter.Key())
178
- if ok {
177
+ if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
178
if seq <= i.seq {
180
- switch t {
181
- case tDel:
179
+ switch kt {
180
+ case ktDel:
181
// Skip deleted key.
182
i.key = append(i.key[:0], ukey...)
183
i.dir = dirForward
185
- case tVal:
184
+ case ktVal:
185
if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
186
i.key = append(i.key[:0], ukey...)
187
i.value = append(i.value[:0], i.iter.Value()...)
@@ -192,7 +191,7 @@ func (i *dbIter) next() bool {
191
}
192
}
193
} else if i.strict {
195
- i.setErr(errInvalidIkey)
194
+ i.setErr(kerr)
195
break
196
}
197
if !i.iter.Next() {
@@ -225,20 +224,19 @@ func (i *dbIter) prev() bool {
224
del := true
225
if i.iter.Valid() {
226
for {
228
- ukey, seq, t, ok := parseIkey(i.iter.Key())
229
- if ok {
227
+ if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
228
if seq <= i.seq {
229
if !del && i.icmp.uCompare(ukey, i.key) < 0 {
230
return true
231
}
234
- del = (t == tDel)
232
+ del = (kt == ktDel)
233
if !del {
234
i.key = append(i.key[:0], ukey...)
235
i.value = append(i.value[:0], i.iter.Value()...)
236
}
237
}
238
} else if i.strict {
241
- i.setErr(errInvalidIkey)
239
+ i.setErr(kerr)
240
return false
241
}
242
if !i.iter.Prev() {
@@ -267,13 +265,12 @@ func (i *dbIter) Prev() bool {
265
return i.Last()
266
case dirForward:
267
for i.iter.Prev() {
270
- ukey, _, _, ok := parseIkey(i.iter.Key())
271
- if ok {
268
+ if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
269
if i.icmp.uCompare(ukey, i.key) < 0 {
270
goto cont
271
}
272
} else if i.strict {
276
- i.setErr(errInvalidIkey)
273
+ i.setErr(kerr)
274
return false
275
}
276
}
@@ -321,9 +318,13 @@ func (i *dbIter) Release() {
318
}
319
320
func (i *dbIter) SetReleaser(releaser util.Releaser) {
324
- if i.dir != dirReleased {
325
- i.releaser = releaser
321
+ if i.dir == dirReleased {
322
+ panic(util.ErrReleased)
323
+ }
324
+ if i.releaser != nil && releaser != nil {
325
+ panic(util.ErrHasReleaser)
326
}
327
+ i.releaser = releaser
328
}
329
330
func (i *dbIter) Error() error {
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go
+52
-38
@@ -7,6 +7,8 @@
7
package leveldb
8
9
import (
10
+ "container/list"
11
+ "fmt"
12
"runtime"
13
"sync"
14
"sync/atomic"
@@ -19,51 +21,41 @@ import (
21
type snapshotElement struct {
22
seq uint64
23
ref int
22
- // Next and previous pointers in the doubly-linked list of elements.
23
- next, prev *snapshotElement
24
-}
25
-
26
-// Initialize the snapshot.
27
-func (db *DB) initSnapshot() {
28
- db.snapsRoot.next = &db.snapsRoot
29
- db.snapsRoot.prev = &db.snapsRoot
24
+ e *list.Element
25
}
26
27
// Acquires a snapshot, based on latest sequence.
28
func (db *DB) acquireSnapshot() *snapshotElement {
29
db.snapsMu.Lock()
30
+ defer db.snapsMu.Unlock()
31
+
32
seq := db.getSeq()
36
- elem := db.snapsRoot.prev
37
- if elem == &db.snapsRoot || elem.seq != seq {
38
- at := db.snapsRoot.prev
39
- next := at.next
40
- elem = &snapshotElement{
41
- seq: seq,
42
- prev: at,
43
- next: next,
33
+
34
+ if e := db.snapsList.Back(); e != nil {
35
+ se := e.Value.(*snapshotElement)
36
+ if se.seq == seq {
37
+ se.ref++
38
+ return se
39
+ } else if seq < se.seq {
40
+ panic("leveldb: sequence number is not increasing")
41
}
45
- at.next = elem
46
- next.prev = elem
42
}
48
- elem.ref++
49
- db.snapsMu.Unlock()
50
- return elem
43
+ se := &snapshotElement{seq: seq, ref: 1}
44
+ se.e = db.snapsList.PushBack(se)
45
+ return se
46
}
47
48
// Releases given snapshot element.
54
-func (db *DB) releaseSnapshot(elem *snapshotElement) {
55
- if !db.isClosed() {
56
- db.snapsMu.Lock()
57
- elem.ref--
58
- if elem.ref == 0 {
59
- elem.prev.next = elem.next
60
- elem.next.prev = elem.prev
61
- elem.next = nil
62
- elem.prev = nil
63
- } else if elem.ref < 0 {
64
- panic("leveldb: Snapshot: negative element reference")
65
- }
66
- db.snapsMu.Unlock()
49
+func (db *DB) releaseSnapshot(se *snapshotElement) {
50
+ db.snapsMu.Lock()
51
+ defer db.snapsMu.Unlock()
52
+
53
+ se.ref--
54
+ if se.ref == 0 {
55
+ db.snapsList.Remove(se.e)
56
+ se.e = nil
57
+ } else if se.ref < 0 {
58
+ panic("leveldb: Snapshot: negative element reference")
59
}
60
}
61
@@ -71,10 +63,11 @@ func (db *DB) releaseSnapshot(elem *snapshotElement) {
63
func (db *DB) minSeq() uint64 {
64
db.snapsMu.Lock()
65
defer db.snapsMu.Unlock()
74
- elem := db.snapsRoot.prev
75
- if elem != &db.snapsRoot {
76
- return elem.seq
66
+
67
+ if e := db.snapsList.Front(); e != nil {
68
+ return e.Value.(*snapshotElement).seq
69
}
70
+
71
return db.getSeq()
72
}
73
@@ -97,8 +90,12 @@ func (db *DB) newSnapshot() *Snapshot {
90
return snap
91
}
92
93
+func (snap *Snapshot) String() string {
94
+ return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
95
+}
96
+
97
// Get gets the value for the given key. It returns ErrNotFound if
101
-// the DB does not contain the key.
98
+// the DB does not contains the key.
99
//
100
// The caller should not modify the contents of the returned slice, but
101
// it is safe to modify the contents of the argument after Get returns.
@@ -116,6 +113,23 @@ func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err er
113
return snap.db.get(key, snap.elem.seq, ro)
114
}
115
116
+// Has returns true if the DB does contains the given key.
117
+//
118
+// It is safe to modify the contents of the argument after Get returns.
119
+func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
120
+ err = snap.db.ok()
121
+ if err != nil {
122
+ return
123
+ }
124
+ snap.mu.RLock()
125
+ defer snap.mu.RUnlock()
126
+ if snap.released {
127
+ err = ErrSnapshotReleased
128
+ return
129
+ }
130
+ return snap.db.has(key, snap.elem.seq, ro)
131
+}
132
+
133
// NewIterator returns an iterator for the snapshot of the uderlying DB.
134
// The returned iterator is not goroutine-safe, but it is safe to use
135
// multiple iterators concurrently, with each in a dedicated goroutine.
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go
+771
-82
@@ -7,6 +7,10 @@
7
package leveldb
8
9
import (
10
+ "bytes"
11
+ "container/list"
12
+ crand "crypto/rand"
13
+ "encoding/binary"
14
"fmt"
15
"math/rand"
16
"os"
@@ -20,6 +24,7 @@ import (
24
"unsafe"
25
26
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
27
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
28
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
29
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
30
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
@@ -148,7 +153,10 @@ func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) {
153
t := h.t
154
db := h.db
155
151
- var res uint64
156
+ var (
157
+ maxOverlaps uint64
158
+ maxLevel int
159
+ )
160
v := db.s.version()
161
for i, tt := range v.tables[1 : len(v.tables)-1] {
162
level := i + 1
@@ -156,15 +164,18 @@ func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) {
164
for _, t := range tt {
165
r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false)
166
sum := r.size()
159
- if sum > res {
160
- res = sum
167
+ if sum > maxOverlaps {
168
+ maxOverlaps = sum
169
+ maxLevel = level
170
}
171
}
172
}
173
v.release()
174
166
- if res > want {
167
- t.Errorf("next level overlapping bytes is more than %d, got=%d", want, res)
175
+ if maxOverlaps > want {
176
+ t.Errorf("next level most overlapping bytes is more than %d, got=%d level=%d", want, maxOverlaps, maxLevel)
177
+ } else {
178
+ t.Logf("next level most overlapping bytes is %d, level=%d want=%d", maxOverlaps, maxLevel, want)
179
}
180
}
181
@@ -237,7 +248,7 @@ func (h *dbHarness) allEntriesFor(key, want string) {
248
db := h.db
249
s := db.s
250
240
- ikey := newIKey([]byte(key), kMaxSeq, tVal)
251
+ ikey := newIkey([]byte(key), kMaxSeq, ktVal)
252
iter := db.newRawIterator(nil, nil)
253
if !iter.Seek(ikey) && iter.Error() != nil {
254
t.Error("AllEntries: error during seek, err: ", iter.Error())
@@ -246,19 +257,18 @@ func (h *dbHarness) allEntriesFor(key, want string) {
257
res := "[ "
258
first := true
259
for iter.Valid() {
249
- rkey := iKey(iter.Key())
250
- if _, t, ok := rkey.parseNum(); ok {
251
- if s.icmp.uCompare(ikey.ukey(), rkey.ukey()) != 0 {
260
+ if ukey, _, kt, kerr := parseIkey(iter.Key()); kerr == nil {
261
+ if s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
262
break
263
}
264
if !first {
265
res += ", "
266
}
267
first = false
258
- switch t {
259
- case tVal:
268
+ switch kt {
269
+ case ktVal:
270
res += string(iter.Value())
261
- case tDel:
271
+ case ktDel:
272
res += "DEL"
273
}
274
} else {
@@ -323,6 +333,8 @@ func (h *dbHarness) compactMem() {
333
t := h.t
334
db := h.db
335
336
+ t.Log("starting memdb compaction")
337
+
338
db.writeLockC <- struct{}{}
339
defer func() {
340
<-db.writeLockC
@@ -338,6 +350,8 @@ func (h *dbHarness) compactMem() {
350
if h.totalTables() == 0 {
351
t.Error("zero tables after mem compaction")
352
}
353
+
354
+ t.Log("memdb compaction done")
355
}
356
357
func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) {
@@ -352,6 +366,8 @@ func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool)
366
_max = []byte(max)
367
}
368
369
+ t.Logf("starting table range compaction: level=%d, min=%q, max=%q", level, min, max)
370
+
371
if err := db.compSendRange(db.tcompCmdC, level, _min, _max); err != nil {
372
if wanterr {
373
t.Log("CompactRangeAt: got error (expected): ", err)
@@ -361,6 +377,8 @@ func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool)
377
} else if wanterr {
378
t.Error("CompactRangeAt: expect error")
379
}
380
+
381
+ t.Log("table range compaction done")
382
}
383
384
func (h *dbHarness) compactRangeAt(level int, min, max string) {
@@ -371,6 +389,8 @@ func (h *dbHarness) compactRange(min, max string) {
389
t := h.t
390
db := h.db
391
392
+ t.Logf("starting DB range compaction: min=%q, max=%q", min, max)
393
+
394
var r util.Range
395
if min != "" {
396
r.Start = []byte(min)
@@ -381,6 +401,8 @@ func (h *dbHarness) compactRange(min, max string) {
401
if err := db.CompactRange(r); err != nil {
402
t.Error("CompactRange: got error: ", err)
403
}
404
+
405
+ t.Log("DB range compaction done")
406
}
407
408
func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) {
@@ -502,13 +524,13 @@ func Test_FieldsAligned(t *testing.T) {
524
p1 := new(DB)
525
testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq))
526
p2 := new(session)
505
- testAligned(t, "session.stFileNum", unsafe.Offsetof(p2.stFileNum))
527
+ testAligned(t, "session.stNextFileNum", unsafe.Offsetof(p2.stNextFileNum))
528
testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum))
529
testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum))
508
- testAligned(t, "session.stSeq", unsafe.Offsetof(p2.stSeq))
530
+ testAligned(t, "session.stSeqNum", unsafe.Offsetof(p2.stSeqNum))
531
}
532
511
-func TestDb_Locking(t *testing.T) {
533
+func TestDB_Locking(t *testing.T) {
534
h := newDbHarness(t)
535
defer h.stor.Close()
536
h.openAssert(false)
@@ -516,7 +538,7 @@ func TestDb_Locking(t *testing.T) {
538
h.openAssert(true)
539
}
540
519
-func TestDb_Empty(t *testing.T) {
541
+func TestDB_Empty(t *testing.T) {
542
trun(t, func(h *dbHarness) {
543
h.get("foo", false)
544
@@ -525,7 +547,7 @@ func TestDb_Empty(t *testing.T) {
547
})
548
}
549
528
-func TestDb_ReadWrite(t *testing.T) {
550
+func TestDB_ReadWrite(t *testing.T) {
551
trun(t, func(h *dbHarness) {
552
h.put("foo", "v1")
553
h.getVal("foo", "v1")
@@ -540,7 +562,7 @@ func TestDb_ReadWrite(t *testing.T) {
562
})
563
}
564
543
-func TestDb_PutDeleteGet(t *testing.T) {
565
+func TestDB_PutDeleteGet(t *testing.T) {
566
trun(t, func(h *dbHarness) {
567
h.put("foo", "v1")
568
h.getVal("foo", "v1")
@@ -554,7 +576,7 @@ func TestDb_PutDeleteGet(t *testing.T) {
576
})
577
}
578
557
-func TestDb_EmptyBatch(t *testing.T) {
579
+func TestDB_EmptyBatch(t *testing.T) {
580
h := newDbHarness(t)
581
defer h.close()
582
@@ -566,7 +588,7 @@ func TestDb_EmptyBatch(t *testing.T) {
588
h.get("foo", false)
589
}
590
569
-func TestDb_GetFromFrozen(t *testing.T) {
591
+func TestDB_GetFromFrozen(t *testing.T) {
592
h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100100})
593
defer h.close()
594
@@ -592,7 +614,7 @@ func TestDb_GetFromFrozen(t *testing.T) {
614
h.get("k2", true)
615
}
616
595
-func TestDb_GetFromTable(t *testing.T) {
617
+func TestDB_GetFromTable(t *testing.T) {
618
trun(t, func(h *dbHarness) {
619
h.put("foo", "v1")
620
h.compactMem()
@@ -600,7 +622,7 @@ func TestDb_GetFromTable(t *testing.T) {
622
})
623
}
624
603
-func TestDb_GetSnapshot(t *testing.T) {
625
+func TestDB_GetSnapshot(t *testing.T) {
626
trun(t, func(h *dbHarness) {
627
bar := strings.Repeat("b", 200)
628
h.put("foo", "v1")
@@ -634,7 +656,7 @@ func TestDb_GetSnapshot(t *testing.T) {
656
})
657
}
658
637
-func TestDb_GetLevel0Ordering(t *testing.T) {
659
+func TestDB_GetLevel0Ordering(t *testing.T) {
660
trun(t, func(h *dbHarness) {
661
for i := 0; i < 4; i++ {
662
h.put("bar", fmt.Sprintf("b%d", i))
@@ -657,7 +679,7 @@ func TestDb_GetLevel0Ordering(t *testing.T) {
679
})
680
}
681
660
-func TestDb_GetOrderedByLevels(t *testing.T) {
682
+func TestDB_GetOrderedByLevels(t *testing.T) {
683
trun(t, func(h *dbHarness) {
684
h.put("foo", "v1")
685
h.compactMem()
@@ -669,7 +691,7 @@ func TestDb_GetOrderedByLevels(t *testing.T) {
691
})
692
}
693
672
-func TestDb_GetPicksCorrectFile(t *testing.T) {
694
+func TestDB_GetPicksCorrectFile(t *testing.T) {
695
trun(t, func(h *dbHarness) {
696
// Arrange to have multiple files in a non-level-0 level.
697
h.put("a", "va")
@@ -693,7 +715,7 @@ func TestDb_GetPicksCorrectFile(t *testing.T) {
715
})
716
}
717
696
-func TestDb_GetEncountersEmptyLevel(t *testing.T) {
718
+func TestDB_GetEncountersEmptyLevel(t *testing.T) {
719
trun(t, func(h *dbHarness) {
720
// Arrange for the following to happen:
721
// * sstable A in level 0
@@ -748,7 +770,7 @@ func TestDb_GetEncountersEmptyLevel(t *testing.T) {
770
})
771
}
772
751
-func TestDb_IterMultiWithDelete(t *testing.T) {
773
+func TestDB_IterMultiWithDelete(t *testing.T) {
774
trun(t, func(h *dbHarness) {
775
h.put("a", "va")
776
h.put("b", "vb")
@@ -774,7 +796,7 @@ func TestDb_IterMultiWithDelete(t *testing.T) {
796
})
797
}
798
777
-func TestDb_IteratorPinsRef(t *testing.T) {
799
+func TestDB_IteratorPinsRef(t *testing.T) {
800
h := newDbHarness(t)
801
defer h.close()
802
@@ -798,7 +820,7 @@ func TestDb_IteratorPinsRef(t *testing.T) {
820
iter.Release()
821
}
822
801
-func TestDb_Recover(t *testing.T) {
823
+func TestDB_Recover(t *testing.T) {
824
trun(t, func(h *dbHarness) {
825
h.put("foo", "v1")
826
h.put("baz", "v5")
@@ -820,7 +842,7 @@ func TestDb_Recover(t *testing.T) {
842
})
843
}
844
823
-func TestDb_RecoverWithEmptyJournal(t *testing.T) {
845
+func TestDB_RecoverWithEmptyJournal(t *testing.T) {
846
trun(t, func(h *dbHarness) {
847
h.put("foo", "v1")
848
h.put("foo", "v2")
@@ -834,7 +856,7 @@ func TestDb_RecoverWithEmptyJournal(t *testing.T) {
856
})
857
}
858
837
-func TestDb_RecoverDuringMemtableCompaction(t *testing.T) {
859
+func TestDB_RecoverDuringMemtableCompaction(t *testing.T) {
860
truno(t, &opt.Options{WriteBuffer: 1000000}, func(h *dbHarness) {
861
862
h.stor.DelaySync(storage.TypeTable)
@@ -850,7 +872,7 @@ func TestDb_RecoverDuringMemtableCompaction(t *testing.T) {
872
})
873
}
874
853
-func TestDb_MinorCompactionsHappen(t *testing.T) {
875
+func TestDB_MinorCompactionsHappen(t *testing.T) {
876
h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 10000})
877
defer h.close()
878
@@ -874,7 +896,7 @@ func TestDb_MinorCompactionsHappen(t *testing.T) {
896
}
897
}
898
877
-func TestDb_RecoverWithLargeJournal(t *testing.T) {
899
+func TestDB_RecoverWithLargeJournal(t *testing.T) {
900
h := newDbHarness(t)
901
defer h.close()
902
@@ -899,7 +921,7 @@ func TestDb_RecoverWithLargeJournal(t *testing.T) {
921
v.release()
922
}
923
902
-func TestDb_CompactionsGenerateMultipleFiles(t *testing.T) {
924
+func TestDB_CompactionsGenerateMultipleFiles(t *testing.T) {
925
h := newDbHarnessWopt(t, &opt.Options{
926
WriteBuffer: 10000000,
927
Compression: opt.NoCompression,
@@ -937,11 +959,11 @@ func TestDb_CompactionsGenerateMultipleFiles(t *testing.T) {
959
}
960
}
961
940
-func TestDb_RepeatedWritesToSameKey(t *testing.T) {
962
+func TestDB_RepeatedWritesToSameKey(t *testing.T) {
963
h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
964
defer h.close()
965
944
- maxTables := kNumLevels + kL0_StopWritesTrigger
966
+ maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger()
967
968
value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
969
for i := 0; i < 5*maxTables; i++ {
@@ -953,13 +975,13 @@ func TestDb_RepeatedWritesToSameKey(t *testing.T) {
975
}
976
}
977
956
-func TestDb_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
978
+func TestDB_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
979
h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
980
defer h.close()
981
982
h.reopenDB()
983
962
- maxTables := kNumLevels + kL0_StopWritesTrigger
984
+ maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger()
985
986
value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
987
for i := 0; i < 5*maxTables; i++ {
@@ -971,11 +993,11 @@ func TestDb_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
993
}
994
}
995
974
-func TestDb_SparseMerge(t *testing.T) {
996
+func TestDB_SparseMerge(t *testing.T) {
997
h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
998
defer h.close()
999
978
- h.putMulti(kNumLevels, "A", "Z")
1000
+ h.putMulti(h.o.GetNumLevel(), "A", "Z")
1001
1002
// Suppose there is:
1003
// small amount of data with prefix A
@@ -999,6 +1021,7 @@ func TestDb_SparseMerge(t *testing.T) {
1021
h.put("C", "vc2")
1022
h.compactMem()
1023
1024
+ h.waitCompaction()
1025
h.maxNextLevelOverlappingBytes(20 * 1048576)
1026
h.compactRangeAt(0, "", "")
1027
h.waitCompaction()
@@ -1008,7 +1031,7 @@ func TestDb_SparseMerge(t *testing.T) {
1031
h.maxNextLevelOverlappingBytes(20 * 1048576)
1032
}
1033
1011
-func TestDb_SizeOf(t *testing.T) {
1034
+func TestDB_SizeOf(t *testing.T) {
1035
h := newDbHarnessWopt(t, &opt.Options{
1036
Compression: opt.NoCompression,
1037
WriteBuffer: 10000000,
@@ -1058,7 +1081,7 @@ func TestDb_SizeOf(t *testing.T) {
1081
}
1082
}
1083
1061
-func TestDb_SizeOf_MixOfSmallAndLarge(t *testing.T) {
1084
+func TestDB_SizeOf_MixOfSmallAndLarge(t *testing.T) {
1085
h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
1086
defer h.close()
1087
@@ -1096,7 +1119,7 @@ func TestDb_SizeOf_MixOfSmallAndLarge(t *testing.T) {
1119
}
1120
}
1121
1099
-func TestDb_Snapshot(t *testing.T) {
1122
+func TestDB_Snapshot(t *testing.T) {
1123
trun(t, func(h *dbHarness) {
1124
h.put("foo", "v1")
1125
s1 := h.getSnapshot()
@@ -1125,13 +1148,51 @@ func TestDb_Snapshot(t *testing.T) {
1148
})
1149
}
1150
1128
-func TestDb_HiddenValuesAreRemoved(t *testing.T) {
1151
+func TestDB_SnapshotList(t *testing.T) {
1152
+ db := &DB{snapsList: list.New()}
1153
+ e0a := db.acquireSnapshot()
1154
+ e0b := db.acquireSnapshot()
1155
+ db.seq = 1
1156
+ e1 := db.acquireSnapshot()
1157
+ db.seq = 2
1158
+ e2 := db.acquireSnapshot()
1159
+
1160
+ if db.minSeq() != 0 {
1161
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1162
+ }
1163
+ db.releaseSnapshot(e0a)
1164
+ if db.minSeq() != 0 {
1165
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1166
+ }
1167
+ db.releaseSnapshot(e2)
1168
+ if db.minSeq() != 0 {
1169
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1170
+ }
1171
+ db.releaseSnapshot(e0b)
1172
+ if db.minSeq() != 1 {
1173
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1174
+ }
1175
+ e2 = db.acquireSnapshot()
1176
+ if db.minSeq() != 1 {
1177
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1178
+ }
1179
+ db.releaseSnapshot(e1)
1180
+ if db.minSeq() != 2 {
1181
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1182
+ }
1183
+ db.releaseSnapshot(e2)
1184
+ if db.minSeq() != 2 {
1185
+ t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1186
+ }
1187
+}
1188
+
1189
+func TestDB_HiddenValuesAreRemoved(t *testing.T) {
1190
trun(t, func(h *dbHarness) {
1191
s := h.db.s
1192
1193
h.put("foo", "v1")
1194
h.compactMem()
1134
- m := kMaxMemCompactLevel
1195
+ m := h.o.GetMaxMemCompationLevel()
1196
v := s.version()
1197
num := v.tLen(m)
1198
v.release()
@@ -1168,14 +1229,14 @@ func TestDb_HiddenValuesAreRemoved(t *testing.T) {
1229
})
1230
}
1231
1171
-func TestDb_DeletionMarkers2(t *testing.T) {
1232
+func TestDB_DeletionMarkers2(t *testing.T) {
1233
h := newDbHarness(t)
1234
defer h.close()
1235
s := h.db.s
1236
1237
h.put("foo", "v1")
1238
h.compactMem()
1178
- m := kMaxMemCompactLevel
1239
+ m := h.o.GetMaxMemCompationLevel()
1240
v := s.version()
1241
num := v.tLen(m)
1242
v.release()
@@ -1209,7 +1270,7 @@ func TestDb_DeletionMarkers2(t *testing.T) {
1270
h.allEntriesFor("foo", "[ ]")
1271
}
1272
1212
-func TestDb_CompactionTableOpenError(t *testing.T) {
1273
+func TestDB_CompactionTableOpenError(t *testing.T) {
1274
h := newDbHarnessWopt(t, &opt.Options{CachedOpenFiles: -1})
1275
defer h.close()
1276
@@ -1228,14 +1289,14 @@ func TestDb_CompactionTableOpenError(t *testing.T) {
1289
t.Errorf("total tables is %d, want %d", n, im)
1290
}
1291
1231
- h.stor.SetOpenErr(storage.TypeTable)
1292
+ h.stor.SetEmuErr(storage.TypeTable, tsOpOpen)
1293
go h.db.CompactRange(util.Range{})
1294
if err := h.db.compSendIdle(h.db.tcompCmdC); err != nil {
1295
t.Log("compaction error: ", err)
1296
}
1297
h.closeDB0()
1298
h.openDB()
1238
- h.stor.SetOpenErr(0)
1299
+ h.stor.SetEmuErr(0, tsOpOpen)
1300
1301
for i := 0; i < im; i++ {
1302
for j := 0; j < jm; j++ {
@@ -1244,9 +1305,9 @@ func TestDb_CompactionTableOpenError(t *testing.T) {
1305
}
1306
}
1307
1247
-func TestDb_OverlapInLevel0(t *testing.T) {
1308
+func TestDB_OverlapInLevel0(t *testing.T) {
1309
trun(t, func(h *dbHarness) {
1249
- if kMaxMemCompactLevel != 2 {
1310
+ if h.o.GetMaxMemCompationLevel() != 2 {
1311
t.Fatal("fix test to reflect the config")
1312
}
1313
@@ -1287,7 +1348,7 @@ func TestDb_OverlapInLevel0(t *testing.T) {
1348
})
1349
}
1350
1290
-func TestDb_L0_CompactionBug_Issue44_a(t *testing.T) {
1351
+func TestDB_L0_CompactionBug_Issue44_a(t *testing.T) {
1352
h := newDbHarness(t)
1353
defer h.close()
1354
@@ -1307,7 +1368,7 @@ func TestDb_L0_CompactionBug_Issue44_a(t *testing.T) {
1368
h.getKeyVal("(a->v)")
1369
}
1370
1310
-func TestDb_L0_CompactionBug_Issue44_b(t *testing.T) {
1371
+func TestDB_L0_CompactionBug_Issue44_b(t *testing.T) {
1372
h := newDbHarness(t)
1373
defer h.close()
1374
@@ -1336,7 +1397,7 @@ func TestDb_L0_CompactionBug_Issue44_b(t *testing.T) {
1397
h.getKeyVal("(->)(c->cv)")
1398
}
1399
1339
-func TestDb_SingleEntryMemCompaction(t *testing.T) {
1400
+func TestDB_SingleEntryMemCompaction(t *testing.T) {
1401
trun(t, func(h *dbHarness) {
1402
for i := 0; i < 10; i++ {
1403
h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer))
@@ -1353,7 +1414,7 @@ func TestDb_SingleEntryMemCompaction(t *testing.T) {
1414
})
1415
}
1416
1356
-func TestDb_ManifestWriteError(t *testing.T) {
1417
+func TestDB_ManifestWriteError(t *testing.T) {
1418
for i := 0; i < 2; i++ {
1419
func() {
1420
h := newDbHarness(t)
@@ -1366,23 +1427,23 @@ func TestDb_ManifestWriteError(t *testing.T) {
1427
h.compactMem()
1428
h.getVal("foo", "bar")
1429
v := h.db.s.version()
1369
- if n := v.tLen(kMaxMemCompactLevel); n != 1 {
1430
+ if n := v.tLen(h.o.GetMaxMemCompationLevel()); n != 1 {
1431
t.Errorf("invalid total tables, want=1 got=%d", n)
1432
}
1433
v.release()
1434
1435
if i == 0 {
1375
- h.stor.SetWriteErr(storage.TypeManifest)
1436
+ h.stor.SetEmuErr(storage.TypeManifest, tsOpWrite)
1437
} else {
1377
- h.stor.SetSyncErr(storage.TypeManifest)
1438
+ h.stor.SetEmuErr(storage.TypeManifest, tsOpSync)
1439
}
1440
1441
// Merging compaction (will fail)
1381
- h.compactRangeAtErr(kMaxMemCompactLevel, "", "", true)
1442
+ h.compactRangeAtErr(h.o.GetMaxMemCompationLevel(), "", "", true)
1443
1444
h.db.Close()
1384
- h.stor.SetWriteErr(0)
1385
- h.stor.SetSyncErr(0)
1445
+ h.stor.SetEmuErr(0, tsOpWrite)
1446
+ h.stor.SetEmuErr(0, tsOpSync)
1447
1448
// Should not lose data
1449
h.openDB()
@@ -1403,7 +1464,7 @@ func assertErr(t *testing.T, err error, wanterr bool) {
1464
}
1465
}
1466
1406
-func TestDb_ClosedIsClosed(t *testing.T) {
1467
+func TestDB_ClosedIsClosed(t *testing.T) {
1468
h := newDbHarness(t)
1469
db := h.db
1470
@@ -1498,7 +1559,7 @@ func (p numberComparer) Compare(a, b []byte) int {
1559
func (numberComparer) Separator(dst, a, b []byte) []byte { return nil }
1560
func (numberComparer) Successor(dst, b []byte) []byte { return nil }
1561
1501
-func TestDb_CustomComparer(t *testing.T) {
1562
+func TestDB_CustomComparer(t *testing.T) {
1563
h := newDbHarnessWopt(t, &opt.Options{
1564
Comparer: numberComparer{},
1565
WriteBuffer: 1000,
@@ -1528,11 +1589,11 @@ func TestDb_CustomComparer(t *testing.T) {
1589
}
1590
}
1591
1531
-func TestDb_ManualCompaction(t *testing.T) {
1592
+func TestDB_ManualCompaction(t *testing.T) {
1593
h := newDbHarness(t)
1594
defer h.close()
1595
1535
- if kMaxMemCompactLevel != 2 {
1596
+ if h.o.GetMaxMemCompationLevel() != 2 {
1597
t.Fatal("fix test to reflect the config")
1598
}
1599
@@ -1566,7 +1627,7 @@ func TestDb_ManualCompaction(t *testing.T) {
1627
h.tablesPerLevel("0,0,1")
1628
}
1629
1569
-func TestDb_BloomFilter(t *testing.T) {
1630
+func TestDB_BloomFilter(t *testing.T) {
1631
h := newDbHarnessWopt(t, &opt.Options{
1632
BlockCache: opt.NoCache,
1633
Filter: filter.NewBloomFilter(10),
@@ -1577,11 +1638,7 @@ func TestDb_BloomFilter(t *testing.T) {
1638
return fmt.Sprintf("key%06d", i)
1639
}
1640
1580
- const (
1581
- n = 10000
1582
- indexOverheat = 19898
1583
- filterOverheat = 19799
1584
- )
1641
+ const n = 10000
1642
1643
// Populate multiple layers
1644
for i := 0; i < n; i++ {
@@ -1605,7 +1662,7 @@ func TestDb_BloomFilter(t *testing.T) {
1662
cnt := int(h.stor.ReadCounter())
1663
t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt)
1664
1608
- if min, max := n+indexOverheat+filterOverheat, n+indexOverheat+filterOverheat+2*n/100; cnt < min || cnt > max {
1665
+ if min, max := n, n+2*n/100; cnt < min || cnt > max {
1666
t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt)
1667
}
1668
@@ -1616,14 +1673,14 @@ func TestDb_BloomFilter(t *testing.T) {
1673
}
1674
cnt = int(h.stor.ReadCounter())
1675
t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt)
1619
- if max := 3*n/100 + indexOverheat + filterOverheat; cnt > max {
1676
+ if max := 3 * n / 100; cnt > max {
1677
t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt)
1678
}
1679
1680
h.stor.ReleaseSync(storage.TypeTable)
1681
}
1682
1626
-func TestDb_Concurrent(t *testing.T) {
1683
+func TestDB_Concurrent(t *testing.T) {
1684
const n, secs, maxkey = 4, 2, 1000
1685
1686
runtime.GOMAXPROCS(n)
@@ -1688,7 +1745,7 @@ func TestDb_Concurrent(t *testing.T) {
1745
runtime.GOMAXPROCS(1)
1746
}
1747
1691
-func TestDb_Concurrent2(t *testing.T) {
1748
+func TestDB_Concurrent2(t *testing.T) {
1749
const n, n2 = 4, 4000
1750
1751
runtime.GOMAXPROCS(n*2 + 2)
@@ -1759,7 +1816,7 @@ func TestDb_Concurrent2(t *testing.T) {
1816
runtime.GOMAXPROCS(1)
1817
}
1818
1762
-func TestDb_CreateReopenDbOnFile(t *testing.T) {
1819
+func TestDB_CreateReopenDbOnFile(t *testing.T) {
1820
dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid()))
1821
if err := os.RemoveAll(dbpath); err != nil {
1822
t.Fatal("cannot remove old db: ", err)
@@ -1787,7 +1844,7 @@ func TestDb_CreateReopenDbOnFile(t *testing.T) {
1844
}
1845
}
1846
1790
-func TestDb_CreateReopenDbOnFile2(t *testing.T) {
1847
+func TestDB_CreateReopenDbOnFile2(t *testing.T) {
1848
dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid()))
1849
if err := os.RemoveAll(dbpath); err != nil {
1850
t.Fatal("cannot remove old db: ", err)
@@ -1808,7 +1865,7 @@ func TestDb_CreateReopenDbOnFile2(t *testing.T) {
1865
}
1866
}
1867
1811
-func TestDb_DeletionMarkersOnMemdb(t *testing.T) {
1868
+func TestDB_DeletionMarkersOnMemdb(t *testing.T) {
1869
h := newDbHarness(t)
1870
defer h.close()
1871
@@ -1819,8 +1876,8 @@ func TestDb_DeletionMarkersOnMemdb(t *testing.T) {
1876
h.getKeyVal("")
1877
}
1878
1822
-func TestDb_LeveldbIssue178(t *testing.T) {
1823
- nKeys := (kMaxTableSize / 30) * 5
1879
+func TestDB_LeveldbIssue178(t *testing.T) {
1880
+ nKeys := (opt.DefaultCompactionTableSize / 30) * 5
1881
key1 := func(i int) string {
1882
return fmt.Sprintf("my_key_%d", i)
1883
}
@@ -1862,7 +1919,7 @@ func TestDb_LeveldbIssue178(t *testing.T) {
1919
h.assertNumKeys(nKeys)
1920
}
1921
1865
-func TestDb_LeveldbIssue200(t *testing.T) {
1922
+func TestDB_LeveldbIssue200(t *testing.T) {
1923
h := newDbHarness(t)
1924
defer h.close()
1925
@@ -1888,3 +1945,635 @@ func TestDb_LeveldbIssue200(t *testing.T) {
1945
iter.Next()
1946
assertBytes(t, []byte("5"), iter.Key())
1947
}
1948
+
1949
+func TestDB_GoleveldbIssue74(t *testing.T) {
1950
+ h := newDbHarnessWopt(t, &opt.Options{
1951
+ WriteBuffer: 1 * opt.MiB,
1952
+ })
1953
+ defer h.close()
1954
+
1955
+ const n, dur = 10000, 5 * time.Second
1956
+
1957
+ runtime.GOMAXPROCS(runtime.NumCPU())
1958
+
1959
+ until := time.Now().Add(dur)
1960
+ wg := new(sync.WaitGroup)
1961
+ wg.Add(2)
1962
+ var done uint32
1963
+ go func() {
1964
+ var i int
1965
+ defer func() {
1966
+ t.Logf("WRITER DONE #%d", i)
1967
+ atomic.StoreUint32(&done, 1)
1968
+ wg.Done()
1969
+ }()
1970
+
1971
+ b := new(Batch)
1972
+ for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
1973
+ iv := fmt.Sprintf("VAL%010d", i)
1974
+ for k := 0; k < n; k++ {
1975
+ key := fmt.Sprintf("KEY%06d", k)
1976
+ b.Put([]byte(key), []byte(key+iv))
1977
+ b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key))
1978
+ }
1979
+ h.write(b)
1980
+
1981
+ b.Reset()
1982
+ snap := h.getSnapshot()
1983
+ iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
1984
+ var k int
1985
+ for ; iter.Next(); k++ {
1986
+ ptrKey := iter.Key()
1987
+ key := iter.Value()
1988
+
1989
+ if _, err := snap.Get(ptrKey, nil); err != nil {
1990
+ t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err)
1991
+ }
1992
+ if value, err := snap.Get(key, nil); err != nil {
1993
+ t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err)
1994
+ } else if string(value) != string(key)+iv {
1995
+ t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value)
1996
+ }
1997
+
1998
+ b.Delete(key)
1999
+ b.Delete(ptrKey)
2000
+ }
2001
+ h.write(b)
2002
+ iter.Release()
2003
+ snap.Release()
2004
+ if k != n {
2005
+ t.Fatalf("#%d %d != %d", i, k, n)
2006
+ }
2007
+ }
2008
+ }()
2009
+ go func() {
2010
+ var i int
2011
+ defer func() {
2012
+ t.Logf("READER DONE #%d", i)
2013
+ atomic.StoreUint32(&done, 1)
2014
+ wg.Done()
2015
+ }()
2016
+ for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2017
+ snap := h.getSnapshot()
2018
+ iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
2019
+ var prevValue string
2020
+ var k int
2021
+ for ; iter.Next(); k++ {
2022
+ ptrKey := iter.Key()
2023
+ key := iter.Value()
2024
+
2025
+ if _, err := snap.Get(ptrKey, nil); err != nil {
2026
+ t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err)
2027
+ }
2028
+
2029
+ if value, err := snap.Get(key, nil); err != nil {
2030
+ t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err)
2031
+ } else if prevValue != "" && string(value) != string(key)+prevValue {
2032
+ t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value)
2033
+ } else {
2034
+ prevValue = string(value[len(key):])
2035
+ }
2036
+ }
2037
+ iter.Release()
2038
+ snap.Release()
2039
+ if k > 0 && k != n {
2040
+ t.Fatalf("#%d %d != %d", i, k, n)
2041
+ }
2042
+ }
2043
+ }()
2044
+ wg.Wait()
2045
+}
2046
+
2047
+func TestDB_GetProperties(t *testing.T) {
2048
+ h := newDbHarness(t)
2049
+ defer h.close()
2050
+
2051
+ _, err := h.db.GetProperty("leveldb.num-files-at-level")
2052
+ if err == nil {
2053
+ t.Error("GetProperty() failed to detect missing level")
2054
+ }
2055
+
2056
+ _, err = h.db.GetProperty("leveldb.num-files-at-level0")
2057
+ if err != nil {
2058
+ t.Error("got unexpected error", err)
2059
+ }
2060
+
2061
+ _, err = h.db.GetProperty("leveldb.num-files-at-level0x")
2062
+ if err == nil {
2063
+ t.Error("GetProperty() failed to detect invalid level")
2064
+ }
2065
+}
2066
+
2067
+func TestDB_GoleveldbIssue72and83(t *testing.T) {
2068
+ h := newDbHarnessWopt(t, &opt.Options{
2069
+ WriteBuffer: 1 * opt.MiB,
2070
+ CachedOpenFiles: 3,
2071
+ })
2072
+ defer h.close()
2073
+
2074
+ const n, wn, dur = 10000, 100, 30 * time.Second
2075
+
2076
+ runtime.GOMAXPROCS(runtime.NumCPU())
2077
+
2078
+ randomData := func(prefix byte, i int) []byte {
2079
+ data := make([]byte, 1+4+32+64+32)
2080
+ _, err := crand.Reader.Read(data[1 : len(data)-8])
2081
+ if err != nil {
2082
+ panic(err)
2083
+ }
2084
+ data[0] = prefix
2085
+ binary.LittleEndian.PutUint32(data[len(data)-8:], uint32(i))
2086
+ binary.LittleEndian.PutUint32(data[len(data)-4:], util.NewCRC(data[:len(data)-4]).Value())
2087
+ return data
2088
+ }
2089
+
2090
+ keys := make([][]byte, n)
2091
+ for i := range keys {
2092
+ keys[i] = randomData(1, 0)
2093
+ }
2094
+
2095
+ until := time.Now().Add(dur)
2096
+ wg := new(sync.WaitGroup)
2097
+ wg.Add(3)
2098
+ var done uint32
2099
+ go func() {
2100
+ i := 0
2101
+ defer func() {
2102
+ t.Logf("WRITER DONE #%d", i)
2103
+ wg.Done()
2104
+ }()
2105
+
2106
+ b := new(Batch)
2107
+ for ; i < wn && atomic.LoadUint32(&done) == 0; i++ {
2108
+ b.Reset()
2109
+ for _, k1 := range keys {
2110
+ k2 := randomData(2, i)
2111
+ b.Put(k2, randomData(42, i))
2112
+ b.Put(k1, k2)
2113
+ }
2114
+ if err := h.db.Write(b, h.wo); err != nil {
2115
+ atomic.StoreUint32(&done, 1)
2116
+ t.Fatalf("WRITER #%d db.Write: %v", i, err)
2117
+ }
2118
+ }
2119
+ }()
2120
+ go func() {
2121
+ var i int
2122
+ defer func() {
2123
+ t.Logf("READER0 DONE #%d", i)
2124
+ atomic.StoreUint32(&done, 1)
2125
+ wg.Done()
2126
+ }()
2127
+ for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2128
+ snap := h.getSnapshot()
2129
+ seq := snap.elem.seq
2130
+ if seq == 0 {
2131
+ snap.Release()
2132
+ continue
2133
+ }
2134
+ iter := snap.NewIterator(util.BytesPrefix([]byte{1}), nil)
2135
+ writei := int(seq/(n*2) - 1)
2136
+ var k int
2137
+ for ; iter.Next(); k++ {
2138
+ k1 := iter.Key()
2139
+ k2 := iter.Value()
2140
+ k1checksum0 := binary.LittleEndian.Uint32(k1[len(k1)-4:])
2141
+ k1checksum1 := util.NewCRC(k1[:len(k1)-4]).Value()
2142
+ if k1checksum0 != k1checksum1 {
2143
+ t.Fatalf("READER0 #%d.%d W#%d invalid K1 checksum: %#x != %#x", i, k, k1checksum0, k1checksum0)
2144
+ }
2145
+ k2checksum0 := binary.LittleEndian.Uint32(k2[len(k2)-4:])
2146
+ k2checksum1 := util.NewCRC(k2[:len(k2)-4]).Value()
2147
+ if k2checksum0 != k2checksum1 {
2148
+ t.Fatalf("READER0 #%d.%d W#%d invalid K2 checksum: %#x != %#x", i, k, k2checksum0, k2checksum1)
2149
+ }
2150
+ kwritei := int(binary.LittleEndian.Uint32(k2[len(k2)-8:]))
2151
+ if writei != kwritei {
2152
+ t.Fatalf("READER0 #%d.%d W#%d invalid write iteration num: %d", i, k, writei, kwritei)
2153
+ }
2154
+ if _, err := snap.Get(k2, nil); err != nil {
2155
+ t.Fatalf("READER0 #%d.%d W#%d snap.Get: %v\nk1: %x\n -> k2: %x", i, k, writei, err, k1, k2)
2156
+ }
2157
+ }
2158
+ if err := iter.Error(); err != nil {
2159
+ t.Fatalf("READER0 #%d.%d W#%d snap.Iterator: %v", i, k, writei, err)
2160
+ }
2161
+ iter.Release()
2162
+ snap.Release()
2163
+ if k > 0 && k != n {
2164
+ t.Fatalf("READER0 #%d W#%d short read, got=%d want=%d", i, writei, k, n)
2165
+ }
2166
+ }
2167
+ }()
2168
+ go func() {
2169
+ var i int
2170
+ defer func() {
2171
+ t.Logf("READER1 DONE #%d", i)
2172
+ atomic.StoreUint32(&done, 1)
2173
+ wg.Done()
2174
+ }()
2175
+ for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2176
+ iter := h.db.NewIterator(nil, nil)
2177
+ seq := iter.(*dbIter).seq
2178
+ if seq == 0 {
2179
+ iter.Release()
2180
+ continue
2181
+ }
2182
+ writei := int(seq/(n*2) - 1)
2183
+ var k int
2184
+ for ok := iter.Last(); ok; ok = iter.Prev() {
2185
+ k++
2186
+ }
2187
+ if err := iter.Error(); err != nil {
2188
+ t.Fatalf("READER1 #%d.%d W#%d db.Iterator: %v", i, k, writei, err)
2189
+ }
2190
+ iter.Release()
2191
+ if m := (writei+1)*n + n; k != m {
2192
+ t.Fatalf("READER1 #%d W#%d short read, got=%d want=%d", i, writei, k, m)
2193
+ }
2194
+ }
2195
+ }()
2196
+
2197
+ wg.Wait()
2198
+}
2199
+
2200
+func TestDB_TransientError(t *testing.T) {
2201
+ h := newDbHarnessWopt(t, &opt.Options{
2202
+ WriteBuffer: 128 * opt.KiB,
2203
+ CachedOpenFiles: 3,
2204
+ DisableCompactionBackoff: true,
2205
+ })
2206
+ defer h.close()
2207
+
2208
+ const (
2209
+ nSnap = 20
2210
+ nKey = 10000
2211
+ )
2212
+
2213
+ var (
2214
+ snaps [nSnap]*Snapshot
2215
+ b = &Batch{}
2216
+ )
2217
+ for i := range snaps {
2218
+ vtail := fmt.Sprintf("VAL%030d", i)
2219
+ b.Reset()
2220
+ for k := 0; k < nKey; k++ {
2221
+ key := fmt.Sprintf("KEY%8d", k)
2222
+ b.Put([]byte(key), []byte(key+vtail))
2223
+ }
2224
+ h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt)
2225
+ if err := h.db.Write(b, nil); err != nil {
2226
+ t.Logf("WRITE #%d error: %v", i, err)
2227
+ h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt, tsOpWrite)
2228
+ for {
2229
+ if err := h.db.Write(b, nil); err == nil {
2230
+ break
2231
+ } else if errors.IsCorrupted(err) {
2232
+ t.Fatalf("WRITE #%d corrupted: %v", i, err)
2233
+ }
2234
+ }
2235
+ }
2236
+
2237
+ snaps[i] = h.db.newSnapshot()
2238
+ b.Reset()
2239
+ for k := 0; k < nKey; k++ {
2240
+ key := fmt.Sprintf("KEY%8d", k)
2241
+ b.Delete([]byte(key))
2242
+ }
2243
+ h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt)
2244
+ if err := h.db.Write(b, nil); err != nil {
2245
+ t.Logf("WRITE #%d error: %v", i, err)
2246
+ h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt)
2247
+ for {
2248
+ if err := h.db.Write(b, nil); err == nil {
2249
+ break
2250
+ } else if errors.IsCorrupted(err) {
2251
+ t.Fatalf("WRITE #%d corrupted: %v", i, err)
2252
+ }
2253
+ }
2254
+ }
2255
+ }
2256
+ h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt)
2257
+
2258
+ runtime.GOMAXPROCS(runtime.NumCPU())
2259
+
2260
+ rnd := rand.New(rand.NewSource(0xecafdaed))
2261
+ wg := &sync.WaitGroup{}
2262
+ for i, snap := range snaps {
2263
+ wg.Add(2)
2264
+
2265
+ go func(i int, snap *Snapshot, sk []int) {
2266
+ defer wg.Done()
2267
+
2268
+ vtail := fmt.Sprintf("VAL%030d", i)
2269
+ for _, k := range sk {
2270
+ key := fmt.Sprintf("KEY%8d", k)
2271
+ xvalue, err := snap.Get([]byte(key), nil)
2272
+ if err != nil {
2273
+ t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err)
2274
+ }
2275
+ value := key + vtail
2276
+ if !bytes.Equal([]byte(value), xvalue) {
2277
+ t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue)
2278
+ }
2279
+ }
2280
+ }(i, snap, rnd.Perm(nKey))
2281
+
2282
+ go func(i int, snap *Snapshot) {
2283
+ defer wg.Done()
2284
+
2285
+ vtail := fmt.Sprintf("VAL%030d", i)
2286
+ iter := snap.NewIterator(nil, nil)
2287
+ defer iter.Release()
2288
+ for k := 0; k < nKey; k++ {
2289
+ if !iter.Next() {
2290
+ if err := iter.Error(); err != nil {
2291
+ t.Fatalf("READER_ITER #%d K%d error: %v", i, k, err)
2292
+ } else {
2293
+ t.Fatalf("READER_ITER #%d K%d eoi", i, k)
2294
+ }
2295
+ }
2296
+ key := fmt.Sprintf("KEY%8d", k)
2297
+ xkey := iter.Key()
2298
+ if !bytes.Equal([]byte(key), xkey) {
2299
+ t.Fatalf("READER_ITER #%d K%d invalid key: want %q, got %q", i, k, key, xkey)
2300
+ }
2301
+ value := key + vtail
2302
+ xvalue := iter.Value()
2303
+ if !bytes.Equal([]byte(value), xvalue) {
2304
+ t.Fatalf("READER_ITER #%d K%d invalid value: want %q, got %q", i, k, value, xvalue)
2305
+ }
2306
+ }
2307
+ }(i, snap)
2308
+ }
2309
+
2310
+ wg.Wait()
2311
+}
2312
+
2313
+func TestDB_UkeyShouldntHopAcrossTable(t *testing.T) {
2314
+ h := newDbHarnessWopt(t, &opt.Options{
2315
+ WriteBuffer: 112 * opt.KiB,
2316
+ CompactionTableSize: 90 * opt.KiB,
2317
+ CompactionExpandLimitFactor: 1,
2318
+ })
2319
+ defer h.close()
2320
+
2321
+ const (
2322
+ nSnap = 190
2323
+ nKey = 140
2324
+ )
2325
+
2326
+ var (
2327
+ snaps [nSnap]*Snapshot
2328
+ b = &Batch{}
2329
+ )
2330
+ for i := range snaps {
2331
+ vtail := fmt.Sprintf("VAL%030d", i)
2332
+ b.Reset()
2333
+ for k := 0; k < nKey; k++ {
2334
+ key := fmt.Sprintf("KEY%08d", k)
2335
+ b.Put([]byte(key), []byte(key+vtail))
2336
+ }
2337
+ if err := h.db.Write(b, nil); err != nil {
2338
+ t.Fatalf("WRITE #%d error: %v", i, err)
2339
+ }
2340
+
2341
+ snaps[i] = h.db.newSnapshot()
2342
+ b.Reset()
2343
+ for k := 0; k < nKey; k++ {
2344
+ key := fmt.Sprintf("KEY%08d", k)
2345
+ b.Delete([]byte(key))
2346
+ }
2347
+ if err := h.db.Write(b, nil); err != nil {
2348
+ t.Fatalf("WRITE #%d error: %v", i, err)
2349
+ }
2350
+ }
2351
+
2352
+ h.compactMem()
2353
+
2354
+ h.waitCompaction()
2355
+ for level, tables := range h.db.s.stVersion.tables {
2356
+ for _, table := range tables {
2357
+ t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2358
+ }
2359
+ }
2360
+
2361
+ h.compactRangeAt(0, "", "")
2362
+ h.waitCompaction()
2363
+ for level, tables := range h.db.s.stVersion.tables {
2364
+ for _, table := range tables {
2365
+ t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2366
+ }
2367
+ }
2368
+ h.compactRangeAt(1, "", "")
2369
+ h.waitCompaction()
2370
+ for level, tables := range h.db.s.stVersion.tables {
2371
+ for _, table := range tables {
2372
+ t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2373
+ }
2374
+ }
2375
+ runtime.GOMAXPROCS(runtime.NumCPU())
2376
+
2377
+ wg := &sync.WaitGroup{}
2378
+ for i, snap := range snaps {
2379
+ wg.Add(1)
2380
+
2381
+ go func(i int, snap *Snapshot) {
2382
+ defer wg.Done()
2383
+
2384
+ vtail := fmt.Sprintf("VAL%030d", i)
2385
+ for k := 0; k < nKey; k++ {
2386
+ key := fmt.Sprintf("KEY%08d", k)
2387
+ xvalue, err := snap.Get([]byte(key), nil)
2388
+ if err != nil {
2389
+ t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err)
2390
+ }
2391
+ value := key + vtail
2392
+ if !bytes.Equal([]byte(value), xvalue) {
2393
+ t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue)
2394
+ }
2395
+ }
2396
+ }(i, snap)
2397
+ }
2398
+
2399
+ wg.Wait()
2400
+}
2401
+
2402
+func TestDB_TableCompactionBuilder(t *testing.T) {
2403
+ stor := newTestStorage(t)
2404
+ defer stor.Close()
2405
+
2406
+ const nSeq = 99
2407
+
2408
+ o := &opt.Options{
2409
+ WriteBuffer: 112 * opt.KiB,
2410
+ CompactionTableSize: 43 * opt.KiB,
2411
+ CompactionExpandLimitFactor: 1,
2412
+ CompactionGPOverlapsFactor: 1,
2413
+ BlockCache: opt.NoCache,
2414
+ }
2415
+ s, err := newSession(stor, o)
2416
+ if err != nil {
2417
+ t.Fatal(err)
2418
+ }
2419
+ if err := s.create(); err != nil {
2420
+ t.Fatal(err)
2421
+ }
2422
+ defer s.close()
2423
+ var (
2424
+ seq uint64
2425
+ targetSize = 5 * o.CompactionTableSize
2426
+ value = bytes.Repeat([]byte{'0'}, 100)
2427
+ )
2428
+ for i := 0; i < 2; i++ {
2429
+ tw, err := s.tops.create()
2430
+ if err != nil {
2431
+ t.Fatal(err)
2432
+ }
2433
+ for k := 0; tw.tw.BytesLen() < targetSize; k++ {
2434
+ key := []byte(fmt.Sprintf("%09d", k))
2435
+ seq += nSeq - 1
2436
+ for x := uint64(0); x < nSeq; x++ {
2437
+ if err := tw.append(newIkey(key, seq-x, ktVal), value); err != nil {
2438
+ t.Fatal(err)
2439
+ }
2440
+ }
2441
+ }
2442
+ tf, err := tw.finish()
2443
+ if err != nil {
2444
+ t.Fatal(err)
2445
+ }
2446
+ rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
2447
+ rec.addTableFile(i, tf)
2448
+ if err := s.commit(rec); err != nil {
2449
+ t.Fatal(err)
2450
+ }
2451
+ }
2452
+
2453
+ // Build grandparent.
2454
+ v := s.version()
2455
+ c := newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
2456
+ rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
2457
+ b := &tableCompactionBuilder{
2458
+ s: s,
2459
+ c: c,
2460
+ rec: rec,
2461
+ stat1: new(cStatsStaging),
2462
+ minSeq: 0,
2463
+ strict: true,
2464
+ tableSize: o.CompactionTableSize/3 + 961,
2465
+ }
2466
+ if err := b.run(new(compactionTransactCounter)); err != nil {
2467
+ t.Fatal(err)
2468
+ }
2469
+ for _, t := range c.tables[0] {
2470
+ rec.delTable(c.level, t.file.Num())
2471
+ }
2472
+ if err := s.commit(rec); err != nil {
2473
+ t.Fatal(err)
2474
+ }
2475
+ c.release()
2476
+
2477
+ // Build level-1.
2478
+ v = s.version()
2479
+ c = newCompaction(s, v, 0, append(tFiles{}, v.tables[0]...))
2480
+ rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
2481
+ b = &tableCompactionBuilder{
2482
+ s: s,
2483
+ c: c,
2484
+ rec: rec,
2485
+ stat1: new(cStatsStaging),
2486
+ minSeq: 0,
2487
+ strict: true,
2488
+ tableSize: o.CompactionTableSize,
2489
+ }
2490
+ if err := b.run(new(compactionTransactCounter)); err != nil {
2491
+ t.Fatal(err)
2492
+ }
2493
+ for _, t := range c.tables[0] {
2494
+ rec.delTable(c.level, t.file.Num())
2495
+ }
2496
+ // Move grandparent to level-3
2497
+ for _, t := range v.tables[2] {
2498
+ rec.delTable(2, t.file.Num())
2499
+ rec.addTableFile(3, t)
2500
+ }
2501
+ if err := s.commit(rec); err != nil {
2502
+ t.Fatal(err)
2503
+ }
2504
+ c.release()
2505
+
2506
+ v = s.version()
2507
+ for level, want := range []bool{false, true, false, true, false} {
2508
+ got := len(v.tables[level]) > 0
2509
+ if want != got {
2510
+ t.Fatalf("invalid level-%d tables len: want %v, got %v", level, want, got)
2511
+ }
2512
+ }
2513
+ for i, f := range v.tables[1][:len(v.tables[1])-1] {
2514
+ nf := v.tables[1][i+1]
2515
+ if bytes.Equal(f.imax.ukey(), nf.imin.ukey()) {
2516
+ t.Fatalf("KEY %q hop across table %d .. %d", f.imax.ukey(), f.file.Num(), nf.file.Num())
2517
+ }
2518
+ }
2519
+ v.release()
2520
+
2521
+ // Compaction with transient error.
2522
+ v = s.version()
2523
+ c = newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
2524
+ rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
2525
+ b = &tableCompactionBuilder{
2526
+ s: s,
2527
+ c: c,
2528
+ rec: rec,
2529
+ stat1: new(cStatsStaging),
2530
+ minSeq: 0,
2531
+ strict: true,
2532
+ tableSize: o.CompactionTableSize,
2533
+ }
2534
+ stor.SetEmuErrOnce(storage.TypeTable, tsOpSync)
2535
+ stor.SetEmuRandErr(storage.TypeTable, tsOpRead, tsOpReadAt, tsOpWrite)
2536
+ stor.SetEmuRandErrProb(0xf0)
2537
+ for {
2538
+ if err := b.run(new(compactionTransactCounter)); err != nil {
2539
+ t.Logf("(expected) b.run: %v", err)
2540
+ } else {
2541
+ break
2542
+ }
2543
+ }
2544
+ if err := s.commit(rec); err != nil {
2545
+ t.Fatal(err)
2546
+ }
2547
+ c.release()
2548
+
2549
+ stor.SetEmuErrOnce(0, tsOpSync)
2550
+ stor.SetEmuRandErr(0, tsOpRead, tsOpReadAt, tsOpWrite)
2551
+
2552
+ v = s.version()
2553
+ if len(v.tables[1]) != len(v.tables[2]) {
2554
+ t.Fatalf("invalid tables length, want %d, got %d", len(v.tables[1]), len(v.tables[2]))
2555
+ }
2556
+ for i, f0 := range v.tables[1] {
2557
+ f1 := v.tables[2][i]
2558
+ iter0 := s.tops.newIterator(f0, nil, nil)
2559
+ iter1 := s.tops.newIterator(f1, nil, nil)
2560
+ for j := 0; true; j++ {
2561
+ next0 := iter0.Next()
2562
+ next1 := iter1.Next()
2563
+ if next0 != next1 {
2564
+ t.Fatalf("#%d.%d invalid eoi: want %v, got %v", i, j, next0, next1)
2565
+ }
2566
+ key0 := iter0.Key()
2567
+ key1 := iter1.Key()
2568
+ if !bytes.Equal(key0, key1) {
2569
+ t.Fatalf("#%d.%d invalid key: want %q, got %q", i, j, key0, key1)
2570
+ }
2571
+ if next0 == false {
2572
+ break
2573
+ }
2574
+ }
2575
+ iter0.Release()
2576
+ iter1.Release()
2577
+ }
2578
+ v.release()
2579
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go
+7
-4
@@ -7,8 +7,7 @@
7
package leveldb
8
9
import (
10
- "errors"
11
-
10
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
12
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
13
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
@@ -38,7 +37,9 @@ func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) }
37
38
// Check and clean files.
39
func (db *DB) checkAndCleanFiles() error {
41
- v := db.s.version_NB()
40
+ v := db.s.version()
41
+ defer v.release()
42
+
43
tablesMap := make(map[uint64]bool)
44
for _, tables := range v.tables {
45
for _, t := range tables {
@@ -78,12 +79,14 @@ func (db *DB) checkAndCleanFiles() error {
79
}
80
81
if nTables != len(tablesMap) {
82
+ var missing []*storage.FileInfo
83
for num, present := range tablesMap {
84
if !present {
85
+ missing = append(missing, &storage.FileInfo{Type: storage.TypeTable, Num: num})
86
db.logf("db@janitor table missing @%d", num)
87
}
88
}
86
- return ErrCorrupted{Type: MissingFiles, Err: errors.New("leveldb: table files missing")}
89
+ return errors.NewErrCorrupted(nil, &errors.ErrMissingFiles{Files: missing})
90
}
91
92
db.logf("db@janitor F·%d G·%d", len(files), len(rem))
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go
+39
-18
@@ -59,7 +59,7 @@ func (db *DB) rotateMem(n int) (mem *memDB, err error) {
59
}
60
61
// Schedule memdb compaction.
62
- db.compTrigger(db.mcompTriggerC)
62
+ db.compSendTrigger(db.mcompCmdC)
63
return
64
}
65
@@ -77,12 +77,12 @@ func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
77
}()
78
nn = mem.mdb.Free()
79
switch {
80
- case v.tLen(0) >= kL0_SlowdownWritesTrigger && !delayed:
80
+ case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed:
81
delayed = true
82
time.Sleep(time.Millisecond)
83
case nn >= n:
84
return false
85
- case v.tLen(0) >= kL0_StopWritesTrigger:
85
+ case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger():
86
delayed = true
87
err = db.compSendIdle(db.tcompCmdC)
88
if err != nil {
@@ -109,7 +109,12 @@ func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
109
for flush() {
110
}
111
if delayed {
112
- db.logf("db@write delayed T·%v", time.Since(start))
112
+ db.writeDelay += time.Since(start)
113
+ db.writeDelayN++
114
+ } else if db.writeDelayN > 0 {
115
+ db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
116
+ db.writeDelay = 0
117
+ db.writeDelayN = 0
118
}
119
return
120
}
@@ -120,28 +125,33 @@ func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
125
// It is safe to modify the contents of the arguments after Write returns.
126
func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
127
err = db.ok()
123
- if err != nil || b == nil || b.len() == 0 {
128
+ if err != nil || b == nil || b.Len() == 0 {
129
return
130
}
131
132
b.init(wo.GetSync())
133
134
// The write happen synchronously.
130
-retry:
135
select {
136
case db.writeC <- b:
137
if <-db.writeMergedC {
138
return <-db.writeAckC
139
}
136
- goto retry
140
case db.writeLockC <- struct{}{}:
141
+ case err = <-db.compPerErrC:
142
+ return
143
case _, _ = <-db.closeC:
144
return ErrClosed
145
}
146
147
merged := 0
148
+ danglingMerge := false
149
defer func() {
144
- <-db.writeLockC
150
+ if danglingMerge {
151
+ db.writeMergedC <- false
152
+ } else {
153
+ <-db.writeLockC
154
+ }
155
for i := 0; i < merged; i++ {
156
db.writeAckC <- err
157
}
@@ -170,7 +180,7 @@ drain:
180
db.writeMergedC <- true
181
merged++
182
} else {
173
- db.writeMergedC <- false
183
+ danglingMerge = true
184
break drain
185
}
186
default:
@@ -185,35 +195,43 @@ drain:
195
if b.size() >= (128 << 10) {
196
// Push the write batch to the journal writer
197
select {
198
+ case db.journalC <- b:
199
+ // Write into memdb
200
+ if berr := b.memReplay(mem.mdb); berr != nil {
201
+ panic(berr)
202
+ }
203
+ case err = <-db.compPerErrC:
204
+ return
205
case _, _ = <-db.closeC:
206
err = ErrClosed
207
return
191
- case db.journalC <- b:
192
- // Write into memdb
193
- b.memReplay(mem.mdb)
208
}
209
// Wait for journal writer
210
select {
197
- case _, _ = <-db.closeC:
198
- err = ErrClosed
199
- return
211
case err = <-db.journalAckC:
212
if err != nil {
213
// Revert memdb if error detected
203
- b.revertMemReplay(mem.mdb)
214
+ if berr := b.revertMemReplay(mem.mdb); berr != nil {
215
+ panic(berr)
216
+ }
217
return
218
}
219
+ case _, _ = <-db.closeC:
220
+ err = ErrClosed
221
+ return
222
}
223
} else {
224
err = db.writeJournal(b)
225
if err != nil {
226
return
227
}
212
- b.memReplay(mem.mdb)
228
+ if berr := b.memReplay(mem.mdb); berr != nil {
229
+ panic(berr)
230
+ }
231
}
232
233
// Set last seq number.
216
- db.addSeq(uint64(b.len()))
234
+ db.addSeq(uint64(b.Len()))
235
236
if b.size() >= memFree {
237
db.rotateMem(0)
@@ -262,8 +280,11 @@ func (db *DB) CompactRange(r util.Range) error {
280
return err
281
}
282
283
+ // Lock writer.
284
select {
285
case db.writeLockC <- struct{}{}:
286
+ case err := <-db.compPerErrC:
287
+ return err
288
case _, _ = <-db.closeC:
289
return ErrClosed
290
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go
renamed
+2
-22
@@ -7,32 +7,12 @@
7
package leveldb
8
9
import (
10
- "errors"
11
-
12
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
10
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11
)
12
13
var (
16
- ErrNotFound = util.ErrNotFound
14
+ ErrNotFound = errors.ErrNotFound
15
ErrSnapshotReleased = errors.New("leveldb: snapshot released")
16
ErrIterReleased = errors.New("leveldb: iterator released")
17
ErrClosed = errors.New("leveldb: closed")
18
)
21
-
22
-type CorruptionType int
23
-
24
-const (
25
- CorruptedManifest CorruptionType = iota
26
- MissingFiles
27
-)
28
-
29
-// ErrCorrupted is the type that wraps errors that indicate corruption in
30
-// the database.
31
-type ErrCorrupted struct {
32
- Type CorruptionType
33
- Err error
34
-}
35
-
36
-func (e ErrCorrupted) Error() string {
37
- return e.Err.Error()
38
-}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go
new
+76
@@ -0,0 +1,76 @@
1
+// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2
+// All rights reserved.
3
+//
4
+// Use of this source code is governed by a BSD-style license that can be
5
+// found in the LICENSE file.
6
+
7
+// Package errors provides common error types used throughout leveldb.
8
+package errors
9
+
10
+import (
11
+ "errors"
12
+ "fmt"
13
+
14
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16
+)
17
+
18
+var (
19
+ ErrNotFound = New("leveldb: not found")
20
+ ErrReleased = util.ErrReleased
21
+ ErrHasReleaser = util.ErrHasReleaser
22
+)
23
+
24
+// New returns an error that formats as the given text.
25
+func New(text string) error {
26
+ return errors.New(text)
27
+}
28
+
29
+// ErrCorrupted is the type that wraps errors that indicate corruption in
30
+// the database.
31
+type ErrCorrupted struct {
32
+ File *storage.FileInfo
33
+ Err error
34
+}
35
+
36
+func (e *ErrCorrupted) Error() string {
37
+ if e.File != nil {
38
+ return fmt.Sprintf("%v [file=%v]", e.Err, e.File)
39
+ } else {
40
+ return e.Err.Error()
41
+ }
42
+}
43
+
44
+// NewErrCorrupted creates new ErrCorrupted error.
45
+func NewErrCorrupted(f storage.File, err error) error {
46
+ return &ErrCorrupted{storage.NewFileInfo(f), err}
47
+}
48
+
49
+// IsCorrupted returns a boolean indicating whether the error is indicating
50
+// a corruption.
51
+func IsCorrupted(err error) bool {
52
+ switch err.(type) {
53
+ case *ErrCorrupted:
54
+ return true
55
+ }
56
+ return false
57
+}
58
+
59
+// ErrMissingFiles is the type that indicating a corruption due to missing
60
+// files.
61
+type ErrMissingFiles struct {
62
+ Files []*storage.FileInfo
63
+}
64
+
65
+func (e *ErrMissingFiles) Error() string { return "file missing" }
66
+
67
+// SetFile sets 'file info' of the given error with the given file.
68
+// Currently only ErrCorrupted is supported, otherwise will do nothing.
69
+func SetFile(err error, f storage.File) error {
70
+ switch x := err.(type) {
71
+ case *ErrCorrupted:
72
+ x.File = storage.NewFileInfo(f)
73
+ return x
74
+ }
75
+ return err
76
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go
+5
-5
@@ -19,11 +19,12 @@ var _ = testutil.Defer(func() {
19
o := &opt.Options{
20
BlockCache: opt.NoCache,
21
BlockRestartInterval: 5,
22
- BlockSize: 50,
22
+ BlockSize: 80,
23
Compression: opt.NoCompression,
24
CachedOpenFiles: -1,
25
Strict: opt.StrictAll,
26
WriteBuffer: 1000,
27
+ CompactionTableSize: 2000,
28
}
29
30
Describe("write test", func() {
@@ -40,18 +41,17 @@ var _ = testutil.Defer(func() {
41
})
42
43
Describe("read test", func() {
43
- testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
44
+ testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
45
// Building the DB.
46
db := newTestingDB(o, nil, nil)
47
kv.IterateShuffled(nil, func(i int, key, value []byte) {
48
err := db.TestPut(key, value)
49
Expect(err).NotTo(HaveOccurred())
50
})
50
- testutil.Defer("teardown", func() {
51
- db.TestClose()
52
- })
51
52
return db
53
+ }, func(db testutil.DB) {
54
+ db.(*testingDB).TestClose()
55
})
56
})
57
})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go
+28
-2
@@ -40,13 +40,19 @@ type basicArrayIterator struct {
40
util.BasicReleaser
41
array BasicArray
42
pos int
43
+ err error
44
}
45
46
func (i *basicArrayIterator) Valid() bool {
46
- return i.pos >= 0 && i.pos < i.array.Len()
47
+ return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
48
}
49
50
func (i *basicArrayIterator) First() bool {
51
+ if i.Released() {
52
+ i.err = ErrIterReleased
53
+ return false
54
+ }
55
+
56
if i.array.Len() == 0 {
57
i.pos = -1
58
return false
@@ -56,6 +62,11 @@ func (i *basicArrayIterator) First() bool {
62
}
63
64
func (i *basicArrayIterator) Last() bool {
65
+ if i.Released() {
66
+ i.err = ErrIterReleased
67
+ return false
68
+ }
69
+
70
n := i.array.Len()
71
if n == 0 {
72
i.pos = 0
@@ -66,6 +77,11 @@ func (i *basicArrayIterator) Last() bool {
77
}
78
79
func (i *basicArrayIterator) Seek(key []byte) bool {
80
+ if i.Released() {
81
+ i.err = ErrIterReleased
82
+ return false
83
+ }
84
+
85
n := i.array.Len()
86
if n == 0 {
87
i.pos = 0
@@ -79,6 +95,11 @@ func (i *basicArrayIterator) Seek(key []byte) bool {
95
}
96
97
func (i *basicArrayIterator) Next() bool {
98
+ if i.Released() {
99
+ i.err = ErrIterReleased
100
+ return false
101
+ }
102
+
103
i.pos++
104
if n := i.array.Len(); i.pos >= n {
105
i.pos = n
@@ -88,6 +109,11 @@ func (i *basicArrayIterator) Next() bool {
109
}
110
111
func (i *basicArrayIterator) Prev() bool {
112
+ if i.Released() {
113
+ i.err = ErrIterReleased
114
+ return false
115
+ }
116
+
117
i.pos--
118
if i.pos < 0 {
119
i.pos = -1
@@ -96,7 +122,7 @@ func (i *basicArrayIterator) Prev() bool {
122
return true
123
}
124
99
-func (i *basicArrayIterator) Error() error { return nil }
125
+func (i *basicArrayIterator) Error() error { return i.err }
126
127
type arrayIterator struct {
128
basicArrayIterator
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go
+47
-26
@@ -7,6 +7,7 @@
7
package iterator
8
9
import (
10
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
12
)
13
@@ -22,13 +23,13 @@ type IteratorIndexer interface {
23
24
type indexedIterator struct {
25
util.BasicReleaser
25
- index IteratorIndexer
26
- strict bool
27
- strictGet bool
26
+ index IteratorIndexer
27
+ strict bool
28
29
- data Iterator
30
- err error
31
- errf func(err error)
29
+ data Iterator
30
+ err error
31
+ errf func(err error)
32
+ closed bool
33
}
34
35
func (i *indexedIterator) setData() {
@@ -36,11 +37,6 @@ func (i *indexedIterator) setData() {
37
i.data.Release()
38
}
39
i.data = i.index.Get()
39
- if i.strictGet {
40
- if err := i.data.Error(); err != nil {
41
- i.err = err
42
- }
43
- }
40
}
41
42
func (i *indexedIterator) clearData() {
@@ -50,14 +46,21 @@ func (i *indexedIterator) clearData() {
46
i.data = nil
47
}
48
53
-func (i *indexedIterator) dataErr() bool {
54
- if i.errf != nil {
55
- if err := i.data.Error(); err != nil {
49
+func (i *indexedIterator) indexErr() {
50
+ if err := i.index.Error(); err != nil {
51
+ if i.errf != nil {
52
i.errf(err)
53
}
54
+ i.err = err
55
}
59
- if i.strict {
60
- if err := i.data.Error(); err != nil {
56
+}
57
+
58
+func (i *indexedIterator) dataErr() bool {
59
+ if err := i.data.Error(); err != nil {
60
+ if i.errf != nil {
61
+ i.errf(err)
62
+ }
63
+ if i.strict || !errors.IsCorrupted(err) {
64
i.err = err
65
return true
66
}
@@ -72,9 +75,13 @@ func (i *indexedIterator) Valid() bool {
75
func (i *indexedIterator) First() bool {
76
if i.err != nil {
77
return false
78
+ } else if i.Released() {
79
+ i.err = ErrIterReleased
80
+ return false
81
}
82
83
if !i.index.First() {
84
+ i.indexErr()
85
i.clearData()
86
return false
87
}
@@ -85,9 +92,13 @@ func (i *indexedIterator) First() bool {
92
func (i *indexedIterator) Last() bool {
93
if i.err != nil {
94
return false
95
+ } else if i.Released() {
96
+ i.err = ErrIterReleased
97
+ return false
98
}
99
100
if !i.index.Last() {
101
+ i.indexErr()
102
i.clearData()
103
return false
104
}
@@ -105,9 +116,13 @@ func (i *indexedIterator) Last() bool {
116
func (i *indexedIterator) Seek(key []byte) bool {
117
if i.err != nil {
118
return false
119
+ } else if i.Released() {
120
+ i.err = ErrIterReleased
121
+ return false
122
}
123
124
if !i.index.Seek(key) {
125
+ i.indexErr()
126
i.clearData()
127
return false
128
}
@@ -125,6 +140,9 @@ func (i *indexedIterator) Seek(key []byte) bool {
140
func (i *indexedIterator) Next() bool {
141
if i.err != nil {
142
return false
143
+ } else if i.Released() {
144
+ i.err = ErrIterReleased
145
+ return false
146
}
147
148
switch {
@@ -136,6 +154,7 @@ func (i *indexedIterator) Next() bool {
154
fallthrough
155
case i.data == nil:
156
if !i.index.Next() {
157
+ i.indexErr()
158
return false
159
}
160
i.setData()
@@ -147,6 +166,9 @@ func (i *indexedIterator) Next() bool {
166
func (i *indexedIterator) Prev() bool {
167
if i.err != nil {
168
return false
169
+ } else if i.Released() {
170
+ i.err = ErrIterReleased
171
+ return false
172
}
173
174
switch {
@@ -158,6 +180,7 @@ func (i *indexedIterator) Prev() bool {
180
fallthrough
181
case i.data == nil:
182
if !i.index.Prev() {
183
+ i.indexErr()
184
return false
185
}
186
i.setData()
@@ -206,16 +229,14 @@ func (i *indexedIterator) SetErrorCallback(f func(err error)) {
229
i.errf = f
230
}
231
209
-// NewIndexedIterator returns an indexed iterator. An index is iterator
210
-// that returns another iterator, a data iterator. A data iterator is the
232
+// NewIndexedIterator returns an 'indexed iterator'. An index is iterator
233
+// that returns another iterator, a 'data iterator'. A 'data iterator' is the
234
// iterator that contains actual key/value pairs.
235
//
213
-// If strict is true then error yield by data iterator will halt the indexed
214
-// iterator, on contrary if strict is false then the indexed iterator will
215
-// ignore those error and move on to the next index. If strictGet is true and
216
-// index.Get() yield an 'error iterator' then the indexed iterator will be halted.
217
-// An 'error iterator' is iterator which its Error() method always return non-nil
218
-// even before any 'seeks method' is called.
219
-func NewIndexedIterator(index IteratorIndexer, strict, strictGet bool) Iterator {
220
- return &indexedIterator{index: index, strict: strict, strictGet: strictGet}
236
+// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
237
+// won't be ignored and will halt 'indexed iterator', otherwise the iterator will
238
+// continue to the next 'data iterator'. Corruption on 'index iterator' will not be
239
+// ignored and will halt the iterator.
240
+func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator {
241
+ return &indexedIterator{index: index, strict: strict}
242
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go
+1
-1
@@ -65,7 +65,7 @@ var _ = testutil.Defer(func() {
65
// Test the iterator.
66
t := testutil.IteratorTesting{
67
KeyValue: kv.Clone(),
68
- Iter: NewIndexedIterator(NewArrayIndexer(index), true, true),
68
+ Iter: NewIndexedIterator(NewArrayIndexer(index), true),
69
}
70
testutil.DoIteratorTesting(&t)
71
done <- true
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go
+8
-19
@@ -14,6 +14,10 @@ import (
14
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
15
)
16
17
+var (
18
+ ErrIterReleased = errors.New("leveldb/iterator: iterator released")
19
+)
20
+
21
// IteratorSeeker is the interface that wraps the 'seeks method'.
22
type IteratorSeeker interface {
23
// First moves the iterator to the first key/value pair. If the iterator
@@ -100,28 +104,13 @@ type ErrorCallbackSetter interface {
104
}
105
106
type emptyIterator struct {
103
- releaser util.Releaser
104
- released bool
105
- err error
107
+ util.BasicReleaser
108
+ err error
109
}
110
111
func (i *emptyIterator) rErr() {
109
- if i.err == nil && i.released {
110
- i.err = errors.New("leveldb/iterator: iterator released")
111
- }
112
-}
113
-
114
-func (i *emptyIterator) Release() {
115
- if i.releaser != nil {
116
- i.releaser.Release()
117
- i.releaser = nil
118
- }
119
- i.released = true
120
-}
121
-
122
-func (i *emptyIterator) SetReleaser(releaser util.Releaser) {
123
- if !i.released {
124
- i.releaser = releaser
112
+ if i.err == nil && i.Released() {
113
+ i.err = ErrIterReleased
114
}
115
}
116
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go
+1
-7
@@ -3,15 +3,9 @@ package iterator_test
3
import (
4
"testing"
5
6
- . "github.com/onsi/ginkgo"
7
- . "github.com/onsi/gomega"
8
-
6
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7
)
8
9
func TestIterator(t *testing.T) {
13
- testutil.RunDefer()
14
-
15
- RegisterFailHandler(Fail)
16
- RunSpecs(t, "Iterator Suite")
10
+ testutil.RunSuite(t, "Iterator Suite")
11
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go
+13
-16
@@ -7,16 +7,11 @@
7
package iterator
8
9
import (
10
- "errors"
11
-
10
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
11
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
12
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
13
)
14
16
-var (
17
- ErrIterReleased = errors.New("leveldb/iterator: iterator released")
18
-)
19
-
15
type dir int
16
17
const (
@@ -48,13 +43,11 @@ func assertKey(key []byte) []byte {
43
}
44
45
func (i *mergedIterator) iterErr(iter Iterator) bool {
51
- if i.errf != nil {
52
- if err := iter.Error(); err != nil {
46
+ if err := iter.Error(); err != nil {
47
+ if i.errf != nil {
48
i.errf(err)
49
}
55
- }
56
- if i.strict {
57
- if err := iter.Error(); err != nil {
50
+ if i.strict || !errors.IsCorrupted(err) {
51
i.err = err
52
return true
53
}
@@ -274,9 +267,13 @@ func (i *mergedIterator) Release() {
267
}
268
269
func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
277
- if i.dir != dirReleased {
278
- i.releaser = releaser
270
+ if i.dir == dirReleased {
271
+ panic(util.ErrReleased)
272
+ }
273
+ if i.releaser != nil && releaser != nil {
274
+ panic(util.ErrHasReleaser)
275
}
276
+ i.releaser = releaser
277
}
278
279
func (i *mergedIterator) Error() error {
@@ -294,9 +291,9 @@ func (i *mergedIterator) SetErrorCallback(f func(err error)) {
291
// keys: if iters[i] contains a key k then iters[j] will not contain that key k.
292
// None of the iters may be nil.
293
//
297
-// If strict is true then error yield by any iterators will halt the merged
298
-// iterator, on contrary if strict is false then the merged iterator will
299
-// ignore those error and move on to the next iterator.
294
+// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
295
+// won't be ignored and will halt 'merged iterator', otherwise the iterator will
296
+// continue to the next 'input iterator'.
297
func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator {
298
return &mergedIterator{
299
iters: iters,
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go
+4
-4
@@ -79,10 +79,10 @@ package journal
79
80
import (
81
"encoding/binary"
82
- "errors"
82
"fmt"
83
"io"
84
85
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
86
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
87
)
88
@@ -109,7 +109,7 @@ type ErrCorrupted struct {
109
Reason string
110
}
111
112
-func (e ErrCorrupted) Error() string {
112
+func (e *ErrCorrupted) Error() string {
113
return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
114
}
115
@@ -162,10 +162,10 @@ var errSkip = errors.New("leveldb/journal: skipped")
162
163
func (r *Reader) corrupt(n int, reason string, skip bool) error {
164
if r.dropper != nil {
165
- r.dropper.Drop(ErrCorrupted{n, reason})
165
+ r.dropper.Drop(&ErrCorrupted{n, reason})
166
}
167
if r.strict && !skip {
168
- r.err = ErrCorrupted{n, reason}
168
+ r.err = errors.NewErrCorrupted(nil, &ErrCorrupted{n, reason})
169
return r.err
170
}
171
return errSkip
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go
+68
-65
@@ -9,15 +9,30 @@ package leveldb
9
import (
10
"encoding/binary"
11
"fmt"
12
+
13
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14
)
15
14
-type vType int
16
+type ErrIkeyCorrupted struct {
17
+ Ikey []byte
18
+ Reason string
19
+}
20
+
21
+func (e *ErrIkeyCorrupted) Error() string {
22
+ return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason)
23
+}
24
+
25
+func newErrIkeyCorrupted(ikey []byte, reason string) error {
26
+ return errors.NewErrCorrupted(nil, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason})
27
+}
28
+
29
+type kType int
30
16
-func (t vType) String() string {
17
- switch t {
18
- case tDel:
31
+func (kt kType) String() string {
32
+ switch kt {
33
+ case ktDel:
34
return "d"
20
- case tVal:
35
+ case ktVal:
36
return "v"
37
}
38
return "x"
@@ -26,16 +41,16 @@ func (t vType) String() string {
41
// Value types encoded as the last component of internal keys.
42
// Don't modify; this value are saved to disk.
43
const (
29
- tDel vType = iota
30
- tVal
44
+ ktDel kType = iota
45
+ ktVal
46
)
47
33
-// tSeek defines the vType that should be passed when constructing an
48
+// ktSeek defines the kType that should be passed when constructing an
49
// internal key for seeking to a particular sequence number (since we
50
// sort sequence numbers in decreasing order and the value type is
51
// embedded as the low 8 bits in the sequence number in internal keys,
52
// we need to use the highest-numbered ValueType, not the lowest).
38
-const tSeek = tVal
53
+const ktSeek = ktVal
54
55
const (
56
// Maximum value possible for sequence number; the 8-bits are
@@ -43,7 +58,7 @@ const (
58
// 64-bit integer.
59
kMaxSeq uint64 = (uint64(1) << 56) - 1
60
// Maximum value possible for packed sequence number and type.
46
- kMaxNum uint64 = (kMaxSeq << 8) | uint64(tSeek)
61
+ kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek)
62
)
63
64
// Maximum number encoded in bytes.
@@ -55,85 +70,73 @@ func init() {
70
71
type iKey []byte
72
58
-func newIKey(ukey []byte, seq uint64, t vType) iKey {
59
- if seq > kMaxSeq || t > tVal {
60
- panic("invalid seq number or value type")
73
+func newIkey(ukey []byte, seq uint64, kt kType) iKey {
74
+ if seq > kMaxSeq {
75
+ panic("leveldb: invalid sequence number")
76
+ } else if kt > ktVal {
77
+ panic("leveldb: invalid type")
78
}
79
63
- b := make(iKey, len(ukey)+8)
64
- copy(b, ukey)
65
- binary.LittleEndian.PutUint64(b[len(ukey):], (seq<<8)|uint64(t))
66
- return b
80
+ ik := make(iKey, len(ukey)+8)
81
+ copy(ik, ukey)
82
+ binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt))
83
+ return ik
84
}
85
69
-func parseIkey(p []byte) (ukey []byte, seq uint64, t vType, ok bool) {
70
- if len(p) < 8 {
71
- return
86
+func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) {
87
+ if len(ik) < 8 {
88
+ return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length")
89
}
73
- num := binary.LittleEndian.Uint64(p[len(p)-8:])
74
- seq, t = uint64(num>>8), vType(num&0xff)
75
- if t > tVal {
76
- return
90
+ num := binary.LittleEndian.Uint64(ik[len(ik)-8:])
91
+ seq, kt = uint64(num>>8), kType(num&0xff)
92
+ if kt > ktVal {
93
+ return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type")
94
}
78
- ukey = p[:len(p)-8]
79
- ok = true
95
+ ukey = ik[:len(ik)-8]
96
return
97
}
98
83
-func validIkey(p []byte) bool {
84
- _, _, _, ok := parseIkey(p)
85
- return ok
99
+func validIkey(ik []byte) bool {
100
+ _, _, _, err := parseIkey(ik)
101
+ return err == nil
102
}
103
88
-func (p iKey) assert() {
89
- if p == nil {
90
- panic("nil iKey")
104
+func (ik iKey) assert() {
105
+ if ik == nil {
106
+ panic("leveldb: nil iKey")
107
}
92
- if len(p) < 8 {
93
- panic(fmt.Sprintf("invalid iKey %q, len=%d", []byte(p), len(p)))
108
+ if len(ik) < 8 {
109
+ panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", []byte(ik), len(ik)))
110
}
111
}
112
97
-func (p iKey) ok() bool {
98
- if len(p) < 8 {
99
- return false
100
- }
101
- _, _, ok := p.parseNum()
102
- return ok
103
-}
104
-
105
-func (p iKey) ukey() []byte {
106
- p.assert()
107
- return p[:len(p)-8]
113
+func (ik iKey) ukey() []byte {
114
+ ik.assert()
115
+ return ik[:len(ik)-8]
116
}
117
110
-func (p iKey) num() uint64 {
111
- p.assert()
112
- return binary.LittleEndian.Uint64(p[len(p)-8:])
118
+func (ik iKey) num() uint64 {
119
+ ik.assert()
120
+ return binary.LittleEndian.Uint64(ik[len(ik)-8:])
121
}
122
115
-func (p iKey) parseNum() (seq uint64, t vType, ok bool) {
116
- if p == nil {
117
- panic("nil iKey")
123
+func (ik iKey) parseNum() (seq uint64, kt kType) {
124
+ num := ik.num()
125
+ seq, kt = uint64(num>>8), kType(num&0xff)
126
+ if kt > ktVal {
127
+ panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt))
128
}
119
- if len(p) < 8 {
120
- return
121
- }
122
- num := p.num()
123
- seq, t = uint64(num>>8), vType(num&0xff)
124
- if t > tVal {
125
- return 0, 0, false
126
- }
127
- ok = true
129
return
130
}
131
131
-func (p iKey) String() string {
132
- if len(p) == 0 {
132
+func (ik iKey) String() string {
133
+ if ik == nil {
134
return "<nil>"
135
}
135
- if seq, t, ok := p.parseNum(); ok {
136
- return fmt.Sprintf("%s,%s%d", shorten(string(p.ukey())), t, seq)
136
+
137
+ if ukey, seq, kt, err := parseIkey(ik); err == nil {
138
+ return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq)
139
+ } else {
140
+ return "<invalid>"
141
}
138
- return "<invalid>"
142
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go
+52
-42
@@ -15,8 +15,8 @@ import (
15
16
var defaultIComparer = &iComparer{comparer.DefaultComparer}
17
18
-func ikey(key string, seq uint64, t vType) iKey {
19
- return newIKey([]byte(key), uint64(seq), t)
18
+func ikey(key string, seq uint64, kt kType) iKey {
19
+ return newIkey([]byte(key), uint64(seq), kt)
20
}
21
22
func shortSep(a, b []byte) []byte {
@@ -37,27 +37,37 @@ func shortSuccessor(b []byte) []byte {
37
return dst
38
}
39
40
-func testSingleKey(t *testing.T, key string, seq uint64, vt vType) {
41
- ik := ikey(key, seq, vt)
40
+func testSingleKey(t *testing.T, key string, seq uint64, kt kType) {
41
+ ik := ikey(key, seq, kt)
42
43
if !bytes.Equal(ik.ukey(), []byte(key)) {
44
t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
45
}
46
47
- if rseq, rt, ok := ik.parseNum(); ok {
47
+ rseq, rt := ik.parseNum()
48
+ if rseq != seq {
49
+ t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
50
+ }
51
+ if rt != kt {
52
+ t.Errorf("type does not equal, got %v, want %v", rt, kt)
53
+ }
54
+
55
+ if rukey, rseq, rt, kerr := parseIkey(ik); kerr == nil {
56
+ if !bytes.Equal(rukey, []byte(key)) {
57
+ t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
58
+ }
59
if rseq != seq {
60
t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
61
}
51
-
52
- if rt != vt {
53
- t.Errorf("type does not equal, got %v, want %v", rt, vt)
62
+ if rt != kt {
63
+ t.Errorf("type does not equal, got %v, want %v", rt, kt)
64
}
65
} else {
56
- t.Error("cannot parse seq and type")
66
+ t.Errorf("key error: %v", kerr)
67
}
68
}
69
60
-func TestIKey_EncodeDecode(t *testing.T) {
70
+func TestIkey_EncodeDecode(t *testing.T) {
71
keys := []string{"", "k", "hello", "longggggggggggggggggggggg"}
72
seqs := []uint64{
73
1, 2, 3,
@@ -67,8 +77,8 @@ func TestIKey_EncodeDecode(t *testing.T) {
77
}
78
for _, key := range keys {
79
for _, seq := range seqs {
70
- testSingleKey(t, key, seq, tVal)
71
- testSingleKey(t, "hello", 1, tDel)
80
+ testSingleKey(t, key, seq, ktVal)
81
+ testSingleKey(t, "hello", 1, ktDel)
82
}
83
}
84
}
@@ -79,45 +89,45 @@ func assertBytes(t *testing.T, want, got []byte) {
89
}
90
}
91
82
-func TestIKeyShortSeparator(t *testing.T) {
92
+func TestIkeyShortSeparator(t *testing.T) {
93
// When user keys are same
84
- assertBytes(t, ikey("foo", 100, tVal),
85
- shortSep(ikey("foo", 100, tVal),
86
- ikey("foo", 99, tVal)))
87
- assertBytes(t, ikey("foo", 100, tVal),
88
- shortSep(ikey("foo", 100, tVal),
89
- ikey("foo", 101, tVal)))
90
- assertBytes(t, ikey("foo", 100, tVal),
91
- shortSep(ikey("foo", 100, tVal),
92
- ikey("foo", 100, tVal)))
93
- assertBytes(t, ikey("foo", 100, tVal),
94
- shortSep(ikey("foo", 100, tVal),
95
- ikey("foo", 100, tDel)))
94
+ assertBytes(t, ikey("foo", 100, ktVal),
95
+ shortSep(ikey("foo", 100, ktVal),
96
+ ikey("foo", 99, ktVal)))
97
+ assertBytes(t, ikey("foo", 100, ktVal),
98
+ shortSep(ikey("foo", 100, ktVal),
99
+ ikey("foo", 101, ktVal)))
100
+ assertBytes(t, ikey("foo", 100, ktVal),
101
+ shortSep(ikey("foo", 100, ktVal),
102
+ ikey("foo", 100, ktVal)))
103
+ assertBytes(t, ikey("foo", 100, ktVal),
104
+ shortSep(ikey("foo", 100, ktVal),
105
+ ikey("foo", 100, ktDel)))
106
107
// When user keys are misordered
98
- assertBytes(t, ikey("foo", 100, tVal),
99
- shortSep(ikey("foo", 100, tVal),
100
- ikey("bar", 99, tVal)))
108
+ assertBytes(t, ikey("foo", 100, ktVal),
109
+ shortSep(ikey("foo", 100, ktVal),
110
+ ikey("bar", 99, ktVal)))
111
112
// When user keys are different, but correctly ordered
103
- assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
104
- shortSep(ikey("foo", 100, tVal),
105
- ikey("hello", 200, tVal)))
113
+ assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
114
+ shortSep(ikey("foo", 100, ktVal),
115
+ ikey("hello", 200, ktVal)))
116
117
// When start user key is prefix of limit user key
108
- assertBytes(t, ikey("foo", 100, tVal),
109
- shortSep(ikey("foo", 100, tVal),
110
- ikey("foobar", 200, tVal)))
118
+ assertBytes(t, ikey("foo", 100, ktVal),
119
+ shortSep(ikey("foo", 100, ktVal),
120
+ ikey("foobar", 200, ktVal)))
121
122
// When limit user key is prefix of start user key
113
- assertBytes(t, ikey("foobar", 100, tVal),
114
- shortSep(ikey("foobar", 100, tVal),
115
- ikey("foo", 200, tVal)))
123
+ assertBytes(t, ikey("foobar", 100, ktVal),
124
+ shortSep(ikey("foobar", 100, ktVal),
125
+ ikey("foo", 200, ktVal)))
126
}
127
118
-func TestIKeyShortestSuccessor(t *testing.T) {
119
- assertBytes(t, ikey("g", uint64(kMaxSeq), tSeek),
120
- shortSuccessor(ikey("foo", 100, tVal)))
121
- assertBytes(t, ikey("\xff\xff", 100, tVal),
122
- shortSuccessor(ikey("\xff\xff", 100, tVal)))
128
+func TestIkeyShortestSuccessor(t *testing.T) {
129
+ assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
130
+ shortSuccessor(ikey("foo", 100, ktVal)))
131
+ assertBytes(t, ikey("\xff\xff", 100, ktVal),
132
+ shortSuccessor(ikey("\xff\xff", 100, ktVal)))
133
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go
+2
-11
@@ -3,18 +3,9 @@ package leveldb
3
import (
4
"testing"
5
6
- . "github.com/onsi/ginkgo"
7
- . "github.com/onsi/gomega"
8
-
6
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7
)
8
12
-func TestLeveldb(t *testing.T) {
13
- testutil.RunDefer()
14
-
15
- RegisterFailHandler(Fail)
16
- RunSpecs(t, "Leveldb Suite")
17
-
18
- RegisterTestingT(t)
19
- testutil.RunDefer("teardown")
9
+func TestLevelDB(t *testing.T) {
10
+ testutil.RunSuite(t, "LevelDB Suite")
11
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go
+23
-7
@@ -12,12 +12,14 @@ import (
12
"sync"
13
14
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18
)
19
20
var (
20
- ErrNotFound = util.ErrNotFound
21
+ ErrNotFound = errors.ErrNotFound
22
+ ErrIterReleased = errors.New("leveldb/memdb: iterator released")
23
)
24
25
const tMaxHeight = 12
@@ -29,6 +31,7 @@ type dbIter struct {
31
node int
32
forward bool
33
key, value []byte
34
+ err error
35
}
36
37
func (i *dbIter) fill(checkStart, checkLimit bool) bool {
@@ -59,6 +62,11 @@ func (i *dbIter) Valid() bool {
62
}
63
64
func (i *dbIter) First() bool {
65
+ if i.Released() {
66
+ i.err = ErrIterReleased
67
+ return false
68
+ }
69
+
70
i.forward = true
71
i.p.mu.RLock()
72
defer i.p.mu.RUnlock()
@@ -71,9 +79,11 @@ func (i *dbIter) First() bool {
79
}
80
81
func (i *dbIter) Last() bool {
74
- if i.p == nil {
82
+ if i.Released() {
83
+ i.err = ErrIterReleased
84
return false
85
}
86
+
87
i.forward = false
88
i.p.mu.RLock()
89
defer i.p.mu.RUnlock()
@@ -86,9 +96,11 @@ func (i *dbIter) Last() bool {
96
}
97
98
func (i *dbIter) Seek(key []byte) bool {
89
- if i.p == nil {
99
+ if i.Released() {
100
+ i.err = ErrIterReleased
101
return false
102
}
103
+
104
i.forward = true
105
i.p.mu.RLock()
106
defer i.p.mu.RUnlock()
@@ -100,9 +112,11 @@ func (i *dbIter) Seek(key []byte) bool {
112
}
113
114
func (i *dbIter) Next() bool {
103
- if i.p == nil {
115
+ if i.Released() {
116
+ i.err = ErrIterReleased
117
return false
118
}
119
+
120
if i.node == 0 {
121
if !i.forward {
122
return i.First()
@@ -117,9 +131,11 @@ func (i *dbIter) Next() bool {
131
}
132
133
func (i *dbIter) Prev() bool {
120
- if i.p == nil {
134
+ if i.Released() {
135
+ i.err = ErrIterReleased
136
return false
137
}
138
+
139
if i.node == 0 {
140
if i.forward {
141
return i.Last()
@@ -141,10 +157,10 @@ func (i *dbIter) Value() []byte {
157
return i.value
158
}
159
144
-func (i *dbIter) Error() error { return nil }
160
+func (i *dbIter) Error() error { return i.err }
161
162
func (i *dbIter) Release() {
147
- if i.p != nil {
163
+ if !i.Released() {
164
i.p = nil
165
i.node = 0
166
i.key = nil
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go
+2
-8
@@ -3,15 +3,9 @@ package memdb
3
import (
4
"testing"
5
6
- . "github.com/onsi/ginkgo"
7
- . "github.com/onsi/gomega"
8
-
6
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7
)
8
12
-func TestMemdb(t *testing.T) {
13
- testutil.RunDefer()
14
-
15
- RegisterFailHandler(Fail)
16
- RunSpecs(t, "Memdb Suite")
9
+func TestMemDB(t *testing.T) {
10
+ testutil.RunSuite(t, "MemDB Suite")
11
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go
+1
-1
@@ -129,7 +129,7 @@ var _ = testutil.Defer(func() {
129
}
130
131
return db
132
- })
132
+ }, nil, nil)
133
})
134
})
135
})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go
+279
-20
@@ -11,6 +11,7 @@ import (
11
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
12
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
13
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
14
+ "math"
15
)
16
17
const (
@@ -20,12 +21,24 @@ const (
21
)
22
23
const (
23
- DefaultBlockCacheSize = 8 * MiB
24
- DefaultBlockRestartInterval = 16
25
- DefaultBlockSize = 4 * KiB
26
- DefaultCompressionType = SnappyCompression
27
- DefaultCachedOpenFiles = 500
28
- DefaultWriteBuffer = 4 * MiB
24
+ DefaultBlockCacheSize = 8 * MiB
25
+ DefaultBlockRestartInterval = 16
26
+ DefaultBlockSize = 4 * KiB
27
+ DefaultCompactionExpandLimitFactor = 25
28
+ DefaultCompactionGPOverlapsFactor = 10
29
+ DefaultCompactionL0Trigger = 4
30
+ DefaultCompactionSourceLimitFactor = 1
31
+ DefaultCompactionTableSize = 2 * MiB
32
+ DefaultCompactionTableSizeMultiplier = 1.0
33
+ DefaultCompactionTotalSize = 10 * MiB
34
+ DefaultCompactionTotalSizeMultiplier = 10.0
35
+ DefaultCompressionType = SnappyCompression
36
+ DefaultCachedOpenFiles = 500
37
+ DefaultMaxMemCompationLevel = 2
38
+ DefaultNumLevel = 7
39
+ DefaultWriteBuffer = 4 * MiB
40
+ DefaultWriteL0PauseTrigger = 12
41
+ DefaultWriteL0SlowdownTrigger = 8
42
)
43
44
type noCache struct{}
@@ -65,34 +78,47 @@ const (
78
nCompression
79
)
80
68
-// Strict is the DB strict level.
81
+// Strict is the DB 'strict level'.
82
type Strict uint
83
84
const (
85
// If present then a corrupted or invalid chunk or block in manifest
73
- // journal will cause an error istead of being dropped.
86
+ // journal will cause an error instead of being dropped.
87
+ // This will prevent database with corrupted manifest to be opened.
88
StrictManifest Strict = 1 << iota
89
76
- // If present then a corrupted or invalid chunk or block in journal
77
- // will cause an error istead of being dropped.
78
- StrictJournal
79
-
90
// If present then journal chunk checksum will be verified.
91
StrictJournalChecksum
92
83
- // If present then an invalid key/value pair will cause an error
84
- // instead of being skipped.
85
- StrictIterator
93
+ // If present then a corrupted or invalid chunk or block in journal
94
+ // will cause an error instead of being dropped.
95
+ // This will prevent database with corrupted journal to be opened.
96
+ StrictJournal
97
98
// If present then 'sorted table' block checksum will be verified.
99
+ // This has effect on both 'read operation' and compaction.
100
StrictBlockChecksum
101
102
+ // If present then a corrupted 'sorted table' will fails compaction.
103
+ // The database will enter read-only mode.
104
+ StrictCompaction
105
+
106
+ // If present then a corrupted 'sorted table' will halts 'read operation'.
107
+ StrictReader
108
+
109
+ // If present then leveldb.Recover will drop corrupted 'sorted table'.
110
+ StrictRecovery
111
+
112
+ // This only applicable for ReadOptions, if present then this ReadOptions
113
+ // 'strict level' will override global ones.
114
+ StrictOverride
115
+
116
// StrictAll enables all strict flags.
91
- StrictAll = StrictManifest | StrictJournal | StrictJournalChecksum | StrictIterator | StrictBlockChecksum
117
+ StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery
118
119
// DefaultStrict is the default strict flags. Specify any strict flags
120
// will override default strict flags as whole (i.e. not OR'ed).
95
- DefaultStrict = StrictJournalChecksum | StrictBlockChecksum
121
+ DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader
122
123
// NoStrict disables all strict flags. Override default strict flags.
124
NoStrict = ^StrictAll
@@ -110,9 +136,14 @@ type Options struct {
136
// BlockCache provides per-block caching for LevelDB. Specify NoCache to
137
// disable block caching.
138
//
113
- // By default LevelDB will create LRU-cache with capacity of 8MiB.
139
+ // By default LevelDB will create LRU-cache with capacity of BlockCacheSize.
140
BlockCache cache.Cache
141
142
+ // BlockCacheSize defines the capacity of the default 'block cache'.
143
+ //
144
+ // The default value is 8MiB.
145
+ BlockCacheSize int
146
+
147
// BlockRestartInterval is the number of keys between restart points for
148
// delta encoding of keys.
149
//
@@ -132,6 +163,73 @@ type Options struct {
163
// The default value is 500.
164
CachedOpenFiles int
165
166
+ // CompactionExpandLimitFactor limits compaction size after expanded.
167
+ // This will be multiplied by table size limit at compaction target level.
168
+ //
169
+ // The default value is 25.
170
+ CompactionExpandLimitFactor int
171
+
172
+ // CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a
173
+ // single 'sorted table' generates.
174
+ // This will be multiplied by table size limit at grandparent level.
175
+ //
176
+ // The default value is 10.
177
+ CompactionGPOverlapsFactor int
178
+
179
+ // CompactionL0Trigger defines number of 'sorted table' at level-0 that will
180
+ // trigger compaction.
181
+ //
182
+ // The default value is 4.
183
+ CompactionL0Trigger int
184
+
185
+ // CompactionSourceLimitFactor limits compaction source size. This doesn't apply to
186
+ // level-0.
187
+ // This will be multiplied by table size limit at compaction target level.
188
+ //
189
+ // The default value is 1.
190
+ CompactionSourceLimitFactor int
191
+
192
+ // CompactionTableSize limits size of 'sorted table' that compaction generates.
193
+ // The limits for each level will be calculated as:
194
+ // CompactionTableSize * (CompactionTableSizeMultiplier ^ Level)
195
+ // The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel.
196
+ //
197
+ // The default value is 2MiB.
198
+ CompactionTableSize int
199
+
200
+ // CompactionTableSizeMultiplier defines multiplier for CompactionTableSize.
201
+ //
202
+ // The default value is 1.
203
+ CompactionTableSizeMultiplier float64
204
+
205
+ // CompactionTableSizeMultiplierPerLevel defines per-level multiplier for
206
+ // CompactionTableSize.
207
+ // Use zero to skip a level.
208
+ //
209
+ // The default value is nil.
210
+ CompactionTableSizeMultiplierPerLevel []float64
211
+
212
+ // CompactionTotalSize limits total size of 'sorted table' for each level.
213
+ // The limits for each level will be calculated as:
214
+ // CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level)
215
+ // The multiplier for each level can also fine-tuned using
216
+ // CompactionTotalSizeMultiplierPerLevel.
217
+ //
218
+ // The default value is 10MiB.
219
+ CompactionTotalSize int
220
+
221
+ // CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize.
222
+ //
223
+ // The default value is 10.
224
+ CompactionTotalSizeMultiplier float64
225
+
226
+ // CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for
227
+ // CompactionTotalSize.
228
+ // Use zero to skip a level.
229
+ //
230
+ // The default value is nil.
231
+ CompactionTotalSizeMultiplierPerLevel []float64
232
+
233
// Comparer defines a total ordering over the space of []byte keys: a 'less
234
// than' relationship. The same comparison algorithm must be used for reads
235
// and writes over the lifetime of the DB.
@@ -144,6 +242,11 @@ type Options struct {
242
// The default value (DefaultCompression) uses snappy compression.
243
Compression Compression
244
245
+ // DisableCompactionBackoff allows disable compaction retry backoff.
246
+ //
247
+ // The default value is false.
248
+ DisableCompactionBackoff bool
249
+
250
// ErrorIfExist defines whether an error should returned if the DB already
251
// exist.
252
//
@@ -172,6 +275,19 @@ type Options struct {
275
// The default value is nil.
276
Filter filter.Filter
277
278
+ // MaxMemCompationLevel defines maximum level a newly compacted 'memdb'
279
+ // will be pushed into if doesn't creates overlap. This should less than
280
+ // NumLevel. Use -1 for level-0.
281
+ //
282
+ // The default is 2.
283
+ MaxMemCompationLevel int
284
+
285
+ // NumLevel defines number of database level. The level shouldn't changed
286
+ // between opens, or the database will panic.
287
+ //
288
+ // The default is 7.
289
+ NumLevel int
290
+
291
// Strict defines the DB strict level.
292
Strict Strict
293
@@ -183,6 +299,18 @@ type Options struct {
299
//
300
// The default value is 4MiB.
301
WriteBuffer int
302
+
303
+ // WriteL0StopTrigger defines number of 'sorted table' at level-0 that will
304
+ // pause write.
305
+ //
306
+ // The default value is 12.
307
+ WriteL0PauseTrigger int
308
+
309
+ // WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that
310
+ // will trigger write slowdown.
311
+ //
312
+ // The default value is 8.
313
+ WriteL0SlowdownTrigger int
314
}
315
316
func (o *Options) GetAltFilters() []filter.Filter {
@@ -199,6 +327,13 @@ func (o *Options) GetBlockCache() cache.Cache {
327
return o.BlockCache
328
}
329
330
+func (o *Options) GetBlockCacheSize() int {
331
+ if o == nil || o.BlockCacheSize <= 0 {
332
+ return DefaultBlockCacheSize
333
+ }
334
+ return o.BlockCacheSize
335
+}
336
+
337
func (o *Options) GetBlockRestartInterval() int {
338
if o == nil || o.BlockRestartInterval <= 0 {
339
return DefaultBlockRestartInterval
@@ -222,6 +357,79 @@ func (o *Options) GetCachedOpenFiles() int {
357
return o.CachedOpenFiles
358
}
359
360
+func (o *Options) GetCompactionExpandLimit(level int) int {
361
+ factor := DefaultCompactionExpandLimitFactor
362
+ if o != nil && o.CompactionExpandLimitFactor > 0 {
363
+ factor = o.CompactionExpandLimitFactor
364
+ }
365
+ return o.GetCompactionTableSize(level+1) * factor
366
+}
367
+
368
+func (o *Options) GetCompactionGPOverlaps(level int) int {
369
+ factor := DefaultCompactionGPOverlapsFactor
370
+ if o != nil && o.CompactionGPOverlapsFactor > 0 {
371
+ factor = o.CompactionGPOverlapsFactor
372
+ }
373
+ return o.GetCompactionTableSize(level+2) * factor
374
+}
375
+
376
+func (o *Options) GetCompactionL0Trigger() int {
377
+ if o == nil || o.CompactionL0Trigger == 0 {
378
+ return DefaultCompactionL0Trigger
379
+ }
380
+ return o.CompactionL0Trigger
381
+}
382
+
383
+func (o *Options) GetCompactionSourceLimit(level int) int {
384
+ factor := DefaultCompactionSourceLimitFactor
385
+ if o != nil && o.CompactionSourceLimitFactor > 0 {
386
+ factor = o.CompactionSourceLimitFactor
387
+ }
388
+ return o.GetCompactionTableSize(level+1) * factor
389
+}
390
+
391
+func (o *Options) GetCompactionTableSize(level int) int {
392
+ var (
393
+ base = DefaultCompactionTableSize
394
+ mult float64
395
+ )
396
+ if o != nil {
397
+ if o.CompactionTableSize > 0 {
398
+ base = o.CompactionTableSize
399
+ }
400
+ if len(o.CompactionTableSizeMultiplierPerLevel) > level && o.CompactionTableSizeMultiplierPerLevel[level] > 0 {
401
+ mult = o.CompactionTableSizeMultiplierPerLevel[level]
402
+ } else if o.CompactionTableSizeMultiplier > 0 {
403
+ mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level))
404
+ }
405
+ }
406
+ if mult == 0 {
407
+ mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level))
408
+ }
409
+ return int(float64(base) * mult)
410
+}
411
+
412
+func (o *Options) GetCompactionTotalSize(level int) int64 {
413
+ var (
414
+ base = DefaultCompactionTotalSize
415
+ mult float64
416
+ )
417
+ if o != nil {
418
+ if o.CompactionTotalSize > 0 {
419
+ base = o.CompactionTotalSize
420
+ }
421
+ if len(o.CompactionTotalSizeMultiplierPerLevel) > level && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 {
422
+ mult = o.CompactionTotalSizeMultiplierPerLevel[level]
423
+ } else if o.CompactionTotalSizeMultiplier > 0 {
424
+ mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level))
425
+ }
426
+ }
427
+ if mult == 0 {
428
+ mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level))
429
+ }
430
+ return int64(float64(base) * mult)
431
+}
432
+
433
func (o *Options) GetComparer() comparer.Comparer {
434
if o == nil || o.Comparer == nil {
435
return comparer.DefaultComparer
@@ -236,6 +444,13 @@ func (o *Options) GetCompression() Compression {
444
return o.Compression
445
}
446
447
+func (o *Options) GetDisableCompactionBackoff() bool {
448
+ if o == nil {
449
+ return false
450
+ }
451
+ return o.DisableCompactionBackoff
452
+}
453
+
454
func (o *Options) GetErrorIfExist() bool {
455
if o == nil {
456
return false
@@ -257,6 +472,28 @@ func (o *Options) GetFilter() filter.Filter {
472
return o.Filter
473
}
474
475
+func (o *Options) GetMaxMemCompationLevel() int {
476
+ level := DefaultMaxMemCompationLevel
477
+ if o != nil {
478
+ if o.MaxMemCompationLevel > 0 {
479
+ level = o.MaxMemCompationLevel
480
+ } else if o.MaxMemCompationLevel == -1 {
481
+ level = 0
482
+ }
483
+ }
484
+ if level >= o.GetNumLevel() {
485
+ return o.GetNumLevel() - 1
486
+ }
487
+ return level
488
+}
489
+
490
+func (o *Options) GetNumLevel() int {
491
+ if o == nil || o.NumLevel <= 0 {
492
+ return DefaultNumLevel
493
+ }
494
+ return o.NumLevel
495
+}
496
+
497
func (o *Options) GetStrict(strict Strict) bool {
498
if o == nil || o.Strict == 0 {
499
return DefaultStrict&strict != 0
@@ -271,6 +508,20 @@ func (o *Options) GetWriteBuffer() int {
508
return o.WriteBuffer
509
}
510
511
+func (o *Options) GetWriteL0PauseTrigger() int {
512
+ if o == nil || o.WriteL0PauseTrigger == 0 {
513
+ return DefaultWriteL0PauseTrigger
514
+ }
515
+ return o.WriteL0PauseTrigger
516
+}
517
+
518
+func (o *Options) GetWriteL0SlowdownTrigger() int {
519
+ if o == nil || o.WriteL0SlowdownTrigger == 0 {
520
+ return DefaultWriteL0SlowdownTrigger
521
+ }
522
+ return o.WriteL0SlowdownTrigger
523
+}
524
+
525
// ReadOptions holds the optional parameters for 'read operation'. The
526
// 'read operation' includes Get, Find and NewIterator.
527
type ReadOptions struct {
@@ -281,8 +532,8 @@ type ReadOptions struct {
532
// The default value is false.
533
DontFillCache bool
534
284
- // Strict overrides global DB strict level. Only StrictIterator and
285
- // StrictBlockChecksum that does have effects here.
535
+ // Strict will be OR'ed with global DB 'strict level' unless StrictOverride
536
+ // is present. Currently only StrictReader that has effect here.
537
Strict Strict
538
}
539
@@ -324,3 +575,11 @@ func (wo *WriteOptions) GetSync() bool {
575
}
576
return wo.Sync
577
}
578
+
579
+func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool {
580
+ if ro.GetStrict(StrictOverride) {
581
+ return ro.GetStrict(strict)
582
+ } else {
583
+ return o.GetStrict(strict) || ro.GetStrict(strict)
584
+ }
585
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go
+68
-9
@@ -12,30 +12,89 @@ import (
12
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
13
)
14
15
-func (s *session) setOptions(o *opt.Options) {
16
- s.o = &opt.Options{}
15
+func dupOptions(o *opt.Options) *opt.Options {
16
+ newo := &opt.Options{}
17
if o != nil {
18
- *s.o = *o
18
+ *newo = *o
19
+ }
20
+ if newo.Strict == 0 {
21
+ newo.Strict = opt.DefaultStrict
22
}
23
+ return newo
24
+}
25
+
26
+func (s *session) setOptions(o *opt.Options) {
27
+ no := dupOptions(o)
28
// Alternative filters.
29
if filters := o.GetAltFilters(); len(filters) > 0 {
22
- s.o.AltFilters = make([]filter.Filter, len(filters))
30
+ no.AltFilters = make([]filter.Filter, len(filters))
31
for i, filter := range filters {
24
- s.o.AltFilters[i] = &iFilter{filter}
32
+ no.AltFilters[i] = &iFilter{filter}
33
}
34
}
35
// Block cache.
36
switch o.GetBlockCache() {
37
case nil:
30
- s.o.BlockCache = cache.NewLRUCache(opt.DefaultBlockCacheSize)
38
+ no.BlockCache = cache.NewLRUCache(o.GetBlockCacheSize())
39
case opt.NoCache:
32
- s.o.BlockCache = nil
40
+ no.BlockCache = nil
41
}
42
// Comparer.
43
s.icmp = &iComparer{o.GetComparer()}
36
- s.o.Comparer = s.icmp
44
+ no.Comparer = s.icmp
45
// Filter.
46
if filter := o.GetFilter(); filter != nil {
39
- s.o.Filter = &iFilter{filter}
47
+ no.Filter = &iFilter{filter}
48
+ }
49
+
50
+ s.o = &cachedOptions{Options: no}
51
+ s.o.cache()
52
+}
53
+
54
+type cachedOptions struct {
55
+ *opt.Options
56
+
57
+ compactionExpandLimit []int
58
+ compactionGPOverlaps []int
59
+ compactionSourceLimit []int
60
+ compactionTableSize []int
61
+ compactionTotalSize []int64
62
+}
63
+
64
+func (co *cachedOptions) cache() {
65
+ numLevel := co.Options.GetNumLevel()
66
+
67
+ co.compactionExpandLimit = make([]int, numLevel)
68
+ co.compactionGPOverlaps = make([]int, numLevel)
69
+ co.compactionSourceLimit = make([]int, numLevel)
70
+ co.compactionTableSize = make([]int, numLevel)
71
+ co.compactionTotalSize = make([]int64, numLevel)
72
+
73
+ for level := 0; level < numLevel; level++ {
74
+ co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level)
75
+ co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level)
76
+ co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level)
77
+ co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level)
78
+ co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level)
79
}
80
}
81
+
82
+func (co *cachedOptions) GetCompactionExpandLimit(level int) int {
83
+ return co.compactionExpandLimit[level]
84
+}
85
+
86
+func (co *cachedOptions) GetCompactionGPOverlaps(level int) int {
87
+ return co.compactionGPOverlaps[level]
88
+}
89
+
90
+func (co *cachedOptions) GetCompactionSourceLimit(level int) int {
91
+ return co.compactionSourceLimit[level]
92
+}
93
+
94
+func (co *cachedOptions) GetCompactionTableSize(level int) int {
95
+ return co.compactionTableSize[level]
96
+}
97
+
98
+func (co *cachedOptions) GetCompactionTotalSize(level int) int64 {
99
+ return co.compactionTotalSize[level]
100
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go
+135
-73
@@ -7,12 +7,13 @@
7
package leveldb
8
9
import (
10
- "errors"
10
+ "fmt"
11
"io"
12
"os"
13
"sync"
14
"sync/atomic"
15
16
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
17
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
18
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
@@ -20,18 +21,31 @@ import (
21
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
22
)
23
24
+type ErrManifestCorrupted struct {
25
+ Field string
26
+ Reason string
27
+}
28
+
29
+func (e *ErrManifestCorrupted) Error() string {
30
+ return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason)
31
+}
32
+
33
+func newErrManifestCorrupted(f storage.File, field, reason string) error {
34
+ return errors.NewErrCorrupted(f, &ErrManifestCorrupted{field, reason})
35
+}
36
+
37
// session represent a persistent database session.
38
type session struct {
39
// Need 64-bit alignment.
26
- stFileNum uint64 // current unused file number
40
+ stNextFileNum uint64 // current unused file number
41
stJournalNum uint64 // current journal file number; need external synchronization
42
stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
29
- stSeq uint64 // last mem compacted seq; need external synchronization
43
+ stSeqNum uint64 // last mem compacted seq; need external synchronization
44
stTempFileNum uint64
45
46
stor storage.Storage
47
storLock util.Releaser
34
- o *opt.Options
48
+ o *cachedOptions
49
icmp *iComparer
50
tops *tOps
51
@@ -39,9 +53,9 @@ type session struct {
53
manifestWriter storage.Writer
54
manifestFile storage.File
55
42
- stCptrs [kNumLevels]iKey // compact pointers; need external synchronization
43
- stVersion *version // current version
44
- vmu sync.Mutex
56
+ stCompPtrs []iKey // compaction pointers; need external synchronization
57
+ stVersion *version // current version
58
+ vmu sync.Mutex
59
}
60
61
// Creates new initialized session instance.
@@ -54,13 +68,14 @@ func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
68
return
69
}
70
s = &session{
57
- stor: stor,
58
- storLock: storLock,
71
+ stor: stor,
72
+ storLock: storLock,
73
+ stCompPtrs: make([]iKey, o.GetNumLevel()),
74
}
75
s.setOptions(o)
76
s.tops = newTableOps(s, s.o.GetCachedOpenFiles())
62
- s.setVersion(&version{s: s})
63
- s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock D·DeletedEntry L·Level Q·SeqNum T·TimeElapsed")
77
+ s.setVersion(newVersion(s))
78
+ s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
79
return
80
}
81
@@ -100,26 +115,26 @@ func (s *session) recover() (err error) {
115
// Don't return os.ErrNotExist if the underlying storage contains
116
// other files that belong to LevelDB. So the DB won't get trashed.
117
if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
103
- err = ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest file missing")}
118
+ err = &errors.ErrCorrupted{File: &storage.FileInfo{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
119
}
120
}
121
}()
122
108
- file, err := s.stor.GetManifest()
123
+ m, err := s.stor.GetManifest()
124
if err != nil {
125
return
126
}
127
113
- reader, err := file.Open()
128
+ reader, err := m.Open()
129
if err != nil {
130
return
131
}
132
defer reader.Close()
133
strict := s.o.GetStrict(opt.StrictManifest)
119
- jr := journal.NewReader(reader, dropper{s, file}, strict, true)
134
+ jr := journal.NewReader(reader, dropper{s, m}, strict, true)
135
121
- staging := s.version_NB().newStaging()
122
- rec := &sessionRecord{}
136
+ staging := s.stVersion.newStaging()
137
+ rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
138
for {
139
var r io.Reader
140
r, err = jr.Next()
@@ -128,51 +143,57 @@ func (s *session) recover() (err error) {
143
err = nil
144
break
145
}
131
- return
146
+ return errors.SetFile(err, m)
147
}
148
149
err = rec.decode(r)
150
if err == nil {
151
// save compact pointers
137
- for _, r := range rec.compactionPointers {
138
- s.stCptrs[r.level] = iKey(r.ikey)
152
+ for _, r := range rec.compPtrs {
153
+ s.stCompPtrs[r.level] = iKey(r.ikey)
154
}
155
// commit record to version staging
156
staging.commit(rec)
142
- } else if strict {
143
- return ErrCorrupted{Type: CorruptedManifest, Err: err}
157
} else {
145
- s.logf("manifest error: %v (skipped)", err)
158
+ err = errors.SetFile(err, m)
159
+ if strict || !errors.IsCorrupted(err) {
160
+ return
161
+ } else {
162
+ s.logf("manifest error: %v (skipped)", errors.SetFile(err, m))
163
+ }
164
}
147
- rec.resetCompactionPointers()
165
+ rec.resetCompPtrs()
166
rec.resetAddedTables()
167
rec.resetDeletedTables()
168
}
169
170
switch {
171
case !rec.has(recComparer):
154
- return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing comparer name")}
172
+ return newErrManifestCorrupted(m, "comparer", "missing")
173
case rec.comparer != s.icmp.uName():
156
- return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: comparer mismatch, " + "want '" + s.icmp.uName() + "', " + "got '" + rec.comparer + "'")}
157
- case !rec.has(recNextNum):
158
- return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing next file number")}
174
+ return newErrManifestCorrupted(m, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
175
+ case !rec.has(recNextFileNum):
176
+ return newErrManifestCorrupted(m, "next-file-num", "missing")
177
case !rec.has(recJournalNum):
160
- return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing journal file number")}
161
- case !rec.has(recSeq):
162
- return ErrCorrupted{Type: CorruptedManifest, Err: errors.New("leveldb: manifest missing seq number")}
178
+ return newErrManifestCorrupted(m, "journal-file-num", "missing")
179
+ case !rec.has(recSeqNum):
180
+ return newErrManifestCorrupted(m, "seq-num", "missing")
181
}
182
165
- s.manifestFile = file
183
+ s.manifestFile = m
184
s.setVersion(staging.finish())
167
- s.setFileNum(rec.nextNum)
185
+ s.setNextFileNum(rec.nextFileNum)
186
s.recordCommited(rec)
187
return nil
188
}
189
190
// Commit session; need external synchronization.
191
func (s *session) commit(r *sessionRecord) (err error) {
192
+ v := s.version()
193
+ defer v.release()
194
+
195
// spawn new version based on current version
175
- nv := s.version_NB().spawn(r)
196
+ nv := v.spawn(r)
197
198
if s.manifest == nil {
199
// manifest journal writer not yet created, create one
@@ -191,13 +212,13 @@ func (s *session) commit(r *sessionRecord) (err error) {
212
213
// Pick a compaction based on current state; need external synchronization.
214
func (s *session) pickCompaction() *compaction {
194
- v := s.version_NB()
215
+ v := s.version()
216
217
var level int
218
var t0 tFiles
219
if v.cScore >= 1 {
220
level = v.cLevel
200
- cptr := s.stCptrs[level]
221
+ cptr := s.stCompPtrs[level]
222
tables := v.tables[level]
223
for _, t := range tables {
224
if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
@@ -214,27 +235,21 @@ func (s *session) pickCompaction() *compaction {
235
level = ts.level
236
t0 = append(t0, ts.table)
237
} else {
238
+ v.release()
239
return nil
240
}
241
}
242
221
- c := &compaction{s: s, v: v, level: level}
222
- if level == 0 {
223
- imin, imax := t0.getRange(s.icmp)
224
- t0 = v.tables[0].getOverlaps(t0[:0], s.icmp, imin.ukey(), imax.ukey(), true)
225
- }
226
-
227
- c.tables[0] = t0
228
- c.expand()
229
- return c
243
+ return newCompaction(s, v, level, t0)
244
}
245
246
// Create compaction from given level and range; need external synchronization.
247
func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
234
- v := s.version_NB()
248
+ v := s.version()
249
250
t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
251
if len(t0) == 0 {
252
+ v.release()
253
return nil
254
}
255
@@ -243,7 +258,7 @@ func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
258
// and we must not pick one file and drop another older file if the
259
// two files overlap.
260
if level > 0 {
246
- limit := uint64(kMaxTableSize)
261
+ limit := uint64(v.s.o.GetCompactionSourceLimit(level))
262
total := uint64(0)
263
for i, t := range t0 {
264
total += t.size
@@ -255,9 +270,20 @@ func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
270
}
271
}
272
258
- c := &compaction{s: s, v: v, level: level}
259
- c.tables[0] = t0
273
+ return newCompaction(s, v, level, t0)
274
+}
275
+
276
+func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
277
+ c := &compaction{
278
+ s: s,
279
+ v: v,
280
+ level: level,
281
+ tables: [2]tFiles{t0, nil},
282
+ maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
283
+ tPtrs: make([]int, s.o.GetNumLevel()),
284
+ }
285
c.expand()
286
+ c.save()
287
return c
288
}
289
@@ -266,25 +292,57 @@ type compaction struct {
292
s *session
293
v *version
294
269
- level int
270
- tables [2]tFiles
295
+ level int
296
+ tables [2]tFiles
297
+ maxGPOverlaps uint64
298
+
299
+ gp tFiles
300
+ gpi int
301
+ seenKey bool
302
+ gpOverlappedBytes uint64
303
+ imin, imax iKey
304
+ tPtrs []int
305
+ released bool
306
+
307
+ snapGPI int
308
+ snapSeenKey bool
309
+ snapGPOverlappedBytes uint64
310
+ snapTPtrs []int
311
+}
312
272
- gp tFiles
273
- gpidx int
274
- seenKey bool
275
- overlappedBytes uint64
276
- imin, imax iKey
313
+func (c *compaction) save() {
314
+ c.snapGPI = c.gpi
315
+ c.snapSeenKey = c.seenKey
316
+ c.snapGPOverlappedBytes = c.gpOverlappedBytes
317
+ c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
318
+}
319
278
- tPtrs [kNumLevels]int
320
+func (c *compaction) restore() {
321
+ c.gpi = c.snapGPI
322
+ c.seenKey = c.snapSeenKey
323
+ c.gpOverlappedBytes = c.snapGPOverlappedBytes
324
+ c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
325
+}
326
+
327
+func (c *compaction) release() {
328
+ if !c.released {
329
+ c.released = true
330
+ c.v.release()
331
+ }
332
}
333
334
// Expand compacted tables; need external synchronization.
335
func (c *compaction) expand() {
283
- level := c.level
284
- vt0, vt1 := c.v.tables[level], c.v.tables[level+1]
336
+ limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
337
+ vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
338
339
t0, t1 := c.tables[0], c.tables[1]
340
imin, imax := t0.getRange(c.s.icmp)
341
+ // We expand t0 here just incase ukey hop across tables.
342
+ t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
343
+ if len(t0) != len(c.tables[0]) {
344
+ imin, imax = t0.getRange(c.s.icmp)
345
+ }
346
t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
347
// Get entire range covered by compaction.
348
amin, amax := append(t0, t1...).getRange(c.s.icmp)
@@ -292,13 +350,13 @@ func (c *compaction) expand() {
350
// See if we can grow the number of inputs in "level" without
351
// changing the number of "level+1" files we pick up.
352
if len(t1) > 0 {
295
- exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), level == 0)
296
- if len(exp0) > len(t0) && t1.size()+exp0.size() < kExpCompactionMaxBytes {
353
+ exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
354
+ if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
355
xmin, xmax := exp0.getRange(c.s.icmp)
356
exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
357
if len(exp1) == len(t1) {
358
c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
301
- level, level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
359
+ c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
360
len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
361
imin, imax = xmin, xmax
362
t0, t1 = exp0, exp1
@@ -309,8 +367,8 @@ func (c *compaction) expand() {
367
368
// Compute the set of grandparent files that overlap this compaction
369
// (parent == level+1; grandparent == level+2)
312
- if level+2 < kNumLevels {
313
- c.gp = c.v.tables[level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
370
+ if c.level+2 < c.s.o.GetNumLevel() {
371
+ c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
372
}
373
374
c.tables[0], c.tables[1] = t0, t1
@@ -319,7 +377,7 @@ func (c *compaction) expand() {
377
378
// Check whether compaction is trivial.
379
func (c *compaction) trivial() bool {
322
- return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= kMaxGrandParentOverlapBytes
380
+ return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
381
}
382
383
func (c *compaction) baseLevelForKey(ukey []byte) bool {
@@ -341,20 +399,20 @@ func (c *compaction) baseLevelForKey(ukey []byte) bool {
399
}
400
401
func (c *compaction) shouldStopBefore(ikey iKey) bool {
344
- for ; c.gpidx < len(c.gp); c.gpidx++ {
345
- gp := c.gp[c.gpidx]
402
+ for ; c.gpi < len(c.gp); c.gpi++ {
403
+ gp := c.gp[c.gpi]
404
if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
405
break
406
}
407
if c.seenKey {
350
- c.overlappedBytes += gp.size
408
+ c.gpOverlappedBytes += gp.size
409
}
410
}
411
c.seenKey = true
412
355
- if c.overlappedBytes > kMaxGrandParentOverlapBytes {
413
+ if c.gpOverlappedBytes > c.maxGPOverlaps {
414
// Too much overlap for current output; start new output.
357
- c.overlappedBytes = 0
415
+ c.gpOverlappedBytes = 0
416
return true
417
}
418
return false
@@ -373,8 +431,12 @@ func (c *compaction) newIterator() iterator.Iterator {
431
// Options.
432
ro := &opt.ReadOptions{
433
DontFillCache: true,
434
+ Strict: opt.StrictOverride,
435
+ }
436
+ strict := c.s.o.GetStrict(opt.StrictCompaction)
437
+ if strict {
438
+ ro.Strict |= opt.StrictReader
439
}
377
- strict := c.s.o.GetStrict(opt.StrictIterator)
440
441
for i, tables := range c.tables {
442
if len(tables) == 0 {
@@ -387,10 +449,10 @@ func (c *compaction) newIterator() iterator.Iterator {
449
its = append(its, c.s.tops.newIterator(t, nil, ro))
450
}
451
} else {
390
- it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict, true)
452
+ it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
453
its = append(its, it)
454
}
455
}
456
395
- return iterator.NewMergedIterator(its, c.s.icmp, true)
457
+ return iterator.NewMergedIterator(its, c.s.icmp, strict)
458
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go
+98
-93
@@ -9,11 +9,11 @@ package leveldb
9
import (
10
"bufio"
11
"encoding/binary"
12
- "errors"
12
"io"
14
-)
13
+ "strings"
14
16
-var errCorruptManifest = errors.New("leveldb: corrupt manifest")
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16
+)
17
18
type byteReader interface {
19
io.Reader
@@ -22,13 +22,13 @@ type byteReader interface {
22
23
// These numbers are written to disk and should not be changed.
24
const (
25
- recComparer = 1
26
- recJournalNum = 2
27
- recNextNum = 3
28
- recSeq = 4
29
- recCompactionPointer = 5
30
- recDeletedTable = 6
31
- recNewTable = 7
25
+ recComparer = 1
26
+ recJournalNum = 2
27
+ recNextFileNum = 3
28
+ recSeqNum = 4
29
+ recCompPtr = 5
30
+ recDelTable = 6
31
+ recAddTable = 7
32
// 8 was used for large value refs
33
recPrevJournalNum = 9
34
)
@@ -38,7 +38,7 @@ type cpRecord struct {
38
ikey iKey
39
}
40
41
-type ntRecord struct {
41
+type atRecord struct {
42
level int
43
num uint64
44
size uint64
@@ -46,27 +46,26 @@ type ntRecord struct {
46
imax iKey
47
}
48
49
-func (r ntRecord) makeFile(s *session) *tFile {
50
- return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
51
-}
52
-
49
type dtRecord struct {
50
level int
51
num uint64
52
}
53
54
type sessionRecord struct {
59
- hasRec int
60
- comparer string
61
- journalNum uint64
62
- prevJournalNum uint64
63
- nextNum uint64
64
- seq uint64
65
- compactionPointers []cpRecord
66
- addedTables []ntRecord
67
- deletedTables []dtRecord
68
- scratch [binary.MaxVarintLen64]byte
69
- err error
55
+ numLevel int
56
+
57
+ hasRec int
58
+ comparer string
59
+ journalNum uint64
60
+ prevJournalNum uint64
61
+ nextFileNum uint64
62
+ seqNum uint64
63
+ compPtrs []cpRecord
64
+ addedTables []atRecord
65
+ deletedTables []dtRecord
66
+
67
+ scratch [binary.MaxVarintLen64]byte
68
+ err error
69
}
70
71
func (p *sessionRecord) has(rec int) bool {
@@ -88,29 +87,29 @@ func (p *sessionRecord) setPrevJournalNum(num uint64) {
87
p.prevJournalNum = num
88
}
89
91
-func (p *sessionRecord) setNextNum(num uint64) {
92
- p.hasRec |= 1 << recNextNum
93
- p.nextNum = num
90
+func (p *sessionRecord) setNextFileNum(num uint64) {
91
+ p.hasRec |= 1 << recNextFileNum
92
+ p.nextFileNum = num
93
}
94
96
-func (p *sessionRecord) setSeq(seq uint64) {
97
- p.hasRec |= 1 << recSeq
98
- p.seq = seq
95
+func (p *sessionRecord) setSeqNum(num uint64) {
96
+ p.hasRec |= 1 << recSeqNum
97
+ p.seqNum = num
98
}
99
101
-func (p *sessionRecord) addCompactionPointer(level int, ikey iKey) {
102
- p.hasRec |= 1 << recCompactionPointer
103
- p.compactionPointers = append(p.compactionPointers, cpRecord{level, ikey})
100
+func (p *sessionRecord) addCompPtr(level int, ikey iKey) {
101
+ p.hasRec |= 1 << recCompPtr
102
+ p.compPtrs = append(p.compPtrs, cpRecord{level, ikey})
103
}
104
106
-func (p *sessionRecord) resetCompactionPointers() {
107
- p.hasRec &= ^(1 << recCompactionPointer)
108
- p.compactionPointers = p.compactionPointers[:0]
105
+func (p *sessionRecord) resetCompPtrs() {
106
+ p.hasRec &= ^(1 << recCompPtr)
107
+ p.compPtrs = p.compPtrs[:0]
108
}
109
110
func (p *sessionRecord) addTable(level int, num, size uint64, imin, imax iKey) {
112
- p.hasRec |= 1 << recNewTable
113
- p.addedTables = append(p.addedTables, ntRecord{level, num, size, imin, imax})
111
+ p.hasRec |= 1 << recAddTable
112
+ p.addedTables = append(p.addedTables, atRecord{level, num, size, imin, imax})
113
}
114
115
func (p *sessionRecord) addTableFile(level int, t *tFile) {
@@ -118,17 +117,17 @@ func (p *sessionRecord) addTableFile(level int, t *tFile) {
117
}
118
119
func (p *sessionRecord) resetAddedTables() {
121
- p.hasRec &= ^(1 << recNewTable)
120
+ p.hasRec &= ^(1 << recAddTable)
121
p.addedTables = p.addedTables[:0]
122
}
123
125
-func (p *sessionRecord) deleteTable(level int, num uint64) {
126
- p.hasRec |= 1 << recDeletedTable
124
+func (p *sessionRecord) delTable(level int, num uint64) {
125
+ p.hasRec |= 1 << recDelTable
126
p.deletedTables = append(p.deletedTables, dtRecord{level, num})
127
}
128
129
func (p *sessionRecord) resetDeletedTables() {
131
- p.hasRec &= ^(1 << recDeletedTable)
130
+ p.hasRec &= ^(1 << recDelTable)
131
p.deletedTables = p.deletedTables[:0]
132
}
133
@@ -161,26 +160,26 @@ func (p *sessionRecord) encode(w io.Writer) error {
160
p.putUvarint(w, recJournalNum)
161
p.putUvarint(w, p.journalNum)
162
}
164
- if p.has(recNextNum) {
165
- p.putUvarint(w, recNextNum)
166
- p.putUvarint(w, p.nextNum)
163
+ if p.has(recNextFileNum) {
164
+ p.putUvarint(w, recNextFileNum)
165
+ p.putUvarint(w, p.nextFileNum)
166
}
168
- if p.has(recSeq) {
169
- p.putUvarint(w, recSeq)
170
- p.putUvarint(w, p.seq)
167
+ if p.has(recSeqNum) {
168
+ p.putUvarint(w, recSeqNum)
169
+ p.putUvarint(w, p.seqNum)
170
}
172
- for _, r := range p.compactionPointers {
173
- p.putUvarint(w, recCompactionPointer)
171
+ for _, r := range p.compPtrs {
172
+ p.putUvarint(w, recCompPtr)
173
p.putUvarint(w, uint64(r.level))
174
p.putBytes(w, r.ikey)
175
}
176
for _, r := range p.deletedTables {
178
- p.putUvarint(w, recDeletedTable)
177
+ p.putUvarint(w, recDelTable)
178
p.putUvarint(w, uint64(r.level))
179
p.putUvarint(w, r.num)
180
}
181
for _, r := range p.addedTables {
183
- p.putUvarint(w, recNewTable)
182
+ p.putUvarint(w, recAddTable)
183
p.putUvarint(w, uint64(r.level))
184
p.putUvarint(w, r.num)
185
p.putUvarint(w, r.size)
@@ -190,14 +189,16 @@ func (p *sessionRecord) encode(w io.Writer) error {
189
return p.err
190
}
191
193
-func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
192
+func (p *sessionRecord) readUvarintMayEOF(field string, r io.ByteReader, mayEOF bool) uint64 {
193
if p.err != nil {
194
return 0
195
}
196
x, err := binary.ReadUvarint(r)
197
if err != nil {
199
- if err == io.EOF {
200
- p.err = errCorruptManifest
198
+ if err == io.ErrUnexpectedEOF || (mayEOF == false && err == io.EOF) {
199
+ p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
200
+ } else if strings.HasPrefix(err.Error(), "binary:") {
201
+ p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, err.Error()})
202
} else {
203
p.err = err
204
}
@@ -206,35 +207,39 @@ func (p *sessionRecord) readUvarint(r io.ByteReader) uint64 {
207
return x
208
}
209
209
-func (p *sessionRecord) readBytes(r byteReader) []byte {
210
+func (p *sessionRecord) readUvarint(field string, r io.ByteReader) uint64 {
211
+ return p.readUvarintMayEOF(field, r, false)
212
+}
213
+
214
+func (p *sessionRecord) readBytes(field string, r byteReader) []byte {
215
if p.err != nil {
216
return nil
217
}
213
- n := p.readUvarint(r)
218
+ n := p.readUvarint(field, r)
219
if p.err != nil {
220
return nil
221
}
222
x := make([]byte, n)
223
_, p.err = io.ReadFull(r, x)
224
if p.err != nil {
220
- if p.err == io.EOF {
221
- p.err = errCorruptManifest
225
+ if p.err == io.ErrUnexpectedEOF {
226
+ p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
227
}
228
return nil
229
}
230
return x
231
}
232
228
-func (p *sessionRecord) readLevel(r io.ByteReader) int {
233
+func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
234
if p.err != nil {
235
return 0
236
}
232
- x := p.readUvarint(r)
237
+ x := p.readUvarint(field, r)
238
if p.err != nil {
239
return 0
240
}
236
- if x >= kNumLevels {
237
- p.err = errCorruptManifest
241
+ if x >= uint64(p.numLevel) {
242
+ p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"})
243
return 0
244
}
245
return int(x)
@@ -247,59 +252,59 @@ func (p *sessionRecord) decode(r io.Reader) error {
252
}
253
p.err = nil
254
for p.err == nil {
250
- rec, err := binary.ReadUvarint(br)
251
- if err != nil {
252
- if err == io.EOF {
253
- err = nil
255
+ rec := p.readUvarintMayEOF("field-header", br, true)
256
+ if p.err != nil {
257
+ if p.err == io.EOF {
258
+ return nil
259
}
255
- return err
260
+ return p.err
261
}
262
switch rec {
263
case recComparer:
259
- x := p.readBytes(br)
264
+ x := p.readBytes("comparer", br)
265
if p.err == nil {
266
p.setComparer(string(x))
267
}
268
case recJournalNum:
264
- x := p.readUvarint(br)
269
+ x := p.readUvarint("journal-num", br)
270
if p.err == nil {
271
p.setJournalNum(x)
272
}
273
case recPrevJournalNum:
269
- x := p.readUvarint(br)
274
+ x := p.readUvarint("prev-journal-num", br)
275
if p.err == nil {
276
p.setPrevJournalNum(x)
277
}
273
- case recNextNum:
274
- x := p.readUvarint(br)
278
+ case recNextFileNum:
279
+ x := p.readUvarint("next-file-num", br)
280
if p.err == nil {
276
- p.setNextNum(x)
281
+ p.setNextFileNum(x)
282
}
278
- case recSeq:
279
- x := p.readUvarint(br)
283
+ case recSeqNum:
284
+ x := p.readUvarint("seq-num", br)
285
if p.err == nil {
281
- p.setSeq(x)
286
+ p.setSeqNum(x)
287
}
283
- case recCompactionPointer:
284
- level := p.readLevel(br)
285
- ikey := p.readBytes(br)
288
+ case recCompPtr:
289
+ level := p.readLevel("comp-ptr.level", br)
290
+ ikey := p.readBytes("comp-ptr.ikey", br)
291
if p.err == nil {
287
- p.addCompactionPointer(level, iKey(ikey))
292
+ p.addCompPtr(level, iKey(ikey))
293
}
289
- case recNewTable:
290
- level := p.readLevel(br)
291
- num := p.readUvarint(br)
292
- size := p.readUvarint(br)
293
- imin := p.readBytes(br)
294
- imax := p.readBytes(br)
294
+ case recAddTable:
295
+ level := p.readLevel("add-table.level", br)
296
+ num := p.readUvarint("add-table.num", br)
297
+ size := p.readUvarint("add-table.size", br)
298
+ imin := p.readBytes("add-table.imin", br)
299
+ imax := p.readBytes("add-table.imax", br)
300
if p.err == nil {
301
p.addTable(level, num, size, imin, imax)
302
}
298
- case recDeletedTable:
299
- level := p.readLevel(br)
300
- num := p.readUvarint(br)
303
+ case recDelTable:
304
+ level := p.readLevel("del-table.level", br)
305
+ num := p.readUvarint("del-table.num", br)
306
if p.err == nil {
302
- p.deleteTable(level, num)
307
+ p.delTable(level, num)
308
}
309
}
310
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go
+10
-8
@@ -9,6 +9,8 @@ package leveldb
9
import (
10
"bytes"
11
"testing"
12
+
13
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14
)
15
16
func decodeEncode(v *sessionRecord) (res bool, err error) {
@@ -17,7 +19,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
19
if err != nil {
20
return
21
}
20
- v2 := new(sessionRecord)
22
+ v2 := &sessionRecord{numLevel: opt.DefaultNumLevel}
23
err = v.decode(b)
24
if err != nil {
25
return
@@ -32,7 +34,7 @@ func decodeEncode(v *sessionRecord) (res bool, err error) {
34
35
func TestSessionRecord_EncodeDecode(t *testing.T) {
36
big := uint64(1) << 50
35
- v := new(sessionRecord)
37
+ v := &sessionRecord{numLevel: opt.DefaultNumLevel}
38
i := uint64(0)
39
test := func() {
40
res, err := decodeEncode(v)
@@ -47,16 +49,16 @@ func TestSessionRecord_EncodeDecode(t *testing.T) {
49
for ; i < 4; i++ {
50
test()
51
v.addTable(3, big+300+i, big+400+i,
50
- newIKey([]byte("foo"), big+500+1, tVal),
51
- newIKey([]byte("zoo"), big+600+1, tDel))
52
- v.deleteTable(4, big+700+i)
53
- v.addCompactionPointer(int(i), newIKey([]byte("x"), big+900+1, tVal))
52
+ newIkey([]byte("foo"), big+500+1, ktVal),
53
+ newIkey([]byte("zoo"), big+600+1, ktDel))
54
+ v.delTable(4, big+700+i)
55
+ v.addCompPtr(int(i), newIkey([]byte("x"), big+900+1, ktVal))
56
}
57
58
v.setComparer("foo")
59
v.setJournalNum(big + 100)
60
v.setPrevJournalNum(big + 99)
59
- v.setNextNum(big + 200)
60
- v.setSeq(big + 1000)
61
+ v.setNextFileNum(big + 200)
62
+ v.setSeqNum(big + 1000)
63
test()
64
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go
+35
-34
@@ -22,7 +22,7 @@ type dropper struct {
22
}
23
24
func (d dropper) Drop(err error) {
25
- if e, ok := err.(journal.ErrCorrupted); ok {
25
+ if e, ok := err.(*journal.ErrCorrupted); ok {
26
d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
27
} else {
28
d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
@@ -51,9 +51,14 @@ func (s *session) newTemp() storage.File {
51
return s.stor.GetFile(num, storage.TypeTemp)
52
}
53
54
+func (s *session) tableFileFromRecord(r atRecord) *tFile {
55
+ return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
56
+}
57
+
58
// Session state.
59
56
-// Get current version.
60
+// Get current version. This will incr version ref, must call
61
+// version.release (exactly once) after use.
62
func (s *session) version() *version {
63
s.vmu.Lock()
64
defer s.vmu.Unlock()
@@ -61,61 +66,56 @@ func (s *session) version() *version {
66
return s.stVersion
67
}
68
64
-// Get current version; no barrier.
65
-func (s *session) version_NB() *version {
66
- return s.stVersion
67
-}
68
-
69
// Set current version to v.
70
func (s *session) setVersion(v *version) {
71
s.vmu.Lock()
72
- v.ref = 1
72
+ v.ref = 1 // Holds by session.
73
if old := s.stVersion; old != nil {
74
- v.ref++
74
+ v.ref++ // Holds by old version.
75
old.next = v
76
- old.release_NB()
76
+ old.releaseNB()
77
}
78
s.stVersion = v
79
s.vmu.Unlock()
80
}
81
82
// Get current unused file number.
83
-func (s *session) fileNum() uint64 {
84
- return atomic.LoadUint64(&s.stFileNum)
83
+func (s *session) nextFileNum() uint64 {
84
+ return atomic.LoadUint64(&s.stNextFileNum)
85
}
86
87
-// Get current unused file number to num.
88
-func (s *session) setFileNum(num uint64) {
89
- atomic.StoreUint64(&s.stFileNum, num)
87
+// Set current unused file number to num.
88
+func (s *session) setNextFileNum(num uint64) {
89
+ atomic.StoreUint64(&s.stNextFileNum, num)
90
}
91
92
// Mark file number as used.
93
func (s *session) markFileNum(num uint64) {
94
- num += 1
94
+ nextFileNum := num + 1
95
for {
96
- old, x := s.stFileNum, num
96
+ old, x := s.stNextFileNum, nextFileNum
97
if old > x {
98
x = old
99
}
100
- if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
100
+ if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
101
break
102
}
103
}
104
}
105
106
// Allocate a file number.
107
-func (s *session) allocFileNum() (num uint64) {
108
- return atomic.AddUint64(&s.stFileNum, 1) - 1
107
+func (s *session) allocFileNum() uint64 {
108
+ return atomic.AddUint64(&s.stNextFileNum, 1) - 1
109
}
110
111
// Reuse given file number.
112
func (s *session) reuseFileNum(num uint64) {
113
for {
114
- old, x := s.stFileNum, num
114
+ old, x := s.stNextFileNum, num
115
if old != x+1 {
116
x = old
117
}
118
- if atomic.CompareAndSwapUint64(&s.stFileNum, old, x) {
118
+ if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
119
break
120
}
121
}
@@ -126,20 +126,20 @@ func (s *session) reuseFileNum(num uint64) {
126
// Fill given session record obj with current states; need external
127
// synchronization.
128
func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
129
- r.setNextNum(s.fileNum())
129
+ r.setNextFileNum(s.nextFileNum())
130
131
if snapshot {
132
if !r.has(recJournalNum) {
133
r.setJournalNum(s.stJournalNum)
134
}
135
136
- if !r.has(recSeq) {
137
- r.setSeq(s.stSeq)
136
+ if !r.has(recSeqNum) {
137
+ r.setSeqNum(s.stSeqNum)
138
}
139
140
- for level, ik := range s.stCptrs {
140
+ for level, ik := range s.stCompPtrs {
141
if ik != nil {
142
- r.addCompactionPointer(level, ik)
142
+ r.addCompPtr(level, ik)
143
}
144
}
145
@@ -147,7 +147,7 @@ func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
147
}
148
}
149
150
-// Mark if record has been commited, this will update session state;
150
+// Mark if record has been committed, this will update session state;
151
// need external synchronization.
152
func (s *session) recordCommited(r *sessionRecord) {
153
if r.has(recJournalNum) {
@@ -158,12 +158,12 @@ func (s *session) recordCommited(r *sessionRecord) {
158
s.stPrevJournalNum = r.prevJournalNum
159
}
160
161
- if r.has(recSeq) {
162
- s.stSeq = r.seq
161
+ if r.has(recSeqNum) {
162
+ s.stSeqNum = r.seqNum
163
}
164
165
- for _, p := range r.compactionPointers {
166
- s.stCptrs[p.level] = iKey(p.ikey)
165
+ for _, p := range r.compPtrs {
166
+ s.stCompPtrs[p.level] = iKey(p.ikey)
167
}
168
}
169
@@ -178,10 +178,11 @@ func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
178
jw := journal.NewWriter(writer)
179
180
if v == nil {
181
- v = s.version_NB()
181
+ v = s.version()
182
+ defer v.release()
183
}
184
if rec == nil {
184
- rec = new(sessionRecord)
185
+ rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
186
}
187
s.fillRecord(rec, true)
188
v.fillRecord(rec)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go
+1
-1
@@ -221,7 +221,7 @@ func (fs *fileStorage) GetManifest() (f File, err error) {
221
fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
222
continue
223
}
224
- if _, e1 := strconv.ParseUint(fn[7:], 10, 0); e1 != nil {
224
+ if _, e1 := strconv.ParseUint(fn[8:], 10, 0); e1 != nil {
225
fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
226
continue
227
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go
+30
@@ -125,3 +125,33 @@ type Storage interface {
125
// Other methods should not be called after the storage has been closed.
126
Close() error
127
}
128
+
129
+// FileInfo wraps basic file info.
130
+type FileInfo struct {
131
+ Type FileType
132
+ Num uint64
133
+}
134
+
135
+func (fi FileInfo) String() string {
136
+ switch fi.Type {
137
+ case TypeManifest:
138
+ return fmt.Sprintf("MANIFEST-%06d", fi.Num)
139
+ case TypeJournal:
140
+ return fmt.Sprintf("%06d.log", fi.Num)
141
+ case TypeTable:
142
+ return fmt.Sprintf("%06d.ldb", fi.Num)
143
+ case TypeTemp:
144
+ return fmt.Sprintf("%06d.tmp", fi.Num)
145
+ default:
146
+ return fmt.Sprintf("%#x-%d", fi.Type, fi.Num)
147
+ }
148
+}
149
+
150
+// NewFileInfo creates new FileInfo from the given File. It will returns nil
151
+// if File is nil.
152
+func NewFileInfo(f File) *FileInfo {
153
+ if f == nil {
154
+ return nil
155
+ }
156
+ return &FileInfo{f.Type(), f.Num()}
157
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go
+119
-39
@@ -11,6 +11,7 @@ import (
11
"fmt"
12
"io"
13
"io/ioutil"
14
+ "math/rand"
15
"os"
16
"path/filepath"
17
"sync"
@@ -28,11 +29,25 @@ var (
29
)
30
31
var (
31
- tsFSEnv = os.Getenv("GOLEVELDB_USEFS")
32
- tsKeepFS = tsFSEnv == "2"
33
- tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
34
- tsMU = &sync.Mutex{}
35
- tsNum = 0
32
+ tsFSEnv = os.Getenv("GOLEVELDB_USEFS")
33
+ tsTempdir = os.Getenv("GOLEVELDB_TEMPDIR")
34
+ tsKeepFS = tsFSEnv == "2"
35
+ tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
36
+ tsMU = &sync.Mutex{}
37
+ tsNum = 0
38
+)
39
+
40
+type tsOp uint
41
+
42
+const (
43
+ tsOpOpen tsOp = iota
44
+ tsOpCreate
45
+ tsOpRead
46
+ tsOpReadAt
47
+ tsOpWrite
48
+ tsOpSync
49
+
50
+ tsOpNum
51
)
52
53
type tsLock struct {
@@ -53,6 +68,9 @@ type tsReader struct {
68
func (tr tsReader) Read(b []byte) (n int, err error) {
69
ts := tr.tf.ts
70
ts.countRead(tr.tf.Type())
71
+ if tr.tf.shouldErrLocked(tsOpRead) {
72
+ return 0, errors.New("leveldb.testStorage: emulated read error")
73
+ }
74
n, err = tr.Reader.Read(b)
75
if err != nil && err != io.EOF {
76
ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
@@ -63,6 +81,9 @@ func (tr tsReader) Read(b []byte) (n int, err error) {
81
func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
82
ts := tr.tf.ts
83
ts.countRead(tr.tf.Type())
84
+ if tr.tf.shouldErrLocked(tsOpReadAt) {
85
+ return 0, errors.New("leveldb.testStorage: emulated readAt error")
86
+ }
87
n, err = tr.Reader.ReadAt(b, off)
88
if err != nil && err != io.EOF {
89
ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
@@ -82,15 +103,12 @@ type tsWriter struct {
103
}
104
105
func (tw tsWriter) Write(b []byte) (n int, err error) {
85
- ts := tw.tf.ts
86
- ts.mu.Lock()
87
- defer ts.mu.Unlock()
88
- if ts.emuWriteErr&tw.tf.Type() != 0 {
106
+ if tw.tf.shouldErrLocked(tsOpWrite) {
107
return 0, errors.New("leveldb.testStorage: emulated write error")
108
}
109
n, err = tw.Writer.Write(b)
110
if err != nil {
93
- ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
111
+ tw.tf.ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
112
}
113
return
114
}
@@ -98,23 +116,23 @@ func (tw tsWriter) Write(b []byte) (n int, err error) {
116
func (tw tsWriter) Sync() (err error) {
117
ts := tw.tf.ts
118
ts.mu.Lock()
101
- defer ts.mu.Unlock()
119
for ts.emuDelaySync&tw.tf.Type() != 0 {
120
ts.cond.Wait()
121
}
105
- if ts.emuSyncErr&tw.tf.Type() != 0 {
122
+ ts.mu.Unlock()
123
+ if tw.tf.shouldErrLocked(tsOpSync) {
124
return errors.New("leveldb.testStorage: emulated sync error")
125
}
126
err = tw.Writer.Sync()
127
if err != nil {
110
- ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
128
+ tw.tf.ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
129
}
130
return
131
}
132
133
func (tw tsWriter) Close() (err error) {
134
err = tw.Writer.Close()
117
- tw.tf.close("reader", err)
135
+ tw.tf.close("writer", err)
136
return
137
}
138
@@ -127,6 +145,16 @@ func (tf tsFile) x() uint64 {
145
return tf.Num()<<typeShift | uint64(tf.Type())
146
}
147
148
+func (tf tsFile) shouldErr(op tsOp) bool {
149
+ return tf.ts.shouldErr(tf, op)
150
+}
151
+
152
+func (tf tsFile) shouldErrLocked(op tsOp) bool {
153
+ tf.ts.mu.Lock()
154
+ defer tf.ts.mu.Unlock()
155
+ return tf.shouldErr(op)
156
+}
157
+
158
func (tf tsFile) checkOpen(m string) error {
159
ts := tf.ts
160
if writer, ok := ts.opens[tf.x()]; ok {
@@ -163,7 +191,7 @@ func (tf tsFile) Open() (r storage.Reader, err error) {
191
if err != nil {
192
return
193
}
166
- if ts.emuOpenErr&tf.Type() != 0 {
194
+ if tf.shouldErr(tsOpOpen) {
195
err = errors.New("leveldb.testStorage: emulated open error")
196
return
197
}
@@ -190,7 +218,7 @@ func (tf tsFile) Create() (w storage.Writer, err error) {
218
if err != nil {
219
return
220
}
193
- if ts.emuCreateErr&tf.Type() != 0 {
221
+ if tf.shouldErr(tsOpCreate) {
222
err = errors.New("leveldb.testStorage: emulated create error")
223
return
224
}
@@ -205,6 +233,23 @@ func (tf tsFile) Create() (w storage.Writer, err error) {
233
return
234
}
235
236
+func (tf tsFile) Replace(newfile storage.File) (err error) {
237
+ ts := tf.ts
238
+ ts.mu.Lock()
239
+ defer ts.mu.Unlock()
240
+ err = tf.checkOpen("replace")
241
+ if err != nil {
242
+ return
243
+ }
244
+ err = tf.File.Replace(newfile.(tsFile).File)
245
+ if err != nil {
246
+ ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
247
+ } else {
248
+ ts.t.Logf("I: file replace, num=%d type=%v", tf.Num(), tf.Type())
249
+ }
250
+ return
251
+}
252
+
253
func (tf tsFile) Remove() (err error) {
254
ts := tf.ts
255
ts.mu.Lock()
@@ -231,51 +276,75 @@ type testStorage struct {
276
cond sync.Cond
277
// Open files, true=writer, false=reader
278
opens map[uint64]bool
234
- emuOpenErr storage.FileType
235
- emuCreateErr storage.FileType
279
emuDelaySync storage.FileType
237
- emuWriteErr storage.FileType
238
- emuSyncErr storage.FileType
280
ignoreOpenErr storage.FileType
281
readCnt uint64
282
readCntEn storage.FileType
283
+
284
+ emuErr [tsOpNum]storage.FileType
285
+ emuErrOnce [tsOpNum]storage.FileType
286
+ emuRandErr [tsOpNum]storage.FileType
287
+ emuRandErrProb int
288
+ emuErrOnceMap map[uint64]uint
289
+ emuRandRand *rand.Rand
290
+}
291
+
292
+func (ts *testStorage) shouldErr(tf tsFile, op tsOp) bool {
293
+ if ts.emuErr[op]&tf.Type() != 0 {
294
+ return true
295
+ } else if ts.emuRandErr[op]&tf.Type() != 0 || ts.emuErrOnce[op]&tf.Type() != 0 {
296
+ sop := uint(1) << op
297
+ eop := ts.emuErrOnceMap[tf.x()]
298
+ if eop&sop == 0 && (ts.emuRandRand.Int()%ts.emuRandErrProb == 0 || ts.emuErrOnce[op]&tf.Type() != 0) {
299
+ ts.emuErrOnceMap[tf.x()] = eop | sop
300
+ ts.t.Logf("I: emulated error: file=%d type=%v op=%v", tf.Num(), tf.Type(), op)
301
+ return true
302
+ }
303
+ }
304
+ return false
305
}
306
244
-func (ts *testStorage) SetOpenErr(t storage.FileType) {
307
+func (ts *testStorage) SetEmuErr(t storage.FileType, ops ...tsOp) {
308
ts.mu.Lock()
246
- ts.emuOpenErr = t
309
+ for _, op := range ops {
310
+ ts.emuErr[op] = t
311
+ }
312
ts.mu.Unlock()
313
}
314
250
-func (ts *testStorage) SetCreateErr(t storage.FileType) {
315
+func (ts *testStorage) SetEmuErrOnce(t storage.FileType, ops ...tsOp) {
316
ts.mu.Lock()
252
- ts.emuCreateErr = t
317
+ for _, op := range ops {
318
+ ts.emuErrOnce[op] = t
319
+ }
320
ts.mu.Unlock()
321
}
322
256
-func (ts *testStorage) DelaySync(t storage.FileType) {
323
+func (ts *testStorage) SetEmuRandErr(t storage.FileType, ops ...tsOp) {
324
ts.mu.Lock()
258
- ts.emuDelaySync |= t
259
- ts.cond.Broadcast()
325
+ for _, op := range ops {
326
+ ts.emuRandErr[op] = t
327
+ }
328
ts.mu.Unlock()
329
}
330
263
-func (ts *testStorage) ReleaseSync(t storage.FileType) {
331
+func (ts *testStorage) SetEmuRandErrProb(prob int) {
332
ts.mu.Lock()
265
- ts.emuDelaySync &= ^t
266
- ts.cond.Broadcast()
333
+ ts.emuRandErrProb = prob
334
ts.mu.Unlock()
335
}
336
270
-func (ts *testStorage) SetWriteErr(t storage.FileType) {
337
+func (ts *testStorage) DelaySync(t storage.FileType) {
338
ts.mu.Lock()
272
- ts.emuWriteErr = t
339
+ ts.emuDelaySync |= t
340
+ ts.cond.Broadcast()
341
ts.mu.Unlock()
342
}
343
276
-func (ts *testStorage) SetSyncErr(t storage.FileType) {
344
+func (ts *testStorage) ReleaseSync(t storage.FileType) {
345
ts.mu.Lock()
278
- ts.emuSyncErr = t
346
+ ts.emuDelaySync &= ^t
347
+ ts.cond.Broadcast()
348
ts.mu.Unlock()
349
}
350
@@ -413,7 +482,11 @@ func newTestStorage(t *testing.T) *testStorage {
482
num := tsNum
483
tsNum++
484
tsMU.Unlock()
416
- path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
485
+ tempdir := tsTempdir
486
+ if tempdir == "" {
487
+ tempdir = os.TempDir()
488
+ }
489
+ path := filepath.Join(tempdir, fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
490
if _, err := os.Stat(path); err != nil {
491
stor, err = storage.OpenFile(path)
492
if err != nil {
@@ -436,6 +509,10 @@ func newTestStorage(t *testing.T) *testStorage {
509
}
510
f.Close()
511
}
512
+ if t.Failed() {
513
+ t.Logf("testing failed, test DB preserved at %s", path)
514
+ return nil
515
+ }
516
if tsKeepFS {
517
return nil
518
}
@@ -449,10 +526,13 @@ func newTestStorage(t *testing.T) *testStorage {
526
stor = storage.NewMemStorage()
527
}
528
ts := &testStorage{
452
- t: t,
453
- Storage: stor,
454
- closeFn: closeFn,
455
- opens: make(map[uint64]bool),
529
+ t: t,
530
+ Storage: stor,
531
+ closeFn: closeFn,
532
+ opens: make(map[uint64]bool),
533
+ emuErrOnceMap: make(map[uint64]uint),
534
+ emuRandErrProb: 0x999,
535
+ emuRandRand: rand.New(rand.NewSource(0xfacedead)),
536
}
537
ts.cond.L = &ts.mu
538
return ts
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go
+58
-22
@@ -7,6 +7,7 @@
7
package leveldb
8
9
import (
10
+ "fmt"
11
"sort"
12
"sync/atomic"
13
@@ -82,6 +83,18 @@ type tFiles []*tFile
83
func (tf tFiles) Len() int { return len(tf) }
84
func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
85
86
+func (tf tFiles) nums() string {
87
+ x := "[ "
88
+ for i, f := range tf {
89
+ if i != 0 {
90
+ x += ", "
91
+ }
92
+ x += fmt.Sprint(f.file.Num())
93
+ }
94
+ x += " ]"
95
+ return x
96
+}
97
+
98
// Returns true if i smallest key is less than j.
99
// This used for sort by key in ascending order.
100
func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
@@ -149,7 +162,7 @@ func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) boo
162
i := 0
163
if len(umin) > 0 {
164
// Find the earliest possible internal key for min.
152
- i = tf.searchMax(icmp, newIKey(umin, kMaxSeq, tSeek))
165
+ i = tf.searchMax(icmp, newIkey(umin, kMaxSeq, ktSeek))
166
}
167
if i >= len(tf) {
168
// Beginning of range is after all files, so no overlap.
@@ -159,24 +172,25 @@ func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) boo
172
}
173
174
// Returns tables whose its key range overlaps with given key range.
162
-// If overlapped is true then the search will be expanded to tables that
163
-// overlaps with each other.
175
+// Range will be expanded if ukey found hop across tables.
176
+// If overlapped is true then the search will be restarted if umax
177
+// expanded.
178
+// The dst content will be overwritten.
179
func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles {
165
- x := len(dst)
180
+ dst = dst[:0]
181
for i := 0; i < len(tf); {
182
t := tf[i]
183
if t.overlaps(icmp, umin, umax) {
169
- if overlapped {
170
- // For overlapped files, check if the newly added file has
171
- // expanded the range. If so, restart search.
172
- if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
173
- umin = t.imin.ukey()
174
- dst = dst[:x]
175
- i = 0
176
- continue
177
- } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
178
- umax = t.imax.ukey()
179
- dst = dst[:x]
184
+ if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
185
+ umin = t.imin.ukey()
186
+ dst = dst[:0]
187
+ i = 0
188
+ continue
189
+ } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
190
+ umax = t.imax.ukey()
191
+ // Restart search if it is overlapped.
192
+ if overlapped {
193
+ dst = dst[:0]
194
i = 0
195
continue
196
}
@@ -289,7 +303,7 @@ func (t *tOps) create() (*tWriter, error) {
303
t: t,
304
file: file,
305
w: fw,
292
- tw: table.NewWriter(fw, t.s.o),
306
+ tw: table.NewWriter(fw, t.s.o.Options),
307
}, nil
308
}
309
@@ -337,7 +351,13 @@ func (t *tOps) open(f *tFile) (ch cache.Handle, err error) {
351
if bc := t.s.o.GetBlockCache(); bc != nil {
352
bcacheNS = bc.GetNamespace(num)
353
}
340
- return 1, table.NewReader(r, int64(f.size), bcacheNS, t.bpool, t.s.o)
354
+ var tr *table.Reader
355
+ tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcacheNS, t.bpool, t.s.o.Options)
356
+ if err != nil {
357
+ r.Close()
358
+ return 0, nil
359
+ }
360
+ return 1, tr
361
})
362
if ch == nil && err == nil {
363
err = ErrClosed
@@ -353,7 +373,17 @@ func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []b
373
return nil, nil, err
374
}
375
defer ch.Release()
356
- return ch.Value().(*table.Reader).Find(key, ro)
376
+ return ch.Value().(*table.Reader).Find(key, true, ro)
377
+}
378
+
379
+// Finds key that is greater than or equal to the given key.
380
+func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) {
381
+ ch, err := t.open(f)
382
+ if err != nil {
383
+ return nil, err
384
+ }
385
+ defer ch.Release()
386
+ return ch.Value().(*table.Reader).FindKey(key, true, ro)
387
}
388
389
// Returns approximate offset of the given key.
@@ -440,28 +470,34 @@ func (w *tWriter) empty() bool {
470
return w.first == nil
471
}
472
473
+// Closes the storage.Writer.
474
+func (w *tWriter) close() {
475
+ if w.w != nil {
476
+ w.w.Close()
477
+ w.w = nil
478
+ }
479
+}
480
+
481
// Finalizes the table and returns table file.
482
func (w *tWriter) finish() (f *tFile, err error) {
483
+ defer w.close()
484
err = w.tw.Close()
485
if err != nil {
486
return
487
}
488
err = w.w.Sync()
489
if err != nil {
451
- w.w.Close()
490
return
491
}
454
- w.w.Close()
492
f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
493
return
494
}
495
496
// Drops the table.
497
func (w *tWriter) drop() {
461
- w.w.Close()
498
+ w.close()
499
w.file.Remove()
500
w.t.s.reuseFileNum(w.file.Num())
464
- w.w = nil
501
w.file = nil
502
w.tw = nil
503
w.first = nil
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go
+19
-11
@@ -19,13 +19,18 @@ import (
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
20
)
21
22
-func (b *block) TestNewIterator(slice *util.Range) iterator.Iterator {
23
- return b.newIterator(slice, false, nil)
22
+type blockTesting struct {
23
+ tr *Reader
24
+ b *block
25
+}
26
+
27
+func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator {
28
+ return t.tr.newBlockIter(t.b, nil, slice, false)
29
}
30
31
var _ = testutil.Defer(func() {
32
Describe("Block", func() {
28
- Build := func(kv *testutil.KeyValue, restartInterval int) *block {
33
+ Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting {
34
// Building the block.
35
bw := &blockWriter{
36
restartInterval: restartInterval,
@@ -39,11 +44,13 @@ var _ = testutil.Defer(func() {
44
// Opening the block.
45
data := bw.buf.Bytes()
46
restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
42
- return &block{
43
- tr: &Reader{cmp: comparer.DefaultComparer},
44
- data: data,
45
- restartsLen: restartsLen,
46
- restartsOffset: len(data) - (restartsLen+1)*4,
47
+ return &blockTesting{
48
+ tr: &Reader{cmp: comparer.DefaultComparer},
49
+ b: &block{
50
+ data: data,
51
+ restartsLen: restartsLen,
52
+ restartsOffset: len(data) - (restartsLen+1)*4,
53
+ },
54
}
55
}
56
@@ -59,7 +66,7 @@ var _ = testutil.Defer(func() {
66
// Make block.
67
br := Build(kv, restartInterval)
68
// Do testing.
62
- testutil.KeyValueTesting(nil, br, kv.Clone())
69
+ testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil)
70
}
71
72
Describe(Text(), Test)
@@ -102,11 +109,11 @@ var _ = testutil.Defer(func() {
109
for restartInterval := 1; restartInterval <= 5; restartInterval++ {
110
Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
111
// Make block.
105
- br := Build(kv, restartInterval)
112
+ bt := Build(kv, restartInterval)
113
114
Test := func(r *util.Range) func(done Done) {
115
return func(done Done) {
109
- iter := br.newIterator(r, false, nil)
116
+ iter := bt.TestNewIterator(r)
117
Expect(iter.Error()).ShouldNot(HaveOccurred())
118
119
t := testutil.IteratorTesting{
@@ -115,6 +122,7 @@ var _ = testutil.Defer(func() {
122
}
123
124
testutil.DoIteratorTesting(&t)
125
+ iter.Release()
126
done <- true
127
}
128
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go
+348
-183
@@ -8,27 +8,41 @@ package table
8
9
import (
10
"encoding/binary"
11
- "errors"
11
"fmt"
12
"io"
13
"sort"
14
"strings"
15
+ "sync"
16
17
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy"
17
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
18
19
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
20
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
21
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
22
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
23
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
24
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
25
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
26
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
27
)
28
29
var (
28
- ErrNotFound = util.ErrNotFound
29
- ErrIterReleased = errors.New("leveldb/table: iterator released")
30
+ ErrNotFound = errors.ErrNotFound
31
+ ErrReaderReleased = errors.New("leveldb/table: reader released")
32
+ ErrIterReleased = errors.New("leveldb/table: iterator released")
33
)
34
35
+type ErrCorrupted struct {
36
+ Pos int64
37
+ Size int64
38
+ Kind string
39
+ Reason string
40
+}
41
+
42
+func (e *ErrCorrupted) Error() string {
43
+ return fmt.Sprintf("leveldb/table: corruption on %s (pos=%d): %s", e.Kind, e.Pos, e.Reason)
44
+}
45
+
46
func max(x, y int) int {
47
if x > y {
48
return x
@@ -37,22 +51,21 @@ func max(x, y int) int {
51
}
52
53
type block struct {
40
- tr *Reader
54
+ bpool *util.BufferPool
55
+ bh blockHandle
56
data []byte
57
restartsLen int
58
restartsOffset int
44
- // Whether checksum is verified and valid.
45
- checksum bool
59
}
60
48
-func (b *block) seek(rstart, rlimit int, key []byte) (index, offset int, err error) {
61
+func (b *block) seek(cmp comparer.Comparer, rstart, rlimit int, key []byte) (index, offset int, err error) {
62
index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
63
offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):]))
64
offset += 1 // shared always zero, since this is a restart point
65
v1, n1 := binary.Uvarint(b.data[offset:]) // key length
66
_, n2 := binary.Uvarint(b.data[offset+n1:]) // value length
67
m := offset + n1 + n2
55
- return b.tr.cmp.Compare(b.data[m:m+int(v1)], key) > 0
68
+ return cmp.Compare(b.data[m:m+int(v1)], key) > 0
69
}) + rstart - 1
70
if index < rstart {
71
// The smallest key is greater-than key sought.
@@ -75,7 +88,7 @@ func (b *block) restartOffset(index int) int {
88
func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) {
89
if offset >= b.restartsOffset {
90
if offset != b.restartsOffset {
78
- err = errors.New("leveldb/table: Reader: BlockEntry: invalid block (block entries offset not aligned)")
91
+ err = &ErrCorrupted{Reason: "entries offset not aligned"}
92
}
93
return
94
}
@@ -85,7 +98,7 @@ func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error)
98
m := n0 + n1 + n2
99
n = m + int(v1) + int(v2)
100
if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset {
88
- err = errors.New("leveldb/table: Reader: invalid block (block entries corrupted)")
101
+ err = &ErrCorrupted{Reason: "entries corrupted"}
102
return
103
}
104
key = b.data[offset+m : offset+m+int(v1)]
@@ -94,50 +107,9 @@ func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error)
107
return
108
}
109
97
-func (b *block) newIterator(slice *util.Range, inclLimit bool, cache util.Releaser) *blockIter {
98
- bi := &blockIter{
99
- block: b,
100
- cache: cache,
101
- // Valid key should never be nil.
102
- key: make([]byte, 0),
103
- dir: dirSOI,
104
- riStart: 0,
105
- riLimit: b.restartsLen,
106
- offsetStart: 0,
107
- offsetRealStart: 0,
108
- offsetLimit: b.restartsOffset,
109
- }
110
- if slice != nil {
111
- if slice.Start != nil {
112
- if bi.Seek(slice.Start) {
113
- bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset)
114
- bi.offsetStart = b.restartOffset(bi.riStart)
115
- bi.offsetRealStart = bi.prevOffset
116
- } else {
117
- bi.riStart = b.restartsLen
118
- bi.offsetStart = b.restartsOffset
119
- bi.offsetRealStart = b.restartsOffset
120
- }
121
- }
122
- if slice.Limit != nil {
123
- if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) {
124
- bi.offsetLimit = bi.prevOffset
125
- bi.riLimit = bi.restartIndex + 1
126
- }
127
- }
128
- bi.reset()
129
- if bi.offsetStart > bi.offsetLimit {
130
- bi.sErr(errors.New("leveldb/table: Reader: invalid slice range"))
131
- }
132
- }
133
- return bi
134
-}
135
-
110
func (b *block) Release() {
137
- if b.tr.bpool != nil {
138
- b.tr.bpool.Put(b.data)
139
- }
140
- b.tr = nil
111
+ b.bpool.Put(b.data)
112
+ b.bpool = nil
113
b.data = nil
114
}
115
@@ -152,10 +124,12 @@ const (
124
)
125
126
type blockIter struct {
155
- block *block
156
- cache, releaser util.Releaser
157
- key, value []byte
158
- offset int
127
+ tr *Reader
128
+ block *block
129
+ blockReleaser util.Releaser
130
+ releaser util.Releaser
131
+ key, value []byte
132
+ offset int
133
// Previous offset, only filled by Next.
134
prevOffset int
135
prevNode []int
@@ -252,7 +226,7 @@ func (i *blockIter) Seek(key []byte) bool {
226
return false
227
}
228
255
- ri, offset, err := i.block.seek(i.riStart, i.riLimit, key)
229
+ ri, offset, err := i.block.seek(i.tr.cmp, i.riStart, i.riLimit, key)
230
if err != nil {
231
i.sErr(err)
232
return false
@@ -263,7 +237,7 @@ func (i *blockIter) Seek(key []byte) bool {
237
i.dir = dirForward
238
}
239
for i.Next() {
266
- if i.block.tr.cmp.Compare(i.key, key) >= 0 {
240
+ if i.tr.cmp.Compare(i.key, key) >= 0 {
241
return true
242
}
243
}
@@ -288,7 +262,7 @@ func (i *blockIter) Next() bool {
262
for i.offset < i.offsetRealStart {
263
key, value, nShared, n, err := i.block.entry(i.offset)
264
if err != nil {
291
- i.sErr(err)
265
+ i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
266
return false
267
}
268
if n == 0 {
@@ -302,13 +276,13 @@ func (i *blockIter) Next() bool {
276
if i.offset >= i.offsetLimit {
277
i.dir = dirEOI
278
if i.offset != i.offsetLimit {
305
- i.sErr(errors.New("leveldb/table: Reader: Next: invalid block (block entries offset not aligned)"))
279
+ i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned"))
280
}
281
return false
282
}
283
key, value, nShared, n, err := i.block.entry(i.offset)
284
if err != nil {
311
- i.sErr(err)
285
+ i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
286
return false
287
}
288
if n == 0 {
@@ -393,7 +367,7 @@ func (i *blockIter) Prev() bool {
367
for {
368
key, value, nShared, n, err := i.block.entry(offset)
369
if err != nil {
396
- i.sErr(err)
370
+ i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
371
return false
372
}
373
if offset >= i.offsetRealStart {
@@ -412,7 +386,7 @@ func (i *blockIter) Prev() bool {
386
// Stop if target offset reached.
387
if offset >= i.offset {
388
if offset != i.offset {
415
- i.sErr(errors.New("leveldb/table: Reader: Prev: invalid block (block entries offset not aligned)"))
389
+ i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned"))
390
return false
391
}
392
@@ -439,16 +413,17 @@ func (i *blockIter) Value() []byte {
413
}
414
415
func (i *blockIter) Release() {
442
- if i.dir > dirReleased {
416
+ if i.dir != dirReleased {
417
+ i.tr = nil
418
i.block = nil
419
i.prevNode = nil
420
i.prevKeys = nil
421
i.key = nil
422
i.value = nil
423
i.dir = dirReleased
449
- if i.cache != nil {
450
- i.cache.Release()
451
- i.cache = nil
424
+ if i.blockReleaser != nil {
425
+ i.blockReleaser.Release()
426
+ i.blockReleaser = nil
427
}
428
if i.releaser != nil {
429
i.releaser.Release()
@@ -458,9 +433,13 @@ func (i *blockIter) Release() {
433
}
434
435
func (i *blockIter) SetReleaser(releaser util.Releaser) {
461
- if i.dir > dirReleased {
462
- i.releaser = releaser
436
+ if i.dir == dirReleased {
437
+ panic(util.ErrReleased)
438
}
439
+ if i.releaser != nil && releaser != nil {
440
+ panic(util.ErrHasReleaser)
441
+ }
442
+ i.releaser = releaser
443
}
444
445
func (i *blockIter) Valid() bool {
@@ -472,21 +451,21 @@ func (i *blockIter) Error() error {
451
}
452
453
type filterBlock struct {
475
- tr *Reader
454
+ bpool *util.BufferPool
455
data []byte
456
oOffset int
457
baseLg uint
458
filtersNum int
459
}
460
482
-func (b *filterBlock) contains(offset uint64, key []byte) bool {
461
+func (b *filterBlock) contains(filter filter.Filter, offset uint64, key []byte) bool {
462
i := int(offset >> b.baseLg)
463
if i < b.filtersNum {
464
o := b.data[b.oOffset+i*4:]
465
n := int(binary.LittleEndian.Uint32(o))
466
m := int(binary.LittleEndian.Uint32(o[4:]))
467
if n < m && m <= b.oOffset {
489
- return b.tr.filter.Contains(b.data[n:m], key)
468
+ return filter.Contains(b.data[n:m], key)
469
} else if n == m {
470
return false
471
}
@@ -495,18 +474,16 @@ func (b *filterBlock) contains(offset uint64, key []byte) bool {
474
}
475
476
func (b *filterBlock) Release() {
498
- if b.tr.bpool != nil {
499
- b.tr.bpool.Put(b.data)
500
- }
501
- b.tr = nil
477
+ b.bpool.Put(b.data)
478
+ b.bpool = nil
479
b.data = nil
480
}
481
482
type indexIter struct {
483
*blockIter
484
+ tr *Reader
485
slice *util.Range
486
// Options
509
- checksum bool
487
fillCache bool
488
}
489
@@ -517,87 +494,124 @@ func (i *indexIter) Get() iterator.Iterator {
494
}
495
dataBH, n := decodeBlockHandle(value)
496
if n == 0 {
520
- return iterator.NewEmptyIterator(errors.New("leveldb/table: Reader: invalid table (bad data block handle)"))
497
+ return iterator.NewEmptyIterator(i.tr.newErrCorruptedBH(i.tr.indexBH, "bad data block handle"))
498
}
499
+
500
var slice *util.Range
501
if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
502
slice = i.slice
503
}
526
- return i.blockIter.block.tr.getDataIter(dataBH, slice, i.checksum, i.fillCache)
504
+ return i.tr.getDataIterErr(dataBH, slice, i.tr.verifyChecksum, i.fillCache)
505
}
506
507
// Reader is a table reader.
508
type Reader struct {
509
+ mu sync.RWMutex
510
+ fi *storage.FileInfo
511
reader io.ReaderAt
512
cache cache.Namespace
513
err error
514
bpool *util.BufferPool
515
// Options
536
- cmp comparer.Comparer
537
- filter filter.Filter
538
- checksum bool
539
- strictIter bool
516
+ o *opt.Options
517
+ cmp comparer.Comparer
518
+ filter filter.Filter
519
+ verifyChecksum bool
520
+
521
+ dataEnd int64
522
+ metaBH, indexBH, filterBH blockHandle
523
+ indexBlock *block
524
+ filterBlock *filterBlock
525
+}
526
+
527
+func (r *Reader) blockKind(bh blockHandle) string {
528
+ switch bh.offset {
529
+ case r.metaBH.offset:
530
+ return "meta-block"
531
+ case r.indexBH.offset:
532
+ return "index-block"
533
+ case r.filterBH.offset:
534
+ if r.filterBH.length > 0 {
535
+ return "filter-block"
536
+ }
537
+ }
538
+ return "data-block"
539
+}
540
+
541
+func (r *Reader) newErrCorrupted(pos, size int64, kind, reason string) error {
542
+ return &errors.ErrCorrupted{File: r.fi, Err: &ErrCorrupted{Pos: pos, Size: size, Kind: kind, Reason: reason}}
543
+}
544
541
- dataEnd int64
542
- indexBH, filterBH blockHandle
545
+func (r *Reader) newErrCorruptedBH(bh blockHandle, reason string) error {
546
+ return r.newErrCorrupted(int64(bh.offset), int64(bh.length), r.blockKind(bh), reason)
547
}
548
545
-func verifyChecksum(data []byte) bool {
546
- n := len(data) - 4
547
- checksum0 := binary.LittleEndian.Uint32(data[n:])
548
- checksum1 := util.NewCRC(data[:n]).Value()
549
- return checksum0 == checksum1
549
+func (r *Reader) fixErrCorruptedBH(bh blockHandle, err error) error {
550
+ if cerr, ok := err.(*ErrCorrupted); ok {
551
+ cerr.Pos = int64(bh.offset)
552
+ cerr.Size = int64(bh.length)
553
+ cerr.Kind = r.blockKind(bh)
554
+ return &errors.ErrCorrupted{File: r.fi, Err: cerr}
555
+ }
556
+ return err
557
}
558
552
-func (r *Reader) readRawBlock(bh blockHandle, checksum bool) ([]byte, error) {
559
+func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, error) {
560
data := r.bpool.Get(int(bh.length + blockTrailerLen))
561
if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
562
return nil, err
563
}
557
- if checksum || r.checksum {
558
- if !verifyChecksum(data) {
564
+
565
+ if verifyChecksum {
566
+ n := bh.length + 1
567
+ checksum0 := binary.LittleEndian.Uint32(data[n:])
568
+ checksum1 := util.NewCRC(data[:n]).Value()
569
+ if checksum0 != checksum1 {
570
r.bpool.Put(data)
560
- return nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
571
+ return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("checksum mismatch, want=%#x got=%#x", checksum0, checksum1))
572
}
573
}
574
+
575
switch data[bh.length] {
576
case blockTypeNoCompression:
577
data = data[:bh.length]
578
case blockTypeSnappyCompression:
579
decLen, err := snappy.DecodedLen(data[:bh.length])
580
if err != nil {
569
- return nil, err
581
+ return nil, r.newErrCorruptedBH(bh, err.Error())
582
}
571
- tmp := data
572
- data, err = snappy.Decode(r.bpool.Get(decLen), tmp[:bh.length])
573
- r.bpool.Put(tmp)
583
+ decData := r.bpool.Get(decLen)
584
+ decData, err = snappy.Decode(decData, data[:bh.length])
585
+ r.bpool.Put(data)
586
if err != nil {
575
- return nil, err
587
+ r.bpool.Put(decData)
588
+ return nil, r.newErrCorruptedBH(bh, err.Error())
589
}
590
+ data = decData
591
default:
592
r.bpool.Put(data)
579
- return nil, fmt.Errorf("leveldb/table: Reader: unknown block compression type: %d", data[bh.length])
593
+ return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("unknown compression type %#x", data[bh.length]))
594
}
595
return data, nil
596
}
597
584
-func (r *Reader) readBlock(bh blockHandle, checksum bool) (*block, error) {
585
- data, err := r.readRawBlock(bh, checksum)
598
+func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) {
599
+ data, err := r.readRawBlock(bh, verifyChecksum)
600
if err != nil {
601
return nil, err
602
}
603
restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
604
b := &block{
591
- tr: r,
605
+ bpool: r.bpool,
606
+ bh: bh,
607
data: data,
608
restartsLen: restartsLen,
609
restartsOffset: len(data) - (restartsLen+1)*4,
595
- checksum: checksum || r.checksum,
610
}
611
return b, nil
612
}
613
600
-func (r *Reader) readBlockCached(bh blockHandle, checksum, fillCache bool) (*block, util.Releaser, error) {
614
+func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) {
615
if r.cache != nil {
616
var err error
617
ch := r.cache.Get(bh.offset, func() (charge int, value interface{}) {
@@ -605,7 +619,7 @@ func (r *Reader) readBlockCached(bh blockHandle, checksum, fillCache bool) (*blo
619
return 0, nil
620
}
621
var b *block
608
- b, err = r.readBlock(bh, checksum)
622
+ b, err = r.readBlock(bh, verifyChecksum)
623
if err != nil {
624
return 0, nil
625
}
@@ -615,14 +629,7 @@ func (r *Reader) readBlockCached(bh blockHandle, checksum, fillCache bool) (*blo
629
b, ok := ch.Value().(*block)
630
if !ok {
631
ch.Release()
618
- return nil, nil, errors.New("leveldb/table: Reader: inconsistent block type")
619
- }
620
- if !b.checksum && (r.checksum || checksum) {
621
- if !verifyChecksum(b.data) {
622
- ch.Release()
623
- return nil, nil, errors.New("leveldb/table: Reader: invalid block (checksum mismatch)")
624
- }
625
- b.checksum = true
632
+ return nil, nil, errors.New("leveldb/table: inconsistent block type")
633
}
634
return b, ch, err
635
} else if err != nil {
@@ -630,7 +637,7 @@ func (r *Reader) readBlockCached(bh blockHandle, checksum, fillCache bool) (*blo
637
}
638
}
639
633
- b, err := r.readBlock(bh, checksum)
640
+ b, err := r.readBlock(bh, verifyChecksum)
641
return b, b, err
642
}
643
@@ -641,15 +648,15 @@ func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) {
648
}
649
n := len(data)
650
if n < 5 {
644
- return nil, errors.New("leveldb/table: Reader: invalid filter block (too short)")
651
+ return nil, r.newErrCorruptedBH(bh, "too short")
652
}
653
m := n - 5
654
oOffset := int(binary.LittleEndian.Uint32(data[m:]))
655
if oOffset > m {
649
- return nil, errors.New("leveldb/table: Reader: invalid filter block (invalid offset)")
656
+ return nil, r.newErrCorruptedBH(bh, "invalid data-offsets offset")
657
}
658
b := &filterBlock{
652
- tr: r,
659
+ bpool: r.bpool,
660
data: data,
661
oOffset: oOffset,
662
baseLg: uint(data[n-1]),
@@ -676,7 +683,7 @@ func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterB
683
b, ok := ch.Value().(*filterBlock)
684
if !ok {
685
ch.Release()
679
- return nil, nil, errors.New("leveldb/table: Reader: inconsistent block type")
686
+ return nil, nil, errors.New("leveldb/table: inconsistent block type")
687
}
688
return b, ch, err
689
} else if err != nil {
@@ -688,12 +695,77 @@ func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterB
695
return b, b, err
696
}
697
691
-func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fillCache bool) iterator.Iterator {
692
- b, rel, err := r.readBlockCached(dataBH, checksum, fillCache)
698
+func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) {
699
+ if r.indexBlock == nil {
700
+ return r.readBlockCached(r.indexBH, true, fillCache)
701
+ }
702
+ return r.indexBlock, util.NoopReleaser{}, nil
703
+}
704
+
705
+func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) {
706
+ if r.filterBlock == nil {
707
+ return r.readFilterBlockCached(r.filterBH, fillCache)
708
+ }
709
+ return r.filterBlock, util.NoopReleaser{}, nil
710
+}
711
+
712
+func (r *Reader) newBlockIter(b *block, bReleaser util.Releaser, slice *util.Range, inclLimit bool) *blockIter {
713
+ bi := &blockIter{
714
+ tr: r,
715
+ block: b,
716
+ blockReleaser: bReleaser,
717
+ // Valid key should never be nil.
718
+ key: make([]byte, 0),
719
+ dir: dirSOI,
720
+ riStart: 0,
721
+ riLimit: b.restartsLen,
722
+ offsetStart: 0,
723
+ offsetRealStart: 0,
724
+ offsetLimit: b.restartsOffset,
725
+ }
726
+ if slice != nil {
727
+ if slice.Start != nil {
728
+ if bi.Seek(slice.Start) {
729
+ bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset)
730
+ bi.offsetStart = b.restartOffset(bi.riStart)
731
+ bi.offsetRealStart = bi.prevOffset
732
+ } else {
733
+ bi.riStart = b.restartsLen
734
+ bi.offsetStart = b.restartsOffset
735
+ bi.offsetRealStart = b.restartsOffset
736
+ }
737
+ }
738
+ if slice.Limit != nil {
739
+ if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) {
740
+ bi.offsetLimit = bi.prevOffset
741
+ bi.riLimit = bi.restartIndex + 1
742
+ }
743
+ }
744
+ bi.reset()
745
+ if bi.offsetStart > bi.offsetLimit {
746
+ bi.sErr(errors.New("leveldb/table: invalid slice range"))
747
+ }
748
+ }
749
+ return bi
750
+}
751
+
752
+func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator {
753
+ b, rel, err := r.readBlockCached(dataBH, verifyChecksum, fillCache)
754
if err != nil {
755
return iterator.NewEmptyIterator(err)
756
}
696
- return b.newIterator(slice, false, rel)
757
+ return r.newBlockIter(b, rel, slice, false)
758
+}
759
+
760
+func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator {
761
+ r.mu.RLock()
762
+ defer r.mu.RUnlock()
763
+
764
+ if r.err != nil {
765
+ return iterator.NewEmptyIterator(r.err)
766
+ }
767
+
768
+ return r.getDataIter(dataBH, slice, verifyChecksum, fillCache)
769
}
770
771
// NewIterator creates an iterator from the table.
@@ -708,43 +780,43 @@ func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, checksum, fi
780
//
781
// Also read Iterator documentation of the leveldb/iterator package.
782
func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
783
+ r.mu.RLock()
784
+ defer r.mu.RUnlock()
785
+
786
if r.err != nil {
787
return iterator.NewEmptyIterator(r.err)
788
}
789
790
fillCache := !ro.GetDontFillCache()
716
- b, rel, err := r.readBlockCached(r.indexBH, true, fillCache)
791
+ indexBlock, rel, err := r.getIndexBlock(fillCache)
792
if err != nil {
793
return iterator.NewEmptyIterator(err)
794
}
795
index := &indexIter{
721
- blockIter: b.newIterator(slice, true, rel),
796
+ blockIter: r.newBlockIter(indexBlock, rel, slice, true),
797
+ tr: r,
798
slice: slice,
723
- checksum: ro.GetStrict(opt.StrictBlockChecksum),
799
fillCache: !ro.GetDontFillCache(),
800
}
726
- return iterator.NewIndexedIterator(index, r.strictIter || ro.GetStrict(opt.StrictIterator), false)
801
+ return iterator.NewIndexedIterator(index, opt.GetStrict(r.o, ro, opt.StrictReader))
802
}
803
729
-// Find finds key/value pair whose key is greater than or equal to the
730
-// given key. It returns ErrNotFound if the table doesn't contain
731
-// such pair.
732
-//
733
-// The caller should not modify the contents of the returned slice, but
734
-// it is safe to modify the contents of the argument after Find returns.
735
-func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err error) {
804
+func (r *Reader) find(key []byte, filtered bool, ro *opt.ReadOptions, noValue bool) (rkey, value []byte, err error) {
805
+ r.mu.RLock()
806
+ defer r.mu.RUnlock()
807
+
808
if r.err != nil {
809
err = r.err
810
return
811
}
812
741
- indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true)
813
+ indexBlock, rel, err := r.getIndexBlock(true)
814
if err != nil {
815
return
816
}
817
defer rel.Release()
818
747
- index := indexBlock.newIterator(nil, true, nil)
819
+ index := r.newBlockIter(indexBlock, nil, nil, true)
820
defer index.Release()
821
if !index.Seek(key) {
822
err = index.Error()
@@ -755,20 +827,23 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
827
}
828
dataBH, n := decodeBlockHandle(index.Value())
829
if n == 0 {
758
- err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
830
+ r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle")
831
return
832
}
761
- if r.filter != nil {
762
- filterBlock, rel, ferr := r.readFilterBlockCached(r.filterBH, true)
833
+ if filtered && r.filter != nil {
834
+ filterBlock, frel, ferr := r.getFilterBlock(true)
835
if ferr == nil {
764
- if !filterBlock.contains(dataBH.offset, key) {
765
- rel.Release()
836
+ if !filterBlock.contains(r.filter, dataBH.offset, key) {
837
+ frel.Release()
838
return nil, nil, ErrNotFound
839
}
768
- rel.Release()
840
+ frel.Release()
841
+ } else if !errors.IsCorrupted(ferr) {
842
+ err = ferr
843
+ return
844
}
845
}
771
- data := r.getDataIter(dataBH, nil, ro.GetStrict(opt.StrictBlockChecksum), !ro.GetDontFillCache())
846
+ data := r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache())
847
defer data.Release()
848
if !data.Seek(key) {
849
err = data.Error()
@@ -779,24 +854,62 @@ func (r *Reader) Find(key []byte, ro *opt.ReadOptions) (rkey, value []byte, err
854
}
855
// Don't use block buffer, no need to copy the buffer.
856
rkey = data.Key()
782
- // Use block buffer, and since the buffer will be recycled, the buffer
783
- // need to be copied.
784
- value = append([]byte{}, data.Value()...)
857
+ if !noValue {
858
+ if r.bpool == nil {
859
+ value = data.Value()
860
+ } else {
861
+ // Use block buffer, and since the buffer will be recycled, the buffer
862
+ // need to be copied.
863
+ value = append([]byte{}, data.Value()...)
864
+ }
865
+ }
866
+ return
867
+}
868
+
869
+// Find finds key/value pair whose key is greater than or equal to the
870
+// given key. It returns ErrNotFound if the table doesn't contain
871
+// such pair.
872
+// If filtered is true then the nearest 'block' will be checked against
873
+// 'filter data' (if present) and will immediately return ErrNotFound if
874
+// 'filter data' indicates that such pair doesn't exist.
875
+//
876
+// The caller may modify the contents of the returned slice as it is its
877
+// own copy.
878
+// It is safe to modify the contents of the argument after Find returns.
879
+func (r *Reader) Find(key []byte, filtered bool, ro *opt.ReadOptions) (rkey, value []byte, err error) {
880
+ return r.find(key, filtered, ro, false)
881
+}
882
+
883
+// Find finds key that is greater than or equal to the given key.
884
+// It returns ErrNotFound if the table doesn't contain such key.
885
+// If filtered is true then the nearest 'block' will be checked against
886
+// 'filter data' (if present) and will immediately return ErrNotFound if
887
+// 'filter data' indicates that such key doesn't exist.
888
+//
889
+// The caller may modify the contents of the returned slice as it is its
890
+// own copy.
891
+// It is safe to modify the contents of the argument after Find returns.
892
+func (r *Reader) FindKey(key []byte, filtered bool, ro *opt.ReadOptions) (rkey []byte, err error) {
893
+ rkey, _, err = r.find(key, filtered, ro, true)
894
return
895
}
896
897
// Get gets the value for the given key. It returns errors.ErrNotFound
898
// if the table does not contain the key.
899
//
791
-// The caller should not modify the contents of the returned slice, but
792
-// it is safe to modify the contents of the argument after Get returns.
900
+// The caller may modify the contents of the returned slice as it is its
901
+// own copy.
902
+// It is safe to modify the contents of the argument after Find returns.
903
func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
904
+ r.mu.RLock()
905
+ defer r.mu.RUnlock()
906
+
907
if r.err != nil {
908
err = r.err
909
return
910
}
911
799
- rkey, value, err := r.Find(key, ro)
912
+ rkey, value, err := r.find(key, false, ro, false)
913
if err == nil && r.cmp.Compare(rkey, key) != 0 {
914
value = nil
915
err = ErrNotFound
@@ -808,6 +921,9 @@ func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
921
//
922
// It is safe to modify the contents of the argument after Get returns.
923
func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
924
+ r.mu.RLock()
925
+ defer r.mu.RUnlock()
926
+
927
if r.err != nil {
928
err = r.err
929
return
@@ -819,12 +935,12 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
935
}
936
defer rel.Release()
937
822
- index := indexBlock.newIterator(nil, true, nil)
938
+ index := r.newBlockIter(indexBlock, nil, nil, true)
939
defer index.Release()
940
if index.Seek(key) {
941
dataBH, n := decodeBlockHandle(index.Value())
942
if n == 0 {
827
- err = errors.New("leveldb/table: Reader: invalid table (bad data block handle)")
943
+ r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle")
944
return
945
}
946
offset = int64(dataBH.offset)
@@ -840,67 +956,91 @@ func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
956
// Release implements util.Releaser.
957
// It also close the file if it is an io.Closer.
958
func (r *Reader) Release() {
959
+ r.mu.Lock()
960
+ defer r.mu.Unlock()
961
+
962
if closer, ok := r.reader.(io.Closer); ok {
963
closer.Close()
964
}
965
+ if r.indexBlock != nil {
966
+ r.indexBlock.Release()
967
+ r.indexBlock = nil
968
+ }
969
+ if r.filterBlock != nil {
970
+ r.filterBlock.Release()
971
+ r.filterBlock = nil
972
+ }
973
r.reader = nil
974
r.cache = nil
975
r.bpool = nil
976
+ r.err = ErrReaderReleased
977
}
978
979
// NewReader creates a new initialized table reader for the file.
852
-// The cache and bpool is optional and can be nil.
980
+// The fi, cache and bpool is optional and can be nil.
981
//
982
// The returned table reader instance is goroutine-safe.
855
-func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) *Reader {
856
- if bpool == nil {
857
- bpool = util.NewBufferPool(o.GetBlockSize() + blockTrailerLen)
983
+func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache cache.Namespace, bpool *util.BufferPool, o *opt.Options) (*Reader, error) {
984
+ if f == nil {
985
+ return nil, errors.New("leveldb/table: nil file")
986
}
987
+
988
r := &Reader{
860
- reader: f,
861
- cache: cache,
862
- bpool: bpool,
863
- cmp: o.GetComparer(),
864
- checksum: o.GetStrict(opt.StrictBlockChecksum),
865
- strictIter: o.GetStrict(opt.StrictIterator),
866
- }
867
- if f == nil {
868
- r.err = errors.New("leveldb/table: Reader: nil file")
869
- return r
989
+ fi: fi,
990
+ reader: f,
991
+ cache: cache,
992
+ bpool: bpool,
993
+ o: o,
994
+ cmp: o.GetComparer(),
995
+ verifyChecksum: o.GetStrict(opt.StrictBlockChecksum),
996
}
997
+
998
if size < footerLen {
872
- r.err = errors.New("leveldb/table: Reader: invalid table (file size is too small)")
873
- return r
999
+ r.err = r.newErrCorrupted(0, size, "table", "too small")
1000
+ return r, nil
1001
}
1002
+
1003
+ footerPos := size - footerLen
1004
var footer [footerLen]byte
876
- if _, err := r.reader.ReadAt(footer[:], size-footerLen); err != nil && err != io.EOF {
877
- r.err = fmt.Errorf("leveldb/table: Reader: invalid table (could not read footer): %v", err)
1005
+ if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF {
1006
+ return nil, err
1007
}
1008
if string(footer[footerLen-len(magic):footerLen]) != magic {
880
- r.err = errors.New("leveldb/table: Reader: invalid table (bad magic number)")
881
- return r
1009
+ r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number")
1010
+ return r, nil
1011
}
1012
+
1013
+ var n int
1014
// Decode the metaindex block handle.
884
- metaBH, n := decodeBlockHandle(footer[:])
1015
+ r.metaBH, n = decodeBlockHandle(footer[:])
1016
if n == 0 {
886
- r.err = errors.New("leveldb/table: Reader: invalid table (bad metaindex block handle)")
887
- return r
1017
+ r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle")
1018
+ return r, nil
1019
}
1020
+
1021
// Decode the index block handle.
1022
r.indexBH, n = decodeBlockHandle(footer[n:])
1023
if n == 0 {
892
- r.err = errors.New("leveldb/table: Reader: invalid table (bad index block handle)")
893
- return r
1024
+ r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle")
1025
+ return r, nil
1026
}
1027
+
1028
// Read metaindex block.
896
- metaBlock, err := r.readBlock(metaBH, true)
1029
+ metaBlock, err := r.readBlock(r.metaBH, true)
1030
if err != nil {
898
- r.err = err
899
- return r
1031
+ if errors.IsCorrupted(err) {
1032
+ r.err = err
1033
+ return r, nil
1034
+ } else {
1035
+ return nil, err
1036
+ }
1037
}
1038
+
1039
// Set data end.
902
- r.dataEnd = int64(metaBH.offset)
903
- metaIter := metaBlock.newIterator(nil, false, nil)
1040
+ r.dataEnd = int64(r.metaBH.offset)
1041
+
1042
+ // Read metaindex.
1043
+ metaIter := r.newBlockIter(metaBlock, nil, nil, true)
1044
for metaIter.Next() {
1045
key := string(metaIter.Key())
1046
if !strings.HasPrefix(key, "filter.") {
@@ -930,5 +1070,30 @@ func NewReader(f io.ReaderAt, size int64, cache cache.Namespace, bpool *util.Buf
1070
}
1071
metaIter.Release()
1072
metaBlock.Release()
933
- return r
1073
+
1074
+ // Cache index and filter block locally, since we don't have global cache.
1075
+ if cache == nil {
1076
+ r.indexBlock, err = r.readBlock(r.indexBH, true)
1077
+ if err != nil {
1078
+ if errors.IsCorrupted(err) {
1079
+ r.err = err
1080
+ return r, nil
1081
+ } else {
1082
+ return nil, err
1083
+ }
1084
+ }
1085
+ if r.filter != nil {
1086
+ r.filterBlock, err = r.readFilterBlock(r.filterBH)
1087
+ if err != nil {
1088
+ if !errors.IsCorrupted(err) {
1089
+ return nil, err
1090
+ }
1091
+
1092
+ // Don't use filter then.
1093
+ r.filter = nil
1094
+ }
1095
+ }
1096
+ }
1097
+
1098
+ return r, nil
1099
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go
+3
-3
@@ -133,9 +133,9 @@ Filter block trailer:
133
134
+- 4-bytes -+
135
/ \
136
- +---------------+---------------+---------------+-------------------------+------------------+
137
- | offset 1 | .... | offset n | filter offset (4-bytes) | base Lg (1-byte) |
138
- +-------------- +---------------+---------------+-------------------------+------------------+
136
+ +---------------+---------------+---------------+-------------------------------+------------------+
137
+ | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) |
138
+ +-------------- +---------------+---------------+-------------------------------+------------------+
139
140
141
NOTE: All fixed-length integer are little-endian.
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go
+1
-7
@@ -3,15 +3,9 @@ package table
3
import (
4
"testing"
5
6
- . "github.com/onsi/ginkgo"
7
- . "github.com/onsi/gomega"
8
-
6
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7
)
8
9
func TestTable(t *testing.T) {
13
- testutil.RunDefer()
14
-
15
- RegisterFailHandler(Fail)
16
- RunSpecs(t, "Table Suite")
10
+ testutil.RunSuite(t, "Table Suite")
11
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go
+6
-5
@@ -23,7 +23,7 @@ type tableWrapper struct {
23
}
24
25
func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) {
26
- return t.Reader.Find(key, nil)
26
+ return t.Reader.Find(key, false, nil)
27
}
28
29
func (t tableWrapper) TestGet(key []byte) (value []byte, err error) {
@@ -59,7 +59,8 @@ var _ = testutil.Defer(func() {
59
It("Should be able to approximate offset of a key correctly", func() {
60
Expect(err).ShouldNot(HaveOccurred())
61
62
- tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
62
+ tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
63
+ Expect(err).ShouldNot(HaveOccurred())
64
CheckOffset := func(key string, expect, threshold int) {
65
offset, err := tr.OffsetOf([]byte(key))
66
Expect(err).ShouldNot(HaveOccurred())
@@ -95,7 +96,7 @@ var _ = testutil.Defer(func() {
96
tw.Close()
97
98
// Opening the table.
98
- tr := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, o)
99
+ tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
100
return tableWrapper{tr}
101
}
102
Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
@@ -104,11 +105,11 @@ var _ = testutil.Defer(func() {
105
if body != nil {
106
body(db.(tableWrapper).Reader)
107
}
107
- testutil.KeyValueTesting(nil, db, *kv)
108
+ testutil.KeyValueTesting(nil, *kv, db, nil, nil)
109
}
110
}
111
111
- testutil.AllKeyValueTesting(nil, Build)
112
+ testutil.AllKeyValueTesting(nil, Build, nil, nil)
113
Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
114
It("should have correct blocks number", func() {
115
indexBlock, err := r.readBlock(r.indexBH, true)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go
+1
-1
@@ -12,7 +12,7 @@ import (
12
"fmt"
13
"io"
14
15
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/snappy-go/snappy"
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
16
17
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
18
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go
+7
-1
@@ -12,6 +12,7 @@ import (
12
13
. "github.com/onsi/gomega"
14
15
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18
)
@@ -34,6 +35,10 @@ type Get interface {
35
TestGet(key []byte) (value []byte, err error)
36
}
37
38
+type Has interface {
39
+ TestHas(key []byte) (ret bool, err error)
40
+}
41
+
42
type NewIterator interface {
43
TestNewIterator(slice *util.Range) iterator.Iterator
44
}
@@ -110,7 +115,7 @@ func (t *DBTesting) TestAllPresent() {
115
116
func (t *DBTesting) TestDeletedKey(key []byte) {
117
_, err := t.DB.TestGet(key)
113
- Expect(err).Should(Equal(util.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
118
+ Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
119
}
120
121
func (t *DBTesting) TestAllDeleted() {
@@ -212,5 +217,6 @@ func DoDBTesting(t *DBTesting) {
217
}
218
219
DoIteratorTesting(&it)
220
+ iter.Release()
221
}
222
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go
new
+21
@@ -0,0 +1,21 @@
1
+package testutil
2
+
3
+import (
4
+ . "github.com/onsi/ginkgo"
5
+ . "github.com/onsi/gomega"
6
+)
7
+
8
+func RunSuite(t GinkgoTestingT, name string) {
9
+ RunDefer()
10
+
11
+ SynchronizedBeforeSuite(func() []byte {
12
+ RunDefer("setup")
13
+ return nil
14
+ }, func(data []byte) {})
15
+ SynchronizedAfterSuite(func() {
16
+ RunDefer("teardown")
17
+ }, func() {})
18
+
19
+ RegisterFailHandler(Fail)
20
+ RunSpecs(t, name)
21
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go
+96
-45
@@ -13,16 +13,28 @@ import (
13
. "github.com/onsi/ginkgo"
14
. "github.com/onsi/gomega"
15
16
+ "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
17
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18
)
19
19
-func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
20
+func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
21
if rnd == nil {
22
rnd = NewRand()
23
}
24
24
- if db, ok := p.(Find); ok {
25
- It("Should find all keys with Find", func() {
25
+ if p == nil {
26
+ BeforeEach(func() {
27
+ p = setup(kv)
28
+ })
29
+ if teardown != nil {
30
+ AfterEach(func() {
31
+ teardown(p)
32
+ })
33
+ }
34
+ }
35
+
36
+ It("Should find all keys with Find", func() {
37
+ if db, ok := p.(Find); ok {
38
ShuffledIndex(nil, kv.Len(), 1, func(i int) {
39
key_, key, value := kv.IndexInexact(i)
40
@@ -38,9 +50,11 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
50
Expect(rkey).Should(Equal(key))
51
Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
52
})
41
- })
53
+ }
54
+ })
55
43
- It("Should return error if the key is not present", func() {
56
+ It("Should return error if the key is not present", func() {
57
+ if db, ok := p.(Find); ok {
58
var key []byte
59
if kv.Len() > 0 {
60
key_, _ := kv.Index(kv.Len() - 1)
@@ -48,12 +62,12 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
62
}
63
rkey, _, err := db.TestFind(key)
64
Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
51
- Expect(err).Should(Equal(util.ErrNotFound))
52
- })
53
- }
65
+ Expect(err).Should(Equal(errors.ErrNotFound))
66
+ }
67
+ })
68
55
- if db, ok := p.(Get); ok {
56
- It("Should only find exact key with Get", func() {
69
+ It("Should only find exact key with Get", func() {
70
+ if db, ok := p.(Get); ok {
71
ShuffledIndex(nil, kv.Len(), 1, func(i int) {
72
key_, key, value := kv.IndexInexact(i)
73
@@ -66,14 +80,34 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
80
if len(key_) > 0 {
81
_, err = db.TestGet(key_)
82
Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
69
- Expect(err).Should(Equal(util.ErrNotFound))
83
+ Expect(err).Should(Equal(errors.ErrNotFound))
84
}
85
})
72
- })
73
- }
86
+ }
87
+ })
88
+
89
+ It("Should only find present key with Has", func() {
90
+ if db, ok := p.(Has); ok {
91
+ ShuffledIndex(nil, kv.Len(), 1, func(i int) {
92
+ key_, key, _ := kv.IndexInexact(i)
93
+
94
+ // Using exact key.
95
+ ret, err := db.TestHas(key)
96
+ Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
97
+ Expect(ret).Should(BeTrue(), "False for key %q", key)
98
75
- if db, ok := p.(NewIterator); ok {
76
- TestIter := func(r *util.Range, _kv KeyValue) {
99
+ // Using inexact key.
100
+ if len(key_) > 0 {
101
+ ret, err = db.TestHas(key_)
102
+ Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_)
103
+ Expect(ret).ShouldNot(BeTrue(), "True for key %q", key)
104
+ }
105
+ })
106
+ }
107
+ })
108
+
109
+ TestIter := func(r *util.Range, _kv KeyValue) {
110
+ if db, ok := p.(NewIterator); ok {
111
iter := db.TestNewIterator(r)
112
Expect(iter.Error()).ShouldNot(HaveOccurred())
113
@@ -83,46 +117,62 @@ func KeyValueTesting(rnd *rand.Rand, p DB, kv KeyValue) {
117
}
118
119
DoIteratorTesting(&t)
120
+ iter.Release()
121
}
122
+ }
123
88
- It("Should iterates and seeks correctly", func(done Done) {
89
- TestIter(nil, kv.Clone())
90
- done <- true
91
- }, 3.0)
92
-
93
- RandomIndex(rnd, kv.Len(), kv.Len(), func(i int) {
94
- type slice struct {
95
- r *util.Range
96
- start, limit int
97
- }
124
+ It("Should iterates and seeks correctly", func(done Done) {
125
+ TestIter(nil, kv.Clone())
126
+ done <- true
127
+ }, 3.0)
128
99
- key_, _, _ := kv.IndexInexact(i)
100
- for _, x := range []slice{
101
- {&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
102
- {&util.Range{Start: nil, Limit: key_}, 0, i},
103
- } {
104
- It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
105
- TestIter(x.r, kv.Slice(x.start, x.limit))
106
- done <- true
107
- }, 3.0)
108
- }
109
- })
129
+ RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) {
130
+ type slice struct {
131
+ r *util.Range
132
+ start, limit int
133
+ }
134
111
- RandomRange(rnd, kv.Len(), kv.Len(), func(start, limit int) {
112
- It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
113
- r := kv.Range(start, limit)
114
- TestIter(&r, kv.Slice(start, limit))
135
+ key_, _, _ := kv.IndexInexact(i)
136
+ for _, x := range []slice{
137
+ {&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
138
+ {&util.Range{Start: nil, Limit: key_}, 0, i},
139
+ } {
140
+ It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
141
+ TestIter(x.r, kv.Slice(x.start, x.limit))
142
done <- true
143
}, 3.0)
117
- })
118
- }
144
+ }
145
+ })
146
+
147
+ RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) {
148
+ It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
149
+ r := kv.Range(start, limit)
150
+ TestIter(&r, kv.Slice(start, limit))
151
+ done <- true
152
+ }, 3.0)
153
+ })
154
}
155
121
-func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
156
+func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
157
Test := func(kv *KeyValue) func() {
158
return func() {
124
- db := body(*kv)
125
- KeyValueTesting(rnd, db, *kv)
159
+ var p DB
160
+ if setup != nil {
161
+ Defer("setup", func() {
162
+ p = setup(*kv)
163
+ })
164
+ }
165
+ if teardown != nil {
166
+ Defer("teardown", func() {
167
+ teardown(p)
168
+ })
169
+ }
170
+ if body != nil {
171
+ p = body(*kv)
172
+ }
173
+ KeyValueTesting(rnd, *kv, p, func(KeyValue) DB {
174
+ return p
175
+ }, nil)
176
}
177
}
178
@@ -133,4 +183,5 @@ func AllKeyValueTesting(rnd *rand.Rand, body func(kv KeyValue) DB) {
183
Describe("with big value", Test(KeyValue_BigValue()))
184
Describe("with special key", Test(KeyValue_SpecialKey()))
185
Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
186
+ Describe("with generated key/value", Test(KeyValue_Generate(nil, 120, 1, 50, 10, 120)))
187
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go
+1
@@ -397,6 +397,7 @@ func (s *Storage) logI(format string, args ...interface{}) {
397
398
func (s *Storage) Log(str string) {
399
s.log(1, "Log: "+str)
400
+ s.Storage.Log(str)
401
}
402
403
func (s *Storage) Lock() (r util.Releaser, err error) {
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go
+14
@@ -155,3 +155,17 @@ func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) {
155
}
156
return
157
}
158
+
159
+func Max(x, y int) int {
160
+ if x > y {
161
+ return x
162
+ }
163
+ return y
164
+}
165
+
166
+func Min(x, y int) int {
167
+ if x < y {
168
+ return x
169
+ }
170
+ return y
171
+}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go
+4
@@ -34,6 +34,10 @@ func (t *testingDB) TestGet(key []byte) (value []byte, err error) {
34
return t.Get(key, t.ro)
35
}
36
37
+func (t *testingDB) TestHas(key []byte) (ret bool, err error) {
38
+ return t.Has(key, t.ro)
39
+}
40
+
41
func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator {
42
return t.NewIterator(slice, t.ro)
43
}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go
+2
-2
@@ -14,10 +14,10 @@ import (
14
)
15
16
func shorten(str string) string {
17
- if len(str) <= 4 {
17
+ if len(str) <= 8 {
18
return str
19
}
20
- return str[:1] + ".." + str[len(str)-1:]
20
+ return str[:3] + ".." + str[len(str)-3:]
21
}
22
23
var bunits = [...]string{"", "Ki", "Mi", "Gi"}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go
+49
-16
@@ -8,6 +8,7 @@ package util
8
9
import (
10
"fmt"
11
+ "sync"
12
"sync/atomic"
13
"time"
14
)
@@ -19,17 +20,16 @@ type buffer struct {
20
21
// BufferPool is a 'buffer pool'.
22
type BufferPool struct {
22
- pool [6]chan []byte
23
- size [5]uint32
24
- sizeMiss [5]uint32
25
- sizeHalf [5]uint32
26
- baseline [4]int
27
- baselinex0 int
28
- baselinex1 int
29
- baseline0 int
30
- baseline1 int
31
- baseline2 int
32
- close chan struct{}
23
+ pool [6]chan []byte
24
+ size [5]uint32
25
+ sizeMiss [5]uint32
26
+ sizeHalf [5]uint32
27
+ baseline [4]int
28
+ baseline0 int
29
+
30
+ mu sync.RWMutex
31
+ closed bool
32
+ closeC chan struct{}
33
34
get uint32
35
put uint32
@@ -54,6 +54,17 @@ func (p *BufferPool) poolNum(n int) int {
54
55
// Get returns buffer with length of n.
56
func (p *BufferPool) Get(n int) []byte {
57
+ if p == nil {
58
+ return make([]byte, n)
59
+ }
60
+
61
+ p.mu.RLock()
62
+ defer p.mu.RUnlock()
63
+
64
+ if p.closed {
65
+ return make([]byte, n)
66
+ }
67
+
68
atomic.AddUint32(&p.get, 1)
69
70
poolNum := p.poolNum(n)
@@ -145,6 +156,17 @@ func (p *BufferPool) Get(n int) []byte {
156
157
// Put adds given buffer to the pool.
158
func (p *BufferPool) Put(b []byte) {
159
+ if p == nil {
160
+ return
161
+ }
162
+
163
+ p.mu.RLock()
164
+ defer p.mu.RUnlock()
165
+
166
+ if p.closed {
167
+ return
168
+ }
169
+
170
atomic.AddUint32(&p.put, 1)
171
172
pool := p.pool[p.poolNum(cap(b))]
@@ -156,13 +178,23 @@ func (p *BufferPool) Put(b []byte) {
178
}
179
180
func (p *BufferPool) Close() {
159
- select {
160
- case p.close <- struct{}{}:
161
- default:
181
+ if p == nil {
182
+ return
183
}
184
+
185
+ p.mu.Lock()
186
+ if !p.closed {
187
+ p.closed = true
188
+ p.closeC <- struct{}{}
189
+ }
190
+ p.mu.Unlock()
191
}
192
193
func (p *BufferPool) String() string {
194
+ if p == nil {
195
+ return "<nil>"
196
+ }
197
+
198
return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}",
199
p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss)
200
}
@@ -178,7 +210,8 @@ func (p *BufferPool) drain() {
210
default:
211
}
212
}
181
- case <-p.close:
213
+ case <-p.closeC:
214
+ close(p.closeC)
215
for _, ch := range p.pool {
216
close(ch)
217
}
@@ -195,7 +228,7 @@ func NewBufferPool(baseline int) *BufferPool {
228
p := &BufferPool{
229
baseline0: baseline,
230
baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4},
198
- close: make(chan struct{}, 1),
231
+ closeC: make(chan struct{}, 1),
232
}
233
for i, cap := range []int{2, 2, 4, 4, 2, 1} {
234
p.pool[i] = make(chan []byte, cap)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go
+28
-4
@@ -12,7 +12,8 @@ import (
12
)
13
14
var (
15
- ErrNotFound = errors.New("leveldb: not found")
15
+ ErrReleased = errors.New("leveldb: resource already relesed")
16
+ ErrHasReleaser = errors.New("leveldb: releaser already defined")
17
)
18
19
// Releaser is the interface that wraps the basic Release method.
@@ -27,23 +28,46 @@ type ReleaseSetter interface {
28
// SetReleaser associates the given releaser to the resources. The
29
// releaser will be called once coresponding resources released.
30
// Calling SetReleaser with nil will clear the releaser.
31
+ //
32
+ // This will panic if a releaser already present or coresponding
33
+ // resource is already released. Releaser should be cleared first
34
+ // before assigned a new one.
35
SetReleaser(releaser Releaser)
36
}
37
38
// BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
39
type BasicReleaser struct {
40
releaser Releaser
41
+ released bool
42
+}
43
+
44
+// Released returns whether Release method already called.
45
+func (r *BasicReleaser) Released() bool {
46
+ return r.released
47
}
48
49
// Release implements Releaser.Release.
50
func (r *BasicReleaser) Release() {
40
- if r.releaser != nil {
41
- r.releaser.Release()
42
- r.releaser = nil
51
+ if !r.released {
52
+ if r.releaser != nil {
53
+ r.releaser.Release()
54
+ r.releaser = nil
55
+ }
56
+ r.released = true
57
}
58
}
59
60
// SetReleaser implements ReleaseSetter.SetReleaser.
61
func (r *BasicReleaser) SetReleaser(releaser Releaser) {
62
+ if r.released {
63
+ panic(ErrReleased)
64
+ }
65
+ if r.releaser != nil && releaser != nil {
66
+ panic(ErrHasReleaser)
67
+ }
68
r.releaser = releaser
69
}
70
+
71
+type NoopReleaser struct{}
72
+
73
+func (NoopReleaser) Release() {}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go
+71
-71
@@ -7,7 +7,6 @@
7
package leveldb
8
9
import (
10
- "errors"
10
"sync/atomic"
11
"unsafe"
12
@@ -16,19 +15,6 @@ import (
15
"github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16
)
17
19
-var levelMaxSize [kNumLevels]float64
20
-
21
-func init() {
22
- // Precompute max size of each level
23
- for level := range levelMaxSize {
24
- res := float64(10 * 1048576)
25
- for n := level; n > 1; n-- {
26
- res *= 10
27
- }
28
- levelMaxSize[level] = res
29
- }
30
-}
31
-
18
type tSet struct {
19
level int
20
table *tFile
@@ -37,7 +23,7 @@ type tSet struct {
23
type version struct {
24
s *session
25
40
- tables [kNumLevels]tFiles
26
+ tables []tFiles
27
28
// Level that should be compacted next and its compaction score.
29
// Score < 1 means compaction is not strictly needed. These fields
@@ -47,11 +33,16 @@ type version struct {
33
34
cSeek unsafe.Pointer
35
50
- ref int
36
+ ref int
37
+ // Succeeding version.
38
next *version
39
}
40
54
-func (v *version) release_NB() {
41
+func newVersion(s *session) *version {
42
+ return &version{s: s, tables: make([]tFiles, s.o.GetNumLevel())}
43
+}
44
+
45
+func (v *version) releaseNB() {
46
v.ref--
47
if v.ref > 0 {
48
return
@@ -77,13 +68,13 @@ func (v *version) release_NB() {
68
}
69
}
70
80
- v.next.release_NB()
71
+ v.next.releaseNB()
72
v.next = nil
73
}
74
75
func (v *version) release() {
76
v.s.vmu.Lock()
86
- v.release_NB()
77
+ v.releaseNB()
78
v.s.vmu.Unlock()
79
}
80
@@ -123,17 +114,18 @@ func (v *version) walkOverlapping(ikey iKey, f func(level int, t *tFile) bool, l
114
}
115
}
116
126
-func (v *version) get(ikey iKey, ro *opt.ReadOptions) (value []byte, tcomp bool, err error) {
117
+func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) {
118
ukey := ikey.ukey()
119
120
var (
121
tset *tSet
122
tseek bool
123
133
- l0found bool
134
- l0seq uint64
135
- l0vt vType
136
- l0val []byte
124
+ // Level-0.
125
+ zfound bool
126
+ zseq uint64
127
+ zkt kType
128
+ zval []byte
129
)
130
131
err = ErrNotFound
@@ -150,55 +142,60 @@ func (v *version) get(ikey iKey, ro *opt.ReadOptions) (value []byte, tcomp bool,
142
}
143
}
144
153
- ikey__, val_, err_ := v.s.tops.find(t, ikey, ro)
154
- switch err_ {
145
+ var (
146
+ fikey, fval []byte
147
+ ferr error
148
+ )
149
+ if noValue {
150
+ fikey, ferr = v.s.tops.findKey(t, ikey, ro)
151
+ } else {
152
+ fikey, fval, ferr = v.s.tops.find(t, ikey, ro)
153
+ }
154
+ switch ferr {
155
case nil:
156
case ErrNotFound:
157
return true
158
default:
159
- err = err_
159
+ err = ferr
160
return false
161
}
162
163
- ikey_ := iKey(ikey__)
164
- if seq, vt, ok := ikey_.parseNum(); ok {
165
- if v.s.icmp.uCompare(ukey, ikey_.ukey()) != 0 {
166
- return true
167
- }
168
-
169
- if level == 0 {
170
- if seq >= l0seq {
171
- l0found = true
172
- l0seq = seq
173
- l0vt = vt
174
- l0val = val_
175
- }
176
- } else {
177
- switch vt {
178
- case tVal:
179
- value = val_
180
- err = nil
181
- case tDel:
182
- default:
183
- panic("leveldb: invalid internal key type")
163
+ if fukey, fseq, fkt, fkerr := parseIkey(fikey); fkerr == nil {
164
+ if v.s.icmp.uCompare(ukey, fukey) == 0 {
165
+ if level == 0 {
166
+ if fseq >= zseq {
167
+ zfound = true
168
+ zseq = fseq
169
+ zkt = fkt
170
+ zval = fval
171
+ }
172
+ } else {
173
+ switch fkt {
174
+ case ktVal:
175
+ value = fval
176
+ err = nil
177
+ case ktDel:
178
+ default:
179
+ panic("leveldb: invalid iKey type")
180
+ }
181
+ return false
182
}
185
- return false
183
}
184
} else {
188
- err = errors.New("leveldb: internal key corrupted")
185
+ err = fkerr
186
return false
187
}
188
189
return true
190
}, func(level int) bool {
194
- if l0found {
195
- switch l0vt {
196
- case tVal:
197
- value = l0val
191
+ if zfound {
192
+ switch zkt {
193
+ case ktVal:
194
+ value = zval
195
err = nil
199
- case tDel:
196
+ case ktDel:
197
default:
201
- panic("leveldb: invalid internal key type")
198
+ panic("leveldb: invalid iKey type")
199
}
200
return false
201
}
@@ -216,13 +213,13 @@ func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []it
213
its = append(its, it)
214
}
215
219
- strict := v.s.o.GetStrict(opt.StrictIterator) || ro.GetStrict(opt.StrictIterator)
216
+ strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader)
217
for _, tables := range v.tables[1:] {
218
if len(tables) == 0 {
219
continue
220
}
221
225
- it := iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict, true)
222
+ it := iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict)
223
its = append(its, it)
224
}
225
@@ -230,7 +227,7 @@ func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []it
227
}
228
229
func (v *version) newStaging() *versionStaging {
233
- return &versionStaging{base: v}
230
+ return &versionStaging{base: v, tables: make([]tablesScratch, v.s.o.GetNumLevel())}
231
}
232
233
// Spawn a new version based on this version.
@@ -285,12 +282,13 @@ func (v *version) offsetOf(ikey iKey) (n uint64, err error) {
282
func (v *version) pickLevel(umin, umax []byte) (level int) {
283
if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) {
284
var overlaps tFiles
288
- for ; level < kMaxMemCompactLevel; level++ {
285
+ maxLevel := v.s.o.GetMaxMemCompationLevel()
286
+ for ; level < maxLevel; level++ {
287
if v.tables[level+1].overlaps(v.s.icmp, umin, umax, false) {
288
break
289
}
290
overlaps = v.tables[level+2].getOverlaps(overlaps, v.s.icmp, umin, umax, false)
293
- if overlaps.size() > kMaxGrandParentOverlapBytes {
291
+ if overlaps.size() > uint64(v.s.o.GetCompactionGPOverlaps(level)) {
292
break
293
}
294
}
@@ -318,9 +316,9 @@ func (v *version) computeCompaction() {
316
// file size is small (perhaps because of a small write-buffer
317
// setting, or very high compression ratios, or lots of
318
// overwrites/deletions).
321
- score = float64(len(tables)) / kL0_CompactionTrigger
319
+ score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger())
320
} else {
323
- score = float64(tables.size()) / levelMaxSize[level]
321
+ score = float64(tables.size()) / float64(v.s.o.GetCompactionTotalSize(level))
322
}
323
324
if score > bestScore {
@@ -337,12 +335,14 @@ func (v *version) needCompaction() bool {
335
return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
336
}
337
338
+type tablesScratch struct {
339
+ added map[uint64]atRecord
340
+ deleted map[uint64]struct{}
341
+}
342
+
343
type versionStaging struct {
344
base *version
342
- tables [kNumLevels]struct {
343
- added map[uint64]ntRecord
344
- deleted map[uint64]struct{}
345
- }
345
+ tables []tablesScratch
346
}
347
348
func (p *versionStaging) commit(r *sessionRecord) {
@@ -367,7 +367,7 @@ func (p *versionStaging) commit(r *sessionRecord) {
367
tm := &(p.tables[r.level])
368
369
if tm.added == nil {
370
- tm.added = make(map[uint64]ntRecord)
370
+ tm.added = make(map[uint64]atRecord)
371
}
372
tm.added[r.num] = r
373
@@ -379,7 +379,7 @@ func (p *versionStaging) commit(r *sessionRecord) {
379
380
func (p *versionStaging) finish() *version {
381
// Build new version.
382
- nv := &version{s: p.base.s}
382
+ nv := newVersion(p.base.s)
383
for level, tm := range p.tables {
384
btables := p.base.tables[level]
385
@@ -402,7 +402,7 @@ func (p *versionStaging) finish() *version {
402
403
// New tables.
404
for _, r := range tm.added {
405
- nt = append(nt, r.makeFile(p.base.s))
405
+ nt = append(nt, p.base.s.tableFileFromRecord(r))
406
}
407
408
// Sort tables.
@@ -429,7 +429,7 @@ func (vr *versionReleaser) Release() {
429
v := vr.v
430
v.s.vmu.Lock()
431
if !vr.once {
432
- v.release_NB()
432
+ v.releaseNB()
433
vr.once = true
434
}
435
v.s.vmu.Unlock()