@cryptotaxi247 / kubo / commits / 35cdc1336

Move goleveldb to gx

License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>

Jakub Sztandera committed Jun 8, 2016 at 19:53 UTC 35cdc1336fb15a529ec44a401a4d30ca918459c3
92 files changed +10 -22364
Godeps/Godeps.json
-4
@@ -61,10 +61,6 @@
61 "ImportPath": "github.com/mtchavez/jenkins",
62 "Rev": "5a816af6ef21ef401bff5e4b7dd255d63400f497"
63 },
64 - {
65 - "ImportPath": "github.com/syndtr/goleveldb/leveldb",
66 - "Rev": "4875955338b0a434238a31165cb87255ab6e9e4a"
67 - },
64 {
65 "ImportPath": "github.com/syndtr/gosnappy/snappy",
66 "Rev": "156a073208e131d7d2e212cb749feae7c339e846"
Godeps/_workspace/src/github.com/ipfs/go-datastore/Godeps/Godeps.json
-4
@@ -55,10 +55,6 @@
55 "ImportPath": "github.com/mattbaird/elastigo/core",
56 "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
57 },
58 - {
59 - "ImportPath": "github.com/syndtr/goleveldb/leveldb",
60 - "Rev": "871eee0a7546bb7d1b2795142e29c4534abc49b3"
61 - },
58 {
59 "ImportPath": "github.com/syndtr/gosnappy/snappy",
60 "Rev": "ce8acff4829e0c2458a67ead32390ac0a381c862"
Godeps/_workspace/src/github.com/ipfs/go-datastore/leveldb/datastore.go
+3 -3
@@ -3,9 +3,9 @@ package leveldb
3 import (
4 ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore"
5 dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/query"
6 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb"
7 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
8 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
6 + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb"
7 + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
8 + "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/util"
9
10 "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
11 )
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch.go deleted
-252
@@ -1,252 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "encoding/binary"
11 - "fmt"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15 -)
16 -
17 -type ErrBatchCorrupted struct {
18 - Reason string
19 -}
20 -
21 -func (e *ErrBatchCorrupted) Error() string {
22 - return fmt.Sprintf("leveldb: batch corrupted: %s", e.Reason)
23 -}
24 -
25 -func newErrBatchCorrupted(reason string) error {
26 - return errors.NewErrCorrupted(nil, &ErrBatchCorrupted{reason})
27 -}
28 -
29 -const (
30 - batchHdrLen = 8 + 4
31 - batchGrowRec = 3000
32 -)
33 -
34 -type BatchReplay interface {
35 - Put(key, value []byte)
36 - Delete(key []byte)
37 -}
38 -
39 -// Batch is a write batch.
40 -type Batch struct {
41 - data []byte
42 - rLen, bLen int
43 - seq uint64
44 - sync bool
45 -}
46 -
47 -func (b *Batch) grow(n int) {
48 - off := len(b.data)
49 - if off == 0 {
50 - off = batchHdrLen
51 - if b.data != nil {
52 - b.data = b.data[:off]
53 - }
54 - }
55 - if cap(b.data)-off < n {
56 - if b.data == nil {
57 - b.data = make([]byte, off, off+n)
58 - } else {
59 - odata := b.data
60 - div := 1
61 - if b.rLen > batchGrowRec {
62 - div = b.rLen / batchGrowRec
63 - }
64 - b.data = make([]byte, off, off+n+(off-batchHdrLen)/div)
65 - copy(b.data, odata)
66 - }
67 - }
68 -}
69 -
70 -func (b *Batch) appendRec(kt kType, key, value []byte) {
71 - n := 1 + binary.MaxVarintLen32 + len(key)
72 - if kt == ktVal {
73 - n += binary.MaxVarintLen32 + len(value)
74 - }
75 - b.grow(n)
76 - off := len(b.data)
77 - data := b.data[:off+n]
78 - data[off] = byte(kt)
79 - off += 1
80 - off += binary.PutUvarint(data[off:], uint64(len(key)))
81 - copy(data[off:], key)
82 - off += len(key)
83 - if kt == ktVal {
84 - off += binary.PutUvarint(data[off:], uint64(len(value)))
85 - copy(data[off:], value)
86 - off += len(value)
87 - }
88 - b.data = data[:off]
89 - b.rLen++
90 - // Include 8-byte ikey header
91 - b.bLen += len(key) + len(value) + 8
92 -}
93 -
94 -// Put appends 'put operation' of the given key/value pair to the batch.
95 -// It is safe to modify the contents of the argument after Put returns.
96 -func (b *Batch) Put(key, value []byte) {
97 - b.appendRec(ktVal, key, value)
98 -}
99 -
100 -// Delete appends 'delete operation' of the given key to the batch.
101 -// It is safe to modify the contents of the argument after Delete returns.
102 -func (b *Batch) Delete(key []byte) {
103 - b.appendRec(ktDel, key, nil)
104 -}
105 -
106 -// Dump dumps batch contents. The returned slice can be loaded into the
107 -// batch using Load method.
108 -// The returned slice is not its own copy, so the contents should not be
109 -// modified.
110 -func (b *Batch) Dump() []byte {
111 - return b.encode()
112 -}
113 -
114 -// Load loads given slice into the batch. Previous contents of the batch
115 -// will be discarded.
116 -// The given slice will not be copied and will be used as batch buffer, so
117 -// it is not safe to modify the contents of the slice.
118 -func (b *Batch) Load(data []byte) error {
119 - return b.decode(0, data)
120 -}
121 -
122 -// Replay replays batch contents.
123 -func (b *Batch) Replay(r BatchReplay) error {
124 - return b.decodeRec(func(i int, kt kType, key, value []byte) {
125 - switch kt {
126 - case ktVal:
127 - r.Put(key, value)
128 - case ktDel:
129 - r.Delete(key)
130 - }
131 - })
132 -}
133 -
134 -// Len returns number of records in the batch.
135 -func (b *Batch) Len() int {
136 - return b.rLen
137 -}
138 -
139 -// Reset resets the batch.
140 -func (b *Batch) Reset() {
141 - b.data = b.data[:0]
142 - b.seq = 0
143 - b.rLen = 0
144 - b.bLen = 0
145 - b.sync = false
146 -}
147 -
148 -func (b *Batch) init(sync bool) {
149 - b.sync = sync
150 -}
151 -
152 -func (b *Batch) append(p *Batch) {
153 - if p.rLen > 0 {
154 - b.grow(len(p.data) - batchHdrLen)
155 - b.data = append(b.data, p.data[batchHdrLen:]...)
156 - b.rLen += p.rLen
157 - }
158 - if p.sync {
159 - b.sync = true
160 - }
161 -}
162 -
163 -// size returns sums of key/value pair length plus 8-bytes ikey.
164 -func (b *Batch) size() int {
165 - return b.bLen
166 -}
167 -
168 -func (b *Batch) encode() []byte {
169 - b.grow(0)
170 - binary.LittleEndian.PutUint64(b.data, b.seq)
171 - binary.LittleEndian.PutUint32(b.data[8:], uint32(b.rLen))
172 -
173 - return b.data
174 -}
175 -
176 -func (b *Batch) decode(prevSeq uint64, data []byte) error {
177 - if len(data) < batchHdrLen {
178 - return newErrBatchCorrupted("too short")
179 - }
180 -
181 - b.seq = binary.LittleEndian.Uint64(data)
182 - if b.seq < prevSeq {
183 - return newErrBatchCorrupted("invalid sequence number")
184 - }
185 - b.rLen = int(binary.LittleEndian.Uint32(data[8:]))
186 - if b.rLen < 0 {
187 - return newErrBatchCorrupted("invalid records length")
188 - }
189 - // No need to be precise at this point, it won't be used anyway
190 - b.bLen = len(data) - batchHdrLen
191 - b.data = data
192 -
193 - return nil
194 -}
195 -
196 -func (b *Batch) decodeRec(f func(i int, kt kType, key, value []byte)) (err error) {
197 - off := batchHdrLen
198 - for i := 0; i < b.rLen; i++ {
199 - if off >= len(b.data) {
200 - return newErrBatchCorrupted("invalid records length")
201 - }
202 -
203 - kt := kType(b.data[off])
204 - if kt > ktVal {
205 - return newErrBatchCorrupted("bad record: invalid type")
206 - }
207 - off += 1
208 -
209 - x, n := binary.Uvarint(b.data[off:])
210 - off += n
211 - if n <= 0 || off+int(x) > len(b.data) {
212 - return newErrBatchCorrupted("bad record: invalid key length")
213 - }
214 - key := b.data[off : off+int(x)]
215 - off += int(x)
216 - var value []byte
217 - if kt == ktVal {
218 - x, n := binary.Uvarint(b.data[off:])
219 - off += n
220 - if n <= 0 || off+int(x) > len(b.data) {
221 - return newErrBatchCorrupted("bad record: invalid value length")
222 - }
223 - value = b.data[off : off+int(x)]
224 - off += int(x)
225 - }
226 -
227 - f(i, kt, key, value)
228 - }
229 -
230 - return nil
231 -}
232 -
233 -func (b *Batch) memReplay(to *memdb.DB) error {
234 - return b.decodeRec(func(i int, kt kType, key, value []byte) {
235 - ikey := newIkey(key, b.seq+uint64(i), kt)
236 - to.Put(ikey, value)
237 - })
238 -}
239 -
240 -func (b *Batch) memDecodeAndReplay(prevSeq uint64, data []byte, to *memdb.DB) error {
241 - if err := b.decode(prevSeq, data); err != nil {
242 - return err
243 - }
244 - return b.memReplay(to)
245 -}
246 -
247 -func (b *Batch) revertMemReplay(to *memdb.DB) error {
248 - return b.decodeRec(func(i int, kt kType, key, value []byte) {
249 - ikey := newIkey(key, b.seq+uint64(i), kt)
250 - to.Delete(ikey)
251 - })
252 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/batch_test.go deleted
-120
@@ -1,120 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "testing"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15 -)
16 -
17 -type tbRec struct {
18 - kt kType
19 - key, value []byte
20 -}
21 -
22 -type testBatch struct {
23 - rec []*tbRec
24 -}
25 -
26 -func (p *testBatch) Put(key, value []byte) {
27 - p.rec = append(p.rec, &tbRec{ktVal, key, value})
28 -}
29 -
30 -func (p *testBatch) Delete(key []byte) {
31 - p.rec = append(p.rec, &tbRec{ktDel, key, nil})
32 -}
33 -
34 -func compareBatch(t *testing.T, b1, b2 *Batch) {
35 - if b1.seq != b2.seq {
36 - t.Errorf("invalid seq number want %d, got %d", b1.seq, b2.seq)
37 - }
38 - if b1.Len() != b2.Len() {
39 - t.Fatalf("invalid record length want %d, got %d", b1.Len(), b2.Len())
40 - }
41 - p1, p2 := new(testBatch), new(testBatch)
42 - err := b1.Replay(p1)
43 - if err != nil {
44 - t.Fatal("error when replaying batch 1: ", err)
45 - }
46 - err = b2.Replay(p2)
47 - if err != nil {
48 - t.Fatal("error when replaying batch 2: ", err)
49 - }
50 - for i := range p1.rec {
51 - r1, r2 := p1.rec[i], p2.rec[i]
52 - if r1.kt != r2.kt {
53 - t.Errorf("invalid type on record '%d' want %d, got %d", i, r1.kt, r2.kt)
54 - }
55 - if !bytes.Equal(r1.key, r2.key) {
56 - t.Errorf("invalid key on record '%d' want %s, got %s", i, string(r1.key), string(r2.key))
57 - }
58 - if r1.kt == ktVal {
59 - if !bytes.Equal(r1.value, r2.value) {
60 - t.Errorf("invalid value on record '%d' want %s, got %s", i, string(r1.value), string(r2.value))
61 - }
62 - }
63 - }
64 -}
65 -
66 -func TestBatch_EncodeDecode(t *testing.T) {
67 - b1 := new(Batch)
68 - b1.seq = 10009
69 - b1.Put([]byte("key1"), []byte("value1"))
70 - b1.Put([]byte("key2"), []byte("value2"))
71 - b1.Delete([]byte("key1"))
72 - b1.Put([]byte("k"), []byte(""))
73 - b1.Put([]byte("zzzzzzzzzzz"), []byte("zzzzzzzzzzzzzzzzzzzzzzzz"))
74 - b1.Delete([]byte("key10000"))
75 - b1.Delete([]byte("k"))
76 - buf := b1.encode()
77 - b2 := new(Batch)
78 - err := b2.decode(0, buf)
79 - if err != nil {
80 - t.Error("error when decoding batch: ", err)
81 - }
82 - compareBatch(t, b1, b2)
83 -}
84 -
85 -func TestBatch_Append(t *testing.T) {
86 - b1 := new(Batch)
87 - b1.seq = 10009
88 - b1.Put([]byte("key1"), []byte("value1"))
89 - b1.Put([]byte("key2"), []byte("value2"))
90 - b1.Delete([]byte("key1"))
91 - b1.Put([]byte("foo"), []byte("foovalue"))
92 - b1.Put([]byte("bar"), []byte("barvalue"))
93 - b2a := new(Batch)
94 - b2a.seq = 10009
95 - b2a.Put([]byte("key1"), []byte("value1"))
96 - b2a.Put([]byte("key2"), []byte("value2"))
97 - b2a.Delete([]byte("key1"))
98 - b2b := new(Batch)
99 - b2b.Put([]byte("foo"), []byte("foovalue"))
100 - b2b.Put([]byte("bar"), []byte("barvalue"))
101 - b2a.append(b2b)
102 - compareBatch(t, b1, b2a)
103 -}
104 -
105 -func TestBatch_Size(t *testing.T) {
106 - b := new(Batch)
107 - for i := 0; i < 2; i++ {
108 - b.Put([]byte("key1"), []byte("value1"))
109 - b.Put([]byte("key2"), []byte("value2"))
110 - b.Delete([]byte("key1"))
111 - b.Put([]byte("foo"), []byte("foovalue"))
112 - b.Put([]byte("bar"), []byte("barvalue"))
113 - mem := memdb.New(&iComparer{comparer.DefaultComparer}, 0)
114 - b.memReplay(mem)
115 - if b.size() != mem.Size() {
116 - t.Errorf("invalid batch size calculation, want=%d got=%d", mem.Size(), b.size())
117 - }
118 - b.Reset()
119 - }
120 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench2_test.go deleted
-58
@@ -1,58 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build !go1.2
8 -
9 -package leveldb
10 -
11 -import (
12 - "sync/atomic"
13 - "testing"
14 -)
15 -
16 -func BenchmarkDBReadConcurrent(b *testing.B) {
17 - p := openDBBench(b, false)
18 - p.populate(b.N)
19 - p.fill()
20 - p.gc()
21 - defer p.close()
22 -
23 - b.ResetTimer()
24 - b.SetBytes(116)
25 -
26 - b.RunParallel(func(pb *testing.PB) {
27 - iter := p.newIter()
28 - defer iter.Release()
29 - for pb.Next() && iter.Next() {
30 - }
31 - })
32 -}
33 -
34 -func BenchmarkDBReadConcurrent2(b *testing.B) {
35 - p := openDBBench(b, false)
36 - p.populate(b.N)
37 - p.fill()
38 - p.gc()
39 - defer p.close()
40 -
41 - b.ResetTimer()
42 - b.SetBytes(116)
43 -
44 - var dir uint32
45 - b.RunParallel(func(pb *testing.PB) {
46 - iter := p.newIter()
47 - defer iter.Release()
48 - if atomic.AddUint32(&dir, 1)%2 == 0 {
49 - for pb.Next() && iter.Next() {
50 - }
51 - } else {
52 - if pb.Next() && iter.Last() {
53 - for pb.Next() && iter.Prev() {
54 - }
55 - }
56 - }
57 - })
58 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/bench_test.go deleted
-464
@@ -1,464 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "fmt"
12 - "math/rand"
13 - "os"
14 - "path/filepath"
15 - "runtime"
16 - "testing"
17 -
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
20 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
21 -)
22 -
23 -func randomString(r *rand.Rand, n int) []byte {
24 - b := new(bytes.Buffer)
25 - for i := 0; i < n; i++ {
26 - b.WriteByte(' ' + byte(r.Intn(95)))
27 - }
28 - return b.Bytes()
29 -}
30 -
31 -func compressibleStr(r *rand.Rand, frac float32, n int) []byte {
32 - nn := int(float32(n) * frac)
33 - rb := randomString(r, nn)
34 - b := make([]byte, 0, n+nn)
35 - for len(b) < n {
36 - b = append(b, rb...)
37 - }
38 - return b[:n]
39 -}
40 -
41 -type valueGen struct {
42 - src []byte
43 - pos int
44 -}
45 -
46 -func newValueGen(frac float32) *valueGen {
47 - v := new(valueGen)
48 - r := rand.New(rand.NewSource(301))
49 - v.src = make([]byte, 0, 1048576+100)
50 - for len(v.src) < 1048576 {
51 - v.src = append(v.src, compressibleStr(r, frac, 100)...)
52 - }
53 - return v
54 -}
55 -
56 -func (v *valueGen) get(n int) []byte {
57 - if v.pos+n > len(v.src) {
58 - v.pos = 0
59 - }
60 - v.pos += n
61 - return v.src[v.pos-n : v.pos]
62 -}
63 -
64 -var benchDB = filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbbench-%d", os.Getuid()))
65 -
66 -type dbBench struct {
67 - b *testing.B
68 - stor storage.Storage
69 - db *DB
70 -
71 - o *opt.Options
72 - ro *opt.ReadOptions
73 - wo *opt.WriteOptions
74 -
75 - keys, values [][]byte
76 -}
77 -
78 -func openDBBench(b *testing.B, noCompress bool) *dbBench {
79 - _, err := os.Stat(benchDB)
80 - if err == nil {
81 - err = os.RemoveAll(benchDB)
82 - if err != nil {
83 - b.Fatal("cannot remove old db: ", err)
84 - }
85 - }
86 -
87 - p := &dbBench{
88 - b: b,
89 - o: &opt.Options{},
90 - ro: &opt.ReadOptions{},
91 - wo: &opt.WriteOptions{},
92 - }
93 - p.stor, err = storage.OpenFile(benchDB)
94 - if err != nil {
95 - b.Fatal("cannot open stor: ", err)
96 - }
97 - if noCompress {
98 - p.o.Compression = opt.NoCompression
99 - }
100 -
101 - p.db, err = Open(p.stor, p.o)
102 - if err != nil {
103 - b.Fatal("cannot open db: ", err)
104 - }
105 -
106 - runtime.GOMAXPROCS(runtime.NumCPU())
107 - return p
108 -}
109 -
110 -func (p *dbBench) reopen() {
111 - p.db.Close()
112 - var err error
113 - p.db, err = Open(p.stor, p.o)
114 - if err != nil {
115 - p.b.Fatal("Reopen: got error: ", err)
116 - }
117 -}
118 -
119 -func (p *dbBench) populate(n int) {
120 - p.keys, p.values = make([][]byte, n), make([][]byte, n)
121 - v := newValueGen(0.5)
122 - for i := range p.keys {
123 - p.keys[i], p.values[i] = []byte(fmt.Sprintf("%016d", i)), v.get(100)
124 - }
125 -}
126 -
127 -func (p *dbBench) randomize() {
128 - m := len(p.keys)
129 - times := m * 2
130 - r1, r2 := rand.New(rand.NewSource(0xdeadbeef)), rand.New(rand.NewSource(0xbeefface))
131 - for n := 0; n < times; n++ {
132 - i, j := r1.Int()%m, r2.Int()%m
133 - if i == j {
134 - continue
135 - }
136 - p.keys[i], p.keys[j] = p.keys[j], p.keys[i]
137 - p.values[i], p.values[j] = p.values[j], p.values[i]
138 - }
139 -}
140 -
141 -func (p *dbBench) writes(perBatch int) {
142 - b := p.b
143 - db := p.db
144 -
145 - n := len(p.keys)
146 - m := n / perBatch
147 - if n%perBatch > 0 {
148 - m++
149 - }
150 - batches := make([]Batch, m)
151 - j := 0
152 - for i := range batches {
153 - first := true
154 - for ; j < n && ((j+1)%perBatch != 0 || first); j++ {
155 - first = false
156 - batches[i].Put(p.keys[j], p.values[j])
157 - }
158 - }
159 - runtime.GC()
160 -
161 - b.ResetTimer()
162 - b.StartTimer()
163 - for i := range batches {
164 - err := db.Write(&(batches[i]), p.wo)
165 - if err != nil {
166 - b.Fatal("write failed: ", err)
167 - }
168 - }
169 - b.StopTimer()
170 - b.SetBytes(116)
171 -}
172 -
173 -func (p *dbBench) gc() {
174 - p.keys, p.values = nil, nil
175 - runtime.GC()
176 -}
177 -
178 -func (p *dbBench) puts() {
179 - b := p.b
180 - db := p.db
181 -
182 - b.ResetTimer()
183 - b.StartTimer()
184 - for i := range p.keys {
185 - err := db.Put(p.keys[i], p.values[i], p.wo)
186 - if err != nil {
187 - b.Fatal("put failed: ", err)
188 - }
189 - }
190 - b.StopTimer()
191 - b.SetBytes(116)
192 -}
193 -
194 -func (p *dbBench) fill() {
195 - b := p.b
196 - db := p.db
197 -
198 - perBatch := 10000
199 - batch := new(Batch)
200 - for i, n := 0, len(p.keys); i < n; {
201 - first := true
202 - for ; i < n && ((i+1)%perBatch != 0 || first); i++ {
203 - first = false
204 - batch.Put(p.keys[i], p.values[i])
205 - }
206 - err := db.Write(batch, p.wo)
207 - if err != nil {
208 - b.Fatal("write failed: ", err)
209 - }
210 - batch.Reset()
211 - }
212 -}
213 -
214 -func (p *dbBench) gets() {
215 - b := p.b
216 - db := p.db
217 -
218 - b.ResetTimer()
219 - for i := range p.keys {
220 - _, err := db.Get(p.keys[i], p.ro)
221 - if err != nil {
222 - b.Error("got error: ", err)
223 - }
224 - }
225 - b.StopTimer()
226 -}
227 -
228 -func (p *dbBench) seeks() {
229 - b := p.b
230 -
231 - iter := p.newIter()
232 - defer iter.Release()
233 - b.ResetTimer()
234 - for i := range p.keys {
235 - if !iter.Seek(p.keys[i]) {
236 - b.Error("value not found for: ", string(p.keys[i]))
237 - }
238 - }
239 - b.StopTimer()
240 -}
241 -
242 -func (p *dbBench) newIter() iterator.Iterator {
243 - iter := p.db.NewIterator(nil, p.ro)
244 - err := iter.Error()
245 - if err != nil {
246 - p.b.Fatal("cannot create iterator: ", err)
247 - }
248 - return iter
249 -}
250 -
251 -func (p *dbBench) close() {
252 - if bp, err := p.db.GetProperty("leveldb.blockpool"); err == nil {
253 - p.b.Log("Block pool stats: ", bp)
254 - }
255 - p.db.Close()
256 - p.stor.Close()
257 - os.RemoveAll(benchDB)
258 - p.db = nil
259 - p.keys = nil
260 - p.values = nil
261 - runtime.GC()
262 - runtime.GOMAXPROCS(1)
263 -}
264 -
265 -func BenchmarkDBWrite(b *testing.B) {
266 - p := openDBBench(b, false)
267 - p.populate(b.N)
268 - p.writes(1)
269 - p.close()
270 -}
271 -
272 -func BenchmarkDBWriteBatch(b *testing.B) {
273 - p := openDBBench(b, false)
274 - p.populate(b.N)
275 - p.writes(1000)
276 - p.close()
277 -}
278 -
279 -func BenchmarkDBWriteUncompressed(b *testing.B) {
280 - p := openDBBench(b, true)
281 - p.populate(b.N)
282 - p.writes(1)
283 - p.close()
284 -}
285 -
286 -func BenchmarkDBWriteBatchUncompressed(b *testing.B) {
287 - p := openDBBench(b, true)
288 - p.populate(b.N)
289 - p.writes(1000)
290 - p.close()
291 -}
292 -
293 -func BenchmarkDBWriteRandom(b *testing.B) {
294 - p := openDBBench(b, false)
295 - p.populate(b.N)
296 - p.randomize()
297 - p.writes(1)
298 - p.close()
299 -}
300 -
301 -func BenchmarkDBWriteRandomSync(b *testing.B) {
302 - p := openDBBench(b, false)
303 - p.wo.Sync = true
304 - p.populate(b.N)
305 - p.writes(1)
306 - p.close()
307 -}
308 -
309 -func BenchmarkDBOverwrite(b *testing.B) {
310 - p := openDBBench(b, false)
311 - p.populate(b.N)
312 - p.writes(1)
313 - p.writes(1)
314 - p.close()
315 -}
316 -
317 -func BenchmarkDBOverwriteRandom(b *testing.B) {
318 - p := openDBBench(b, false)
319 - p.populate(b.N)
320 - p.writes(1)
321 - p.randomize()
322 - p.writes(1)
323 - p.close()
324 -}
325 -
326 -func BenchmarkDBPut(b *testing.B) {
327 - p := openDBBench(b, false)
328 - p.populate(b.N)
329 - p.puts()
330 - p.close()
331 -}
332 -
333 -func BenchmarkDBRead(b *testing.B) {
334 - p := openDBBench(b, false)
335 - p.populate(b.N)
336 - p.fill()
337 - p.gc()
338 -
339 - iter := p.newIter()
340 - b.ResetTimer()
341 - for iter.Next() {
342 - }
343 - iter.Release()
344 - b.StopTimer()
345 - b.SetBytes(116)
346 - p.close()
347 -}
348 -
349 -func BenchmarkDBReadGC(b *testing.B) {
350 - p := openDBBench(b, false)
351 - p.populate(b.N)
352 - p.fill()
353 -
354 - iter := p.newIter()
355 - b.ResetTimer()
356 - for iter.Next() {
357 - }
358 - iter.Release()
359 - b.StopTimer()
360 - b.SetBytes(116)
361 - p.close()
362 -}
363 -
364 -func BenchmarkDBReadUncompressed(b *testing.B) {
365 - p := openDBBench(b, true)
366 - p.populate(b.N)
367 - p.fill()
368 - p.gc()
369 -
370 - iter := p.newIter()
371 - b.ResetTimer()
372 - for iter.Next() {
373 - }
374 - iter.Release()
375 - b.StopTimer()
376 - b.SetBytes(116)
377 - p.close()
378 -}
379 -
380 -func BenchmarkDBReadTable(b *testing.B) {
381 - p := openDBBench(b, false)
382 - p.populate(b.N)
383 - p.fill()
384 - p.reopen()
385 - p.gc()
386 -
387 - iter := p.newIter()
388 - b.ResetTimer()
389 - for iter.Next() {
390 - }
391 - iter.Release()
392 - b.StopTimer()
393 - b.SetBytes(116)
394 - p.close()
395 -}
396 -
397 -func BenchmarkDBReadReverse(b *testing.B) {
398 - p := openDBBench(b, false)
399 - p.populate(b.N)
400 - p.fill()
401 - p.gc()
402 -
403 - iter := p.newIter()
404 - b.ResetTimer()
405 - iter.Last()
406 - for iter.Prev() {
407 - }
408 - iter.Release()
409 - b.StopTimer()
410 - b.SetBytes(116)
411 - p.close()
412 -}
413 -
414 -func BenchmarkDBReadReverseTable(b *testing.B) {
415 - p := openDBBench(b, false)
416 - p.populate(b.N)
417 - p.fill()
418 - p.reopen()
419 - p.gc()
420 -
421 - iter := p.newIter()
422 - b.ResetTimer()
423 - iter.Last()
424 - for iter.Prev() {
425 - }
426 - iter.Release()
427 - b.StopTimer()
428 - b.SetBytes(116)
429 - p.close()
430 -}
431 -
432 -func BenchmarkDBSeek(b *testing.B) {
433 - p := openDBBench(b, false)
434 - p.populate(b.N)
435 - p.fill()
436 - p.seeks()
437 - p.close()
438 -}
439 -
440 -func BenchmarkDBSeekRandom(b *testing.B) {
441 - p := openDBBench(b, false)
442 - p.populate(b.N)
443 - p.fill()
444 - p.randomize()
445 - p.seeks()
446 - p.close()
447 -}
448 -
449 -func BenchmarkDBGet(b *testing.B) {
450 - p := openDBBench(b, false)
451 - p.populate(b.N)
452 - p.fill()
453 - p.gets()
454 - p.close()
455 -}
456 -
457 -func BenchmarkDBGetRandom(b *testing.B) {
458 - p := openDBBench(b, false)
459 - p.populate(b.N)
460 - p.fill()
461 - p.randomize()
462 - p.gets()
463 - p.close()
464 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/bench2_test.go deleted
-30
@@ -1,30 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build !go1.2
8 -
9 -package cache
10 -
11 -import (
12 - "math/rand"
13 - "testing"
14 -)
15 -
16 -func BenchmarkLRUCache(b *testing.B) {
17 - c := NewCache(NewLRU(10000))
18 -
19 - b.SetParallelism(10)
20 - b.RunParallel(func(pb *testing.PB) {
21 - r := rand.New(rand.NewSource(time.Now().UnixNano()))
22 -
23 - for pb.Next() {
24 - key := uint64(r.Intn(1000000))
25 - c.Get(0, key, func() (int, Value) {
26 - return 1, key
27 - }).Release()
28 - }
29 - })
30 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache.go deleted
-676
@@ -1,676 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package cache provides interface and implementation of a cache algorithms.
8 -package cache
9 -
10 -import (
11 - "sync"
12 - "sync/atomic"
13 - "unsafe"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -// Cacher provides interface to implements a caching functionality.
19 -// An implementation must be goroutine-safe.
20 -type Cacher interface {
21 - // Capacity returns cache capacity.
22 - Capacity() int
23 -
24 - // SetCapacity sets cache capacity.
25 - SetCapacity(capacity int)
26 -
27 - // Promote promotes the 'cache node'.
28 - Promote(n *Node)
29 -
30 - // Ban evicts the 'cache node' and prevent subsequent 'promote'.
31 - Ban(n *Node)
32 -
33 - // Evict evicts the 'cache node'.
34 - Evict(n *Node)
35 -
36 - // EvictNS evicts 'cache node' with the given namespace.
37 - EvictNS(ns uint64)
38 -
39 - // EvictAll evicts all 'cache node'.
40 - EvictAll()
41 -
42 - // Close closes the 'cache tree'
43 - Close() error
44 -}
45 -
46 -// Value is a 'cacheable object'. It may implements util.Releaser, if
47 -// so the the Release method will be called once object is released.
48 -type Value interface{}
49 -
50 -type CacheGetter struct {
51 - Cache *Cache
52 - NS uint64
53 -}
54 -
55 -func (g *CacheGetter) Get(key uint64, setFunc func() (size int, value Value)) *Handle {
56 - return g.Cache.Get(g.NS, key, setFunc)
57 -}
58 -
59 -// The hash tables implementation is based on:
60 -// "Dynamic-Sized Nonblocking Hash Tables", by Yujie Liu, Kunlong Zhang, and Michael Spear. ACM Symposium on Principles of Distributed Computing, Jul 2014.
61 -
62 -const (
63 - mInitialSize = 1 << 4
64 - mOverflowThreshold = 1 << 5
65 - mOverflowGrowThreshold = 1 << 7
66 -)
67 -
68 -type mBucket struct {
69 - mu sync.Mutex
70 - node []*Node
71 - frozen bool
72 -}
73 -
74 -func (b *mBucket) freeze() []*Node {
75 - b.mu.Lock()
76 - defer b.mu.Unlock()
77 - if !b.frozen {
78 - b.frozen = true
79 - }
80 - return b.node
81 -}
82 -
83 -func (b *mBucket) get(r *Cache, h *mNode, hash uint32, ns, key uint64, noset bool) (done, added bool, n *Node) {
84 - b.mu.Lock()
85 -
86 - if b.frozen {
87 - b.mu.Unlock()
88 - return
89 - }
90 -
91 - // Scan the node.
92 - for _, n := range b.node {
93 - if n.hash == hash && n.ns == ns && n.key == key {
94 - atomic.AddInt32(&n.ref, 1)
95 - b.mu.Unlock()
96 - return true, false, n
97 - }
98 - }
99 -
100 - // Get only.
101 - if noset {
102 - b.mu.Unlock()
103 - return true, false, nil
104 - }
105 -
106 - // Create node.
107 - n = &Node{
108 - r: r,
109 - hash: hash,
110 - ns: ns,
111 - key: key,
112 - ref: 1,
113 - }
114 - // Add node to bucket.
115 - b.node = append(b.node, n)
116 - bLen := len(b.node)
117 - b.mu.Unlock()
118 -
119 - // Update counter.
120 - grow := atomic.AddInt32(&r.nodes, 1) >= h.growThreshold
121 - if bLen > mOverflowThreshold {
122 - grow = grow || atomic.AddInt32(&h.overflow, 1) >= mOverflowGrowThreshold
123 - }
124 -
125 - // Grow.
126 - if grow && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
127 - nhLen := len(h.buckets) << 1
128 - nh := &mNode{
129 - buckets: make([]unsafe.Pointer, nhLen),
130 - mask: uint32(nhLen) - 1,
131 - pred: unsafe.Pointer(h),
132 - growThreshold: int32(nhLen * mOverflowThreshold),
133 - shrinkThreshold: int32(nhLen >> 1),
134 - }
135 - ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
136 - if !ok {
137 - panic("BUG: failed swapping head")
138 - }
139 - go nh.initBuckets()
140 - }
141 -
142 - return true, true, n
143 -}
144 -
145 -func (b *mBucket) delete(r *Cache, h *mNode, hash uint32, ns, key uint64) (done, deleted bool) {
146 - b.mu.Lock()
147 -
148 - if b.frozen {
149 - b.mu.Unlock()
150 - return
151 - }
152 -
153 - // Scan the node.
154 - var (
155 - n *Node
156 - bLen int
157 - )
158 - for i := range b.node {
159 - n = b.node[i]
160 - if n.ns == ns && n.key == key {
161 - if atomic.LoadInt32(&n.ref) == 0 {
162 - deleted = true
163 -
164 - // Call releaser.
165 - if n.value != nil {
166 - if r, ok := n.value.(util.Releaser); ok {
167 - r.Release()
168 - }
169 - n.value = nil
170 - }
171 -
172 - // Remove node from bucket.
173 - b.node = append(b.node[:i], b.node[i+1:]...)
174 - bLen = len(b.node)
175 - }
176 - break
177 - }
178 - }
179 - b.mu.Unlock()
180 -
181 - if deleted {
182 - // Call OnDel.
183 - for _, f := range n.onDel {
184 - f()
185 - }
186 -
187 - // Update counter.
188 - atomic.AddInt32(&r.size, int32(n.size)*-1)
189 - shrink := atomic.AddInt32(&r.nodes, -1) < h.shrinkThreshold
190 - if bLen >= mOverflowThreshold {
191 - atomic.AddInt32(&h.overflow, -1)
192 - }
193 -
194 - // Shrink.
195 - if shrink && len(h.buckets) > mInitialSize && atomic.CompareAndSwapInt32(&h.resizeInProgess, 0, 1) {
196 - nhLen := len(h.buckets) >> 1
197 - nh := &mNode{
198 - buckets: make([]unsafe.Pointer, nhLen),
199 - mask: uint32(nhLen) - 1,
200 - pred: unsafe.Pointer(h),
201 - growThreshold: int32(nhLen * mOverflowThreshold),
202 - shrinkThreshold: int32(nhLen >> 1),
203 - }
204 - ok := atomic.CompareAndSwapPointer(&r.mHead, unsafe.Pointer(h), unsafe.Pointer(nh))
205 - if !ok {
206 - panic("BUG: failed swapping head")
207 - }
208 - go nh.initBuckets()
209 - }
210 - }
211 -
212 - return true, deleted
213 -}
214 -
215 -type mNode struct {
216 - buckets []unsafe.Pointer // []*mBucket
217 - mask uint32
218 - pred unsafe.Pointer // *mNode
219 - resizeInProgess int32
220 -
221 - overflow int32
222 - growThreshold int32
223 - shrinkThreshold int32
224 -}
225 -
226 -func (n *mNode) initBucket(i uint32) *mBucket {
227 - if b := (*mBucket)(atomic.LoadPointer(&n.buckets[i])); b != nil {
228 - return b
229 - }
230 -
231 - p := (*mNode)(atomic.LoadPointer(&n.pred))
232 - if p != nil {
233 - var node []*Node
234 - if n.mask > p.mask {
235 - // Grow.
236 - pb := (*mBucket)(atomic.LoadPointer(&p.buckets[i&p.mask]))
237 - if pb == nil {
238 - pb = p.initBucket(i & p.mask)
239 - }
240 - m := pb.freeze()
241 - // Split nodes.
242 - for _, x := range m {
243 - if x.hash&n.mask == i {
244 - node = append(node, x)
245 - }
246 - }
247 - } else {
248 - // Shrink.
249 - pb0 := (*mBucket)(atomic.LoadPointer(&p.buckets[i]))
250 - if pb0 == nil {
251 - pb0 = p.initBucket(i)
252 - }
253 - pb1 := (*mBucket)(atomic.LoadPointer(&p.buckets[i+uint32(len(n.buckets))]))
254 - if pb1 == nil {
255 - pb1 = p.initBucket(i + uint32(len(n.buckets)))
256 - }
257 - m0 := pb0.freeze()
258 - m1 := pb1.freeze()
259 - // Merge nodes.
260 - node = make([]*Node, 0, len(m0)+len(m1))
261 - node = append(node, m0...)
262 - node = append(node, m1...)
263 - }
264 - b := &mBucket{node: node}
265 - if atomic.CompareAndSwapPointer(&n.buckets[i], nil, unsafe.Pointer(b)) {
266 - if len(node) > mOverflowThreshold {
267 - atomic.AddInt32(&n.overflow, int32(len(node)-mOverflowThreshold))
268 - }
269 - return b
270 - }
271 - }
272 -
273 - return (*mBucket)(atomic.LoadPointer(&n.buckets[i]))
274 -}
275 -
276 -func (n *mNode) initBuckets() {
277 - for i := range n.buckets {
278 - n.initBucket(uint32(i))
279 - }
280 - atomic.StorePointer(&n.pred, nil)
281 -}
282 -
283 -// Cache is a 'cache map'.
284 -type Cache struct {
285 - mu sync.RWMutex
286 - mHead unsafe.Pointer // *mNode
287 - nodes int32
288 - size int32
289 - cacher Cacher
290 - closed bool
291 -}
292 -
293 -// NewCache creates a new 'cache map'. The cacher is optional and
294 -// may be nil.
295 -func NewCache(cacher Cacher) *Cache {
296 - h := &mNode{
297 - buckets: make([]unsafe.Pointer, mInitialSize),
298 - mask: mInitialSize - 1,
299 - growThreshold: int32(mInitialSize * mOverflowThreshold),
300 - shrinkThreshold: 0,
301 - }
302 - for i := range h.buckets {
303 - h.buckets[i] = unsafe.Pointer(&mBucket{})
304 - }
305 - r := &Cache{
306 - mHead: unsafe.Pointer(h),
307 - cacher: cacher,
308 - }
309 - return r
310 -}
311 -
312 -func (r *Cache) getBucket(hash uint32) (*mNode, *mBucket) {
313 - h := (*mNode)(atomic.LoadPointer(&r.mHead))
314 - i := hash & h.mask
315 - b := (*mBucket)(atomic.LoadPointer(&h.buckets[i]))
316 - if b == nil {
317 - b = h.initBucket(i)
318 - }
319 - return h, b
320 -}
321 -
322 -func (r *Cache) delete(n *Node) bool {
323 - for {
324 - h, b := r.getBucket(n.hash)
325 - done, deleted := b.delete(r, h, n.hash, n.ns, n.key)
326 - if done {
327 - return deleted
328 - }
329 - }
330 - return false
331 -}
332 -
333 -// Nodes returns number of 'cache node' in the map.
334 -func (r *Cache) Nodes() int {
335 - return int(atomic.LoadInt32(&r.nodes))
336 -}
337 -
338 -// Size returns sums of 'cache node' size in the map.
339 -func (r *Cache) Size() int {
340 - return int(atomic.LoadInt32(&r.size))
341 -}
342 -
343 -// Capacity returns cache capacity.
344 -func (r *Cache) Capacity() int {
345 - if r.cacher == nil {
346 - return 0
347 - }
348 - return r.cacher.Capacity()
349 -}
350 -
351 -// SetCapacity sets cache capacity.
352 -func (r *Cache) SetCapacity(capacity int) {
353 - if r.cacher != nil {
354 - r.cacher.SetCapacity(capacity)
355 - }
356 -}
357 -
358 -// Get gets 'cache node' with the given namespace and key.
359 -// If cache node is not found and setFunc is not nil, Get will atomically creates
360 -// the 'cache node' by calling setFunc. Otherwise Get will returns nil.
361 -//
362 -// The returned 'cache handle' should be released after use by calling Release
363 -// method.
364 -func (r *Cache) Get(ns, key uint64, setFunc func() (size int, value Value)) *Handle {
365 - r.mu.RLock()
366 - defer r.mu.RUnlock()
367 - if r.closed {
368 - return nil
369 - }
370 -
371 - hash := murmur32(ns, key, 0xf00)
372 - for {
373 - h, b := r.getBucket(hash)
374 - done, _, n := b.get(r, h, hash, ns, key, setFunc == nil)
375 - if done {
376 - if n != nil {
377 - n.mu.Lock()
378 - if n.value == nil {
379 - if setFunc == nil {
380 - n.mu.Unlock()
381 - n.unref()
382 - return nil
383 - }
384 -
385 - n.size, n.value = setFunc()
386 - if n.value == nil {
387 - n.size = 0
388 - n.mu.Unlock()
389 - n.unref()
390 - return nil
391 - }
392 - atomic.AddInt32(&r.size, int32(n.size))
393 - }
394 - n.mu.Unlock()
395 - if r.cacher != nil {
396 - r.cacher.Promote(n)
397 - }
398 - return &Handle{unsafe.Pointer(n)}
399 - }
400 -
401 - break
402 - }
403 - }
404 - return nil
405 -}
406 -
407 -// Delete removes and ban 'cache node' with the given namespace and key.
408 -// A banned 'cache node' will never inserted into the 'cache tree'. Ban
409 -// only attributed to the particular 'cache node', so when a 'cache node'
410 -// is recreated it will not be banned.
411 -//
412 -// If onDel is not nil, then it will be executed if such 'cache node'
413 -// doesn't exist or once the 'cache node' is released.
414 -//
415 -// Delete return true is such 'cache node' exist.
416 -func (r *Cache) Delete(ns, key uint64, onDel func()) bool {
417 - r.mu.RLock()
418 - defer r.mu.RUnlock()
419 - if r.closed {
420 - return false
421 - }
422 -
423 - hash := murmur32(ns, key, 0xf00)
424 - for {
425 - h, b := r.getBucket(hash)
426 - done, _, n := b.get(r, h, hash, ns, key, true)
427 - if done {
428 - if n != nil {
429 - if onDel != nil {
430 - n.mu.Lock()
431 - n.onDel = append(n.onDel, onDel)
432 - n.mu.Unlock()
433 - }
434 - if r.cacher != nil {
435 - r.cacher.Ban(n)
436 - }
437 - n.unref()
438 - return true
439 - }
440 -
441 - break
442 - }
443 - }
444 -
445 - if onDel != nil {
446 - onDel()
447 - }
448 -
449 - return false
450 -}
451 -
452 -// Evict evicts 'cache node' with the given namespace and key. This will
453 -// simply call Cacher.Evict.
454 -//
455 -// Evict return true is such 'cache node' exist.
456 -func (r *Cache) Evict(ns, key uint64) bool {
457 - r.mu.RLock()
458 - defer r.mu.RUnlock()
459 - if r.closed {
460 - return false
461 - }
462 -
463 - hash := murmur32(ns, key, 0xf00)
464 - for {
465 - h, b := r.getBucket(hash)
466 - done, _, n := b.get(r, h, hash, ns, key, true)
467 - if done {
468 - if n != nil {
469 - if r.cacher != nil {
470 - r.cacher.Evict(n)
471 - }
472 - n.unref()
473 - return true
474 - }
475 -
476 - break
477 - }
478 - }
479 -
480 - return false
481 -}
482 -
483 -// EvictNS evicts 'cache node' with the given namespace. This will
484 -// simply call Cacher.EvictNS.
485 -func (r *Cache) EvictNS(ns uint64) {
486 - r.mu.RLock()
487 - defer r.mu.RUnlock()
488 - if r.closed {
489 - return
490 - }
491 -
492 - if r.cacher != nil {
493 - r.cacher.EvictNS(ns)
494 - }
495 -}
496 -
497 -// EvictAll evicts all 'cache node'. This will simply call Cacher.EvictAll.
498 -func (r *Cache) EvictAll() {
499 - r.mu.RLock()
500 - defer r.mu.RUnlock()
501 - if r.closed {
502 - return
503 - }
504 -
505 - if r.cacher != nil {
506 - r.cacher.EvictAll()
507 - }
508 -}
509 -
510 -// Close closes the 'cache map' and releases all 'cache node'.
511 -func (r *Cache) Close() error {
512 - r.mu.Lock()
513 - if !r.closed {
514 - r.closed = true
515 -
516 - if r.cacher != nil {
517 - if err := r.cacher.Close(); err != nil {
518 - return err
519 - }
520 - }
521 -
522 - h := (*mNode)(r.mHead)
523 - h.initBuckets()
524 -
525 - for i := range h.buckets {
526 - b := (*mBucket)(h.buckets[i])
527 - for _, n := range b.node {
528 - // Call releaser.
529 - if n.value != nil {
530 - if r, ok := n.value.(util.Releaser); ok {
531 - r.Release()
532 - }
533 - n.value = nil
534 - }
535 -
536 - // Call OnDel.
537 - for _, f := range n.onDel {
538 - f()
539 - }
540 - }
541 - }
542 - }
543 - r.mu.Unlock()
544 - return nil
545 -}
546 -
547 -// Node is a 'cache node'.
548 -type Node struct {
549 - r *Cache
550 -
551 - hash uint32
552 - ns, key uint64
553 -
554 - mu sync.Mutex
555 - size int
556 - value Value
557 -
558 - ref int32
559 - onDel []func()
560 -
561 - CacheData unsafe.Pointer
562 -}
563 -
564 -// NS returns this 'cache node' namespace.
565 -func (n *Node) NS() uint64 {
566 - return n.ns
567 -}
568 -
569 -// Key returns this 'cache node' key.
570 -func (n *Node) Key() uint64 {
571 - return n.key
572 -}
573 -
574 -// Size returns this 'cache node' size.
575 -func (n *Node) Size() int {
576 - return n.size
577 -}
578 -
579 -// Value returns this 'cache node' value.
580 -func (n *Node) Value() Value {
581 - return n.value
582 -}
583 -
584 -// Ref returns this 'cache node' ref counter.
585 -func (n *Node) Ref() int32 {
586 - return atomic.LoadInt32(&n.ref)
587 -}
588 -
589 -// GetHandle returns an handle for this 'cache node'.
590 -func (n *Node) GetHandle() *Handle {
591 - if atomic.AddInt32(&n.ref, 1) <= 1 {
592 - panic("BUG: Node.GetHandle on zero ref")
593 - }
594 - return &Handle{unsafe.Pointer(n)}
595 -}
596 -
597 -func (n *Node) unref() {
598 - if atomic.AddInt32(&n.ref, -1) == 0 {
599 - n.r.delete(n)
600 - }
601 -}
602 -
603 -func (n *Node) unrefLocked() {
604 - if atomic.AddInt32(&n.ref, -1) == 0 {
605 - n.r.mu.RLock()
606 - if !n.r.closed {
607 - n.r.delete(n)
608 - }
609 - n.r.mu.RUnlock()
610 - }
611 -}
612 -
613 -type Handle struct {
614 - n unsafe.Pointer // *Node
615 -}
616 -
617 -func (h *Handle) Value() Value {
618 - n := (*Node)(atomic.LoadPointer(&h.n))
619 - if n != nil {
620 - return n.value
621 - }
622 - return nil
623 -}
624 -
625 -func (h *Handle) Release() {
626 - nPtr := atomic.LoadPointer(&h.n)
627 - if nPtr != nil && atomic.CompareAndSwapPointer(&h.n, nPtr, nil) {
628 - n := (*Node)(nPtr)
629 - n.unrefLocked()
630 - }
631 -}
632 -
633 -func murmur32(ns, key uint64, seed uint32) uint32 {
634 - const (
635 - m = uint32(0x5bd1e995)
636 - r = 24
637 - )
638 -
639 - k1 := uint32(ns >> 32)
640 - k2 := uint32(ns)
641 - k3 := uint32(key >> 32)
642 - k4 := uint32(key)
643 -
644 - k1 *= m
645 - k1 ^= k1 >> r
646 - k1 *= m
647 -
648 - k2 *= m
649 - k2 ^= k2 >> r
650 - k2 *= m
651 -
652 - k3 *= m
653 - k3 ^= k3 >> r
654 - k3 *= m
655 -
656 - k4 *= m
657 - k4 ^= k4 >> r
658 - k4 *= m
659 -
660 - h := seed
661 -
662 - h *= m
663 - h ^= k1
664 - h *= m
665 - h ^= k2
666 - h *= m
667 - h ^= k3
668 - h *= m
669 - h ^= k4
670 -
671 - h ^= h >> 13
672 - h *= m
673 - h ^= h >> 15
674 -
675 - return h
676 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/cache_test.go deleted
-554
@@ -1,554 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package cache
8 -
9 -import (
10 - "math/rand"
11 - "runtime"
12 - "sync"
13 - "sync/atomic"
14 - "testing"
15 - "time"
16 - "unsafe"
17 -)
18 -
19 -type int32o int32
20 -
21 -func (o *int32o) acquire() {
22 - if atomic.AddInt32((*int32)(o), 1) != 1 {
23 - panic("BUG: invalid ref")
24 - }
25 -}
26 -
27 -func (o *int32o) Release() {
28 - if atomic.AddInt32((*int32)(o), -1) != 0 {
29 - panic("BUG: invalid ref")
30 - }
31 -}
32 -
33 -type releaserFunc struct {
34 - fn func()
35 - value Value
36 -}
37 -
38 -func (r releaserFunc) Release() {
39 - if r.fn != nil {
40 - r.fn()
41 - }
42 -}
43 -
44 -func set(c *Cache, ns, key uint64, value Value, charge int, relf func()) *Handle {
45 - return c.Get(ns, key, func() (int, Value) {
46 - if relf != nil {
47 - return charge, releaserFunc{relf, value}
48 - } else {
49 - return charge, value
50 - }
51 - })
52 -}
53 -
54 -func TestCacheMap(t *testing.T) {
55 - runtime.GOMAXPROCS(runtime.NumCPU())
56 -
57 - nsx := []struct {
58 - nobjects, nhandles, concurrent, repeat int
59 - }{
60 - {10000, 400, 50, 3},
61 - {100000, 1000, 100, 10},
62 - }
63 -
64 - var (
65 - objects [][]int32o
66 - handles [][]unsafe.Pointer
67 - )
68 -
69 - for _, x := range nsx {
70 - objects = append(objects, make([]int32o, x.nobjects))
71 - handles = append(handles, make([]unsafe.Pointer, x.nhandles))
72 - }
73 -
74 - c := NewCache(nil)
75 -
76 - wg := new(sync.WaitGroup)
77 - var done int32
78 -
79 - for ns, x := range nsx {
80 - for i := 0; i < x.concurrent; i++ {
81 - wg.Add(1)
82 - go func(ns, i, repeat int, objects []int32o, handles []unsafe.Pointer) {
83 - defer wg.Done()
84 - r := rand.New(rand.NewSource(time.Now().UnixNano()))
85 -
86 - for j := len(objects) * repeat; j >= 0; j-- {
87 - key := uint64(r.Intn(len(objects)))
88 - h := c.Get(uint64(ns), key, func() (int, Value) {
89 - o := &objects[key]
90 - o.acquire()
91 - return 1, o
92 - })
93 - if v := h.Value().(*int32o); v != &objects[key] {
94 - t.Fatalf("#%d invalid value: want=%p got=%p", ns, &objects[key], v)
95 - }
96 - if objects[key] != 1 {
97 - t.Fatalf("#%d invalid object %d: %d", ns, key, objects[key])
98 - }
99 - if !atomic.CompareAndSwapPointer(&handles[r.Intn(len(handles))], nil, unsafe.Pointer(h)) {
100 - h.Release()
101 - }
102 - }
103 - }(ns, i, x.repeat, objects[ns], handles[ns])
104 - }
105 -
106 - go func(handles []unsafe.Pointer) {
107 - r := rand.New(rand.NewSource(time.Now().UnixNano()))
108 -
109 - for atomic.LoadInt32(&done) == 0 {
110 - i := r.Intn(len(handles))
111 - h := (*Handle)(atomic.LoadPointer(&handles[i]))
112 - if h != nil && atomic.CompareAndSwapPointer(&handles[i], unsafe.Pointer(h), nil) {
113 - h.Release()
114 - }
115 - time.Sleep(time.Millisecond)
116 - }
117 - }(handles[ns])
118 - }
119 -
120 - go func() {
121 - handles := make([]*Handle, 100000)
122 - for atomic.LoadInt32(&done) == 0 {
123 - for i := range handles {
124 - handles[i] = c.Get(999999999, uint64(i), func() (int, Value) {
125 - return 1, 1
126 - })
127 - }
128 - for _, h := range handles {
129 - h.Release()
130 - }
131 - }
132 - }()
133 -
134 - wg.Wait()
135 -
136 - atomic.StoreInt32(&done, 1)
137 -
138 - for _, handles0 := range handles {
139 - for i := range handles0 {
140 - h := (*Handle)(atomic.LoadPointer(&handles0[i]))
141 - if h != nil && atomic.CompareAndSwapPointer(&handles0[i], unsafe.Pointer(h), nil) {
142 - h.Release()
143 - }
144 - }
145 - }
146 -
147 - for ns, objects0 := range objects {
148 - for i, o := range objects0 {
149 - if o != 0 {
150 - t.Fatalf("invalid object #%d.%d: ref=%d", ns, i, o)
151 - }
152 - }
153 - }
154 -}
155 -
156 -func TestCacheMap_NodesAndSize(t *testing.T) {
157 - c := NewCache(nil)
158 - if c.Nodes() != 0 {
159 - t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
160 - }
161 - if c.Size() != 0 {
162 - t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
163 - }
164 - set(c, 0, 1, 1, 1, nil)
165 - set(c, 0, 2, 2, 2, nil)
166 - set(c, 1, 1, 3, 3, nil)
167 - set(c, 2, 1, 4, 1, nil)
168 - if c.Nodes() != 4 {
169 - t.Errorf("invalid nodes counter: want=%d got=%d", 4, c.Nodes())
170 - }
171 - if c.Size() != 7 {
172 - t.Errorf("invalid size counter: want=%d got=%d", 4, c.Size())
173 - }
174 -}
175 -
176 -func TestLRUCache_Capacity(t *testing.T) {
177 - c := NewCache(NewLRU(10))
178 - if c.Capacity() != 10 {
179 - t.Errorf("invalid capacity: want=%d got=%d", 10, c.Capacity())
180 - }
181 - set(c, 0, 1, 1, 1, nil).Release()
182 - set(c, 0, 2, 2, 2, nil).Release()
183 - set(c, 1, 1, 3, 3, nil).Release()
184 - set(c, 2, 1, 4, 1, nil).Release()
185 - set(c, 2, 2, 5, 1, nil).Release()
186 - set(c, 2, 3, 6, 1, nil).Release()
187 - set(c, 2, 4, 7, 1, nil).Release()
188 - set(c, 2, 5, 8, 1, nil).Release()
189 - if c.Nodes() != 7 {
190 - t.Errorf("invalid nodes counter: want=%d got=%d", 7, c.Nodes())
191 - }
192 - if c.Size() != 10 {
193 - t.Errorf("invalid size counter: want=%d got=%d", 10, c.Size())
194 - }
195 - c.SetCapacity(9)
196 - if c.Capacity() != 9 {
197 - t.Errorf("invalid capacity: want=%d got=%d", 9, c.Capacity())
198 - }
199 - if c.Nodes() != 6 {
200 - t.Errorf("invalid nodes counter: want=%d got=%d", 6, c.Nodes())
201 - }
202 - if c.Size() != 8 {
203 - t.Errorf("invalid size counter: want=%d got=%d", 8, c.Size())
204 - }
205 -}
206 -
207 -func TestCacheMap_NilValue(t *testing.T) {
208 - c := NewCache(NewLRU(10))
209 - h := c.Get(0, 0, func() (size int, value Value) {
210 - return 1, nil
211 - })
212 - if h != nil {
213 - t.Error("cache handle is non-nil")
214 - }
215 - if c.Nodes() != 0 {
216 - t.Errorf("invalid nodes counter: want=%d got=%d", 0, c.Nodes())
217 - }
218 - if c.Size() != 0 {
219 - t.Errorf("invalid size counter: want=%d got=%d", 0, c.Size())
220 - }
221 -}
222 -
223 -func TestLRUCache_GetLatency(t *testing.T) {
224 - runtime.GOMAXPROCS(runtime.NumCPU())
225 -
226 - const (
227 - concurrentSet = 30
228 - concurrentGet = 3
229 - duration = 3 * time.Second
230 - delay = 3 * time.Millisecond
231 - maxkey = 100000
232 - )
233 -
234 - var (
235 - set, getHit, getAll int32
236 - getMaxLatency, getDuration int64
237 - )
238 -
239 - c := NewCache(NewLRU(5000))
240 - wg := &sync.WaitGroup{}
241 - until := time.Now().Add(duration)
242 - for i := 0; i < concurrentSet; i++ {
243 - wg.Add(1)
244 - go func(i int) {
245 - defer wg.Done()
246 - r := rand.New(rand.NewSource(time.Now().UnixNano()))
247 - for time.Now().Before(until) {
248 - c.Get(0, uint64(r.Intn(maxkey)), func() (int, Value) {
249 - time.Sleep(delay)
250 - atomic.AddInt32(&set, 1)
251 - return 1, 1
252 - }).Release()
253 - }
254 - }(i)
255 - }
256 - for i := 0; i < concurrentGet; i++ {
257 - wg.Add(1)
258 - go func(i int) {
259 - defer wg.Done()
260 - r := rand.New(rand.NewSource(time.Now().UnixNano()))
261 - for {
262 - mark := time.Now()
263 - if mark.Before(until) {
264 - h := c.Get(0, uint64(r.Intn(maxkey)), nil)
265 - latency := int64(time.Now().Sub(mark))
266 - m := atomic.LoadInt64(&getMaxLatency)
267 - if latency > m {
268 - atomic.CompareAndSwapInt64(&getMaxLatency, m, latency)
269 - }
270 - atomic.AddInt64(&getDuration, latency)
271 - if h != nil {
272 - atomic.AddInt32(&getHit, 1)
273 - h.Release()
274 - }
275 - atomic.AddInt32(&getAll, 1)
276 - } else {
277 - break
278 - }
279 - }
280 - }(i)
281 - }
282 -
283 - wg.Wait()
284 - getAvglatency := time.Duration(getDuration) / time.Duration(getAll)
285 - t.Logf("set=%d getHit=%d getAll=%d getMaxLatency=%v getAvgLatency=%v",
286 - set, getHit, getAll, time.Duration(getMaxLatency), getAvglatency)
287 -
288 - if getAvglatency > delay/3 {
289 - t.Errorf("get avg latency > %v: got=%v", delay/3, getAvglatency)
290 - }
291 -}
292 -
293 -func TestLRUCache_HitMiss(t *testing.T) {
294 - cases := []struct {
295 - key uint64
296 - value string
297 - }{
298 - {1, "vvvvvvvvv"},
299 - {100, "v1"},
300 - {0, "v2"},
301 - {12346, "v3"},
302 - {777, "v4"},
303 - {999, "v5"},
304 - {7654, "v6"},
305 - {2, "v7"},
306 - {3, "v8"},
307 - {9, "v9"},
308 - }
309 -
310 - setfin := 0
311 - c := NewCache(NewLRU(1000))
312 - for i, x := range cases {
313 - set(c, 0, x.key, x.value, len(x.value), func() {
314 - setfin++
315 - }).Release()
316 - for j, y := range cases {
317 - h := c.Get(0, y.key, nil)
318 - if j <= i {
319 - // should hit
320 - if h == nil {
321 - t.Errorf("case '%d' iteration '%d' is miss", i, j)
322 - } else {
323 - if x := h.Value().(releaserFunc).value.(string); x != y.value {
324 - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
325 - }
326 - }
327 - } else {
328 - // should miss
329 - if h != nil {
330 - t.Errorf("case '%d' iteration '%d' is hit , value '%s'", i, j, h.Value().(releaserFunc).value.(string))
331 - }
332 - }
333 - if h != nil {
334 - h.Release()
335 - }
336 - }
337 - }
338 -
339 - for i, x := range cases {
340 - finalizerOk := false
341 - c.Delete(0, x.key, func() {
342 - finalizerOk = true
343 - })
344 -
345 - if !finalizerOk {
346 - t.Errorf("case %d delete finalizer not executed", i)
347 - }
348 -
349 - for j, y := range cases {
350 - h := c.Get(0, y.key, nil)
351 - if j > i {
352 - // should hit
353 - if h == nil {
354 - t.Errorf("case '%d' iteration '%d' is miss", i, j)
355 - } else {
356 - if x := h.Value().(releaserFunc).value.(string); x != y.value {
357 - t.Errorf("case '%d' iteration '%d' has invalid value got '%s', want '%s'", i, j, x, y.value)
358 - }
359 - }
360 - } else {
361 - // should miss
362 - if h != nil {
363 - t.Errorf("case '%d' iteration '%d' is hit, value '%s'", i, j, h.Value().(releaserFunc).value.(string))
364 - }
365 - }
366 - if h != nil {
367 - h.Release()
368 - }
369 - }
370 - }
371 -
372 - if setfin != len(cases) {
373 - t.Errorf("some set finalizer may not be executed, want=%d got=%d", len(cases), setfin)
374 - }
375 -}
376 -
377 -func TestLRUCache_Eviction(t *testing.T) {
378 - c := NewCache(NewLRU(12))
379 - o1 := set(c, 0, 1, 1, 1, nil)
380 - set(c, 0, 2, 2, 1, nil).Release()
381 - set(c, 0, 3, 3, 1, nil).Release()
382 - set(c, 0, 4, 4, 1, nil).Release()
383 - set(c, 0, 5, 5, 1, nil).Release()
384 - if h := c.Get(0, 2, nil); h != nil { // 1,3,4,5,2
385 - h.Release()
386 - }
387 - set(c, 0, 9, 9, 10, nil).Release() // 5,2,9
388 -
389 - for _, key := range []uint64{9, 2, 5, 1} {
390 - h := c.Get(0, key, nil)
391 - if h == nil {
392 - t.Errorf("miss for key '%d'", key)
393 - } else {
394 - if x := h.Value().(int); x != int(key) {
395 - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
396 - }
397 - h.Release()
398 - }
399 - }
400 - o1.Release()
401 - for _, key := range []uint64{1, 2, 5} {
402 - h := c.Get(0, key, nil)
403 - if h == nil {
404 - t.Errorf("miss for key '%d'", key)
405 - } else {
406 - if x := h.Value().(int); x != int(key) {
407 - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
408 - }
409 - h.Release()
410 - }
411 - }
412 - for _, key := range []uint64{3, 4, 9} {
413 - h := c.Get(0, key, nil)
414 - if h != nil {
415 - t.Errorf("hit for key '%d'", key)
416 - if x := h.Value().(int); x != int(key) {
417 - t.Errorf("invalid value for key '%d' want '%d', got '%d'", key, key, x)
418 - }
419 - h.Release()
420 - }
421 - }
422 -}
423 -
424 -func TestLRUCache_Evict(t *testing.T) {
425 - c := NewCache(NewLRU(6))
426 - set(c, 0, 1, 1, 1, nil).Release()
427 - set(c, 0, 2, 2, 1, nil).Release()
428 - set(c, 1, 1, 4, 1, nil).Release()
429 - set(c, 1, 2, 5, 1, nil).Release()
430 - set(c, 2, 1, 6, 1, nil).Release()
431 - set(c, 2, 2, 7, 1, nil).Release()
432 -
433 - for ns := 0; ns < 3; ns++ {
434 - for key := 1; key < 3; key++ {
435 - if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
436 - h.Release()
437 - } else {
438 - t.Errorf("Cache.Get on #%d.%d return nil", ns, key)
439 - }
440 - }
441 - }
442 -
443 - if ok := c.Evict(0, 1); !ok {
444 - t.Error("first Cache.Evict on #0.1 return false")
445 - }
446 - if ok := c.Evict(0, 1); ok {
447 - t.Error("second Cache.Evict on #0.1 return true")
448 - }
449 - if h := c.Get(0, 1, nil); h != nil {
450 - t.Errorf("Cache.Get on #0.1 return non-nil: %v", h.Value())
451 - }
452 -
453 - c.EvictNS(1)
454 - if h := c.Get(1, 1, nil); h != nil {
455 - t.Errorf("Cache.Get on #1.1 return non-nil: %v", h.Value())
456 - }
457 - if h := c.Get(1, 2, nil); h != nil {
458 - t.Errorf("Cache.Get on #1.2 return non-nil: %v", h.Value())
459 - }
460 -
461 - c.EvictAll()
462 - for ns := 0; ns < 3; ns++ {
463 - for key := 1; key < 3; key++ {
464 - if h := c.Get(uint64(ns), uint64(key), nil); h != nil {
465 - t.Errorf("Cache.Get on #%d.%d return non-nil: %v", ns, key, h.Value())
466 - }
467 - }
468 - }
469 -}
470 -
471 -func TestLRUCache_Delete(t *testing.T) {
472 - delFuncCalled := 0
473 - delFunc := func() {
474 - delFuncCalled++
475 - }
476 -
477 - c := NewCache(NewLRU(2))
478 - set(c, 0, 1, 1, 1, nil).Release()
479 - set(c, 0, 2, 2, 1, nil).Release()
480 -
481 - if ok := c.Delete(0, 1, delFunc); !ok {
482 - t.Error("Cache.Delete on #1 return false")
483 - }
484 - if h := c.Get(0, 1, nil); h != nil {
485 - t.Errorf("Cache.Get on #1 return non-nil: %v", h.Value())
486 - }
487 - if ok := c.Delete(0, 1, delFunc); ok {
488 - t.Error("Cache.Delete on #1 return true")
489 - }
490 -
491 - h2 := c.Get(0, 2, nil)
492 - if h2 == nil {
493 - t.Error("Cache.Get on #2 return nil")
494 - }
495 - if ok := c.Delete(0, 2, delFunc); !ok {
496 - t.Error("(1) Cache.Delete on #2 return false")
497 - }
498 - if ok := c.Delete(0, 2, delFunc); !ok {
499 - t.Error("(2) Cache.Delete on #2 return false")
500 - }
501 -
502 - set(c, 0, 3, 3, 1, nil).Release()
503 - set(c, 0, 4, 4, 1, nil).Release()
504 - c.Get(0, 2, nil).Release()
505 -
506 - for key := 2; key <= 4; key++ {
507 - if h := c.Get(0, uint64(key), nil); h != nil {
508 - h.Release()
509 - } else {
510 - t.Errorf("Cache.Get on #%d return nil", key)
511 - }
512 - }
513 -
514 - h2.Release()
515 - if h := c.Get(0, 2, nil); h != nil {
516 - t.Errorf("Cache.Get on #2 return non-nil: %v", h.Value())
517 - }
518 -
519 - if delFuncCalled != 4 {
520 - t.Errorf("delFunc isn't called 4 times: got=%d", delFuncCalled)
521 - }
522 -}
523 -
524 -func TestLRUCache_Close(t *testing.T) {
525 - relFuncCalled := 0
526 - relFunc := func() {
527 - relFuncCalled++
528 - }
529 - delFuncCalled := 0
530 - delFunc := func() {
531 - delFuncCalled++
532 - }
533 -
534 - c := NewCache(NewLRU(2))
535 - set(c, 0, 1, 1, 1, relFunc).Release()
536 - set(c, 0, 2, 2, 1, relFunc).Release()
537 -
538 - h3 := set(c, 0, 3, 3, 1, relFunc)
539 - if h3 == nil {
540 - t.Error("Cache.Get on #3 return nil")
541 - }
542 - if ok := c.Delete(0, 3, delFunc); !ok {
543 - t.Error("Cache.Delete on #3 return false")
544 - }
545 -
546 - c.Close()
547 -
548 - if relFuncCalled != 3 {
549 - t.Errorf("relFunc isn't called 3 times: got=%d", relFuncCalled)
550 - }
551 - if delFuncCalled != 1 {
552 - t.Errorf("delFunc isn't called 1 times: got=%d", delFuncCalled)
553 - }
554 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache/lru.go deleted
-195
@@ -1,195 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package cache
8 -
9 -import (
10 - "sync"
11 - "unsafe"
12 -)
13 -
14 -type lruNode struct {
15 - n *Node
16 - h *Handle
17 - ban bool
18 -
19 - next, prev *lruNode
20 -}
21 -
22 -func (n *lruNode) insert(at *lruNode) {
23 - x := at.next
24 - at.next = n
25 - n.prev = at
26 - n.next = x
27 - x.prev = n
28 -}
29 -
30 -func (n *lruNode) remove() {
31 - if n.prev != nil {
32 - n.prev.next = n.next
33 - n.next.prev = n.prev
34 - n.prev = nil
35 - n.next = nil
36 - } else {
37 - panic("BUG: removing removed node")
38 - }
39 -}
40 -
41 -type lru struct {
42 - mu sync.Mutex
43 - capacity int
44 - used int
45 - recent lruNode
46 -}
47 -
48 -func (r *lru) reset() {
49 - r.recent.next = &r.recent
50 - r.recent.prev = &r.recent
51 - r.used = 0
52 -}
53 -
54 -func (r *lru) Capacity() int {
55 - r.mu.Lock()
56 - defer r.mu.Unlock()
57 - return r.capacity
58 -}
59 -
60 -func (r *lru) SetCapacity(capacity int) {
61 - var evicted []*lruNode
62 -
63 - r.mu.Lock()
64 - r.capacity = capacity
65 - for r.used > r.capacity {
66 - rn := r.recent.prev
67 - if rn == nil {
68 - panic("BUG: invalid LRU used or capacity counter")
69 - }
70 - rn.remove()
71 - rn.n.CacheData = nil
72 - r.used -= rn.n.Size()
73 - evicted = append(evicted, rn)
74 - }
75 - r.mu.Unlock()
76 -
77 - for _, rn := range evicted {
78 - rn.h.Release()
79 - }
80 -}
81 -
82 -func (r *lru) Promote(n *Node) {
83 - var evicted []*lruNode
84 -
85 - r.mu.Lock()
86 - if n.CacheData == nil {
87 - if n.Size() <= r.capacity {
88 - rn := &lruNode{n: n, h: n.GetHandle()}
89 - rn.insert(&r.recent)
90 - n.CacheData = unsafe.Pointer(rn)
91 - r.used += n.Size()
92 -
93 - for r.used > r.capacity {
94 - rn := r.recent.prev
95 - if rn == nil {
96 - panic("BUG: invalid LRU used or capacity counter")
97 - }
98 - rn.remove()
99 - rn.n.CacheData = nil
100 - r.used -= rn.n.Size()
101 - evicted = append(evicted, rn)
102 - }
103 - }
104 - } else {
105 - rn := (*lruNode)(n.CacheData)
106 - if !rn.ban {
107 - rn.remove()
108 - rn.insert(&r.recent)
109 - }
110 - }
111 - r.mu.Unlock()
112 -
113 - for _, rn := range evicted {
114 - rn.h.Release()
115 - }
116 -}
117 -
118 -func (r *lru) Ban(n *Node) {
119 - r.mu.Lock()
120 - if n.CacheData == nil {
121 - n.CacheData = unsafe.Pointer(&lruNode{n: n, ban: true})
122 - } else {
123 - rn := (*lruNode)(n.CacheData)
124 - if !rn.ban {
125 - rn.remove()
126 - rn.ban = true
127 - r.used -= rn.n.Size()
128 - r.mu.Unlock()
129 -
130 - rn.h.Release()
131 - rn.h = nil
132 - return
133 - }
134 - }
135 - r.mu.Unlock()
136 -}
137 -
138 -func (r *lru) Evict(n *Node) {
139 - r.mu.Lock()
140 - rn := (*lruNode)(n.CacheData)
141 - if rn == nil || rn.ban {
142 - r.mu.Unlock()
143 - return
144 - }
145 - n.CacheData = nil
146 - r.mu.Unlock()
147 -
148 - rn.h.Release()
149 -}
150 -
151 -func (r *lru) EvictNS(ns uint64) {
152 - var evicted []*lruNode
153 -
154 - r.mu.Lock()
155 - for e := r.recent.prev; e != &r.recent; {
156 - rn := e
157 - e = e.prev
158 - if rn.n.NS() == ns {
159 - rn.remove()
160 - rn.n.CacheData = nil
161 - r.used -= rn.n.Size()
162 - evicted = append(evicted, rn)
163 - }
164 - }
165 - r.mu.Unlock()
166 -
167 - for _, rn := range evicted {
168 - rn.h.Release()
169 - }
170 -}
171 -
172 -func (r *lru) EvictAll() {
173 - r.mu.Lock()
174 - back := r.recent.prev
175 - for rn := back; rn != &r.recent; rn = rn.prev {
176 - rn.n.CacheData = nil
177 - }
178 - r.reset()
179 - r.mu.Unlock()
180 -
181 - for rn := back; rn != &r.recent; rn = rn.prev {
182 - rn.h.Release()
183 - }
184 -}
185 -
186 -func (r *lru) Close() error {
187 - return nil
188 -}
189 -
190 -// NewLRU create a new LRU-cache.
191 -func NewLRU(capacity int) Cacher {
192 - r := &lru{capacity: capacity}
193 - r.reset()
194 - return r
195 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer.go deleted
-75
@@ -1,75 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
10 -
11 -type iComparer struct {
12 - ucmp comparer.Comparer
13 -}
14 -
15 -func (icmp *iComparer) uName() string {
16 - return icmp.ucmp.Name()
17 -}
18 -
19 -func (icmp *iComparer) uCompare(a, b []byte) int {
20 - return icmp.ucmp.Compare(a, b)
21 -}
22 -
23 -func (icmp *iComparer) uSeparator(dst, a, b []byte) []byte {
24 - return icmp.ucmp.Separator(dst, a, b)
25 -}
26 -
27 -func (icmp *iComparer) uSuccessor(dst, b []byte) []byte {
28 - return icmp.ucmp.Successor(dst, b)
29 -}
30 -
31 -func (icmp *iComparer) Name() string {
32 - return icmp.uName()
33 -}
34 -
35 -func (icmp *iComparer) Compare(a, b []byte) int {
36 - x := icmp.ucmp.Compare(iKey(a).ukey(), iKey(b).ukey())
37 - if x == 0 {
38 - if m, n := iKey(a).num(), iKey(b).num(); m > n {
39 - x = -1
40 - } else if m < n {
41 - x = 1
42 - }
43 - }
44 - return x
45 -}
46 -
47 -func (icmp *iComparer) Separator(dst, a, b []byte) []byte {
48 - ua, ub := iKey(a).ukey(), iKey(b).ukey()
49 - dst = icmp.ucmp.Separator(dst, ua, ub)
50 - if dst == nil {
51 - return nil
52 - }
53 - if len(dst) < len(ua) && icmp.uCompare(ua, dst) < 0 {
54 - dst = append(dst, kMaxNumBytes...)
55 - } else {
56 - // Did not close possibilities that n maybe longer than len(ub).
57 - dst = append(dst, a[len(a)-8:]...)
58 - }
59 - return dst
60 -}
61 -
62 -func (icmp *iComparer) Successor(dst, b []byte) []byte {
63 - ub := iKey(b).ukey()
64 - dst = icmp.ucmp.Successor(dst, ub)
65 - if dst == nil {
66 - return nil
67 - }
68 - if len(dst) < len(ub) && icmp.uCompare(ub, dst) < 0 {
69 - dst = append(dst, kMaxNumBytes...)
70 - } else {
71 - // Did not close possibilities that n maybe longer than len(ub).
72 - dst = append(dst, b[len(b)-8:]...)
73 - }
74 - return dst
75 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer/bytes_comparer.go deleted
-51
@@ -1,51 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package comparer
8 -
9 -import "bytes"
10 -
11 -type bytesComparer struct{}
12 -
13 -func (bytesComparer) Compare(a, b []byte) int {
14 - return bytes.Compare(a, b)
15 -}
16 -
17 -func (bytesComparer) Name() string {
18 - return "leveldb.BytewiseComparator"
19 -}
20 -
21 -func (bytesComparer) Separator(dst, a, b []byte) []byte {
22 - i, n := 0, len(a)
23 - if n > len(b) {
24 - n = len(b)
25 - }
26 - for ; i < n && a[i] == b[i]; i++ {
27 - }
28 - if i >= n {
29 - // Do not shorten if one string is a prefix of the other
30 - } else if c := a[i]; c < 0xff && c+1 < b[i] {
31 - dst = append(dst, a[:i+1]...)
32 - dst[i]++
33 - return dst
34 - }
35 - return nil
36 -}
37 -
38 -func (bytesComparer) Successor(dst, b []byte) []byte {
39 - for i, c := range b {
40 - if c != 0xff {
41 - dst = append(dst, b[:i+1]...)
42 - dst[i]++
43 - return dst
44 - }
45 - }
46 - return nil
47 -}
48 -
49 -// DefaultComparer are default implementation of the Comparer interface.
50 -// It uses the natural ordering, consistent with bytes.Compare.
51 -var DefaultComparer = bytesComparer{}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer/comparer.go deleted
-57
@@ -1,57 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package comparer provides interface and implementation for ordering
8 -// sets of data.
9 -package comparer
10 -
11 -// BasicComparer is the interface that wraps the basic Compare method.
12 -type BasicComparer interface {
13 - // Compare returns -1, 0, or +1 depending on whether a is 'less than',
14 - // 'equal to' or 'greater than' b. The two arguments can only be 'equal'
15 - // if their contents are exactly equal. Furthermore, the empty slice
16 - // must be 'less than' any non-empty slice.
17 - Compare(a, b []byte) int
18 -}
19 -
20 -// Comparer defines a total ordering over the space of []byte keys: a 'less
21 -// than' relationship.
22 -type Comparer interface {
23 - BasicComparer
24 -
25 - // Name returns name of the comparer.
26 - //
27 - // The Level-DB on-disk format stores the comparer name, and opening a
28 - // database with a different comparer from the one it was created with
29 - // will result in an error.
30 - //
31 - // An implementation to a new name whenever the comparer implementation
32 - // changes in a way that will cause the relative ordering of any two keys
33 - // to change.
34 - //
35 - // Names starting with "leveldb." are reserved and should not be used
36 - // by any users of this package.
37 - Name() string
38 -
39 - // Bellow are advanced functions used used to reduce the space requirements
40 - // for internal data structures such as index blocks.
41 -
42 - // Separator appends a sequence of bytes x to dst such that a <= x && x < b,
43 - // where 'less than' is consistent with Compare. An implementation should
44 - // return nil if x equal to a.
45 - //
46 - // Either contents of a or b should not by any means modified. Doing so
47 - // may cause corruption on the internal state.
48 - Separator(dst, a, b []byte) []byte
49 -
50 - // Successor appends a sequence of bytes x to dst such that x >= b, where
51 - // 'less than' is consistent with Compare. An implementation should return
52 - // nil if x equal to b.
53 - //
54 - // Contents of b should not by any means modified. Doing so may cause
55 - // corruption on the internal state.
56 - Successor(dst, b []byte) []byte
57 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/corrupt_test.go deleted
-500
@@ -1,500 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "fmt"
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
15 - "io"
16 - "math/rand"
17 - "testing"
18 -)
19 -
20 -const ctValSize = 1000
21 -
22 -type dbCorruptHarness struct {
23 - dbHarness
24 -}
25 -
26 -func newDbCorruptHarnessWopt(t *testing.T, o *opt.Options) *dbCorruptHarness {
27 - h := new(dbCorruptHarness)
28 - h.init(t, o)
29 - return h
30 -}
31 -
32 -func newDbCorruptHarness(t *testing.T) *dbCorruptHarness {
33 - return newDbCorruptHarnessWopt(t, &opt.Options{
34 - BlockCacheCapacity: 100,
35 - Strict: opt.StrictJournalChecksum,
36 - })
37 -}
38 -
39 -func (h *dbCorruptHarness) recover() {
40 - p := &h.dbHarness
41 - t := p.t
42 -
43 - var err error
44 - p.db, err = Recover(h.stor, h.o)
45 - if err != nil {
46 - t.Fatal("Repair: got error: ", err)
47 - }
48 -}
49 -
50 -func (h *dbCorruptHarness) build(n int) {
51 - p := &h.dbHarness
52 - t := p.t
53 - db := p.db
54 -
55 - batch := new(Batch)
56 - for i := 0; i < n; i++ {
57 - batch.Reset()
58 - batch.Put(tkey(i), tval(i, ctValSize))
59 - err := db.Write(batch, p.wo)
60 - if err != nil {
61 - t.Fatal("write error: ", err)
62 - }
63 - }
64 -}
65 -
66 -func (h *dbCorruptHarness) buildShuffled(n int, rnd *rand.Rand) {
67 - p := &h.dbHarness
68 - t := p.t
69 - db := p.db
70 -
71 - batch := new(Batch)
72 - for i := range rnd.Perm(n) {
73 - batch.Reset()
74 - batch.Put(tkey(i), tval(i, ctValSize))
75 - err := db.Write(batch, p.wo)
76 - if err != nil {
77 - t.Fatal("write error: ", err)
78 - }
79 - }
80 -}
81 -
82 -func (h *dbCorruptHarness) deleteRand(n, max int, rnd *rand.Rand) {
83 - p := &h.dbHarness
84 - t := p.t
85 - db := p.db
86 -
87 - batch := new(Batch)
88 - for i := 0; i < n; i++ {
89 - batch.Reset()
90 - batch.Delete(tkey(rnd.Intn(max)))
91 - err := db.Write(batch, p.wo)
92 - if err != nil {
93 - t.Fatal("write error: ", err)
94 - }
95 - }
96 -}
97 -
98 -func (h *dbCorruptHarness) corrupt(ft storage.FileType, fi, offset, n int) {
99 - p := &h.dbHarness
100 - t := p.t
101 -
102 - ff, _ := p.stor.GetFiles(ft)
103 - sff := files(ff)
104 - sff.sort()
105 - if fi < 0 {
106 - fi = len(sff) - 1
107 - }
108 - if fi >= len(sff) {
109 - t.Fatalf("no such file with type %q with index %d", ft, fi)
110 - }
111 -
112 - file := sff[fi]
113 -
114 - r, err := file.Open()
115 - if err != nil {
116 - t.Fatal("cannot open file: ", err)
117 - }
118 - x, err := r.Seek(0, 2)
119 - if err != nil {
120 - t.Fatal("cannot query file size: ", err)
121 - }
122 - m := int(x)
123 - if _, err := r.Seek(0, 0); err != nil {
124 - t.Fatal(err)
125 - }
126 -
127 - if offset < 0 {
128 - if -offset > m {
129 - offset = 0
130 - } else {
131 - offset = m + offset
132 - }
133 - }
134 - if offset > m {
135 - offset = m
136 - }
137 - if offset+n > m {
138 - n = m - offset
139 - }
140 -
141 - buf := make([]byte, m)
142 - _, err = io.ReadFull(r, buf)
143 - if err != nil {
144 - t.Fatal("cannot read file: ", err)
145 - }
146 - r.Close()
147 -
148 - for i := 0; i < n; i++ {
149 - buf[offset+i] ^= 0x80
150 - }
151 -
152 - err = file.Remove()
153 - if err != nil {
154 - t.Fatal("cannot remove old file: ", err)
155 - }
156 - w, err := file.Create()
157 - if err != nil {
158 - t.Fatal("cannot create new file: ", err)
159 - }
160 - _, err = w.Write(buf)
161 - if err != nil {
162 - t.Fatal("cannot write new file: ", err)
163 - }
164 - w.Close()
165 -}
166 -
167 -func (h *dbCorruptHarness) removeAll(ft storage.FileType) {
168 - ff, err := h.stor.GetFiles(ft)
169 - if err != nil {
170 - h.t.Fatal("get files: ", err)
171 - }
172 - for _, f := range ff {
173 - if err := f.Remove(); err != nil {
174 - h.t.Error("remove file: ", err)
175 - }
176 - }
177 -}
178 -
179 -func (h *dbCorruptHarness) removeOne(ft storage.FileType) {
180 - ff, err := h.stor.GetFiles(ft)
181 - if err != nil {
182 - h.t.Fatal("get files: ", err)
183 - }
184 - f := ff[rand.Intn(len(ff))]
185 - h.t.Logf("removing file @%d", f.Num())
186 - if err := f.Remove(); err != nil {
187 - h.t.Error("remove file: ", err)
188 - }
189 -}
190 -
191 -func (h *dbCorruptHarness) check(min, max int) {
192 - p := &h.dbHarness
193 - t := p.t
194 - db := p.db
195 -
196 - var n, badk, badv, missed, good int
197 - iter := db.NewIterator(nil, p.ro)
198 - for iter.Next() {
199 - k := 0
200 - fmt.Sscanf(string(iter.Key()), "%d", &k)
201 - if k < n {
202 - badk++
203 - continue
204 - }
205 - missed += k - n
206 - n = k + 1
207 - if !bytes.Equal(iter.Value(), tval(k, ctValSize)) {
208 - badv++
209 - } else {
210 - good++
211 - }
212 - }
213 - err := iter.Error()
214 - iter.Release()
215 - t.Logf("want=%d..%d got=%d badkeys=%d badvalues=%d missed=%d, err=%v",
216 - min, max, good, badk, badv, missed, err)
217 - if good < min || good > max {
218 - t.Errorf("good entries number not in range")
219 - }
220 -}
221 -
222 -func TestCorruptDB_Journal(t *testing.T) {
223 - h := newDbCorruptHarness(t)
224 -
225 - h.build(100)
226 - h.check(100, 100)
227 - h.closeDB()
228 - h.corrupt(storage.TypeJournal, -1, 19, 1)
229 - h.corrupt(storage.TypeJournal, -1, 32*1024+1000, 1)
230 -
231 - h.openDB()
232 - h.check(36, 36)
233 -
234 - h.close()
235 -}
236 -
237 -func TestCorruptDB_Table(t *testing.T) {
238 - h := newDbCorruptHarness(t)
239 -
240 - h.build(100)
241 - h.compactMem()
242 - h.compactRangeAt(0, "", "")
243 - h.compactRangeAt(1, "", "")
244 - h.closeDB()
245 - h.corrupt(storage.TypeTable, -1, 100, 1)
246 -
247 - h.openDB()
248 - h.check(99, 99)
249 -
250 - h.close()
251 -}
252 -
253 -func TestCorruptDB_TableIndex(t *testing.T) {
254 - h := newDbCorruptHarness(t)
255 -
256 - h.build(10000)
257 - h.compactMem()
258 - h.closeDB()
259 - h.corrupt(storage.TypeTable, -1, -2000, 500)
260 -
261 - h.openDB()
262 - h.check(5000, 9999)
263 -
264 - h.close()
265 -}
266 -
267 -func TestCorruptDB_MissingManifest(t *testing.T) {
268 - rnd := rand.New(rand.NewSource(0x0badda7a))
269 - h := newDbCorruptHarnessWopt(t, &opt.Options{
270 - BlockCacheCapacity: 100,
271 - Strict: opt.StrictJournalChecksum,
272 - WriteBuffer: 1000 * 60,
273 - })
274 -
275 - h.build(1000)
276 - h.compactMem()
277 - h.buildShuffled(1000, rnd)
278 - h.compactMem()
279 - h.deleteRand(500, 1000, rnd)
280 - h.compactMem()
281 - h.buildShuffled(1000, rnd)
282 - h.compactMem()
283 - h.deleteRand(500, 1000, rnd)
284 - h.compactMem()
285 - h.buildShuffled(1000, rnd)
286 - h.compactMem()
287 - h.closeDB()
288 -
289 - h.stor.SetIgnoreOpenErr(storage.TypeManifest)
290 - h.removeAll(storage.TypeManifest)
291 - h.openAssert(false)
292 - h.stor.SetIgnoreOpenErr(0)
293 -
294 - h.recover()
295 - h.check(1000, 1000)
296 - h.build(1000)
297 - h.compactMem()
298 - h.compactRange("", "")
299 - h.closeDB()
300 -
301 - h.recover()
302 - h.check(1000, 1000)
303 -
304 - h.close()
305 -}
306 -
307 -func TestCorruptDB_SequenceNumberRecovery(t *testing.T) {
308 - h := newDbCorruptHarness(t)
309 -
310 - h.put("foo", "v1")
311 - h.put("foo", "v2")
312 - h.put("foo", "v3")
313 - h.put("foo", "v4")
314 - h.put("foo", "v5")
315 - h.closeDB()
316 -
317 - h.recover()
318 - h.getVal("foo", "v5")
319 - h.put("foo", "v6")
320 - h.getVal("foo", "v6")
321 -
322 - h.reopenDB()
323 - h.getVal("foo", "v6")
324 -
325 - h.close()
326 -}
327 -
328 -func TestCorruptDB_SequenceNumberRecoveryTable(t *testing.T) {
329 - h := newDbCorruptHarness(t)
330 -
331 - h.put("foo", "v1")
332 - h.put("foo", "v2")
333 - h.put("foo", "v3")
334 - h.compactMem()
335 - h.put("foo", "v4")
336 - h.put("foo", "v5")
337 - h.compactMem()
338 - h.closeDB()
339 -
340 - h.recover()
341 - h.getVal("foo", "v5")
342 - h.put("foo", "v6")
343 - h.getVal("foo", "v6")
344 -
345 - h.reopenDB()
346 - h.getVal("foo", "v6")
347 -
348 - h.close()
349 -}
350 -
351 -func TestCorruptDB_CorruptedManifest(t *testing.T) {
352 - h := newDbCorruptHarness(t)
353 -
354 - h.put("foo", "hello")
355 - h.compactMem()
356 - h.compactRange("", "")
357 - h.closeDB()
358 - h.corrupt(storage.TypeManifest, -1, 0, 1000)
359 - h.openAssert(false)
360 -
361 - h.recover()
362 - h.getVal("foo", "hello")
363 -
364 - h.close()
365 -}
366 -
367 -func TestCorruptDB_CompactionInputError(t *testing.T) {
368 - h := newDbCorruptHarness(t)
369 -
370 - h.build(10)
371 - h.compactMem()
372 - h.closeDB()
373 - h.corrupt(storage.TypeTable, -1, 100, 1)
374 -
375 - h.openDB()
376 - h.check(9, 9)
377 -
378 - h.build(10000)
379 - h.check(10000, 10000)
380 -
381 - h.close()
382 -}
383 -
384 -func TestCorruptDB_UnrelatedKeys(t *testing.T) {
385 - h := newDbCorruptHarness(t)
386 -
387 - h.build(10)
388 - h.compactMem()
389 - h.closeDB()
390 - h.corrupt(storage.TypeTable, -1, 100, 1)
391 -
392 - h.openDB()
393 - h.put(string(tkey(1000)), string(tval(1000, ctValSize)))
394 - h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
395 - h.compactMem()
396 - h.getVal(string(tkey(1000)), string(tval(1000, ctValSize)))
397 -
398 - h.close()
399 -}
400 -
401 -func TestCorruptDB_Level0NewerFileHasOlderSeqnum(t *testing.T) {
402 - h := newDbCorruptHarness(t)
403 -
404 - h.put("a", "v1")
405 - h.put("b", "v1")
406 - h.compactMem()
407 - h.put("a", "v2")
408 - h.put("b", "v2")
409 - h.compactMem()
410 - h.put("a", "v3")
411 - h.put("b", "v3")
412 - h.compactMem()
413 - h.put("c", "v0")
414 - h.put("d", "v0")
415 - h.compactMem()
416 - h.compactRangeAt(1, "", "")
417 - h.closeDB()
418 -
419 - h.recover()
420 - h.getVal("a", "v3")
421 - h.getVal("b", "v3")
422 - h.getVal("c", "v0")
423 - h.getVal("d", "v0")
424 -
425 - h.close()
426 -}
427 -
428 -func TestCorruptDB_RecoverInvalidSeq_Issue53(t *testing.T) {
429 - h := newDbCorruptHarness(t)
430 -
431 - h.put("a", "v1")
432 - h.put("b", "v1")
433 - h.compactMem()
434 - h.put("a", "v2")
435 - h.put("b", "v2")
436 - h.compactMem()
437 - h.put("a", "v3")
438 - h.put("b", "v3")
439 - h.compactMem()
440 - h.put("c", "v0")
441 - h.put("d", "v0")
442 - h.compactMem()
443 - h.compactRangeAt(0, "", "")
444 - h.closeDB()
445 -
446 - h.recover()
447 - h.getVal("a", "v3")
448 - h.getVal("b", "v3")
449 - h.getVal("c", "v0")
450 - h.getVal("d", "v0")
451 -
452 - h.close()
453 -}
454 -
455 -func TestCorruptDB_MissingTableFiles(t *testing.T) {
456 - h := newDbCorruptHarness(t)
457 -
458 - h.put("a", "v1")
459 - h.put("b", "v1")
460 - h.compactMem()
461 - h.put("c", "v2")
462 - h.put("d", "v2")
463 - h.compactMem()
464 - h.put("e", "v3")
465 - h.put("f", "v3")
466 - h.closeDB()
467 -
468 - h.removeOne(storage.TypeTable)
469 - h.openAssert(false)
470 -
471 - h.close()
472 -}
473 -
474 -func TestCorruptDB_RecoverTable(t *testing.T) {
475 - h := newDbCorruptHarnessWopt(t, &opt.Options{
476 - WriteBuffer: 112 * opt.KiB,
477 - CompactionTableSize: 90 * opt.KiB,
478 - Filter: filter.NewBloomFilter(10),
479 - })
480 -
481 - h.build(1000)
482 - h.compactMem()
483 - h.compactRangeAt(0, "", "")
484 - h.compactRangeAt(1, "", "")
485 - seq := h.db.seq
486 - h.closeDB()
487 - h.corrupt(storage.TypeTable, 0, 1000, 1)
488 - h.corrupt(storage.TypeTable, 3, 10000, 1)
489 - // Corrupted filter shouldn't affect recovery.
490 - h.corrupt(storage.TypeTable, 3, 113888, 10)
491 - h.corrupt(storage.TypeTable, -1, 20000, 1)
492 -
493 - h.recover()
494 - if h.db.seq != seq {
495 - t.Errorf("invalid seq, want=%d got=%d", seq, h.db.seq)
496 - }
497 - h.check(985, 985)
498 -
499 - h.close()
500 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db.go deleted
-945
@@ -1,945 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "container/list"
11 - "fmt"
12 - "io"
13 - "os"
14 - "runtime"
15 - "strings"
16 - "sync"
17 - "sync/atomic"
18 - "time"
19 -
20 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
22 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
23 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
24 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
25 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
26 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table"
27 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
28 -)
29 -
30 -type DB struct // DB is a LevelDB database.
31 -{
32 - // Need 64-bit alignment.
33 - seq uint64
34 -
35 - // Session.
36 - s *session
37 -
38 - // MemDB.
39 - memMu sync.RWMutex
40 - memPool chan *memdb.DB
41 - mem, frozenMem *memDB
42 - journal *journal.Writer
43 - journalWriter storage.Writer
44 - journalFile storage.File
45 - frozenJournalFile storage.File
46 - frozenSeq uint64
47 -
48 - // Snapshot.
49 - snapsMu sync.Mutex
50 - snapsList *list.List
51 -
52 - // Stats.
53 - aliveSnaps, aliveIters int32
54 -
55 - // Write.
56 - writeC chan *Batch
57 - writeMergedC chan bool
58 - writeLockC chan struct{}
59 - writeAckC chan error
60 - writeDelay time.Duration
61 - writeDelayN int
62 - journalC chan *Batch
63 - journalAckC chan error
64 -
65 - // Compaction.
66 - tcompCmdC chan cCmd
67 - tcompPauseC chan chan<- struct{}
68 - mcompCmdC chan cCmd
69 - compErrC chan error
70 - compPerErrC chan error
71 - compErrSetC chan error
72 - compStats []cStats
73 -
74 - // Close.
75 - closeW sync.WaitGroup
76 - closeC chan struct{}
77 - closed uint32
78 - closer io.Closer
79 -}
80 -
81 -func openDB(s *session) (*DB, error) {
82 - s.log("db@open opening")
83 - start := time.Now()
84 - db := &DB{
85 - s: s,
86 - // Initial sequence
87 - seq: s.stSeqNum,
88 - // MemDB
89 - memPool: make(chan *memdb.DB, 1),
90 - // Snapshot
91 - snapsList: list.New(),
92 - // Write
93 - writeC: make(chan *Batch),
94 - writeMergedC: make(chan bool),
95 - writeLockC: make(chan struct{}, 1),
96 - writeAckC: make(chan error),
97 - journalC: make(chan *Batch),
98 - journalAckC: make(chan error),
99 - // Compaction
100 - tcompCmdC: make(chan cCmd),
101 - tcompPauseC: make(chan chan<- struct{}),
102 - mcompCmdC: make(chan cCmd),
103 - compErrC: make(chan error),
104 - compPerErrC: make(chan error),
105 - compErrSetC: make(chan error),
106 - compStats: make([]cStats, s.o.GetNumLevel()),
107 - // Close
108 - closeC: make(chan struct{}),
109 - }
110 -
111 - if err := db.recoverJournal(); err != nil {
112 - return nil, err
113 - }
114 -
115 - // Remove any obsolete files.
116 - if err := db.checkAndCleanFiles(); err != nil {
117 - // Close journal.
118 - if db.journal != nil {
119 - db.journal.Close()
120 - db.journalWriter.Close()
121 - }
122 - return nil, err
123 - }
124 -
125 - // Doesn't need to be included in the wait group.
126 - go db.compactionError()
127 - go db.mpoolDrain()
128 -
129 - db.closeW.Add(3)
130 - go db.tCompaction()
131 - go db.mCompaction()
132 - go db.jWriter()
133 -
134 - s.logf("db@open done T·%v", time.Since(start))
135 -
136 - runtime.SetFinalizer(db, (*DB).Close)
137 - return db, nil
138 -}
139 -
140 -// Open opens or creates a DB for the given storage.
141 -// The DB will be created if not exist, unless ErrorIfMissing is true.
142 -// Also, if ErrorIfExist is true and the DB exist Open will returns
143 -// os.ErrExist error.
144 -//
145 -// Open will return an error with type of ErrCorrupted if corruption
146 -// detected in the DB. Corrupted DB can be recovered with Recover
147 -// function.
148 -//
149 -// The returned DB instance is goroutine-safe.
150 -// The DB must be closed after use, by calling Close method.
151 -func Open(stor storage.Storage, o *opt.Options) (db *DB, err error) {
152 - s, err := newSession(stor, o)
153 - if err != nil {
154 - return
155 - }
156 - defer func() {
157 - if err != nil {
158 - s.close()
159 - s.release()
160 - }
161 - }()
162 -
163 - err = s.recover()
164 - if err != nil {
165 - if !os.IsNotExist(err) || s.o.GetErrorIfMissing() {
166 - return
167 - }
168 - err = s.create()
169 - if err != nil {
170 - return
171 - }
172 - } else if s.o.GetErrorIfExist() {
173 - err = os.ErrExist
174 - return
175 - }
176 -
177 - return openDB(s)
178 -}
179 -
180 -// OpenFile opens or creates a DB for the given path.
181 -// The DB will be created if not exist, unless ErrorIfMissing is true.
182 -// Also, if ErrorIfExist is true and the DB exist OpenFile will returns
183 -// os.ErrExist error.
184 -//
185 -// OpenFile uses standard file-system backed storage implementation as
186 -// desribed in the leveldb/storage package.
187 -//
188 -// OpenFile will return an error with type of ErrCorrupted if corruption
189 -// detected in the DB. Corrupted DB can be recovered with Recover
190 -// function.
191 -//
192 -// The returned DB instance is goroutine-safe.
193 -// The DB must be closed after use, by calling Close method.
194 -func OpenFile(path string, o *opt.Options) (db *DB, err error) {
195 - stor, err := storage.OpenFile(path)
196 - if err != nil {
197 - return
198 - }
199 - db, err = Open(stor, o)
200 - if err != nil {
201 - stor.Close()
202 - } else {
203 - db.closer = stor
204 - }
205 - return
206 -}
207 -
208 -// Recover recovers and opens a DB with missing or corrupted manifest files
209 -// for the given storage. It will ignore any manifest files, valid or not.
210 -// The DB must already exist or it will returns an error.
211 -// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
212 -//
213 -// The returned DB instance is goroutine-safe.
214 -// The DB must be closed after use, by calling Close method.
215 -func Recover(stor storage.Storage, o *opt.Options) (db *DB, err error) {
216 - s, err := newSession(stor, o)
217 - if err != nil {
218 - return
219 - }
220 - defer func() {
221 - if err != nil {
222 - s.close()
223 - s.release()
224 - }
225 - }()
226 -
227 - err = recoverTable(s, o)
228 - if err != nil {
229 - return
230 - }
231 - return openDB(s)
232 -}
233 -
234 -// RecoverFile recovers and opens a DB with missing or corrupted manifest files
235 -// for the given path. It will ignore any manifest files, valid or not.
236 -// The DB must already exist or it will returns an error.
237 -// Also, Recover will ignore ErrorIfMissing and ErrorIfExist options.
238 -//
239 -// RecoverFile uses standard file-system backed storage implementation as desribed
240 -// in the leveldb/storage package.
241 -//
242 -// The returned DB instance is goroutine-safe.
243 -// The DB must be closed after use, by calling Close method.
244 -func RecoverFile(path string, o *opt.Options) (db *DB, err error) {
245 - stor, err := storage.OpenFile(path)
246 - if err != nil {
247 - return
248 - }
249 - db, err = Recover(stor, o)
250 - if err != nil {
251 - stor.Close()
252 - } else {
253 - db.closer = stor
254 - }
255 - return
256 -}
257 -
258 -func recoverTable(s *session, o *opt.Options) error {
259 - o = dupOptions(o)
260 - // Mask StrictReader, lets StrictRecovery doing its job.
261 - o.Strict &= ^opt.StrictReader
262 -
263 - // Get all tables and sort it by file number.
264 - tableFiles_, err := s.getFiles(storage.TypeTable)
265 - if err != nil {
266 - return err
267 - }
268 - tableFiles := files(tableFiles_)
269 - tableFiles.sort()
270 -
271 - var (
272 - maxSeq uint64
273 - recoveredKey, goodKey, corruptedKey, corruptedBlock, droppedTable int
274 -
275 - // We will drop corrupted table.
276 - strict = o.GetStrict(opt.StrictRecovery)
277 -
278 - rec = &sessionRecord{numLevel: o.GetNumLevel()}
279 - bpool = util.NewBufferPool(o.GetBlockSize() + 5)
280 - )
281 - buildTable := func(iter iterator.Iterator) (tmp storage.File, size int64, err error) {
282 - tmp = s.newTemp()
283 - writer, err := tmp.Create()
284 - if err != nil {
285 - return
286 - }
287 - defer func() {
288 - writer.Close()
289 - if err != nil {
290 - tmp.Remove()
291 - tmp = nil
292 - }
293 - }()
294 -
295 - // Copy entries.
296 - tw := table.NewWriter(writer, o)
297 - for iter.Next() {
298 - key := iter.Key()
299 - if validIkey(key) {
300 - err = tw.Append(key, iter.Value())
301 - if err != nil {
302 - return
303 - }
304 - }
305 - }
306 - err = iter.Error()
307 - if err != nil {
308 - return
309 - }
310 - err = tw.Close()
311 - if err != nil {
312 - return
313 - }
314 - err = writer.Sync()
315 - if err != nil {
316 - return
317 - }
318 - size = int64(tw.BytesLen())
319 - return
320 - }
321 - recoverTable := func(file storage.File) error {
322 - s.logf("table@recovery recovering @%d", file.Num())
323 - reader, err := file.Open()
324 - if err != nil {
325 - return err
326 - }
327 - var closed bool
328 - defer func() {
329 - if !closed {
330 - reader.Close()
331 - }
332 - }()
333 -
334 - // Get file size.
335 - size, err := reader.Seek(0, 2)
336 - if err != nil {
337 - return err
338 - }
339 -
340 - var (
341 - tSeq uint64
342 - tgoodKey, tcorruptedKey, tcorruptedBlock int
343 - imin, imax []byte
344 - )
345 - tr, err := table.NewReader(reader, size, storage.NewFileInfo(file), nil, bpool, o)
346 - if err != nil {
347 - return err
348 - }
349 - iter := tr.NewIterator(nil, nil)
350 - if itererr, ok := iter.(iterator.ErrorCallbackSetter); ok {
351 - itererr.SetErrorCallback(func(err error) {
352 - if errors.IsCorrupted(err) {
353 - s.logf("table@recovery block corruption @%d %q", file.Num(), err)
354 - tcorruptedBlock++
355 - }
356 - })
357 - }
358 -
359 - // Scan the table.
360 - for iter.Next() {
361 - key := iter.Key()
362 - _, seq, _, kerr := parseIkey(key)
363 - if kerr != nil {
364 - tcorruptedKey++
365 - continue
366 - }
367 - tgoodKey++
368 - if seq > tSeq {
369 - tSeq = seq
370 - }
371 - if imin == nil {
372 - imin = append([]byte{}, key...)
373 - }
374 - imax = append(imax[:0], key...)
375 - }
376 - if err := iter.Error(); err != nil {
377 - iter.Release()
378 - return err
379 - }
380 - iter.Release()
381 -
382 - goodKey += tgoodKey
383 - corruptedKey += tcorruptedKey
384 - corruptedBlock += tcorruptedBlock
385 -
386 - if strict && (tcorruptedKey > 0 || tcorruptedBlock > 0) {
387 - droppedTable++
388 - s.logf("table@recovery dropped @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
389 - return nil
390 - }
391 -
392 - if tgoodKey > 0 {
393 - if tcorruptedKey > 0 || tcorruptedBlock > 0 {
394 - // Rebuild the table.
395 - s.logf("table@recovery rebuilding @%d", file.Num())
396 - iter := tr.NewIterator(nil, nil)
397 - tmp, newSize, err := buildTable(iter)
398 - iter.Release()
399 - if err != nil {
400 - return err
401 - }
402 - closed = true
403 - reader.Close()
404 - if err := file.Replace(tmp); err != nil {
405 - return err
406 - }
407 - size = newSize
408 - }
409 - if tSeq > maxSeq {
410 - maxSeq = tSeq
411 - }
412 - recoveredKey += tgoodKey
413 - // Add table to level 0.
414 - rec.addTable(0, file.Num(), uint64(size), imin, imax)
415 - s.logf("table@recovery recovered @%d Gk·%d Ck·%d Cb·%d S·%d Q·%d", file.Num(), tgoodKey, tcorruptedKey, tcorruptedBlock, size, tSeq)
416 - } else {
417 - droppedTable++
418 - s.logf("table@recovery unrecoverable @%d Ck·%d Cb·%d S·%d", file.Num(), tcorruptedKey, tcorruptedBlock, size)
419 - }
420 -
421 - return nil
422 - }
423 -
424 - // Recover all tables.
425 - if len(tableFiles) > 0 {
426 - s.logf("table@recovery F·%d", len(tableFiles))
427 -
428 - // Mark file number as used.
429 - s.markFileNum(tableFiles[len(tableFiles)-1].Num())
430 -
431 - for _, file := range tableFiles {
432 - if err := recoverTable(file); err != nil {
433 - return err
434 - }
435 - }
436 -
437 - s.logf("table@recovery recovered F·%d N·%d Gk·%d Ck·%d Q·%d", len(tableFiles), recoveredKey, goodKey, corruptedKey, maxSeq)
438 - }
439 -
440 - // Set sequence number.
441 - rec.setSeqNum(maxSeq)
442 -
443 - // Create new manifest.
444 - if err := s.create(); err != nil {
445 - return err
446 - }
447 -
448 - // Commit.
449 - return s.commit(rec)
450 -}
451 -
452 -func (db *DB) recoverJournal() error {
453 - // Get all tables and sort it by file number.
454 - journalFiles_, err := db.s.getFiles(storage.TypeJournal)
455 - if err != nil {
456 - return err
457 - }
458 - journalFiles := files(journalFiles_)
459 - journalFiles.sort()
460 -
461 - // Discard older journal.
462 - prev := -1
463 - for i, file := range journalFiles {
464 - if file.Num() >= db.s.stJournalNum {
465 - if prev >= 0 {
466 - i--
467 - journalFiles[i] = journalFiles[prev]
468 - }
469 - journalFiles = journalFiles[i:]
470 - break
471 - } else if file.Num() == db.s.stPrevJournalNum {
472 - prev = i
473 - }
474 - }
475 -
476 - var jr *journal.Reader
477 - var of storage.File
478 - var mem *memdb.DB
479 - batch := new(Batch)
480 - cm := newCMem(db.s)
481 - buf := new(util.Buffer)
482 - // Options.
483 - strict := db.s.o.GetStrict(opt.StrictJournal)
484 - checksum := db.s.o.GetStrict(opt.StrictJournalChecksum)
485 - writeBuffer := db.s.o.GetWriteBuffer()
486 - recoverJournal := func(file storage.File) error {
487 - db.logf("journal@recovery recovering @%d", file.Num())
488 - reader, err := file.Open()
489 - if err != nil {
490 - return err
491 - }
492 - defer reader.Close()
493 -
494 - // Create/reset journal reader instance.
495 - if jr == nil {
496 - jr = journal.NewReader(reader, dropper{db.s, file}, strict, checksum)
497 - } else {
498 - jr.Reset(reader, dropper{db.s, file}, strict, checksum)
499 - }
500 -
501 - // Flush memdb and remove obsolete journal file.
502 - if of != nil {
503 - if mem.Len() > 0 {
504 - if err := cm.flush(mem, 0); err != nil {
505 - return err
506 - }
507 - }
508 - if err := cm.commit(file.Num(), db.seq); err != nil {
509 - return err
510 - }
511 - cm.reset()
512 - of.Remove()
513 - of = nil
514 - }
515 -
516 - // Replay journal to memdb.
517 - mem.Reset()
518 - for {
519 - r, err := jr.Next()
520 - if err != nil {
521 - if err == io.EOF {
522 - break
523 - }
524 - return errors.SetFile(err, file)
525 - }
526 -
527 - buf.Reset()
528 - if _, err := buf.ReadFrom(r); err != nil {
529 - if err == io.ErrUnexpectedEOF {
530 - // This is error returned due to corruption, with strict == false.
531 - continue
532 - } else {
533 - return errors.SetFile(err, file)
534 - }
535 - }
536 - if err := batch.memDecodeAndReplay(db.seq, buf.Bytes(), mem); err != nil {
537 - if strict || !errors.IsCorrupted(err) {
538 - return errors.SetFile(err, file)
539 - } else {
540 - db.s.logf("journal error: %v (skipped)", err)
541 - // We won't apply sequence number as it might be corrupted.
542 - continue
543 - }
544 - }
545 -
546 - // Save sequence number.
547 - db.seq = batch.seq + uint64(batch.Len())
548 -
549 - // Flush it if large enough.
550 - if mem.Size() >= writeBuffer {
551 - if err := cm.flush(mem, 0); err != nil {
552 - return err
553 - }
554 - mem.Reset()
555 - }
556 - }
557 -
558 - of = file
559 - return nil
560 - }
561 -
562 - // Recover all journals.
563 - if len(journalFiles) > 0 {
564 - db.logf("journal@recovery F·%d", len(journalFiles))
565 -
566 - // Mark file number as used.
567 - db.s.markFileNum(journalFiles[len(journalFiles)-1].Num())
568 -
569 - mem = memdb.New(db.s.icmp, writeBuffer)
570 - for _, file := range journalFiles {
571 - if err := recoverJournal(file); err != nil {
572 - return err
573 - }
574 - }
575 -
576 - // Flush the last journal.
577 - if mem.Len() > 0 {
578 - if err := cm.flush(mem, 0); err != nil {
579 - return err
580 - }
581 - }
582 - }
583 -
584 - // Create a new journal.
585 - if _, err := db.newMem(0); err != nil {
586 - return err
587 - }
588 -
589 - // Commit.
590 - if err := cm.commit(db.journalFile.Num(), db.seq); err != nil {
591 - // Close journal.
592 - if db.journal != nil {
593 - db.journal.Close()
594 - db.journalWriter.Close()
595 - }
596 - return err
597 - }
598 -
599 - // Remove the last obsolete journal file.
600 - if of != nil {
601 - of.Remove()
602 - }
603 -
604 - return nil
605 -}
606 -
607 -func (db *DB) get(key []byte, seq uint64, ro *opt.ReadOptions) (value []byte, err error) {
608 - ikey := newIkey(key, seq, ktSeek)
609 -
610 - em, fm := db.getMems()
611 - for _, m := range [...]*memDB{em, fm} {
612 - if m == nil {
613 - continue
614 - }
615 - defer m.decref()
616 -
617 - mk, mv, me := m.mdb.Find(ikey)
618 - if me == nil {
619 - ukey, _, kt, kerr := parseIkey(mk)
620 - if kerr != nil {
621 - // Shouldn't have had happen.
622 - panic(kerr)
623 - }
624 - if db.s.icmp.uCompare(ukey, key) == 0 {
625 - if kt == ktDel {
626 - return nil, ErrNotFound
627 - }
628 - return append([]byte{}, mv...), nil
629 - }
630 - } else if me != ErrNotFound {
631 - return nil, me
632 - }
633 - }
634 -
635 - v := db.s.version()
636 - value, cSched, err := v.get(ikey, ro, false)
637 - v.release()
638 - if cSched {
639 - // Trigger table compaction.
640 - db.compSendTrigger(db.tcompCmdC)
641 - }
642 - return
643 -}
644 -
645 -func (db *DB) has(key []byte, seq uint64, ro *opt.ReadOptions) (ret bool, err error) {
646 - ikey := newIkey(key, seq, ktSeek)
647 -
648 - em, fm := db.getMems()
649 - for _, m := range [...]*memDB{em, fm} {
650 - if m == nil {
651 - continue
652 - }
653 - defer m.decref()
654 -
655 - mk, _, me := m.mdb.Find(ikey)
656 - if me == nil {
657 - ukey, _, kt, kerr := parseIkey(mk)
658 - if kerr != nil {
659 - // Shouldn't have had happen.
660 - panic(kerr)
661 - }
662 - if db.s.icmp.uCompare(ukey, key) == 0 {
663 - if kt == ktDel {
664 - return false, nil
665 - }
666 - return true, nil
667 - }
668 - } else if me != ErrNotFound {
669 - return false, me
670 - }
671 - }
672 -
673 - v := db.s.version()
674 - _, cSched, err := v.get(ikey, ro, true)
675 - v.release()
676 - if cSched {
677 - // Trigger table compaction.
678 - db.compSendTrigger(db.tcompCmdC)
679 - }
680 - if err == nil {
681 - ret = true
682 - } else if err == ErrNotFound {
683 - err = nil
684 - }
685 - return
686 -}
687 -
688 -// Get gets the value for the given key. It returns ErrNotFound if the
689 -// DB does not contains the key.
690 -//
691 -// The returned slice is its own copy, it is safe to modify the contents
692 -// of the returned slice.
693 -// It is safe to modify the contents of the argument after Get returns.
694 -func (db *DB) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
695 - err = db.ok()
696 - if err != nil {
697 - return
698 - }
699 -
700 - se := db.acquireSnapshot()
701 - defer db.releaseSnapshot(se)
702 - return db.get(key, se.seq, ro)
703 -}
704 -
705 -// Has returns true if the DB does contains the given key.
706 -//
707 -// It is safe to modify the contents of the argument after Get returns.
708 -func (db *DB) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
709 - err = db.ok()
710 - if err != nil {
711 - return
712 - }
713 -
714 - se := db.acquireSnapshot()
715 - defer db.releaseSnapshot(se)
716 - return db.has(key, se.seq, ro)
717 -}
718 -
719 -// NewIterator returns an iterator for the latest snapshot of the
720 -// uderlying DB.
721 -// The returned iterator is not goroutine-safe, but it is safe to use
722 -// multiple iterators concurrently, with each in a dedicated goroutine.
723 -// It is also safe to use an iterator concurrently with modifying its
724 -// underlying DB. The resultant key/value pairs are guaranteed to be
725 -// consistent.
726 -//
727 -// Slice allows slicing the iterator to only contains keys in the given
728 -// range. A nil Range.Start is treated as a key before all keys in the
729 -// DB. And a nil Range.Limit is treated as a key after all keys in
730 -// the DB.
731 -//
732 -// The iterator must be released after use, by calling Release method.
733 -//
734 -// Also read Iterator documentation of the leveldb/iterator package.
735 -func (db *DB) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
736 - if err := db.ok(); err != nil {
737 - return iterator.NewEmptyIterator(err)
738 - }
739 -
740 - se := db.acquireSnapshot()
741 - defer db.releaseSnapshot(se)
742 - // Iterator holds 'version' lock, 'version' is immutable so snapshot
743 - // can be released after iterator created.
744 - return db.newIterator(se.seq, slice, ro)
745 -}
746 -
747 -// GetSnapshot returns a latest snapshot of the underlying DB. A snapshot
748 -// is a frozen snapshot of a DB state at a particular point in time. The
749 -// content of snapshot are guaranteed to be consistent.
750 -//
751 -// The snapshot must be released after use, by calling Release method.
752 -func (db *DB) GetSnapshot() (*Snapshot, error) {
753 - if err := db.ok(); err != nil {
754 - return nil, err
755 - }
756 -
757 - return db.newSnapshot(), nil
758 -}
759 -
760 -// GetProperty returns value of the given property name.
761 -//
762 -// Property names:
763 -// leveldb.num-files-at-level{n}
764 -// Returns the number of files at level 'n'.
765 -// leveldb.stats
766 -// Returns statistics of the underlying DB.
767 -// leveldb.sstables
768 -// Returns sstables list for each level.
769 -// leveldb.blockpool
770 -// Returns block pool stats.
771 -// leveldb.cachedblock
772 -// Returns size of cached block.
773 -// leveldb.openedtables
774 -// Returns number of opened tables.
775 -// leveldb.alivesnaps
776 -// Returns number of alive snapshots.
777 -// leveldb.aliveiters
778 -// Returns number of alive iterators.
779 -func (db *DB) GetProperty(name string) (value string, err error) {
780 - err = db.ok()
781 - if err != nil {
782 - return
783 - }
784 -
785 - const prefix = "leveldb."
786 - if !strings.HasPrefix(name, prefix) {
787 - return "", errors.New("leveldb: GetProperty: unknown property: " + name)
788 - }
789 - p := name[len(prefix):]
790 -
791 - v := db.s.version()
792 - defer v.release()
793 -
794 - numFilesPrefix := "num-files-at-level"
795 - switch {
796 - case strings.HasPrefix(p, numFilesPrefix):
797 - var level uint
798 - var rest string
799 - n, _ := fmt.Sscanf(p[len(numFilesPrefix):], "%d%s", &level, &rest)
800 - if n != 1 || int(level) >= db.s.o.GetNumLevel() {
801 - err = errors.New("leveldb: GetProperty: invalid property: " + name)
802 - } else {
803 - value = fmt.Sprint(v.tLen(int(level)))
804 - }
805 - case p == "stats":
806 - value = "Compactions\n" +
807 - " Level | Tables | Size(MB) | Time(sec) | Read(MB) | Write(MB)\n" +
808 - "-------+------------+---------------+---------------+---------------+---------------\n"
809 - for level, tables := range v.tables {
810 - duration, read, write := db.compStats[level].get()
811 - if len(tables) == 0 && duration == 0 {
812 - continue
813 - }
814 - value += fmt.Sprintf(" %3d | %10d | %13.5f | %13.5f | %13.5f | %13.5f\n",
815 - level, len(tables), float64(tables.size())/1048576.0, duration.Seconds(),
816 - float64(read)/1048576.0, float64(write)/1048576.0)
817 - }
818 - case p == "sstables":
819 - for level, tables := range v.tables {
820 - value += fmt.Sprintf("--- level %d ---\n", level)
821 - for _, t := range tables {
822 - value += fmt.Sprintf("%d:%d[%q .. %q]\n", t.file.Num(), t.size, t.imin, t.imax)
823 - }
824 - }
825 - case p == "blockpool":
826 - value = fmt.Sprintf("%v", db.s.tops.bpool)
827 - case p == "cachedblock":
828 - if db.s.tops.bcache != nil {
829 - value = fmt.Sprintf("%d", db.s.tops.bcache.Size())
830 - } else {
831 - value = "<nil>"
832 - }
833 - case p == "openedtables":
834 - value = fmt.Sprintf("%d", db.s.tops.cache.Size())
835 - case p == "alivesnaps":
836 - value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveSnaps))
837 - case p == "aliveiters":
838 - value = fmt.Sprintf("%d", atomic.LoadInt32(&db.aliveIters))
839 - default:
840 - err = errors.New("leveldb: GetProperty: unknown property: " + name)
841 - }
842 -
843 - return
844 -}
845 -
846 -// SizeOf calculates approximate sizes of the given key ranges.
847 -// The length of the returned sizes are equal with the length of the given
848 -// ranges. The returned sizes measure storage space usage, so if the user
849 -// data compresses by a factor of ten, the returned sizes will be one-tenth
850 -// the size of the corresponding user data size.
851 -// The results may not include the sizes of recently written data.
852 -func (db *DB) SizeOf(ranges []util.Range) (Sizes, error) {
853 - if err := db.ok(); err != nil {
854 - return nil, err
855 - }
856 -
857 - v := db.s.version()
858 - defer v.release()
859 -
860 - sizes := make(Sizes, 0, len(ranges))
861 - for _, r := range ranges {
862 - imin := newIkey(r.Start, kMaxSeq, ktSeek)
863 - imax := newIkey(r.Limit, kMaxSeq, ktSeek)
864 - start, err := v.offsetOf(imin)
865 - if err != nil {
866 - return nil, err
867 - }
868 - limit, err := v.offsetOf(imax)
869 - if err != nil {
870 - return nil, err
871 - }
872 - var size uint64
873 - if limit >= start {
874 - size = limit - start
875 - }
876 - sizes = append(sizes, size)
877 - }
878 -
879 - return sizes, nil
880 -}
881 -
882 -// Close closes the DB. This will also releases any outstanding snapshot and
883 -// abort any in-flight compaction.
884 -//
885 -// It is not safe to close a DB until all outstanding iterators are released.
886 -// It is valid to call Close multiple times. Other methods should not be
887 -// called after the DB has been closed.
888 -func (db *DB) Close() error {
889 - if !db.setClosed() {
890 - return ErrClosed
891 - }
892 -
893 - start := time.Now()
894 - db.log("db@close closing")
895 -
896 - // Clear the finalizer.
897 - runtime.SetFinalizer(db, nil)
898 -
899 - // Get compaction error.
900 - var err error
901 - select {
902 - case err = <-db.compErrC:
903 - default:
904 - }
905 -
906 - // Signal all goroutines.
907 - close(db.closeC)
908 -
909 - // Wait for all gorotines to exit.
910 - db.closeW.Wait()
911 -
912 - // Lock writer and closes journal.
913 - db.writeLockC <- struct{}{}
914 - if db.journal != nil {
915 - db.journal.Close()
916 - db.journalWriter.Close()
917 - }
918 -
919 - if db.writeDelayN > 0 {
920 - db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
921 - }
922 -
923 - // Close session.
924 - db.s.close()
925 - db.logf("db@close done T·%v", time.Since(start))
926 - db.s.release()
927 -
928 - if db.closer != nil {
929 - if err1 := db.closer.Close(); err == nil {
930 - err = err1
931 - }
932 - }
933 -
934 - // NIL'ing pointers.
935 - db.s = nil
936 - db.mem = nil
937 - db.frozenMem = nil
938 - db.journal = nil
939 - db.journalWriter = nil
940 - db.journalFile = nil
941 - db.frozenJournalFile = nil
942 - db.closer = nil
943 -
944 - return err
945 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_compaction.go deleted
-835
@@ -1,835 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "sync"
11 - "time"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
16 -)
17 -
18 -var (
19 - errCompactionTransactExiting = errors.New("leveldb: compaction transact exiting")
20 -)
21 -
22 -type cStats struct {
23 - sync.Mutex
24 - duration time.Duration
25 - read uint64
26 - write uint64
27 -}
28 -
29 -func (p *cStats) add(n *cStatsStaging) {
30 - p.Lock()
31 - p.duration += n.duration
32 - p.read += n.read
33 - p.write += n.write
34 - p.Unlock()
35 -}
36 -
37 -func (p *cStats) get() (duration time.Duration, read, write uint64) {
38 - p.Lock()
39 - defer p.Unlock()
40 - return p.duration, p.read, p.write
41 -}
42 -
43 -type cStatsStaging struct {
44 - start time.Time
45 - duration time.Duration
46 - on bool
47 - read uint64
48 - write uint64
49 -}
50 -
51 -func (p *cStatsStaging) startTimer() {
52 - if !p.on {
53 - p.start = time.Now()
54 - p.on = true
55 - }
56 -}
57 -
58 -func (p *cStatsStaging) stopTimer() {
59 - if p.on {
60 - p.duration += time.Since(p.start)
61 - p.on = false
62 - }
63 -}
64 -
65 -type cMem struct {
66 - s *session
67 - level int
68 - rec *sessionRecord
69 -}
70 -
71 -func newCMem(s *session) *cMem {
72 - return &cMem{s: s, rec: &sessionRecord{numLevel: s.o.GetNumLevel()}}
73 -}
74 -
75 -func (c *cMem) flush(mem *memdb.DB, level int) error {
76 - s := c.s
77 -
78 - // Write memdb to table.
79 - iter := mem.NewIterator(nil)
80 - defer iter.Release()
81 - t, n, err := s.tops.createFrom(iter)
82 - if err != nil {
83 - return err
84 - }
85 -
86 - // Pick level.
87 - if level < 0 {
88 - v := s.version()
89 - level = v.pickLevel(t.imin.ukey(), t.imax.ukey())
90 - v.release()
91 - }
92 - c.rec.addTableFile(level, t)
93 -
94 - s.logf("mem@flush created L%d@%d N·%d S·%s %q:%q", level, t.file.Num(), n, shortenb(int(t.size)), t.imin, t.imax)
95 -
96 - c.level = level
97 - return nil
98 -}
99 -
100 -func (c *cMem) reset() {
101 - c.rec = &sessionRecord{numLevel: c.s.o.GetNumLevel()}
102 -}
103 -
104 -func (c *cMem) commit(journal, seq uint64) error {
105 - c.rec.setJournalNum(journal)
106 - c.rec.setSeqNum(seq)
107 -
108 - // Commit changes.
109 - return c.s.commit(c.rec)
110 -}
111 -
112 -func (db *DB) compactionError() {
113 - var (
114 - err error
115 - wlocked bool
116 - )
117 -noerr:
118 - // No error.
119 - for {
120 - select {
121 - case err = <-db.compErrSetC:
122 - switch {
123 - case err == nil:
124 - case errors.IsCorrupted(err):
125 - goto hasperr
126 - default:
127 - goto haserr
128 - }
129 - case _, _ = <-db.closeC:
130 - return
131 - }
132 - }
133 -haserr:
134 - // Transient error.
135 - for {
136 - select {
137 - case db.compErrC <- err:
138 - case err = <-db.compErrSetC:
139 - switch {
140 - case err == nil:
141 - goto noerr
142 - case errors.IsCorrupted(err):
143 - goto hasperr
144 - default:
145 - }
146 - case _, _ = <-db.closeC:
147 - return
148 - }
149 - }
150 -hasperr:
151 - // Persistent error.
152 - for {
153 - select {
154 - case db.compErrC <- err:
155 - case db.compPerErrC <- err:
156 - case db.writeLockC <- struct{}{}:
157 - // Hold write lock, so that write won't pass-through.
158 - wlocked = true
159 - case _, _ = <-db.closeC:
160 - if wlocked {
161 - // We should release the lock or Close will hang.
162 - <-db.writeLockC
163 - }
164 - return
165 - }
166 - }
167 -}
168 -
169 -type compactionTransactCounter int
170 -
171 -func (cnt *compactionTransactCounter) incr() {
172 - *cnt++
173 -}
174 -
175 -type compactionTransactInterface interface {
176 - run(cnt *compactionTransactCounter) error
177 - revert() error
178 -}
179 -
180 -func (db *DB) compactionTransact(name string, t compactionTransactInterface) {
181 - defer func() {
182 - if x := recover(); x != nil {
183 - if x == errCompactionTransactExiting {
184 - if err := t.revert(); err != nil {
185 - db.logf("%s revert error %q", name, err)
186 - }
187 - }
188 - panic(x)
189 - }
190 - }()
191 -
192 - const (
193 - backoffMin = 1 * time.Second
194 - backoffMax = 8 * time.Second
195 - backoffMul = 2 * time.Second
196 - )
197 - var (
198 - backoff = backoffMin
199 - backoffT = time.NewTimer(backoff)
200 - lastCnt = compactionTransactCounter(0)
201 -
202 - disableBackoff = db.s.o.GetDisableCompactionBackoff()
203 - )
204 - for n := 0; ; n++ {
205 - // Check wether the DB is closed.
206 - if db.isClosed() {
207 - db.logf("%s exiting", name)
208 - db.compactionExitTransact()
209 - } else if n > 0 {
210 - db.logf("%s retrying N·%d", name, n)
211 - }
212 -
213 - // Execute.
214 - cnt := compactionTransactCounter(0)
215 - err := t.run(&cnt)
216 - if err != nil {
217 - db.logf("%s error I·%d %q", name, cnt, err)
218 - }
219 -
220 - // Set compaction error status.
221 - select {
222 - case db.compErrSetC <- err:
223 - case perr := <-db.compPerErrC:
224 - if err != nil {
225 - db.logf("%s exiting (persistent error %q)", name, perr)
226 - db.compactionExitTransact()
227 - }
228 - case _, _ = <-db.closeC:
229 - db.logf("%s exiting", name)
230 - db.compactionExitTransact()
231 - }
232 - if err == nil {
233 - return
234 - }
235 - if errors.IsCorrupted(err) {
236 - db.logf("%s exiting (corruption detected)", name)
237 - db.compactionExitTransact()
238 - }
239 -
240 - if !disableBackoff {
241 - // Reset backoff duration if counter is advancing.
242 - if cnt > lastCnt {
243 - backoff = backoffMin
244 - lastCnt = cnt
245 - }
246 -
247 - // Backoff.
248 - backoffT.Reset(backoff)
249 - if backoff < backoffMax {
250 - backoff *= backoffMul
251 - if backoff > backoffMax {
252 - backoff = backoffMax
253 - }
254 - }
255 - select {
256 - case <-backoffT.C:
257 - case _, _ = <-db.closeC:
258 - db.logf("%s exiting", name)
259 - db.compactionExitTransact()
260 - }
261 - }
262 - }
263 -}
264 -
265 -type compactionTransactFunc struct {
266 - runFunc func(cnt *compactionTransactCounter) error
267 - revertFunc func() error
268 -}
269 -
270 -func (t *compactionTransactFunc) run(cnt *compactionTransactCounter) error {
271 - return t.runFunc(cnt)
272 -}
273 -
274 -func (t *compactionTransactFunc) revert() error {
275 - if t.revertFunc != nil {
276 - return t.revertFunc()
277 - }
278 - return nil
279 -}
280 -
281 -func (db *DB) compactionTransactFunc(name string, run func(cnt *compactionTransactCounter) error, revert func() error) {
282 - db.compactionTransact(name, &compactionTransactFunc{run, revert})
283 -}
284 -
285 -func (db *DB) compactionExitTransact() {
286 - panic(errCompactionTransactExiting)
287 -}
288 -
289 -func (db *DB) memCompaction() {
290 - mem := db.getFrozenMem()
291 - if mem == nil {
292 - return
293 - }
294 - defer mem.decref()
295 -
296 - c := newCMem(db.s)
297 - stats := new(cStatsStaging)
298 -
299 - db.logf("mem@flush N·%d S·%s", mem.mdb.Len(), shortenb(mem.mdb.Size()))
300 -
301 - // Don't compact empty memdb.
302 - if mem.mdb.Len() == 0 {
303 - db.logf("mem@flush skipping")
304 - // drop frozen mem
305 - db.dropFrozenMem()
306 - return
307 - }
308 -
309 - // Pause table compaction.
310 - resumeC := make(chan struct{})
311 - select {
312 - case db.tcompPauseC <- (chan<- struct{})(resumeC):
313 - case <-db.compPerErrC:
314 - close(resumeC)
315 - resumeC = nil
316 - case _, _ = <-db.closeC:
317 - return
318 - }
319 -
320 - db.compactionTransactFunc("mem@flush", func(cnt *compactionTransactCounter) (err error) {
321 - stats.startTimer()
322 - defer stats.stopTimer()
323 - return c.flush(mem.mdb, -1)
324 - }, func() error {
325 - for _, r := range c.rec.addedTables {
326 - db.logf("mem@flush revert @%d", r.num)
327 - f := db.s.getTableFile(r.num)
328 - if err := f.Remove(); err != nil {
329 - return err
330 - }
331 - }
332 - return nil
333 - })
334 -
335 - db.compactionTransactFunc("mem@commit", func(cnt *compactionTransactCounter) (err error) {
336 - stats.startTimer()
337 - defer stats.stopTimer()
338 - return c.commit(db.journalFile.Num(), db.frozenSeq)
339 - }, nil)
340 -
341 - db.logf("mem@flush committed F·%d T·%v", len(c.rec.addedTables), stats.duration)
342 -
343 - for _, r := range c.rec.addedTables {
344 - stats.write += r.size
345 - }
346 - db.compStats[c.level].add(stats)
347 -
348 - // Drop frozen mem.
349 - db.dropFrozenMem()
350 -
351 - // Resume table compaction.
352 - if resumeC != nil {
353 - select {
354 - case <-resumeC:
355 - close(resumeC)
356 - case _, _ = <-db.closeC:
357 - return
358 - }
359 - }
360 -
361 - // Trigger table compaction.
362 - db.compSendTrigger(db.tcompCmdC)
363 -}
364 -
365 -type tableCompactionBuilder struct {
366 - db *DB
367 - s *session
368 - c *compaction
369 - rec *sessionRecord
370 - stat0, stat1 *cStatsStaging
371 -
372 - snapHasLastUkey bool
373 - snapLastUkey []byte
374 - snapLastSeq uint64
375 - snapIter int
376 - snapKerrCnt int
377 - snapDropCnt int
378 -
379 - kerrCnt int
380 - dropCnt int
381 -
382 - minSeq uint64
383 - strict bool
384 - tableSize int
385 -
386 - tw *tWriter
387 -}
388 -
389 -func (b *tableCompactionBuilder) appendKV(key, value []byte) error {
390 - // Create new table if not already.
391 - if b.tw == nil {
392 - // Check for pause event.
393 - if b.db != nil {
394 - select {
395 - case ch := <-b.db.tcompPauseC:
396 - b.db.pauseCompaction(ch)
397 - case _, _ = <-b.db.closeC:
398 - b.db.compactionExitTransact()
399 - default:
400 - }
401 - }
402 -
403 - // Create new table.
404 - var err error
405 - b.tw, err = b.s.tops.create()
406 - if err != nil {
407 - return err
408 - }
409 - }
410 -
411 - // Write key/value into table.
412 - return b.tw.append(key, value)
413 -}
414 -
415 -func (b *tableCompactionBuilder) needFlush() bool {
416 - return b.tw.tw.BytesLen() >= b.tableSize
417 -}
418 -
419 -func (b *tableCompactionBuilder) flush() error {
420 - t, err := b.tw.finish()
421 - if err != nil {
422 - return err
423 - }
424 - b.rec.addTableFile(b.c.level+1, t)
425 - b.stat1.write += t.size
426 - b.s.logf("table@build created L%d@%d N·%d S·%s %q:%q", b.c.level+1, t.file.Num(), b.tw.tw.EntriesLen(), shortenb(int(t.size)), t.imin, t.imax)
427 - b.tw = nil
428 - return nil
429 -}
430 -
431 -func (b *tableCompactionBuilder) cleanup() {
432 - if b.tw != nil {
433 - b.tw.drop()
434 - b.tw = nil
435 - }
436 -}
437 -
438 -func (b *tableCompactionBuilder) run(cnt *compactionTransactCounter) error {
439 - snapResumed := b.snapIter > 0
440 - hasLastUkey := b.snapHasLastUkey // The key might has zero length, so this is necessary.
441 - lastUkey := append([]byte{}, b.snapLastUkey...)
442 - lastSeq := b.snapLastSeq
443 - b.kerrCnt = b.snapKerrCnt
444 - b.dropCnt = b.snapDropCnt
445 - // Restore compaction state.
446 - b.c.restore()
447 -
448 - defer b.cleanup()
449 -
450 - b.stat1.startTimer()
451 - defer b.stat1.stopTimer()
452 -
453 - iter := b.c.newIterator()
454 - defer iter.Release()
455 - for i := 0; iter.Next(); i++ {
456 - // Incr transact counter.
457 - cnt.incr()
458 -
459 - // Skip until last state.
460 - if i < b.snapIter {
461 - continue
462 - }
463 -
464 - resumed := false
465 - if snapResumed {
466 - resumed = true
467 - snapResumed = false
468 - }
469 -
470 - ikey := iter.Key()
471 - ukey, seq, kt, kerr := parseIkey(ikey)
472 -
473 - if kerr == nil {
474 - shouldStop := !resumed && b.c.shouldStopBefore(ikey)
475 -
476 - if !hasLastUkey || b.s.icmp.uCompare(lastUkey, ukey) != 0 {
477 - // First occurrence of this user key.
478 -
479 - // Only rotate tables if ukey doesn't hop across.
480 - if b.tw != nil && (shouldStop || b.needFlush()) {
481 - if err := b.flush(); err != nil {
482 - return err
483 - }
484 -
485 - // Creates snapshot of the state.
486 - b.c.save()
487 - b.snapHasLastUkey = hasLastUkey
488 - b.snapLastUkey = append(b.snapLastUkey[:0], lastUkey...)
489 - b.snapLastSeq = lastSeq
490 - b.snapIter = i
491 - b.snapKerrCnt = b.kerrCnt
492 - b.snapDropCnt = b.dropCnt
493 - }
494 -
495 - hasLastUkey = true
496 - lastUkey = append(lastUkey[:0], ukey...)
497 - lastSeq = kMaxSeq
498 - }
499 -
500 - switch {
501 - case lastSeq <= b.minSeq:
502 - // Dropped because newer entry for same user key exist
503 - fallthrough // (A)
504 - case kt == ktDel && seq <= b.minSeq && b.c.baseLevelForKey(lastUkey):
505 - // For this user key:
506 - // (1) there is no data in higher levels
507 - // (2) data in lower levels will have larger seq numbers
508 - // (3) data in layers that are being compacted here and have
509 - // smaller seq numbers will be dropped in the next
510 - // few iterations of this loop (by rule (A) above).
511 - // Therefore this deletion marker is obsolete and can be dropped.
512 - lastSeq = seq
513 - b.dropCnt++
514 - continue
515 - default:
516 - lastSeq = seq
517 - }
518 - } else {
519 - if b.strict {
520 - return kerr
521 - }
522 -
523 - // Don't drop corrupted keys.
524 - hasLastUkey = false
525 - lastUkey = lastUkey[:0]
526 - lastSeq = kMaxSeq
527 - b.kerrCnt++
528 - }
529 -
530 - if err := b.appendKV(ikey, iter.Value()); err != nil {
531 - return err
532 - }
533 - }
534 -
535 - if err := iter.Error(); err != nil {
536 - return err
537 - }
538 -
539 - // Finish last table.
540 - if b.tw != nil && !b.tw.empty() {
541 - return b.flush()
542 - }
543 - return nil
544 -}
545 -
546 -func (b *tableCompactionBuilder) revert() error {
547 - for _, at := range b.rec.addedTables {
548 - b.s.logf("table@build revert @%d", at.num)
549 - f := b.s.getTableFile(at.num)
550 - if err := f.Remove(); err != nil {
551 - return err
552 - }
553 - }
554 - return nil
555 -}
556 -
557 -func (db *DB) tableCompaction(c *compaction, noTrivial bool) {
558 - defer c.release()
559 -
560 - rec := &sessionRecord{numLevel: db.s.o.GetNumLevel()}
561 - rec.addCompPtr(c.level, c.imax)
562 -
563 - if !noTrivial && c.trivial() {
564 - t := c.tables[0][0]
565 - db.logf("table@move L%d@%d -> L%d", c.level, t.file.Num(), c.level+1)
566 - rec.delTable(c.level, t.file.Num())
567 - rec.addTableFile(c.level+1, t)
568 - db.compactionTransactFunc("table@move", func(cnt *compactionTransactCounter) (err error) {
569 - return db.s.commit(rec)
570 - }, nil)
571 - return
572 - }
573 -
574 - var stats [2]cStatsStaging
575 - for i, tables := range c.tables {
576 - for _, t := range tables {
577 - stats[i].read += t.size
578 - // Insert deleted tables into record
579 - rec.delTable(c.level+i, t.file.Num())
580 - }
581 - }
582 - sourceSize := int(stats[0].read + stats[1].read)
583 - minSeq := db.minSeq()
584 - db.logf("table@compaction L%d·%d -> L%d·%d S·%s Q·%d", c.level, len(c.tables[0]), c.level+1, len(c.tables[1]), shortenb(sourceSize), minSeq)
585 -
586 - b := &tableCompactionBuilder{
587 - db: db,
588 - s: db.s,
589 - c: c,
590 - rec: rec,
591 - stat1: &stats[1],
592 - minSeq: minSeq,
593 - strict: db.s.o.GetStrict(opt.StrictCompaction),
594 - tableSize: db.s.o.GetCompactionTableSize(c.level + 1),
595 - }
596 - db.compactionTransact("table@build", b)
597 -
598 - // Commit changes
599 - db.compactionTransactFunc("table@commit", func(cnt *compactionTransactCounter) (err error) {
600 - stats[1].startTimer()
601 - defer stats[1].stopTimer()
602 - return db.s.commit(rec)
603 - }, nil)
604 -
605 - resultSize := int(stats[1].write)
606 - db.logf("table@compaction committed F%s S%s Ke·%d D·%d T·%v", sint(len(rec.addedTables)-len(rec.deletedTables)), sshortenb(resultSize-sourceSize), b.kerrCnt, b.dropCnt, stats[1].duration)
607 -
608 - // Save compaction stats
609 - for i := range stats {
610 - db.compStats[c.level+1].add(&stats[i])
611 - }
612 -}
613 -
614 -func (db *DB) tableRangeCompaction(level int, umin, umax []byte) {
615 - db.logf("table@compaction range L%d %q:%q", level, umin, umax)
616 -
617 - if level >= 0 {
618 - if c := db.s.getCompactionRange(level, umin, umax); c != nil {
619 - db.tableCompaction(c, true)
620 - }
621 - } else {
622 - v := db.s.version()
623 - m := 1
624 - for i, t := range v.tables[1:] {
625 - if t.overlaps(db.s.icmp, umin, umax, false) {
626 - m = i + 1
627 - }
628 - }
629 - v.release()
630 -
631 - for level := 0; level < m; level++ {
632 - if c := db.s.getCompactionRange(level, umin, umax); c != nil {
633 - db.tableCompaction(c, true)
634 - }
635 - }
636 - }
637 -}
638 -
639 -func (db *DB) tableAutoCompaction() {
640 - if c := db.s.pickCompaction(); c != nil {
641 - db.tableCompaction(c, false)
642 - }
643 -}
644 -
645 -func (db *DB) tableNeedCompaction() bool {
646 - v := db.s.version()
647 - defer v.release()
648 - return v.needCompaction()
649 -}
650 -
651 -func (db *DB) pauseCompaction(ch chan<- struct{}) {
652 - select {
653 - case ch <- struct{}{}:
654 - case _, _ = <-db.closeC:
655 - db.compactionExitTransact()
656 - }
657 -}
658 -
659 -type cCmd interface {
660 - ack(err error)
661 -}
662 -
663 -type cIdle struct {
664 - ackC chan<- error
665 -}
666 -
667 -func (r cIdle) ack(err error) {
668 - if r.ackC != nil {
669 - defer func() {
670 - recover()
671 - }()
672 - r.ackC <- err
673 - }
674 -}
675 -
676 -type cRange struct {
677 - level int
678 - min, max []byte
679 - ackC chan<- error
680 -}
681 -
682 -func (r cRange) ack(err error) {
683 - if r.ackC != nil {
684 - defer func() {
685 - recover()
686 - }()
687 - r.ackC <- err
688 - }
689 -}
690 -
691 -// This will trigger auto compation and/or wait for all compaction to be done.
692 -func (db *DB) compSendIdle(compC chan<- cCmd) (err error) {
693 - ch := make(chan error)
694 - defer close(ch)
695 - // Send cmd.
696 - select {
697 - case compC <- cIdle{ch}:
698 - case err = <-db.compErrC:
699 - return
700 - case _, _ = <-db.closeC:
701 - return ErrClosed
702 - }
703 - // Wait cmd.
704 - select {
705 - case err = <-ch:
706 - case err = <-db.compErrC:
707 - case _, _ = <-db.closeC:
708 - return ErrClosed
709 - }
710 - return err
711 -}
712 -
713 -// This will trigger auto compaction but will not wait for it.
714 -func (db *DB) compSendTrigger(compC chan<- cCmd) {
715 - select {
716 - case compC <- cIdle{}:
717 - default:
718 - }
719 -}
720 -
721 -// Send range compaction request.
722 -func (db *DB) compSendRange(compC chan<- cCmd, level int, min, max []byte) (err error) {
723 - ch := make(chan error)
724 - defer close(ch)
725 - // Send cmd.
726 - select {
727 - case compC <- cRange{level, min, max, ch}:
728 - case err := <-db.compErrC:
729 - return err
730 - case _, _ = <-db.closeC:
731 - return ErrClosed
732 - }
733 - // Wait cmd.
734 - select {
735 - case err = <-ch:
736 - case err = <-db.compErrC:
737 - case _, _ = <-db.closeC:
738 - return ErrClosed
739 - }
740 - return err
741 -}
742 -
743 -func (db *DB) mCompaction() {
744 - var x cCmd
745 -
746 - defer func() {
747 - if x := recover(); x != nil {
748 - if x != errCompactionTransactExiting {
749 - panic(x)
750 - }
751 - }
752 - if x != nil {
753 - x.ack(ErrClosed)
754 - }
755 - db.closeW.Done()
756 - }()
757 -
758 - for {
759 - select {
760 - case x = <-db.mcompCmdC:
761 - switch x.(type) {
762 - case cIdle:
763 - db.memCompaction()
764 - x.ack(nil)
765 - x = nil
766 - default:
767 - panic("leveldb: unknown command")
768 - }
769 - case _, _ = <-db.closeC:
770 - return
771 - }
772 - }
773 -}
774 -
775 -func (db *DB) tCompaction() {
776 - var x cCmd
777 - var ackQ []cCmd
778 -
779 - defer func() {
780 - if x := recover(); x != nil {
781 - if x != errCompactionTransactExiting {
782 - panic(x)
783 - }
784 - }
785 - for i := range ackQ {
786 - ackQ[i].ack(ErrClosed)
787 - ackQ[i] = nil
788 - }
789 - if x != nil {
790 - x.ack(ErrClosed)
791 - }
792 - db.closeW.Done()
793 - }()
794 -
795 - for {
796 - if db.tableNeedCompaction() {
797 - select {
798 - case x = <-db.tcompCmdC:
799 - case ch := <-db.tcompPauseC:
800 - db.pauseCompaction(ch)
801 - continue
802 - case _, _ = <-db.closeC:
803 - return
804 - default:
805 - }
806 - } else {
807 - for i := range ackQ {
808 - ackQ[i].ack(nil)
809 - ackQ[i] = nil
810 - }
811 - ackQ = ackQ[:0]
812 - select {
813 - case x = <-db.tcompCmdC:
814 - case ch := <-db.tcompPauseC:
815 - db.pauseCompaction(ch)
816 - continue
817 - case _, _ = <-db.closeC:
818 - return
819 - }
820 - }
821 - if x != nil {
822 - switch cmd := x.(type) {
823 - case cIdle:
824 - ackQ = append(ackQ, x)
825 - case cRange:
826 - db.tableRangeCompaction(cmd.level, cmd.min, cmd.max)
827 - x.ack(nil)
828 - default:
829 - panic("leveldb: unknown command")
830 - }
831 - x = nil
832 - }
833 - db.tableAutoCompaction()
834 - }
835 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_iter.go deleted
-350
@@ -1,350 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "errors"
11 - "math/rand"
12 - "runtime"
13 - "sync"
14 - "sync/atomic"
15 -
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
19 -)
20 -
21 -var (
22 - errInvalidIkey = errors.New("leveldb: Iterator: invalid internal key")
23 -)
24 -
25 -type memdbReleaser struct {
26 - once sync.Once
27 - m *memDB
28 -}
29 -
30 -func (mr *memdbReleaser) Release() {
31 - mr.once.Do(func() {
32 - mr.m.decref()
33 - })
34 -}
35 -
36 -func (db *DB) newRawIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
37 - em, fm := db.getMems()
38 - v := db.s.version()
39 -
40 - ti := v.getIterators(slice, ro)
41 - n := len(ti) + 2
42 - i := make([]iterator.Iterator, 0, n)
43 - emi := em.mdb.NewIterator(slice)
44 - emi.SetReleaser(&memdbReleaser{m: em})
45 - i = append(i, emi)
46 - if fm != nil {
47 - fmi := fm.mdb.NewIterator(slice)
48 - fmi.SetReleaser(&memdbReleaser{m: fm})
49 - i = append(i, fmi)
50 - }
51 - i = append(i, ti...)
52 - strict := opt.GetStrict(db.s.o.Options, ro, opt.StrictReader)
53 - mi := iterator.NewMergedIterator(i, db.s.icmp, strict)
54 - mi.SetReleaser(&versionReleaser{v: v})
55 - return mi
56 -}
57 -
58 -func (db *DB) newIterator(seq uint64, slice *util.Range, ro *opt.ReadOptions) *dbIter {
59 - var islice *util.Range
60 - if slice != nil {
61 - islice = &util.Range{}
62 - if slice.Start != nil {
63 - islice.Start = newIkey(slice.Start, kMaxSeq, ktSeek)
64 - }
65 - if slice.Limit != nil {
66 - islice.Limit = newIkey(slice.Limit, kMaxSeq, ktSeek)
67 - }
68 - }
69 - rawIter := db.newRawIterator(islice, ro)
70 - iter := &dbIter{
71 - db: db,
72 - icmp: db.s.icmp,
73 - iter: rawIter,
74 - seq: seq,
75 - strict: opt.GetStrict(db.s.o.Options, ro, opt.StrictReader),
76 - key: make([]byte, 0),
77 - value: make([]byte, 0),
78 - }
79 - atomic.AddInt32(&db.aliveIters, 1)
80 - runtime.SetFinalizer(iter, (*dbIter).Release)
81 - return iter
82 -}
83 -
84 -func (db *DB) iterSamplingRate() int {
85 - return rand.Intn(2 * db.s.o.GetIteratorSamplingRate())
86 -}
87 -
88 -type dir int
89 -
90 -const (
91 - dirReleased dir = iota - 1
92 - dirSOI
93 - dirEOI
94 - dirBackward
95 - dirForward
96 -)
97 -
98 -// dbIter represent an interator states over a database session.
99 -type dbIter struct {
100 - db *DB
101 - icmp *iComparer
102 - iter iterator.Iterator
103 - seq uint64
104 - strict bool
105 -
106 - smaplingGap int
107 - dir dir
108 - key []byte
109 - value []byte
110 - err error
111 - releaser util.Releaser
112 -}
113 -
114 -func (i *dbIter) sampleSeek() {
115 - ikey := i.iter.Key()
116 - i.smaplingGap -= len(ikey) + len(i.iter.Value())
117 - for i.smaplingGap < 0 {
118 - i.smaplingGap += i.db.iterSamplingRate()
119 - i.db.sampleSeek(ikey)
120 - }
121 -}
122 -
123 -func (i *dbIter) setErr(err error) {
124 - i.err = err
125 - i.key = nil
126 - i.value = nil
127 -}
128 -
129 -func (i *dbIter) iterErr() {
130 - if err := i.iter.Error(); err != nil {
131 - i.setErr(err)
132 - }
133 -}
134 -
135 -func (i *dbIter) Valid() bool {
136 - return i.err == nil && i.dir > dirEOI
137 -}
138 -
139 -func (i *dbIter) First() bool {
140 - if i.err != nil {
141 - return false
142 - } else if i.dir == dirReleased {
143 - i.err = ErrIterReleased
144 - return false
145 - }
146 -
147 - if i.iter.First() {
148 - i.dir = dirSOI
149 - return i.next()
150 - }
151 - i.dir = dirEOI
152 - i.iterErr()
153 - return false
154 -}
155 -
156 -func (i *dbIter) Last() bool {
157 - if i.err != nil {
158 - return false
159 - } else if i.dir == dirReleased {
160 - i.err = ErrIterReleased
161 - return false
162 - }
163 -
164 - if i.iter.Last() {
165 - return i.prev()
166 - }
167 - i.dir = dirSOI
168 - i.iterErr()
169 - return false
170 -}
171 -
172 -func (i *dbIter) Seek(key []byte) bool {
173 - if i.err != nil {
174 - return false
175 - } else if i.dir == dirReleased {
176 - i.err = ErrIterReleased
177 - return false
178 - }
179 -
180 - ikey := newIkey(key, i.seq, ktSeek)
181 - if i.iter.Seek(ikey) {
182 - i.dir = dirSOI
183 - return i.next()
184 - }
185 - i.dir = dirEOI
186 - i.iterErr()
187 - return false
188 -}
189 -
190 -func (i *dbIter) next() bool {
191 - for {
192 - if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
193 - i.sampleSeek()
194 - if seq <= i.seq {
195 - switch kt {
196 - case ktDel:
197 - // Skip deleted key.
198 - i.key = append(i.key[:0], ukey...)
199 - i.dir = dirForward
200 - case ktVal:
201 - if i.dir == dirSOI || i.icmp.uCompare(ukey, i.key) > 0 {
202 - i.key = append(i.key[:0], ukey...)
203 - i.value = append(i.value[:0], i.iter.Value()...)
204 - i.dir = dirForward
205 - return true
206 - }
207 - }
208 - }
209 - } else if i.strict {
210 - i.setErr(kerr)
211 - break
212 - }
213 - if !i.iter.Next() {
214 - i.dir = dirEOI
215 - i.iterErr()
216 - break
217 - }
218 - }
219 - return false
220 -}
221 -
222 -func (i *dbIter) Next() bool {
223 - if i.dir == dirEOI || i.err != nil {
224 - return false
225 - } else if i.dir == dirReleased {
226 - i.err = ErrIterReleased
227 - return false
228 - }
229 -
230 - if !i.iter.Next() || (i.dir == dirBackward && !i.iter.Next()) {
231 - i.dir = dirEOI
232 - i.iterErr()
233 - return false
234 - }
235 - return i.next()
236 -}
237 -
238 -func (i *dbIter) prev() bool {
239 - i.dir = dirBackward
240 - del := true
241 - if i.iter.Valid() {
242 - for {
243 - if ukey, seq, kt, kerr := parseIkey(i.iter.Key()); kerr == nil {
244 - i.sampleSeek()
245 - if seq <= i.seq {
246 - if !del && i.icmp.uCompare(ukey, i.key) < 0 {
247 - return true
248 - }
249 - del = (kt == ktDel)
250 - if !del {
251 - i.key = append(i.key[:0], ukey...)
252 - i.value = append(i.value[:0], i.iter.Value()...)
253 - }
254 - }
255 - } else if i.strict {
256 - i.setErr(kerr)
257 - return false
258 - }
259 - if !i.iter.Prev() {
260 - break
261 - }
262 - }
263 - }
264 - if del {
265 - i.dir = dirSOI
266 - i.iterErr()
267 - return false
268 - }
269 - return true
270 -}
271 -
272 -func (i *dbIter) Prev() bool {
273 - if i.dir == dirSOI || i.err != nil {
274 - return false
275 - } else if i.dir == dirReleased {
276 - i.err = ErrIterReleased
277 - return false
278 - }
279 -
280 - switch i.dir {
281 - case dirEOI:
282 - return i.Last()
283 - case dirForward:
284 - for i.iter.Prev() {
285 - if ukey, _, _, kerr := parseIkey(i.iter.Key()); kerr == nil {
286 - i.sampleSeek()
287 - if i.icmp.uCompare(ukey, i.key) < 0 {
288 - goto cont
289 - }
290 - } else if i.strict {
291 - i.setErr(kerr)
292 - return false
293 - }
294 - }
295 - i.dir = dirSOI
296 - i.iterErr()
297 - return false
298 - }
299 -
300 -cont:
301 - return i.prev()
302 -}
303 -
304 -func (i *dbIter) Key() []byte {
305 - if i.err != nil || i.dir <= dirEOI {
306 - return nil
307 - }
308 - return i.key
309 -}
310 -
311 -func (i *dbIter) Value() []byte {
312 - if i.err != nil || i.dir <= dirEOI {
313 - return nil
314 - }
315 - return i.value
316 -}
317 -
318 -func (i *dbIter) Release() {
319 - if i.dir != dirReleased {
320 - // Clear the finalizer.
321 - runtime.SetFinalizer(i, nil)
322 -
323 - if i.releaser != nil {
324 - i.releaser.Release()
325 - i.releaser = nil
326 - }
327 -
328 - i.dir = dirReleased
329 - i.key = nil
330 - i.value = nil
331 - i.iter.Release()
332 - i.iter = nil
333 - atomic.AddInt32(&i.db.aliveIters, -1)
334 - i.db = nil
335 - }
336 -}
337 -
338 -func (i *dbIter) SetReleaser(releaser util.Releaser) {
339 - if i.dir == dirReleased {
340 - panic(util.ErrReleased)
341 - }
342 - if i.releaser != nil && releaser != nil {
343 - panic(util.ErrHasReleaser)
344 - }
345 - i.releaser = releaser
346 -}
347 -
348 -func (i *dbIter) Error() error {
349 - return i.err
350 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_snapshot.go deleted
-183
@@ -1,183 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "container/list"
11 - "fmt"
12 - "runtime"
13 - "sync"
14 - "sync/atomic"
15 -
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
19 -)
20 -
21 -type snapshotElement struct {
22 - seq uint64
23 - ref int
24 - e *list.Element
25 -}
26 -
27 -// Acquires a snapshot, based on latest sequence.
28 -func (db *DB) acquireSnapshot() *snapshotElement {
29 - db.snapsMu.Lock()
30 - defer db.snapsMu.Unlock()
31 -
32 - seq := db.getSeq()
33 -
34 - if e := db.snapsList.Back(); e != nil {
35 - se := e.Value.(*snapshotElement)
36 - if se.seq == seq {
37 - se.ref++
38 - return se
39 - } else if seq < se.seq {
40 - panic("leveldb: sequence number is not increasing")
41 - }
42 - }
43 - se := &snapshotElement{seq: seq, ref: 1}
44 - se.e = db.snapsList.PushBack(se)
45 - return se
46 -}
47 -
48 -// Releases given snapshot element.
49 -func (db *DB) releaseSnapshot(se *snapshotElement) {
50 - db.snapsMu.Lock()
51 - defer db.snapsMu.Unlock()
52 -
53 - se.ref--
54 - if se.ref == 0 {
55 - db.snapsList.Remove(se.e)
56 - se.e = nil
57 - } else if se.ref < 0 {
58 - panic("leveldb: Snapshot: negative element reference")
59 - }
60 -}
61 -
62 -// Gets minimum sequence that not being snapshoted.
63 -func (db *DB) minSeq() uint64 {
64 - db.snapsMu.Lock()
65 - defer db.snapsMu.Unlock()
66 -
67 - if e := db.snapsList.Front(); e != nil {
68 - return e.Value.(*snapshotElement).seq
69 - }
70 -
71 - return db.getSeq()
72 -}
73 -
74 -// Snapshot is a DB snapshot.
75 -type Snapshot struct {
76 - db *DB
77 - elem *snapshotElement
78 - mu sync.RWMutex
79 - released bool
80 -}
81 -
82 -// Creates new snapshot object.
83 -func (db *DB) newSnapshot() *Snapshot {
84 - snap := &Snapshot{
85 - db: db,
86 - elem: db.acquireSnapshot(),
87 - }
88 - atomic.AddInt32(&db.aliveSnaps, 1)
89 - runtime.SetFinalizer(snap, (*Snapshot).Release)
90 - return snap
91 -}
92 -
93 -func (snap *Snapshot) String() string {
94 - return fmt.Sprintf("leveldb.Snapshot{%d}", snap.elem.seq)
95 -}
96 -
97 -// Get gets the value for the given key. It returns ErrNotFound if
98 -// the DB does not contains the key.
99 -//
100 -// The caller should not modify the contents of the returned slice, but
101 -// it is safe to modify the contents of the argument after Get returns.
102 -func (snap *Snapshot) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
103 - err = snap.db.ok()
104 - if err != nil {
105 - return
106 - }
107 - snap.mu.RLock()
108 - defer snap.mu.RUnlock()
109 - if snap.released {
110 - err = ErrSnapshotReleased
111 - return
112 - }
113 - return snap.db.get(key, snap.elem.seq, ro)
114 -}
115 -
116 -// Has returns true if the DB does contains the given key.
117 -//
118 -// It is safe to modify the contents of the argument after Get returns.
119 -func (snap *Snapshot) Has(key []byte, ro *opt.ReadOptions) (ret bool, err error) {
120 - err = snap.db.ok()
121 - if err != nil {
122 - return
123 - }
124 - snap.mu.RLock()
125 - defer snap.mu.RUnlock()
126 - if snap.released {
127 - err = ErrSnapshotReleased
128 - return
129 - }
130 - return snap.db.has(key, snap.elem.seq, ro)
131 -}
132 -
133 -// NewIterator returns an iterator for the snapshot of the uderlying DB.
134 -// The returned iterator is not goroutine-safe, but it is safe to use
135 -// multiple iterators concurrently, with each in a dedicated goroutine.
136 -// It is also safe to use an iterator concurrently with modifying its
137 -// underlying DB. The resultant key/value pairs are guaranteed to be
138 -// consistent.
139 -//
140 -// Slice allows slicing the iterator to only contains keys in the given
141 -// range. A nil Range.Start is treated as a key before all keys in the
142 -// DB. And a nil Range.Limit is treated as a key after all keys in
143 -// the DB.
144 -//
145 -// The iterator must be released after use, by calling Release method.
146 -// Releasing the snapshot doesn't mean releasing the iterator too, the
147 -// iterator would be still valid until released.
148 -//
149 -// Also read Iterator documentation of the leveldb/iterator package.
150 -func (snap *Snapshot) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
151 - if err := snap.db.ok(); err != nil {
152 - return iterator.NewEmptyIterator(err)
153 - }
154 - snap.mu.Lock()
155 - defer snap.mu.Unlock()
156 - if snap.released {
157 - return iterator.NewEmptyIterator(ErrSnapshotReleased)
158 - }
159 - // Since iterator already hold version ref, it doesn't need to
160 - // hold snapshot ref.
161 - return snap.db.newIterator(snap.elem.seq, slice, ro)
162 -}
163 -
164 -// Release releases the snapshot. This will not release any returned
165 -// iterators, the iterators would still be valid until released or the
166 -// underlying DB is closed.
167 -//
168 -// Other methods should not be called after the snapshot has been released.
169 -func (snap *Snapshot) Release() {
170 - snap.mu.Lock()
171 - defer snap.mu.Unlock()
172 -
173 - if !snap.released {
174 - // Clear the finalizer.
175 - runtime.SetFinalizer(snap, nil)
176 -
177 - snap.released = true
178 - snap.db.releaseSnapshot(snap.elem)
179 - atomic.AddInt32(&snap.db.aliveSnaps, -1)
180 - snap.db = nil
181 - snap.elem = nil
182 - }
183 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_state.go deleted
-211
@@ -1,211 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "sync/atomic"
11 - "time"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
15 -)
16 -
17 -type memDB struct {
18 - db *DB
19 - mdb *memdb.DB
20 - ref int32
21 -}
22 -
23 -func (m *memDB) incref() {
24 - atomic.AddInt32(&m.ref, 1)
25 -}
26 -
27 -func (m *memDB) decref() {
28 - if ref := atomic.AddInt32(&m.ref, -1); ref == 0 {
29 - // Only put back memdb with std capacity.
30 - if m.mdb.Capacity() == m.db.s.o.GetWriteBuffer() {
31 - m.mdb.Reset()
32 - m.db.mpoolPut(m.mdb)
33 - }
34 - m.db = nil
35 - m.mdb = nil
36 - } else if ref < 0 {
37 - panic("negative memdb ref")
38 - }
39 -}
40 -
41 -// Get latest sequence number.
42 -func (db *DB) getSeq() uint64 {
43 - return atomic.LoadUint64(&db.seq)
44 -}
45 -
46 -// Atomically adds delta to seq.
47 -func (db *DB) addSeq(delta uint64) {
48 - atomic.AddUint64(&db.seq, delta)
49 -}
50 -
51 -func (db *DB) sampleSeek(ikey iKey) {
52 - v := db.s.version()
53 - if v.sampleSeek(ikey) {
54 - // Trigger table compaction.
55 - db.compSendTrigger(db.tcompCmdC)
56 - }
57 - v.release()
58 -}
59 -
60 -func (db *DB) mpoolPut(mem *memdb.DB) {
61 - defer func() {
62 - recover()
63 - }()
64 - select {
65 - case db.memPool <- mem:
66 - default:
67 - }
68 -}
69 -
70 -func (db *DB) mpoolGet() *memdb.DB {
71 - select {
72 - case mem := <-db.memPool:
73 - return mem
74 - default:
75 - return nil
76 - }
77 -}
78 -
79 -func (db *DB) mpoolDrain() {
80 - ticker := time.NewTicker(30 * time.Second)
81 - for {
82 - select {
83 - case <-ticker.C:
84 - select {
85 - case <-db.memPool:
86 - default:
87 - }
88 - case _, _ = <-db.closeC:
89 - close(db.memPool)
90 - return
91 - }
92 - }
93 -}
94 -
95 -// Create new memdb and froze the old one; need external synchronization.
96 -// newMem only called synchronously by the writer.
97 -func (db *DB) newMem(n int) (mem *memDB, err error) {
98 - num := db.s.allocFileNum()
99 - file := db.s.getJournalFile(num)
100 - w, err := file.Create()
101 - if err != nil {
102 - db.s.reuseFileNum(num)
103 - return
104 - }
105 -
106 - db.memMu.Lock()
107 - defer db.memMu.Unlock()
108 -
109 - if db.frozenMem != nil {
110 - panic("still has frozen mem")
111 - }
112 -
113 - if db.journal == nil {
114 - db.journal = journal.NewWriter(w)
115 - } else {
116 - db.journal.Reset(w)
117 - db.journalWriter.Close()
118 - db.frozenJournalFile = db.journalFile
119 - }
120 - db.journalWriter = w
121 - db.journalFile = file
122 - db.frozenMem = db.mem
123 - mdb := db.mpoolGet()
124 - if mdb == nil || mdb.Capacity() < n {
125 - mdb = memdb.New(db.s.icmp, maxInt(db.s.o.GetWriteBuffer(), n))
126 - }
127 - mem = &memDB{
128 - db: db,
129 - mdb: mdb,
130 - ref: 2,
131 - }
132 - db.mem = mem
133 - // The seq only incremented by the writer. And whoever called newMem
134 - // should hold write lock, so no need additional synchronization here.
135 - db.frozenSeq = db.seq
136 - return
137 -}
138 -
139 -// Get all memdbs.
140 -func (db *DB) getMems() (e, f *memDB) {
141 - db.memMu.RLock()
142 - defer db.memMu.RUnlock()
143 - if db.mem == nil {
144 - panic("nil effective mem")
145 - }
146 - db.mem.incref()
147 - if db.frozenMem != nil {
148 - db.frozenMem.incref()
149 - }
150 - return db.mem, db.frozenMem
151 -}
152 -
153 -// Get frozen memdb.
154 -func (db *DB) getEffectiveMem() *memDB {
155 - db.memMu.RLock()
156 - defer db.memMu.RUnlock()
157 - if db.mem == nil {
158 - panic("nil effective mem")
159 - }
160 - db.mem.incref()
161 - return db.mem
162 -}
163 -
164 -// Check whether we has frozen memdb.
165 -func (db *DB) hasFrozenMem() bool {
166 - db.memMu.RLock()
167 - defer db.memMu.RUnlock()
168 - return db.frozenMem != nil
169 -}
170 -
171 -// Get frozen memdb.
172 -func (db *DB) getFrozenMem() *memDB {
173 - db.memMu.RLock()
174 - defer db.memMu.RUnlock()
175 - if db.frozenMem != nil {
176 - db.frozenMem.incref()
177 - }
178 - return db.frozenMem
179 -}
180 -
181 -// Drop frozen memdb; assume that frozen memdb isn't nil.
182 -func (db *DB) dropFrozenMem() {
183 - db.memMu.Lock()
184 - if err := db.frozenJournalFile.Remove(); err != nil {
185 - db.logf("journal@remove removing @%d %q", db.frozenJournalFile.Num(), err)
186 - } else {
187 - db.logf("journal@remove removed @%d", db.frozenJournalFile.Num())
188 - }
189 - db.frozenJournalFile = nil
190 - db.frozenMem.decref()
191 - db.frozenMem = nil
192 - db.memMu.Unlock()
193 -}
194 -
195 -// Set closed flag; return true if not already closed.
196 -func (db *DB) setClosed() bool {
197 - return atomic.CompareAndSwapUint32(&db.closed, 0, 1)
198 -}
199 -
200 -// Check whether DB was closed.
201 -func (db *DB) isClosed() bool {
202 - return atomic.LoadUint32(&db.closed) != 0
203 -}
204 -
205 -// Check read ok status.
206 -func (db *DB) ok() error {
207 - if db.isClosed() {
208 - return ErrClosed
209 - }
210 - return nil
211 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_test.go deleted
-2665
@@ -1,2665 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "container/list"
12 - crand "crypto/rand"
13 - "encoding/binary"
14 - "fmt"
15 - "math/rand"
16 - "os"
17 - "path/filepath"
18 - "runtime"
19 - "strings"
20 - "sync"
21 - "sync/atomic"
22 - "testing"
23 - "time"
24 - "unsafe"
25 -
26 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
27 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
28 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
29 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
30 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
31 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
32 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
33 -)
34 -
35 -func tkey(i int) []byte {
36 - return []byte(fmt.Sprintf("%016d", i))
37 -}
38 -
39 -func tval(seed, n int) []byte {
40 - r := rand.New(rand.NewSource(int64(seed)))
41 - return randomString(r, n)
42 -}
43 -
44 -type dbHarness struct {
45 - t *testing.T
46 -
47 - stor *testStorage
48 - db *DB
49 - o *opt.Options
50 - ro *opt.ReadOptions
51 - wo *opt.WriteOptions
52 -}
53 -
54 -func newDbHarnessWopt(t *testing.T, o *opt.Options) *dbHarness {
55 - h := new(dbHarness)
56 - h.init(t, o)
57 - return h
58 -}
59 -
60 -func newDbHarness(t *testing.T) *dbHarness {
61 - return newDbHarnessWopt(t, &opt.Options{})
62 -}
63 -
64 -func (h *dbHarness) init(t *testing.T, o *opt.Options) {
65 - h.t = t
66 - h.stor = newTestStorage(t)
67 - h.o = o
68 - h.ro = nil
69 - h.wo = nil
70 -
71 - if err := h.openDB0(); err != nil {
72 - // So that it will come after fatal message.
73 - defer h.stor.Close()
74 - h.t.Fatal("Open (init): got error: ", err)
75 - }
76 -}
77 -
78 -func (h *dbHarness) openDB0() (err error) {
79 - h.t.Log("opening DB")
80 - h.db, err = Open(h.stor, h.o)
81 - return
82 -}
83 -
84 -func (h *dbHarness) openDB() {
85 - if err := h.openDB0(); err != nil {
86 - h.t.Fatal("Open: got error: ", err)
87 - }
88 -}
89 -
90 -func (h *dbHarness) closeDB0() error {
91 - h.t.Log("closing DB")
92 - return h.db.Close()
93 -}
94 -
95 -func (h *dbHarness) closeDB() {
96 - if err := h.closeDB0(); err != nil {
97 - h.t.Error("Close: got error: ", err)
98 - }
99 - h.stor.CloseCheck()
100 - runtime.GC()
101 -}
102 -
103 -func (h *dbHarness) reopenDB() {
104 - h.closeDB()
105 - h.openDB()
106 -}
107 -
108 -func (h *dbHarness) close() {
109 - h.closeDB0()
110 - h.db = nil
111 - h.stor.Close()
112 - h.stor = nil
113 - runtime.GC()
114 -}
115 -
116 -func (h *dbHarness) openAssert(want bool) {
117 - db, err := Open(h.stor, h.o)
118 - if err != nil {
119 - if want {
120 - h.t.Error("Open: assert: got error: ", err)
121 - } else {
122 - h.t.Log("Open: assert: got error (expected): ", err)
123 - }
124 - } else {
125 - if !want {
126 - h.t.Error("Open: assert: expect error")
127 - }
128 - db.Close()
129 - }
130 -}
131 -
132 -func (h *dbHarness) write(batch *Batch) {
133 - if err := h.db.Write(batch, h.wo); err != nil {
134 - h.t.Error("Write: got error: ", err)
135 - }
136 -}
137 -
138 -func (h *dbHarness) put(key, value string) {
139 - if err := h.db.Put([]byte(key), []byte(value), h.wo); err != nil {
140 - h.t.Error("Put: got error: ", err)
141 - }
142 -}
143 -
144 -func (h *dbHarness) putMulti(n int, low, hi string) {
145 - for i := 0; i < n; i++ {
146 - h.put(low, "begin")
147 - h.put(hi, "end")
148 - h.compactMem()
149 - }
150 -}
151 -
152 -func (h *dbHarness) maxNextLevelOverlappingBytes(want uint64) {
153 - t := h.t
154 - db := h.db
155 -
156 - var (
157 - maxOverlaps uint64
158 - maxLevel int
159 - )
160 - v := db.s.version()
161 - for i, tt := range v.tables[1 : len(v.tables)-1] {
162 - level := i + 1
163 - next := v.tables[level+1]
164 - for _, t := range tt {
165 - r := next.getOverlaps(nil, db.s.icmp, t.imin.ukey(), t.imax.ukey(), false)
166 - sum := r.size()
167 - if sum > maxOverlaps {
168 - maxOverlaps = sum
169 - maxLevel = level
170 - }
171 - }
172 - }
173 - v.release()
174 -
175 - if maxOverlaps > want {
176 - t.Errorf("next level most overlapping bytes is more than %d, got=%d level=%d", want, maxOverlaps, maxLevel)
177 - } else {
178 - t.Logf("next level most overlapping bytes is %d, level=%d want=%d", maxOverlaps, maxLevel, want)
179 - }
180 -}
181 -
182 -func (h *dbHarness) delete(key string) {
183 - t := h.t
184 - db := h.db
185 -
186 - err := db.Delete([]byte(key), h.wo)
187 - if err != nil {
188 - t.Error("Delete: got error: ", err)
189 - }
190 -}
191 -
192 -func (h *dbHarness) assertNumKeys(want int) {
193 - iter := h.db.NewIterator(nil, h.ro)
194 - defer iter.Release()
195 - got := 0
196 - for iter.Next() {
197 - got++
198 - }
199 - if err := iter.Error(); err != nil {
200 - h.t.Error("assertNumKeys: ", err)
201 - }
202 - if want != got {
203 - h.t.Errorf("assertNumKeys: want=%d got=%d", want, got)
204 - }
205 -}
206 -
207 -func (h *dbHarness) getr(db Reader, key string, expectFound bool) (found bool, v []byte) {
208 - t := h.t
209 - v, err := db.Get([]byte(key), h.ro)
210 - switch err {
211 - case ErrNotFound:
212 - if expectFound {
213 - t.Errorf("Get: key '%s' not found, want found", key)
214 - }
215 - case nil:
216 - found = true
217 - if !expectFound {
218 - t.Errorf("Get: key '%s' found, want not found", key)
219 - }
220 - default:
221 - t.Error("Get: got error: ", err)
222 - }
223 - return
224 -}
225 -
226 -func (h *dbHarness) get(key string, expectFound bool) (found bool, v []byte) {
227 - return h.getr(h.db, key, expectFound)
228 -}
229 -
230 -func (h *dbHarness) getValr(db Reader, key, value string) {
231 - t := h.t
232 - found, r := h.getr(db, key, true)
233 - if !found {
234 - return
235 - }
236 - rval := string(r)
237 - if rval != value {
238 - t.Errorf("Get: invalid value, got '%s', want '%s'", rval, value)
239 - }
240 -}
241 -
242 -func (h *dbHarness) getVal(key, value string) {
243 - h.getValr(h.db, key, value)
244 -}
245 -
246 -func (h *dbHarness) allEntriesFor(key, want string) {
247 - t := h.t
248 - db := h.db
249 - s := db.s
250 -
251 - ikey := newIkey([]byte(key), kMaxSeq, ktVal)
252 - iter := db.newRawIterator(nil, nil)
253 - if !iter.Seek(ikey) && iter.Error() != nil {
254 - t.Error("AllEntries: error during seek, err: ", iter.Error())
255 - return
256 - }
257 - res := "[ "
258 - first := true
259 - for iter.Valid() {
260 - if ukey, _, kt, kerr := parseIkey(iter.Key()); kerr == nil {
261 - if s.icmp.uCompare(ikey.ukey(), ukey) != 0 {
262 - break
263 - }
264 - if !first {
265 - res += ", "
266 - }
267 - first = false
268 - switch kt {
269 - case ktVal:
270 - res += string(iter.Value())
271 - case ktDel:
272 - res += "DEL"
273 - }
274 - } else {
275 - if !first {
276 - res += ", "
277 - }
278 - first = false
279 - res += "CORRUPTED"
280 - }
281 - iter.Next()
282 - }
283 - if !first {
284 - res += " "
285 - }
286 - res += "]"
287 - if res != want {
288 - t.Errorf("AllEntries: assert failed for key %q, got=%q want=%q", key, res, want)
289 - }
290 -}
291 -
292 -// Return a string that contains all key,value pairs in order,
293 -// formatted like "(k1->v1)(k2->v2)".
294 -func (h *dbHarness) getKeyVal(want string) {
295 - t := h.t
296 - db := h.db
297 -
298 - s, err := db.GetSnapshot()
299 - if err != nil {
300 - t.Fatal("GetSnapshot: got error: ", err)
301 - }
302 - res := ""
303 - iter := s.NewIterator(nil, nil)
304 - for iter.Next() {
305 - res += fmt.Sprintf("(%s->%s)", string(iter.Key()), string(iter.Value()))
306 - }
307 - iter.Release()
308 -
309 - if res != want {
310 - t.Errorf("GetKeyVal: invalid key/value pair, got=%q want=%q", res, want)
311 - }
312 - s.Release()
313 -}
314 -
315 -func (h *dbHarness) waitCompaction() {
316 - t := h.t
317 - db := h.db
318 - if err := db.compSendIdle(db.tcompCmdC); err != nil {
319 - t.Error("compaction error: ", err)
320 - }
321 -}
322 -
323 -func (h *dbHarness) waitMemCompaction() {
324 - t := h.t
325 - db := h.db
326 -
327 - if err := db.compSendIdle(db.mcompCmdC); err != nil {
328 - t.Error("compaction error: ", err)
329 - }
330 -}
331 -
332 -func (h *dbHarness) compactMem() {
333 - t := h.t
334 - db := h.db
335 -
336 - t.Log("starting memdb compaction")
337 -
338 - db.writeLockC <- struct{}{}
339 - defer func() {
340 - <-db.writeLockC
341 - }()
342 -
343 - if _, err := db.rotateMem(0); err != nil {
344 - t.Error("compaction error: ", err)
345 - }
346 - if err := db.compSendIdle(db.mcompCmdC); err != nil {
347 - t.Error("compaction error: ", err)
348 - }
349 -
350 - if h.totalTables() == 0 {
351 - t.Error("zero tables after mem compaction")
352 - }
353 -
354 - t.Log("memdb compaction done")
355 -}
356 -
357 -func (h *dbHarness) compactRangeAtErr(level int, min, max string, wanterr bool) {
358 - t := h.t
359 - db := h.db
360 -
361 - var _min, _max []byte
362 - if min != "" {
363 - _min = []byte(min)
364 - }
365 - if max != "" {
366 - _max = []byte(max)
367 - }
368 -
369 - t.Logf("starting table range compaction: level=%d, min=%q, max=%q", level, min, max)
370 -
371 - if err := db.compSendRange(db.tcompCmdC, level, _min, _max); err != nil {
372 - if wanterr {
373 - t.Log("CompactRangeAt: got error (expected): ", err)
374 - } else {
375 - t.Error("CompactRangeAt: got error: ", err)
376 - }
377 - } else if wanterr {
378 - t.Error("CompactRangeAt: expect error")
379 - }
380 -
381 - t.Log("table range compaction done")
382 -}
383 -
384 -func (h *dbHarness) compactRangeAt(level int, min, max string) {
385 - h.compactRangeAtErr(level, min, max, false)
386 -}
387 -
388 -func (h *dbHarness) compactRange(min, max string) {
389 - t := h.t
390 - db := h.db
391 -
392 - t.Logf("starting DB range compaction: min=%q, max=%q", min, max)
393 -
394 - var r util.Range
395 - if min != "" {
396 - r.Start = []byte(min)
397 - }
398 - if max != "" {
399 - r.Limit = []byte(max)
400 - }
401 - if err := db.CompactRange(r); err != nil {
402 - t.Error("CompactRange: got error: ", err)
403 - }
404 -
405 - t.Log("DB range compaction done")
406 -}
407 -
408 -func (h *dbHarness) sizeOf(start, limit string) uint64 {
409 - sz, err := h.db.SizeOf([]util.Range{
410 - {[]byte(start), []byte(limit)},
411 - })
412 - if err != nil {
413 - h.t.Error("SizeOf: got error: ", err)
414 - }
415 - return sz.Sum()
416 -}
417 -
418 -func (h *dbHarness) sizeAssert(start, limit string, low, hi uint64) {
419 - sz := h.sizeOf(start, limit)
420 - if sz < low || sz > hi {
421 - h.t.Errorf("sizeOf %q to %q not in range, want %d - %d, got %d",
422 - shorten(start), shorten(limit), low, hi, sz)
423 - }
424 -}
425 -
426 -func (h *dbHarness) getSnapshot() (s *Snapshot) {
427 - s, err := h.db.GetSnapshot()
428 - if err != nil {
429 - h.t.Fatal("GetSnapshot: got error: ", err)
430 - }
431 - return
432 -}
433 -func (h *dbHarness) tablesPerLevel(want string) {
434 - res := ""
435 - nz := 0
436 - v := h.db.s.version()
437 - for level, tt := range v.tables {
438 - if level > 0 {
439 - res += ","
440 - }
441 - res += fmt.Sprint(len(tt))
442 - if len(tt) > 0 {
443 - nz = len(res)
444 - }
445 - }
446 - v.release()
447 - res = res[:nz]
448 - if res != want {
449 - h.t.Errorf("invalid tables len, want=%s, got=%s", want, res)
450 - }
451 -}
452 -
453 -func (h *dbHarness) totalTables() (n int) {
454 - v := h.db.s.version()
455 - for _, tt := range v.tables {
456 - n += len(tt)
457 - }
458 - v.release()
459 - return
460 -}
461 -
462 -type keyValue interface {
463 - Key() []byte
464 - Value() []byte
465 -}
466 -
467 -func testKeyVal(t *testing.T, kv keyValue, want string) {
468 - res := string(kv.Key()) + "->" + string(kv.Value())
469 - if res != want {
470 - t.Errorf("invalid key/value, want=%q, got=%q", want, res)
471 - }
472 -}
473 -
474 -func numKey(num int) string {
475 - return fmt.Sprintf("key%06d", num)
476 -}
477 -
478 -var _bloom_filter = filter.NewBloomFilter(10)
479 -
480 -func truno(t *testing.T, o *opt.Options, f func(h *dbHarness)) {
481 - for i := 0; i < 4; i++ {
482 - func() {
483 - switch i {
484 - case 0:
485 - case 1:
486 - if o == nil {
487 - o = &opt.Options{Filter: _bloom_filter}
488 - } else {
489 - old := o
490 - o = &opt.Options{}
491 - *o = *old
492 - o.Filter = _bloom_filter
493 - }
494 - case 2:
495 - if o == nil {
496 - o = &opt.Options{Compression: opt.NoCompression}
497 - } else {
498 - old := o
499 - o = &opt.Options{}
500 - *o = *old
501 - o.Compression = opt.NoCompression
502 - }
503 - }
504 - h := newDbHarnessWopt(t, o)
505 - defer h.close()
506 - switch i {
507 - case 3:
508 - h.reopenDB()
509 - }
510 - f(h)
511 - }()
512 - }
513 -}
514 -
515 -func trun(t *testing.T, f func(h *dbHarness)) {
516 - truno(t, nil, f)
517 -}
518 -
519 -func testAligned(t *testing.T, name string, offset uintptr) {
520 - if offset%8 != 0 {
521 - t.Errorf("field %s offset is not 64-bit aligned", name)
522 - }
523 -}
524 -
525 -func Test_FieldsAligned(t *testing.T) {
526 - p1 := new(DB)
527 - testAligned(t, "DB.seq", unsafe.Offsetof(p1.seq))
528 - p2 := new(session)
529 - testAligned(t, "session.stNextFileNum", unsafe.Offsetof(p2.stNextFileNum))
530 - testAligned(t, "session.stJournalNum", unsafe.Offsetof(p2.stJournalNum))
531 - testAligned(t, "session.stPrevJournalNum", unsafe.Offsetof(p2.stPrevJournalNum))
532 - testAligned(t, "session.stSeqNum", unsafe.Offsetof(p2.stSeqNum))
533 -}
534 -
535 -func TestDB_Locking(t *testing.T) {
536 - h := newDbHarness(t)
537 - defer h.stor.Close()
538 - h.openAssert(false)
539 - h.closeDB()
540 - h.openAssert(true)
541 -}
542 -
543 -func TestDB_Empty(t *testing.T) {
544 - trun(t, func(h *dbHarness) {
545 - h.get("foo", false)
546 -
547 - h.reopenDB()
548 - h.get("foo", false)
549 - })
550 -}
551 -
552 -func TestDB_ReadWrite(t *testing.T) {
553 - trun(t, func(h *dbHarness) {
554 - h.put("foo", "v1")
555 - h.getVal("foo", "v1")
556 - h.put("bar", "v2")
557 - h.put("foo", "v3")
558 - h.getVal("foo", "v3")
559 - h.getVal("bar", "v2")
560 -
561 - h.reopenDB()
562 - h.getVal("foo", "v3")
563 - h.getVal("bar", "v2")
564 - })
565 -}
566 -
567 -func TestDB_PutDeleteGet(t *testing.T) {
568 - trun(t, func(h *dbHarness) {
569 - h.put("foo", "v1")
570 - h.getVal("foo", "v1")
571 - h.put("foo", "v2")
572 - h.getVal("foo", "v2")
573 - h.delete("foo")
574 - h.get("foo", false)
575 -
576 - h.reopenDB()
577 - h.get("foo", false)
578 - })
579 -}
580 -
581 -func TestDB_EmptyBatch(t *testing.T) {
582 - h := newDbHarness(t)
583 - defer h.close()
584 -
585 - h.get("foo", false)
586 - err := h.db.Write(new(Batch), h.wo)
587 - if err != nil {
588 - t.Error("writing empty batch yield error: ", err)
589 - }
590 - h.get("foo", false)
591 -}
592 -
593 -func TestDB_GetFromFrozen(t *testing.T) {
594 - h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100100})
595 - defer h.close()
596 -
597 - h.put("foo", "v1")
598 - h.getVal("foo", "v1")
599 -
600 - h.stor.DelaySync(storage.TypeTable) // Block sync calls
601 - h.put("k1", strings.Repeat("x", 100000)) // Fill memtable
602 - h.put("k2", strings.Repeat("y", 100000)) // Trigger compaction
603 - for i := 0; h.db.getFrozenMem() == nil && i < 100; i++ {
604 - time.Sleep(10 * time.Microsecond)
605 - }
606 - if h.db.getFrozenMem() == nil {
607 - h.stor.ReleaseSync(storage.TypeTable)
608 - t.Fatal("No frozen mem")
609 - }
610 - h.getVal("foo", "v1")
611 - h.stor.ReleaseSync(storage.TypeTable) // Release sync calls
612 -
613 - h.reopenDB()
614 - h.getVal("foo", "v1")
615 - h.get("k1", true)
616 - h.get("k2", true)
617 -}
618 -
619 -func TestDB_GetFromTable(t *testing.T) {
620 - trun(t, func(h *dbHarness) {
621 - h.put("foo", "v1")
622 - h.compactMem()
623 - h.getVal("foo", "v1")
624 - })
625 -}
626 -
627 -func TestDB_GetSnapshot(t *testing.T) {
628 - trun(t, func(h *dbHarness) {
629 - bar := strings.Repeat("b", 200)
630 - h.put("foo", "v1")
631 - h.put(bar, "v1")
632 -
633 - snap, err := h.db.GetSnapshot()
634 - if err != nil {
635 - t.Fatal("GetSnapshot: got error: ", err)
636 - }
637 -
638 - h.put("foo", "v2")
639 - h.put(bar, "v2")
640 -
641 - h.getVal("foo", "v2")
642 - h.getVal(bar, "v2")
643 - h.getValr(snap, "foo", "v1")
644 - h.getValr(snap, bar, "v1")
645 -
646 - h.compactMem()
647 -
648 - h.getVal("foo", "v2")
649 - h.getVal(bar, "v2")
650 - h.getValr(snap, "foo", "v1")
651 - h.getValr(snap, bar, "v1")
652 -
653 - snap.Release()
654 -
655 - h.reopenDB()
656 - h.getVal("foo", "v2")
657 - h.getVal(bar, "v2")
658 - })
659 -}
660 -
661 -func TestDB_GetLevel0Ordering(t *testing.T) {
662 - trun(t, func(h *dbHarness) {
663 - for i := 0; i < 4; i++ {
664 - h.put("bar", fmt.Sprintf("b%d", i))
665 - h.put("foo", fmt.Sprintf("v%d", i))
666 - h.compactMem()
667 - }
668 - h.getVal("foo", "v3")
669 - h.getVal("bar", "b3")
670 -
671 - v := h.db.s.version()
672 - t0len := v.tLen(0)
673 - v.release()
674 - if t0len < 2 {
675 - t.Errorf("level-0 tables is less than 2, got %d", t0len)
676 - }
677 -
678 - h.reopenDB()
679 - h.getVal("foo", "v3")
680 - h.getVal("bar", "b3")
681 - })
682 -}
683 -
684 -func TestDB_GetOrderedByLevels(t *testing.T) {
685 - trun(t, func(h *dbHarness) {
686 - h.put("foo", "v1")
687 - h.compactMem()
688 - h.compactRange("a", "z")
689 - h.getVal("foo", "v1")
690 - h.put("foo", "v2")
691 - h.compactMem()
692 - h.getVal("foo", "v2")
693 - })
694 -}
695 -
696 -func TestDB_GetPicksCorrectFile(t *testing.T) {
697 - trun(t, func(h *dbHarness) {
698 - // Arrange to have multiple files in a non-level-0 level.
699 - h.put("a", "va")
700 - h.compactMem()
701 - h.compactRange("a", "b")
702 - h.put("x", "vx")
703 - h.compactMem()
704 - h.compactRange("x", "y")
705 - h.put("f", "vf")
706 - h.compactMem()
707 - h.compactRange("f", "g")
708 -
709 - h.getVal("a", "va")
710 - h.getVal("f", "vf")
711 - h.getVal("x", "vx")
712 -
713 - h.compactRange("", "")
714 - h.getVal("a", "va")
715 - h.getVal("f", "vf")
716 - h.getVal("x", "vx")
717 - })
718 -}
719 -
720 -func TestDB_GetEncountersEmptyLevel(t *testing.T) {
721 - trun(t, func(h *dbHarness) {
722 - // Arrange for the following to happen:
723 - // * sstable A in level 0
724 - // * nothing in level 1
725 - // * sstable B in level 2
726 - // Then do enough Get() calls to arrange for an automatic compaction
727 - // of sstable A. A bug would cause the compaction to be marked as
728 - // occuring at level 1 (instead of the correct level 0).
729 -
730 - // Step 1: First place sstables in levels 0 and 2
731 - for i := 0; ; i++ {
732 - if i >= 100 {
733 - t.Fatal("could not fill levels-0 and level-2")
734 - }
735 - v := h.db.s.version()
736 - if v.tLen(0) > 0 && v.tLen(2) > 0 {
737 - v.release()
738 - break
739 - }
740 - v.release()
741 - h.put("a", "begin")
742 - h.put("z", "end")
743 - h.compactMem()
744 -
745 - h.getVal("a", "begin")
746 - h.getVal("z", "end")
747 - }
748 -
749 - // Step 2: clear level 1 if necessary.
750 - h.compactRangeAt(1, "", "")
751 - h.tablesPerLevel("1,0,1")
752 -
753 - h.getVal("a", "begin")
754 - h.getVal("z", "end")
755 -
756 - // Step 3: read a bunch of times
757 - for i := 0; i < 200; i++ {
758 - h.get("missing", false)
759 - }
760 -
761 - // Step 4: Wait for compaction to finish
762 - h.waitCompaction()
763 -
764 - v := h.db.s.version()
765 - if v.tLen(0) > 0 {
766 - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
767 - }
768 - v.release()
769 -
770 - h.getVal("a", "begin")
771 - h.getVal("z", "end")
772 - })
773 -}
774 -
775 -func TestDB_IterMultiWithDelete(t *testing.T) {
776 - trun(t, func(h *dbHarness) {
777 - h.put("a", "va")
778 - h.put("b", "vb")
779 - h.put("c", "vc")
780 - h.delete("b")
781 - h.get("b", false)
782 -
783 - iter := h.db.NewIterator(nil, nil)
784 - iter.Seek([]byte("c"))
785 - testKeyVal(t, iter, "c->vc")
786 - iter.Prev()
787 - testKeyVal(t, iter, "a->va")
788 - iter.Release()
789 -
790 - h.compactMem()
791 -
792 - iter = h.db.NewIterator(nil, nil)
793 - iter.Seek([]byte("c"))
794 - testKeyVal(t, iter, "c->vc")
795 - iter.Prev()
796 - testKeyVal(t, iter, "a->va")
797 - iter.Release()
798 - })
799 -}
800 -
801 -func TestDB_IteratorPinsRef(t *testing.T) {
802 - h := newDbHarness(t)
803 - defer h.close()
804 -
805 - h.put("foo", "hello")
806 -
807 - // Get iterator that will yield the current contents of the DB.
808 - iter := h.db.NewIterator(nil, nil)
809 -
810 - // Write to force compactions
811 - h.put("foo", "newvalue1")
812 - for i := 0; i < 100; i++ {
813 - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
814 - }
815 - h.put("foo", "newvalue2")
816 -
817 - iter.First()
818 - testKeyVal(t, iter, "foo->hello")
819 - if iter.Next() {
820 - t.Errorf("expect eof")
821 - }
822 - iter.Release()
823 -}
824 -
825 -func TestDB_Recover(t *testing.T) {
826 - trun(t, func(h *dbHarness) {
827 - h.put("foo", "v1")
828 - h.put("baz", "v5")
829 -
830 - h.reopenDB()
831 - h.getVal("foo", "v1")
832 -
833 - h.getVal("foo", "v1")
834 - h.getVal("baz", "v5")
835 - h.put("bar", "v2")
836 - h.put("foo", "v3")
837 -
838 - h.reopenDB()
839 - h.getVal("foo", "v3")
840 - h.put("foo", "v4")
841 - h.getVal("foo", "v4")
842 - h.getVal("bar", "v2")
843 - h.getVal("baz", "v5")
844 - })
845 -}
846 -
847 -func TestDB_RecoverWithEmptyJournal(t *testing.T) {
848 - trun(t, func(h *dbHarness) {
849 - h.put("foo", "v1")
850 - h.put("foo", "v2")
851 -
852 - h.reopenDB()
853 - h.reopenDB()
854 - h.put("foo", "v3")
855 -
856 - h.reopenDB()
857 - h.getVal("foo", "v3")
858 - })
859 -}
860 -
861 -func TestDB_RecoverDuringMemtableCompaction(t *testing.T) {
862 - truno(t, &opt.Options{WriteBuffer: 1000000}, func(h *dbHarness) {
863 -
864 - h.stor.DelaySync(storage.TypeTable)
865 - h.put("big1", strings.Repeat("x", 10000000))
866 - h.put("big2", strings.Repeat("y", 1000))
867 - h.put("bar", "v2")
868 - h.stor.ReleaseSync(storage.TypeTable)
869 -
870 - h.reopenDB()
871 - h.getVal("bar", "v2")
872 - h.getVal("big1", strings.Repeat("x", 10000000))
873 - h.getVal("big2", strings.Repeat("y", 1000))
874 - })
875 -}
876 -
877 -func TestDB_MinorCompactionsHappen(t *testing.T) {
878 - h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 10000})
879 - defer h.close()
880 -
881 - n := 500
882 -
883 - key := func(i int) string {
884 - return fmt.Sprintf("key%06d", i)
885 - }
886 -
887 - for i := 0; i < n; i++ {
888 - h.put(key(i), key(i)+strings.Repeat("v", 1000))
889 - }
890 -
891 - for i := 0; i < n; i++ {
892 - h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
893 - }
894 -
895 - h.reopenDB()
896 - for i := 0; i < n; i++ {
897 - h.getVal(key(i), key(i)+strings.Repeat("v", 1000))
898 - }
899 -}
900 -
901 -func TestDB_RecoverWithLargeJournal(t *testing.T) {
902 - h := newDbHarness(t)
903 - defer h.close()
904 -
905 - h.put("big1", strings.Repeat("1", 200000))
906 - h.put("big2", strings.Repeat("2", 200000))
907 - h.put("small3", strings.Repeat("3", 10))
908 - h.put("small4", strings.Repeat("4", 10))
909 - h.tablesPerLevel("")
910 -
911 - // Make sure that if we re-open with a small write buffer size that
912 - // we flush table files in the middle of a large journal file.
913 - h.o.WriteBuffer = 100000
914 - h.reopenDB()
915 - h.getVal("big1", strings.Repeat("1", 200000))
916 - h.getVal("big2", strings.Repeat("2", 200000))
917 - h.getVal("small3", strings.Repeat("3", 10))
918 - h.getVal("small4", strings.Repeat("4", 10))
919 - v := h.db.s.version()
920 - if v.tLen(0) <= 1 {
921 - t.Errorf("tables-0 less than one")
922 - }
923 - v.release()
924 -}
925 -
926 -func TestDB_CompactionsGenerateMultipleFiles(t *testing.T) {
927 - h := newDbHarnessWopt(t, &opt.Options{
928 - WriteBuffer: 10000000,
929 - Compression: opt.NoCompression,
930 - })
931 - defer h.close()
932 -
933 - v := h.db.s.version()
934 - if v.tLen(0) > 0 {
935 - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
936 - }
937 - v.release()
938 -
939 - n := 80
940 -
941 - // Write 8MB (80 values, each 100K)
942 - for i := 0; i < n; i++ {
943 - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
944 - }
945 -
946 - // Reopening moves updates to level-0
947 - h.reopenDB()
948 - h.compactRangeAt(0, "", "")
949 -
950 - v = h.db.s.version()
951 - if v.tLen(0) > 0 {
952 - t.Errorf("level-0 tables more than 0, got %d", v.tLen(0))
953 - }
954 - if v.tLen(1) <= 1 {
955 - t.Errorf("level-1 tables less than 1, got %d", v.tLen(1))
956 - }
957 - v.release()
958 -
959 - for i := 0; i < n; i++ {
960 - h.getVal(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), 100000/10))
961 - }
962 -}
963 -
964 -func TestDB_RepeatedWritesToSameKey(t *testing.T) {
965 - h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
966 - defer h.close()
967 -
968 - maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger()
969 -
970 - value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
971 - for i := 0; i < 5*maxTables; i++ {
972 - h.put("key", value)
973 - n := h.totalTables()
974 - if n > maxTables {
975 - t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
976 - }
977 - }
978 -}
979 -
980 -func TestDB_RepeatedWritesToSameKeyAfterReopen(t *testing.T) {
981 - h := newDbHarnessWopt(t, &opt.Options{WriteBuffer: 100000})
982 - defer h.close()
983 -
984 - h.reopenDB()
985 -
986 - maxTables := h.o.GetNumLevel() + h.o.GetWriteL0PauseTrigger()
987 -
988 - value := strings.Repeat("v", 2*h.o.GetWriteBuffer())
989 - for i := 0; i < 5*maxTables; i++ {
990 - h.put("key", value)
991 - n := h.totalTables()
992 - if n > maxTables {
993 - t.Errorf("total tables exceed %d, got=%d, iter=%d", maxTables, n, i)
994 - }
995 - }
996 -}
997 -
998 -func TestDB_SparseMerge(t *testing.T) {
999 - h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
1000 - defer h.close()
1001 -
1002 - h.putMulti(h.o.GetNumLevel(), "A", "Z")
1003 -
1004 - // Suppose there is:
1005 - // small amount of data with prefix A
1006 - // large amount of data with prefix B
1007 - // small amount of data with prefix C
1008 - // and that recent updates have made small changes to all three prefixes.
1009 - // Check that we do not do a compaction that merges all of B in one shot.
1010 - h.put("A", "va")
1011 - value := strings.Repeat("x", 1000)
1012 - for i := 0; i < 100000; i++ {
1013 - h.put(fmt.Sprintf("B%010d", i), value)
1014 - }
1015 - h.put("C", "vc")
1016 - h.compactMem()
1017 - h.compactRangeAt(0, "", "")
1018 - h.waitCompaction()
1019 -
1020 - // Make sparse update
1021 - h.put("A", "va2")
1022 - h.put("B100", "bvalue2")
1023 - h.put("C", "vc2")
1024 - h.compactMem()
1025 -
1026 - h.waitCompaction()
1027 - h.maxNextLevelOverlappingBytes(20 * 1048576)
1028 - h.compactRangeAt(0, "", "")
1029 - h.waitCompaction()
1030 - h.maxNextLevelOverlappingBytes(20 * 1048576)
1031 - h.compactRangeAt(1, "", "")
1032 - h.waitCompaction()
1033 - h.maxNextLevelOverlappingBytes(20 * 1048576)
1034 -}
1035 -
1036 -func TestDB_SizeOf(t *testing.T) {
1037 - h := newDbHarnessWopt(t, &opt.Options{
1038 - Compression: opt.NoCompression,
1039 - WriteBuffer: 10000000,
1040 - })
1041 - defer h.close()
1042 -
1043 - h.sizeAssert("", "xyz", 0, 0)
1044 - h.reopenDB()
1045 - h.sizeAssert("", "xyz", 0, 0)
1046 -
1047 - // Write 8MB (80 values, each 100K)
1048 - n := 80
1049 - s1 := 100000
1050 - s2 := 105000
1051 -
1052 - for i := 0; i < n; i++ {
1053 - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), s1/10))
1054 - }
1055 -
1056 - // 0 because SizeOf() does not account for memtable space
1057 - h.sizeAssert("", numKey(50), 0, 0)
1058 -
1059 - for r := 0; r < 3; r++ {
1060 - h.reopenDB()
1061 -
1062 - for cs := 0; cs < n; cs += 10 {
1063 - for i := 0; i < n; i += 10 {
1064 - h.sizeAssert("", numKey(i), uint64(s1*i), uint64(s2*i))
1065 - h.sizeAssert("", numKey(i)+".suffix", uint64(s1*(i+1)), uint64(s2*(i+1)))
1066 - h.sizeAssert(numKey(i), numKey(i+10), uint64(s1*10), uint64(s2*10))
1067 - }
1068 -
1069 - h.sizeAssert("", numKey(50), uint64(s1*50), uint64(s2*50))
1070 - h.sizeAssert("", numKey(50)+".suffix", uint64(s1*50), uint64(s2*50))
1071 -
1072 - h.compactRangeAt(0, numKey(cs), numKey(cs+9))
1073 - }
1074 -
1075 - v := h.db.s.version()
1076 - if v.tLen(0) != 0 {
1077 - t.Errorf("level-0 tables was not zero, got %d", v.tLen(0))
1078 - }
1079 - if v.tLen(1) == 0 {
1080 - t.Error("level-1 tables was zero")
1081 - }
1082 - v.release()
1083 - }
1084 -}
1085 -
1086 -func TestDB_SizeOf_MixOfSmallAndLarge(t *testing.T) {
1087 - h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
1088 - defer h.close()
1089 -
1090 - sizes := []uint64{
1091 - 10000,
1092 - 10000,
1093 - 100000,
1094 - 10000,
1095 - 100000,
1096 - 10000,
1097 - 300000,
1098 - 10000,
1099 - }
1100 -
1101 - for i, n := range sizes {
1102 - h.put(numKey(i), strings.Repeat(fmt.Sprintf("v%09d", i), int(n)/10))
1103 - }
1104 -
1105 - for r := 0; r < 3; r++ {
1106 - h.reopenDB()
1107 -
1108 - var x uint64
1109 - for i, n := range sizes {
1110 - y := x
1111 - if i > 0 {
1112 - y += 1000
1113 - }
1114 - h.sizeAssert("", numKey(i), x, y)
1115 - x += n
1116 - }
1117 -
1118 - h.sizeAssert(numKey(3), numKey(5), 110000, 111000)
1119 -
1120 - h.compactRangeAt(0, "", "")
1121 - }
1122 -}
1123 -
1124 -func TestDB_Snapshot(t *testing.T) {
1125 - trun(t, func(h *dbHarness) {
1126 - h.put("foo", "v1")
1127 - s1 := h.getSnapshot()
1128 - h.put("foo", "v2")
1129 - s2 := h.getSnapshot()
1130 - h.put("foo", "v3")
1131 - s3 := h.getSnapshot()
1132 - h.put("foo", "v4")
1133 -
1134 - h.getValr(s1, "foo", "v1")
1135 - h.getValr(s2, "foo", "v2")
1136 - h.getValr(s3, "foo", "v3")
1137 - h.getVal("foo", "v4")
1138 -
1139 - s3.Release()
1140 - h.getValr(s1, "foo", "v1")
1141 - h.getValr(s2, "foo", "v2")
1142 - h.getVal("foo", "v4")
1143 -
1144 - s1.Release()
1145 - h.getValr(s2, "foo", "v2")
1146 - h.getVal("foo", "v4")
1147 -
1148 - s2.Release()
1149 - h.getVal("foo", "v4")
1150 - })
1151 -}
1152 -
1153 -func TestDB_SnapshotList(t *testing.T) {
1154 - db := &DB{snapsList: list.New()}
1155 - e0a := db.acquireSnapshot()
1156 - e0b := db.acquireSnapshot()
1157 - db.seq = 1
1158 - e1 := db.acquireSnapshot()
1159 - db.seq = 2
1160 - e2 := db.acquireSnapshot()
1161 -
1162 - if db.minSeq() != 0 {
1163 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1164 - }
1165 - db.releaseSnapshot(e0a)
1166 - if db.minSeq() != 0 {
1167 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1168 - }
1169 - db.releaseSnapshot(e2)
1170 - if db.minSeq() != 0 {
1171 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1172 - }
1173 - db.releaseSnapshot(e0b)
1174 - if db.minSeq() != 1 {
1175 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1176 - }
1177 - e2 = db.acquireSnapshot()
1178 - if db.minSeq() != 1 {
1179 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1180 - }
1181 - db.releaseSnapshot(e1)
1182 - if db.minSeq() != 2 {
1183 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1184 - }
1185 - db.releaseSnapshot(e2)
1186 - if db.minSeq() != 2 {
1187 - t.Fatalf("invalid sequence number, got=%d", db.minSeq())
1188 - }
1189 -}
1190 -
1191 -func TestDB_HiddenValuesAreRemoved(t *testing.T) {
1192 - trun(t, func(h *dbHarness) {
1193 - s := h.db.s
1194 -
1195 - h.put("foo", "v1")
1196 - h.compactMem()
1197 - m := h.o.GetMaxMemCompationLevel()
1198 - v := s.version()
1199 - num := v.tLen(m)
1200 - v.release()
1201 - if num != 1 {
1202 - t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
1203 - }
1204 -
1205 - // Place a table at level last-1 to prevent merging with preceding mutation
1206 - h.put("a", "begin")
1207 - h.put("z", "end")
1208 - h.compactMem()
1209 - v = s.version()
1210 - if v.tLen(m) != 1 {
1211 - t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
1212 - }
1213 - if v.tLen(m-1) != 1 {
1214 - t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
1215 - }
1216 - v.release()
1217 -
1218 - h.delete("foo")
1219 - h.put("foo", "v2")
1220 - h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
1221 - h.compactMem()
1222 - h.allEntriesFor("foo", "[ v2, DEL, v1 ]")
1223 - h.compactRangeAt(m-2, "", "z")
1224 - // DEL eliminated, but v1 remains because we aren't compacting that level
1225 - // (DEL can be eliminated because v2 hides v1).
1226 - h.allEntriesFor("foo", "[ v2, v1 ]")
1227 - h.compactRangeAt(m-1, "", "")
1228 - // Merging last-1 w/ last, so we are the base level for "foo", so
1229 - // DEL is removed. (as is v1).
1230 - h.allEntriesFor("foo", "[ v2 ]")
1231 - })
1232 -}
1233 -
1234 -func TestDB_DeletionMarkers2(t *testing.T) {
1235 - h := newDbHarness(t)
1236 - defer h.close()
1237 - s := h.db.s
1238 -
1239 - h.put("foo", "v1")
1240 - h.compactMem()
1241 - m := h.o.GetMaxMemCompationLevel()
1242 - v := s.version()
1243 - num := v.tLen(m)
1244 - v.release()
1245 - if num != 1 {
1246 - t.Errorf("invalid level-%d len, want=1 got=%d", m, num)
1247 - }
1248 -
1249 - // Place a table at level last-1 to prevent merging with preceding mutation
1250 - h.put("a", "begin")
1251 - h.put("z", "end")
1252 - h.compactMem()
1253 - v = s.version()
1254 - if v.tLen(m) != 1 {
1255 - t.Errorf("invalid level-%d len, want=1 got=%d", m, v.tLen(m))
1256 - }
1257 - if v.tLen(m-1) != 1 {
1258 - t.Errorf("invalid level-%d len, want=1 got=%d", m-1, v.tLen(m-1))
1259 - }
1260 - v.release()
1261 -
1262 - h.delete("foo")
1263 - h.allEntriesFor("foo", "[ DEL, v1 ]")
1264 - h.compactMem() // Moves to level last-2
1265 - h.allEntriesFor("foo", "[ DEL, v1 ]")
1266 - h.compactRangeAt(m-2, "", "")
1267 - // DEL kept: "last" file overlaps
1268 - h.allEntriesFor("foo", "[ DEL, v1 ]")
1269 - h.compactRangeAt(m-1, "", "")
1270 - // Merging last-1 w/ last, so we are the base level for "foo", so
1271 - // DEL is removed. (as is v1).
1272 - h.allEntriesFor("foo", "[ ]")
1273 -}
1274 -
1275 -func TestDB_CompactionTableOpenError(t *testing.T) {
1276 - h := newDbHarnessWopt(t, &opt.Options{OpenFilesCacheCapacity: -1})
1277 - defer h.close()
1278 -
1279 - im := 10
1280 - jm := 10
1281 - for r := 0; r < 2; r++ {
1282 - for i := 0; i < im; i++ {
1283 - for j := 0; j < jm; j++ {
1284 - h.put(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
1285 - }
1286 - h.compactMem()
1287 - }
1288 - }
1289 -
1290 - if n := h.totalTables(); n != im*2 {
1291 - t.Errorf("total tables is %d, want %d", n, im)
1292 - }
1293 -
1294 - h.stor.SetEmuErr(storage.TypeTable, tsOpOpen)
1295 - go h.db.CompactRange(util.Range{})
1296 - if err := h.db.compSendIdle(h.db.tcompCmdC); err != nil {
1297 - t.Log("compaction error: ", err)
1298 - }
1299 - h.closeDB0()
1300 - h.openDB()
1301 - h.stor.SetEmuErr(0, tsOpOpen)
1302 -
1303 - for i := 0; i < im; i++ {
1304 - for j := 0; j < jm; j++ {
1305 - h.getVal(fmt.Sprintf("k%d,%d", i, j), fmt.Sprintf("v%d,%d", i, j))
1306 - }
1307 - }
1308 -}
1309 -
1310 -func TestDB_OverlapInLevel0(t *testing.T) {
1311 - trun(t, func(h *dbHarness) {
1312 - if h.o.GetMaxMemCompationLevel() != 2 {
1313 - t.Fatal("fix test to reflect the config")
1314 - }
1315 -
1316 - // Fill levels 1 and 2 to disable the pushing of new memtables to levels > 0.
1317 - h.put("100", "v100")
1318 - h.put("999", "v999")
1319 - h.compactMem()
1320 - h.delete("100")
1321 - h.delete("999")
1322 - h.compactMem()
1323 - h.tablesPerLevel("0,1,1")
1324 -
1325 - // Make files spanning the following ranges in level-0:
1326 - // files[0] 200 .. 900
1327 - // files[1] 300 .. 500
1328 - // Note that files are sorted by min key.
1329 - h.put("300", "v300")
1330 - h.put("500", "v500")
1331 - h.compactMem()
1332 - h.put("200", "v200")
1333 - h.put("600", "v600")
1334 - h.put("900", "v900")
1335 - h.compactMem()
1336 - h.tablesPerLevel("2,1,1")
1337 -
1338 - // Compact away the placeholder files we created initially
1339 - h.compactRangeAt(1, "", "")
1340 - h.compactRangeAt(2, "", "")
1341 - h.tablesPerLevel("2")
1342 -
1343 - // Do a memtable compaction. Before bug-fix, the compaction would
1344 - // not detect the overlap with level-0 files and would incorrectly place
1345 - // the deletion in a deeper level.
1346 - h.delete("600")
1347 - h.compactMem()
1348 - h.tablesPerLevel("3")
1349 - h.get("600", false)
1350 - })
1351 -}
1352 -
1353 -func TestDB_L0_CompactionBug_Issue44_a(t *testing.T) {
1354 - h := newDbHarness(t)
1355 - defer h.close()
1356 -
1357 - h.reopenDB()
1358 - h.put("b", "v")
1359 - h.reopenDB()
1360 - h.delete("b")
1361 - h.delete("a")
1362 - h.reopenDB()
1363 - h.delete("a")
1364 - h.reopenDB()
1365 - h.put("a", "v")
1366 - h.reopenDB()
1367 - h.reopenDB()
1368 - h.getKeyVal("(a->v)")
1369 - h.waitCompaction()
1370 - h.getKeyVal("(a->v)")
1371 -}
1372 -
1373 -func TestDB_L0_CompactionBug_Issue44_b(t *testing.T) {
1374 - h := newDbHarness(t)
1375 - defer h.close()
1376 -
1377 - h.reopenDB()
1378 - h.put("", "")
1379 - h.reopenDB()
1380 - h.delete("e")
1381 - h.put("", "")
1382 - h.reopenDB()
1383 - h.put("c", "cv")
1384 - h.reopenDB()
1385 - h.put("", "")
1386 - h.reopenDB()
1387 - h.put("", "")
1388 - h.waitCompaction()
1389 - h.reopenDB()
1390 - h.put("d", "dv")
1391 - h.reopenDB()
1392 - h.put("", "")
1393 - h.reopenDB()
1394 - h.delete("d")
1395 - h.delete("b")
1396 - h.reopenDB()
1397 - h.getKeyVal("(->)(c->cv)")
1398 - h.waitCompaction()
1399 - h.getKeyVal("(->)(c->cv)")
1400 -}
1401 -
1402 -func TestDB_SingleEntryMemCompaction(t *testing.T) {
1403 - trun(t, func(h *dbHarness) {
1404 - for i := 0; i < 10; i++ {
1405 - h.put("big", strings.Repeat("v", opt.DefaultWriteBuffer))
1406 - h.compactMem()
1407 - h.put("key", strings.Repeat("v", opt.DefaultBlockSize))
1408 - h.compactMem()
1409 - h.put("k", "v")
1410 - h.compactMem()
1411 - h.put("", "")
1412 - h.compactMem()
1413 - h.put("verybig", strings.Repeat("v", opt.DefaultWriteBuffer*2))
1414 - h.compactMem()
1415 - }
1416 - })
1417 -}
1418 -
1419 -func TestDB_ManifestWriteError(t *testing.T) {
1420 - for i := 0; i < 2; i++ {
1421 - func() {
1422 - h := newDbHarness(t)
1423 - defer h.close()
1424 -
1425 - h.put("foo", "bar")
1426 - h.getVal("foo", "bar")
1427 -
1428 - // Mem compaction (will succeed)
1429 - h.compactMem()
1430 - h.getVal("foo", "bar")
1431 - v := h.db.s.version()
1432 - if n := v.tLen(h.o.GetMaxMemCompationLevel()); n != 1 {
1433 - t.Errorf("invalid total tables, want=1 got=%d", n)
1434 - }
1435 - v.release()
1436 -
1437 - if i == 0 {
1438 - h.stor.SetEmuErr(storage.TypeManifest, tsOpWrite)
1439 - } else {
1440 - h.stor.SetEmuErr(storage.TypeManifest, tsOpSync)
1441 - }
1442 -
1443 - // Merging compaction (will fail)
1444 - h.compactRangeAtErr(h.o.GetMaxMemCompationLevel(), "", "", true)
1445 -
1446 - h.db.Close()
1447 - h.stor.SetEmuErr(0, tsOpWrite)
1448 - h.stor.SetEmuErr(0, tsOpSync)
1449 -
1450 - // Should not lose data
1451 - h.openDB()
1452 - h.getVal("foo", "bar")
1453 - }()
1454 - }
1455 -}
1456 -
1457 -func assertErr(t *testing.T, err error, wanterr bool) {
1458 - if err != nil {
1459 - if wanterr {
1460 - t.Log("AssertErr: got error (expected): ", err)
1461 - } else {
1462 - t.Error("AssertErr: got error: ", err)
1463 - }
1464 - } else if wanterr {
1465 - t.Error("AssertErr: expect error")
1466 - }
1467 -}
1468 -
1469 -func TestDB_ClosedIsClosed(t *testing.T) {
1470 - h := newDbHarness(t)
1471 - db := h.db
1472 -
1473 - var iter, iter2 iterator.Iterator
1474 - var snap *Snapshot
1475 - func() {
1476 - defer h.close()
1477 -
1478 - h.put("k", "v")
1479 - h.getVal("k", "v")
1480 -
1481 - iter = db.NewIterator(nil, h.ro)
1482 - iter.Seek([]byte("k"))
1483 - testKeyVal(t, iter, "k->v")
1484 -
1485 - var err error
1486 - snap, err = db.GetSnapshot()
1487 - if err != nil {
1488 - t.Fatal("GetSnapshot: got error: ", err)
1489 - }
1490 -
1491 - h.getValr(snap, "k", "v")
1492 -
1493 - iter2 = snap.NewIterator(nil, h.ro)
1494 - iter2.Seek([]byte("k"))
1495 - testKeyVal(t, iter2, "k->v")
1496 -
1497 - h.put("foo", "v2")
1498 - h.delete("foo")
1499 -
1500 - // closing DB
1501 - iter.Release()
1502 - iter2.Release()
1503 - }()
1504 -
1505 - assertErr(t, db.Put([]byte("x"), []byte("y"), h.wo), true)
1506 - _, err := db.Get([]byte("k"), h.ro)
1507 - assertErr(t, err, true)
1508 -
1509 - if iter.Valid() {
1510 - t.Errorf("iter.Valid should false")
1511 - }
1512 - assertErr(t, iter.Error(), false)
1513 - testKeyVal(t, iter, "->")
1514 - if iter.Seek([]byte("k")) {
1515 - t.Errorf("iter.Seek should false")
1516 - }
1517 - assertErr(t, iter.Error(), true)
1518 -
1519 - assertErr(t, iter2.Error(), false)
1520 -
1521 - _, err = snap.Get([]byte("k"), h.ro)
1522 - assertErr(t, err, true)
1523 -
1524 - _, err = db.GetSnapshot()
1525 - assertErr(t, err, true)
1526 -
1527 - iter3 := db.NewIterator(nil, h.ro)
1528 - assertErr(t, iter3.Error(), true)
1529 -
1530 - iter3 = snap.NewIterator(nil, h.ro)
1531 - assertErr(t, iter3.Error(), true)
1532 -
1533 - assertErr(t, db.Delete([]byte("k"), h.wo), true)
1534 -
1535 - _, err = db.GetProperty("leveldb.stats")
1536 - assertErr(t, err, true)
1537 -
1538 - _, err = db.SizeOf([]util.Range{{[]byte("a"), []byte("z")}})
1539 - assertErr(t, err, true)
1540 -
1541 - assertErr(t, db.CompactRange(util.Range{}), true)
1542 -
1543 - assertErr(t, db.Close(), true)
1544 -}
1545 -
1546 -type numberComparer struct{}
1547 -
1548 -func (numberComparer) num(x []byte) (n int) {
1549 - fmt.Sscan(string(x[1:len(x)-1]), &n)
1550 - return
1551 -}
1552 -
1553 -func (numberComparer) Name() string {
1554 - return "test.NumberComparer"
1555 -}
1556 -
1557 -func (p numberComparer) Compare(a, b []byte) int {
1558 - return p.num(a) - p.num(b)
1559 -}
1560 -
1561 -func (numberComparer) Separator(dst, a, b []byte) []byte { return nil }
1562 -func (numberComparer) Successor(dst, b []byte) []byte { return nil }
1563 -
1564 -func TestDB_CustomComparer(t *testing.T) {
1565 - h := newDbHarnessWopt(t, &opt.Options{
1566 - Comparer: numberComparer{},
1567 - WriteBuffer: 1000,
1568 - })
1569 - defer h.close()
1570 -
1571 - h.put("[10]", "ten")
1572 - h.put("[0x14]", "twenty")
1573 - for i := 0; i < 2; i++ {
1574 - h.getVal("[10]", "ten")
1575 - h.getVal("[0xa]", "ten")
1576 - h.getVal("[20]", "twenty")
1577 - h.getVal("[0x14]", "twenty")
1578 - h.get("[15]", false)
1579 - h.get("[0xf]", false)
1580 - h.compactMem()
1581 - h.compactRange("[0]", "[9999]")
1582 - }
1583 -
1584 - for n := 0; n < 2; n++ {
1585 - for i := 0; i < 100; i++ {
1586 - v := fmt.Sprintf("[%d]", i*10)
1587 - h.put(v, v)
1588 - }
1589 - h.compactMem()
1590 - h.compactRange("[0]", "[1000000]")
1591 - }
1592 -}
1593 -
1594 -func TestDB_ManualCompaction(t *testing.T) {
1595 - h := newDbHarness(t)
1596 - defer h.close()
1597 -
1598 - if h.o.GetMaxMemCompationLevel() != 2 {
1599 - t.Fatal("fix test to reflect the config")
1600 - }
1601 -
1602 - h.putMulti(3, "p", "q")
1603 - h.tablesPerLevel("1,1,1")
1604 -
1605 - // Compaction range falls before files
1606 - h.compactRange("", "c")
1607 - h.tablesPerLevel("1,1,1")
1608 -
1609 - // Compaction range falls after files
1610 - h.compactRange("r", "z")
1611 - h.tablesPerLevel("1,1,1")
1612 -
1613 - // Compaction range overlaps files
1614 - h.compactRange("p1", "p9")
1615 - h.tablesPerLevel("0,0,1")
1616 -
1617 - // Populate a different range
1618 - h.putMulti(3, "c", "e")
1619 - h.tablesPerLevel("1,1,2")
1620 -
1621 - // Compact just the new range
1622 - h.compactRange("b", "f")
1623 - h.tablesPerLevel("0,0,2")
1624 -
1625 - // Compact all
1626 - h.putMulti(1, "a", "z")
1627 - h.tablesPerLevel("0,1,2")
1628 - h.compactRange("", "")
1629 - h.tablesPerLevel("0,0,1")
1630 -}
1631 -
1632 -func TestDB_BloomFilter(t *testing.T) {
1633 - h := newDbHarnessWopt(t, &opt.Options{
1634 - DisableBlockCache: true,
1635 - Filter: filter.NewBloomFilter(10),
1636 - })
1637 - defer h.close()
1638 -
1639 - key := func(i int) string {
1640 - return fmt.Sprintf("key%06d", i)
1641 - }
1642 -
1643 - const n = 10000
1644 -
1645 - // Populate multiple layers
1646 - for i := 0; i < n; i++ {
1647 - h.put(key(i), key(i))
1648 - }
1649 - h.compactMem()
1650 - h.compactRange("a", "z")
1651 - for i := 0; i < n; i += 100 {
1652 - h.put(key(i), key(i))
1653 - }
1654 - h.compactMem()
1655 -
1656 - // Prevent auto compactions triggered by seeks
1657 - h.stor.DelaySync(storage.TypeTable)
1658 -
1659 - // Lookup present keys. Should rarely read from small sstable.
1660 - h.stor.SetReadCounter(storage.TypeTable)
1661 - for i := 0; i < n; i++ {
1662 - h.getVal(key(i), key(i))
1663 - }
1664 - cnt := int(h.stor.ReadCounter())
1665 - t.Logf("lookup of %d present keys yield %d sstable I/O reads", n, cnt)
1666 -
1667 - if min, max := n, n+2*n/100; cnt < min || cnt > max {
1668 - t.Errorf("num of sstable I/O reads of present keys not in range of %d - %d, got %d", min, max, cnt)
1669 - }
1670 -
1671 - // Lookup missing keys. Should rarely read from either sstable.
1672 - h.stor.ResetReadCounter()
1673 - for i := 0; i < n; i++ {
1674 - h.get(key(i)+".missing", false)
1675 - }
1676 - cnt = int(h.stor.ReadCounter())
1677 - t.Logf("lookup of %d missing keys yield %d sstable I/O reads", n, cnt)
1678 - if max := 3 * n / 100; cnt > max {
1679 - t.Errorf("num of sstable I/O reads of missing keys was more than %d, got %d", max, cnt)
1680 - }
1681 -
1682 - h.stor.ReleaseSync(storage.TypeTable)
1683 -}
1684 -
1685 -func TestDB_Concurrent(t *testing.T) {
1686 - const n, secs, maxkey = 4, 2, 1000
1687 -
1688 - runtime.GOMAXPROCS(n)
1689 - trun(t, func(h *dbHarness) {
1690 - var closeWg sync.WaitGroup
1691 - var stop uint32
1692 - var cnt [n]uint32
1693 -
1694 - for i := 0; i < n; i++ {
1695 - closeWg.Add(1)
1696 - go func(i int) {
1697 - var put, get, found uint
1698 - defer func() {
1699 - t.Logf("goroutine %d stopped after %d ops, put=%d get=%d found=%d missing=%d",
1700 - i, cnt[i], put, get, found, get-found)
1701 - closeWg.Done()
1702 - }()
1703 -
1704 - rnd := rand.New(rand.NewSource(int64(1000 + i)))
1705 - for atomic.LoadUint32(&stop) == 0 {
1706 - x := cnt[i]
1707 -
1708 - k := rnd.Intn(maxkey)
1709 - kstr := fmt.Sprintf("%016d", k)
1710 -
1711 - if (rnd.Int() % 2) > 0 {
1712 - put++
1713 - h.put(kstr, fmt.Sprintf("%d.%d.%-1000d", k, i, x))
1714 - } else {
1715 - get++
1716 - v, err := h.db.Get([]byte(kstr), h.ro)
1717 - if err == nil {
1718 - found++
1719 - rk, ri, rx := 0, -1, uint32(0)
1720 - fmt.Sscanf(string(v), "%d.%d.%d", &rk, &ri, &rx)
1721 - if rk != k {
1722 - t.Errorf("invalid key want=%d got=%d", k, rk)
1723 - }
1724 - if ri < 0 || ri >= n {
1725 - t.Error("invalid goroutine number: ", ri)
1726 - } else {
1727 - tx := atomic.LoadUint32(&(cnt[ri]))
1728 - if rx > tx {
1729 - t.Errorf("invalid seq number, %d > %d ", rx, tx)
1730 - }
1731 - }
1732 - } else if err != ErrNotFound {
1733 - t.Error("Get: got error: ", err)
1734 - return
1735 - }
1736 - }
1737 - atomic.AddUint32(&cnt[i], 1)
1738 - }
1739 - }(i)
1740 - }
1741 -
1742 - time.Sleep(secs * time.Second)
1743 - atomic.StoreUint32(&stop, 1)
1744 - closeWg.Wait()
1745 - })
1746 -
1747 - runtime.GOMAXPROCS(1)
1748 -}
1749 -
1750 -func TestDB_Concurrent2(t *testing.T) {
1751 - const n, n2 = 4, 4000
1752 -
1753 - runtime.GOMAXPROCS(n*2 + 2)
1754 - truno(t, &opt.Options{WriteBuffer: 30}, func(h *dbHarness) {
1755 - var closeWg sync.WaitGroup
1756 - var stop uint32
1757 -
1758 - for i := 0; i < n; i++ {
1759 - closeWg.Add(1)
1760 - go func(i int) {
1761 - for k := 0; atomic.LoadUint32(&stop) == 0; k++ {
1762 - h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
1763 - }
1764 - closeWg.Done()
1765 - }(i)
1766 - }
1767 -
1768 - for i := 0; i < n; i++ {
1769 - closeWg.Add(1)
1770 - go func(i int) {
1771 - for k := 1000000; k < 0 || atomic.LoadUint32(&stop) == 0; k-- {
1772 - h.put(fmt.Sprintf("k%d", k), fmt.Sprintf("%d.%d.", k, i)+strings.Repeat("x", 10))
1773 - }
1774 - closeWg.Done()
1775 - }(i)
1776 - }
1777 -
1778 - cmp := comparer.DefaultComparer
1779 - for i := 0; i < n2; i++ {
1780 - closeWg.Add(1)
1781 - go func(i int) {
1782 - it := h.db.NewIterator(nil, nil)
1783 - var pk []byte
1784 - for it.Next() {
1785 - kk := it.Key()
1786 - if cmp.Compare(kk, pk) <= 0 {
1787 - t.Errorf("iter %d: %q is successor of %q", i, pk, kk)
1788 - }
1789 - pk = append(pk[:0], kk...)
1790 - var k, vk, vi int
1791 - if n, err := fmt.Sscanf(string(it.Key()), "k%d", &k); err != nil {
1792 - t.Errorf("iter %d: Scanf error on key %q: %v", i, it.Key(), err)
1793 - } else if n < 1 {
1794 - t.Errorf("iter %d: Cannot parse key %q", i, it.Key())
1795 - }
1796 - if n, err := fmt.Sscanf(string(it.Value()), "%d.%d", &vk, &vi); err != nil {
1797 - t.Errorf("iter %d: Scanf error on value %q: %v", i, it.Value(), err)
1798 - } else if n < 2 {
1799 - t.Errorf("iter %d: Cannot parse value %q", i, it.Value())
1800 - }
1801 -
1802 - if vk != k {
1803 - t.Errorf("iter %d: invalid value i=%d, want=%d got=%d", i, vi, k, vk)
1804 - }
1805 - }
1806 - if err := it.Error(); err != nil {
1807 - t.Errorf("iter %d: Got error: %v", i, err)
1808 - }
1809 - it.Release()
1810 - closeWg.Done()
1811 - }(i)
1812 - }
1813 -
1814 - atomic.StoreUint32(&stop, 1)
1815 - closeWg.Wait()
1816 - })
1817 -
1818 - runtime.GOMAXPROCS(1)
1819 -}
1820 -
1821 -func TestDB_CreateReopenDbOnFile(t *testing.T) {
1822 - dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile-%d", os.Getuid()))
1823 - if err := os.RemoveAll(dbpath); err != nil {
1824 - t.Fatal("cannot remove old db: ", err)
1825 - }
1826 - defer os.RemoveAll(dbpath)
1827 -
1828 - for i := 0; i < 3; i++ {
1829 - stor, err := storage.OpenFile(dbpath)
1830 - if err != nil {
1831 - t.Fatalf("(%d) cannot open storage: %s", i, err)
1832 - }
1833 - db, err := Open(stor, nil)
1834 - if err != nil {
1835 - t.Fatalf("(%d) cannot open db: %s", i, err)
1836 - }
1837 - if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
1838 - t.Fatalf("(%d) cannot write to db: %s", i, err)
1839 - }
1840 - if err := db.Close(); err != nil {
1841 - t.Fatalf("(%d) cannot close db: %s", i, err)
1842 - }
1843 - if err := stor.Close(); err != nil {
1844 - t.Fatalf("(%d) cannot close storage: %s", i, err)
1845 - }
1846 - }
1847 -}
1848 -
1849 -func TestDB_CreateReopenDbOnFile2(t *testing.T) {
1850 - dbpath := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestCreateReopenDbOnFile2-%d", os.Getuid()))
1851 - if err := os.RemoveAll(dbpath); err != nil {
1852 - t.Fatal("cannot remove old db: ", err)
1853 - }
1854 - defer os.RemoveAll(dbpath)
1855 -
1856 - for i := 0; i < 3; i++ {
1857 - db, err := OpenFile(dbpath, nil)
1858 - if err != nil {
1859 - t.Fatalf("(%d) cannot open db: %s", i, err)
1860 - }
1861 - if err := db.Put([]byte("foo"), []byte("bar"), nil); err != nil {
1862 - t.Fatalf("(%d) cannot write to db: %s", i, err)
1863 - }
1864 - if err := db.Close(); err != nil {
1865 - t.Fatalf("(%d) cannot close db: %s", i, err)
1866 - }
1867 - }
1868 -}
1869 -
1870 -func TestDB_DeletionMarkersOnMemdb(t *testing.T) {
1871 - h := newDbHarness(t)
1872 - defer h.close()
1873 -
1874 - h.put("foo", "v1")
1875 - h.compactMem()
1876 - h.delete("foo")
1877 - h.get("foo", false)
1878 - h.getKeyVal("")
1879 -}
1880 -
1881 -func TestDB_LeveldbIssue178(t *testing.T) {
1882 - nKeys := (opt.DefaultCompactionTableSize / 30) * 5
1883 - key1 := func(i int) string {
1884 - return fmt.Sprintf("my_key_%d", i)
1885 - }
1886 - key2 := func(i int) string {
1887 - return fmt.Sprintf("my_key_%d_xxx", i)
1888 - }
1889 -
1890 - // Disable compression since it affects the creation of layers and the
1891 - // code below is trying to test against a very specific scenario.
1892 - h := newDbHarnessWopt(t, &opt.Options{Compression: opt.NoCompression})
1893 - defer h.close()
1894 -
1895 - // Create first key range.
1896 - batch := new(Batch)
1897 - for i := 0; i < nKeys; i++ {
1898 - batch.Put([]byte(key1(i)), []byte("value for range 1 key"))
1899 - }
1900 - h.write(batch)
1901 -
1902 - // Create second key range.
1903 - batch.Reset()
1904 - for i := 0; i < nKeys; i++ {
1905 - batch.Put([]byte(key2(i)), []byte("value for range 2 key"))
1906 - }
1907 - h.write(batch)
1908 -
1909 - // Delete second key range.
1910 - batch.Reset()
1911 - for i := 0; i < nKeys; i++ {
1912 - batch.Delete([]byte(key2(i)))
1913 - }
1914 - h.write(batch)
1915 - h.waitMemCompaction()
1916 -
1917 - // Run manual compaction.
1918 - h.compactRange(key1(0), key1(nKeys-1))
1919 -
1920 - // Checking the keys.
1921 - h.assertNumKeys(nKeys)
1922 -}
1923 -
1924 -func TestDB_LeveldbIssue200(t *testing.T) {
1925 - h := newDbHarness(t)
1926 - defer h.close()
1927 -
1928 - h.put("1", "b")
1929 - h.put("2", "c")
1930 - h.put("3", "d")
1931 - h.put("4", "e")
1932 - h.put("5", "f")
1933 -
1934 - iter := h.db.NewIterator(nil, h.ro)
1935 -
1936 - // Add an element that should not be reflected in the iterator.
1937 - h.put("25", "cd")
1938 -
1939 - iter.Seek([]byte("5"))
1940 - assertBytes(t, []byte("5"), iter.Key())
1941 - iter.Prev()
1942 - assertBytes(t, []byte("4"), iter.Key())
1943 - iter.Prev()
1944 - assertBytes(t, []byte("3"), iter.Key())
1945 - iter.Next()
1946 - assertBytes(t, []byte("4"), iter.Key())
1947 - iter.Next()
1948 - assertBytes(t, []byte("5"), iter.Key())
1949 -}
1950 -
1951 -func TestDB_GoleveldbIssue74(t *testing.T) {
1952 - h := newDbHarnessWopt(t, &opt.Options{
1953 - WriteBuffer: 1 * opt.MiB,
1954 - })
1955 - defer h.close()
1956 -
1957 - const n, dur = 10000, 5 * time.Second
1958 -
1959 - runtime.GOMAXPROCS(runtime.NumCPU())
1960 -
1961 - until := time.Now().Add(dur)
1962 - wg := new(sync.WaitGroup)
1963 - wg.Add(2)
1964 - var done uint32
1965 - go func() {
1966 - var i int
1967 - defer func() {
1968 - t.Logf("WRITER DONE #%d", i)
1969 - atomic.StoreUint32(&done, 1)
1970 - wg.Done()
1971 - }()
1972 -
1973 - b := new(Batch)
1974 - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
1975 - iv := fmt.Sprintf("VAL%010d", i)
1976 - for k := 0; k < n; k++ {
1977 - key := fmt.Sprintf("KEY%06d", k)
1978 - b.Put([]byte(key), []byte(key+iv))
1979 - b.Put([]byte(fmt.Sprintf("PTR%06d", k)), []byte(key))
1980 - }
1981 - h.write(b)
1982 -
1983 - b.Reset()
1984 - snap := h.getSnapshot()
1985 - iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
1986 - var k int
1987 - for ; iter.Next(); k++ {
1988 - ptrKey := iter.Key()
1989 - key := iter.Value()
1990 -
1991 - if _, err := snap.Get(ptrKey, nil); err != nil {
1992 - t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, ptrKey, err)
1993 - }
1994 - if value, err := snap.Get(key, nil); err != nil {
1995 - t.Fatalf("WRITER #%d snapshot.Get %q: %v", i, key, err)
1996 - } else if string(value) != string(key)+iv {
1997 - t.Fatalf("WRITER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+iv, value)
1998 - }
1999 -
2000 - b.Delete(key)
2001 - b.Delete(ptrKey)
2002 - }
2003 - h.write(b)
2004 - iter.Release()
2005 - snap.Release()
2006 - if k != n {
2007 - t.Fatalf("#%d %d != %d", i, k, n)
2008 - }
2009 - }
2010 - }()
2011 - go func() {
2012 - var i int
2013 - defer func() {
2014 - t.Logf("READER DONE #%d", i)
2015 - atomic.StoreUint32(&done, 1)
2016 - wg.Done()
2017 - }()
2018 - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2019 - snap := h.getSnapshot()
2020 - iter := snap.NewIterator(util.BytesPrefix([]byte("PTR")), nil)
2021 - var prevValue string
2022 - var k int
2023 - for ; iter.Next(); k++ {
2024 - ptrKey := iter.Key()
2025 - key := iter.Value()
2026 -
2027 - if _, err := snap.Get(ptrKey, nil); err != nil {
2028 - t.Fatalf("READER #%d snapshot.Get %q: %v", i, ptrKey, err)
2029 - }
2030 -
2031 - if value, err := snap.Get(key, nil); err != nil {
2032 - t.Fatalf("READER #%d snapshot.Get %q: %v", i, key, err)
2033 - } else if prevValue != "" && string(value) != string(key)+prevValue {
2034 - t.Fatalf("READER #%d snapshot.Get %q got invalid value, want %q got %q", i, key, string(key)+prevValue, value)
2035 - } else {
2036 - prevValue = string(value[len(key):])
2037 - }
2038 - }
2039 - iter.Release()
2040 - snap.Release()
2041 - if k > 0 && k != n {
2042 - t.Fatalf("#%d %d != %d", i, k, n)
2043 - }
2044 - }
2045 - }()
2046 - wg.Wait()
2047 -}
2048 -
2049 -func TestDB_GetProperties(t *testing.T) {
2050 - h := newDbHarness(t)
2051 - defer h.close()
2052 -
2053 - _, err := h.db.GetProperty("leveldb.num-files-at-level")
2054 - if err == nil {
2055 - t.Error("GetProperty() failed to detect missing level")
2056 - }
2057 -
2058 - _, err = h.db.GetProperty("leveldb.num-files-at-level0")
2059 - if err != nil {
2060 - t.Error("got unexpected error", err)
2061 - }
2062 -
2063 - _, err = h.db.GetProperty("leveldb.num-files-at-level0x")
2064 - if err == nil {
2065 - t.Error("GetProperty() failed to detect invalid level")
2066 - }
2067 -}
2068 -
2069 -func TestDB_GoleveldbIssue72and83(t *testing.T) {
2070 - h := newDbHarnessWopt(t, &opt.Options{
2071 - WriteBuffer: 1 * opt.MiB,
2072 - OpenFilesCacheCapacity: 3,
2073 - })
2074 - defer h.close()
2075 -
2076 - const n, wn, dur = 10000, 100, 30 * time.Second
2077 -
2078 - runtime.GOMAXPROCS(runtime.NumCPU())
2079 -
2080 - randomData := func(prefix byte, i int) []byte {
2081 - data := make([]byte, 1+4+32+64+32)
2082 - _, err := crand.Reader.Read(data[1 : len(data)-8])
2083 - if err != nil {
2084 - panic(err)
2085 - }
2086 - data[0] = prefix
2087 - binary.LittleEndian.PutUint32(data[len(data)-8:], uint32(i))
2088 - binary.LittleEndian.PutUint32(data[len(data)-4:], util.NewCRC(data[:len(data)-4]).Value())
2089 - return data
2090 - }
2091 -
2092 - keys := make([][]byte, n)
2093 - for i := range keys {
2094 - keys[i] = randomData(1, 0)
2095 - }
2096 -
2097 - until := time.Now().Add(dur)
2098 - wg := new(sync.WaitGroup)
2099 - wg.Add(3)
2100 - var done uint32
2101 - go func() {
2102 - i := 0
2103 - defer func() {
2104 - t.Logf("WRITER DONE #%d", i)
2105 - wg.Done()
2106 - }()
2107 -
2108 - b := new(Batch)
2109 - for ; i < wn && atomic.LoadUint32(&done) == 0; i++ {
2110 - b.Reset()
2111 - for _, k1 := range keys {
2112 - k2 := randomData(2, i)
2113 - b.Put(k2, randomData(42, i))
2114 - b.Put(k1, k2)
2115 - }
2116 - if err := h.db.Write(b, h.wo); err != nil {
2117 - atomic.StoreUint32(&done, 1)
2118 - t.Fatalf("WRITER #%d db.Write: %v", i, err)
2119 - }
2120 - }
2121 - }()
2122 - go func() {
2123 - var i int
2124 - defer func() {
2125 - t.Logf("READER0 DONE #%d", i)
2126 - atomic.StoreUint32(&done, 1)
2127 - wg.Done()
2128 - }()
2129 - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2130 - snap := h.getSnapshot()
2131 - seq := snap.elem.seq
2132 - if seq == 0 {
2133 - snap.Release()
2134 - continue
2135 - }
2136 - iter := snap.NewIterator(util.BytesPrefix([]byte{1}), nil)
2137 - writei := int(seq/(n*2) - 1)
2138 - var k int
2139 - for ; iter.Next(); k++ {
2140 - k1 := iter.Key()
2141 - k2 := iter.Value()
2142 - k1checksum0 := binary.LittleEndian.Uint32(k1[len(k1)-4:])
2143 - k1checksum1 := util.NewCRC(k1[:len(k1)-4]).Value()
2144 - if k1checksum0 != k1checksum1 {
2145 - t.Fatalf("READER0 #%d.%d W#%d invalid K1 checksum: %#x != %#x", i, k, k1checksum0, k1checksum0)
2146 - }
2147 - k2checksum0 := binary.LittleEndian.Uint32(k2[len(k2)-4:])
2148 - k2checksum1 := util.NewCRC(k2[:len(k2)-4]).Value()
2149 - if k2checksum0 != k2checksum1 {
2150 - t.Fatalf("READER0 #%d.%d W#%d invalid K2 checksum: %#x != %#x", i, k, k2checksum0, k2checksum1)
2151 - }
2152 - kwritei := int(binary.LittleEndian.Uint32(k2[len(k2)-8:]))
2153 - if writei != kwritei {
2154 - t.Fatalf("READER0 #%d.%d W#%d invalid write iteration num: %d", i, k, writei, kwritei)
2155 - }
2156 - if _, err := snap.Get(k2, nil); err != nil {
2157 - t.Fatalf("READER0 #%d.%d W#%d snap.Get: %v\nk1: %x\n -> k2: %x", i, k, writei, err, k1, k2)
2158 - }
2159 - }
2160 - if err := iter.Error(); err != nil {
2161 - t.Fatalf("READER0 #%d.%d W#%d snap.Iterator: %v", i, k, writei, err)
2162 - }
2163 - iter.Release()
2164 - snap.Release()
2165 - if k > 0 && k != n {
2166 - t.Fatalf("READER0 #%d W#%d short read, got=%d want=%d", i, writei, k, n)
2167 - }
2168 - }
2169 - }()
2170 - go func() {
2171 - var i int
2172 - defer func() {
2173 - t.Logf("READER1 DONE #%d", i)
2174 - atomic.StoreUint32(&done, 1)
2175 - wg.Done()
2176 - }()
2177 - for ; time.Now().Before(until) && atomic.LoadUint32(&done) == 0; i++ {
2178 - iter := h.db.NewIterator(nil, nil)
2179 - seq := iter.(*dbIter).seq
2180 - if seq == 0 {
2181 - iter.Release()
2182 - continue
2183 - }
2184 - writei := int(seq/(n*2) - 1)
2185 - var k int
2186 - for ok := iter.Last(); ok; ok = iter.Prev() {
2187 - k++
2188 - }
2189 - if err := iter.Error(); err != nil {
2190 - t.Fatalf("READER1 #%d.%d W#%d db.Iterator: %v", i, k, writei, err)
2191 - }
2192 - iter.Release()
2193 - if m := (writei+1)*n + n; k != m {
2194 - t.Fatalf("READER1 #%d W#%d short read, got=%d want=%d", i, writei, k, m)
2195 - }
2196 - }
2197 - }()
2198 -
2199 - wg.Wait()
2200 -}
2201 -
2202 -func TestDB_TransientError(t *testing.T) {
2203 - h := newDbHarnessWopt(t, &opt.Options{
2204 - WriteBuffer: 128 * opt.KiB,
2205 - OpenFilesCacheCapacity: 3,
2206 - DisableCompactionBackoff: true,
2207 - })
2208 - defer h.close()
2209 -
2210 - const (
2211 - nSnap = 20
2212 - nKey = 10000
2213 - )
2214 -
2215 - var (
2216 - snaps [nSnap]*Snapshot
2217 - b = &Batch{}
2218 - )
2219 - for i := range snaps {
2220 - vtail := fmt.Sprintf("VAL%030d", i)
2221 - b.Reset()
2222 - for k := 0; k < nKey; k++ {
2223 - key := fmt.Sprintf("KEY%8d", k)
2224 - b.Put([]byte(key), []byte(key+vtail))
2225 - }
2226 - h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt)
2227 - if err := h.db.Write(b, nil); err != nil {
2228 - t.Logf("WRITE #%d error: %v", i, err)
2229 - h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt, tsOpWrite)
2230 - for {
2231 - if err := h.db.Write(b, nil); err == nil {
2232 - break
2233 - } else if errors.IsCorrupted(err) {
2234 - t.Fatalf("WRITE #%d corrupted: %v", i, err)
2235 - }
2236 - }
2237 - }
2238 -
2239 - snaps[i] = h.db.newSnapshot()
2240 - b.Reset()
2241 - for k := 0; k < nKey; k++ {
2242 - key := fmt.Sprintf("KEY%8d", k)
2243 - b.Delete([]byte(key))
2244 - }
2245 - h.stor.SetEmuRandErr(storage.TypeTable, tsOpOpen, tsOpRead, tsOpReadAt)
2246 - if err := h.db.Write(b, nil); err != nil {
2247 - t.Logf("WRITE #%d error: %v", i, err)
2248 - h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt)
2249 - for {
2250 - if err := h.db.Write(b, nil); err == nil {
2251 - break
2252 - } else if errors.IsCorrupted(err) {
2253 - t.Fatalf("WRITE #%d corrupted: %v", i, err)
2254 - }
2255 - }
2256 - }
2257 - }
2258 - h.stor.SetEmuRandErr(0, tsOpOpen, tsOpRead, tsOpReadAt)
2259 -
2260 - runtime.GOMAXPROCS(runtime.NumCPU())
2261 -
2262 - rnd := rand.New(rand.NewSource(0xecafdaed))
2263 - wg := &sync.WaitGroup{}
2264 - for i, snap := range snaps {
2265 - wg.Add(2)
2266 -
2267 - go func(i int, snap *Snapshot, sk []int) {
2268 - defer wg.Done()
2269 -
2270 - vtail := fmt.Sprintf("VAL%030d", i)
2271 - for _, k := range sk {
2272 - key := fmt.Sprintf("KEY%8d", k)
2273 - xvalue, err := snap.Get([]byte(key), nil)
2274 - if err != nil {
2275 - t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err)
2276 - }
2277 - value := key + vtail
2278 - if !bytes.Equal([]byte(value), xvalue) {
2279 - t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue)
2280 - }
2281 - }
2282 - }(i, snap, rnd.Perm(nKey))
2283 -
2284 - go func(i int, snap *Snapshot) {
2285 - defer wg.Done()
2286 -
2287 - vtail := fmt.Sprintf("VAL%030d", i)
2288 - iter := snap.NewIterator(nil, nil)
2289 - defer iter.Release()
2290 - for k := 0; k < nKey; k++ {
2291 - if !iter.Next() {
2292 - if err := iter.Error(); err != nil {
2293 - t.Fatalf("READER_ITER #%d K%d error: %v", i, k, err)
2294 - } else {
2295 - t.Fatalf("READER_ITER #%d K%d eoi", i, k)
2296 - }
2297 - }
2298 - key := fmt.Sprintf("KEY%8d", k)
2299 - xkey := iter.Key()
2300 - if !bytes.Equal([]byte(key), xkey) {
2301 - t.Fatalf("READER_ITER #%d K%d invalid key: want %q, got %q", i, k, key, xkey)
2302 - }
2303 - value := key + vtail
2304 - xvalue := iter.Value()
2305 - if !bytes.Equal([]byte(value), xvalue) {
2306 - t.Fatalf("READER_ITER #%d K%d invalid value: want %q, got %q", i, k, value, xvalue)
2307 - }
2308 - }
2309 - }(i, snap)
2310 - }
2311 -
2312 - wg.Wait()
2313 -}
2314 -
2315 -func TestDB_UkeyShouldntHopAcrossTable(t *testing.T) {
2316 - h := newDbHarnessWopt(t, &opt.Options{
2317 - WriteBuffer: 112 * opt.KiB,
2318 - CompactionTableSize: 90 * opt.KiB,
2319 - CompactionExpandLimitFactor: 1,
2320 - })
2321 - defer h.close()
2322 -
2323 - const (
2324 - nSnap = 190
2325 - nKey = 140
2326 - )
2327 -
2328 - var (
2329 - snaps [nSnap]*Snapshot
2330 - b = &Batch{}
2331 - )
2332 - for i := range snaps {
2333 - vtail := fmt.Sprintf("VAL%030d", i)
2334 - b.Reset()
2335 - for k := 0; k < nKey; k++ {
2336 - key := fmt.Sprintf("KEY%08d", k)
2337 - b.Put([]byte(key), []byte(key+vtail))
2338 - }
2339 - if err := h.db.Write(b, nil); err != nil {
2340 - t.Fatalf("WRITE #%d error: %v", i, err)
2341 - }
2342 -
2343 - snaps[i] = h.db.newSnapshot()
2344 - b.Reset()
2345 - for k := 0; k < nKey; k++ {
2346 - key := fmt.Sprintf("KEY%08d", k)
2347 - b.Delete([]byte(key))
2348 - }
2349 - if err := h.db.Write(b, nil); err != nil {
2350 - t.Fatalf("WRITE #%d error: %v", i, err)
2351 - }
2352 - }
2353 -
2354 - h.compactMem()
2355 -
2356 - h.waitCompaction()
2357 - for level, tables := range h.db.s.stVersion.tables {
2358 - for _, table := range tables {
2359 - t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2360 - }
2361 - }
2362 -
2363 - h.compactRangeAt(0, "", "")
2364 - h.waitCompaction()
2365 - for level, tables := range h.db.s.stVersion.tables {
2366 - for _, table := range tables {
2367 - t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2368 - }
2369 - }
2370 - h.compactRangeAt(1, "", "")
2371 - h.waitCompaction()
2372 - for level, tables := range h.db.s.stVersion.tables {
2373 - for _, table := range tables {
2374 - t.Logf("L%d@%d %q:%q", level, table.file.Num(), table.imin, table.imax)
2375 - }
2376 - }
2377 - runtime.GOMAXPROCS(runtime.NumCPU())
2378 -
2379 - wg := &sync.WaitGroup{}
2380 - for i, snap := range snaps {
2381 - wg.Add(1)
2382 -
2383 - go func(i int, snap *Snapshot) {
2384 - defer wg.Done()
2385 -
2386 - vtail := fmt.Sprintf("VAL%030d", i)
2387 - for k := 0; k < nKey; k++ {
2388 - key := fmt.Sprintf("KEY%08d", k)
2389 - xvalue, err := snap.Get([]byte(key), nil)
2390 - if err != nil {
2391 - t.Fatalf("READER_GET #%d SEQ=%d K%d error: %v", i, snap.elem.seq, k, err)
2392 - }
2393 - value := key + vtail
2394 - if !bytes.Equal([]byte(value), xvalue) {
2395 - t.Fatalf("READER_GET #%d SEQ=%d K%d invalid value: want %q, got %q", i, snap.elem.seq, k, value, xvalue)
2396 - }
2397 - }
2398 - }(i, snap)
2399 - }
2400 -
2401 - wg.Wait()
2402 -}
2403 -
2404 -func TestDB_TableCompactionBuilder(t *testing.T) {
2405 - stor := newTestStorage(t)
2406 - defer stor.Close()
2407 -
2408 - const nSeq = 99
2409 -
2410 - o := &opt.Options{
2411 - WriteBuffer: 112 * opt.KiB,
2412 - CompactionTableSize: 43 * opt.KiB,
2413 - CompactionExpandLimitFactor: 1,
2414 - CompactionGPOverlapsFactor: 1,
2415 - DisableBlockCache: true,
2416 - }
2417 - s, err := newSession(stor, o)
2418 - if err != nil {
2419 - t.Fatal(err)
2420 - }
2421 - if err := s.create(); err != nil {
2422 - t.Fatal(err)
2423 - }
2424 - defer s.close()
2425 - var (
2426 - seq uint64
2427 - targetSize = 5 * o.CompactionTableSize
2428 - value = bytes.Repeat([]byte{'0'}, 100)
2429 - )
2430 - for i := 0; i < 2; i++ {
2431 - tw, err := s.tops.create()
2432 - if err != nil {
2433 - t.Fatal(err)
2434 - }
2435 - for k := 0; tw.tw.BytesLen() < targetSize; k++ {
2436 - key := []byte(fmt.Sprintf("%09d", k))
2437 - seq += nSeq - 1
2438 - for x := uint64(0); x < nSeq; x++ {
2439 - if err := tw.append(newIkey(key, seq-x, ktVal), value); err != nil {
2440 - t.Fatal(err)
2441 - }
2442 - }
2443 - }
2444 - tf, err := tw.finish()
2445 - if err != nil {
2446 - t.Fatal(err)
2447 - }
2448 - rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
2449 - rec.addTableFile(i, tf)
2450 - if err := s.commit(rec); err != nil {
2451 - t.Fatal(err)
2452 - }
2453 - }
2454 -
2455 - // Build grandparent.
2456 - v := s.version()
2457 - c := newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
2458 - rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
2459 - b := &tableCompactionBuilder{
2460 - s: s,
2461 - c: c,
2462 - rec: rec,
2463 - stat1: new(cStatsStaging),
2464 - minSeq: 0,
2465 - strict: true,
2466 - tableSize: o.CompactionTableSize/3 + 961,
2467 - }
2468 - if err := b.run(new(compactionTransactCounter)); err != nil {
2469 - t.Fatal(err)
2470 - }
2471 - for _, t := range c.tables[0] {
2472 - rec.delTable(c.level, t.file.Num())
2473 - }
2474 - if err := s.commit(rec); err != nil {
2475 - t.Fatal(err)
2476 - }
2477 - c.release()
2478 -
2479 - // Build level-1.
2480 - v = s.version()
2481 - c = newCompaction(s, v, 0, append(tFiles{}, v.tables[0]...))
2482 - rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
2483 - b = &tableCompactionBuilder{
2484 - s: s,
2485 - c: c,
2486 - rec: rec,
2487 - stat1: new(cStatsStaging),
2488 - minSeq: 0,
2489 - strict: true,
2490 - tableSize: o.CompactionTableSize,
2491 - }
2492 - if err := b.run(new(compactionTransactCounter)); err != nil {
2493 - t.Fatal(err)
2494 - }
2495 - for _, t := range c.tables[0] {
2496 - rec.delTable(c.level, t.file.Num())
2497 - }
2498 - // Move grandparent to level-3
2499 - for _, t := range v.tables[2] {
2500 - rec.delTable(2, t.file.Num())
2501 - rec.addTableFile(3, t)
2502 - }
2503 - if err := s.commit(rec); err != nil {
2504 - t.Fatal(err)
2505 - }
2506 - c.release()
2507 -
2508 - v = s.version()
2509 - for level, want := range []bool{false, true, false, true, false} {
2510 - got := len(v.tables[level]) > 0
2511 - if want != got {
2512 - t.Fatalf("invalid level-%d tables len: want %v, got %v", level, want, got)
2513 - }
2514 - }
2515 - for i, f := range v.tables[1][:len(v.tables[1])-1] {
2516 - nf := v.tables[1][i+1]
2517 - if bytes.Equal(f.imax.ukey(), nf.imin.ukey()) {
2518 - t.Fatalf("KEY %q hop across table %d .. %d", f.imax.ukey(), f.file.Num(), nf.file.Num())
2519 - }
2520 - }
2521 - v.release()
2522 -
2523 - // Compaction with transient error.
2524 - v = s.version()
2525 - c = newCompaction(s, v, 1, append(tFiles{}, v.tables[1]...))
2526 - rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
2527 - b = &tableCompactionBuilder{
2528 - s: s,
2529 - c: c,
2530 - rec: rec,
2531 - stat1: new(cStatsStaging),
2532 - minSeq: 0,
2533 - strict: true,
2534 - tableSize: o.CompactionTableSize,
2535 - }
2536 - stor.SetEmuErrOnce(storage.TypeTable, tsOpSync)
2537 - stor.SetEmuRandErr(storage.TypeTable, tsOpRead, tsOpReadAt, tsOpWrite)
2538 - stor.SetEmuRandErrProb(0xf0)
2539 - for {
2540 - if err := b.run(new(compactionTransactCounter)); err != nil {
2541 - t.Logf("(expected) b.run: %v", err)
2542 - } else {
2543 - break
2544 - }
2545 - }
2546 - if err := s.commit(rec); err != nil {
2547 - t.Fatal(err)
2548 - }
2549 - c.release()
2550 -
2551 - stor.SetEmuErrOnce(0, tsOpSync)
2552 - stor.SetEmuRandErr(0, tsOpRead, tsOpReadAt, tsOpWrite)
2553 -
2554 - v = s.version()
2555 - if len(v.tables[1]) != len(v.tables[2]) {
2556 - t.Fatalf("invalid tables length, want %d, got %d", len(v.tables[1]), len(v.tables[2]))
2557 - }
2558 - for i, f0 := range v.tables[1] {
2559 - f1 := v.tables[2][i]
2560 - iter0 := s.tops.newIterator(f0, nil, nil)
2561 - iter1 := s.tops.newIterator(f1, nil, nil)
2562 - for j := 0; true; j++ {
2563 - next0 := iter0.Next()
2564 - next1 := iter1.Next()
2565 - if next0 != next1 {
2566 - t.Fatalf("#%d.%d invalid eoi: want %v, got %v", i, j, next0, next1)
2567 - }
2568 - key0 := iter0.Key()
2569 - key1 := iter1.Key()
2570 - if !bytes.Equal(key0, key1) {
2571 - t.Fatalf("#%d.%d invalid key: want %q, got %q", i, j, key0, key1)
2572 - }
2573 - if next0 == false {
2574 - break
2575 - }
2576 - }
2577 - iter0.Release()
2578 - iter1.Release()
2579 - }
2580 - v.release()
2581 -}
2582 -
2583 -func testDB_IterTriggeredCompaction(t *testing.T, limitDiv int) {
2584 - const (
2585 - vSize = 200 * opt.KiB
2586 - tSize = 100 * opt.MiB
2587 - mIter = 100
2588 - n = tSize / vSize
2589 - )
2590 -
2591 - h := newDbHarnessWopt(t, &opt.Options{
2592 - Compression: opt.NoCompression,
2593 - DisableBlockCache: true,
2594 - })
2595 - defer h.close()
2596 -
2597 - key := func(x int) string {
2598 - return fmt.Sprintf("v%06d", x)
2599 - }
2600 -
2601 - // Fill.
2602 - value := strings.Repeat("x", vSize)
2603 - for i := 0; i < n; i++ {
2604 - h.put(key(i), value)
2605 - }
2606 - h.compactMem()
2607 -
2608 - // Delete all.
2609 - for i := 0; i < n; i++ {
2610 - h.delete(key(i))
2611 - }
2612 - h.compactMem()
2613 -
2614 - var (
2615 - limit = n / limitDiv
2616 -
2617 - startKey = key(0)
2618 - limitKey = key(limit)
2619 - maxKey = key(n)
2620 - slice = &util.Range{Limit: []byte(limitKey)}
2621 -
2622 - initialSize0 = h.sizeOf(startKey, limitKey)
2623 - initialSize1 = h.sizeOf(limitKey, maxKey)
2624 - )
2625 -
2626 - t.Logf("inital size %s [rest %s]", shortenb(int(initialSize0)), shortenb(int(initialSize1)))
2627 -
2628 - for r := 0; true; r++ {
2629 - if r >= mIter {
2630 - t.Fatal("taking too long to compact")
2631 - }
2632 -
2633 - // Iterates.
2634 - iter := h.db.NewIterator(slice, h.ro)
2635 - for iter.Next() {
2636 - }
2637 - if err := iter.Error(); err != nil {
2638 - t.Fatalf("Iter err: %v", err)
2639 - }
2640 - iter.Release()
2641 -
2642 - // Wait compaction.
2643 - h.waitCompaction()
2644 -
2645 - // Check size.
2646 - size0 := h.sizeOf(startKey, limitKey)
2647 - size1 := h.sizeOf(limitKey, maxKey)
2648 - t.Logf("#%03d size %s [rest %s]", r, shortenb(int(size0)), shortenb(int(size1)))
2649 - if size0 < initialSize0/10 {
2650 - break
2651 - }
2652 - }
2653 -
2654 - if initialSize1 > 0 {
2655 - h.sizeAssert(limitKey, maxKey, initialSize1/4-opt.MiB, initialSize1+opt.MiB)
2656 - }
2657 -}
2658 -
2659 -func TestDB_IterTriggeredCompaction(t *testing.T) {
2660 - testDB_IterTriggeredCompaction(t, 1)
2661 -}
2662 -
2663 -func TestDB_IterTriggeredCompactionHalf(t *testing.T) {
2664 - testDB_IterTriggeredCompaction(t, 2)
2665 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_util.go deleted
-100
@@ -1,100 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
15 -)
16 -
17 -// Reader is the interface that wraps basic Get and NewIterator methods.
18 -// This interface implemented by both DB and Snapshot.
19 -type Reader interface {
20 - Get(key []byte, ro *opt.ReadOptions) (value []byte, err error)
21 - NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator
22 -}
23 -
24 -type Sizes []uint64
25 -
26 -// Sum returns sum of the sizes.
27 -func (p Sizes) Sum() (n uint64) {
28 - for _, s := range p {
29 - n += s
30 - }
31 - return n
32 -}
33 -
34 -// Logging.
35 -func (db *DB) log(v ...interface{}) { db.s.log(v...) }
36 -func (db *DB) logf(format string, v ...interface{}) { db.s.logf(format, v...) }
37 -
38 -// Check and clean files.
39 -func (db *DB) checkAndCleanFiles() error {
40 - v := db.s.version()
41 - defer v.release()
42 -
43 - tablesMap := make(map[uint64]bool)
44 - for _, tables := range v.tables {
45 - for _, t := range tables {
46 - tablesMap[t.file.Num()] = false
47 - }
48 - }
49 -
50 - files, err := db.s.getFiles(storage.TypeAll)
51 - if err != nil {
52 - return err
53 - }
54 -
55 - var nTables int
56 - var rem []storage.File
57 - for _, f := range files {
58 - keep := true
59 - switch f.Type() {
60 - case storage.TypeManifest:
61 - keep = f.Num() >= db.s.manifestFile.Num()
62 - case storage.TypeJournal:
63 - if db.frozenJournalFile != nil {
64 - keep = f.Num() >= db.frozenJournalFile.Num()
65 - } else {
66 - keep = f.Num() >= db.journalFile.Num()
67 - }
68 - case storage.TypeTable:
69 - _, keep = tablesMap[f.Num()]
70 - if keep {
71 - tablesMap[f.Num()] = true
72 - nTables++
73 - }
74 - }
75 -
76 - if !keep {
77 - rem = append(rem, f)
78 - }
79 - }
80 -
81 - if nTables != len(tablesMap) {
82 - var missing []*storage.FileInfo
83 - for num, present := range tablesMap {
84 - if !present {
85 - missing = append(missing, &storage.FileInfo{Type: storage.TypeTable, Num: num})
86 - db.logf("db@janitor table missing @%d", num)
87 - }
88 - }
89 - return errors.NewErrCorrupted(nil, &errors.ErrMissingFiles{Files: missing})
90 - }
91 -
92 - db.logf("db@janitor F·%d G·%d", len(files), len(rem))
93 - for _, f := range rem {
94 - db.logf("db@janitor removing %s-%d", f.Type(), f.Num())
95 - if err := f.Remove(); err != nil {
96 - return err
97 - }
98 - }
99 - return nil
100 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/db_write.go deleted
-311
@@ -1,311 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "time"
11 -
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
15 -)
16 -
17 -func (db *DB) writeJournal(b *Batch) error {
18 - w, err := db.journal.Next()
19 - if err != nil {
20 - return err
21 - }
22 - if _, err := w.Write(b.encode()); err != nil {
23 - return err
24 - }
25 - if err := db.journal.Flush(); err != nil {
26 - return err
27 - }
28 - if b.sync {
29 - return db.journalWriter.Sync()
30 - }
31 - return nil
32 -}
33 -
34 -func (db *DB) jWriter() {
35 - defer db.closeW.Done()
36 - for {
37 - select {
38 - case b := <-db.journalC:
39 - if b != nil {
40 - db.journalAckC <- db.writeJournal(b)
41 - }
42 - case _, _ = <-db.closeC:
43 - return
44 - }
45 - }
46 -}
47 -
48 -func (db *DB) rotateMem(n int) (mem *memDB, err error) {
49 - // Wait for pending memdb compaction.
50 - err = db.compSendIdle(db.mcompCmdC)
51 - if err != nil {
52 - return
53 - }
54 -
55 - // Create new memdb and journal.
56 - mem, err = db.newMem(n)
57 - if err != nil {
58 - return
59 - }
60 -
61 - // Schedule memdb compaction.
62 - db.compSendTrigger(db.mcompCmdC)
63 - return
64 -}
65 -
66 -func (db *DB) flush(n int) (mem *memDB, nn int, err error) {
67 - delayed := false
68 - flush := func() (retry bool) {
69 - v := db.s.version()
70 - defer v.release()
71 - mem = db.getEffectiveMem()
72 - defer func() {
73 - if retry {
74 - mem.decref()
75 - mem = nil
76 - }
77 - }()
78 - nn = mem.mdb.Free()
79 - switch {
80 - case v.tLen(0) >= db.s.o.GetWriteL0SlowdownTrigger() && !delayed:
81 - delayed = true
82 - time.Sleep(time.Millisecond)
83 - case nn >= n:
84 - return false
85 - case v.tLen(0) >= db.s.o.GetWriteL0PauseTrigger():
86 - delayed = true
87 - err = db.compSendIdle(db.tcompCmdC)
88 - if err != nil {
89 - return false
90 - }
91 - default:
92 - // Allow memdb to grow if it has no entry.
93 - if mem.mdb.Len() == 0 {
94 - nn = n
95 - } else {
96 - mem.decref()
97 - mem, err = db.rotateMem(n)
98 - if err == nil {
99 - nn = mem.mdb.Free()
100 - } else {
101 - nn = 0
102 - }
103 - }
104 - return false
105 - }
106 - return true
107 - }
108 - start := time.Now()
109 - for flush() {
110 - }
111 - if delayed {
112 - db.writeDelay += time.Since(start)
113 - db.writeDelayN++
114 - } else if db.writeDelayN > 0 {
115 - db.logf("db@write was delayed N·%d T·%v", db.writeDelayN, db.writeDelay)
116 - db.writeDelay = 0
117 - db.writeDelayN = 0
118 - }
119 - return
120 -}
121 -
122 -// Write apply the given batch to the DB. The batch will be applied
123 -// sequentially.
124 -//
125 -// It is safe to modify the contents of the arguments after Write returns.
126 -func (db *DB) Write(b *Batch, wo *opt.WriteOptions) (err error) {
127 - err = db.ok()
128 - if err != nil || b == nil || b.Len() == 0 {
129 - return
130 - }
131 -
132 - b.init(wo.GetSync())
133 -
134 - // The write happen synchronously.
135 - select {
136 - case db.writeC <- b:
137 - if <-db.writeMergedC {
138 - return <-db.writeAckC
139 - }
140 - case db.writeLockC <- struct{}{}:
141 - case err = <-db.compPerErrC:
142 - return
143 - case _, _ = <-db.closeC:
144 - return ErrClosed
145 - }
146 -
147 - merged := 0
148 - danglingMerge := false
149 - defer func() {
150 - if danglingMerge {
151 - db.writeMergedC <- false
152 - } else {
153 - <-db.writeLockC
154 - }
155 - for i := 0; i < merged; i++ {
156 - db.writeAckC <- err
157 - }
158 - }()
159 -
160 - mem, memFree, err := db.flush(b.size())
161 - if err != nil {
162 - return
163 - }
164 - defer mem.decref()
165 -
166 - // Calculate maximum size of the batch.
167 - m := 1 << 20
168 - if x := b.size(); x <= 128<<10 {
169 - m = x + (128 << 10)
170 - }
171 - m = minInt(m, memFree)
172 -
173 - // Merge with other batch.
174 -drain:
175 - for b.size() < m && !b.sync {
176 - select {
177 - case nb := <-db.writeC:
178 - if b.size()+nb.size() <= m {
179 - b.append(nb)
180 - db.writeMergedC <- true
181 - merged++
182 - } else {
183 - danglingMerge = true
184 - break drain
185 - }
186 - default:
187 - break drain
188 - }
189 - }
190 -
191 - // Set batch first seq number relative from last seq.
192 - b.seq = db.seq + 1
193 -
194 - // Write journal concurrently if it is large enough.
195 - if b.size() >= (128 << 10) {
196 - // Push the write batch to the journal writer
197 - select {
198 - case db.journalC <- b:
199 - // Write into memdb
200 - if berr := b.memReplay(mem.mdb); berr != nil {
201 - panic(berr)
202 - }
203 - case err = <-db.compPerErrC:
204 - return
205 - case _, _ = <-db.closeC:
206 - err = ErrClosed
207 - return
208 - }
209 - // Wait for journal writer
210 - select {
211 - case err = <-db.journalAckC:
212 - if err != nil {
213 - // Revert memdb if error detected
214 - if berr := b.revertMemReplay(mem.mdb); berr != nil {
215 - panic(berr)
216 - }
217 - return
218 - }
219 - case _, _ = <-db.closeC:
220 - err = ErrClosed
221 - return
222 - }
223 - } else {
224 - err = db.writeJournal(b)
225 - if err != nil {
226 - return
227 - }
228 - if berr := b.memReplay(mem.mdb); berr != nil {
229 - panic(berr)
230 - }
231 - }
232 -
233 - // Set last seq number.
234 - db.addSeq(uint64(b.Len()))
235 -
236 - if b.size() >= memFree {
237 - db.rotateMem(0)
238 - }
239 - return
240 -}
241 -
242 -// Put sets the value for the given key. It overwrites any previous value
243 -// for that key; a DB is not a multi-map.
244 -//
245 -// It is safe to modify the contents of the arguments after Put returns.
246 -func (db *DB) Put(key, value []byte, wo *opt.WriteOptions) error {
247 - b := new(Batch)
248 - b.Put(key, value)
249 - return db.Write(b, wo)
250 -}
251 -
252 -// Delete deletes the value for the given key. It returns ErrNotFound if
253 -// the DB does not contain the key.
254 -//
255 -// It is safe to modify the contents of the arguments after Delete returns.
256 -func (db *DB) Delete(key []byte, wo *opt.WriteOptions) error {
257 - b := new(Batch)
258 - b.Delete(key)
259 - return db.Write(b, wo)
260 -}
261 -
262 -func isMemOverlaps(icmp *iComparer, mem *memdb.DB, min, max []byte) bool {
263 - iter := mem.NewIterator(nil)
264 - defer iter.Release()
265 - return (max == nil || (iter.First() && icmp.uCompare(max, iKey(iter.Key()).ukey()) >= 0)) &&
266 - (min == nil || (iter.Last() && icmp.uCompare(min, iKey(iter.Key()).ukey()) <= 0))
267 -}
268 -
269 -// CompactRange compacts the underlying DB for the given key range.
270 -// In particular, deleted and overwritten versions are discarded,
271 -// and the data is rearranged to reduce the cost of operations
272 -// needed to access the data. This operation should typically only
273 -// be invoked by users who understand the underlying implementation.
274 -//
275 -// A nil Range.Start is treated as a key before all keys in the DB.
276 -// And a nil Range.Limit is treated as a key after all keys in the DB.
277 -// Therefore if both is nil then it will compact entire DB.
278 -func (db *DB) CompactRange(r util.Range) error {
279 - if err := db.ok(); err != nil {
280 - return err
281 - }
282 -
283 - // Lock writer.
284 - select {
285 - case db.writeLockC <- struct{}{}:
286 - case err := <-db.compPerErrC:
287 - return err
288 - case _, _ = <-db.closeC:
289 - return ErrClosed
290 - }
291 -
292 - // Check for overlaps in memdb.
293 - mem := db.getEffectiveMem()
294 - defer mem.decref()
295 - if isMemOverlaps(db.s.icmp, mem.mdb, r.Start, r.Limit) {
296 - // Memdb compaction.
297 - if _, err := db.rotateMem(0); err != nil {
298 - <-db.writeLockC
299 - return err
300 - }
301 - <-db.writeLockC
302 - if err := db.compSendIdle(db.mcompCmdC); err != nil {
303 - return err
304 - }
305 - } else {
306 - <-db.writeLockC
307 - }
308 -
309 - // Table compaction.
310 - return db.compSendRange(db.tcompCmdC, -1, r.Start, r.Limit)
311 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/doc.go deleted
-90
@@ -1,90 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package leveldb provides implementation of LevelDB key/value database.
8 -//
9 -// Create or open a database:
10 -//
11 -// db, err := leveldb.OpenFile("path/to/db", nil)
12 -// ...
13 -// defer db.Close()
14 -// ...
15 -//
16 -// Read or modify the database content:
17 -//
18 -// // Remember that the contents of the returned slice should not be modified.
19 -// data, err := db.Get([]byte("key"), nil)
20 -// ...
21 -// err = db.Put([]byte("key"), []byte("value"), nil)
22 -// ...
23 -// err = db.Delete([]byte("key"), nil)
24 -// ...
25 -//
26 -// Iterate over database content:
27 -//
28 -// iter := db.NewIterator(nil, nil)
29 -// for iter.Next() {
30 -// // Remember that the contents of the returned slice should not be modified, and
31 -// // only valid until the next call to Next.
32 -// key := iter.Key()
33 -// value := iter.Value()
34 -// ...
35 -// }
36 -// iter.Release()
37 -// err = iter.Error()
38 -// ...
39 -//
40 -// Iterate over subset of database content with a particular prefix:
41 -// iter := db.NewIterator(util.BytesPrefix([]byte("foo-")), nil)
42 -// for iter.Next() {
43 -// // Use key/value.
44 -// ...
45 -// }
46 -// iter.Release()
47 -// err = iter.Error()
48 -// ...
49 -//
50 -// Seek-then-Iterate:
51 -//
52 -// iter := db.NewIterator(nil, nil)
53 -// for ok := iter.Seek(key); ok; ok = iter.Next() {
54 -// // Use key/value.
55 -// ...
56 -// }
57 -// iter.Release()
58 -// err = iter.Error()
59 -// ...
60 -//
61 -// Iterate over subset of database content:
62 -//
63 -// iter := db.NewIterator(&util.Range{Start: []byte("foo"), Limit: []byte("xoo")}, nil)
64 -// for iter.Next() {
65 -// // Use key/value.
66 -// ...
67 -// }
68 -// iter.Release()
69 -// err = iter.Error()
70 -// ...
71 -//
72 -// Batch writes:
73 -//
74 -// batch := new(leveldb.Batch)
75 -// batch.Put([]byte("foo"), []byte("value"))
76 -// batch.Put([]byte("bar"), []byte("another value"))
77 -// batch.Delete([]byte("baz"))
78 -// err = db.Write(batch, nil)
79 -// ...
80 -//
81 -// Use bloom filter:
82 -//
83 -// o := &opt.Options{
84 -// Filter: filter.NewBloomFilter(10),
85 -// }
86 -// db, err := leveldb.OpenFile("path/to/db", o)
87 -// ...
88 -// defer db.Close()
89 -// ...
90 -package leveldb
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors.go deleted
-18
@@ -1,18 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11 -)
12 -
13 -var (
14 - ErrNotFound = errors.ErrNotFound
15 - ErrSnapshotReleased = errors.New("leveldb: snapshot released")
16 - ErrIterReleased = errors.New("leveldb: iterator released")
17 - ErrClosed = errors.New("leveldb: closed")
18 -)
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors/errors.go deleted
-76
@@ -1,76 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package errors provides common error types used throughout leveldb.
8 -package errors
9 -
10 -import (
11 - "errors"
12 - "fmt"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -var (
19 - ErrNotFound = New("leveldb: not found")
20 - ErrReleased = util.ErrReleased
21 - ErrHasReleaser = util.ErrHasReleaser
22 -)
23 -
24 -// New returns an error that formats as the given text.
25 -func New(text string) error {
26 - return errors.New(text)
27 -}
28 -
29 -// ErrCorrupted is the type that wraps errors that indicate corruption in
30 -// the database.
31 -type ErrCorrupted struct {
32 - File *storage.FileInfo
33 - Err error
34 -}
35 -
36 -func (e *ErrCorrupted) Error() string {
37 - if e.File != nil {
38 - return fmt.Sprintf("%v [file=%v]", e.Err, e.File)
39 - } else {
40 - return e.Err.Error()
41 - }
42 -}
43 -
44 -// NewErrCorrupted creates new ErrCorrupted error.
45 -func NewErrCorrupted(f storage.File, err error) error {
46 - return &ErrCorrupted{storage.NewFileInfo(f), err}
47 -}
48 -
49 -// IsCorrupted returns a boolean indicating whether the error is indicating
50 -// a corruption.
51 -func IsCorrupted(err error) bool {
52 - switch err.(type) {
53 - case *ErrCorrupted:
54 - return true
55 - }
56 - return false
57 -}
58 -
59 -// ErrMissingFiles is the type that indicating a corruption due to missing
60 -// files.
61 -type ErrMissingFiles struct {
62 - Files []*storage.FileInfo
63 -}
64 -
65 -func (e *ErrMissingFiles) Error() string { return "file missing" }
66 -
67 -// SetFile sets 'file info' of the given error with the given file.
68 -// Currently only ErrCorrupted is supported, otherwise will do nothing.
69 -func SetFile(err error, f storage.File) error {
70 - switch x := err.(type) {
71 - case *ErrCorrupted:
72 - x.File = storage.NewFileInfo(f)
73 - return x
74 - }
75 - return err
76 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/external_test.go deleted
-58
@@ -1,58 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - . "github.com/onsi/ginkgo"
11 - . "github.com/onsi/gomega"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
15 -)
16 -
17 -var _ = testutil.Defer(func() {
18 - Describe("Leveldb external", func() {
19 - o := &opt.Options{
20 - DisableBlockCache: true,
21 - BlockRestartInterval: 5,
22 - BlockSize: 80,
23 - Compression: opt.NoCompression,
24 - OpenFilesCacheCapacity: -1,
25 - Strict: opt.StrictAll,
26 - WriteBuffer: 1000,
27 - CompactionTableSize: 2000,
28 - }
29 -
30 - Describe("write test", func() {
31 - It("should do write correctly", func(done Done) {
32 - db := newTestingDB(o, nil, nil)
33 - t := testutil.DBTesting{
34 - DB: db,
35 - Deleted: testutil.KeyValue_Generate(nil, 500, 1, 50, 5, 5).Clone(),
36 - }
37 - testutil.DoDBTesting(&t)
38 - db.TestClose()
39 - done <- true
40 - }, 20.0)
41 - })
42 -
43 - Describe("read test", func() {
44 - testutil.AllKeyValueTesting(nil, nil, func(kv testutil.KeyValue) testutil.DB {
45 - // Building the DB.
46 - db := newTestingDB(o, nil, nil)
47 - kv.IterateShuffled(nil, func(i int, key, value []byte) {
48 - err := db.TestPut(key, value)
49 - Expect(err).NotTo(HaveOccurred())
50 - })
51 -
52 - return db
53 - }, func(db testutil.DB) {
54 - db.(*testingDB).TestClose()
55 - })
56 - })
57 - })
58 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter.go deleted
-31
@@ -1,31 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
11 -)
12 -
13 -type iFilter struct {
14 - filter.Filter
15 -}
16 -
17 -func (f iFilter) Contains(filter, key []byte) bool {
18 - return f.Filter.Contains(filter, iKey(key).ukey())
19 -}
20 -
21 -func (f iFilter) NewGenerator() filter.FilterGenerator {
22 - return iFilterGenerator{f.Filter.NewGenerator()}
23 -}
24 -
25 -type iFilterGenerator struct {
26 - filter.FilterGenerator
27 -}
28 -
29 -func (g iFilterGenerator) Add(key []byte) {
30 - g.FilterGenerator.Add(iKey(key).ukey())
31 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/bloom.go deleted
-116
@@ -1,116 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package filter
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
11 -)
12 -
13 -func bloomHash(key []byte) uint32 {
14 - return util.Hash(key, 0xbc9f1d34)
15 -}
16 -
17 -type bloomFilter int
18 -
19 -// The bloom filter serializes its parameters and is backward compatible
20 -// with respect to them. Therefor, its parameters are not added to its
21 -// name.
22 -func (bloomFilter) Name() string {
23 - return "leveldb.BuiltinBloomFilter"
24 -}
25 -
26 -func (f bloomFilter) Contains(filter, key []byte) bool {
27 - nBytes := len(filter) - 1
28 - if nBytes < 1 {
29 - return false
30 - }
31 - nBits := uint32(nBytes * 8)
32 -
33 - // Use the encoded k so that we can read filters generated by
34 - // bloom filters created using different parameters.
35 - k := filter[nBytes]
36 - if k > 30 {
37 - // Reserved for potentially new encodings for short bloom filters.
38 - // Consider it a match.
39 - return true
40 - }
41 -
42 - kh := bloomHash(key)
43 - delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
44 - for j := uint8(0); j < k; j++ {
45 - bitpos := kh % nBits
46 - if (uint32(filter[bitpos/8]) & (1 << (bitpos % 8))) == 0 {
47 - return false
48 - }
49 - kh += delta
50 - }
51 - return true
52 -}
53 -
54 -func (f bloomFilter) NewGenerator() FilterGenerator {
55 - // Round down to reduce probing cost a little bit.
56 - k := uint8(f * 69 / 100) // 0.69 =~ ln(2)
57 - if k < 1 {
58 - k = 1
59 - } else if k > 30 {
60 - k = 30
61 - }
62 - return &bloomFilterGenerator{
63 - n: int(f),
64 - k: k,
65 - }
66 -}
67 -
68 -type bloomFilterGenerator struct {
69 - n int
70 - k uint8
71 -
72 - keyHashes []uint32
73 -}
74 -
75 -func (g *bloomFilterGenerator) Add(key []byte) {
76 - // Use double-hashing to generate a sequence of hash values.
77 - // See analysis in [Kirsch,Mitzenmacher 2006].
78 - g.keyHashes = append(g.keyHashes, bloomHash(key))
79 -}
80 -
81 -func (g *bloomFilterGenerator) Generate(b Buffer) {
82 - // Compute bloom filter size (in both bits and bytes)
83 - nBits := uint32(len(g.keyHashes) * g.n)
84 - // For small n, we can see a very high false positive rate. Fix it
85 - // by enforcing a minimum bloom filter length.
86 - if nBits < 64 {
87 - nBits = 64
88 - }
89 - nBytes := (nBits + 7) / 8
90 - nBits = nBytes * 8
91 -
92 - dest := b.Alloc(int(nBytes) + 1)
93 - dest[nBytes] = g.k
94 - for _, kh := range g.keyHashes {
95 - delta := (kh >> 17) | (kh << 15) // Rotate right 17 bits
96 - for j := uint8(0); j < g.k; j++ {
97 - bitpos := kh % nBits
98 - dest[bitpos/8] |= (1 << (bitpos % 8))
99 - kh += delta
100 - }
101 - }
102 -
103 - g.keyHashes = g.keyHashes[:0]
104 -}
105 -
106 -// NewBloomFilter creates a new initialized bloom filter for given
107 -// bitsPerKey.
108 -//
109 -// Since bitsPerKey is persisted individually for each bloom filter
110 -// serialization, bloom filters are backwards compatible with respect to
111 -// changing bitsPerKey. This means that no big performance penalty will
112 -// be experienced when changing the parameter. See documentation for
113 -// opt.Options.Filter for more information.
114 -func NewBloomFilter(bitsPerKey int) Filter {
115 - return bloomFilter(bitsPerKey)
116 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/bloom_test.go deleted
-142
@@ -1,142 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package filter
8 -
9 -import (
10 - "encoding/binary"
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
12 - "testing"
13 -)
14 -
15 -type harness struct {
16 - t *testing.T
17 -
18 - bloom Filter
19 - generator FilterGenerator
20 - filter []byte
21 -}
22 -
23 -func newHarness(t *testing.T) *harness {
24 - bloom := NewBloomFilter(10)
25 - return &harness{
26 - t: t,
27 - bloom: bloom,
28 - generator: bloom.NewGenerator(),
29 - }
30 -}
31 -
32 -func (h *harness) add(key []byte) {
33 - h.generator.Add(key)
34 -}
35 -
36 -func (h *harness) addNum(key uint32) {
37 - var b [4]byte
38 - binary.LittleEndian.PutUint32(b[:], key)
39 - h.add(b[:])
40 -}
41 -
42 -func (h *harness) build() {
43 - b := &util.Buffer{}
44 - h.generator.Generate(b)
45 - h.filter = b.Bytes()
46 -}
47 -
48 -func (h *harness) reset() {
49 - h.filter = nil
50 -}
51 -
52 -func (h *harness) filterLen() int {
53 - return len(h.filter)
54 -}
55 -
56 -func (h *harness) assert(key []byte, want, silent bool) bool {
57 - got := h.bloom.Contains(h.filter, key)
58 - if !silent && got != want {
59 - h.t.Errorf("assert on '%v' failed got '%v', want '%v'", key, got, want)
60 - }
61 - return got
62 -}
63 -
64 -func (h *harness) assertNum(key uint32, want, silent bool) bool {
65 - var b [4]byte
66 - binary.LittleEndian.PutUint32(b[:], key)
67 - return h.assert(b[:], want, silent)
68 -}
69 -
70 -func TestBloomFilter_Empty(t *testing.T) {
71 - h := newHarness(t)
72 - h.build()
73 - h.assert([]byte("hello"), false, false)
74 - h.assert([]byte("world"), false, false)
75 -}
76 -
77 -func TestBloomFilter_Small(t *testing.T) {
78 - h := newHarness(t)
79 - h.add([]byte("hello"))
80 - h.add([]byte("world"))
81 - h.build()
82 - h.assert([]byte("hello"), true, false)
83 - h.assert([]byte("world"), true, false)
84 - h.assert([]byte("x"), false, false)
85 - h.assert([]byte("foo"), false, false)
86 -}
87 -
88 -func nextN(n int) int {
89 - switch {
90 - case n < 10:
91 - n += 1
92 - case n < 100:
93 - n += 10
94 - case n < 1000:
95 - n += 100
96 - default:
97 - n += 1000
98 - }
99 - return n
100 -}
101 -
102 -func TestBloomFilter_VaryingLengths(t *testing.T) {
103 - h := newHarness(t)
104 - var mediocre, good int
105 - for n := 1; n < 10000; n = nextN(n) {
106 - h.reset()
107 - for i := 0; i < n; i++ {
108 - h.addNum(uint32(i))
109 - }
110 - h.build()
111 -
112 - got := h.filterLen()
113 - want := (n * 10 / 8) + 40
114 - if got > want {
115 - t.Errorf("filter len test failed, '%d' > '%d'", got, want)
116 - }
117 -
118 - for i := 0; i < n; i++ {
119 - h.assertNum(uint32(i), true, false)
120 - }
121 -
122 - var rate float32
123 - for i := 0; i < 10000; i++ {
124 - if h.assertNum(uint32(i+1000000000), true, true) {
125 - rate++
126 - }
127 - }
128 - rate /= 10000
129 - if rate > 0.02 {
130 - t.Errorf("false positive rate is more than 2%%, got %v, at len %d", rate, n)
131 - }
132 - if rate > 0.0125 {
133 - mediocre++
134 - } else {
135 - good++
136 - }
137 - }
138 - t.Logf("false positive rate: %d good, %d mediocre", good, mediocre)
139 - if mediocre > good/5 {
140 - t.Error("mediocre false positive rate is more than expected")
141 - }
142 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter/filter.go deleted
-60
@@ -1,60 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package filter provides interface and implementation of probabilistic
8 -// data structure.
9 -//
10 -// The filter is resposible for creating small filter from a set of keys.
11 -// These filter will then used to test whether a key is a member of the set.
12 -// In many cases, a filter can cut down the number of disk seeks from a
13 -// handful to a single disk seek per DB.Get call.
14 -package filter
15 -
16 -// Buffer is the interface that wraps basic Alloc, Write and WriteByte methods.
17 -type Buffer interface {
18 - // Alloc allocs n bytes of slice from the buffer. This also advancing
19 - // write offset.
20 - Alloc(n int) []byte
21 -
22 - // Write appends the contents of p to the buffer.
23 - Write(p []byte) (n int, err error)
24 -
25 - // WriteByte appends the byte c to the buffer.
26 - WriteByte(c byte) error
27 -}
28 -
29 -// Filter is the filter.
30 -type Filter interface {
31 - // Name returns the name of this policy.
32 - //
33 - // Note that if the filter encoding changes in an incompatible way,
34 - // the name returned by this method must be changed. Otherwise, old
35 - // incompatible filters may be passed to methods of this type.
36 - Name() string
37 -
38 - // NewGenerator creates a new filter generator.
39 - NewGenerator() FilterGenerator
40 -
41 - // Contains returns true if the filter contains the given key.
42 - //
43 - // The filter are filters generated by the filter generator.
44 - Contains(filter, key []byte) bool
45 -}
46 -
47 -// FilterGenerator is the filter generator.
48 -type FilterGenerator interface {
49 - // Add adds a key to the filter generator.
50 - //
51 - // The key may become invalid after call to this method end, therefor
52 - // key must be copied if implementation require keeping key for later
53 - // use. The key should not modified directly, doing so may cause
54 - // undefined results.
55 - Add(key []byte)
56 -
57 - // Generate generates filters based on keys passed so far. After call
58 - // to Generate the filter generator maybe resetted, depends on implementation.
59 - Generate(b Buffer)
60 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter.go deleted
-184
@@ -1,184 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
11 -)
12 -
13 -// BasicArray is the interface that wraps basic Len and Search method.
14 -type BasicArray interface {
15 - // Len returns length of the array.
16 - Len() int
17 -
18 - // Search finds smallest index that point to a key that is greater
19 - // than or equal to the given key.
20 - Search(key []byte) int
21 -}
22 -
23 -// Array is the interface that wraps BasicArray and basic Index method.
24 -type Array interface {
25 - BasicArray
26 -
27 - // Index returns key/value pair with index of i.
28 - Index(i int) (key, value []byte)
29 -}
30 -
31 -// Array is the interface that wraps BasicArray and basic Get method.
32 -type ArrayIndexer interface {
33 - BasicArray
34 -
35 - // Get returns a new data iterator with index of i.
36 - Get(i int) Iterator
37 -}
38 -
39 -type basicArrayIterator struct {
40 - util.BasicReleaser
41 - array BasicArray
42 - pos int
43 - err error
44 -}
45 -
46 -func (i *basicArrayIterator) Valid() bool {
47 - return i.pos >= 0 && i.pos < i.array.Len() && !i.Released()
48 -}
49 -
50 -func (i *basicArrayIterator) First() bool {
51 - if i.Released() {
52 - i.err = ErrIterReleased
53 - return false
54 - }
55 -
56 - if i.array.Len() == 0 {
57 - i.pos = -1
58 - return false
59 - }
60 - i.pos = 0
61 - return true
62 -}
63 -
64 -func (i *basicArrayIterator) Last() bool {
65 - if i.Released() {
66 - i.err = ErrIterReleased
67 - return false
68 - }
69 -
70 - n := i.array.Len()
71 - if n == 0 {
72 - i.pos = 0
73 - return false
74 - }
75 - i.pos = n - 1
76 - return true
77 -}
78 -
79 -func (i *basicArrayIterator) Seek(key []byte) bool {
80 - if i.Released() {
81 - i.err = ErrIterReleased
82 - return false
83 - }
84 -
85 - n := i.array.Len()
86 - if n == 0 {
87 - i.pos = 0
88 - return false
89 - }
90 - i.pos = i.array.Search(key)
91 - if i.pos >= n {
92 - return false
93 - }
94 - return true
95 -}
96 -
97 -func (i *basicArrayIterator) Next() bool {
98 - if i.Released() {
99 - i.err = ErrIterReleased
100 - return false
101 - }
102 -
103 - i.pos++
104 - if n := i.array.Len(); i.pos >= n {
105 - i.pos = n
106 - return false
107 - }
108 - return true
109 -}
110 -
111 -func (i *basicArrayIterator) Prev() bool {
112 - if i.Released() {
113 - i.err = ErrIterReleased
114 - return false
115 - }
116 -
117 - i.pos--
118 - if i.pos < 0 {
119 - i.pos = -1
120 - return false
121 - }
122 - return true
123 -}
124 -
125 -func (i *basicArrayIterator) Error() error { return i.err }
126 -
127 -type arrayIterator struct {
128 - basicArrayIterator
129 - array Array
130 - pos int
131 - key, value []byte
132 -}
133 -
134 -func (i *arrayIterator) updateKV() {
135 - if i.pos == i.basicArrayIterator.pos {
136 - return
137 - }
138 - i.pos = i.basicArrayIterator.pos
139 - if i.Valid() {
140 - i.key, i.value = i.array.Index(i.pos)
141 - } else {
142 - i.key = nil
143 - i.value = nil
144 - }
145 -}
146 -
147 -func (i *arrayIterator) Key() []byte {
148 - i.updateKV()
149 - return i.key
150 -}
151 -
152 -func (i *arrayIterator) Value() []byte {
153 - i.updateKV()
154 - return i.value
155 -}
156 -
157 -type arrayIteratorIndexer struct {
158 - basicArrayIterator
159 - array ArrayIndexer
160 -}
161 -
162 -func (i *arrayIteratorIndexer) Get() Iterator {
163 - if i.Valid() {
164 - return i.array.Get(i.basicArrayIterator.pos)
165 - }
166 - return nil
167 -}
168 -
169 -// NewArrayIterator returns an iterator from the given array.
170 -func NewArrayIterator(array Array) Iterator {
171 - return &arrayIterator{
172 - basicArrayIterator: basicArrayIterator{array: array, pos: -1},
173 - array: array,
174 - pos: -1,
175 - }
176 -}
177 -
178 -// NewArrayIndexer returns an index iterator from the given array.
179 -func NewArrayIndexer(array ArrayIndexer) IteratorIndexer {
180 - return &arrayIteratorIndexer{
181 - basicArrayIterator: basicArrayIterator{array: array, pos: -1},
182 - array: array,
183 - }
184 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/array_iter_test.go deleted
-30
@@ -1,30 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator_test
8 -
9 -import (
10 - . "github.com/onsi/ginkgo"
11 -
12 - . "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
14 -)
15 -
16 -var _ = testutil.Defer(func() {
17 - Describe("Array iterator", func() {
18 - It("Should iterates and seeks correctly", func() {
19 - // Build key/value.
20 - kv := testutil.KeyValue_Generate(nil, 70, 1, 5, 3, 3)
21 -
22 - // Test the iterator.
23 - t := testutil.IteratorTesting{
24 - KeyValue: kv.Clone(),
25 - Iter: NewArrayIterator(kv),
26 - }
27 - testutil.DoIteratorTesting(&t)
28 - })
29 - })
30 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter.go deleted
-242
@@ -1,242 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
12 -)
13 -
14 -// IteratorIndexer is the interface that wraps CommonIterator and basic Get
15 -// method. IteratorIndexer provides index for indexed iterator.
16 -type IteratorIndexer interface {
17 - CommonIterator
18 -
19 - // Get returns a new data iterator for the current position, or nil if
20 - // done.
21 - Get() Iterator
22 -}
23 -
24 -type indexedIterator struct {
25 - util.BasicReleaser
26 - index IteratorIndexer
27 - strict bool
28 -
29 - data Iterator
30 - err error
31 - errf func(err error)
32 - closed bool
33 -}
34 -
35 -func (i *indexedIterator) setData() {
36 - if i.data != nil {
37 - i.data.Release()
38 - }
39 - i.data = i.index.Get()
40 -}
41 -
42 -func (i *indexedIterator) clearData() {
43 - if i.data != nil {
44 - i.data.Release()
45 - }
46 - i.data = nil
47 -}
48 -
49 -func (i *indexedIterator) indexErr() {
50 - if err := i.index.Error(); err != nil {
51 - if i.errf != nil {
52 - i.errf(err)
53 - }
54 - i.err = err
55 - }
56 -}
57 -
58 -func (i *indexedIterator) dataErr() bool {
59 - if err := i.data.Error(); err != nil {
60 - if i.errf != nil {
61 - i.errf(err)
62 - }
63 - if i.strict || !errors.IsCorrupted(err) {
64 - i.err = err
65 - return true
66 - }
67 - }
68 - return false
69 -}
70 -
71 -func (i *indexedIterator) Valid() bool {
72 - return i.data != nil && i.data.Valid()
73 -}
74 -
75 -func (i *indexedIterator) First() bool {
76 - if i.err != nil {
77 - return false
78 - } else if i.Released() {
79 - i.err = ErrIterReleased
80 - return false
81 - }
82 -
83 - if !i.index.First() {
84 - i.indexErr()
85 - i.clearData()
86 - return false
87 - }
88 - i.setData()
89 - return i.Next()
90 -}
91 -
92 -func (i *indexedIterator) Last() bool {
93 - if i.err != nil {
94 - return false
95 - } else if i.Released() {
96 - i.err = ErrIterReleased
97 - return false
98 - }
99 -
100 - if !i.index.Last() {
101 - i.indexErr()
102 - i.clearData()
103 - return false
104 - }
105 - i.setData()
106 - if !i.data.Last() {
107 - if i.dataErr() {
108 - return false
109 - }
110 - i.clearData()
111 - return i.Prev()
112 - }
113 - return true
114 -}
115 -
116 -func (i *indexedIterator) Seek(key []byte) bool {
117 - if i.err != nil {
118 - return false
119 - } else if i.Released() {
120 - i.err = ErrIterReleased
121 - return false
122 - }
123 -
124 - if !i.index.Seek(key) {
125 - i.indexErr()
126 - i.clearData()
127 - return false
128 - }
129 - i.setData()
130 - if !i.data.Seek(key) {
131 - if i.dataErr() {
132 - return false
133 - }
134 - i.clearData()
135 - return i.Next()
136 - }
137 - return true
138 -}
139 -
140 -func (i *indexedIterator) Next() bool {
141 - if i.err != nil {
142 - return false
143 - } else if i.Released() {
144 - i.err = ErrIterReleased
145 - return false
146 - }
147 -
148 - switch {
149 - case i.data != nil && !i.data.Next():
150 - if i.dataErr() {
151 - return false
152 - }
153 - i.clearData()
154 - fallthrough
155 - case i.data == nil:
156 - if !i.index.Next() {
157 - i.indexErr()
158 - return false
159 - }
160 - i.setData()
161 - return i.Next()
162 - }
163 - return true
164 -}
165 -
166 -func (i *indexedIterator) Prev() bool {
167 - if i.err != nil {
168 - return false
169 - } else if i.Released() {
170 - i.err = ErrIterReleased
171 - return false
172 - }
173 -
174 - switch {
175 - case i.data != nil && !i.data.Prev():
176 - if i.dataErr() {
177 - return false
178 - }
179 - i.clearData()
180 - fallthrough
181 - case i.data == nil:
182 - if !i.index.Prev() {
183 - i.indexErr()
184 - return false
185 - }
186 - i.setData()
187 - if !i.data.Last() {
188 - if i.dataErr() {
189 - return false
190 - }
191 - i.clearData()
192 - return i.Prev()
193 - }
194 - }
195 - return true
196 -}
197 -
198 -func (i *indexedIterator) Key() []byte {
199 - if i.data == nil {
200 - return nil
201 - }
202 - return i.data.Key()
203 -}
204 -
205 -func (i *indexedIterator) Value() []byte {
206 - if i.data == nil {
207 - return nil
208 - }
209 - return i.data.Value()
210 -}
211 -
212 -func (i *indexedIterator) Release() {
213 - i.clearData()
214 - i.index.Release()
215 - i.BasicReleaser.Release()
216 -}
217 -
218 -func (i *indexedIterator) Error() error {
219 - if i.err != nil {
220 - return i.err
221 - }
222 - if err := i.index.Error(); err != nil {
223 - return err
224 - }
225 - return nil
226 -}
227 -
228 -func (i *indexedIterator) SetErrorCallback(f func(err error)) {
229 - i.errf = f
230 -}
231 -
232 -// NewIndexedIterator returns an 'indexed iterator'. An index is iterator
233 -// that returns another iterator, a 'data iterator'. A 'data iterator' is the
234 -// iterator that contains actual key/value pairs.
235 -//
236 -// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
237 -// won't be ignored and will halt 'indexed iterator', otherwise the iterator will
238 -// continue to the next 'data iterator'. Corruption on 'index iterator' will not be
239 -// ignored and will halt the iterator.
240 -func NewIndexedIterator(index IteratorIndexer, strict bool) Iterator {
241 - return &indexedIterator{index: index, strict: strict}
242 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/indexed_iter_test.go deleted
-83
@@ -1,83 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator_test
8 -
9 -import (
10 - "sort"
11 -
12 - . "github.com/onsi/ginkgo"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
15 - . "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
17 -)
18 -
19 -type keyValue struct {
20 - key []byte
21 - testutil.KeyValue
22 -}
23 -
24 -type keyValueIndex []keyValue
25 -
26 -func (x keyValueIndex) Search(key []byte) int {
27 - return sort.Search(x.Len(), func(i int) bool {
28 - return comparer.DefaultComparer.Compare(x[i].key, key) >= 0
29 - })
30 -}
31 -
32 -func (x keyValueIndex) Len() int { return len(x) }
33 -func (x keyValueIndex) Index(i int) (key, value []byte) { return x[i].key, nil }
34 -func (x keyValueIndex) Get(i int) Iterator { return NewArrayIterator(x[i]) }
35 -
36 -var _ = testutil.Defer(func() {
37 - Describe("Indexed iterator", func() {
38 - Test := func(n ...int) func() {
39 - if len(n) == 0 {
40 - rnd := testutil.NewRand()
41 - n = make([]int, rnd.Intn(17)+3)
42 - for i := range n {
43 - n[i] = rnd.Intn(19) + 1
44 - }
45 - }
46 -
47 - return func() {
48 - It("Should iterates and seeks correctly", func(done Done) {
49 - // Build key/value.
50 - index := make(keyValueIndex, len(n))
51 - sum := 0
52 - for _, x := range n {
53 - sum += x
54 - }
55 - kv := testutil.KeyValue_Generate(nil, sum, 1, 10, 4, 4)
56 - for i, j := 0, 0; i < len(n); i++ {
57 - for x := n[i]; x > 0; x-- {
58 - key, value := kv.Index(j)
59 - index[i].key = key
60 - index[i].Put(key, value)
61 - j++
62 - }
63 - }
64 -
65 - // Test the iterator.
66 - t := testutil.IteratorTesting{
67 - KeyValue: kv.Clone(),
68 - Iter: NewIndexedIterator(NewArrayIndexer(index), true),
69 - }
70 - testutil.DoIteratorTesting(&t)
71 - done <- true
72 - }, 1.5)
73 - }
74 - }
75 -
76 - Describe("with 100 keys", Test(100))
77 - Describe("with 50-50 keys", Test(50, 50))
78 - Describe("with 50-1 keys", Test(50, 1))
79 - Describe("with 50-1-50 keys", Test(50, 1, 50))
80 - Describe("with 1-50 keys", Test(1, 50))
81 - Describe("with random N-keys", Test())
82 - })
83 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter.go deleted
-131
@@ -1,131 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package iterator provides interface and implementation to traverse over
8 -// contents of a database.
9 -package iterator
10 -
11 -import (
12 - "errors"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
15 -)
16 -
17 -var (
18 - ErrIterReleased = errors.New("leveldb/iterator: iterator released")
19 -)
20 -
21 -// IteratorSeeker is the interface that wraps the 'seeks method'.
22 -type IteratorSeeker interface {
23 - // First moves the iterator to the first key/value pair. If the iterator
24 - // only contains one key/value pair then First and Last whould moves
25 - // to the same key/value pair.
26 - // It returns whether such pair exist.
27 - First() bool
28 -
29 - // Last moves the iterator to the last key/value pair. If the iterator
30 - // only contains one key/value pair then First and Last whould moves
31 - // to the same key/value pair.
32 - // It returns whether such pair exist.
33 - Last() bool
34 -
35 - // Seek moves the iterator to the first key/value pair whose key is greater
36 - // than or equal to the given key.
37 - // It returns whether such pair exist.
38 - //
39 - // It is safe to modify the contents of the argument after Seek returns.
40 - Seek(key []byte) bool
41 -
42 - // Next moves the iterator to the next key/value pair.
43 - // It returns whether the iterator is exhausted.
44 - Next() bool
45 -
46 - // Prev moves the iterator to the previous key/value pair.
47 - // It returns whether the iterator is exhausted.
48 - Prev() bool
49 -}
50 -
51 -// CommonIterator is the interface that wraps common interator methods.
52 -type CommonIterator interface {
53 - IteratorSeeker
54 -
55 - // util.Releaser is the interface that wraps basic Release method.
56 - // When called Release will releases any resources associated with the
57 - // iterator.
58 - util.Releaser
59 -
60 - // util.ReleaseSetter is the interface that wraps the basic SetReleaser
61 - // method.
62 - util.ReleaseSetter
63 -
64 - // TODO: Remove this when ready.
65 - Valid() bool
66 -
67 - // Error returns any accumulated error. Exhausting all the key/value pairs
68 - // is not considered to be an error.
69 - Error() error
70 -}
71 -
72 -// Iterator iterates over a DB's key/value pairs in key order.
73 -//
74 -// When encouter an error any 'seeks method' will return false and will
75 -// yield no key/value pairs. The error can be queried by calling the Error
76 -// method. Calling Release is still necessary.
77 -//
78 -// An iterator must be released after use, but it is not necessary to read
79 -// an iterator until exhaustion.
80 -// Also, an iterator is not necessarily goroutine-safe, but it is safe to use
81 -// multiple iterators concurrently, with each in a dedicated goroutine.
82 -type Iterator interface {
83 - CommonIterator
84 -
85 - // Key returns the key of the current key/value pair, or nil if done.
86 - // The caller should not modify the contents of the returned slice, and
87 - // its contents may change on the next call to any 'seeks method'.
88 - Key() []byte
89 -
90 - // Value returns the key of the current key/value pair, or nil if done.
91 - // The caller should not modify the contents of the returned slice, and
92 - // its contents may change on the next call to any 'seeks method'.
93 - Value() []byte
94 -}
95 -
96 -// ErrorCallbackSetter is the interface that wraps basic SetErrorCallback
97 -// method.
98 -//
99 -// ErrorCallbackSetter implemented by indexed and merged iterator.
100 -type ErrorCallbackSetter interface {
101 - // SetErrorCallback allows set an error callback of the coresponding
102 - // iterator. Use nil to clear the callback.
103 - SetErrorCallback(f func(err error))
104 -}
105 -
106 -type emptyIterator struct {
107 - util.BasicReleaser
108 - err error
109 -}
110 -
111 -func (i *emptyIterator) rErr() {
112 - if i.err == nil && i.Released() {
113 - i.err = ErrIterReleased
114 - }
115 -}
116 -
117 -func (*emptyIterator) Valid() bool { return false }
118 -func (i *emptyIterator) First() bool { i.rErr(); return false }
119 -func (i *emptyIterator) Last() bool { i.rErr(); return false }
120 -func (i *emptyIterator) Seek(key []byte) bool { i.rErr(); return false }
121 -func (i *emptyIterator) Next() bool { i.rErr(); return false }
122 -func (i *emptyIterator) Prev() bool { i.rErr(); return false }
123 -func (*emptyIterator) Key() []byte { return nil }
124 -func (*emptyIterator) Value() []byte { return nil }
125 -func (i *emptyIterator) Error() error { return i.err }
126 -
127 -// NewEmptyIterator creates an empty iterator. The err parameter can be
128 -// nil, but if not nil the given err will be returned by Error method.
129 -func NewEmptyIterator(err error) Iterator {
130 - return &emptyIterator{err: err}
131 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/iter_suite_test.go deleted
-11
@@ -1,11 +0,0 @@
1 -package iterator_test
2 -
3 -import (
4 - "testing"
5 -
6 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7 -)
8 -
9 -func TestIterator(t *testing.T) {
10 - testutil.RunSuite(t, "Iterator Suite")
11 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter.go deleted
-304
@@ -1,304 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
13 -)
14 -
15 -type dir int
16 -
17 -const (
18 - dirReleased dir = iota - 1
19 - dirSOI
20 - dirEOI
21 - dirBackward
22 - dirForward
23 -)
24 -
25 -type mergedIterator struct {
26 - cmp comparer.Comparer
27 - iters []Iterator
28 - strict bool
29 -
30 - keys [][]byte
31 - index int
32 - dir dir
33 - err error
34 - errf func(err error)
35 - releaser util.Releaser
36 -}
37 -
38 -func assertKey(key []byte) []byte {
39 - if key == nil {
40 - panic("leveldb/iterator: nil key")
41 - }
42 - return key
43 -}
44 -
45 -func (i *mergedIterator) iterErr(iter Iterator) bool {
46 - if err := iter.Error(); err != nil {
47 - if i.errf != nil {
48 - i.errf(err)
49 - }
50 - if i.strict || !errors.IsCorrupted(err) {
51 - i.err = err
52 - return true
53 - }
54 - }
55 - return false
56 -}
57 -
58 -func (i *mergedIterator) Valid() bool {
59 - return i.err == nil && i.dir > dirEOI
60 -}
61 -
62 -func (i *mergedIterator) First() bool {
63 - if i.err != nil {
64 - return false
65 - } else if i.dir == dirReleased {
66 - i.err = ErrIterReleased
67 - return false
68 - }
69 -
70 - for x, iter := range i.iters {
71 - switch {
72 - case iter.First():
73 - i.keys[x] = assertKey(iter.Key())
74 - case i.iterErr(iter):
75 - return false
76 - default:
77 - i.keys[x] = nil
78 - }
79 - }
80 - i.dir = dirSOI
81 - return i.next()
82 -}
83 -
84 -func (i *mergedIterator) Last() bool {
85 - if i.err != nil {
86 - return false
87 - } else if i.dir == dirReleased {
88 - i.err = ErrIterReleased
89 - return false
90 - }
91 -
92 - for x, iter := range i.iters {
93 - switch {
94 - case iter.Last():
95 - i.keys[x] = assertKey(iter.Key())
96 - case i.iterErr(iter):
97 - return false
98 - default:
99 - i.keys[x] = nil
100 - }
101 - }
102 - i.dir = dirEOI
103 - return i.prev()
104 -}
105 -
106 -func (i *mergedIterator) Seek(key []byte) bool {
107 - if i.err != nil {
108 - return false
109 - } else if i.dir == dirReleased {
110 - i.err = ErrIterReleased
111 - return false
112 - }
113 -
114 - for x, iter := range i.iters {
115 - switch {
116 - case iter.Seek(key):
117 - i.keys[x] = assertKey(iter.Key())
118 - case i.iterErr(iter):
119 - return false
120 - default:
121 - i.keys[x] = nil
122 - }
123 - }
124 - i.dir = dirSOI
125 - return i.next()
126 -}
127 -
128 -func (i *mergedIterator) next() bool {
129 - var key []byte
130 - if i.dir == dirForward {
131 - key = i.keys[i.index]
132 - }
133 - for x, tkey := range i.keys {
134 - if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) < 0) {
135 - key = tkey
136 - i.index = x
137 - }
138 - }
139 - if key == nil {
140 - i.dir = dirEOI
141 - return false
142 - }
143 - i.dir = dirForward
144 - return true
145 -}
146 -
147 -func (i *mergedIterator) Next() bool {
148 - if i.dir == dirEOI || i.err != nil {
149 - return false
150 - } else if i.dir == dirReleased {
151 - i.err = ErrIterReleased
152 - return false
153 - }
154 -
155 - switch i.dir {
156 - case dirSOI:
157 - return i.First()
158 - case dirBackward:
159 - key := append([]byte{}, i.keys[i.index]...)
160 - if !i.Seek(key) {
161 - return false
162 - }
163 - return i.Next()
164 - }
165 -
166 - x := i.index
167 - iter := i.iters[x]
168 - switch {
169 - case iter.Next():
170 - i.keys[x] = assertKey(iter.Key())
171 - case i.iterErr(iter):
172 - return false
173 - default:
174 - i.keys[x] = nil
175 - }
176 - return i.next()
177 -}
178 -
179 -func (i *mergedIterator) prev() bool {
180 - var key []byte
181 - if i.dir == dirBackward {
182 - key = i.keys[i.index]
183 - }
184 - for x, tkey := range i.keys {
185 - if tkey != nil && (key == nil || i.cmp.Compare(tkey, key) > 0) {
186 - key = tkey
187 - i.index = x
188 - }
189 - }
190 - if key == nil {
191 - i.dir = dirSOI
192 - return false
193 - }
194 - i.dir = dirBackward
195 - return true
196 -}
197 -
198 -func (i *mergedIterator) Prev() bool {
199 - if i.dir == dirSOI || i.err != nil {
200 - return false
201 - } else if i.dir == dirReleased {
202 - i.err = ErrIterReleased
203 - return false
204 - }
205 -
206 - switch i.dir {
207 - case dirEOI:
208 - return i.Last()
209 - case dirForward:
210 - key := append([]byte{}, i.keys[i.index]...)
211 - for x, iter := range i.iters {
212 - if x == i.index {
213 - continue
214 - }
215 - seek := iter.Seek(key)
216 - switch {
217 - case seek && iter.Prev(), !seek && iter.Last():
218 - i.keys[x] = assertKey(iter.Key())
219 - case i.iterErr(iter):
220 - return false
221 - default:
222 - i.keys[x] = nil
223 - }
224 - }
225 - }
226 -
227 - x := i.index
228 - iter := i.iters[x]
229 - switch {
230 - case iter.Prev():
231 - i.keys[x] = assertKey(iter.Key())
232 - case i.iterErr(iter):
233 - return false
234 - default:
235 - i.keys[x] = nil
236 - }
237 - return i.prev()
238 -}
239 -
240 -func (i *mergedIterator) Key() []byte {
241 - if i.err != nil || i.dir <= dirEOI {
242 - return nil
243 - }
244 - return i.keys[i.index]
245 -}
246 -
247 -func (i *mergedIterator) Value() []byte {
248 - if i.err != nil || i.dir <= dirEOI {
249 - return nil
250 - }
251 - return i.iters[i.index].Value()
252 -}
253 -
254 -func (i *mergedIterator) Release() {
255 - if i.dir != dirReleased {
256 - i.dir = dirReleased
257 - for _, iter := range i.iters {
258 - iter.Release()
259 - }
260 - i.iters = nil
261 - i.keys = nil
262 - if i.releaser != nil {
263 - i.releaser.Release()
264 - i.releaser = nil
265 - }
266 - }
267 -}
268 -
269 -func (i *mergedIterator) SetReleaser(releaser util.Releaser) {
270 - if i.dir == dirReleased {
271 - panic(util.ErrReleased)
272 - }
273 - if i.releaser != nil && releaser != nil {
274 - panic(util.ErrHasReleaser)
275 - }
276 - i.releaser = releaser
277 -}
278 -
279 -func (i *mergedIterator) Error() error {
280 - return i.err
281 -}
282 -
283 -func (i *mergedIterator) SetErrorCallback(f func(err error)) {
284 - i.errf = f
285 -}
286 -
287 -// NewMergedIterator returns an iterator that merges its input. Walking the
288 -// resultant iterator will return all key/value pairs of all input iterators
289 -// in strictly increasing key order, as defined by cmp.
290 -// The input's key ranges may overlap, but there are assumed to be no duplicate
291 -// keys: if iters[i] contains a key k then iters[j] will not contain that key k.
292 -// None of the iters may be nil.
293 -//
294 -// If strict is true the any 'corruption errors' (i.e errors.IsCorrupted(err) == true)
295 -// won't be ignored and will halt 'merged iterator', otherwise the iterator will
296 -// continue to the next 'input iterator'.
297 -func NewMergedIterator(iters []Iterator, cmp comparer.Comparer, strict bool) Iterator {
298 - return &mergedIterator{
299 - iters: iters,
300 - cmp: cmp,
301 - strict: strict,
302 - keys: make([][]byte, len(iters)),
303 - }
304 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator/merged_iter_test.go deleted
-60
@@ -1,60 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package iterator_test
8 -
9 -import (
10 - . "github.com/onsi/ginkgo"
11 - . "github.com/onsi/gomega"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
14 - . "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
16 -)
17 -
18 -var _ = testutil.Defer(func() {
19 - Describe("Merged iterator", func() {
20 - Test := func(filled int, empty int) func() {
21 - return func() {
22 - It("Should iterates and seeks correctly", func(done Done) {
23 - rnd := testutil.NewRand()
24 -
25 - // Build key/value.
26 - filledKV := make([]testutil.KeyValue, filled)
27 - kv := testutil.KeyValue_Generate(nil, 100, 1, 10, 4, 4)
28 - kv.Iterate(func(i int, key, value []byte) {
29 - filledKV[rnd.Intn(filled)].Put(key, value)
30 - })
31 -
32 - // Create itearators.
33 - iters := make([]Iterator, filled+empty)
34 - for i := range iters {
35 - if empty == 0 || (rnd.Int()%2 == 0 && filled > 0) {
36 - filled--
37 - Expect(filledKV[filled].Len()).ShouldNot(BeZero())
38 - iters[i] = NewArrayIterator(filledKV[filled])
39 - } else {
40 - empty--
41 - iters[i] = NewEmptyIterator(nil)
42 - }
43 - }
44 -
45 - // Test the iterator.
46 - t := testutil.IteratorTesting{
47 - KeyValue: kv.Clone(),
48 - Iter: NewMergedIterator(iters, comparer.DefaultComparer, true),
49 - }
50 - testutil.DoIteratorTesting(&t)
51 - done <- true
52 - }, 1.5)
53 - }
54 - }
55 -
56 - Describe("with three, all filled iterators", Test(3, 0))
57 - Describe("with one filled, one empty iterators", Test(1, 1))
58 - Describe("with one filled, two empty iterators", Test(1, 2))
59 - })
60 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal.go deleted
-520
@@ -1,520 +0,0 @@
1 -// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record.go?r=1d5ccbe03246da926391ee12d1c6caae054ff4b0
6 -// License, authors and contributors informations can be found at bellow URLs respectively:
7 -// https://code.google.com/p/leveldb-go/source/browse/LICENSE
8 -// https://code.google.com/p/leveldb-go/source/browse/AUTHORS
9 -// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
10 -
11 -// Package journal reads and writes sequences of journals. Each journal is a stream
12 -// of bytes that completes before the next journal starts.
13 -//
14 -// When reading, call Next to obtain an io.Reader for the next journal. Next will
15 -// return io.EOF when there are no more journals. It is valid to call Next
16 -// without reading the current journal to exhaustion.
17 -//
18 -// When writing, call Next to obtain an io.Writer for the next journal. Calling
19 -// Next finishes the current journal. Call Close to finish the final journal.
20 -//
21 -// Optionally, call Flush to finish the current journal and flush the underlying
22 -// writer without starting a new journal. To start a new journal after flushing,
23 -// call Next.
24 -//
25 -// Neither Readers or Writers are safe to use concurrently.
26 -//
27 -// Example code:
28 -// func read(r io.Reader) ([]string, error) {
29 -// var ss []string
30 -// journals := journal.NewReader(r, nil, true, true)
31 -// for {
32 -// j, err := journals.Next()
33 -// if err == io.EOF {
34 -// break
35 -// }
36 -// if err != nil {
37 -// return nil, err
38 -// }
39 -// s, err := ioutil.ReadAll(j)
40 -// if err != nil {
41 -// return nil, err
42 -// }
43 -// ss = append(ss, string(s))
44 -// }
45 -// return ss, nil
46 -// }
47 -//
48 -// func write(w io.Writer, ss []string) error {
49 -// journals := journal.NewWriter(w)
50 -// for _, s := range ss {
51 -// j, err := journals.Next()
52 -// if err != nil {
53 -// return err
54 -// }
55 -// if _, err := j.Write([]byte(s)), err != nil {
56 -// return err
57 -// }
58 -// }
59 -// return journals.Close()
60 -// }
61 -//
62 -// The wire format is that the stream is divided into 32KiB blocks, and each
63 -// block contains a number of tightly packed chunks. Chunks cannot cross block
64 -// boundaries. The last block may be shorter than 32 KiB. Any unused bytes in a
65 -// block must be zero.
66 -//
67 -// A journal maps to one or more chunks. Each chunk has a 7 byte header (a 4
68 -// byte checksum, a 2 byte little-endian uint16 length, and a 1 byte chunk type)
69 -// followed by a payload. The checksum is over the chunk type and the payload.
70 -//
71 -// There are four chunk types: whether the chunk is the full journal, or the
72 -// first, middle or last chunk of a multi-chunk journal. A multi-chunk journal
73 -// has one first chunk, zero or more middle chunks, and one last chunk.
74 -//
75 -// The wire format allows for limited recovery in the face of data corruption:
76 -// on a format error (such as a checksum mismatch), the reader moves to the
77 -// next block and looks for the next full or first chunk.
78 -package journal
79 -
80 -import (
81 - "encoding/binary"
82 - "fmt"
83 - "io"
84 -
85 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
86 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
87 -)
88 -
89 -// These constants are part of the wire format and should not be changed.
90 -const (
91 - fullChunkType = 1
92 - firstChunkType = 2
93 - middleChunkType = 3
94 - lastChunkType = 4
95 -)
96 -
97 -const (
98 - blockSize = 32 * 1024
99 - headerSize = 7
100 -)
101 -
102 -type flusher interface {
103 - Flush() error
104 -}
105 -
106 -// ErrCorrupted is the error type that generated by corrupted block or chunk.
107 -type ErrCorrupted struct {
108 - Size int
109 - Reason string
110 -}
111 -
112 -func (e *ErrCorrupted) Error() string {
113 - return fmt.Sprintf("leveldb/journal: block/chunk corrupted: %s (%d bytes)", e.Reason, e.Size)
114 -}
115 -
116 -// Dropper is the interface that wrap simple Drop method. The Drop
117 -// method will be called when the journal reader dropping a block or chunk.
118 -type Dropper interface {
119 - Drop(err error)
120 -}
121 -
122 -// Reader reads journals from an underlying io.Reader.
123 -type Reader struct {
124 - // r is the underlying reader.
125 - r io.Reader
126 - // the dropper.
127 - dropper Dropper
128 - // strict flag.
129 - strict bool
130 - // checksum flag.
131 - checksum bool
132 - // seq is the sequence number of the current journal.
133 - seq int
134 - // buf[i:j] is the unread portion of the current chunk's payload.
135 - // The low bound, i, excludes the chunk header.
136 - i, j int
137 - // n is the number of bytes of buf that are valid. Once reading has started,
138 - // only the final block can have n < blockSize.
139 - n int
140 - // last is whether the current chunk is the last chunk of the journal.
141 - last bool
142 - // err is any accumulated error.
143 - err error
144 - // buf is the buffer.
145 - buf [blockSize]byte
146 -}
147 -
148 -// NewReader returns a new reader. The dropper may be nil, and if
149 -// strict is true then corrupted or invalid chunk will halt the journal
150 -// reader entirely.
151 -func NewReader(r io.Reader, dropper Dropper, strict, checksum bool) *Reader {
152 - return &Reader{
153 - r: r,
154 - dropper: dropper,
155 - strict: strict,
156 - checksum: checksum,
157 - last: true,
158 - }
159 -}
160 -
161 -var errSkip = errors.New("leveldb/journal: skipped")
162 -
163 -func (r *Reader) corrupt(n int, reason string, skip bool) error {
164 - if r.dropper != nil {
165 - r.dropper.Drop(&ErrCorrupted{n, reason})
166 - }
167 - if r.strict && !skip {
168 - r.err = errors.NewErrCorrupted(nil, &ErrCorrupted{n, reason})
169 - return r.err
170 - }
171 - return errSkip
172 -}
173 -
174 -// nextChunk sets r.buf[r.i:r.j] to hold the next chunk's payload, reading the
175 -// next block into the buffer if necessary.
176 -func (r *Reader) nextChunk(first bool) error {
177 - for {
178 - if r.j+headerSize <= r.n {
179 - checksum := binary.LittleEndian.Uint32(r.buf[r.j+0 : r.j+4])
180 - length := binary.LittleEndian.Uint16(r.buf[r.j+4 : r.j+6])
181 - chunkType := r.buf[r.j+6]
182 -
183 - if checksum == 0 && length == 0 && chunkType == 0 {
184 - // Drop entire block.
185 - m := r.n - r.j
186 - r.i = r.n
187 - r.j = r.n
188 - return r.corrupt(m, "zero header", false)
189 - } else {
190 - m := r.n - r.j
191 - r.i = r.j + headerSize
192 - r.j = r.j + headerSize + int(length)
193 - if r.j > r.n {
194 - // Drop entire block.
195 - r.i = r.n
196 - r.j = r.n
197 - return r.corrupt(m, "chunk length overflows block", false)
198 - } else if r.checksum && checksum != util.NewCRC(r.buf[r.i-1:r.j]).Value() {
199 - // Drop entire block.
200 - r.i = r.n
201 - r.j = r.n
202 - return r.corrupt(m, "checksum mismatch", false)
203 - }
204 - }
205 - if first && chunkType != fullChunkType && chunkType != firstChunkType {
206 - m := r.j - r.i
207 - r.i = r.j
208 - // Report the error, but skip it.
209 - return r.corrupt(m+headerSize, "orphan chunk", true)
210 - }
211 - r.last = chunkType == fullChunkType || chunkType == lastChunkType
212 - return nil
213 - }
214 -
215 - // The last block.
216 - if r.n < blockSize && r.n > 0 {
217 - if !first {
218 - return r.corrupt(0, "missing chunk part", false)
219 - }
220 - r.err = io.EOF
221 - return r.err
222 - }
223 -
224 - // Read block.
225 - n, err := io.ReadFull(r.r, r.buf[:])
226 - if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
227 - return err
228 - }
229 - if n == 0 {
230 - if !first {
231 - return r.corrupt(0, "missing chunk part", false)
232 - }
233 - r.err = io.EOF
234 - return r.err
235 - }
236 - r.i, r.j, r.n = 0, 0, n
237 - }
238 -}
239 -
240 -// Next returns a reader for the next journal. It returns io.EOF if there are no
241 -// more journals. The reader returned becomes stale after the next Next call,
242 -// and should no longer be used. If strict is false, the reader will returns
243 -// io.ErrUnexpectedEOF error when found corrupted journal.
244 -func (r *Reader) Next() (io.Reader, error) {
245 - r.seq++
246 - if r.err != nil {
247 - return nil, r.err
248 - }
249 - r.i = r.j
250 - for {
251 - if err := r.nextChunk(true); err == nil {
252 - break
253 - } else if err != errSkip {
254 - return nil, err
255 - }
256 - }
257 - return &singleReader{r, r.seq, nil}, nil
258 -}
259 -
260 -// Reset resets the journal reader, allows reuse of the journal reader. Reset returns
261 -// last accumulated error.
262 -func (r *Reader) Reset(reader io.Reader, dropper Dropper, strict, checksum bool) error {
263 - r.seq++
264 - err := r.err
265 - r.r = reader
266 - r.dropper = dropper
267 - r.strict = strict
268 - r.checksum = checksum
269 - r.i = 0
270 - r.j = 0
271 - r.n = 0
272 - r.last = true
273 - r.err = nil
274 - return err
275 -}
276 -
277 -type singleReader struct {
278 - r *Reader
279 - seq int
280 - err error
281 -}
282 -
283 -func (x *singleReader) Read(p []byte) (int, error) {
284 - r := x.r
285 - if r.seq != x.seq {
286 - return 0, errors.New("leveldb/journal: stale reader")
287 - }
288 - if x.err != nil {
289 - return 0, x.err
290 - }
291 - if r.err != nil {
292 - return 0, r.err
293 - }
294 - for r.i == r.j {
295 - if r.last {
296 - return 0, io.EOF
297 - }
298 - x.err = r.nextChunk(false)
299 - if x.err != nil {
300 - if x.err == errSkip {
301 - x.err = io.ErrUnexpectedEOF
302 - }
303 - return 0, x.err
304 - }
305 - }
306 - n := copy(p, r.buf[r.i:r.j])
307 - r.i += n
308 - return n, nil
309 -}
310 -
311 -func (x *singleReader) ReadByte() (byte, error) {
312 - r := x.r
313 - if r.seq != x.seq {
314 - return 0, errors.New("leveldb/journal: stale reader")
315 - }
316 - if x.err != nil {
317 - return 0, x.err
318 - }
319 - if r.err != nil {
320 - return 0, r.err
321 - }
322 - for r.i == r.j {
323 - if r.last {
324 - return 0, io.EOF
325 - }
326 - x.err = r.nextChunk(false)
327 - if x.err != nil {
328 - if x.err == errSkip {
329 - x.err = io.ErrUnexpectedEOF
330 - }
331 - return 0, x.err
332 - }
333 - }
334 - c := r.buf[r.i]
335 - r.i++
336 - return c, nil
337 -}
338 -
339 -// Writer writes journals to an underlying io.Writer.
340 -type Writer struct {
341 - // w is the underlying writer.
342 - w io.Writer
343 - // seq is the sequence number of the current journal.
344 - seq int
345 - // f is w as a flusher.
346 - f flusher
347 - // buf[i:j] is the bytes that will become the current chunk.
348 - // The low bound, i, includes the chunk header.
349 - i, j int
350 - // buf[:written] has already been written to w.
351 - // written is zero unless Flush has been called.
352 - written int
353 - // first is whether the current chunk is the first chunk of the journal.
354 - first bool
355 - // pending is whether a chunk is buffered but not yet written.
356 - pending bool
357 - // err is any accumulated error.
358 - err error
359 - // buf is the buffer.
360 - buf [blockSize]byte
361 -}
362 -
363 -// NewWriter returns a new Writer.
364 -func NewWriter(w io.Writer) *Writer {
365 - f, _ := w.(flusher)
366 - return &Writer{
367 - w: w,
368 - f: f,
369 - }
370 -}
371 -
372 -// fillHeader fills in the header for the pending chunk.
373 -func (w *Writer) fillHeader(last bool) {
374 - if w.i+headerSize > w.j || w.j > blockSize {
375 - panic("leveldb/journal: bad writer state")
376 - }
377 - if last {
378 - if w.first {
379 - w.buf[w.i+6] = fullChunkType
380 - } else {
381 - w.buf[w.i+6] = lastChunkType
382 - }
383 - } else {
384 - if w.first {
385 - w.buf[w.i+6] = firstChunkType
386 - } else {
387 - w.buf[w.i+6] = middleChunkType
388 - }
389 - }
390 - binary.LittleEndian.PutUint32(w.buf[w.i+0:w.i+4], util.NewCRC(w.buf[w.i+6:w.j]).Value())
391 - binary.LittleEndian.PutUint16(w.buf[w.i+4:w.i+6], uint16(w.j-w.i-headerSize))
392 -}
393 -
394 -// writeBlock writes the buffered block to the underlying writer, and reserves
395 -// space for the next chunk's header.
396 -func (w *Writer) writeBlock() {
397 - _, w.err = w.w.Write(w.buf[w.written:])
398 - w.i = 0
399 - w.j = headerSize
400 - w.written = 0
401 -}
402 -
403 -// writePending finishes the current journal and writes the buffer to the
404 -// underlying writer.
405 -func (w *Writer) writePending() {
406 - if w.err != nil {
407 - return
408 - }
409 - if w.pending {
410 - w.fillHeader(true)
411 - w.pending = false
412 - }
413 - _, w.err = w.w.Write(w.buf[w.written:w.j])
414 - w.written = w.j
415 -}
416 -
417 -// Close finishes the current journal and closes the writer.
418 -func (w *Writer) Close() error {
419 - w.seq++
420 - w.writePending()
421 - if w.err != nil {
422 - return w.err
423 - }
424 - w.err = errors.New("leveldb/journal: closed Writer")
425 - return nil
426 -}
427 -
428 -// Flush finishes the current journal, writes to the underlying writer, and
429 -// flushes it if that writer implements interface{ Flush() error }.
430 -func (w *Writer) Flush() error {
431 - w.seq++
432 - w.writePending()
433 - if w.err != nil {
434 - return w.err
435 - }
436 - if w.f != nil {
437 - w.err = w.f.Flush()
438 - return w.err
439 - }
440 - return nil
441 -}
442 -
443 -// Reset resets the journal writer, allows reuse of the journal writer. Reset
444 -// will also closes the journal writer if not already.
445 -func (w *Writer) Reset(writer io.Writer) (err error) {
446 - w.seq++
447 - if w.err == nil {
448 - w.writePending()
449 - err = w.err
450 - }
451 - w.w = writer
452 - w.f, _ = writer.(flusher)
453 - w.i = 0
454 - w.j = 0
455 - w.written = 0
456 - w.first = false
457 - w.pending = false
458 - w.err = nil
459 - return
460 -}
461 -
462 -// Next returns a writer for the next journal. The writer returned becomes stale
463 -// after the next Close, Flush or Next call, and should no longer be used.
464 -func (w *Writer) Next() (io.Writer, error) {
465 - w.seq++
466 - if w.err != nil {
467 - return nil, w.err
468 - }
469 - if w.pending {
470 - w.fillHeader(true)
471 - }
472 - w.i = w.j
473 - w.j = w.j + headerSize
474 - // Check if there is room in the block for the header.
475 - if w.j > blockSize {
476 - // Fill in the rest of the block with zeroes.
477 - for k := w.i; k < blockSize; k++ {
478 - w.buf[k] = 0
479 - }
480 - w.writeBlock()
481 - if w.err != nil {
482 - return nil, w.err
483 - }
484 - }
485 - w.first = true
486 - w.pending = true
487 - return singleWriter{w, w.seq}, nil
488 -}
489 -
490 -type singleWriter struct {
491 - w *Writer
492 - seq int
493 -}
494 -
495 -func (x singleWriter) Write(p []byte) (int, error) {
496 - w := x.w
497 - if w.seq != x.seq {
498 - return 0, errors.New("leveldb/journal: stale writer")
499 - }
500 - if w.err != nil {
501 - return 0, w.err
502 - }
503 - n0 := len(p)
504 - for len(p) > 0 {
505 - // Write a block, if it is full.
506 - if w.j == blockSize {
507 - w.fillHeader(false)
508 - w.writeBlock()
509 - if w.err != nil {
510 - return 0, w.err
511 - }
512 - w.first = false
513 - }
514 - // Copy bytes into the buffer.
515 - n := copy(w.buf[w.j:], p)
516 - w.j += n
517 - p = p[n:]
518 - }
519 - return n0, nil
520 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal/journal_test.go deleted
-818
@@ -1,818 +0,0 @@
1 -// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -// Taken from: https://code.google.com/p/leveldb-go/source/browse/leveldb/record/record_test.go?r=df1fa28f7f3be6c3935548169002309c12967135
6 -// License, authors and contributors informations can be found at bellow URLs respectively:
7 -// https://code.google.com/p/leveldb-go/source/browse/LICENSE
8 -// https://code.google.com/p/leveldb-go/source/browse/AUTHORS
9 -// https://code.google.com/p/leveldb-go/source/browse/CONTRIBUTORS
10 -
11 -package journal
12 -
13 -import (
14 - "bytes"
15 - "encoding/binary"
16 - "fmt"
17 - "io"
18 - "io/ioutil"
19 - "math/rand"
20 - "strings"
21 - "testing"
22 -)
23 -
24 -type dropper struct {
25 - t *testing.T
26 -}
27 -
28 -func (d dropper) Drop(err error) {
29 - d.t.Log(err)
30 -}
31 -
32 -func short(s string) string {
33 - if len(s) < 64 {
34 - return s
35 - }
36 - return fmt.Sprintf("%s...(skipping %d bytes)...%s", s[:20], len(s)-40, s[len(s)-20:])
37 -}
38 -
39 -// big returns a string of length n, composed of repetitions of partial.
40 -func big(partial string, n int) string {
41 - return strings.Repeat(partial, n/len(partial)+1)[:n]
42 -}
43 -
44 -func TestEmpty(t *testing.T) {
45 - buf := new(bytes.Buffer)
46 - r := NewReader(buf, dropper{t}, true, true)
47 - if _, err := r.Next(); err != io.EOF {
48 - t.Fatalf("got %v, want %v", err, io.EOF)
49 - }
50 -}
51 -
52 -func testGenerator(t *testing.T, reset func(), gen func() (string, bool)) {
53 - buf := new(bytes.Buffer)
54 -
55 - reset()
56 - w := NewWriter(buf)
57 - for {
58 - s, ok := gen()
59 - if !ok {
60 - break
61 - }
62 - ww, err := w.Next()
63 - if err != nil {
64 - t.Fatal(err)
65 - }
66 - if _, err := ww.Write([]byte(s)); err != nil {
67 - t.Fatal(err)
68 - }
69 - }
70 - if err := w.Close(); err != nil {
71 - t.Fatal(err)
72 - }
73 -
74 - reset()
75 - r := NewReader(buf, dropper{t}, true, true)
76 - for {
77 - s, ok := gen()
78 - if !ok {
79 - break
80 - }
81 - rr, err := r.Next()
82 - if err != nil {
83 - t.Fatal(err)
84 - }
85 - x, err := ioutil.ReadAll(rr)
86 - if err != nil {
87 - t.Fatal(err)
88 - }
89 - if string(x) != s {
90 - t.Fatalf("got %q, want %q", short(string(x)), short(s))
91 - }
92 - }
93 - if _, err := r.Next(); err != io.EOF {
94 - t.Fatalf("got %v, want %v", err, io.EOF)
95 - }
96 -}
97 -
98 -func testLiterals(t *testing.T, s []string) {
99 - var i int
100 - reset := func() {
101 - i = 0
102 - }
103 - gen := func() (string, bool) {
104 - if i == len(s) {
105 - return "", false
106 - }
107 - i++
108 - return s[i-1], true
109 - }
110 - testGenerator(t, reset, gen)
111 -}
112 -
113 -func TestMany(t *testing.T) {
114 - const n = 1e5
115 - var i int
116 - reset := func() {
117 - i = 0
118 - }
119 - gen := func() (string, bool) {
120 - if i == n {
121 - return "", false
122 - }
123 - i++
124 - return fmt.Sprintf("%d.", i-1), true
125 - }
126 - testGenerator(t, reset, gen)
127 -}
128 -
129 -func TestRandom(t *testing.T) {
130 - const n = 1e2
131 - var (
132 - i int
133 - r *rand.Rand
134 - )
135 - reset := func() {
136 - i, r = 0, rand.New(rand.NewSource(0))
137 - }
138 - gen := func() (string, bool) {
139 - if i == n {
140 - return "", false
141 - }
142 - i++
143 - return strings.Repeat(string(uint8(i)), r.Intn(2*blockSize+16)), true
144 - }
145 - testGenerator(t, reset, gen)
146 -}
147 -
148 -func TestBasic(t *testing.T) {
149 - testLiterals(t, []string{
150 - strings.Repeat("a", 1000),
151 - strings.Repeat("b", 97270),
152 - strings.Repeat("c", 8000),
153 - })
154 -}
155 -
156 -func TestBoundary(t *testing.T) {
157 - for i := blockSize - 16; i < blockSize+16; i++ {
158 - s0 := big("abcd", i)
159 - for j := blockSize - 16; j < blockSize+16; j++ {
160 - s1 := big("ABCDE", j)
161 - testLiterals(t, []string{s0, s1})
162 - testLiterals(t, []string{s0, "", s1})
163 - testLiterals(t, []string{s0, "x", s1})
164 - }
165 - }
166 -}
167 -
168 -func TestFlush(t *testing.T) {
169 - buf := new(bytes.Buffer)
170 - w := NewWriter(buf)
171 - // Write a couple of records. Everything should still be held
172 - // in the record.Writer buffer, so that buf.Len should be 0.
173 - w0, _ := w.Next()
174 - w0.Write([]byte("0"))
175 - w1, _ := w.Next()
176 - w1.Write([]byte("11"))
177 - if got, want := buf.Len(), 0; got != want {
178 - t.Fatalf("buffer length #0: got %d want %d", got, want)
179 - }
180 - // Flush the record.Writer buffer, which should yield 17 bytes.
181 - // 17 = 2*7 + 1 + 2, which is two headers and 1 + 2 payload bytes.
182 - if err := w.Flush(); err != nil {
183 - t.Fatal(err)
184 - }
185 - if got, want := buf.Len(), 17; got != want {
186 - t.Fatalf("buffer length #1: got %d want %d", got, want)
187 - }
188 - // Do another write, one that isn't large enough to complete the block.
189 - // The write should not have flowed through to buf.
190 - w2, _ := w.Next()
191 - w2.Write(bytes.Repeat([]byte("2"), 10000))
192 - if got, want := buf.Len(), 17; got != want {
193 - t.Fatalf("buffer length #2: got %d want %d", got, want)
194 - }
195 - // Flushing should get us up to 10024 bytes written.
196 - // 10024 = 17 + 7 + 10000.
197 - if err := w.Flush(); err != nil {
198 - t.Fatal(err)
199 - }
200 - if got, want := buf.Len(), 10024; got != want {
201 - t.Fatalf("buffer length #3: got %d want %d", got, want)
202 - }
203 - // Do a bigger write, one that completes the current block.
204 - // We should now have 32768 bytes (a complete block), without
205 - // an explicit flush.
206 - w3, _ := w.Next()
207 - w3.Write(bytes.Repeat([]byte("3"), 40000))
208 - if got, want := buf.Len(), 32768; got != want {
209 - t.Fatalf("buffer length #4: got %d want %d", got, want)
210 - }
211 - // Flushing should get us up to 50038 bytes written.
212 - // 50038 = 10024 + 2*7 + 40000. There are two headers because
213 - // the one record was split into two chunks.
214 - if err := w.Flush(); err != nil {
215 - t.Fatal(err)
216 - }
217 - if got, want := buf.Len(), 50038; got != want {
218 - t.Fatalf("buffer length #5: got %d want %d", got, want)
219 - }
220 - // Check that reading those records give the right lengths.
221 - r := NewReader(buf, dropper{t}, true, true)
222 - wants := []int64{1, 2, 10000, 40000}
223 - for i, want := range wants {
224 - rr, _ := r.Next()
225 - n, err := io.Copy(ioutil.Discard, rr)
226 - if err != nil {
227 - t.Fatalf("read #%d: %v", i, err)
228 - }
229 - if n != want {
230 - t.Fatalf("read #%d: got %d bytes want %d", i, n, want)
231 - }
232 - }
233 -}
234 -
235 -func TestNonExhaustiveRead(t *testing.T) {
236 - const n = 100
237 - buf := new(bytes.Buffer)
238 - p := make([]byte, 10)
239 - rnd := rand.New(rand.NewSource(1))
240 -
241 - w := NewWriter(buf)
242 - for i := 0; i < n; i++ {
243 - length := len(p) + rnd.Intn(3*blockSize)
244 - s := string(uint8(i)) + "123456789abcdefgh"
245 - ww, _ := w.Next()
246 - ww.Write([]byte(big(s, length)))
247 - }
248 - if err := w.Close(); err != nil {
249 - t.Fatal(err)
250 - }
251 -
252 - r := NewReader(buf, dropper{t}, true, true)
253 - for i := 0; i < n; i++ {
254 - rr, _ := r.Next()
255 - _, err := io.ReadFull(rr, p)
256 - if err != nil {
257 - t.Fatal(err)
258 - }
259 - want := string(uint8(i)) + "123456789"
260 - if got := string(p); got != want {
261 - t.Fatalf("read #%d: got %q want %q", i, got, want)
262 - }
263 - }
264 -}
265 -
266 -func TestStaleReader(t *testing.T) {
267 - buf := new(bytes.Buffer)
268 -
269 - w := NewWriter(buf)
270 - w0, err := w.Next()
271 - if err != nil {
272 - t.Fatal(err)
273 - }
274 - w0.Write([]byte("0"))
275 - w1, err := w.Next()
276 - if err != nil {
277 - t.Fatal(err)
278 - }
279 - w1.Write([]byte("11"))
280 - if err := w.Close(); err != nil {
281 - t.Fatal(err)
282 - }
283 -
284 - r := NewReader(buf, dropper{t}, true, true)
285 - r0, err := r.Next()
286 - if err != nil {
287 - t.Fatal(err)
288 - }
289 - r1, err := r.Next()
290 - if err != nil {
291 - t.Fatal(err)
292 - }
293 - p := make([]byte, 1)
294 - if _, err := r0.Read(p); err == nil || !strings.Contains(err.Error(), "stale") {
295 - t.Fatalf("stale read #0: unexpected error: %v", err)
296 - }
297 - if _, err := r1.Read(p); err != nil {
298 - t.Fatalf("fresh read #1: got %v want nil error", err)
299 - }
300 - if p[0] != '1' {
301 - t.Fatalf("fresh read #1: byte contents: got '%c' want '1'", p[0])
302 - }
303 -}
304 -
305 -func TestStaleWriter(t *testing.T) {
306 - buf := new(bytes.Buffer)
307 -
308 - w := NewWriter(buf)
309 - w0, err := w.Next()
310 - if err != nil {
311 - t.Fatal(err)
312 - }
313 - w1, err := w.Next()
314 - if err != nil {
315 - t.Fatal(err)
316 - }
317 - if _, err := w0.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
318 - t.Fatalf("stale write #0: unexpected error: %v", err)
319 - }
320 - if _, err := w1.Write([]byte("11")); err != nil {
321 - t.Fatalf("fresh write #1: got %v want nil error", err)
322 - }
323 - if err := w.Flush(); err != nil {
324 - t.Fatalf("flush: %v", err)
325 - }
326 - if _, err := w1.Write([]byte("0")); err == nil || !strings.Contains(err.Error(), "stale") {
327 - t.Fatalf("stale write #1: unexpected error: %v", err)
328 - }
329 -}
330 -
331 -func TestCorrupt_MissingLastBlock(t *testing.T) {
332 - buf := new(bytes.Buffer)
333 -
334 - w := NewWriter(buf)
335 -
336 - // First record.
337 - ww, err := w.Next()
338 - if err != nil {
339 - t.Fatal(err)
340 - }
341 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-1024)); err != nil {
342 - t.Fatalf("write #0: unexpected error: %v", err)
343 - }
344 -
345 - // Second record.
346 - ww, err = w.Next()
347 - if err != nil {
348 - t.Fatal(err)
349 - }
350 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
351 - t.Fatalf("write #1: unexpected error: %v", err)
352 - }
353 -
354 - if err := w.Close(); err != nil {
355 - t.Fatal(err)
356 - }
357 -
358 - // Cut the last block.
359 - b := buf.Bytes()[:blockSize]
360 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
361 -
362 - // First read.
363 - rr, err := r.Next()
364 - if err != nil {
365 - t.Fatal(err)
366 - }
367 - n, err := io.Copy(ioutil.Discard, rr)
368 - if err != nil {
369 - t.Fatalf("read #0: %v", err)
370 - }
371 - if n != blockSize-1024 {
372 - t.Fatalf("read #0: got %d bytes want %d", n, blockSize-1024)
373 - }
374 -
375 - // Second read.
376 - rr, err = r.Next()
377 - if err != nil {
378 - t.Fatal(err)
379 - }
380 - n, err = io.Copy(ioutil.Discard, rr)
381 - if err != io.ErrUnexpectedEOF {
382 - t.Fatalf("read #1: unexpected error: %v", err)
383 - }
384 -
385 - if _, err := r.Next(); err != io.EOF {
386 - t.Fatalf("last next: unexpected error: %v", err)
387 - }
388 -}
389 -
390 -func TestCorrupt_CorruptedFirstBlock(t *testing.T) {
391 - buf := new(bytes.Buffer)
392 -
393 - w := NewWriter(buf)
394 -
395 - // First record.
396 - ww, err := w.Next()
397 - if err != nil {
398 - t.Fatal(err)
399 - }
400 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
401 - t.Fatalf("write #0: unexpected error: %v", err)
402 - }
403 -
404 - // Second record.
405 - ww, err = w.Next()
406 - if err != nil {
407 - t.Fatal(err)
408 - }
409 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
410 - t.Fatalf("write #1: unexpected error: %v", err)
411 - }
412 -
413 - // Third record.
414 - ww, err = w.Next()
415 - if err != nil {
416 - t.Fatal(err)
417 - }
418 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
419 - t.Fatalf("write #2: unexpected error: %v", err)
420 - }
421 -
422 - // Fourth record.
423 - ww, err = w.Next()
424 - if err != nil {
425 - t.Fatal(err)
426 - }
427 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
428 - t.Fatalf("write #3: unexpected error: %v", err)
429 - }
430 -
431 - if err := w.Close(); err != nil {
432 - t.Fatal(err)
433 - }
434 -
435 - b := buf.Bytes()
436 - // Corrupting block #0.
437 - for i := 0; i < 1024; i++ {
438 - b[i] = '1'
439 - }
440 -
441 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
442 -
443 - // First read (third record).
444 - rr, err := r.Next()
445 - if err != nil {
446 - t.Fatal(err)
447 - }
448 - n, err := io.Copy(ioutil.Discard, rr)
449 - if err != nil {
450 - t.Fatalf("read #0: %v", err)
451 - }
452 - if want := int64(blockSize-headerSize) + 1; n != want {
453 - t.Fatalf("read #0: got %d bytes want %d", n, want)
454 - }
455 -
456 - // Second read (fourth record).
457 - rr, err = r.Next()
458 - if err != nil {
459 - t.Fatal(err)
460 - }
461 - n, err = io.Copy(ioutil.Discard, rr)
462 - if err != nil {
463 - t.Fatalf("read #1: %v", err)
464 - }
465 - if want := int64(blockSize-headerSize) + 2; n != want {
466 - t.Fatalf("read #1: got %d bytes want %d", n, want)
467 - }
468 -
469 - if _, err := r.Next(); err != io.EOF {
470 - t.Fatalf("last next: unexpected error: %v", err)
471 - }
472 -}
473 -
474 -func TestCorrupt_CorruptedMiddleBlock(t *testing.T) {
475 - buf := new(bytes.Buffer)
476 -
477 - w := NewWriter(buf)
478 -
479 - // First record.
480 - ww, err := w.Next()
481 - if err != nil {
482 - t.Fatal(err)
483 - }
484 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
485 - t.Fatalf("write #0: unexpected error: %v", err)
486 - }
487 -
488 - // Second record.
489 - ww, err = w.Next()
490 - if err != nil {
491 - t.Fatal(err)
492 - }
493 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
494 - t.Fatalf("write #1: unexpected error: %v", err)
495 - }
496 -
497 - // Third record.
498 - ww, err = w.Next()
499 - if err != nil {
500 - t.Fatal(err)
501 - }
502 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
503 - t.Fatalf("write #2: unexpected error: %v", err)
504 - }
505 -
506 - // Fourth record.
507 - ww, err = w.Next()
508 - if err != nil {
509 - t.Fatal(err)
510 - }
511 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
512 - t.Fatalf("write #3: unexpected error: %v", err)
513 - }
514 -
515 - if err := w.Close(); err != nil {
516 - t.Fatal(err)
517 - }
518 -
519 - b := buf.Bytes()
520 - // Corrupting block #1.
521 - for i := 0; i < 1024; i++ {
522 - b[blockSize+i] = '1'
523 - }
524 -
525 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
526 -
527 - // First read (first record).
528 - rr, err := r.Next()
529 - if err != nil {
530 - t.Fatal(err)
531 - }
532 - n, err := io.Copy(ioutil.Discard, rr)
533 - if err != nil {
534 - t.Fatalf("read #0: %v", err)
535 - }
536 - if want := int64(blockSize / 2); n != want {
537 - t.Fatalf("read #0: got %d bytes want %d", n, want)
538 - }
539 -
540 - // Second read (second record).
541 - rr, err = r.Next()
542 - if err != nil {
543 - t.Fatal(err)
544 - }
545 - n, err = io.Copy(ioutil.Discard, rr)
546 - if err != io.ErrUnexpectedEOF {
547 - t.Fatalf("read #1: unexpected error: %v", err)
548 - }
549 -
550 - // Third read (fourth record).
551 - rr, err = r.Next()
552 - if err != nil {
553 - t.Fatal(err)
554 - }
555 - n, err = io.Copy(ioutil.Discard, rr)
556 - if err != nil {
557 - t.Fatalf("read #2: %v", err)
558 - }
559 - if want := int64(blockSize-headerSize) + 2; n != want {
560 - t.Fatalf("read #2: got %d bytes want %d", n, want)
561 - }
562 -
563 - if _, err := r.Next(); err != io.EOF {
564 - t.Fatalf("last next: unexpected error: %v", err)
565 - }
566 -}
567 -
568 -func TestCorrupt_CorruptedLastBlock(t *testing.T) {
569 - buf := new(bytes.Buffer)
570 -
571 - w := NewWriter(buf)
572 -
573 - // First record.
574 - ww, err := w.Next()
575 - if err != nil {
576 - t.Fatal(err)
577 - }
578 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
579 - t.Fatalf("write #0: unexpected error: %v", err)
580 - }
581 -
582 - // Second record.
583 - ww, err = w.Next()
584 - if err != nil {
585 - t.Fatal(err)
586 - }
587 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
588 - t.Fatalf("write #1: unexpected error: %v", err)
589 - }
590 -
591 - // Third record.
592 - ww, err = w.Next()
593 - if err != nil {
594 - t.Fatal(err)
595 - }
596 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
597 - t.Fatalf("write #2: unexpected error: %v", err)
598 - }
599 -
600 - // Fourth record.
601 - ww, err = w.Next()
602 - if err != nil {
603 - t.Fatal(err)
604 - }
605 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+2)); err != nil {
606 - t.Fatalf("write #3: unexpected error: %v", err)
607 - }
608 -
609 - if err := w.Close(); err != nil {
610 - t.Fatal(err)
611 - }
612 -
613 - b := buf.Bytes()
614 - // Corrupting block #3.
615 - for i := len(b) - 1; i > len(b)-1024; i-- {
616 - b[i] = '1'
617 - }
618 -
619 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
620 -
621 - // First read (first record).
622 - rr, err := r.Next()
623 - if err != nil {
624 - t.Fatal(err)
625 - }
626 - n, err := io.Copy(ioutil.Discard, rr)
627 - if err != nil {
628 - t.Fatalf("read #0: %v", err)
629 - }
630 - if want := int64(blockSize / 2); n != want {
631 - t.Fatalf("read #0: got %d bytes want %d", n, want)
632 - }
633 -
634 - // Second read (second record).
635 - rr, err = r.Next()
636 - if err != nil {
637 - t.Fatal(err)
638 - }
639 - n, err = io.Copy(ioutil.Discard, rr)
640 - if err != nil {
641 - t.Fatalf("read #1: %v", err)
642 - }
643 - if want := int64(blockSize - headerSize); n != want {
644 - t.Fatalf("read #1: got %d bytes want %d", n, want)
645 - }
646 -
647 - // Third read (third record).
648 - rr, err = r.Next()
649 - if err != nil {
650 - t.Fatal(err)
651 - }
652 - n, err = io.Copy(ioutil.Discard, rr)
653 - if err != nil {
654 - t.Fatalf("read #2: %v", err)
655 - }
656 - if want := int64(blockSize-headerSize) + 1; n != want {
657 - t.Fatalf("read #2: got %d bytes want %d", n, want)
658 - }
659 -
660 - // Fourth read (fourth record).
661 - rr, err = r.Next()
662 - if err != nil {
663 - t.Fatal(err)
664 - }
665 - n, err = io.Copy(ioutil.Discard, rr)
666 - if err != io.ErrUnexpectedEOF {
667 - t.Fatalf("read #3: unexpected error: %v", err)
668 - }
669 -
670 - if _, err := r.Next(); err != io.EOF {
671 - t.Fatalf("last next: unexpected error: %v", err)
672 - }
673 -}
674 -
675 -func TestCorrupt_FirstChuckLengthOverflow(t *testing.T) {
676 - buf := new(bytes.Buffer)
677 -
678 - w := NewWriter(buf)
679 -
680 - // First record.
681 - ww, err := w.Next()
682 - if err != nil {
683 - t.Fatal(err)
684 - }
685 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
686 - t.Fatalf("write #0: unexpected error: %v", err)
687 - }
688 -
689 - // Second record.
690 - ww, err = w.Next()
691 - if err != nil {
692 - t.Fatal(err)
693 - }
694 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
695 - t.Fatalf("write #1: unexpected error: %v", err)
696 - }
697 -
698 - // Third record.
699 - ww, err = w.Next()
700 - if err != nil {
701 - t.Fatal(err)
702 - }
703 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
704 - t.Fatalf("write #2: unexpected error: %v", err)
705 - }
706 -
707 - if err := w.Close(); err != nil {
708 - t.Fatal(err)
709 - }
710 -
711 - b := buf.Bytes()
712 - // Corrupting record #1.
713 - x := blockSize
714 - binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
715 -
716 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
717 -
718 - // First read (first record).
719 - rr, err := r.Next()
720 - if err != nil {
721 - t.Fatal(err)
722 - }
723 - n, err := io.Copy(ioutil.Discard, rr)
724 - if err != nil {
725 - t.Fatalf("read #0: %v", err)
726 - }
727 - if want := int64(blockSize / 2); n != want {
728 - t.Fatalf("read #0: got %d bytes want %d", n, want)
729 - }
730 -
731 - // Second read (second record).
732 - rr, err = r.Next()
733 - if err != nil {
734 - t.Fatal(err)
735 - }
736 - n, err = io.Copy(ioutil.Discard, rr)
737 - if err != io.ErrUnexpectedEOF {
738 - t.Fatalf("read #1: unexpected error: %v", err)
739 - }
740 -
741 - if _, err := r.Next(); err != io.EOF {
742 - t.Fatalf("last next: unexpected error: %v", err)
743 - }
744 -}
745 -
746 -func TestCorrupt_MiddleChuckLengthOverflow(t *testing.T) {
747 - buf := new(bytes.Buffer)
748 -
749 - w := NewWriter(buf)
750 -
751 - // First record.
752 - ww, err := w.Next()
753 - if err != nil {
754 - t.Fatal(err)
755 - }
756 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize/2)); err != nil {
757 - t.Fatalf("write #0: unexpected error: %v", err)
758 - }
759 -
760 - // Second record.
761 - ww, err = w.Next()
762 - if err != nil {
763 - t.Fatal(err)
764 - }
765 - if _, err := ww.Write(bytes.Repeat([]byte("0"), blockSize-headerSize)); err != nil {
766 - t.Fatalf("write #1: unexpected error: %v", err)
767 - }
768 -
769 - // Third record.
770 - ww, err = w.Next()
771 - if err != nil {
772 - t.Fatal(err)
773 - }
774 - if _, err := ww.Write(bytes.Repeat([]byte("0"), (blockSize-headerSize)+1)); err != nil {
775 - t.Fatalf("write #2: unexpected error: %v", err)
776 - }
777 -
778 - if err := w.Close(); err != nil {
779 - t.Fatal(err)
780 - }
781 -
782 - b := buf.Bytes()
783 - // Corrupting record #1.
784 - x := blockSize/2 + headerSize
785 - binary.LittleEndian.PutUint16(b[x+4:], 0xffff)
786 -
787 - r := NewReader(bytes.NewReader(b), dropper{t}, false, true)
788 -
789 - // First read (first record).
790 - rr, err := r.Next()
791 - if err != nil {
792 - t.Fatal(err)
793 - }
794 - n, err := io.Copy(ioutil.Discard, rr)
795 - if err != nil {
796 - t.Fatalf("read #0: %v", err)
797 - }
798 - if want := int64(blockSize / 2); n != want {
799 - t.Fatalf("read #0: got %d bytes want %d", n, want)
800 - }
801 -
802 - // Second read (third record).
803 - rr, err = r.Next()
804 - if err != nil {
805 - t.Fatal(err)
806 - }
807 - n, err = io.Copy(ioutil.Discard, rr)
808 - if err != nil {
809 - t.Fatalf("read #1: %v", err)
810 - }
811 - if want := int64(blockSize-headerSize) + 1; n != want {
812 - t.Fatalf("read #1: got %d bytes want %d", n, want)
813 - }
814 -
815 - if _, err := r.Next(); err != io.EOF {
816 - t.Fatalf("last next: unexpected error: %v", err)
817 - }
818 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key.go deleted
-142
@@ -1,142 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "encoding/binary"
11 - "fmt"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
14 -)
15 -
16 -type ErrIkeyCorrupted struct {
17 - Ikey []byte
18 - Reason string
19 -}
20 -
21 -func (e *ErrIkeyCorrupted) Error() string {
22 - return fmt.Sprintf("leveldb: iKey %q corrupted: %s", e.Ikey, e.Reason)
23 -}
24 -
25 -func newErrIkeyCorrupted(ikey []byte, reason string) error {
26 - return errors.NewErrCorrupted(nil, &ErrIkeyCorrupted{append([]byte{}, ikey...), reason})
27 -}
28 -
29 -type kType int
30 -
31 -func (kt kType) String() string {
32 - switch kt {
33 - case ktDel:
34 - return "d"
35 - case ktVal:
36 - return "v"
37 - }
38 - return "x"
39 -}
40 -
41 -// Value types encoded as the last component of internal keys.
42 -// Don't modify; this value are saved to disk.
43 -const (
44 - ktDel kType = iota
45 - ktVal
46 -)
47 -
48 -// ktSeek defines the kType that should be passed when constructing an
49 -// internal key for seeking to a particular sequence number (since we
50 -// sort sequence numbers in decreasing order and the value type is
51 -// embedded as the low 8 bits in the sequence number in internal keys,
52 -// we need to use the highest-numbered ValueType, not the lowest).
53 -const ktSeek = ktVal
54 -
55 -const (
56 - // Maximum value possible for sequence number; the 8-bits are
57 - // used by value type, so its can packed together in single
58 - // 64-bit integer.
59 - kMaxSeq uint64 = (uint64(1) << 56) - 1
60 - // Maximum value possible for packed sequence number and type.
61 - kMaxNum uint64 = (kMaxSeq << 8) | uint64(ktSeek)
62 -)
63 -
64 -// Maximum number encoded in bytes.
65 -var kMaxNumBytes = make([]byte, 8)
66 -
67 -func init() {
68 - binary.LittleEndian.PutUint64(kMaxNumBytes, kMaxNum)
69 -}
70 -
71 -type iKey []byte
72 -
73 -func newIkey(ukey []byte, seq uint64, kt kType) iKey {
74 - if seq > kMaxSeq {
75 - panic("leveldb: invalid sequence number")
76 - } else if kt > ktVal {
77 - panic("leveldb: invalid type")
78 - }
79 -
80 - ik := make(iKey, len(ukey)+8)
81 - copy(ik, ukey)
82 - binary.LittleEndian.PutUint64(ik[len(ukey):], (seq<<8)|uint64(kt))
83 - return ik
84 -}
85 -
86 -func parseIkey(ik []byte) (ukey []byte, seq uint64, kt kType, err error) {
87 - if len(ik) < 8 {
88 - return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid length")
89 - }
90 - num := binary.LittleEndian.Uint64(ik[len(ik)-8:])
91 - seq, kt = uint64(num>>8), kType(num&0xff)
92 - if kt > ktVal {
93 - return nil, 0, 0, newErrIkeyCorrupted(ik, "invalid type")
94 - }
95 - ukey = ik[:len(ik)-8]
96 - return
97 -}
98 -
99 -func validIkey(ik []byte) bool {
100 - _, _, _, err := parseIkey(ik)
101 - return err == nil
102 -}
103 -
104 -func (ik iKey) assert() {
105 - if ik == nil {
106 - panic("leveldb: nil iKey")
107 - }
108 - if len(ik) < 8 {
109 - panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid length", []byte(ik), len(ik)))
110 - }
111 -}
112 -
113 -func (ik iKey) ukey() []byte {
114 - ik.assert()
115 - return ik[:len(ik)-8]
116 -}
117 -
118 -func (ik iKey) num() uint64 {
119 - ik.assert()
120 - return binary.LittleEndian.Uint64(ik[len(ik)-8:])
121 -}
122 -
123 -func (ik iKey) parseNum() (seq uint64, kt kType) {
124 - num := ik.num()
125 - seq, kt = uint64(num>>8), kType(num&0xff)
126 - if kt > ktVal {
127 - panic(fmt.Sprintf("leveldb: iKey %q, len=%d: invalid type %#x", []byte(ik), len(ik), kt))
128 - }
129 - return
130 -}
131 -
132 -func (ik iKey) String() string {
133 - if ik == nil {
134 - return "<nil>"
135 - }
136 -
137 - if ukey, seq, kt, err := parseIkey(ik); err == nil {
138 - return fmt.Sprintf("%s,%s%d", shorten(string(ukey)), kt, seq)
139 - } else {
140 - return "<invalid>"
141 - }
142 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/key_test.go deleted
-133
@@ -1,133 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "testing"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
14 -)
15 -
16 -var defaultIComparer = &iComparer{comparer.DefaultComparer}
17 -
18 -func ikey(key string, seq uint64, kt kType) iKey {
19 - return newIkey([]byte(key), uint64(seq), kt)
20 -}
21 -
22 -func shortSep(a, b []byte) []byte {
23 - dst := make([]byte, len(a))
24 - dst = defaultIComparer.Separator(dst[:0], a, b)
25 - if dst == nil {
26 - return a
27 - }
28 - return dst
29 -}
30 -
31 -func shortSuccessor(b []byte) []byte {
32 - dst := make([]byte, len(b))
33 - dst = defaultIComparer.Successor(dst[:0], b)
34 - if dst == nil {
35 - return b
36 - }
37 - return dst
38 -}
39 -
40 -func testSingleKey(t *testing.T, key string, seq uint64, kt kType) {
41 - ik := ikey(key, seq, kt)
42 -
43 - if !bytes.Equal(ik.ukey(), []byte(key)) {
44 - t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
45 - }
46 -
47 - rseq, rt := ik.parseNum()
48 - if rseq != seq {
49 - t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
50 - }
51 - if rt != kt {
52 - t.Errorf("type does not equal, got %v, want %v", rt, kt)
53 - }
54 -
55 - if rukey, rseq, rt, kerr := parseIkey(ik); kerr == nil {
56 - if !bytes.Equal(rukey, []byte(key)) {
57 - t.Errorf("user key does not equal, got %v, want %v", string(ik.ukey()), key)
58 - }
59 - if rseq != seq {
60 - t.Errorf("seq number does not equal, got %v, want %v", rseq, seq)
61 - }
62 - if rt != kt {
63 - t.Errorf("type does not equal, got %v, want %v", rt, kt)
64 - }
65 - } else {
66 - t.Errorf("key error: %v", kerr)
67 - }
68 -}
69 -
70 -func TestIkey_EncodeDecode(t *testing.T) {
71 - keys := []string{"", "k", "hello", "longggggggggggggggggggggg"}
72 - seqs := []uint64{
73 - 1, 2, 3,
74 - (1 << 8) - 1, 1 << 8, (1 << 8) + 1,
75 - (1 << 16) - 1, 1 << 16, (1 << 16) + 1,
76 - (1 << 32) - 1, 1 << 32, (1 << 32) + 1,
77 - }
78 - for _, key := range keys {
79 - for _, seq := range seqs {
80 - testSingleKey(t, key, seq, ktVal)
81 - testSingleKey(t, "hello", 1, ktDel)
82 - }
83 - }
84 -}
85 -
86 -func assertBytes(t *testing.T, want, got []byte) {
87 - if !bytes.Equal(got, want) {
88 - t.Errorf("assert failed, got %v, want %v", got, want)
89 - }
90 -}
91 -
92 -func TestIkeyShortSeparator(t *testing.T) {
93 - // When user keys are same
94 - assertBytes(t, ikey("foo", 100, ktVal),
95 - shortSep(ikey("foo", 100, ktVal),
96 - ikey("foo", 99, ktVal)))
97 - assertBytes(t, ikey("foo", 100, ktVal),
98 - shortSep(ikey("foo", 100, ktVal),
99 - ikey("foo", 101, ktVal)))
100 - assertBytes(t, ikey("foo", 100, ktVal),
101 - shortSep(ikey("foo", 100, ktVal),
102 - ikey("foo", 100, ktVal)))
103 - assertBytes(t, ikey("foo", 100, ktVal),
104 - shortSep(ikey("foo", 100, ktVal),
105 - ikey("foo", 100, ktDel)))
106 -
107 - // When user keys are misordered
108 - assertBytes(t, ikey("foo", 100, ktVal),
109 - shortSep(ikey("foo", 100, ktVal),
110 - ikey("bar", 99, ktVal)))
111 -
112 - // When user keys are different, but correctly ordered
113 - assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
114 - shortSep(ikey("foo", 100, ktVal),
115 - ikey("hello", 200, ktVal)))
116 -
117 - // When start user key is prefix of limit user key
118 - assertBytes(t, ikey("foo", 100, ktVal),
119 - shortSep(ikey("foo", 100, ktVal),
120 - ikey("foobar", 200, ktVal)))
121 -
122 - // When limit user key is prefix of start user key
123 - assertBytes(t, ikey("foobar", 100, ktVal),
124 - shortSep(ikey("foobar", 100, ktVal),
125 - ikey("foo", 200, ktVal)))
126 -}
127 -
128 -func TestIkeyShortestSuccessor(t *testing.T) {
129 - assertBytes(t, ikey("g", uint64(kMaxSeq), ktSeek),
130 - shortSuccessor(ikey("foo", 100, ktVal)))
131 - assertBytes(t, ikey("\xff\xff", 100, ktVal),
132 - shortSuccessor(ikey("\xff\xff", 100, ktVal)))
133 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/leveldb_suite_test.go deleted
-11
@@ -1,11 +0,0 @@
1 -package leveldb
2 -
3 -import (
4 - "testing"
5 -
6 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7 -)
8 -
9 -func TestLevelDB(t *testing.T) {
10 - testutil.RunSuite(t, "LevelDB Suite")
11 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/bench_test.go deleted
-75
@@ -1,75 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package memdb
8 -
9 -import (
10 - "encoding/binary"
11 - "math/rand"
12 - "testing"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
15 -)
16 -
17 -func BenchmarkPut(b *testing.B) {
18 - buf := make([][4]byte, b.N)
19 - for i := range buf {
20 - binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
21 - }
22 -
23 - b.ResetTimer()
24 - p := New(comparer.DefaultComparer, 0)
25 - for i := range buf {
26 - p.Put(buf[i][:], nil)
27 - }
28 -}
29 -
30 -func BenchmarkPutRandom(b *testing.B) {
31 - buf := make([][4]byte, b.N)
32 - for i := range buf {
33 - binary.LittleEndian.PutUint32(buf[i][:], uint32(rand.Int()))
34 - }
35 -
36 - b.ResetTimer()
37 - p := New(comparer.DefaultComparer, 0)
38 - for i := range buf {
39 - p.Put(buf[i][:], nil)
40 - }
41 -}
42 -
43 -func BenchmarkGet(b *testing.B) {
44 - buf := make([][4]byte, b.N)
45 - for i := range buf {
46 - binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
47 - }
48 -
49 - p := New(comparer.DefaultComparer, 0)
50 - for i := range buf {
51 - p.Put(buf[i][:], nil)
52 - }
53 -
54 - b.ResetTimer()
55 - for i := range buf {
56 - p.Get(buf[i][:])
57 - }
58 -}
59 -
60 -func BenchmarkGetRandom(b *testing.B) {
61 - buf := make([][4]byte, b.N)
62 - for i := range buf {
63 - binary.LittleEndian.PutUint32(buf[i][:], uint32(i))
64 - }
65 -
66 - p := New(comparer.DefaultComparer, 0)
67 - for i := range buf {
68 - p.Put(buf[i][:], nil)
69 - }
70 -
71 - b.ResetTimer()
72 - for i := 0; i < b.N; i++ {
73 - p.Get(buf[rand.Int()%b.N][:])
74 - }
75 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb.go deleted
-468
@@ -1,468 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package memdb provides in-memory key/value database implementation.
8 -package memdb
9 -
10 -import (
11 - "math/rand"
12 - "sync"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18 -)
19 -
20 -var (
21 - ErrNotFound = errors.ErrNotFound
22 - ErrIterReleased = errors.New("leveldb/memdb: iterator released")
23 -)
24 -
25 -const tMaxHeight = 12
26 -
27 -type dbIter struct {
28 - util.BasicReleaser
29 - p *DB
30 - slice *util.Range
31 - node int
32 - forward bool
33 - key, value []byte
34 - err error
35 -}
36 -
37 -func (i *dbIter) fill(checkStart, checkLimit bool) bool {
38 - if i.node != 0 {
39 - n := i.p.nodeData[i.node]
40 - m := n + i.p.nodeData[i.node+nKey]
41 - i.key = i.p.kvData[n:m]
42 - if i.slice != nil {
43 - switch {
44 - case checkLimit && i.slice.Limit != nil && i.p.cmp.Compare(i.key, i.slice.Limit) >= 0:
45 - fallthrough
46 - case checkStart && i.slice.Start != nil && i.p.cmp.Compare(i.key, i.slice.Start) < 0:
47 - i.node = 0
48 - goto bail
49 - }
50 - }
51 - i.value = i.p.kvData[m : m+i.p.nodeData[i.node+nVal]]
52 - return true
53 - }
54 -bail:
55 - i.key = nil
56 - i.value = nil
57 - return false
58 -}
59 -
60 -func (i *dbIter) Valid() bool {
61 - return i.node != 0
62 -}
63 -
64 -func (i *dbIter) First() bool {
65 - if i.Released() {
66 - i.err = ErrIterReleased
67 - return false
68 - }
69 -
70 - i.forward = true
71 - i.p.mu.RLock()
72 - defer i.p.mu.RUnlock()
73 - if i.slice != nil && i.slice.Start != nil {
74 - i.node, _ = i.p.findGE(i.slice.Start, false)
75 - } else {
76 - i.node = i.p.nodeData[nNext]
77 - }
78 - return i.fill(false, true)
79 -}
80 -
81 -func (i *dbIter) Last() bool {
82 - if i.Released() {
83 - i.err = ErrIterReleased
84 - return false
85 - }
86 -
87 - i.forward = false
88 - i.p.mu.RLock()
89 - defer i.p.mu.RUnlock()
90 - if i.slice != nil && i.slice.Limit != nil {
91 - i.node = i.p.findLT(i.slice.Limit)
92 - } else {
93 - i.node = i.p.findLast()
94 - }
95 - return i.fill(true, false)
96 -}
97 -
98 -func (i *dbIter) Seek(key []byte) bool {
99 - if i.Released() {
100 - i.err = ErrIterReleased
101 - return false
102 - }
103 -
104 - i.forward = true
105 - i.p.mu.RLock()
106 - defer i.p.mu.RUnlock()
107 - if i.slice != nil && i.slice.Start != nil && i.p.cmp.Compare(key, i.slice.Start) < 0 {
108 - key = i.slice.Start
109 - }
110 - i.node, _ = i.p.findGE(key, false)
111 - return i.fill(false, true)
112 -}
113 -
114 -func (i *dbIter) Next() bool {
115 - if i.Released() {
116 - i.err = ErrIterReleased
117 - return false
118 - }
119 -
120 - if i.node == 0 {
121 - if !i.forward {
122 - return i.First()
123 - }
124 - return false
125 - }
126 - i.forward = true
127 - i.p.mu.RLock()
128 - defer i.p.mu.RUnlock()
129 - i.node = i.p.nodeData[i.node+nNext]
130 - return i.fill(false, true)
131 -}
132 -
133 -func (i *dbIter) Prev() bool {
134 - if i.Released() {
135 - i.err = ErrIterReleased
136 - return false
137 - }
138 -
139 - if i.node == 0 {
140 - if i.forward {
141 - return i.Last()
142 - }
143 - return false
144 - }
145 - i.forward = false
146 - i.p.mu.RLock()
147 - defer i.p.mu.RUnlock()
148 - i.node = i.p.findLT(i.key)
149 - return i.fill(true, false)
150 -}
151 -
152 -func (i *dbIter) Key() []byte {
153 - return i.key
154 -}
155 -
156 -func (i *dbIter) Value() []byte {
157 - return i.value
158 -}
159 -
160 -func (i *dbIter) Error() error { return i.err }
161 -
162 -func (i *dbIter) Release() {
163 - if !i.Released() {
164 - i.p = nil
165 - i.node = 0
166 - i.key = nil
167 - i.value = nil
168 - i.BasicReleaser.Release()
169 - }
170 -}
171 -
172 -const (
173 - nKV = iota
174 - nKey
175 - nVal
176 - nHeight
177 - nNext
178 -)
179 -
180 -// DB is an in-memory key/value database.
181 -type DB struct {
182 - cmp comparer.BasicComparer
183 - rnd *rand.Rand
184 -
185 - mu sync.RWMutex
186 - kvData []byte
187 - // Node data:
188 - // [0] : KV offset
189 - // [1] : Key length
190 - // [2] : Value length
191 - // [3] : Height
192 - // [3..height] : Next nodes
193 - nodeData []int
194 - prevNode [tMaxHeight]int
195 - maxHeight int
196 - n int
197 - kvSize int
198 -}
199 -
200 -func (p *DB) randHeight() (h int) {
201 - const branching = 4
202 - h = 1
203 - for h < tMaxHeight && p.rnd.Int()%branching == 0 {
204 - h++
205 - }
206 - return
207 -}
208 -
209 -func (p *DB) findGE(key []byte, prev bool) (int, bool) {
210 - node := 0
211 - h := p.maxHeight - 1
212 - for {
213 - next := p.nodeData[node+nNext+h]
214 - cmp := 1
215 - if next != 0 {
216 - o := p.nodeData[next]
217 - cmp = p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key)
218 - }
219 - if cmp < 0 {
220 - // Keep searching in this list
221 - node = next
222 - } else {
223 - if prev {
224 - p.prevNode[h] = node
225 - } else if cmp == 0 {
226 - return next, true
227 - }
228 - if h == 0 {
229 - return next, cmp == 0
230 - }
231 - h--
232 - }
233 - }
234 -}
235 -
236 -func (p *DB) findLT(key []byte) int {
237 - node := 0
238 - h := p.maxHeight - 1
239 - for {
240 - next := p.nodeData[node+nNext+h]
241 - o := p.nodeData[next]
242 - if next == 0 || p.cmp.Compare(p.kvData[o:o+p.nodeData[next+nKey]], key) >= 0 {
243 - if h == 0 {
244 - break
245 - }
246 - h--
247 - } else {
248 - node = next
249 - }
250 - }
251 - return node
252 -}
253 -
254 -func (p *DB) findLast() int {
255 - node := 0
256 - h := p.maxHeight - 1
257 - for {
258 - next := p.nodeData[node+nNext+h]
259 - if next == 0 {
260 - if h == 0 {
261 - break
262 - }
263 - h--
264 - } else {
265 - node = next
266 - }
267 - }
268 - return node
269 -}
270 -
271 -// Put sets the value for the given key. It overwrites any previous value
272 -// for that key; a DB is not a multi-map.
273 -//
274 -// It is safe to modify the contents of the arguments after Put returns.
275 -func (p *DB) Put(key []byte, value []byte) error {
276 - p.mu.Lock()
277 - defer p.mu.Unlock()
278 -
279 - if node, exact := p.findGE(key, true); exact {
280 - kvOffset := len(p.kvData)
281 - p.kvData = append(p.kvData, key...)
282 - p.kvData = append(p.kvData, value...)
283 - p.nodeData[node] = kvOffset
284 - m := p.nodeData[node+nVal]
285 - p.nodeData[node+nVal] = len(value)
286 - p.kvSize += len(value) - m
287 - return nil
288 - }
289 -
290 - h := p.randHeight()
291 - if h > p.maxHeight {
292 - for i := p.maxHeight; i < h; i++ {
293 - p.prevNode[i] = 0
294 - }
295 - p.maxHeight = h
296 - }
297 -
298 - kvOffset := len(p.kvData)
299 - p.kvData = append(p.kvData, key...)
300 - p.kvData = append(p.kvData, value...)
301 - // Node
302 - node := len(p.nodeData)
303 - p.nodeData = append(p.nodeData, kvOffset, len(key), len(value), h)
304 - for i, n := range p.prevNode[:h] {
305 - m := n + 4 + i
306 - p.nodeData = append(p.nodeData, p.nodeData[m])
307 - p.nodeData[m] = node
308 - }
309 -
310 - p.kvSize += len(key) + len(value)
311 - p.n++
312 - return nil
313 -}
314 -
315 -// Delete deletes the value for the given key. It returns ErrNotFound if
316 -// the DB does not contain the key.
317 -//
318 -// It is safe to modify the contents of the arguments after Delete returns.
319 -func (p *DB) Delete(key []byte) error {
320 - p.mu.Lock()
321 - defer p.mu.Unlock()
322 -
323 - node, exact := p.findGE(key, true)
324 - if !exact {
325 - return ErrNotFound
326 - }
327 -
328 - h := p.nodeData[node+nHeight]
329 - for i, n := range p.prevNode[:h] {
330 - m := n + 4 + i
331 - p.nodeData[m] = p.nodeData[p.nodeData[m]+nNext+i]
332 - }
333 -
334 - p.kvSize -= p.nodeData[node+nKey] + p.nodeData[node+nVal]
335 - p.n--
336 - return nil
337 -}
338 -
339 -// Contains returns true if the given key are in the DB.
340 -//
341 -// It is safe to modify the contents of the arguments after Contains returns.
342 -func (p *DB) Contains(key []byte) bool {
343 - p.mu.RLock()
344 - _, exact := p.findGE(key, false)
345 - p.mu.RUnlock()
346 - return exact
347 -}
348 -
349 -// Get gets the value for the given key. It returns error.ErrNotFound if the
350 -// DB does not contain the key.
351 -//
352 -// The caller should not modify the contents of the returned slice, but
353 -// it is safe to modify the contents of the argument after Get returns.
354 -func (p *DB) Get(key []byte) (value []byte, err error) {
355 - p.mu.RLock()
356 - if node, exact := p.findGE(key, false); exact {
357 - o := p.nodeData[node] + p.nodeData[node+nKey]
358 - value = p.kvData[o : o+p.nodeData[node+nVal]]
359 - } else {
360 - err = ErrNotFound
361 - }
362 - p.mu.RUnlock()
363 - return
364 -}
365 -
366 -// Find finds key/value pair whose key is greater than or equal to the
367 -// given key. It returns ErrNotFound if the table doesn't contain
368 -// such pair.
369 -//
370 -// The caller should not modify the contents of the returned slice, but
371 -// it is safe to modify the contents of the argument after Find returns.
372 -func (p *DB) Find(key []byte) (rkey, value []byte, err error) {
373 - p.mu.RLock()
374 - if node, _ := p.findGE(key, false); node != 0 {
375 - n := p.nodeData[node]
376 - m := n + p.nodeData[node+nKey]
377 - rkey = p.kvData[n:m]
378 - value = p.kvData[m : m+p.nodeData[node+nVal]]
379 - } else {
380 - err = ErrNotFound
381 - }
382 - p.mu.RUnlock()
383 - return
384 -}
385 -
386 -// NewIterator returns an iterator of the DB.
387 -// The returned iterator is not goroutine-safe, but it is safe to use
388 -// multiple iterators concurrently, with each in a dedicated goroutine.
389 -// It is also safe to use an iterator concurrently with modifying its
390 -// underlying DB. However, the resultant key/value pairs are not guaranteed
391 -// to be a consistent snapshot of the DB at a particular point in time.
392 -//
393 -// Slice allows slicing the iterator to only contains keys in the given
394 -// range. A nil Range.Start is treated as a key before all keys in the
395 -// DB. And a nil Range.Limit is treated as a key after all keys in
396 -// the DB.
397 -//
398 -// The iterator must be released after use, by calling Release method.
399 -//
400 -// Also read Iterator documentation of the leveldb/iterator package.
401 -func (p *DB) NewIterator(slice *util.Range) iterator.Iterator {
402 - return &dbIter{p: p, slice: slice}
403 -}
404 -
405 -// Capacity returns keys/values buffer capacity.
406 -func (p *DB) Capacity() int {
407 - p.mu.RLock()
408 - defer p.mu.RUnlock()
409 - return cap(p.kvData)
410 -}
411 -
412 -// Size returns sum of keys and values length. Note that deleted
413 -// key/value will not be accouted for, but it will still consume
414 -// the buffer, since the buffer is append only.
415 -func (p *DB) Size() int {
416 - p.mu.RLock()
417 - defer p.mu.RUnlock()
418 - return p.kvSize
419 -}
420 -
421 -// Free returns keys/values free buffer before need to grow.
422 -func (p *DB) Free() int {
423 - p.mu.RLock()
424 - defer p.mu.RUnlock()
425 - return cap(p.kvData) - len(p.kvData)
426 -}
427 -
428 -// Len returns the number of entries in the DB.
429 -func (p *DB) Len() int {
430 - p.mu.RLock()
431 - defer p.mu.RUnlock()
432 - return p.n
433 -}
434 -
435 -// Reset resets the DB to initial empty state. Allows reuse the buffer.
436 -func (p *DB) Reset() {
437 - p.rnd = rand.New(rand.NewSource(0xdeadbeef))
438 - p.maxHeight = 1
439 - p.n = 0
440 - p.kvSize = 0
441 - p.kvData = p.kvData[:0]
442 - p.nodeData = p.nodeData[:4+tMaxHeight]
443 - p.nodeData[nKV] = 0
444 - p.nodeData[nKey] = 0
445 - p.nodeData[nVal] = 0
446 - p.nodeData[nHeight] = tMaxHeight
447 - for n := 0; n < tMaxHeight; n++ {
448 - p.nodeData[4+n] = 0
449 - p.prevNode[n] = 0
450 - }
451 -}
452 -
453 -// New creates a new initalized in-memory key/value DB. The capacity
454 -// is the initial key/value buffer capacity. The capacity is advisory,
455 -// not enforced.
456 -//
457 -// The returned DB instance is goroutine-safe.
458 -func New(cmp comparer.BasicComparer, capacity int) *DB {
459 - p := &DB{
460 - cmp: cmp,
461 - rnd: rand.New(rand.NewSource(0xdeadbeef)),
462 - maxHeight: 1,
463 - kvData: make([]byte, 0, capacity),
464 - nodeData: make([]int, 4+tMaxHeight),
465 - }
466 - p.nodeData[nHeight] = tMaxHeight
467 - return p
468 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_suite_test.go deleted
-11
@@ -1,11 +0,0 @@
1 -package memdb
2 -
3 -import (
4 - "testing"
5 -
6 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7 -)
8 -
9 -func TestMemDB(t *testing.T) {
10 - testutil.RunSuite(t, "MemDB Suite")
11 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/memdb/memdb_test.go deleted
-135
@@ -1,135 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package memdb
8 -
9 -import (
10 - . "github.com/onsi/ginkgo"
11 - . "github.com/onsi/gomega"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
17 -)
18 -
19 -func (p *DB) TestFindLT(key []byte) (rkey, value []byte, err error) {
20 - p.mu.RLock()
21 - if node := p.findLT(key); node != 0 {
22 - n := p.nodeData[node]
23 - m := n + p.nodeData[node+nKey]
24 - rkey = p.kvData[n:m]
25 - value = p.kvData[m : m+p.nodeData[node+nVal]]
26 - } else {
27 - err = ErrNotFound
28 - }
29 - p.mu.RUnlock()
30 - return
31 -}
32 -
33 -func (p *DB) TestFindLast() (rkey, value []byte, err error) {
34 - p.mu.RLock()
35 - if node := p.findLast(); node != 0 {
36 - n := p.nodeData[node]
37 - m := n + p.nodeData[node+nKey]
38 - rkey = p.kvData[n:m]
39 - value = p.kvData[m : m+p.nodeData[node+nVal]]
40 - } else {
41 - err = ErrNotFound
42 - }
43 - p.mu.RUnlock()
44 - return
45 -}
46 -
47 -func (p *DB) TestPut(key []byte, value []byte) error {
48 - p.Put(key, value)
49 - return nil
50 -}
51 -
52 -func (p *DB) TestDelete(key []byte) error {
53 - p.Delete(key)
54 - return nil
55 -}
56 -
57 -func (p *DB) TestFind(key []byte) (rkey, rvalue []byte, err error) {
58 - return p.Find(key)
59 -}
60 -
61 -func (p *DB) TestGet(key []byte) (value []byte, err error) {
62 - return p.Get(key)
63 -}
64 -
65 -func (p *DB) TestNewIterator(slice *util.Range) iterator.Iterator {
66 - return p.NewIterator(slice)
67 -}
68 -
69 -var _ = testutil.Defer(func() {
70 - Describe("Memdb", func() {
71 - Describe("write test", func() {
72 - It("should do write correctly", func() {
73 - db := New(comparer.DefaultComparer, 0)
74 - t := testutil.DBTesting{
75 - DB: db,
76 - Deleted: testutil.KeyValue_Generate(nil, 1000, 1, 30, 5, 5).Clone(),
77 - PostFn: func(t *testutil.DBTesting) {
78 - Expect(db.Len()).Should(Equal(t.Present.Len()))
79 - Expect(db.Size()).Should(Equal(t.Present.Size()))
80 - switch t.Act {
81 - case testutil.DBPut, testutil.DBOverwrite:
82 - Expect(db.Contains(t.ActKey)).Should(BeTrue())
83 - default:
84 - Expect(db.Contains(t.ActKey)).Should(BeFalse())
85 - }
86 - },
87 - }
88 - testutil.DoDBTesting(&t)
89 - })
90 - })
91 -
92 - Describe("read test", func() {
93 - testutil.AllKeyValueTesting(nil, func(kv testutil.KeyValue) testutil.DB {
94 - // Building the DB.
95 - db := New(comparer.DefaultComparer, 0)
96 - kv.IterateShuffled(nil, func(i int, key, value []byte) {
97 - db.Put(key, value)
98 - })
99 -
100 - if kv.Len() > 1 {
101 - It("Should find correct keys with findLT", func() {
102 - testutil.ShuffledIndex(nil, kv.Len()-1, 1, func(i int) {
103 - key_, key, _ := kv.IndexInexact(i + 1)
104 - expectedKey, expectedValue := kv.Index(i)
105 -
106 - // Using key that exist.
107 - rkey, rvalue, err := db.TestFindLT(key)
108 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q -> %q", key, expectedKey)
109 - Expect(rkey).Should(Equal(expectedKey), "Key")
110 - Expect(rvalue).Should(Equal(expectedValue), "Value for key %q -> %q", key, expectedKey)
111 -
112 - // Using key that doesn't exist.
113 - rkey, rvalue, err = db.TestFindLT(key_)
114 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q) -> %q", key_, key, expectedKey)
115 - Expect(rkey).Should(Equal(expectedKey))
116 - Expect(rvalue).Should(Equal(expectedValue), "Value for key %q (%q) -> %q", key_, key, expectedKey)
117 - })
118 - })
119 - }
120 -
121 - if kv.Len() > 0 {
122 - It("Should find last key with findLast", func() {
123 - key, value := kv.Index(kv.Len() - 1)
124 - rkey, rvalue, err := db.TestFindLast()
125 - Expect(err).ShouldNot(HaveOccurred())
126 - Expect(rkey).Should(Equal(key))
127 - Expect(rvalue).Should(Equal(value))
128 - })
129 - }
130 -
131 - return db
132 - }, nil, nil)
133 - })
134 - })
135 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt/options.go deleted
-639
@@ -1,639 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package opt provides sets of options used by LevelDB.
8 -package opt
9 -
10 -import (
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
14 - "math"
15 -)
16 -
17 -const (
18 - KiB = 1024
19 - MiB = KiB * 1024
20 - GiB = MiB * 1024
21 -)
22 -
23 -var (
24 - DefaultBlockCacher = LRUCacher
25 - DefaultBlockCacheCapacity = 8 * MiB
26 - DefaultBlockRestartInterval = 16
27 - DefaultBlockSize = 4 * KiB
28 - DefaultCompactionExpandLimitFactor = 25
29 - DefaultCompactionGPOverlapsFactor = 10
30 - DefaultCompactionL0Trigger = 4
31 - DefaultCompactionSourceLimitFactor = 1
32 - DefaultCompactionTableSize = 2 * MiB
33 - DefaultCompactionTableSizeMultiplier = 1.0
34 - DefaultCompactionTotalSize = 10 * MiB
35 - DefaultCompactionTotalSizeMultiplier = 10.0
36 - DefaultCompressionType = SnappyCompression
37 - DefaultIteratorSamplingRate = 1 * MiB
38 - DefaultMaxMemCompationLevel = 2
39 - DefaultNumLevel = 7
40 - DefaultOpenFilesCacher = LRUCacher
41 - DefaultOpenFilesCacheCapacity = 500
42 - DefaultWriteBuffer = 4 * MiB
43 - DefaultWriteL0PauseTrigger = 12
44 - DefaultWriteL0SlowdownTrigger = 8
45 -)
46 -
47 -// Cacher is a caching algorithm.
48 -type Cacher interface {
49 - New(capacity int) cache.Cacher
50 -}
51 -
52 -type CacherFunc struct {
53 - NewFunc func(capacity int) cache.Cacher
54 -}
55 -
56 -func (f *CacherFunc) New(capacity int) cache.Cacher {
57 - if f.NewFunc != nil {
58 - return f.NewFunc(capacity)
59 - }
60 - return nil
61 -}
62 -
63 -func noCacher(int) cache.Cacher { return nil }
64 -
65 -var (
66 - // LRUCacher is the LRU-cache algorithm.
67 - LRUCacher = &CacherFunc{cache.NewLRU}
68 -
69 - // NoCacher is the value to disable caching algorithm.
70 - NoCacher = &CacherFunc{}
71 -)
72 -
73 -// Compression is the 'sorted table' block compression algorithm to use.
74 -type Compression uint
75 -
76 -func (c Compression) String() string {
77 - switch c {
78 - case DefaultCompression:
79 - return "default"
80 - case NoCompression:
81 - return "none"
82 - case SnappyCompression:
83 - return "snappy"
84 - }
85 - return "invalid"
86 -}
87 -
88 -const (
89 - DefaultCompression Compression = iota
90 - NoCompression
91 - SnappyCompression
92 - nCompression
93 -)
94 -
95 -// Strict is the DB 'strict level'.
96 -type Strict uint
97 -
98 -const (
99 - // If present then a corrupted or invalid chunk or block in manifest
100 - // journal will cause an error instead of being dropped.
101 - // This will prevent database with corrupted manifest to be opened.
102 - StrictManifest Strict = 1 << iota
103 -
104 - // If present then journal chunk checksum will be verified.
105 - StrictJournalChecksum
106 -
107 - // If present then a corrupted or invalid chunk or block in journal
108 - // will cause an error instead of being dropped.
109 - // This will prevent database with corrupted journal to be opened.
110 - StrictJournal
111 -
112 - // If present then 'sorted table' block checksum will be verified.
113 - // This has effect on both 'read operation' and compaction.
114 - StrictBlockChecksum
115 -
116 - // If present then a corrupted 'sorted table' will fails compaction.
117 - // The database will enter read-only mode.
118 - StrictCompaction
119 -
120 - // If present then a corrupted 'sorted table' will halts 'read operation'.
121 - StrictReader
122 -
123 - // If present then leveldb.Recover will drop corrupted 'sorted table'.
124 - StrictRecovery
125 -
126 - // This only applicable for ReadOptions, if present then this ReadOptions
127 - // 'strict level' will override global ones.
128 - StrictOverride
129 -
130 - // StrictAll enables all strict flags.
131 - StrictAll = StrictManifest | StrictJournalChecksum | StrictJournal | StrictBlockChecksum | StrictCompaction | StrictReader | StrictRecovery
132 -
133 - // DefaultStrict is the default strict flags. Specify any strict flags
134 - // will override default strict flags as whole (i.e. not OR'ed).
135 - DefaultStrict = StrictJournalChecksum | StrictBlockChecksum | StrictCompaction | StrictReader
136 -
137 - // NoStrict disables all strict flags. Override default strict flags.
138 - NoStrict = ^StrictAll
139 -)
140 -
141 -// Options holds the optional parameters for the DB at large.
142 -type Options struct {
143 - // AltFilters defines one or more 'alternative filters'.
144 - // 'alternative filters' will be used during reads if a filter block
145 - // does not match with the 'effective filter'.
146 - //
147 - // The default value is nil
148 - AltFilters []filter.Filter
149 -
150 - // BlockCacher provides cache algorithm for LevelDB 'sorted table' block caching.
151 - // Specify NoCacher to disable caching algorithm.
152 - //
153 - // The default value is LRUCacher.
154 - BlockCacher Cacher
155 -
156 - // BlockCacheCapacity defines the capacity of the 'sorted table' block caching.
157 - // Use -1 for zero, this has same effect as specifying NoCacher to BlockCacher.
158 - //
159 - // The default value is 8MiB.
160 - BlockCacheCapacity int
161 -
162 - // BlockRestartInterval is the number of keys between restart points for
163 - // delta encoding of keys.
164 - //
165 - // The default value is 16.
166 - BlockRestartInterval int
167 -
168 - // BlockSize is the minimum uncompressed size in bytes of each 'sorted table'
169 - // block.
170 - //
171 - // The default value is 4KiB.
172 - BlockSize int
173 -
174 - // CompactionExpandLimitFactor limits compaction size after expanded.
175 - // This will be multiplied by table size limit at compaction target level.
176 - //
177 - // The default value is 25.
178 - CompactionExpandLimitFactor int
179 -
180 - // CompactionGPOverlapsFactor limits overlaps in grandparent (Level + 2) that a
181 - // single 'sorted table' generates.
182 - // This will be multiplied by table size limit at grandparent level.
183 - //
184 - // The default value is 10.
185 - CompactionGPOverlapsFactor int
186 -
187 - // CompactionL0Trigger defines number of 'sorted table' at level-0 that will
188 - // trigger compaction.
189 - //
190 - // The default value is 4.
191 - CompactionL0Trigger int
192 -
193 - // CompactionSourceLimitFactor limits compaction source size. This doesn't apply to
194 - // level-0.
195 - // This will be multiplied by table size limit at compaction target level.
196 - //
197 - // The default value is 1.
198 - CompactionSourceLimitFactor int
199 -
200 - // CompactionTableSize limits size of 'sorted table' that compaction generates.
201 - // The limits for each level will be calculated as:
202 - // CompactionTableSize * (CompactionTableSizeMultiplier ^ Level)
203 - // The multiplier for each level can also fine-tuned using CompactionTableSizeMultiplierPerLevel.
204 - //
205 - // The default value is 2MiB.
206 - CompactionTableSize int
207 -
208 - // CompactionTableSizeMultiplier defines multiplier for CompactionTableSize.
209 - //
210 - // The default value is 1.
211 - CompactionTableSizeMultiplier float64
212 -
213 - // CompactionTableSizeMultiplierPerLevel defines per-level multiplier for
214 - // CompactionTableSize.
215 - // Use zero to skip a level.
216 - //
217 - // The default value is nil.
218 - CompactionTableSizeMultiplierPerLevel []float64
219 -
220 - // CompactionTotalSize limits total size of 'sorted table' for each level.
221 - // The limits for each level will be calculated as:
222 - // CompactionTotalSize * (CompactionTotalSizeMultiplier ^ Level)
223 - // The multiplier for each level can also fine-tuned using
224 - // CompactionTotalSizeMultiplierPerLevel.
225 - //
226 - // The default value is 10MiB.
227 - CompactionTotalSize int
228 -
229 - // CompactionTotalSizeMultiplier defines multiplier for CompactionTotalSize.
230 - //
231 - // The default value is 10.
232 - CompactionTotalSizeMultiplier float64
233 -
234 - // CompactionTotalSizeMultiplierPerLevel defines per-level multiplier for
235 - // CompactionTotalSize.
236 - // Use zero to skip a level.
237 - //
238 - // The default value is nil.
239 - CompactionTotalSizeMultiplierPerLevel []float64
240 -
241 - // Comparer defines a total ordering over the space of []byte keys: a 'less
242 - // than' relationship. The same comparison algorithm must be used for reads
243 - // and writes over the lifetime of the DB.
244 - //
245 - // The default value uses the same ordering as bytes.Compare.
246 - Comparer comparer.Comparer
247 -
248 - // Compression defines the 'sorted table' block compression to use.
249 - //
250 - // The default value (DefaultCompression) uses snappy compression.
251 - Compression Compression
252 -
253 - // DisableBlockCache allows disable use of cache.Cache functionality on
254 - // 'sorted table' block.
255 - //
256 - // The default value is false.
257 - DisableBlockCache bool
258 -
259 - // DisableCompactionBackoff allows disable compaction retry backoff.
260 - //
261 - // The default value is false.
262 - DisableCompactionBackoff bool
263 -
264 - // ErrorIfExist defines whether an error should returned if the DB already
265 - // exist.
266 - //
267 - // The default value is false.
268 - ErrorIfExist bool
269 -
270 - // ErrorIfMissing defines whether an error should returned if the DB is
271 - // missing. If false then the database will be created if missing, otherwise
272 - // an error will be returned.
273 - //
274 - // The default value is false.
275 - ErrorIfMissing bool
276 -
277 - // Filter defines an 'effective filter' to use. An 'effective filter'
278 - // if defined will be used to generate per-table filter block.
279 - // The filter name will be stored on disk.
280 - // During reads LevelDB will try to find matching filter from
281 - // 'effective filter' and 'alternative filters'.
282 - //
283 - // Filter can be changed after a DB has been created. It is recommended
284 - // to put old filter to the 'alternative filters' to mitigate lack of
285 - // filter during transition period.
286 - //
287 - // A filter is used to reduce disk reads when looking for a specific key.
288 - //
289 - // The default value is nil.
290 - Filter filter.Filter
291 -
292 - // IteratorSamplingRate defines approximate gap (in bytes) between read
293 - // sampling of an iterator. The samples will be used to determine when
294 - // compaction should be triggered.
295 - //
296 - // The default is 1MiB.
297 - IteratorSamplingRate int
298 -
299 - // MaxMemCompationLevel defines maximum level a newly compacted 'memdb'
300 - // will be pushed into if doesn't creates overlap. This should less than
301 - // NumLevel. Use -1 for level-0.
302 - //
303 - // The default is 2.
304 - MaxMemCompationLevel int
305 -
306 - // NumLevel defines number of database level. The level shouldn't changed
307 - // between opens, or the database will panic.
308 - //
309 - // The default is 7.
310 - NumLevel int
311 -
312 - // OpenFilesCacher provides cache algorithm for open files caching.
313 - // Specify NoCacher to disable caching algorithm.
314 - //
315 - // The default value is LRUCacher.
316 - OpenFilesCacher Cacher
317 -
318 - // OpenFilesCacheCapacity defines the capacity of the open files caching.
319 - // Use -1 for zero, this has same effect as specifying NoCacher to OpenFilesCacher.
320 - //
321 - // The default value is 500.
322 - OpenFilesCacheCapacity int
323 -
324 - // Strict defines the DB strict level.
325 - Strict Strict
326 -
327 - // WriteBuffer defines maximum size of a 'memdb' before flushed to
328 - // 'sorted table'. 'memdb' is an in-memory DB backed by an on-disk
329 - // unsorted journal.
330 - //
331 - // LevelDB may held up to two 'memdb' at the same time.
332 - //
333 - // The default value is 4MiB.
334 - WriteBuffer int
335 -
336 - // WriteL0StopTrigger defines number of 'sorted table' at level-0 that will
337 - // pause write.
338 - //
339 - // The default value is 12.
340 - WriteL0PauseTrigger int
341 -
342 - // WriteL0SlowdownTrigger defines number of 'sorted table' at level-0 that
343 - // will trigger write slowdown.
344 - //
345 - // The default value is 8.
346 - WriteL0SlowdownTrigger int
347 -}
348 -
349 -func (o *Options) GetAltFilters() []filter.Filter {
350 - if o == nil {
351 - return nil
352 - }
353 - return o.AltFilters
354 -}
355 -
356 -func (o *Options) GetBlockCacher() Cacher {
357 - if o == nil || o.BlockCacher == nil {
358 - return DefaultBlockCacher
359 - } else if o.BlockCacher == NoCacher {
360 - return nil
361 - }
362 - return o.BlockCacher
363 -}
364 -
365 -func (o *Options) GetBlockCacheCapacity() int {
366 - if o == nil || o.BlockCacheCapacity == 0 {
367 - return DefaultBlockCacheCapacity
368 - } else if o.BlockCacheCapacity < 0 {
369 - return 0
370 - }
371 - return o.BlockCacheCapacity
372 -}
373 -
374 -func (o *Options) GetBlockRestartInterval() int {
375 - if o == nil || o.BlockRestartInterval <= 0 {
376 - return DefaultBlockRestartInterval
377 - }
378 - return o.BlockRestartInterval
379 -}
380 -
381 -func (o *Options) GetBlockSize() int {
382 - if o == nil || o.BlockSize <= 0 {
383 - return DefaultBlockSize
384 - }
385 - return o.BlockSize
386 -}
387 -
388 -func (o *Options) GetCompactionExpandLimit(level int) int {
389 - factor := DefaultCompactionExpandLimitFactor
390 - if o != nil && o.CompactionExpandLimitFactor > 0 {
391 - factor = o.CompactionExpandLimitFactor
392 - }
393 - return o.GetCompactionTableSize(level+1) * factor
394 -}
395 -
396 -func (o *Options) GetCompactionGPOverlaps(level int) int {
397 - factor := DefaultCompactionGPOverlapsFactor
398 - if o != nil && o.CompactionGPOverlapsFactor > 0 {
399 - factor = o.CompactionGPOverlapsFactor
400 - }
401 - return o.GetCompactionTableSize(level+2) * factor
402 -}
403 -
404 -func (o *Options) GetCompactionL0Trigger() int {
405 - if o == nil || o.CompactionL0Trigger == 0 {
406 - return DefaultCompactionL0Trigger
407 - }
408 - return o.CompactionL0Trigger
409 -}
410 -
411 -func (o *Options) GetCompactionSourceLimit(level int) int {
412 - factor := DefaultCompactionSourceLimitFactor
413 - if o != nil && o.CompactionSourceLimitFactor > 0 {
414 - factor = o.CompactionSourceLimitFactor
415 - }
416 - return o.GetCompactionTableSize(level+1) * factor
417 -}
418 -
419 -func (o *Options) GetCompactionTableSize(level int) int {
420 - var (
421 - base = DefaultCompactionTableSize
422 - mult float64
423 - )
424 - if o != nil {
425 - if o.CompactionTableSize > 0 {
426 - base = o.CompactionTableSize
427 - }
428 - if len(o.CompactionTableSizeMultiplierPerLevel) > level && o.CompactionTableSizeMultiplierPerLevel[level] > 0 {
429 - mult = o.CompactionTableSizeMultiplierPerLevel[level]
430 - } else if o.CompactionTableSizeMultiplier > 0 {
431 - mult = math.Pow(o.CompactionTableSizeMultiplier, float64(level))
432 - }
433 - }
434 - if mult == 0 {
435 - mult = math.Pow(DefaultCompactionTableSizeMultiplier, float64(level))
436 - }
437 - return int(float64(base) * mult)
438 -}
439 -
440 -func (o *Options) GetCompactionTotalSize(level int) int64 {
441 - var (
442 - base = DefaultCompactionTotalSize
443 - mult float64
444 - )
445 - if o != nil {
446 - if o.CompactionTotalSize > 0 {
447 - base = o.CompactionTotalSize
448 - }
449 - if len(o.CompactionTotalSizeMultiplierPerLevel) > level && o.CompactionTotalSizeMultiplierPerLevel[level] > 0 {
450 - mult = o.CompactionTotalSizeMultiplierPerLevel[level]
451 - } else if o.CompactionTotalSizeMultiplier > 0 {
452 - mult = math.Pow(o.CompactionTotalSizeMultiplier, float64(level))
453 - }
454 - }
455 - if mult == 0 {
456 - mult = math.Pow(DefaultCompactionTotalSizeMultiplier, float64(level))
457 - }
458 - return int64(float64(base) * mult)
459 -}
460 -
461 -func (o *Options) GetComparer() comparer.Comparer {
462 - if o == nil || o.Comparer == nil {
463 - return comparer.DefaultComparer
464 - }
465 - return o.Comparer
466 -}
467 -
468 -func (o *Options) GetCompression() Compression {
469 - if o == nil || o.Compression <= DefaultCompression || o.Compression >= nCompression {
470 - return DefaultCompressionType
471 - }
472 - return o.Compression
473 -}
474 -
475 -func (o *Options) GetDisableCompactionBackoff() bool {
476 - if o == nil {
477 - return false
478 - }
479 - return o.DisableCompactionBackoff
480 -}
481 -
482 -func (o *Options) GetErrorIfExist() bool {
483 - if o == nil {
484 - return false
485 - }
486 - return o.ErrorIfExist
487 -}
488 -
489 -func (o *Options) GetErrorIfMissing() bool {
490 - if o == nil {
491 - return false
492 - }
493 - return o.ErrorIfMissing
494 -}
495 -
496 -func (o *Options) GetFilter() filter.Filter {
497 - if o == nil {
498 - return nil
499 - }
500 - return o.Filter
501 -}
502 -
503 -func (o *Options) GetIteratorSamplingRate() int {
504 - if o == nil || o.IteratorSamplingRate <= 0 {
505 - return DefaultIteratorSamplingRate
506 - }
507 - return o.IteratorSamplingRate
508 -}
509 -
510 -func (o *Options) GetMaxMemCompationLevel() int {
511 - level := DefaultMaxMemCompationLevel
512 - if o != nil {
513 - if o.MaxMemCompationLevel > 0 {
514 - level = o.MaxMemCompationLevel
515 - } else if o.MaxMemCompationLevel < 0 {
516 - level = 0
517 - }
518 - }
519 - if level >= o.GetNumLevel() {
520 - return o.GetNumLevel() - 1
521 - }
522 - return level
523 -}
524 -
525 -func (o *Options) GetNumLevel() int {
526 - if o == nil || o.NumLevel <= 0 {
527 - return DefaultNumLevel
528 - }
529 - return o.NumLevel
530 -}
531 -
532 -func (o *Options) GetOpenFilesCacher() Cacher {
533 - if o == nil || o.OpenFilesCacher == nil {
534 - return DefaultOpenFilesCacher
535 - }
536 - if o.OpenFilesCacher == NoCacher {
537 - return nil
538 - }
539 - return o.OpenFilesCacher
540 -}
541 -
542 -func (o *Options) GetOpenFilesCacheCapacity() int {
543 - if o == nil || o.OpenFilesCacheCapacity == 0 {
544 - return DefaultOpenFilesCacheCapacity
545 - } else if o.OpenFilesCacheCapacity < 0 {
546 - return 0
547 - }
548 - return o.OpenFilesCacheCapacity
549 -}
550 -
551 -func (o *Options) GetStrict(strict Strict) bool {
552 - if o == nil || o.Strict == 0 {
553 - return DefaultStrict&strict != 0
554 - }
555 - return o.Strict&strict != 0
556 -}
557 -
558 -func (o *Options) GetWriteBuffer() int {
559 - if o == nil || o.WriteBuffer <= 0 {
560 - return DefaultWriteBuffer
561 - }
562 - return o.WriteBuffer
563 -}
564 -
565 -func (o *Options) GetWriteL0PauseTrigger() int {
566 - if o == nil || o.WriteL0PauseTrigger == 0 {
567 - return DefaultWriteL0PauseTrigger
568 - }
569 - return o.WriteL0PauseTrigger
570 -}
571 -
572 -func (o *Options) GetWriteL0SlowdownTrigger() int {
573 - if o == nil || o.WriteL0SlowdownTrigger == 0 {
574 - return DefaultWriteL0SlowdownTrigger
575 - }
576 - return o.WriteL0SlowdownTrigger
577 -}
578 -
579 -// ReadOptions holds the optional parameters for 'read operation'. The
580 -// 'read operation' includes Get, Find and NewIterator.
581 -type ReadOptions struct {
582 - // DontFillCache defines whether block reads for this 'read operation'
583 - // should be cached. If false then the block will be cached. This does
584 - // not affects already cached block.
585 - //
586 - // The default value is false.
587 - DontFillCache bool
588 -
589 - // Strict will be OR'ed with global DB 'strict level' unless StrictOverride
590 - // is present. Currently only StrictReader that has effect here.
591 - Strict Strict
592 -}
593 -
594 -func (ro *ReadOptions) GetDontFillCache() bool {
595 - if ro == nil {
596 - return false
597 - }
598 - return ro.DontFillCache
599 -}
600 -
601 -func (ro *ReadOptions) GetStrict(strict Strict) bool {
602 - if ro == nil {
603 - return false
604 - }
605 - return ro.Strict&strict != 0
606 -}
607 -
608 -// WriteOptions holds the optional parameters for 'write operation'. The
609 -// 'write operation' includes Write, Put and Delete.
610 -type WriteOptions struct {
611 - // Sync is whether to sync underlying writes from the OS buffer cache
612 - // through to actual disk, if applicable. Setting Sync can result in
613 - // slower writes.
614 - //
615 - // If false, and the machine crashes, then some recent writes may be lost.
616 - // Note that if it is just the process that crashes (and the machine does
617 - // not) then no writes will be lost.
618 - //
619 - // In other words, Sync being false has the same semantics as a write
620 - // system call. Sync being true means write followed by fsync.
621 - //
622 - // The default value is false.
623 - Sync bool
624 -}
625 -
626 -func (wo *WriteOptions) GetSync() bool {
627 - if wo == nil {
628 - return false
629 - }
630 - return wo.Sync
631 -}
632 -
633 -func GetStrict(o *Options, ro *ReadOptions, strict Strict) bool {
634 - if ro.GetStrict(StrictOverride) {
635 - return ro.GetStrict(strict)
636 - } else {
637 - return o.GetStrict(strict) || ro.GetStrict(strict)
638 - }
639 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/options.go deleted
-92
@@ -1,92 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
11 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
12 -)
13 -
14 -func dupOptions(o *opt.Options) *opt.Options {
15 - newo := &opt.Options{}
16 - if o != nil {
17 - *newo = *o
18 - }
19 - if newo.Strict == 0 {
20 - newo.Strict = opt.DefaultStrict
21 - }
22 - return newo
23 -}
24 -
25 -func (s *session) setOptions(o *opt.Options) {
26 - no := dupOptions(o)
27 - // Alternative filters.
28 - if filters := o.GetAltFilters(); len(filters) > 0 {
29 - no.AltFilters = make([]filter.Filter, len(filters))
30 - for i, filter := range filters {
31 - no.AltFilters[i] = &iFilter{filter}
32 - }
33 - }
34 - // Comparer.
35 - s.icmp = &iComparer{o.GetComparer()}
36 - no.Comparer = s.icmp
37 - // Filter.
38 - if filter := o.GetFilter(); filter != nil {
39 - no.Filter = &iFilter{filter}
40 - }
41 -
42 - s.o = &cachedOptions{Options: no}
43 - s.o.cache()
44 -}
45 -
46 -type cachedOptions struct {
47 - *opt.Options
48 -
49 - compactionExpandLimit []int
50 - compactionGPOverlaps []int
51 - compactionSourceLimit []int
52 - compactionTableSize []int
53 - compactionTotalSize []int64
54 -}
55 -
56 -func (co *cachedOptions) cache() {
57 - numLevel := co.Options.GetNumLevel()
58 -
59 - co.compactionExpandLimit = make([]int, numLevel)
60 - co.compactionGPOverlaps = make([]int, numLevel)
61 - co.compactionSourceLimit = make([]int, numLevel)
62 - co.compactionTableSize = make([]int, numLevel)
63 - co.compactionTotalSize = make([]int64, numLevel)
64 -
65 - for level := 0; level < numLevel; level++ {
66 - co.compactionExpandLimit[level] = co.Options.GetCompactionExpandLimit(level)
67 - co.compactionGPOverlaps[level] = co.Options.GetCompactionGPOverlaps(level)
68 - co.compactionSourceLimit[level] = co.Options.GetCompactionSourceLimit(level)
69 - co.compactionTableSize[level] = co.Options.GetCompactionTableSize(level)
70 - co.compactionTotalSize[level] = co.Options.GetCompactionTotalSize(level)
71 - }
72 -}
73 -
74 -func (co *cachedOptions) GetCompactionExpandLimit(level int) int {
75 - return co.compactionExpandLimit[level]
76 -}
77 -
78 -func (co *cachedOptions) GetCompactionGPOverlaps(level int) int {
79 - return co.compactionGPOverlaps[level]
80 -}
81 -
82 -func (co *cachedOptions) GetCompactionSourceLimit(level int) int {
83 - return co.compactionSourceLimit[level]
84 -}
85 -
86 -func (co *cachedOptions) GetCompactionTableSize(level int) int {
87 - return co.compactionTableSize[level]
88 -}
89 -
90 -func (co *cachedOptions) GetCompactionTotalSize(level int) int64 {
91 - return co.compactionTotalSize[level]
92 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session.go deleted
-455
@@ -1,455 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "fmt"
11 - "io"
12 - "os"
13 - "sync"
14 - "sync/atomic"
15 -
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
20 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
22 -)
23 -
24 -type ErrManifestCorrupted struct {
25 - Field string
26 - Reason string
27 -}
28 -
29 -func (e *ErrManifestCorrupted) Error() string {
30 - return fmt.Sprintf("leveldb: manifest corrupted (field '%s'): %s", e.Field, e.Reason)
31 -}
32 -
33 -func newErrManifestCorrupted(f storage.File, field, reason string) error {
34 - return errors.NewErrCorrupted(f, &ErrManifestCorrupted{field, reason})
35 -}
36 -
37 -// session represent a persistent database session.
38 -type session struct {
39 - // Need 64-bit alignment.
40 - stNextFileNum uint64 // current unused file number
41 - stJournalNum uint64 // current journal file number; need external synchronization
42 - stPrevJournalNum uint64 // prev journal file number; no longer used; for compatibility with older version of leveldb
43 - stSeqNum uint64 // last mem compacted seq; need external synchronization
44 - stTempFileNum uint64
45 -
46 - stor storage.Storage
47 - storLock util.Releaser
48 - o *cachedOptions
49 - icmp *iComparer
50 - tops *tOps
51 -
52 - manifest *journal.Writer
53 - manifestWriter storage.Writer
54 - manifestFile storage.File
55 -
56 - stCompPtrs []iKey // compaction pointers; need external synchronization
57 - stVersion *version // current version
58 - vmu sync.Mutex
59 -}
60 -
61 -// Creates new initialized session instance.
62 -func newSession(stor storage.Storage, o *opt.Options) (s *session, err error) {
63 - if stor == nil {
64 - return nil, os.ErrInvalid
65 - }
66 - storLock, err := stor.Lock()
67 - if err != nil {
68 - return
69 - }
70 - s = &session{
71 - stor: stor,
72 - storLock: storLock,
73 - stCompPtrs: make([]iKey, o.GetNumLevel()),
74 - }
75 - s.setOptions(o)
76 - s.tops = newTableOps(s)
77 - s.setVersion(newVersion(s))
78 - s.log("log@legend F·NumFile S·FileSize N·Entry C·BadEntry B·BadBlock Ke·KeyError D·DroppedEntry L·Level Q·SeqNum T·TimeElapsed")
79 - return
80 -}
81 -
82 -// Close session.
83 -func (s *session) close() {
84 - s.tops.close()
85 - if s.manifest != nil {
86 - s.manifest.Close()
87 - }
88 - if s.manifestWriter != nil {
89 - s.manifestWriter.Close()
90 - }
91 - s.manifest = nil
92 - s.manifestWriter = nil
93 - s.manifestFile = nil
94 - s.stVersion = nil
95 -}
96 -
97 -// Release session lock.
98 -func (s *session) release() {
99 - s.storLock.Release()
100 -}
101 -
102 -// Create a new database session; need external synchronization.
103 -func (s *session) create() error {
104 - // create manifest
105 - return s.newManifest(nil, nil)
106 -}
107 -
108 -// Recover a database session; need external synchronization.
109 -func (s *session) recover() (err error) {
110 - defer func() {
111 - if os.IsNotExist(err) {
112 - // Don't return os.ErrNotExist if the underlying storage contains
113 - // other files that belong to LevelDB. So the DB won't get trashed.
114 - if files, _ := s.stor.GetFiles(storage.TypeAll); len(files) > 0 {
115 - err = &errors.ErrCorrupted{File: &storage.FileInfo{Type: storage.TypeManifest}, Err: &errors.ErrMissingFiles{}}
116 - }
117 - }
118 - }()
119 -
120 - m, err := s.stor.GetManifest()
121 - if err != nil {
122 - return
123 - }
124 -
125 - reader, err := m.Open()
126 - if err != nil {
127 - return
128 - }
129 - defer reader.Close()
130 - strict := s.o.GetStrict(opt.StrictManifest)
131 - jr := journal.NewReader(reader, dropper{s, m}, strict, true)
132 -
133 - staging := s.stVersion.newStaging()
134 - rec := &sessionRecord{numLevel: s.o.GetNumLevel()}
135 - for {
136 - var r io.Reader
137 - r, err = jr.Next()
138 - if err != nil {
139 - if err == io.EOF {
140 - err = nil
141 - break
142 - }
143 - return errors.SetFile(err, m)
144 - }
145 -
146 - err = rec.decode(r)
147 - if err == nil {
148 - // save compact pointers
149 - for _, r := range rec.compPtrs {
150 - s.stCompPtrs[r.level] = iKey(r.ikey)
151 - }
152 - // commit record to version staging
153 - staging.commit(rec)
154 - } else {
155 - err = errors.SetFile(err, m)
156 - if strict || !errors.IsCorrupted(err) {
157 - return
158 - } else {
159 - s.logf("manifest error: %v (skipped)", errors.SetFile(err, m))
160 - }
161 - }
162 - rec.resetCompPtrs()
163 - rec.resetAddedTables()
164 - rec.resetDeletedTables()
165 - }
166 -
167 - switch {
168 - case !rec.has(recComparer):
169 - return newErrManifestCorrupted(m, "comparer", "missing")
170 - case rec.comparer != s.icmp.uName():
171 - return newErrManifestCorrupted(m, "comparer", fmt.Sprintf("mismatch: want '%s', got '%s'", s.icmp.uName(), rec.comparer))
172 - case !rec.has(recNextFileNum):
173 - return newErrManifestCorrupted(m, "next-file-num", "missing")
174 - case !rec.has(recJournalNum):
175 - return newErrManifestCorrupted(m, "journal-file-num", "missing")
176 - case !rec.has(recSeqNum):
177 - return newErrManifestCorrupted(m, "seq-num", "missing")
178 - }
179 -
180 - s.manifestFile = m
181 - s.setVersion(staging.finish())
182 - s.setNextFileNum(rec.nextFileNum)
183 - s.recordCommited(rec)
184 - return nil
185 -}
186 -
187 -// Commit session; need external synchronization.
188 -func (s *session) commit(r *sessionRecord) (err error) {
189 - v := s.version()
190 - defer v.release()
191 -
192 - // spawn new version based on current version
193 - nv := v.spawn(r)
194 -
195 - if s.manifest == nil {
196 - // manifest journal writer not yet created, create one
197 - err = s.newManifest(r, nv)
198 - } else {
199 - err = s.flushManifest(r)
200 - }
201 -
202 - // finally, apply new version if no error rise
203 - if err == nil {
204 - s.setVersion(nv)
205 - }
206 -
207 - return
208 -}
209 -
210 -// Pick a compaction based on current state; need external synchronization.
211 -func (s *session) pickCompaction() *compaction {
212 - v := s.version()
213 -
214 - var level int
215 - var t0 tFiles
216 - if v.cScore >= 1 {
217 - level = v.cLevel
218 - cptr := s.stCompPtrs[level]
219 - tables := v.tables[level]
220 - for _, t := range tables {
221 - if cptr == nil || s.icmp.Compare(t.imax, cptr) > 0 {
222 - t0 = append(t0, t)
223 - break
224 - }
225 - }
226 - if len(t0) == 0 {
227 - t0 = append(t0, tables[0])
228 - }
229 - } else {
230 - if p := atomic.LoadPointer(&v.cSeek); p != nil {
231 - ts := (*tSet)(p)
232 - level = ts.level
233 - t0 = append(t0, ts.table)
234 - } else {
235 - v.release()
236 - return nil
237 - }
238 - }
239 -
240 - return newCompaction(s, v, level, t0)
241 -}
242 -
243 -// Create compaction from given level and range; need external synchronization.
244 -func (s *session) getCompactionRange(level int, umin, umax []byte) *compaction {
245 - v := s.version()
246 -
247 - t0 := v.tables[level].getOverlaps(nil, s.icmp, umin, umax, level == 0)
248 - if len(t0) == 0 {
249 - v.release()
250 - return nil
251 - }
252 -
253 - // Avoid compacting too much in one shot in case the range is large.
254 - // But we cannot do this for level-0 since level-0 files can overlap
255 - // and we must not pick one file and drop another older file if the
256 - // two files overlap.
257 - if level > 0 {
258 - limit := uint64(v.s.o.GetCompactionSourceLimit(level))
259 - total := uint64(0)
260 - for i, t := range t0 {
261 - total += t.size
262 - if total >= limit {
263 - s.logf("table@compaction limiting F·%d -> F·%d", len(t0), i+1)
264 - t0 = t0[:i+1]
265 - break
266 - }
267 - }
268 - }
269 -
270 - return newCompaction(s, v, level, t0)
271 -}
272 -
273 -func newCompaction(s *session, v *version, level int, t0 tFiles) *compaction {
274 - c := &compaction{
275 - s: s,
276 - v: v,
277 - level: level,
278 - tables: [2]tFiles{t0, nil},
279 - maxGPOverlaps: uint64(s.o.GetCompactionGPOverlaps(level)),
280 - tPtrs: make([]int, s.o.GetNumLevel()),
281 - }
282 - c.expand()
283 - c.save()
284 - return c
285 -}
286 -
287 -// compaction represent a compaction state.
288 -type compaction struct {
289 - s *session
290 - v *version
291 -
292 - level int
293 - tables [2]tFiles
294 - maxGPOverlaps uint64
295 -
296 - gp tFiles
297 - gpi int
298 - seenKey bool
299 - gpOverlappedBytes uint64
300 - imin, imax iKey
301 - tPtrs []int
302 - released bool
303 -
304 - snapGPI int
305 - snapSeenKey bool
306 - snapGPOverlappedBytes uint64
307 - snapTPtrs []int
308 -}
309 -
310 -func (c *compaction) save() {
311 - c.snapGPI = c.gpi
312 - c.snapSeenKey = c.seenKey
313 - c.snapGPOverlappedBytes = c.gpOverlappedBytes
314 - c.snapTPtrs = append(c.snapTPtrs[:0], c.tPtrs...)
315 -}
316 -
317 -func (c *compaction) restore() {
318 - c.gpi = c.snapGPI
319 - c.seenKey = c.snapSeenKey
320 - c.gpOverlappedBytes = c.snapGPOverlappedBytes
321 - c.tPtrs = append(c.tPtrs[:0], c.snapTPtrs...)
322 -}
323 -
324 -func (c *compaction) release() {
325 - if !c.released {
326 - c.released = true
327 - c.v.release()
328 - }
329 -}
330 -
331 -// Expand compacted tables; need external synchronization.
332 -func (c *compaction) expand() {
333 - limit := uint64(c.s.o.GetCompactionExpandLimit(c.level))
334 - vt0, vt1 := c.v.tables[c.level], c.v.tables[c.level+1]
335 -
336 - t0, t1 := c.tables[0], c.tables[1]
337 - imin, imax := t0.getRange(c.s.icmp)
338 - // We expand t0 here just incase ukey hop across tables.
339 - t0 = vt0.getOverlaps(t0, c.s.icmp, imin.ukey(), imax.ukey(), c.level == 0)
340 - if len(t0) != len(c.tables[0]) {
341 - imin, imax = t0.getRange(c.s.icmp)
342 - }
343 - t1 = vt1.getOverlaps(t1, c.s.icmp, imin.ukey(), imax.ukey(), false)
344 - // Get entire range covered by compaction.
345 - amin, amax := append(t0, t1...).getRange(c.s.icmp)
346 -
347 - // See if we can grow the number of inputs in "level" without
348 - // changing the number of "level+1" files we pick up.
349 - if len(t1) > 0 {
350 - exp0 := vt0.getOverlaps(nil, c.s.icmp, amin.ukey(), amax.ukey(), c.level == 0)
351 - if len(exp0) > len(t0) && t1.size()+exp0.size() < limit {
352 - xmin, xmax := exp0.getRange(c.s.icmp)
353 - exp1 := vt1.getOverlaps(nil, c.s.icmp, xmin.ukey(), xmax.ukey(), false)
354 - if len(exp1) == len(t1) {
355 - c.s.logf("table@compaction expanding L%d+L%d (F·%d S·%s)+(F·%d S·%s) -> (F·%d S·%s)+(F·%d S·%s)",
356 - c.level, c.level+1, len(t0), shortenb(int(t0.size())), len(t1), shortenb(int(t1.size())),
357 - len(exp0), shortenb(int(exp0.size())), len(exp1), shortenb(int(exp1.size())))
358 - imin, imax = xmin, xmax
359 - t0, t1 = exp0, exp1
360 - amin, amax = append(t0, t1...).getRange(c.s.icmp)
361 - }
362 - }
363 - }
364 -
365 - // Compute the set of grandparent files that overlap this compaction
366 - // (parent == level+1; grandparent == level+2)
367 - if c.level+2 < c.s.o.GetNumLevel() {
368 - c.gp = c.v.tables[c.level+2].getOverlaps(c.gp, c.s.icmp, amin.ukey(), amax.ukey(), false)
369 - }
370 -
371 - c.tables[0], c.tables[1] = t0, t1
372 - c.imin, c.imax = imin, imax
373 -}
374 -
375 -// Check whether compaction is trivial.
376 -func (c *compaction) trivial() bool {
377 - return len(c.tables[0]) == 1 && len(c.tables[1]) == 0 && c.gp.size() <= c.maxGPOverlaps
378 -}
379 -
380 -func (c *compaction) baseLevelForKey(ukey []byte) bool {
381 - for level, tables := range c.v.tables[c.level+2:] {
382 - for c.tPtrs[level] < len(tables) {
383 - t := tables[c.tPtrs[level]]
384 - if c.s.icmp.uCompare(ukey, t.imax.ukey()) <= 0 {
385 - // We've advanced far enough.
386 - if c.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
387 - // Key falls in this file's range, so definitely not base level.
388 - return false
389 - }
390 - break
391 - }
392 - c.tPtrs[level]++
393 - }
394 - }
395 - return true
396 -}
397 -
398 -func (c *compaction) shouldStopBefore(ikey iKey) bool {
399 - for ; c.gpi < len(c.gp); c.gpi++ {
400 - gp := c.gp[c.gpi]
401 - if c.s.icmp.Compare(ikey, gp.imax) <= 0 {
402 - break
403 - }
404 - if c.seenKey {
405 - c.gpOverlappedBytes += gp.size
406 - }
407 - }
408 - c.seenKey = true
409 -
410 - if c.gpOverlappedBytes > c.maxGPOverlaps {
411 - // Too much overlap for current output; start new output.
412 - c.gpOverlappedBytes = 0
413 - return true
414 - }
415 - return false
416 -}
417 -
418 -// Creates an iterator.
419 -func (c *compaction) newIterator() iterator.Iterator {
420 - // Creates iterator slice.
421 - icap := len(c.tables)
422 - if c.level == 0 {
423 - // Special case for level-0
424 - icap = len(c.tables[0]) + 1
425 - }
426 - its := make([]iterator.Iterator, 0, icap)
427 -
428 - // Options.
429 - ro := &opt.ReadOptions{
430 - DontFillCache: true,
431 - Strict: opt.StrictOverride,
432 - }
433 - strict := c.s.o.GetStrict(opt.StrictCompaction)
434 - if strict {
435 - ro.Strict |= opt.StrictReader
436 - }
437 -
438 - for i, tables := range c.tables {
439 - if len(tables) == 0 {
440 - continue
441 - }
442 -
443 - // Level-0 is not sorted and may overlaps each other.
444 - if c.level+i == 0 {
445 - for _, t := range tables {
446 - its = append(its, c.s.tops.newIterator(t, nil, ro))
447 - }
448 - } else {
449 - it := iterator.NewIndexedIterator(tables.newIndexIterator(c.s.tops, c.s.icmp, nil, ro), strict)
450 - its = append(its, it)
451 - }
452 - }
453 -
454 - return iterator.NewMergedIterator(its, c.s.icmp, strict)
455 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record.go deleted
-313
@@ -1,313 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bufio"
11 - "encoding/binary"
12 - "io"
13 - "strings"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16 -)
17 -
18 -type byteReader interface {
19 - io.Reader
20 - io.ByteReader
21 -}
22 -
23 -// These numbers are written to disk and should not be changed.
24 -const (
25 - recComparer = 1
26 - recJournalNum = 2
27 - recNextFileNum = 3
28 - recSeqNum = 4
29 - recCompPtr = 5
30 - recDelTable = 6
31 - recAddTable = 7
32 - // 8 was used for large value refs
33 - recPrevJournalNum = 9
34 -)
35 -
36 -type cpRecord struct {
37 - level int
38 - ikey iKey
39 -}
40 -
41 -type atRecord struct {
42 - level int
43 - num uint64
44 - size uint64
45 - imin iKey
46 - imax iKey
47 -}
48 -
49 -type dtRecord struct {
50 - level int
51 - num uint64
52 -}
53 -
54 -type sessionRecord struct {
55 - numLevel int
56 -
57 - hasRec int
58 - comparer string
59 - journalNum uint64
60 - prevJournalNum uint64
61 - nextFileNum uint64
62 - seqNum uint64
63 - compPtrs []cpRecord
64 - addedTables []atRecord
65 - deletedTables []dtRecord
66 -
67 - scratch [binary.MaxVarintLen64]byte
68 - err error
69 -}
70 -
71 -func (p *sessionRecord) has(rec int) bool {
72 - return p.hasRec&(1<<uint(rec)) != 0
73 -}
74 -
75 -func (p *sessionRecord) setComparer(name string) {
76 - p.hasRec |= 1 << recComparer
77 - p.comparer = name
78 -}
79 -
80 -func (p *sessionRecord) setJournalNum(num uint64) {
81 - p.hasRec |= 1 << recJournalNum
82 - p.journalNum = num
83 -}
84 -
85 -func (p *sessionRecord) setPrevJournalNum(num uint64) {
86 - p.hasRec |= 1 << recPrevJournalNum
87 - p.prevJournalNum = num
88 -}
89 -
90 -func (p *sessionRecord) setNextFileNum(num uint64) {
91 - p.hasRec |= 1 << recNextFileNum
92 - p.nextFileNum = num
93 -}
94 -
95 -func (p *sessionRecord) setSeqNum(num uint64) {
96 - p.hasRec |= 1 << recSeqNum
97 - p.seqNum = num
98 -}
99 -
100 -func (p *sessionRecord) addCompPtr(level int, ikey iKey) {
101 - p.hasRec |= 1 << recCompPtr
102 - p.compPtrs = append(p.compPtrs, cpRecord{level, ikey})
103 -}
104 -
105 -func (p *sessionRecord) resetCompPtrs() {
106 - p.hasRec &= ^(1 << recCompPtr)
107 - p.compPtrs = p.compPtrs[:0]
108 -}
109 -
110 -func (p *sessionRecord) addTable(level int, num, size uint64, imin, imax iKey) {
111 - p.hasRec |= 1 << recAddTable
112 - p.addedTables = append(p.addedTables, atRecord{level, num, size, imin, imax})
113 -}
114 -
115 -func (p *sessionRecord) addTableFile(level int, t *tFile) {
116 - p.addTable(level, t.file.Num(), t.size, t.imin, t.imax)
117 -}
118 -
119 -func (p *sessionRecord) resetAddedTables() {
120 - p.hasRec &= ^(1 << recAddTable)
121 - p.addedTables = p.addedTables[:0]
122 -}
123 -
124 -func (p *sessionRecord) delTable(level int, num uint64) {
125 - p.hasRec |= 1 << recDelTable
126 - p.deletedTables = append(p.deletedTables, dtRecord{level, num})
127 -}
128 -
129 -func (p *sessionRecord) resetDeletedTables() {
130 - p.hasRec &= ^(1 << recDelTable)
131 - p.deletedTables = p.deletedTables[:0]
132 -}
133 -
134 -func (p *sessionRecord) putUvarint(w io.Writer, x uint64) {
135 - if p.err != nil {
136 - return
137 - }
138 - n := binary.PutUvarint(p.scratch[:], x)
139 - _, p.err = w.Write(p.scratch[:n])
140 -}
141 -
142 -func (p *sessionRecord) putBytes(w io.Writer, x []byte) {
143 - if p.err != nil {
144 - return
145 - }
146 - p.putUvarint(w, uint64(len(x)))
147 - if p.err != nil {
148 - return
149 - }
150 - _, p.err = w.Write(x)
151 -}
152 -
153 -func (p *sessionRecord) encode(w io.Writer) error {
154 - p.err = nil
155 - if p.has(recComparer) {
156 - p.putUvarint(w, recComparer)
157 - p.putBytes(w, []byte(p.comparer))
158 - }
159 - if p.has(recJournalNum) {
160 - p.putUvarint(w, recJournalNum)
161 - p.putUvarint(w, p.journalNum)
162 - }
163 - if p.has(recNextFileNum) {
164 - p.putUvarint(w, recNextFileNum)
165 - p.putUvarint(w, p.nextFileNum)
166 - }
167 - if p.has(recSeqNum) {
168 - p.putUvarint(w, recSeqNum)
169 - p.putUvarint(w, p.seqNum)
170 - }
171 - for _, r := range p.compPtrs {
172 - p.putUvarint(w, recCompPtr)
173 - p.putUvarint(w, uint64(r.level))
174 - p.putBytes(w, r.ikey)
175 - }
176 - for _, r := range p.deletedTables {
177 - p.putUvarint(w, recDelTable)
178 - p.putUvarint(w, uint64(r.level))
179 - p.putUvarint(w, r.num)
180 - }
181 - for _, r := range p.addedTables {
182 - p.putUvarint(w, recAddTable)
183 - p.putUvarint(w, uint64(r.level))
184 - p.putUvarint(w, r.num)
185 - p.putUvarint(w, r.size)
186 - p.putBytes(w, r.imin)
187 - p.putBytes(w, r.imax)
188 - }
189 - return p.err
190 -}
191 -
192 -func (p *sessionRecord) readUvarintMayEOF(field string, r io.ByteReader, mayEOF bool) uint64 {
193 - if p.err != nil {
194 - return 0
195 - }
196 - x, err := binary.ReadUvarint(r)
197 - if err != nil {
198 - if err == io.ErrUnexpectedEOF || (mayEOF == false && err == io.EOF) {
199 - p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
200 - } else if strings.HasPrefix(err.Error(), "binary:") {
201 - p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, err.Error()})
202 - } else {
203 - p.err = err
204 - }
205 - return 0
206 - }
207 - return x
208 -}
209 -
210 -func (p *sessionRecord) readUvarint(field string, r io.ByteReader) uint64 {
211 - return p.readUvarintMayEOF(field, r, false)
212 -}
213 -
214 -func (p *sessionRecord) readBytes(field string, r byteReader) []byte {
215 - if p.err != nil {
216 - return nil
217 - }
218 - n := p.readUvarint(field, r)
219 - if p.err != nil {
220 - return nil
221 - }
222 - x := make([]byte, n)
223 - _, p.err = io.ReadFull(r, x)
224 - if p.err != nil {
225 - if p.err == io.ErrUnexpectedEOF {
226 - p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "short read"})
227 - }
228 - return nil
229 - }
230 - return x
231 -}
232 -
233 -func (p *sessionRecord) readLevel(field string, r io.ByteReader) int {
234 - if p.err != nil {
235 - return 0
236 - }
237 - x := p.readUvarint(field, r)
238 - if p.err != nil {
239 - return 0
240 - }
241 - if x >= uint64(p.numLevel) {
242 - p.err = errors.NewErrCorrupted(nil, &ErrManifestCorrupted{field, "invalid level number"})
243 - return 0
244 - }
245 - return int(x)
246 -}
247 -
248 -func (p *sessionRecord) decode(r io.Reader) error {
249 - br, ok := r.(byteReader)
250 - if !ok {
251 - br = bufio.NewReader(r)
252 - }
253 - p.err = nil
254 - for p.err == nil {
255 - rec := p.readUvarintMayEOF("field-header", br, true)
256 - if p.err != nil {
257 - if p.err == io.EOF {
258 - return nil
259 - }
260 - return p.err
261 - }
262 - switch rec {
263 - case recComparer:
264 - x := p.readBytes("comparer", br)
265 - if p.err == nil {
266 - p.setComparer(string(x))
267 - }
268 - case recJournalNum:
269 - x := p.readUvarint("journal-num", br)
270 - if p.err == nil {
271 - p.setJournalNum(x)
272 - }
273 - case recPrevJournalNum:
274 - x := p.readUvarint("prev-journal-num", br)
275 - if p.err == nil {
276 - p.setPrevJournalNum(x)
277 - }
278 - case recNextFileNum:
279 - x := p.readUvarint("next-file-num", br)
280 - if p.err == nil {
281 - p.setNextFileNum(x)
282 - }
283 - case recSeqNum:
284 - x := p.readUvarint("seq-num", br)
285 - if p.err == nil {
286 - p.setSeqNum(x)
287 - }
288 - case recCompPtr:
289 - level := p.readLevel("comp-ptr.level", br)
290 - ikey := p.readBytes("comp-ptr.ikey", br)
291 - if p.err == nil {
292 - p.addCompPtr(level, iKey(ikey))
293 - }
294 - case recAddTable:
295 - level := p.readLevel("add-table.level", br)
296 - num := p.readUvarint("add-table.num", br)
297 - size := p.readUvarint("add-table.size", br)
298 - imin := p.readBytes("add-table.imin", br)
299 - imax := p.readBytes("add-table.imax", br)
300 - if p.err == nil {
301 - p.addTable(level, num, size, imin, imax)
302 - }
303 - case recDelTable:
304 - level := p.readLevel("del-table.level", br)
305 - num := p.readUvarint("del-table.num", br)
306 - if p.err == nil {
307 - p.delTable(level, num)
308 - }
309 - }
310 - }
311 -
312 - return p.err
313 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_record_test.go deleted
-64
@@ -1,64 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "bytes"
11 - "testing"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14 -)
15 -
16 -func decodeEncode(v *sessionRecord) (res bool, err error) {
17 - b := new(bytes.Buffer)
18 - err = v.encode(b)
19 - if err != nil {
20 - return
21 - }
22 - v2 := &sessionRecord{numLevel: opt.DefaultNumLevel}
23 - err = v.decode(b)
24 - if err != nil {
25 - return
26 - }
27 - b2 := new(bytes.Buffer)
28 - err = v2.encode(b2)
29 - if err != nil {
30 - return
31 - }
32 - return bytes.Equal(b.Bytes(), b2.Bytes()), nil
33 -}
34 -
35 -func TestSessionRecord_EncodeDecode(t *testing.T) {
36 - big := uint64(1) << 50
37 - v := &sessionRecord{numLevel: opt.DefaultNumLevel}
38 - i := uint64(0)
39 - test := func() {
40 - res, err := decodeEncode(v)
41 - if err != nil {
42 - t.Fatalf("error when testing encode/decode sessionRecord: %v", err)
43 - }
44 - if !res {
45 - t.Error("encode/decode test failed at iteration:", i)
46 - }
47 - }
48 -
49 - for ; i < 4; i++ {
50 - test()
51 - v.addTable(3, big+300+i, big+400+i,
52 - newIkey([]byte("foo"), big+500+1, ktVal),
53 - newIkey([]byte("zoo"), big+600+1, ktDel))
54 - v.delTable(4, big+700+i)
55 - v.addCompPtr(int(i), newIkey([]byte("x"), big+900+1, ktVal))
56 - }
57 -
58 - v.setComparer("foo")
59 - v.setJournalNum(big + 100)
60 - v.setPrevJournalNum(big + 99)
61 - v.setNextFileNum(big + 200)
62 - v.setSeqNum(big + 1000)
63 - test()
64 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/session_util.go deleted
-247
@@ -1,247 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "fmt"
11 - "sync/atomic"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/journal"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
15 -)
16 -
17 -type dropper struct {
18 - s *session // Logging.
19 - file storage.File
20 -}
21 -
22 -func (d dropper) Drop(err error) {
23 - if e, ok := err.(*journal.ErrCorrupted); ok {
24 - d.s.logf("journal@drop %s-%d S·%s %q", d.file.Type(), d.file.Num(), shortenb(e.Size), e.Reason)
25 - } else {
26 - d.s.logf("journal@drop %s-%d %q", d.file.Type(), d.file.Num(), err)
27 - }
28 -}
29 -
30 -func (s *session) log(v ...interface{}) { s.stor.Log(fmt.Sprint(v...)) }
31 -func (s *session) logf(format string, v ...interface{}) { s.stor.Log(fmt.Sprintf(format, v...)) }
32 -
33 -// File utils.
34 -
35 -func (s *session) getJournalFile(num uint64) storage.File {
36 - return s.stor.GetFile(num, storage.TypeJournal)
37 -}
38 -
39 -func (s *session) getTableFile(num uint64) storage.File {
40 - return s.stor.GetFile(num, storage.TypeTable)
41 -}
42 -
43 -func (s *session) getFiles(t storage.FileType) ([]storage.File, error) {
44 - return s.stor.GetFiles(t)
45 -}
46 -
47 -func (s *session) newTemp() storage.File {
48 - num := atomic.AddUint64(&s.stTempFileNum, 1) - 1
49 - return s.stor.GetFile(num, storage.TypeTemp)
50 -}
51 -
52 -func (s *session) tableFileFromRecord(r atRecord) *tFile {
53 - return newTableFile(s.getTableFile(r.num), r.size, r.imin, r.imax)
54 -}
55 -
56 -// Session state.
57 -
58 -// Get current version. This will incr version ref, must call
59 -// version.release (exactly once) after use.
60 -func (s *session) version() *version {
61 - s.vmu.Lock()
62 - defer s.vmu.Unlock()
63 - s.stVersion.ref++
64 - return s.stVersion
65 -}
66 -
67 -// Set current version to v.
68 -func (s *session) setVersion(v *version) {
69 - s.vmu.Lock()
70 - v.ref = 1 // Holds by session.
71 - if old := s.stVersion; old != nil {
72 - v.ref++ // Holds by old version.
73 - old.next = v
74 - old.releaseNB()
75 - }
76 - s.stVersion = v
77 - s.vmu.Unlock()
78 -}
79 -
80 -// Get current unused file number.
81 -func (s *session) nextFileNum() uint64 {
82 - return atomic.LoadUint64(&s.stNextFileNum)
83 -}
84 -
85 -// Set current unused file number to num.
86 -func (s *session) setNextFileNum(num uint64) {
87 - atomic.StoreUint64(&s.stNextFileNum, num)
88 -}
89 -
90 -// Mark file number as used.
91 -func (s *session) markFileNum(num uint64) {
92 - nextFileNum := num + 1
93 - for {
94 - old, x := s.stNextFileNum, nextFileNum
95 - if old > x {
96 - x = old
97 - }
98 - if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
99 - break
100 - }
101 - }
102 -}
103 -
104 -// Allocate a file number.
105 -func (s *session) allocFileNum() uint64 {
106 - return atomic.AddUint64(&s.stNextFileNum, 1) - 1
107 -}
108 -
109 -// Reuse given file number.
110 -func (s *session) reuseFileNum(num uint64) {
111 - for {
112 - old, x := s.stNextFileNum, num
113 - if old != x+1 {
114 - x = old
115 - }
116 - if atomic.CompareAndSwapUint64(&s.stNextFileNum, old, x) {
117 - break
118 - }
119 - }
120 -}
121 -
122 -// Manifest related utils.
123 -
124 -// Fill given session record obj with current states; need external
125 -// synchronization.
126 -func (s *session) fillRecord(r *sessionRecord, snapshot bool) {
127 - r.setNextFileNum(s.nextFileNum())
128 -
129 - if snapshot {
130 - if !r.has(recJournalNum) {
131 - r.setJournalNum(s.stJournalNum)
132 - }
133 -
134 - if !r.has(recSeqNum) {
135 - r.setSeqNum(s.stSeqNum)
136 - }
137 -
138 - for level, ik := range s.stCompPtrs {
139 - if ik != nil {
140 - r.addCompPtr(level, ik)
141 - }
142 - }
143 -
144 - r.setComparer(s.icmp.uName())
145 - }
146 -}
147 -
148 -// Mark if record has been committed, this will update session state;
149 -// need external synchronization.
150 -func (s *session) recordCommited(r *sessionRecord) {
151 - if r.has(recJournalNum) {
152 - s.stJournalNum = r.journalNum
153 - }
154 -
155 - if r.has(recPrevJournalNum) {
156 - s.stPrevJournalNum = r.prevJournalNum
157 - }
158 -
159 - if r.has(recSeqNum) {
160 - s.stSeqNum = r.seqNum
161 - }
162 -
163 - for _, p := range r.compPtrs {
164 - s.stCompPtrs[p.level] = iKey(p.ikey)
165 - }
166 -}
167 -
168 -// Create a new manifest file; need external synchronization.
169 -func (s *session) newManifest(rec *sessionRecord, v *version) (err error) {
170 - num := s.allocFileNum()
171 - file := s.stor.GetFile(num, storage.TypeManifest)
172 - writer, err := file.Create()
173 - if err != nil {
174 - return
175 - }
176 - jw := journal.NewWriter(writer)
177 -
178 - if v == nil {
179 - v = s.version()
180 - defer v.release()
181 - }
182 - if rec == nil {
183 - rec = &sessionRecord{numLevel: s.o.GetNumLevel()}
184 - }
185 - s.fillRecord(rec, true)
186 - v.fillRecord(rec)
187 -
188 - defer func() {
189 - if err == nil {
190 - s.recordCommited(rec)
191 - if s.manifest != nil {
192 - s.manifest.Close()
193 - }
194 - if s.manifestWriter != nil {
195 - s.manifestWriter.Close()
196 - }
197 - if s.manifestFile != nil {
198 - s.manifestFile.Remove()
199 - }
200 - s.manifestFile = file
201 - s.manifestWriter = writer
202 - s.manifest = jw
203 - } else {
204 - writer.Close()
205 - file.Remove()
206 - s.reuseFileNum(num)
207 - }
208 - }()
209 -
210 - w, err := jw.Next()
211 - if err != nil {
212 - return
213 - }
214 - err = rec.encode(w)
215 - if err != nil {
216 - return
217 - }
218 - err = jw.Flush()
219 - if err != nil {
220 - return
221 - }
222 - err = s.stor.SetManifest(file)
223 - return
224 -}
225 -
226 -// Flush record to disk.
227 -func (s *session) flushManifest(rec *sessionRecord) (err error) {
228 - s.fillRecord(rec, false)
229 - w, err := s.manifest.Next()
230 - if err != nil {
231 - return
232 - }
233 - err = rec.encode(w)
234 - if err != nil {
235 - return
236 - }
237 - err = s.manifest.Flush()
238 - if err != nil {
239 - return
240 - }
241 - err = s.manifestWriter.Sync()
242 - if err != nil {
243 - return
244 - }
245 - s.recordCommited(rec)
246 - return
247 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage.go deleted
-534
@@ -1,534 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reservefs.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "errors"
11 - "fmt"
12 - "io/ioutil"
13 - "os"
14 - "path/filepath"
15 - "runtime"
16 - "strconv"
17 - "strings"
18 - "sync"
19 - "time"
20 -
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
22 -)
23 -
24 -var errFileOpen = errors.New("leveldb/storage: file still open")
25 -
26 -type fileLock interface {
27 - release() error
28 -}
29 -
30 -type fileStorageLock struct {
31 - fs *fileStorage
32 -}
33 -
34 -func (lock *fileStorageLock) Release() {
35 - fs := lock.fs
36 - fs.mu.Lock()
37 - defer fs.mu.Unlock()
38 - if fs.slock == lock {
39 - fs.slock = nil
40 - }
41 - return
42 -}
43 -
44 -// fileStorage is a file-system backed storage.
45 -type fileStorage struct {
46 - path string
47 -
48 - mu sync.Mutex
49 - flock fileLock
50 - slock *fileStorageLock
51 - logw *os.File
52 - buf []byte
53 - // Opened file counter; if open < 0 means closed.
54 - open int
55 - day int
56 -}
57 -
58 -// OpenFile returns a new filesytem-backed storage implementation with the given
59 -// path. This also hold a file lock, so any subsequent attempt to open the same
60 -// path will fail.
61 -//
62 -// The storage must be closed after use, by calling Close method.
63 -func OpenFile(path string) (Storage, error) {
64 - if err := os.MkdirAll(path, 0755); err != nil {
65 - return nil, err
66 - }
67 -
68 - flock, err := newFileLock(filepath.Join(path, "LOCK"))
69 - if err != nil {
70 - return nil, err
71 - }
72 -
73 - defer func() {
74 - if err != nil {
75 - flock.release()
76 - }
77 - }()
78 -
79 - rename(filepath.Join(path, "LOG"), filepath.Join(path, "LOG.old"))
80 - logw, err := os.OpenFile(filepath.Join(path, "LOG"), os.O_WRONLY|os.O_CREATE, 0644)
81 - if err != nil {
82 - return nil, err
83 - }
84 -
85 - fs := &fileStorage{path: path, flock: flock, logw: logw}
86 - runtime.SetFinalizer(fs, (*fileStorage).Close)
87 - return fs, nil
88 -}
89 -
90 -func (fs *fileStorage) Lock() (util.Releaser, error) {
91 - fs.mu.Lock()
92 - defer fs.mu.Unlock()
93 - if fs.open < 0 {
94 - return nil, ErrClosed
95 - }
96 - if fs.slock != nil {
97 - return nil, ErrLocked
98 - }
99 - fs.slock = &fileStorageLock{fs: fs}
100 - return fs.slock, nil
101 -}
102 -
103 -func itoa(buf []byte, i int, wid int) []byte {
104 - var u uint = uint(i)
105 - if u == 0 && wid <= 1 {
106 - return append(buf, '0')
107 - }
108 -
109 - // Assemble decimal in reverse order.
110 - var b [32]byte
111 - bp := len(b)
112 - for ; u > 0 || wid > 0; u /= 10 {
113 - bp--
114 - wid--
115 - b[bp] = byte(u%10) + '0'
116 - }
117 - return append(buf, b[bp:]...)
118 -}
119 -
120 -func (fs *fileStorage) printDay(t time.Time) {
121 - if fs.day == t.Day() {
122 - return
123 - }
124 - fs.day = t.Day()
125 - fs.logw.Write([]byte("=============== " + t.Format("Jan 2, 2006 (MST)") + " ===============\n"))
126 -}
127 -
128 -func (fs *fileStorage) doLog(t time.Time, str string) {
129 - fs.printDay(t)
130 - hour, min, sec := t.Clock()
131 - msec := t.Nanosecond() / 1e3
132 - // time
133 - fs.buf = itoa(fs.buf[:0], hour, 2)
134 - fs.buf = append(fs.buf, ':')
135 - fs.buf = itoa(fs.buf, min, 2)
136 - fs.buf = append(fs.buf, ':')
137 - fs.buf = itoa(fs.buf, sec, 2)
138 - fs.buf = append(fs.buf, '.')
139 - fs.buf = itoa(fs.buf, msec, 6)
140 - fs.buf = append(fs.buf, ' ')
141 - // write
142 - fs.buf = append(fs.buf, []byte(str)...)
143 - fs.buf = append(fs.buf, '\n')
144 - fs.logw.Write(fs.buf)
145 -}
146 -
147 -func (fs *fileStorage) Log(str string) {
148 - t := time.Now()
149 - fs.mu.Lock()
150 - defer fs.mu.Unlock()
151 - if fs.open < 0 {
152 - return
153 - }
154 - fs.doLog(t, str)
155 -}
156 -
157 -func (fs *fileStorage) log(str string) {
158 - fs.doLog(time.Now(), str)
159 -}
160 -
161 -func (fs *fileStorage) GetFile(num uint64, t FileType) File {
162 - return &file{fs: fs, num: num, t: t}
163 -}
164 -
165 -func (fs *fileStorage) GetFiles(t FileType) (ff []File, err error) {
166 - fs.mu.Lock()
167 - defer fs.mu.Unlock()
168 - if fs.open < 0 {
169 - return nil, ErrClosed
170 - }
171 - dir, err := os.Open(fs.path)
172 - if err != nil {
173 - return
174 - }
175 - fnn, err := dir.Readdirnames(0)
176 - // Close the dir first before checking for Readdirnames error.
177 - if err := dir.Close(); err != nil {
178 - fs.log(fmt.Sprintf("close dir: %v", err))
179 - }
180 - if err != nil {
181 - return
182 - }
183 - f := &file{fs: fs}
184 - for _, fn := range fnn {
185 - if f.parse(fn) && (f.t&t) != 0 {
186 - ff = append(ff, f)
187 - f = &file{fs: fs}
188 - }
189 - }
190 - return
191 -}
192 -
193 -func (fs *fileStorage) GetManifest() (f File, err error) {
194 - fs.mu.Lock()
195 - defer fs.mu.Unlock()
196 - if fs.open < 0 {
197 - return nil, ErrClosed
198 - }
199 - dir, err := os.Open(fs.path)
200 - if err != nil {
201 - return
202 - }
203 - fnn, err := dir.Readdirnames(0)
204 - // Close the dir first before checking for Readdirnames error.
205 - if err := dir.Close(); err != nil {
206 - fs.log(fmt.Sprintf("close dir: %v", err))
207 - }
208 - if err != nil {
209 - return
210 - }
211 - // Find latest CURRENT file.
212 - var rem []string
213 - var pend bool
214 - var cerr error
215 - for _, fn := range fnn {
216 - if strings.HasPrefix(fn, "CURRENT") {
217 - pend1 := len(fn) > 7
218 - // Make sure it is valid name for a CURRENT file, otherwise skip it.
219 - if pend1 {
220 - if fn[7] != '.' || len(fn) < 9 {
221 - fs.log(fmt.Sprintf("skipping %s: invalid file name", fn))
222 - continue
223 - }
224 - if _, e1 := strconv.ParseUint(fn[8:], 10, 0); e1 != nil {
225 - fs.log(fmt.Sprintf("skipping %s: invalid file num: %v", fn, e1))
226 - continue
227 - }
228 - }
229 - path := filepath.Join(fs.path, fn)
230 - r, e1 := os.OpenFile(path, os.O_RDONLY, 0)
231 - if e1 != nil {
232 - return nil, e1
233 - }
234 - b, e1 := ioutil.ReadAll(r)
235 - if e1 != nil {
236 - r.Close()
237 - return nil, e1
238 - }
239 - f1 := &file{fs: fs}
240 - if len(b) < 1 || b[len(b)-1] != '\n' || !f1.parse(string(b[:len(b)-1])) {
241 - fs.log(fmt.Sprintf("skipping %s: corrupted or incomplete", fn))
242 - if pend1 {
243 - rem = append(rem, fn)
244 - }
245 - if !pend1 || cerr == nil {
246 - cerr = fmt.Errorf("leveldb/storage: corrupted or incomplete %s file", fn)
247 - }
248 - } else if f != nil && f1.Num() < f.Num() {
249 - fs.log(fmt.Sprintf("skipping %s: obsolete", fn))
250 - if pend1 {
251 - rem = append(rem, fn)
252 - }
253 - } else {
254 - f = f1
255 - pend = pend1
256 - }
257 - if err := r.Close(); err != nil {
258 - fs.log(fmt.Sprintf("close %s: %v", fn, err))
259 - }
260 - }
261 - }
262 - // Don't remove any files if there is no valid CURRENT file.
263 - if f == nil {
264 - if cerr != nil {
265 - err = cerr
266 - } else {
267 - err = os.ErrNotExist
268 - }
269 - return
270 - }
271 - // Rename pending CURRENT file to an effective CURRENT.
272 - if pend {
273 - path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f.Num())
274 - if err := rename(path, filepath.Join(fs.path, "CURRENT")); err != nil {
275 - fs.log(fmt.Sprintf("CURRENT.%d -> CURRENT: %v", f.Num(), err))
276 - }
277 - }
278 - // Remove obsolete or incomplete pending CURRENT files.
279 - for _, fn := range rem {
280 - path := filepath.Join(fs.path, fn)
281 - if err := os.Remove(path); err != nil {
282 - fs.log(fmt.Sprintf("remove %s: %v", fn, err))
283 - }
284 - }
285 - return
286 -}
287 -
288 -func (fs *fileStorage) SetManifest(f File) (err error) {
289 - fs.mu.Lock()
290 - defer fs.mu.Unlock()
291 - if fs.open < 0 {
292 - return ErrClosed
293 - }
294 - f2, ok := f.(*file)
295 - if !ok || f2.t != TypeManifest {
296 - return ErrInvalidFile
297 - }
298 - defer func() {
299 - if err != nil {
300 - fs.log(fmt.Sprintf("CURRENT: %v", err))
301 - }
302 - }()
303 - path := fmt.Sprintf("%s.%d", filepath.Join(fs.path, "CURRENT"), f2.Num())
304 - w, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
305 - if err != nil {
306 - return err
307 - }
308 - _, err = fmt.Fprintln(w, f2.name())
309 - // Close the file first.
310 - if err := w.Close(); err != nil {
311 - fs.log(fmt.Sprintf("close CURRENT.%d: %v", f2.num, err))
312 - }
313 - if err != nil {
314 - return err
315 - }
316 - return rename(path, filepath.Join(fs.path, "CURRENT"))
317 -}
318 -
319 -func (fs *fileStorage) Close() error {
320 - fs.mu.Lock()
321 - defer fs.mu.Unlock()
322 - if fs.open < 0 {
323 - return ErrClosed
324 - }
325 - // Clear the finalizer.
326 - runtime.SetFinalizer(fs, nil)
327 -
328 - if fs.open > 0 {
329 - fs.log(fmt.Sprintf("refuse to close, %d files still open", fs.open))
330 - return fmt.Errorf("leveldb/storage: cannot close, %d files still open", fs.open)
331 - }
332 - fs.open = -1
333 - e1 := fs.logw.Close()
334 - err := fs.flock.release()
335 - if err == nil {
336 - err = e1
337 - }
338 - return err
339 -}
340 -
341 -type fileWrap struct {
342 - *os.File
343 - f *file
344 -}
345 -
346 -func (fw fileWrap) Sync() error {
347 - if err := fw.File.Sync(); err != nil {
348 - return err
349 - }
350 - if fw.f.Type() == TypeManifest {
351 - // Also sync parent directory if file type is manifest.
352 - // See: https://code.google.com/p/leveldb/issues/detail?id=190.
353 - if err := syncDir(fw.f.fs.path); err != nil {
354 - return err
355 - }
356 - }
357 - return nil
358 -}
359 -
360 -func (fw fileWrap) Close() error {
361 - f := fw.f
362 - f.fs.mu.Lock()
363 - defer f.fs.mu.Unlock()
364 - if !f.open {
365 - return ErrClosed
366 - }
367 - f.open = false
368 - f.fs.open--
369 - err := fw.File.Close()
370 - if err != nil {
371 - f.fs.log(fmt.Sprintf("close %s.%d: %v", f.Type(), f.Num(), err))
372 - }
373 - return err
374 -}
375 -
376 -type file struct {
377 - fs *fileStorage
378 - num uint64
379 - t FileType
380 - open bool
381 -}
382 -
383 -func (f *file) Open() (Reader, error) {
384 - f.fs.mu.Lock()
385 - defer f.fs.mu.Unlock()
386 - if f.fs.open < 0 {
387 - return nil, ErrClosed
388 - }
389 - if f.open {
390 - return nil, errFileOpen
391 - }
392 - of, err := os.OpenFile(f.path(), os.O_RDONLY, 0)
393 - if err != nil {
394 - if f.hasOldName() && os.IsNotExist(err) {
395 - of, err = os.OpenFile(f.oldPath(), os.O_RDONLY, 0)
396 - if err == nil {
397 - goto ok
398 - }
399 - }
400 - return nil, err
401 - }
402 -ok:
403 - f.open = true
404 - f.fs.open++
405 - return fileWrap{of, f}, nil
406 -}
407 -
408 -func (f *file) Create() (Writer, error) {
409 - f.fs.mu.Lock()
410 - defer f.fs.mu.Unlock()
411 - if f.fs.open < 0 {
412 - return nil, ErrClosed
413 - }
414 - if f.open {
415 - return nil, errFileOpen
416 - }
417 - of, err := os.OpenFile(f.path(), os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0644)
418 - if err != nil {
419 - return nil, err
420 - }
421 - f.open = true
422 - f.fs.open++
423 - return fileWrap{of, f}, nil
424 -}
425 -
426 -func (f *file) Replace(newfile File) error {
427 - f.fs.mu.Lock()
428 - defer f.fs.mu.Unlock()
429 - if f.fs.open < 0 {
430 - return ErrClosed
431 - }
432 - newfile2, ok := newfile.(*file)
433 - if !ok {
434 - return ErrInvalidFile
435 - }
436 - if f.open || newfile2.open {
437 - return errFileOpen
438 - }
439 - return rename(newfile2.path(), f.path())
440 -}
441 -
442 -func (f *file) Type() FileType {
443 - return f.t
444 -}
445 -
446 -func (f *file) Num() uint64 {
447 - return f.num
448 -}
449 -
450 -func (f *file) Remove() error {
451 - f.fs.mu.Lock()
452 - defer f.fs.mu.Unlock()
453 - if f.fs.open < 0 {
454 - return ErrClosed
455 - }
456 - if f.open {
457 - return errFileOpen
458 - }
459 - err := os.Remove(f.path())
460 - if err != nil {
461 - f.fs.log(fmt.Sprintf("remove %s.%d: %v", f.Type(), f.Num(), err))
462 - }
463 - // Also try remove file with old name, just in case.
464 - if f.hasOldName() {
465 - if e1 := os.Remove(f.oldPath()); !os.IsNotExist(e1) {
466 - f.fs.log(fmt.Sprintf("remove %s.%d: %v (old name)", f.Type(), f.Num(), err))
467 - err = e1
468 - }
469 - }
470 - return err
471 -}
472 -
473 -func (f *file) hasOldName() bool {
474 - return f.t == TypeTable
475 -}
476 -
477 -func (f *file) oldName() string {
478 - switch f.t {
479 - case TypeTable:
480 - return fmt.Sprintf("%06d.sst", f.num)
481 - }
482 - return f.name()
483 -}
484 -
485 -func (f *file) oldPath() string {
486 - return filepath.Join(f.fs.path, f.oldName())
487 -}
488 -
489 -func (f *file) name() string {
490 - switch f.t {
491 - case TypeManifest:
492 - return fmt.Sprintf("MANIFEST-%06d", f.num)
493 - case TypeJournal:
494 - return fmt.Sprintf("%06d.log", f.num)
495 - case TypeTable:
496 - return fmt.Sprintf("%06d.ldb", f.num)
497 - case TypeTemp:
498 - return fmt.Sprintf("%06d.tmp", f.num)
499 - default:
500 - panic("invalid file type")
501 - }
502 -}
503 -
504 -func (f *file) path() string {
505 - return filepath.Join(f.fs.path, f.name())
506 -}
507 -
508 -func (f *file) parse(name string) bool {
509 - var num uint64
510 - var tail string
511 - _, err := fmt.Sscanf(name, "%d.%s", &num, &tail)
512 - if err == nil {
513 - switch tail {
514 - case "log":
515 - f.t = TypeJournal
516 - case "ldb", "sst":
517 - f.t = TypeTable
518 - case "tmp":
519 - f.t = TypeTemp
520 - default:
521 - return false
522 - }
523 - f.num = num
524 - return true
525 - }
526 - n, _ := fmt.Sscanf(name, "MANIFEST-%d%s", &num, &tail)
527 - if n == 1 {
528 - f.t = TypeManifest
529 - f.num = num
530 - return true
531 - }
532 -
533 - return false
534 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_plan9.go deleted
-52
@@ -1,52 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "os"
11 - "path/filepath"
12 -)
13 -
14 -type plan9FileLock struct {
15 - f *os.File
16 -}
17 -
18 -func (fl *plan9FileLock) release() error {
19 - return fl.f.Close()
20 -}
21 -
22 -func newFileLock(path string) (fl fileLock, err error) {
23 - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, os.ModeExclusive|0644)
24 - if err != nil {
25 - return
26 - }
27 - fl = &plan9FileLock{f: f}
28 - return
29 -}
30 -
31 -func rename(oldpath, newpath string) error {
32 - if _, err := os.Stat(newpath); err == nil {
33 - if err := os.Remove(newpath); err != nil {
34 - return err
35 - }
36 - }
37 -
38 - _, fname := filepath.Split(newpath)
39 - return os.Rename(oldpath, fname)
40 -}
41 -
42 -func syncDir(name string) error {
43 - f, err := os.Open(name)
44 - if err != nil {
45 - return err
46 - }
47 - defer f.Close()
48 - if err := f.Sync(); err != nil {
49 - return err
50 - }
51 - return nil
52 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_solaris.go deleted
-68
@@ -1,68 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build solaris
8 -
9 -package storage
10 -
11 -import (
12 - "os"
13 - "syscall"
14 -)
15 -
16 -type unixFileLock struct {
17 - f *os.File
18 -}
19 -
20 -func (fl *unixFileLock) release() error {
21 - if err := setFileLock(fl.f, false); err != nil {
22 - return err
23 - }
24 - return fl.f.Close()
25 -}
26 -
27 -func newFileLock(path string) (fl fileLock, err error) {
28 - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
29 - if err != nil {
30 - return
31 - }
32 - err = setFileLock(f, true)
33 - if err != nil {
34 - f.Close()
35 - return
36 - }
37 - fl = &unixFileLock{f: f}
38 - return
39 -}
40 -
41 -func setFileLock(f *os.File, lock bool) error {
42 - flock := syscall.Flock_t{
43 - Type: syscall.F_UNLCK,
44 - Start: 0,
45 - Len: 0,
46 - Whence: 1,
47 - }
48 - if lock {
49 - flock.Type = syscall.F_WRLCK
50 - }
51 - return syscall.FcntlFlock(f.Fd(), syscall.F_SETLK, &flock)
52 -}
53 -
54 -func rename(oldpath, newpath string) error {
55 - return os.Rename(oldpath, newpath)
56 -}
57 -
58 -func syncDir(name string) error {
59 - f, err := os.Open(name)
60 - if err != nil {
61 - return err
62 - }
63 - defer f.Close()
64 - if err := f.Sync(); err != nil {
65 - return err
66 - }
67 - return nil
68 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_test.go deleted
-142
@@ -1,142 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "fmt"
11 - "os"
12 - "path/filepath"
13 - "testing"
14 -)
15 -
16 -var cases = []struct {
17 - oldName []string
18 - name string
19 - ftype FileType
20 - num uint64
21 -}{
22 - {nil, "000100.log", TypeJournal, 100},
23 - {nil, "000000.log", TypeJournal, 0},
24 - {[]string{"000000.sst"}, "000000.ldb", TypeTable, 0},
25 - {nil, "MANIFEST-000002", TypeManifest, 2},
26 - {nil, "MANIFEST-000007", TypeManifest, 7},
27 - {nil, "18446744073709551615.log", TypeJournal, 18446744073709551615},
28 - {nil, "000100.tmp", TypeTemp, 100},
29 -}
30 -
31 -var invalidCases = []string{
32 - "",
33 - "foo",
34 - "foo-dx-100.log",
35 - ".log",
36 - "",
37 - "manifest",
38 - "CURREN",
39 - "CURRENTX",
40 - "MANIFES",
41 - "MANIFEST",
42 - "MANIFEST-",
43 - "XMANIFEST-3",
44 - "MANIFEST-3x",
45 - "LOC",
46 - "LOCKx",
47 - "LO",
48 - "LOGx",
49 - "18446744073709551616.log",
50 - "184467440737095516150.log",
51 - "100",
52 - "100.",
53 - "100.lop",
54 -}
55 -
56 -func TestFileStorage_CreateFileName(t *testing.T) {
57 - for _, c := range cases {
58 - f := &file{num: c.num, t: c.ftype}
59 - if f.name() != c.name {
60 - t.Errorf("invalid filename got '%s', want '%s'", f.name(), c.name)
61 - }
62 - }
63 -}
64 -
65 -func TestFileStorage_ParseFileName(t *testing.T) {
66 - for _, c := range cases {
67 - for _, name := range append([]string{c.name}, c.oldName...) {
68 - f := new(file)
69 - if !f.parse(name) {
70 - t.Errorf("cannot parse filename '%s'", name)
71 - continue
72 - }
73 - if f.Type() != c.ftype {
74 - t.Errorf("filename '%s' invalid type got '%d', want '%d'", name, f.Type(), c.ftype)
75 - }
76 - if f.Num() != c.num {
77 - t.Errorf("filename '%s' invalid number got '%d', want '%d'", name, f.Num(), c.num)
78 - }
79 - }
80 - }
81 -}
82 -
83 -func TestFileStorage_InvalidFileName(t *testing.T) {
84 - for _, name := range invalidCases {
85 - f := new(file)
86 - if f.parse(name) {
87 - t.Errorf("filename '%s' should be invalid", name)
88 - }
89 - }
90 -}
91 -
92 -func TestFileStorage_Locking(t *testing.T) {
93 - path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldbtestfd-%d", os.Getuid()))
94 -
95 - _, err := os.Stat(path)
96 - if err == nil {
97 - err = os.RemoveAll(path)
98 - if err != nil {
99 - t.Fatal("RemoveAll: got error: ", err)
100 - }
101 - }
102 -
103 - p1, err := OpenFile(path)
104 - if err != nil {
105 - t.Fatal("OpenFile(1): got error: ", err)
106 - }
107 -
108 - defer os.RemoveAll(path)
109 -
110 - p2, err := OpenFile(path)
111 - if err != nil {
112 - t.Logf("OpenFile(2): got error: %s (expected)", err)
113 - } else {
114 - p2.Close()
115 - p1.Close()
116 - t.Fatal("OpenFile(2): expect error")
117 - }
118 -
119 - p1.Close()
120 -
121 - p3, err := OpenFile(path)
122 - if err != nil {
123 - t.Fatal("OpenFile(3): got error: ", err)
124 - }
125 - defer p3.Close()
126 -
127 - l, err := p3.Lock()
128 - if err != nil {
129 - t.Fatal("storage lock failed(1): ", err)
130 - }
131 - _, err = p3.Lock()
132 - if err == nil {
133 - t.Fatal("expect error for second storage lock attempt")
134 - } else {
135 - t.Logf("storage lock got error: %s (expected)", err)
136 - }
137 - l.Release()
138 - _, err = p3.Lock()
139 - if err != nil {
140 - t.Fatal("storage lock failed(2): ", err)
141 - }
142 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_unix.go deleted
-63
@@ -1,63 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build darwin dragonfly freebsd linux netbsd openbsd
8 -
9 -package storage
10 -
11 -import (
12 - "os"
13 - "syscall"
14 -)
15 -
16 -type unixFileLock struct {
17 - f *os.File
18 -}
19 -
20 -func (fl *unixFileLock) release() error {
21 - if err := setFileLock(fl.f, false); err != nil {
22 - return err
23 - }
24 - return fl.f.Close()
25 -}
26 -
27 -func newFileLock(path string) (fl fileLock, err error) {
28 - f, err := os.OpenFile(path, os.O_RDWR|os.O_CREATE, 0644)
29 - if err != nil {
30 - return
31 - }
32 - err = setFileLock(f, true)
33 - if err != nil {
34 - f.Close()
35 - return
36 - }
37 - fl = &unixFileLock{f: f}
38 - return
39 -}
40 -
41 -func setFileLock(f *os.File, lock bool) error {
42 - how := syscall.LOCK_UN
43 - if lock {
44 - how = syscall.LOCK_EX
45 - }
46 - return syscall.Flock(int(f.Fd()), how|syscall.LOCK_NB)
47 -}
48 -
49 -func rename(oldpath, newpath string) error {
50 - return os.Rename(oldpath, newpath)
51 -}
52 -
53 -func syncDir(name string) error {
54 - f, err := os.Open(name)
55 - if err != nil {
56 - return err
57 - }
58 - defer f.Close()
59 - if err := f.Sync(); err != nil {
60 - return err
61 - }
62 - return nil
63 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/file_storage_windows.go deleted
-69
@@ -1,69 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "syscall"
11 - "unsafe"
12 -)
13 -
14 -var (
15 - modkernel32 = syscall.NewLazyDLL("kernel32.dll")
16 -
17 - procMoveFileExW = modkernel32.NewProc("MoveFileExW")
18 -)
19 -
20 -const (
21 - _MOVEFILE_REPLACE_EXISTING = 1
22 -)
23 -
24 -type windowsFileLock struct {
25 - fd syscall.Handle
26 -}
27 -
28 -func (fl *windowsFileLock) release() error {
29 - return syscall.Close(fl.fd)
30 -}
31 -
32 -func newFileLock(path string) (fl fileLock, err error) {
33 - pathp, err := syscall.UTF16PtrFromString(path)
34 - if err != nil {
35 - return
36 - }
37 - fd, err := syscall.CreateFile(pathp, syscall.GENERIC_READ|syscall.GENERIC_WRITE, 0, nil, syscall.CREATE_ALWAYS, syscall.FILE_ATTRIBUTE_NORMAL, 0)
38 - if err != nil {
39 - return
40 - }
41 - fl = &windowsFileLock{fd: fd}
42 - return
43 -}
44 -
45 -func moveFileEx(from *uint16, to *uint16, flags uint32) error {
46 - r1, _, e1 := syscall.Syscall(procMoveFileExW.Addr(), 3, uintptr(unsafe.Pointer(from)), uintptr(unsafe.Pointer(to)), uintptr(flags))
47 - if r1 == 0 {
48 - if e1 != 0 {
49 - return error(e1)
50 - } else {
51 - return syscall.EINVAL
52 - }
53 - }
54 - return nil
55 -}
56 -
57 -func rename(oldpath, newpath string) error {
58 - from, err := syscall.UTF16PtrFromString(oldpath)
59 - if err != nil {
60 - return err
61 - }
62 - to, err := syscall.UTF16PtrFromString(newpath)
63 - if err != nil {
64 - return err
65 - }
66 - return moveFileEx(from, to, _MOVEFILE_REPLACE_EXISTING)
67 -}
68 -
69 -func syncDir(name string) error { return nil }
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/mem_storage.go deleted
-203
@@ -1,203 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "bytes"
11 - "os"
12 - "sync"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
15 -)
16 -
17 -const typeShift = 3
18 -
19 -type memStorageLock struct {
20 - ms *memStorage
21 -}
22 -
23 -func (lock *memStorageLock) Release() {
24 - ms := lock.ms
25 - ms.mu.Lock()
26 - defer ms.mu.Unlock()
27 - if ms.slock == lock {
28 - ms.slock = nil
29 - }
30 - return
31 -}
32 -
33 -// memStorage is a memory-backed storage.
34 -type memStorage struct {
35 - mu sync.Mutex
36 - slock *memStorageLock
37 - files map[uint64]*memFile
38 - manifest *memFilePtr
39 -}
40 -
41 -// NewMemStorage returns a new memory-backed storage implementation.
42 -func NewMemStorage() Storage {
43 - return &memStorage{
44 - files: make(map[uint64]*memFile),
45 - }
46 -}
47 -
48 -func (ms *memStorage) Lock() (util.Releaser, error) {
49 - ms.mu.Lock()
50 - defer ms.mu.Unlock()
51 - if ms.slock != nil {
52 - return nil, ErrLocked
53 - }
54 - ms.slock = &memStorageLock{ms: ms}
55 - return ms.slock, nil
56 -}
57 -
58 -func (*memStorage) Log(str string) {}
59 -
60 -func (ms *memStorage) GetFile(num uint64, t FileType) File {
61 - return &memFilePtr{ms: ms, num: num, t: t}
62 -}
63 -
64 -func (ms *memStorage) GetFiles(t FileType) ([]File, error) {
65 - ms.mu.Lock()
66 - var ff []File
67 - for x, _ := range ms.files {
68 - num, mt := x>>typeShift, FileType(x)&TypeAll
69 - if mt&t == 0 {
70 - continue
71 - }
72 - ff = append(ff, &memFilePtr{ms: ms, num: num, t: mt})
73 - }
74 - ms.mu.Unlock()
75 - return ff, nil
76 -}
77 -
78 -func (ms *memStorage) GetManifest() (File, error) {
79 - ms.mu.Lock()
80 - defer ms.mu.Unlock()
81 - if ms.manifest == nil {
82 - return nil, os.ErrNotExist
83 - }
84 - return ms.manifest, nil
85 -}
86 -
87 -func (ms *memStorage) SetManifest(f File) error {
88 - fm, ok := f.(*memFilePtr)
89 - if !ok || fm.t != TypeManifest {
90 - return ErrInvalidFile
91 - }
92 - ms.mu.Lock()
93 - ms.manifest = fm
94 - ms.mu.Unlock()
95 - return nil
96 -}
97 -
98 -func (*memStorage) Close() error { return nil }
99 -
100 -type memReader struct {
101 - *bytes.Reader
102 - m *memFile
103 -}
104 -
105 -func (mr *memReader) Close() error {
106 - return mr.m.Close()
107 -}
108 -
109 -type memFile struct {
110 - bytes.Buffer
111 - ms *memStorage
112 - open bool
113 -}
114 -
115 -func (*memFile) Sync() error { return nil }
116 -func (m *memFile) Close() error {
117 - m.ms.mu.Lock()
118 - m.open = false
119 - m.ms.mu.Unlock()
120 - return nil
121 -}
122 -
123 -type memFilePtr struct {
124 - ms *memStorage
125 - num uint64
126 - t FileType
127 -}
128 -
129 -func (p *memFilePtr) x() uint64 {
130 - return p.Num()<<typeShift | uint64(p.Type())
131 -}
132 -
133 -func (p *memFilePtr) Open() (Reader, error) {
134 - ms := p.ms
135 - ms.mu.Lock()
136 - defer ms.mu.Unlock()
137 - if m, exist := ms.files[p.x()]; exist {
138 - if m.open {
139 - return nil, errFileOpen
140 - }
141 - m.open = true
142 - return &memReader{Reader: bytes.NewReader(m.Bytes()), m: m}, nil
143 - }
144 - return nil, os.ErrNotExist
145 -}
146 -
147 -func (p *memFilePtr) Create() (Writer, error) {
148 - ms := p.ms
149 - ms.mu.Lock()
150 - defer ms.mu.Unlock()
151 - m, exist := ms.files[p.x()]
152 - if exist {
153 - if m.open {
154 - return nil, errFileOpen
155 - }
156 - m.Reset()
157 - } else {
158 - m = &memFile{ms: ms}
159 - ms.files[p.x()] = m
160 - }
161 - m.open = true
162 - return m, nil
163 -}
164 -
165 -func (p *memFilePtr) Replace(newfile File) error {
166 - p1, ok := newfile.(*memFilePtr)
167 - if !ok {
168 - return ErrInvalidFile
169 - }
170 - ms := p.ms
171 - ms.mu.Lock()
172 - defer ms.mu.Unlock()
173 - m1, exist := ms.files[p1.x()]
174 - if !exist {
175 - return os.ErrNotExist
176 - }
177 - m0, exist := ms.files[p.x()]
178 - if (exist && m0.open) || m1.open {
179 - return errFileOpen
180 - }
181 - delete(ms.files, p1.x())
182 - ms.files[p.x()] = m1
183 - return nil
184 -}
185 -
186 -func (p *memFilePtr) Type() FileType {
187 - return p.t
188 -}
189 -
190 -func (p *memFilePtr) Num() uint64 {
191 - return p.num
192 -}
193 -
194 -func (p *memFilePtr) Remove() error {
195 - ms := p.ms
196 - ms.mu.Lock()
197 - defer ms.mu.Unlock()
198 - if _, exist := ms.files[p.x()]; exist {
199 - delete(ms.files, p.x())
200 - return nil
201 - }
202 - return os.ErrNotExist
203 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/mem_storage_test.go deleted
-66
@@ -1,66 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package storage
8 -
9 -import (
10 - "bytes"
11 - "testing"
12 -)
13 -
14 -func TestMemStorage(t *testing.T) {
15 - m := NewMemStorage()
16 -
17 - l, err := m.Lock()
18 - if err != nil {
19 - t.Fatal("storage lock failed(1): ", err)
20 - }
21 - _, err = m.Lock()
22 - if err == nil {
23 - t.Fatal("expect error for second storage lock attempt")
24 - } else {
25 - t.Logf("storage lock got error: %s (expected)", err)
26 - }
27 - l.Release()
28 - _, err = m.Lock()
29 - if err != nil {
30 - t.Fatal("storage lock failed(2): ", err)
31 - }
32 -
33 - f := m.GetFile(1, TypeTable)
34 - if f.Num() != 1 && f.Type() != TypeTable {
35 - t.Fatal("invalid file number and type")
36 - }
37 - w, _ := f.Create()
38 - w.Write([]byte("abc"))
39 - w.Close()
40 - if ff, _ := m.GetFiles(TypeAll); len(ff) != 1 {
41 - t.Fatal("invalid GetFiles len")
42 - }
43 - buf := new(bytes.Buffer)
44 - r, err := f.Open()
45 - if err != nil {
46 - t.Fatal("Open: got error: ", err)
47 - }
48 - buf.ReadFrom(r)
49 - r.Close()
50 - if got := buf.String(); got != "abc" {
51 - t.Fatalf("Read: invalid value, want=abc got=%s", got)
52 - }
53 - if _, err := f.Open(); err != nil {
54 - t.Fatal("Open: got error: ", err)
55 - }
56 - if _, err := m.GetFile(1, TypeTable).Open(); err == nil {
57 - t.Fatal("expecting error")
58 - }
59 - f.Remove()
60 - if ff, _ := m.GetFiles(TypeAll); len(ff) != 0 {
61 - t.Fatal("invalid GetFiles len", len(ff))
62 - }
63 - if _, err := f.Open(); err == nil {
64 - t.Fatal("expecting error")
65 - }
66 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage/storage.go deleted
-157
@@ -1,157 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package storage provides storage abstraction for LevelDB.
8 -package storage
9 -
10 -import (
11 - "errors"
12 - "fmt"
13 - "io"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -type FileType uint32
19 -
20 -const (
21 - TypeManifest FileType = 1 << iota
22 - TypeJournal
23 - TypeTable
24 - TypeTemp
25 -
26 - TypeAll = TypeManifest | TypeJournal | TypeTable | TypeTemp
27 -)
28 -
29 -func (t FileType) String() string {
30 - switch t {
31 - case TypeManifest:
32 - return "manifest"
33 - case TypeJournal:
34 - return "journal"
35 - case TypeTable:
36 - return "table"
37 - case TypeTemp:
38 - return "temp"
39 - }
40 - return fmt.Sprintf("<unknown:%d>", t)
41 -}
42 -
43 -var (
44 - ErrInvalidFile = errors.New("leveldb/storage: invalid file for argument")
45 - ErrLocked = errors.New("leveldb/storage: already locked")
46 - ErrClosed = errors.New("leveldb/storage: closed")
47 -)
48 -
49 -// Syncer is the interface that wraps basic Sync method.
50 -type Syncer interface {
51 - // Sync commits the current contents of the file to stable storage.
52 - Sync() error
53 -}
54 -
55 -// Reader is the interface that groups the basic Read, Seek, ReadAt and Close
56 -// methods.
57 -type Reader interface {
58 - io.ReadSeeker
59 - io.ReaderAt
60 - io.Closer
61 -}
62 -
63 -// Writer is the interface that groups the basic Write, Sync and Close
64 -// methods.
65 -type Writer interface {
66 - io.WriteCloser
67 - Syncer
68 -}
69 -
70 -// File is the file. A file instance must be goroutine-safe.
71 -type File interface {
72 - // Open opens the file for read. Returns os.ErrNotExist error
73 - // if the file does not exist.
74 - // Returns ErrClosed if the underlying storage is closed.
75 - Open() (r Reader, err error)
76 -
77 - // Create creates the file for writting. Truncate the file if
78 - // already exist.
79 - // Returns ErrClosed if the underlying storage is closed.
80 - Create() (w Writer, err error)
81 -
82 - // Replace replaces file with newfile.
83 - // Returns ErrClosed if the underlying storage is closed.
84 - Replace(newfile File) error
85 -
86 - // Type returns the file type
87 - Type() FileType
88 -
89 - // Num returns the file number.
90 - Num() uint64
91 -
92 - // Remove removes the file.
93 - // Returns ErrClosed if the underlying storage is closed.
94 - Remove() error
95 -}
96 -
97 -// Storage is the storage. A storage instance must be goroutine-safe.
98 -type Storage interface {
99 - // Lock locks the storage. Any subsequent attempt to call Lock will fail
100 - // until the last lock released.
101 - // After use the caller should call the Release method.
102 - Lock() (l util.Releaser, err error)
103 -
104 - // Log logs a string. This is used for logging. An implementation
105 - // may write to a file, stdout or simply do nothing.
106 - Log(str string)
107 -
108 - // GetFile returns a file for the given number and type. GetFile will never
109 - // returns nil, even if the underlying storage is closed.
110 - GetFile(num uint64, t FileType) File
111 -
112 - // GetFiles returns a slice of files that match the given file types.
113 - // The file types may be OR'ed together.
114 - GetFiles(t FileType) ([]File, error)
115 -
116 - // GetManifest returns a manifest file. Returns os.ErrNotExist if manifest
117 - // file does not exist.
118 - GetManifest() (File, error)
119 -
120 - // SetManifest sets the given file as manifest file. The given file should
121 - // be a manifest file type or error will be returned.
122 - SetManifest(f File) error
123 -
124 - // Close closes the storage. It is valid to call Close multiple times.
125 - // Other methods should not be called after the storage has been closed.
126 - Close() error
127 -}
128 -
129 -// FileInfo wraps basic file info.
130 -type FileInfo struct {
131 - Type FileType
132 - Num uint64
133 -}
134 -
135 -func (fi FileInfo) String() string {
136 - switch fi.Type {
137 - case TypeManifest:
138 - return fmt.Sprintf("MANIFEST-%06d", fi.Num)
139 - case TypeJournal:
140 - return fmt.Sprintf("%06d.log", fi.Num)
141 - case TypeTable:
142 - return fmt.Sprintf("%06d.ldb", fi.Num)
143 - case TypeTemp:
144 - return fmt.Sprintf("%06d.tmp", fi.Num)
145 - default:
146 - return fmt.Sprintf("%#x-%d", fi.Type, fi.Num)
147 - }
148 -}
149 -
150 -// NewFileInfo creates new FileInfo from the given File. It will returns nil
151 -// if File is nil.
152 -func NewFileInfo(f File) *FileInfo {
153 - if f == nil {
154 - return nil
155 - }
156 - return &FileInfo{f.Type(), f.Num()}
157 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage_test.go deleted
-539
@@ -1,539 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "errors"
11 - "fmt"
12 - "io"
13 - "io/ioutil"
14 - "math/rand"
15 - "os"
16 - "path/filepath"
17 - "sync"
18 - "testing"
19 -
20 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
22 -)
23 -
24 -const typeShift = 4
25 -
26 -var (
27 - tsErrInvalidFile = errors.New("leveldb.testStorage: invalid file for argument")
28 - tsErrFileOpen = errors.New("leveldb.testStorage: file still open")
29 -)
30 -
31 -var (
32 - tsFSEnv = os.Getenv("GOLEVELDB_USEFS")
33 - tsTempdir = os.Getenv("GOLEVELDB_TEMPDIR")
34 - tsKeepFS = tsFSEnv == "2"
35 - tsFS = tsKeepFS || tsFSEnv == "" || tsFSEnv == "1"
36 - tsMU = &sync.Mutex{}
37 - tsNum = 0
38 -)
39 -
40 -type tsOp uint
41 -
42 -const (
43 - tsOpOpen tsOp = iota
44 - tsOpCreate
45 - tsOpRead
46 - tsOpReadAt
47 - tsOpWrite
48 - tsOpSync
49 -
50 - tsOpNum
51 -)
52 -
53 -type tsLock struct {
54 - ts *testStorage
55 - r util.Releaser
56 -}
57 -
58 -func (l tsLock) Release() {
59 - l.r.Release()
60 - l.ts.t.Log("I: storage lock released")
61 -}
62 -
63 -type tsReader struct {
64 - tf tsFile
65 - storage.Reader
66 -}
67 -
68 -func (tr tsReader) Read(b []byte) (n int, err error) {
69 - ts := tr.tf.ts
70 - ts.countRead(tr.tf.Type())
71 - if tr.tf.shouldErrLocked(tsOpRead) {
72 - return 0, errors.New("leveldb.testStorage: emulated read error")
73 - }
74 - n, err = tr.Reader.Read(b)
75 - if err != nil && err != io.EOF {
76 - ts.t.Errorf("E: read error, num=%d type=%v n=%d: %v", tr.tf.Num(), tr.tf.Type(), n, err)
77 - }
78 - return
79 -}
80 -
81 -func (tr tsReader) ReadAt(b []byte, off int64) (n int, err error) {
82 - ts := tr.tf.ts
83 - ts.countRead(tr.tf.Type())
84 - if tr.tf.shouldErrLocked(tsOpReadAt) {
85 - return 0, errors.New("leveldb.testStorage: emulated readAt error")
86 - }
87 - n, err = tr.Reader.ReadAt(b, off)
88 - if err != nil && err != io.EOF {
89 - ts.t.Errorf("E: readAt error, num=%d type=%v off=%d n=%d: %v", tr.tf.Num(), tr.tf.Type(), off, n, err)
90 - }
91 - return
92 -}
93 -
94 -func (tr tsReader) Close() (err error) {
95 - err = tr.Reader.Close()
96 - tr.tf.close("reader", err)
97 - return
98 -}
99 -
100 -type tsWriter struct {
101 - tf tsFile
102 - storage.Writer
103 -}
104 -
105 -func (tw tsWriter) Write(b []byte) (n int, err error) {
106 - if tw.tf.shouldErrLocked(tsOpWrite) {
107 - return 0, errors.New("leveldb.testStorage: emulated write error")
108 - }
109 - n, err = tw.Writer.Write(b)
110 - if err != nil {
111 - tw.tf.ts.t.Errorf("E: write error, num=%d type=%v n=%d: %v", tw.tf.Num(), tw.tf.Type(), n, err)
112 - }
113 - return
114 -}
115 -
116 -func (tw tsWriter) Sync() (err error) {
117 - ts := tw.tf.ts
118 - ts.mu.Lock()
119 - for ts.emuDelaySync&tw.tf.Type() != 0 {
120 - ts.cond.Wait()
121 - }
122 - ts.mu.Unlock()
123 - if tw.tf.shouldErrLocked(tsOpSync) {
124 - return errors.New("leveldb.testStorage: emulated sync error")
125 - }
126 - err = tw.Writer.Sync()
127 - if err != nil {
128 - tw.tf.ts.t.Errorf("E: sync error, num=%d type=%v: %v", tw.tf.Num(), tw.tf.Type(), err)
129 - }
130 - return
131 -}
132 -
133 -func (tw tsWriter) Close() (err error) {
134 - err = tw.Writer.Close()
135 - tw.tf.close("writer", err)
136 - return
137 -}
138 -
139 -type tsFile struct {
140 - ts *testStorage
141 - storage.File
142 -}
143 -
144 -func (tf tsFile) x() uint64 {
145 - return tf.Num()<<typeShift | uint64(tf.Type())
146 -}
147 -
148 -func (tf tsFile) shouldErr(op tsOp) bool {
149 - return tf.ts.shouldErr(tf, op)
150 -}
151 -
152 -func (tf tsFile) shouldErrLocked(op tsOp) bool {
153 - tf.ts.mu.Lock()
154 - defer tf.ts.mu.Unlock()
155 - return tf.shouldErr(op)
156 -}
157 -
158 -func (tf tsFile) checkOpen(m string) error {
159 - ts := tf.ts
160 - if writer, ok := ts.opens[tf.x()]; ok {
161 - if writer {
162 - ts.t.Errorf("E: cannot %s file, num=%d type=%v: a writer still open", m, tf.Num(), tf.Type())
163 - } else {
164 - ts.t.Errorf("E: cannot %s file, num=%d type=%v: a reader still open", m, tf.Num(), tf.Type())
165 - }
166 - return tsErrFileOpen
167 - }
168 - return nil
169 -}
170 -
171 -func (tf tsFile) close(m string, err error) {
172 - ts := tf.ts
173 - ts.mu.Lock()
174 - defer ts.mu.Unlock()
175 - if _, ok := ts.opens[tf.x()]; !ok {
176 - ts.t.Errorf("E: %s: redudant file closing, num=%d type=%v", m, tf.Num(), tf.Type())
177 - } else if err == nil {
178 - ts.t.Logf("I: %s: file closed, num=%d type=%v", m, tf.Num(), tf.Type())
179 - }
180 - delete(ts.opens, tf.x())
181 - if err != nil {
182 - ts.t.Errorf("E: %s: cannot close file, num=%d type=%v: %v", m, tf.Num(), tf.Type(), err)
183 - }
184 -}
185 -
186 -func (tf tsFile) Open() (r storage.Reader, err error) {
187 - ts := tf.ts
188 - ts.mu.Lock()
189 - defer ts.mu.Unlock()
190 - err = tf.checkOpen("open")
191 - if err != nil {
192 - return
193 - }
194 - if tf.shouldErr(tsOpOpen) {
195 - err = errors.New("leveldb.testStorage: emulated open error")
196 - return
197 - }
198 - r, err = tf.File.Open()
199 - if err != nil {
200 - if ts.ignoreOpenErr&tf.Type() != 0 {
201 - ts.t.Logf("I: cannot open file, num=%d type=%v: %v (ignored)", tf.Num(), tf.Type(), err)
202 - } else {
203 - ts.t.Errorf("E: cannot open file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
204 - }
205 - } else {
206 - ts.t.Logf("I: file opened, num=%d type=%v", tf.Num(), tf.Type())
207 - ts.opens[tf.x()] = false
208 - r = tsReader{tf, r}
209 - }
210 - return
211 -}
212 -
213 -func (tf tsFile) Create() (w storage.Writer, err error) {
214 - ts := tf.ts
215 - ts.mu.Lock()
216 - defer ts.mu.Unlock()
217 - err = tf.checkOpen("create")
218 - if err != nil {
219 - return
220 - }
221 - if tf.shouldErr(tsOpCreate) {
222 - err = errors.New("leveldb.testStorage: emulated create error")
223 - return
224 - }
225 - w, err = tf.File.Create()
226 - if err != nil {
227 - ts.t.Errorf("E: cannot create file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
228 - } else {
229 - ts.t.Logf("I: file created, num=%d type=%v", tf.Num(), tf.Type())
230 - ts.opens[tf.x()] = true
231 - w = tsWriter{tf, w}
232 - }
233 - return
234 -}
235 -
236 -func (tf tsFile) Replace(newfile storage.File) (err error) {
237 - ts := tf.ts
238 - ts.mu.Lock()
239 - defer ts.mu.Unlock()
240 - err = tf.checkOpen("replace")
241 - if err != nil {
242 - return
243 - }
244 - err = tf.File.Replace(newfile.(tsFile).File)
245 - if err != nil {
246 - ts.t.Errorf("E: cannot replace file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
247 - } else {
248 - ts.t.Logf("I: file replace, num=%d type=%v", tf.Num(), tf.Type())
249 - }
250 - return
251 -}
252 -
253 -func (tf tsFile) Remove() (err error) {
254 - ts := tf.ts
255 - ts.mu.Lock()
256 - defer ts.mu.Unlock()
257 - err = tf.checkOpen("remove")
258 - if err != nil {
259 - return
260 - }
261 - err = tf.File.Remove()
262 - if err != nil {
263 - ts.t.Errorf("E: cannot remove file, num=%d type=%v: %v", tf.Num(), tf.Type(), err)
264 - } else {
265 - ts.t.Logf("I: file removed, num=%d type=%v", tf.Num(), tf.Type())
266 - }
267 - return
268 -}
269 -
270 -type testStorage struct {
271 - t *testing.T
272 - storage.Storage
273 - closeFn func() error
274 -
275 - mu sync.Mutex
276 - cond sync.Cond
277 - // Open files, true=writer, false=reader
278 - opens map[uint64]bool
279 - emuDelaySync storage.FileType
280 - ignoreOpenErr storage.FileType
281 - readCnt uint64
282 - readCntEn storage.FileType
283 -
284 - emuErr [tsOpNum]storage.FileType
285 - emuErrOnce [tsOpNum]storage.FileType
286 - emuRandErr [tsOpNum]storage.FileType
287 - emuRandErrProb int
288 - emuErrOnceMap map[uint64]uint
289 - emuRandRand *rand.Rand
290 -}
291 -
292 -func (ts *testStorage) shouldErr(tf tsFile, op tsOp) bool {
293 - if ts.emuErr[op]&tf.Type() != 0 {
294 - return true
295 - } else if ts.emuRandErr[op]&tf.Type() != 0 || ts.emuErrOnce[op]&tf.Type() != 0 {
296 - sop := uint(1) << op
297 - eop := ts.emuErrOnceMap[tf.x()]
298 - if eop&sop == 0 && (ts.emuRandRand.Int()%ts.emuRandErrProb == 0 || ts.emuErrOnce[op]&tf.Type() != 0) {
299 - ts.emuErrOnceMap[tf.x()] = eop | sop
300 - ts.t.Logf("I: emulated error: file=%d type=%v op=%v", tf.Num(), tf.Type(), op)
301 - return true
302 - }
303 - }
304 - return false
305 -}
306 -
307 -func (ts *testStorage) SetEmuErr(t storage.FileType, ops ...tsOp) {
308 - ts.mu.Lock()
309 - for _, op := range ops {
310 - ts.emuErr[op] = t
311 - }
312 - ts.mu.Unlock()
313 -}
314 -
315 -func (ts *testStorage) SetEmuErrOnce(t storage.FileType, ops ...tsOp) {
316 - ts.mu.Lock()
317 - for _, op := range ops {
318 - ts.emuErrOnce[op] = t
319 - }
320 - ts.mu.Unlock()
321 -}
322 -
323 -func (ts *testStorage) SetEmuRandErr(t storage.FileType, ops ...tsOp) {
324 - ts.mu.Lock()
325 - for _, op := range ops {
326 - ts.emuRandErr[op] = t
327 - }
328 - ts.mu.Unlock()
329 -}
330 -
331 -func (ts *testStorage) SetEmuRandErrProb(prob int) {
332 - ts.mu.Lock()
333 - ts.emuRandErrProb = prob
334 - ts.mu.Unlock()
335 -}
336 -
337 -func (ts *testStorage) DelaySync(t storage.FileType) {
338 - ts.mu.Lock()
339 - ts.emuDelaySync |= t
340 - ts.cond.Broadcast()
341 - ts.mu.Unlock()
342 -}
343 -
344 -func (ts *testStorage) ReleaseSync(t storage.FileType) {
345 - ts.mu.Lock()
346 - ts.emuDelaySync &= ^t
347 - ts.cond.Broadcast()
348 - ts.mu.Unlock()
349 -}
350 -
351 -func (ts *testStorage) ReadCounter() uint64 {
352 - ts.mu.Lock()
353 - defer ts.mu.Unlock()
354 - return ts.readCnt
355 -}
356 -
357 -func (ts *testStorage) ResetReadCounter() {
358 - ts.mu.Lock()
359 - ts.readCnt = 0
360 - ts.mu.Unlock()
361 -}
362 -
363 -func (ts *testStorage) SetReadCounter(t storage.FileType) {
364 - ts.mu.Lock()
365 - ts.readCntEn = t
366 - ts.mu.Unlock()
367 -}
368 -
369 -func (ts *testStorage) countRead(t storage.FileType) {
370 - ts.mu.Lock()
371 - if ts.readCntEn&t != 0 {
372 - ts.readCnt++
373 - }
374 - ts.mu.Unlock()
375 -}
376 -
377 -func (ts *testStorage) SetIgnoreOpenErr(t storage.FileType) {
378 - ts.ignoreOpenErr = t
379 -}
380 -
381 -func (ts *testStorage) Lock() (r util.Releaser, err error) {
382 - r, err = ts.Storage.Lock()
383 - if err != nil {
384 - ts.t.Logf("W: storage locking failed: %v", err)
385 - } else {
386 - ts.t.Log("I: storage locked")
387 - r = tsLock{ts, r}
388 - }
389 - return
390 -}
391 -
392 -func (ts *testStorage) Log(str string) {
393 - ts.t.Log("L: " + str)
394 - ts.Storage.Log(str)
395 -}
396 -
397 -func (ts *testStorage) GetFile(num uint64, t storage.FileType) storage.File {
398 - return tsFile{ts, ts.Storage.GetFile(num, t)}
399 -}
400 -
401 -func (ts *testStorage) GetFiles(t storage.FileType) (ff []storage.File, err error) {
402 - ff0, err := ts.Storage.GetFiles(t)
403 - if err != nil {
404 - ts.t.Errorf("E: get files failed: %v", err)
405 - return
406 - }
407 - ff = make([]storage.File, len(ff0))
408 - for i, f := range ff0 {
409 - ff[i] = tsFile{ts, f}
410 - }
411 - ts.t.Logf("I: get files, type=0x%x count=%d", int(t), len(ff))
412 - return
413 -}
414 -
415 -func (ts *testStorage) GetManifest() (f storage.File, err error) {
416 - f0, err := ts.Storage.GetManifest()
417 - if err != nil {
418 - if !os.IsNotExist(err) {
419 - ts.t.Errorf("E: get manifest failed: %v", err)
420 - }
421 - return
422 - }
423 - f = tsFile{ts, f0}
424 - ts.t.Logf("I: get manifest, num=%d", f.Num())
425 - return
426 -}
427 -
428 -func (ts *testStorage) SetManifest(f storage.File) error {
429 - tf, ok := f.(tsFile)
430 - if !ok {
431 - ts.t.Error("E: set manifest failed: type assertion failed")
432 - return tsErrInvalidFile
433 - } else if tf.Type() != storage.TypeManifest {
434 - ts.t.Errorf("E: set manifest failed: invalid file type: %s", tf.Type())
435 - return tsErrInvalidFile
436 - }
437 - err := ts.Storage.SetManifest(tf.File)
438 - if err != nil {
439 - ts.t.Errorf("E: set manifest failed: %v", err)
440 - } else {
441 - ts.t.Logf("I: set manifest, num=%d", tf.Num())
442 - }
443 - return err
444 -}
445 -
446 -func (ts *testStorage) Close() error {
447 - ts.CloseCheck()
448 - err := ts.Storage.Close()
449 - if err != nil {
450 - ts.t.Errorf("E: closing storage failed: %v", err)
451 - } else {
452 - ts.t.Log("I: storage closed")
453 - }
454 - if ts.closeFn != nil {
455 - if err := ts.closeFn(); err != nil {
456 - ts.t.Errorf("E: close function: %v", err)
457 - }
458 - }
459 - return err
460 -}
461 -
462 -func (ts *testStorage) CloseCheck() {
463 - ts.mu.Lock()
464 - if len(ts.opens) == 0 {
465 - ts.t.Log("I: all files are closed")
466 - } else {
467 - ts.t.Errorf("E: %d files still open", len(ts.opens))
468 - for x, writer := range ts.opens {
469 - num, tt := x>>typeShift, storage.FileType(x)&storage.TypeAll
470 - ts.t.Errorf("E: * num=%d type=%v writer=%v", num, tt, writer)
471 - }
472 - }
473 - ts.mu.Unlock()
474 -}
475 -
476 -func newTestStorage(t *testing.T) *testStorage {
477 - var stor storage.Storage
478 - var closeFn func() error
479 - if tsFS {
480 - for {
481 - tsMU.Lock()
482 - num := tsNum
483 - tsNum++
484 - tsMU.Unlock()
485 - tempdir := tsTempdir
486 - if tempdir == "" {
487 - tempdir = os.TempDir()
488 - }
489 - path := filepath.Join(tempdir, fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
490 - if _, err := os.Stat(path); err != nil {
491 - stor, err = storage.OpenFile(path)
492 - if err != nil {
493 - t.Fatalf("F: cannot create storage: %v", err)
494 - }
495 - t.Logf("I: storage created: %s", path)
496 - closeFn = func() error {
497 - for _, name := range []string{"LOG.old", "LOG"} {
498 - f, err := os.Open(filepath.Join(path, name))
499 - if err != nil {
500 - continue
501 - }
502 - if log, err := ioutil.ReadAll(f); err != nil {
503 - t.Logf("---------------------- %s ----------------------", name)
504 - t.Logf("cannot read log: %v", err)
505 - t.Logf("---------------------- %s ----------------------", name)
506 - } else if len(log) > 0 {
507 - t.Logf("---------------------- %s ----------------------\n%s", name, string(log))
508 - t.Logf("---------------------- %s ----------------------", name)
509 - }
510 - f.Close()
511 - }
512 - if t.Failed() {
513 - t.Logf("testing failed, test DB preserved at %s", path)
514 - return nil
515 - }
516 - if tsKeepFS {
517 - return nil
518 - }
519 - return os.RemoveAll(path)
520 - }
521 -
522 - break
523 - }
524 - }
525 - } else {
526 - stor = storage.NewMemStorage()
527 - }
528 - ts := &testStorage{
529 - t: t,
530 - Storage: stor,
531 - closeFn: closeFn,
532 - opens: make(map[uint64]bool),
533 - emuErrOnceMap: make(map[uint64]uint),
534 - emuRandErrProb: 0x999,
535 - emuRandRand: rand.New(rand.NewSource(0xfacedead)),
536 - }
537 - ts.cond.L = &ts.mu
538 - return ts
539 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table.go deleted
-521
@@ -1,521 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "fmt"
11 - "sort"
12 - "sync/atomic"
13 -
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
20 -)
21 -
22 -// tFile holds basic information about a table.
23 -type tFile struct {
24 - file storage.File
25 - seekLeft int32
26 - size uint64
27 - imin, imax iKey
28 -}
29 -
30 -// Returns true if given key is after largest key of this table.
31 -func (t *tFile) after(icmp *iComparer, ukey []byte) bool {
32 - return ukey != nil && icmp.uCompare(ukey, t.imax.ukey()) > 0
33 -}
34 -
35 -// Returns true if given key is before smallest key of this table.
36 -func (t *tFile) before(icmp *iComparer, ukey []byte) bool {
37 - return ukey != nil && icmp.uCompare(ukey, t.imin.ukey()) < 0
38 -}
39 -
40 -// Returns true if given key range overlaps with this table key range.
41 -func (t *tFile) overlaps(icmp *iComparer, umin, umax []byte) bool {
42 - return !t.after(icmp, umin) && !t.before(icmp, umax)
43 -}
44 -
45 -// Cosumes one seek and return current seeks left.
46 -func (t *tFile) consumeSeek() int32 {
47 - return atomic.AddInt32(&t.seekLeft, -1)
48 -}
49 -
50 -// Creates new tFile.
51 -func newTableFile(file storage.File, size uint64, imin, imax iKey) *tFile {
52 - f := &tFile{
53 - file: file,
54 - size: size,
55 - imin: imin,
56 - imax: imax,
57 - }
58 -
59 - // We arrange to automatically compact this file after
60 - // a certain number of seeks. Let's assume:
61 - // (1) One seek costs 10ms
62 - // (2) Writing or reading 1MB costs 10ms (100MB/s)
63 - // (3) A compaction of 1MB does 25MB of IO:
64 - // 1MB read from this level
65 - // 10-12MB read from next level (boundaries may be misaligned)
66 - // 10-12MB written to next level
67 - // This implies that 25 seeks cost the same as the compaction
68 - // of 1MB of data. I.e., one seek costs approximately the
69 - // same as the compaction of 40KB of data. We are a little
70 - // conservative and allow approximately one seek for every 16KB
71 - // of data before triggering a compaction.
72 - f.seekLeft = int32(size / 16384)
73 - if f.seekLeft < 100 {
74 - f.seekLeft = 100
75 - }
76 -
77 - return f
78 -}
79 -
80 -// tFiles hold multiple tFile.
81 -type tFiles []*tFile
82 -
83 -func (tf tFiles) Len() int { return len(tf) }
84 -func (tf tFiles) Swap(i, j int) { tf[i], tf[j] = tf[j], tf[i] }
85 -
86 -func (tf tFiles) nums() string {
87 - x := "[ "
88 - for i, f := range tf {
89 - if i != 0 {
90 - x += ", "
91 - }
92 - x += fmt.Sprint(f.file.Num())
93 - }
94 - x += " ]"
95 - return x
96 -}
97 -
98 -// Returns true if i smallest key is less than j.
99 -// This used for sort by key in ascending order.
100 -func (tf tFiles) lessByKey(icmp *iComparer, i, j int) bool {
101 - a, b := tf[i], tf[j]
102 - n := icmp.Compare(a.imin, b.imin)
103 - if n == 0 {
104 - return a.file.Num() < b.file.Num()
105 - }
106 - return n < 0
107 -}
108 -
109 -// Returns true if i file number is greater than j.
110 -// This used for sort by file number in descending order.
111 -func (tf tFiles) lessByNum(i, j int) bool {
112 - return tf[i].file.Num() > tf[j].file.Num()
113 -}
114 -
115 -// Sorts tables by key in ascending order.
116 -func (tf tFiles) sortByKey(icmp *iComparer) {
117 - sort.Sort(&tFilesSortByKey{tFiles: tf, icmp: icmp})
118 -}
119 -
120 -// Sorts tables by file number in descending order.
121 -func (tf tFiles) sortByNum() {
122 - sort.Sort(&tFilesSortByNum{tFiles: tf})
123 -}
124 -
125 -// Returns sum of all tables size.
126 -func (tf tFiles) size() (sum uint64) {
127 - for _, t := range tf {
128 - sum += t.size
129 - }
130 - return sum
131 -}
132 -
133 -// Searches smallest index of tables whose its smallest
134 -// key is after or equal with given key.
135 -func (tf tFiles) searchMin(icmp *iComparer, ikey iKey) int {
136 - return sort.Search(len(tf), func(i int) bool {
137 - return icmp.Compare(tf[i].imin, ikey) >= 0
138 - })
139 -}
140 -
141 -// Searches smallest index of tables whose its largest
142 -// key is after or equal with given key.
143 -func (tf tFiles) searchMax(icmp *iComparer, ikey iKey) int {
144 - return sort.Search(len(tf), func(i int) bool {
145 - return icmp.Compare(tf[i].imax, ikey) >= 0
146 - })
147 -}
148 -
149 -// Returns true if given key range overlaps with one or more
150 -// tables key range. If unsorted is true then binary search will not be used.
151 -func (tf tFiles) overlaps(icmp *iComparer, umin, umax []byte, unsorted bool) bool {
152 - if unsorted {
153 - // Check against all files.
154 - for _, t := range tf {
155 - if t.overlaps(icmp, umin, umax) {
156 - return true
157 - }
158 - }
159 - return false
160 - }
161 -
162 - i := 0
163 - if len(umin) > 0 {
164 - // Find the earliest possible internal key for min.
165 - i = tf.searchMax(icmp, newIkey(umin, kMaxSeq, ktSeek))
166 - }
167 - if i >= len(tf) {
168 - // Beginning of range is after all files, so no overlap.
169 - return false
170 - }
171 - return !tf[i].before(icmp, umax)
172 -}
173 -
174 -// Returns tables whose its key range overlaps with given key range.
175 -// Range will be expanded if ukey found hop across tables.
176 -// If overlapped is true then the search will be restarted if umax
177 -// expanded.
178 -// The dst content will be overwritten.
179 -func (tf tFiles) getOverlaps(dst tFiles, icmp *iComparer, umin, umax []byte, overlapped bool) tFiles {
180 - dst = dst[:0]
181 - for i := 0; i < len(tf); {
182 - t := tf[i]
183 - if t.overlaps(icmp, umin, umax) {
184 - if umin != nil && icmp.uCompare(t.imin.ukey(), umin) < 0 {
185 - umin = t.imin.ukey()
186 - dst = dst[:0]
187 - i = 0
188 - continue
189 - } else if umax != nil && icmp.uCompare(t.imax.ukey(), umax) > 0 {
190 - umax = t.imax.ukey()
191 - // Restart search if it is overlapped.
192 - if overlapped {
193 - dst = dst[:0]
194 - i = 0
195 - continue
196 - }
197 - }
198 -
199 - dst = append(dst, t)
200 - }
201 - i++
202 - }
203 -
204 - return dst
205 -}
206 -
207 -// Returns tables key range.
208 -func (tf tFiles) getRange(icmp *iComparer) (imin, imax iKey) {
209 - for i, t := range tf {
210 - if i == 0 {
211 - imin, imax = t.imin, t.imax
212 - continue
213 - }
214 - if icmp.Compare(t.imin, imin) < 0 {
215 - imin = t.imin
216 - }
217 - if icmp.Compare(t.imax, imax) > 0 {
218 - imax = t.imax
219 - }
220 - }
221 -
222 - return
223 -}
224 -
225 -// Creates iterator index from tables.
226 -func (tf tFiles) newIndexIterator(tops *tOps, icmp *iComparer, slice *util.Range, ro *opt.ReadOptions) iterator.IteratorIndexer {
227 - if slice != nil {
228 - var start, limit int
229 - if slice.Start != nil {
230 - start = tf.searchMax(icmp, iKey(slice.Start))
231 - }
232 - if slice.Limit != nil {
233 - limit = tf.searchMin(icmp, iKey(slice.Limit))
234 - } else {
235 - limit = tf.Len()
236 - }
237 - tf = tf[start:limit]
238 - }
239 - return iterator.NewArrayIndexer(&tFilesArrayIndexer{
240 - tFiles: tf,
241 - tops: tops,
242 - icmp: icmp,
243 - slice: slice,
244 - ro: ro,
245 - })
246 -}
247 -
248 -// Tables iterator index.
249 -type tFilesArrayIndexer struct {
250 - tFiles
251 - tops *tOps
252 - icmp *iComparer
253 - slice *util.Range
254 - ro *opt.ReadOptions
255 -}
256 -
257 -func (a *tFilesArrayIndexer) Search(key []byte) int {
258 - return a.searchMax(a.icmp, iKey(key))
259 -}
260 -
261 -func (a *tFilesArrayIndexer) Get(i int) iterator.Iterator {
262 - if i == 0 || i == a.Len()-1 {
263 - return a.tops.newIterator(a.tFiles[i], a.slice, a.ro)
264 - }
265 - return a.tops.newIterator(a.tFiles[i], nil, a.ro)
266 -}
267 -
268 -// Helper type for sortByKey.
269 -type tFilesSortByKey struct {
270 - tFiles
271 - icmp *iComparer
272 -}
273 -
274 -func (x *tFilesSortByKey) Less(i, j int) bool {
275 - return x.lessByKey(x.icmp, i, j)
276 -}
277 -
278 -// Helper type for sortByNum.
279 -type tFilesSortByNum struct {
280 - tFiles
281 -}
282 -
283 -func (x *tFilesSortByNum) Less(i, j int) bool {
284 - return x.lessByNum(i, j)
285 -}
286 -
287 -// Table operations.
288 -type tOps struct {
289 - s *session
290 - cache *cache.Cache
291 - bcache *cache.Cache
292 - bpool *util.BufferPool
293 -}
294 -
295 -// Creates an empty table and returns table writer.
296 -func (t *tOps) create() (*tWriter, error) {
297 - file := t.s.getTableFile(t.s.allocFileNum())
298 - fw, err := file.Create()
299 - if err != nil {
300 - return nil, err
301 - }
302 - return &tWriter{
303 - t: t,
304 - file: file,
305 - w: fw,
306 - tw: table.NewWriter(fw, t.s.o.Options),
307 - }, nil
308 -}
309 -
310 -// Builds table from src iterator.
311 -func (t *tOps) createFrom(src iterator.Iterator) (f *tFile, n int, err error) {
312 - w, err := t.create()
313 - if err != nil {
314 - return
315 - }
316 -
317 - defer func() {
318 - if err != nil {
319 - w.drop()
320 - }
321 - }()
322 -
323 - for src.Next() {
324 - err = w.append(src.Key(), src.Value())
325 - if err != nil {
326 - return
327 - }
328 - }
329 - err = src.Error()
330 - if err != nil {
331 - return
332 - }
333 -
334 - n = w.tw.EntriesLen()
335 - f, err = w.finish()
336 - return
337 -}
338 -
339 -// Opens table. It returns a cache handle, which should
340 -// be released after use.
341 -func (t *tOps) open(f *tFile) (ch *cache.Handle, err error) {
342 - num := f.file.Num()
343 - ch = t.cache.Get(0, num, func() (size int, value cache.Value) {
344 - var r storage.Reader
345 - r, err = f.file.Open()
346 - if err != nil {
347 - return 0, nil
348 - }
349 -
350 - var bcache *cache.CacheGetter
351 - if t.bcache != nil {
352 - bcache = &cache.CacheGetter{Cache: t.bcache, NS: num}
353 - }
354 -
355 - var tr *table.Reader
356 - tr, err = table.NewReader(r, int64(f.size), storage.NewFileInfo(f.file), bcache, t.bpool, t.s.o.Options)
357 - if err != nil {
358 - r.Close()
359 - return 0, nil
360 - }
361 - return 1, tr
362 -
363 - })
364 - if ch == nil && err == nil {
365 - err = ErrClosed
366 - }
367 - return
368 -}
369 -
370 -// Finds key/value pair whose key is greater than or equal to the
371 -// given key.
372 -func (t *tOps) find(f *tFile, key []byte, ro *opt.ReadOptions) (rkey, rvalue []byte, err error) {
373 - ch, err := t.open(f)
374 - if err != nil {
375 - return nil, nil, err
376 - }
377 - defer ch.Release()
378 - return ch.Value().(*table.Reader).Find(key, true, ro)
379 -}
380 -
381 -// Finds key that is greater than or equal to the given key.
382 -func (t *tOps) findKey(f *tFile, key []byte, ro *opt.ReadOptions) (rkey []byte, err error) {
383 - ch, err := t.open(f)
384 - if err != nil {
385 - return nil, err
386 - }
387 - defer ch.Release()
388 - return ch.Value().(*table.Reader).FindKey(key, true, ro)
389 -}
390 -
391 -// Returns approximate offset of the given key.
392 -func (t *tOps) offsetOf(f *tFile, key []byte) (offset uint64, err error) {
393 - ch, err := t.open(f)
394 - if err != nil {
395 - return
396 - }
397 - defer ch.Release()
398 - offset_, err := ch.Value().(*table.Reader).OffsetOf(key)
399 - return uint64(offset_), err
400 -}
401 -
402 -// Creates an iterator from the given table.
403 -func (t *tOps) newIterator(f *tFile, slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
404 - ch, err := t.open(f)
405 - if err != nil {
406 - return iterator.NewEmptyIterator(err)
407 - }
408 - iter := ch.Value().(*table.Reader).NewIterator(slice, ro)
409 - iter.SetReleaser(ch)
410 - return iter
411 -}
412 -
413 -// Removes table from persistent storage. It waits until
414 -// no one use the the table.
415 -func (t *tOps) remove(f *tFile) {
416 - num := f.file.Num()
417 - t.cache.Delete(0, num, func() {
418 - if err := f.file.Remove(); err != nil {
419 - t.s.logf("table@remove removing @%d %q", num, err)
420 - } else {
421 - t.s.logf("table@remove removed @%d", num)
422 - }
423 - if t.bcache != nil {
424 - t.bcache.EvictNS(num)
425 - }
426 - })
427 -}
428 -
429 -// Closes the table ops instance. It will close all tables,
430 -// regadless still used or not.
431 -func (t *tOps) close() {
432 - t.bpool.Close()
433 - t.cache.Close()
434 - if t.bcache != nil {
435 - t.bcache.Close()
436 - }
437 -}
438 -
439 -// Creates new initialized table ops instance.
440 -func newTableOps(s *session) *tOps {
441 - var (
442 - cacher cache.Cacher
443 - bcache *cache.Cache
444 - )
445 - if s.o.GetOpenFilesCacheCapacity() > 0 {
446 - cacher = cache.NewLRU(s.o.GetOpenFilesCacheCapacity())
447 - }
448 - if !s.o.DisableBlockCache {
449 - var bcacher cache.Cacher
450 - if s.o.GetBlockCacheCapacity() > 0 {
451 - bcacher = cache.NewLRU(s.o.GetBlockCacheCapacity())
452 - }
453 - bcache = cache.NewCache(bcacher)
454 - }
455 - return &tOps{
456 - s: s,
457 - cache: cache.NewCache(cacher),
458 - bcache: bcache,
459 - bpool: util.NewBufferPool(s.o.GetBlockSize() + 5),
460 - }
461 -}
462 -
463 -// tWriter wraps the table writer. It keep track of file descriptor
464 -// and added key range.
465 -type tWriter struct {
466 - t *tOps
467 -
468 - file storage.File
469 - w storage.Writer
470 - tw *table.Writer
471 -
472 - first, last []byte
473 -}
474 -
475 -// Append key/value pair to the table.
476 -func (w *tWriter) append(key, value []byte) error {
477 - if w.first == nil {
478 - w.first = append([]byte{}, key...)
479 - }
480 - w.last = append(w.last[:0], key...)
481 - return w.tw.Append(key, value)
482 -}
483 -
484 -// Returns true if the table is empty.
485 -func (w *tWriter) empty() bool {
486 - return w.first == nil
487 -}
488 -
489 -// Closes the storage.Writer.
490 -func (w *tWriter) close() {
491 - if w.w != nil {
492 - w.w.Close()
493 - w.w = nil
494 - }
495 -}
496 -
497 -// Finalizes the table and returns table file.
498 -func (w *tWriter) finish() (f *tFile, err error) {
499 - defer w.close()
500 - err = w.tw.Close()
501 - if err != nil {
502 - return
503 - }
504 - err = w.w.Sync()
505 - if err != nil {
506 - return
507 - }
508 - f = newTableFile(w.file, uint64(w.tw.BytesLen()), iKey(w.first), iKey(w.last))
509 - return
510 -}
511 -
512 -// Drops the table.
513 -func (w *tWriter) drop() {
514 - w.close()
515 - w.file.Remove()
516 - w.t.s.reuseFileNum(w.file.Num())
517 - w.file = nil
518 - w.tw = nil
519 - w.first = nil
520 - w.last = nil
521 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/block_test.go deleted
-139
@@ -1,139 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package table
8 -
9 -import (
10 - "encoding/binary"
11 - "fmt"
12 -
13 - . "github.com/onsi/ginkgo"
14 - . "github.com/onsi/gomega"
15 -
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
20 -)
21 -
22 -type blockTesting struct {
23 - tr *Reader
24 - b *block
25 -}
26 -
27 -func (t *blockTesting) TestNewIterator(slice *util.Range) iterator.Iterator {
28 - return t.tr.newBlockIter(t.b, nil, slice, false)
29 -}
30 -
31 -var _ = testutil.Defer(func() {
32 - Describe("Block", func() {
33 - Build := func(kv *testutil.KeyValue, restartInterval int) *blockTesting {
34 - // Building the block.
35 - bw := &blockWriter{
36 - restartInterval: restartInterval,
37 - scratch: make([]byte, 30),
38 - }
39 - kv.Iterate(func(i int, key, value []byte) {
40 - bw.append(key, value)
41 - })
42 - bw.finish()
43 -
44 - // Opening the block.
45 - data := bw.buf.Bytes()
46 - restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
47 - return &blockTesting{
48 - tr: &Reader{cmp: comparer.DefaultComparer},
49 - b: &block{
50 - data: data,
51 - restartsLen: restartsLen,
52 - restartsOffset: len(data) - (restartsLen+1)*4,
53 - },
54 - }
55 - }
56 -
57 - Describe("read test", func() {
58 - for restartInterval := 1; restartInterval <= 5; restartInterval++ {
59 - Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
60 - kv := &testutil.KeyValue{}
61 - Text := func() string {
62 - return fmt.Sprintf("and %d keys", kv.Len())
63 - }
64 -
65 - Test := func() {
66 - // Make block.
67 - br := Build(kv, restartInterval)
68 - // Do testing.
69 - testutil.KeyValueTesting(nil, kv.Clone(), br, nil, nil)
70 - }
71 -
72 - Describe(Text(), Test)
73 -
74 - kv.PutString("", "empty")
75 - Describe(Text(), Test)
76 -
77 - kv.PutString("a1", "foo")
78 - Describe(Text(), Test)
79 -
80 - kv.PutString("a2", "v")
81 - Describe(Text(), Test)
82 -
83 - kv.PutString("a3qqwrkks", "hello")
84 - Describe(Text(), Test)
85 -
86 - kv.PutString("a4", "bar")
87 - Describe(Text(), Test)
88 -
89 - kv.PutString("a5111111", "v5")
90 - kv.PutString("a6", "")
91 - kv.PutString("a7", "v7")
92 - kv.PutString("a8", "vvvvvvvvvvvvvvvvvvvvvv8")
93 - kv.PutString("b", "v9")
94 - kv.PutString("c9", "v9")
95 - kv.PutString("c91", "v9")
96 - kv.PutString("d0", "v9")
97 - Describe(Text(), Test)
98 - })
99 - }
100 - })
101 -
102 - Describe("out-of-bound slice test", func() {
103 - kv := &testutil.KeyValue{}
104 - kv.PutString("k1", "v1")
105 - kv.PutString("k2", "v2")
106 - kv.PutString("k3abcdefgg", "v3")
107 - kv.PutString("k4", "v4")
108 - kv.PutString("k5", "v5")
109 - for restartInterval := 1; restartInterval <= 5; restartInterval++ {
110 - Describe(fmt.Sprintf("with restart interval of %d", restartInterval), func() {
111 - // Make block.
112 - bt := Build(kv, restartInterval)
113 -
114 - Test := func(r *util.Range) func(done Done) {
115 - return func(done Done) {
116 - iter := bt.TestNewIterator(r)
117 - Expect(iter.Error()).ShouldNot(HaveOccurred())
118 -
119 - t := testutil.IteratorTesting{
120 - KeyValue: kv.Clone(),
121 - Iter: iter,
122 - }
123 -
124 - testutil.DoIteratorTesting(&t)
125 - iter.Release()
126 - done <- true
127 - }
128 - }
129 -
130 - It("Should do iterations and seeks correctly #0",
131 - Test(&util.Range{Start: []byte("k0"), Limit: []byte("k6")}), 2.0)
132 -
133 - It("Should do iterations and seeks correctly #1",
134 - Test(&util.Range{Start: []byte(""), Limit: []byte("zzzzzzz")}), 2.0)
135 - })
136 - }
137 - })
138 - })
139 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/reader.go deleted
-1106
@@ -1,1106 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package table
8 -
9 -import (
10 - "encoding/binary"
11 - "fmt"
12 - "io"
13 - "sort"
14 - "strings"
15 - "sync"
16 -
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/cache"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
20 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
22 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
23 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
24 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
25 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
26 -)
27 -
28 -var (
29 - ErrNotFound = errors.ErrNotFound
30 - ErrReaderReleased = errors.New("leveldb/table: reader released")
31 - ErrIterReleased = errors.New("leveldb/table: iterator released")
32 -)
33 -
34 -type ErrCorrupted struct {
35 - Pos int64
36 - Size int64
37 - Kind string
38 - Reason string
39 -}
40 -
41 -func (e *ErrCorrupted) Error() string {
42 - return fmt.Sprintf("leveldb/table: corruption on %s (pos=%d): %s", e.Kind, e.Pos, e.Reason)
43 -}
44 -
45 -func max(x, y int) int {
46 - if x > y {
47 - return x
48 - }
49 - return y
50 -}
51 -
52 -type block struct {
53 - bpool *util.BufferPool
54 - bh blockHandle
55 - data []byte
56 - restartsLen int
57 - restartsOffset int
58 -}
59 -
60 -func (b *block) seek(cmp comparer.Comparer, rstart, rlimit int, key []byte) (index, offset int, err error) {
61 - index = sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
62 - offset := int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):]))
63 - offset += 1 // shared always zero, since this is a restart point
64 - v1, n1 := binary.Uvarint(b.data[offset:]) // key length
65 - _, n2 := binary.Uvarint(b.data[offset+n1:]) // value length
66 - m := offset + n1 + n2
67 - return cmp.Compare(b.data[m:m+int(v1)], key) > 0
68 - }) + rstart - 1
69 - if index < rstart {
70 - // The smallest key is greater-than key sought.
71 - index = rstart
72 - }
73 - offset = int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:]))
74 - return
75 -}
76 -
77 -func (b *block) restartIndex(rstart, rlimit, offset int) int {
78 - return sort.Search(b.restartsLen-rstart-(b.restartsLen-rlimit), func(i int) bool {
79 - return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*(rstart+i):])) > offset
80 - }) + rstart - 1
81 -}
82 -
83 -func (b *block) restartOffset(index int) int {
84 - return int(binary.LittleEndian.Uint32(b.data[b.restartsOffset+4*index:]))
85 -}
86 -
87 -func (b *block) entry(offset int) (key, value []byte, nShared, n int, err error) {
88 - if offset >= b.restartsOffset {
89 - if offset != b.restartsOffset {
90 - err = &ErrCorrupted{Reason: "entries offset not aligned"}
91 - }
92 - return
93 - }
94 - v0, n0 := binary.Uvarint(b.data[offset:]) // Shared prefix length
95 - v1, n1 := binary.Uvarint(b.data[offset+n0:]) // Key length
96 - v2, n2 := binary.Uvarint(b.data[offset+n0+n1:]) // Value length
97 - m := n0 + n1 + n2
98 - n = m + int(v1) + int(v2)
99 - if n0 <= 0 || n1 <= 0 || n2 <= 0 || offset+n > b.restartsOffset {
100 - err = &ErrCorrupted{Reason: "entries corrupted"}
101 - return
102 - }
103 - key = b.data[offset+m : offset+m+int(v1)]
104 - value = b.data[offset+m+int(v1) : offset+n]
105 - nShared = int(v0)
106 - return
107 -}
108 -
109 -func (b *block) Release() {
110 - b.bpool.Put(b.data)
111 - b.bpool = nil
112 - b.data = nil
113 -}
114 -
115 -type dir int
116 -
117 -const (
118 - dirReleased dir = iota - 1
119 - dirSOI
120 - dirEOI
121 - dirBackward
122 - dirForward
123 -)
124 -
125 -type blockIter struct {
126 - tr *Reader
127 - block *block
128 - blockReleaser util.Releaser
129 - releaser util.Releaser
130 - key, value []byte
131 - offset int
132 - // Previous offset, only filled by Next.
133 - prevOffset int
134 - prevNode []int
135 - prevKeys []byte
136 - restartIndex int
137 - // Iterator direction.
138 - dir dir
139 - // Restart index slice range.
140 - riStart int
141 - riLimit int
142 - // Offset slice range.
143 - offsetStart int
144 - offsetRealStart int
145 - offsetLimit int
146 - // Error.
147 - err error
148 -}
149 -
150 -func (i *blockIter) sErr(err error) {
151 - i.err = err
152 - i.key = nil
153 - i.value = nil
154 - i.prevNode = nil
155 - i.prevKeys = nil
156 -}
157 -
158 -func (i *blockIter) reset() {
159 - if i.dir == dirBackward {
160 - i.prevNode = i.prevNode[:0]
161 - i.prevKeys = i.prevKeys[:0]
162 - }
163 - i.restartIndex = i.riStart
164 - i.offset = i.offsetStart
165 - i.dir = dirSOI
166 - i.key = i.key[:0]
167 - i.value = nil
168 -}
169 -
170 -func (i *blockIter) isFirst() bool {
171 - switch i.dir {
172 - case dirForward:
173 - return i.prevOffset == i.offsetRealStart
174 - case dirBackward:
175 - return len(i.prevNode) == 1 && i.restartIndex == i.riStart
176 - }
177 - return false
178 -}
179 -
180 -func (i *blockIter) isLast() bool {
181 - switch i.dir {
182 - case dirForward, dirBackward:
183 - return i.offset == i.offsetLimit
184 - }
185 - return false
186 -}
187 -
188 -func (i *blockIter) First() bool {
189 - if i.err != nil {
190 - return false
191 - } else if i.dir == dirReleased {
192 - i.err = ErrIterReleased
193 - return false
194 - }
195 -
196 - if i.dir == dirBackward {
197 - i.prevNode = i.prevNode[:0]
198 - i.prevKeys = i.prevKeys[:0]
199 - }
200 - i.dir = dirSOI
201 - return i.Next()
202 -}
203 -
204 -func (i *blockIter) Last() bool {
205 - if i.err != nil {
206 - return false
207 - } else if i.dir == dirReleased {
208 - i.err = ErrIterReleased
209 - return false
210 - }
211 -
212 - if i.dir == dirBackward {
213 - i.prevNode = i.prevNode[:0]
214 - i.prevKeys = i.prevKeys[:0]
215 - }
216 - i.dir = dirEOI
217 - return i.Prev()
218 -}
219 -
220 -func (i *blockIter) Seek(key []byte) bool {
221 - if i.err != nil {
222 - return false
223 - } else if i.dir == dirReleased {
224 - i.err = ErrIterReleased
225 - return false
226 - }
227 -
228 - ri, offset, err := i.block.seek(i.tr.cmp, i.riStart, i.riLimit, key)
229 - if err != nil {
230 - i.sErr(err)
231 - return false
232 - }
233 - i.restartIndex = ri
234 - i.offset = max(i.offsetStart, offset)
235 - if i.dir == dirSOI || i.dir == dirEOI {
236 - i.dir = dirForward
237 - }
238 - for i.Next() {
239 - if i.tr.cmp.Compare(i.key, key) >= 0 {
240 - return true
241 - }
242 - }
243 - return false
244 -}
245 -
246 -func (i *blockIter) Next() bool {
247 - if i.dir == dirEOI || i.err != nil {
248 - return false
249 - } else if i.dir == dirReleased {
250 - i.err = ErrIterReleased
251 - return false
252 - }
253 -
254 - if i.dir == dirSOI {
255 - i.restartIndex = i.riStart
256 - i.offset = i.offsetStart
257 - } else if i.dir == dirBackward {
258 - i.prevNode = i.prevNode[:0]
259 - i.prevKeys = i.prevKeys[:0]
260 - }
261 - for i.offset < i.offsetRealStart {
262 - key, value, nShared, n, err := i.block.entry(i.offset)
263 - if err != nil {
264 - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
265 - return false
266 - }
267 - if n == 0 {
268 - i.dir = dirEOI
269 - return false
270 - }
271 - i.key = append(i.key[:nShared], key...)
272 - i.value = value
273 - i.offset += n
274 - }
275 - if i.offset >= i.offsetLimit {
276 - i.dir = dirEOI
277 - if i.offset != i.offsetLimit {
278 - i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned"))
279 - }
280 - return false
281 - }
282 - key, value, nShared, n, err := i.block.entry(i.offset)
283 - if err != nil {
284 - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
285 - return false
286 - }
287 - if n == 0 {
288 - i.dir = dirEOI
289 - return false
290 - }
291 - i.key = append(i.key[:nShared], key...)
292 - i.value = value
293 - i.prevOffset = i.offset
294 - i.offset += n
295 - i.dir = dirForward
296 - return true
297 -}
298 -
299 -func (i *blockIter) Prev() bool {
300 - if i.dir == dirSOI || i.err != nil {
301 - return false
302 - } else if i.dir == dirReleased {
303 - i.err = ErrIterReleased
304 - return false
305 - }
306 -
307 - var ri int
308 - if i.dir == dirForward {
309 - // Change direction.
310 - i.offset = i.prevOffset
311 - if i.offset == i.offsetRealStart {
312 - i.dir = dirSOI
313 - return false
314 - }
315 - ri = i.block.restartIndex(i.restartIndex, i.riLimit, i.offset)
316 - i.dir = dirBackward
317 - } else if i.dir == dirEOI {
318 - // At the end of iterator.
319 - i.restartIndex = i.riLimit
320 - i.offset = i.offsetLimit
321 - if i.offset == i.offsetRealStart {
322 - i.dir = dirSOI
323 - return false
324 - }
325 - ri = i.riLimit - 1
326 - i.dir = dirBackward
327 - } else if len(i.prevNode) == 1 {
328 - // This is the end of a restart range.
329 - i.offset = i.prevNode[0]
330 - i.prevNode = i.prevNode[:0]
331 - if i.restartIndex == i.riStart {
332 - i.dir = dirSOI
333 - return false
334 - }
335 - i.restartIndex--
336 - ri = i.restartIndex
337 - } else {
338 - // In the middle of restart range, get from cache.
339 - n := len(i.prevNode) - 3
340 - node := i.prevNode[n:]
341 - i.prevNode = i.prevNode[:n]
342 - // Get the key.
343 - ko := node[0]
344 - i.key = append(i.key[:0], i.prevKeys[ko:]...)
345 - i.prevKeys = i.prevKeys[:ko]
346 - // Get the value.
347 - vo := node[1]
348 - vl := vo + node[2]
349 - i.value = i.block.data[vo:vl]
350 - i.offset = vl
351 - return true
352 - }
353 - // Build entries cache.
354 - i.key = i.key[:0]
355 - i.value = nil
356 - offset := i.block.restartOffset(ri)
357 - if offset == i.offset {
358 - ri -= 1
359 - if ri < 0 {
360 - i.dir = dirSOI
361 - return false
362 - }
363 - offset = i.block.restartOffset(ri)
364 - }
365 - i.prevNode = append(i.prevNode, offset)
366 - for {
367 - key, value, nShared, n, err := i.block.entry(offset)
368 - if err != nil {
369 - i.sErr(i.tr.fixErrCorruptedBH(i.block.bh, err))
370 - return false
371 - }
372 - if offset >= i.offsetRealStart {
373 - if i.value != nil {
374 - // Appends 3 variables:
375 - // 1. Previous keys offset
376 - // 2. Value offset in the data block
377 - // 3. Value length
378 - i.prevNode = append(i.prevNode, len(i.prevKeys), offset-len(i.value), len(i.value))
379 - i.prevKeys = append(i.prevKeys, i.key...)
380 - }
381 - i.value = value
382 - }
383 - i.key = append(i.key[:nShared], key...)
384 - offset += n
385 - // Stop if target offset reached.
386 - if offset >= i.offset {
387 - if offset != i.offset {
388 - i.sErr(i.tr.newErrCorruptedBH(i.block.bh, "entries offset not aligned"))
389 - return false
390 - }
391 -
392 - break
393 - }
394 - }
395 - i.restartIndex = ri
396 - i.offset = offset
397 - return true
398 -}
399 -
400 -func (i *blockIter) Key() []byte {
401 - if i.err != nil || i.dir <= dirEOI {
402 - return nil
403 - }
404 - return i.key
405 -}
406 -
407 -func (i *blockIter) Value() []byte {
408 - if i.err != nil || i.dir <= dirEOI {
409 - return nil
410 - }
411 - return i.value
412 -}
413 -
414 -func (i *blockIter) Release() {
415 - if i.dir != dirReleased {
416 - i.tr = nil
417 - i.block = nil
418 - i.prevNode = nil
419 - i.prevKeys = nil
420 - i.key = nil
421 - i.value = nil
422 - i.dir = dirReleased
423 - if i.blockReleaser != nil {
424 - i.blockReleaser.Release()
425 - i.blockReleaser = nil
426 - }
427 - if i.releaser != nil {
428 - i.releaser.Release()
429 - i.releaser = nil
430 - }
431 - }
432 -}
433 -
434 -func (i *blockIter) SetReleaser(releaser util.Releaser) {
435 - if i.dir == dirReleased {
436 - panic(util.ErrReleased)
437 - }
438 - if i.releaser != nil && releaser != nil {
439 - panic(util.ErrHasReleaser)
440 - }
441 - i.releaser = releaser
442 -}
443 -
444 -func (i *blockIter) Valid() bool {
445 - return i.err == nil && (i.dir == dirBackward || i.dir == dirForward)
446 -}
447 -
448 -func (i *blockIter) Error() error {
449 - return i.err
450 -}
451 -
452 -type filterBlock struct {
453 - bpool *util.BufferPool
454 - data []byte
455 - oOffset int
456 - baseLg uint
457 - filtersNum int
458 -}
459 -
460 -func (b *filterBlock) contains(filter filter.Filter, offset uint64, key []byte) bool {
461 - i := int(offset >> b.baseLg)
462 - if i < b.filtersNum {
463 - o := b.data[b.oOffset+i*4:]
464 - n := int(binary.LittleEndian.Uint32(o))
465 - m := int(binary.LittleEndian.Uint32(o[4:]))
466 - if n < m && m <= b.oOffset {
467 - return filter.Contains(b.data[n:m], key)
468 - } else if n == m {
469 - return false
470 - }
471 - }
472 - return true
473 -}
474 -
475 -func (b *filterBlock) Release() {
476 - b.bpool.Put(b.data)
477 - b.bpool = nil
478 - b.data = nil
479 -}
480 -
481 -type indexIter struct {
482 - *blockIter
483 - tr *Reader
484 - slice *util.Range
485 - // Options
486 - fillCache bool
487 -}
488 -
489 -func (i *indexIter) Get() iterator.Iterator {
490 - value := i.Value()
491 - if value == nil {
492 - return nil
493 - }
494 - dataBH, n := decodeBlockHandle(value)
495 - if n == 0 {
496 - return iterator.NewEmptyIterator(i.tr.newErrCorruptedBH(i.tr.indexBH, "bad data block handle"))
497 - }
498 -
499 - var slice *util.Range
500 - if i.slice != nil && (i.blockIter.isFirst() || i.blockIter.isLast()) {
501 - slice = i.slice
502 - }
503 - return i.tr.getDataIterErr(dataBH, slice, i.tr.verifyChecksum, i.fillCache)
504 -}
505 -
506 -// Reader is a table reader.
507 -type Reader struct {
508 - mu sync.RWMutex
509 - fi *storage.FileInfo
510 - reader io.ReaderAt
511 - cache *cache.CacheGetter
512 - err error
513 - bpool *util.BufferPool
514 - // Options
515 - o *opt.Options
516 - cmp comparer.Comparer
517 - filter filter.Filter
518 - verifyChecksum bool
519 -
520 - dataEnd int64
521 - metaBH, indexBH, filterBH blockHandle
522 - indexBlock *block
523 - filterBlock *filterBlock
524 -}
525 -
526 -func (r *Reader) blockKind(bh blockHandle) string {
527 - switch bh.offset {
528 - case r.metaBH.offset:
529 - return "meta-block"
530 - case r.indexBH.offset:
531 - return "index-block"
532 - case r.filterBH.offset:
533 - if r.filterBH.length > 0 {
534 - return "filter-block"
535 - }
536 - }
537 - return "data-block"
538 -}
539 -
540 -func (r *Reader) newErrCorrupted(pos, size int64, kind, reason string) error {
541 - return &errors.ErrCorrupted{File: r.fi, Err: &ErrCorrupted{Pos: pos, Size: size, Kind: kind, Reason: reason}}
542 -}
543 -
544 -func (r *Reader) newErrCorruptedBH(bh blockHandle, reason string) error {
545 - return r.newErrCorrupted(int64(bh.offset), int64(bh.length), r.blockKind(bh), reason)
546 -}
547 -
548 -func (r *Reader) fixErrCorruptedBH(bh blockHandle, err error) error {
549 - if cerr, ok := err.(*ErrCorrupted); ok {
550 - cerr.Pos = int64(bh.offset)
551 - cerr.Size = int64(bh.length)
552 - cerr.Kind = r.blockKind(bh)
553 - return &errors.ErrCorrupted{File: r.fi, Err: cerr}
554 - }
555 - return err
556 -}
557 -
558 -func (r *Reader) readRawBlock(bh blockHandle, verifyChecksum bool) ([]byte, error) {
559 - data := r.bpool.Get(int(bh.length + blockTrailerLen))
560 - if _, err := r.reader.ReadAt(data, int64(bh.offset)); err != nil && err != io.EOF {
561 - return nil, err
562 - }
563 -
564 - if verifyChecksum {
565 - n := bh.length + 1
566 - checksum0 := binary.LittleEndian.Uint32(data[n:])
567 - checksum1 := util.NewCRC(data[:n]).Value()
568 - if checksum0 != checksum1 {
569 - r.bpool.Put(data)
570 - return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("checksum mismatch, want=%#x got=%#x", checksum0, checksum1))
571 - }
572 - }
573 -
574 - switch data[bh.length] {
575 - case blockTypeNoCompression:
576 - data = data[:bh.length]
577 - case blockTypeSnappyCompression:
578 - decLen, err := snappy.DecodedLen(data[:bh.length])
579 - if err != nil {
580 - return nil, r.newErrCorruptedBH(bh, err.Error())
581 - }
582 - decData := r.bpool.Get(decLen)
583 - decData, err = snappy.Decode(decData, data[:bh.length])
584 - r.bpool.Put(data)
585 - if err != nil {
586 - r.bpool.Put(decData)
587 - return nil, r.newErrCorruptedBH(bh, err.Error())
588 - }
589 - data = decData
590 - default:
591 - r.bpool.Put(data)
592 - return nil, r.newErrCorruptedBH(bh, fmt.Sprintf("unknown compression type %#x", data[bh.length]))
593 - }
594 - return data, nil
595 -}
596 -
597 -func (r *Reader) readBlock(bh blockHandle, verifyChecksum bool) (*block, error) {
598 - data, err := r.readRawBlock(bh, verifyChecksum)
599 - if err != nil {
600 - return nil, err
601 - }
602 - restartsLen := int(binary.LittleEndian.Uint32(data[len(data)-4:]))
603 - b := &block{
604 - bpool: r.bpool,
605 - bh: bh,
606 - data: data,
607 - restartsLen: restartsLen,
608 - restartsOffset: len(data) - (restartsLen+1)*4,
609 - }
610 - return b, nil
611 -}
612 -
613 -func (r *Reader) readBlockCached(bh blockHandle, verifyChecksum, fillCache bool) (*block, util.Releaser, error) {
614 - if r.cache != nil {
615 - var (
616 - err error
617 - ch *cache.Handle
618 - )
619 - if fillCache {
620 - ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) {
621 - var b *block
622 - b, err = r.readBlock(bh, verifyChecksum)
623 - if err != nil {
624 - return 0, nil
625 - }
626 - return cap(b.data), b
627 - })
628 - } else {
629 - ch = r.cache.Get(bh.offset, nil)
630 - }
631 - if ch != nil {
632 - b, ok := ch.Value().(*block)
633 - if !ok {
634 - ch.Release()
635 - return nil, nil, errors.New("leveldb/table: inconsistent block type")
636 - }
637 - return b, ch, err
638 - } else if err != nil {
639 - return nil, nil, err
640 - }
641 - }
642 -
643 - b, err := r.readBlock(bh, verifyChecksum)
644 - return b, b, err
645 -}
646 -
647 -func (r *Reader) readFilterBlock(bh blockHandle) (*filterBlock, error) {
648 - data, err := r.readRawBlock(bh, true)
649 - if err != nil {
650 - return nil, err
651 - }
652 - n := len(data)
653 - if n < 5 {
654 - return nil, r.newErrCorruptedBH(bh, "too short")
655 - }
656 - m := n - 5
657 - oOffset := int(binary.LittleEndian.Uint32(data[m:]))
658 - if oOffset > m {
659 - return nil, r.newErrCorruptedBH(bh, "invalid data-offsets offset")
660 - }
661 - b := &filterBlock{
662 - bpool: r.bpool,
663 - data: data,
664 - oOffset: oOffset,
665 - baseLg: uint(data[n-1]),
666 - filtersNum: (m - oOffset) / 4,
667 - }
668 - return b, nil
669 -}
670 -
671 -func (r *Reader) readFilterBlockCached(bh blockHandle, fillCache bool) (*filterBlock, util.Releaser, error) {
672 - if r.cache != nil {
673 - var (
674 - err error
675 - ch *cache.Handle
676 - )
677 - if fillCache {
678 - ch = r.cache.Get(bh.offset, func() (size int, value cache.Value) {
679 - var b *filterBlock
680 - b, err = r.readFilterBlock(bh)
681 - if err != nil {
682 - return 0, nil
683 - }
684 - return cap(b.data), b
685 - })
686 - } else {
687 - ch = r.cache.Get(bh.offset, nil)
688 - }
689 - if ch != nil {
690 - b, ok := ch.Value().(*filterBlock)
691 - if !ok {
692 - ch.Release()
693 - return nil, nil, errors.New("leveldb/table: inconsistent block type")
694 - }
695 - return b, ch, err
696 - } else if err != nil {
697 - return nil, nil, err
698 - }
699 - }
700 -
701 - b, err := r.readFilterBlock(bh)
702 - return b, b, err
703 -}
704 -
705 -func (r *Reader) getIndexBlock(fillCache bool) (b *block, rel util.Releaser, err error) {
706 - if r.indexBlock == nil {
707 - return r.readBlockCached(r.indexBH, true, fillCache)
708 - }
709 - return r.indexBlock, util.NoopReleaser{}, nil
710 -}
711 -
712 -func (r *Reader) getFilterBlock(fillCache bool) (*filterBlock, util.Releaser, error) {
713 - if r.filterBlock == nil {
714 - return r.readFilterBlockCached(r.filterBH, fillCache)
715 - }
716 - return r.filterBlock, util.NoopReleaser{}, nil
717 -}
718 -
719 -func (r *Reader) newBlockIter(b *block, bReleaser util.Releaser, slice *util.Range, inclLimit bool) *blockIter {
720 - bi := &blockIter{
721 - tr: r,
722 - block: b,
723 - blockReleaser: bReleaser,
724 - // Valid key should never be nil.
725 - key: make([]byte, 0),
726 - dir: dirSOI,
727 - riStart: 0,
728 - riLimit: b.restartsLen,
729 - offsetStart: 0,
730 - offsetRealStart: 0,
731 - offsetLimit: b.restartsOffset,
732 - }
733 - if slice != nil {
734 - if slice.Start != nil {
735 - if bi.Seek(slice.Start) {
736 - bi.riStart = b.restartIndex(bi.restartIndex, b.restartsLen, bi.prevOffset)
737 - bi.offsetStart = b.restartOffset(bi.riStart)
738 - bi.offsetRealStart = bi.prevOffset
739 - } else {
740 - bi.riStart = b.restartsLen
741 - bi.offsetStart = b.restartsOffset
742 - bi.offsetRealStart = b.restartsOffset
743 - }
744 - }
745 - if slice.Limit != nil {
746 - if bi.Seek(slice.Limit) && (!inclLimit || bi.Next()) {
747 - bi.offsetLimit = bi.prevOffset
748 - bi.riLimit = bi.restartIndex + 1
749 - }
750 - }
751 - bi.reset()
752 - if bi.offsetStart > bi.offsetLimit {
753 - bi.sErr(errors.New("leveldb/table: invalid slice range"))
754 - }
755 - }
756 - return bi
757 -}
758 -
759 -func (r *Reader) getDataIter(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator {
760 - b, rel, err := r.readBlockCached(dataBH, verifyChecksum, fillCache)
761 - if err != nil {
762 - return iterator.NewEmptyIterator(err)
763 - }
764 - return r.newBlockIter(b, rel, slice, false)
765 -}
766 -
767 -func (r *Reader) getDataIterErr(dataBH blockHandle, slice *util.Range, verifyChecksum, fillCache bool) iterator.Iterator {
768 - r.mu.RLock()
769 - defer r.mu.RUnlock()
770 -
771 - if r.err != nil {
772 - return iterator.NewEmptyIterator(r.err)
773 - }
774 -
775 - return r.getDataIter(dataBH, slice, verifyChecksum, fillCache)
776 -}
777 -
778 -// NewIterator creates an iterator from the table.
779 -//
780 -// Slice allows slicing the iterator to only contains keys in the given
781 -// range. A nil Range.Start is treated as a key before all keys in the
782 -// table. And a nil Range.Limit is treated as a key after all keys in
783 -// the table.
784 -//
785 -// The returned iterator is not goroutine-safe and should be released
786 -// when not used.
787 -//
788 -// Also read Iterator documentation of the leveldb/iterator package.
789 -func (r *Reader) NewIterator(slice *util.Range, ro *opt.ReadOptions) iterator.Iterator {
790 - r.mu.RLock()
791 - defer r.mu.RUnlock()
792 -
793 - if r.err != nil {
794 - return iterator.NewEmptyIterator(r.err)
795 - }
796 -
797 - fillCache := !ro.GetDontFillCache()
798 - indexBlock, rel, err := r.getIndexBlock(fillCache)
799 - if err != nil {
800 - return iterator.NewEmptyIterator(err)
801 - }
802 - index := &indexIter{
803 - blockIter: r.newBlockIter(indexBlock, rel, slice, true),
804 - tr: r,
805 - slice: slice,
806 - fillCache: !ro.GetDontFillCache(),
807 - }
808 - return iterator.NewIndexedIterator(index, opt.GetStrict(r.o, ro, opt.StrictReader))
809 -}
810 -
811 -func (r *Reader) find(key []byte, filtered bool, ro *opt.ReadOptions, noValue bool) (rkey, value []byte, err error) {
812 - r.mu.RLock()
813 - defer r.mu.RUnlock()
814 -
815 - if r.err != nil {
816 - err = r.err
817 - return
818 - }
819 -
820 - indexBlock, rel, err := r.getIndexBlock(true)
821 - if err != nil {
822 - return
823 - }
824 - defer rel.Release()
825 -
826 - index := r.newBlockIter(indexBlock, nil, nil, true)
827 - defer index.Release()
828 - if !index.Seek(key) {
829 - err = index.Error()
830 - if err == nil {
831 - err = ErrNotFound
832 - }
833 - return
834 - }
835 - dataBH, n := decodeBlockHandle(index.Value())
836 - if n == 0 {
837 - r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle")
838 - return
839 - }
840 - if filtered && r.filter != nil {
841 - filterBlock, frel, ferr := r.getFilterBlock(true)
842 - if ferr == nil {
843 - if !filterBlock.contains(r.filter, dataBH.offset, key) {
844 - frel.Release()
845 - return nil, nil, ErrNotFound
846 - }
847 - frel.Release()
848 - } else if !errors.IsCorrupted(ferr) {
849 - err = ferr
850 - return
851 - }
852 - }
853 - data := r.getDataIter(dataBH, nil, r.verifyChecksum, !ro.GetDontFillCache())
854 - defer data.Release()
855 - if !data.Seek(key) {
856 - err = data.Error()
857 - if err == nil {
858 - err = ErrNotFound
859 - }
860 - return
861 - }
862 - // Don't use block buffer, no need to copy the buffer.
863 - rkey = data.Key()
864 - if !noValue {
865 - if r.bpool == nil {
866 - value = data.Value()
867 - } else {
868 - // Use block buffer, and since the buffer will be recycled, the buffer
869 - // need to be copied.
870 - value = append([]byte{}, data.Value()...)
871 - }
872 - }
873 - return
874 -}
875 -
876 -// Find finds key/value pair whose key is greater than or equal to the
877 -// given key. It returns ErrNotFound if the table doesn't contain
878 -// such pair.
879 -// If filtered is true then the nearest 'block' will be checked against
880 -// 'filter data' (if present) and will immediately return ErrNotFound if
881 -// 'filter data' indicates that such pair doesn't exist.
882 -//
883 -// The caller may modify the contents of the returned slice as it is its
884 -// own copy.
885 -// It is safe to modify the contents of the argument after Find returns.
886 -func (r *Reader) Find(key []byte, filtered bool, ro *opt.ReadOptions) (rkey, value []byte, err error) {
887 - return r.find(key, filtered, ro, false)
888 -}
889 -
890 -// Find finds key that is greater than or equal to the given key.
891 -// It returns ErrNotFound if the table doesn't contain such key.
892 -// If filtered is true then the nearest 'block' will be checked against
893 -// 'filter data' (if present) and will immediately return ErrNotFound if
894 -// 'filter data' indicates that such key doesn't exist.
895 -//
896 -// The caller may modify the contents of the returned slice as it is its
897 -// own copy.
898 -// It is safe to modify the contents of the argument after Find returns.
899 -func (r *Reader) FindKey(key []byte, filtered bool, ro *opt.ReadOptions) (rkey []byte, err error) {
900 - rkey, _, err = r.find(key, filtered, ro, true)
901 - return
902 -}
903 -
904 -// Get gets the value for the given key. It returns errors.ErrNotFound
905 -// if the table does not contain the key.
906 -//
907 -// The caller may modify the contents of the returned slice as it is its
908 -// own copy.
909 -// It is safe to modify the contents of the argument after Find returns.
910 -func (r *Reader) Get(key []byte, ro *opt.ReadOptions) (value []byte, err error) {
911 - r.mu.RLock()
912 - defer r.mu.RUnlock()
913 -
914 - if r.err != nil {
915 - err = r.err
916 - return
917 - }
918 -
919 - rkey, value, err := r.find(key, false, ro, false)
920 - if err == nil && r.cmp.Compare(rkey, key) != 0 {
921 - value = nil
922 - err = ErrNotFound
923 - }
924 - return
925 -}
926 -
927 -// OffsetOf returns approximate offset for the given key.
928 -//
929 -// It is safe to modify the contents of the argument after Get returns.
930 -func (r *Reader) OffsetOf(key []byte) (offset int64, err error) {
931 - r.mu.RLock()
932 - defer r.mu.RUnlock()
933 -
934 - if r.err != nil {
935 - err = r.err
936 - return
937 - }
938 -
939 - indexBlock, rel, err := r.readBlockCached(r.indexBH, true, true)
940 - if err != nil {
941 - return
942 - }
943 - defer rel.Release()
944 -
945 - index := r.newBlockIter(indexBlock, nil, nil, true)
946 - defer index.Release()
947 - if index.Seek(key) {
948 - dataBH, n := decodeBlockHandle(index.Value())
949 - if n == 0 {
950 - r.err = r.newErrCorruptedBH(r.indexBH, "bad data block handle")
951 - return
952 - }
953 - offset = int64(dataBH.offset)
954 - return
955 - }
956 - err = index.Error()
957 - if err == nil {
958 - offset = r.dataEnd
959 - }
960 - return
961 -}
962 -
963 -// Release implements util.Releaser.
964 -// It also close the file if it is an io.Closer.
965 -func (r *Reader) Release() {
966 - r.mu.Lock()
967 - defer r.mu.Unlock()
968 -
969 - if closer, ok := r.reader.(io.Closer); ok {
970 - closer.Close()
971 - }
972 - if r.indexBlock != nil {
973 - r.indexBlock.Release()
974 - r.indexBlock = nil
975 - }
976 - if r.filterBlock != nil {
977 - r.filterBlock.Release()
978 - r.filterBlock = nil
979 - }
980 - r.reader = nil
981 - r.cache = nil
982 - r.bpool = nil
983 - r.err = ErrReaderReleased
984 -}
985 -
986 -// NewReader creates a new initialized table reader for the file.
987 -// The fi, cache and bpool is optional and can be nil.
988 -//
989 -// The returned table reader instance is goroutine-safe.
990 -func NewReader(f io.ReaderAt, size int64, fi *storage.FileInfo, cache *cache.CacheGetter, bpool *util.BufferPool, o *opt.Options) (*Reader, error) {
991 - if f == nil {
992 - return nil, errors.New("leveldb/table: nil file")
993 - }
994 -
995 - r := &Reader{
996 - fi: fi,
997 - reader: f,
998 - cache: cache,
999 - bpool: bpool,
1000 - o: o,
1001 - cmp: o.GetComparer(),
1002 - verifyChecksum: o.GetStrict(opt.StrictBlockChecksum),
1003 - }
1004 -
1005 - if size < footerLen {
1006 - r.err = r.newErrCorrupted(0, size, "table", "too small")
1007 - return r, nil
1008 - }
1009 -
1010 - footerPos := size - footerLen
1011 - var footer [footerLen]byte
1012 - if _, err := r.reader.ReadAt(footer[:], footerPos); err != nil && err != io.EOF {
1013 - return nil, err
1014 - }
1015 - if string(footer[footerLen-len(magic):footerLen]) != magic {
1016 - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad magic number")
1017 - return r, nil
1018 - }
1019 -
1020 - var n int
1021 - // Decode the metaindex block handle.
1022 - r.metaBH, n = decodeBlockHandle(footer[:])
1023 - if n == 0 {
1024 - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad metaindex block handle")
1025 - return r, nil
1026 - }
1027 -
1028 - // Decode the index block handle.
1029 - r.indexBH, n = decodeBlockHandle(footer[n:])
1030 - if n == 0 {
1031 - r.err = r.newErrCorrupted(footerPos, footerLen, "table-footer", "bad index block handle")
1032 - return r, nil
1033 - }
1034 -
1035 - // Read metaindex block.
1036 - metaBlock, err := r.readBlock(r.metaBH, true)
1037 - if err != nil {
1038 - if errors.IsCorrupted(err) {
1039 - r.err = err
1040 - return r, nil
1041 - } else {
1042 - return nil, err
1043 - }
1044 - }
1045 -
1046 - // Set data end.
1047 - r.dataEnd = int64(r.metaBH.offset)
1048 -
1049 - // Read metaindex.
1050 - metaIter := r.newBlockIter(metaBlock, nil, nil, true)
1051 - for metaIter.Next() {
1052 - key := string(metaIter.Key())
1053 - if !strings.HasPrefix(key, "filter.") {
1054 - continue
1055 - }
1056 - fn := key[7:]
1057 - if f0 := o.GetFilter(); f0 != nil && f0.Name() == fn {
1058 - r.filter = f0
1059 - } else {
1060 - for _, f0 := range o.GetAltFilters() {
1061 - if f0.Name() == fn {
1062 - r.filter = f0
1063 - break
1064 - }
1065 - }
1066 - }
1067 - if r.filter != nil {
1068 - filterBH, n := decodeBlockHandle(metaIter.Value())
1069 - if n == 0 {
1070 - continue
1071 - }
1072 - r.filterBH = filterBH
1073 - // Update data end.
1074 - r.dataEnd = int64(filterBH.offset)
1075 - break
1076 - }
1077 - }
1078 - metaIter.Release()
1079 - metaBlock.Release()
1080 -
1081 - // Cache index and filter block locally, since we don't have global cache.
1082 - if cache == nil {
1083 - r.indexBlock, err = r.readBlock(r.indexBH, true)
1084 - if err != nil {
1085 - if errors.IsCorrupted(err) {
1086 - r.err = err
1087 - return r, nil
1088 - } else {
1089 - return nil, err
1090 - }
1091 - }
1092 - if r.filter != nil {
1093 - r.filterBlock, err = r.readFilterBlock(r.filterBH)
1094 - if err != nil {
1095 - if !errors.IsCorrupted(err) {
1096 - return nil, err
1097 - }
1098 -
1099 - // Don't use filter then.
1100 - r.filter = nil
1101 - }
1102 - }
1103 - }
1104 -
1105 - return r, nil
1106 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table.go deleted
-177
@@ -1,177 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package table allows read and write sorted key/value.
8 -package table
9 -
10 -import (
11 - "encoding/binary"
12 -)
13 -
14 -/*
15 -Table:
16 -
17 -Table is consist of one or more data blocks, an optional filter block
18 -a metaindex block, an index block and a table footer. Metaindex block
19 -is a special block used to keep parameters of the table, such as filter
20 -block name and its block handle. Index block is a special block used to
21 -keep record of data blocks offset and length, index block use one as
22 -restart interval. The key used by index block are the last key of preceding
23 -block, shorter separator of adjacent blocks or shorter successor of the
24 -last key of the last block. Filter block is an optional block contains
25 -sequence of filter data generated by a filter generator.
26 -
27 -Table data structure:
28 - + optional
29 - /
30 - +--------------+--------------+--------------+------+-------+-----------------+-------------+--------+
31 - | data block 1 | ... | data block n | filter block | metaindex block | index block | footer |
32 - +--------------+--------------+--------------+--------------+-----------------+-------------+--------+
33 -
34 - Each block followed by a 5-bytes trailer contains compression type and checksum.
35 -
36 -Table block trailer:
37 -
38 - +---------------------------+-------------------+
39 - | compression type (1-byte) | checksum (4-byte) |
40 - +---------------------------+-------------------+
41 -
42 - The checksum is a CRC-32 computed using Castagnoli's polynomial. Compression
43 - type also included in the checksum.
44 -
45 -Table footer:
46 -
47 - +------------------- 40-bytes -------------------+
48 - / \
49 - +------------------------+--------------------+------+-----------------+
50 - | metaindex block handle / index block handle / ---- | magic (8-bytes) |
51 - +------------------------+--------------------+------+-----------------+
52 -
53 - The magic are first 64-bit of SHA-1 sum of "http://code.google.com/p/leveldb/".
54 -
55 -NOTE: All fixed-length integer are little-endian.
56 -*/
57 -
58 -/*
59 -Block:
60 -
61 -Block is consist of one or more key/value entries and a block trailer.
62 -Block entry shares key prefix with its preceding key until a restart
63 -point reached. A block should contains at least one restart point.
64 -First restart point are always zero.
65 -
66 -Block data structure:
67 -
68 - + restart point + restart point (depends on restart interval)
69 - / /
70 - +---------------+---------------+---------------+---------------+---------+
71 - | block entry 1 | block entry 2 | ... | block entry n | trailer |
72 - +---------------+---------------+---------------+---------------+---------+
73 -
74 -Key/value entry:
75 -
76 - +---- key len ----+
77 - / \
78 - +-------+---------+-----------+---------+--------------------+--------------+----------------+
79 - | shared (varint) | not shared (varint) | value len (varint) | key (varlen) | value (varlen) |
80 - +-----------------+---------------------+--------------------+--------------+----------------+
81 -
82 - Block entry shares key prefix with its preceding key:
83 - Conditions:
84 - restart_interval=2
85 - entry one : key=deck,value=v1
86 - entry two : key=dock,value=v2
87 - entry three: key=duck,value=v3
88 - The entries will be encoded as follow:
89 -
90 - + restart point (offset=0) + restart point (offset=16)
91 - / /
92 - +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
93 - | 0 | 4 | 2 | "deck" | "v1" | 1 | 3 | 2 | "ock" | "v2" | 0 | 4 | 2 | "duck" | "v3" |
94 - +-----+-----+-----+----------+--------+-----+-----+-----+---------+--------+-----+-----+-----+----------+--------+
95 - \ / \ / \ /
96 - +----------- entry one -----------+ +----------- entry two ----------+ +---------- entry three ----------+
97 -
98 - The block trailer will contains two restart points:
99 -
100 - +------------+-----------+--------+
101 - | 0 | 16 | 2 |
102 - +------------+-----------+---+----+
103 - \ / \
104 - +-- restart points --+ + restart points length
105 -
106 -Block trailer:
107 -
108 - +-- 4-bytes --+
109 - / \
110 - +-----------------+-----------------+-----------------+------------------------------+
111 - | restart point 1 | .... | restart point n | restart points len (4-bytes) |
112 - +-----------------+-----------------+-----------------+------------------------------+
113 -
114 -
115 -NOTE: All fixed-length integer are little-endian.
116 -*/
117 -
118 -/*
119 -Filter block:
120 -
121 -Filter block consist of one or more filter data and a filter block trailer.
122 -The trailer contains filter data offsets, a trailer offset and a 1-byte base Lg.
123 -
124 -Filter block data structure:
125 -
126 - + offset 1 + offset 2 + offset n + trailer offset
127 - / / / /
128 - +---------------+---------------+---------------+---------+
129 - | filter data 1 | ... | filter data n | trailer |
130 - +---------------+---------------+---------------+---------+
131 -
132 -Filter block trailer:
133 -
134 - +- 4-bytes -+
135 - / \
136 - +---------------+---------------+---------------+-------------------------------+------------------+
137 - | data 1 offset | .... | data n offset | data-offsets offset (4-bytes) | base Lg (1-byte) |
138 - +-------------- +---------------+---------------+-------------------------------+------------------+
139 -
140 -
141 -NOTE: All fixed-length integer are little-endian.
142 -*/
143 -
144 -const (
145 - blockTrailerLen = 5
146 - footerLen = 48
147 -
148 - magic = "\x57\xfb\x80\x8b\x24\x75\x47\xdb"
149 -
150 - // The block type gives the per-block compression format.
151 - // These constants are part of the file format and should not be changed.
152 - blockTypeNoCompression = 0
153 - blockTypeSnappyCompression = 1
154 -
155 - // Generate new filter every 2KB of data
156 - filterBaseLg = 11
157 - filterBase = 1 << filterBaseLg
158 -)
159 -
160 -type blockHandle struct {
161 - offset, length uint64
162 -}
163 -
164 -func decodeBlockHandle(src []byte) (blockHandle, int) {
165 - offset, n := binary.Uvarint(src)
166 - length, m := binary.Uvarint(src[n:])
167 - if n == 0 || m == 0 {
168 - return blockHandle{}, 0
169 - }
170 - return blockHandle{offset, length}, n + m
171 -}
172 -
173 -func encodeBlockHandle(dst []byte, b blockHandle) int {
174 - n := binary.PutUvarint(dst, b.offset)
175 - m := binary.PutUvarint(dst[n:], b.length)
176 - return n + m
177 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_suite_test.go deleted
-11
@@ -1,11 +0,0 @@
1 -package table
2 -
3 -import (
4 - "testing"
5 -
6 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
7 -)
8 -
9 -func TestTable(t *testing.T) {
10 - testutil.RunSuite(t, "Table Suite")
11 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/table_test.go deleted
-122
@@ -1,122 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package table
8 -
9 -import (
10 - "bytes"
11 -
12 - . "github.com/onsi/ginkgo"
13 - . "github.com/onsi/gomega"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
19 -)
20 -
21 -type tableWrapper struct {
22 - *Reader
23 -}
24 -
25 -func (t tableWrapper) TestFind(key []byte) (rkey, rvalue []byte, err error) {
26 - return t.Reader.Find(key, false, nil)
27 -}
28 -
29 -func (t tableWrapper) TestGet(key []byte) (value []byte, err error) {
30 - return t.Reader.Get(key, nil)
31 -}
32 -
33 -func (t tableWrapper) TestNewIterator(slice *util.Range) iterator.Iterator {
34 - return t.Reader.NewIterator(slice, nil)
35 -}
36 -
37 -var _ = testutil.Defer(func() {
38 - Describe("Table", func() {
39 - Describe("approximate offset test", func() {
40 - var (
41 - buf = &bytes.Buffer{}
42 - o = &opt.Options{
43 - BlockSize: 1024,
44 - Compression: opt.NoCompression,
45 - }
46 - )
47 -
48 - // Building the table.
49 - tw := NewWriter(buf, o)
50 - tw.Append([]byte("k01"), []byte("hello"))
51 - tw.Append([]byte("k02"), []byte("hello2"))
52 - tw.Append([]byte("k03"), bytes.Repeat([]byte{'x'}, 10000))
53 - tw.Append([]byte("k04"), bytes.Repeat([]byte{'x'}, 200000))
54 - tw.Append([]byte("k05"), bytes.Repeat([]byte{'x'}, 300000))
55 - tw.Append([]byte("k06"), []byte("hello3"))
56 - tw.Append([]byte("k07"), bytes.Repeat([]byte{'x'}, 100000))
57 - err := tw.Close()
58 -
59 - It("Should be able to approximate offset of a key correctly", func() {
60 - Expect(err).ShouldNot(HaveOccurred())
61 -
62 - tr, err := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
63 - Expect(err).ShouldNot(HaveOccurred())
64 - CheckOffset := func(key string, expect, threshold int) {
65 - offset, err := tr.OffsetOf([]byte(key))
66 - Expect(err).ShouldNot(HaveOccurred())
67 - Expect(offset).Should(BeNumerically("~", expect, threshold), "Offset of key %q", key)
68 - }
69 -
70 - CheckOffset("k0", 0, 0)
71 - CheckOffset("k01a", 0, 0)
72 - CheckOffset("k02", 0, 0)
73 - CheckOffset("k03", 0, 0)
74 - CheckOffset("k04", 10000, 1000)
75 - CheckOffset("k04a", 210000, 1000)
76 - CheckOffset("k05", 210000, 1000)
77 - CheckOffset("k06", 510000, 1000)
78 - CheckOffset("k07", 510000, 1000)
79 - CheckOffset("xyz", 610000, 2000)
80 - })
81 - })
82 -
83 - Describe("read test", func() {
84 - Build := func(kv testutil.KeyValue) testutil.DB {
85 - o := &opt.Options{
86 - BlockSize: 512,
87 - BlockRestartInterval: 3,
88 - }
89 - buf := &bytes.Buffer{}
90 -
91 - // Building the table.
92 - tw := NewWriter(buf, o)
93 - kv.Iterate(func(i int, key, value []byte) {
94 - tw.Append(key, value)
95 - })
96 - tw.Close()
97 -
98 - // Opening the table.
99 - tr, _ := NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()), nil, nil, nil, o)
100 - return tableWrapper{tr}
101 - }
102 - Test := func(kv *testutil.KeyValue, body func(r *Reader)) func() {
103 - return func() {
104 - db := Build(*kv)
105 - if body != nil {
106 - body(db.(tableWrapper).Reader)
107 - }
108 - testutil.KeyValueTesting(nil, *kv, db, nil, nil)
109 - }
110 - }
111 -
112 - testutil.AllKeyValueTesting(nil, Build, nil, nil)
113 - Describe("with one key per block", Test(testutil.KeyValue_Generate(nil, 9, 1, 10, 512, 512), func(r *Reader) {
114 - It("should have correct blocks number", func() {
115 - indexBlock, err := r.readBlock(r.indexBH, true)
116 - Expect(err).To(BeNil())
117 - Expect(indexBlock.restartsLen).Should(Equal(9))
118 - })
119 - }))
120 - })
121 - })
122 -})
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/table/writer.go deleted
-378
@@ -1,378 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package table
8 -
9 -import (
10 - "encoding/binary"
11 - "errors"
12 - "fmt"
13 - "io"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/filter"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
19 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/gosnappy/snappy"
20 -)
21 -
22 -func sharedPrefixLen(a, b []byte) int {
23 - i, n := 0, len(a)
24 - if n > len(b) {
25 - n = len(b)
26 - }
27 - for i < n && a[i] == b[i] {
28 - i++
29 - }
30 - return i
31 -}
32 -
33 -type blockWriter struct {
34 - restartInterval int
35 - buf util.Buffer
36 - nEntries int
37 - prevKey []byte
38 - restarts []uint32
39 - scratch []byte
40 -}
41 -
42 -func (w *blockWriter) append(key, value []byte) {
43 - nShared := 0
44 - if w.nEntries%w.restartInterval == 0 {
45 - w.restarts = append(w.restarts, uint32(w.buf.Len()))
46 - } else {
47 - nShared = sharedPrefixLen(w.prevKey, key)
48 - }
49 - n := binary.PutUvarint(w.scratch[0:], uint64(nShared))
50 - n += binary.PutUvarint(w.scratch[n:], uint64(len(key)-nShared))
51 - n += binary.PutUvarint(w.scratch[n:], uint64(len(value)))
52 - w.buf.Write(w.scratch[:n])
53 - w.buf.Write(key[nShared:])
54 - w.buf.Write(value)
55 - w.prevKey = append(w.prevKey[:0], key...)
56 - w.nEntries++
57 -}
58 -
59 -func (w *blockWriter) finish() {
60 - // Write restarts entry.
61 - if w.nEntries == 0 {
62 - // Must have at least one restart entry.
63 - w.restarts = append(w.restarts, 0)
64 - }
65 - w.restarts = append(w.restarts, uint32(len(w.restarts)))
66 - for _, x := range w.restarts {
67 - buf4 := w.buf.Alloc(4)
68 - binary.LittleEndian.PutUint32(buf4, x)
69 - }
70 -}
71 -
72 -func (w *blockWriter) reset() {
73 - w.buf.Reset()
74 - w.nEntries = 0
75 - w.restarts = w.restarts[:0]
76 -}
77 -
78 -func (w *blockWriter) bytesLen() int {
79 - restartsLen := len(w.restarts)
80 - if restartsLen == 0 {
81 - restartsLen = 1
82 - }
83 - return w.buf.Len() + 4*restartsLen + 4
84 -}
85 -
86 -type filterWriter struct {
87 - generator filter.FilterGenerator
88 - buf util.Buffer
89 - nKeys int
90 - offsets []uint32
91 -}
92 -
93 -func (w *filterWriter) add(key []byte) {
94 - if w.generator == nil {
95 - return
96 - }
97 - w.generator.Add(key)
98 - w.nKeys++
99 -}
100 -
101 -func (w *filterWriter) flush(offset uint64) {
102 - if w.generator == nil {
103 - return
104 - }
105 - for x := int(offset / filterBase); x > len(w.offsets); {
106 - w.generate()
107 - }
108 -}
109 -
110 -func (w *filterWriter) finish() {
111 - if w.generator == nil {
112 - return
113 - }
114 - // Generate last keys.
115 -
116 - if w.nKeys > 0 {
117 - w.generate()
118 - }
119 - w.offsets = append(w.offsets, uint32(w.buf.Len()))
120 - for _, x := range w.offsets {
121 - buf4 := w.buf.Alloc(4)
122 - binary.LittleEndian.PutUint32(buf4, x)
123 - }
124 - w.buf.WriteByte(filterBaseLg)
125 -}
126 -
127 -func (w *filterWriter) generate() {
128 - // Record offset.
129 - w.offsets = append(w.offsets, uint32(w.buf.Len()))
130 - // Generate filters.
131 - if w.nKeys > 0 {
132 - w.generator.Generate(&w.buf)
133 - w.nKeys = 0
134 - }
135 -}
136 -
137 -// Writer is a table writer.
138 -type Writer struct {
139 - writer io.Writer
140 - err error
141 - // Options
142 - cmp comparer.Comparer
143 - filter filter.Filter
144 - compression opt.Compression
145 - blockSize int
146 -
147 - dataBlock blockWriter
148 - indexBlock blockWriter
149 - filterBlock filterWriter
150 - pendingBH blockHandle
151 - offset uint64
152 - nEntries int
153 - // Scratch allocated enough for 5 uvarint. Block writer should not use
154 - // first 20-bytes since it will be used to encode block handle, which
155 - // then passed to the block writer itself.
156 - scratch [50]byte
157 - comparerScratch []byte
158 - compressionScratch []byte
159 -}
160 -
161 -func (w *Writer) writeBlock(buf *util.Buffer, compression opt.Compression) (bh blockHandle, err error) {
162 - // Compress the buffer if necessary.
163 - var b []byte
164 - if compression == opt.SnappyCompression {
165 - // Allocate scratch enough for compression and block trailer.
166 - if n := snappy.MaxEncodedLen(buf.Len()) + blockTrailerLen; len(w.compressionScratch) < n {
167 - w.compressionScratch = make([]byte, n)
168 - }
169 - var compressed []byte
170 - compressed, err = snappy.Encode(w.compressionScratch, buf.Bytes())
171 - if err != nil {
172 - return
173 - }
174 - n := len(compressed)
175 - b = compressed[:n+blockTrailerLen]
176 - b[n] = blockTypeSnappyCompression
177 - } else {
178 - tmp := buf.Alloc(blockTrailerLen)
179 - tmp[0] = blockTypeNoCompression
180 - b = buf.Bytes()
181 - }
182 -
183 - // Calculate the checksum.
184 - n := len(b) - 4
185 - checksum := util.NewCRC(b[:n]).Value()
186 - binary.LittleEndian.PutUint32(b[n:], checksum)
187 -
188 - // Write the buffer to the file.
189 - _, err = w.writer.Write(b)
190 - if err != nil {
191 - return
192 - }
193 - bh = blockHandle{w.offset, uint64(len(b) - blockTrailerLen)}
194 - w.offset += uint64(len(b))
195 - return
196 -}
197 -
198 -func (w *Writer) flushPendingBH(key []byte) {
199 - if w.pendingBH.length == 0 {
200 - return
201 - }
202 - var separator []byte
203 - if len(key) == 0 {
204 - separator = w.cmp.Successor(w.comparerScratch[:0], w.dataBlock.prevKey)
205 - } else {
206 - separator = w.cmp.Separator(w.comparerScratch[:0], w.dataBlock.prevKey, key)
207 - }
208 - if separator == nil {
209 - separator = w.dataBlock.prevKey
210 - } else {
211 - w.comparerScratch = separator
212 - }
213 - n := encodeBlockHandle(w.scratch[:20], w.pendingBH)
214 - // Append the block handle to the index block.
215 - w.indexBlock.append(separator, w.scratch[:n])
216 - // Reset prev key of the data block.
217 - w.dataBlock.prevKey = w.dataBlock.prevKey[:0]
218 - // Clear pending block handle.
219 - w.pendingBH = blockHandle{}
220 -}
221 -
222 -func (w *Writer) finishBlock() error {
223 - w.dataBlock.finish()
224 - bh, err := w.writeBlock(&w.dataBlock.buf, w.compression)
225 - if err != nil {
226 - return err
227 - }
228 - w.pendingBH = bh
229 - // Reset the data block.
230 - w.dataBlock.reset()
231 - // Flush the filter block.
232 - w.filterBlock.flush(w.offset)
233 - return nil
234 -}
235 -
236 -// Append appends key/value pair to the table. The keys passed must
237 -// be in increasing order.
238 -//
239 -// It is safe to modify the contents of the arguments after Append returns.
240 -func (w *Writer) Append(key, value []byte) error {
241 - if w.err != nil {
242 - return w.err
243 - }
244 - if w.nEntries > 0 && w.cmp.Compare(w.dataBlock.prevKey, key) >= 0 {
245 - w.err = fmt.Errorf("leveldb/table: Writer: keys are not in increasing order: %q, %q", w.dataBlock.prevKey, key)
246 - return w.err
247 - }
248 -
249 - w.flushPendingBH(key)
250 - // Append key/value pair to the data block.
251 - w.dataBlock.append(key, value)
252 - // Add key to the filter block.
253 - w.filterBlock.add(key)
254 -
255 - // Finish the data block if block size target reached.
256 - if w.dataBlock.bytesLen() >= w.blockSize {
257 - if err := w.finishBlock(); err != nil {
258 - w.err = err
259 - return w.err
260 - }
261 - }
262 - w.nEntries++
263 - return nil
264 -}
265 -
266 -// BlocksLen returns number of blocks written so far.
267 -func (w *Writer) BlocksLen() int {
268 - n := w.indexBlock.nEntries
269 - if w.pendingBH.length > 0 {
270 - // Includes the pending block.
271 - n++
272 - }
273 - return n
274 -}
275 -
276 -// EntriesLen returns number of entries added so far.
277 -func (w *Writer) EntriesLen() int {
278 - return w.nEntries
279 -}
280 -
281 -// BytesLen returns number of bytes written so far.
282 -func (w *Writer) BytesLen() int {
283 - return int(w.offset)
284 -}
285 -
286 -// Close will finalize the table. Calling Append is not possible
287 -// after Close, but calling BlocksLen, EntriesLen and BytesLen
288 -// is still possible.
289 -func (w *Writer) Close() error {
290 - if w.err != nil {
291 - return w.err
292 - }
293 -
294 - // Write the last data block. Or empty data block if there
295 - // aren't any data blocks at all.
296 - if w.dataBlock.nEntries > 0 || w.nEntries == 0 {
297 - if err := w.finishBlock(); err != nil {
298 - w.err = err
299 - return w.err
300 - }
301 - }
302 - w.flushPendingBH(nil)
303 -
304 - // Write the filter block.
305 - var filterBH blockHandle
306 - w.filterBlock.finish()
307 - if buf := &w.filterBlock.buf; buf.Len() > 0 {
308 - filterBH, w.err = w.writeBlock(buf, opt.NoCompression)
309 - if w.err != nil {
310 - return w.err
311 - }
312 - }
313 -
314 - // Write the metaindex block.
315 - if filterBH.length > 0 {
316 - key := []byte("filter." + w.filter.Name())
317 - n := encodeBlockHandle(w.scratch[:20], filterBH)
318 - w.dataBlock.append(key, w.scratch[:n])
319 - }
320 - w.dataBlock.finish()
321 - metaindexBH, err := w.writeBlock(&w.dataBlock.buf, w.compression)
322 - if err != nil {
323 - w.err = err
324 - return w.err
325 - }
326 -
327 - // Write the index block.
328 - w.indexBlock.finish()
329 - indexBH, err := w.writeBlock(&w.indexBlock.buf, w.compression)
330 - if err != nil {
331 - w.err = err
332 - return w.err
333 - }
334 -
335 - // Write the table footer.
336 - footer := w.scratch[:footerLen]
337 - for i := range footer {
338 - footer[i] = 0
339 - }
340 - n := encodeBlockHandle(footer, metaindexBH)
341 - encodeBlockHandle(footer[n:], indexBH)
342 - copy(footer[footerLen-len(magic):], magic)
343 - if _, err := w.writer.Write(footer); err != nil {
344 - w.err = err
345 - return w.err
346 - }
347 - w.offset += footerLen
348 -
349 - w.err = errors.New("leveldb/table: writer is closed")
350 - return nil
351 -}
352 -
353 -// NewWriter creates a new initialized table writer for the file.
354 -//
355 -// Table writer is not goroutine-safe.
356 -func NewWriter(f io.Writer, o *opt.Options) *Writer {
357 - w := &Writer{
358 - writer: f,
359 - cmp: o.GetComparer(),
360 - filter: o.GetFilter(),
361 - compression: o.GetCompression(),
362 - blockSize: o.GetBlockSize(),
363 - comparerScratch: make([]byte, 0),
364 - }
365 - // data block
366 - w.dataBlock.restartInterval = o.GetBlockRestartInterval()
367 - // The first 20-bytes are used for encoding block handle.
368 - w.dataBlock.scratch = w.scratch[20:]
369 - // index block
370 - w.indexBlock.restartInterval = 1
371 - w.indexBlock.scratch = w.scratch[20:]
372 - // filter block
373 - if w.filter != nil {
374 - w.filterBlock.generator = w.filter.NewGenerator()
375 - w.filterBlock.flush(0)
376 - }
377 - return w
378 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/db.go deleted
-222
@@ -1,222 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "fmt"
11 - "math/rand"
12 -
13 - . "github.com/onsi/gomega"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18 -)
19 -
20 -type DB interface{}
21 -
22 -type Put interface {
23 - TestPut(key []byte, value []byte) error
24 -}
25 -
26 -type Delete interface {
27 - TestDelete(key []byte) error
28 -}
29 -
30 -type Find interface {
31 - TestFind(key []byte) (rkey, rvalue []byte, err error)
32 -}
33 -
34 -type Get interface {
35 - TestGet(key []byte) (value []byte, err error)
36 -}
37 -
38 -type Has interface {
39 - TestHas(key []byte) (ret bool, err error)
40 -}
41 -
42 -type NewIterator interface {
43 - TestNewIterator(slice *util.Range) iterator.Iterator
44 -}
45 -
46 -type DBAct int
47 -
48 -func (a DBAct) String() string {
49 - switch a {
50 - case DBNone:
51 - return "none"
52 - case DBPut:
53 - return "put"
54 - case DBOverwrite:
55 - return "overwrite"
56 - case DBDelete:
57 - return "delete"
58 - case DBDeleteNA:
59 - return "delete_na"
60 - }
61 - return "unknown"
62 -}
63 -
64 -const (
65 - DBNone DBAct = iota
66 - DBPut
67 - DBOverwrite
68 - DBDelete
69 - DBDeleteNA
70 -)
71 -
72 -type DBTesting struct {
73 - Rand *rand.Rand
74 - DB interface {
75 - Get
76 - Put
77 - Delete
78 - }
79 - PostFn func(t *DBTesting)
80 - Deleted, Present KeyValue
81 - Act, LastAct DBAct
82 - ActKey, LastActKey []byte
83 -}
84 -
85 -func (t *DBTesting) post() {
86 - if t.PostFn != nil {
87 - t.PostFn(t)
88 - }
89 -}
90 -
91 -func (t *DBTesting) setAct(act DBAct, key []byte) {
92 - t.LastAct, t.Act = t.Act, act
93 - t.LastActKey, t.ActKey = t.ActKey, key
94 -}
95 -
96 -func (t *DBTesting) text() string {
97 - return fmt.Sprintf("last action was <%v> %q, <%v> %q", t.LastAct, t.LastActKey, t.Act, t.ActKey)
98 -}
99 -
100 -func (t *DBTesting) Text() string {
101 - return "DBTesting " + t.text()
102 -}
103 -
104 -func (t *DBTesting) TestPresentKV(key, value []byte) {
105 - rvalue, err := t.DB.TestGet(key)
106 - Expect(err).ShouldNot(HaveOccurred(), "Get on key %q, %s", key, t.text())
107 - Expect(rvalue).Should(Equal(value), "Value for key %q, %s", key, t.text())
108 -}
109 -
110 -func (t *DBTesting) TestAllPresent() {
111 - t.Present.IterateShuffled(t.Rand, func(i int, key, value []byte) {
112 - t.TestPresentKV(key, value)
113 - })
114 -}
115 -
116 -func (t *DBTesting) TestDeletedKey(key []byte) {
117 - _, err := t.DB.TestGet(key)
118 - Expect(err).Should(Equal(errors.ErrNotFound), "Get on deleted key %q, %s", key, t.text())
119 -}
120 -
121 -func (t *DBTesting) TestAllDeleted() {
122 - t.Deleted.IterateShuffled(t.Rand, func(i int, key, value []byte) {
123 - t.TestDeletedKey(key)
124 - })
125 -}
126 -
127 -func (t *DBTesting) TestAll() {
128 - dn := t.Deleted.Len()
129 - pn := t.Present.Len()
130 - ShuffledIndex(t.Rand, dn+pn, 1, func(i int) {
131 - if i >= dn {
132 - key, value := t.Present.Index(i - dn)
133 - t.TestPresentKV(key, value)
134 - } else {
135 - t.TestDeletedKey(t.Deleted.KeyAt(i))
136 - }
137 - })
138 -}
139 -
140 -func (t *DBTesting) Put(key, value []byte) {
141 - if new := t.Present.PutU(key, value); new {
142 - t.setAct(DBPut, key)
143 - } else {
144 - t.setAct(DBOverwrite, key)
145 - }
146 - t.Deleted.Delete(key)
147 - err := t.DB.TestPut(key, value)
148 - Expect(err).ShouldNot(HaveOccurred(), t.Text())
149 - t.TestPresentKV(key, value)
150 - t.post()
151 -}
152 -
153 -func (t *DBTesting) PutRandom() bool {
154 - if t.Deleted.Len() > 0 {
155 - i := t.Rand.Intn(t.Deleted.Len())
156 - key, value := t.Deleted.Index(i)
157 - t.Put(key, value)
158 - return true
159 - }
160 - return false
161 -}
162 -
163 -func (t *DBTesting) Delete(key []byte) {
164 - if exist, value := t.Present.Delete(key); exist {
165 - t.setAct(DBDelete, key)
166 - t.Deleted.PutU(key, value)
167 - } else {
168 - t.setAct(DBDeleteNA, key)
169 - }
170 - err := t.DB.TestDelete(key)
171 - Expect(err).ShouldNot(HaveOccurred(), t.Text())
172 - t.TestDeletedKey(key)
173 - t.post()
174 -}
175 -
176 -func (t *DBTesting) DeleteRandom() bool {
177 - if t.Present.Len() > 0 {
178 - i := t.Rand.Intn(t.Present.Len())
179 - t.Delete(t.Present.KeyAt(i))
180 - return true
181 - }
182 - return false
183 -}
184 -
185 -func (t *DBTesting) RandomAct(round int) {
186 - for i := 0; i < round; i++ {
187 - if t.Rand.Int()%2 == 0 {
188 - t.PutRandom()
189 - } else {
190 - t.DeleteRandom()
191 - }
192 - }
193 -}
194 -
195 -func DoDBTesting(t *DBTesting) {
196 - if t.Rand == nil {
197 - t.Rand = NewRand()
198 - }
199 -
200 - t.DeleteRandom()
201 - t.PutRandom()
202 - t.DeleteRandom()
203 - t.DeleteRandom()
204 - for i := t.Deleted.Len() / 2; i >= 0; i-- {
205 - t.PutRandom()
206 - }
207 - t.RandomAct((t.Deleted.Len() + t.Present.Len()) * 10)
208 -
209 - // Additional iterator testing
210 - if db, ok := t.DB.(NewIterator); ok {
211 - iter := db.TestNewIterator(nil)
212 - Expect(iter.Error()).NotTo(HaveOccurred())
213 -
214 - it := IteratorTesting{
215 - KeyValue: t.Present,
216 - Iter: iter,
217 - }
218 -
219 - DoIteratorTesting(&it)
220 - iter.Release()
221 - }
222 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/ginkgo.go deleted
-21
@@ -1,21 +0,0 @@
1 -package testutil
2 -
3 -import (
4 - . "github.com/onsi/ginkgo"
5 - . "github.com/onsi/gomega"
6 -)
7 -
8 -func RunSuite(t GinkgoTestingT, name string) {
9 - RunDefer()
10 -
11 - SynchronizedBeforeSuite(func() []byte {
12 - RunDefer("setup")
13 - return nil
14 - }, func(data []byte) {})
15 - SynchronizedAfterSuite(func() {
16 - RunDefer("teardown")
17 - }, func() {})
18 -
19 - RegisterFailHandler(Fail)
20 - RunSpecs(t, name)
21 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/iter.go deleted
-327
@@ -1,327 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "fmt"
11 - "math/rand"
12 -
13 - . "github.com/onsi/gomega"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
16 -)
17 -
18 -type IterAct int
19 -
20 -func (a IterAct) String() string {
21 - switch a {
22 - case IterNone:
23 - return "none"
24 - case IterFirst:
25 - return "first"
26 - case IterLast:
27 - return "last"
28 - case IterPrev:
29 - return "prev"
30 - case IterNext:
31 - return "next"
32 - case IterSeek:
33 - return "seek"
34 - case IterSOI:
35 - return "soi"
36 - case IterEOI:
37 - return "eoi"
38 - }
39 - return "unknown"
40 -}
41 -
42 -const (
43 - IterNone IterAct = iota
44 - IterFirst
45 - IterLast
46 - IterPrev
47 - IterNext
48 - IterSeek
49 - IterSOI
50 - IterEOI
51 -)
52 -
53 -type IteratorTesting struct {
54 - KeyValue
55 - Iter iterator.Iterator
56 - Rand *rand.Rand
57 - PostFn func(t *IteratorTesting)
58 - Pos int
59 - Act, LastAct IterAct
60 -
61 - once bool
62 -}
63 -
64 -func (t *IteratorTesting) init() {
65 - if !t.once {
66 - t.Pos = -1
67 - t.once = true
68 - }
69 -}
70 -
71 -func (t *IteratorTesting) post() {
72 - if t.PostFn != nil {
73 - t.PostFn(t)
74 - }
75 -}
76 -
77 -func (t *IteratorTesting) setAct(act IterAct) {
78 - t.LastAct, t.Act = t.Act, act
79 -}
80 -
81 -func (t *IteratorTesting) text() string {
82 - return fmt.Sprintf("at pos %d and last action was <%v> -> <%v>", t.Pos, t.LastAct, t.Act)
83 -}
84 -
85 -func (t *IteratorTesting) Text() string {
86 - return "IteratorTesting is " + t.text()
87 -}
88 -
89 -func (t *IteratorTesting) IsFirst() bool {
90 - t.init()
91 - return t.Len() > 0 && t.Pos == 0
92 -}
93 -
94 -func (t *IteratorTesting) IsLast() bool {
95 - t.init()
96 - return t.Len() > 0 && t.Pos == t.Len()-1
97 -}
98 -
99 -func (t *IteratorTesting) TestKV() {
100 - t.init()
101 - key, value := t.Index(t.Pos)
102 - Expect(t.Iter.Key()).NotTo(BeNil())
103 - Expect(t.Iter.Key()).Should(Equal(key), "Key is invalid, %s", t.text())
104 - Expect(t.Iter.Value()).Should(Equal(value), "Value for key %q, %s", key, t.text())
105 -}
106 -
107 -func (t *IteratorTesting) First() {
108 - t.init()
109 - t.setAct(IterFirst)
110 -
111 - ok := t.Iter.First()
112 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
113 - if t.Len() > 0 {
114 - t.Pos = 0
115 - Expect(ok).Should(BeTrue(), t.Text())
116 - t.TestKV()
117 - } else {
118 - t.Pos = -1
119 - Expect(ok).ShouldNot(BeTrue(), t.Text())
120 - }
121 - t.post()
122 -}
123 -
124 -func (t *IteratorTesting) Last() {
125 - t.init()
126 - t.setAct(IterLast)
127 -
128 - ok := t.Iter.Last()
129 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
130 - if t.Len() > 0 {
131 - t.Pos = t.Len() - 1
132 - Expect(ok).Should(BeTrue(), t.Text())
133 - t.TestKV()
134 - } else {
135 - t.Pos = 0
136 - Expect(ok).ShouldNot(BeTrue(), t.Text())
137 - }
138 - t.post()
139 -}
140 -
141 -func (t *IteratorTesting) Next() {
142 - t.init()
143 - t.setAct(IterNext)
144 -
145 - ok := t.Iter.Next()
146 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
147 - if t.Pos < t.Len()-1 {
148 - t.Pos++
149 - Expect(ok).Should(BeTrue(), t.Text())
150 - t.TestKV()
151 - } else {
152 - t.Pos = t.Len()
153 - Expect(ok).ShouldNot(BeTrue(), t.Text())
154 - }
155 - t.post()
156 -}
157 -
158 -func (t *IteratorTesting) Prev() {
159 - t.init()
160 - t.setAct(IterPrev)
161 -
162 - ok := t.Iter.Prev()
163 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
164 - if t.Pos > 0 {
165 - t.Pos--
166 - Expect(ok).Should(BeTrue(), t.Text())
167 - t.TestKV()
168 - } else {
169 - t.Pos = -1
170 - Expect(ok).ShouldNot(BeTrue(), t.Text())
171 - }
172 - t.post()
173 -}
174 -
175 -func (t *IteratorTesting) Seek(i int) {
176 - t.init()
177 - t.setAct(IterSeek)
178 -
179 - key, _ := t.Index(i)
180 - oldKey, _ := t.IndexOrNil(t.Pos)
181 -
182 - ok := t.Iter.Seek(key)
183 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
184 - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q, to pos %d, %s", oldKey, key, i, t.text()))
185 -
186 - t.Pos = i
187 - t.TestKV()
188 - t.post()
189 -}
190 -
191 -func (t *IteratorTesting) SeekInexact(i int) {
192 - t.init()
193 - t.setAct(IterSeek)
194 - var key0 []byte
195 - key1, _ := t.Index(i)
196 - if i > 0 {
197 - key0, _ = t.Index(i - 1)
198 - }
199 - key := BytesSeparator(key0, key1)
200 - oldKey, _ := t.IndexOrNil(t.Pos)
201 -
202 - ok := t.Iter.Seek(key)
203 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
204 - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key1, i, t.text()))
205 -
206 - t.Pos = i
207 - t.TestKV()
208 - t.post()
209 -}
210 -
211 -func (t *IteratorTesting) SeekKey(key []byte) {
212 - t.init()
213 - t.setAct(IterSeek)
214 - oldKey, _ := t.IndexOrNil(t.Pos)
215 - i := t.Search(key)
216 -
217 - ok := t.Iter.Seek(key)
218 - Expect(t.Iter.Error()).ShouldNot(HaveOccurred())
219 - if i < t.Len() {
220 - key_, _ := t.Index(i)
221 - Expect(ok).Should(BeTrue(), fmt.Sprintf("Seek from key %q to %q (%q), to pos %d, %s", oldKey, key, key_, i, t.text()))
222 - t.Pos = i
223 - t.TestKV()
224 - } else {
225 - Expect(ok).ShouldNot(BeTrue(), fmt.Sprintf("Seek from key %q to %q, %s", oldKey, key, t.text()))
226 - }
227 -
228 - t.Pos = i
229 - t.post()
230 -}
231 -
232 -func (t *IteratorTesting) SOI() {
233 - t.init()
234 - t.setAct(IterSOI)
235 - Expect(t.Pos).Should(BeNumerically("<=", 0), t.Text())
236 - for i := 0; i < 3; i++ {
237 - t.Prev()
238 - }
239 - t.post()
240 -}
241 -
242 -func (t *IteratorTesting) EOI() {
243 - t.init()
244 - t.setAct(IterEOI)
245 - Expect(t.Pos).Should(BeNumerically(">=", t.Len()-1), t.Text())
246 - for i := 0; i < 3; i++ {
247 - t.Next()
248 - }
249 - t.post()
250 -}
251 -
252 -func (t *IteratorTesting) WalkPrev(fn func(t *IteratorTesting)) {
253 - t.init()
254 - for old := t.Pos; t.Pos > 0; old = t.Pos {
255 - fn(t)
256 - Expect(t.Pos).Should(BeNumerically("<", old), t.Text())
257 - }
258 -}
259 -
260 -func (t *IteratorTesting) WalkNext(fn func(t *IteratorTesting)) {
261 - t.init()
262 - for old := t.Pos; t.Pos < t.Len()-1; old = t.Pos {
263 - fn(t)
264 - Expect(t.Pos).Should(BeNumerically(">", old), t.Text())
265 - }
266 -}
267 -
268 -func (t *IteratorTesting) PrevAll() {
269 - t.WalkPrev(func(t *IteratorTesting) {
270 - t.Prev()
271 - })
272 -}
273 -
274 -func (t *IteratorTesting) NextAll() {
275 - t.WalkNext(func(t *IteratorTesting) {
276 - t.Next()
277 - })
278 -}
279 -
280 -func DoIteratorTesting(t *IteratorTesting) {
281 - if t.Rand == nil {
282 - t.Rand = NewRand()
283 - }
284 - t.SOI()
285 - t.NextAll()
286 - t.First()
287 - t.SOI()
288 - t.NextAll()
289 - t.EOI()
290 - t.PrevAll()
291 - t.Last()
292 - t.EOI()
293 - t.PrevAll()
294 - t.SOI()
295 -
296 - t.NextAll()
297 - t.PrevAll()
298 - t.NextAll()
299 - t.Last()
300 - t.PrevAll()
301 - t.First()
302 - t.NextAll()
303 - t.EOI()
304 -
305 - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
306 - t.Seek(i)
307 - })
308 -
309 - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
310 - t.SeekInexact(i)
311 - })
312 -
313 - ShuffledIndex(t.Rand, t.Len(), 1, func(i int) {
314 - t.Seek(i)
315 - if i%2 != 0 {
316 - t.PrevAll()
317 - t.SOI()
318 - } else {
319 - t.NextAll()
320 - t.EOI()
321 - }
322 - })
323 -
324 - for _, key := range []string{"", "foo", "bar", "\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff"} {
325 - t.SeekKey([]byte(key))
326 - }
327 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kv.go deleted
-352
@@ -1,352 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "fmt"
11 - "math/rand"
12 - "sort"
13 - "strings"
14 -
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -type KeyValueEntry struct {
19 - key, value []byte
20 -}
21 -
22 -type KeyValue struct {
23 - entries []KeyValueEntry
24 - nbytes int
25 -}
26 -
27 -func (kv *KeyValue) Put(key, value []byte) {
28 - if n := len(kv.entries); n > 0 && cmp.Compare(kv.entries[n-1].key, key) >= 0 {
29 - panic(fmt.Sprintf("Put: keys are not in increasing order: %q, %q", kv.entries[n-1].key, key))
30 - }
31 - kv.entries = append(kv.entries, KeyValueEntry{key, value})
32 - kv.nbytes += len(key) + len(value)
33 -}
34 -
35 -func (kv *KeyValue) PutString(key, value string) {
36 - kv.Put([]byte(key), []byte(value))
37 -}
38 -
39 -func (kv *KeyValue) PutU(key, value []byte) bool {
40 - if i, exist := kv.Get(key); !exist {
41 - if i < kv.Len() {
42 - kv.entries = append(kv.entries[:i+1], kv.entries[i:]...)
43 - kv.entries[i] = KeyValueEntry{key, value}
44 - } else {
45 - kv.entries = append(kv.entries, KeyValueEntry{key, value})
46 - }
47 - kv.nbytes += len(key) + len(value)
48 - return true
49 - } else {
50 - kv.nbytes += len(value) - len(kv.ValueAt(i))
51 - kv.entries[i].value = value
52 - }
53 - return false
54 -}
55 -
56 -func (kv *KeyValue) PutUString(key, value string) bool {
57 - return kv.PutU([]byte(key), []byte(value))
58 -}
59 -
60 -func (kv *KeyValue) Delete(key []byte) (exist bool, value []byte) {
61 - i, exist := kv.Get(key)
62 - if exist {
63 - value = kv.entries[i].value
64 - kv.DeleteIndex(i)
65 - }
66 - return
67 -}
68 -
69 -func (kv *KeyValue) DeleteIndex(i int) bool {
70 - if i < kv.Len() {
71 - kv.nbytes -= len(kv.KeyAt(i)) + len(kv.ValueAt(i))
72 - kv.entries = append(kv.entries[:i], kv.entries[i+1:]...)
73 - return true
74 - }
75 - return false
76 -}
77 -
78 -func (kv KeyValue) Len() int {
79 - return len(kv.entries)
80 -}
81 -
82 -func (kv *KeyValue) Size() int {
83 - return kv.nbytes
84 -}
85 -
86 -func (kv KeyValue) KeyAt(i int) []byte {
87 - return kv.entries[i].key
88 -}
89 -
90 -func (kv KeyValue) ValueAt(i int) []byte {
91 - return kv.entries[i].value
92 -}
93 -
94 -func (kv KeyValue) Index(i int) (key, value []byte) {
95 - if i < 0 || i >= len(kv.entries) {
96 - panic(fmt.Sprintf("Index #%d: out of range", i))
97 - }
98 - return kv.entries[i].key, kv.entries[i].value
99 -}
100 -
101 -func (kv KeyValue) IndexInexact(i int) (key_, key, value []byte) {
102 - key, value = kv.Index(i)
103 - var key0 []byte
104 - var key1 = kv.KeyAt(i)
105 - if i > 0 {
106 - key0 = kv.KeyAt(i - 1)
107 - }
108 - key_ = BytesSeparator(key0, key1)
109 - return
110 -}
111 -
112 -func (kv KeyValue) IndexOrNil(i int) (key, value []byte) {
113 - if i >= 0 && i < len(kv.entries) {
114 - return kv.entries[i].key, kv.entries[i].value
115 - }
116 - return nil, nil
117 -}
118 -
119 -func (kv KeyValue) IndexString(i int) (key, value string) {
120 - key_, _value := kv.Index(i)
121 - return string(key_), string(_value)
122 -}
123 -
124 -func (kv KeyValue) Search(key []byte) int {
125 - return sort.Search(kv.Len(), func(i int) bool {
126 - return cmp.Compare(kv.KeyAt(i), key) >= 0
127 - })
128 -}
129 -
130 -func (kv KeyValue) SearchString(key string) int {
131 - return kv.Search([]byte(key))
132 -}
133 -
134 -func (kv KeyValue) Get(key []byte) (i int, exist bool) {
135 - i = kv.Search(key)
136 - if i < kv.Len() && cmp.Compare(kv.KeyAt(i), key) == 0 {
137 - exist = true
138 - }
139 - return
140 -}
141 -
142 -func (kv KeyValue) GetString(key string) (i int, exist bool) {
143 - return kv.Get([]byte(key))
144 -}
145 -
146 -func (kv KeyValue) Iterate(fn func(i int, key, value []byte)) {
147 - for i, x := range kv.entries {
148 - fn(i, x.key, x.value)
149 - }
150 -}
151 -
152 -func (kv KeyValue) IterateString(fn func(i int, key, value string)) {
153 - kv.Iterate(func(i int, key, value []byte) {
154 - fn(i, string(key), string(value))
155 - })
156 -}
157 -
158 -func (kv KeyValue) IterateShuffled(rnd *rand.Rand, fn func(i int, key, value []byte)) {
159 - ShuffledIndex(rnd, kv.Len(), 1, func(i int) {
160 - fn(i, kv.entries[i].key, kv.entries[i].value)
161 - })
162 -}
163 -
164 -func (kv KeyValue) IterateShuffledString(rnd *rand.Rand, fn func(i int, key, value string)) {
165 - kv.IterateShuffled(rnd, func(i int, key, value []byte) {
166 - fn(i, string(key), string(value))
167 - })
168 -}
169 -
170 -func (kv KeyValue) IterateInexact(fn func(i int, key_, key, value []byte)) {
171 - for i := range kv.entries {
172 - key_, key, value := kv.IndexInexact(i)
173 - fn(i, key_, key, value)
174 - }
175 -}
176 -
177 -func (kv KeyValue) IterateInexactString(fn func(i int, key_, key, value string)) {
178 - kv.IterateInexact(func(i int, key_, key, value []byte) {
179 - fn(i, string(key_), string(key), string(value))
180 - })
181 -}
182 -
183 -func (kv KeyValue) Clone() KeyValue {
184 - return KeyValue{append([]KeyValueEntry{}, kv.entries...), kv.nbytes}
185 -}
186 -
187 -func (kv KeyValue) Slice(start, limit int) KeyValue {
188 - if start < 0 || limit > kv.Len() {
189 - panic(fmt.Sprintf("Slice %d .. %d: out of range", start, limit))
190 - } else if limit < start {
191 - panic(fmt.Sprintf("Slice %d .. %d: invalid range", start, limit))
192 - }
193 - return KeyValue{append([]KeyValueEntry{}, kv.entries[start:limit]...), kv.nbytes}
194 -}
195 -
196 -func (kv KeyValue) SliceKey(start, limit []byte) KeyValue {
197 - start_ := 0
198 - limit_ := kv.Len()
199 - if start != nil {
200 - start_ = kv.Search(start)
201 - }
202 - if limit != nil {
203 - limit_ = kv.Search(limit)
204 - }
205 - return kv.Slice(start_, limit_)
206 -}
207 -
208 -func (kv KeyValue) SliceKeyString(start, limit string) KeyValue {
209 - return kv.SliceKey([]byte(start), []byte(limit))
210 -}
211 -
212 -func (kv KeyValue) SliceRange(r *util.Range) KeyValue {
213 - if r != nil {
214 - return kv.SliceKey(r.Start, r.Limit)
215 - }
216 - return kv.Clone()
217 -}
218 -
219 -func (kv KeyValue) Range(start, limit int) (r util.Range) {
220 - if kv.Len() > 0 {
221 - if start == kv.Len() {
222 - r.Start = BytesAfter(kv.KeyAt(start - 1))
223 - } else {
224 - r.Start = kv.KeyAt(start)
225 - }
226 - }
227 - if limit < kv.Len() {
228 - r.Limit = kv.KeyAt(limit)
229 - }
230 - return
231 -}
232 -
233 -func KeyValue_EmptyKey() *KeyValue {
234 - kv := &KeyValue{}
235 - kv.PutString("", "v")
236 - return kv
237 -}
238 -
239 -func KeyValue_EmptyValue() *KeyValue {
240 - kv := &KeyValue{}
241 - kv.PutString("abc", "")
242 - kv.PutString("abcd", "")
243 - return kv
244 -}
245 -
246 -func KeyValue_OneKeyValue() *KeyValue {
247 - kv := &KeyValue{}
248 - kv.PutString("abc", "v")
249 - return kv
250 -}
251 -
252 -func KeyValue_BigValue() *KeyValue {
253 - kv := &KeyValue{}
254 - kv.PutString("big1", strings.Repeat("1", 200000))
255 - return kv
256 -}
257 -
258 -func KeyValue_SpecialKey() *KeyValue {
259 - kv := &KeyValue{}
260 - kv.PutString("\xff\xff", "v3")
261 - return kv
262 -}
263 -
264 -func KeyValue_MultipleKeyValue() *KeyValue {
265 - kv := &KeyValue{}
266 - kv.PutString("a", "v")
267 - kv.PutString("aa", "v1")
268 - kv.PutString("aaa", "v2")
269 - kv.PutString("aaacccccccccc", "v2")
270 - kv.PutString("aaaccccccccccd", "v3")
271 - kv.PutString("aaaccccccccccf", "v4")
272 - kv.PutString("aaaccccccccccfg", "v5")
273 - kv.PutString("ab", "v6")
274 - kv.PutString("abc", "v7")
275 - kv.PutString("abcd", "v8")
276 - kv.PutString("accccccccccccccc", "v9")
277 - kv.PutString("b", "v10")
278 - kv.PutString("bb", "v11")
279 - kv.PutString("bc", "v12")
280 - kv.PutString("c", "v13")
281 - kv.PutString("c1", "v13")
282 - kv.PutString("czzzzzzzzzzzzzz", "v14")
283 - kv.PutString("fffffffffffffff", "v15")
284 - kv.PutString("g11", "v15")
285 - kv.PutString("g111", "v15")
286 - kv.PutString("g111\xff", "v15")
287 - kv.PutString("zz", "v16")
288 - kv.PutString("zzzzzzz", "v16")
289 - kv.PutString("zzzzzzzzzzzzzzzz", "v16")
290 - return kv
291 -}
292 -
293 -var keymap = []byte("012345678ABCDEFGHIJKLMNOPQRSTUVWXYabcdefghijklmnopqrstuvwxy")
294 -
295 -func KeyValue_Generate(rnd *rand.Rand, n, minlen, maxlen, vminlen, vmaxlen int) *KeyValue {
296 - if rnd == nil {
297 - rnd = NewRand()
298 - }
299 - if maxlen < minlen {
300 - panic("max len should >= min len")
301 - }
302 -
303 - rrand := func(min, max int) int {
304 - if min == max {
305 - return max
306 - }
307 - return rnd.Intn(max-min) + min
308 - }
309 -
310 - kv := &KeyValue{}
311 - endC := byte(len(keymap) - 1)
312 - gen := make([]byte, 0, maxlen)
313 - for i := 0; i < n; i++ {
314 - m := rrand(minlen, maxlen)
315 - last := gen
316 - retry:
317 - gen = last[:m]
318 - if k := len(last); m > k {
319 - for j := k; j < m; j++ {
320 - gen[j] = 0
321 - }
322 - } else {
323 - for j := m - 1; j >= 0; j-- {
324 - c := last[j]
325 - if c == endC {
326 - continue
327 - }
328 - gen[j] = c + 1
329 - for j += 1; j < m; j++ {
330 - gen[j] = 0
331 - }
332 - goto ok
333 - }
334 - if m < maxlen {
335 - m++
336 - goto retry
337 - }
338 - panic(fmt.Sprintf("only able to generate %d keys out of %d keys, try increasing max len", kv.Len(), n))
339 - ok:
340 - }
341 - key := make([]byte, m)
342 - for j := 0; j < m; j++ {
343 - key[j] = keymap[gen[j]]
344 - }
345 - value := make([]byte, rrand(vminlen, vmaxlen))
346 - for n := copy(value, []byte(fmt.Sprintf("v%d", i))); n < len(value); n++ {
347 - value[n] = 'x'
348 - }
349 - kv.Put(key, value)
350 - }
351 - return kv
352 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/kvtest.go deleted
-187
@@ -1,187 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "fmt"
11 - "math/rand"
12 -
13 - . "github.com/onsi/ginkgo"
14 - . "github.com/onsi/gomega"
15 -
16 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/errors"
17 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
18 -)
19 -
20 -func KeyValueTesting(rnd *rand.Rand, kv KeyValue, p DB, setup func(KeyValue) DB, teardown func(DB)) {
21 - if rnd == nil {
22 - rnd = NewRand()
23 - }
24 -
25 - if p == nil {
26 - BeforeEach(func() {
27 - p = setup(kv)
28 - })
29 - if teardown != nil {
30 - AfterEach(func() {
31 - teardown(p)
32 - })
33 - }
34 - }
35 -
36 - It("Should find all keys with Find", func() {
37 - if db, ok := p.(Find); ok {
38 - ShuffledIndex(nil, kv.Len(), 1, func(i int) {
39 - key_, key, value := kv.IndexInexact(i)
40 -
41 - // Using exact key.
42 - rkey, rvalue, err := db.TestFind(key)
43 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
44 - Expect(rkey).Should(Equal(key), "Key")
45 - Expect(rvalue).Should(Equal(value), "Value for key %q", key)
46 -
47 - // Using inexact key.
48 - rkey, rvalue, err = db.TestFind(key_)
49 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q (%q)", key_, key)
50 - Expect(rkey).Should(Equal(key))
51 - Expect(rvalue).Should(Equal(value), "Value for key %q (%q)", key_, key)
52 - })
53 - }
54 - })
55 -
56 - It("Should return error if the key is not present", func() {
57 - if db, ok := p.(Find); ok {
58 - var key []byte
59 - if kv.Len() > 0 {
60 - key_, _ := kv.Index(kv.Len() - 1)
61 - key = BytesAfter(key_)
62 - }
63 - rkey, _, err := db.TestFind(key)
64 - Expect(err).Should(HaveOccurred(), "Find for key %q yield key %q", key, rkey)
65 - Expect(err).Should(Equal(errors.ErrNotFound))
66 - }
67 - })
68 -
69 - It("Should only find exact key with Get", func() {
70 - if db, ok := p.(Get); ok {
71 - ShuffledIndex(nil, kv.Len(), 1, func(i int) {
72 - key_, key, value := kv.IndexInexact(i)
73 -
74 - // Using exact key.
75 - rvalue, err := db.TestGet(key)
76 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
77 - Expect(rvalue).Should(Equal(value), "Value for key %q", key)
78 -
79 - // Using inexact key.
80 - if len(key_) > 0 {
81 - _, err = db.TestGet(key_)
82 - Expect(err).Should(HaveOccurred(), "Error for key %q", key_)
83 - Expect(err).Should(Equal(errors.ErrNotFound))
84 - }
85 - })
86 - }
87 - })
88 -
89 - It("Should only find present key with Has", func() {
90 - if db, ok := p.(Has); ok {
91 - ShuffledIndex(nil, kv.Len(), 1, func(i int) {
92 - key_, key, _ := kv.IndexInexact(i)
93 -
94 - // Using exact key.
95 - ret, err := db.TestHas(key)
96 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key)
97 - Expect(ret).Should(BeTrue(), "False for key %q", key)
98 -
99 - // Using inexact key.
100 - if len(key_) > 0 {
101 - ret, err = db.TestHas(key_)
102 - Expect(err).ShouldNot(HaveOccurred(), "Error for key %q", key_)
103 - Expect(ret).ShouldNot(BeTrue(), "True for key %q", key)
104 - }
105 - })
106 - }
107 - })
108 -
109 - TestIter := func(r *util.Range, _kv KeyValue) {
110 - if db, ok := p.(NewIterator); ok {
111 - iter := db.TestNewIterator(r)
112 - Expect(iter.Error()).ShouldNot(HaveOccurred())
113 -
114 - t := IteratorTesting{
115 - KeyValue: _kv,
116 - Iter: iter,
117 - }
118 -
119 - DoIteratorTesting(&t)
120 - iter.Release()
121 - }
122 - }
123 -
124 - It("Should iterates and seeks correctly", func(done Done) {
125 - TestIter(nil, kv.Clone())
126 - done <- true
127 - }, 3.0)
128 -
129 - RandomIndex(rnd, kv.Len(), Min(kv.Len(), 50), func(i int) {
130 - type slice struct {
131 - r *util.Range
132 - start, limit int
133 - }
134 -
135 - key_, _, _ := kv.IndexInexact(i)
136 - for _, x := range []slice{
137 - {&util.Range{Start: key_, Limit: nil}, i, kv.Len()},
138 - {&util.Range{Start: nil, Limit: key_}, 0, i},
139 - } {
140 - It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", x.start, x.limit), func(done Done) {
141 - TestIter(x.r, kv.Slice(x.start, x.limit))
142 - done <- true
143 - }, 3.0)
144 - }
145 - })
146 -
147 - RandomRange(rnd, kv.Len(), Min(kv.Len(), 50), func(start, limit int) {
148 - It(fmt.Sprintf("Should iterates and seeks correctly of a slice %d .. %d", start, limit), func(done Done) {
149 - r := kv.Range(start, limit)
150 - TestIter(&r, kv.Slice(start, limit))
151 - done <- true
152 - }, 3.0)
153 - })
154 -}
155 -
156 -func AllKeyValueTesting(rnd *rand.Rand, body, setup func(KeyValue) DB, teardown func(DB)) {
157 - Test := func(kv *KeyValue) func() {
158 - return func() {
159 - var p DB
160 - if setup != nil {
161 - Defer("setup", func() {
162 - p = setup(*kv)
163 - })
164 - }
165 - if teardown != nil {
166 - Defer("teardown", func() {
167 - teardown(p)
168 - })
169 - }
170 - if body != nil {
171 - p = body(*kv)
172 - }
173 - KeyValueTesting(rnd, *kv, p, func(KeyValue) DB {
174 - return p
175 - }, nil)
176 - }
177 - }
178 -
179 - Describe("with no key/value (empty)", Test(&KeyValue{}))
180 - Describe("with empty key", Test(KeyValue_EmptyKey()))
181 - Describe("with empty value", Test(KeyValue_EmptyValue()))
182 - Describe("with one key/value", Test(KeyValue_OneKeyValue()))
183 - Describe("with big value", Test(KeyValue_BigValue()))
184 - Describe("with special key", Test(KeyValue_SpecialKey()))
185 - Describe("with multiple key/value", Test(KeyValue_MultipleKeyValue()))
186 - Describe("with generated key/value", Test(KeyValue_Generate(nil, 120, 1, 50, 10, 120)))
187 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/storage.go deleted
-586
@@ -1,586 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "bytes"
11 - "fmt"
12 - "io"
13 - "os"
14 - "path/filepath"
15 - "runtime"
16 - "strings"
17 - "sync"
18 -
19 - . "github.com/onsi/gomega"
20 -
21 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
22 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
23 -)
24 -
25 -var (
26 - storageMu sync.Mutex
27 - storageUseFS bool = true
28 - storageKeepFS bool = false
29 - storageNum int
30 -)
31 -
32 -type StorageMode int
33 -
34 -const (
35 - ModeOpen StorageMode = 1 << iota
36 - ModeCreate
37 - ModeRemove
38 - ModeRead
39 - ModeWrite
40 - ModeSync
41 - ModeClose
42 -)
43 -
44 -const (
45 - modeOpen = iota
46 - modeCreate
47 - modeRemove
48 - modeRead
49 - modeWrite
50 - modeSync
51 - modeClose
52 -
53 - modeCount
54 -)
55 -
56 -const (
57 - typeManifest = iota
58 - typeJournal
59 - typeTable
60 - typeTemp
61 -
62 - typeCount
63 -)
64 -
65 -const flattenCount = modeCount * typeCount
66 -
67 -func flattenType(m StorageMode, t storage.FileType) int {
68 - var x int
69 - switch m {
70 - case ModeOpen:
71 - x = modeOpen
72 - case ModeCreate:
73 - x = modeCreate
74 - case ModeRemove:
75 - x = modeRemove
76 - case ModeRead:
77 - x = modeRead
78 - case ModeWrite:
79 - x = modeWrite
80 - case ModeSync:
81 - x = modeSync
82 - case ModeClose:
83 - x = modeClose
84 - default:
85 - panic("invalid storage mode")
86 - }
87 - x *= typeCount
88 - switch t {
89 - case storage.TypeManifest:
90 - return x + typeManifest
91 - case storage.TypeJournal:
92 - return x + typeJournal
93 - case storage.TypeTable:
94 - return x + typeTable
95 - case storage.TypeTemp:
96 - return x + typeTemp
97 - default:
98 - panic("invalid file type")
99 - }
100 -}
101 -
102 -func listFlattenType(m StorageMode, t storage.FileType) []int {
103 - ret := make([]int, 0, flattenCount)
104 - add := func(x int) {
105 - x *= typeCount
106 - switch {
107 - case t&storage.TypeManifest != 0:
108 - ret = append(ret, x+typeManifest)
109 - case t&storage.TypeJournal != 0:
110 - ret = append(ret, x+typeJournal)
111 - case t&storage.TypeTable != 0:
112 - ret = append(ret, x+typeTable)
113 - case t&storage.TypeTemp != 0:
114 - ret = append(ret, x+typeTemp)
115 - }
116 - }
117 - switch {
118 - case m&ModeOpen != 0:
119 - add(modeOpen)
120 - case m&ModeCreate != 0:
121 - add(modeCreate)
122 - case m&ModeRemove != 0:
123 - add(modeRemove)
124 - case m&ModeRead != 0:
125 - add(modeRead)
126 - case m&ModeWrite != 0:
127 - add(modeWrite)
128 - case m&ModeSync != 0:
129 - add(modeSync)
130 - case m&ModeClose != 0:
131 - add(modeClose)
132 - }
133 - return ret
134 -}
135 -
136 -func packFile(num uint64, t storage.FileType) uint64 {
137 - if num>>(64-typeCount) != 0 {
138 - panic("overflow")
139 - }
140 - return num<<typeCount | uint64(t)
141 -}
142 -
143 -func unpackFile(x uint64) (uint64, storage.FileType) {
144 - return x >> typeCount, storage.FileType(x) & storage.TypeAll
145 -}
146 -
147 -type emulatedError struct {
148 - err error
149 -}
150 -
151 -func (err emulatedError) Error() string {
152 - return fmt.Sprintf("emulated storage error: %v", err.err)
153 -}
154 -
155 -type storageLock struct {
156 - s *Storage
157 - r util.Releaser
158 -}
159 -
160 -func (l storageLock) Release() {
161 - l.r.Release()
162 - l.s.logI("storage lock released")
163 -}
164 -
165 -type reader struct {
166 - f *file
167 - storage.Reader
168 -}
169 -
170 -func (r *reader) Read(p []byte) (n int, err error) {
171 - err = r.f.s.emulateError(ModeRead, r.f.Type())
172 - if err == nil {
173 - r.f.s.stall(ModeRead, r.f.Type())
174 - n, err = r.Reader.Read(p)
175 - }
176 - r.f.s.count(ModeRead, r.f.Type(), n)
177 - if err != nil && err != io.EOF {
178 - r.f.s.logI("read error, num=%d type=%v n=%d err=%v", r.f.Num(), r.f.Type(), n, err)
179 - }
180 - return
181 -}
182 -
183 -func (r *reader) ReadAt(p []byte, off int64) (n int, err error) {
184 - err = r.f.s.emulateError(ModeRead, r.f.Type())
185 - if err == nil {
186 - r.f.s.stall(ModeRead, r.f.Type())
187 - n, err = r.Reader.ReadAt(p, off)
188 - }
189 - r.f.s.count(ModeRead, r.f.Type(), n)
190 - if err != nil && err != io.EOF {
191 - r.f.s.logI("readAt error, num=%d type=%v offset=%d n=%d err=%v", r.f.Num(), r.f.Type(), off, n, err)
192 - }
193 - return
194 -}
195 -
196 -func (r *reader) Close() (err error) {
197 - return r.f.doClose(r.Reader)
198 -}
199 -
200 -type writer struct {
201 - f *file
202 - storage.Writer
203 -}
204 -
205 -func (w *writer) Write(p []byte) (n int, err error) {
206 - err = w.f.s.emulateError(ModeWrite, w.f.Type())
207 - if err == nil {
208 - w.f.s.stall(ModeWrite, w.f.Type())
209 - n, err = w.Writer.Write(p)
210 - }
211 - w.f.s.count(ModeWrite, w.f.Type(), n)
212 - if err != nil && err != io.EOF {
213 - w.f.s.logI("write error, num=%d type=%v n=%d err=%v", w.f.Num(), w.f.Type(), n, err)
214 - }
215 - return
216 -}
217 -
218 -func (w *writer) Sync() (err error) {
219 - err = w.f.s.emulateError(ModeSync, w.f.Type())
220 - if err == nil {
221 - w.f.s.stall(ModeSync, w.f.Type())
222 - err = w.Writer.Sync()
223 - }
224 - w.f.s.count(ModeSync, w.f.Type(), 0)
225 - if err != nil {
226 - w.f.s.logI("sync error, num=%d type=%v err=%v", w.f.Num(), w.f.Type(), err)
227 - }
228 - return
229 -}
230 -
231 -func (w *writer) Close() (err error) {
232 - return w.f.doClose(w.Writer)
233 -}
234 -
235 -type file struct {
236 - s *Storage
237 - storage.File
238 -}
239 -
240 -func (f *file) pack() uint64 {
241 - return packFile(f.Num(), f.Type())
242 -}
243 -
244 -func (f *file) assertOpen() {
245 - ExpectWithOffset(2, f.s.opens).NotTo(HaveKey(f.pack()), "File open, num=%d type=%v writer=%v", f.Num(), f.Type(), f.s.opens[f.pack()])
246 -}
247 -
248 -func (f *file) doClose(closer io.Closer) (err error) {
249 - err = f.s.emulateError(ModeClose, f.Type())
250 - if err == nil {
251 - f.s.stall(ModeClose, f.Type())
252 - }
253 - f.s.mu.Lock()
254 - defer f.s.mu.Unlock()
255 - if err == nil {
256 - ExpectWithOffset(2, f.s.opens).To(HaveKey(f.pack()), "File closed, num=%d type=%v", f.Num(), f.Type())
257 - err = closer.Close()
258 - }
259 - f.s.countNB(ModeClose, f.Type(), 0)
260 - writer := f.s.opens[f.pack()]
261 - if err != nil {
262 - f.s.logISkip(1, "file close failed, num=%d type=%v writer=%v err=%v", f.Num(), f.Type(), writer, err)
263 - } else {
264 - f.s.logISkip(1, "file closed, num=%d type=%v writer=%v", f.Num(), f.Type(), writer)
265 - delete(f.s.opens, f.pack())
266 - }
267 - return
268 -}
269 -
270 -func (f *file) Open() (r storage.Reader, err error) {
271 - err = f.s.emulateError(ModeOpen, f.Type())
272 - if err == nil {
273 - f.s.stall(ModeOpen, f.Type())
274 - }
275 - f.s.mu.Lock()
276 - defer f.s.mu.Unlock()
277 - if err == nil {
278 - f.assertOpen()
279 - f.s.countNB(ModeOpen, f.Type(), 0)
280 - r, err = f.File.Open()
281 - }
282 - if err != nil {
283 - f.s.logI("file open failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
284 - } else {
285 - f.s.logI("file opened, num=%d type=%v", f.Num(), f.Type())
286 - f.s.opens[f.pack()] = false
287 - r = &reader{f, r}
288 - }
289 - return
290 -}
291 -
292 -func (f *file) Create() (w storage.Writer, err error) {
293 - err = f.s.emulateError(ModeCreate, f.Type())
294 - if err == nil {
295 - f.s.stall(ModeCreate, f.Type())
296 - }
297 - f.s.mu.Lock()
298 - defer f.s.mu.Unlock()
299 - if err == nil {
300 - f.assertOpen()
301 - f.s.countNB(ModeCreate, f.Type(), 0)
302 - w, err = f.File.Create()
303 - }
304 - if err != nil {
305 - f.s.logI("file create failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
306 - } else {
307 - f.s.logI("file created, num=%d type=%v", f.Num(), f.Type())
308 - f.s.opens[f.pack()] = true
309 - w = &writer{f, w}
310 - }
311 - return
312 -}
313 -
314 -func (f *file) Remove() (err error) {
315 - err = f.s.emulateError(ModeRemove, f.Type())
316 - if err == nil {
317 - f.s.stall(ModeRemove, f.Type())
318 - }
319 - f.s.mu.Lock()
320 - defer f.s.mu.Unlock()
321 - if err == nil {
322 - f.assertOpen()
323 - f.s.countNB(ModeRemove, f.Type(), 0)
324 - err = f.File.Remove()
325 - }
326 - if err != nil {
327 - f.s.logI("file remove failed, num=%d type=%v err=%v", f.Num(), f.Type(), err)
328 - } else {
329 - f.s.logI("file removed, num=%d type=%v", f.Num(), f.Type())
330 - }
331 - return
332 -}
333 -
334 -type Storage struct {
335 - storage.Storage
336 - closeFn func() error
337 -
338 - lmu sync.Mutex
339 - lb bytes.Buffer
340 -
341 - mu sync.Mutex
342 - // Open files, true=writer, false=reader
343 - opens map[uint64]bool
344 - counters [flattenCount]int
345 - bytesCounter [flattenCount]int64
346 - emulatedError [flattenCount]error
347 - stallCond sync.Cond
348 - stalled [flattenCount]bool
349 -}
350 -
351 -func (s *Storage) log(skip int, str string) {
352 - s.lmu.Lock()
353 - defer s.lmu.Unlock()
354 - _, file, line, ok := runtime.Caller(skip + 2)
355 - if ok {
356 - // Truncate file name at last file name separator.
357 - if index := strings.LastIndex(file, "/"); index >= 0 {
358 - file = file[index+1:]
359 - } else if index = strings.LastIndex(file, "\\"); index >= 0 {
360 - file = file[index+1:]
361 - }
362 - } else {
363 - file = "???"
364 - line = 1
365 - }
366 - fmt.Fprintf(&s.lb, "%s:%d: ", file, line)
367 - lines := strings.Split(str, "\n")
368 - if l := len(lines); l > 1 && lines[l-1] == "" {
369 - lines = lines[:l-1]
370 - }
371 - for i, line := range lines {
372 - if i > 0 {
373 - s.lb.WriteString("\n\t")
374 - }
375 - s.lb.WriteString(line)
376 - }
377 - s.lb.WriteByte('\n')
378 -}
379 -
380 -func (s *Storage) logISkip(skip int, format string, args ...interface{}) {
381 - pc, _, _, ok := runtime.Caller(skip + 1)
382 - if ok {
383 - if f := runtime.FuncForPC(pc); f != nil {
384 - fname := f.Name()
385 - if index := strings.LastIndex(fname, "."); index >= 0 {
386 - fname = fname[index+1:]
387 - }
388 - format = fname + ": " + format
389 - }
390 - }
391 - s.log(skip+1, fmt.Sprintf(format, args...))
392 -}
393 -
394 -func (s *Storage) logI(format string, args ...interface{}) {
395 - s.logISkip(1, format, args...)
396 -}
397 -
398 -func (s *Storage) Log(str string) {
399 - s.log(1, "Log: "+str)
400 - s.Storage.Log(str)
401 -}
402 -
403 -func (s *Storage) Lock() (r util.Releaser, err error) {
404 - r, err = s.Storage.Lock()
405 - if err != nil {
406 - s.logI("storage locking failed, err=%v", err)
407 - } else {
408 - s.logI("storage locked")
409 - r = storageLock{s, r}
410 - }
411 - return
412 -}
413 -
414 -func (s *Storage) GetFile(num uint64, t storage.FileType) storage.File {
415 - return &file{s, s.Storage.GetFile(num, t)}
416 -}
417 -
418 -func (s *Storage) GetFiles(t storage.FileType) (files []storage.File, err error) {
419 - rfiles, err := s.Storage.GetFiles(t)
420 - if err != nil {
421 - s.logI("get files failed, err=%v", err)
422 - return
423 - }
424 - files = make([]storage.File, len(rfiles))
425 - for i, f := range rfiles {
426 - files[i] = &file{s, f}
427 - }
428 - s.logI("get files, type=0x%x count=%d", int(t), len(files))
429 - return
430 -}
431 -
432 -func (s *Storage) GetManifest() (f storage.File, err error) {
433 - manifest, err := s.Storage.GetManifest()
434 - if err != nil {
435 - if !os.IsNotExist(err) {
436 - s.logI("get manifest failed, err=%v", err)
437 - }
438 - return
439 - }
440 - s.logI("get manifest, num=%d", manifest.Num())
441 - return &file{s, manifest}, nil
442 -}
443 -
444 -func (s *Storage) SetManifest(f storage.File) error {
445 - f_, ok := f.(*file)
446 - ExpectWithOffset(1, ok).To(BeTrue())
447 - ExpectWithOffset(1, f_.Type()).To(Equal(storage.TypeManifest))
448 - err := s.Storage.SetManifest(f_.File)
449 - if err != nil {
450 - s.logI("set manifest failed, err=%v", err)
451 - } else {
452 - s.logI("set manifest, num=%d", f_.Num())
453 - }
454 - return err
455 -}
456 -
457 -func (s *Storage) openFiles() string {
458 - out := "Open files:"
459 - for x, writer := range s.opens {
460 - num, t := unpackFile(x)
461 - out += fmt.Sprintf("\n · num=%d type=%v writer=%v", num, t, writer)
462 - }
463 - return out
464 -}
465 -
466 -func (s *Storage) Close() error {
467 - s.mu.Lock()
468 - defer s.mu.Unlock()
469 - ExpectWithOffset(1, s.opens).To(BeEmpty(), s.openFiles())
470 - err := s.Storage.Close()
471 - if err != nil {
472 - s.logI("storage closing failed, err=%v", err)
473 - } else {
474 - s.logI("storage closed")
475 - }
476 - if s.closeFn != nil {
477 - if err1 := s.closeFn(); err1 != nil {
478 - s.logI("close func error, err=%v", err1)
479 - }
480 - }
481 - return err
482 -}
483 -
484 -func (s *Storage) countNB(m StorageMode, t storage.FileType, n int) {
485 - s.counters[flattenType(m, t)]++
486 - s.bytesCounter[flattenType(m, t)] += int64(n)
487 -}
488 -
489 -func (s *Storage) count(m StorageMode, t storage.FileType, n int) {
490 - s.mu.Lock()
491 - defer s.mu.Unlock()
492 - s.countNB(m, t, n)
493 -}
494 -
495 -func (s *Storage) ResetCounter(m StorageMode, t storage.FileType) {
496 - for _, x := range listFlattenType(m, t) {
497 - s.counters[x] = 0
498 - s.bytesCounter[x] = 0
499 - }
500 -}
501 -
502 -func (s *Storage) Counter(m StorageMode, t storage.FileType) (count int, bytes int64) {
503 - for _, x := range listFlattenType(m, t) {
504 - count += s.counters[x]
505 - bytes += s.bytesCounter[x]
506 - }
507 - return
508 -}
509 -
510 -func (s *Storage) emulateError(m StorageMode, t storage.FileType) error {
511 - s.mu.Lock()
512 - defer s.mu.Unlock()
513 - err := s.emulatedError[flattenType(m, t)]
514 - if err != nil {
515 - return emulatedError{err}
516 - }
517 - return nil
518 -}
519 -
520 -func (s *Storage) EmulateError(m StorageMode, t storage.FileType, err error) {
521 - s.mu.Lock()
522 - defer s.mu.Unlock()
523 - for _, x := range listFlattenType(m, t) {
524 - s.emulatedError[x] = err
525 - }
526 -}
527 -
528 -func (s *Storage) stall(m StorageMode, t storage.FileType) {
529 - x := flattenType(m, t)
530 - s.mu.Lock()
531 - defer s.mu.Unlock()
532 - for s.stalled[x] {
533 - s.stallCond.Wait()
534 - }
535 -}
536 -
537 -func (s *Storage) Stall(m StorageMode, t storage.FileType) {
538 - s.mu.Lock()
539 - defer s.mu.Unlock()
540 - for _, x := range listFlattenType(m, t) {
541 - s.stalled[x] = true
542 - }
543 -}
544 -
545 -func (s *Storage) Release(m StorageMode, t storage.FileType) {
546 - s.mu.Lock()
547 - defer s.mu.Unlock()
548 - for _, x := range listFlattenType(m, t) {
549 - s.stalled[x] = false
550 - }
551 - s.stallCond.Broadcast()
552 -}
553 -
554 -func NewStorage() *Storage {
555 - var stor storage.Storage
556 - var closeFn func() error
557 - if storageUseFS {
558 - for {
559 - storageMu.Lock()
560 - num := storageNum
561 - storageNum++
562 - storageMu.Unlock()
563 - path := filepath.Join(os.TempDir(), fmt.Sprintf("goleveldb-test%d0%d0%d", os.Getuid(), os.Getpid(), num))
564 - if _, err := os.Stat(path); os.IsNotExist(err) {
565 - stor, err = storage.OpenFile(path)
566 - ExpectWithOffset(1, err).NotTo(HaveOccurred(), "creating storage at %s", path)
567 - closeFn = func() error {
568 - if storageKeepFS {
569 - return nil
570 - }
571 - return os.RemoveAll(path)
572 - }
573 - break
574 - }
575 - }
576 - } else {
577 - stor = storage.NewMemStorage()
578 - }
579 - s := &Storage{
580 - Storage: stor,
581 - closeFn: closeFn,
582 - opens: make(map[uint64]bool),
583 - }
584 - s.stallCond.L = &s.mu
585 - return s
586 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil/util.go deleted
-171
@@ -1,171 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package testutil
8 -
9 -import (
10 - "bytes"
11 - "flag"
12 - "math/rand"
13 - "reflect"
14 - "sync"
15 -
16 - "github.com/onsi/ginkgo/config"
17 -
18 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/comparer"
19 -)
20 -
21 -var (
22 - runfn = make(map[string][]func())
23 - runmu sync.Mutex
24 -)
25 -
26 -func Defer(args ...interface{}) bool {
27 - var (
28 - group string
29 - fn func()
30 - )
31 - for _, arg := range args {
32 - v := reflect.ValueOf(arg)
33 - switch v.Kind() {
34 - case reflect.String:
35 - group = v.String()
36 - case reflect.Func:
37 - r := reflect.ValueOf(&fn).Elem()
38 - r.Set(v)
39 - }
40 - }
41 - if fn != nil {
42 - runmu.Lock()
43 - runfn[group] = append(runfn[group], fn)
44 - runmu.Unlock()
45 - }
46 - return true
47 -}
48 -
49 -func RunDefer(groups ...string) bool {
50 - if len(groups) == 0 {
51 - groups = append(groups, "")
52 - }
53 - runmu.Lock()
54 - var runfn_ []func()
55 - for _, group := range groups {
56 - runfn_ = append(runfn_, runfn[group]...)
57 - delete(runfn, group)
58 - }
59 - runmu.Unlock()
60 - for _, fn := range runfn_ {
61 - fn()
62 - }
63 - return runfn_ != nil
64 -}
65 -
66 -func RandomSeed() int64 {
67 - if !flag.Parsed() {
68 - panic("random seed not initialized")
69 - }
70 - return config.GinkgoConfig.RandomSeed
71 -}
72 -
73 -func NewRand() *rand.Rand {
74 - return rand.New(rand.NewSource(RandomSeed()))
75 -}
76 -
77 -var cmp = comparer.DefaultComparer
78 -
79 -func BytesSeparator(a, b []byte) []byte {
80 - if bytes.Equal(a, b) {
81 - return b
82 - }
83 - i, n := 0, len(a)
84 - if n > len(b) {
85 - n = len(b)
86 - }
87 - for ; i < n && (a[i] == b[i]); i++ {
88 - }
89 - x := append([]byte{}, a[:i]...)
90 - if i < n {
91 - if c := a[i] + 1; c < b[i] {
92 - return append(x, c)
93 - }
94 - x = append(x, a[i])
95 - i++
96 - }
97 - for ; i < len(a); i++ {
98 - if c := a[i]; c < 0xff {
99 - return append(x, c+1)
100 - } else {
101 - x = append(x, c)
102 - }
103 - }
104 - if len(b) > i && b[i] > 0 {
105 - return append(x, b[i]-1)
106 - }
107 - return append(x, 'x')
108 -}
109 -
110 -func BytesAfter(b []byte) []byte {
111 - var x []byte
112 - for _, c := range b {
113 - if c < 0xff {
114 - return append(x, c+1)
115 - } else {
116 - x = append(x, c)
117 - }
118 - }
119 - return append(x, 'x')
120 -}
121 -
122 -func RandomIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
123 - if rnd == nil {
124 - rnd = NewRand()
125 - }
126 - for x := 0; x < round; x++ {
127 - fn(rnd.Intn(n))
128 - }
129 - return
130 -}
131 -
132 -func ShuffledIndex(rnd *rand.Rand, n, round int, fn func(i int)) {
133 - if rnd == nil {
134 - rnd = NewRand()
135 - }
136 - for x := 0; x < round; x++ {
137 - for _, i := range rnd.Perm(n) {
138 - fn(i)
139 - }
140 - }
141 - return
142 -}
143 -
144 -func RandomRange(rnd *rand.Rand, n, round int, fn func(start, limit int)) {
145 - if rnd == nil {
146 - rnd = NewRand()
147 - }
148 - for x := 0; x < round; x++ {
149 - start := rnd.Intn(n)
150 - length := 0
151 - if j := n - start; j > 0 {
152 - length = rnd.Intn(j)
153 - }
154 - fn(start, start+length)
155 - }
156 - return
157 -}
158 -
159 -func Max(x, y int) int {
160 - if x > y {
161 - return x
162 - }
163 - return y
164 -}
165 -
166 -func Min(x, y int) int {
167 - if x < y {
168 - return x
169 - }
170 - return y
171 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil_test.go deleted
-63
@@ -1,63 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - . "github.com/onsi/gomega"
11 -
12 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/testutil"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -type testingDB struct {
19 - *DB
20 - ro *opt.ReadOptions
21 - wo *opt.WriteOptions
22 - stor *testutil.Storage
23 -}
24 -
25 -func (t *testingDB) TestPut(key []byte, value []byte) error {
26 - return t.Put(key, value, t.wo)
27 -}
28 -
29 -func (t *testingDB) TestDelete(key []byte) error {
30 - return t.Delete(key, t.wo)
31 -}
32 -
33 -func (t *testingDB) TestGet(key []byte) (value []byte, err error) {
34 - return t.Get(key, t.ro)
35 -}
36 -
37 -func (t *testingDB) TestHas(key []byte) (ret bool, err error) {
38 - return t.Has(key, t.ro)
39 -}
40 -
41 -func (t *testingDB) TestNewIterator(slice *util.Range) iterator.Iterator {
42 - return t.NewIterator(slice, t.ro)
43 -}
44 -
45 -func (t *testingDB) TestClose() {
46 - err := t.Close()
47 - ExpectWithOffset(1, err).NotTo(HaveOccurred())
48 - err = t.stor.Close()
49 - ExpectWithOffset(1, err).NotTo(HaveOccurred())
50 -}
51 -
52 -func newTestingDB(o *opt.Options, ro *opt.ReadOptions, wo *opt.WriteOptions) *testingDB {
53 - stor := testutil.NewStorage()
54 - db, err := Open(stor, o)
55 - // FIXME: This may be called from outside It, which may cause panic.
56 - Expect(err).NotTo(HaveOccurred())
57 - return &testingDB{
58 - DB: db,
59 - ro: ro,
60 - wo: wo,
61 - stor: stor,
62 - }
63 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util.go deleted
-91
@@ -1,91 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "fmt"
11 - "sort"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/storage"
14 -)
15 -
16 -func shorten(str string) string {
17 - if len(str) <= 8 {
18 - return str
19 - }
20 - return str[:3] + ".." + str[len(str)-3:]
21 -}
22 -
23 -var bunits = [...]string{"", "Ki", "Mi", "Gi"}
24 -
25 -func shortenb(bytes int) string {
26 - i := 0
27 - for ; bytes > 1024 && i < 4; i++ {
28 - bytes /= 1024
29 - }
30 - return fmt.Sprintf("%d%sB", bytes, bunits[i])
31 -}
32 -
33 -func sshortenb(bytes int) string {
34 - if bytes == 0 {
35 - return "~"
36 - }
37 - sign := "+"
38 - if bytes < 0 {
39 - sign = "-"
40 - bytes *= -1
41 - }
42 - i := 0
43 - for ; bytes > 1024 && i < 4; i++ {
44 - bytes /= 1024
45 - }
46 - return fmt.Sprintf("%s%d%sB", sign, bytes, bunits[i])
47 -}
48 -
49 -func sint(x int) string {
50 - if x == 0 {
51 - return "~"
52 - }
53 - sign := "+"
54 - if x < 0 {
55 - sign = "-"
56 - x *= -1
57 - }
58 - return fmt.Sprintf("%s%d", sign, x)
59 -}
60 -
61 -func minInt(a, b int) int {
62 - if a < b {
63 - return a
64 - }
65 - return b
66 -}
67 -
68 -func maxInt(a, b int) int {
69 - if a > b {
70 - return a
71 - }
72 - return b
73 -}
74 -
75 -type files []storage.File
76 -
77 -func (p files) Len() int {
78 - return len(p)
79 -}
80 -
81 -func (p files) Less(i, j int) bool {
82 - return p[i].Num() < p[j].Num()
83 -}
84 -
85 -func (p files) Swap(i, j int) {
86 - p[i], p[j] = p[j], p[i]
87 -}
88 -
89 -func (p files) sort() {
90 - sort.Sort(p)
91 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer.go deleted
-293
@@ -1,293 +0,0 @@
1 -// Copyright 2009 The Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -package util
6 -
7 -// This a copy of Go std bytes.Buffer with some modification
8 -// and some features stripped.
9 -
10 -import (
11 - "bytes"
12 - "io"
13 -)
14 -
15 -// A Buffer is a variable-sized buffer of bytes with Read and Write methods.
16 -// The zero value for Buffer is an empty buffer ready to use.
17 -type Buffer struct {
18 - buf []byte // contents are the bytes buf[off : len(buf)]
19 - off int // read at &buf[off], write at &buf[len(buf)]
20 - bootstrap [64]byte // memory to hold first slice; helps small buffers (Printf) avoid allocation.
21 -}
22 -
23 -// Bytes returns a slice of the contents of the unread portion of the buffer;
24 -// len(b.Bytes()) == b.Len(). If the caller changes the contents of the
25 -// returned slice, the contents of the buffer will change provided there
26 -// are no intervening method calls on the Buffer.
27 -func (b *Buffer) Bytes() []byte { return b.buf[b.off:] }
28 -
29 -// String returns the contents of the unread portion of the buffer
30 -// as a string. If the Buffer is a nil pointer, it returns "<nil>".
31 -func (b *Buffer) String() string {
32 - if b == nil {
33 - // Special case, useful in debugging.
34 - return "<nil>"
35 - }
36 - return string(b.buf[b.off:])
37 -}
38 -
39 -// Len returns the number of bytes of the unread portion of the buffer;
40 -// b.Len() == len(b.Bytes()).
41 -func (b *Buffer) Len() int { return len(b.buf) - b.off }
42 -
43 -// Truncate discards all but the first n unread bytes from the buffer.
44 -// It panics if n is negative or greater than the length of the buffer.
45 -func (b *Buffer) Truncate(n int) {
46 - switch {
47 - case n < 0 || n > b.Len():
48 - panic("leveldb/util.Buffer: truncation out of range")
49 - case n == 0:
50 - // Reuse buffer space.
51 - b.off = 0
52 - }
53 - b.buf = b.buf[0 : b.off+n]
54 -}
55 -
56 -// Reset resets the buffer so it has no content.
57 -// b.Reset() is the same as b.Truncate(0).
58 -func (b *Buffer) Reset() { b.Truncate(0) }
59 -
60 -// grow grows the buffer to guarantee space for n more bytes.
61 -// It returns the index where bytes should be written.
62 -// If the buffer can't grow it will panic with bytes.ErrTooLarge.
63 -func (b *Buffer) grow(n int) int {
64 - m := b.Len()
65 - // If buffer is empty, reset to recover space.
66 - if m == 0 && b.off != 0 {
67 - b.Truncate(0)
68 - }
69 - if len(b.buf)+n > cap(b.buf) {
70 - var buf []byte
71 - if b.buf == nil && n <= len(b.bootstrap) {
72 - buf = b.bootstrap[0:]
73 - } else if m+n <= cap(b.buf)/2 {
74 - // We can slide things down instead of allocating a new
75 - // slice. We only need m+n <= cap(b.buf) to slide, but
76 - // we instead let capacity get twice as large so we
77 - // don't spend all our time copying.
78 - copy(b.buf[:], b.buf[b.off:])
79 - buf = b.buf[:m]
80 - } else {
81 - // not enough space anywhere
82 - buf = makeSlice(2*cap(b.buf) + n)
83 - copy(buf, b.buf[b.off:])
84 - }
85 - b.buf = buf
86 - b.off = 0
87 - }
88 - b.buf = b.buf[0 : b.off+m+n]
89 - return b.off + m
90 -}
91 -
92 -// Alloc allocs n bytes of slice from the buffer, growing the buffer as
93 -// needed. If n is negative, Alloc will panic.
94 -// If the buffer can't grow it will panic with bytes.ErrTooLarge.
95 -func (b *Buffer) Alloc(n int) []byte {
96 - if n < 0 {
97 - panic("leveldb/util.Buffer.Alloc: negative count")
98 - }
99 - m := b.grow(n)
100 - return b.buf[m:]
101 -}
102 -
103 -// Grow grows the buffer's capacity, if necessary, to guarantee space for
104 -// another n bytes. After Grow(n), at least n bytes can be written to the
105 -// buffer without another allocation.
106 -// If n is negative, Grow will panic.
107 -// If the buffer can't grow it will panic with bytes.ErrTooLarge.
108 -func (b *Buffer) Grow(n int) {
109 - if n < 0 {
110 - panic("leveldb/util.Buffer.Grow: negative count")
111 - }
112 - m := b.grow(n)
113 - b.buf = b.buf[0:m]
114 -}
115 -
116 -// Write appends the contents of p to the buffer, growing the buffer as
117 -// needed. The return value n is the length of p; err is always nil. If the
118 -// buffer becomes too large, Write will panic with bytes.ErrTooLarge.
119 -func (b *Buffer) Write(p []byte) (n int, err error) {
120 - m := b.grow(len(p))
121 - return copy(b.buf[m:], p), nil
122 -}
123 -
124 -// MinRead is the minimum slice size passed to a Read call by
125 -// Buffer.ReadFrom. As long as the Buffer has at least MinRead bytes beyond
126 -// what is required to hold the contents of r, ReadFrom will not grow the
127 -// underlying buffer.
128 -const MinRead = 512
129 -
130 -// ReadFrom reads data from r until EOF and appends it to the buffer, growing
131 -// the buffer as needed. The return value n is the number of bytes read. Any
132 -// error except io.EOF encountered during the read is also returned. If the
133 -// buffer becomes too large, ReadFrom will panic with bytes.ErrTooLarge.
134 -func (b *Buffer) ReadFrom(r io.Reader) (n int64, err error) {
135 - // If buffer is empty, reset to recover space.
136 - if b.off >= len(b.buf) {
137 - b.Truncate(0)
138 - }
139 - for {
140 - if free := cap(b.buf) - len(b.buf); free < MinRead {
141 - // not enough space at end
142 - newBuf := b.buf
143 - if b.off+free < MinRead {
144 - // not enough space using beginning of buffer;
145 - // double buffer capacity
146 - newBuf = makeSlice(2*cap(b.buf) + MinRead)
147 - }
148 - copy(newBuf, b.buf[b.off:])
149 - b.buf = newBuf[:len(b.buf)-b.off]
150 - b.off = 0
151 - }
152 - m, e := r.Read(b.buf[len(b.buf):cap(b.buf)])
153 - b.buf = b.buf[0 : len(b.buf)+m]
154 - n += int64(m)
155 - if e == io.EOF {
156 - break
157 - }
158 - if e != nil {
159 - return n, e
160 - }
161 - }
162 - return n, nil // err is EOF, so return nil explicitly
163 -}
164 -
165 -// makeSlice allocates a slice of size n. If the allocation fails, it panics
166 -// with bytes.ErrTooLarge.
167 -func makeSlice(n int) []byte {
168 - // If the make fails, give a known error.
169 - defer func() {
170 - if recover() != nil {
171 - panic(bytes.ErrTooLarge)
172 - }
173 - }()
174 - return make([]byte, n)
175 -}
176 -
177 -// WriteTo writes data to w until the buffer is drained or an error occurs.
178 -// The return value n is the number of bytes written; it always fits into an
179 -// int, but it is int64 to match the io.WriterTo interface. Any error
180 -// encountered during the write is also returned.
181 -func (b *Buffer) WriteTo(w io.Writer) (n int64, err error) {
182 - if b.off < len(b.buf) {
183 - nBytes := b.Len()
184 - m, e := w.Write(b.buf[b.off:])
185 - if m > nBytes {
186 - panic("leveldb/util.Buffer.WriteTo: invalid Write count")
187 - }
188 - b.off += m
189 - n = int64(m)
190 - if e != nil {
191 - return n, e
192 - }
193 - // all bytes should have been written, by definition of
194 - // Write method in io.Writer
195 - if m != nBytes {
196 - return n, io.ErrShortWrite
197 - }
198 - }
199 - // Buffer is now empty; reset.
200 - b.Truncate(0)
201 - return
202 -}
203 -
204 -// WriteByte appends the byte c to the buffer, growing the buffer as needed.
205 -// The returned error is always nil, but is included to match bufio.Writer's
206 -// WriteByte. If the buffer becomes too large, WriteByte will panic with
207 -// bytes.ErrTooLarge.
208 -func (b *Buffer) WriteByte(c byte) error {
209 - m := b.grow(1)
210 - b.buf[m] = c
211 - return nil
212 -}
213 -
214 -// Read reads the next len(p) bytes from the buffer or until the buffer
215 -// is drained. The return value n is the number of bytes read. If the
216 -// buffer has no data to return, err is io.EOF (unless len(p) is zero);
217 -// otherwise it is nil.
218 -func (b *Buffer) Read(p []byte) (n int, err error) {
219 - if b.off >= len(b.buf) {
220 - // Buffer is empty, reset to recover space.
221 - b.Truncate(0)
222 - if len(p) == 0 {
223 - return
224 - }
225 - return 0, io.EOF
226 - }
227 - n = copy(p, b.buf[b.off:])
228 - b.off += n
229 - return
230 -}
231 -
232 -// Next returns a slice containing the next n bytes from the buffer,
233 -// advancing the buffer as if the bytes had been returned by Read.
234 -// If there are fewer than n bytes in the buffer, Next returns the entire buffer.
235 -// The slice is only valid until the next call to a read or write method.
236 -func (b *Buffer) Next(n int) []byte {
237 - m := b.Len()
238 - if n > m {
239 - n = m
240 - }
241 - data := b.buf[b.off : b.off+n]
242 - b.off += n
243 - return data
244 -}
245 -
246 -// ReadByte reads and returns the next byte from the buffer.
247 -// If no byte is available, it returns error io.EOF.
248 -func (b *Buffer) ReadByte() (c byte, err error) {
249 - if b.off >= len(b.buf) {
250 - // Buffer is empty, reset to recover space.
251 - b.Truncate(0)
252 - return 0, io.EOF
253 - }
254 - c = b.buf[b.off]
255 - b.off++
256 - return c, nil
257 -}
258 -
259 -// ReadBytes reads until the first occurrence of delim in the input,
260 -// returning a slice containing the data up to and including the delimiter.
261 -// If ReadBytes encounters an error before finding a delimiter,
262 -// it returns the data read before the error and the error itself (often io.EOF).
263 -// ReadBytes returns err != nil if and only if the returned data does not end in
264 -// delim.
265 -func (b *Buffer) ReadBytes(delim byte) (line []byte, err error) {
266 - slice, err := b.readSlice(delim)
267 - // return a copy of slice. The buffer's backing array may
268 - // be overwritten by later calls.
269 - line = append(line, slice...)
270 - return
271 -}
272 -
273 -// readSlice is like ReadBytes but returns a reference to internal buffer data.
274 -func (b *Buffer) readSlice(delim byte) (line []byte, err error) {
275 - i := bytes.IndexByte(b.buf[b.off:], delim)
276 - end := b.off + i + 1
277 - if i < 0 {
278 - end = len(b.buf)
279 - err = io.EOF
280 - }
281 - line = b.buf[b.off:end]
282 - b.off = end
283 - return line, err
284 -}
285 -
286 -// NewBuffer creates and initializes a new Buffer using buf as its initial
287 -// contents. It is intended to prepare a Buffer to read existing data. It
288 -// can also be used to size the internal buffer for writing. To do that,
289 -// buf should have the desired capacity but a length of zero.
290 -//
291 -// In most cases, new(Buffer) (or just declaring a Buffer variable) is
292 -// sufficient to initialize a Buffer.
293 -func NewBuffer(buf []byte) *Buffer { return &Buffer{buf: buf} }
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go deleted
-238
@@ -1,238 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package util
8 -
9 -import (
10 - "fmt"
11 - "sync"
12 - "sync/atomic"
13 - "time"
14 -)
15 -
16 -type buffer struct {
17 - b []byte
18 - miss int
19 -}
20 -
21 -// BufferPool is a 'buffer pool'.
22 -type BufferPool struct {
23 - pool [6]chan []byte
24 - size [5]uint32
25 - sizeMiss [5]uint32
26 - sizeHalf [5]uint32
27 - baseline [4]int
28 - baseline0 int
29 -
30 - mu sync.RWMutex
31 - closed bool
32 - closeC chan struct{}
33 -
34 - get uint32
35 - put uint32
36 - half uint32
37 - less uint32
38 - equal uint32
39 - greater uint32
40 - miss uint32
41 -}
42 -
43 -func (p *BufferPool) poolNum(n int) int {
44 - if n <= p.baseline0 && n > p.baseline0/2 {
45 - return 0
46 - }
47 - for i, x := range p.baseline {
48 - if n <= x {
49 - return i + 1
50 - }
51 - }
52 - return len(p.baseline) + 1
53 -}
54 -
55 -// Get returns buffer with length of n.
56 -func (p *BufferPool) Get(n int) []byte {
57 - if p == nil {
58 - return make([]byte, n)
59 - }
60 -
61 - p.mu.RLock()
62 - defer p.mu.RUnlock()
63 -
64 - if p.closed {
65 - return make([]byte, n)
66 - }
67 -
68 - atomic.AddUint32(&p.get, 1)
69 -
70 - poolNum := p.poolNum(n)
71 - pool := p.pool[poolNum]
72 - if poolNum == 0 {
73 - // Fast path.
74 - select {
75 - case b := <-pool:
76 - switch {
77 - case cap(b) > n:
78 - if cap(b)-n >= n {
79 - atomic.AddUint32(&p.half, 1)
80 - select {
81 - case pool <- b:
82 - default:
83 - }
84 - return make([]byte, n)
85 - } else {
86 - atomic.AddUint32(&p.less, 1)
87 - return b[:n]
88 - }
89 - case cap(b) == n:
90 - atomic.AddUint32(&p.equal, 1)
91 - return b[:n]
92 - default:
93 - atomic.AddUint32(&p.greater, 1)
94 - }
95 - default:
96 - atomic.AddUint32(&p.miss, 1)
97 - }
98 -
99 - return make([]byte, n, p.baseline0)
100 - } else {
101 - sizePtr := &p.size[poolNum-1]
102 -
103 - select {
104 - case b := <-pool:
105 - switch {
106 - case cap(b) > n:
107 - if cap(b)-n >= n {
108 - atomic.AddUint32(&p.half, 1)
109 - sizeHalfPtr := &p.sizeHalf[poolNum-1]
110 - if atomic.AddUint32(sizeHalfPtr, 1) == 20 {
111 - atomic.StoreUint32(sizePtr, uint32(cap(b)/2))
112 - atomic.StoreUint32(sizeHalfPtr, 0)
113 - } else {
114 - select {
115 - case pool <- b:
116 - default:
117 - }
118 - }
119 - return make([]byte, n)
120 - } else {
121 - atomic.AddUint32(&p.less, 1)
122 - return b[:n]
123 - }
124 - case cap(b) == n:
125 - atomic.AddUint32(&p.equal, 1)
126 - return b[:n]
127 - default:
128 - atomic.AddUint32(&p.greater, 1)
129 - if uint32(cap(b)) >= atomic.LoadUint32(sizePtr) {
130 - select {
131 - case pool <- b:
132 - default:
133 - }
134 - }
135 - }
136 - default:
137 - atomic.AddUint32(&p.miss, 1)
138 - }
139 -
140 - if size := atomic.LoadUint32(sizePtr); uint32(n) > size {
141 - if size == 0 {
142 - atomic.CompareAndSwapUint32(sizePtr, 0, uint32(n))
143 - } else {
144 - sizeMissPtr := &p.sizeMiss[poolNum-1]
145 - if atomic.AddUint32(sizeMissPtr, 1) == 20 {
146 - atomic.StoreUint32(sizePtr, uint32(n))
147 - atomic.StoreUint32(sizeMissPtr, 0)
148 - }
149 - }
150 - return make([]byte, n)
151 - } else {
152 - return make([]byte, n, size)
153 - }
154 - }
155 -}
156 -
157 -// Put adds given buffer to the pool.
158 -func (p *BufferPool) Put(b []byte) {
159 - if p == nil {
160 - return
161 - }
162 -
163 - p.mu.RLock()
164 - defer p.mu.RUnlock()
165 -
166 - if p.closed {
167 - return
168 - }
169 -
170 - atomic.AddUint32(&p.put, 1)
171 -
172 - pool := p.pool[p.poolNum(cap(b))]
173 - select {
174 - case pool <- b:
175 - default:
176 - }
177 -
178 -}
179 -
180 -func (p *BufferPool) Close() {
181 - if p == nil {
182 - return
183 - }
184 -
185 - p.mu.Lock()
186 - if !p.closed {
187 - p.closed = true
188 - p.closeC <- struct{}{}
189 - }
190 - p.mu.Unlock()
191 -}
192 -
193 -func (p *BufferPool) String() string {
194 - if p == nil {
195 - return "<nil>"
196 - }
197 -
198 - return fmt.Sprintf("BufferPool{B·%d Z·%v Zm·%v Zh·%v G·%d P·%d H·%d <·%d =·%d >·%d M·%d}",
199 - p.baseline0, p.size, p.sizeMiss, p.sizeHalf, p.get, p.put, p.half, p.less, p.equal, p.greater, p.miss)
200 -}
201 -
202 -func (p *BufferPool) drain() {
203 - ticker := time.NewTicker(2 * time.Second)
204 - for {
205 - select {
206 - case <-ticker.C:
207 - for _, ch := range p.pool {
208 - select {
209 - case <-ch:
210 - default:
211 - }
212 - }
213 - case <-p.closeC:
214 - close(p.closeC)
215 - for _, ch := range p.pool {
216 - close(ch)
217 - }
218 - return
219 - }
220 - }
221 -}
222 -
223 -// NewBufferPool creates a new initialized 'buffer pool'.
224 -func NewBufferPool(baseline int) *BufferPool {
225 - if baseline <= 0 {
226 - panic("baseline can't be <= 0")
227 - }
228 - p := &BufferPool{
229 - baseline0: baseline,
230 - baseline: [...]int{baseline / 4, baseline / 2, baseline * 2, baseline * 4},
231 - closeC: make(chan struct{}, 1),
232 - }
233 - for i, cap := range []int{2, 2, 4, 4, 2, 1} {
234 - p.pool[i] = make(chan []byte, cap)
235 - }
236 - go p.drain()
237 - return p
238 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/buffer_test.go deleted
-369
@@ -1,369 +0,0 @@
1 -// Copyright 2009 The Go Authors. All rights reserved.
2 -// Use of this source code is governed by a BSD-style
3 -// license that can be found in the LICENSE file.
4 -
5 -package util
6 -
7 -import (
8 - "bytes"
9 - "io"
10 - "math/rand"
11 - "runtime"
12 - "testing"
13 -)
14 -
15 -const N = 10000 // make this bigger for a larger (and slower) test
16 -var data string // test data for write tests
17 -var testBytes []byte // test data; same as data but as a slice.
18 -
19 -func init() {
20 - testBytes = make([]byte, N)
21 - for i := 0; i < N; i++ {
22 - testBytes[i] = 'a' + byte(i%26)
23 - }
24 - data = string(testBytes)
25 -}
26 -
27 -// Verify that contents of buf match the string s.
28 -func check(t *testing.T, testname string, buf *Buffer, s string) {
29 - bytes := buf.Bytes()
30 - str := buf.String()
31 - if buf.Len() != len(bytes) {
32 - t.Errorf("%s: buf.Len() == %d, len(buf.Bytes()) == %d", testname, buf.Len(), len(bytes))
33 - }
34 -
35 - if buf.Len() != len(str) {
36 - t.Errorf("%s: buf.Len() == %d, len(buf.String()) == %d", testname, buf.Len(), len(str))
37 - }
38 -
39 - if buf.Len() != len(s) {
40 - t.Errorf("%s: buf.Len() == %d, len(s) == %d", testname, buf.Len(), len(s))
41 - }
42 -
43 - if string(bytes) != s {
44 - t.Errorf("%s: string(buf.Bytes()) == %q, s == %q", testname, string(bytes), s)
45 - }
46 -}
47 -
48 -// Fill buf through n writes of byte slice fub.
49 -// The initial contents of buf corresponds to the string s;
50 -// the result is the final contents of buf returned as a string.
51 -func fillBytes(t *testing.T, testname string, buf *Buffer, s string, n int, fub []byte) string {
52 - check(t, testname+" (fill 1)", buf, s)
53 - for ; n > 0; n-- {
54 - m, err := buf.Write(fub)
55 - if m != len(fub) {
56 - t.Errorf(testname+" (fill 2): m == %d, expected %d", m, len(fub))
57 - }
58 - if err != nil {
59 - t.Errorf(testname+" (fill 3): err should always be nil, found err == %s", err)
60 - }
61 - s += string(fub)
62 - check(t, testname+" (fill 4)", buf, s)
63 - }
64 - return s
65 -}
66 -
67 -func TestNewBuffer(t *testing.T) {
68 - buf := NewBuffer(testBytes)
69 - check(t, "NewBuffer", buf, data)
70 -}
71 -
72 -// Empty buf through repeated reads into fub.
73 -// The initial contents of buf corresponds to the string s.
74 -func empty(t *testing.T, testname string, buf *Buffer, s string, fub []byte) {
75 - check(t, testname+" (empty 1)", buf, s)
76 -
77 - for {
78 - n, err := buf.Read(fub)
79 - if n == 0 {
80 - break
81 - }
82 - if err != nil {
83 - t.Errorf(testname+" (empty 2): err should always be nil, found err == %s", err)
84 - }
85 - s = s[n:]
86 - check(t, testname+" (empty 3)", buf, s)
87 - }
88 -
89 - check(t, testname+" (empty 4)", buf, "")
90 -}
91 -
92 -func TestBasicOperations(t *testing.T) {
93 - var buf Buffer
94 -
95 - for i := 0; i < 5; i++ {
96 - check(t, "TestBasicOperations (1)", &buf, "")
97 -
98 - buf.Reset()
99 - check(t, "TestBasicOperations (2)", &buf, "")
100 -
101 - buf.Truncate(0)
102 - check(t, "TestBasicOperations (3)", &buf, "")
103 -
104 - n, err := buf.Write([]byte(data[0:1]))
105 - if n != 1 {
106 - t.Errorf("wrote 1 byte, but n == %d", n)
107 - }
108 - if err != nil {
109 - t.Errorf("err should always be nil, but err == %s", err)
110 - }
111 - check(t, "TestBasicOperations (4)", &buf, "a")
112 -
113 - buf.WriteByte(data[1])
114 - check(t, "TestBasicOperations (5)", &buf, "ab")
115 -
116 - n, err = buf.Write([]byte(data[2:26]))
117 - if n != 24 {
118 - t.Errorf("wrote 25 bytes, but n == %d", n)
119 - }
120 - check(t, "TestBasicOperations (6)", &buf, string(data[0:26]))
121 -
122 - buf.Truncate(26)
123 - check(t, "TestBasicOperations (7)", &buf, string(data[0:26]))
124 -
125 - buf.Truncate(20)
126 - check(t, "TestBasicOperations (8)", &buf, string(data[0:20]))
127 -
128 - empty(t, "TestBasicOperations (9)", &buf, string(data[0:20]), make([]byte, 5))
129 - empty(t, "TestBasicOperations (10)", &buf, "", make([]byte, 100))
130 -
131 - buf.WriteByte(data[1])
132 - c, err := buf.ReadByte()
133 - if err != nil {
134 - t.Error("ReadByte unexpected eof")
135 - }
136 - if c != data[1] {
137 - t.Errorf("ReadByte wrong value c=%v", c)
138 - }
139 - c, err = buf.ReadByte()
140 - if err == nil {
141 - t.Error("ReadByte unexpected not eof")
142 - }
143 - }
144 -}
145 -
146 -func TestLargeByteWrites(t *testing.T) {
147 - var buf Buffer
148 - limit := 30
149 - if testing.Short() {
150 - limit = 9
151 - }
152 - for i := 3; i < limit; i += 3 {
153 - s := fillBytes(t, "TestLargeWrites (1)", &buf, "", 5, testBytes)
154 - empty(t, "TestLargeByteWrites (2)", &buf, s, make([]byte, len(data)/i))
155 - }
156 - check(t, "TestLargeByteWrites (3)", &buf, "")
157 -}
158 -
159 -func TestLargeByteReads(t *testing.T) {
160 - var buf Buffer
161 - for i := 3; i < 30; i += 3 {
162 - s := fillBytes(t, "TestLargeReads (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
163 - empty(t, "TestLargeReads (2)", &buf, s, make([]byte, len(data)))
164 - }
165 - check(t, "TestLargeByteReads (3)", &buf, "")
166 -}
167 -
168 -func TestMixedReadsAndWrites(t *testing.T) {
169 - var buf Buffer
170 - s := ""
171 - for i := 0; i < 50; i++ {
172 - wlen := rand.Intn(len(data))
173 - s = fillBytes(t, "TestMixedReadsAndWrites (1)", &buf, s, 1, testBytes[0:wlen])
174 - rlen := rand.Intn(len(data))
175 - fub := make([]byte, rlen)
176 - n, _ := buf.Read(fub)
177 - s = s[n:]
178 - }
179 - empty(t, "TestMixedReadsAndWrites (2)", &buf, s, make([]byte, buf.Len()))
180 -}
181 -
182 -func TestNil(t *testing.T) {
183 - var b *Buffer
184 - if b.String() != "<nil>" {
185 - t.Errorf("expected <nil>; got %q", b.String())
186 - }
187 -}
188 -
189 -func TestReadFrom(t *testing.T) {
190 - var buf Buffer
191 - for i := 3; i < 30; i += 3 {
192 - s := fillBytes(t, "TestReadFrom (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
193 - var b Buffer
194 - b.ReadFrom(&buf)
195 - empty(t, "TestReadFrom (2)", &b, s, make([]byte, len(data)))
196 - }
197 -}
198 -
199 -func TestWriteTo(t *testing.T) {
200 - var buf Buffer
201 - for i := 3; i < 30; i += 3 {
202 - s := fillBytes(t, "TestWriteTo (1)", &buf, "", 5, testBytes[0:len(testBytes)/i])
203 - var b Buffer
204 - buf.WriteTo(&b)
205 - empty(t, "TestWriteTo (2)", &b, s, make([]byte, len(data)))
206 - }
207 -}
208 -
209 -func TestNext(t *testing.T) {
210 - b := []byte{0, 1, 2, 3, 4}
211 - tmp := make([]byte, 5)
212 - for i := 0; i <= 5; i++ {
213 - for j := i; j <= 5; j++ {
214 - for k := 0; k <= 6; k++ {
215 - // 0 <= i <= j <= 5; 0 <= k <= 6
216 - // Check that if we start with a buffer
217 - // of length j at offset i and ask for
218 - // Next(k), we get the right bytes.
219 - buf := NewBuffer(b[0:j])
220 - n, _ := buf.Read(tmp[0:i])
221 - if n != i {
222 - t.Fatalf("Read %d returned %d", i, n)
223 - }
224 - bb := buf.Next(k)
225 - want := k
226 - if want > j-i {
227 - want = j - i
228 - }
229 - if len(bb) != want {
230 - t.Fatalf("in %d,%d: len(Next(%d)) == %d", i, j, k, len(bb))
231 - }
232 - for l, v := range bb {
233 - if v != byte(l+i) {
234 - t.Fatalf("in %d,%d: Next(%d)[%d] = %d, want %d", i, j, k, l, v, l+i)
235 - }
236 - }
237 - }
238 - }
239 - }
240 -}
241 -
242 -var readBytesTests = []struct {
243 - buffer string
244 - delim byte
245 - expected []string
246 - err error
247 -}{
248 - {"", 0, []string{""}, io.EOF},
249 - {"a\x00", 0, []string{"a\x00"}, nil},
250 - {"abbbaaaba", 'b', []string{"ab", "b", "b", "aaab"}, nil},
251 - {"hello\x01world", 1, []string{"hello\x01"}, nil},
252 - {"foo\nbar", 0, []string{"foo\nbar"}, io.EOF},
253 - {"alpha\nbeta\ngamma\n", '\n', []string{"alpha\n", "beta\n", "gamma\n"}, nil},
254 - {"alpha\nbeta\ngamma", '\n', []string{"alpha\n", "beta\n", "gamma"}, io.EOF},
255 -}
256 -
257 -func TestReadBytes(t *testing.T) {
258 - for _, test := range readBytesTests {
259 - buf := NewBuffer([]byte(test.buffer))
260 - var err error
261 - for _, expected := range test.expected {
262 - var bytes []byte
263 - bytes, err = buf.ReadBytes(test.delim)
264 - if string(bytes) != expected {
265 - t.Errorf("expected %q, got %q", expected, bytes)
266 - }
267 - if err != nil {
268 - break
269 - }
270 - }
271 - if err != test.err {
272 - t.Errorf("expected error %v, got %v", test.err, err)
273 - }
274 - }
275 -}
276 -
277 -func TestGrow(t *testing.T) {
278 - x := []byte{'x'}
279 - y := []byte{'y'}
280 - tmp := make([]byte, 72)
281 - for _, startLen := range []int{0, 100, 1000, 10000, 100000} {
282 - xBytes := bytes.Repeat(x, startLen)
283 - for _, growLen := range []int{0, 100, 1000, 10000, 100000} {
284 - buf := NewBuffer(xBytes)
285 - // If we read, this affects buf.off, which is good to test.
286 - readBytes, _ := buf.Read(tmp)
287 - buf.Grow(growLen)
288 - yBytes := bytes.Repeat(y, growLen)
289 - // Check no allocation occurs in write, as long as we're single-threaded.
290 - var m1, m2 runtime.MemStats
291 - runtime.ReadMemStats(&m1)
292 - buf.Write(yBytes)
293 - runtime.ReadMemStats(&m2)
294 - if runtime.GOMAXPROCS(-1) == 1 && m1.Mallocs != m2.Mallocs {
295 - t.Errorf("allocation occurred during write")
296 - }
297 - // Check that buffer has correct data.
298 - if !bytes.Equal(buf.Bytes()[0:startLen-readBytes], xBytes[readBytes:]) {
299 - t.Errorf("bad initial data at %d %d", startLen, growLen)
300 - }
301 - if !bytes.Equal(buf.Bytes()[startLen-readBytes:startLen-readBytes+growLen], yBytes) {
302 - t.Errorf("bad written data at %d %d", startLen, growLen)
303 - }
304 - }
305 - }
306 -}
307 -
308 -// Was a bug: used to give EOF reading empty slice at EOF.
309 -func TestReadEmptyAtEOF(t *testing.T) {
310 - b := new(Buffer)
311 - slice := make([]byte, 0)
312 - n, err := b.Read(slice)
313 - if err != nil {
314 - t.Errorf("read error: %v", err)
315 - }
316 - if n != 0 {
317 - t.Errorf("wrong count; got %d want 0", n)
318 - }
319 -}
320 -
321 -// Tests that we occasionally compact. Issue 5154.
322 -func TestBufferGrowth(t *testing.T) {
323 - var b Buffer
324 - buf := make([]byte, 1024)
325 - b.Write(buf[0:1])
326 - var cap0 int
327 - for i := 0; i < 5<<10; i++ {
328 - b.Write(buf)
329 - b.Read(buf)
330 - if i == 0 {
331 - cap0 = cap(b.buf)
332 - }
333 - }
334 - cap1 := cap(b.buf)
335 - // (*Buffer).grow allows for 2x capacity slop before sliding,
336 - // so set our error threshold at 3x.
337 - if cap1 > cap0*3 {
338 - t.Errorf("buffer cap = %d; too big (grew from %d)", cap1, cap0)
339 - }
340 -}
341 -
342 -// From Issue 5154.
343 -func BenchmarkBufferNotEmptyWriteRead(b *testing.B) {
344 - buf := make([]byte, 1024)
345 - for i := 0; i < b.N; i++ {
346 - var b Buffer
347 - b.Write(buf[0:1])
348 - for i := 0; i < 5<<10; i++ {
349 - b.Write(buf)
350 - b.Read(buf)
351 - }
352 - }
353 -}
354 -
355 -// Check that we don't compact too often. From Issue 5154.
356 -func BenchmarkBufferFullSmallReads(b *testing.B) {
357 - buf := make([]byte, 1024)
358 - for i := 0; i < b.N; i++ {
359 - var b Buffer
360 - b.Write(buf)
361 - for b.Len()+20 < cap(b.buf) {
362 - b.Write(buf[:10])
363 - }
364 - for i := 0; i < 5<<10; i++ {
365 - b.Read(buf[:1])
366 - b.Write(buf[:1])
367 - }
368 - }
369 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/crc32.go deleted
-30
@@ -1,30 +0,0 @@
1 -// Copyright 2011 The LevelDB-Go Authors. All rights reserved.
2 -//
3 -// Use of this source code is governed by a BSD-style license that can be
4 -// found in the LICENSE file.
5 -
6 -package util
7 -
8 -import (
9 - "hash/crc32"
10 -)
11 -
12 -var table = crc32.MakeTable(crc32.Castagnoli)
13 -
14 -// CRC is a CRC-32 checksum computed using Castagnoli's polynomial.
15 -type CRC uint32
16 -
17 -// NewCRC creates a new crc based on the given bytes.
18 -func NewCRC(b []byte) CRC {
19 - return CRC(0).Update(b)
20 -}
21 -
22 -// Update updates the crc with the given bytes.
23 -func (c CRC) Update(b []byte) CRC {
24 - return CRC(crc32.Update(uint32(c), table, b))
25 -}
26 -
27 -// Value returns a masked crc.
28 -func (c CRC) Value() uint32 {
29 - return uint32(c>>15|c<<17) + 0xa282ead8
30 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/hash.go deleted
-48
@@ -1,48 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package util
8 -
9 -import (
10 - "bytes"
11 - "encoding/binary"
12 -)
13 -
14 -// Hash return hash of the given data.
15 -func Hash(data []byte, seed uint32) uint32 {
16 - // Similar to murmur hash
17 - var m uint32 = 0xc6a4a793
18 - var r uint32 = 24
19 - h := seed ^ (uint32(len(data)) * m)
20 -
21 - buf := bytes.NewBuffer(data)
22 - for buf.Len() >= 4 {
23 - var w uint32
24 - binary.Read(buf, binary.LittleEndian, &w)
25 - h += w
26 - h *= m
27 - h ^= (h >> 16)
28 - }
29 -
30 - rest := buf.Bytes()
31 - switch len(rest) {
32 - default:
33 - panic("not reached")
34 - case 3:
35 - h += uint32(rest[2]) << 16
36 - fallthrough
37 - case 2:
38 - h += uint32(rest[1]) << 8
39 - fallthrough
40 - case 1:
41 - h += uint32(rest[0])
42 - h *= m
43 - h ^= (h >> r)
44 - case 0:
45 - }
46 -
47 - return h
48 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool.go deleted
-21
@@ -1,21 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build go1.3
8 -
9 -package util
10 -
11 -import (
12 - "sync"
13 -)
14 -
15 -type Pool struct {
16 - sync.Pool
17 -}
18 -
19 -func NewPool(cap int) *Pool {
20 - return &Pool{}
21 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/pool_legacy.go deleted
-33
@@ -1,33 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// +build !go1.3
8 -
9 -package util
10 -
11 -type Pool struct {
12 - pool chan interface{}
13 -}
14 -
15 -func (p *Pool) Get() interface{} {
16 - select {
17 - case x := <-p.pool:
18 - return x
19 - default:
20 - return nil
21 - }
22 -}
23 -
24 -func (p *Pool) Put(x interface{}) {
25 - select {
26 - case p.pool <- x:
27 - default:
28 - }
29 -}
30 -
31 -func NewPool(cap int) *Pool {
32 - return &Pool{pool: make(chan interface{}, cap)}
33 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/range.go deleted
-32
@@ -1,32 +0,0 @@
1 -// Copyright (c) 2014, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package util
8 -
9 -// Range is a key range.
10 -type Range struct {
11 - // Start of the key range, include in the range.
12 - Start []byte
13 -
14 - // Limit of the key range, not include in the range.
15 - Limit []byte
16 -}
17 -
18 -// BytesPrefix returns key range that satisfy the given prefix.
19 -// This only applicable for the standard 'bytes comparer'.
20 -func BytesPrefix(prefix []byte) *Range {
21 - var limit []byte
22 - for i := len(prefix) - 1; i >= 0; i-- {
23 - c := prefix[i]
24 - if c < 0xff {
25 - limit = make([]byte, i+1)
26 - copy(limit, prefix)
27 - limit[i] = c + 1
28 - break
29 - }
30 - }
31 - return &Range{prefix, limit}
32 -}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util/util.go deleted
-73
@@ -1,73 +0,0 @@
1 -// Copyright (c) 2013, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -// Package util provides utilities used throughout leveldb.
8 -package util
9 -
10 -import (
11 - "errors"
12 -)
13 -
14 -var (
15 - ErrReleased = errors.New("leveldb: resource already relesed")
16 - ErrHasReleaser = errors.New("leveldb: releaser already defined")
17 -)
18 -
19 -// Releaser is the interface that wraps the basic Release method.
20 -type Releaser interface {
21 - // Release releases associated resources. Release should always success
22 - // and can be called multipe times without causing error.
23 - Release()
24 -}
25 -
26 -// ReleaseSetter is the interface that wraps the basic SetReleaser method.
27 -type ReleaseSetter interface {
28 - // SetReleaser associates the given releaser to the resources. The
29 - // releaser will be called once coresponding resources released.
30 - // Calling SetReleaser with nil will clear the releaser.
31 - //
32 - // This will panic if a releaser already present or coresponding
33 - // resource is already released. Releaser should be cleared first
34 - // before assigned a new one.
35 - SetReleaser(releaser Releaser)
36 -}
37 -
38 -// BasicReleaser provides basic implementation of Releaser and ReleaseSetter.
39 -type BasicReleaser struct {
40 - releaser Releaser
41 - released bool
42 -}
43 -
44 -// Released returns whether Release method already called.
45 -func (r *BasicReleaser) Released() bool {
46 - return r.released
47 -}
48 -
49 -// Release implements Releaser.Release.
50 -func (r *BasicReleaser) Release() {
51 - if !r.released {
52 - if r.releaser != nil {
53 - r.releaser.Release()
54 - r.releaser = nil
55 - }
56 - r.released = true
57 - }
58 -}
59 -
60 -// SetReleaser implements ReleaseSetter.SetReleaser.
61 -func (r *BasicReleaser) SetReleaser(releaser Releaser) {
62 - if r.released {
63 - panic(ErrReleased)
64 - }
65 - if r.releaser != nil && releaser != nil {
66 - panic(ErrHasReleaser)
67 - }
68 - r.releaser = releaser
69 -}
70 -
71 -type NoopReleaser struct{}
72 -
73 -func (NoopReleaser) Release() {}
Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/version.go deleted
-457
@@ -1,457 +0,0 @@
1 -// Copyright (c) 2012, Suryandaru Triandana <syndtr@gmail.com>
2 -// All rights reserved.
3 -//
4 -// Use of this source code is governed by a BSD-style license that can be
5 -// found in the LICENSE file.
6 -
7 -package leveldb
8 -
9 -import (
10 - "sync/atomic"
11 - "unsafe"
12 -
13 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/iterator"
14 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
15 - "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
16 -)
17 -
18 -type tSet struct {
19 - level int
20 - table *tFile
21 -}
22 -
23 -type version struct {
24 - s *session
25 -
26 - tables []tFiles
27 -
28 - // Level that should be compacted next and its compaction score.
29 - // Score < 1 means compaction is not strictly needed. These fields
30 - // are initialized by computeCompaction()
31 - cLevel int
32 - cScore float64
33 -
34 - cSeek unsafe.Pointer
35 -
36 - ref int
37 - // Succeeding version.
38 - next *version
39 -}
40 -
41 -func newVersion(s *session) *version {
42 - return &version{s: s, tables: make([]tFiles, s.o.GetNumLevel())}
43 -}
44 -
45 -func (v *version) releaseNB() {
46 - v.ref--
47 - if v.ref > 0 {
48 - return
49 - }
50 - if v.ref < 0 {
51 - panic("negative version ref")
52 - }
53 -
54 - tables := make(map[uint64]bool)
55 - for _, tt := range v.next.tables {
56 - for _, t := range tt {
57 - num := t.file.Num()
58 - tables[num] = true
59 - }
60 - }
61 -
62 - for _, tt := range v.tables {
63 - for _, t := range tt {
64 - num := t.file.Num()
65 - if _, ok := tables[num]; !ok {
66 - v.s.tops.remove(t)
67 - }
68 - }
69 - }
70 -
71 - v.next.releaseNB()
72 - v.next = nil
73 -}
74 -
75 -func (v *version) release() {
76 - v.s.vmu.Lock()
77 - v.releaseNB()
78 - v.s.vmu.Unlock()
79 -}
80 -
81 -func (v *version) walkOverlapping(ikey iKey, f func(level int, t *tFile) bool, lf func(level int) bool) {
82 - ukey := ikey.ukey()
83 -
84 - // Walk tables level-by-level.
85 - for level, tables := range v.tables {
86 - if len(tables) == 0 {
87 - continue
88 - }
89 -
90 - if level == 0 {
91 - // Level-0 files may overlap each other. Find all files that
92 - // overlap ukey.
93 - for _, t := range tables {
94 - if t.overlaps(v.s.icmp, ukey, ukey) {
95 - if !f(level, t) {
96 - return
97 - }
98 - }
99 - }
100 - } else {
101 - if i := tables.searchMax(v.s.icmp, ikey); i < len(tables) {
102 - t := tables[i]
103 - if v.s.icmp.uCompare(ukey, t.imin.ukey()) >= 0 {
104 - if !f(level, t) {
105 - return
106 - }
107 - }
108 - }
109 - }
110 -
111 - if lf != nil && !lf(level) {
112 - return
113 - }
114 - }
115 -}
116 -
117 -func (v *version) get(ikey iKey, ro *opt.ReadOptions, noValue bool) (value []byte, tcomp bool, err error) {
118 - ukey := ikey.ukey()
119 -
120 - var (
121 - tset *tSet
122 - tseek bool
123 -
124 - // Level-0.
125 - zfound bool
126 - zseq uint64
127 - zkt kType
128 - zval []byte
129 - )
130 -
131 - err = ErrNotFound
132 -
133 - // Since entries never hope across level, finding key/value
134 - // in smaller level make later levels irrelevant.
135 - v.walkOverlapping(ikey, func(level int, t *tFile) bool {
136 - if !tseek {
137 - if tset == nil {
138 - tset = &tSet{level, t}
139 - } else {
140 - tseek = true
141 - }
142 - }
143 -
144 - var (
145 - fikey, fval []byte
146 - ferr error
147 - )
148 - if noValue {
149 - fikey, ferr = v.s.tops.findKey(t, ikey, ro)
150 - } else {
151 - fikey, fval, ferr = v.s.tops.find(t, ikey, ro)
152 - }
153 - switch ferr {
154 - case nil:
155 - case ErrNotFound:
156 - return true
157 - default:
158 - err = ferr
159 - return false
160 - }
161 -
162 - if fukey, fseq, fkt, fkerr := parseIkey(fikey); fkerr == nil {
163 - if v.s.icmp.uCompare(ukey, fukey) == 0 {
164 - if level == 0 {
165 - if fseq >= zseq {
166 - zfound = true
167 - zseq = fseq
168 - zkt = fkt
169 - zval = fval
170 - }
171 - } else {
172 - switch fkt {
173 - case ktVal:
174 - value = fval
175 - err = nil
176 - case ktDel:
177 - default:
178 - panic("leveldb: invalid iKey type")
179 - }
180 - return false
181 - }
182 - }
183 - } else {
184 - err = fkerr
185 - return false
186 - }
187 -
188 - return true
189 - }, func(level int) bool {
190 - if zfound {
191 - switch zkt {
192 - case ktVal:
193 - value = zval
194 - err = nil
195 - case ktDel:
196 - default:
197 - panic("leveldb: invalid iKey type")
198 - }
199 - return false
200 - }
201 -
202 - return true
203 - })
204 -
205 - if tseek && tset.table.consumeSeek() <= 0 {
206 - tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
207 - }
208 -
209 - return
210 -}
211 -
212 -func (v *version) sampleSeek(ikey iKey) (tcomp bool) {
213 - var tset *tSet
214 -
215 - v.walkOverlapping(ikey, func(level int, t *tFile) bool {
216 - if tset == nil {
217 - tset = &tSet{level, t}
218 - return true
219 - } else {
220 - if tset.table.consumeSeek() <= 0 {
221 - tcomp = atomic.CompareAndSwapPointer(&v.cSeek, nil, unsafe.Pointer(tset))
222 - }
223 - return false
224 - }
225 - }, nil)
226 -
227 - return
228 -}
229 -
230 -func (v *version) getIterators(slice *util.Range, ro *opt.ReadOptions) (its []iterator.Iterator) {
231 - // Merge all level zero files together since they may overlap
232 - for _, t := range v.tables[0] {
233 - it := v.s.tops.newIterator(t, slice, ro)
234 - its = append(its, it)
235 - }
236 -
237 - strict := opt.GetStrict(v.s.o.Options, ro, opt.StrictReader)
238 - for _, tables := range v.tables[1:] {
239 - if len(tables) == 0 {
240 - continue
241 - }
242 -
243 - it := iterator.NewIndexedIterator(tables.newIndexIterator(v.s.tops, v.s.icmp, slice, ro), strict)
244 - its = append(its, it)
245 - }
246 -
247 - return
248 -}
249 -
250 -func (v *version) newStaging() *versionStaging {
251 - return &versionStaging{base: v, tables: make([]tablesScratch, v.s.o.GetNumLevel())}
252 -}
253 -
254 -// Spawn a new version based on this version.
255 -func (v *version) spawn(r *sessionRecord) *version {
256 - staging := v.newStaging()
257 - staging.commit(r)
258 - return staging.finish()
259 -}
260 -
261 -func (v *version) fillRecord(r *sessionRecord) {
262 - for level, ts := range v.tables {
263 - for _, t := range ts {
264 - r.addTableFile(level, t)
265 - }
266 - }
267 -}
268 -
269 -func (v *version) tLen(level int) int {
270 - return len(v.tables[level])
271 -}
272 -
273 -func (v *version) offsetOf(ikey iKey) (n uint64, err error) {
274 - for level, tables := range v.tables {
275 - for _, t := range tables {
276 - if v.s.icmp.Compare(t.imax, ikey) <= 0 {
277 - // Entire file is before "ikey", so just add the file size
278 - n += t.size
279 - } else if v.s.icmp.Compare(t.imin, ikey) > 0 {
280 - // Entire file is after "ikey", so ignore
281 - if level > 0 {
282 - // Files other than level 0 are sorted by meta->min, so
283 - // no further files in this level will contain data for
284 - // "ikey".
285 - break
286 - }
287 - } else {
288 - // "ikey" falls in the range for this table. Add the
289 - // approximate offset of "ikey" within the table.
290 - var nn uint64
291 - nn, err = v.s.tops.offsetOf(t, ikey)
292 - if err != nil {
293 - return 0, err
294 - }
295 - n += nn
296 - }
297 - }
298 - }
299 -
300 - return
301 -}
302 -
303 -func (v *version) pickLevel(umin, umax []byte) (level int) {
304 - if !v.tables[0].overlaps(v.s.icmp, umin, umax, true) {
305 - var overlaps tFiles
306 - maxLevel := v.s.o.GetMaxMemCompationLevel()
307 - for ; level < maxLevel; level++ {
308 - if v.tables[level+1].overlaps(v.s.icmp, umin, umax, false) {
309 - break
310 - }
311 - overlaps = v.tables[level+2].getOverlaps(overlaps, v.s.icmp, umin, umax, false)
312 - if overlaps.size() > uint64(v.s.o.GetCompactionGPOverlaps(level)) {
313 - break
314 - }
315 - }
316 - }
317 -
318 - return
319 -}
320 -
321 -func (v *version) computeCompaction() {
322 - // Precomputed best level for next compaction
323 - var bestLevel int = -1
324 - var bestScore float64 = -1
325 -
326 - for level, tables := range v.tables {
327 - var score float64
328 - if level == 0 {
329 - // We treat level-0 specially by bounding the number of files
330 - // instead of number of bytes for two reasons:
331 - //
332 - // (1) With larger write-buffer sizes, it is nice not to do too
333 - // many level-0 compactions.
334 - //
335 - // (2) The files in level-0 are merged on every read and
336 - // therefore we wish to avoid too many files when the individual
337 - // file size is small (perhaps because of a small write-buffer
338 - // setting, or very high compression ratios, or lots of
339 - // overwrites/deletions).
340 - score = float64(len(tables)) / float64(v.s.o.GetCompactionL0Trigger())
341 - } else {
342 - score = float64(tables.size()) / float64(v.s.o.GetCompactionTotalSize(level))
343 - }
344 -
345 - if score > bestScore {
346 - bestLevel = level
347 - bestScore = score
348 - }
349 - }
350 -
351 - v.cLevel = bestLevel
352 - v.cScore = bestScore
353 -}
354 -
355 -func (v *version) needCompaction() bool {
356 - return v.cScore >= 1 || atomic.LoadPointer(&v.cSeek) != nil
357 -}
358 -
359 -type tablesScratch struct {
360 - added map[uint64]atRecord
361 - deleted map[uint64]struct{}
362 -}
363 -
364 -type versionStaging struct {
365 - base *version
366 - tables []tablesScratch
367 -}
368 -
369 -func (p *versionStaging) commit(r *sessionRecord) {
370 - // Deleted tables.
371 - for _, r := range r.deletedTables {
372 - tm := &(p.tables[r.level])
373 -
374 - if len(p.base.tables[r.level]) > 0 {
375 - if tm.deleted == nil {
376 - tm.deleted = make(map[uint64]struct{})
377 - }
378 - tm.deleted[r.num] = struct{}{}
379 - }
380 -
381 - if tm.added != nil {
382 - delete(tm.added, r.num)
383 - }
384 - }
385 -
386 - // New tables.
387 - for _, r := range r.addedTables {
388 - tm := &(p.tables[r.level])
389 -
390 - if tm.added == nil {
391 - tm.added = make(map[uint64]atRecord)
392 - }
393 - tm.added[r.num] = r
394 -
395 - if tm.deleted != nil {
396 - delete(tm.deleted, r.num)
397 - }
398 - }
399 -}
400 -
401 -func (p *versionStaging) finish() *version {
402 - // Build new version.
403 - nv := newVersion(p.base.s)
404 - for level, tm := range p.tables {
405 - btables := p.base.tables[level]
406 -
407 - n := len(btables) + len(tm.added) - len(tm.deleted)
408 - if n < 0 {
409 - n = 0
410 - }
411 - nt := make(tFiles, 0, n)
412 -
413 - // Base tables.
414 - for _, t := range btables {
415 - if _, ok := tm.deleted[t.file.Num()]; ok {
416 - continue
417 - }
418 - if _, ok := tm.added[t.file.Num()]; ok {
419 - continue
420 - }
421 - nt = append(nt, t)
422 - }
423 -
424 - // New tables.
425 - for _, r := range tm.added {
426 - nt = append(nt, p.base.s.tableFileFromRecord(r))
427 - }
428 -
429 - // Sort tables.
430 - if level == 0 {
431 - nt.sortByNum()
432 - } else {
433 - nt.sortByKey(p.base.s.icmp)
434 - }
435 - nv.tables[level] = nt
436 - }
437 -
438 - // Compute compaction score for new version.
439 - nv.computeCompaction()
440 -
441 - return nv
442 -}
443 -
444 -type versionReleaser struct {
445 - v *version
446 - once bool
447 -}
448 -
449 -func (vr *versionReleaser) Release() {
450 - v := vr.v
451 - v.s.vmu.Lock()
452 - if !vr.once {
453 - v.releaseNB()
454 - vr.once = true
455 - }
456 - v.s.vmu.Unlock()
457 -}
package.json
+6
@@ -123,6 +123,12 @@
123 "hash": "QmdYwCmx8pZRkzdcd8MhmLJqYVoVTC1aGsy5Q4reMGLNLg",
124 "name": "atomicfile",
125 "version": "0.0.0"
126 + },
127 + {
128 + "author": "syndtr",
129 + "hash": "QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g",
130 + "name": "goleveldb",
131 + "version": "0.0.1"
132 }
133 ],
134 "gxVersion": "0.4.0",
repo/fsrepo/defaultds.go
+1 -1
@@ -9,10 +9,10 @@ import (
9 levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/leveldb"
10 "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/measure"
11 mount "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/ipfs/go-datastore/syncmount"
12 - ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
12 repo "github.com/ipfs/go-ipfs/repo"
13 config "github.com/ipfs/go-ipfs/repo/config"
14 "github.com/ipfs/go-ipfs/thirdparty/dir"
15 + ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
16 )
17
18 const (