blocks: AllKeys + tests
Juan Batiz-Benet committed
Jan 9, 2015 at 17:39 UTC
da976a5f2159151859e6de4b70e96a50023b6496
4 files changed
+103
-4
blocks/blockstore/blockstore.go
+33
-2
@@ -6,12 +6,17 @@ import (
6
"errors"
7
8
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9
+ dsns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
10
+ dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
11
mh "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-multihash"
12
13
blocks "github.com/jbenet/go-ipfs/blocks"
14
u "github.com/jbenet/go-ipfs/util"
15
)
16
17
+// BlockPrefix namespaces blockstore datastores
18
+var BlockPrefix = ds.NewKey("blocks")
19
+
20
var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
21
22
var ErrNotFound = errors.New("blockstore: block not found")
@@ -22,16 +27,20 @@ type Blockstore interface {
27
Has(u.Key) (bool, error)
28
Get(u.Key) (*blocks.Block, error)
29
Put(*blocks.Block) error
30
+ AllKeys(offset int, limit int) ([]u.Key, error)
31
}
32
33
func NewBlockstore(d ds.ThreadSafeDatastore) Blockstore {
34
+ dd := dsns.Wrap(d, BlockPrefix)
35
return &blockstore{
29
- datastore: d,
36
+ datastore: dd,
37
}
38
}
39
40
type blockstore struct {
34
- datastore ds.ThreadSafeDatastore
41
+ datastore ds.Datastore
42
+ // cant be ThreadSafeDatastore cause namespace.Datastore doesnt support it.
43
+ // we do check it on `NewBlockstore` though.
44
}
45
46
func (bs *blockstore) Get(k u.Key) (*blocks.Block, error) {
@@ -67,3 +76,25 @@ func (bs *blockstore) Has(k u.Key) (bool, error) {
76
func (s *blockstore) DeleteBlock(k u.Key) error {
77
return s.datastore.Delete(k.DsKey())
78
}
79
+
80
+// AllKeys runs a query for keys from the blockstore.
81
+// this is very simplistic, in the future, take dsq.Query as a param?
82
+// if offset and limit are 0, they are ignored.
83
+func (bs *blockstore) AllKeys(offset int, limit int) ([]u.Key, error) {
84
+ var keys []u.Key
85
+
86
+ // TODO make async inside ds/leveldb.Query
87
+ // KeysOnly, because that would be _a lot_ of data.
88
+ q := dsq.Query{KeysOnly: true, Offset: offset, Limit: limit}
89
+ res, err := bs.datastore.Query(q)
90
+ if err != nil {
91
+ return nil, err
92
+ }
93
+
94
+ for e := range res.Entries() {
95
+ // need to convert to u.Key using u.KeyFromDsKey.
96
+ k := u.KeyFromDsKey(ds.NewKey(e.Key))
97
+ keys = append(keys, k)
98
+ }
99
+ return keys, nil
100
+}
blocks/blockstore/blockstore_test.go
+58
-1
@@ -2,6 +2,7 @@ package blockstore
2
3
import (
4
"bytes"
5
+ "fmt"
6
"testing"
7
8
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
@@ -41,11 +42,49 @@ func TestPutThenGetBlock(t *testing.T) {
42
}
43
}
44
45
+func TestAllKeys(t *testing.T) {
46
+ bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
47
+ N := 100
48
+
49
+ keys := make([]u.Key, N)
50
+ for i := 0; i < N; i++ {
51
+ block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
52
+ err := bs.Put(block)
53
+ if err != nil {
54
+ t.Fatal(err)
55
+ }
56
+ keys[i] = block.Key()
57
+ }
58
+
59
+ keys2, err := bs.AllKeys(0, 0)
60
+ if err != nil {
61
+ t.Fatal(err)
62
+ }
63
+ // for _, k2 := range keys2 {
64
+ // t.Log("found ", k2.Pretty())
65
+ // }
66
+
67
+ expectMatches(t, keys, keys2)
68
+
69
+ keys3, err := bs.AllKeys(N/3, N/3)
70
+ if err != nil {
71
+ t.Fatal(err)
72
+ }
73
+ for _, k3 := range keys3 {
74
+ t.Log("found ", k3.Pretty())
75
+ }
76
+ if len(keys3) != N/3 {
77
+ t.Errorf("keys3 should be: %d != %d", N/3, len(keys3))
78
+ }
79
+
80
+}
81
+
82
func TestValueTypeMismatch(t *testing.T) {
83
block := blocks.NewBlock([]byte("some data"))
84
85
datastore := ds.NewMapDatastore()
48
- datastore.Put(block.Key().DsKey(), "data that isn't a block!")
86
+ k := BlockPrefix.Child(block.Key().DsKey())
87
+ datastore.Put(k, "data that isn't a block!")
88
89
blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
90
@@ -54,3 +93,21 @@ func TestValueTypeMismatch(t *testing.T) {
93
t.Fatal(err)
94
}
95
}
96
+
97
+func expectMatches(t *testing.T, expect, actual []u.Key) {
98
+
99
+ if len(expect) != len(actual) {
100
+ t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
101
+ }
102
+ for _, ek := range expect {
103
+ found := false
104
+ for _, ak := range actual {
105
+ if ek == ak {
106
+ found = true
107
+ }
108
+ }
109
+ if !found {
110
+ t.Error("expected key not found: ", ek)
111
+ }
112
+ }
113
+}
blocks/blockstore/write_cache.go
+4
@@ -43,3 +43,7 @@ func (w *writecache) Put(b *blocks.Block) error {
43
w.cache.Add(b.Key(), struct{}{})
44
return w.blockstore.Put(b)
45
}
46
+
47
+func (w *writecache) AllKeys(offset int, limit int) ([]u.Key, error) {
48
+ return w.blockstore.AllKeys(offset, limit)
49
+}
util/key.go
+8
-1
@@ -71,7 +71,7 @@ func (k *Key) Loggable() map[string]interface{} {
71
72
// KeyFromDsKey returns a Datastore key
73
func KeyFromDsKey(dsk ds.Key) Key {
74
- return Key(dsk.BaseNamespace())
74
+ return Key(dsk.String()[1:])
75
}
76
77
// B58KeyConverter -- for KeyTransform datastores
@@ -131,3 +131,10 @@ func XOR(a, b []byte) []byte {
131
}
132
return c
133
}
134
+
135
+// KeySlice is used for sorting Keys
136
+type KeySlice []Key
137
+
138
+func (es KeySlice) Len() int { return len(es) }
139
+func (es KeySlice) Swap(i, j int) { es[i], es[j] = es[j], es[i] }
140
+func (es KeySlice) Less(i, j int) bool { return es[i] < es[j] }