@cryptotaxi247 / kubo / commits / f9ca67ef0

updated datastore (Query)

Juan Batiz-Benet committed Jan 9, 2015 at 16:37 UTC f9ca67ef04296d0081fa03b767261f5cdadbd2e5
25 files changed +836 -65
Godeps/Godeps.json
+1 -1
@@ -110,7 +110,7 @@
110 },
111 {
112 "ImportPath": "github.com/jbenet/go-datastore",
113 - "Rev": "6a1c83bda2a71a9bdc936749fdb507df958ed949"
113 + "Rev": "8a8988d1a4e174274bd4a9dd55c4837f46fdf323"
114 },
115 {
116 "ImportPath": "github.com/jbenet/go-fuse-version",
Godeps/_workspace/src/github.com/jbenet/go-datastore/.travis.yml new
+11
@@ -0,0 +1,11 @@
1 +language: go
2 +
3 +go:
4 + - 1.3
5 + - release
6 + - tip
7 +
8 +script:
9 + - make test
10 +
11 +env: TEST_NO_FUSE=1 TEST_VERBOSE=1
Godeps/_workspace/src/github.com/jbenet/go-datastore/Makefile
+3
@@ -1,6 +1,9 @@
1 build:
2 go build
3
4 +test:
5 + go test ./...
6 +
7 # saves/vendors third-party dependencies to Godeps/_workspace
8 # -r flag rewrites import paths to use the vendored path
9 # ./... performs operation on all packages in tree
Godeps/_workspace/src/github.com/jbenet/go-datastore/basic_ds.go
+20 -14
@@ -1,6 +1,10 @@
1 package datastore
2
3 -import "log"
3 +import (
4 + "log"
5 +
6 + query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
7 +)
8
9 // Here are some basic datastore implementations.
10
@@ -45,13 +49,15 @@ func (d *MapDatastore) Delete(key Key) (err error) {
49 return nil
50 }
51
48 -// KeyList implements Datastore.KeyList
49 -func (d *MapDatastore) KeyList() ([]Key, error) {
50 - var keys []Key
51 - for k := range d.values {
52 - keys = append(keys, k)
52 +// Query implements Datastore.Query
53 +func (d *MapDatastore) Query(q query.Query) (*query.Results, error) {
54 + re := make([]query.Entry, 0, len(d.values))
55 + for k, v := range d.values {
56 + re = append(re, query.Entry{Key: k.String(), Value: v})
57 }
54 - return keys, nil
58 + r := query.ResultsWithEntries(q, re)
59 + r = q.ApplyTo(r)
60 + return r, nil
61 }
62
63 // NullDatastore stores nothing, but conforms to the API.
@@ -84,9 +90,9 @@ func (d *NullDatastore) Delete(key Key) (err error) {
90 return nil
91 }
92
87 -// KeyList implements Datastore.KeyList
88 -func (d *NullDatastore) KeyList() ([]Key, error) {
89 - return nil, nil
93 +// Query implements Datastore.Query
94 +func (d *NullDatastore) Query(q query.Query) (*query.Results, error) {
95 + return query.ResultsWithEntries(q, nil), nil
96 }
97
98 // LogDatastore logs all accesses through the datastore.
@@ -140,8 +146,8 @@ func (d *LogDatastore) Delete(key Key) (err error) {
146 return d.child.Delete(key)
147 }
148
143 -// KeyList implements Datastore.KeyList
144 -func (d *LogDatastore) KeyList() ([]Key, error) {
145 - log.Printf("%s: Get KeyList\n", d.Name)
146 - return d.child.KeyList()
149 +// Query implements Datastore.Query
150 +func (d *LogDatastore) Query(q query.Query) (*query.Results, error) {
151 + log.Printf("%s: Query\n", d.Name)
152 + return d.child.Query(q)
153 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/datastore.go
+15 -2
@@ -2,6 +2,8 @@ package datastore
2
3 import (
4 "errors"
5 +
6 + query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
7 )
8
9 /*
@@ -52,8 +54,19 @@ type Datastore interface {
54 // Delete removes the value for given `key`.
55 Delete(key Key) (err error)
56
55 - // KeyList returns a list of keys in the datastore
56 - KeyList() ([]Key, error)
57 + // Query searches the datastore and returns a query result. This function
58 + // may return before the query actually runs. To wait for the query:
59 + //
60 + // result, _ := ds.Query(q)
61 + //
62 + // // use the channel interface; result may come in at different times
63 + // for entry := range result.Entries() { ... }
64 + //
65 + // // or wait for the query to be completely done
66 + // result.Wait()
67 + // result.AllEntries()
68 + //
69 + Query(q query.Query) (*query.Results, error)
70 }
71
72 // ThreadSafeDatastore is an interface that all threadsafe datastore should
Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go
+2 -1
@@ -8,6 +8,7 @@ import (
8
9 "github.com/codahale/blake2"
10 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 + query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 "github.com/mattbaird/elastigo/api"
13 "github.com/mattbaird/elastigo/core"
14 )
@@ -112,7 +113,7 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
113 return nil
114 }
115
115 -func (d *Datastore) KeyList() ([]ds.Key, error) {
116 +func (d *Datastore) Query(query.Query) (*query.Results, error) {
117 return nil, errors.New("Not yet implemented!")
118 }
119
Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs.go
+18 -7
@@ -8,8 +8,11 @@ import (
8 "strings"
9
10 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 + query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 )
13
14 +var ObjectKeySuffix = ".dsobject"
15 +
16 // Datastore uses a standard Go map for internal storage.
17 type Datastore struct {
18 path string
@@ -26,7 +29,7 @@ func NewDatastore(path string) (ds.Datastore, error) {
29
30 // KeyFilename returns the filename associated with `key`
31 func (d *Datastore) KeyFilename(key ds.Key) string {
29 - return filepath.Join(d.path, key.String(), ".dsobject")
32 + return filepath.Join(d.path, key.String(), ObjectKeySuffix)
33 }
34
35 // Put stores the given value.
@@ -79,10 +82,10 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
82 return os.Remove(fn)
83 }
84
82 -// KeyList returns a list of all keys in the datastore
83 -func (d *Datastore) KeyList() ([]ds.Key, error) {
85 +// Query implements Datastore.Query
86 +func (d *Datastore) Query(q query.Query) (*query.Results, error) {
87
85 - keys := []ds.Key{}
88 + entries := make(chan query.Entry)
89
90 walkFn := func(path string, info os.FileInfo, err error) error {
91 // remove ds path prefix
@@ -91,14 +94,22 @@ func (d *Datastore) KeyList() ([]ds.Key, error) {
94 }
95
96 if !info.IsDir() {
97 + if strings.HasSuffix(path, ObjectKeySuffix) {
98 + path = path[:len(path)-len(ObjectKeySuffix)]
99 + }
100 key := ds.NewKey(path)
95 - keys = append(keys, key)
101 + entries <- query.Entry{Key: key.String(), Value: query.NotFetched}
102 }
103 return nil
104 }
105
100 - filepath.Walk(d.path, walkFn)
101 - return keys, nil
106 + go func() {
107 + filepath.Walk(d.path, walkFn)
108 + close(entries)
109 + }()
110 + r := query.ResultsWithEntriesChan(q, entries)
111 + r = q.ApplyTo(r)
112 + return r, nil
113 }
114
115 // isDir returns whether given path is a directory
Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go
+29 -1
@@ -4,9 +4,11 @@ import (
4 "bytes"
5 "testing"
6
7 + . "launchpad.net/gocheck"
8 +
9 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 fs "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/fs"
9 - . "launchpad.net/gocheck"
11 + query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 )
13
14 // Hook up gocheck into the "go test" runner.
@@ -54,6 +56,32 @@ func (ks *DSSuite) TestBasic(c *C) {
56 c.Check(err, Equals, nil)
57 c.Check(bytes.Equal(v.([]byte), []byte(k.String())), Equals, true)
58 }
59 +
60 + r, err := ks.ds.Query(query.Query{Prefix: "/foo/bar/"})
61 + if err != nil {
62 + c.Check(err, Equals, nil)
63 + }
64 +
65 + expect := []string{
66 + "/foo/bar/baz",
67 + "/foo/bar/bazb",
68 + "/foo/bar/baz/barb",
69 + }
70 + all := r.AllEntries()
71 + c.Check(len(all), Equals, len(expect))
72 +
73 + for _, k := range expect {
74 + found := false
75 + for _, e := range all {
76 + if e.Key == k {
77 + found = true
78 + }
79 + }
80 +
81 + if !found {
82 + c.Error("did not find expected key: ", k)
83 + }
84 + }
85 }
86
87 func strsToKeys(strs []string) []ds.Key {
Godeps/_workspace/src/github.com/jbenet/go-datastore/key.go
+11
@@ -5,6 +5,8 @@ import (
5 "strings"
6
7 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid"
8 +
9 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
10 )
11
12 /*
@@ -239,3 +241,12 @@ type KeySlice []Key
241 func (p KeySlice) Len() int { return len(p) }
242 func (p KeySlice) Less(i, j int) bool { return p[i].Less(p[j]) }
243 func (p KeySlice) Swap(i, j int) { p[i], p[j] = p[j], p[i] }
244 +
245 +// EntryKeys
246 +func EntryKeys(e []dsq.Entry) []Key {
247 + ks := make([]Key, len(e))
248 + for i, e := range e {
249 + ks[i] = NewKey(e.Key)
250 + }
251 + return ks
252 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform/keytransform.go
+20 -8
@@ -1,6 +1,9 @@
1 package keytransform
2
3 -import ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
3 +import (
4 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
5 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
6 +)
7
8 type Pair struct {
9 Convert KeyMapping
@@ -48,16 +51,25 @@ func (d *ktds) Delete(key ds.Key) (err error) {
51 return d.child.Delete(d.ConvertKey(key))
52 }
53
51 -// KeyList returns a list of all keys in the datastore, transforming keys out.
52 -func (d *ktds) KeyList() ([]ds.Key, error) {
54 +// Query implements Query, inverting keys on the way back out.
55 +func (d *ktds) Query(q dsq.Query) (*dsq.Results, error) {
56
54 - keys, err := d.child.KeyList()
57 + q2 := q
58 + q2.Prefix = d.ConvertKey(ds.NewKey(q2.Prefix)).String()
59 + r, err := d.child.Query(q2)
60 if err != nil {
61 return nil, err
62 }
63
59 - for i, k := range keys {
60 - keys[i] = d.InvertKey(k)
61 - }
62 - return keys, nil
64 + ch := make(chan dsq.Entry)
65 + go func() {
66 + for e := range r.Entries() {
67 + e.Key = d.InvertKey(ds.NewKey(e.Key)).String()
68 + ch <- e
69 + }
70 + close(ch)
71 + }()
72 +
73 + r2 := dsq.ResultsWithEntriesChan(q, ch)
74 + return r2, nil
75 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform/keytransform_test.go
+11 -3
@@ -5,9 +5,11 @@ import (
5 "sort"
6 "testing"
7
8 + . "launchpad.net/gocheck"
9 +
10 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 kt "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform"
10 - . "launchpad.net/gocheck"
12 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
13 )
14
15 // Hook up gocheck into the "go test" runner.
@@ -60,10 +62,13 @@ func (ks *DSSuite) TestBasic(c *C) {
62 c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
63 }
64
63 - listA, errA := mpds.KeyList()
64 - listB, errB := ktds.KeyList()
65 + listAr, errA := mpds.Query(dsq.Query{})
66 + listBr, errB := ktds.Query(dsq.Query{})
67 c.Check(errA, Equals, nil)
68 c.Check(errB, Equals, nil)
69 +
70 + listA := ds.EntryKeys(listAr.AllEntries())
71 + listB := ds.EntryKeys(listBr.AllEntries())
72 c.Check(len(listA), Equals, len(listB))
73
74 // sort them cause yeah.
@@ -75,6 +80,9 @@ func (ks *DSSuite) TestBasic(c *C) {
80 c.Check(pair.Invert(kA), Equals, kB)
81 c.Check(kA, Equals, pair.Convert(kB))
82 }
83 +
84 + c.Log("listA: ", listA)
85 + c.Log("listB: ", listB)
86 }
87
88 func strsToKeys(strs []string) []ds.Key {
Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb/datastore.go
+50 -6
@@ -3,9 +3,12 @@ package leveldb
3 import (
4 "io"
5
6 - ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
6 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb"
7 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
8 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
9 +
10 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 )
13
14 type Datastore interface {
@@ -69,13 +72,54 @@ func (d *datastore) Delete(key ds.Key) (err error) {
72 return err
73 }
74
72 -func (d *datastore) KeyList() ([]ds.Key, error) {
73 - i := d.DB.NewIterator(nil, nil)
74 - var keys []ds.Key
75 +func (d *datastore) Query(q dsq.Query) (*dsq.Results, error) {
76 + var rnge *util.Range
77 + if q.Prefix != "" {
78 + rnge = util.BytesPrefix([]byte(q.Prefix))
79 + }
80 + i := d.DB.NewIterator(rnge, nil)
81 +
82 + // offset
83 + if q.Offset > 0 {
84 + for j := 0; j < q.Offset; j++ {
85 + i.Next()
86 + }
87 + }
88 +
89 + var es []dsq.Entry
90 for i.Next() {
76 - keys = append(keys, ds.NewKey(string(i.Key())))
91 +
92 + // limit
93 + if q.Limit > 0 && len(es) >= q.Limit {
94 + break
95 + }
96 +
97 + k := ds.NewKey(string(i.Key())).String()
98 + e := dsq.Entry{Key: k}
99 +
100 + if !q.KeysOnly {
101 + buf := make([]byte, len(i.Value()))
102 + copy(buf, i.Value())
103 + e.Value = buf
104 + }
105 +
106 + es = append(es, e)
107 + }
108 + i.Release()
109 + if err := i.Error(); err != nil {
110 + return nil, err
111 }
78 - return keys, nil
112 +
113 + // Now, apply remaining pieces.
114 + q2 := q
115 + q2.Offset = 0 // already applied
116 + q2.Limit = 0 // already applied
117 + // TODO: make this async with:
118 + // qr := dsq.ResultsWithEntriesChan(q, ch)
119 + qr := dsq.ResultsWithEntries(q, es)
120 + qr = q2.ApplyTo(qr)
121 + qr.Query = q // set it back
122 + return qr, nil
123 }
124
125 // LevelDB needs to be closed.
Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb/ds_test.go new
+99
@@ -0,0 +1,99 @@
1 +package leveldb
2 +
3 +import (
4 + "io/ioutil"
5 + "os"
6 + "testing"
7 +
8 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
10 +)
11 +
12 +var testcases = map[string]string{
13 + "/a": "a",
14 + "/a/b": "ab",
15 + "/a/b/c": "abc",
16 + "/a/b/d": "a/b/d",
17 + "/a/c": "ac",
18 + "/a/d": "ad",
19 + "/e": "e",
20 + "/f": "f",
21 +}
22 +
23 +func TestQuery(t *testing.T) {
24 + path, err := ioutil.TempDir("/tmp", "testing_leveldb_")
25 + if err != nil {
26 + t.Fatal(err)
27 + }
28 + defer func() {
29 + os.RemoveAll(path)
30 + }()
31 +
32 + d, err := NewDatastore(path, nil)
33 + if err != nil {
34 + t.Fatal(err)
35 + }
36 + defer d.Close()
37 +
38 + for k, v := range testcases {
39 + dsk := ds.NewKey(k)
40 + if err := d.Put(dsk, []byte(v)); err != nil {
41 + t.Fatal(err)
42 + }
43 + }
44 +
45 + for k, v := range testcases {
46 + dsk := ds.NewKey(k)
47 + v2, err := d.Get(dsk)
48 + if err != nil {
49 + t.Fatal(err)
50 + }
51 + v2b := v2.([]byte)
52 + if string(v2b) != v {
53 + t.Errorf("%s values differ: %s != %s", k, v, v2)
54 + }
55 + }
56 +
57 + rs, err := d.Query(dsq.Query{Prefix: "/a/"})
58 + if err != nil {
59 + t.Fatal(err)
60 + }
61 +
62 + expectMatches(t, []string{
63 + "/a/b",
64 + "/a/b/c",
65 + "/a/b/d",
66 + "/a/c",
67 + "/a/d",
68 + }, rs.AllEntries())
69 +
70 + // test offset and limit
71 +
72 + rs, err = d.Query(dsq.Query{Prefix: "/a/", Offset: 2, Limit: 2})
73 + if err != nil {
74 + t.Fatal(err)
75 + }
76 +
77 + expectMatches(t, []string{
78 + "/a/b/d",
79 + "/a/c",
80 + }, rs.AllEntries())
81 +
82 +}
83 +
84 +func expectMatches(t *testing.T, expect []string, actual []dsq.Entry) {
85 + if len(actual) != len(expect) {
86 + t.Error("not enough", expect, actual)
87 + }
88 + for _, k := range expect {
89 + found := false
90 + for _, e := range actual {
91 + if e.Key == k {
92 + found = true
93 + }
94 + }
95 + if !found {
96 + t.Error(k, "not found")
97 + }
98 + }
99 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore.go
+3 -1
@@ -4,7 +4,9 @@ import (
4 "errors"
5
6 lru "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/golang-lru"
7 +
8 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
10 )
11
12 // Datastore uses golang-lru for internal storage.
@@ -49,6 +51,6 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
51 }
52
53 // KeyList returns a list of keys in the datastore
52 -func (d *Datastore) KeyList() ([]ds.Key, error) {
54 +func (d *Datastore) Query(q dsq.Query) (*dsq.Results, error) {
55 return nil, errors.New("KeyList not implemented.")
56 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace/namespace_test.go
+8 -3
@@ -5,9 +5,11 @@ import (
5 "sort"
6 "testing"
7
8 + . "launchpad.net/gocheck"
9 +
10 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 ns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
10 - . "launchpad.net/gocheck"
12 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
13 )
14
15 // Hook up gocheck into the "go test" runner.
@@ -46,10 +48,13 @@ func (ks *DSSuite) TestBasic(c *C) {
48 c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
49 }
50
49 - listA, errA := mpds.KeyList()
50 - listB, errB := nsds.KeyList()
51 + listAr, errA := mpds.Query(dsq.Query{})
52 + listBr, errB := nsds.Query(dsq.Query{})
53 c.Check(errA, Equals, nil)
54 c.Check(errB, Equals, nil)
55 +
56 + listA := ds.EntryKeys(listAr.AllEntries())
57 + listB := ds.EntryKeys(listBr.AllEntries())
58 c.Check(len(listA), Equals, len(listB))
59
60 // sort them cause yeah.
Godeps/_workspace/src/github.com/jbenet/go-datastore/panic/panic.go
+5 -4
@@ -5,6 +5,7 @@ import (
5 "os"
6
7 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9 )
10
11 type datastore struct {
@@ -57,11 +58,11 @@ func (d *datastore) Delete(key ds.Key) error {
58 return nil
59 }
60
60 -func (d *datastore) KeyList() ([]ds.Key, error) {
61 - kl, err := d.child.KeyList()
61 +func (d *datastore) Query(q dsq.Query) (*dsq.Results, error) {
62 + r, err := d.child.Query(q)
63 if err != nil {
64 fmt.Fprintf(os.Stdout, "panic datastore: %s", err)
64 - panic("panic datastore: KeyList failed")
65 + panic("panic datastore: Query failed")
66 }
66 - return kl, nil
67 + return r, nil
68 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/filter.go new
+86
@@ -0,0 +1,86 @@
1 +package query
2 +
3 +import (
4 + "fmt"
5 + "reflect"
6 + "strings"
7 +)
8 +
9 +// Filter is an object that tests ResultEntries
10 +type Filter interface {
11 + // Filter returns whether an entry passes the filter
12 + Filter(e Entry) bool
13 +}
14 +
15 +// Op is a comparison operator
16 +type Op string
17 +
18 +var (
19 + Equal = Op("==")
20 + NotEqual = Op("!=")
21 + GreaterThan = Op(">")
22 + GreaterThanOrEqual = Op(">=")
23 + LessThan = Op("<")
24 + LessThanOrEqual = Op("<=")
25 +)
26 +
27 +// FilterValueCompare is used to signal to datastores they
28 +// should apply internal comparisons. unfortunately, there
29 +// is no way to apply comparisons* to interface{} types in
30 +// Go, so if the datastore doesnt have a special way to
31 +// handle these comparisons, you must provided the
32 +// TypedFilter to actually do filtering.
33 +//
34 +// [*] other than == and !=, which use reflect.DeepEqual.
35 +type FilterValueCompare struct {
36 + Op Op
37 + Value interface{}
38 + TypedFilter Filter
39 +}
40 +
41 +func (f FilterValueCompare) Filter(e Entry) bool {
42 + if f.TypedFilter != nil {
43 + return f.TypedFilter.Filter(e)
44 + }
45 +
46 + switch f.Op {
47 + case Equal:
48 + return reflect.DeepEqual(f.Value, e.Value)
49 + case NotEqual:
50 + return !reflect.DeepEqual(f.Value, e.Value)
51 + default:
52 + panic(fmt.Errorf("cannot apply op '%s' to interface{}.", f.Op))
53 + }
54 +}
55 +
56 +type FilterKeyCompare struct {
57 + Op Op
58 + Key string
59 +}
60 +
61 +func (f FilterKeyCompare) Filter(e Entry) bool {
62 + switch f.Op {
63 + case Equal:
64 + return e.Key == f.Key
65 + case NotEqual:
66 + return e.Key != f.Key
67 + case GreaterThan:
68 + return e.Key > f.Key
69 + case GreaterThanOrEqual:
70 + return e.Key >= f.Key
71 + case LessThan:
72 + return e.Key < f.Key
73 + case LessThanOrEqual:
74 + return e.Key <= f.Key
75 + default:
76 + panic(fmt.Errorf("unknown op '%s'", f.Op))
77 + }
78 +}
79 +
80 +type FilterKeyPrefix struct {
81 + Prefix string
82 +}
83 +
84 +func (f FilterKeyPrefix) Filter(e Entry) bool {
85 + return strings.HasPrefix(e.Key, f.Prefix)
86 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/filter_test.go new
+75
@@ -0,0 +1,75 @@
1 +package query
2 +
3 +import (
4 + "strings"
5 + "testing"
6 +)
7 +
8 +var sampleKeys = []string{
9 + "/ab/c",
10 + "/ab/cd",
11 + "/a",
12 + "/abce",
13 + "/abcf",
14 + "/ab",
15 +}
16 +
17 +type filterTestCase struct {
18 + filter Filter
19 + keys []string
20 + expect []string
21 +}
22 +
23 +func testKeyFilter(t *testing.T, f Filter, keys []string, expect []string) {
24 + e := make([]Entry, len(keys))
25 + for i, k := range keys {
26 + e[i] = Entry{Key: k}
27 + }
28 +
29 + res := ResultsWithEntries(Query{}, e)
30 + res = NaiveFilter(res, f)
31 + actualE := res.AllEntries()
32 + actual := make([]string, len(actualE))
33 + for i, e := range actualE {
34 + actual[i] = e.Key
35 + }
36 +
37 + if len(actual) != len(expect) {
38 + t.Error("expect != actual.", expect, actual)
39 + }
40 +
41 + if strings.Join(actual, "") != strings.Join(expect, "") {
42 + t.Error("expect != actual.", expect, actual)
43 + }
44 +}
45 +
46 +func TestFilterKeyCompare(t *testing.T) {
47 +
48 + testKeyFilter(t, FilterKeyCompare{Equal, "/ab"}, sampleKeys, []string{"/ab"})
49 + testKeyFilter(t, FilterKeyCompare{GreaterThan, "/ab"}, sampleKeys, []string{
50 + "/ab/c",
51 + "/ab/cd",
52 + "/abce",
53 + "/abcf",
54 + })
55 + testKeyFilter(t, FilterKeyCompare{LessThanOrEqual, "/ab"}, sampleKeys, []string{
56 + "/a",
57 + "/ab",
58 + })
59 +}
60 +
61 +func TestFilterKeyPrefix(t *testing.T) {
62 +
63 + testKeyFilter(t, FilterKeyPrefix{"/a"}, sampleKeys, []string{
64 + "/ab/c",
65 + "/ab/cd",
66 + "/a",
67 + "/abce",
68 + "/abcf",
69 + "/ab",
70 + })
71 + testKeyFilter(t, FilterKeyPrefix{"/ab/"}, sampleKeys, []string{
72 + "/ab/c",
73 + "/ab/cd",
74 + })
75 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/order.go new
+66
@@ -0,0 +1,66 @@
1 +package query
2 +
3 +import (
4 + "sort"
5 +)
6 +
7 +// Order is an object used to order objects
8 +type Order interface {
9 +
10 + // Sort sorts the Entry slice according to
11 + // the Order criteria.
12 + Sort([]Entry)
13 +}
14 +
15 +// OrderByValue is used to signal to datastores they
16 +// should apply internal orderings. unfortunately, there
17 +// is no way to apply order comparisons to interface{} types
18 +// in Go, so if the datastore doesnt have a special way to
19 +// handle these comparisons, you must provide an Order
20 +// implementation that casts to the correct type.
21 +type OrderByValue struct {
22 + TypedOrder Order
23 +}
24 +
25 +func (o OrderByValue) Sort(res []Entry) {
26 + if o.TypedOrder == nil {
27 + panic("cannot order interface{} by value. see query docs.")
28 + }
29 + o.TypedOrder.Sort(res)
30 +}
31 +
32 +// OrderByValueDescending is used to signal to datastores they
33 +// should apply internal orderings. unfortunately, there
34 +// is no way to apply order comparisons to interface{} types
35 +// in Go, so if the datastore doesnt have a special way to
36 +// handle these comparisons, you are SOL.
37 +type OrderByValueDescending struct {
38 + TypedOrder Order
39 +}
40 +
41 +func (o OrderByValueDescending) Sort(res []Entry) {
42 + if o.TypedOrder == nil {
43 + panic("cannot order interface{} by value. see query docs.")
44 + }
45 + o.TypedOrder.Sort(res)
46 +}
47 +
48 +// OrderByKey
49 +type OrderByKey struct{}
50 +
51 +func (o OrderByKey) Sort(res []Entry) {
52 + sort.Stable(reByKey(res))
53 +}
54 +
55 +// OrderByKeyDescending
56 +type OrderByKeyDescending struct{}
57 +
58 +func (o OrderByKeyDescending) Sort(res []Entry) {
59 + sort.Stable(sort.Reverse(reByKey(res)))
60 +}
61 +
62 +type reByKey []Entry
63 +
64 +func (s reByKey) Len() int { return len(s) }
65 +func (s reByKey) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
66 +func (s reByKey) Less(i, j int) bool { return s[i].Key < s[j].Key }
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/order_test.go new
+55
@@ -0,0 +1,55 @@
1 +package query
2 +
3 +import (
4 + "strings"
5 + "testing"
6 +)
7 +
8 +type orderTestCase struct {
9 + order Order
10 + keys []string
11 + expect []string
12 +}
13 +
14 +func testKeyOrder(t *testing.T, f Order, keys []string, expect []string) {
15 + e := make([]Entry, len(keys))
16 + for i, k := range keys {
17 + e[i] = Entry{Key: k}
18 + }
19 +
20 + res := ResultsWithEntries(Query{}, e)
21 + res = NaiveOrder(res, f)
22 + actualE := res.AllEntries()
23 + actual := make([]string, len(actualE))
24 + for i, e := range actualE {
25 + actual[i] = e.Key
26 + }
27 +
28 + if len(actual) != len(expect) {
29 + t.Error("expect != actual.", expect, actual)
30 + }
31 +
32 + if strings.Join(actual, "") != strings.Join(expect, "") {
33 + t.Error("expect != actual.", expect, actual)
34 + }
35 +}
36 +
37 +func TestOrderByKey(t *testing.T) {
38 +
39 + testKeyOrder(t, OrderByKey{}, sampleKeys, []string{
40 + "/a",
41 + "/ab",
42 + "/ab/c",
43 + "/ab/cd",
44 + "/abce",
45 + "/abcf",
46 + })
47 + testKeyOrder(t, OrderByKeyDescending{}, sampleKeys, []string{
48 + "/abcf",
49 + "/abce",
50 + "/ab/cd",
51 + "/ab/c",
52 + "/ab",
53 + "/a",
54 + })
55 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/query.go new
+145
@@ -0,0 +1,145 @@
1 +package query
2 +
3 +/*
4 +Query represents storage for any key-value pair.
5 +
6 +tl;dr:
7 +
8 + queries are supported across datastores.
9 + Cheap on top of relational dbs, and expensive otherwise.
10 + Pick the right tool for the job!
11 +
12 +In addition to the key-value store get and set semantics, datastore
13 +provides an interface to retrieve multiple records at a time through
14 +the use of queries. The datastore Query model gleans a common set of
15 +operations performed when querying. To avoid pasting here years of
16 +database research, let’s summarize the operations datastore supports.
17 +
18 +Query Operations:
19 +
20 + * namespace - scope the query, usually by object type
21 + * filters - select a subset of values by applying constraints
22 + * orders - sort the results by applying sort conditions
23 + * limit - impose a numeric limit on the number of results
24 + * offset - skip a number of results (for efficient pagination)
25 +
26 +datastore combines these operations into a simple Query class that allows
27 +applications to define their constraints in a simple, generic, way without
28 +introducing datastore specific calls, languages, etc.
29 +
30 +Of course, different datastores provide relational query support across a
31 +wide spectrum, from full support in traditional databases to none at all in
32 +most key-value stores. Datastore aims to provide a common, simple interface
33 +for the sake of application evolution over time and keeping large code bases
34 +free of tool-specific code. It would be ridiculous to claim to support high-
35 +performance queries on architectures that obviously do not. Instead, datastore
36 +provides the interface, ideally translating queries to their native form
37 +(e.g. into SQL for MySQL).
38 +
39 +However, on the wrong datastore, queries can potentially incur the high cost
40 +of performing the aforemantioned query operations on the data set directly in
41 +Go. It is the client’s responsibility to select the right tool for the job:
42 +pick a data storage solution that fits the application’s needs now, and wrap
43 +it with a datastore implementation. As the needs change, swap out datastore
44 +implementations to support your new use cases. Some applications, particularly
45 +in early development stages, can afford to incurr the cost of queries on non-
46 +relational databases (e.g. using a FSDatastore and not worry about a database
47 +at all). When it comes time to switch the tool for performance, updating the
48 +application code can be as simple as swapping the datastore in one place, not
49 +all over the application code base. This gain in engineering time, both at
50 +initial development and during later iterations, can significantly offset the
51 +cost of the layer of abstraction.
52 +
53 +*/
54 +type Query struct {
55 + Prefix string // namespaces the query to results whose keys have Prefix
56 + Filters []Filter // filter results. apply sequentially
57 + Orders []Order // order results. apply sequentially
58 + Limit int // maximum number of results
59 + Offset int // skip given number of results
60 + KeysOnly bool // return only keys.
61 +}
62 +
63 +// NotFetched is a special type that signals whether or not the value
64 +// of an Entry has been fetched or not. This is needed because
65 +// datastore implementations get to decide whether Query returns values
66 +// or only keys. nil is not a good signal, as real values may be nil.
67 +var NotFetched = struct{}{}
68 +
69 +// Entry is a query result entry.
70 +type Entry struct {
71 + Key string // cant be ds.Key because circular imports ...!!!
72 + Value interface{}
73 +}
74 +
75 +// Results is a set of Query results
76 +type Results struct {
77 + Query Query // the query these Results correspond to
78 +
79 + done chan struct{}
80 + res chan Entry
81 + all []Entry
82 +}
83 +
84 +// ResultsWithEntriesChan returns a Results object from a
85 +// channel of ResultEntries. It's merely an encapsulation
86 +// that provides for AllEntries() functionality.
87 +func ResultsWithEntriesChan(q Query, res <-chan Entry) *Results {
88 + r := &Results{
89 + Query: q,
90 + done: make(chan struct{}),
91 + res: make(chan Entry),
92 + all: []Entry{},
93 + }
94 +
95 + // go consume all the results and add them to the results.
96 + go func() {
97 + for e := range res {
98 + r.all = append(r.all, e)
99 + r.res <- e
100 + }
101 + close(r.res)
102 + close(r.done)
103 + }()
104 + return r
105 +}
106 +
107 +// ResultsWithEntries returns a Results object from a
108 +// channel of ResultEntries. It's merely an encapsulation
109 +// that provides for AllEntries() functionality.
110 +func ResultsWithEntries(q Query, res []Entry) *Results {
111 + r := &Results{
112 + Query: q,
113 + done: make(chan struct{}),
114 + res: make(chan Entry),
115 + all: res,
116 + }
117 +
118 + // go add all the results
119 + go func() {
120 + for _, e := range res {
121 + r.res <- e
122 + }
123 + close(r.res)
124 + close(r.done)
125 + }()
126 + return r
127 +}
128 +
129 +// Entries() returns results through a channel.
130 +// Results may arrive at any time.
131 +// The channel may or may not be buffered.
132 +// The channel may or may not rate limit the query processing.
133 +func (r *Results) Entries() <-chan Entry {
134 + return r.res
135 +}
136 +
137 +// AllEntries returns all the entries in Results.
138 +// It blocks until all the results have come in.
139 +func (r *Results) AllEntries() []Entry {
140 + for e := range r.res {
141 + _ = e
142 + }
143 + <-r.done
144 + return r.all
145 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/query_impl.go new
+85
@@ -0,0 +1,85 @@
1 +package query
2 +
3 +// NaiveFilter applies a filter to the results
4 +func NaiveFilter(qr *Results, filter Filter) *Results {
5 + ch := make(chan Entry)
6 + go func() {
7 + defer close(ch)
8 +
9 + for e := range qr.Entries() {
10 + if filter.Filter(e) {
11 + ch <- e
12 + }
13 + }
14 + }()
15 + return ResultsWithEntriesChan(qr.Query, ch)
16 +}
17 +
18 +// NaiveLimit truncates the results to a given int limit
19 +func NaiveLimit(qr *Results, limit int) *Results {
20 + ch := make(chan Entry)
21 + go func() {
22 + defer close(ch)
23 +
24 + for l := 0; l < limit; l++ {
25 + e, more := <-qr.Entries()
26 + if !more {
27 + return
28 + }
29 + ch <- e
30 + }
31 + }()
32 + return ResultsWithEntriesChan(qr.Query, ch)
33 +}
34 +
35 +// NaiveOffset skips a given number of results
36 +func NaiveOffset(qr *Results, offset int) *Results {
37 + ch := make(chan Entry)
38 + go func() {
39 + defer close(ch)
40 +
41 + for l := 0; l < offset; l++ {
42 + <-qr.Entries() // discard
43 + }
44 +
45 + for e := range qr.Entries() {
46 + ch <- e
47 + }
48 + }()
49 + return ResultsWithEntriesChan(qr.Query, ch)
50 +}
51 +
52 +// NaiveOrder reorders results according to given Order.
53 +// WARNING: this is the only non-stream friendly operation!
54 +func NaiveOrder(qr *Results, o Order) *Results {
55 + e := qr.AllEntries()
56 + o.Sort(e)
57 + return ResultsWithEntries(qr.Query, e)
58 +}
59 +
60 +func (q Query) ApplyTo(qr *Results) *Results {
61 + if q.Prefix != "" {
62 + qr = NaiveFilter(qr, FilterKeyPrefix{q.Prefix})
63 + }
64 + for _, f := range q.Filters {
65 + qr = NaiveFilter(qr, f)
66 + }
67 + for _, o := range q.Orders {
68 + qr = NaiveOrder(qr, o)
69 + }
70 + if q.Offset != 0 {
71 + qr = NaiveOffset(qr, q.Offset)
72 + }
73 + if q.Limit != 0 {
74 + qr = NaiveLimit(qr, q.Offset)
75 + }
76 + return qr
77 +}
78 +
79 +func ResultEntriesFrom(keys []string, vals []interface{}) []Entry {
80 + re := make([]Entry, len(keys))
81 + for i, k := range keys {
82 + re[i] = Entry{Key: k, Value: vals[i]}
83 + }
84 + return re
85 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/sync/sync.go
+3 -2
@@ -4,6 +4,7 @@ import (
4 "sync"
5
6 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
8 )
9
10 // MutexDatastore contains a child datastire and a mutex.
@@ -57,8 +58,8 @@ func (d *MutexDatastore) Delete(key ds.Key) (err error) {
58 }
59
60 // KeyList implements Datastore.KeyList
60 -func (d *MutexDatastore) KeyList() ([]ds.Key, error) {
61 +func (d *MutexDatastore) Query(q dsq.Query) (*dsq.Results, error) {
62 d.RLock()
63 defer d.RUnlock()
63 - return d.child.KeyList()
64 + return d.child.Query(q)
65 }
blocks/blockstore/write_cache_test.go
+3 -2
@@ -4,6 +4,7 @@ import (
4 "testing"
5
6 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
8 syncds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
9 "github.com/jbenet/go-ipfs/blocks"
10 )
@@ -83,7 +84,7 @@ func (c *callbackDatastore) Delete(key ds.Key) (err error) {
84 return c.ds.Delete(key)
85 }
86
86 -func (c *callbackDatastore) KeyList() ([]ds.Key, error) {
87 +func (c *callbackDatastore) Query(q dsq.Query) (*dsq.Results, error) {
88 c.f()
88 - return c.ds.KeyList()
89 + return c.ds.Query(q)
90 }
util/datastore2/delayed.go
+12 -10
@@ -1,42 +1,44 @@
1 package datastore2
2
3 import (
4 - datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
4 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
5 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
6 +
7 delay "github.com/jbenet/go-ipfs/util/delay"
8 )
9
8 -func WithDelay(ds datastore.Datastore, delay delay.D) datastore.Datastore {
10 +func WithDelay(ds ds.Datastore, delay delay.D) ds.Datastore {
11 return &delayed{ds: ds, delay: delay}
12 }
13
14 type delayed struct {
13 - ds datastore.Datastore
15 + ds ds.Datastore
16 delay delay.D
17 }
18
17 -func (dds *delayed) Put(key datastore.Key, value interface{}) (err error) {
19 +func (dds *delayed) Put(key ds.Key, value interface{}) (err error) {
20 dds.delay.Wait()
21 return dds.ds.Put(key, value)
22 }
23
22 -func (dds *delayed) Get(key datastore.Key) (value interface{}, err error) {
24 +func (dds *delayed) Get(key ds.Key) (value interface{}, err error) {
25 dds.delay.Wait()
26 return dds.ds.Get(key)
27 }
28
27 -func (dds *delayed) Has(key datastore.Key) (exists bool, err error) {
29 +func (dds *delayed) Has(key ds.Key) (exists bool, err error) {
30 dds.delay.Wait()
31 return dds.ds.Has(key)
32 }
33
32 -func (dds *delayed) Delete(key datastore.Key) (err error) {
34 +func (dds *delayed) Delete(key ds.Key) (err error) {
35 dds.delay.Wait()
36 return dds.ds.Delete(key)
37 }
38
37 -func (dds *delayed) KeyList() ([]datastore.Key, error) {
39 +func (dds *delayed) Query(q dsq.Query) (*dsq.Results, error) {
40 dds.delay.Wait()
39 - return dds.ds.KeyList()
41 + return dds.ds.Query(q)
42 }
43
42 -var _ datastore.Datastore = &delayed{}
44 +var _ ds.Datastore = &delayed{}