@cryptotaxi247 / kubo / commits / 92e8a7bcd

updated datastore for proper query handling

Queries now can be cancelled and the resources collected

Juan Batiz-Benet committed Jan 10, 2015 at 14:31 UTC 92e8a7bcd56a38ae722b556fc5b5e0345e1906ac
27 files changed +780 -196
Godeps/Godeps.json
+1 -1
@@ -110,7 +110,7 @@
110 },
111 {
112 "ImportPath": "github.com/jbenet/go-datastore",
113 - "Rev": "8a8988d1a4e174274bd4a9dd55c4837f46fdf323"
113 + "Rev": "35738aceb35505bd3c77c2a618fb1947ca3f72da"
114 },
115 {
116 "ImportPath": "github.com/jbenet/go-fuse-version",
Godeps/_workspace/src/github.com/jbenet/go-datastore/Godeps/Godeps.json
+4
@@ -18,6 +18,10 @@
18 "ImportPath": "github.com/hashicorp/golang-lru",
19 "Rev": "4dfff096c4973178c8f35cf6dd1a732a0a139370"
20 },
21 + {
22 + "ImportPath": "github.com/jbenet/goprocess",
23 + "Rev": "b4b4178efcf2404ce9db72438c9c49db2fb399d8"
24 + },
25 {
26 "ImportPath": "github.com/mattbaird/elastigo/api",
27 "Rev": "041b88c1fcf6489a5721ede24378ce1253b9159d"
Godeps/_workspace/src/github.com/jbenet/go-datastore/basic_ds.go
+9 -9
@@ -3,7 +3,7 @@ package datastore
3 import (
4 "log"
5
6 - query "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
6 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
7 )
8
9 // Here are some basic datastore implementations.
@@ -50,13 +50,13 @@ func (d *MapDatastore) Delete(key Key) (err error) {
50 }
51
52 // Query implements Datastore.Query
53 -func (d *MapDatastore) Query(q query.Query) (*query.Results, error) {
54 - re := make([]query.Entry, 0, len(d.values))
53 +func (d *MapDatastore) Query(q dsq.Query) (dsq.Results, error) {
54 + re := make([]dsq.Entry, 0, len(d.values))
55 for k, v := range d.values {
56 - re = append(re, query.Entry{Key: k.String(), Value: v})
56 + re = append(re, dsq.Entry{Key: k.String(), Value: v})
57 }
58 - r := query.ResultsWithEntries(q, re)
59 - r = q.ApplyTo(r)
58 + r := dsq.ResultsWithEntries(q, re)
59 + r = dsq.NaiveQueryApply(q, r)
60 return r, nil
61 }
62
@@ -91,8 +91,8 @@ func (d *NullDatastore) Delete(key Key) (err error) {
91 }
92
93 // Query implements Datastore.Query
94 -func (d *NullDatastore) Query(q query.Query) (*query.Results, error) {
95 - return query.ResultsWithEntries(q, nil), nil
94 +func (d *NullDatastore) Query(q dsq.Query) (dsq.Results, error) {
95 + return dsq.ResultsWithEntries(q, nil), nil
96 }
97
98 // LogDatastore logs all accesses through the datastore.
@@ -147,7 +147,7 @@ func (d *LogDatastore) Delete(key Key) (err error) {
147 }
148
149 // Query implements Datastore.Query
150 -func (d *LogDatastore) Query(q query.Query) (*query.Results, error) {
150 +func (d *LogDatastore) Query(q dsq.Query) (dsq.Results, error) {
151 log.Printf("%s: Query\n", d.Name)
152 return d.child.Query(q)
153 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/datastore.go
+1 -1
@@ -66,7 +66,7 @@ type Datastore interface {
66 // result.Wait()
67 // result.AllEntries()
68 //
69 - Query(q query.Query) (*query.Results, error)
69 + Query(q query.Query) (query.Results, error)
70 }
71
72 // ThreadSafeDatastore is an interface that all threadsafe datastore should
Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go
+1 -1
@@ -113,7 +113,7 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
113 return nil
114 }
115
116 -func (d *Datastore) Query(query.Query) (*query.Results, error) {
116 +func (d *Datastore) Query(query.Query) (query.Results, error) {
117 return nil, errors.New("Not yet implemented!")
118 }
119
Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs.go
+7 -6
@@ -83,9 +83,9 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
83 }
84
85 // Query implements Datastore.Query
86 -func (d *Datastore) Query(q query.Query) (*query.Results, error) {
86 +func (d *Datastore) Query(q query.Query) (query.Results, error) {
87
88 - entries := make(chan query.Entry)
88 + results := make(chan query.Result)
89
90 walkFn := func(path string, info os.FileInfo, err error) error {
91 // remove ds path prefix
@@ -98,17 +98,18 @@ func (d *Datastore) Query(q query.Query) (*query.Results, error) {
98 path = path[:len(path)-len(ObjectKeySuffix)]
99 }
100 key := ds.NewKey(path)
101 - entries <- query.Entry{Key: key.String(), Value: query.NotFetched}
101 + entry := query.Entry{Key: key.String(), Value: query.NotFetched}
102 + results <- query.Result{Entry: entry}
103 }
104 return nil
105 }
106
107 go func() {
108 filepath.Walk(d.path, walkFn)
108 - close(entries)
109 + close(results)
110 }()
110 - r := query.ResultsWithEntriesChan(q, entries)
111 - r = q.ApplyTo(r)
111 + r := query.ResultsWithChan(q, results)
112 + r = query.NaiveQueryApply(q, r)
113 return r, nil
114 }
115
Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs_test.go
+4 -1
@@ -67,7 +67,10 @@ func (ks *DSSuite) TestBasic(c *C) {
67 "/foo/bar/bazb",
68 "/foo/bar/baz/barb",
69 }
70 - all := r.AllEntries()
70 + all, err := r.Rest()
71 + if err != nil {
72 + c.Fatal(err)
73 + }
74 c.Check(len(all), Equals, len(expect))
75
76 for _, k := range expect {
Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform/keytransform.go
+12 -12
@@ -52,24 +52,24 @@ func (d *ktds) Delete(key ds.Key) (err error) {
52 }
53
54 // Query implements Query, inverting keys on the way back out.
55 -func (d *ktds) Query(q dsq.Query) (*dsq.Results, error) {
56 -
57 - q2 := q
58 - q2.Prefix = d.ConvertKey(ds.NewKey(q2.Prefix)).String()
59 - r, err := d.child.Query(q2)
55 +func (d *ktds) Query(q dsq.Query) (dsq.Results, error) {
56 + qr, err := d.child.Query(q)
57 if err != nil {
58 return nil, err
59 }
60
64 - ch := make(chan dsq.Entry)
61 + ch := make(chan dsq.Result)
62 go func() {
66 - for e := range r.Entries() {
67 - e.Key = d.InvertKey(ds.NewKey(e.Key)).String()
68 - ch <- e
63 + defer close(ch)
64 + defer qr.Close()
65 +
66 + for r := range qr.Next() {
67 + if r.Error == nil {
68 + r.Entry.Key = d.InvertKey(ds.NewKey(r.Entry.Key)).String()
69 + }
70 + ch <- r
71 }
70 - close(ch)
72 }()
73
73 - r2 := dsq.ResultsWithEntriesChan(q, ch)
74 - return r2, nil
74 + return dsq.DerivedResults(qr, ch), nil
75 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform/keytransform_test.go
+11 -6
@@ -62,13 +62,18 @@ func (ks *DSSuite) TestBasic(c *C) {
62 c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
63 }
64
65 - listAr, errA := mpds.Query(dsq.Query{})
66 - listBr, errB := ktds.Query(dsq.Query{})
67 - c.Check(errA, Equals, nil)
68 - c.Check(errB, Equals, nil)
65 + run := func(d ds.Datastore, q dsq.Query) []ds.Key {
66 + r, err := d.Query(q)
67 + c.Check(err, Equals, nil)
68 +
69 + e, err := r.Rest()
70 + c.Check(err, Equals, nil)
71 +
72 + return ds.EntryKeys(e)
73 + }
74
70 - listA := ds.EntryKeys(listAr.AllEntries())
71 - listB := ds.EntryKeys(listBr.AllEntries())
75 + listA := run(mpds, dsq.Query{})
76 + listB := run(ktds, dsq.Query{})
77 c.Check(len(listA), Equals, len(listB))
78
79 // sort them cause yeah.
Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb/datastore.go
+55 -29
@@ -3,12 +3,12 @@ package leveldb
3 import (
4 "io"
5
6 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
8 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
9 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb"
10 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
11 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/util"
9 -
10 - ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
11 - dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 )
13
14 type Datastore interface {
@@ -72,54 +72,80 @@ func (d *datastore) Delete(key ds.Key) (err error) {
72 return err
73 }
74
75 -func (d *datastore) Query(q dsq.Query) (*dsq.Results, error) {
75 +func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
76 +
77 + // we can use multiple iterators concurrently. see:
78 + // https://godoc.org/github.com/syndtr/goleveldb/leveldb#DB.NewIterator
79 + // advance the iterator only if the reader reads
80 + //
81 + // run query in own sub-process tied to Results.Process(), so that
82 + // it waits for us to finish AND so that clients can signal to us
83 + // that resources should be reclaimed.
84 + qrb := dsq.NewResultBuilder(q)
85 + qrb.Process.Go(func(worker goprocess.Process) {
86 + d.runQuery(worker, qrb)
87 + })
88 +
89 + // go wait on the worker (without signaling close)
90 + go qrb.Process.CloseAfterChildren()
91 +
92 + // Now, apply remaining things (filters, order)
93 + qr := qrb.Results()
94 + for _, f := range q.Filters {
95 + qr = dsq.NaiveFilter(qr, f)
96 + }
97 + for _, o := range q.Orders {
98 + qr = dsq.NaiveOrder(qr, o)
99 + }
100 + return qr, nil
101 +}
102 +
103 +func (d *datastore) runQuery(worker goprocess.Process, qrb *dsq.ResultBuilder) {
104 +
105 var rnge *util.Range
77 - if q.Prefix != "" {
78 - rnge = util.BytesPrefix([]byte(q.Prefix))
106 + if qrb.Query.Prefix != "" {
107 + rnge = util.BytesPrefix([]byte(qrb.Query.Prefix))
108 }
109 i := d.DB.NewIterator(rnge, nil)
110 + defer i.Release()
111
82 - // offset
83 - if q.Offset > 0 {
84 - for j := 0; j < q.Offset; j++ {
112 + // advance iterator for offset
113 + if qrb.Query.Offset > 0 {
114 + for j := 0; j < qrb.Query.Offset; j++ {
115 i.Next()
116 }
117 }
118
89 - var es []dsq.Entry
90 - for i.Next() {
91 -
92 - // limit
93 - if q.Limit > 0 && len(es) >= q.Limit {
119 + // iterate, and handle limit, too
120 + for sent := 0; i.Next(); sent++ {
121 + // end early if we hit the limit
122 + if qrb.Query.Limit > 0 && sent >= qrb.Query.Limit {
123 break
124 }
125
126 k := ds.NewKey(string(i.Key())).String()
127 e := dsq.Entry{Key: k}
128
100 - if !q.KeysOnly {
129 + if !qrb.Query.KeysOnly {
130 buf := make([]byte, len(i.Value()))
131 copy(buf, i.Value())
132 e.Value = buf
133 }
134
106 - es = append(es, e)
135 + select {
136 + case qrb.Output <- dsq.Result{Entry: e}: // we sent it out
137 + case <-worker.Closing(): // client told us to end early.
138 + break
139 + }
140 }
108 - i.Release()
141 +
142 if err := i.Error(); err != nil {
110 - return nil, err
143 + select {
144 + case qrb.Output <- dsq.Result{Error: err}: // client read our error
145 + case <-worker.Closing(): // client told us to end.
146 + return
147 + }
148 }
112 -
113 - // Now, apply remaining pieces.
114 - q2 := q
115 - q2.Offset = 0 // already applied
116 - q2.Limit = 0 // already applied
117 - // TODO: make this async with:
118 - // qr := dsq.ResultsWithEntriesChan(q, ch)
119 - qr := dsq.ResultsWithEntries(q, es)
120 - qr = q2.ApplyTo(qr)
121 - qr.Query = q // set it back
122 - return qr, nil
149 }
150
151 // LevelDB needs to be closed.
Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb/ds_test.go
+33 -8
@@ -20,21 +20,28 @@ var testcases = map[string]string{
20 "/f": "f",
21 }
22
23 -func TestQuery(t *testing.T) {
23 +// returns datastore, and a function to call on exit.
24 +// (this garbage collects). So:
25 +//
26 +// d, close := newDS(t)
27 +// defer close()
28 +func newDS(t *testing.T) (Datastore, func()) {
29 path, err := ioutil.TempDir("/tmp", "testing_leveldb_")
30 if err != nil {
31 t.Fatal(err)
32 }
28 - defer func() {
29 - os.RemoveAll(path)
30 - }()
33
34 d, err := NewDatastore(path, nil)
35 if err != nil {
36 t.Fatal(err)
37 }
36 - defer d.Close()
38 + return d, func() {
39 + os.RemoveAll(path)
40 + d.Close()
41 + }
42 +}
43
44 +func addTestCases(t *testing.T, d Datastore, testcases map[string]string) {
45 for k, v := range testcases {
46 dsk := ds.NewKey(k)
47 if err := d.Put(dsk, []byte(v)); err != nil {
@@ -54,6 +61,13 @@ func TestQuery(t *testing.T) {
61 }
62 }
63
64 +}
65 +
66 +func TestQuery(t *testing.T) {
67 + d, close := newDS(t)
68 + defer close()
69 + addTestCases(t, d, testcases)
70 +
71 rs, err := d.Query(dsq.Query{Prefix: "/a/"})
72 if err != nil {
73 t.Fatal(err)
@@ -65,7 +79,7 @@ func TestQuery(t *testing.T) {
79 "/a/b/d",
80 "/a/c",
81 "/a/d",
68 - }, rs.AllEntries())
82 + }, rs)
83
84 // test offset and limit
85
@@ -77,11 +91,22 @@ func TestQuery(t *testing.T) {
91 expectMatches(t, []string{
92 "/a/b/d",
93 "/a/c",
80 - }, rs.AllEntries())
94 + }, rs)
95 +
96 +}
97
98 +func TestQueryRespectsProcess(t *testing.T) {
99 + d, close := newDS(t)
100 + defer close()
101 + addTestCases(t, d, testcases)
102 }
103
84 -func expectMatches(t *testing.T, expect []string, actual []dsq.Entry) {
104 +func expectMatches(t *testing.T, expect []string, actualR dsq.Results) {
105 + actual, err := actualR.Rest()
106 + if err != nil {
107 + t.Error(err)
108 + }
109 +
110 if len(actual) != len(expect) {
111 t.Error("not enough", expect, actual)
112 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore.go
+1 -1
@@ -51,6 +51,6 @@ func (d *Datastore) Delete(key ds.Key) (err error) {
51 }
52
53 // KeyList returns a list of keys in the datastore
54 -func (d *Datastore) Query(q dsq.Query) (*dsq.Results, error) {
54 +func (d *Datastore) Query(q dsq.Query) (dsq.Results, error) {
55 return nil, errors.New("KeyList not implemented.")
56 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace/namespace.go
+40 -1
@@ -6,6 +6,7 @@ import (
6
7 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 ktds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform"
9 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
10 )
11
12 // PrefixTransform constructs a KeyTransform with a pair of functions that
@@ -40,5 +41,43 @@ func Wrap(child ds.Datastore, prefix ds.Key) ktds.Datastore {
41 panic("child (ds.Datastore) is nil")
42 }
43
43 - return ktds.Wrap(child, PrefixTransform(prefix))
44 + d := ktds.Wrap(child, PrefixTransform(prefix))
45 + return &datastore{Datastore: d, raw: child, prefix: prefix}
46 +}
47 +
48 +type datastore struct {
49 + prefix ds.Key
50 + raw ds.Datastore
51 + ktds.Datastore
52 +}
53 +
54 +// Query implements Query, inverting keys on the way back out.
55 +func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
56 + qr, err := d.raw.Query(q)
57 + if err != nil {
58 + return nil, err
59 + }
60 +
61 + ch := make(chan dsq.Result)
62 + go func() {
63 + defer close(ch)
64 + defer qr.Close()
65 +
66 + for r := range qr.Next() {
67 + if r.Error != nil {
68 + ch <- r
69 + continue
70 + }
71 +
72 + k := ds.NewKey(r.Entry.Key)
73 + if !d.prefix.IsAncestorOf(k) {
74 + continue
75 + }
76 +
77 + r.Entry.Key = d.Datastore.InvertKey(k).String()
78 + ch <- r
79 + }
80 + }()
81 +
82 + return dsq.DerivedResults(qr, ch), nil
83 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace/namespace_test.go
+11 -6
@@ -48,13 +48,18 @@ func (ks *DSSuite) TestBasic(c *C) {
48 c.Check(bytes.Equal(v2.([]byte), []byte(k.String())), Equals, true)
49 }
50
51 - listAr, errA := mpds.Query(dsq.Query{})
52 - listBr, errB := nsds.Query(dsq.Query{})
53 - c.Check(errA, Equals, nil)
54 - c.Check(errB, Equals, nil)
51 + run := func(d ds.Datastore, q dsq.Query) []ds.Key {
52 + r, err := d.Query(q)
53 + c.Check(err, Equals, nil)
54 +
55 + e, err := r.Rest()
56 + c.Check(err, Equals, nil)
57 +
58 + return ds.EntryKeys(e)
59 + }
60
56 - listA := ds.EntryKeys(listAr.AllEntries())
57 - listB := ds.EntryKeys(listBr.AllEntries())
61 + listA := run(mpds, dsq.Query{})
62 + listB := run(nsds, dsq.Query{})
63 c.Check(len(listA), Equals, len(listB))
64
65 // sort them cause yeah.
Godeps/_workspace/src/github.com/jbenet/go-datastore/panic/panic.go
+1 -1
@@ -58,7 +58,7 @@ func (d *datastore) Delete(key ds.Key) error {
58 return nil
59 }
60
61 -func (d *datastore) Query(q dsq.Query) (*dsq.Results, error) {
61 +func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
62 r, err := d.child.Query(q)
63 if err != nil {
64 fmt.Fprintf(os.Stdout, "panic datastore: %s", err)
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/filter_test.go
+4 -10
@@ -5,15 +5,6 @@ import (
5 "testing"
6 )
7
8 -var sampleKeys = []string{
9 - "/ab/c",
10 - "/ab/cd",
11 - "/a",
12 - "/abce",
13 - "/abcf",
14 - "/ab",
15 -}
16 -
8 type filterTestCase struct {
9 filter Filter
10 keys []string
@@ -28,7 +19,10 @@ func testKeyFilter(t *testing.T, f Filter, keys []string, expect []string) {
19
20 res := ResultsWithEntries(Query{}, e)
21 res = NaiveFilter(res, f)
31 - actualE := res.AllEntries()
22 + actualE, err := res.Rest()
23 + if err != nil {
24 + t.Fatal(err)
25 + }
26 actual := make([]string, len(actualE))
27 for i, e := range actualE {
28 actual[i] = e.Key
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/order_test.go
+5 -1
@@ -19,7 +19,11 @@ func testKeyOrder(t *testing.T, f Order, keys []string, expect []string) {
19
20 res := ResultsWithEntries(Query{}, e)
21 res = NaiveOrder(res, f)
22 - actualE := res.AllEntries()
22 + actualE, err := res.Rest()
23 + if err != nil {
24 + t.Fatal(err)
25 + }
26 +
27 actual := make([]string, len(actualE))
28 for i, e := range actualE {
29 actual[i] = e.Key
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/query.go
+163 -58
@@ -1,5 +1,9 @@
1 package query
2
3 +import (
4 + goprocess "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/goprocess"
5 +)
6 +
7 /*
8 Query represents storage for any key-value pair.
9
@@ -64,7 +68,7 @@ type Query struct {
68 // of an Entry has been fetched or not. This is needed because
69 // datastore implementations get to decide whether Query returns values
70 // or only keys. nil is not a good signal, as real values may be nil.
67 -var NotFetched = struct{}{}
71 +const NotFetched int = iota
72
73 // Entry is a query result entry.
74 type Entry struct {
@@ -72,74 +76,175 @@ type Entry struct {
76 Value interface{}
77 }
78
75 -// Results is a set of Query results
76 -type Results struct {
77 - Query Query // the query these Results correspond to
79 +// Result is a special entry that includes an error, so that the client
80 +// may be warned about internal errors.
81 +type Result struct {
82 + Entry
83
79 - done chan struct{}
80 - res chan Entry
81 - all []Entry
84 + Error error
85 }
86
84 -// ResultsWithEntriesChan returns a Results object from a
85 -// channel of ResultEntries. It's merely an encapsulation
86 -// that provides for AllEntries() functionality.
87 -func ResultsWithEntriesChan(q Query, res <-chan Entry) *Results {
88 - r := &Results{
89 - Query: q,
90 - done: make(chan struct{}),
91 - res: make(chan Entry),
92 - all: []Entry{},
93 - }
87 +// Results is a set of Query results. This is the interface for clients.
88 +// Example:
89 +//
90 +// qr, _ := myds.Query(q)
91 +// for r := range qr.Next() {
92 +// if r.Error != nil {
93 +// // handle.
94 +// break
95 +// }
96 +//
97 +// fmt.Println(r.Entry.Key, r.Entry.Value)
98 +// }
99 +//
100 +// or, wait on all results at once:
101 +//
102 +// qr, _ := myds.Query(q)
103 +// es, _ := qr.Rest()
104 +// for _, e := range es {
105 +// fmt.Println(e.Key, e.Value)
106 +// }
107 +//
108 +type Results interface {
109 + Query() Query // the query these Results correspond to
110 + Next() <-chan Result // returns a channel to wait for the next result
111 + Rest() ([]Entry, error) // waits till processing finishes, returns all entries at once.
112 + Close() error // client may call Close to signal early exit
113 +
114 + // Process returns a goprocess.Process associated with these results.
115 + // most users will not need this function (Close is all they want),
116 + // but it's here in case you want to connect the results to other
117 + // goprocess-friendly things.
118 + Process() goprocess.Process
119 +}
120 +
121 +// results implements Results
122 +type results struct {
123 + query Query
124 + proc goprocess.Process
125 + res <-chan Result
126 +}
127 +
128 +func (r *results) Next() <-chan Result {
129 + return r.res
130 +}
131
95 - // go consume all the results and add them to the results.
96 - go func() {
97 - for e := range res {
98 - r.all = append(r.all, e)
99 - r.res <- e
132 +func (r *results) Rest() ([]Entry, error) {
133 + var es []Entry
134 + for e := range r.res {
135 + if e.Error != nil {
136 + return es, e.Error
137 }
101 - close(r.res)
102 - close(r.done)
103 - }()
104 - return r
105 -}
106 -
107 -// ResultsWithEntries returns a Results object from a
108 -// channel of ResultEntries. It's merely an encapsulation
109 -// that provides for AllEntries() functionality.
110 -func ResultsWithEntries(q Query, res []Entry) *Results {
111 - r := &Results{
112 - Query: q,
113 - done: make(chan struct{}),
114 - res: make(chan Entry),
115 - all: res,
138 + es = append(es, e.Entry)
139 }
140 + <-r.proc.Closed() // wait till the processing finishes.
141 + return es, nil
142 +}
143
118 - // go add all the results
119 - go func() {
120 - for _, e := range res {
121 - r.res <- e
144 +func (r *results) Process() goprocess.Process {
145 + return r.proc
146 +}
147 +
148 +func (r *results) Close() error {
149 + return r.proc.Close()
150 +}
151 +
152 +func (r *results) Query() Query {
153 + return r.query
154 +}
155 +
156 +// ResultBuilder is what implementors use to construct results
157 +// Implementors of datastores and their clients must respect the
158 +// Process of the Request:
159 +//
160 +// * clients must call r.Process().Close() on an early exit, so
161 +// implementations can reclaim resources.
162 +// * if the Entries are read to completion (channel closed), Process
163 +// should be closed automatically.
164 +// * datastores must respect <-Process.Closing(), which intermediates
165 +// an early close signal from the client.
166 +//
167 +type ResultBuilder struct {
168 + Query Query
169 + Process goprocess.Process
170 + Output chan Result
171 +}
172 +
173 +// Results returns a Results to to this builder.
174 +func (rb *ResultBuilder) Results() Results {
175 + return &results{
176 + query: rb.Query,
177 + proc: rb.Process,
178 + res: rb.Output,
179 + }
180 +}
181 +
182 +func NewResultBuilder(q Query) *ResultBuilder {
183 + b := &ResultBuilder{
184 + Query: q,
185 + Output: make(chan Result),
186 + }
187 + b.Process = goprocess.WithTeardown(func() error {
188 + close(b.Output)
189 + return nil
190 + })
191 + return b
192 +}
193 +
194 +// ResultsWithChan returns a Results object from a channel
195 +// of Result entries. Respects its own Close()
196 +func ResultsWithChan(q Query, res <-chan Result) Results {
197 + b := NewResultBuilder(q)
198 +
199 + // go consume all the entries and add them to the results.
200 + b.Process.Go(func(worker goprocess.Process) {
201 + for {
202 + select {
203 + case <-worker.Closing(): // client told us to close early
204 + return
205 + case e, more := <-res:
206 + if !more {
207 + return
208 + }
209 +
210 + select {
211 + case b.Output <- e:
212 + case <-worker.Closing(): // client told us to close early
213 + return
214 + }
215 + }
216 }
123 - close(r.res)
124 - close(r.done)
125 - }()
126 - return r
217 + return
218 + })
219 +
220 + go b.Process.CloseAfterChildren()
221 + return b.Results()
222 }
223
129 -// Entries() returns results through a channel.
130 -// Results may arrive at any time.
131 -// The channel may or may not be buffered.
132 -// The channel may or may not rate limit the query processing.
133 -func (r *Results) Entries() <-chan Entry {
134 - return r.res
224 +// ResultsWithEntries returns a Results object from a list of entries
225 +func ResultsWithEntries(q Query, res []Entry) Results {
226 + b := NewResultBuilder(q)
227 +
228 + // go consume all the entries and add them to the results.
229 + b.Process.Go(func(worker goprocess.Process) {
230 + for _, e := range res {
231 + select {
232 + case b.Output <- Result{Entry: e}:
233 + case <-worker.Closing(): // client told us to close early
234 + return
235 + }
236 + }
237 + return
238 + })
239 +
240 + go b.Process.CloseAfterChildren()
241 + return b.Results()
242 }
243
137 -// AllEntries returns all the entries in Results.
138 -// It blocks until all the results have come in.
139 -func (r *Results) AllEntries() []Entry {
140 - for e := range r.res {
141 - _ = e
244 +func ResultsReplaceQuery(r Results, q Query) Results {
245 + return &results{
246 + query: q,
247 + proc: r.Process(),
248 + res: r.Next(),
249 }
143 - <-r.done
144 - return r.all
250 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/query_impl.go
+67 -25
@@ -1,63 +1,105 @@
1 package query
2
3 -// NaiveFilter applies a filter to the results
4 -func NaiveFilter(qr *Results, filter Filter) *Results {
5 - ch := make(chan Entry)
3 +func DerivedResults(qr Results, ch <-chan Result) Results {
4 + return &results{
5 + query: qr.Query(),
6 + proc: qr.Process(),
7 + res: ch,
8 + }
9 +}
10 +
11 +// NaiveFilter applies a filter to the results.
12 +func NaiveFilter(qr Results, filter Filter) Results {
13 + ch := make(chan Result)
14 go func() {
15 defer close(ch)
16 + defer qr.Close()
17
9 - for e := range qr.Entries() {
10 - if filter.Filter(e) {
18 + for e := range qr.Next() {
19 + if e.Error != nil || filter.Filter(e.Entry) {
20 ch <- e
21 }
22 }
23 }()
15 - return ResultsWithEntriesChan(qr.Query, ch)
24 +
25 + return DerivedResults(qr, ch)
26 }
27
28 // NaiveLimit truncates the results to a given int limit
19 -func NaiveLimit(qr *Results, limit int) *Results {
20 - ch := make(chan Entry)
29 +func NaiveLimit(qr Results, limit int) Results {
30 + ch := make(chan Result)
31 go func() {
32 defer close(ch)
33 + defer qr.Close()
34
24 - for l := 0; l < limit; l++ {
25 - e, more := <-qr.Entries()
26 - if !more {
27 - return
35 + l := 0
36 + for e := range qr.Next() {
37 + if e.Error != nil {
38 + ch <- e
39 + continue
40 }
41 ch <- e
42 + l++
43 + if limit > 0 && l >= limit {
44 + break
45 + }
46 }
47 }()
32 - return ResultsWithEntriesChan(qr.Query, ch)
48 +
49 + return DerivedResults(qr, ch)
50 }
51
52 // NaiveOffset skips a given number of results
36 -func NaiveOffset(qr *Results, offset int) *Results {
37 - ch := make(chan Entry)
53 +func NaiveOffset(qr Results, offset int) Results {
54 + ch := make(chan Result)
55 go func() {
56 defer close(ch)
57 + defer qr.Close()
58
41 - for l := 0; l < offset; l++ {
42 - <-qr.Entries() // discard
43 - }
59 + sent := 0
60 + for e := range qr.Next() {
61 + if e.Error != nil {
62 + ch <- e
63 + }
64
45 - for e := range qr.Entries() {
65 + if sent < offset {
66 + sent++
67 + continue
68 + }
69 ch <- e
70 }
71 }()
49 - return ResultsWithEntriesChan(qr.Query, ch)
72 +
73 + return DerivedResults(qr, ch)
74 }
75
76 // NaiveOrder reorders results according to given Order.
77 // WARNING: this is the only non-stream friendly operation!
54 -func NaiveOrder(qr *Results, o Order) *Results {
55 - e := qr.AllEntries()
56 - o.Sort(e)
57 - return ResultsWithEntries(qr.Query, e)
78 +func NaiveOrder(qr Results, o Order) Results {
79 + ch := make(chan Result)
80 + var entries []Entry
81 + go func() {
82 + defer close(ch)
83 + defer qr.Close()
84 +
85 + for e := range qr.Next() {
86 + if e.Error != nil {
87 + ch <- e
88 + }
89 +
90 + entries = append(entries, e.Entry)
91 + }
92 +
93 + o.Sort(entries)
94 + for _, e := range entries {
95 + ch <- Result{Entry: e}
96 + }
97 + }()
98 +
99 + return DerivedResults(qr, ch)
100 }
101
60 -func (q Query) ApplyTo(qr *Results) *Results {
102 +func NaiveQueryApply(q Query, qr Results) Results {
103 if q.Prefix != "" {
104 qr = NaiveFilter(qr, FilterKeyPrefix{q.Prefix})
105 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/query/query_test.go new
+109
@@ -0,0 +1,109 @@
1 +package query
2 +
3 +import (
4 + "strings"
5 + "testing"
6 +)
7 +
8 +var sampleKeys = []string{
9 + "/ab/c",
10 + "/ab/cd",
11 + "/a",
12 + "/abce",
13 + "/abcf",
14 + "/ab",
15 +}
16 +
17 +type testCase struct {
18 + keys []string
19 + expect []string
20 +}
21 +
22 +func testResults(t *testing.T, res Results, expect []string) {
23 + actualE, err := res.Rest()
24 + if err != nil {
25 + t.Fatal(err)
26 + }
27 +
28 + actual := make([]string, len(actualE))
29 + for i, e := range actualE {
30 + actual[i] = e.Key
31 + }
32 +
33 + if len(actual) != len(expect) {
34 + t.Error("expect != actual.", expect, actual)
35 + }
36 +
37 + if strings.Join(actual, "") != strings.Join(expect, "") {
38 + t.Error("expect != actual.", expect, actual)
39 + }
40 +}
41 +
42 +func TestLimit(t *testing.T) {
43 + testKeyLimit := func(t *testing.T, limit int, keys []string, expect []string) {
44 + e := make([]Entry, len(keys))
45 + for i, k := range keys {
46 + e[i] = Entry{Key: k}
47 + }
48 +
49 + res := ResultsWithEntries(Query{}, e)
50 + res = NaiveLimit(res, limit)
51 + testResults(t, res, expect)
52 + }
53 +
54 + testKeyLimit(t, 0, sampleKeys, []string{ // none
55 + "/ab/c",
56 + "/ab/cd",
57 + "/a",
58 + "/abce",
59 + "/abcf",
60 + "/ab",
61 + })
62 +
63 + testKeyLimit(t, 10, sampleKeys, []string{ // large
64 + "/ab/c",
65 + "/ab/cd",
66 + "/a",
67 + "/abce",
68 + "/abcf",
69 + "/ab",
70 + })
71 +
72 + testKeyLimit(t, 2, sampleKeys, []string{
73 + "/ab/c",
74 + "/ab/cd",
75 + })
76 +}
77 +
78 +func TestOffset(t *testing.T) {
79 +
80 + testOffset := func(t *testing.T, offset int, keys []string, expect []string) {
81 + e := make([]Entry, len(keys))
82 + for i, k := range keys {
83 + e[i] = Entry{Key: k}
84 + }
85 +
86 + res := ResultsWithEntries(Query{}, e)
87 + res = NaiveOffset(res, offset)
88 + testResults(t, res, expect)
89 + }
90 +
91 + testOffset(t, 0, sampleKeys, []string{ // none
92 + "/ab/c",
93 + "/ab/cd",
94 + "/a",
95 + "/abce",
96 + "/abcf",
97 + "/ab",
98 + })
99 +
100 + testOffset(t, 10, sampleKeys, []string{ // large
101 + })
102 +
103 + testOffset(t, 2, sampleKeys, []string{
104 + "/a",
105 + "/abce",
106 + "/abcf",
107 + "/ab",
108 + })
109 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/sync/sync.go
+1 -1
@@ -58,7 +58,7 @@ func (d *MutexDatastore) Delete(key ds.Key) (err error) {
58 }
59
60 // KeyList implements Datastore.KeyList
61 -func (d *MutexDatastore) Query(q dsq.Query) (*dsq.Results, error) {
61 +func (d *MutexDatastore) Query(q dsq.Query) (dsq.Results, error) {
62 d.RLock()
63 defer d.RUnlock()
64 return d.child.Query(q)
blocks/blockstore/blockstore.go
+69 -8
@@ -5,6 +5,7 @@ package blockstore
5 import (
6 "errors"
7
8 + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
9 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 dsns "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/namespace"
11 dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
@@ -12,8 +13,11 @@ import (
13
14 blocks "github.com/jbenet/go-ipfs/blocks"
15 u "github.com/jbenet/go-ipfs/util"
16 + eventlog "github.com/jbenet/go-ipfs/util/eventlog"
17 )
18
19 +var log = eventlog.Logger("blockstore")
20 +
21 // BlockPrefix namespaces blockstore datastores
22 var BlockPrefix = ds.NewKey("blocks")
23
@@ -27,7 +31,9 @@ type Blockstore interface {
31 Has(u.Key) (bool, error)
32 Get(u.Key) (*blocks.Block, error)
33 Put(*blocks.Block) error
30 - AllKeys(offset int, limit int) ([]u.Key, error)
34 +
35 + AllKeys(ctx context.Context, offset int, limit int) ([]u.Key, error)
36 + AllKeysChan(ctx context.Context, offset int, limit int) (<-chan u.Key, error)
37 }
38
39 func NewBlockstore(d ds.ThreadSafeDatastore) Blockstore {
@@ -80,10 +86,29 @@ func (s *blockstore) DeleteBlock(k u.Key) error {
86 // AllKeys runs a query for keys from the blockstore.
87 // this is very simplistic, in the future, take dsq.Query as a param?
88 // if offset and limit are 0, they are ignored.
83 -func (bs *blockstore) AllKeys(offset int, limit int) ([]u.Key, error) {
89 +//
90 +// AllKeys respects context
91 +func (bs *blockstore) AllKeys(ctx context.Context, offset int, limit int) ([]u.Key, error) {
92 +
93 + ch, err := bs.AllKeysChan(ctx, offset, limit)
94 + if err != nil {
95 + return nil, err
96 + }
97 +
98 var keys []u.Key
99 + for k := range ch {
100 + keys = append(keys, k)
101 + }
102 + return keys, nil
103 +}
104 +
105 +// AllKeys runs a query for keys from the blockstore.
106 +// this is very simplistic, in the future, take dsq.Query as a param?
107 +// if offset and limit are 0, they are ignored.
108 +//
109 +// AllKeys respects context
110 +func (bs *blockstore) AllKeysChan(ctx context.Context, offset int, limit int) (<-chan u.Key, error) {
111
86 - // TODO make async inside ds/leveldb.Query
112 // KeysOnly, because that would be _a lot_ of data.
113 q := dsq.Query{KeysOnly: true, Offset: offset, Limit: limit}
114 res, err := bs.datastore.Query(q)
@@ -91,10 +116,46 @@ func (bs *blockstore) AllKeys(offset int, limit int) ([]u.Key, error) {
116 return nil, err
117 }
118
94 - for e := range res.Entries() {
95 - // need to convert to u.Key using u.KeyFromDsKey.
96 - k := u.KeyFromDsKey(ds.NewKey(e.Key))
97 - keys = append(keys, k)
119 + // this function is here to compartmentalize
120 + get := func() (k u.Key, ok bool) {
121 + select {
122 + case <-ctx.Done():
123 + return k, false
124 + case e, more := <-res.Next():
125 + if !more {
126 + return k, false
127 + }
128 + if e.Error != nil {
129 + log.Debug("blockstore.AllKeysChan got err:", e.Error)
130 + return k, false
131 + }
132 +
133 + // need to convert to u.Key using u.KeyFromDsKey.
134 + k = u.KeyFromDsKey(ds.NewKey(e.Key))
135 + return k, true
136 + }
137 }
99 - return keys, nil
138 +
139 + output := make(chan u.Key)
140 + go func() {
141 + defer func() {
142 + res.Process().Close() // ensure exit (signals early exit, too)
143 + close(output)
144 + }()
145 +
146 + for {
147 + k, ok := get()
148 + if !ok {
149 + return
150 + }
151 +
152 + select {
153 + case <-ctx.Done():
154 + return
155 + case output <- k:
156 + }
157 + }
158 + }()
159 +
160 + return output, nil
161 }
blocks/blockstore/blockstore_test.go
+160 -5
@@ -5,8 +5,11 @@ import (
5 "fmt"
6 "testing"
7
8 + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
9 ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
11 ds_sync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
12 +
13 blocks "github.com/jbenet/go-ipfs/blocks"
14 u "github.com/jbenet/go-ipfs/util"
15 )
@@ -42,9 +45,11 @@ func TestPutThenGetBlock(t *testing.T) {
45 }
46 }
47
45 -func TestAllKeys(t *testing.T) {
46 - bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
47 - N := 100
48 +func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []u.Key) {
49 + if d == nil {
50 + d = ds.NewMapDatastore()
51 + }
52 + bs := NewBlockstore(ds_sync.MutexWrap(d))
53
54 keys := make([]u.Key, N)
55 for i := 0; i < N; i++ {
@@ -55,8 +60,14 @@ func TestAllKeys(t *testing.T) {
60 }
61 keys[i] = block.Key()
62 }
63 + return bs, keys
64 +}
65 +
66 +func TestAllKeysSimple(t *testing.T) {
67 + bs, keys := newBlockStoreWithKeys(t, nil, 100)
68
59 - keys2, err := bs.AllKeys(0, 0)
69 + ctx := context.Background()
70 + keys2, err := bs.AllKeys(ctx, 0, 0)
71 if err != nil {
72 t.Fatal(err)
73 }
@@ -65,8 +76,14 @@ func TestAllKeys(t *testing.T) {
76 // }
77
78 expectMatches(t, keys, keys2)
79 +}
80
69 - keys3, err := bs.AllKeys(N/3, N/3)
81 +func TestAllKeysOffsetAndLimit(t *testing.T) {
82 + N := 30
83 + bs, _ := newBlockStoreWithKeys(t, nil, N)
84 +
85 + ctx := context.Background()
86 + keys3, err := bs.AllKeys(ctx, N/3, N/3)
87 if err != nil {
88 t.Fatal(err)
89 }
@@ -76,6 +93,114 @@ func TestAllKeys(t *testing.T) {
93 if len(keys3) != N/3 {
94 t.Errorf("keys3 should be: %d != %d", N/3, len(keys3))
95 }
96 +}
97 +
98 +func TestAllKeysRespectsContext(t *testing.T) {
99 + N := 100
100 +
101 + d := &queryTestDS{ds: ds.NewMapDatastore()}
102 + bs, _ := newBlockStoreWithKeys(t, d, N)
103 +
104 + started := make(chan struct{}, 1)
105 + done := make(chan struct{}, 1)
106 + errors := make(chan error, 100)
107 +
108 + getKeys := func(ctx context.Context) {
109 + started <- struct{}{}
110 + _, err := bs.AllKeys(ctx, 0, 0) // once without cancelling
111 + if err != nil {
112 + errors <- err
113 + }
114 + done <- struct{}{}
115 + errors <- nil // a nil one to signal break
116 + }
117 +
118 + // Once without context, to make sure it all works
119 + {
120 + var results dsq.Results
121 + resultChan := make(chan dsq.Result)
122 + d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
123 + results = dsq.ResultsWithChan(q, resultChan)
124 + return results, nil
125 + })
126 +
127 + go getKeys(context.Background())
128 +
129 + // make sure it's waiting.
130 + <-started
131 + select {
132 + case <-done:
133 + t.Fatal("sync is wrong")
134 + case <-results.Process().Closing():
135 + t.Fatal("should not be closing")
136 + case <-results.Process().Closed():
137 + t.Fatal("should not be closed")
138 + default:
139 + }
140 +
141 + e := dsq.Entry{Key: BlockPrefix.ChildString("foo").String()}
142 + resultChan <- dsq.Result{Entry: e} // let it go.
143 + close(resultChan)
144 + <-done // should be done now.
145 + <-results.Process().Closed() // should be closed now
146 +
147 + // print any errors
148 + for err := range errors {
149 + if err == nil {
150 + break
151 + }
152 + t.Error(err)
153 + }
154 + }
155 +
156 + // Once with
157 + {
158 + var results dsq.Results
159 + resultChan := make(chan dsq.Result)
160 + d.SetFunc(func(q dsq.Query) (dsq.Results, error) {
161 + results = dsq.ResultsWithChan(q, resultChan)
162 + return results, nil
163 + })
164 +
165 + ctx, cancel := context.WithCancel(context.Background())
166 + go getKeys(ctx)
167 +
168 + // make sure it's waiting.
169 + <-started
170 + select {
171 + case <-done:
172 + t.Fatal("sync is wrong")
173 + case <-results.Process().Closing():
174 + t.Fatal("should not be closing")
175 + case <-results.Process().Closed():
176 + t.Fatal("should not be closed")
177 + default:
178 + }
179 +
180 + cancel() // let it go.
181 +
182 + select {
183 + case <-done:
184 + t.Fatal("sync is wrong")
185 + case <-results.Process().Closed():
186 + t.Fatal("should not be closed") // should not be closed yet.
187 + case <-results.Process().Closing():
188 + // should be closing now!
189 + t.Log("closing correctly at this point.")
190 + }
191 +
192 + close(resultChan)
193 + <-done // should be done now.
194 + <-results.Process().Closed() // should be closed now
195 +
196 + // print any errors
197 + for err := range errors {
198 + if err == nil {
199 + break
200 + }
201 + t.Error(err)
202 + }
203 + }
204
205 }
206
@@ -111,3 +236,33 @@ func expectMatches(t *testing.T, expect, actual []u.Key) {
236 }
237 }
238 }
239 +
240 +type queryTestDS struct {
241 + cb func(q dsq.Query) (dsq.Results, error)
242 + ds ds.Datastore
243 +}
244 +
245 +func (c *queryTestDS) SetFunc(f func(dsq.Query) (dsq.Results, error)) { c.cb = f }
246 +
247 +func (c *queryTestDS) Put(key ds.Key, value interface{}) (err error) {
248 + return c.ds.Put(key, value)
249 +}
250 +
251 +func (c *queryTestDS) Get(key ds.Key) (value interface{}, err error) {
252 + return c.ds.Get(key)
253 +}
254 +
255 +func (c *queryTestDS) Has(key ds.Key) (exists bool, err error) {
256 + return c.ds.Has(key)
257 +}
258 +
259 +func (c *queryTestDS) Delete(key ds.Key) (err error) {
260 + return c.ds.Delete(key)
261 +}
262 +
263 +func (c *queryTestDS) Query(q dsq.Query) (dsq.Results, error) {
264 + if c.cb != nil {
265 + return c.cb(q)
266 + }
267 + return c.ds.Query(q)
268 +}
blocks/blockstore/write_cache.go
+8 -2
@@ -1,7 +1,9 @@
1 package blockstore
2
3 import (
4 + context "github.com/jbenet/go-ipfs/Godeps/_workspace/src/code.google.com/p/go.net/context"
5 "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/hashicorp/golang-lru"
6 +
7 "github.com/jbenet/go-ipfs/blocks"
8 u "github.com/jbenet/go-ipfs/util"
9 )
@@ -44,6 +46,10 @@ func (w *writecache) Put(b *blocks.Block) error {
46 return w.blockstore.Put(b)
47 }
48
47 -func (w *writecache) AllKeys(offset int, limit int) ([]u.Key, error) {
48 - return w.blockstore.AllKeys(offset, limit)
49 +func (w *writecache) AllKeys(ctx context.Context, offset int, limit int) ([]u.Key, error) {
50 + return w.blockstore.AllKeys(ctx, offset, limit)
51 +}
52 +
53 +func (w *writecache) AllKeysChan(ctx context.Context, offset int, limit int) (<-chan u.Key, error) {
54 + return w.blockstore.AllKeysChan(ctx, offset, limit)
55 }
blocks/blockstore/write_cache_test.go
+1 -1
@@ -84,7 +84,7 @@ func (c *callbackDatastore) Delete(key ds.Key) (err error) {
84 return c.ds.Delete(key)
85 }
86
87 -func (c *callbackDatastore) Query(q dsq.Query) (*dsq.Results, error) {
87 +func (c *callbackDatastore) Query(q dsq.Query) (dsq.Results, error) {
88 c.f()
89 return c.ds.Query(q)
90 }
core/commands/refs.go
+1 -1
@@ -128,7 +128,7 @@ Displays the hashes of all local objects.
128 }
129
130 // todo: make async
131 - allKeys, err := n.Blockstore.AllKeys(0, 0)
131 + allKeys, err := n.Blockstore.AllKeys(context.TODO(), 0, 0)
132 if err != nil {
133 return nil, err
134 }
util/datastore2/delayed.go
+1 -1
@@ -36,7 +36,7 @@ func (dds *delayed) Delete(key ds.Key) (err error) {
36 return dds.ds.Delete(key)
37 }
38
39 -func (dds *delayed) Query(q dsq.Query) (*dsq.Results, error) {
39 +func (dds *delayed) Query(q dsq.Query) (dsq.Results, error) {
40 dds.delay.Wait()
41 return dds.ds.Query(q)
42 }