@cryptotaxi247 / kubo / commits / 4f4b89462

Update vendored go-datastore, now has flatfs and mount

Tommi Virtanen committed Mar 16, 2015 at 13:59 UTC 4f4b894627d56e0c1a88f1dd3905b0d017fc10ae
19 files changed +1742 -10
Godeps/Godeps.json
+1 -1
@@ -141,7 +141,7 @@
141 },
142 {
143 "ImportPath": "github.com/jbenet/go-datastore",
144 - "Rev": "35738aceb35505bd3c77c2a618fb1947ca3f72da"
144 + "Rev": "f1a0a0fd88f23b67589957f02b7500372aca186f"
145 },
146 {
147 "ImportPath": "github.com/jbenet/go-detect-race",
Godeps/_workspace/src/github.com/jbenet/go-datastore/Godeps/Godeps.json
+2 -2
@@ -1,6 +1,6 @@
1 {
2 "ImportPath": "github.com/jbenet/go-datastore",
3 - "GoVersion": "go1.3",
3 + "GoVersion": "go1.4",
4 "Packages": [
5 "./..."
6 ],
@@ -20,7 +20,7 @@
20 },
21 {
22 "ImportPath": "github.com/jbenet/goprocess",
23 - "Rev": "b4b4178efcf2404ce9db72438c9c49db2fb399d8"
23 + "Rev": "5b02f8d275a2dd882fb06f8bbdf74347795ff3b1"
24 },
25 {
26 "ImportPath": "github.com/mattbaird/elastigo/api",
Godeps/_workspace/src/github.com/jbenet/go-datastore/Makefile
+2 -2
@@ -1,8 +1,8 @@
1 build:
2 go build
3
4 -test:
5 - go test ./...
4 +test: build
5 + go test -race -cpu=5 -v ./...
6
7 # saves/vendors third-party dependencies to Godeps/_workspace
8 # -r flag rewrites import paths to use the vendored path
Godeps/_workspace/src/github.com/jbenet/go-datastore/README.md
+1 -1
@@ -8,7 +8,7 @@ Based on [datastore.py](https://github.com/datastore/datastore).
8
9 ### Documentation
10
11 -https://godoc.org/github.com/datastore/go-datastore
11 +https://godoc.org/github.com/jbenet/go-datastore
12
13 ### License
14
Godeps/_workspace/src/github.com/jbenet/go-datastore/basic_ds.go
+3
@@ -45,6 +45,9 @@ func (d *MapDatastore) Has(key Key) (exists bool, err error) {
45
46 // Delete implements Datastore.Delete
47 func (d *MapDatastore) Delete(key Key) (err error) {
48 + if _, found := d.values[key]; !found {
49 + return ErrNotFound
50 + }
51 delete(d.values, key)
52 return nil
53 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/callback/callback.go new
+42
@@ -0,0 +1,42 @@
1 +package callback
2 +
3 +import (
4 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
5 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
6 +)
7 +
8 +type Datastore struct {
9 + D ds.Datastore
10 + F func()
11 +}
12 +
13 +func Wrap(ds ds.Datastore, f func()) ds.Datastore {
14 + return &Datastore{ds, f}
15 +}
16 +
17 +func (c *Datastore) SetFunc(f func()) { c.F = f }
18 +
19 +func (c *Datastore) Put(key ds.Key, value interface{}) (err error) {
20 + c.F()
21 + return c.D.Put(key, value)
22 +}
23 +
24 +func (c *Datastore) Get(key ds.Key) (value interface{}, err error) {
25 + c.F()
26 + return c.D.Get(key)
27 +}
28 +
29 +func (c *Datastore) Has(key ds.Key) (exists bool, err error) {
30 + c.F()
31 + return c.D.Has(key)
32 +}
33 +
34 +func (c *Datastore) Delete(key ds.Key) (err error) {
35 + c.F()
36 + return c.D.Delete(key)
37 +}
38 +
39 +func (c *Datastore) Query(q dsq.Query) (dsq.Results, error) {
40 + c.F()
41 + return c.D.Query(q)
42 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go new
+126
@@ -0,0 +1,126 @@
1 +package coalesce
2 +
3 +import (
4 + "sync"
5 +
6 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
8 +)
9 +
10 +var (
11 + putKey = "put"
12 + getKey = // parent keys
13 + "get"
14 + hasKey = "has"
15 + deleteKey = "delete"
16 +)
17 +
18 +type keySync struct {
19 + op string
20 + k ds.Key
21 + value interface{}
22 +}
23 +
24 +type valSync struct {
25 + val interface{}
26 + err error
27 + done chan struct{}
28 +}
29 +
30 +// Datastore uses golang-lru for internal storage.
31 +type datastore struct {
32 + child ds.Datastore
33 +
34 + reqmu sync.Mutex
35 + req map[keySync]*valSync
36 +}
37 +
38 +// Wrap wraps a given datastore with a coalescing datastore.
39 +// All simultaenous requests which have the same keys will
40 +// yield the exact same result. Note that this shares
41 +// memory. It is not possible to copy a generic interface{}
42 +func Wrap(d ds.Datastore) ds.Datastore {
43 + return &datastore{child: d, req: make(map[keySync]*valSync)}
44 +}
45 +
46 +// sync synchronizes requests for a given key.
47 +func (d *datastore) sync(k keySync) (vs *valSync, found bool) {
48 + d.reqmu.Lock()
49 + vs, found = d.req[k]
50 + if !found {
51 + vs = &valSync{done: make(chan struct{})}
52 + d.req[k] = vs
53 + }
54 + d.reqmu.Unlock()
55 +
56 + // if we did find one, wait till it's done.
57 + if found {
58 + <-vs.done
59 + }
60 + return vs, found
61 +}
62 +
63 +// sync synchronizes requests for a given key.
64 +func (d *datastore) syncDone(k keySync) {
65 +
66 + d.reqmu.Lock()
67 + vs, found := d.req[k]
68 + if !found {
69 + panic("attempt to syncDone non-existent request")
70 + }
71 + delete(d.req, k)
72 + d.reqmu.Unlock()
73 +
74 + // release all the waiters.
75 + close(vs.done)
76 +}
77 +
78 +// Put stores the object `value` named by `key`.
79 +func (d *datastore) Put(key ds.Key, value interface{}) (err error) {
80 + ks := keySync{putKey, key, value}
81 + vs, found := d.sync(ks)
82 + if !found {
83 + vs.err = d.child.Put(key, value)
84 + d.syncDone(ks)
85 + }
86 + return err
87 +}
88 +
89 +// Get retrieves the object `value` named by `key`.
90 +func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
91 + ks := keySync{getKey, key, nil}
92 + vs, found := d.sync(ks)
93 + if !found {
94 + vs.val, vs.err = d.child.Get(key)
95 + d.syncDone(ks)
96 + }
97 + return vs.val, vs.err
98 +}
99 +
100 +// Has returns whether the `key` is mapped to a `value`.
101 +func (d *datastore) Has(key ds.Key) (exists bool, err error) {
102 + ks := keySync{hasKey, key, nil}
103 + vs, found := d.sync(ks)
104 + if !found {
105 + vs.val, vs.err = d.child.Has(key)
106 + d.syncDone(ks)
107 + }
108 + return vs.val.(bool), vs.err
109 +}
110 +
111 +// Delete removes the value for given `key`.
112 +func (d *datastore) Delete(key ds.Key) (err error) {
113 + ks := keySync{deleteKey, key, nil}
114 + vs, found := d.sync(ks)
115 + if !found {
116 + vs.err = d.child.Delete(key)
117 + d.syncDone(ks)
118 + }
119 + return vs.err
120 +}
121 +
122 +// Query returns a list of keys in the datastore
123 +func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
124 + // query not coalesced yet.
125 + return d.child.Query(q)
126 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce_test.go new
+299
@@ -0,0 +1,299 @@
1 +package coalesce
2 +
3 +import (
4 + "fmt"
5 + "sync"
6 + "testing"
7 + "time"
8 +
9 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 + dscb "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/callback"
11 + dssync "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/sync"
12 +)
13 +
14 +type mock struct {
15 + sync.Mutex
16 +
17 + inside int
18 + outside int
19 + ds ds.Datastore
20 +}
21 +
22 +func setup() *mock {
23 + m := &mock{}
24 +
25 + mp := ds.NewMapDatastore()
26 + ts := dssync.MutexWrap(mp)
27 + cb1 := dscb.Wrap(ts, func() {
28 + m.Lock()
29 + m.inside++
30 + m.Unlock()
31 + <-time.After(20 * time.Millisecond)
32 + })
33 + cd := Wrap(cb1)
34 + cb2 := dscb.Wrap(cd, func() {
35 + m.Lock()
36 + m.outside++
37 + m.Unlock()
38 + })
39 +
40 + m.ds = cb2
41 + return m
42 +}
43 +
44 +func TestCoalesceSamePut(t *testing.T) {
45 + m := setup()
46 + done := make(chan struct{})
47 +
48 + go func() {
49 + m.ds.Put(ds.NewKey("foo"), "bar")
50 + done <- struct{}{}
51 + }()
52 + go func() {
53 + m.ds.Put(ds.NewKey("foo"), "bar")
54 + done <- struct{}{}
55 + }()
56 + go func() {
57 + m.ds.Put(ds.NewKey("foo"), "bar")
58 + done <- struct{}{}
59 + }()
60 +
61 + <-done
62 + <-done
63 + <-done
64 +
65 + if m.inside != 1 {
66 + t.Error("incalls should be 1", m.inside)
67 + }
68 +
69 + if m.outside != 3 {
70 + t.Error("outcalls should be 3", m.outside)
71 + }
72 +}
73 +
74 +func TestCoalesceSamePutDiffPut(t *testing.T) {
75 + m := setup()
76 + done := make(chan struct{})
77 +
78 + go func() {
79 + m.ds.Put(ds.NewKey("foo"), "bar")
80 + done <- struct{}{}
81 + }()
82 + go func() {
83 + m.ds.Put(ds.NewKey("foo"), "bar")
84 + done <- struct{}{}
85 + }()
86 + go func() {
87 + m.ds.Put(ds.NewKey("foo"), "bar2")
88 + done <- struct{}{}
89 + }()
90 + go func() {
91 + m.ds.Put(ds.NewKey("foo"), "bar3")
92 + done <- struct{}{}
93 + }()
94 +
95 + <-done
96 + <-done
97 + <-done
98 + <-done
99 +
100 + if m.inside != 3 {
101 + t.Error("incalls should be 3", m.inside)
102 + }
103 +
104 + if m.outside != 4 {
105 + t.Error("outcalls should be 4", m.outside)
106 + }
107 +}
108 +
109 +func TestCoalesceSameGet(t *testing.T) {
110 + m := setup()
111 + done := make(chan struct{})
112 + errs := make(chan error, 30)
113 +
114 + m.ds.Put(ds.NewKey("foo1"), "bar")
115 + m.ds.Put(ds.NewKey("foo2"), "baz")
116 +
117 + for i := 0; i < 10; i++ {
118 + go func() {
119 + v, err := m.ds.Get(ds.NewKey("foo1"))
120 + if err != nil {
121 + errs <- err
122 + }
123 + if v != "bar" {
124 + errs <- fmt.Errorf("v is not bar", v)
125 + }
126 + done <- struct{}{}
127 + }()
128 + }
129 + for i := 0; i < 10; i++ {
130 + go func() {
131 + v, err := m.ds.Get(ds.NewKey("foo2"))
132 + if err != nil {
133 + errs <- err
134 + }
135 + if v != "baz" {
136 + errs <- fmt.Errorf("v is not baz", v)
137 + }
138 + done <- struct{}{}
139 + }()
140 + }
141 + for i := 0; i < 10; i++ {
142 + go func() {
143 + _, err := m.ds.Get(ds.NewKey("foo3"))
144 + if err == nil {
145 + errs <- fmt.Errorf("no error")
146 + }
147 + done <- struct{}{}
148 + }()
149 + }
150 +
151 + for i := 0; i < 30; i++ {
152 + <-done
153 + }
154 +
155 + if m.inside != 5 {
156 + t.Error("incalls should be 3", m.inside)
157 + }
158 +
159 + if m.outside != 32 {
160 + t.Error("outcalls should be 30", m.outside)
161 + }
162 +}
163 +
164 +func TestCoalesceHas(t *testing.T) {
165 + m := setup()
166 + done := make(chan struct{})
167 + errs := make(chan error, 30)
168 +
169 + m.ds.Put(ds.NewKey("foo1"), "bar")
170 + m.ds.Put(ds.NewKey("foo2"), "baz")
171 +
172 + for i := 0; i < 10; i++ {
173 + go func() {
174 + v, err := m.ds.Has(ds.NewKey("foo1"))
175 + if err != nil {
176 + errs <- err
177 + }
178 + if !v {
179 + errs <- fmt.Errorf("should have foo1")
180 + }
181 + done <- struct{}{}
182 + }()
183 + }
184 + for i := 0; i < 10; i++ {
185 + go func() {
186 + v, err := m.ds.Has(ds.NewKey("foo2"))
187 + if err != nil {
188 + errs <- err
189 + }
190 + if !v {
191 + errs <- fmt.Errorf("should have foo2")
192 + }
193 + done <- struct{}{}
194 + }()
195 + }
196 + for i := 0; i < 10; i++ {
197 + go func() {
198 + v, err := m.ds.Has(ds.NewKey("foo3"))
199 + if err != nil {
200 + errs <- err
201 + }
202 + if v {
203 + errs <- fmt.Errorf("should not have foo3")
204 + }
205 + done <- struct{}{}
206 + }()
207 + }
208 +
209 + for i := 0; i < 30; i++ {
210 + <-done
211 + }
212 +
213 + if m.inside != 5 {
214 + t.Error("incalls should be 3", m.inside)
215 + }
216 +
217 + if m.outside != 32 {
218 + t.Error("outcalls should be 30", m.outside)
219 + }
220 +}
221 +
222 +func TestCoalesceDelete(t *testing.T) {
223 + m := setup()
224 + done := make(chan struct{})
225 + errs := make(chan error, 30)
226 +
227 + m.ds.Put(ds.NewKey("foo1"), "bar1")
228 + m.ds.Put(ds.NewKey("foo2"), "bar2")
229 + m.ds.Put(ds.NewKey("foo3"), "bar3")
230 +
231 + for i := 0; i < 10; i++ {
232 + go func() {
233 + err := m.ds.Delete(ds.NewKey("foo1"))
234 + if err != nil {
235 + errs <- err
236 + }
237 + has, err := m.ds.Has(ds.NewKey("foo1"))
238 + if err != nil {
239 + errs <- err
240 + }
241 + if has {
242 + t.Error("still have it after deleting")
243 + }
244 + done <- struct{}{}
245 + }()
246 + }
247 + for i := 0; i < 10; i++ {
248 + go func() {
249 + err := m.ds.Delete(ds.NewKey("foo2"))
250 + if err != nil {
251 + errs <- err
252 + }
253 + has, err := m.ds.Has(ds.NewKey("foo2"))
254 + if err != nil {
255 + errs <- err
256 + }
257 + if has {
258 + t.Error("still have it after deleting")
259 + }
260 + done <- struct{}{}
261 + }()
262 + }
263 + for i := 0; i < 10; i++ {
264 + go func() {
265 + has, err := m.ds.Has(ds.NewKey("foo3"))
266 + if err != nil {
267 + errs <- err
268 + }
269 + if !has {
270 + t.Error("should still have foo3")
271 + }
272 + done <- struct{}{}
273 + }()
274 + }
275 + for i := 0; i < 10; i++ {
276 + go func() {
277 + has, err := m.ds.Has(ds.NewKey("foo4"))
278 + if err != nil {
279 + errs <- err
280 + }
281 + if has {
282 + t.Error("should not have foo4")
283 + }
284 + done <- struct{}{}
285 + }()
286 + }
287 +
288 + for i := 0; i < 40; i++ {
289 + <-done
290 + }
291 +
292 + if m.inside != 9 {
293 + t.Error("incalls should be 9", m.inside)
294 + }
295 +
296 + if m.outside != 63 {
297 + t.Error("outcalls should be 63", m.outside)
298 + }
299 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/datastore.go
+2 -2
@@ -39,7 +39,7 @@ type Datastore interface {
39 // Ultimately, the lowest-level datastore will need to do some value checking
40 // or risk getting incorrect values. It may also be useful to expose a more
41 // type-safe interface to your application, and do the checking up-front.
42 - Put(key Key, value interface{}) (err error)
42 + Put(key Key, value interface{}) error
43
44 // Get retrieves the object `value` named by `key`.
45 // Get will return ErrNotFound if the key is not mapped to a value.
@@ -52,7 +52,7 @@ type Datastore interface {
52 Has(key Key) (exists bool, err error)
53
54 // Delete removes the value for given `key`.
55 - Delete(key Key) (err error)
55 + Delete(key Key) error
56
57 // Query searches the datastore and returns a query result. This function
58 // may return before the query actually runs. To wait for the query:
Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go new
+241
@@ -0,0 +1,241 @@
1 +// Package flatfs is a Datastore implementation that stores all
2 +// objects in a two-level directory structure in the local file
3 +// system, regardless of the hierarchy of the keys.
4 +package flatfs
5 +
6 +import (
7 + "encoding/hex"
8 + "errors"
9 + "io/ioutil"
10 + "os"
11 + "path"
12 + "strings"
13 +
14 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
15 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
16 +)
17 +
18 +const (
19 + extension = ".data"
20 + maxPrefixLen = 16
21 +)
22 +
23 +var (
24 + ErrBadPrefixLen = errors.New("bad prefix length")
25 +)
26 +
27 +type Datastore struct {
28 + path string
29 + // length of the dir splay prefix, in bytes of hex digits
30 + hexPrefixLen int
31 +}
32 +
33 +var _ datastore.Datastore = (*Datastore)(nil)
34 +
35 +func New(path string, prefixLen int) (*Datastore, error) {
36 + if prefixLen <= 0 || prefixLen > maxPrefixLen {
37 + return nil, ErrBadPrefixLen
38 + }
39 + fs := &Datastore{
40 + path: path,
41 + // convert from binary bytes to bytes of hex encoding
42 + hexPrefixLen: prefixLen * hex.EncodedLen(1),
43 + }
44 + return fs, nil
45 +}
46 +
47 +var padding = strings.Repeat("_", maxPrefixLen*hex.EncodedLen(1))
48 +
49 +func (fs *Datastore) encode(key datastore.Key) (dir, file string) {
50 + safe := hex.EncodeToString(key.Bytes())
51 + prefix := (safe + padding)[:fs.hexPrefixLen]
52 + dir = path.Join(fs.path, prefix)
53 + file = path.Join(dir, safe+extension)
54 + return dir, file
55 +}
56 +
57 +func (fs *Datastore) decode(file string) (key datastore.Key, ok bool) {
58 + if path.Ext(file) != extension {
59 + return datastore.Key{}, false
60 + }
61 + name := file[:len(file)-len(extension)]
62 + k, err := hex.DecodeString(name)
63 + if err != nil {
64 + return datastore.Key{}, false
65 + }
66 + return datastore.NewKey(string(k)), true
67 +}
68 +
69 +func (fs *Datastore) makePrefixDir(dir string) error {
70 + if err := os.Mkdir(dir, 0777); err != nil {
71 + // EEXIST is safe to ignore here, that just means the prefix
72 + // directory already existed.
73 + if !os.IsExist(err) {
74 + return err
75 + }
76 + }
77 +
78 + // In theory, if we create a new prefix dir and add a file to
79 + // it, the creation of the prefix dir itself might not be
80 + // durable yet. Sync the root dir after a successful mkdir of
81 + // a prefix dir, just to be paranoid.
82 + f, err := os.Open(fs.path)
83 + if err != nil {
84 + return err
85 + }
86 + defer f.Close()
87 + if err := f.Sync(); err != nil {
88 + return err
89 + }
90 + return nil
91 +}
92 +
93 +func (fs *Datastore) Put(key datastore.Key, value interface{}) error {
94 + val, ok := value.([]byte)
95 + if !ok {
96 + return datastore.ErrInvalidType
97 + }
98 +
99 + dir, path := fs.encode(key)
100 + if err := fs.makePrefixDir(dir); err != nil {
101 + return err
102 + }
103 +
104 + dirF, err := os.Open(dir)
105 + if err != nil {
106 + return err
107 + }
108 + defer dirF.Close()
109 +
110 + tmp, err := ioutil.TempFile(dir, "put-")
111 + if err != nil {
112 + return err
113 + }
114 + closed := false
115 + removed := false
116 + defer func() {
117 + if !closed {
118 + // silence errcheck
119 + _ = tmp.Close()
120 + }
121 + if !removed {
122 + // silence errcheck
123 + _ = os.Remove(tmp.Name())
124 + }
125 + }()
126 +
127 + if _, err := tmp.Write(val); err != nil {
128 + return err
129 + }
130 + if err := tmp.Sync(); err != nil {
131 + return err
132 + }
133 + if err := tmp.Close(); err != nil {
134 + return err
135 + }
136 + closed = true
137 +
138 + err = os.Rename(tmp.Name(), path)
139 + if err != nil {
140 + return err
141 + }
142 + removed = true
143 +
144 + if err := dirF.Sync(); err != nil {
145 + return err
146 + }
147 +
148 + return nil
149 +}
150 +
151 +func (fs *Datastore) Get(key datastore.Key) (value interface{}, err error) {
152 + _, path := fs.encode(key)
153 + data, err := ioutil.ReadFile(path)
154 + if err != nil {
155 + if os.IsNotExist(err) {
156 + return nil, datastore.ErrNotFound
157 + }
158 + // no specific error to return, so just pass it through
159 + return nil, err
160 + }
161 + return data, nil
162 +}
163 +
164 +func (fs *Datastore) Has(key datastore.Key) (exists bool, err error) {
165 + _, path := fs.encode(key)
166 + switch _, err := os.Stat(path); {
167 + case err == nil:
168 + return true, nil
169 + case os.IsNotExist(err):
170 + return false, nil
171 + default:
172 + return false, err
173 + }
174 +}
175 +
176 +func (fs *Datastore) Delete(key datastore.Key) error {
177 + _, path := fs.encode(key)
178 + switch err := os.Remove(path); {
179 + case err == nil:
180 + return nil
181 + case os.IsNotExist(err):
182 + return datastore.ErrNotFound
183 + default:
184 + return err
185 + }
186 +}
187 +
188 +func (fs *Datastore) Query(q query.Query) (query.Results, error) {
189 + if (q.Prefix != "" && q.Prefix != "/") ||
190 + len(q.Filters) > 0 ||
191 + len(q.Orders) > 0 ||
192 + q.Limit > 0 ||
193 + q.Offset > 0 ||
194 + !q.KeysOnly {
195 + // TODO this is overly simplistic, but the only caller is
196 + // `ipfs refs local` for now, and this gets us moving.
197 + return nil, errors.New("flatfs only supports listing all keys in random order")
198 + }
199 +
200 + // TODO this dumb implementation gathers all keys into a single slice.
201 + root, err := os.Open(fs.path)
202 + if err != nil {
203 + return nil, err
204 + }
205 + defer root.Close()
206 +
207 + var res []query.Entry
208 + prefixes, err := root.Readdir(0)
209 + if err != nil {
210 + return nil, err
211 + }
212 + for _, fi := range prefixes {
213 + if !fi.IsDir() || fi.Name()[0] == '.' {
214 + continue
215 + }
216 + child, err := os.Open(path.Join(fs.path, fi.Name()))
217 + if err != nil {
218 + return nil, err
219 + }
220 + defer child.Close()
221 + objs, err := child.Readdir(0)
222 + if err != nil {
223 + return nil, err
224 + }
225 + for _, fi := range objs {
226 + if !fi.Mode().IsRegular() || fi.Name()[0] == '.' {
227 + continue
228 + }
229 + key, ok := fs.decode(fi.Name())
230 + if !ok {
231 + continue
232 + }
233 + res = append(res, query.Entry{Key: key.String()})
234 + }
235 + }
236 + return query.ResultsWithEntries(q, res), nil
237 +}
238 +
239 +var _ datastore.ThreadSafeDatastore = (*Datastore)(nil)
240 +
241 +func (*Datastore) IsThreadSafe() {}
Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go new
+315
@@ -0,0 +1,315 @@
1 +package flatfs_test
2 +
3 +import (
4 + "io/ioutil"
5 + "os"
6 + "path/filepath"
7 + "testing"
8 +
9 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
11 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 +)
13 +
14 +func tempdir(t testing.TB) (path string, cleanup func()) {
15 + path, err := ioutil.TempDir("", "test-datastore-flatfs-")
16 + if err != nil {
17 + t.Fatalf("cannot create temp directory: %v", err)
18 + }
19 +
20 + cleanup = func() {
21 + if err := os.RemoveAll(path); err != nil {
22 + t.Errorf("tempdir cleanup failed: %v", err)
23 + }
24 + }
25 + return path, cleanup
26 +}
27 +
28 +func TestBadPrefixLen(t *testing.T) {
29 + temp, cleanup := tempdir(t)
30 + defer cleanup()
31 +
32 + for i := 0; i > -3; i-- {
33 + _, err := flatfs.New(temp, 0)
34 + if g, e := err, flatfs.ErrBadPrefixLen; g != e {
35 + t.Errorf("expected ErrBadPrefixLen, got: %v", g)
36 + }
37 + }
38 +}
39 +
40 +func TestPutBadValueType(t *testing.T) {
41 + temp, cleanup := tempdir(t)
42 + defer cleanup()
43 +
44 + fs, err := flatfs.New(temp, 2)
45 + if err != nil {
46 + t.Fatalf("New fail: %v\n", err)
47 + }
48 +
49 + err = fs.Put(datastore.NewKey("quux"), 22)
50 + if g, e := err, datastore.ErrInvalidType; g != e {
51 + t.Fatalf("expected ErrInvalidType, got: %v\n", g)
52 + }
53 +}
54 +
55 +func TestPut(t *testing.T) {
56 + temp, cleanup := tempdir(t)
57 + defer cleanup()
58 +
59 + fs, err := flatfs.New(temp, 2)
60 + if err != nil {
61 + t.Fatalf("New fail: %v\n", err)
62 + }
63 +
64 + err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
65 + if err != nil {
66 + t.Fatalf("Put fail: %v\n", err)
67 + }
68 +}
69 +
70 +func TestGet(t *testing.T) {
71 + temp, cleanup := tempdir(t)
72 + defer cleanup()
73 +
74 + fs, err := flatfs.New(temp, 2)
75 + if err != nil {
76 + t.Fatalf("New fail: %v\n", err)
77 + }
78 +
79 + const input = "foobar"
80 + err = fs.Put(datastore.NewKey("quux"), []byte(input))
81 + if err != nil {
82 + t.Fatalf("Put fail: %v\n", err)
83 + }
84 +
85 + data, err := fs.Get(datastore.NewKey("quux"))
86 + if err != nil {
87 + t.Fatalf("Get failed: %v", err)
88 + }
89 + buf, ok := data.([]byte)
90 + if !ok {
91 + t.Fatalf("expected []byte from Get, got %T: %v", data, data)
92 + }
93 + if g, e := string(buf), input; g != e {
94 + t.Fatalf("Get gave wrong content: %q != %q", g, e)
95 + }
96 +}
97 +
98 +func TestPutOverwrite(t *testing.T) {
99 + temp, cleanup := tempdir(t)
100 + defer cleanup()
101 +
102 + fs, err := flatfs.New(temp, 2)
103 + if err != nil {
104 + t.Fatalf("New fail: %v\n", err)
105 + }
106 +
107 + const (
108 + loser = "foobar"
109 + winner = "xyzzy"
110 + )
111 + err = fs.Put(datastore.NewKey("quux"), []byte(loser))
112 + if err != nil {
113 + t.Fatalf("Put fail: %v\n", err)
114 + }
115 +
116 + err = fs.Put(datastore.NewKey("quux"), []byte(winner))
117 + if err != nil {
118 + t.Fatalf("Put fail: %v\n", err)
119 + }
120 +
121 + data, err := fs.Get(datastore.NewKey("quux"))
122 + if err != nil {
123 + t.Fatalf("Get failed: %v", err)
124 + }
125 + if g, e := string(data.([]byte)), winner; g != e {
126 + t.Fatalf("Get gave wrong content: %q != %q", g, e)
127 + }
128 +}
129 +
130 +func TestGetNotFoundError(t *testing.T) {
131 + temp, cleanup := tempdir(t)
132 + defer cleanup()
133 +
134 + fs, err := flatfs.New(temp, 2)
135 + if err != nil {
136 + t.Fatalf("New fail: %v\n", err)
137 + }
138 +
139 + _, err = fs.Get(datastore.NewKey("quux"))
140 + if g, e := err, datastore.ErrNotFound; g != e {
141 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
142 + }
143 +}
144 +
145 +func TestStorage(t *testing.T) {
146 + temp, cleanup := tempdir(t)
147 + defer cleanup()
148 +
149 + const prefixLen = 2
150 + const prefix = "2f71"
151 + const target = prefix + "/2f71757578.data"
152 + fs, err := flatfs.New(temp, prefixLen)
153 + if err != nil {
154 + t.Fatalf("New fail: %v\n", err)
155 + }
156 +
157 + err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
158 + if err != nil {
159 + t.Fatalf("Put fail: %v\n", err)
160 + }
161 +
162 + seen := false
163 + walk := func(absPath string, fi os.FileInfo, err error) error {
164 + if err != nil {
165 + return err
166 + }
167 + path, err := filepath.Rel(temp, absPath)
168 + if err != nil {
169 + return err
170 + }
171 + switch path {
172 + case ".", "..":
173 + // ignore
174 + case prefix:
175 + if !fi.IsDir() {
176 + t.Errorf("prefix directory is not a file? %v", fi.Mode())
177 + }
178 + // we know it's there if we see the file, nothing more to
179 + // do here
180 + case target:
181 + seen = true
182 + if !fi.Mode().IsRegular() {
183 + t.Errorf("expected a regular file, mode: %04o", fi.Mode())
184 + }
185 + if g, e := fi.Mode()&os.ModePerm&0007, os.FileMode(0000); g != e {
186 + t.Errorf("file should not be world accessible: %04o", fi.Mode())
187 + }
188 + default:
189 + t.Errorf("saw unexpected directory entry: %q %v", path, fi.Mode())
190 + }
191 + return nil
192 + }
193 + if err := filepath.Walk(temp, walk); err != nil {
194 + t.Fatal("walk: %v", err)
195 + }
196 + if !seen {
197 + t.Error("did not see the data file")
198 + }
199 +}
200 +
201 +func TestHasNotFound(t *testing.T) {
202 + temp, cleanup := tempdir(t)
203 + defer cleanup()
204 +
205 + fs, err := flatfs.New(temp, 2)
206 + if err != nil {
207 + t.Fatalf("New fail: %v\n", err)
208 + }
209 +
210 + found, err := fs.Has(datastore.NewKey("quux"))
211 + if err != nil {
212 + t.Fatalf("Has fail: %v\n", err)
213 + }
214 + if g, e := found, false; g != e {
215 + t.Fatalf("wrong Has: %v != %v", g, e)
216 + }
217 +}
218 +
219 +func TestHasFound(t *testing.T) {
220 + temp, cleanup := tempdir(t)
221 + defer cleanup()
222 +
223 + fs, err := flatfs.New(temp, 2)
224 + if err != nil {
225 + t.Fatalf("New fail: %v\n", err)
226 + }
227 + err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
228 + if err != nil {
229 + t.Fatalf("Put fail: %v\n", err)
230 + }
231 +
232 + found, err := fs.Has(datastore.NewKey("quux"))
233 + if err != nil {
234 + t.Fatalf("Has fail: %v\n", err)
235 + }
236 + if g, e := found, true; g != e {
237 + t.Fatalf("wrong Has: %v != %v", g, e)
238 + }
239 +}
240 +
241 +func TestDeleteNotFound(t *testing.T) {
242 + temp, cleanup := tempdir(t)
243 + defer cleanup()
244 +
245 + fs, err := flatfs.New(temp, 2)
246 + if err != nil {
247 + t.Fatalf("New fail: %v\n", err)
248 + }
249 +
250 + err = fs.Delete(datastore.NewKey("quux"))
251 + if g, e := err, datastore.ErrNotFound; g != e {
252 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
253 + }
254 +}
255 +
256 +func TestDeleteFound(t *testing.T) {
257 + temp, cleanup := tempdir(t)
258 + defer cleanup()
259 +
260 + fs, err := flatfs.New(temp, 2)
261 + if err != nil {
262 + t.Fatalf("New fail: %v\n", err)
263 + }
264 + err = fs.Put(datastore.NewKey("quux"), []byte("foobar"))
265 + if err != nil {
266 + t.Fatalf("Put fail: %v\n", err)
267 + }
268 +
269 + err = fs.Delete(datastore.NewKey("quux"))
270 + if err != nil {
271 + t.Fatalf("Delete fail: %v\n", err)
272 + }
273 +
274 + // check that it's gone
275 + _, err = fs.Get(datastore.NewKey("quux"))
276 + if g, e := err, datastore.ErrNotFound; g != e {
277 + t.Fatalf("expected Get after Delete to give ErrNotFound, got: %v\n", g)
278 + }
279 +}
280 +
281 +func TestQuerySimple(t *testing.T) {
282 + temp, cleanup := tempdir(t)
283 + defer cleanup()
284 +
285 + fs, err := flatfs.New(temp, 2)
286 + if err != nil {
287 + t.Fatalf("New fail: %v\n", err)
288 + }
289 + const myKey = "quux"
290 + err = fs.Put(datastore.NewKey(myKey), []byte("foobar"))
291 + if err != nil {
292 + t.Fatalf("Put fail: %v\n", err)
293 + }
294 +
295 + res, err := fs.Query(query.Query{KeysOnly: true})
296 + if err != nil {
297 + t.Fatalf("Query fail: %v\n", err)
298 + }
299 + entries, err := res.Rest()
300 + if err != nil {
301 + t.Fatalf("Query Results.Rest fail: %v\n", err)
302 + }
303 + seen := false
304 + for _, e := range entries {
305 + switch e.Key {
306 + case datastore.NewKey(myKey).String():
307 + seen = true
308 + default:
309 + t.Errorf("saw unexpected key: %q", e.Key)
310 + }
311 + }
312 + if !seen {
313 + t.Errorf("did not see wanted key %q in %+v", myKey, entries)
314 + }
315 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/fs/fs.go
+18 -1
@@ -1,3 +1,20 @@
1 +// Package fs is a simple Datastore implementation that stores keys
2 +// are directories and files, mirroring the key. That is, the key
3 +// "/foo/bar" is stored as file "PATH/foo/bar/.dsobject".
4 +//
5 +// This means key some segments will not work. For example, the
6 +// following keys will result in unwanted behavior:
7 +//
8 +// - "/foo/./bar"
9 +// - "/foo/../bar"
10 +// - "/foo\x00bar"
11 +//
12 +// Keys that only differ in case may be confused with each other on
13 +// case insensitive file systems, for example in OS X.
14 +//
15 +// This package is intended for exploratory use, where the user would
16 +// examine the file system manually, and should only be used with
17 +// human-friendly, trusted keys. You have been warned.
18 package fs
19
20 import (
@@ -13,7 +30,7 @@ import (
30
31 var ObjectKeySuffix = ".dsobject"
32
16 -// Datastore uses a standard Go map for internal storage.
33 +// Datastore uses a uses a file per key to store values.
34 type Datastore struct {
35 path string
36 }
Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go
-1
@@ -9,7 +9,6 @@ import (
9 . "gopkg.in/check.v1"
10 )
11
12 -// Hook up gocheck into the "go test" runner.
12 func Test(t *testing.T) { TestingT(t) }
13
14 type DSSuite struct{}
Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go new
+116
@@ -0,0 +1,116 @@
1 +// Package mount provides a Datastore that has other Datastores
2 +// mounted at various key prefixes.
3 +package mount
4 +
5 +import (
6 + "errors"
7 + "strings"
8 +
9 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform"
11 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12 +)
13 +
14 +var (
15 + ErrNoMount = errors.New("no datastore mounted for this key")
16 +)
17 +
18 +type Mount struct {
19 + Prefix datastore.Key
20 + Datastore datastore.Datastore
21 +}
22 +
23 +func New(mounts []Mount) *Datastore {
24 + // make a copy so we're sure it doesn't mutate
25 + m := make([]Mount, len(mounts))
26 + for i, v := range mounts {
27 + m[i] = v
28 + }
29 + return &Datastore{mounts: m}
30 +}
31 +
32 +type Datastore struct {
33 + mounts []Mount
34 +}
35 +
36 +var _ datastore.Datastore = (*Datastore)(nil)
37 +
38 +func (d *Datastore) lookup(key datastore.Key) (ds datastore.Datastore, mountpoint, rest datastore.Key) {
39 + for _, m := range d.mounts {
40 + if m.Prefix.Equal(key) || m.Prefix.IsAncestorOf(key) {
41 + s := strings.TrimPrefix(key.String(), m.Prefix.String())
42 + k := datastore.NewKey(s)
43 + return m.Datastore, m.Prefix, k
44 + }
45 + }
46 + return nil, datastore.NewKey("/"), key
47 +}
48 +
49 +func (d *Datastore) Put(key datastore.Key, value interface{}) error {
50 + ds, _, k := d.lookup(key)
51 + if ds == nil {
52 + return ErrNoMount
53 + }
54 + return ds.Put(k, value)
55 +}
56 +
57 +func (d *Datastore) Get(key datastore.Key) (value interface{}, err error) {
58 + ds, _, k := d.lookup(key)
59 + if ds == nil {
60 + return nil, datastore.ErrNotFound
61 + }
62 + return ds.Get(k)
63 +}
64 +
65 +func (d *Datastore) Has(key datastore.Key) (exists bool, err error) {
66 + ds, _, k := d.lookup(key)
67 + if ds == nil {
68 + return false, nil
69 + }
70 + return ds.Has(k)
71 +}
72 +
73 +func (d *Datastore) Delete(key datastore.Key) error {
74 + ds, _, k := d.lookup(key)
75 + if ds == nil {
76 + return datastore.ErrNotFound
77 + }
78 + return ds.Delete(k)
79 +}
80 +
81 +func (d *Datastore) Query(q query.Query) (query.Results, error) {
82 + if len(q.Filters) > 0 ||
83 + len(q.Orders) > 0 ||
84 + q.Limit > 0 ||
85 + q.Offset > 0 {
86 + // TODO this is overly simplistic, but the only caller is
87 + // `ipfs refs local` for now, and this gets us moving.
88 + return nil, errors.New("mount only supports listing all prefixed keys in random order")
89 + }
90 + key := datastore.NewKey(q.Prefix)
91 + ds, mount, k := d.lookup(key)
92 + if ds == nil {
93 + return nil, errors.New("mount only supports listing a mount point")
94 + }
95 + // TODO support listing cross mount points too
96 +
97 + // delegate the query to the mounted datastore, while adjusting
98 + // keys in and out
99 + q2 := q
100 + q2.Prefix = k.String()
101 + wrapDS := keytransform.Wrap(ds, &keytransform.Pair{
102 + Convert: func(datastore.Key) datastore.Key {
103 + panic("this should never be called")
104 + },
105 + Invert: func(k datastore.Key) datastore.Key {
106 + return mount.Child(k)
107 + },
108 + })
109 +
110 + r, err := wrapDS.Query(q2)
111 + if err != nil {
112 + return nil, err
113 + }
114 + r = query.ResultsReplaceQuery(r, q)
115 + return r, nil
116 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount_test.go new
+241
@@ -0,0 +1,241 @@
1 +package mount_test
2 +
3 +import (
4 + "testing"
5 +
6 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount"
8 + "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9 +)
10 +
11 +func TestPutBadNothing(t *testing.T) {
12 + m := mount.New(nil)
13 +
14 + err := m.Put(datastore.NewKey("quux"), []byte("foobar"))
15 + if g, e := err, mount.ErrNoMount; g != e {
16 + t.Fatalf("Put got wrong error: %v != %v", g, e)
17 + }
18 +}
19 +
20 +func TestPutBadNoMount(t *testing.T) {
21 + mapds := datastore.NewMapDatastore()
22 + m := mount.New([]mount.Mount{
23 + {Prefix: datastore.NewKey("/redherring"), Datastore: mapds},
24 + })
25 +
26 + err := m.Put(datastore.NewKey("/quux/thud"), []byte("foobar"))
27 + if g, e := err, mount.ErrNoMount; g != e {
28 + t.Fatalf("expected ErrNoMount, got: %v\n", g)
29 + }
30 +}
31 +
32 +func TestPut(t *testing.T) {
33 + mapds := datastore.NewMapDatastore()
34 + m := mount.New([]mount.Mount{
35 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
36 + })
37 +
38 + if err := m.Put(datastore.NewKey("/quux/thud"), []byte("foobar")); err != nil {
39 + t.Fatalf("Put error: %v", err)
40 + }
41 +
42 + val, err := mapds.Get(datastore.NewKey("/thud"))
43 + if err != nil {
44 + t.Fatalf("Get error: %v", err)
45 + }
46 + buf, ok := val.([]byte)
47 + if !ok {
48 + t.Fatalf("Get value is not []byte: %T %v", val, val)
49 + }
50 + if g, e := string(buf), "foobar"; g != e {
51 + t.Errorf("wrong value: %q != %q", g, e)
52 + }
53 +}
54 +
55 +func TestGetBadNothing(t *testing.T) {
56 + m := mount.New([]mount.Mount{})
57 +
58 + _, err := m.Get(datastore.NewKey("/quux/thud"))
59 + if g, e := err, datastore.ErrNotFound; g != e {
60 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
61 + }
62 +}
63 +
64 +func TestGetBadNoMount(t *testing.T) {
65 + mapds := datastore.NewMapDatastore()
66 + m := mount.New([]mount.Mount{
67 + {Prefix: datastore.NewKey("/redherring"), Datastore: mapds},
68 + })
69 +
70 + _, err := m.Get(datastore.NewKey("/quux/thud"))
71 + if g, e := err, datastore.ErrNotFound; g != e {
72 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
73 + }
74 +}
75 +
76 +func TestGetNotFound(t *testing.T) {
77 + mapds := datastore.NewMapDatastore()
78 + m := mount.New([]mount.Mount{
79 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
80 + })
81 +
82 + _, err := m.Get(datastore.NewKey("/quux/thud"))
83 + if g, e := err, datastore.ErrNotFound; g != e {
84 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
85 + }
86 +}
87 +
88 +func TestGet(t *testing.T) {
89 + mapds := datastore.NewMapDatastore()
90 + m := mount.New([]mount.Mount{
91 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
92 + })
93 +
94 + if err := mapds.Put(datastore.NewKey("/thud"), []byte("foobar")); err != nil {
95 + t.Fatalf("Get error: %v", err)
96 + }
97 +
98 + val, err := m.Get(datastore.NewKey("/quux/thud"))
99 + if err != nil {
100 + t.Fatalf("Put error: %v", err)
101 + }
102 +
103 + buf, ok := val.([]byte)
104 + if !ok {
105 + t.Fatalf("Get value is not []byte: %T %v", val, val)
106 + }
107 + if g, e := string(buf), "foobar"; g != e {
108 + t.Errorf("wrong value: %q != %q", g, e)
109 + }
110 +}
111 +
112 +func TestHasBadNothing(t *testing.T) {
113 + m := mount.New([]mount.Mount{})
114 +
115 + found, err := m.Has(datastore.NewKey("/quux/thud"))
116 + if err != nil {
117 + t.Fatalf("Has error: %v", err)
118 + }
119 + if g, e := found, false; g != e {
120 + t.Fatalf("wrong value: %v != %v", g, e)
121 + }
122 +}
123 +
124 +func TestHasBadNoMount(t *testing.T) {
125 + mapds := datastore.NewMapDatastore()
126 + m := mount.New([]mount.Mount{
127 + {Prefix: datastore.NewKey("/redherring"), Datastore: mapds},
128 + })
129 +
130 + found, err := m.Has(datastore.NewKey("/quux/thud"))
131 + if err != nil {
132 + t.Fatalf("Has error: %v", err)
133 + }
134 + if g, e := found, false; g != e {
135 + t.Fatalf("wrong value: %v != %v", g, e)
136 + }
137 +}
138 +
139 +func TestHasNotFound(t *testing.T) {
140 + mapds := datastore.NewMapDatastore()
141 + m := mount.New([]mount.Mount{
142 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
143 + })
144 +
145 + found, err := m.Has(datastore.NewKey("/quux/thud"))
146 + if err != nil {
147 + t.Fatalf("Has error: %v", err)
148 + }
149 + if g, e := found, false; g != e {
150 + t.Fatalf("wrong value: %v != %v", g, e)
151 + }
152 +}
153 +
154 +func TestHas(t *testing.T) {
155 + mapds := datastore.NewMapDatastore()
156 + m := mount.New([]mount.Mount{
157 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
158 + })
159 +
160 + if err := mapds.Put(datastore.NewKey("/thud"), []byte("foobar")); err != nil {
161 + t.Fatalf("Put error: %v", err)
162 + }
163 +
164 + found, err := m.Has(datastore.NewKey("/quux/thud"))
165 + if err != nil {
166 + t.Fatalf("Has error: %v", err)
167 + }
168 + if g, e := found, true; g != e {
169 + t.Fatalf("wrong value: %v != %v", g, e)
170 + }
171 +}
172 +
173 +func TestDeleteNotFound(t *testing.T) {
174 + mapds := datastore.NewMapDatastore()
175 + m := mount.New([]mount.Mount{
176 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
177 + })
178 +
179 + err := m.Delete(datastore.NewKey("/quux/thud"))
180 + if g, e := err, datastore.ErrNotFound; g != e {
181 + t.Fatalf("expected ErrNotFound, got: %v\n", g)
182 + }
183 +}
184 +
185 +func TestDelete(t *testing.T) {
186 + mapds := datastore.NewMapDatastore()
187 + m := mount.New([]mount.Mount{
188 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
189 + })
190 +
191 + if err := mapds.Put(datastore.NewKey("/thud"), []byte("foobar")); err != nil {
192 + t.Fatalf("Put error: %v", err)
193 + }
194 +
195 + err := m.Delete(datastore.NewKey("/quux/thud"))
196 + if err != nil {
197 + t.Fatalf("Delete error: %v", err)
198 + }
199 +
200 + // make sure it disappeared
201 + found, err := mapds.Has(datastore.NewKey("/thud"))
202 + if err != nil {
203 + t.Fatalf("Has error: %v", err)
204 + }
205 + if g, e := found, false; g != e {
206 + t.Fatalf("wrong value: %v != %v", g, e)
207 + }
208 +}
209 +
210 +func TestQuerySimple(t *testing.T) {
211 + mapds := datastore.NewMapDatastore()
212 + m := mount.New([]mount.Mount{
213 + {Prefix: datastore.NewKey("/quux"), Datastore: mapds},
214 + })
215 +
216 + const myKey = "/quux/thud"
217 + if err := m.Put(datastore.NewKey(myKey), []byte("foobar")); err != nil {
218 + t.Fatalf("Put error: %v", err)
219 + }
220 +
221 + res, err := m.Query(query.Query{Prefix: "/quux"})
222 + if err != nil {
223 + t.Fatalf("Query fail: %v\n", err)
224 + }
225 + entries, err := res.Rest()
226 + if err != nil {
227 + t.Fatalf("Query Results.Rest fail: %v\n", err)
228 + }
229 + seen := false
230 + for _, e := range entries {
231 + switch e.Key {
232 + case datastore.NewKey(myKey).String():
233 + seen = true
234 + default:
235 + t.Errorf("saw unexpected key: %q", e.Key)
236 + }
237 + }
238 + if !seen {
239 + t.Errorf("did not see wanted key %q in %+v", myKey, entries)
240 + }
241 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/tiered/tiered.go new
+94
@@ -0,0 +1,94 @@
1 +package tiered
2 +
3 +import (
4 + "fmt"
5 + "sync"
6 +
7 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9 +)
10 +
11 +type tiered []ds.Datastore
12 +
13 +// New returns a tiered datastore. Puts and Deletes will write-through to
14 +// all datastores, Has and Get will try each datastore sequentially, and
15 +// Query will always try the last one (most complete) first.
16 +func New(dses ...ds.Datastore) ds.Datastore {
17 + return tiered(dses)
18 +}
19 +
20 +// Put stores the object `value` named by `key`.
21 +func (d tiered) Put(key ds.Key, value interface{}) (err error) {
22 + errs := make(chan error, len(d))
23 +
24 + var wg sync.WaitGroup
25 + for _, cd := range d {
26 + wg.Add(1)
27 + go func(cd ds.Datastore) {
28 + defer wg.Done()
29 + if err := cd.Put(key, value); err != nil {
30 + errs <- err
31 + }
32 + }(cd)
33 + }
34 + wg.Wait()
35 +
36 + close(errs)
37 + for err := range errs {
38 + return err
39 + }
40 + return nil
41 +}
42 +
43 +// Get retrieves the object `value` named by `key`.
44 +func (d tiered) Get(key ds.Key) (value interface{}, err error) {
45 + err = fmt.Errorf("no datastores")
46 + for _, cd := range d {
47 + value, err = cd.Get(key)
48 + if err == nil {
49 + break
50 + }
51 + }
52 + return
53 +}
54 +
55 +// Has returns whether the `key` is mapped to a `value`.
56 +func (d tiered) Has(key ds.Key) (exists bool, err error) {
57 + err = fmt.Errorf("no datastores")
58 + for _, cd := range d {
59 + exists, err = cd.Has(key)
60 + if err == nil && exists {
61 + break
62 + }
63 + }
64 + return
65 +}
66 +
67 +// Delete removes the value for given `key`.
68 +func (d tiered) Delete(key ds.Key) (err error) {
69 + errs := make(chan error, len(d))
70 +
71 + var wg sync.WaitGroup
72 + for _, cd := range d {
73 + wg.Add(1)
74 + go func(cd ds.Datastore) {
75 + defer wg.Done()
76 + if err := cd.Delete(key); err != nil {
77 + errs <- err
78 + }
79 + }(cd)
80 + }
81 + wg.Wait()
82 +
83 + close(errs)
84 + for err := range errs {
85 + return err
86 + }
87 + return nil
88 +}
89 +
90 +// Query returns a list of keys in the datastore
91 +func (d tiered) Query(q dsq.Query) (dsq.Results, error) {
92 + // query always the last (most complete) one
93 + return d[len(d)-1].Query(q)
94 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/tiered/tiered_test.go new
+79
@@ -0,0 +1,79 @@
1 +package tiered
2 +
3 +import (
4 + "testing"
5 +
6 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7 + dscb "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/callback"
8 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9 +)
10 +
11 +func testHas(t *testing.T, dses []ds.Datastore, k ds.Key, v interface{}) {
12 + // all under should have it
13 + for _, d := range dses {
14 + if v2, err := d.Get(k); err != nil {
15 + t.Error(err)
16 + } else if v2 != v {
17 + t.Error("value incorrect", d, k, v, v2)
18 + }
19 +
20 + if has, err := d.Has(k); err != nil {
21 + t.Error(err)
22 + } else if !has {
23 + t.Error("should have it", d, k, v)
24 + }
25 + }
26 +}
27 +
28 +func testNotHas(t *testing.T, dses []ds.Datastore, k ds.Key) {
29 + // all under should not have it
30 + for _, d := range dses {
31 + if _, err := d.Get(k); err == nil {
32 + t.Error("should not have it", d, k)
33 + }
34 +
35 + if has, err := d.Has(k); err != nil {
36 + t.Error(err)
37 + } else if has {
38 + t.Error("should not have it", d, k)
39 + }
40 + }
41 +}
42 +
43 +func TestTiered(t *testing.T) {
44 + d1 := ds.NewMapDatastore()
45 + d2 := ds.NewMapDatastore()
46 + d3 := ds.NewMapDatastore()
47 + d4 := ds.NewMapDatastore()
48 +
49 + td := New(d1, d2, d3, d4)
50 + td.Put(ds.NewKey("foo"), "bar")
51 + testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
52 + testHas(t, td.(tiered), ds.NewKey("foo"), "bar") // all children
53 +
54 + // remove it from, say, caches.
55 + d1.Delete(ds.NewKey("foo"))
56 + d2.Delete(ds.NewKey("foo"))
57 + testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar")
58 + testHas(t, td.(tiered)[2:], ds.NewKey("foo"), "bar")
59 + testNotHas(t, td.(tiered)[:2], ds.NewKey("foo"))
60 +
61 + // write it again.
62 + td.Put(ds.NewKey("foo"), "bar2")
63 + testHas(t, []ds.Datastore{td}, ds.NewKey("foo"), "bar2")
64 + testHas(t, td.(tiered), ds.NewKey("foo"), "bar2")
65 +}
66 +
67 +func TestQueryCallsLast(t *testing.T) {
68 + var d1n, d2n, d3n int
69 + d1 := dscb.Wrap(ds.NewMapDatastore(), func() { d1n++ })
70 + d2 := dscb.Wrap(ds.NewMapDatastore(), func() { d2n++ })
71 + d3 := dscb.Wrap(ds.NewMapDatastore(), func() { d3n++ })
72 +
73 + td := New(d1, d2, d3)
74 +
75 + td.Query(dsq.Query{})
76 + if d3n < 1 {
77 + t.Error("should call last")
78 + }
79 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go new
+96
@@ -0,0 +1,96 @@
1 +package timecache
2 +
3 +import (
4 + "sync"
5 + "time"
6 +
7 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 + dsq "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9 +)
10 +
11 +var (
12 + putKey = "put"
13 + getKey = // op keys
14 + "get"
15 + hasKey = "has"
16 + deleteKey = "delete"
17 +)
18 +
19 +type datastore struct {
20 + cache ds.Datastore
21 + ttl time.Duration
22 +
23 + ttlmu sync.Mutex
24 + ttls map[ds.Key]time.Time
25 +}
26 +
27 +func WithTTL(ttl time.Duration) ds.Datastore {
28 + return WithCache(ds.NewMapDatastore(), ttl)
29 +}
30 +
31 +// WithCache wraps a given datastore as a timecache.
32 +// Get + Has requests are considered expired after a TTL.
33 +func WithCache(d ds.Datastore, ttl time.Duration) ds.Datastore {
34 + return &datastore{cache: d, ttl: ttl, ttls: make(map[ds.Key]time.Time)}
35 +}
36 +
37 +func (d *datastore) gc() {
38 + var now = time.Now()
39 + var del []ds.Key
40 +
41 + // remove all expired ttls.
42 + d.ttlmu.Lock()
43 + for k, ttl := range d.ttls {
44 + if now.After(ttl) {
45 + delete(d.ttls, k)
46 + del = append(del, k)
47 + }
48 + }
49 + d.ttlmu.Unlock()
50 +
51 + for _, k := range del {
52 + d.cache.Delete(k)
53 + }
54 +}
55 +
56 +func (d *datastore) ttlPut(key ds.Key) {
57 + d.ttlmu.Lock()
58 + d.ttls[key] = time.Now().Add(d.ttl)
59 + d.ttlmu.Unlock()
60 +}
61 +
62 +func (d *datastore) ttlDelete(key ds.Key) {
63 + d.ttlmu.Lock()
64 + delete(d.ttls, key)
65 + d.ttlmu.Unlock()
66 +}
67 +
68 +// Put stores the object `value` named by `key`.
69 +func (d *datastore) Put(key ds.Key, value interface{}) (err error) {
70 + err = d.cache.Put(key, value)
71 + d.ttlPut(key)
72 + return err
73 +}
74 +
75 +// Get retrieves the object `value` named by `key`.
76 +func (d *datastore) Get(key ds.Key) (value interface{}, err error) {
77 + d.gc()
78 + return d.cache.Get(key)
79 +}
80 +
81 +// Has returns whether the `key` is mapped to a `value`.
82 +func (d *datastore) Has(key ds.Key) (exists bool, err error) {
83 + d.gc()
84 + return d.cache.Has(key)
85 +}
86 +
87 +// Delete removes the value for given `key`.
88 +func (d *datastore) Delete(key ds.Key) (err error) {
89 + d.ttlDelete(key)
90 + return d.cache.Delete(key)
91 +}
92 +
93 +// Query returns a list of keys in the datastore
94 +func (d *datastore) Query(q dsq.Query) (dsq.Results, error) {
95 + return d.cache.Query(q)
96 +}
Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache_test.go new
+64
@@ -0,0 +1,64 @@
1 +package timecache
2 +
3 +import (
4 + "testing"
5 + "time"
6 +
7 + ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8 +)
9 +
10 +func testHas(t *testing.T, d ds.Datastore, k ds.Key, v interface{}) {
11 + if v2, err := d.Get(k); err != nil {
12 + t.Error(err)
13 + } else if v2 != v {
14 + t.Error("value incorrect", d, k, v, v2)
15 + }
16 +
17 + if has, err := d.Has(k); err != nil {
18 + t.Error(err)
19 + } else if !has {
20 + t.Error("should have it", d, k, v)
21 + }
22 +}
23 +
24 +func testNotHas(t *testing.T, d ds.Datastore, k ds.Key) {
25 + if _, err := d.Get(k); err == nil {
26 + t.Error("should not have it", d, k)
27 + }
28 +
29 + if has, err := d.Has(k); err != nil {
30 + t.Error(err)
31 + } else if has {
32 + t.Error("should not have it", d, k)
33 + }
34 +}
35 +
36 +func TestTimeCache(t *testing.T) {
37 + ttl := time.Millisecond * 100
38 + cache := WithTTL(ttl)
39 + cache.Put(ds.NewKey("foo1"), "bar1")
40 + cache.Put(ds.NewKey("foo2"), "bar2")
41 +
42 + <-time.After(ttl / 2)
43 + cache.Put(ds.NewKey("foo3"), "bar3")
44 + cache.Put(ds.NewKey("foo4"), "bar4")
45 + testHas(t, cache, ds.NewKey("foo1"), "bar1")
46 + testHas(t, cache, ds.NewKey("foo2"), "bar2")
47 + testHas(t, cache, ds.NewKey("foo3"), "bar3")
48 + testHas(t, cache, ds.NewKey("foo4"), "bar4")
49 +
50 + <-time.After(ttl / 2)
51 + testNotHas(t, cache, ds.NewKey("foo1"))
52 + testNotHas(t, cache, ds.NewKey("foo2"))
53 + testHas(t, cache, ds.NewKey("foo3"), "bar3")
54 + testHas(t, cache, ds.NewKey("foo4"), "bar4")
55 +
56 + cache.Delete(ds.NewKey("foo3"))
57 + testNotHas(t, cache, ds.NewKey("foo3"))
58 +
59 + <-time.After(ttl / 2)
60 + testNotHas(t, cache, ds.NewKey("foo1"))
61 + testNotHas(t, cache, ds.NewKey("foo2"))
62 + testNotHas(t, cache, ds.NewKey("foo3"))
63 + testNotHas(t, cache, ds.NewKey("foo4"))
64 +}