fsrepo components simplification: directly use datastore
Tommi Virtanen committed
Mar 11, 2015 at 12:44 UTC
24a32af3f161d4e1b7d5dc62053bea5cdc55b9b3
5 files changed
+79
-212
repo/fsrepo/component/component.go
deleted
-15
@@ -1,15 +0,0 @@
1
-package component
2
-
3
-import (
4
- "io"
5
-
6
- "github.com/jbenet/go-ipfs/repo/config"
7
-)
8
-
9
-type Component interface {
10
- Open(*config.Config) error
11
- io.Closer
12
- SetPath(string)
13
-}
14
-type Initializer func(path string, conf *config.Config) error
15
-type InitializationChecker func(path string) bool
repo/fsrepo/component/datastore.go
deleted
-113
@@ -1,113 +0,0 @@
1
-package component
2
-
3
-import (
4
- "errors"
5
- "path"
6
- "sync"
7
-
8
- datastore "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
9
- levelds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
10
- ldbopts "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
11
- config "github.com/jbenet/go-ipfs/repo/config"
12
- counter "github.com/jbenet/go-ipfs/repo/fsrepo/counter"
13
- dir "github.com/jbenet/go-ipfs/thirdparty/dir"
14
- util "github.com/jbenet/go-ipfs/util"
15
- ds2 "github.com/jbenet/go-ipfs/util/datastore2"
16
- debugerror "github.com/jbenet/go-ipfs/util/debugerror"
17
-)
18
-
19
-const (
20
- DefaultDataStoreDirectory = "datastore"
21
-)
22
-
23
-var (
24
- _ Component = &DatastoreComponent{}
25
- _ Initializer = InitDatastoreComponent
26
- _ InitializationChecker = DatastoreComponentIsInitialized
27
-
28
- dsLock sync.Mutex // protects openersCounter and datastores
29
- openersCounter *counter.Openers
30
- datastores map[string]ds2.ThreadSafeDatastoreCloser
31
-)
32
-
33
-func init() {
34
- openersCounter = counter.NewOpenersCounter()
35
- datastores = make(map[string]ds2.ThreadSafeDatastoreCloser)
36
-}
37
-
38
-func InitDatastoreComponent(dspath string, conf *config.Config) error {
39
- // The actual datastore contents are initialized lazily when Opened.
40
- // During Init, we merely check that the directory is writeable.
41
- p := path.Join(dspath, DefaultDataStoreDirectory)
42
- if err := dir.Writable(p); err != nil {
43
- return debugerror.Errorf("datastore: %s", err)
44
- }
45
- return nil
46
-}
47
-
48
-// DatastoreComponentIsInitialized returns true if the datastore dir exists.
49
-func DatastoreComponentIsInitialized(dspath string) bool {
50
- if !util.FileExists(path.Join(dspath, DefaultDataStoreDirectory)) {
51
- return false
52
- }
53
- return true
54
-}
55
-
56
-// DatastoreComponent abstracts the datastore component of the FSRepo.
57
-type DatastoreComponent struct {
58
- path string // required
59
- ds ds2.ThreadSafeDatastoreCloser // assigned when repo is opened
60
-}
61
-
62
-func (dsc *DatastoreComponent) SetPath(p string) {
63
- dsc.path = path.Join(p, DefaultDataStoreDirectory)
64
-}
65
-
66
-func (dsc *DatastoreComponent) Datastore() datastore.ThreadSafeDatastore { return dsc.ds }
67
-
68
-// Open returns an error if the config file is not present.
69
-func (dsc *DatastoreComponent) Open(*config.Config) error {
70
-
71
- dsLock.Lock()
72
- defer dsLock.Unlock()
73
-
74
- // if no other goroutines have the datastore Open, initialize it and assign
75
- // it to the package-scoped map for the goroutines that follow.
76
- if openersCounter.NumOpeners(dsc.path) == 0 {
77
- ds, err := levelds.NewDatastore(dsc.path, &levelds.Options{
78
- Compression: ldbopts.NoCompression,
79
- })
80
- if err != nil {
81
- return debugerror.New("unable to open leveldb datastore")
82
- }
83
- datastores[dsc.path] = ds
84
- }
85
-
86
- // get the datastore from the package-scoped map and record self as an
87
- // opener.
88
- ds, dsIsPresent := datastores[dsc.path]
89
- if !dsIsPresent {
90
- // This indicates a programmer error has occurred.
91
- return errors.New("datastore should be available, but it isn't")
92
- }
93
- dsc.ds = ds
94
- openersCounter.AddOpener(dsc.path) // only after success
95
- return nil
96
-}
97
-
98
-func (dsc *DatastoreComponent) Close() error {
99
-
100
- dsLock.Lock()
101
- defer dsLock.Unlock()
102
-
103
- // decrement the Opener count. if this goroutine is the last, also close
104
- // the underlying datastore (and remove its reference from the map)
105
-
106
- openersCounter.RemoveOpener(dsc.path)
107
-
108
- if openersCounter.NumOpeners(dsc.path) == 0 {
109
- delete(datastores, dsc.path) // remove the reference
110
- return dsc.ds.Close()
111
- }
112
- return nil
113
-}
repo/fsrepo/component/datastore_test.go
deleted
-30
@@ -1,30 +0,0 @@
1
-package component
2
-
3
-import (
4
- "io/ioutil"
5
- "path/filepath"
6
- "testing"
7
-
8
- "github.com/jbenet/go-ipfs/thirdparty/assert"
9
-)
10
-
11
-// swap arg order
12
-func testRepoPath(t *testing.T, path ...string) string {
13
- name, err := ioutil.TempDir("", filepath.Join(path...))
14
- if err != nil {
15
- t.Fatal(err)
16
- }
17
- return name
18
-}
19
-
20
-func TestOpenMoreThanOnceInSameProcess(t *testing.T) {
21
- t.Parallel()
22
- path := testRepoPath(t)
23
- dsc1 := DatastoreComponent{path: path}
24
- dsc2 := DatastoreComponent{path: path}
25
- assert.Nil(dsc1.Open(nil), t, "first repo should open successfully")
26
- assert.Nil(dsc2.Open(nil), t, "second repo should open successfully")
27
-
28
- assert.Nil(dsc1.Close(), t)
29
- assert.Nil(dsc2.Close(), t)
30
-}
repo/fsrepo/fsrepo.go
+78
-54
@@ -10,10 +10,11 @@ import (
10
"sync"
11
12
ds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
13
+ levelds "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
14
+ ldbopts "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
15
repo "github.com/jbenet/go-ipfs/repo"
16
"github.com/jbenet/go-ipfs/repo/common"
17
config "github.com/jbenet/go-ipfs/repo/config"
16
- component "github.com/jbenet/go-ipfs/repo/fsrepo/component"
18
counter "github.com/jbenet/go-ipfs/repo/fsrepo/counter"
19
lockfile "github.com/jbenet/go-ipfs/repo/fsrepo/lock"
20
serialize "github.com/jbenet/go-ipfs/repo/fsrepo/serialize"
@@ -21,9 +22,14 @@ import (
22
"github.com/jbenet/go-ipfs/thirdparty/eventlog"
23
u "github.com/jbenet/go-ipfs/util"
24
util "github.com/jbenet/go-ipfs/util"
25
+ ds2 "github.com/jbenet/go-ipfs/util/datastore2"
26
debugerror "github.com/jbenet/go-ipfs/util/debugerror"
27
)
28
29
+const (
30
+ defaultDataStoreDirectory = "datastore"
31
+)
32
+
33
var (
34
35
// packageLock must be held to while performing any operation that modifies an
@@ -40,11 +46,19 @@ var (
46
// If an operation is used when repo is Open and the operation does not
47
// change the repo's state, the package lock does not need to be acquired.
48
openersCounter *counter.Openers
49
+
50
+ // protects dsOpenersCounter and datastores
51
+ dsLock sync.Mutex
52
+ dsOpenersCounter *counter.Openers
53
+ datastores map[string]ds2.ThreadSafeDatastoreCloser
54
)
55
56
func init() {
57
openersCounter = counter.NewOpenersCounter()
58
lockfiles = make(map[string]io.Closer)
59
+
60
+ dsOpenersCounter = counter.NewOpenersCounter()
61
+ datastores = make(map[string]ds2.ThreadSafeDatastoreCloser)
62
}
63
64
// FSRepo represents an IPFS FileSystem Repo. It is safe for use by multiple
@@ -56,19 +70,12 @@ type FSRepo struct {
70
path string
71
// config is set on Open, guarded by packageLock
72
config *config.Config
59
-
60
- // TODO test
61
- datastoreComponent component.DatastoreComponent
73
+ // ds is set on Open
74
+ ds ds2.ThreadSafeDatastoreCloser
75
}
76
77
var _ repo.Repo = (*FSRepo)(nil)
78
66
-type componentBuilder struct {
67
- Init component.Initializer
68
- IsInitialized component.InitializationChecker
69
- OpenHandler func(*FSRepo) error
70
-}
71
-
79
// At returns a handle to an FSRepo at the provided |path|.
80
func At(repoPath string) *FSRepo {
81
// This method must not have side-effects.
@@ -141,10 +148,11 @@ func Init(repoPath string, conf *config.Config) error {
148
return err
149
}
150
144
- for _, b := range componentBuilders() {
145
- if err := b.Init(repoPath, conf); err != nil {
146
- return err
147
- }
151
+ // The actual datastore contents are initialized lazily when Opened.
152
+ // During Init, we merely check that the directory is writeable.
153
+ p := path.Join(repoPath, defaultDataStoreDirectory)
154
+ if err := dir.Writable(p); err != nil {
155
+ return debugerror.Errorf("datastore: %s", err)
156
}
157
158
if err := dir.Writable(path.Join(repoPath, "logs")); err != nil {
@@ -196,6 +204,37 @@ func (r *FSRepo) openConfig() error {
204
return nil
205
}
206
207
+// openDatastore returns an error if the config file is not present.
208
+func (r *FSRepo) openDatastore() error {
209
+ dsLock.Lock()
210
+ defer dsLock.Unlock()
211
+
212
+ dsPath := path.Join(r.path, defaultDataStoreDirectory)
213
+
214
+ // if no other goroutines have the datastore Open, initialize it and assign
215
+ // it to the package-scoped map for the goroutines that follow.
216
+ if dsOpenersCounter.NumOpeners(dsPath) == 0 {
217
+ ds, err := levelds.NewDatastore(dsPath, &levelds.Options{
218
+ Compression: ldbopts.NoCompression,
219
+ })
220
+ if err != nil {
221
+ return debugerror.New("unable to open leveldb datastore")
222
+ }
223
+ datastores[dsPath] = ds
224
+ }
225
+
226
+ // get the datastore from the package-scoped map and record self as an
227
+ // opener.
228
+ ds, dsIsPresent := datastores[dsPath]
229
+ if !dsIsPresent {
230
+ // This indicates a programmer error has occurred.
231
+ return errors.New("datastore should be available, but it isn't")
232
+ }
233
+ r.ds = ds
234
+ dsOpenersCounter.AddOpener(dsPath) // only after success
235
+ return nil
236
+}
237
+
238
func configureEventLoggerAtRepoPath(c *config.Config, repoPath string) {
239
eventlog.Configure(eventlog.LevelInfo)
240
eventlog.Configure(eventlog.LdJSONFormatter)
@@ -240,10 +279,8 @@ func (r *FSRepo) Open() error {
279
return err
280
}
281
243
- for _, b := range componentBuilders() {
244
- if err := b.OpenHandler(r); err != nil {
245
- return err
246
- }
282
+ if err := r.openDatastore(); err != nil {
283
+ return err
284
}
285
286
// log.Debugf("writing eventlogs to ...", c.path)
@@ -252,6 +289,24 @@ func (r *FSRepo) Open() error {
289
return r.transitionToOpened()
290
}
291
292
+func (r *FSRepo) closeDatastore() error {
293
+ dsLock.Lock()
294
+ defer dsLock.Unlock()
295
+
296
+ dsPath := path.Join(r.path, defaultDataStoreDirectory)
297
+
298
+ // decrement the Opener count. if this goroutine is the last, also close
299
+ // the underlying datastore (and remove its reference from the map)
300
+
301
+ dsOpenersCounter.RemoveOpener(dsPath)
302
+
303
+ if dsOpenersCounter.NumOpeners(dsPath) == 0 {
304
+ delete(datastores, dsPath) // remove the reference
305
+ return r.ds.Close()
306
+ }
307
+ return nil
308
+}
309
+
310
// Close closes the FSRepo, releasing held resources.
311
func (r *FSRepo) Close() error {
312
packageLock.Lock()
@@ -261,10 +316,8 @@ func (r *FSRepo) Close() error {
316
return debugerror.Errorf("repo is %s", r.state)
317
}
318
264
- for _, closer := range r.components() {
265
- if err := closer.Close(); err != nil {
266
- return err
267
- }
319
+ if err := r.closeDatastore(); err != nil {
320
+ return err
321
}
322
323
// This code existed in the previous versions, but
@@ -395,7 +448,7 @@ func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
448
// is undefined.
449
func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
450
packageLock.Lock()
398
- d := r.datastoreComponent.Datastore()
451
+ d := r.ds
452
packageLock.Unlock()
453
return d
454
}
@@ -421,10 +474,8 @@ func isInitializedUnsynced(repoPath string) bool {
474
if !configIsInitialized(repoPath) {
475
return false
476
}
424
- for _, b := range componentBuilders() {
425
- if !b.IsInitialized(repoPath) {
426
- return false
427
- }
477
+ if !util.FileExists(path.Join(repoPath, defaultDataStoreDirectory)) {
478
+ return false
479
}
480
return true
481
}
@@ -462,30 +513,3 @@ func (r *FSRepo) transitionToClosed() error {
513
}
514
return nil
515
}
465
-
466
-// components returns the FSRepo's constituent components
467
-func (r *FSRepo) components() []component.Component {
468
- return []component.Component{
469
- &r.datastoreComponent,
470
- }
471
-}
472
-
473
-func componentBuilders() []componentBuilder {
474
- return []componentBuilder{
475
-
476
- // DatastoreComponent
477
- componentBuilder{
478
- Init: component.InitDatastoreComponent,
479
- IsInitialized: component.DatastoreComponentIsInitialized,
480
- OpenHandler: func(r *FSRepo) error {
481
- c := component.DatastoreComponent{}
482
- c.SetPath(r.path)
483
- if err := c.Open(r.config); err != nil {
484
- return err
485
- }
486
- r.datastoreComponent = c
487
- return nil
488
- },
489
- },
490
- }
491
-}
repo/fsrepo/fsrepo_test.go
+1
@@ -135,6 +135,7 @@ func TestOpenMoreThanOnceInSameProcess(t *testing.T) {
135
r2 := At(path)
136
assert.Nil(r1.Open(), t, "first repo should open successfully")
137
assert.Nil(r2.Open(), t, "second repo should open successfully")
138
+ assert.True(r1.ds == r2.ds, t, "repos should share the datastore")
139
140
assert.Nil(r1.Close(), t)
141
assert.Nil(r2.Close(), t)