Use DatastoreConfig abstraction to create datastores.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Jul 4, 2017 at 22:15 UTC
1f97170ef9aeb266a29ca6624490266ee0b3774c
3 files changed
+195
-84
repo/fsrepo/config_test.go
+3
-11
@@ -7,10 +7,6 @@ import (
7
"reflect"
8
"testing"
9
10
- //"fmt"
11
-
12
- syncmount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount"
13
- //levelds "gx/ipfs/QmaHHmfEozrrotyhyN44omJouyuEtx6ahddqV6W5yRaUSQ/go-ds-leveldb"
10
config "github.com/ipfs/go-ipfs/repo/config"
11
)
12
@@ -90,10 +86,6 @@ func TestDefaultDatastoreConfig(t *testing.T) {
86
if err != nil {
87
t.Fatal(err)
88
}
93
- _, ok := ds.(*syncmount.Datastore)
94
- if !ok {
95
- t.Fatal("expected mount datastore at top level")
96
- }
89
if typ := reflect.TypeOf(ds).String(); typ != "*syncmount.Datastore" {
90
t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
91
}
@@ -122,7 +114,7 @@ func TestLevelDbConfig(t *testing.T) {
114
t.Fatal(err)
115
}
116
if typ := reflect.TypeOf(ds).String(); typ != "*leveldb.datastore" {
125
- t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
117
+ t.Errorf("expected '*leveldb.datastore' got '%s'", typ)
118
}
119
}
120
@@ -149,7 +141,7 @@ func TestFlatfsConfig(t *testing.T) {
141
t.Fatal(err)
142
}
143
if typ := reflect.TypeOf(ds).String(); typ != "*flatfs.Datastore" {
152
- t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
144
+ t.Errorf("expected '*flatfs.Datastore' got '%s'", typ)
145
}
146
}
147
@@ -176,6 +168,6 @@ func TestMeasureConfig(t *testing.T) {
168
t.Fatal(err)
169
}
170
if typ := reflect.TypeOf(ds).String(); typ != "*measure.measure" {
179
- t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
171
+ t.Errorf("expected '*measure.measure' got '%s'", typ)
172
}
173
}
repo/fsrepo/datastores.go
+184
-72
@@ -14,63 +14,65 @@ import (
14
ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
15
)
16
17
-func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) {
18
- switch params["type"] {
19
- case "mount":
20
- mounts, ok := params["mounts"].([]interface{})
21
- if !ok {
22
- return nil, fmt.Errorf("'mounts' field is missing or not an array")
23
- }
17
+// ConfigFromMap creates a new datastore config from a map
18
+type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error)
19
25
- return r.openMountDatastore(mounts)
26
- case "flatfs":
27
- return r.openFlatfsDatastore(params)
28
- case "mem":
29
- return ds.NewMapDatastore(), nil
30
- case "log":
31
- childField, ok := params["child"].(map[string]interface{})
32
- if !ok {
33
- return nil, fmt.Errorf("'child' field is missing or not a map")
34
- }
35
- child, err := r.constructDatastore(childField)
36
- if err != nil {
37
- return nil, err
38
- }
39
- nameField, ok := params["name"].(string)
40
- if !ok {
41
- return nil, fmt.Errorf("'name' field was missing or not a string")
42
- }
43
- return ds.NewLogDatastore(child, nameField), nil
44
- case "measure":
45
- childField, ok := params["child"].(map[string]interface{})
46
- if !ok {
47
- return nil, fmt.Errorf("'child' field was missing or not a map")
48
- }
49
- child, err := r.constructDatastore(childField)
50
- if err != nil {
51
- return nil, err
52
- }
20
+type DatastoreConfig interface {
21
+ // DiskId is a unique id representing the Datastore config as stored on disk, runtime config values are not
22
+ // part of this Id. No length limit.
23
+ //DiskId() string
24
54
- prefix, ok := params["prefix"].(string)
55
- if !ok {
56
- return nil, fmt.Errorf("'prefix' field was missing or not a string")
57
- }
25
+ // Create instantiate a new datastore from this config
26
+ Create(path string) (repo.Datastore, error)
27
+}
28
59
- return r.openMeasureDB(prefix, child)
29
+var datastores map[string]ConfigFromMap
30
61
- case "levelds":
62
- return r.openLeveldbDatastore(params)
63
- default:
64
- return nil, fmt.Errorf("unknown datastore type: %s", params["type"])
31
+func init() {
32
+ datastores = map[string]ConfigFromMap{
33
+ "mount": MountDatastoreConfig,
34
+ "flatfs": FlatfsDatastoreConfig,
35
+ "levelds": LeveldsDatastoreConfig,
36
+ "mem": MemDatastoreConfig,
37
+ "log": LogDatastoreConfig,
38
+ "measure": MeasureDatastoreConfig,
39
+ }
40
+}
41
+
42
+func AnyDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
43
+ which, ok := params["type"].(string)
44
+ if !ok {
45
+ return nil, fmt.Errorf("'type' field missing or not a string")
46
+ }
47
+ fun, ok := datastores[which]
48
+ if !ok {
49
+ return nil, fmt.Errorf("unknown datastore type: %s", which)
50
}
51
+ return fun(params)
52
}
53
68
-func (r *FSRepo) openMountDatastore(mountcfg []interface{}) (repo.Datastore, error) {
69
- var mounts []mount.Mount
70
- for _, iface := range mountcfg {
71
- cfg := iface.(map[string]interface{})
54
+type mountDatastoreConfig struct {
55
+ mounts []premount
56
+}
57
+
58
+type premount struct {
59
+ ds DatastoreConfig
60
+ prefix ds.Key
61
+}
62
+
63
+func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
64
+ var res mountDatastoreConfig
65
+ mounts, ok := params["mounts"].([]interface{})
66
+ if !ok {
67
+ return nil, fmt.Errorf("'mounts' field is missing or not an array")
68
+ }
69
+ for _, iface := range mounts {
70
+ cfg, ok := iface.(map[string]interface{})
71
+ if !ok {
72
+ return nil, fmt.Errorf("expected map for mountpoint")
73
+ }
74
73
- child, err := r.constructDatastore(cfg)
75
+ child, err := AnyDatastoreConfig(cfg)
76
if err != nil {
77
return nil, err
78
}
@@ -80,65 +82,175 @@ func (r *FSRepo) openMountDatastore(mountcfg []interface{}) (repo.Datastore, err
82
return nil, fmt.Errorf("no 'mountpoint' on mount")
83
}
84
83
- mounts = append(mounts, mount.Mount{
84
- Datastore: child,
85
- Prefix: ds.NewKey(prefix.(string)),
85
+ res.mounts = append(res.mounts, premount{
86
+ ds: child,
87
+ prefix: ds.NewKey(prefix.(string)),
88
})
89
}
90
91
+ return &res, nil
92
+}
93
+
94
+func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
95
+ mounts := make([]mount.Mount, len(c.mounts))
96
+ for i, m := range c.mounts {
97
+ ds, err := m.ds.Create(path)
98
+ if err != nil {
99
+ return nil, err
100
+ }
101
+ mounts[i].Datastore = ds
102
+ mounts[i].Prefix = m.prefix
103
+ }
104
return mount.New(mounts), nil
105
}
106
92
-func (r *FSRepo) openFlatfsDatastore(params map[string]interface{}) (repo.Datastore, error) {
93
- p, ok := params["path"].(string)
107
+type flatfsDatastoreConfig struct {
108
+ path string
109
+ shardFun *flatfs.ShardIdV1
110
+ syncField bool
111
+}
112
+
113
+func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
114
+ var c flatfsDatastoreConfig
115
+ var ok bool
116
+ var err error
117
+
118
+ c.path, ok = params["path"].(string)
119
if !ok {
120
return nil, fmt.Errorf("'path' field is missing or not boolean")
121
}
97
- if !filepath.IsAbs(p) {
98
- p = filepath.Join(r.path, p)
99
- }
122
123
sshardFun, ok := params["shardFunc"].(string)
124
if !ok {
125
return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
126
}
105
- shardFun, err := flatfs.ParseShardFunc(sshardFun)
127
+ c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
128
if err != nil {
129
return nil, err
130
}
131
110
- syncField, ok := params["sync"].(bool)
132
+ c.syncField, ok = params["sync"].(bool)
133
if !ok {
134
return nil, fmt.Errorf("'sync' field is missing or not boolean")
135
}
114
- return flatfs.CreateOrOpen(p, shardFun, syncField)
136
+ return &c, nil
137
+}
138
+
139
+func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
140
+ p := c.path
141
+ if !filepath.IsAbs(p) {
142
+ p = filepath.Join(path, p)
143
+ }
144
+
145
+ return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
146
}
147
117
-func (r *FSRepo) openLeveldbDatastore(params map[string]interface{}) (repo.Datastore, error) {
118
- p, ok := params["path"].(string)
148
+type leveldsDatastoreConfig struct {
149
+ path string
150
+ compression ldbopts.Compression
151
+}
152
+
153
+func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
154
+ var c leveldsDatastoreConfig
155
+ var ok bool
156
+
157
+ c.path, ok = params["path"].(string)
158
if !ok {
159
return nil, fmt.Errorf("'path' field is missing or not string")
160
}
122
- if !filepath.IsAbs(p) {
123
- p = filepath.Join(r.path, p)
124
- }
161
126
- var c ldbopts.Compression
162
switch params["compression"].(string) {
163
case "none":
129
- c = ldbopts.NoCompression
164
+ c.compression = ldbopts.NoCompression
165
case "snappy":
131
- c = ldbopts.SnappyCompression
166
+ c.compression = ldbopts.SnappyCompression
167
case "":
168
fallthrough
169
default:
135
- c = ldbopts.DefaultCompression
170
+ c.compression = ldbopts.DefaultCompression
171
+ }
172
+
173
+ return &c, nil
174
+}
175
+
176
+func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
177
+ p := c.path
178
+ if !filepath.IsAbs(p) {
179
+ p = filepath.Join(path, p)
180
}
181
+
182
return levelds.NewDatastore(p, &levelds.Options{
138
- Compression: c,
183
+ Compression: c.compression,
184
})
185
}
186
142
-func (r *FSRepo) openMeasureDB(prefix string, child repo.Datastore) (repo.Datastore, error) {
143
- return measure.New(prefix, child), nil
187
+type memDatastoreConfig struct {
188
+ cfg map[string]interface{}
189
+}
190
+
191
+func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
192
+ return &memDatastoreConfig{params}, nil
193
+}
194
+
195
+func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
196
+ return ds.NewMapDatastore(), nil
197
+}
198
+
199
+type logDatastoreConfig struct {
200
+ child DatastoreConfig
201
+ name string
202
+}
203
+
204
+func LogDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
205
+ childField, ok := params["child"].(map[string]interface{})
206
+ if !ok {
207
+ return nil, fmt.Errorf("'child' field is missing or not a map")
208
+ }
209
+ child, err := AnyDatastoreConfig(childField)
210
+ if err != nil {
211
+ return nil, err
212
+ }
213
+ name, ok := params["name"].(string)
214
+ if !ok {
215
+ return nil, fmt.Errorf("'name' field was missing or not a string")
216
+ }
217
+ return &logDatastoreConfig{child, name}, nil
218
+
219
+}
220
+
221
+func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
222
+ child, err := c.child.Create(path)
223
+ if err != nil {
224
+ return nil, err
225
+ }
226
+ return ds.NewLogDatastore(child, c.name), nil
227
+}
228
+
229
+type measureDatastoreConfig struct {
230
+ child DatastoreConfig
231
+ prefix string
232
+}
233
+
234
+func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
235
+ childField, ok := params["child"].(map[string]interface{})
236
+ if !ok {
237
+ return nil, fmt.Errorf("'child' field is missing or not a map")
238
+ }
239
+ child, err := AnyDatastoreConfig(childField)
240
+ if err != nil {
241
+ return nil, err
242
+ }
243
+ prefix, ok := params["prefix"].(string)
244
+ if !ok {
245
+ return nil, fmt.Errorf("'prefix' field was missing or not a string")
246
+ }
247
+ return &measureDatastoreConfig{child, prefix}, nil
248
+}
249
+
250
+func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
251
+ child, err := c.child.Create(path)
252
+ if err != nil {
253
+ return nil, err
254
+ }
255
+ return measure.New(c.prefix, child), nil
256
}
repo/fsrepo/fsrepo.go
+8
-1
@@ -362,7 +362,6 @@ func (r *FSRepo) openDatastore() error {
362
if err != nil {
363
return err
364
}
365
-
365
r.ds = d
366
} else if r.config.Datastore.Type != "" || r.config.Datastore.Path != "" {
367
return fmt.Errorf("old style datatstore config detected")
@@ -377,6 +376,14 @@ func (r *FSRepo) openDatastore() error {
376
return nil
377
}
378
379
+func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) {
380
+ cfg, err := AnyDatastoreConfig(params)
381
+ if err != nil {
382
+ return nil, err
383
+ }
384
+ return cfg.Create(r.path)
385
+}
386
+
387
// Close closes the FSRepo, releasing held resources.
388
func (r *FSRepo) Close() error {
389
packageLock.Lock()