Move datastores to plugins
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Oct 30, 2018 at 04:26 UTC
c154f7f01e933a1120801fdc4c4b3fae6013a33b
7 files changed
+310
-188
plugin/loader/preload.go
+6
@@ -2,7 +2,10 @@ package loader
2
3
import (
4
"github.com/ipfs/go-ipfs/plugin"
5
+ pluginbadgerds "github.com/ipfs/go-ipfs/plugin/plugins/badgerds"
6
+ pluginflatfs "github.com/ipfs/go-ipfs/plugin/plugins/flatfs"
7
pluginipldgit "github.com/ipfs/go-ipfs/plugin/plugins/git"
8
+ pluginlevelds "github.com/ipfs/go-ipfs/plugin/plugins/levelds"
9
)
10
11
// DO NOT EDIT THIS FILE
@@ -11,4 +14,7 @@ import (
14
15
var preloadPlugins = []plugin.Plugin{
16
pluginipldgit.Plugins[0],
17
+ pluginbadgerds.Plugins[0],
18
+ pluginflatfs.Plugins[0],
19
+ pluginlevelds.Plugins[0],
20
}
plugin/loader/preload_list
+4
@@ -4,3 +4,7 @@
4
# name go-path number of the sub-plugin
5
6
ipldgit github.com/ipfs/go-ipfs/plugin/plugins/git 0
7
+
8
+badgerds github.com/ipfs/go-ipfs/plugin/plugins/badgerds 0
9
+flatfs github.com/ipfs/go-ipfs/plugin/plugins/flatfs 0
10
+levelds github.com/ipfs/go-ipfs/plugin/plugins/levelds 0
plugin/plugins/Rules.mk
+1
-1
@@ -1,6 +1,6 @@
1
include mk/header.mk
2
3
-$(d)_plugins:=$(d)/git
3
+$(d)_plugins:=$(d)/git $(d)/badgerds $(d)/flatfs $(d)/levelds
4
$(d)_plugins_so:=$(addsuffix .so,$($(d)_plugins))
5
$(d)_plugins_main:=$(addsuffix /main/main.go,$($(d)_plugins))
6
plugin/plugins/badgerds/badgerds.go
new
+114
@@ -0,0 +1,114 @@
1
+package badgerds
2
+
3
+import (
4
+ "fmt"
5
+ "os"
6
+ "path/filepath"
7
+
8
+ "github.com/ipfs/go-ipfs/plugin"
9
+ "github.com/ipfs/go-ipfs/repo"
10
+ "github.com/ipfs/go-ipfs/repo/fsrepo"
11
+
12
+ humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
13
+ badgerds "gx/ipfs/QmaixNkKwtinV3umL5VD1VDD5CQjnZhXY31awM2YHTzbui/go-ds-badger"
14
+)
15
+
16
+// Plugins is exported list of plugins that will be loaded
17
+var Plugins = []plugin.Plugin{
18
+ &badgerdsPlugin{},
19
+}
20
+
21
+type badgerdsPlugin struct{}
22
+
23
+var _ plugin.PluginDatastore = (*badgerdsPlugin)(nil)
24
+
25
+func (*badgerdsPlugin) Name() string {
26
+ return "ds-badgerds"
27
+}
28
+
29
+func (*badgerdsPlugin) Version() string {
30
+ return "0.1.0"
31
+}
32
+
33
+func (*badgerdsPlugin) Init() error {
34
+ return nil
35
+}
36
+
37
+func (*badgerdsPlugin) DatastoreTypeName() string {
38
+ return "badgerds"
39
+}
40
+
41
+type datastoreConfig struct {
42
+ path string
43
+ syncWrites bool
44
+
45
+ vlogFileSize int64
46
+}
47
+
48
+// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
49
+// from the given parameters
50
+func (*badgerdsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
51
+ return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
52
+ var c datastoreConfig
53
+ var ok bool
54
+
55
+ c.path, ok = params["path"].(string)
56
+ if !ok {
57
+ return nil, fmt.Errorf("'path' field is missing or not string")
58
+ }
59
+
60
+ sw, ok := params["syncWrites"]
61
+ if !ok {
62
+ c.syncWrites = true
63
+ } else {
64
+ if swb, ok := sw.(bool); ok {
65
+ c.syncWrites = swb
66
+ } else {
67
+ return nil, fmt.Errorf("'syncWrites' field was not a boolean")
68
+ }
69
+ }
70
+
71
+ vls, ok := params["vlogFileSize"]
72
+ if !ok {
73
+ // default to 1GiB
74
+ c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
75
+ } else {
76
+ if vlogSize, ok := vls.(string); ok {
77
+ s, err := humanize.ParseBytes(vlogSize)
78
+ if err != nil {
79
+ return nil, err
80
+ }
81
+ c.vlogFileSize = int64(s)
82
+ } else {
83
+ return nil, fmt.Errorf("'vlogFileSize' field was not a string")
84
+ }
85
+ }
86
+
87
+ return &c, nil
88
+ }
89
+}
90
+
91
+func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
92
+ return map[string]interface{}{
93
+ "type": "badgerds",
94
+ "path": c.path,
95
+ }
96
+}
97
+
98
+func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
99
+ p := c.path
100
+ if !filepath.IsAbs(p) {
101
+ p = filepath.Join(path, p)
102
+ }
103
+
104
+ err := os.MkdirAll(p, 0755)
105
+ if err != nil {
106
+ return nil, err
107
+ }
108
+
109
+ defopts := badgerds.DefaultOptions
110
+ defopts.SyncWrites = c.syncWrites
111
+ defopts.ValueLogFileSize = c.vlogFileSize
112
+
113
+ return badgerds.NewDatastore(p, &defopts)
114
+}
plugin/plugins/flatfs/flatfs.go
new
+90
@@ -0,0 +1,90 @@
1
+package flatfs
2
+
3
+import (
4
+ "fmt"
5
+ "path/filepath"
6
+
7
+ "github.com/ipfs/go-ipfs/plugin"
8
+ "github.com/ipfs/go-ipfs/repo"
9
+ "github.com/ipfs/go-ipfs/repo/fsrepo"
10
+
11
+ flatfs "gx/ipfs/QmVFboKxbVJZMJAoFdvX6q4hzvXFkbWCE8DejnqrQV4ZtN/go-ds-flatfs"
12
+)
13
+
14
+// Plugins is exported list of plugins that will be loaded
15
+var Plugins = []plugin.Plugin{
16
+ &flatfsPlugin{},
17
+}
18
+
19
+type flatfsPlugin struct{}
20
+
21
+var _ plugin.PluginDatastore = (*flatfsPlugin)(nil)
22
+
23
+func (*flatfsPlugin) Name() string {
24
+ return "ds-flatfs"
25
+}
26
+
27
+func (*flatfsPlugin) Version() string {
28
+ return "0.1.0"
29
+}
30
+
31
+func (*flatfsPlugin) Init() error {
32
+ return nil
33
+}
34
+
35
+func (*flatfsPlugin) DatastoreTypeName() string {
36
+ return "flatfs"
37
+}
38
+
39
+type datastoreConfig struct {
40
+ path string
41
+ shardFun *flatfs.ShardIdV1
42
+ syncField bool
43
+}
44
+
45
+// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
46
+// from the given parameters
47
+func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48
+ return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
49
+ var c datastoreConfig
50
+ var ok bool
51
+ var err error
52
+
53
+ c.path, ok = params["path"].(string)
54
+ if !ok {
55
+ return nil, fmt.Errorf("'path' field is missing or not boolean")
56
+ }
57
+
58
+ sshardFun, ok := params["shardFunc"].(string)
59
+ if !ok {
60
+ return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
61
+ }
62
+ c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
63
+ if err != nil {
64
+ return nil, err
65
+ }
66
+
67
+ c.syncField, ok = params["sync"].(bool)
68
+ if !ok {
69
+ return nil, fmt.Errorf("'sync' field is missing or not boolean")
70
+ }
71
+ return &c, nil
72
+ }
73
+}
74
+
75
+func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
76
+ return map[string]interface{}{
77
+ "type": "flatfs",
78
+ "path": c.path,
79
+ "shardFunc": c.shardFun.String(),
80
+ }
81
+}
82
+
83
+func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
84
+ p := c.path
85
+ if !filepath.IsAbs(p) {
86
+ p = filepath.Join(path, p)
87
+ }
88
+
89
+ return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
90
+}
plugin/plugins/levelds/levelds.go
new
+88
@@ -0,0 +1,88 @@
1
+package levelds
2
+
3
+import (
4
+ "fmt"
5
+ "path/filepath"
6
+
7
+ "github.com/ipfs/go-ipfs/plugin"
8
+ "github.com/ipfs/go-ipfs/repo"
9
+ "github.com/ipfs/go-ipfs/repo/fsrepo"
10
+
11
+ ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
12
+ levelds "gx/ipfs/QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a/go-ds-leveldb"
13
+)
14
+
15
+// Plugins is exported list of plugins that will be loaded
16
+var Plugins = []plugin.Plugin{
17
+ &leveldsPlugin{},
18
+}
19
+
20
+type leveldsPlugin struct{}
21
+
22
+var _ plugin.PluginDatastore = (*leveldsPlugin)(nil)
23
+
24
+func (*leveldsPlugin) Name() string {
25
+ return "ds-level"
26
+}
27
+
28
+func (*leveldsPlugin) Version() string {
29
+ return "0.1.0"
30
+}
31
+
32
+func (*leveldsPlugin) Init() error {
33
+ return nil
34
+}
35
+
36
+func (*leveldsPlugin) DatastoreTypeName() string {
37
+ return "levelds"
38
+}
39
+
40
+type datastoreConfig struct {
41
+ path string
42
+ compression ldbopts.Compression
43
+}
44
+
45
+// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
46
+// from the given parameters
47
+func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48
+ return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
49
+ var c datastoreConfig
50
+ var ok bool
51
+
52
+ c.path, ok = params["path"].(string)
53
+ if !ok {
54
+ return nil, fmt.Errorf("'path' field is missing or not string")
55
+ }
56
+
57
+ switch cm := params["compression"].(string); cm {
58
+ case "none":
59
+ c.compression = ldbopts.NoCompression
60
+ case "snappy":
61
+ c.compression = ldbopts.SnappyCompression
62
+ case "":
63
+ c.compression = ldbopts.DefaultCompression
64
+ default:
65
+ return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
66
+ }
67
+
68
+ return &c, nil
69
+ }
70
+}
71
+
72
+func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
73
+ return map[string]interface{}{
74
+ "type": "levelds",
75
+ "path": c.path,
76
+ }
77
+}
78
+
79
+func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
80
+ p := c.path
81
+ if !filepath.IsAbs(p) {
82
+ p = filepath.Join(path, p)
83
+ }
84
+
85
+ return levelds.NewDatastore(p, &levelds.Options{
86
+ Compression: c.compression,
87
+ })
88
+}
repo/fsrepo/datastores.go
+7
-187
@@ -4,20 +4,13 @@ import (
4
"bytes"
5
"encoding/json"
6
"fmt"
7
- "os"
8
- "path/filepath"
7
"sort"
8
11
- repo "github.com/ipfs/go-ipfs/repo"
9
+ "github.com/ipfs/go-ipfs/repo"
10
13
- humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14
- measure "gx/ipfs/QmQS6UXi1R87y9nEgnCNmG6YfMzvBSLir7xUheMNFP3hoe/go-ds-measure"
15
- flatfs "gx/ipfs/QmVFboKxbVJZMJAoFdvX6q4hzvXFkbWCE8DejnqrQV4ZtN/go-ds-flatfs"
11
+ "gx/ipfs/QmQS6UXi1R87y9nEgnCNmG6YfMzvBSLir7xUheMNFP3hoe/go-ds-measure"
12
ds "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore"
17
- mount "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore/mount"
18
- badgerds "gx/ipfs/QmaixNkKwtinV3umL5VD1VDD5CQjnZhXY31awM2YHTzbui/go-ds-badger"
19
- ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
20
- levelds "gx/ipfs/QmccqjKZUTqp4ikWNyAbjBuP5HEdqSqRuAr9mcEhYab54a/go-ds-leveldb"
13
+ "gx/ipfs/QmaRb5yNXKonhbkpNxNawoydk4N6es6b4fPj19sjEKsh5D/go-datastore/mount"
14
)
15
16
// ConfigFromMap creates a new datastore config from a map
@@ -63,13 +56,10 @@ var datastores map[string]ConfigFromMap
56
57
func init() {
58
datastores = map[string]ConfigFromMap{
66
- "mount": MountDatastoreConfig,
67
- "flatfs": FlatfsDatastoreConfig,
68
- "levelds": LeveldsDatastoreConfig,
69
- "badgerds": BadgerdsDatastoreConfig,
70
- "mem": MemDatastoreConfig,
71
- "log": LogDatastoreConfig,
72
- "measure": MeasureDatastoreConfig,
59
+ "mount": MountDatastoreConfig,
60
+ "mem": MemDatastoreConfig,
61
+ "log": LogDatastoreConfig,
62
+ "measure": MeasureDatastoreConfig,
63
}
64
}
65
@@ -170,103 +160,6 @@ func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
160
return mount.New(mounts), nil
161
}
162
173
-type flatfsDatastoreConfig struct {
174
- path string
175
- shardFun *flatfs.ShardIdV1
176
- syncField bool
177
-}
178
-
179
-// FlatfsDatastoreConfig returns a flatfs DatastoreConfig from a spec
180
-func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
181
- var c flatfsDatastoreConfig
182
- var ok bool
183
- var err error
184
-
185
- c.path, ok = params["path"].(string)
186
- if !ok {
187
- return nil, fmt.Errorf("'path' field is missing or not boolean")
188
- }
189
-
190
- sshardFun, ok := params["shardFunc"].(string)
191
- if !ok {
192
- return nil, fmt.Errorf("'shardFunc' field is missing or not a string")
193
- }
194
- c.shardFun, err = flatfs.ParseShardFunc(sshardFun)
195
- if err != nil {
196
- return nil, err
197
- }
198
-
199
- c.syncField, ok = params["sync"].(bool)
200
- if !ok {
201
- return nil, fmt.Errorf("'sync' field is missing or not boolean")
202
- }
203
- return &c, nil
204
-}
205
-
206
-func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
207
- return map[string]interface{}{
208
- "type": "flatfs",
209
- "path": c.path,
210
- "shardFunc": c.shardFun.String(),
211
- }
212
-}
213
-
214
-func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
215
- p := c.path
216
- if !filepath.IsAbs(p) {
217
- p = filepath.Join(path, p)
218
- }
219
-
220
- return flatfs.CreateOrOpen(p, c.shardFun, c.syncField)
221
-}
222
-
223
-type leveldsDatastoreConfig struct {
224
- path string
225
- compression ldbopts.Compression
226
-}
227
-
228
-// LeveldsDatastoreConfig returns a levelds DatastoreConfig from a spec
229
-func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
230
- var c leveldsDatastoreConfig
231
- var ok bool
232
-
233
- c.path, ok = params["path"].(string)
234
- if !ok {
235
- return nil, fmt.Errorf("'path' field is missing or not string")
236
- }
237
-
238
- switch cm := params["compression"].(string); cm {
239
- case "none":
240
- c.compression = ldbopts.NoCompression
241
- case "snappy":
242
- c.compression = ldbopts.SnappyCompression
243
- case "":
244
- c.compression = ldbopts.DefaultCompression
245
- default:
246
- return nil, fmt.Errorf("unrecognized value for compression: %s", cm)
247
- }
248
-
249
- return &c, nil
250
-}
251
-
252
-func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
253
- return map[string]interface{}{
254
- "type": "levelds",
255
- "path": c.path,
256
- }
257
-}
258
-
259
-func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
260
- p := c.path
261
- if !filepath.IsAbs(p) {
262
- p = filepath.Join(path, p)
263
- }
264
-
265
- return levelds.NewDatastore(p, &levelds.Options{
266
- Compression: c.compression,
267
- })
268
-}
269
-
163
type memDatastoreConfig struct {
164
cfg map[string]interface{}
165
}
@@ -352,76 +245,3 @@ func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
245
}
246
return measure.New(c.prefix, child), nil
247
}
355
-
356
-type badgerdsDatastoreConfig struct {
357
- path string
358
- syncWrites bool
359
-
360
- vlogFileSize int64
361
-}
362
-
363
-// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
364
-// from the given parameters
365
-func BadgerdsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error) {
366
- var c badgerdsDatastoreConfig
367
- var ok bool
368
-
369
- c.path, ok = params["path"].(string)
370
- if !ok {
371
- return nil, fmt.Errorf("'path' field is missing or not string")
372
- }
373
-
374
- sw, ok := params["syncWrites"]
375
- if !ok {
376
- c.syncWrites = true
377
- } else {
378
- if swb, ok := sw.(bool); ok {
379
- c.syncWrites = swb
380
- } else {
381
- return nil, fmt.Errorf("'syncWrites' field was not a boolean")
382
- }
383
- }
384
-
385
- vls, ok := params["vlogFileSize"]
386
- if !ok {
387
- // default to 1GiB
388
- c.vlogFileSize = badgerds.DefaultOptions.ValueLogFileSize
389
- } else {
390
- if vlogSize, ok := vls.(string); ok {
391
- s, err := humanize.ParseBytes(vlogSize)
392
- if err != nil {
393
- return nil, err
394
- }
395
- c.vlogFileSize = int64(s)
396
- } else {
397
- return nil, fmt.Errorf("'vlogFileSize' field was not a string")
398
- }
399
- }
400
-
401
- return &c, nil
402
-}
403
-
404
-func (c *badgerdsDatastoreConfig) DiskSpec() DiskSpec {
405
- return map[string]interface{}{
406
- "type": "badgerds",
407
- "path": c.path,
408
- }
409
-}
410
-
411
-func (c *badgerdsDatastoreConfig) Create(path string) (repo.Datastore, error) {
412
- p := c.path
413
- if !filepath.IsAbs(p) {
414
- p = filepath.Join(path, p)
415
- }
416
-
417
- err := os.MkdirAll(p, 0755)
418
- if err != nil {
419
- return nil, err
420
- }
421
-
422
- defopts := badgerds.DefaultOptions
423
- defopts.SyncWrites = c.syncWrites
424
- defopts.ValueLogFileSize = c.vlogFileSize
425
-
426
- return badgerds.NewDatastore(p, &defopts)
427
-}