make datastore configuration nicely customizable
License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> make things super customizable License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> better json format License: MIT Signed-off-by: Jeromy <jeromyj@gmail.com> Migrate to new flatfs License: MIT Signed-off-by: Jakub Sztandera <kubuxu@protonmail.ch>
Jeromy committed
Mar 15, 2016 at 13:46 UTC
7875674caeb84acb52e4bc47598e40adbe2f401e
4 files changed
+181
-30
repo/config/datastore.go
+19
-16
@@ -1,40 +1,43 @@
1
package config
2
3
-import (
4
- "encoding/json"
5
-)
6
-
3
// DefaultDataStoreDirectory is the directory to store all the local IPFS data.
4
const DefaultDataStoreDirectory = "datastore"
5
6
// Datastore tracks the configuration of the datastore.
7
type Datastore struct {
12
- Type string
13
- Path string
8
StorageMax string // in B, kB, kiB, MB, ...
9
StorageGCWatermark int64 // in percentage to multiply on StorageMax
10
GCPeriod string // in ns, us, ms, s, m, h
11
+ Path string
12
+ NoSync bool // deprecated
13
+
14
+ Spec map[string]interface{}
15
18
- Params *json.RawMessage
19
- NoSync bool
16
HashOnRead bool
17
BloomFilterSize int
18
}
19
24
-func (d *Datastore) ParamData() []byte {
25
- if d.Params == nil {
26
- return nil
27
- }
28
-
29
- return []byte(*d.Params)
30
-}
31
-
20
type S3Datastore struct {
21
Region string `json:"region"`
22
Bucket string `json:"bucket"`
23
ACL string `json:"acl"`
24
}
25
26
+type FlatDS struct {
27
+ Path string
28
+ ShardFunc string
29
+ Sync bool
30
+}
31
+
32
+type LevelDB struct {
33
+ Path string
34
+ Compression string
35
+}
36
+
37
+type SbsDS struct {
38
+ Path string
39
+}
40
+
41
// DataStorePath returns the default data store path given a configuration root
42
// (set an empty string to have the default configuration root)
43
func DataStorePath(configroot string) (string, error) {
repo/config/init.go
+29
-10
@@ -42,7 +42,7 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
42
Gateway: "/ip4/127.0.0.1/tcp/8080",
43
},
44
45
- Datastore: datastore,
45
+ Datastore: *datastore,
46
Bootstrap: BootstrapPeerStrings(bootstrapPeers),
47
Identity: identity,
48
Discovery: Discovery{MDNS{
@@ -79,19 +79,38 @@ func Init(out io.Writer, nBitsForKeypair int) (*Config, error) {
79
return conf, nil
80
}
81
82
-func datastoreConfig() (Datastore, error) {
83
- dspath, err := DataStorePath("")
84
- if err != nil {
85
- return Datastore{}, err
86
- }
87
- return Datastore{
88
- Path: dspath,
89
- Type: "leveldb",
82
+func datastoreConfig() (*Datastore, error) {
83
+ return &Datastore{
84
StorageMax: "10GB",
85
StorageGCWatermark: 90, // 90%
86
GCPeriod: "1h",
93
- HashOnRead: false,
87
BloomFilterSize: 0,
88
+ Spec: map[string]interface{}{
89
+ "type": "mount",
90
+ "mounts": []interface{}{
91
+ map[string]interface{}{
92
+ "mountpoint": "/blocks",
93
+ "type": "measure",
94
+ "prefix": "flatfs.datastore",
95
+ "child": map[string]interface{}{
96
+ "type": "flatfs",
97
+ "path": "blocks",
98
+ "nosync": false,
99
+ "shardFunc": "/repo/flatfs/shard/v1/next-to-last/2",
100
+ },
101
+ },
102
+ map[string]interface{}{
103
+ "mountpoint": "/",
104
+ "type": "measure",
105
+ "prefix": "leveldb.datastore",
106
+ "child": map[string]interface{}{
107
+ "type": "levelds",
108
+ "path": "datastore",
109
+ "compression": "none",
110
+ },
111
+ },
112
+ },
113
+ },
114
}, nil
115
}
116
repo/fsrepo/datastores.go
new
+124
@@ -0,0 +1,124 @@
1
+package fsrepo
2
+
3
+import (
4
+ "encoding/json"
5
+ "fmt"
6
+ "path/filepath"
7
+
8
+ repo "github.com/ipfs/go-ipfs/repo"
9
+
10
+ levelds "gx/ipfs/QmPdvXuXWAR6gtxxqZw42RtSADMwz4ijVmYHGS542b6cMz/go-ds-leveldb"
11
+ measure "gx/ipfs/QmSb95iHExSSb47zpmyn5CyY5PZidVWSjyKyDqgYQrnKor/go-ds-measure"
12
+ flatfs "gx/ipfs/QmUTshC2PP4ZDqkrFfDU4JGJFMWjYnunxPgkQ6ZCA2hGqh/go-ds-flatfs"
13
+ ds "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore"
14
+ mount "gx/ipfs/QmVSase1JP7cq9QkPT46oNwdp9pT6kBkG3oqS14y3QcZjG/go-datastore/syncmount"
15
+ ldbopts "gx/ipfs/QmbBhyDKsY4mbY6xsKt3qu9Y7FPvMJ6qbD8AMjYYvPRw1g/goleveldb/leveldb/opt"
16
+)
17
+
18
+func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) {
19
+ switch params["type"] {
20
+ case "mount":
21
+ mounts, ok := params["mounts"].([]interface{})
22
+ if !ok {
23
+ return nil, fmt.Errorf("mounts field wasnt an array")
24
+ }
25
+
26
+ return r.openMountDatastore(mounts)
27
+ case "flatfs":
28
+ return r.openFlatfsDatastore(params)
29
+ case "mem":
30
+ return ds.NewMapDatastore(), nil
31
+ case "log":
32
+ child, err := r.constructDatastore(params["child"].(map[string]interface{}))
33
+ if err != nil {
34
+ return nil, err
35
+ }
36
+
37
+ return ds.NewLogDatastore(child, params["name"].(string)), nil
38
+ case "measure":
39
+ child, err := r.constructDatastore(params["child"].(map[string]interface{}))
40
+ if err != nil {
41
+ return nil, err
42
+ }
43
+
44
+ prefix := params["prefix"].(string)
45
+
46
+ return r.openMeasureDB(prefix, child)
47
+
48
+ case "levelds":
49
+ return r.openLeveldbDatastore(params)
50
+ default:
51
+ return nil, fmt.Errorf("unknown datastore type: %s", params["type"])
52
+ }
53
+}
54
+
55
+type mountConfig struct {
56
+ Path string
57
+ ChildType string
58
+ Child *json.RawMessage
59
+}
60
+
61
+func (r *FSRepo) openMountDatastore(mountcfg []interface{}) (repo.Datastore, error) {
62
+ var mounts []mount.Mount
63
+ for _, iface := range mountcfg {
64
+ cfg := iface.(map[string]interface{})
65
+
66
+ child, err := r.constructDatastore(cfg)
67
+ if err != nil {
68
+ return nil, err
69
+ }
70
+
71
+ prefix, found := cfg["mountpoint"]
72
+ if !found {
73
+ return nil, fmt.Errorf("no 'mountpoint' on mount")
74
+ }
75
+
76
+ mounts = append(mounts, mount.Mount{
77
+ Datastore: child,
78
+ Prefix: ds.NewKey(prefix.(string)),
79
+ })
80
+ }
81
+
82
+ return mount.New(mounts), nil
83
+}
84
+
85
+func (r *FSRepo) openFlatfsDatastore(params map[string]interface{}) (repo.Datastore, error) {
86
+ p := params["path"].(string)
87
+ if !filepath.IsAbs(p) {
88
+ p = filepath.Join(r.path, p)
89
+ }
90
+
91
+ sshardFun := params["shardFunc"].(string)
92
+ shardFun, err := flatfs.ParseShardFunc(sshardFun)
93
+ if err != nil {
94
+ return nil, err
95
+ }
96
+
97
+ return flatfs.CreateOrOpen(p, shardFun, params["nosync"].(bool))
98
+}
99
+
100
+func (r *FSRepo) openLeveldbDatastore(params map[string]interface{}) (repo.Datastore, error) {
101
+ p := params["path"].(string)
102
+ if !filepath.IsAbs(p) {
103
+ p = filepath.Join(r.path, p)
104
+ }
105
+
106
+ var c ldbopts.Compression
107
+ switch params["compression"].(string) {
108
+ case "none":
109
+ c = ldbopts.NoCompression
110
+ case "snappy":
111
+ c = ldbopts.SnappyCompression
112
+ case "":
113
+ fallthrough
114
+ default:
115
+ c = ldbopts.DefaultCompression
116
+ }
117
+ return levelds.NewDatastore(p, &levelds.Options{
118
+ Compression: c,
119
+ })
120
+}
121
+
122
+func (r *FSRepo) openMeasureDB(prefix string, child repo.Datastore) (repo.Datastore, error) {
123
+ return measure.New(prefix, child), nil
124
+}
repo/fsrepo/fsrepo.go
+9
-4
@@ -361,15 +361,20 @@ func (r *FSRepo) openKeystore() error {
361
362
// openDatastore returns an error if the config file is not present.
363
func (r *FSRepo) openDatastore() error {
364
- switch r.config.Datastore.Type {
365
- case "default", "leveldb", "":
364
+ if r.config.Datastore.Spec != nil {
365
+ d, err := r.constructDatastore(r.config.Datastore.Spec)
366
+ if err != nil {
367
+ return err
368
+ }
369
+
370
+ r.ds = d
371
+ } else {
372
+ // TODO: This is for legacy configs, remove in the future
373
d, err := openDefaultDatastore(r)
374
if err != nil {
375
return err
376
}
377
r.ds = d
371
- default:
372
- return fmt.Errorf("unknown datastore type: %s", r.config.Datastore.Type)
378
}
379
380
// Wrap it with metrics gathering