Use the json repr. of the minimal config as the DiskId.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Jul 14, 2017 at 00:42 UTC
d64ab3ce0f355de6fca955f1899f40a4859c7f90
3 files changed
+71
-74
repo/fsrepo/config_test.go
+12
-12
@@ -87,9 +87,9 @@ func TestDefaultDatastoreConfig(t *testing.T) {
87
t.Fatal(err)
88
}
89
90
- expected := "/blocks:{flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2};/:{levelds;datastore};"
91
- if dsc.DiskId() != expected {
92
- t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
90
+ expected := `{"mounts":[{"mountpoint":{"string":"/blocks"},"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"},{"mountpoint":{"string":"/"},"path":"datastore","type":"levelds"}],"type":"mount"}`
91
+ if dsc.DiskSpec().String() != expected {
92
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
93
}
94
95
ds, err := dsc.Create(dir)
@@ -125,9 +125,9 @@ func TestLevelDbConfig(t *testing.T) {
125
t.Fatal(err)
126
}
127
128
- expected := "levelds;datastore"
129
- if dsc.DiskId() != expected {
130
- t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
128
+ expected := `{"path":"datastore","type":"levelds"}`
129
+ if dsc.DiskSpec().String() != expected {
130
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
131
}
132
133
ds, err := dsc.Create(dir)
@@ -163,9 +163,9 @@ func TestFlatfsConfig(t *testing.T) {
163
t.Fatal(err)
164
}
165
166
- expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2"
167
- if dsc.DiskId() != expected {
168
- t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
166
+ expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}`
167
+ if dsc.DiskSpec().String() != expected {
168
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
169
}
170
171
ds, err := dsc.Create(dir)
@@ -201,9 +201,9 @@ func TestMeasureConfig(t *testing.T) {
201
t.Fatal(err)
202
}
203
204
- expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2"
205
- if dsc.DiskId() != expected {
206
- t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
204
+ expected := `{"path":"blocks","shardFunc":"/repo/flatfs/shard/v1/next-to-last/2","type":"flatfs"}`
205
+ if dsc.DiskSpec().String() != expected {
206
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskSpec().String())
207
}
208
209
ds, err := dsc.Create(dir)
repo/fsrepo/datastores.go
+46
-20
@@ -2,6 +2,7 @@ package fsrepo
2
3
import (
4
"bytes"
5
+ "encoding/json"
6
"fmt"
7
"path/filepath"
8
@@ -18,17 +19,28 @@ import (
19
// ConfigFromMap creates a new datastore config from a map
20
type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error)
21
22
+type DiskSpec map[string]interface{}
23
+
24
type DatastoreConfig interface {
22
- // DiskId is a unique id representing the Datastore config as
23
- // stored on disk, runtime config values are not part of this Id.
24
- // Returns an empty string if the datastore does not have an on
25
- // disk representation. No length limit.
26
- DiskId() string
25
+ // DiskSpec returns a minimal configuration of the datastore
26
+ // represting what is stored on disk. Run time values are
27
+ // excluded.
28
+ DiskSpec() DiskSpec
29
30
// Create instantiate a new datastore from this config
31
Create(path string) (repo.Datastore, error)
32
}
33
34
+func (spec DiskSpec) String() string {
35
+ b, err := json.Marshal(spec)
36
+ if err != nil {
37
+ // should not happen
38
+ panic(err)
39
+ }
40
+ b = bytes.TrimSpace(b)
41
+ return string(b)
42
+}
43
+
44
var datastores map[string]ConfigFromMap
45
46
func init() {
@@ -94,12 +106,19 @@ func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error
106
return &res, nil
107
}
108
97
-func (c *mountDatastoreConfig) DiskId() string {
98
- buf := new(bytes.Buffer)
99
- for _, m := range c.mounts {
100
- fmt.Fprintf(buf, "%s:{%s};", m.prefix.String(), m.ds.DiskId())
109
+func (c *mountDatastoreConfig) DiskSpec() DiskSpec {
110
+ cfg := map[string]interface{}{"type": "mount"}
111
+ mounts := make([]interface{}, len(c.mounts))
112
+ for i, m := range c.mounts {
113
+ c := m.ds.DiskSpec()
114
+ if c == nil {
115
+ c = make(map[string]interface{})
116
+ }
117
+ c["mountpoint"] = m.prefix
118
+ mounts[i] = c
119
}
102
- return buf.String()
120
+ cfg["mounts"] = mounts
121
+ return cfg
122
}
123
124
func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
@@ -147,8 +166,12 @@ func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, erro
166
return &c, nil
167
}
168
150
-func (c *flatfsDatastoreConfig) DiskId() string {
151
- return fmt.Sprintf("flatfs;%s;%s", c.path, c.shardFun.String())
169
+func (c *flatfsDatastoreConfig) DiskSpec() DiskSpec {
170
+ return map[string]interface{}{
171
+ "type": "flatfs",
172
+ "path": c.path,
173
+ "shardFunc": c.shardFun.String(),
174
+ }
175
}
176
177
func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
@@ -188,8 +211,11 @@ func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err
211
return &c, nil
212
}
213
191
-func (c *leveldsDatastoreConfig) DiskId() string {
192
- return fmt.Sprintf("levelds;%s", c.path)
214
+func (c *leveldsDatastoreConfig) DiskSpec() DiskSpec {
215
+ return map[string]interface{}{
216
+ "type": "levelds",
217
+ "path": c.path,
218
+ }
219
}
220
221
func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
@@ -211,8 +237,8 @@ func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error)
237
return &memDatastoreConfig{params}, nil
238
}
239
214
-func (c *memDatastoreConfig) DiskId() string {
215
- return ""
240
+func (c *memDatastoreConfig) DiskSpec() DiskSpec {
241
+ return nil
242
}
243
244
func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
@@ -249,8 +275,8 @@ func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
275
return ds.NewLogDatastore(child, c.name), nil
276
}
277
252
-func (c *logDatastoreConfig) DiskId() string {
253
- return c.child.DiskId()
278
+func (c *logDatastoreConfig) DiskSpec() DiskSpec {
279
+ return c.child.DiskSpec()
280
}
281
282
type measureDatastoreConfig struct {
@@ -274,8 +300,8 @@ func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err
300
return &measureDatastoreConfig{child, prefix}, nil
301
}
302
277
-func (c *measureDatastoreConfig) DiskId() string {
278
- return c.child.DiskId()
303
+func (c *measureDatastoreConfig) DiskSpec() DiskSpec {
304
+ return c.child.DiskSpec()
305
}
306
307
func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
repo/fsrepo/fsrepo.go
+13
-42
@@ -1,7 +1,6 @@
1
package fsrepo
2
3
import (
4
- "encoding/json"
4
"errors"
5
"fmt"
6
"io"
@@ -368,16 +367,16 @@ func (r *FSRepo) openDatastore() error {
367
if err != nil {
368
return err
369
}
371
- diskId := dsc.DiskId()
370
+ spec := dsc.DiskSpec()
371
373
- oldId, _, err := r.readSpec()
372
+ oldSpec, err := r.readSpec()
373
if err == nil {
375
- if oldId != diskId {
374
+ if oldSpec != spec.String() {
375
return fmt.Errorf("Datastore configuration of '%s' does not match what is on disk '%s'",
377
- oldId, diskId)
376
+ oldSpec, spec.String())
377
}
378
} else if os.IsNotExist(err) {
380
- err := r.writeSpec(diskId, r.config.Datastore.Spec)
379
+ err := r.writeSpec(spec.String())
380
if err != nil {
381
return err
382
}
@@ -398,55 +397,27 @@ func (r *FSRepo) openDatastore() error {
397
return nil
398
}
399
401
-var SpecFn = "spec"
400
+var SpecFn = "datastore_spec"
401
403
-func (r *FSRepo) readSpec() (string, map[string]interface{}, error) {
402
+func (r *FSRepo) readSpec() (string, error) {
403
fn, err := config.Path(r.path, SpecFn)
404
if err != nil {
406
- return "", nil, err
405
+ return "", err
406
}
407
b, err := ioutil.ReadFile(fn)
408
if err != nil {
410
- return "", nil, err
409
+ return "", err
410
}
412
- idspec := make(map[string]interface{})
413
- err = json.Unmarshal(b, &idspec)
414
- if err != nil {
415
- return "", nil, err
416
- }
417
- id, ok := idspec["id"].(string)
418
- if !ok {
419
- return "", nil, fmt.Errorf("could not retrieve 'id' field from spec file")
420
- }
421
- spec, ok := idspec["spec"].(map[string]interface{})
422
- if !ok {
423
- return "", nil, fmt.Errorf("could not retrieve 'spec' field from spec file")
424
- }
425
-
426
- dsc, err := AnyDatastoreConfig(spec)
427
- if err != nil {
428
- return "", nil, err
429
- }
430
- computedId := dsc.DiskId()
431
- if computedId != id {
432
- return "", nil, fmt.Errorf("bad spec file, computed id (%s) does not match given (%s)",
433
- computedId, id)
434
- }
435
-
436
- return id, spec, nil
411
+ return strings.TrimSpace(string(b)), nil
412
}
413
439
-func (r *FSRepo) writeSpec(id string, spec map[string]interface{}) error {
414
+func (r *FSRepo) writeSpec(spec string) error {
415
fn, err := config.Path(r.path, SpecFn)
416
if err != nil {
417
return err
418
}
444
- idspec := map[string]interface{}{
445
- "id": id,
446
- "spec": spec,
447
- }
448
- b, err := json.Marshal(idspec)
449
- err = ioutil.WriteFile(fn, b, 0666)
419
+ b := []byte(spec)
420
+ err = ioutil.WriteFile(fn, b, 0600)
421
if err != nil {
422
return err
423
}