Implement DiskId()
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Jul 5, 2017 at 18:32 UTC
5027b4b14f2fc60e808a39d3b3efc9316d8e511d
3 files changed
+137
-21
repo/fsrepo/config_test.go
+52
-8
@@ -75,17 +75,28 @@ func TestDefaultDatastoreConfig(t *testing.T) {
75
t.Fatal(err)
76
}
77
defer os.RemoveAll(dir) // clean up
78
- repo := FSRepo{path: dir}
78
79
config := new(config.Datastore)
80
err = json.Unmarshal(defaultConfig, config)
81
if err != nil {
82
t.Fatal(err)
83
}
85
- ds, err := repo.constructDatastore(config.Spec)
84
+
85
+ dsc, err := AnyDatastoreConfig(config.Spec)
86
+ if err != nil {
87
+ t.Fatal(err)
88
+ }
89
+
90
+ expected := "/blocks:{flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2};/:{levelds;datastore};"
91
+ if dsc.DiskId() != expected {
92
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
93
+ }
94
+
95
+ ds, err := dsc.Create(dir)
96
if err != nil {
97
t.Fatal(err)
98
}
99
+
100
if typ := reflect.TypeOf(ds).String(); typ != "*syncmount.Datastore" {
101
t.Errorf("expected '*syncmount.Datastore' got '%s'", typ)
102
}
@@ -102,17 +113,28 @@ func TestLevelDbConfig(t *testing.T) {
113
t.Fatal(err)
114
}
115
defer os.RemoveAll(dir) // clean up
105
- repo := FSRepo{path: dir}
116
117
spec := make(map[string]interface{})
118
err = json.Unmarshal(leveldbConfig, &spec)
119
if err != nil {
120
t.Fatal(err)
121
}
112
- ds, err := repo.constructDatastore(spec)
122
+
123
+ dsc, err := AnyDatastoreConfig(spec)
124
+ if err != nil {
125
+ t.Fatal(err)
126
+ }
127
+
128
+ expected := "levelds;datastore"
129
+ if dsc.DiskId() != expected {
130
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
131
+ }
132
+
133
+ ds, err := dsc.Create(dir)
134
if err != nil {
135
t.Fatal(err)
136
}
137
+
138
if typ := reflect.TypeOf(ds).String(); typ != "*leveldb.datastore" {
139
t.Errorf("expected '*leveldb.datastore' got '%s'", typ)
140
}
@@ -129,17 +151,28 @@ func TestFlatfsConfig(t *testing.T) {
151
t.Fatal(err)
152
}
153
defer os.RemoveAll(dir) // clean up
132
- repo := FSRepo{path: dir}
154
155
spec := make(map[string]interface{})
156
err = json.Unmarshal(flatfsConfig, &spec)
157
if err != nil {
158
t.Fatal(err)
159
}
139
- ds, err := repo.constructDatastore(spec)
160
+
161
+ dsc, err := AnyDatastoreConfig(spec)
162
+ if err != nil {
163
+ t.Fatal(err)
164
+ }
165
+
166
+ expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2"
167
+ if dsc.DiskId() != expected {
168
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
169
+ }
170
+
171
+ ds, err := dsc.Create(dir)
172
if err != nil {
173
t.Fatal(err)
174
}
175
+
176
if typ := reflect.TypeOf(ds).String(); typ != "*flatfs.Datastore" {
177
t.Errorf("expected '*flatfs.Datastore' got '%s'", typ)
178
}
@@ -156,17 +189,28 @@ func TestMeasureConfig(t *testing.T) {
189
t.Fatal(err)
190
}
191
defer os.RemoveAll(dir) // clean up
159
- repo := FSRepo{path: dir}
192
193
spec := make(map[string]interface{})
194
err = json.Unmarshal(measureConfig, &spec)
195
if err != nil {
196
t.Fatal(err)
197
}
166
- ds, err := repo.constructDatastore(spec)
198
+
199
+ dsc, err := AnyDatastoreConfig(spec)
200
+ if err != nil {
201
+ t.Fatal(err)
202
+ }
203
+
204
+ expected := "flatfs;blocks;/repo/flatfs/shard/v1/next-to-last/2"
205
+ if dsc.DiskId() != expected {
206
+ t.Errorf("expected '%s' got '%s' as DiskId", expected, dsc.DiskId())
207
+ }
208
+
209
+ ds, err := dsc.Create(dir)
210
if err != nil {
211
t.Fatal(err)
212
}
213
+
214
if typ := reflect.TypeOf(ds).String(); typ != "*measure.measure" {
215
t.Errorf("expected '*measure.measure' got '%s'", typ)
216
}
repo/fsrepo/datastores.go
+34
-3
@@ -1,6 +1,7 @@
1
package fsrepo
2
3
import (
4
+ "bytes"
5
"fmt"
6
"path/filepath"
7
@@ -18,9 +19,11 @@ import (
19
type ConfigFromMap func(map[string]interface{}) (DatastoreConfig, error)
20
21
type DatastoreConfig interface {
21
- // DiskId is a unique id representing the Datastore config as stored on disk, runtime config values are not
22
- // part of this Id. No length limit.
23
- //DiskId() string
22
+ // DiskId is a unique id representing the Datastore config as
23
+ // stored on disk, runtime config values are not part of this Id.
24
+ // Returns an empty string if the datastore does not have an on
25
+ // disk representation. No length limit.
26
+ DiskId() string
27
28
// Create instantiate a new datastore from this config
29
Create(path string) (repo.Datastore, error)
@@ -91,6 +94,14 @@ func MountDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error
94
return &res, nil
95
}
96
97
+func (c *mountDatastoreConfig) DiskId() string {
98
+ buf := new(bytes.Buffer)
99
+ for _, m := range c.mounts {
100
+ fmt.Fprintf(buf, "%s:{%s};", m.prefix.String(), m.ds.DiskId())
101
+ }
102
+ return buf.String()
103
+}
104
+
105
func (c *mountDatastoreConfig) Create(path string) (repo.Datastore, error) {
106
mounts := make([]mount.Mount, len(c.mounts))
107
for i, m := range c.mounts {
@@ -136,6 +147,10 @@ func FlatfsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, erro
147
return &c, nil
148
}
149
150
+func (c *flatfsDatastoreConfig) DiskId() string {
151
+ return fmt.Sprintf("flatfs;%s;%s", c.path, c.shardFun.String())
152
+}
153
+
154
func (c *flatfsDatastoreConfig) Create(path string) (repo.Datastore, error) {
155
p := c.path
156
if !filepath.IsAbs(p) {
@@ -173,6 +188,10 @@ func LeveldsDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err
188
return &c, nil
189
}
190
191
+func (c *leveldsDatastoreConfig) DiskId() string {
192
+ return fmt.Sprintf("levelds;%s", c.path)
193
+}
194
+
195
func (c *leveldsDatastoreConfig) Create(path string) (repo.Datastore, error) {
196
p := c.path
197
if !filepath.IsAbs(p) {
@@ -192,6 +211,10 @@ func MemDatastoreConfig(params map[string]interface{}) (DatastoreConfig, error)
211
return &memDatastoreConfig{params}, nil
212
}
213
214
+func (c *memDatastoreConfig) DiskId() string {
215
+ return ""
216
+}
217
+
218
func (c *memDatastoreConfig) Create(string) (repo.Datastore, error) {
219
return ds.NewMapDatastore(), nil
220
}
@@ -226,6 +249,10 @@ func (c *logDatastoreConfig) Create(path string) (repo.Datastore, error) {
249
return ds.NewLogDatastore(child, c.name), nil
250
}
251
252
+func (c *logDatastoreConfig) DiskId() string {
253
+ return c.child.DiskId()
254
+}
255
+
256
type measureDatastoreConfig struct {
257
child DatastoreConfig
258
prefix string
@@ -247,6 +274,10 @@ func MeasureDatastoreConfig(params map[string]interface{}) (DatastoreConfig, err
274
return &measureDatastoreConfig{child, prefix}, nil
275
}
276
277
+func (c *measureDatastoreConfig) DiskId() string {
278
+ return c.child.DiskId()
279
+}
280
+
281
func (c measureDatastoreConfig) Create(path string) (repo.Datastore, error) {
282
child, err := c.child.Create(path)
283
if err != nil {
repo/fsrepo/fsrepo.go
+51
-10
@@ -1,6 +1,7 @@
1
package fsrepo
2
3
import (
4
+ "bytes"
5
"errors"
6
"fmt"
7
"io"
@@ -357,18 +358,39 @@ func (r *FSRepo) openKeystore() error {
358
359
// openDatastore returns an error if the config file is not present.
360
func (r *FSRepo) openDatastore() error {
360
- if r.config.Datastore.Spec != nil {
361
- d, err := r.constructDatastore(r.config.Datastore.Spec)
361
+ if r.config.Datastore.Type != "" || r.config.Datastore.Path != "" {
362
+ return fmt.Errorf("old style datatstore config detected")
363
+ } else if r.config.Datastore.Spec == nil {
364
+ return fmt.Errorf("required Datastore.Spec entry missing form config file")
365
+ }
366
+
367
+ dsc, err := AnyDatastoreConfig(r.config.Datastore.Spec)
368
+ if err != nil {
369
+ return err
370
+ }
371
+ diskId := dsc.DiskId()
372
+
373
+ oldDiskId, err := r.readDiskId()
374
+ if err == nil {
375
+ if oldDiskId != diskId {
376
+ return fmt.Errorf("Datastore configuration of '%s' does not match what is on disk '%s'",
377
+ oldDiskId, diskId)
378
+ }
379
+ } else if os.IsNotExist(err) {
380
+ err := r.writeDiskId(diskId)
381
if err != nil {
382
return err
383
}
365
- r.ds = d
366
- } else if r.config.Datastore.Type != "" || r.config.Datastore.Path != "" {
367
- return fmt.Errorf("old style datatstore config detected")
384
} else {
369
- return fmt.Errorf("required Datastore.Spec entry missing form config file")
385
+ return err
386
}
387
388
+ d, err := dsc.Create(r.path)
389
+ if err != nil {
390
+ return err
391
+ }
392
+ r.ds = d
393
+
394
// Wrap it with metrics gathering
395
prefix := "ipfs.fsrepo.datastore"
396
r.ds = measure.New(prefix, r.ds)
@@ -376,12 +398,31 @@ func (r *FSRepo) openDatastore() error {
398
return nil
399
}
400
379
-func (r *FSRepo) constructDatastore(params map[string]interface{}) (repo.Datastore, error) {
380
- cfg, err := AnyDatastoreConfig(params)
401
+var DiskIdFn = "dsid"
402
+
403
+func (r *FSRepo) readDiskId() (string, error) {
404
+ fn, err := config.Path(r.path, DiskIdFn)
405
if err != nil {
382
- return nil, err
406
+ return "", err
407
+ }
408
+ b, err := ioutil.ReadFile(fn)
409
+ if err != nil {
410
+ return "", err
411
+ }
412
+ b = bytes.TrimSpace(b)
413
+ return string(b), nil
414
+}
415
+
416
+func (r *FSRepo) writeDiskId(newId string) error {
417
+ fn, err := config.Path(r.path, DiskIdFn)
418
+ if err != nil {
419
+ return err
420
}
384
- return cfg.Create(r.path)
421
+ err = ioutil.WriteFile(fn, []byte(newId), 0666)
422
+ if err != nil {
423
+ return err
424
+ }
425
+ return nil
426
}
427
428
// Close closes the FSRepo, releasing held resources.