Add config option for flatfs no-sync
License: MIT Signed-off-by: rht <rhtbot@gmail.com>
rht committed
Nov 12, 2015 at 22:28 UTC
8abb12e7602ddb9d89a7d14e41ab17498c666101
9 files changed
+63
-43
Godeps/Godeps.json
+1
-1
@@ -166,7 +166,7 @@
166
},
167
{
168
"ImportPath": "github.com/jbenet/go-datastore",
169
- "Rev": "c835c30f206c1e97172e428f052e225adab9abde"
169
+ "Rev": "bec407bccea1cfaf56ee946e947642e3ac5a9258"
170
},
171
{
172
"ImportPath": "github.com/jbenet/go-detect-race",
Godeps/_workspace/src/github.com/jbenet/go-datastore/coalesce/coalesce.go
+3
-3
@@ -8,10 +8,10 @@ import (
8
dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9
)
10
11
+// parent keys
12
var (
12
- putKey = "put"
13
- getKey = // parent keys
14
- "get"
13
+ putKey = "put"
14
+ getKey = "get"
15
hasKey = "has"
16
deleteKey = "delete"
17
)
Godeps/_workspace/src/github.com/jbenet/go-datastore/elastigo/datastore.go
+2
-1
@@ -6,9 +6,10 @@ import (
6
"net/url"
7
"strings"
8
9
- "github.com/codahale/blake2"
9
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
query "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
11
+
12
+ "github.com/codahale/blake2"
13
"github.com/mattbaird/elastigo/api"
14
"github.com/mattbaird/elastigo/core"
15
)
Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go
+31
-16
@@ -15,6 +15,7 @@ import (
15
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
16
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
17
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-os-rename"
18
+
19
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
20
)
21
@@ -33,11 +34,14 @@ type Datastore struct {
34
path string
35
// length of the dir splay prefix, in bytes of hex digits
36
hexPrefixLen int
37
+
38
+ // sychronize all writes and directory changes for added safety
39
+ sync bool
40
}
41
42
var _ datastore.Datastore = (*Datastore)(nil)
43
40
-func New(path string, prefixLen int) (*Datastore, error) {
44
+func New(path string, prefixLen int, sync bool) (*Datastore, error) {
45
if prefixLen <= 0 || prefixLen > maxPrefixLen {
46
return nil, ErrBadPrefixLen
47
}
@@ -45,6 +49,7 @@ func New(path string, prefixLen int) (*Datastore, error) {
49
path: path,
50
// convert from binary bytes to bytes of hex encoding
51
hexPrefixLen: prefixLen * hex.EncodedLen(1),
52
+ sync: sync,
53
}
54
return fs, nil
55
}
@@ -80,8 +85,10 @@ func (fs *Datastore) makePrefixDir(dir string) error {
85
// it, the creation of the prefix dir itself might not be
86
// durable yet. Sync the root dir after a successful mkdir of
87
// a prefix dir, just to be paranoid.
83
- if err := syncDir(fs.path); err != nil {
84
- return err
88
+ if fs.sync {
89
+ if err := syncDir(fs.path); err != nil {
90
+ return err
91
+ }
92
}
93
return nil
94
}
@@ -148,8 +155,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
155
if _, err := tmp.Write(val); err != nil {
156
return err
157
}
151
- if err := tmp.Sync(); err != nil {
152
- return err
158
+ if fs.sync {
159
+ if err := tmp.Sync(); err != nil {
160
+ return err
161
+ }
162
}
163
if err := tmp.Close(); err != nil {
164
return err
@@ -162,8 +171,10 @@ func (fs *Datastore) doPut(key datastore.Key, val []byte) error {
171
}
172
removed = true
173
165
- if err := syncDir(dir); err != nil {
166
- return err
174
+ if fs.sync {
175
+ if err := syncDir(dir); err != nil {
176
+ return err
177
+ }
178
}
179
return nil
180
}
@@ -213,8 +224,10 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
224
// Now we sync everything
225
// sync and close files
226
for fi, _ := range files {
216
- if err := fi.Sync(); err != nil {
217
- return err
227
+ if fs.sync {
228
+ if err := fi.Sync(); err != nil {
229
+ return err
230
+ }
231
}
232
233
if err := fi.Close(); err != nil {
@@ -236,15 +249,17 @@ func (fs *Datastore) putMany(data map[datastore.Key]interface{}) error {
249
}
250
251
// now sync the dirs for those files
239
- for _, dir := range dirsToSync {
240
- if err := syncDir(dir); err != nil {
241
- return err
252
+ if fs.sync {
253
+ for _, dir := range dirsToSync {
254
+ if err := syncDir(dir); err != nil {
255
+ return err
256
+ }
257
}
243
- }
258
245
- // sync top flatfs dir
246
- if err := syncDir(fs.path); err != nil {
247
- return err
259
+ // sync top flatfs dir
260
+ if err := syncDir(fs.path); err != nil {
261
+ return err
262
+ }
263
}
264
265
return nil
Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs_test.go
+18
-17
@@ -8,11 +8,12 @@ import (
8
"runtime"
9
"testing"
10
11
- rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo"
11
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
12
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
13
"github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
14
dstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/test"
15
+
16
+ rand "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/dustin/randbo"
17
)
18
19
func tempdir(t testing.TB) (path string, cleanup func()) {
@@ -34,7 +35,7 @@ func TestBadPrefixLen(t *testing.T) {
35
defer cleanup()
36
37
for i := 0; i > -3; i-- {
37
- _, err := flatfs.New(temp, 0)
38
+ _, err := flatfs.New(temp, i, false)
39
if g, e := err, flatfs.ErrBadPrefixLen; g != e {
40
t.Errorf("expected ErrBadPrefixLen, got: %v", g)
41
}
@@ -45,7 +46,7 @@ func TestPutBadValueType(t *testing.T) {
46
temp, cleanup := tempdir(t)
47
defer cleanup()
48
48
- fs, err := flatfs.New(temp, 2)
49
+ fs, err := flatfs.New(temp, 2, false)
50
if err != nil {
51
t.Fatalf("New fail: %v\n", err)
52
}
@@ -60,7 +61,7 @@ func TestPut(t *testing.T) {
61
temp, cleanup := tempdir(t)
62
defer cleanup()
63
63
- fs, err := flatfs.New(temp, 2)
64
+ fs, err := flatfs.New(temp, 2, false)
65
if err != nil {
66
t.Fatalf("New fail: %v\n", err)
67
}
@@ -75,7 +76,7 @@ func TestGet(t *testing.T) {
76
temp, cleanup := tempdir(t)
77
defer cleanup()
78
78
- fs, err := flatfs.New(temp, 2)
79
+ fs, err := flatfs.New(temp, 2, false)
80
if err != nil {
81
t.Fatalf("New fail: %v\n", err)
82
}
@@ -103,7 +104,7 @@ func TestPutOverwrite(t *testing.T) {
104
temp, cleanup := tempdir(t)
105
defer cleanup()
106
106
- fs, err := flatfs.New(temp, 2)
107
+ fs, err := flatfs.New(temp, 2, false)
108
if err != nil {
109
t.Fatalf("New fail: %v\n", err)
110
}
@@ -135,7 +136,7 @@ func TestGetNotFoundError(t *testing.T) {
136
temp, cleanup := tempdir(t)
137
defer cleanup()
138
138
- fs, err := flatfs.New(temp, 2)
139
+ fs, err := flatfs.New(temp, 2, false)
140
if err != nil {
141
t.Fatalf("New fail: %v\n", err)
142
}
@@ -153,7 +154,7 @@ func TestStorage(t *testing.T) {
154
const prefixLen = 2
155
const prefix = "7175"
156
const target = prefix + string(os.PathSeparator) + "71757578.data"
156
- fs, err := flatfs.New(temp, prefixLen)
157
+ fs, err := flatfs.New(temp, prefixLen, false)
158
if err != nil {
159
t.Fatalf("New fail: %v\n", err)
160
}
@@ -208,7 +209,7 @@ func TestHasNotFound(t *testing.T) {
209
temp, cleanup := tempdir(t)
210
defer cleanup()
211
211
- fs, err := flatfs.New(temp, 2)
212
+ fs, err := flatfs.New(temp, 2, false)
213
if err != nil {
214
t.Fatalf("New fail: %v\n", err)
215
}
@@ -226,7 +227,7 @@ func TestHasFound(t *testing.T) {
227
temp, cleanup := tempdir(t)
228
defer cleanup()
229
229
- fs, err := flatfs.New(temp, 2)
230
+ fs, err := flatfs.New(temp, 2, false)
231
if err != nil {
232
t.Fatalf("New fail: %v\n", err)
233
}
@@ -248,7 +249,7 @@ func TestDeleteNotFound(t *testing.T) {
249
temp, cleanup := tempdir(t)
250
defer cleanup()
251
251
- fs, err := flatfs.New(temp, 2)
252
+ fs, err := flatfs.New(temp, 2, false)
253
if err != nil {
254
t.Fatalf("New fail: %v\n", err)
255
}
@@ -263,7 +264,7 @@ func TestDeleteFound(t *testing.T) {
264
temp, cleanup := tempdir(t)
265
defer cleanup()
266
266
- fs, err := flatfs.New(temp, 2)
267
+ fs, err := flatfs.New(temp, 2, false)
268
if err != nil {
269
t.Fatalf("New fail: %v\n", err)
270
}
@@ -288,7 +289,7 @@ func TestQuerySimple(t *testing.T) {
289
temp, cleanup := tempdir(t)
290
defer cleanup()
291
291
- fs, err := flatfs.New(temp, 2)
292
+ fs, err := flatfs.New(temp, 2, false)
293
if err != nil {
294
t.Fatalf("New fail: %v\n", err)
295
}
@@ -324,7 +325,7 @@ func TestBatchPut(t *testing.T) {
325
temp, cleanup := tempdir(t)
326
defer cleanup()
327
327
- fs, err := flatfs.New(temp, 2)
328
+ fs, err := flatfs.New(temp, 2, false)
329
if err != nil {
330
t.Fatalf("New fail: %v\n", err)
331
}
@@ -336,7 +337,7 @@ func TestBatchDelete(t *testing.T) {
337
temp, cleanup := tempdir(t)
338
defer cleanup()
339
339
- fs, err := flatfs.New(temp, 2)
340
+ fs, err := flatfs.New(temp, 2, false)
341
if err != nil {
342
t.Fatalf("New fail: %v\n", err)
343
}
@@ -359,7 +360,7 @@ func BenchmarkConsecutivePut(b *testing.B) {
360
temp, cleanup := tempdir(b)
361
defer cleanup()
362
362
- fs, err := flatfs.New(temp, 2)
363
+ fs, err := flatfs.New(temp, 2, false)
364
if err != nil {
365
b.Fatalf("New fail: %v\n", err)
366
}
@@ -389,7 +390,7 @@ func BenchmarkBatchedPut(b *testing.B) {
390
temp, cleanup := tempdir(b)
391
defer cleanup()
392
392
- fs, err := flatfs.New(temp, 2)
393
+ fs, err := flatfs.New(temp, 2, false)
394
if err != nil {
395
b.Fatalf("New fail: %v\n", err)
396
}
Godeps/_workspace/src/github.com/jbenet/go-datastore/lru/datastore_test.go
+2
-1
@@ -5,10 +5,11 @@ import (
5
"testing"
6
7
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8
- lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru" // Hook up gocheck into the "go test" runner.
8
+ lru "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/lru"
9
. "gopkg.in/check.v1"
10
)
11
12
+// Hook up gocheck into the "go test" runner.
13
func Test(t *testing.T) { TestingT(t) }
14
15
type DSSuite struct{}
Godeps/_workspace/src/github.com/jbenet/go-datastore/timecache/timecache.go
+3
-3
@@ -9,10 +9,10 @@ import (
9
dsq "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
10
)
11
12
+// op keys
13
var (
13
- putKey = "put"
14
- getKey = // op keys
15
- "get"
14
+ putKey = "put"
15
+ getKey = "get"
16
hasKey = "has"
17
deleteKey = "delete"
18
)
repo/config/datastore.go
+1
@@ -16,6 +16,7 @@ type Datastore struct {
16
GCPeriod string // in ns, us, ms, s, m, h
17
18
Params *json.RawMessage
19
+ NoSync bool
20
}
21
22
func (d *Datastore) ParamData() []byte {
repo/fsrepo/defaultds.go
+2
-1
@@ -39,7 +39,8 @@ func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
39
// including "/" from datastore.Key and 2 bytes from multihash. To
40
// reach a uniform 256-way split, we need approximately 4 bytes of
41
// prefix.
42
- blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
42
+ syncfs := !r.config.Datastore.NoSync
43
+ blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4, syncfs)
44
if err != nil {
45
return nil, fmt.Errorf("unable to open flatfs datastore: %v", err)
46
}