Use flatfs to store objects under /blocks outside of LevelDB
WARNING: No migration performed! That needs to come in a separate commit, perhaps amended into this one. Migration must move keyspace "/b" from leveldb to the flatfs subdir, while removing the "b" prefix (keys should start with just "/").
Tommi Virtanen committed
Mar 16, 2015 at 14:03 UTC
24daeec70c7c600e3f1c413fe6bf11cb57d5b243
5 files changed
+57
-7
Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs/flatfs.go
+2
-2
@@ -11,8 +11,8 @@ import (
11
"path"
12
"strings"
13
14
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
15
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
14
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
15
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
16
)
17
18
const (
Godeps/_workspace/src/github.com/jbenet/go-datastore/mount/mount.go
+3
-3
@@ -6,9 +6,9 @@ import (
6
"errors"
7
"strings"
8
9
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform"
11
- "github.com/jbenet/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
9
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
10
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/keytransform"
11
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/query"
12
)
13
14
var (
blocks/blockstore/blockstore.go
+3
-1
@@ -18,7 +18,7 @@ import (
18
var log = eventlog.Logger("blockstore")
19
20
// BlockPrefix namespaces blockstore datastores
21
-var BlockPrefix = ds.NewKey("b")
21
+var BlockPrefix = ds.NewKey("blocks")
22
23
var ValueTypeMismatch = errors.New("The retrieved value is not a Block")
24
@@ -89,6 +89,8 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan u.Key, error) {
89
90
// KeysOnly, because that would be _a lot_ of data.
91
q := dsq.Query{KeysOnly: true}
92
+ // datastore/namespace does *NOT* fix up Query.Prefix
93
+ q.Prefix = BlockPrefix.String()
94
res, err := bs.datastore.Query(q)
95
if err != nil {
96
return nil, err
repo/fsrepo/fsrepo.go
+34
-1
@@ -10,7 +10,9 @@ import (
10
"sync"
11
12
ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
13
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
14
levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
15
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount"
16
ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
17
repo "github.com/ipfs/go-ipfs/repo"
18
"github.com/ipfs/go-ipfs/repo/common"
@@ -26,6 +28,7 @@ import (
28
29
const (
30
leveldbDirectory = "datastore"
31
+ flatfsDirectory = "blocks"
32
)
33
34
var (
@@ -196,6 +199,11 @@ func Init(repoPath string, conf *config.Config) error {
199
return fmt.Errorf("datastore: %s", err)
200
}
201
202
+ flatfsPath := path.Join(repoPath, flatfsDirectory)
203
+ if err := dir.Writable(flatfsPath); err != nil {
204
+ return fmt.Errorf("datastore: %s", err)
205
+ }
206
+
207
if err := dir.Writable(path.Join(repoPath, "logs")); err != nil {
208
return err
209
}
@@ -246,7 +254,32 @@ func (r *FSRepo) openDatastore() error {
254
if err != nil {
255
return errors.New("unable to open leveldb datastore")
256
}
249
- r.ds = r.leveldbDS
257
+
258
+ // 4TB of 256kB objects ~=17M objects, splitting that 256-way
259
+ // leads to ~66k objects per dir, splitting 256*256-way leads to
260
+ // only 256.
261
+ //
262
+ // The keys seen by the block store have predictable prefixes,
263
+ // including "/" from datastore.Key and 2 bytes from multihash. To
264
+ // reach a uniform 256-way split, we need approximately 4 bytes of
265
+ // prefix.
266
+ blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
267
+ if err != nil {
268
+ return errors.New("unable to open flatfs datastore")
269
+ }
270
+
271
+ mountDS := mount.New([]mount.Mount{
272
+ {Prefix: ds.NewKey("/blocks"), Datastore: blocksDS},
273
+ {Prefix: ds.NewKey("/"), Datastore: r.leveldbDS},
274
+ })
275
+ // Make sure it's ok to claim the virtual datastore from mount as
276
+ // threadsafe. There's no clean way to make mount itself provide
277
+ // this information without copy-pasting the code into two
278
+ // variants. This is the same dilemma as the `[].byte` attempt at
279
+ // introducing const types to Go.
280
+ var _ ds.ThreadSafeDatastore = blocksDS
281
+ var _ ds.ThreadSafeDatastore = r.leveldbDS
282
+ r.ds = ds2.ClaimThreadSafe{mountDS}
283
return nil
284
}
285
util/datastore2/threadsafe.go
new
+15
@@ -0,0 +1,15 @@
1
+package datastore2
2
+
3
+import (
4
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
5
+)
6
+
7
+// ClaimThreadSafe claims that a Datastore is threadsafe, even when
8
+// it's type does not guarantee this. Use carefully.
9
+type ClaimThreadSafe struct {
10
+ datastore.Datastore
11
+}
12
+
13
+var _ datastore.ThreadSafeDatastore = ClaimThreadSafe{}
14
+
15
+func (ClaimThreadSafe) IsThreadSafe() {}