fsrepo: Refactor to extract datastore internals
License: MIT Signed-off-by: Tommi Virtanen <tv@eagain.net>
Tommi Virtanen committed
May 20, 2015 at 08:50 UTC
b76581d6c7b10e3b95eda758129a8cc8af8d7767
11 files changed
+140
-96
blocks/blockstore/blockstore.go
+1
-1
@@ -51,7 +51,7 @@ type GCBlockstore interface {
51
PinLock() func()
52
}
53
54
-func NewBlockstore(d ds.ThreadSafeDatastore) *blockstore {
54
+func NewBlockstore(d ds.Datastore) *blockstore {
55
dd := dsns.Wrap(d, BlockPrefix)
56
return &blockstore{
57
datastore: dd,
core/builder.go
+1
-1
@@ -63,7 +63,7 @@ func (cfg *BuildCfg) fillDefaults() error {
63
return nil
64
}
65
66
-func defaultRepo(dstore ds.ThreadSafeDatastore) (repo.Repo, error) {
66
+func defaultRepo(dstore repo.Datastore) (repo.Repo, error) {
67
c := cfg.Config{}
68
priv, pub, err := ci.GenerateKeyPairWithReader(ci.RSA, 1024, rand.Reader)
69
if err != nil {
core/core.go
+2
-2
@@ -570,14 +570,14 @@ func startListening(ctx context.Context, host p2phost.Host, cfg *config.Config)
570
return nil
571
}
572
573
-func constructDHTRouting(ctx context.Context, host p2phost.Host, dstore ds.ThreadSafeDatastore) (routing.IpfsRouting, error) {
573
+func constructDHTRouting(ctx context.Context, host p2phost.Host, dstore ds.Datastore) (routing.IpfsRouting, error) {
574
dhtRouting := dht.NewDHT(ctx, host, dstore)
575
dhtRouting.Validator[IpnsValidatorTag] = namesys.IpnsRecordValidator
576
dhtRouting.Selector[IpnsValidatorTag] = namesys.IpnsSelectorFunc
577
return dhtRouting, nil
578
}
579
580
-type RoutingOption func(context.Context, p2phost.Host, ds.ThreadSafeDatastore) (routing.IpfsRouting, error)
580
+type RoutingOption func(context.Context, p2phost.Host, ds.Datastore) (routing.IpfsRouting, error)
581
582
type DiscoveryOption func(p2phost.Host) (discovery.Service, error)
583
core/corerouting/core.go
+2
-2
@@ -28,7 +28,7 @@ var (
28
// routing records to the provided datastore. Only routing records are store in
29
// the datastore.
30
func SupernodeServer(recordSource ds.ThreadSafeDatastore) core.RoutingOption {
31
- return func(ctx context.Context, ph host.Host, dstore ds.ThreadSafeDatastore) (routing.IpfsRouting, error) {
31
+ return func(ctx context.Context, ph host.Host, dstore ds.Datastore) (routing.IpfsRouting, error) {
32
server, err := supernode.NewServer(recordSource, ph.Peerstore(), ph.ID())
33
if err != nil {
34
return nil, err
@@ -44,7 +44,7 @@ func SupernodeServer(recordSource ds.ThreadSafeDatastore) core.RoutingOption {
44
45
// TODO doc
46
func SupernodeClient(remotes ...peer.PeerInfo) core.RoutingOption {
47
- return func(ctx context.Context, ph host.Host, dstore ds.ThreadSafeDatastore) (routing.IpfsRouting, error) {
47
+ return func(ctx context.Context, ph host.Host, dstore ds.Datastore) (routing.IpfsRouting, error) {
48
if len(remotes) < 1 {
49
return nil, errServersMissing
50
}
pin/pin.go
+3
-3
@@ -64,11 +64,11 @@ type pinner struct {
64
// not delete them.
65
internalPin map[key.Key]struct{}
66
dserv mdag.DAGService
67
- dstore ds.ThreadSafeDatastore
67
+ dstore ds.Datastore
68
}
69
70
// NewPinner creates a new pinner using the given datastore as a backend
71
-func NewPinner(dstore ds.ThreadSafeDatastore, serv mdag.DAGService) Pinner {
71
+func NewPinner(dstore ds.Datastore, serv mdag.DAGService) Pinner {
72
73
// Load set from given datastore...
74
rcset := set.NewSimpleBlockSet()
@@ -207,7 +207,7 @@ func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
207
}
208
209
// LoadPinner loads a pinner and its keysets from the given datastore
210
-func LoadPinner(d ds.ThreadSafeDatastore, dserv mdag.DAGService) (Pinner, error) {
210
+func LoadPinner(d ds.Datastore, dserv mdag.DAGService) (Pinner, error) {
211
p := new(pinner)
212
213
rootKeyI, err := d.Get(pinDatastoreKey)
repo/fsrepo/defaultds.go
new
+105
@@ -0,0 +1,105 @@
1
+package fsrepo
2
+
3
+import (
4
+ "fmt"
5
+ "path"
6
+
7
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
9
+ levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
10
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/measure"
11
+ "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount"
12
+ ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
13
+ repo "github.com/ipfs/go-ipfs/repo"
14
+ config "github.com/ipfs/go-ipfs/repo/config"
15
+ "github.com/ipfs/go-ipfs/thirdparty/dir"
16
+)
17
+
18
+const (
19
+ leveldbDirectory = "datastore"
20
+ flatfsDirectory = "blocks"
21
+)
22
+
23
+type defaultDatastore struct {
24
+ repo.Datastore
25
+
26
+ // tracked separately for use in Close; do not use directly.
27
+ leveldbDS repo.Datastore
28
+ metricsBlocks repo.Datastore
29
+ metricsLevelDB repo.Datastore
30
+}
31
+
32
+func openDefaultDatastore(r *FSRepo) (repo.Datastore, error) {
33
+ d := &defaultDatastore{}
34
+
35
+ leveldbPath := path.Join(r.path, leveldbDirectory)
36
+ var err error
37
+ // save leveldb reference so it can be neatly closed afterward
38
+ d.leveldbDS, err = levelds.NewDatastore(leveldbPath, &levelds.Options{
39
+ Compression: ldbopts.NoCompression,
40
+ })
41
+ if err != nil {
42
+ return nil, fmt.Errorf("unable to open leveldb datastore: %v", err)
43
+ }
44
+
45
+ // 4TB of 256kB objects ~=17M objects, splitting that 256-way
46
+ // leads to ~66k objects per dir, splitting 256*256-way leads to
47
+ // only 256.
48
+ //
49
+ // The keys seen by the block store have predictable prefixes,
50
+ // including "/" from datastore.Key and 2 bytes from multihash. To
51
+ // reach a uniform 256-way split, we need approximately 4 bytes of
52
+ // prefix.
53
+ blocksDS, err := flatfs.New(path.Join(r.path, flatfsDirectory), 4)
54
+ if err != nil {
55
+ return nil, fmt.Errorf("unable to open flatfs datastore: %v", err)
56
+ }
57
+
58
+ // Add our PeerID to metrics paths to keep them unique
59
+ //
60
+ // As some tests just pass a zero-value Config to fsrepo.Init,
61
+ // cope with missing PeerID.
62
+ id := r.config.Identity.PeerID
63
+ if id == "" {
64
+ // the tests pass in a zero Config; cope with it
65
+ id = fmt.Sprintf("uninitialized_%p", r)
66
+ }
67
+ prefix := "fsrepo." + id + ".datastore."
68
+ d.metricsBlocks = measure.New(prefix+"blocks", blocksDS)
69
+ d.metricsLevelDB = measure.New(prefix+"leveldb", d.leveldbDS)
70
+ mountDS := mount.New([]mount.Mount{
71
+ {
72
+ Prefix: ds.NewKey("/blocks"),
73
+ Datastore: d.metricsBlocks,
74
+ },
75
+ {
76
+ Prefix: ds.NewKey("/"),
77
+ Datastore: d.metricsLevelDB,
78
+ },
79
+ })
80
+ // Make sure it's ok to claim the virtual datastore from mount as
81
+ // threadsafe. There's no clean way to make mount itself provide
82
+ // this information without copy-pasting the code into two
83
+ // variants. This is the same dilemma as the `[].byte` attempt at
84
+ // introducing const types to Go.
85
+ d.Datastore = mountDS
86
+
87
+ return d, nil
88
+}
89
+
90
+func initDefaultDatastore(repoPath string, conf *config.Config) error {
91
+ // The actual datastore contents are initialized lazily when Opened.
92
+ // During Init, we merely check that the directory is writeable.
93
+ leveldbPath := path.Join(repoPath, leveldbDirectory)
94
+ if err := dir.Writable(leveldbPath); err != nil {
95
+ return fmt.Errorf("datastore: %s", err)
96
+ }
97
+
98
+ flatfsPath := path.Join(repoPath, flatfsDirectory)
99
+ if err := dir.Writable(flatfsPath); err != nil {
100
+ return fmt.Errorf("datastore: %s", err)
101
+ }
102
+ return nil
103
+}
104
+
105
+var _ repo.Datastore = (*defaultDatastore)(nil)
repo/fsrepo/fsrepo.go
+12
-79
@@ -10,12 +10,6 @@ import (
10
"strings"
11
"sync"
12
13
- ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
14
- "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/flatfs"
15
- levelds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/leveldb"
16
- "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/measure"
17
- "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore/mount"
18
- ldbopts "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/syndtr/goleveldb/leveldb/opt"
13
repo "github.com/ipfs/go-ipfs/repo"
14
"github.com/ipfs/go-ipfs/repo/common"
15
config "github.com/ipfs/go-ipfs/repo/config"
@@ -24,7 +18,6 @@ import (
18
serialize "github.com/ipfs/go-ipfs/repo/fsrepo/serialize"
19
dir "github.com/ipfs/go-ipfs/thirdparty/dir"
20
util "github.com/ipfs/go-ipfs/util"
27
- ds2 "github.com/ipfs/go-ipfs/util/datastore2"
21
logging "github.com/ipfs/go-ipfs/vendor/QmQg1J6vikuXF9oDvm4wpdeAUvvkVEKW1EYDw9HhTMnP2b/go-log"
22
)
23
@@ -56,11 +49,7 @@ func (err NoRepoError) Error() string {
49
return fmt.Sprintf("no ipfs repo found in %s.\nplease run: ipfs init", err.Path)
50
}
51
59
-const (
60
- leveldbDirectory = "datastore"
61
- flatfsDirectory = "blocks"
62
- apiFile = "api"
63
-)
52
+const apiFile = "api"
53
54
var (
55
@@ -94,7 +83,7 @@ type FSRepo struct {
83
// the same fsrepo path concurrently
84
lockfile io.Closer
85
config *config.Config
97
- ds ds.ThreadSafeDatastore
86
+ ds repo.Datastore
87
}
88
89
var _ repo.Repo = (*FSRepo)(nil)
@@ -247,16 +236,8 @@ func Init(repoPath string, conf *config.Config) error {
236
return err
237
}
238
250
- // The actual datastore contents are initialized lazily when Opened.
251
- // During Init, we merely check that the directory is writeable.
252
- leveldbPath := filepath.Join(repoPath, leveldbDirectory)
253
- if err := dir.Writable(leveldbPath); err != nil {
254
- return fmt.Errorf("datastore: %s", err)
255
- }
256
-
257
- flatfsPath := filepath.Join(repoPath, flatfsDirectory)
258
- if err := dir.Writable(flatfsPath); err != nil {
259
- return fmt.Errorf("datastore: %s", err)
239
+ if err := initDefaultDatastore(repoPath, conf); err != nil {
240
+ return err
241
}
242
243
if err := dir.Writable(filepath.Join(repoPath, "logs")); err != nil {
@@ -343,59 +324,11 @@ func (r *FSRepo) openConfig() error {
324
325
// openDatastore returns an error if the config file is not present.
326
func (r *FSRepo) openDatastore() error {
346
- leveldbPath := filepath.Join(r.path, leveldbDirectory)
347
- var err error
348
- // save leveldb reference so it can be neatly closed afterward
349
- leveldbDS, err := levelds.NewDatastore(leveldbPath, &levelds.Options{
350
- Compression: ldbopts.NoCompression,
351
- })
327
+ d, err := openDefaultDatastore(r)
328
if err != nil {
353
- return errors.New("unable to open leveldb datastore")
354
- }
355
-
356
- // 4TB of 256kB objects ~=17M objects, splitting that 256-way
357
- // leads to ~66k objects per dir, splitting 256*256-way leads to
358
- // only 256.
359
- //
360
- // The keys seen by the block store have predictable prefixes,
361
- // including "/" from datastore.Key and 2 bytes from multihash. To
362
- // reach a uniform 256-way split, we need approximately 4 bytes of
363
- // prefix.
364
- blocksDS, err := flatfs.New(filepath.Join(r.path, flatfsDirectory), 4)
365
- if err != nil {
366
- return errors.New("unable to open flatfs datastore")
329
+ return err
330
}
368
-
369
- // Add our PeerID to metrics paths to keep them unique
370
- //
371
- // As some tests just pass a zero-value Config to fsrepo.Init,
372
- // cope with missing PeerID.
373
- id := r.config.Identity.PeerID
374
- if id == "" {
375
- // the tests pass in a zero Config; cope with it
376
- id = fmt.Sprintf("uninitialized_%p", r)
377
- }
378
- prefix := "fsrepo." + id + ".datastore."
379
- metricsBlocks := measure.New(prefix+"blocks", blocksDS)
380
- metricsLevelDB := measure.New(prefix+"leveldb", leveldbDS)
381
- mountDS := mount.New([]mount.Mount{
382
- {
383
- Prefix: ds.NewKey("/blocks"),
384
- Datastore: metricsBlocks,
385
- },
386
- {
387
- Prefix: ds.NewKey("/"),
388
- Datastore: metricsLevelDB,
389
- },
390
- })
391
- // Make sure it's ok to claim the virtual datastore from mount as
392
- // threadsafe. There's no clean way to make mount itself provide
393
- // this information without copy-pasting the code into two
394
- // variants. This is the same dilemma as the `[].byte` attempt at
395
- // introducing const types to Go.
396
- var _ ds.ThreadSafeDatastore = blocksDS
397
- var _ ds.ThreadSafeDatastore = leveldbDS
398
- r.ds = ds2.ClaimThreadSafe{mountDS}
331
+ r.ds = d
332
return nil
333
}
334
@@ -408,15 +341,15 @@ func (r *FSRepo) Close() error {
341
return errors.New("repo is closed")
342
}
343
411
- if err := r.ds.(io.Closer).Close(); err != nil {
412
- return err
413
- }
414
-
344
err := os.Remove(filepath.Join(r.path, apiFile))
345
if err != nil {
346
log.Warning("error removing api file: ", err)
347
}
348
349
+ if err := r.ds.Close(); err != nil {
350
+ return err
351
+ }
352
+
353
// This code existed in the previous versions, but
354
// EventlogComponent.Close was never called. Preserving here
355
// pending further discussion.
@@ -579,7 +512,7 @@ func (r *FSRepo) SetConfigKey(key string, value interface{}) error {
512
513
// Datastore returns a repo-owned datastore. If FSRepo is Closed, return value
514
// is undefined.
582
-func (r *FSRepo) Datastore() ds.ThreadSafeDatastore {
515
+func (r *FSRepo) Datastore() repo.Datastore {
516
packageLock.Lock()
517
d := r.ds
518
packageLock.Unlock()
repo/mock.go
+2
-3
@@ -3,7 +3,6 @@ package repo
3
import (
4
"errors"
5
6
- ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
6
"github.com/ipfs/go-ipfs/repo/config"
7
)
8
@@ -12,7 +11,7 @@ var errTODO = errors.New("TODO")
11
// Mock is not thread-safe
12
type Mock struct {
13
C config.Config
15
- D ds.ThreadSafeDatastore
14
+ D Datastore
15
}
16
17
func (m *Mock) Config() (*config.Config, error) {
@@ -32,7 +31,7 @@ func (m *Mock) GetConfigKey(key string) (interface{}, error) {
31
return nil, errTODO
32
}
33
35
-func (m *Mock) Datastore() ds.ThreadSafeDatastore { return m.D }
34
+func (m *Mock) Datastore() Datastore { return m.D }
35
36
func (m *Mock) GetStorageUsage() (uint64, error) { return 0, nil }
37
repo/repo.go
+9
-2
@@ -4,7 +4,7 @@ import (
4
"errors"
5
"io"
6
7
- datastore "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
7
+ ds "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-datastore"
8
9
config "github.com/ipfs/go-ipfs/repo/config"
10
)
@@ -20,7 +20,7 @@ type Repo interface {
20
SetConfigKey(key string, value interface{}) error
21
GetConfigKey(key string) (interface{}, error)
22
23
- Datastore() datastore.ThreadSafeDatastore
23
+ Datastore() Datastore
24
GetStorageUsage() (uint64, error)
25
26
// SetAPIAddr sets the API address in the repo.
@@ -28,3 +28,10 @@ type Repo interface {
28
29
io.Closer
30
}
31
+
32
+// Datastore is the interface required from a datastore to be
33
+// acceptable to FSRepo.
34
+type Datastore interface {
35
+ ds.Datastore // should be threadsafe, just be careful
36
+ io.Closer
37
+}
routing/dht/dht.go
+2
-2
@@ -44,7 +44,7 @@ type IpfsDHT struct {
44
self peer.ID // Local peer (yourself)
45
peerstore peer.Peerstore // Peer Registry
46
47
- datastore ds.ThreadSafeDatastore // Local data
47
+ datastore ds.Datastore // Local data
48
49
routingTable *kb.RoutingTable // Array of routing tables for differently distanced nodes
50
providers *ProviderManager
@@ -60,7 +60,7 @@ type IpfsDHT struct {
60
}
61
62
// NewDHT creates a new DHT object with the given peer as the 'local' host
63
-func NewDHT(ctx context.Context, h host.Host, dstore ds.ThreadSafeDatastore) *IpfsDHT {
63
+func NewDHT(ctx context.Context, h host.Host, dstore ds.Datastore) *IpfsDHT {
64
dht := new(IpfsDHT)
65
dht.datastore = dstore
66
dht.self = h.ID()
routing/none/none_client.go
+1
-1
@@ -47,7 +47,7 @@ func (c *nilclient) Bootstrap(_ context.Context) error {
47
return nil
48
}
49
50
-func ConstructNilRouting(_ context.Context, _ p2phost.Host, _ ds.ThreadSafeDatastore) (routing.IpfsRouting, error) {
50
+func ConstructNilRouting(_ context.Context, _ p2phost.Host, _ ds.Datastore) (routing.IpfsRouting, error) {
51
return &nilclient{}, nil
52
}
53