Filestore: make golint happy
Comments for exported functions and little else. License: MIT Signed-off-by: Hector Sanjuan <hector@protocol.ai>
Hector Sanjuan committed
Mar 24, 2017 at 20:36 UTC
8c40bd91946399e53b5a8c3f78f3d8b96c1802b1
3 files changed
+94
-9
filestore/filestore.go
+29
@@ -1,3 +1,10 @@
1
+// Package filestore implements a Blockstore which is able to read certain
2
+// blocks of data directly from its original location in the filesystem.
3
+//
4
+// In a Filestore, object leaves are stored as FilestoreNodes. FilestoreNodes
5
+// include a filesystem path and an offset, allowing a Blockstore dealing with
6
+// such blocks to avoid storing the whole contents and reading them from their
7
+// filesystem location instead.
8
package filestore
9
10
import (
@@ -14,23 +21,31 @@ import (
21
22
var log = logging.Logger("filestore")
23
24
+// Filestore implements a Blockstore by combining a standard Blockstore
25
+// to store regular blocks and a special Blockstore called
26
+// FileManager to store blocks which data exists in an external file.
27
type Filestore struct {
28
fm *FileManager
29
bs blockstore.Blockstore
30
}
31
32
+// FileManager returns the FileManager in Filestore.
33
func (f *Filestore) FileManager() *FileManager {
34
return f.fm
35
}
36
37
+// MainBlockstore returns the standard Blockstore in the Filestore.
38
func (f *Filestore) MainBlockstore() blockstore.Blockstore {
39
return f.bs
40
}
41
42
+// NewFilestore creates one using the given Blockstore and FileManager.
43
func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore {
44
return &Filestore{fm, bs}
45
}
46
47
+// AllKeysChan returns a channel from which to read the keys stored in
48
+// the blockstore. If the given context is cancelled the channel will be closed.
49
func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
50
ctx, cancel := context.WithCancel(ctx)
51
@@ -93,6 +108,10 @@ func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
108
return out, nil
109
}
110
111
+// DeleteBlock deletes the block with the given key from the
112
+// blockstore. As expected, in the case of FileManager blocks, only the
113
+// reference is deleted, not its contents. It may return
114
+// ErrNotFound when the block is not stored.
115
func (f *Filestore) DeleteBlock(c *cid.Cid) error {
116
err1 := f.bs.DeleteBlock(c)
117
if err1 != nil && err1 != blockstore.ErrNotFound {
@@ -116,6 +135,8 @@ func (f *Filestore) DeleteBlock(c *cid.Cid) error {
135
}
136
}
137
138
+// Get retrieves the block with the given Cid. It may return
139
+// ErrNotFound when the block is not stored.
140
func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) {
141
blk, err := f.bs.Get(c)
142
switch err {
@@ -130,6 +151,8 @@ func (f *Filestore) Get(c *cid.Cid) (blocks.Block, error) {
151
return f.fm.Get(c)
152
}
153
154
+// Has returns true if the block with the given Cid is
155
+// stored in the Filestore.
156
func (f *Filestore) Has(c *cid.Cid) (bool, error) {
157
has, err := f.bs.Has(c)
158
if err != nil {
@@ -143,6 +166,10 @@ func (f *Filestore) Has(c *cid.Cid) (bool, error) {
166
return f.fm.Has(c)
167
}
168
169
+// Put stores a block in the Filestore. For blocks of
170
+// underlying type FilestoreNode, the operation is
171
+// delegated to the FileManager, while the rest of blocks
172
+// are handled by the regular blockstore.
173
func (f *Filestore) Put(b blocks.Block) error {
174
has, err := f.Has(b.Cid())
175
if err != nil {
@@ -161,6 +188,8 @@ func (f *Filestore) Put(b blocks.Block) error {
188
}
189
}
190
191
+// PutMany is like Put(), but takes a slice of blocks, allowing
192
+// the underlying blockstore to perform batch transactions.
193
func (f *Filestore) PutMany(bs []blocks.Block) error {
194
var normals []blocks.Block
195
var fstores []*posinfo.FilestoreNode
filestore/fsrefstore.go
+29
@@ -20,26 +20,43 @@ import (
20
cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
21
)
22
23
+// FilestorePrefix identifies the key prefix for FileManager blocks.
24
var FilestorePrefix = ds.NewKey("filestore")
25
26
+// FileManager is a blockstore implementation which stores special
27
+// blocks FilestoreNode type. These nodes only contain a reference
28
+// to the actual location of the block data in the filesystem
29
+// (a path and an offset).
30
type FileManager struct {
31
ds ds.Batching
32
root string
33
}
34
35
+// CorruptReferenceError implements the error interface.
36
+// It is used to indicate that the block contents pointed
37
+// by the referencing blocks cannot be retrieved (i.e. the
38
+// file is not found, or the data changed as it was being read).
39
type CorruptReferenceError struct {
40
Code Status
41
Err error
42
}
43
44
+// Error() returns the error message in the CorruptReferenceError
45
+// as a string.
46
func (c CorruptReferenceError) Error() string {
47
return c.Err.Error()
48
}
49
50
+// NewFileManager initializes a new file manager with the given
51
+// datastore and root. All FilestoreNodes paths are relative to the
52
+// root path given here, which is prepended for any operations.
53
func NewFileManager(ds ds.Batching, root string) *FileManager {
54
return &FileManager{dsns.Wrap(ds, FilestorePrefix), root}
55
}
56
57
+// AllKeysChan returns a channel from which to read the keys stored in
58
+// the FileManager. If the given context is cancelled the channel will be
59
+// closed.
60
func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
61
q := dsq.Query{KeysOnly: true}
62
q.Prefix = FilestorePrefix.String()
@@ -76,6 +93,8 @@ func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error)
93
return out, nil
94
}
95
96
+// DeleteBlock deletes the reference-block from the underlying
97
+// datastore. It does not touch the referenced data.
98
func (f *FileManager) DeleteBlock(c *cid.Cid) error {
99
err := f.ds.Delete(dshelp.CidToDsKey(c))
100
if err == ds.ErrNotFound {
@@ -84,6 +103,10 @@ func (f *FileManager) DeleteBlock(c *cid.Cid) error {
103
return err
104
}
105
106
+// Get reads a block from the datastore. Reading a block
107
+// is done in two steps: the first step retrieves the reference
108
+// block from the datastore. The second step uses the stored
109
+// path and offsets to read the raw block data directly from disk.
110
func (f *FileManager) Get(c *cid.Cid) (blocks.Block, error) {
111
dobj, err := f.getDataObj(c)
112
if err != nil {
@@ -165,6 +188,8 @@ func (f *FileManager) readDataObj(c *cid.Cid, d *pb.DataObj) ([]byte, error) {
188
return outbuf, nil
189
}
190
191
+// Has returns if the FileManager is storing a block reference. It does not
192
+// validate the data, nor checks if the reference is valid.
193
func (f *FileManager) Has(c *cid.Cid) (bool, error) {
194
// NOTE: interesting thing to consider. Has doesnt validate the data.
195
// So the data on disk could be invalid, and we could think we have it.
@@ -176,6 +201,8 @@ type putter interface {
201
Put(ds.Key, interface{}) error
202
}
203
204
+// Put adds a new reference block to the FileManager. It does not check
205
+// that the reference is valid.
206
func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
207
return f.putTo(b, f.ds)
208
}
@@ -204,6 +231,8 @@ func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
231
return to.Put(dshelp.CidToDsKey(b.Cid()), data)
232
}
233
234
+// PutMany is like Put() but takes a slice of blocks instead,
235
+// allowing it to create a batch transaction.
236
func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error {
237
batch, err := f.ds.Batch()
238
if err != nil {
filestore/util.go
+36
-9
@@ -12,8 +12,11 @@ import (
12
cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
13
)
14
15
+// Status is used to identify the state of the block data referenced
16
+// by a FilestoreNode. Among other places, it is used by CorruptReferenceError.
17
type Status int32
18
19
+// These are the supported Status codes.
20
const (
21
StatusOk Status = 0
22
StatusFileError Status = 10 // Backing File Error
@@ -23,6 +26,7 @@ const (
26
StatusKeyNotFound Status = 30
27
)
28
29
+// String provides a human-readable representation for Status codes.
30
func (s Status) String() string {
31
switch s {
32
case StatusOk:
@@ -42,10 +46,16 @@ func (s Status) String() string {
46
}
47
}
48
49
+// Format returns the status formatted as a string
50
+// with leading 0s.
51
func (s Status) Format() string {
52
return fmt.Sprintf("%-7s", s.String())
53
}
54
55
+// ListRes wraps the response of the List*() functions, which
56
+// allows to obtain and verify blocks stored by the FileManager
57
+// of a Filestore. It includes information about the referenced
58
+// block.
59
type ListRes struct {
60
Status Status
61
ErrorMsg string
@@ -55,6 +65,7 @@ type ListRes struct {
65
Size uint64
66
}
67
68
+// FormatLong returns a human readable string for a ListRes object.
69
func (r *ListRes) FormatLong() string {
70
switch {
71
case r.Key == nil:
@@ -66,18 +77,34 @@ func (r *ListRes) FormatLong() string {
77
}
78
}
79
80
+// List fetches the block with the given key from the Filemanager
81
+// of the given Filestore and returns a ListRes object with the information.
82
+// List does not verify that the reference is valid or whether the
83
+// raw data is accesible. See Verify().
84
func List(fs *Filestore, key *cid.Cid) *ListRes {
85
return list(fs, false, key)
86
}
87
88
+// ListAll returns a function as an iterator which, once invoked, returns
89
+// one by one each block in the Filestore's FileManager.
90
+// ListAll does not verify that the references are valid or whether
91
+// the raw data is accessible. See VerifyAll().
92
func ListAll(fs *Filestore) (func() *ListRes, error) {
93
return listAll(fs, false)
94
}
95
96
+// Verify fetches the block with the given key from the Filemanager
97
+// of the given Filestore and returns a ListRes object with the information.
98
+// Verify makes sure that the reference is valid and the block data can be
99
+// read.
100
func Verify(fs *Filestore, key *cid.Cid) *ListRes {
101
return list(fs, true, key)
102
}
103
104
+// VerifyAll returns a function as an iterator which, once invoked,
105
+// returns one by one each block in the Filestore's FileManager.
106
+// VerifyAll checks that the reference is valid and that the block data
107
+// can be read.
108
func VerifyAll(fs *Filestore) (func() *ListRes, error) {
109
return listAll(fs, true)
110
}
@@ -150,14 +177,14 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
177
ErrorMsg: errorMsg,
178
Key: c,
179
}
153
- } else {
154
- return &ListRes{
155
- Status: status,
156
- ErrorMsg: errorMsg,
157
- Key: c,
158
- FilePath: *d.FilePath,
159
- Size: *d.Size_,
160
- Offset: *d.Offset,
161
- }
180
+ }
181
+
182
+ return &ListRes{
183
+ Status: status,
184
+ ErrorMsg: errorMsg,
185
+ Key: c,
186
+ FilePath: *d.FilePath,
187
+ Size: *d.Size_,
188
+ Offset: *d.Offset,
189
}
190
}