Extract filestore
Łukasz Magiera committed
Jul 15, 2019 at 15:11 UTC
4b125b96720a3057978d9134272b3fb49d908260
7 files changed
-1461
filestore/filestore.go
deleted
-251
@@ -1,251 +0,0 @@
1
-// Package filestore implements a Blockstore which is able to read certain
2
-// blocks of data directly from its original location in the filesystem.
3
-//
4
-// In a Filestore, object leaves are stored as FilestoreNodes. FilestoreNodes
5
-// include a filesystem path and an offset, allowing a Blockstore dealing with
6
-// such blocks to avoid storing the whole contents and reading them from their
7
-// filesystem location instead.
8
-package filestore
9
-
10
-import (
11
- "context"
12
- "errors"
13
-
14
- blocks "github.com/ipfs/go-block-format"
15
- cid "github.com/ipfs/go-cid"
16
- dsq "github.com/ipfs/go-datastore/query"
17
- blockstore "github.com/ipfs/go-ipfs-blockstore"
18
- posinfo "github.com/ipfs/go-ipfs-posinfo"
19
- logging "github.com/ipfs/go-log"
20
-)
21
-
22
-var log = logging.Logger("filestore")
23
-
24
-var ErrFilestoreNotEnabled = errors.New("filestore is not enabled, see https://git.io/vNItf")
25
-var ErrUrlstoreNotEnabled = errors.New("urlstore is not enabled")
26
-
27
-// Filestore implements a Blockstore by combining a standard Blockstore
28
-// to store regular blocks and a special Blockstore called
29
-// FileManager to store blocks which data exists in an external file.
30
-type Filestore struct {
31
- fm *FileManager
32
- bs blockstore.Blockstore
33
-}
34
-
35
-// FileManager returns the FileManager in Filestore.
36
-func (f *Filestore) FileManager() *FileManager {
37
- return f.fm
38
-}
39
-
40
-// MainBlockstore returns the standard Blockstore in the Filestore.
41
-func (f *Filestore) MainBlockstore() blockstore.Blockstore {
42
- return f.bs
43
-}
44
-
45
-// NewFilestore creates one using the given Blockstore and FileManager.
46
-func NewFilestore(bs blockstore.Blockstore, fm *FileManager) *Filestore {
47
- return &Filestore{fm, bs}
48
-}
49
-
50
-// AllKeysChan returns a channel from which to read the keys stored in
51
-// the blockstore. If the given context is cancelled the channel will be closed.
52
-func (f *Filestore) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
53
- ctx, cancel := context.WithCancel(ctx)
54
-
55
- a, err := f.bs.AllKeysChan(ctx)
56
- if err != nil {
57
- cancel()
58
- return nil, err
59
- }
60
-
61
- out := make(chan cid.Cid, dsq.KeysOnlyBufSize)
62
- go func() {
63
- defer cancel()
64
- defer close(out)
65
-
66
- var done bool
67
- for !done {
68
- select {
69
- case c, ok := <-a:
70
- if !ok {
71
- done = true
72
- continue
73
- }
74
- select {
75
- case out <- c:
76
- case <-ctx.Done():
77
- return
78
- }
79
- case <-ctx.Done():
80
- return
81
- }
82
- }
83
-
84
- // Can't do these at the same time because the abstractions around
85
- // leveldb make us query leveldb for both operations. We apparently
86
- // cant query leveldb concurrently
87
- b, err := f.fm.AllKeysChan(ctx)
88
- if err != nil {
89
- log.Error("error querying filestore: ", err)
90
- return
91
- }
92
-
93
- done = false
94
- for !done {
95
- select {
96
- case c, ok := <-b:
97
- if !ok {
98
- done = true
99
- continue
100
- }
101
- select {
102
- case out <- c:
103
- case <-ctx.Done():
104
- return
105
- }
106
- case <-ctx.Done():
107
- return
108
- }
109
- }
110
- }()
111
- return out, nil
112
-}
113
-
114
-// DeleteBlock deletes the block with the given key from the
115
-// blockstore. As expected, in the case of FileManager blocks, only the
116
-// reference is deleted, not its contents. It may return
117
-// ErrNotFound when the block is not stored.
118
-func (f *Filestore) DeleteBlock(c cid.Cid) error {
119
- err1 := f.bs.DeleteBlock(c)
120
- if err1 != nil && err1 != blockstore.ErrNotFound {
121
- return err1
122
- }
123
-
124
- err2 := f.fm.DeleteBlock(c)
125
- // if we successfully removed something from the blockstore, but the
126
- // filestore didnt have it, return success
127
-
128
- switch err2 {
129
- case nil:
130
- return nil
131
- case blockstore.ErrNotFound:
132
- if err1 == blockstore.ErrNotFound {
133
- return blockstore.ErrNotFound
134
- }
135
- return nil
136
- default:
137
- return err2
138
- }
139
-}
140
-
141
-// Get retrieves the block with the given Cid. It may return
142
-// ErrNotFound when the block is not stored.
143
-func (f *Filestore) Get(c cid.Cid) (blocks.Block, error) {
144
- blk, err := f.bs.Get(c)
145
- switch err {
146
- case nil:
147
- return blk, nil
148
- case blockstore.ErrNotFound:
149
- return f.fm.Get(c)
150
- default:
151
- return nil, err
152
- }
153
-}
154
-
155
-// GetSize returns the size of the requested block. It may return ErrNotFound
156
-// when the block is not stored.
157
-func (f *Filestore) GetSize(c cid.Cid) (int, error) {
158
- size, err := f.bs.GetSize(c)
159
- switch err {
160
- case nil:
161
- return size, nil
162
- case blockstore.ErrNotFound:
163
- return f.fm.GetSize(c)
164
- default:
165
- return -1, err
166
- }
167
-}
168
-
169
-// Has returns true if the block with the given Cid is
170
-// stored in the Filestore.
171
-func (f *Filestore) Has(c cid.Cid) (bool, error) {
172
- has, err := f.bs.Has(c)
173
- if err != nil {
174
- return false, err
175
- }
176
-
177
- if has {
178
- return true, nil
179
- }
180
-
181
- return f.fm.Has(c)
182
-}
183
-
184
-// Put stores a block in the Filestore. For blocks of
185
-// underlying type FilestoreNode, the operation is
186
-// delegated to the FileManager, while the rest of blocks
187
-// are handled by the regular blockstore.
188
-func (f *Filestore) Put(b blocks.Block) error {
189
- has, err := f.Has(b.Cid())
190
- if err != nil {
191
- return err
192
- }
193
-
194
- if has {
195
- return nil
196
- }
197
-
198
- switch b := b.(type) {
199
- case *posinfo.FilestoreNode:
200
- return f.fm.Put(b)
201
- default:
202
- return f.bs.Put(b)
203
- }
204
-}
205
-
206
-// PutMany is like Put(), but takes a slice of blocks, allowing
207
-// the underlying blockstore to perform batch transactions.
208
-func (f *Filestore) PutMany(bs []blocks.Block) error {
209
- var normals []blocks.Block
210
- var fstores []*posinfo.FilestoreNode
211
-
212
- for _, b := range bs {
213
- has, err := f.Has(b.Cid())
214
- if err != nil {
215
- return err
216
- }
217
-
218
- if has {
219
- continue
220
- }
221
-
222
- switch b := b.(type) {
223
- case *posinfo.FilestoreNode:
224
- fstores = append(fstores, b)
225
- default:
226
- normals = append(normals, b)
227
- }
228
- }
229
-
230
- if len(normals) > 0 {
231
- err := f.bs.PutMany(normals)
232
- if err != nil {
233
- return err
234
- }
235
- }
236
-
237
- if len(fstores) > 0 {
238
- err := f.fm.PutMany(fstores)
239
- if err != nil {
240
- return err
241
- }
242
- }
243
- return nil
244
-}
245
-
246
-// HashOnRead calls blockstore.HashOnRead.
247
-func (f *Filestore) HashOnRead(enabled bool) {
248
- f.bs.HashOnRead(enabled)
249
-}
250
-
251
-var _ blockstore.Blockstore = (*Filestore)(nil)
filestore/filestore_test.go
deleted
-177
@@ -1,177 +0,0 @@
1
-package filestore
2
-
3
-import (
4
- "bytes"
5
- "context"
6
- "io/ioutil"
7
- "math/rand"
8
- "testing"
9
-
10
- dag "github.com/ipfs/go-merkledag"
11
-
12
- cid "github.com/ipfs/go-cid"
13
- ds "github.com/ipfs/go-datastore"
14
- blockstore "github.com/ipfs/go-ipfs-blockstore"
15
- posinfo "github.com/ipfs/go-ipfs-posinfo"
16
-)
17
-
18
-func newTestFilestore(t *testing.T) (string, *Filestore) {
19
- mds := ds.NewMapDatastore()
20
-
21
- testdir, err := ioutil.TempDir("", "filestore-test")
22
- if err != nil {
23
- t.Fatal(err)
24
- }
25
- fm := NewFileManager(mds, testdir)
26
- fm.AllowFiles = true
27
-
28
- bs := blockstore.NewBlockstore(mds)
29
- fstore := NewFilestore(bs, fm)
30
- return testdir, fstore
31
-}
32
-
33
-func makeFile(dir string, data []byte) (string, error) {
34
- f, err := ioutil.TempFile(dir, "file")
35
- if err != nil {
36
- return "", err
37
- }
38
-
39
- _, err = f.Write(data)
40
- if err != nil {
41
- return "", err
42
- }
43
-
44
- return f.Name(), nil
45
-}
46
-
47
-func TestBasicFilestore(t *testing.T) {
48
- dir, fs := newTestFilestore(t)
49
-
50
- buf := make([]byte, 1000)
51
- rand.Read(buf)
52
-
53
- fname, err := makeFile(dir, buf)
54
- if err != nil {
55
- t.Fatal(err)
56
- }
57
-
58
- var cids []cid.Cid
59
- for i := 0; i < 100; i++ {
60
- n := &posinfo.FilestoreNode{
61
- PosInfo: &posinfo.PosInfo{
62
- FullPath: fname,
63
- Offset: uint64(i * 10),
64
- },
65
- Node: dag.NewRawNode(buf[i*10 : (i+1)*10]),
66
- }
67
-
68
- err := fs.Put(n)
69
- if err != nil {
70
- t.Fatal(err)
71
- }
72
- cids = append(cids, n.Node.Cid())
73
- }
74
-
75
- for i, c := range cids {
76
- blk, err := fs.Get(c)
77
- if err != nil {
78
- t.Fatal(err)
79
- }
80
-
81
- if !bytes.Equal(blk.RawData(), buf[i*10:(i+1)*10]) {
82
- t.Fatal("data didnt match on the way out")
83
- }
84
- }
85
-
86
- kch, err := fs.AllKeysChan(context.Background())
87
- if err != nil {
88
- t.Fatal(err)
89
- }
90
-
91
- out := make(map[string]struct{})
92
- for c := range kch {
93
- out[c.KeyString()] = struct{}{}
94
- }
95
-
96
- if len(out) != len(cids) {
97
- t.Fatal("mismatch in number of entries")
98
- }
99
-
100
- for _, c := range cids {
101
- if _, ok := out[c.KeyString()]; !ok {
102
- t.Fatal("missing cid: ", c)
103
- }
104
- }
105
-}
106
-
107
-func randomFileAdd(t *testing.T, fs *Filestore, dir string, size int) (string, []cid.Cid) {
108
- buf := make([]byte, size)
109
- rand.Read(buf)
110
-
111
- fname, err := makeFile(dir, buf)
112
- if err != nil {
113
- t.Fatal(err)
114
- }
115
-
116
- var out []cid.Cid
117
- for i := 0; i < size/10; i++ {
118
- n := &posinfo.FilestoreNode{
119
- PosInfo: &posinfo.PosInfo{
120
- FullPath: fname,
121
- Offset: uint64(i * 10),
122
- },
123
- Node: dag.NewRawNode(buf[i*10 : (i+1)*10]),
124
- }
125
- err := fs.Put(n)
126
- if err != nil {
127
- t.Fatal(err)
128
- }
129
- out = append(out, n.Cid())
130
- }
131
-
132
- return fname, out
133
-}
134
-
135
-func TestDeletes(t *testing.T) {
136
- dir, fs := newTestFilestore(t)
137
- _, cids := randomFileAdd(t, fs, dir, 100)
138
- todelete := cids[:4]
139
- for _, c := range todelete {
140
- err := fs.DeleteBlock(c)
141
- if err != nil {
142
- t.Fatal(err)
143
- }
144
- }
145
-
146
- deleted := make(map[string]bool)
147
- for _, c := range todelete {
148
- _, err := fs.Get(c)
149
- if err != blockstore.ErrNotFound {
150
- t.Fatal("expected blockstore not found error")
151
- }
152
- deleted[c.KeyString()] = true
153
- }
154
-
155
- keys, err := fs.AllKeysChan(context.Background())
156
- if err != nil {
157
- t.Fatal(err)
158
- }
159
-
160
- for c := range keys {
161
- if deleted[c.KeyString()] {
162
- t.Fatal("shouldnt have reference to this key anymore")
163
- }
164
- }
165
-}
166
-
167
-func TestIsURL(t *testing.T) {
168
- if !IsURL("http://www.example.com") {
169
- t.Fatal("IsURL failed: http://www.example.com")
170
- }
171
- if !IsURL("https://www.example.com") {
172
- t.Fatal("IsURL failed: https://www.example.com")
173
- }
174
- if IsURL("adir/afile") || IsURL("http:/ /afile") || IsURL("http:/a/file") {
175
- t.Fatal("IsURL recognized non-url")
176
- }
177
-}
filestore/fsrefstore.go
deleted
-330
@@ -1,330 +0,0 @@
1
-package filestore
2
-
3
-import (
4
- "context"
5
- "fmt"
6
- "io"
7
- "net/http"
8
- "os"
9
- "path/filepath"
10
-
11
- pb "github.com/ipfs/go-ipfs/filestore/pb"
12
-
13
- proto "github.com/gogo/protobuf/proto"
14
- blocks "github.com/ipfs/go-block-format"
15
- cid "github.com/ipfs/go-cid"
16
- ds "github.com/ipfs/go-datastore"
17
- dsns "github.com/ipfs/go-datastore/namespace"
18
- dsq "github.com/ipfs/go-datastore/query"
19
- blockstore "github.com/ipfs/go-ipfs-blockstore"
20
- dshelp "github.com/ipfs/go-ipfs-ds-help"
21
- posinfo "github.com/ipfs/go-ipfs-posinfo"
22
-)
23
-
24
-// FilestorePrefix identifies the key prefix for FileManager blocks.
25
-var FilestorePrefix = ds.NewKey("filestore")
26
-
27
-// FileManager is a blockstore implementation which stores special
28
-// blocks FilestoreNode type. These nodes only contain a reference
29
-// to the actual location of the block data in the filesystem
30
-// (a path and an offset).
31
-type FileManager struct {
32
- AllowFiles bool
33
- AllowUrls bool
34
- ds ds.Batching
35
- root string
36
-}
37
-
38
-// CorruptReferenceError implements the error interface.
39
-// It is used to indicate that the block contents pointed
40
-// by the referencing blocks cannot be retrieved (i.e. the
41
-// file is not found, or the data changed as it was being read).
42
-type CorruptReferenceError struct {
43
- Code Status
44
- Err error
45
-}
46
-
47
-// Error() returns the error message in the CorruptReferenceError
48
-// as a string.
49
-func (c CorruptReferenceError) Error() string {
50
- return c.Err.Error()
51
-}
52
-
53
-// NewFileManager initializes a new file manager with the given
54
-// datastore and root. All FilestoreNodes paths are relative to the
55
-// root path given here, which is prepended for any operations.
56
-func NewFileManager(ds ds.Batching, root string) *FileManager {
57
- return &FileManager{ds: dsns.Wrap(ds, FilestorePrefix), root: root}
58
-}
59
-
60
-// AllKeysChan returns a channel from which to read the keys stored in
61
-// the FileManager. If the given context is cancelled the channel will be
62
-// closed.
63
-func (f *FileManager) AllKeysChan(ctx context.Context) (<-chan cid.Cid, error) {
64
- q := dsq.Query{KeysOnly: true}
65
-
66
- res, err := f.ds.Query(q)
67
- if err != nil {
68
- return nil, err
69
- }
70
-
71
- out := make(chan cid.Cid, dsq.KeysOnlyBufSize)
72
- go func() {
73
- defer close(out)
74
- for {
75
- v, ok := res.NextSync()
76
- if !ok {
77
- return
78
- }
79
-
80
- k := ds.RawKey(v.Key)
81
- c, err := dshelp.DsKeyToCid(k)
82
- if err != nil {
83
- log.Errorf("decoding cid from filestore: %s", err)
84
- continue
85
- }
86
-
87
- select {
88
- case out <- c:
89
- case <-ctx.Done():
90
- return
91
- }
92
- }
93
- }()
94
-
95
- return out, nil
96
-}
97
-
98
-// DeleteBlock deletes the reference-block from the underlying
99
-// datastore. It does not touch the referenced data.
100
-func (f *FileManager) DeleteBlock(c cid.Cid) error {
101
- err := f.ds.Delete(dshelp.CidToDsKey(c))
102
- if err == ds.ErrNotFound {
103
- return blockstore.ErrNotFound
104
- }
105
- return err
106
-}
107
-
108
-// Get reads a block from the datastore. Reading a block
109
-// is done in two steps: the first step retrieves the reference
110
-// block from the datastore. The second step uses the stored
111
-// path and offsets to read the raw block data directly from disk.
112
-func (f *FileManager) Get(c cid.Cid) (blocks.Block, error) {
113
- dobj, err := f.getDataObj(c)
114
- if err != nil {
115
- return nil, err
116
- }
117
- out, err := f.readDataObj(c, dobj)
118
- if err != nil {
119
- return nil, err
120
- }
121
-
122
- return blocks.NewBlockWithCid(out, c)
123
-}
124
-
125
-// GetSize gets the size of the block from the datastore.
126
-//
127
-// This method may successfully return the size even if returning the block
128
-// would fail because the associated file is no longer available.
129
-func (f *FileManager) GetSize(c cid.Cid) (int, error) {
130
- dobj, err := f.getDataObj(c)
131
- if err != nil {
132
- return -1, err
133
- }
134
- return int(dobj.GetSize_()), nil
135
-}
136
-
137
-func (f *FileManager) readDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) {
138
- if IsURL(d.GetFilePath()) {
139
- return f.readURLDataObj(c, d)
140
- }
141
- return f.readFileDataObj(c, d)
142
-}
143
-
144
-func (f *FileManager) getDataObj(c cid.Cid) (*pb.DataObj, error) {
145
- o, err := f.ds.Get(dshelp.CidToDsKey(c))
146
- switch err {
147
- case ds.ErrNotFound:
148
- return nil, blockstore.ErrNotFound
149
- default:
150
- return nil, err
151
- case nil:
152
- //
153
- }
154
-
155
- return unmarshalDataObj(o)
156
-}
157
-
158
-func unmarshalDataObj(data []byte) (*pb.DataObj, error) {
159
- var dobj pb.DataObj
160
- if err := proto.Unmarshal(data, &dobj); err != nil {
161
- return nil, err
162
- }
163
-
164
- return &dobj, nil
165
-}
166
-
167
-func (f *FileManager) readFileDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) {
168
- if !f.AllowFiles {
169
- return nil, ErrFilestoreNotEnabled
170
- }
171
-
172
- p := filepath.FromSlash(d.GetFilePath())
173
- abspath := filepath.Join(f.root, p)
174
-
175
- fi, err := os.Open(abspath)
176
- if os.IsNotExist(err) {
177
- return nil, &CorruptReferenceError{StatusFileNotFound, err}
178
- } else if err != nil {
179
- return nil, &CorruptReferenceError{StatusFileError, err}
180
- }
181
- defer fi.Close()
182
-
183
- _, err = fi.Seek(int64(d.GetOffset()), io.SeekStart)
184
- if err != nil {
185
- return nil, &CorruptReferenceError{StatusFileError, err}
186
- }
187
-
188
- outbuf := make([]byte, d.GetSize_())
189
- _, err = io.ReadFull(fi, outbuf)
190
- if err == io.EOF || err == io.ErrUnexpectedEOF {
191
- return nil, &CorruptReferenceError{StatusFileChanged, err}
192
- } else if err != nil {
193
- return nil, &CorruptReferenceError{StatusFileError, err}
194
- }
195
-
196
- outcid, err := c.Prefix().Sum(outbuf)
197
- if err != nil {
198
- return nil, err
199
- }
200
-
201
- if !c.Equals(outcid) {
202
- return nil, &CorruptReferenceError{StatusFileChanged,
203
- fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())}
204
- }
205
-
206
- return outbuf, nil
207
-}
208
-
209
-// reads and verifies the block from URL
210
-func (f *FileManager) readURLDataObj(c cid.Cid, d *pb.DataObj) ([]byte, error) {
211
- if !f.AllowUrls {
212
- return nil, ErrUrlstoreNotEnabled
213
- }
214
-
215
- req, err := http.NewRequest("GET", d.GetFilePath(), nil)
216
- if err != nil {
217
- return nil, err
218
- }
219
-
220
- req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", d.GetOffset(), d.GetOffset()+d.GetSize_()-1))
221
-
222
- res, err := http.DefaultClient.Do(req)
223
- if err != nil {
224
- return nil, &CorruptReferenceError{StatusFileError, err}
225
- }
226
- if res.StatusCode != http.StatusOK && res.StatusCode != http.StatusPartialContent {
227
- return nil, &CorruptReferenceError{StatusFileError,
228
- fmt.Errorf("expected HTTP 200 or 206 got %d", res.StatusCode)}
229
- }
230
-
231
- outbuf := make([]byte, d.GetSize_())
232
- _, err = io.ReadFull(res.Body, outbuf)
233
- if err == io.EOF || err == io.ErrUnexpectedEOF {
234
- return nil, &CorruptReferenceError{StatusFileChanged, err}
235
- } else if err != nil {
236
- return nil, &CorruptReferenceError{StatusFileError, err}
237
- }
238
- res.Body.Close()
239
-
240
- outcid, err := c.Prefix().Sum(outbuf)
241
- if err != nil {
242
- return nil, err
243
- }
244
-
245
- if !c.Equals(outcid) {
246
- return nil, &CorruptReferenceError{StatusFileChanged,
247
- fmt.Errorf("data in file did not match. %s offset %d", d.GetFilePath(), d.GetOffset())}
248
- }
249
-
250
- return outbuf, nil
251
-}
252
-
253
-// Has returns if the FileManager is storing a block reference. It does not
254
-// validate the data, nor checks if the reference is valid.
255
-func (f *FileManager) Has(c cid.Cid) (bool, error) {
256
- // NOTE: interesting thing to consider. Has doesnt validate the data.
257
- // So the data on disk could be invalid, and we could think we have it.
258
- dsk := dshelp.CidToDsKey(c)
259
- return f.ds.Has(dsk)
260
-}
261
-
262
-type putter interface {
263
- Put(ds.Key, []byte) error
264
-}
265
-
266
-// Put adds a new reference block to the FileManager. It does not check
267
-// that the reference is valid.
268
-func (f *FileManager) Put(b *posinfo.FilestoreNode) error {
269
- return f.putTo(b, f.ds)
270
-}
271
-
272
-func (f *FileManager) putTo(b *posinfo.FilestoreNode, to putter) error {
273
- var dobj pb.DataObj
274
-
275
- if IsURL(b.PosInfo.FullPath) {
276
- if !f.AllowUrls {
277
- return ErrUrlstoreNotEnabled
278
- }
279
- dobj.FilePath = b.PosInfo.FullPath
280
- } else {
281
- if !f.AllowFiles {
282
- return ErrFilestoreNotEnabled
283
- }
284
- if !filepath.HasPrefix(b.PosInfo.FullPath, f.root) { //nolint:staticcheck
285
- return fmt.Errorf("cannot add filestore references outside ipfs root (%s)", f.root)
286
- }
287
-
288
- p, err := filepath.Rel(f.root, b.PosInfo.FullPath)
289
- if err != nil {
290
- return err
291
- }
292
-
293
- dobj.FilePath = filepath.ToSlash(p)
294
- }
295
- dobj.Offset = b.PosInfo.Offset
296
- dobj.Size_ = uint64(len(b.RawData()))
297
-
298
- data, err := proto.Marshal(&dobj)
299
- if err != nil {
300
- return err
301
- }
302
-
303
- return to.Put(dshelp.CidToDsKey(b.Cid()), data)
304
-}
305
-
306
-// PutMany is like Put() but takes a slice of blocks instead,
307
-// allowing it to create a batch transaction.
308
-func (f *FileManager) PutMany(bs []*posinfo.FilestoreNode) error {
309
- batch, err := f.ds.Batch()
310
- if err != nil {
311
- return err
312
- }
313
-
314
- for _, b := range bs {
315
- if err := f.putTo(b, batch); err != nil {
316
- return err
317
- }
318
- }
319
-
320
- return batch.Commit()
321
-}
322
-
323
-// IsURL returns true if the string represents a valid URL that the
324
-// urlstore can handle. More specifically it returns true if a string
325
-// begins with 'http://' or 'https://'.
326
-func IsURL(str string) bool {
327
- return (len(str) > 7 && str[0] == 'h' && str[1] == 't' && str[2] == 't' && str[3] == 'p') &&
328
- ((len(str) > 8 && str[4] == 's' && str[5] == ':' && str[6] == '/' && str[7] == '/') ||
329
- (str[4] == ':' && str[5] == '/' && str[6] == '/'))
330
-}
filestore/pb/Rules.mk
deleted
-8
@@ -1,8 +0,0 @@
1
-include mk/header.mk
2
-
3
-PB_$(d) = $(wildcard $(d)/*.proto)
4
-TGTS_$(d) = $(PB_$(d):.proto=.pb.go)
5
-
6
-#DEPS_GO += $(TGTS_$(d))
7
-
8
-include mk/footer.mk
filestore/pb/dataobj.pb.go
deleted
-399
@@ -1,399 +0,0 @@
1
-// Code generated by protoc-gen-gogo. DO NOT EDIT.
2
-// source: filestore/pb/dataobj.proto
3
-
4
-package datastore_pb
5
-
6
-import (
7
- fmt "fmt"
8
- proto "github.com/gogo/protobuf/proto"
9
- io "io"
10
- math "math"
11
-)
12
-
13
-// Reference imports to suppress errors if they are not otherwise used.
14
-var _ = proto.Marshal
15
-var _ = fmt.Errorf
16
-var _ = math.Inf
17
-
18
-// This is a compile-time assertion to ensure that this generated file
19
-// is compatible with the proto package it is being compiled against.
20
-// A compilation error at this line likely means your copy of the
21
-// proto package needs to be updated.
22
-const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
23
-
24
-type DataObj struct {
25
- FilePath string `protobuf:"bytes,1,opt,name=FilePath" json:"FilePath"`
26
- Offset uint64 `protobuf:"varint,2,opt,name=Offset" json:"Offset"`
27
- Size_ uint64 `protobuf:"varint,3,opt,name=Size" json:"Size"`
28
-}
29
-
30
-func (m *DataObj) Reset() { *m = DataObj{} }
31
-func (m *DataObj) String() string { return proto.CompactTextString(m) }
32
-func (*DataObj) ProtoMessage() {}
33
-func (*DataObj) Descriptor() ([]byte, []int) {
34
- return fileDescriptor_86a3613fbaff9a6c, []int{0}
35
-}
36
-func (m *DataObj) XXX_Unmarshal(b []byte) error {
37
- return m.Unmarshal(b)
38
-}
39
-func (m *DataObj) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
40
- if deterministic {
41
- return xxx_messageInfo_DataObj.Marshal(b, m, deterministic)
42
- } else {
43
- b = b[:cap(b)]
44
- n, err := m.MarshalTo(b)
45
- if err != nil {
46
- return nil, err
47
- }
48
- return b[:n], nil
49
- }
50
-}
51
-func (m *DataObj) XXX_Merge(src proto.Message) {
52
- xxx_messageInfo_DataObj.Merge(m, src)
53
-}
54
-func (m *DataObj) XXX_Size() int {
55
- return m.Size()
56
-}
57
-func (m *DataObj) XXX_DiscardUnknown() {
58
- xxx_messageInfo_DataObj.DiscardUnknown(m)
59
-}
60
-
61
-var xxx_messageInfo_DataObj proto.InternalMessageInfo
62
-
63
-func (m *DataObj) GetFilePath() string {
64
- if m != nil {
65
- return m.FilePath
66
- }
67
- return ""
68
-}
69
-
70
-func (m *DataObj) GetOffset() uint64 {
71
- if m != nil {
72
- return m.Offset
73
- }
74
- return 0
75
-}
76
-
77
-func (m *DataObj) GetSize_() uint64 {
78
- if m != nil {
79
- return m.Size_
80
- }
81
- return 0
82
-}
83
-
84
-func init() {
85
- proto.RegisterType((*DataObj)(nil), "datastore.pb.DataObj")
86
-}
87
-
88
-func init() { proto.RegisterFile("filestore/pb/dataobj.proto", fileDescriptor_86a3613fbaff9a6c) }
89
-
90
-var fileDescriptor_86a3613fbaff9a6c = []byte{
91
- // 160 bytes of a gzipped FileDescriptorProto
92
- 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0x92, 0x4a, 0xcb, 0xcc, 0x49,
93
- 0x2d, 0x2e, 0xc9, 0x2f, 0x4a, 0xd5, 0x2f, 0x48, 0xd2, 0x4f, 0x49, 0x2c, 0x49, 0xcc, 0x4f, 0xca,
94
- 0xd2, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x17, 0xe2, 0x01, 0x71, 0xc1, 0x72, 0x7a, 0x05, 0x49, 0x4a,
95
- 0xc9, 0x5c, 0xec, 0x2e, 0x89, 0x25, 0x89, 0xfe, 0x49, 0x59, 0x42, 0x0a, 0x5c, 0x1c, 0x6e, 0x99,
96
- 0x39, 0xa9, 0x01, 0x89, 0x25, 0x19, 0x12, 0x8c, 0x0a, 0x8c, 0x1a, 0x9c, 0x4e, 0x2c, 0x27, 0xee,
97
- 0xc9, 0x33, 0x04, 0xc1, 0x45, 0x85, 0x64, 0xb8, 0xd8, 0xfc, 0xd3, 0xd2, 0x8a, 0x53, 0x4b, 0x24,
98
- 0x98, 0x14, 0x18, 0x35, 0x58, 0xa0, 0xf2, 0x50, 0x31, 0x21, 0x09, 0x2e, 0x96, 0xe0, 0xcc, 0xaa,
99
- 0x54, 0x09, 0x66, 0x24, 0x39, 0xb0, 0x88, 0x93, 0xc4, 0x89, 0x47, 0x72, 0x8c, 0x17, 0x1e, 0xc9,
100
- 0x31, 0x3e, 0x78, 0x24, 0xc7, 0x38, 0xe1, 0xb1, 0x1c, 0xc3, 0x85, 0xc7, 0x72, 0x0c, 0x37, 0x1e,
101
- 0xcb, 0x31, 0x00, 0x02, 0x00, 0x00, 0xff, 0xff, 0x7f, 0x87, 0xf5, 0x88, 0xa9, 0x00, 0x00, 0x00,
102
-}
103
-
104
-func (m *DataObj) Marshal() (dAtA []byte, err error) {
105
- size := m.Size()
106
- dAtA = make([]byte, size)
107
- n, err := m.MarshalTo(dAtA)
108
- if err != nil {
109
- return nil, err
110
- }
111
- return dAtA[:n], nil
112
-}
113
-
114
-func (m *DataObj) MarshalTo(dAtA []byte) (int, error) {
115
- var i int
116
- _ = i
117
- var l int
118
- _ = l
119
- dAtA[i] = 0xa
120
- i++
121
- i = encodeVarintDataobj(dAtA, i, uint64(len(m.FilePath)))
122
- i += copy(dAtA[i:], m.FilePath)
123
- dAtA[i] = 0x10
124
- i++
125
- i = encodeVarintDataobj(dAtA, i, uint64(m.Offset))
126
- dAtA[i] = 0x18
127
- i++
128
- i = encodeVarintDataobj(dAtA, i, uint64(m.Size_))
129
- return i, nil
130
-}
131
-
132
-func encodeVarintDataobj(dAtA []byte, offset int, v uint64) int {
133
- for v >= 1<<7 {
134
- dAtA[offset] = uint8(v&0x7f | 0x80)
135
- v >>= 7
136
- offset++
137
- }
138
- dAtA[offset] = uint8(v)
139
- return offset + 1
140
-}
141
-func (m *DataObj) Size() (n int) {
142
- if m == nil {
143
- return 0
144
- }
145
- var l int
146
- _ = l
147
- l = len(m.FilePath)
148
- n += 1 + l + sovDataobj(uint64(l))
149
- n += 1 + sovDataobj(uint64(m.Offset))
150
- n += 1 + sovDataobj(uint64(m.Size_))
151
- return n
152
-}
153
-
154
-func sovDataobj(x uint64) (n int) {
155
- for {
156
- n++
157
- x >>= 7
158
- if x == 0 {
159
- break
160
- }
161
- }
162
- return n
163
-}
164
-func sozDataobj(x uint64) (n int) {
165
- return sovDataobj(uint64((x << 1) ^ uint64((int64(x) >> 63))))
166
-}
167
-func (m *DataObj) Unmarshal(dAtA []byte) error {
168
- l := len(dAtA)
169
- iNdEx := 0
170
- for iNdEx < l {
171
- preIndex := iNdEx
172
- var wire uint64
173
- for shift := uint(0); ; shift += 7 {
174
- if shift >= 64 {
175
- return ErrIntOverflowDataobj
176
- }
177
- if iNdEx >= l {
178
- return io.ErrUnexpectedEOF
179
- }
180
- b := dAtA[iNdEx]
181
- iNdEx++
182
- wire |= uint64(b&0x7F) << shift
183
- if b < 0x80 {
184
- break
185
- }
186
- }
187
- fieldNum := int32(wire >> 3)
188
- wireType := int(wire & 0x7)
189
- if wireType == 4 {
190
- return fmt.Errorf("proto: DataObj: wiretype end group for non-group")
191
- }
192
- if fieldNum <= 0 {
193
- return fmt.Errorf("proto: DataObj: illegal tag %d (wire type %d)", fieldNum, wire)
194
- }
195
- switch fieldNum {
196
- case 1:
197
- if wireType != 2 {
198
- return fmt.Errorf("proto: wrong wireType = %d for field FilePath", wireType)
199
- }
200
- var stringLen uint64
201
- for shift := uint(0); ; shift += 7 {
202
- if shift >= 64 {
203
- return ErrIntOverflowDataobj
204
- }
205
- if iNdEx >= l {
206
- return io.ErrUnexpectedEOF
207
- }
208
- b := dAtA[iNdEx]
209
- iNdEx++
210
- stringLen |= uint64(b&0x7F) << shift
211
- if b < 0x80 {
212
- break
213
- }
214
- }
215
- intStringLen := int(stringLen)
216
- if intStringLen < 0 {
217
- return ErrInvalidLengthDataobj
218
- }
219
- postIndex := iNdEx + intStringLen
220
- if postIndex < 0 {
221
- return ErrInvalidLengthDataobj
222
- }
223
- if postIndex > l {
224
- return io.ErrUnexpectedEOF
225
- }
226
- m.FilePath = string(dAtA[iNdEx:postIndex])
227
- iNdEx = postIndex
228
- case 2:
229
- if wireType != 0 {
230
- return fmt.Errorf("proto: wrong wireType = %d for field Offset", wireType)
231
- }
232
- m.Offset = 0
233
- for shift := uint(0); ; shift += 7 {
234
- if shift >= 64 {
235
- return ErrIntOverflowDataobj
236
- }
237
- if iNdEx >= l {
238
- return io.ErrUnexpectedEOF
239
- }
240
- b := dAtA[iNdEx]
241
- iNdEx++
242
- m.Offset |= uint64(b&0x7F) << shift
243
- if b < 0x80 {
244
- break
245
- }
246
- }
247
- case 3:
248
- if wireType != 0 {
249
- return fmt.Errorf("proto: wrong wireType = %d for field Size_", wireType)
250
- }
251
- m.Size_ = 0
252
- for shift := uint(0); ; shift += 7 {
253
- if shift >= 64 {
254
- return ErrIntOverflowDataobj
255
- }
256
- if iNdEx >= l {
257
- return io.ErrUnexpectedEOF
258
- }
259
- b := dAtA[iNdEx]
260
- iNdEx++
261
- m.Size_ |= uint64(b&0x7F) << shift
262
- if b < 0x80 {
263
- break
264
- }
265
- }
266
- default:
267
- iNdEx = preIndex
268
- skippy, err := skipDataobj(dAtA[iNdEx:])
269
- if err != nil {
270
- return err
271
- }
272
- if skippy < 0 {
273
- return ErrInvalidLengthDataobj
274
- }
275
- if (iNdEx + skippy) < 0 {
276
- return ErrInvalidLengthDataobj
277
- }
278
- if (iNdEx + skippy) > l {
279
- return io.ErrUnexpectedEOF
280
- }
281
- iNdEx += skippy
282
- }
283
- }
284
-
285
- if iNdEx > l {
286
- return io.ErrUnexpectedEOF
287
- }
288
- return nil
289
-}
290
-func skipDataobj(dAtA []byte) (n int, err error) {
291
- l := len(dAtA)
292
- iNdEx := 0
293
- for iNdEx < l {
294
- var wire uint64
295
- for shift := uint(0); ; shift += 7 {
296
- if shift >= 64 {
297
- return 0, ErrIntOverflowDataobj
298
- }
299
- if iNdEx >= l {
300
- return 0, io.ErrUnexpectedEOF
301
- }
302
- b := dAtA[iNdEx]
303
- iNdEx++
304
- wire |= (uint64(b) & 0x7F) << shift
305
- if b < 0x80 {
306
- break
307
- }
308
- }
309
- wireType := int(wire & 0x7)
310
- switch wireType {
311
- case 0:
312
- for shift := uint(0); ; shift += 7 {
313
- if shift >= 64 {
314
- return 0, ErrIntOverflowDataobj
315
- }
316
- if iNdEx >= l {
317
- return 0, io.ErrUnexpectedEOF
318
- }
319
- iNdEx++
320
- if dAtA[iNdEx-1] < 0x80 {
321
- break
322
- }
323
- }
324
- return iNdEx, nil
325
- case 1:
326
- iNdEx += 8
327
- return iNdEx, nil
328
- case 2:
329
- var length int
330
- for shift := uint(0); ; shift += 7 {
331
- if shift >= 64 {
332
- return 0, ErrIntOverflowDataobj
333
- }
334
- if iNdEx >= l {
335
- return 0, io.ErrUnexpectedEOF
336
- }
337
- b := dAtA[iNdEx]
338
- iNdEx++
339
- length |= (int(b) & 0x7F) << shift
340
- if b < 0x80 {
341
- break
342
- }
343
- }
344
- if length < 0 {
345
- return 0, ErrInvalidLengthDataobj
346
- }
347
- iNdEx += length
348
- if iNdEx < 0 {
349
- return 0, ErrInvalidLengthDataobj
350
- }
351
- return iNdEx, nil
352
- case 3:
353
- for {
354
- var innerWire uint64
355
- var start int = iNdEx
356
- for shift := uint(0); ; shift += 7 {
357
- if shift >= 64 {
358
- return 0, ErrIntOverflowDataobj
359
- }
360
- if iNdEx >= l {
361
- return 0, io.ErrUnexpectedEOF
362
- }
363
- b := dAtA[iNdEx]
364
- iNdEx++
365
- innerWire |= (uint64(b) & 0x7F) << shift
366
- if b < 0x80 {
367
- break
368
- }
369
- }
370
- innerWireType := int(innerWire & 0x7)
371
- if innerWireType == 4 {
372
- break
373
- }
374
- next, err := skipDataobj(dAtA[start:])
375
- if err != nil {
376
- return 0, err
377
- }
378
- iNdEx = start + next
379
- if iNdEx < 0 {
380
- return 0, ErrInvalidLengthDataobj
381
- }
382
- }
383
- return iNdEx, nil
384
- case 4:
385
- return iNdEx, nil
386
- case 5:
387
- iNdEx += 4
388
- return iNdEx, nil
389
- default:
390
- return 0, fmt.Errorf("proto: illegal wireType %d", wireType)
391
- }
392
- }
393
- panic("unreachable")
394
-}
395
-
396
-var (
397
- ErrInvalidLengthDataobj = fmt.Errorf("proto: negative length found during unmarshaling")
398
- ErrIntOverflowDataobj = fmt.Errorf("proto: integer overflow")
399
-)
filestore/pb/dataobj.proto
deleted
-9
@@ -1,9 +0,0 @@
1
-syntax = "proto2";
2
-
3
-package datastore.pb;
4
-
5
-message DataObj {
6
- optional string FilePath = 1;
7
- optional uint64 Offset = 2;
8
- optional uint64 Size = 3;
9
-}
filestore/util.go
deleted
-287
@@ -1,287 +0,0 @@
1
-package filestore
2
-
3
-import (
4
- "fmt"
5
- "sort"
6
-
7
- pb "github.com/ipfs/go-ipfs/filestore/pb"
8
-
9
- cid "github.com/ipfs/go-cid"
10
- ds "github.com/ipfs/go-datastore"
11
- dsq "github.com/ipfs/go-datastore/query"
12
- blockstore "github.com/ipfs/go-ipfs-blockstore"
13
- dshelp "github.com/ipfs/go-ipfs-ds-help"
14
-)
15
-
16
-// Status is used to identify the state of the block data referenced
17
-// by a FilestoreNode. Among other places, it is used by CorruptReferenceError.
18
-type Status int32
19
-
20
-// These are the supported Status codes.
21
-const (
22
- StatusOk Status = 0
23
- StatusFileError Status = 10 // Backing File Error
24
- StatusFileNotFound Status = 11 // Backing File Not Found
25
- StatusFileChanged Status = 12 // Contents of the file changed
26
- StatusOtherError Status = 20 // Internal Error, likely corrupt entry
27
- StatusKeyNotFound Status = 30
28
-)
29
-
30
-// String provides a human-readable representation for Status codes.
31
-func (s Status) String() string {
32
- switch s {
33
- case StatusOk:
34
- return "ok"
35
- case StatusFileError:
36
- return "error"
37
- case StatusFileNotFound:
38
- return "no-file"
39
- case StatusFileChanged:
40
- return "changed"
41
- case StatusOtherError:
42
- return "ERROR"
43
- case StatusKeyNotFound:
44
- return "missing"
45
- default:
46
- return "???"
47
- }
48
-}
49
-
50
-// Format returns the status formatted as a string
51
-// with leading 0s.
52
-func (s Status) Format() string {
53
- return fmt.Sprintf("%-7s", s.String())
54
-}
55
-
56
-// ListRes wraps the response of the List*() functions, which
57
-// allows to obtain and verify blocks stored by the FileManager
58
-// of a Filestore. It includes information about the referenced
59
-// block.
60
-type ListRes struct {
61
- Status Status
62
- ErrorMsg string
63
- Key cid.Cid
64
- FilePath string
65
- Offset uint64
66
- Size uint64
67
-}
68
-
69
-// FormatLong returns a human readable string for a ListRes object
70
-func (r *ListRes) FormatLong(enc func(cid.Cid) string) string {
71
- if enc == nil {
72
- enc = (cid.Cid).String
73
- }
74
- switch {
75
- case !r.Key.Defined():
76
- return "<corrupt key>"
77
- case r.FilePath == "":
78
- return r.Key.String()
79
- default:
80
- return fmt.Sprintf("%-50s %6d %s %d", enc(r.Key), r.Size, r.FilePath, r.Offset)
81
- }
82
-}
83
-
84
-// List fetches the block with the given key from the Filemanager
85
-// of the given Filestore and returns a ListRes object with the information.
86
-// List does not verify that the reference is valid or whether the
87
-// raw data is accesible. See Verify().
88
-func List(fs *Filestore, key cid.Cid) *ListRes {
89
- return list(fs, false, key)
90
-}
91
-
92
-// ListAll returns a function as an iterator which, once invoked, returns
93
-// one by one each block in the Filestore's FileManager.
94
-// ListAll does not verify that the references are valid or whether
95
-// the raw data is accessible. See VerifyAll().
96
-func ListAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) {
97
- if fileOrder {
98
- return listAllFileOrder(fs, false)
99
- }
100
- return listAll(fs, false)
101
-}
102
-
103
-// Verify fetches the block with the given key from the Filemanager
104
-// of the given Filestore and returns a ListRes object with the information.
105
-// Verify makes sure that the reference is valid and the block data can be
106
-// read.
107
-func Verify(fs *Filestore, key cid.Cid) *ListRes {
108
- return list(fs, true, key)
109
-}
110
-
111
-// VerifyAll returns a function as an iterator which, once invoked,
112
-// returns one by one each block in the Filestore's FileManager.
113
-// VerifyAll checks that the reference is valid and that the block data
114
-// can be read.
115
-func VerifyAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) {
116
- if fileOrder {
117
- return listAllFileOrder(fs, true)
118
- }
119
- return listAll(fs, true)
120
-}
121
-
122
-func list(fs *Filestore, verify bool, key cid.Cid) *ListRes {
123
- dobj, err := fs.fm.getDataObj(key)
124
- if err != nil {
125
- return mkListRes(key, nil, err)
126
- }
127
- if verify {
128
- _, err = fs.fm.readDataObj(key, dobj)
129
- }
130
- return mkListRes(key, dobj, err)
131
-}
132
-
133
-func listAll(fs *Filestore, verify bool) (func() *ListRes, error) {
134
- q := dsq.Query{}
135
- qr, err := fs.fm.ds.Query(q)
136
- if err != nil {
137
- return nil, err
138
- }
139
-
140
- return func() *ListRes {
141
- cid, dobj, err := next(qr)
142
- if dobj == nil && err == nil {
143
- return nil
144
- } else if err == nil && verify {
145
- _, err = fs.fm.readDataObj(cid, dobj)
146
- }
147
- return mkListRes(cid, dobj, err)
148
- }, nil
149
-}
150
-
151
-func next(qr dsq.Results) (cid.Cid, *pb.DataObj, error) {
152
- v, ok := qr.NextSync()
153
- if !ok {
154
- return cid.Cid{}, nil, nil
155
- }
156
-
157
- k := ds.RawKey(v.Key)
158
- c, err := dshelp.DsKeyToCid(k)
159
- if err != nil {
160
- return cid.Cid{}, nil, fmt.Errorf("decoding cid from filestore: %s", err)
161
- }
162
-
163
- dobj, err := unmarshalDataObj(v.Value)
164
- if err != nil {
165
- return c, nil, err
166
- }
167
-
168
- return c, dobj, nil
169
-}
170
-
171
-func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) {
172
- q := dsq.Query{}
173
- qr, err := fs.fm.ds.Query(q)
174
- if err != nil {
175
- return nil, err
176
- }
177
-
178
- var entries listEntries
179
-
180
- for {
181
- v, ok := qr.NextSync()
182
- if !ok {
183
- break
184
- }
185
- dobj, err := unmarshalDataObj(v.Value)
186
- if err != nil {
187
- entries = append(entries, &listEntry{
188
- dsKey: v.Key,
189
- err: err,
190
- })
191
- } else {
192
- entries = append(entries, &listEntry{
193
- dsKey: v.Key,
194
- filePath: dobj.GetFilePath(),
195
- offset: dobj.GetOffset(),
196
- size: dobj.GetSize_(),
197
- })
198
- }
199
- }
200
- sort.Sort(entries)
201
-
202
- i := 0
203
- return func() *ListRes {
204
- if i >= len(entries) {
205
- return nil
206
- }
207
- v := entries[i]
208
- i++
209
- // attempt to convert the datastore key to a CID,
210
- // store the error but don't use it yet
211
- cid, keyErr := dshelp.DsKeyToCid(ds.RawKey(v.dsKey))
212
- // first if they listRes already had an error return that error
213
- if v.err != nil {
214
- return mkListRes(cid, nil, v.err)
215
- }
216
- // now reconstruct the DataObj
217
- dobj := pb.DataObj{
218
- FilePath: v.filePath,
219
- Offset: v.offset,
220
- Size_: v.size,
221
- }
222
- // now if we could not convert the datastore key return that
223
- // error
224
- if keyErr != nil {
225
- return mkListRes(cid, &dobj, keyErr)
226
- }
227
- // finally verify the dataobj if requested
228
- var err error
229
- if verify {
230
- _, err = fs.fm.readDataObj(cid, &dobj)
231
- }
232
- return mkListRes(cid, &dobj, err)
233
- }, nil
234
-}
235
-
236
-type listEntry struct {
237
- filePath string
238
- offset uint64
239
- dsKey string
240
- size uint64
241
- err error
242
-}
243
-
244
-type listEntries []*listEntry
245
-
246
-func (l listEntries) Len() int { return len(l) }
247
-func (l listEntries) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
248
-func (l listEntries) Less(i, j int) bool {
249
- if l[i].filePath == l[j].filePath {
250
- if l[i].offset == l[j].offset {
251
- return l[i].dsKey < l[j].dsKey
252
- }
253
- return l[i].offset < l[j].offset
254
- }
255
- return l[i].filePath < l[j].filePath
256
-}
257
-
258
-func mkListRes(c cid.Cid, d *pb.DataObj, err error) *ListRes {
259
- status := StatusOk
260
- errorMsg := ""
261
- if err != nil {
262
- if err == ds.ErrNotFound || err == blockstore.ErrNotFound {
263
- status = StatusKeyNotFound
264
- } else if err, ok := err.(*CorruptReferenceError); ok {
265
- status = err.Code
266
- } else {
267
- status = StatusOtherError
268
- }
269
- errorMsg = err.Error()
270
- }
271
- if d == nil {
272
- return &ListRes{
273
- Status: status,
274
- ErrorMsg: errorMsg,
275
- Key: c,
276
- }
277
- }
278
-
279
- return &ListRes{
280
- Status: status,
281
- ErrorMsg: errorMsg,
282
- Key: c,
283
- FilePath: d.FilePath,
284
- Size: d.Size_,
285
- Offset: d.Offset,
286
- }
287
-}