filestore: add "--file-order" option to "filestore ls" and "verify"
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
May 23, 2017 at 17:40 UTC
332a7967f97786faea0ee1d2b9dfb0785b8d8565
3 files changed
+128
-12
core/commands/filestore.go
+10
-2
@@ -40,6 +40,9 @@ The output is:
40
Arguments: []cmds.Argument{
41
cmds.StringArg("obj", false, true, "Cid of objects to list."),
42
},
43
+ Options: []cmds.Option{
44
+ cmds.BoolOption("file-order", "sort the results based on the path of the backing file"),
45
+ },
46
Run: func(req cmds.Request, res cmds.Response) {
47
_, fs, err := getFilestore(req)
48
if err != nil {
@@ -53,7 +56,8 @@ The output is:
56
}, req.Context())
57
res.SetOutput(out)
58
} else {
56
- next, err := filestore.ListAll(fs)
59
+ fileOrder, _, _ := req.Option("file-order").Bool()
60
+ next, err := filestore.ListAll(fs, fileOrder)
61
if err != nil {
62
res.SetError(err, cmds.ErrNormal)
63
return
@@ -114,6 +118,9 @@ For ERROR entries the error will also be printed to stderr.
118
Arguments: []cmds.Argument{
119
cmds.StringArg("obj", false, true, "Cid of objects to verify."),
120
},
121
+ Options: []cmds.Option{
122
+ cmds.BoolOption("file-order", "verify the objects based on the order of the backing file"),
123
+ },
124
Run: func(req cmds.Request, res cmds.Response) {
125
_, fs, err := getFilestore(req)
126
if err != nil {
@@ -127,7 +134,8 @@ For ERROR entries the error will also be printed to stderr.
134
}, req.Context())
135
res.SetOutput(out)
136
} else {
130
- next, err := filestore.VerifyAll(fs)
137
+ fileOrder, _, _ := req.Option("file-order").Bool()
138
+ next, err := filestore.VerifyAll(fs, fileOrder)
139
if err != nil {
140
res.SetError(err, cmds.ErrNormal)
141
return
filestore/util.go
+96
-2
@@ -2,6 +2,7 @@ package filestore
2
3
import (
4
"fmt"
5
+ "sort"
6
7
"github.com/ipfs/go-ipfs/blocks/blockstore"
8
pb "github.com/ipfs/go-ipfs/filestore/pb"
@@ -89,7 +90,10 @@ func List(fs *Filestore, key *cid.Cid) *ListRes {
90
// one by one each block in the Filestore's FileManager.
91
// ListAll does not verify that the references are valid or whether
92
// the raw data is accessible. See VerifyAll().
92
-func ListAll(fs *Filestore) (func() *ListRes, error) {
93
+func ListAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) {
94
+ if fileOrder {
95
+ return listAllFileOrder(fs, false)
96
+ }
97
return listAll(fs, false)
98
}
99
@@ -105,7 +109,10 @@ func Verify(fs *Filestore, key *cid.Cid) *ListRes {
109
// returns one by one each block in the Filestore's FileManager.
110
// VerifyAll checks that the reference is valid and that the block data
111
// can be read.
108
-func VerifyAll(fs *Filestore) (func() *ListRes, error) {
112
+func VerifyAll(fs *Filestore, fileOrder bool) (func() *ListRes, error) {
113
+ if fileOrder {
114
+ return listAllFileOrder(fs, true)
115
+ }
116
return listAll(fs, true)
117
}
118
@@ -158,6 +165,93 @@ func next(qr dsq.Results) (*cid.Cid, *pb.DataObj, error) {
165
return c, dobj, nil
166
}
167
168
+func listAllFileOrder(fs *Filestore, verify bool) (func() *ListRes, error) {
169
+ q := dsq.Query{}
170
+ qr, err := fs.fm.ds.Query(q)
171
+ if err != nil {
172
+ return nil, err
173
+ }
174
+
175
+ var entries listEntries
176
+
177
+ for {
178
+ v, ok := qr.NextSync()
179
+ if !ok {
180
+ break
181
+ }
182
+ dobj, err := unmarshalDataObj(v.Value)
183
+ if err != nil {
184
+ entries = append(entries, &listEntry{
185
+ dsKey: v.Key,
186
+ err: err,
187
+ })
188
+ } else {
189
+ entries = append(entries, &listEntry{
190
+ dsKey: v.Key,
191
+ filePath: dobj.GetFilePath(),
192
+ offset: dobj.GetOffset(),
193
+ size: dobj.GetSize_(),
194
+ })
195
+ }
196
+ }
197
+ sort.Sort(entries)
198
+
199
+ i := 0
200
+ return func() *ListRes {
201
+ if i >= len(entries) {
202
+ return nil
203
+ }
204
+ v := entries[i]
205
+ i++
206
+ // attempt to convert the datastore key to a CID,
207
+ // store the error but don't use it yet
208
+ cid, keyErr := dshelp.DsKeyToCid(ds.RawKey(v.dsKey))
209
+ // first if they listRes already had an error return that error
210
+ if v.err != nil {
211
+ return mkListRes(cid, nil, v.err)
212
+ }
213
+ // now reconstruct the DataObj
214
+ dobj := pb.DataObj{
215
+ FilePath: &v.filePath,
216
+ Offset: &v.offset,
217
+ Size_: &v.size,
218
+ }
219
+ // now if we could not convert the datastore key return that
220
+ // error
221
+ if keyErr != nil {
222
+ return mkListRes(cid, &dobj, keyErr)
223
+ }
224
+ // finally verify the dataobj if requested
225
+ var err error
226
+ if verify {
227
+ _, err = fs.fm.readDataObj(cid, &dobj)
228
+ }
229
+ return mkListRes(cid, &dobj, err)
230
+ }, nil
231
+}
232
+
233
+type listEntry struct {
234
+ filePath string
235
+ offset uint64
236
+ dsKey string
237
+ size uint64
238
+ err error
239
+}
240
+
241
+type listEntries []*listEntry
242
+
243
+func (l listEntries) Len() int { return len(l) }
244
+func (l listEntries) Swap(i, j int) { l[i], l[j] = l[j], l[i] }
245
+func (l listEntries) Less(i, j int) bool {
246
+ if l[i].filePath == l[j].filePath {
247
+ if l[i].offset == l[j].offset {
248
+ return l[i].dsKey < l[j].dsKey
249
+ }
250
+ return l[i].offset < l[j].offset
251
+ }
252
+ return l[i].filePath < l[j].filePath
253
+}
254
+
255
func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
256
status := StatusOk
257
errorMsg := ""
test/sharness/t0271-filestore-utils.sh
+22
-8
@@ -37,28 +37,32 @@ test_init() {
37
38
EXPHASH="QmRueCuPMYYvdxWz1vWncF7wzCScEx4qasZXo5aVBb1R4V"
39
40
-cat <<EOF > ls_expect
41
-zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0
42
-zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288
40
+cat <<EOF > ls_expect_file_order
41
zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR 1000 somedir/file1 0
42
+zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0
43
zb2rhe28UqCDm7TFib7PRyQYEkvuq8iahcXA2AbgaxCLvNhfk 262144 somedir/file3 0
44
zb2rhebtyTTuHKyTbJPnkDUSruU5Uma4DN8t2EkvYZ6fP36mm 262144 somedir/file3 262144
45
+zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288
46
zb2rhm9VTrX2mfatggYUk8mHLz78XBxVUTTzLvM2N3d6frdAU 213568 somedir/file3 786432
47
EOF
48
49
+sort < ls_expect_file_order > ls_expect_key_order
50
+
51
FILE1_HASH=zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR
52
FILE2_HASH=zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo
53
FILE3_HASH=QmfE4SDQazxTD7u8VTYs9AJqQL8rrJPUAorLeJXKSZrVf9
54
53
-cat <<EOF > verify_expect
54
-ok zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0
55
-ok zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288
55
+cat <<EOF > verify_expect_file_order
56
ok zb2rhbcZ3aUXYcrbhhDH1JyrpDcpdw1KFJ5Xs5covjnvMpxDR 1000 somedir/file1 0
57
+ok zb2rhaPkR7ZF9BzSC2BfqbcGivi9QMdauermW9YB6NvS7FZMo 10000 somedir/file2 0
58
ok zb2rhe28UqCDm7TFib7PRyQYEkvuq8iahcXA2AbgaxCLvNhfk 262144 somedir/file3 0
59
ok zb2rhebtyTTuHKyTbJPnkDUSruU5Uma4DN8t2EkvYZ6fP36mm 262144 somedir/file3 262144
60
+ok zb2rhav4wcdvNXtaKDTWHYAqtUHMEpygT1cxqMsfK7QrDuHxH 262144 somedir/file3 524288
61
ok zb2rhm9VTrX2mfatggYUk8mHLz78XBxVUTTzLvM2N3d6frdAU 213568 somedir/file3 786432
62
EOF
63
64
+sort < verify_expect_file_order > verify_expect_key_order
65
+
66
test_filestore_adds() {
67
test_expect_success "nocopy add succeeds" '
68
HASH=$(ipfs add --raw-leaves --nocopy -r -q somedir | tail -n1)
@@ -70,7 +74,12 @@ test_filestore_adds() {
74
75
test_expect_success "'ipfs filestore ls' output looks good'" '
76
ipfs filestore ls | sort > ls_actual &&
73
- test_cmp ls_expect ls_actual
77
+ test_cmp ls_expect_key_order ls_actual
78
+ '
79
+
80
+ test_expect_success "'ipfs filestore ls --file-order' output looks good'" '
81
+ ipfs filestore ls --file-order > ls_actual &&
82
+ test_cmp ls_expect_file_order ls_actual
83
'
84
85
test_expect_success "'ipfs filestore ls HASH' works" '
@@ -88,13 +97,18 @@ test_filestore_adds() {
97
test_filestore_state() {
98
test_expect_success "ipfs filestore verify' output looks good'" '
99
ipfs filestore verify | LC_ALL=C sort > verify_actual
91
- test_cmp verify_expect verify_actual
100
+ test_cmp verify_expect_key_order verify_actual
101
'
102
}
103
104
test_filestore_verify() {
105
test_filestore_state
106
107
+ test_expect_success "ipfs filestore verify --file-order' output looks good'" '
108
+ ipfs filestore verify --file-order > verify_actual
109
+ test_cmp verify_expect_file_order verify_actual
110
+ '
111
+
112
test_expect_success "'ipfs filestore verify HASH' works" '
113
ipfs filestore verify $FILE1_HASH > verify_actual &&
114
grep -q somedir/file1 verify_actual