filestore util: allow listing/verifying of individual blocks.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Mar 7, 2017 at 00:48 UTC
13f617d139687a1f6ffa7cb9c0caba45693bd685
2 files changed
+94
-16
core/commands/filestore.go
+64
-13
@@ -8,6 +8,7 @@ import (
8
"github.com/ipfs/go-ipfs/core"
9
"github.com/ipfs/go-ipfs/filestore"
10
u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
11
+ cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
12
)
13
14
var FileStoreCmd = &cmds.Command{
@@ -24,19 +25,30 @@ var lsFileStore = &cmds.Command{
25
Helptext: cmds.HelpText{
26
Tagline: "List objects in filestore.",
27
},
28
+ Arguments: []cmds.Argument{
29
+ cmds.StringArg("obj", false, true, "Cid of objects to list."),
30
+ },
31
Run: func(req cmds.Request, res cmds.Response) {
32
_, fs, err := getFilestore(req)
33
if err != nil {
34
res.SetError(err, cmds.ErrNormal)
35
return
36
}
33
- next, err := filestore.ListAll(fs)
34
- if err != nil {
35
- res.SetError(err, cmds.ErrNormal)
36
- return
37
+ args := req.Arguments()
38
+ if len(args) > 0 {
39
+ out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes {
40
+ return filestore.List(fs, c)
41
+ }, req.Context())
42
+ res.SetOutput(out)
43
+ } else {
44
+ next, err := filestore.ListAll(fs)
45
+ if err != nil {
46
+ res.SetError(err, cmds.ErrNormal)
47
+ return
48
+ }
49
+ out := listResToChan(next, req.Context())
50
+ res.SetOutput(out)
51
}
38
- out := listResAsChan(next, req.Context())
39
- res.SetOutput(out)
52
},
53
PostRun: func(req cmds.Request, res cmds.Response) {
54
if res.Error() != nil {
@@ -52,6 +64,7 @@ var lsFileStore = &cmds.Command{
64
for r0 := range outChan {
65
r := r0.(*filestore.ListRes)
66
if r.ErrorMsg != "" {
67
+ errors = true
68
fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg)
69
} else {
70
fmt.Fprintf(res.Stdout(), "%s\n", r.FormatLong())
@@ -68,19 +81,30 @@ var verifyFileStore = &cmds.Command{
81
Helptext: cmds.HelpText{
82
Tagline: "Verify objects in filestore.",
83
},
84
+ Arguments: []cmds.Argument{
85
+ cmds.StringArg("obj", false, true, "Cid of objects to verify."),
86
+ },
87
Run: func(req cmds.Request, res cmds.Response) {
88
_, fs, err := getFilestore(req)
89
if err != nil {
90
res.SetError(err, cmds.ErrNormal)
91
return
92
}
77
- next, err := filestore.VerifyAll(fs)
78
- if err != nil {
79
- res.SetError(err, cmds.ErrNormal)
80
- return
93
+ args := req.Arguments()
94
+ if len(args) > 0 {
95
+ out := perKeyActionToChan(args, func(c *cid.Cid) *filestore.ListRes {
96
+ return filestore.Verify(fs, c)
97
+ }, req.Context())
98
+ res.SetOutput(out)
99
+ } else {
100
+ next, err := filestore.VerifyAll(fs)
101
+ if err != nil {
102
+ res.SetError(err, cmds.ErrNormal)
103
+ return
104
+ }
105
+ out := listResToChan(next, req.Context())
106
+ res.SetOutput(out)
107
}
82
- out := listResAsChan(next, req.Context())
83
- res.SetOutput(out)
108
},
109
PostRun: func(req cmds.Request, res cmds.Response) {
110
if res.Error() != nil {
@@ -94,6 +118,9 @@ var verifyFileStore = &cmds.Command{
118
res.SetOutput(nil)
119
for r0 := range outChan {
120
r := r0.(*filestore.ListRes)
121
+ if r.Status == filestore.StatusOtherError {
122
+ fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg)
123
+ }
124
fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong())
125
}
126
},
@@ -112,7 +139,7 @@ func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error
139
return n, fs, err
140
}
141
115
-func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} {
142
+func listResToChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} {
143
out := make(chan interface{}, 128)
144
go func() {
145
defer close(out)
@@ -130,3 +157,27 @@ func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan i
157
}()
158
return out
159
}
160
+
161
+func perKeyActionToChan(args []string, action func(*cid.Cid) *filestore.ListRes, ctx context.Context) <-chan interface{} {
162
+ out := make(chan interface{}, 128)
163
+ go func() {
164
+ defer close(out)
165
+ for _, arg := range args {
166
+ c, err := cid.Decode(arg)
167
+ if err != nil {
168
+ out <- &filestore.ListRes{
169
+ Status: filestore.StatusOtherError,
170
+ ErrorMsg: fmt.Sprintf("%s: %v", arg, err),
171
+ }
172
+ continue
173
+ }
174
+ r := action(c)
175
+ select {
176
+ case out <- r:
177
+ case <-ctx.Done():
178
+ return
179
+ }
180
+ }
181
+ }()
182
+ return out
183
+}
filestore/util.go
+30
-3
@@ -3,6 +3,7 @@ package filestore
3
import (
4
"fmt"
5
6
+ "github.com/ipfs/go-ipfs/blocks/blockstore"
7
pb "github.com/ipfs/go-ipfs/filestore/pb"
8
dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
9
@@ -19,7 +20,8 @@ const (
20
StatusFileError Status = 10 // Backing File Error
21
//StatusFileNotFound Status = 11 // Backing File Not Found
22
//StatusFileChanged Status = 12 // Contents of the file changed
22
- StatusOtherError Status = 20 // Internal Error, likely corrupt entry
23
+ StatusOtherError Status = 20 // Internal Error, likely corrupt entry
24
+ StatusKeyNotFound Status = 30
25
)
26
27
func (s Status) String() string {
@@ -30,13 +32,15 @@ func (s Status) String() string {
32
return "error"
33
case StatusOtherError:
34
return "ERROR"
35
+ case StatusKeyNotFound:
36
+ return "missing"
37
default:
38
return "???"
39
}
40
}
41
42
func (s Status) Format() string {
39
- return fmt.Sprintf("%-5s", s.String())
43
+ return fmt.Sprintf("%-7s", s.String())
44
}
45
46
type ListRes struct {
@@ -52,19 +56,40 @@ func (r *ListRes) FormatLong() string {
56
switch {
57
case r.Key == nil:
58
return "?????????????????????????????????????????????????"
59
+ case r.FilePath == "":
60
+ return r.Key.String()
61
default:
62
return fmt.Sprintf("%-50s %6d %s %d", r.Key, r.Size, r.FilePath, r.Offset)
63
}
64
}
65
66
+func List(fs *Filestore, key *cid.Cid) *ListRes {
67
+ return list(fs, false, key)
68
+}
69
+
70
func ListAll(fs *Filestore) (func() *ListRes, error) {
71
return listAll(fs, false)
72
}
73
74
+func Verify(fs *Filestore, key *cid.Cid) *ListRes {
75
+ return list(fs, true, key)
76
+}
77
+
78
func VerifyAll(fs *Filestore) (func() *ListRes, error) {
79
return listAll(fs, true)
80
}
81
82
+func list(fs *Filestore, verify bool, key *cid.Cid) *ListRes {
83
+ dobj, err := fs.fm.getDataObj(key)
84
+ if err != nil {
85
+ return mkListRes(key, nil, err)
86
+ }
87
+ if verify {
88
+ _, err = fs.fm.readDataObj(key, dobj)
89
+ }
90
+ return mkListRes(key, dobj, err)
91
+}
92
+
93
func listAll(fs *Filestore, verify bool) (func() *ListRes, error) {
94
q := dsq.Query{}
95
qr, err := fs.fm.ds.Query(q)
@@ -112,7 +137,9 @@ func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
137
status := StatusOk
138
errorMsg := ""
139
if err != nil {
115
- if _, ok := err.(*CorruptReferenceError); ok {
140
+ if err == ds.ErrNotFound || err == blockstore.ErrNotFound {
141
+ status = StatusKeyNotFound
142
+ } else if _, ok := err.(*CorruptReferenceError); ok {
143
status = StatusFileError
144
} else {
145
status = StatusOtherError