filestore util: basic filestore commands.
License: MIT Signed-off-by: Kevin Atkinson <k@kevina.org>
Kevin Atkinson committed
Feb 3, 2017 at 16:20 UTC
4bf43047d8a342b0e3c34aa736831f7677676b0f
3 files changed
+272
core/commands/filestore.go
new
+132
@@ -0,0 +1,132 @@
1
+package commands
2
+
3
+import (
4
+ "context"
5
+ "fmt"
6
+
7
+ cmds "github.com/ipfs/go-ipfs/commands"
8
+ "github.com/ipfs/go-ipfs/core"
9
+ "github.com/ipfs/go-ipfs/filestore"
10
+ u "gx/ipfs/QmZuY8aV7zbNXVy6DyN9SmnuH3o9nG852F4aTiSBpts8d1/go-ipfs-util"
11
+)
12
+
13
+var FileStoreCmd = &cmds.Command{
14
+ Helptext: cmds.HelpText{
15
+ Tagline: "Interact with filestore objects.",
16
+ },
17
+ Subcommands: map[string]*cmds.Command{
18
+ "ls": lsFileStore,
19
+ "verify": verifyFileStore,
20
+ },
21
+}
22
+
23
+var lsFileStore = &cmds.Command{
24
+ Helptext: cmds.HelpText{
25
+ Tagline: "List objects in filestore.",
26
+ },
27
+ Run: func(req cmds.Request, res cmds.Response) {
28
+ _, fs, err := getFilestore(req)
29
+ if err != nil {
30
+ res.SetError(err, cmds.ErrNormal)
31
+ return
32
+ }
33
+ next, err := filestore.ListAll(fs)
34
+ if err != nil {
35
+ res.SetError(err, cmds.ErrNormal)
36
+ return
37
+ }
38
+ out := listResAsChan(next, req.Context())
39
+ res.SetOutput(out)
40
+ },
41
+ PostRun: func(req cmds.Request, res cmds.Response) {
42
+ if res.Error() != nil {
43
+ return
44
+ }
45
+ outChan, ok := res.Output().(<-chan interface{})
46
+ if !ok {
47
+ res.SetError(u.ErrCast(), cmds.ErrNormal)
48
+ return
49
+ }
50
+ res.SetOutput(nil)
51
+ errors := false
52
+ for r0 := range outChan {
53
+ r := r0.(*filestore.ListRes)
54
+ if r.ErrorMsg != "" {
55
+ fmt.Fprintf(res.Stderr(), "%s\n", r.ErrorMsg)
56
+ } else {
57
+ fmt.Fprintf(res.Stdout(), "%s\n", r.FormatLong())
58
+ }
59
+ }
60
+ if errors {
61
+ res.SetError(fmt.Errorf("errors while displaying some entries"), cmds.ErrNormal)
62
+ }
63
+ },
64
+ Type: filestore.ListRes{},
65
+}
66
+
67
+var verifyFileStore = &cmds.Command{
68
+ Helptext: cmds.HelpText{
69
+ Tagline: "Verify objects in filestore.",
70
+ },
71
+ Run: func(req cmds.Request, res cmds.Response) {
72
+ _, fs, err := getFilestore(req)
73
+ if err != nil {
74
+ res.SetError(err, cmds.ErrNormal)
75
+ return
76
+ }
77
+ next, err := filestore.VerifyAll(fs)
78
+ if err != nil {
79
+ res.SetError(err, cmds.ErrNormal)
80
+ return
81
+ }
82
+ out := listResAsChan(next, req.Context())
83
+ res.SetOutput(out)
84
+ },
85
+ PostRun: func(req cmds.Request, res cmds.Response) {
86
+ if res.Error() != nil {
87
+ return
88
+ }
89
+ outChan, ok := res.Output().(<-chan interface{})
90
+ if !ok {
91
+ res.SetError(u.ErrCast(), cmds.ErrNormal)
92
+ return
93
+ }
94
+ res.SetOutput(nil)
95
+ for r0 := range outChan {
96
+ r := r0.(*filestore.ListRes)
97
+ fmt.Fprintf(res.Stdout(), "%s %s\n", r.Status.Format(), r.FormatLong())
98
+ }
99
+ },
100
+ Type: filestore.ListRes{},
101
+}
102
+
103
+func getFilestore(req cmds.Request) (*core.IpfsNode, *filestore.Filestore, error) {
104
+ n, err := req.InvocContext().GetNode()
105
+ if err != nil {
106
+ return nil, nil, err
107
+ }
108
+ fs := n.Filestore
109
+ if fs == nil {
110
+ return n, nil, fmt.Errorf("filestore not enabled")
111
+ }
112
+ return n, fs, err
113
+}
114
+
115
+func listResAsChan(next func() *filestore.ListRes, ctx context.Context) <-chan interface{} {
116
+ out := make(chan interface{}, 128)
117
+ go func() {
118
+ defer close(out)
119
+ for {
120
+ r := next()
121
+ if r == nil {
122
+ return
123
+ }
124
+ select {
125
+ case out <- r:
126
+ case <-ctx.Done():
127
+ return
128
+ }
129
+ }
130
+ }()
131
+ return out
132
+}
core/commands/root.go
+2
@@ -47,6 +47,7 @@ ADVANCED COMMANDS
47
pin Pin objects to local storage
48
repo Manipulate the IPFS repository
49
stats Various operational stats
50
+ filestore Manage the filestore (experimental)
51
52
NETWORK COMMANDS
53
id Show info about IPFS peers
@@ -124,6 +125,7 @@ var rootSubcommands = map[string]*cmds.Command{
125
"update": ExternalBinary(),
126
"version": VersionCmd,
127
"bitswap": BitswapCmd,
128
+ "filestore": FileStoreCmd,
129
}
130
131
// RootRO is the readonly version of Root
filestore/util.go
new
+138
@@ -0,0 +1,138 @@
1
+package filestore
2
+
3
+import (
4
+ "fmt"
5
+
6
+ pb "github.com/ipfs/go-ipfs/filestore/pb"
7
+ dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
8
+
9
+ ds "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore"
10
+ dsq "gx/ipfs/QmRWDav6mzWseLWeYfVd5fvUKiVe9xNH29YfMF438fG364/go-datastore/query"
11
+ proto "gx/ipfs/QmT6n4mspWYEya864BhCUJEgyxiRfmiSY9ruQwTUNpRKaM/protobuf/proto"
12
+ cid "gx/ipfs/QmV5gPoRsjN1Gid3LMdNZTyfCtP2DsvqEbMAmz82RmmiGk/go-cid"
13
+)
14
+
15
+type Status int32
16
+
17
+const (
18
+ StatusOk Status = 0
19
+ StatusFileError Status = 10 // Backing File Error
20
+ //StatusFileNotFound Status = 11 // Backing File Not Found
21
+ //StatusFileChanged Status = 12 // Contents of the file changed
22
+ StatusOtherError Status = 20 // Internal Error, likely corrupt entry
23
+)
24
+
25
+func (s Status) String() string {
26
+ switch s {
27
+ case StatusOk:
28
+ return "ok"
29
+ case StatusFileError:
30
+ return "error"
31
+ case StatusOtherError:
32
+ return "ERROR"
33
+ default:
34
+ return "???"
35
+ }
36
+}
37
+
38
+func (s Status) Format() string {
39
+ return fmt.Sprintf("%-5s", s.String())
40
+}
41
+
42
+type ListRes struct {
43
+ Status Status
44
+ ErrorMsg string
45
+ Key *cid.Cid
46
+ FilePath string
47
+ Offset uint64
48
+ Size uint64
49
+}
50
+
51
+func (r *ListRes) FormatLong() string {
52
+ switch {
53
+ case r.Key == nil:
54
+ return "?????????????????????????????????????????????????"
55
+ default:
56
+ return fmt.Sprintf("%-50s %6d %s %d", r.Key, r.Size, r.FilePath, r.Offset)
57
+ }
58
+}
59
+
60
+func ListAll(fs *Filestore) (func() *ListRes, error) {
61
+ return listAll(fs, false)
62
+}
63
+
64
+func VerifyAll(fs *Filestore) (func() *ListRes, error) {
65
+ return listAll(fs, true)
66
+}
67
+
68
+func listAll(fs *Filestore, verify bool) (func() *ListRes, error) {
69
+ q := dsq.Query{}
70
+ qr, err := fs.fm.ds.Query(q)
71
+ if err != nil {
72
+ return nil, err
73
+ }
74
+
75
+ return func() *ListRes {
76
+ cid, dobj, err := next(qr)
77
+ if dobj == nil && err == nil {
78
+ return nil
79
+ } else if err == nil && verify {
80
+ _, err = fs.fm.readDataObj(cid, dobj)
81
+ }
82
+ return mkListRes(cid, dobj, err)
83
+ }, nil
84
+}
85
+
86
+func next(qr dsq.Results) (*cid.Cid, *pb.DataObj, error) {
87
+ v, ok := qr.NextSync()
88
+ if !ok {
89
+ return nil, nil, nil
90
+ }
91
+
92
+ k := ds.RawKey(v.Key)
93
+ c, err := dshelp.DsKeyToCid(k)
94
+ if err != nil {
95
+ return nil, nil, fmt.Errorf("decoding cid from filestore: %s", err)
96
+ }
97
+
98
+ data, ok := v.Value.([]byte)
99
+ if !ok {
100
+ return c, nil, fmt.Errorf("stored filestore dataobj was not a []byte")
101
+ }
102
+
103
+ var dobj pb.DataObj
104
+ if err := proto.Unmarshal(data, &dobj); err != nil {
105
+ return c, nil, err
106
+ }
107
+
108
+ return c, &dobj, nil
109
+}
110
+
111
+func mkListRes(c *cid.Cid, d *pb.DataObj, err error) *ListRes {
112
+ status := StatusOk
113
+ errorMsg := ""
114
+ if err != nil {
115
+ if _, ok := err.(*CorruptReferenceError); ok {
116
+ status = StatusFileError
117
+ } else {
118
+ status = StatusOtherError
119
+ }
120
+ errorMsg = err.Error()
121
+ }
122
+ if d == nil {
123
+ return &ListRes{
124
+ Status: status,
125
+ ErrorMsg: errorMsg,
126
+ Key: c,
127
+ }
128
+ } else {
129
+ return &ListRes{
130
+ Status: status,
131
+ ErrorMsg: errorMsg,
132
+ Key: c,
133
+ FilePath: *d.FilePath,
134
+ Size: *d.Size_,
135
+ Offset: *d.Offset,
136
+ }
137
+ }
138
+}