dag: Support multiple files in put
License: MIT Signed-off-by: Łukasz Magiera <magik6k@gmail.com>
Łukasz Magiera committed
Sep 20, 2017 at 14:30 UTC
eb7da51203ea9bbb6fe0cce8997c9e96d2b99821
2 files changed
+84
-40
core/commands/dag/dag.go
+69
-38
@@ -8,12 +8,15 @@ import (
8
"strings"
9
10
cmds "github.com/ipfs/go-ipfs/commands"
11
+ files "github.com/ipfs/go-ipfs/commands/files"
12
coredag "github.com/ipfs/go-ipfs/core/coredag"
13
path "github.com/ipfs/go-ipfs/path"
14
pin "github.com/ipfs/go-ipfs/pin"
15
16
cid "gx/ipfs/QmNp85zy9RLrQ5oQD4hPyS39ezrrXpcaa7R4Y9kxdWQLLQ/go-cid"
17
+ u "gx/ipfs/QmSU6eubNdhXjFBJBSksTp8kv8YRub8mGAPv8tVJHmL2EU/go-ipfs-util"
18
mh "gx/ipfs/QmU9a9NV9RdPNwZQDYd5uKsm6N6LJLSvLbywDDYFbaaC6P/go-multihash"
19
+ "reflect"
20
)
21
22
var DagCmd = &cmds.Command{
@@ -53,7 +56,7 @@ into an object of the specified format.
56
`,
57
},
58
Arguments: []cmds.Argument{
56
- cmds.FileArg("object data", true, false, "The object to put").EnableStdin(),
59
+ cmds.FileArg("object data", true, true, "The object to put").EnableStdin(),
60
},
61
Options: []cmds.Option{
62
cmds.StringOption("format", "f", "Format that the object will be added as.").Default("cbor"),
@@ -68,12 +71,6 @@ into an object of the specified format.
71
return
72
}
73
71
- fi, err := req.Files().NextFile()
72
- if err != nil {
73
- res.SetError(err, cmds.ErrNormal)
74
- return
75
- }
76
-
74
ienc, _, _ := req.Option("input-enc").String()
75
format, _, _ := req.Option("format").String()
76
hash, _, err := req.Option("hash").String()
@@ -100,52 +97,86 @@ into an object of the specified format.
97
defer n.Blockstore.PinLock().Unlock()
98
}
99
103
- nds, err := coredag.ParseInputs(ienc, format, fi, mhType, -1)
104
- if err != nil {
105
- res.SetError(err, cmds.ErrNormal)
106
- return
107
- }
108
- if len(nds) == 0 {
109
- res.SetError(fmt.Errorf("no node returned from ParseInputs"), cmds.ErrNormal)
110
- return
111
- }
100
+ outChan := make(chan interface{}, 8)
101
+ res.SetOutput((<-chan interface{})(outChan))
102
113
- b := n.DAG.Batch()
114
- for _, nd := range nds {
115
- _, err := b.Add(nd)
116
- if err != nil {
117
- res.SetError(err, cmds.ErrNormal)
118
- return
103
+ addAllAndPin := func(f files.File) error {
104
+ for {
105
+ file, err := f.NextFile()
106
+ if err == io.EOF {
107
+ // Finished the list of files.
108
+ break
109
+ } else if err != nil {
110
+ return err
111
+ }
112
+
113
+ nds, err := coredag.ParseInputs(ienc, format, file, mhType, -1)
114
+ if err != nil {
115
+ return err
116
+ }
117
+ if len(nds) == 0 {
118
+ return fmt.Errorf("no node returned from ParseInputs")
119
+ }
120
+
121
+ b := n.DAG.Batch()
122
+ for _, nd := range nds {
123
+ _, err := b.Add(nd)
124
+ if err != nil {
125
+ return err
126
+ }
127
+ }
128
+
129
+ if err := b.Commit(); err != nil {
130
+ return err
131
+ }
132
+
133
+ root := nds[0].Cid()
134
+ if dopin {
135
+ n.Pinning.PinWithMode(root, pin.Recursive)
136
+
137
+ err := n.Pinning.Flush()
138
+ if err != nil {
139
+ return err
140
+ }
141
+ }
142
+
143
+ outChan <- &OutputObject{Cid: root}
144
}
120
- }
145
122
- if err := b.Commit(); err != nil {
123
- res.SetError(err, cmds.ErrNormal)
124
- return
146
+ return nil
147
}
148
127
- root := nds[0].Cid()
128
- if dopin {
129
- n.Pinning.PinWithMode(root, pin.Recursive)
130
-
131
- err := n.Pinning.Flush()
132
- if err != nil {
149
+ go func() {
150
+ defer close(outChan)
151
+ if err := addAllAndPin(req.Files()); err != nil {
152
res.SetError(err, cmds.ErrNormal)
153
return
154
}
136
- }
137
-
138
- res.SetOutput(&OutputObject{Cid: root})
155
+ }()
156
},
157
Type: OutputObject{},
158
Marshalers: cmds.MarshalerMap{
159
cmds.Text: func(res cmds.Response) (io.Reader, error) {
143
- oobj, ok := res.Output().(*OutputObject)
160
+ outChan, ok := res.Output().(<-chan interface{})
161
if !ok {
145
- return nil, fmt.Errorf("expected a different object in marshaler")
162
+ fmt.Println(reflect.TypeOf(res.Output()))
163
+ return nil, u.ErrCast()
164
+ }
165
+
166
+ marshal := func(v interface{}) (io.Reader, error) {
167
+ obj, ok := v.(*OutputObject)
168
+ if !ok {
169
+ return nil, u.ErrCast()
170
+ }
171
+
172
+ return strings.NewReader(obj.Cid.String() + "\n"), nil
173
}
174
148
- return strings.NewReader(oobj.Cid.String()), nil
175
+ return &cmds.ChannelMarshaler{
176
+ Channel: outChan,
177
+ Marshaler: marshal,
178
+ Res: res,
179
+ }, nil
180
},
181
},
182
}
test/sharness/t0053-dag.sh
+15
-2
@@ -153,7 +153,7 @@ test_dag_cmd() {
153
'
154
155
test_expect_success "dag put with dag-pb works output looks good" '
156
- printf $HASH > dag_put_exp &&
156
+ echo $HASH > dag_put_exp &&
157
test_cmp dag_put_exp dag_put_out
158
'
159
@@ -163,7 +163,20 @@ test_dag_cmd() {
163
'
164
165
test_expect_success "dag put with dag-pb works output looks good" '
166
- printf $HASH > dag_put_exp &&
166
+ echo $HASH > dag_put_exp &&
167
+ test_cmp dag_put_exp dag_put_out
168
+ '
169
+
170
+ test_expect_success "dag put multiple files" '
171
+ printf {\"foo\":\"bar\"} > a.json &&
172
+ printf {\"foo\":\"baz\"} > b.json &&
173
+ ipfs dag put a.json b.json > dag_put_out
174
+ '
175
+
176
+ test_expect_success "dag put multiple files output looks good" '
177
+ echo zdpuAoKMEvka7gKGSjF9B3of1F5gE5MyMMywxTC13wCmouQrf > dag_put_exp &&
178
+ echo zdpuAogmDEvpvGjMFsNTGDEU1JMYe6v69oxR8nG81EurmGHMj >> dag_put_exp &&
179
+
180
test_cmp dag_put_exp dag_put_out
181
'
182