fix: multibase in pubsub http rpc (#8183)
* multibase encoding on pubsub * emit multibase for json clients * refactor(pubsub): base64url for all URL args This makes it easier to reason about. Also added better helptext to each command explaining how the binary data is encoded on the wire, and how to process it in userland. * refactor: remove ndpayload and lenpayload Those output formats are undocumented and seem to be only used in tests. This change removes their implementation and replaces it with error message to use JSON instead. I also refactored tests to test the --enc=json response format instead of imaginary one, making tests more useful as they also act as regression tests for HTTP RPC. * test(pubsub): go-ipfs-api Testing against compatible version from https://github.com/ipfs/go-ipfs-api/pull/255 * refactor: safeTextListEncoder Making it clear what it does and why * refactor(pubsub): unify peerids This ensures `ipfs pubsub sub` returns the same peerids in the `From` field as `ipfs pubsub peers`. libp2p already uses base encoding, no need to double wrap or use custom multibase. * test(pubsub): go-ipfs-http-client * refactor(pubsub): make pub command read from a file We want to send payload in the body as multipart so users can use existing tools like curl for publishing arbitrary bytes to a topic. StringArg was created for "one message per line" use case, and if data has `\n` or `\r\n` byte sequences, it will cause payload to be split. It is not possible to undo this, because mentioned sequences are lost, so we are not able to tell if it was `\n` or `\r\n` We already avoid this problem in `block put` and `dht put` by reading payload via FileArg which does not mangle binary data and send it as-is. It feel like `pubsub pub` should be using it in the first place anyway, so this commit replaces StringArg with FileArg. This also closes https://github.com/ipfs/go-ipfs/issues/8454 and makes rpc in go-ipfs easier to code against. * test(pubsub): publishing with line breaks Making sure we don't see regressions in the future. Ref. https://github.com/ipfs/go-ipfs/issues/7939 * chore: disable pubsub interop for now See https://github.com/ipfs/interop/commit/344f692d8cdc68fabe424814214dfb43c716edac * test: t0322-pubsub-http-rpc.sh - Adds HTTP RPC regression test that ensures topic is encoded as URL-safe multibase. - Moves pubsub tests to live in unique range ./t032x * fix(ci): js-ipfs with fixed pubsub wire format uses js-ipfs from https://github.com/ipfs/js-ipfs/pull/3922 until js-ipfs release can ship with dependency on go-ipfs 0.11.0-rc1 Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Adin Schmahmann <adin.schmahmann@gmail.com>
Cory Schwartz committed
Nov 29, 2021 at 14:06 UTC
a43e506d7fd4365979df53c353579d417058c39c
6 files changed
+254
-109
.circleci/main.yml
+12
@@ -234,6 +234,15 @@ jobs:
234
- run:
235
name: Running tests
236
command: |
237
+ WORKDIR=$(pwd)
238
+ cd /tmp
239
+ git clone https://github.com/ipfs/js-ipfs.git
240
+ cd js-ipfs
241
+ git checkout 1dcac76f56972fc3519526e93567e39d685033dd
242
+ npm install
243
+ npm run build
244
+ npm run link
245
+ cd $WORKDIR
246
mkdir -p /tmp/test-results/interop/
247
export MOCHA_FILE="$(mktemp /tmp/test-results/interop/unit.XXXXXX.xml)"
248
npx ipfs-interop -- -t node -f $(sed -n -e "s|^require('\(.*\)')$|test/\1|p" node_modules/ipfs-interop/test/node.js | circleci tests split) -- --reporter mocha-circleci-reporter
@@ -242,6 +251,9 @@ jobs:
251
LIBP2P_TCP_REUSEPORT: false
252
LIBP2P_ALLOW_WEAK_RSA_KEYS: 1
253
IPFS_GO_EXEC: /tmp/circleci-workspace/bin/ipfs
254
+ IPFS_JS_EXEC: /tmp/js-ipfs/packages/ipfs/src/cli.js
255
+ IPFS_JS_MODULE: /tmp/js-ipfs/packages/ipfs/dist/cjs/src/index.js
256
+ IPFS_JS_HTTP_MODULE: /tmp/js-ipfs/packages/ipfs-http-client/dist/cjs/src/index.js
257
- store_test_results:
258
path: /tmp/test-results
259
go-ipfs-api:
core/commands/pubsub.go
+171
-67
@@ -2,13 +2,15 @@ package commands
2
3
import (
4
"context"
5
- "encoding/binary"
5
"fmt"
6
"io"
7
+ "io/ioutil"
8
"net/http"
9
"sort"
10
11
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
12
+ mbase "github.com/multiformats/go-multibase"
13
+ "github.com/pkg/errors"
14
15
cmds "github.com/ipfs/go-ipfs-cmds"
16
options "github.com/ipfs/interface-go-ipfs-core/options"
@@ -21,10 +23,11 @@ var PubsubCmd = &cmds.Command{
23
ipfs pubsub allows you to publish messages to a given topic, and also to
24
subscribe to new messages on a given topic.
25
24
-This is an experimental feature. It is not intended in its current state
25
-to be used in a production environment.
26
+EXPERIMENTAL FEATURE
27
27
-To use, the daemon must be run with '--enable-pubsub-experiment'.
28
+ It is not intended in its current state to be used in a production
29
+ environment. To use, the daemon must be run with
30
+ '--enable-pubsub-experiment'.
31
`,
32
},
33
Subcommands: map[string]*cmds.Command{
@@ -35,14 +38,10 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
38
},
39
}
40
38
-const (
39
- pubsubDiscoverOptionName = "discover"
40
-)
41
-
41
type pubsubMessage struct {
43
- From []byte `json:"from,omitempty"`
44
- Data []byte `json:"data,omitempty"`
45
- Seqno []byte `json:"seqno,omitempty"`
42
+ From string `json:"from,omitempty"`
43
+ Data string `json:"data,omitempty"`
44
+ Seqno string `json:"seqno,omitempty"`
45
TopicIDs []string `json:"topicIDs,omitempty"`
46
}
47
@@ -52,37 +51,42 @@ var PubsubSubCmd = &cmds.Command{
51
ShortDescription: `
52
ipfs pubsub sub subscribes to messages on a given topic.
53
55
-This is an experimental feature. It is not intended in its current state
56
-to be used in a production environment.
54
+EXPERIMENTAL FEATURE
55
58
-To use, the daemon must be run with '--enable-pubsub-experiment'.
59
-`,
60
- LongDescription: `
61
-ipfs pubsub sub subscribes to messages on a given topic.
56
+ It is not intended in its current state to be used in a production
57
+ environment. To use, the daemon must be run with
58
+ '--enable-pubsub-experiment'.
59
63
-This is an experimental feature. It is not intended in its current state
64
-to be used in a production environment.
60
+PEER ENCODING
61
66
-To use, the daemon must be run with '--enable-pubsub-experiment'.
62
+ Peer IDs in From fields are encoded using the default text representation
63
+ from go-libp2p. This ensures the same string values as in 'ipfs pubsub peers'.
64
68
-This command outputs data in the following encodings:
69
- * "json"
70
-(Specified by the "--encoding" or "--enc" flag)
65
+TOPIC AND DATA ENCODING
66
+
67
+ Topics, Data and Seqno are binary data. To ensure all bytes are transferred
68
+ correctly the RPC client and server will use multibase encoding behind
69
+ the scenes.
70
+
71
+ You can inspect the format by passing --enc=json. The ipfs multibase commands
72
+ can be used for encoding/decoding multibase strings in the userland.
73
`,
74
},
75
Arguments: []cmds.Argument{
74
- cmds.StringArg("topic", true, false, "String name of topic to subscribe to."),
75
- },
76
- Options: []cmds.Option{
77
- cmds.BoolOption(pubsubDiscoverOptionName, "Deprecated option to instruct pubsub to discovery peers for the topic. Discovery is now built into pubsub."),
76
+ cmds.StringArg("topic", true, false, "Name of topic to subscribe to."),
77
},
78
+ PreRun: urlArgsEncoder,
79
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
80
api, err := cmdenv.GetApi(env, req)
81
if err != nil {
82
return err
83
}
84
+ if err := urlArgsDecoder(req, env); err != nil {
85
+ return err
86
+ }
87
88
topic := req.Arguments[0]
89
+
90
sub, err := api.PubSub().Subscribe(req.Context, topic)
91
if err != nil {
92
return err
@@ -101,33 +105,39 @@ This command outputs data in the following encodings:
105
return err
106
}
107
104
- if err := res.Emit(&pubsubMessage{
105
- Data: msg.Data(),
106
- From: []byte(msg.From()),
107
- Seqno: msg.Seq(),
108
- TopicIDs: msg.Topics(),
109
- }); err != nil {
108
+ // turn bytes into strings
109
+ encoder, _ := mbase.EncoderByName("base64url")
110
+ psm := pubsubMessage{
111
+ Data: encoder.Encode(msg.Data()),
112
+ From: msg.From().Pretty(),
113
+ Seqno: encoder.Encode(msg.Seq()),
114
+ }
115
+ for _, topic := range msg.Topics() {
116
+ psm.TopicIDs = append(psm.TopicIDs, encoder.Encode([]byte(topic)))
117
+ }
118
+ if err := res.Emit(&psm); err != nil {
119
return err
120
}
121
}
122
},
123
Encoders: cmds.EncoderMap{
124
cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
116
- _, err := w.Write(psm.Data)
125
+ _, dec, err := mbase.Decode(psm.Data)
126
+ if err != nil {
127
+ return err
128
+ }
129
+ _, err = w.Write(dec)
130
return err
131
}),
132
+ // DEPRECATED, undocumented format we used in tests, but not anymore
133
+ // <message.payload>\n<message.payload>\n
134
"ndpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
120
- psm.Data = append(psm.Data, '\n')
121
- _, err := w.Write(psm.Data)
122
- return err
135
+ return errors.New("--enc=ndpayload was removed, use --enc=json instead")
136
}),
137
+ // DEPRECATED, uncodumented format we used in tests, but not anymore
138
+ // <varint-len><message.payload><varint-len><message.payload>
139
"lenpayload": cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, psm *pubsubMessage) error {
125
- buf := make([]byte, 8, len(psm.Data)+8)
126
-
127
- n := binary.PutUvarint(buf, uint64(len(psm.Data)))
128
- buf = append(buf[:n], psm.Data...)
129
- _, err := w.Write(buf)
130
- return err
140
+ return errors.New("--enc=lenpayload was removed, use --enc=json instead")
141
}),
142
},
143
Type: pubsubMessage{},
@@ -135,40 +145,56 @@ This command outputs data in the following encodings:
145
146
var PubsubPubCmd = &cmds.Command{
147
Helptext: cmds.HelpText{
138
- Tagline: "Publish a message to a given pubsub topic.",
148
+ Tagline: "Publish data to a given pubsub topic.",
149
ShortDescription: `
150
ipfs pubsub pub publishes a message to a specified topic.
151
+It reads binary data from stdin or a file.
152
+
153
+EXPERIMENTAL FEATURE
154
+
155
+ It is not intended in its current state to be used in a production
156
+ environment. To use, the daemon must be run with
157
+ '--enable-pubsub-experiment'.
158
142
-This is an experimental feature. It is not intended in its current state
143
-to be used in a production environment.
159
+HTTP RPC ENCODING
160
+
161
+ The data to be published is sent in HTTP request body as multipart/form-data.
162
+
163
+ Topic names are binary data too. To ensure all bytes are transferred
164
+ correctly via URL params, the RPC client and server will use multibase
165
+ encoding behind the scenes.
166
145
-To use, the daemon must be run with '--enable-pubsub-experiment'.
167
`,
168
},
169
Arguments: []cmds.Argument{
170
cmds.StringArg("topic", true, false, "Topic to publish to."),
150
- cmds.StringArg("data", true, true, "Payload of message to publish.").EnableStdin(),
171
+ cmds.FileArg("data", true, false, "The data to be published.").EnableStdin(),
172
},
173
+ PreRun: urlArgsEncoder,
174
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
175
api, err := cmdenv.GetApi(env, req)
176
if err != nil {
177
return err
178
}
179
+ if err := urlArgsDecoder(req, env); err != nil {
180
+ return err
181
+ }
182
183
topic := req.Arguments[0]
184
160
- err = req.ParseBodyArgs()
185
+ // read data passed as a file
186
+ file, err := cmdenv.GetFileArg(req.Files.Entries())
187
if err != nil {
188
return err
189
}
164
-
165
- for _, data := range req.Arguments[1:] {
166
- if err := api.PubSub().Publish(req.Context, topic, []byte(data)); err != nil {
167
- return err
168
- }
190
+ defer file.Close()
191
+ data, err := ioutil.ReadAll(file)
192
+ if err != nil {
193
+ return err
194
}
195
171
- return nil
196
+ // publish
197
+ return api.PubSub().Publish(req.Context, topic, data)
198
},
199
}
200
@@ -178,10 +204,20 @@ var PubsubLsCmd = &cmds.Command{
204
ShortDescription: `
205
ipfs pubsub ls lists out the names of topics you are currently subscribed to.
206
181
-This is an experimental feature. It is not intended in its current state
182
-to be used in a production environment.
207
+EXPERIMENTAL FEATURE
208
+
209
+ It is not intended in its current state to be used in a production
210
+ environment. To use, the daemon must be run with
211
+ '--enable-pubsub-experiment'.
212
184
-To use, the daemon must be run with '--enable-pubsub-experiment'.
213
+TOPIC ENCODING
214
+
215
+ Topic names are a binary data. To ensure all bytes are transferred
216
+ correctly RPC client and server will use multibase encoding behind
217
+ the scenes.
218
+
219
+ You can inspect the format by passing --enc=json. ipfs multibase commands
220
+ can be used for encoding/decoding multibase strings in the userland.
221
`,
222
},
223
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
@@ -195,15 +231,35 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
231
return err
232
}
233
234
+ // emit topics encoded in multibase
235
+ encoder, _ := mbase.EncoderByName("base64url")
236
+ for n, topic := range l {
237
+ l[n] = encoder.Encode([]byte(topic))
238
+ }
239
+
240
return cmds.EmitOnce(res, stringList{l})
241
},
242
Type: stringList{},
243
Encoders: cmds.EncoderMap{
202
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
244
+ cmds.Text: cmds.MakeTypedEncoder(multibaseDecodedStringListEncoder),
245
},
246
}
247
206
-func stringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
248
+func multibaseDecodedStringListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
249
+ for n, mb := range list.Strings {
250
+ _, data, err := mbase.Decode(mb)
251
+ if err != nil {
252
+ return err
253
+ }
254
+ list.Strings[n] = string(data)
255
+ }
256
+ return safeTextListEncoder(req, w, list)
257
+}
258
+
259
+// converts list of strings to text representation where each string is placed
260
+// in separate line with non-printable/unsafe characters escaped
261
+// (this protects terminal output from being mangled by non-ascii topic names)
262
+func safeTextListEncoder(req *cmds.Request, w io.Writer, list *stringList) error {
263
for _, str := range list.Strings {
264
_, err := fmt.Fprintf(w, "%s\n", cmdenv.EscNonPrint(str))
265
if err != nil {
@@ -218,23 +274,37 @@ var PubsubPeersCmd = &cmds.Command{
274
Tagline: "List peers we are currently pubsubbing with.",
275
ShortDescription: `
276
ipfs pubsub peers with no arguments lists out the pubsub peers you are
221
-currently connected to. If given a topic, it will list connected
222
-peers who are subscribed to the named topic.
277
+currently connected to. If given a topic, it will list connected peers who are
278
+subscribed to the named topic.
279
+
280
+EXPERIMENTAL FEATURE
281
224
-This is an experimental feature. It is not intended in its current state
225
-to be used in a production environment.
282
+ It is not intended in its current state to be used in a production
283
+ environment. To use, the daemon must be run with
284
+ '--enable-pubsub-experiment'.
285
227
-To use, the daemon must be run with '--enable-pubsub-experiment'.
286
+TOPIC AND DATA ENCODING
287
+
288
+ Topic names are a binary data. To ensure all bytes are transferred
289
+ correctly RPC client and server will use multibase encoding behind
290
+ the scenes.
291
+
292
+ You can inspect the format by passing --enc=json. ipfs multibase commands
293
+ can be used for encoding/decoding multibase strings in the userland.
294
`,
295
},
296
Arguments: []cmds.Argument{
231
- cmds.StringArg("topic", false, false, "topic to list connected peers of"),
297
+ cmds.StringArg("topic", false, false, "Topic to list connected peers of."),
298
},
299
+ PreRun: urlArgsEncoder,
300
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
301
api, err := cmdenv.GetApi(env, req)
302
if err != nil {
303
return err
304
}
305
+ if err := urlArgsDecoder(req, env); err != nil {
306
+ return err
307
+ }
308
309
var topic string
310
if len(req.Arguments) == 1 {
@@ -256,6 +326,40 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
326
},
327
Type: stringList{},
328
Encoders: cmds.EncoderMap{
259
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
329
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
330
},
331
}
332
+
333
+// TODO: move to cmdenv?
334
+// Encode binary data to be passed as multibase string in URL arguments.
335
+// (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
336
+func urlArgsEncoder(req *cmds.Request, env cmds.Environment) error {
337
+ encoder, _ := mbase.EncoderByName("base64url")
338
+ for n, arg := range req.Arguments {
339
+ req.Arguments[n] = encoder.Encode([]byte(arg))
340
+ }
341
+ return nil
342
+}
343
+
344
+// Decode binary data passed as multibase string in URL arguments.
345
+// (avoiding issues described in https://github.com/ipfs/go-ipfs/issues/7939)
346
+func urlArgsDecoder(req *cmds.Request, env cmds.Environment) error {
347
+ for n, arg := range req.Arguments {
348
+ encoding, data, err := mbase.Decode(arg)
349
+ if err != nil {
350
+ return errors.Wrap(err, "URL arg must be multibase encoded")
351
+ }
352
+
353
+ // Enforce URL-safe encoding is used for data passed via URL arguments
354
+ // - without this we get data corruption similar to https://github.com/ipfs/go-ipfs/issues/7939
355
+ // - we can't just deny base64, because there may be other bases that
356
+ // are not URL-safe – better to force base64url which is known to be
357
+ // safe in URL context
358
+ if encoding != mbase.Base64url {
359
+ return errors.New("URL arg must be base64url encoded")
360
+ }
361
+
362
+ req.Arguments[n] = string(data)
363
+ }
364
+ return nil
365
+}
core/commands/swarm.go
+7
-7
@@ -453,7 +453,7 @@ var swarmAddrsLocalCmd = &cmds.Command{
453
},
454
Type: stringList{},
455
Encoders: cmds.EncoderMap{
456
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
456
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
457
},
458
}
459
@@ -485,7 +485,7 @@ var swarmAddrsListenCmd = &cmds.Command{
485
},
486
Type: stringList{},
487
Encoders: cmds.EncoderMap{
488
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
488
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
489
},
490
}
491
@@ -535,7 +535,7 @@ ipfs swarm connect /ip4/104.131.131.82/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N
535
return cmds.EmitOnce(res, &stringList{output})
536
},
537
Encoders: cmds.EncoderMap{
538
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
538
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
539
},
540
Type: stringList{},
541
}
@@ -600,7 +600,7 @@ it will reconnect.
600
return cmds.EmitOnce(res, &stringList{output})
601
},
602
Encoders: cmds.EncoderMap{
603
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
603
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
604
},
605
Type: stringList{},
606
}
@@ -722,7 +722,7 @@ Filters default to those specified under the "Swarm.AddrFilters" config key.
722
return cmds.EmitOnce(res, &stringList{output})
723
},
724
Encoders: cmds.EncoderMap{
725
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
725
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
726
},
727
Type: stringList{},
728
}
@@ -778,7 +778,7 @@ var swarmFiltersAddCmd = &cmds.Command{
778
return cmds.EmitOnce(res, &stringList{added})
779
},
780
Encoders: cmds.EncoderMap{
781
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
781
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
782
},
783
Type: stringList{},
784
}
@@ -844,7 +844,7 @@ var swarmFiltersRmCmd = &cmds.Command{
844
return cmds.EmitOnce(res, &stringList{removed})
845
},
846
Encoders: cmds.EncoderMap{
847
- cmds.Text: cmds.MakeTypedEncoder(stringListEncoder),
847
+ cmds.Text: cmds.MakeTypedEncoder(safeTextListEncoder),
848
},
849
Type: stringList{},
850
}
test/sharness/t0320-pubsub.sh
renamed
+25
-25
@@ -19,72 +19,72 @@ run_pubsub_tests() {
19
PEERID_0=$(iptb attr get 0 id) &&
20
PEERID_2=$(iptb attr get 2 id)
21
'
22
-
22
+
23
# ipfs pubsub sub
24
test_expect_success 'pubsub' '
25
- echo "testOK" > expected &&
25
+ echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected &&
26
touch empty &&
27
mkfifo wait ||
28
test_fsh echo init fail
29
-
29
+
30
# ipfs pubsub sub is long-running so we need to start it in the background and
31
# wait put its output somewhere where we can access it
32
(
33
- ipfsi 0 pubsub sub --enc=ndpayload testTopic | if read line; then
34
- echo $line > actual &&
33
+ ipfsi 0 pubsub sub --enc=json testTopic | if read line; then
34
+ echo $line | jq -j .data > actual &&
35
echo > wait
36
fi
37
) &
38
'
39
-
39
+
40
test_expect_success "wait until ipfs pubsub sub is ready to do work" '
41
go-sleep 500ms
42
'
43
-
43
+
44
test_expect_success "can see peer subscribed to testTopic" '
45
ipfsi 1 pubsub peers testTopic > peers_out
46
'
47
-
47
+
48
test_expect_success "output looks good" '
49
echo $PEERID_0 > peers_exp &&
50
test_cmp peers_exp peers_out
51
'
52
-
53
- test_expect_success "publish something" '
54
- ipfsi 1 pubsub pub testTopic "testOK" &> pubErr
52
+
53
+ test_expect_success "publish something from file" '
54
+ echo -n -e "test\nOK" > payload-file &&
55
+ ipfsi 1 pubsub pub testTopic payload-file &> pubErr
56
'
56
-
57
+
58
test_expect_success "wait until echo > wait executed" '
59
cat wait &&
60
test_cmp pubErr empty &&
61
test_cmp expected actual
62
'
62
-
63
+
64
test_expect_success "wait for another pubsub message" '
64
- echo "testOK2" > expected &&
65
+ echo -n -e "test\nOK\r\n2" | ipfs multibase encode -b base64url > expected &&
66
mkfifo wait2 ||
67
test_fsh echo init fail
67
-
68
+
69
# ipfs pubsub sub is long-running so we need to start it in the background and
70
# wait put its output somewhere where we can access it
71
(
71
- ipfsi 2 pubsub sub --enc=ndpayload testTopic | if read line; then
72
- echo $line > actual &&
72
+ ipfsi 2 pubsub sub --enc=json testTopic | if read line; then
73
+ echo $line | jq -j .data > actual &&
74
echo > wait2
75
fi
76
) &
77
'
77
-
78
+
79
test_expect_success "wait until ipfs pubsub sub is ready to do work" '
80
go-sleep 500ms
81
'
81
-
82
- test_expect_success "publish something" '
83
- echo "testOK2" | ipfsi 3 pubsub pub testTopic &> pubErr
82
+
83
+ test_expect_success "publish something from stdin" '
84
+ echo -n -e "test\nOK\r\n2" | ipfsi 3 pubsub pub testTopic &> pubErr
85
'
85
-
86
+
87
test_expect_success "wait until echo > wait executed" '
87
- echo "testOK2" > expected &&
88
cat wait2 &&
89
test_cmp pubErr empty &&
90
test_cmp expected actual
@@ -93,7 +93,7 @@ run_pubsub_tests() {
93
test_expect_success 'cleanup fifos' '
94
rm -f wait wait2
95
'
96
-
96
+
97
}
98
99
# Normal tests
@@ -114,7 +114,7 @@ startup_cluster $NUM_NODES --enable-pubsub-experiment
114
115
test_expect_success 'set node 4 to listen on testTopic' '
116
rm -f node4_actual &&
117
- ipfsi 4 pubsub sub --enc=ndpayload testTopic > node4_actual &
117
+ ipfsi 4 pubsub sub --enc=json testTopic > node4_actual &
118
'
119
120
run_pubsub_tests
test/sharness/t0321-pubsub-gossipsub.sh
renamed
+10
-10
@@ -25,7 +25,7 @@ test_expect_success 'peer ids' '
25
'
26
27
test_expect_success 'pubsub' '
28
- echo "testOK" > expected &&
28
+ echo -n -e "test\nOK" | ipfs multibase encode -b base64url > expected &&
29
touch empty &&
30
mkfifo wait ||
31
test_fsh echo init fail
@@ -33,8 +33,8 @@ test_expect_success 'pubsub' '
33
# ipfs pubsub sub is long-running so we need to start it in the background and
34
# wait put its output somewhere where we can access it
35
(
36
- ipfsi 0 pubsub sub --enc=ndpayload testTopic | if read line; then
37
- echo $line > actual &&
36
+ ipfsi 0 pubsub sub --enc=json testTopic | if read line; then
37
+ echo $line | jq -j .data > actual &&
38
echo > wait
39
fi
40
) &
@@ -53,8 +53,9 @@ test_expect_success "output looks good" '
53
test_cmp peers_exp peers_out
54
'
55
56
-test_expect_success "publish something" '
57
- ipfsi 1 pubsub pub testTopic "testOK" &> pubErr
56
+test_expect_success "publish something from a file" '
57
+ echo -n -e "test\nOK" > payload-file &&
58
+ ipfsi 1 pubsub pub testTopic payload-file &> pubErr
59
'
60
61
test_expect_success "wait until echo > wait executed" '
@@ -64,15 +65,15 @@ test_expect_success "wait until echo > wait executed" '
65
'
66
67
test_expect_success "wait for another pubsub message" '
67
- echo "testOK2" > expected &&
68
+ echo -n -e "test\nOK2" | ipfs multibase encode -b base64url > expected &&
69
mkfifo wait2 ||
70
test_fsh echo init fail
71
72
# ipfs pubsub sub is long-running so we need to start it in the background and
73
# wait put its output somewhere where we can access it
74
(
74
- ipfsi 2 pubsub sub --enc=ndpayload testTopic | if read line; then
75
- echo $line > actual &&
75
+ ipfsi 2 pubsub sub --enc=json testTopic | if read line; then
76
+ echo $line | jq -j .data > actual &&
77
echo > wait2
78
fi
79
) &
@@ -83,11 +84,10 @@ test_expect_success "wait until ipfs pubsub sub is ready to do work" '
84
'
85
86
test_expect_success "publish something" '
86
- echo "testOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
87
+ echo -n -e "test\nOK2" | ipfsi 1 pubsub pub testTopic &> pubErr
88
'
89
90
test_expect_success "wait until echo > wait executed" '
90
- echo "testOK2" > expected &&
91
cat wait2 &&
92
test_cmp pubErr empty &&
93
test_cmp expected actual
test/sharness/t0322-pubsub-http-rpc.sh
new
+29
@@ -0,0 +1,29 @@
1
+#!/usr/bin/env bash
2
+
3
+test_description="Test pubsub command behavior over HTTP RPC API"
4
+
5
+. lib/test-lib.sh
6
+
7
+test_init_ipfs
8
+test_launch_ipfs_daemon --enable-pubsub-experiment
9
+
10
+# Require topic as multibase
11
+# https://github.com/ipfs/go-ipfs/pull/8183
12
+test_expect_success "/api/v0/pubsub/pub URL arg must be multibase encoded" '
13
+ echo test > data.txt &&
14
+ curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=foobar" > result &&
15
+ test_should_contain "error" result &&
16
+ test_should_contain "URL arg must be multibase encoded" result
17
+'
18
+
19
+# Use URL-safe multibase
20
+# base64 should produce error when used in URL args, base64url should be used
21
+test_expect_success "/api/v0/pubsub/pub URL arg must be in URL-safe multibase" '
22
+ echo test > data.txt &&
23
+ curl -s -X POST -F "data=@data.txt" "$API_ADDR/api/v0/pubsub/pub?arg=mZm9vYmFyCg" > result &&
24
+ test_should_contain "error" result &&
25
+ test_should_contain "URL arg must be base64url encoded" result
26
+'
27
+
28
+test_kill_ipfs_daemon
29
+test_done