test: port twonode test to Go, remove multinode test
The multinode test is effectively the same as the twonode test. There are some problems with it too: it *looks* like it's testing the Websocket transport with the "listentype,ws" IPTB attribute, but that attribute doesn't actually exist in ipfs/iptb-plugins, so it does nothing, so that test actually just runs the same test twice (Yamux disabled). Furthermore, this is just the same test as in the mplex twonode test. So this just removes the useless multinode test entirely. Also, this removes the part of the twonode test that checks the amount of data transferred over Bitswap. This is an implementation detail of Bitswap, it's not appropriate to test this in an end-to-end test as it depends on algorithmic details of how Bitswap works, and has nothing to do with transports. This is probably more appropriate as a perf or benchmark test of Bitswap. This also moves equivalent functionality from jbenet/go-random-files into the testutils package. This just copies the code and modifies it slightly for better ergonomics.
Gus Eggert committed
Apr 7, 2023 at 08:57 UTC
03a98280e3e642774776cd3d0435ab53e5dfa867
4 files changed
+243
-293
test/cli/testutils/random_files.go
new
+116
@@ -0,0 +1,116 @@
1
+package testutils
2
+
3
+import (
4
+ "fmt"
5
+ "io"
6
+ "math/rand"
7
+ "os"
8
+ "path"
9
+ "time"
10
+)
11
+
12
+var AlphabetEasy = []rune("abcdefghijklmnopqrstuvwxyz01234567890-_")
13
+var AlphabetHard = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ01234567890!@#$%^&*()-_+= ;.,<>'\"[]{}() ")
14
+
15
+type RandFiles struct {
16
+ Rand *rand.Rand
17
+ FileSize int // the size per file.
18
+ FilenameSize int
19
+ Alphabet []rune // for filenames
20
+
21
+ FanoutDepth int // how deep the hierarchy goes
22
+ FanoutFiles int // how many files per dir
23
+ FanoutDirs int // how many dirs per dir
24
+
25
+ RandomSize bool // randomize file sizes
26
+ RandomFanout bool // randomize fanout numbers
27
+}
28
+
29
+func NewRandFiles() *RandFiles {
30
+ return &RandFiles{
31
+ Rand: rand.New(rand.NewSource(time.Now().UnixNano())),
32
+ FileSize: 4096,
33
+ FilenameSize: 16,
34
+ Alphabet: AlphabetEasy,
35
+ FanoutDepth: 2,
36
+ FanoutDirs: 5,
37
+ FanoutFiles: 10,
38
+ RandomSize: true,
39
+ }
40
+}
41
+
42
+func (r *RandFiles) WriteRandomFiles(root string, depth int) error {
43
+ numfiles := r.FanoutFiles
44
+ if r.RandomFanout {
45
+ numfiles = rand.Intn(r.FanoutFiles) + 1
46
+ }
47
+
48
+ for i := 0; i < numfiles; i++ {
49
+ if err := r.WriteRandomFile(root); err != nil {
50
+ return err
51
+ }
52
+ }
53
+
54
+ if depth+1 <= r.FanoutDepth {
55
+ numdirs := r.FanoutDirs
56
+ if r.RandomFanout {
57
+ numdirs = r.Rand.Intn(numdirs) + 1
58
+ }
59
+
60
+ for i := 0; i < numdirs; i++ {
61
+ if err := r.WriteRandomDir(root, depth+1); err != nil {
62
+ return err
63
+ }
64
+ }
65
+ }
66
+
67
+ return nil
68
+}
69
+
70
+func (r *RandFiles) RandomFilename(length int) string {
71
+ b := make([]rune, length)
72
+ for i := range b {
73
+ b[i] = r.Alphabet[r.Rand.Intn(len(r.Alphabet))]
74
+ }
75
+ return string(b)
76
+}
77
+
78
+func (r *RandFiles) WriteRandomFile(root string) error {
79
+ filesize := int64(r.FileSize)
80
+ if r.RandomSize {
81
+ filesize = r.Rand.Int63n(filesize) + 1
82
+ }
83
+
84
+ n := rand.Intn(r.FilenameSize-4) + 4
85
+ name := r.RandomFilename(n)
86
+ filepath := path.Join(root, name)
87
+ f, err := os.Create(filepath)
88
+ if err != nil {
89
+ return fmt.Errorf("creating random file: %w", err)
90
+ }
91
+
92
+ if _, err := io.CopyN(f, r.Rand, filesize); err != nil {
93
+ return fmt.Errorf("copying random file: %w", err)
94
+ }
95
+
96
+ return f.Close()
97
+}
98
+
99
+func (r *RandFiles) WriteRandomDir(root string, depth int) error {
100
+ if depth > r.FanoutDepth {
101
+ return nil
102
+ }
103
+
104
+ n := rand.Intn(r.FilenameSize-4) + 4
105
+ name := r.RandomFilename(n)
106
+ root = path.Join(root, name)
107
+ if err := os.MkdirAll(root, 0755); err != nil {
108
+ return fmt.Errorf("creating random dir: %w", err)
109
+ }
110
+
111
+ err := r.WriteRandomFiles(root, depth)
112
+ if err != nil {
113
+ return fmt.Errorf("writing random files in random dir: %w", err)
114
+ }
115
+ return nil
116
+}
test/cli/transports_test.go
new
+127
@@ -0,0 +1,127 @@
1
+package cli
2
+
3
+import (
4
+ "os"
5
+ "path/filepath"
6
+ "testing"
7
+
8
+ "github.com/ipfs/kubo/config"
9
+ "github.com/ipfs/kubo/test/cli/harness"
10
+ "github.com/ipfs/kubo/test/cli/testutils"
11
+ "github.com/stretchr/testify/assert"
12
+ "github.com/stretchr/testify/require"
13
+)
14
+
15
+func TestTransports(t *testing.T) {
16
+ disableRouting := func(nodes harness.Nodes) {
17
+ nodes.ForEachPar(func(n *harness.Node) {
18
+ n.UpdateConfig(func(cfg *config.Config) {
19
+ cfg.Routing.Type = config.NewOptionalString("none")
20
+ cfg.Bootstrap = nil
21
+ })
22
+ })
23
+ }
24
+ checkSingleFile := func(nodes harness.Nodes) {
25
+ s := testutils.RandomStr(100)
26
+ hash := nodes[0].IPFSAddStr(s)
27
+ nodes.ForEachPar(func(n *harness.Node) {
28
+ val := n.IPFS("cat", hash).Stdout.String()
29
+ assert.Equal(t, s, val)
30
+ })
31
+ }
32
+ checkRandomDir := func(nodes harness.Nodes) {
33
+ randDir := filepath.Join(nodes[0].Dir, "foobar")
34
+ require.NoError(t, os.Mkdir(randDir, 0777))
35
+ rf := testutils.NewRandFiles()
36
+ rf.FanoutDirs = 3
37
+ rf.FanoutFiles = 6
38
+ require.NoError(t, rf.WriteRandomFiles(randDir, 4))
39
+
40
+ hash := nodes[1].IPFS("add", "-r", "-Q", randDir).Stdout.Trimmed()
41
+ nodes.ForEachPar(func(n *harness.Node) {
42
+ res := n.RunIPFS("refs", "-r", hash)
43
+ assert.Equal(t, 0, res.ExitCode())
44
+ })
45
+ }
46
+
47
+ runTests := func(nodes harness.Nodes) {
48
+ checkSingleFile(nodes)
49
+ checkRandomDir(nodes)
50
+ }
51
+
52
+ tcpNodes := func(t *testing.T) harness.Nodes {
53
+ nodes := harness.NewT(t).NewNodes(2).Init()
54
+ nodes.ForEachPar(func(n *harness.Node) {
55
+ n.UpdateConfig(func(cfg *config.Config) {
56
+ cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/tcp/0"}
57
+ cfg.Swarm.Transports.Network.QUIC = config.False
58
+ cfg.Swarm.Transports.Network.Relay = config.False
59
+ cfg.Swarm.Transports.Network.WebTransport = config.False
60
+ cfg.Swarm.Transports.Network.Websocket = config.False
61
+ })
62
+ })
63
+ disableRouting(nodes)
64
+ return nodes
65
+ }
66
+
67
+ t.Run("tcp", func(t *testing.T) {
68
+ t.Parallel()
69
+ nodes := tcpNodes(t).StartDaemons().Connect()
70
+ runTests(nodes)
71
+ })
72
+
73
+ t.Run("tcp with mplex", func(t *testing.T) {
74
+ t.Parallel()
75
+ nodes := tcpNodes(t)
76
+ nodes.ForEachPar(func(n *harness.Node) {
77
+ n.UpdateConfig(func(cfg *config.Config) {
78
+ cfg.Swarm.Transports.Multiplexers.Yamux = config.Disabled
79
+ })
80
+ })
81
+ nodes.StartDaemons().Connect()
82
+ runTests(nodes)
83
+ })
84
+
85
+ t.Run("tcp with NOISE", func(t *testing.T) {
86
+ t.Parallel()
87
+ nodes := tcpNodes(t)
88
+ nodes.ForEachPar(func(n *harness.Node) {
89
+ n.UpdateConfig(func(cfg *config.Config) {
90
+ cfg.Swarm.Transports.Security.TLS = config.Disabled
91
+ })
92
+ })
93
+ nodes.StartDaemons().Connect()
94
+ runTests(nodes)
95
+ })
96
+
97
+ t.Run("QUIC", func(t *testing.T) {
98
+ t.Parallel()
99
+ nodes := harness.NewT(t).NewNodes(5).Init()
100
+ nodes.ForEachPar(func(n *harness.Node) {
101
+ n.UpdateConfig(func(cfg *config.Config) {
102
+ cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1"}
103
+ cfg.Swarm.Transports.Network.QUIC = config.True
104
+ cfg.Swarm.Transports.Network.TCP = config.False
105
+ })
106
+ })
107
+ disableRouting(nodes)
108
+ nodes.StartDaemons().Connect()
109
+ runTests(nodes)
110
+ })
111
+
112
+ t.Run("QUIC", func(t *testing.T) {
113
+ t.Parallel()
114
+ nodes := harness.NewT(t).NewNodes(5).Init()
115
+ nodes.ForEachPar(func(n *harness.Node) {
116
+ n.UpdateConfig(func(cfg *config.Config) {
117
+ cfg.Addresses.Swarm = []string{"/ip4/127.0.0.1/udp/0/quic-v1/webtransport"}
118
+ cfg.Swarm.Transports.Network.QUIC = config.True
119
+ cfg.Swarm.Transports.Network.WebTransport = config.True
120
+ })
121
+ })
122
+ disableRouting(nodes)
123
+ nodes.StartDaemons().Connect()
124
+ runTests(nodes)
125
+ })
126
+
127
+}
test/sharness/t0125-twonode.sh
deleted
-178
@@ -1,178 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Copyright (c) 2017 Jeromy Johnson
4
-# MIT Licensed; see the LICENSE file in this repository.
5
-#
6
-
7
-test_description="Test two ipfs nodes transferring a file"
8
-
9
-. lib/test-lib.sh
10
-
11
-check_file_fetch() {
12
- node=$1
13
- fhash=$2
14
- fname=$3
15
-
16
- test_expect_success "can fetch file" '
17
- ipfsi $node cat $fhash > fetch_out
18
- '
19
-
20
- test_expect_success "file looks good" '
21
- test_cmp $fname fetch_out
22
- '
23
-}
24
-
25
-check_dir_fetch() {
26
- node=$1
27
- ref=$2
28
-
29
- test_expect_success "node can fetch all refs for dir" '
30
- ipfsi $node refs -r $ref > /dev/null
31
- '
32
-}
33
-
34
-run_single_file_test() {
35
- test_expect_success "add a file on node1" '
36
- random 1000000 > filea &&
37
- FILEA_HASH=$(ipfsi 1 add -q filea)
38
- '
39
-
40
- check_file_fetch 0 $FILEA_HASH filea
41
-}
42
-
43
-run_random_dir_test() {
44
- test_expect_success "create a bunch of random files" '
45
- random-files -depth=3 -dirs=4 -files=5 -seed=5 foobar > /dev/null
46
- '
47
-
48
- test_expect_success "add those on node 0" '
49
- DIR_HASH=$(ipfsi 0 add -r -Q foobar)
50
- '
51
-
52
- check_dir_fetch 1 $DIR_HASH
53
-}
54
-
55
-flaky_advanced_test() {
56
- startup_cluster 2 "$@"
57
-
58
- test_expect_success "clean repo before test" '
59
- ipfsi 0 repo gc > /dev/null &&
60
- ipfsi 1 repo gc > /dev/null
61
- '
62
-
63
- run_single_file_test
64
-
65
- run_random_dir_test
66
-
67
- test_expect_success "gather bitswap stats" '
68
- ipfsi 0 bitswap stat -v > stat0 &&
69
- ipfsi 1 bitswap stat -v > stat1
70
- '
71
-
72
- test_expect_success "shut down nodes" '
73
- iptb stop && iptb_wait_stop
74
- '
75
-
76
- # NOTE: data transferred stats checks are flaky
77
- # trying to debug them by printing out the stats hides the flakiness
78
- # my theory is that the extra time cat calls take to print out the stats
79
- # allow for proper cleanup to happen
80
- go-sleep 1s
81
-}
82
-
83
-run_advanced_test() {
84
- # TODO: investigate why flaky_advanced_test is flaky
85
- # Context: https://github.com/ipfs/kubo/pull/9486
86
- # sometimes, bitswap status returns unexpected block transfers
87
- # and everyone has been re-running circleci until is passes for at least a year.
88
- # this re-runs test until it passes or a timeout hits
89
-
90
- BLOCKS_0=126
91
- BLOCKS_1=5
92
- DATA_0=228113
93
- DATA_1=1000256
94
- for i in $(test_seq 1 600); do
95
- flaky_advanced_test
96
- (grep -q "$DATA_0" stat0 && grep -q "$DATA_1" stat1) && break
97
- go-sleep 100ms
98
- done
99
-
100
- test_expect_success "node0 data transferred looks correct" '
101
- test_should_contain "blocks sent: $BLOCKS_0" stat0 &&
102
- test_should_contain "blocks received: $BLOCKS_1" stat0 &&
103
- test_should_contain "data sent: $DATA_0" stat0 &&
104
- test_should_contain "data received: $DATA_1" stat0
105
- '
106
-
107
- test_expect_success "node1 data transferred looks correct" '
108
- test_should_contain "blocks received: $BLOCKS_0" stat1 &&
109
- test_should_contain "blocks sent: $BLOCKS_1" stat1 &&
110
- test_should_contain "data received: $DATA_0" stat1 &&
111
- test_should_contain "data sent: $DATA_1" stat1
112
- '
113
-
114
-}
115
-
116
-test_expect_success "set up tcp testbed" '
117
- iptb testbed create -type localipfs -count 2 -force -init
118
-'
119
-
120
-test_expect_success "disable routing, use direct peering" '
121
- iptb run -- ipfs config Routing.Type none &&
122
- iptb run -- ipfs config --json Bootstrap "[]"
123
-'
124
-
125
-# Test TCP transport
126
-echo "Testing TCP"
127
-addrs='"[\"/ip4/127.0.0.1/tcp/0\"]"'
128
-test_expect_success "use TCP only" '
129
- iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
130
- iptb run -- ipfs config --json Swarm.Transports.Network.QUIC false &&
131
- iptb run -- ipfs config --json Swarm.Transports.Network.Relay false &&
132
- iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport false &&
133
- iptb run -- ipfs config --json Swarm.Transports.Network.Websocket false
134
-'
135
-run_advanced_test
136
-
137
-# test multiplex muxer
138
-echo "Running TCP tests with mplex"
139
-test_expect_success "disable yamux" '
140
- iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
141
-'
142
-run_advanced_test
143
-
144
-test_expect_success "re-enable yamux" '
145
- iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux null
146
-'
147
-# test Noise
148
-echo "Running TCP tests with NOISE"
149
-test_expect_success "use noise only" '
150
- iptb run -- ipfs config --json Swarm.Transports.Security.TLS false
151
-'
152
-run_advanced_test
153
-
154
-test_expect_success "re-enable TLS" '
155
- iptb run -- ipfs config --json Swarm.Transports.Security.TLS null
156
-'
157
-
158
-# test QUIC
159
-echo "Running advanced tests over QUIC"
160
-addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1\"]"'
161
-test_expect_success "use QUIC only" '
162
- iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
163
- iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true &&
164
- iptb run -- ipfs config --json Swarm.Transports.Network.TCP false
165
-'
166
-run_advanced_test
167
-
168
-# test WebTransport
169
-echo "Running advanced tests over WebTransport"
170
-addrs='"[\"/ip4/127.0.0.1/udp/0/quic-v1/webtransport\"]"'
171
-test_expect_success "use WebTransport only" '
172
- iptb run -- ipfs config --json Addresses.Swarm '"${addrs}"' &&
173
- iptb run -- ipfs config --json Swarm.Transports.Network.QUIC true &&
174
- iptb run -- ipfs config --json Swarm.Transports.Network.WebTransport true
175
-'
176
-run_advanced_test
177
-
178
-test_done
test/sharness/t0130-multinode.sh
deleted
-115
@@ -1,115 +0,0 @@
1
-#!/usr/bin/env bash
2
-#
3
-# Copyright (c) 2015 Jeromy Johnson
4
-# MIT Licensed; see the LICENSE file in this repository.
5
-#
6
-
7
-test_description="Test multiple ipfs nodes"
8
-
9
-. lib/test-lib.sh
10
-
11
-check_file_fetch() {
12
- node=$1
13
- fhash=$2
14
- fname=$3
15
-
16
- test_expect_success "can fetch file" '
17
- ipfsi $node cat $fhash > fetch_out
18
- '
19
-
20
- test_expect_success "file looks good" '
21
- test_cmp $fname fetch_out
22
- '
23
-}
24
-
25
-check_dir_fetch() {
26
- node=$1
27
- ref=$2
28
-
29
- test_expect_success "node can fetch all refs for dir" '
30
- ipfsi $node refs -r $ref > /dev/null
31
- '
32
-}
33
-
34
-run_single_file_test() {
35
- test_expect_success "add a file on node1" '
36
- random 1000000 > filea &&
37
- FILEA_HASH=$(ipfsi 1 add -q filea)
38
- '
39
-
40
- check_file_fetch 4 $FILEA_HASH filea
41
- check_file_fetch 3 $FILEA_HASH filea
42
- check_file_fetch 2 $FILEA_HASH filea
43
- check_file_fetch 1 $FILEA_HASH filea
44
- check_file_fetch 0 $FILEA_HASH filea
45
-}
46
-
47
-run_random_dir_test() {
48
- test_expect_success "create a bunch of random files" '
49
- random-files -depth=4 -dirs=3 -files=6 foobar > /dev/null
50
- '
51
-
52
- test_expect_success "add those on node 2" '
53
- DIR_HASH=$(ipfsi 2 add -r -Q foobar)
54
- '
55
-
56
- check_dir_fetch 0 $DIR_HASH
57
- check_dir_fetch 1 $DIR_HASH
58
- check_dir_fetch 2 $DIR_HASH
59
- check_dir_fetch 3 $DIR_HASH
60
- check_dir_fetch 4 $DIR_HASH
61
-}
62
-
63
-
64
-run_basic_test() {
65
- startup_cluster 5
66
-
67
- run_single_file_test
68
-
69
- test_expect_success "shut down nodes" '
70
- iptb stop && iptb_wait_stop
71
- '
72
-}
73
-
74
-run_advanced_test() {
75
- startup_cluster 5 "$@"
76
-
77
- run_single_file_test
78
-
79
- run_random_dir_test
80
-
81
- test_expect_success "shut down nodes" '
82
- iptb stop && iptb_wait_stop ||
83
- test_fsh tail -n +1 .iptb/testbeds/default/*/daemon.std*
84
- '
85
-}
86
-
87
-test_expect_success "set up /tcp testbed" '
88
- iptb testbed create -type localipfs -count 5 -force -init
89
-'
90
-
91
-# test default configuration
92
-run_advanced_test
93
-
94
-# test multiplex muxer
95
-test_expect_success "disable yamux" '
96
- iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
97
-'
98
-run_advanced_test
99
-
100
-test_expect_success "set up /ws testbed" '
101
- iptb testbed create -type localipfs -count 5 -attr listentype,ws -force -init
102
-'
103
-
104
-# test default configuration
105
-run_advanced_test
106
-
107
-# test multiplex muxer
108
-test_expect_success "disable yamux" '
109
- iptb run -- ipfs config --json Swarm.Transports.Multiplexers.Yamux false
110
-'
111
-
112
-run_advanced_test
113
-
114
-
115
-test_done