tests for kbucket and some code cleanup
Jeromy committed
Aug 3, 2014 at 15:04 UTC
bade1aa277b8531ccead82384930adab8eb841f0
4 files changed
+249
-127
routing/dht/bucket.go
new
+63
@@ -0,0 +1,63 @@
1
+package dht
2
+
3
+import (
4
+ "container/list"
5
+
6
+ peer "github.com/jbenet/go-ipfs/peer"
7
+)
8
+// Bucket holds a list of peers.
9
+type Bucket list.List
10
+
11
+func (b *Bucket) Find(id peer.ID) *list.Element {
12
+ bucket_list := (*list.List)(b)
13
+ for e := bucket_list.Front(); e != nil; e = e.Next() {
14
+ if e.Value.(*peer.Peer).ID.Equal(id) {
15
+ return e
16
+ }
17
+ }
18
+ return nil
19
+}
20
+
21
+func (b *Bucket) MoveToFront(e *list.Element) {
22
+ bucket_list := (*list.List)(b)
23
+ bucket_list.MoveToFront(e)
24
+}
25
+
26
+func (b *Bucket) PushFront(p *peer.Peer) {
27
+ bucket_list := (*list.List)(b)
28
+ bucket_list.PushFront(p)
29
+}
30
+
31
+func (b *Bucket) PopBack() *peer.Peer {
32
+ bucket_list := (*list.List)(b)
33
+ last := bucket_list.Back()
34
+ bucket_list.Remove(last)
35
+ return last.Value.(*peer.Peer)
36
+}
37
+
38
+func (b *Bucket) Len() int {
39
+ bucket_list := (*list.List)(b)
40
+ return bucket_list.Len()
41
+}
42
+
43
+// Splits a buckets peers into two buckets, the methods receiver will have
44
+// peers with CPL equal to cpl, the returned bucket will have peers with CPL
45
+// greater than cpl (returned bucket has closer peers)
46
+func (b *Bucket) Split(cpl int, target ID) *Bucket {
47
+ bucket_list := (*list.List)(b)
48
+ out := list.New()
49
+ e := bucket_list.Front()
50
+ for e != nil {
51
+ peer_id := convertPeerID(e.Value.(*peer.Peer).ID)
52
+ peer_cpl := xor(peer_id, target).commonPrefixLen()
53
+ if peer_cpl > cpl {
54
+ cur := e
55
+ out.PushBack(e.Value)
56
+ e = e.Next()
57
+ bucket_list.Remove(cur)
58
+ continue
59
+ }
60
+ e = e.Next()
61
+ }
62
+ return (*Bucket)(out)
63
+}
routing/dht/table.go
+16
-127
@@ -1,76 +1,12 @@
1
package dht
2
3
import (
4
- "bytes"
4
"container/list"
5
"sort"
6
8
- "crypto/sha256"
9
-
7
peer "github.com/jbenet/go-ipfs/peer"
11
- u "github.com/jbenet/go-ipfs/util"
8
)
9
14
-// ID for IpfsDHT should be a byte slice, to allow for simpler operations
15
-// (xor). DHT ids are based on the peer.IDs.
16
-//
17
-// NOTE: peer.IDs are biased because they are multihashes (first bytes
18
-// biased). Thus, may need to re-hash keys (uniform dist). TODO(jbenet)
19
-type ID []byte
20
-
21
-// Bucket holds a list of peers.
22
-type Bucket list.List
23
-
24
-func (b *Bucket) Find(id peer.ID) *list.Element {
25
- bucket_list := (*list.List)(b)
26
- for e := bucket_list.Front(); e != nil; e = e.Next() {
27
- if e.Value.(*peer.Peer).ID.Equal(id) {
28
- return e
29
- }
30
- }
31
- return nil
32
-}
33
-
34
-func (b *Bucket) MoveToFront(e *list.Element) {
35
- bucket_list := (*list.List)(b)
36
- bucket_list.MoveToFront(e)
37
-}
38
-
39
-func (b *Bucket) PushFront(p *peer.Peer) {
40
- bucket_list := (*list.List)(b)
41
- bucket_list.PushFront(p)
42
-}
43
-
44
-func (b *Bucket) PopBack() *peer.Peer {
45
- bucket_list := (*list.List)(b)
46
- last := bucket_list.Back()
47
- bucket_list.Remove(last)
48
- return last.Value.(*peer.Peer)
49
-}
50
-
51
-func (b *Bucket) Len() int {
52
- bucket_list := (*list.List)(b)
53
- return bucket_list.Len()
54
-}
55
-
56
-func (b *Bucket) Split(cpl int, target ID) *Bucket {
57
- bucket_list := (*list.List)(b)
58
- out := list.New()
59
- e := bucket_list.Front()
60
- for e != nil {
61
- peer_id := convertPeerID(e.Value.(*peer.Peer).ID)
62
- peer_cpl := xor(peer_id, target).commonPrefixLen()
63
- if peer_cpl > cpl {
64
- cur := e
65
- out.PushBack(e.Value)
66
- e = e.Next()
67
- bucket_list.Remove(cur)
68
- continue
69
- }
70
- }
71
- return (*Bucket)(out)
72
-}
73
-
10
// RoutingTable defines the routing table.
11
type RoutingTable struct {
12
@@ -82,14 +18,12 @@ type RoutingTable struct {
18
bucketsize int
19
}
20
85
-func convertPeerID(id peer.ID) ID {
86
- hash := sha256.Sum256(id)
87
- return hash[:]
88
-}
89
-
90
-func convertKey(id u.Key) ID {
91
- hash := sha256.Sum256([]byte(id))
92
- return hash[:]
21
+func NewRoutingTable(bucketsize int, local_id ID) *RoutingTable {
22
+ rt := new(RoutingTable)
23
+ rt.Buckets = []*Bucket{new(Bucket)}
24
+ rt.bucketsize = bucketsize
25
+ rt.local = local_id
26
+ return rt
27
}
28
29
// Update adds or moves the given peer to the front of its respective bucket
@@ -114,6 +48,10 @@ func (rt *RoutingTable) Update(p *peer.Peer) *peer.Peer {
48
if b_id == len(rt.Buckets) - 1 {
49
new_bucket := bucket.Split(b_id, rt.local)
50
rt.Buckets = append(rt.Buckets, new_bucket)
51
+ if new_bucket.Len() > rt.bucketsize {
52
+ // This is another very rare and annoying case
53
+ panic("Case not handled.")
54
+ }
55
56
// If all elements were on left side of split...
57
if bucket.Len() > rt.bucketsize {
@@ -139,20 +77,23 @@ type peerDistance struct {
77
p *peer.Peer
78
distance ID
79
}
80
+
81
+// peerSorterArr implements sort.Interface to sort peers by xor distance
82
type peerSorterArr []*peerDistance
83
func (p peerSorterArr) Len() int {return len(p)}
84
func (p peerSorterArr) Swap(a, b int) {p[a],p[b] = p[b],p[a]}
85
func (p peerSorterArr) Less(a, b int) bool {
146
- return p[a].distance.Less(p[b])
86
+ return p[a].distance.Less(p[b].distance)
87
}
88
//
89
90
+// Returns a single peer that is nearest to the given ID
91
func (rt *RoutingTable) NearestPeer(id ID) *peer.Peer {
92
peers := rt.NearestPeers(id, 1)
93
return peers[0]
94
}
95
155
-//TODO: make this accept an ID, requires method of converting keys to IDs
96
+// Returns a list of the 'count' closest peers to the given ID
97
func (rt *RoutingTable) NearestPeers(id ID, count int) []*peer.Peer {
98
cpl := xor(id, rt.local).commonPrefixLen()
99
@@ -170,7 +111,6 @@ func (rt *RoutingTable) NearestPeers(id ID, count int) []*peer.Peer {
111
}
112
113
var peerArr peerSorterArr
173
-
114
plist := (*list.List)(bucket)
115
for e := plist.Front();e != nil; e = e.Next() {
116
p := e.Value.(*peer.Peer)
@@ -182,6 +122,7 @@ func (rt *RoutingTable) NearestPeers(id ID, count int) []*peer.Peer {
122
peerArr = append(peerArr, &pd)
123
}
124
125
+ // Sort by distance to local peer
126
sort.Sort(peerArr)
127
128
var out []*peer.Peer
@@ -191,55 +132,3 @@ func (rt *RoutingTable) NearestPeers(id ID, count int) []*peer.Peer {
132
133
return out
134
}
194
-
195
-func (id ID) Equal(other ID) bool {
196
- return bytes.Equal(id, other)
197
-}
198
-
199
-func (id ID) Less(other interface{}) bool {
200
- a, b := equalizeSizes(id, other.(ID))
201
- for i := 0; i < len(a); i++ {
202
- if a[i] != b[i] {
203
- return a[i] < b[i]
204
- }
205
- }
206
- return len(a) < len(b)
207
-}
208
-
209
-func (id ID) commonPrefixLen() int {
210
- for i := 0; i < len(id); i++ {
211
- for j := 0; j < 8; j++ {
212
- if (id[i]>>uint8(7-j))&0x1 != 0 {
213
- return i*8 + j
214
- }
215
- }
216
- }
217
- return len(id)*8 - 1
218
-}
219
-
220
-func xor(a, b ID) ID {
221
- a, b = equalizeSizes(a, b)
222
-
223
- c := make(ID, len(a))
224
- for i := 0; i < len(a); i++ {
225
- c[i] = a[i] ^ b[i]
226
- }
227
- return c
228
-}
229
-
230
-func equalizeSizes(a, b ID) (ID, ID) {
231
- la := len(a)
232
- lb := len(b)
233
-
234
- if la < lb {
235
- na := make([]byte, lb)
236
- copy(na, a)
237
- a = na
238
- } else if lb < la {
239
- nb := make([]byte, la)
240
- copy(nb, b)
241
- b = nb
242
- }
243
-
244
- return a, b
245
-}
routing/dht/table_test.go
new
+92
@@ -0,0 +1,92 @@
1
+package dht
2
+
3
+import (
4
+ crand "crypto/rand"
5
+ "crypto/sha256"
6
+ "math/rand"
7
+ "container/list"
8
+ "testing"
9
+
10
+ peer "github.com/jbenet/go-ipfs/peer"
11
+)
12
+
13
+func _randPeer() *peer.Peer {
14
+ p := new(peer.Peer)
15
+ p.ID = make(peer.ID, 16)
16
+ crand.Read(p.ID)
17
+ return p
18
+}
19
+
20
+func _randID() ID {
21
+ buf := make([]byte, 16)
22
+ crand.Read(buf)
23
+
24
+ hash := sha256.Sum256(buf)
25
+ return ID(hash[:])
26
+}
27
+
28
+// Test basic features of the bucket struct
29
+func TestBucket(t *testing.T) {
30
+ b := new(Bucket)
31
+
32
+ peers := make([]*peer.Peer, 100)
33
+ for i := 0; i < 100; i++ {
34
+ peers[i] = _randPeer()
35
+ b.PushFront(peers[i])
36
+ }
37
+
38
+ local := _randPeer()
39
+ local_id := convertPeerID(local.ID)
40
+
41
+ i := rand.Intn(len(peers))
42
+ e := b.Find(peers[i].ID)
43
+ if e == nil {
44
+ t.Errorf("Failed to find peer: %v", peers[i])
45
+ }
46
+
47
+ spl := b.Split(0, convertPeerID(local.ID))
48
+ llist := (*list.List)(b)
49
+ for e := llist.Front(); e != nil; e = e.Next() {
50
+ p := convertPeerID(e.Value.(*peer.Peer).ID)
51
+ cpl := xor(p, local_id).commonPrefixLen()
52
+ if cpl > 0 {
53
+ t.Fatalf("Split failed. found id with cpl > 0 in 0 bucket")
54
+ }
55
+ }
56
+
57
+ rlist := (*list.List)(spl)
58
+ for e := rlist.Front(); e != nil; e = e.Next() {
59
+ p := convertPeerID(e.Value.(*peer.Peer).ID)
60
+ cpl := xor(p, local_id).commonPrefixLen()
61
+ if cpl == 0 {
62
+ t.Fatalf("Split failed. found id with cpl == 0 in non 0 bucket")
63
+ }
64
+ }
65
+}
66
+
67
+// Right now, this just makes sure that it doesnt hang or crash
68
+func TestTableUpdate(t *testing.T) {
69
+ local := _randPeer()
70
+ rt := NewRoutingTable(10, convertPeerID(local.ID))
71
+
72
+ peers := make([]*peer.Peer, 100)
73
+ for i := 0; i < 100; i++ {
74
+ peers[i] = _randPeer()
75
+ }
76
+
77
+ // Testing Update
78
+ for i := 0; i < 10000; i++ {
79
+ p := rt.Update(peers[rand.Intn(len(peers))])
80
+ if p != nil {
81
+ t.Log("evicted peer.")
82
+ }
83
+ }
84
+
85
+ for i := 0; i < 100; i++ {
86
+ id := _randID()
87
+ ret := rt.NearestPeers(id, 5)
88
+ if len(ret) == 0 {
89
+ t.Fatal("Failed to find node near ID.")
90
+ }
91
+ }
92
+}
routing/dht/util.go
new
+78
@@ -0,0 +1,78 @@
1
+package dht
2
+
3
+import (
4
+ "bytes"
5
+ "crypto/sha256"
6
+
7
+ peer "github.com/jbenet/go-ipfs/peer"
8
+ u "github.com/jbenet/go-ipfs/util"
9
+)
10
+
11
+// ID for IpfsDHT should be a byte slice, to allow for simpler operations
12
+// (xor). DHT ids are based on the peer.IDs.
13
+//
14
+// NOTE: peer.IDs are biased because they are multihashes (first bytes
15
+// biased). Thus, may need to re-hash keys (uniform dist). TODO(jbenet)
16
+type ID []byte
17
+
18
+func (id ID) Equal(other ID) bool {
19
+ return bytes.Equal(id, other)
20
+}
21
+
22
+func (id ID) Less(other interface{}) bool {
23
+ a, b := equalizeSizes(id, other.(ID))
24
+ for i := 0; i < len(a); i++ {
25
+ if a[i] != b[i] {
26
+ return a[i] < b[i]
27
+ }
28
+ }
29
+ return len(a) < len(b)
30
+}
31
+
32
+func (id ID) commonPrefixLen() int {
33
+ for i := 0; i < len(id); i++ {
34
+ for j := 0; j < 8; j++ {
35
+ if (id[i]>>uint8(7-j))&0x1 != 0 {
36
+ return i*8 + j
37
+ }
38
+ }
39
+ }
40
+ return len(id)*8 - 1
41
+}
42
+
43
+func xor(a, b ID) ID {
44
+ a, b = equalizeSizes(a, b)
45
+
46
+ c := make(ID, len(a))
47
+ for i := 0; i < len(a); i++ {
48
+ c[i] = a[i] ^ b[i]
49
+ }
50
+ return c
51
+}
52
+
53
+func equalizeSizes(a, b ID) (ID, ID) {
54
+ la := len(a)
55
+ lb := len(b)
56
+
57
+ if la < lb {
58
+ na := make([]byte, lb)
59
+ copy(na, a)
60
+ a = na
61
+ } else if lb < la {
62
+ nb := make([]byte, la)
63
+ copy(nb, b)
64
+ b = nb
65
+ }
66
+
67
+ return a, b
68
+}
69
+
70
+func convertPeerID(id peer.ID) ID {
71
+ hash := sha256.Sum256(id)
72
+ return hash[:]
73
+}
74
+
75
+func convertKey(id u.Key) ID {
76
+ hash := sha256.Sum256([]byte(id))
77
+ return hash[:]
78
+}