cid: integrate cid into bitswap and blockstores
License: MIT Signed-off-by: Jeromy <why@ipfs.io>
Jeromy committed
Oct 7, 2016 at 11:14 UTC
282bdc4816ff445295bd490fb880421bf437f483
52 files changed
+544
-535
blocks/blocks.go
+14
-19
@@ -6,8 +6,6 @@ import (
6
"errors"
7
"fmt"
8
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
10
-
9
mh "gx/ipfs/QmYDds3421prZgqKbLpEK7T9Aa2eVdQ7o3YarX1LVLdP2J/go-multihash"
10
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
11
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
@@ -18,37 +16,39 @@ var ErrWrongHash = errors.New("data did not match given hash!")
16
type Block interface {
17
Multihash() mh.Multihash
18
RawData() []byte
21
- Key() key.Key
19
+ Cid() *cid.Cid
20
String() string
21
Loggable() map[string]interface{}
22
}
23
24
// Block is a singular block of data in ipfs
25
type BasicBlock struct {
28
- multihash mh.Multihash
29
- data []byte
26
+ cid *cid.Cid
27
+ data []byte
28
}
29
30
// NewBlock creates a Block object from opaque data. It will hash the data.
31
func NewBlock(data []byte) *BasicBlock {
34
- return &BasicBlock{data: data, multihash: u.Hash(data)}
32
+ // TODO: fix assumptions
33
+ return &BasicBlock{data: data, cid: cid.NewCidV0(u.Hash(data))}
34
}
35
36
// NewBlockWithHash creates a new block when the hash of the data
37
// is already known, this is used to save time in situations where
38
// we are able to be confident that the data is correct
40
-func NewBlockWithHash(data []byte, h mh.Multihash) (*BasicBlock, error) {
39
+func NewBlockWithCid(data []byte, c *cid.Cid) (*BasicBlock, error) {
40
if u.Debug {
42
- chk := u.Hash(data)
43
- if string(chk) != string(h) {
41
+ // TODO: fix assumptions
42
+ chkc := cid.NewCidV0(u.Hash(data))
43
+ if !chkc.Equals(c) {
44
return nil, ErrWrongHash
45
}
46
}
47
- return &BasicBlock{data: data, multihash: h}, nil
47
+ return &BasicBlock{data: data, cid: c}, nil
48
}
49
50
func (b *BasicBlock) Multihash() mh.Multihash {
51
- return b.multihash
51
+ return b.cid.Hash()
52
}
53
54
func (b *BasicBlock) RawData() []byte {
@@ -56,20 +56,15 @@ func (b *BasicBlock) RawData() []byte {
56
}
57
58
func (b *BasicBlock) Cid() *cid.Cid {
59
- return cid.NewCidV0(b.multihash)
60
-}
61
-
62
-// Key returns the block's Multihash as a Key value.
63
-func (b *BasicBlock) Key() key.Key {
64
- return key.Key(b.multihash)
59
+ return b.cid
60
}
61
62
func (b *BasicBlock) String() string {
68
- return fmt.Sprintf("[Block %s]", b.Key())
63
+ return fmt.Sprintf("[Block %s]", b.Cid())
64
}
65
66
func (b *BasicBlock) Loggable() map[string]interface{} {
67
return map[string]interface{}{
73
- "block": b.Key().String(),
68
+ "block": b.Cid().String(),
69
}
70
}
blocks/blocks_test.go
+9
-6
@@ -5,6 +5,7 @@ import (
5
"testing"
6
7
mh "gx/ipfs/QmYDds3421prZgqKbLpEK7T9Aa2eVdQ7o3YarX1LVLdP2J/go-multihash"
8
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
9
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
10
)
11
@@ -44,12 +45,12 @@ func TestHash(t *testing.T) {
45
}
46
}
47
47
-func TestKey(t *testing.T) {
48
+func TestCid(t *testing.T) {
49
data := []byte("yet another data")
50
block := NewBlock(data)
50
- key := block.Key()
51
+ c := block.Cid()
52
52
- if !bytes.Equal(block.Multihash(), key.ToMultihash()) {
53
+ if !bytes.Equal(block.Multihash(), c.Hash()) {
54
t.Error("key contains wrong data")
55
}
56
}
@@ -66,8 +67,10 @@ func TestManualHash(t *testing.T) {
67
t.Fatal(err)
68
}
69
70
+ c := cid.NewCidV0(hash)
71
+
72
u.Debug = false
70
- block, err := NewBlockWithHash(data, hash)
73
+ block, err := NewBlockWithCid(data, c)
74
if err != nil {
75
t.Fatal(err)
76
}
@@ -77,7 +80,7 @@ func TestManualHash(t *testing.T) {
80
}
81
82
data[5] = byte((uint32(data[5]) + 5) % 256) // Transfrom hash to be different
80
- block, err = NewBlockWithHash(data, hash)
83
+ block, err = NewBlockWithCid(data, c)
84
if err != nil {
85
t.Fatal(err)
86
}
@@ -88,7 +91,7 @@ func TestManualHash(t *testing.T) {
91
92
u.Debug = true
93
91
- block, err = NewBlockWithHash(data, hash)
94
+ block, err = NewBlockWithCid(data, c)
95
if err != ErrWrongHash {
96
t.Fatal(err)
97
}
blocks/blockstore/arc_cache.go
+27
-17
@@ -1,13 +1,13 @@
1
package blockstore
2
3
import (
4
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
4
+ "context"
5
6
"github.com/ipfs/go-ipfs/blocks"
7
8
- context "context"
8
"gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
9
lru "gx/ipfs/QmVYxfoJQiZijTgPNHCHgHELvQpbsJNTg6Crmc3dQkj3yy/golang-lru"
10
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
11
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
12
)
13
@@ -31,7 +31,7 @@ func newARCCachedBS(ctx context.Context, bs Blockstore, lruSize int) (*arccache,
31
return c, nil
32
}
33
34
-func (b *arccache) DeleteBlock(k key.Key) error {
34
+func (b *arccache) DeleteBlock(k *cid.Cid) error {
35
if has, ok := b.hasCached(k); ok && !has {
36
return ErrNotFound
37
}
@@ -40,7 +40,7 @@ func (b *arccache) DeleteBlock(k key.Key) error {
40
err := b.blockstore.DeleteBlock(k)
41
switch err {
42
case nil, ds.ErrNotFound, ErrNotFound:
43
- b.arc.Add(k, false)
43
+ b.addCache(k, false)
44
return err
45
default:
46
return err
@@ -49,15 +49,16 @@ func (b *arccache) DeleteBlock(k key.Key) error {
49
50
// if ok == false has is inconclusive
51
// if ok == true then has respons to question: is it contained
52
-func (b *arccache) hasCached(k key.Key) (has bool, ok bool) {
52
+func (b *arccache) hasCached(k *cid.Cid) (has bool, ok bool) {
53
b.total.Inc()
54
- if k == "" {
54
+ if k == nil {
55
+ log.Error("nil cid in arccache")
56
// Return cache invalid so the call to blockstore happens
57
// in case of invalid key and correct error is created.
58
return false, false
59
}
60
60
- h, ok := b.arc.Get(k)
61
+ h, ok := b.arc.Get(k.KeyString())
62
if ok {
63
b.hits.Inc()
64
return h.(bool), true
@@ -65,40 +66,45 @@ func (b *arccache) hasCached(k key.Key) (has bool, ok bool) {
66
return false, false
67
}
68
68
-func (b *arccache) Has(k key.Key) (bool, error) {
69
+func (b *arccache) Has(k *cid.Cid) (bool, error) {
70
if has, ok := b.hasCached(k); ok {
71
return has, nil
72
}
73
74
res, err := b.blockstore.Has(k)
75
if err == nil {
75
- b.arc.Add(k, res)
76
+ b.addCache(k, res)
77
}
78
return res, err
79
}
80
80
-func (b *arccache) Get(k key.Key) (blocks.Block, error) {
81
+func (b *arccache) Get(k *cid.Cid) (blocks.Block, error) {
82
+ if k == nil {
83
+ log.Error("nil cid in arc cache")
84
+ return nil, ErrNotFound
85
+ }
86
+
87
if has, ok := b.hasCached(k); ok && !has {
88
return nil, ErrNotFound
89
}
90
91
bl, err := b.blockstore.Get(k)
92
if bl == nil && err == ErrNotFound {
87
- b.arc.Add(k, false)
93
+ b.addCache(k, false)
94
} else if bl != nil {
89
- b.arc.Add(k, true)
95
+ b.addCache(k, true)
96
}
97
return bl, err
98
}
99
100
func (b *arccache) Put(bl blocks.Block) error {
95
- if has, ok := b.hasCached(bl.Key()); ok && has {
101
+ if has, ok := b.hasCached(bl.Cid()); ok && has {
102
return nil
103
}
104
105
err := b.blockstore.Put(bl)
106
if err == nil {
101
- b.arc.Add(bl.Key(), true)
107
+ b.addCache(bl.Cid(), true)
108
}
109
return err
110
}
@@ -108,7 +114,7 @@ func (b *arccache) PutMany(bs []blocks.Block) error {
114
for _, block := range bs {
115
// call put on block if result is inconclusive or we are sure that
116
// the block isn't in storage
111
- if has, ok := b.hasCached(block.Key()); !ok || (ok && !has) {
117
+ if has, ok := b.hasCached(block.Cid()); !ok || (ok && !has) {
118
good = append(good, block)
119
}
120
}
@@ -117,12 +123,16 @@ func (b *arccache) PutMany(bs []blocks.Block) error {
123
return err
124
}
125
for _, block := range good {
120
- b.arc.Add(block.Key(), true)
126
+ b.addCache(block.Cid(), true)
127
}
128
return nil
129
}
130
125
-func (b *arccache) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
131
+func (b *arccache) addCache(c *cid.Cid, has bool) {
132
+ b.arc.Add(c.KeyString(), has)
133
+}
134
+
135
+func (b *arccache) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
136
return b.blockstore.AllKeysChan(ctx)
137
}
138
blocks/blockstore/arc_cache_test.go
+35
-17
@@ -1,12 +1,12 @@
1
package blockstore
2
3
import (
4
+ "context"
5
"testing"
6
7
"github.com/ipfs/go-ipfs/blocks"
7
- "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
8
9
- context "context"
9
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
10
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
11
syncds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
12
)
@@ -60,7 +60,7 @@ func TestRemoveCacheEntryOnDelete(t *testing.T) {
60
writeHitTheDatastore = true
61
})
62
63
- arc.DeleteBlock(exampleBlock.Key())
63
+ arc.DeleteBlock(exampleBlock.Cid())
64
arc.Put(exampleBlock)
65
if !writeHitTheDatastore {
66
t.Fail()
@@ -78,9 +78,9 @@ func TestElideDuplicateWrite(t *testing.T) {
78
func TestHasRequestTriggersCache(t *testing.T) {
79
arc, _, cd := createStores(t)
80
81
- arc.Has(exampleBlock.Key())
81
+ arc.Has(exampleBlock.Cid())
82
trap("has hit datastore", cd, t)
83
- if has, err := arc.Has(exampleBlock.Key()); has || err != nil {
83
+ if has, err := arc.Has(exampleBlock.Cid()); has || err != nil {
84
t.Fatal("has was true but there is no such block")
85
}
86
@@ -92,7 +92,7 @@ func TestHasRequestTriggersCache(t *testing.T) {
92
93
trap("has hit datastore", cd, t)
94
95
- if has, err := arc.Has(exampleBlock.Key()); !has || err != nil {
95
+ if has, err := arc.Has(exampleBlock.Cid()); !has || err != nil {
96
t.Fatal("has returned invalid result")
97
}
98
}
@@ -100,13 +100,13 @@ func TestHasRequestTriggersCache(t *testing.T) {
100
func TestGetFillsCache(t *testing.T) {
101
arc, _, cd := createStores(t)
102
103
- if bl, err := arc.Get(exampleBlock.Key()); bl != nil || err == nil {
103
+ if bl, err := arc.Get(exampleBlock.Cid()); bl != nil || err == nil {
104
t.Fatal("block was found or there was no error")
105
}
106
107
trap("has hit datastore", cd, t)
108
109
- if has, err := arc.Has(exampleBlock.Key()); has || err != nil {
109
+ if has, err := arc.Has(exampleBlock.Cid()); has || err != nil {
110
t.Fatal("has was true but there is no such block")
111
}
112
@@ -118,7 +118,7 @@ func TestGetFillsCache(t *testing.T) {
118
119
trap("has hit datastore", cd, t)
120
121
- if has, err := arc.Has(exampleBlock.Key()); !has || err != nil {
121
+ if has, err := arc.Has(exampleBlock.Cid()); !has || err != nil {
122
t.Fatal("has returned invalid result")
123
}
124
}
@@ -126,15 +126,15 @@ func TestGetFillsCache(t *testing.T) {
126
func TestGetAndDeleteFalseShortCircuit(t *testing.T) {
127
arc, _, cd := createStores(t)
128
129
- arc.Has(exampleBlock.Key())
129
+ arc.Has(exampleBlock.Cid())
130
131
trap("get hit datastore", cd, t)
132
133
- if bl, err := arc.Get(exampleBlock.Key()); bl != nil || err != ErrNotFound {
133
+ if bl, err := arc.Get(exampleBlock.Cid()); bl != nil || err != ErrNotFound {
134
t.Fatal("get returned invalid result")
135
}
136
137
- if arc.DeleteBlock(exampleBlock.Key()) != ErrNotFound {
137
+ if arc.DeleteBlock(exampleBlock.Cid()) != ErrNotFound {
138
t.Fatal("expected ErrNotFound error")
139
}
140
}
@@ -148,7 +148,7 @@ func TestArcCreationFailure(t *testing.T) {
148
func TestInvalidKey(t *testing.T) {
149
arc, _, _ := createStores(t)
150
151
- bl, err := arc.Get(key.Key(""))
151
+ bl, err := arc.Get(nil)
152
153
if bl != nil {
154
t.Fatal("blocks should be nil")
@@ -163,10 +163,28 @@ func TestHasAfterSucessfulGetIsCached(t *testing.T) {
163
164
bs.Put(exampleBlock)
165
166
- arc.Get(exampleBlock.Key())
166
+ arc.Get(exampleBlock.Cid())
167
168
trap("has hit datastore", cd, t)
169
- arc.Has(exampleBlock.Key())
169
+ arc.Has(exampleBlock.Cid())
170
+}
171
+
172
+func TestDifferentKeyObjectsWork(t *testing.T) {
173
+ arc, bs, cd := createStores(t)
174
+
175
+ bs.Put(exampleBlock)
176
+
177
+ arc.Get(exampleBlock.Cid())
178
+
179
+ trap("has hit datastore", cd, t)
180
+ cidstr := exampleBlock.Cid().String()
181
+
182
+ ncid, err := cid.Decode(cidstr)
183
+ if err != nil {
184
+ t.Fatal(err)
185
+ }
186
+
187
+ arc.Has(ncid)
188
}
189
190
func TestPutManyCaches(t *testing.T) {
@@ -174,9 +192,9 @@ func TestPutManyCaches(t *testing.T) {
192
arc.PutMany([]blocks.Block{exampleBlock})
193
194
trap("has hit datastore", cd, t)
177
- arc.Has(exampleBlock.Key())
195
+ arc.Has(exampleBlock.Cid())
196
untrap(cd)
179
- arc.DeleteBlock(exampleBlock.Key())
197
+ arc.DeleteBlock(exampleBlock.Cid())
198
199
arc.Put(exampleBlock)
200
trap("PunMany has hit datastore", cd, t)
blocks/blockstore/blockstore.go
+34
-33
@@ -3,15 +3,16 @@
3
package blockstore
4
5
import (
6
+ "context"
7
"errors"
8
"sync"
9
"sync/atomic"
10
10
- context "context"
11
blocks "github.com/ipfs/go-ipfs/blocks"
12
+ dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
13
+
14
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
13
- mh "gx/ipfs/QmYDds3421prZgqKbLpEK7T9Aa2eVdQ7o3YarX1LVLdP2J/go-multihash"
14
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
15
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
16
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
17
dsns "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/namespace"
18
dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query"
@@ -29,13 +30,13 @@ var ErrNotFound = errors.New("blockstore: block not found")
30
31
// Blockstore wraps a Datastore
32
type Blockstore interface {
32
- DeleteBlock(key.Key) error
33
- Has(key.Key) (bool, error)
34
- Get(key.Key) (blocks.Block, error)
33
+ DeleteBlock(*cid.Cid) error
34
+ Has(*cid.Cid) (bool, error)
35
+ Get(*cid.Cid) (blocks.Block, error)
36
Put(blocks.Block) error
37
PutMany([]blocks.Block) error
38
38
- AllKeysChan(ctx context.Context) (<-chan key.Key, error)
39
+ AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error)
40
}
41
42
type GCBlockstore interface {
@@ -80,12 +81,13 @@ func (bs *blockstore) HashOnRead(enabled bool) {
81
bs.rehash = enabled
82
}
83
83
-func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
84
- if k == "" {
84
+func (bs *blockstore) Get(k *cid.Cid) (blocks.Block, error) {
85
+ if k == nil {
86
+ log.Error("nil cid in blockstore")
87
return nil, ErrNotFound
88
}
89
88
- maybeData, err := bs.datastore.Get(k.DsKey())
90
+ maybeData, err := bs.datastore.Get(dshelp.NewKeyFromBinary(k.KeyString()))
91
if err == ds.ErrNotFound {
92
return nil, ErrNotFound
93
}
@@ -99,18 +101,18 @@ func (bs *blockstore) Get(k key.Key) (blocks.Block, error) {
101
102
if bs.rehash {
103
rb := blocks.NewBlock(bdata)
102
- if rb.Key() != k {
104
+ if !rb.Cid().Equals(k) {
105
return nil, ErrHashMismatch
106
} else {
107
return rb, nil
108
}
109
} else {
108
- return blocks.NewBlockWithHash(bdata, mh.Multihash(k))
110
+ return blocks.NewBlockWithCid(bdata, k)
111
}
112
}
113
114
func (bs *blockstore) Put(block blocks.Block) error {
113
- k := block.Key().DsKey()
115
+ k := dshelp.NewKeyFromBinary(block.Cid().KeyString())
116
117
// Has is cheaper than Put, so see if we already have it
118
exists, err := bs.datastore.Has(k)
@@ -126,7 +128,7 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
128
return err
129
}
130
for _, b := range blocks {
129
- k := b.Key().DsKey()
131
+ k := dshelp.NewKeyFromBinary(b.Cid().KeyString())
132
exists, err := bs.datastore.Has(k)
133
if err == nil && exists {
134
continue
@@ -140,19 +142,19 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
142
return t.Commit()
143
}
144
143
-func (bs *blockstore) Has(k key.Key) (bool, error) {
144
- return bs.datastore.Has(k.DsKey())
145
+func (bs *blockstore) Has(k *cid.Cid) (bool, error) {
146
+ return bs.datastore.Has(dshelp.NewKeyFromBinary(k.KeyString()))
147
}
148
147
-func (s *blockstore) DeleteBlock(k key.Key) error {
148
- return s.datastore.Delete(k.DsKey())
149
+func (s *blockstore) DeleteBlock(k *cid.Cid) error {
150
+ return s.datastore.Delete(dshelp.NewKeyFromBinary(k.KeyString()))
151
}
152
153
// AllKeysChan runs a query for keys from the blockstore.
154
// this is very simplistic, in the future, take dsq.Query as a param?
155
//
156
// AllKeysChan respects context
155
-func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
157
+func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
158
159
// KeysOnly, because that would be _a lot_ of data.
160
q := dsq.Query{KeysOnly: true}
@@ -164,39 +166,38 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
166
}
167
168
// this function is here to compartmentalize
167
- get := func() (key.Key, bool) {
169
+ get := func() (*cid.Cid, bool) {
170
select {
171
case <-ctx.Done():
170
- return "", false
172
+ return nil, false
173
case e, more := <-res.Next():
174
if !more {
173
- return "", false
175
+ return nil, false
176
}
177
if e.Error != nil {
178
log.Debug("blockstore.AllKeysChan got err:", e.Error)
177
- return "", false
179
+ return nil, false
180
}
181
182
// need to convert to key.Key using key.KeyFromDsKey.
181
- k, err := key.KeyFromDsKey(ds.NewKey(e.Key))
183
+ kb, err := dshelp.BinaryFromDsKey(ds.NewKey(e.Key)) // TODO: calling NewKey isnt free
184
if err != nil {
185
log.Warningf("error parsing key from DsKey: ", err)
184
- return "", true
186
+ return nil, true
187
}
186
- log.Debug("blockstore: query got key", k)
188
188
- // key must be a multihash. else ignore it.
189
- _, err = mh.Cast([]byte(k))
189
+ c, err := cid.Cast(kb)
190
if err != nil {
191
- log.Warningf("key from datastore was not a multihash: ", err)
192
- return "", true
191
+ log.Warning("error parsing cid from decoded DsKey: ", err)
192
+ return nil, true
193
}
194
+ log.Debug("blockstore: query got key", c)
195
195
- return k, true
196
+ return c, true
197
}
198
}
199
199
- output := make(chan key.Key, dsq.KeysOnlyBufSize)
200
+ output := make(chan *cid.Cid, dsq.KeysOnlyBufSize)
201
go func() {
202
defer func() {
203
res.Process().Close() // ensure exit (signals early exit, too)
@@ -208,7 +209,7 @@ func (bs *blockstore) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
209
if !ok {
210
return
211
}
211
- if k == "" {
212
+ if k == nil {
213
continue
214
}
215
blocks/blockstore/blockstore_test.go
+22
-20
@@ -2,22 +2,24 @@ package blockstore
2
3
import (
4
"bytes"
5
+ "context"
6
"fmt"
7
"testing"
8
8
- context "context"
9
+ blocks "github.com/ipfs/go-ipfs/blocks"
10
+ dshelp "github.com/ipfs/go-ipfs/thirdparty/ds-help"
11
+
12
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
13
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
14
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
15
dsq "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/query"
16
ds_sync "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
13
-
14
- blocks "github.com/ipfs/go-ipfs/blocks"
15
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
17
)
18
19
func TestGetWhenKeyNotPresent(t *testing.T) {
20
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
20
- bl, err := bs.Get(key.Key("not present"))
21
+ c := cid.NewCidV0(u.Hash([]byte("stuff")))
22
+ bl, err := bs.Get(c)
23
24
if bl != nil {
25
t.Error("nil block expected")
@@ -27,9 +29,9 @@ func TestGetWhenKeyNotPresent(t *testing.T) {
29
}
30
}
31
30
-func TestGetWhenKeyIsEmptyString(t *testing.T) {
32
+func TestGetWhenKeyIsNil(t *testing.T) {
33
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
32
- _, err := bs.Get(key.Key(""))
34
+ _, err := bs.Get(nil)
35
if err != ErrNotFound {
36
t.Fail()
37
}
@@ -44,7 +46,7 @@ func TestPutThenGetBlock(t *testing.T) {
46
t.Fatal(err)
47
}
48
47
- blockFromBlockstore, err := bs.Get(block.Key())
49
+ blockFromBlockstore, err := bs.Get(block.Cid())
50
if err != nil {
51
t.Fatal(err)
52
}
@@ -62,7 +64,7 @@ func TestHashOnRead(t *testing.T) {
64
65
bs := NewBlockstore(ds_sync.MutexWrap(ds.NewMapDatastore()))
66
bl := blocks.NewBlock([]byte("some data"))
65
- blBad, err := blocks.NewBlockWithHash([]byte("some other data"), bl.Key().ToMultihash())
67
+ blBad, err := blocks.NewBlockWithCid([]byte("some other data"), bl.Cid())
68
if err != nil {
69
t.Fatal("debug is off, still got an error")
70
}
@@ -71,35 +73,35 @@ func TestHashOnRead(t *testing.T) {
73
bs.Put(bl2)
74
bs.HashOnRead(true)
75
74
- if _, err := bs.Get(bl.Key()); err != ErrHashMismatch {
76
+ if _, err := bs.Get(bl.Cid()); err != ErrHashMismatch {
77
t.Fatalf("expected '%v' got '%v'\n", ErrHashMismatch, err)
78
}
79
78
- if b, err := bs.Get(bl2.Key()); err != nil || b.String() != bl2.String() {
80
+ if b, err := bs.Get(bl2.Cid()); err != nil || b.String() != bl2.String() {
81
t.Fatal("got wrong blocks")
82
}
83
}
84
83
-func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []key.Key) {
85
+func newBlockStoreWithKeys(t *testing.T, d ds.Datastore, N int) (Blockstore, []*cid.Cid) {
86
if d == nil {
87
d = ds.NewMapDatastore()
88
}
89
bs := NewBlockstore(ds_sync.MutexWrap(d))
90
89
- keys := make([]key.Key, N)
91
+ keys := make([]*cid.Cid, N)
92
for i := 0; i < N; i++ {
93
block := blocks.NewBlock([]byte(fmt.Sprintf("some data %d", i)))
94
err := bs.Put(block)
95
if err != nil {
96
t.Fatal(err)
97
}
96
- keys[i] = block.Key()
98
+ keys[i] = block.Cid()
99
}
100
return bs, keys
101
}
102
101
-func collect(ch <-chan key.Key) []key.Key {
102
- var keys []key.Key
103
+func collect(ch <-chan *cid.Cid) []*cid.Cid {
104
+ var keys []*cid.Cid
105
for k := range ch {
106
keys = append(keys, k)
107
}
@@ -188,18 +190,18 @@ func TestValueTypeMismatch(t *testing.T) {
190
block := blocks.NewBlock([]byte("some data"))
191
192
datastore := ds.NewMapDatastore()
191
- k := BlockPrefix.Child(block.Key().DsKey())
193
+ k := BlockPrefix.Child(dshelp.NewKeyFromBinary(block.Cid().KeyString()))
194
datastore.Put(k, "data that isn't a block!")
195
196
blockstore := NewBlockstore(ds_sync.MutexWrap(datastore))
197
196
- _, err := blockstore.Get(block.Key())
198
+ _, err := blockstore.Get(block.Cid())
199
if err != ValueTypeMismatch {
200
t.Fatal(err)
201
}
202
}
203
202
-func expectMatches(t *testing.T, expect, actual []key.Key) {
204
+func expectMatches(t *testing.T, expect, actual []*cid.Cid) {
205
206
if len(expect) != len(actual) {
207
t.Errorf("expect and actual differ: %d != %d", len(expect), len(actual))
@@ -207,7 +209,7 @@ func expectMatches(t *testing.T, expect, actual []key.Key) {
209
for _, ek := range expect {
210
found := false
211
for _, ak := range actual {
210
- if ek == ak {
212
+ if ek.Equals(ak) {
213
found = true
214
}
215
}
blocks/blockstore/bloom_cache.go
+14
-13
@@ -1,14 +1,14 @@
1
package blockstore
2
3
import (
4
+ "context"
5
"sync/atomic"
6
"time"
7
8
"github.com/ipfs/go-ipfs/blocks"
8
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
10
- context "context"
10
"gx/ipfs/QmRg1gKTHzc3CZXSKzem8aR4E3TubFhbgXwfVuWnSK5CC5/go-metrics-interface"
11
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
bloom "gx/ipfs/QmeiMCBkYHxkDkDfnDadzz4YxY5ruL5Pj499essE4vRsGM/bbloom"
13
)
14
@@ -84,7 +84,7 @@ func (b *bloomcache) Rebuild(ctx context.Context) {
84
select {
85
case key, ok := <-ch:
86
if ok {
87
- b.bloom.AddTS([]byte(key)) // Use binary key, the more compact the better
87
+ b.bloom.AddTS(key.Bytes()) // Use binary key, the more compact the better
88
} else {
89
finish = true
90
}
@@ -97,7 +97,7 @@ func (b *bloomcache) Rebuild(ctx context.Context) {
97
atomic.StoreInt32(&b.active, 1)
98
}
99
100
-func (b *bloomcache) DeleteBlock(k key.Key) error {
100
+func (b *bloomcache) DeleteBlock(k *cid.Cid) error {
101
if has, ok := b.hasCached(k); ok && !has {
102
return ErrNotFound
103
}
@@ -107,15 +107,16 @@ func (b *bloomcache) DeleteBlock(k key.Key) error {
107
108
// if ok == false has is inconclusive
109
// if ok == true then has respons to question: is it contained
110
-func (b *bloomcache) hasCached(k key.Key) (has bool, ok bool) {
110
+func (b *bloomcache) hasCached(k *cid.Cid) (has bool, ok bool) {
111
b.total.Inc()
112
- if k == "" {
112
+ if k == nil {
113
+ log.Error("nil cid in bloom cache")
114
// Return cache invalid so call to blockstore
115
// in case of invalid key is forwarded deeper
116
return false, false
117
}
118
if b.BloomActive() {
118
- blr := b.bloom.HasTS([]byte(k))
119
+ blr := b.bloom.HasTS(k.Bytes())
120
if blr == false { // not contained in bloom is only conclusive answer bloom gives
121
b.hits.Inc()
122
return false, true
@@ -124,7 +125,7 @@ func (b *bloomcache) hasCached(k key.Key) (has bool, ok bool) {
125
return false, false
126
}
127
127
-func (b *bloomcache) Has(k key.Key) (bool, error) {
128
+func (b *bloomcache) Has(k *cid.Cid) (bool, error) {
129
if has, ok := b.hasCached(k); ok {
130
return has, nil
131
}
@@ -132,7 +133,7 @@ func (b *bloomcache) Has(k key.Key) (bool, error) {
133
return b.blockstore.Has(k)
134
}
135
135
-func (b *bloomcache) Get(k key.Key) (blocks.Block, error) {
136
+func (b *bloomcache) Get(k *cid.Cid) (blocks.Block, error) {
137
if has, ok := b.hasCached(k); ok && !has {
138
return nil, ErrNotFound
139
}
@@ -141,13 +142,13 @@ func (b *bloomcache) Get(k key.Key) (blocks.Block, error) {
142
}
143
144
func (b *bloomcache) Put(bl blocks.Block) error {
144
- if has, ok := b.hasCached(bl.Key()); ok && has {
145
+ if has, ok := b.hasCached(bl.Cid()); ok && has {
146
return nil
147
}
148
149
err := b.blockstore.Put(bl)
150
if err == nil {
150
- b.bloom.AddTS([]byte(bl.Key()))
151
+ b.bloom.AddTS(bl.Cid().Bytes())
152
}
153
return err
154
}
@@ -162,12 +163,12 @@ func (b *bloomcache) PutMany(bs []blocks.Block) error {
163
return err
164
}
165
for _, bl := range bs {
165
- b.bloom.AddTS([]byte(bl.Key()))
166
+ b.bloom.AddTS(bl.Cid().Bytes())
167
}
168
return nil
169
}
170
170
-func (b *bloomcache) AllKeysChan(ctx context.Context) (<-chan key.Key, error) {
171
+func (b *bloomcache) AllKeysChan(ctx context.Context) (<-chan *cid.Cid, error) {
172
return b.blockstore.AllKeysChan(ctx)
173
}
174
blocks/blockstore/bloom_cache_test.go
+5
-5
@@ -44,7 +44,7 @@ func TestPutManyAddsToBloom(t *testing.T) {
44
block2 := blocks.NewBlock([]byte("bar"))
45
46
cachedbs.PutMany([]blocks.Block{block1})
47
- has, err := cachedbs.Has(block1.Key())
47
+ has, err := cachedbs.Has(block1.Cid())
48
if err != nil {
49
t.Fatal(err)
50
}
@@ -52,7 +52,7 @@ func TestPutManyAddsToBloom(t *testing.T) {
52
t.Fatal("added block is reported missing")
53
}
54
55
- has, err = cachedbs.Has(block2.Key())
55
+ has, err = cachedbs.Has(block2.Cid())
56
if err != nil {
57
t.Fatal(err)
58
}
@@ -93,7 +93,7 @@ func TestHasIsBloomCached(t *testing.T) {
93
})
94
95
for i := 0; i < 1000; i++ {
96
- cachedbs.Has(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Key())
96
+ cachedbs.Has(blocks.NewBlock([]byte(fmt.Sprintf("data: %d", i+2000))).Cid())
97
}
98
99
if float64(cacheFails)/float64(1000) > float64(0.05) {
@@ -112,11 +112,11 @@ func TestHasIsBloomCached(t *testing.T) {
112
t.Fatalf("expected datastore hit: %d", cacheFails)
113
}
114
115
- if has, err := cachedbs.Has(block.Key()); !has || err != nil {
115
+ if has, err := cachedbs.Has(block.Cid()); !has || err != nil {
116
t.Fatal("has gave wrong response")
117
}
118
119
- bl, err := cachedbs.Get(block.Key())
119
+ bl, err := cachedbs.Get(block.Cid())
120
if bl.String() != block.String() {
121
t.Fatal("block data doesn't match")
122
}
blocks/blockstore/util/remove.go
+1
-2
@@ -6,7 +6,6 @@ import (
6
7
bs "github.com/ipfs/go-ipfs/blocks/blockstore"
8
"github.com/ipfs/go-ipfs/pin"
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
10
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
11
)
@@ -38,7 +37,7 @@ func RmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, c
37
stillOkay := FilterPinned(pins, out, cids)
38
39
for _, c := range stillOkay {
41
- err := blocks.DeleteBlock(key.Key(c.Hash()))
40
+ err := blocks.DeleteBlock(c)
41
if err != nil && opts.Force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
42
// ignore non-existent blocks
43
} else if err != nil {
blocks/set/set.go
+20
-25
@@ -4,62 +4,57 @@ package set
4
import (
5
"github.com/ipfs/go-ipfs/blocks/bloom"
6
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
7
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
7
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
8
)
9
10
var log = logging.Logger("blockset")
11
12
// BlockSet represents a mutable set of keyed blocks
13
type BlockSet interface {
14
- AddBlock(key.Key)
15
- RemoveBlock(key.Key)
16
- HasKey(key.Key) bool
14
+ AddBlock(*cid.Cid)
15
+ RemoveBlock(*cid.Cid)
16
+ HasKey(*cid.Cid) bool
17
GetBloomFilter() bloom.Filter
18
19
- GetKeys() []key.Key
19
+ GetKeys() []*cid.Cid
20
}
21
22
-func SimpleSetFromKeys(keys []key.Key) BlockSet {
23
- sbs := &simpleBlockSet{blocks: make(map[key.Key]struct{})}
22
+func SimpleSetFromKeys(keys []*cid.Cid) BlockSet {
23
+ sbs := &simpleBlockSet{blocks: cid.NewSet()}
24
for _, k := range keys {
25
- sbs.blocks[k] = struct{}{}
25
+ sbs.AddBlock(k)
26
}
27
return sbs
28
}
29
30
func NewSimpleBlockSet() BlockSet {
31
- return &simpleBlockSet{blocks: make(map[key.Key]struct{})}
31
+ return &simpleBlockSet{blocks: cid.NewSet()}
32
}
33
34
type simpleBlockSet struct {
35
- blocks map[key.Key]struct{}
35
+ blocks *cid.Set
36
}
37
38
-func (b *simpleBlockSet) AddBlock(k key.Key) {
39
- b.blocks[k] = struct{}{}
38
+func (b *simpleBlockSet) AddBlock(k *cid.Cid) {
39
+ b.blocks.Add(k)
40
}
41
42
-func (b *simpleBlockSet) RemoveBlock(k key.Key) {
43
- delete(b.blocks, k)
42
+func (b *simpleBlockSet) RemoveBlock(k *cid.Cid) {
43
+ b.blocks.Remove(k)
44
}
45
46
-func (b *simpleBlockSet) HasKey(k key.Key) bool {
47
- _, has := b.blocks[k]
48
- return has
46
+func (b *simpleBlockSet) HasKey(k *cid.Cid) bool {
47
+ return b.blocks.Has(k)
48
}
49
50
func (b *simpleBlockSet) GetBloomFilter() bloom.Filter {
51
f := bloom.BasicFilter()
53
- for k := range b.blocks {
54
- f.Add([]byte(k))
52
+ for _, k := range b.blocks.Keys() {
53
+ f.Add(k.Bytes())
54
}
55
return f
56
}
57
59
-func (b *simpleBlockSet) GetKeys() []key.Key {
60
- var out []key.Key
61
- for k := range b.blocks {
62
- out = append(out, k)
63
- }
64
- return out
58
+func (b *simpleBlockSet) GetKeys() []*cid.Cid {
59
+ return b.blocks.Keys()
60
}
blocks/set/set_test.go
+7
-6
@@ -4,7 +4,8 @@ import (
4
"testing"
5
6
bu "github.com/ipfs/go-ipfs/blocks/blocksutil"
7
- k "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
7
+
8
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
9
)
10
11
const (
@@ -13,15 +14,15 @@ const (
14
tReAdd
15
)
16
16
-func exampleKeys() []k.Key {
17
- res := make([]k.Key, 1<<8)
17
+func exampleKeys() []*cid.Cid {
18
+ res := make([]*cid.Cid, 1<<8)
19
gen := bu.NewBlockGenerator()
20
for i := uint64(0); i < 1<<8; i++ {
20
- res[i] = gen.Next().Key()
21
+ res[i] = gen.Next().Cid()
22
}
23
return res
24
}
24
-func checkSet(set BlockSet, keySlice []k.Key, t *testing.T) {
25
+func checkSet(set BlockSet, keySlice []*cid.Cid, t *testing.T) {
26
for i, key := range keySlice {
27
if i&tReAdd == 0 {
28
if set.HasKey(key) == false {
@@ -69,7 +70,7 @@ func TestSetWorks(t *testing.T) {
70
bloom := set.GetBloomFilter()
71
72
for _, key := range addedKeys {
72
- if bloom.Find([]byte(key)) == false {
73
+ if bloom.Find(key.Bytes()) == false {
74
t.Error("bloom doesn't contain expected key")
75
}
76
}
blockservice/blockservice.go
+12
-25
@@ -10,7 +10,6 @@ import (
10
blocks "github.com/ipfs/go-ipfs/blocks"
11
"github.com/ipfs/go-ipfs/blocks/blockstore"
12
exchange "github.com/ipfs/go-ipfs/exchange"
13
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
13
14
context "context"
15
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
@@ -30,12 +29,6 @@ type BlockService struct {
29
Exchange exchange.Interface
30
}
31
33
-// an Object is simply a typed block
34
-type Object interface {
35
- Cid() *cid.Cid
36
- blocks.Block
37
-}
38
-
32
// NewBlockService creates a BlockService with given datastore instance.
33
func New(bs blockstore.Blockstore, rem exchange.Interface) *BlockService {
34
if rem == nil {
@@ -50,14 +43,14 @@ func New(bs blockstore.Blockstore, rem exchange.Interface) *BlockService {
43
44
// AddBlock adds a particular block to the service, Putting it into the datastore.
45
// TODO pass a context into this if the remote.HasBlock is going to remain here.
53
-func (s *BlockService) AddObject(o Object) (*cid.Cid, error) {
46
+func (s *BlockService) AddBlock(o blocks.Block) (*cid.Cid, error) {
47
// TODO: while this is a great optimization, we should think about the
48
// possibility of streaming writes directly to disk. If we can pass this object
49
// all the way down to the datastore without having to 'buffer' its data,
50
// we could implement a `WriteTo` method on it that could do a streaming write
51
// of the content, saving us (probably) considerable memory.
52
c := o.Cid()
60
- has, err := s.Blockstore.Has(key.Key(c.Hash()))
53
+ has, err := s.Blockstore.Has(c)
54
if err != nil {
55
return nil, err
56
}
@@ -78,13 +71,10 @@ func (s *BlockService) AddObject(o Object) (*cid.Cid, error) {
71
return c, nil
72
}
73
81
-func (s *BlockService) AddObjects(bs []Object) ([]*cid.Cid, error) {
74
+func (s *BlockService) AddBlocks(bs []blocks.Block) ([]*cid.Cid, error) {
75
var toput []blocks.Block
83
- var toputcids []*cid.Cid
76
for _, b := range bs {
85
- c := b.Cid()
86
-
87
- has, err := s.Blockstore.Has(key.Key(c.Hash()))
77
+ has, err := s.Blockstore.Has(b.Cid())
78
if err != nil {
79
return nil, err
80
}
@@ -94,7 +84,6 @@ func (s *BlockService) AddObjects(bs []Object) ([]*cid.Cid, error) {
84
}
85
86
toput = append(toput, b)
97
- toputcids = append(toputcids, c)
87
}
88
89
err := s.Blockstore.PutMany(toput)
@@ -108,8 +97,7 @@ func (s *BlockService) AddObjects(bs []Object) ([]*cid.Cid, error) {
97
return nil, fmt.Errorf("blockservice is closed (%s)", err)
98
}
99
111
- c := o.(Object).Cid() // cast is safe, we created these
112
- ks = append(ks, c)
100
+ ks = append(ks, o.Cid())
101
}
102
return ks, nil
103
}
@@ -119,7 +107,7 @@ func (s *BlockService) AddObjects(bs []Object) ([]*cid.Cid, error) {
107
func (s *BlockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) {
108
log.Debugf("BlockService GetBlock: '%s'", c)
109
122
- block, err := s.Blockstore.Get(key.Key(c.Hash()))
110
+ block, err := s.Blockstore.Get(c)
111
if err == nil {
112
return block, nil
113
}
@@ -128,7 +116,7 @@ func (s *BlockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block,
116
// TODO be careful checking ErrNotFound. If the underlying
117
// implementation changes, this will break.
118
log.Debug("Blockservice: Searching bitswap")
131
- blk, err := s.Exchange.GetBlock(ctx, key.Key(c.Hash()))
119
+ blk, err := s.Exchange.GetBlock(ctx, c)
120
if err != nil {
121
if err == blockstore.ErrNotFound {
122
return nil, ErrNotFound
@@ -153,12 +141,11 @@ func (s *BlockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan bloc
141
out := make(chan blocks.Block, 0)
142
go func() {
143
defer close(out)
156
- var misses []key.Key
144
+ var misses []*cid.Cid
145
for _, c := range ks {
158
- k := key.Key(c.Hash())
159
- hit, err := s.Blockstore.Get(k)
146
+ hit, err := s.Blockstore.Get(c)
147
if err != nil {
161
- misses = append(misses, k)
148
+ misses = append(misses, c)
149
continue
150
}
151
log.Debug("Blockservice: Got data in datastore")
@@ -191,8 +178,8 @@ func (s *BlockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan bloc
178
}
179
180
// DeleteBlock deletes a block in the blockservice from the datastore
194
-func (s *BlockService) DeleteObject(o Object) error {
195
- return s.Blockstore.DeleteBlock(o.Key())
181
+func (s *BlockService) DeleteBlock(o blocks.Block) error {
182
+ return s.Blockstore.DeleteBlock(o.Cid())
183
}
184
185
func (s *BlockService) Close() error {
blockservice/test/blocks_test.go
+8
-9
@@ -2,6 +2,7 @@ package bstest
2
3
import (
4
"bytes"
5
+ "context"
6
"fmt"
7
"testing"
8
"time"
@@ -10,9 +11,7 @@ import (
11
blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
12
. "github.com/ipfs/go-ipfs/blockservice"
13
offline "github.com/ipfs/go-ipfs/exchange/offline"
13
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
14
15
- "context"
15
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
16
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
17
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
@@ -44,11 +43,11 @@ func TestBlocks(t *testing.T) {
43
t.Error("Block Multihash and data multihash not equal")
44
}
45
47
- if o.Key() != key.Key(h) {
46
+ if !o.Cid().Equals(cid.NewCidV0(h)) {
47
t.Error("Block key and data multihash key not equal")
48
}
49
51
- k, err := bs.AddObject(o)
50
+ k, err := bs.AddBlock(o)
51
if err != nil {
52
t.Error("failed to add block to BlockService", err)
53
return
@@ -66,7 +65,7 @@ func TestBlocks(t *testing.T) {
65
return
66
}
67
69
- if o.Key() != b2.Key() {
68
+ if !o.Cid().Equals(b2.Cid()) {
69
t.Error("Block keys not equal.")
70
}
71
@@ -93,7 +92,7 @@ func TestGetBlocksSequential(t *testing.T) {
92
var cids []*cid.Cid
93
for _, o := range objs {
94
cids = append(cids, o.Cid())
96
- servs[0].AddObject(o)
95
+ servs[0].AddBlock(o)
96
}
97
98
t.Log("one instance at a time, get blocks concurrently")
@@ -102,12 +101,12 @@ func TestGetBlocksSequential(t *testing.T) {
101
ctx, cancel := context.WithTimeout(context.Background(), time.Second*50)
102
defer cancel()
103
out := servs[i].GetBlocks(ctx, cids)
105
- gotten := make(map[key.Key]blocks.Block)
104
+ gotten := make(map[string]blocks.Block)
105
for blk := range out {
107
- if _, ok := gotten[blk.Key()]; ok {
106
+ if _, ok := gotten[blk.Cid().KeyString()]; ok {
107
t.Fatal("Got duplicate block!")
108
}
110
- gotten[blk.Key()] = blk
109
+ gotten[blk.Cid().KeyString()] = blk
110
}
111
if len(gotten) != len(objs) {
112
t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs))
core/commands/bitswap.go
+3
-4
@@ -8,7 +8,6 @@ import (
8
cmds "github.com/ipfs/go-ipfs/commands"
9
bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
10
decision "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
11
12
"gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
13
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
@@ -54,7 +53,7 @@ var unwantCmd = &cmds.Command{
53
return
54
}
55
57
- var ks []key.Key
56
+ var ks []*cid.Cid
57
for _, arg := range req.Arguments() {
58
c, err := cid.Decode(arg)
59
if err != nil {
@@ -62,7 +61,7 @@ var unwantCmd = &cmds.Command{
61
return
62
}
63
65
- ks = append(ks, key.Key(c.Hash()))
64
+ ks = append(ks, c)
65
}
66
67
bs.CancelWants(ks)
@@ -164,7 +163,7 @@ var bitswapStatCmd = &cmds.Command{
163
fmt.Fprintf(buf, "\tdup data received: %s\n", humanize.Bytes(out.DupDataReceived))
164
fmt.Fprintf(buf, "\twantlist [%d keys]\n", len(out.Wantlist))
165
for _, k := range out.Wantlist {
167
- fmt.Fprintf(buf, "\t\t%s\n", k.B58String())
166
+ fmt.Fprintf(buf, "\t\t%s\n", k.String())
167
}
168
fmt.Fprintf(buf, "\tpartners [%d]\n", len(out.Peers))
169
for _, p := range out.Peers {
core/commands/block.go
+4
-4
@@ -66,7 +66,7 @@ on raw ipfs blocks. It outputs the following to stdout:
66
}
67
68
res.SetOutput(&BlockStat{
69
- Key: b.Key().B58String(),
69
+ Key: b.Cid().String(),
70
Size: len(b.RawData()),
71
})
72
},
@@ -140,9 +140,9 @@ It reads from stdin, and <key> is a base58 encoded multihash.
140
}
141
142
b := blocks.NewBlock(data)
143
- log.Debugf("BlockPut key: '%q'", b.Key())
143
+ log.Debugf("BlockPut key: '%q'", b.Cid())
144
145
- k, err := n.Blocks.AddObject(b)
145
+ k, err := n.Blocks.AddBlock(b)
146
if err != nil {
147
res.SetError(err, cmds.ErrNormal)
148
return
@@ -182,7 +182,7 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
182
return nil, err
183
}
184
185
- log.Debugf("ipfs block: got block with key: %q", b.Key())
185
+ log.Debugf("ipfs block: got block with key: %s", b.Cid())
186
return b, nil
187
}
188
core/commands/dht.go
+1
-2
@@ -17,7 +17,6 @@ import (
17
routing "gx/ipfs/QmXKuGUzLcgoQvp8M6ZEJzupWUNmx8NoqXEbYLMDjL4rjj/go-libp2p-routing"
18
notif "gx/ipfs/QmXKuGUzLcgoQvp8M6ZEJzupWUNmx8NoqXEbYLMDjL4rjj/go-libp2p-routing/notifications"
19
pstore "gx/ipfs/QmXXCcQ7CLg5a81Ui9TTR35QcR4y7ZyihxwfjqaHfUVcVo/go-libp2p-peerstore"
20
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
20
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
21
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
22
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
@@ -276,7 +275,7 @@ var provideRefDhtCmd = &cmds.Command{
275
return
276
}
277
279
- has, err := n.Blockstore.Has(key.Key(c.Hash()))
278
+ has, err := n.Blockstore.Has(c)
279
if err != nil {
280
res.SetError(err, cmds.ErrNormal)
281
return
core/commands/files/files.go
+6
-1
@@ -609,7 +609,12 @@ stat' on the file or any of its ancestors.
609
return
610
}
611
612
- defer wfd.Close()
612
+ defer func() {
613
+ err := wfd.Close()
614
+ if err != nil {
615
+ res.SetError(err, cmds.ErrNormal)
616
+ }
617
+ }()
618
619
if trunc {
620
if err := wfd.Truncate(0); err != nil {
core/commands/ls.go
+3
-2
@@ -12,7 +12,8 @@ import (
12
path "github.com/ipfs/go-ipfs/path"
13
unixfs "github.com/ipfs/go-ipfs/unixfs"
14
unixfspb "github.com/ipfs/go-ipfs/unixfs/pb"
15
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
15
+
16
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
17
)
18
19
type LsLink struct {
@@ -90,7 +91,7 @@ The JSON output contains type information.
91
for j, link := range dagnode.Links {
92
var linkNode *merkledag.Node
93
t := unixfspb.Data_DataType(-1)
93
- linkKey := key.Key(link.Hash)
94
+ linkKey := cid.NewCidV0(link.Hash)
95
if ok, err := node.Blockstore.Has(linkKey); ok && err == nil {
96
b, err := node.Blockstore.Get(linkKey)
97
if err != nil {
core/commands/pubsub.go
+1
-1
@@ -106,7 +106,7 @@ To use, the daemon must be run with '--enable-pubsub-experiment'.
106
if discover {
107
go func() {
108
blk := blocks.NewBlock([]byte("floodsub:" + topic))
109
- cid, err := n.Blocks.AddObject(blk)
109
+ cid, err := n.Blocks.AddBlock(blk)
110
if err != nil {
111
log.Error("pubsub discovery: ", err)
112
return
core/commands/refs.go
+4
-5
@@ -2,6 +2,7 @@ package commands
2
3
import (
4
"bytes"
5
+ "context"
6
"errors"
7
"io"
8
"strings"
@@ -10,16 +11,14 @@ import (
11
"github.com/ipfs/go-ipfs/core"
12
dag "github.com/ipfs/go-ipfs/merkledag"
13
path "github.com/ipfs/go-ipfs/path"
13
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
14
15
- context "context"
15
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
16
u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
17
)
18
19
// KeyList is a general type for outputting lists of keys
20
type KeyList struct {
22
- Keys []key.Key
21
+ Keys []*cid.Cid
22
}
23
24
// KeyListTextMarshaler outputs a KeyList as plaintext, one key per line
@@ -27,7 +26,7 @@ func KeyListTextMarshaler(res cmds.Response) (io.Reader, error) {
26
output := res.Output().(*KeyList)
27
buf := new(bytes.Buffer)
28
for _, key := range output.Keys {
30
- buf.WriteString(key.B58String() + "\n")
29
+ buf.WriteString(key.String() + "\n")
30
}
31
return buf, nil
32
}
@@ -160,7 +159,7 @@ Displays the hashes of all local objects.
159
defer close(out)
160
161
for k := range allKeys {
163
- out <- &RefWrapper{Ref: k.B58String()}
162
+ out <- &RefWrapper{Ref: k.String()}
163
}
164
}()
165
},
core/commands/repo.go
+1
-1
@@ -95,7 +95,7 @@ order to reclaim hard disk space.
95
96
buf := new(bytes.Buffer)
97
if quiet {
98
- buf = bytes.NewBufferString(string(obj.Key) + "\n")
98
+ buf = bytes.NewBufferString(obj.Key.String() + "\n")
99
} else {
100
buf = bytes.NewBufferString(fmt.Sprintf("removed %s\n", obj.Key))
101
}
core/corerepo/gc.go
+2
-3
@@ -1,6 +1,7 @@
1
package corerepo
2
3
import (
4
+ "context"
5
"errors"
6
"time"
7
@@ -8,9 +9,7 @@ import (
9
mfs "github.com/ipfs/go-ipfs/mfs"
10
gc "github.com/ipfs/go-ipfs/pin/gc"
11
repo "github.com/ipfs/go-ipfs/repo"
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
12
13
- context "context"
13
humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
15
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
@@ -21,7 +20,7 @@ var log = logging.Logger("corerepo")
20
var ErrMaxStorageExceeded = errors.New("Maximum storage limit exceeded. Maybe unpin some files?")
21
22
type KeyRemoved struct {
24
- Key key.Key
23
+ Key *cid.Cid
24
}
25
26
type GC struct {
core/coreunix/add_test.go
+2
-3
@@ -14,7 +14,6 @@ import (
14
"github.com/ipfs/go-ipfs/repo"
15
"github.com/ipfs/go-ipfs/repo/config"
16
"github.com/ipfs/go-ipfs/thirdparty/testutil"
17
- "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
17
18
"context"
19
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
@@ -94,7 +93,7 @@ func TestAddGCLive(t *testing.T) {
93
t.Fatal("add shouldnt complete yet")
94
}
95
97
- var gcout <-chan key.Key
96
+ var gcout <-chan *cid.Cid
97
gcstarted := make(chan struct{})
98
go func() {
99
defer close(gcstarted)
@@ -139,7 +138,7 @@ func TestAddGCLive(t *testing.T) {
138
}
139
140
for k := range gcout {
142
- if _, ok := addedHashes[k.B58String()]; ok {
141
+ if _, ok := addedHashes[k.String()]; ok {
142
t.Fatal("gc'ed a hash we just added")
143
}
144
}
exchange/bitswap/bitswap.go
+37
-41
@@ -3,13 +3,12 @@
3
package bitswap
4
5
import (
6
+ "context"
7
"errors"
8
"math"
9
"sync"
10
"time"
11
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
12
-
12
blocks "github.com/ipfs/go-ipfs/blocks"
13
blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
14
exchange "github.com/ipfs/go-ipfs/exchange"
@@ -19,12 +18,12 @@ import (
18
notifications "github.com/ipfs/go-ipfs/exchange/bitswap/notifications"
19
flags "github.com/ipfs/go-ipfs/flags"
20
"github.com/ipfs/go-ipfs/thirdparty/delay"
22
- loggables "gx/ipfs/QmTMy4hVSY28DdwJ9kBz6y7q6MuioFzPcpM3Ma3aPjo1i3/go-libp2p-loggables"
21
24
- context "context"
22
process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess"
23
procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context"
24
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
25
+ loggables "gx/ipfs/QmTMy4hVSY28DdwJ9kBz6y7q6MuioFzPcpM3Ma3aPjo1i3/go-libp2p-loggables"
26
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
27
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
28
)
29
@@ -90,8 +89,8 @@ func New(parent context.Context, p peer.ID, network bsnet.BitSwapNetwork,
89
network: network,
90
findKeys: make(chan *blockRequest, sizeBatchRequestChan),
91
process: px,
93
- newBlocks: make(chan key.Key, HasBlockBufferSize),
94
- provideKeys: make(chan key.Key, provideKeysBufferSize),
92
+ newBlocks: make(chan *cid.Cid, HasBlockBufferSize),
93
+ provideKeys: make(chan *cid.Cid, provideKeysBufferSize),
94
wm: NewWantManager(ctx, network),
95
}
96
go bs.wm.Run()
@@ -137,9 +136,9 @@ type Bitswap struct {
136
137
process process.Process
138
140
- newBlocks chan key.Key
139
+ newBlocks chan *cid.Cid
140
142
- provideKeys chan key.Key
141
+ provideKeys chan *cid.Cid
142
143
counterLk sync.Mutex
144
blocksRecvd int
@@ -148,14 +147,15 @@ type Bitswap struct {
147
}
148
149
type blockRequest struct {
151
- Key key.Key
150
+ Cid *cid.Cid
151
Ctx context.Context
152
}
153
154
// GetBlock attempts to retrieve a particular block from peers within the
155
// deadline enforced by the context.
157
-func (bs *Bitswap) GetBlock(parent context.Context, k key.Key) (blocks.Block, error) {
158
- if k == "" {
156
+func (bs *Bitswap) GetBlock(parent context.Context, k *cid.Cid) (blocks.Block, error) {
157
+ if k == nil {
158
+ log.Error("nil cid in GetBlock")
159
return nil, blockstore.ErrNotFound
160
}
161
@@ -165,18 +165,17 @@ func (bs *Bitswap) GetBlock(parent context.Context, k key.Key) (blocks.Block, er
165
// functions called by this one. Otherwise those functions won't return
166
// when this context's cancel func is executed. This is difficult to
167
// enforce. May this comment keep you safe.
168
-
168
ctx, cancelFunc := context.WithCancel(parent)
169
170
ctx = logging.ContextWithLoggable(ctx, loggables.Uuid("GetBlockRequest"))
172
- log.Event(ctx, "Bitswap.GetBlockRequest.Start", &k)
173
- defer log.Event(ctx, "Bitswap.GetBlockRequest.End", &k)
171
+ log.Event(ctx, "Bitswap.GetBlockRequest.Start", k)
172
+ defer log.Event(ctx, "Bitswap.GetBlockRequest.End", k)
173
174
defer func() {
175
cancelFunc()
176
}()
177
179
- promise, err := bs.GetBlocks(ctx, []key.Key{k})
178
+ promise, err := bs.GetBlocks(ctx, []*cid.Cid{k})
179
if err != nil {
180
return nil, err
181
}
@@ -197,10 +196,10 @@ func (bs *Bitswap) GetBlock(parent context.Context, k key.Key) (blocks.Block, er
196
}
197
}
198
200
-func (bs *Bitswap) WantlistForPeer(p peer.ID) []key.Key {
201
- var out []key.Key
199
+func (bs *Bitswap) WantlistForPeer(p peer.ID) []*cid.Cid {
200
+ var out []*cid.Cid
201
for _, e := range bs.engine.WantlistForPeer(p) {
203
- out = append(out, e.Key)
202
+ out = append(out, e.Cid)
203
}
204
return out
205
}
@@ -216,7 +215,7 @@ func (bs *Bitswap) LedgerForPeer(p peer.ID) *decision.Receipt {
215
// NB: Your request remains open until the context expires. To conserve
216
// resources, provide a context with a reasonably short deadline (ie. not one
217
// that lasts throughout the lifetime of the server)
219
-func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks.Block, error) {
218
+func (bs *Bitswap) GetBlocks(ctx context.Context, keys []*cid.Cid) (<-chan blocks.Block, error) {
219
if len(keys) == 0 {
220
out := make(chan blocks.Block)
221
close(out)
@@ -231,7 +230,7 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
230
promise := bs.notifications.Subscribe(ctx, keys...)
231
232
for _, k := range keys {
234
- log.Event(ctx, "Bitswap.GetBlockRequest.Start", &k)
233
+ log.Event(ctx, "Bitswap.GetBlockRequest.Start", k)
234
}
235
236
bs.wm.WantBlocks(ctx, keys)
@@ -240,13 +239,13 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
239
// be able to provide for all keys. This currently holds true in most
240
// every situation. Later, this assumption may not hold as true.
241
req := &blockRequest{
243
- Key: keys[0],
242
+ Cid: keys[0],
243
Ctx: ctx,
244
}
245
247
- remaining := make(map[key.Key]struct{})
246
+ remaining := cid.NewSet()
247
for _, k := range keys {
249
- remaining[k] = struct{}{}
248
+ remaining.Add(k)
249
}
250
251
out := make(chan blocks.Block)
@@ -255,11 +254,8 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
254
defer cancel()
255
defer close(out)
256
defer func() {
258
- var toCancel []key.Key
259
- for k, _ := range remaining {
260
- toCancel = append(toCancel, k)
261
- }
262
- bs.CancelWants(toCancel)
257
+ // can't just defer this call on its own, arguments are resolved *when* the defer is created
258
+ bs.CancelWants(remaining.Keys())
259
}()
260
for {
261
select {
@@ -268,7 +264,7 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
264
return
265
}
266
271
- delete(remaining, blk.Key())
267
+ remaining.Remove(blk.Cid())
268
select {
269
case out <- blk:
270
case <-ctx.Done():
@@ -289,8 +285,8 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
285
}
286
287
// CancelWant removes a given key from the wantlist
292
-func (bs *Bitswap) CancelWants(keys []key.Key) {
293
- bs.wm.CancelWants(keys)
288
+func (bs *Bitswap) CancelWants(cids []*cid.Cid) {
289
+ bs.wm.CancelWants(cids)
290
}
291
292
// HasBlock announces the existance of a block to this bitswap service. The
@@ -318,7 +314,7 @@ func (bs *Bitswap) HasBlock(blk blocks.Block) error {
314
bs.engine.AddBlock(blk)
315
316
select {
321
- case bs.newBlocks <- blk.Key():
317
+ case bs.newBlocks <- blk.Cid():
318
// send block off to be reprovided
319
case <-bs.process.Closing():
320
return bs.process.Close()
@@ -340,13 +336,13 @@ func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg
336
}
337
338
// quickly send out cancels, reduces chances of duplicate block receives
343
- var keys []key.Key
339
+ var keys []*cid.Cid
340
for _, block := range iblocks {
345
- if _, found := bs.wm.wl.Contains(block.Key()); !found {
341
+ if _, found := bs.wm.wl.Contains(block.Cid()); !found {
342
log.Infof("received un-asked-for %s from %s", block, p)
343
continue
344
}
349
- keys = append(keys, block.Key())
345
+ keys = append(keys, block.Cid())
346
}
347
bs.wm.CancelWants(keys)
348
@@ -360,8 +356,8 @@ func (bs *Bitswap) ReceiveMessage(ctx context.Context, p peer.ID, incoming bsmsg
356
return // ignore error, is either logged previously, or ErrAlreadyHaveBlock
357
}
358
363
- k := b.Key()
364
- log.Event(ctx, "Bitswap.GetBlockRequest.End", &k)
359
+ k := b.Cid()
360
+ log.Event(ctx, "Bitswap.GetBlockRequest.End", k)
361
362
log.Debugf("got block %s from %s", b, p)
363
if err := bs.HasBlock(b); err != nil {
@@ -378,7 +374,7 @@ func (bs *Bitswap) updateReceiveCounters(b blocks.Block) error {
374
bs.counterLk.Lock()
375
defer bs.counterLk.Unlock()
376
bs.blocksRecvd++
381
- has, err := bs.blockstore.Has(b.Key())
377
+ has, err := bs.blockstore.Has(b.Cid())
378
if err != nil {
379
log.Infof("blockstore.Has error: %s", err)
380
return err
@@ -415,10 +411,10 @@ func (bs *Bitswap) Close() error {
411
return bs.process.Close()
412
}
413
418
-func (bs *Bitswap) GetWantlist() []key.Key {
419
- var out []key.Key
414
+func (bs *Bitswap) GetWantlist() []*cid.Cid {
415
+ var out []*cid.Cid
416
for _, e := range bs.wm.wl.Entries() {
421
- out = append(out, e.Key)
417
+ out = append(out, e.Cid)
418
}
419
return out
420
}
exchange/bitswap/bitswap_test.go
+21
-21
@@ -2,21 +2,21 @@ package bitswap
2
3
import (
4
"bytes"
5
+ "context"
6
"sync"
7
"testing"
8
"time"
9
9
- context "context"
10
- detectrace "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-detect-race"
11
- travis "github.com/ipfs/go-ipfs/thirdparty/testutil/ci/travis"
12
-
10
blocks "github.com/ipfs/go-ipfs/blocks"
11
blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
12
blocksutil "github.com/ipfs/go-ipfs/blocks/blocksutil"
13
tn "github.com/ipfs/go-ipfs/exchange/bitswap/testnet"
14
mockrouting "github.com/ipfs/go-ipfs/routing/mock"
15
delay "github.com/ipfs/go-ipfs/thirdparty/delay"
19
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
16
+ travis "github.com/ipfs/go-ipfs/thirdparty/testutil/ci/travis"
17
+
18
+ detectrace "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/jbenet/go-detect-race"
19
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
20
p2ptestutil "gx/ipfs/QmcRa2qn6iCmap9bjp8jAwkvYAq13AUfxdY3rrYiaJbLum/go-libp2p/p2p/test/util"
21
)
22
@@ -38,7 +38,7 @@ func TestClose(t *testing.T) {
38
bitswap := sesgen.Next()
39
40
bitswap.Exchange.Close()
41
- bitswap.Exchange.GetBlock(context.Background(), block.Key())
41
+ bitswap.Exchange.GetBlock(context.Background(), block.Cid())
42
}
43
44
func TestProviderForKeyButNetworkCannotFind(t *testing.T) { // TODO revisit this
@@ -57,7 +57,7 @@ func TestProviderForKeyButNetworkCannotFind(t *testing.T) { // TODO revisit this
57
58
ctx, cancel := context.WithTimeout(context.Background(), time.Nanosecond)
59
defer cancel()
60
- _, err := solo.Exchange.GetBlock(ctx, block.Key())
60
+ _, err := solo.Exchange.GetBlock(ctx, block.Cid())
61
62
if err != context.DeadlineExceeded {
63
t.Fatal("Expected DeadlineExceeded error")
@@ -84,7 +84,7 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {
84
85
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
86
defer cancel()
87
- received, err := wantsBlock.Exchange.GetBlock(ctx, block.Key())
87
+ received, err := wantsBlock.Exchange.GetBlock(ctx, block.Cid())
88
if err != nil {
89
t.Log(err)
90
t.Fatal("Expected to succeed")
@@ -176,10 +176,10 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
176
}
177
}
178
179
- var blkeys []key.Key
179
+ var blkeys []*cid.Cid
180
first := instances[0]
181
for _, b := range blocks {
182
- blkeys = append(blkeys, b.Key())
182
+ blkeys = append(blkeys, b.Cid())
183
first.Exchange.HasBlock(b)
184
}
185
@@ -216,7 +216,7 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
216
217
for _, inst := range instances {
218
for _, b := range blocks {
219
- if _, err := inst.Blockstore().Get(b.Key()); err != nil {
219
+ if _, err := inst.Blockstore().Get(b.Cid()); err != nil {
220
t.Fatal(err)
221
}
222
}
@@ -224,8 +224,8 @@ func PerformDistributionTest(t *testing.T, numInstances, numBlocks int) {
224
}
225
226
func getOrFail(bitswap Instance, b blocks.Block, t *testing.T, wg *sync.WaitGroup) {
227
- if _, err := bitswap.Blockstore().Get(b.Key()); err != nil {
228
- _, err := bitswap.Exchange.GetBlock(context.Background(), b.Key())
227
+ if _, err := bitswap.Blockstore().Get(b.Cid()); err != nil {
228
+ _, err := bitswap.Exchange.GetBlock(context.Background(), b.Cid())
229
if err != nil {
230
t.Fatal(err)
231
}
@@ -260,7 +260,7 @@ func TestSendToWantingPeer(t *testing.T) {
260
// peerA requests and waits for block alpha
261
ctx, cancel := context.WithTimeout(context.Background(), waitTime)
262
defer cancel()
263
- alphaPromise, err := peerA.Exchange.GetBlocks(ctx, []key.Key{alpha.Key()})
263
+ alphaPromise, err := peerA.Exchange.GetBlocks(ctx, []*cid.Cid{alpha.Cid()})
264
if err != nil {
265
t.Fatal(err)
266
}
@@ -277,7 +277,7 @@ func TestSendToWantingPeer(t *testing.T) {
277
t.Fatal("context timed out and broke promise channel!")
278
}
279
280
- if blkrecvd.Key() != alpha.Key() {
280
+ if !blkrecvd.Cid().Equals(alpha.Cid()) {
281
t.Fatal("Wrong block!")
282
}
283
@@ -292,7 +292,7 @@ func TestEmptyKey(t *testing.T) {
292
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
293
defer cancel()
294
295
- _, err := bs.GetBlock(ctx, key.Key(""))
295
+ _, err := bs.GetBlock(ctx, nil)
296
if err != blockstore.ErrNotFound {
297
t.Error("empty str key should return ErrNotFound")
298
}
@@ -315,7 +315,7 @@ func TestBasicBitswap(t *testing.T) {
315
316
ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
317
defer cancel()
318
- blk, err := instances[1].Exchange.GetBlock(ctx, blocks[0].Key())
318
+ blk, err := instances[1].Exchange.GetBlock(ctx, blocks[0].Cid())
319
if err != nil {
320
t.Fatal(err)
321
}
@@ -341,7 +341,7 @@ func TestDoubleGet(t *testing.T) {
341
blocks := bg.Blocks(1)
342
343
ctx1, cancel1 := context.WithCancel(context.Background())
344
- blkch1, err := instances[1].Exchange.GetBlocks(ctx1, []key.Key{blocks[0].Key()})
344
+ blkch1, err := instances[1].Exchange.GetBlocks(ctx1, []*cid.Cid{blocks[0].Cid()})
345
if err != nil {
346
t.Fatal(err)
347
}
@@ -349,7 +349,7 @@ func TestDoubleGet(t *testing.T) {
349
ctx2, cancel2 := context.WithCancel(context.Background())
350
defer cancel2()
351
352
- blkch2, err := instances[1].Exchange.GetBlocks(ctx2, []key.Key{blocks[0].Key()})
352
+ blkch2, err := instances[1].Exchange.GetBlocks(ctx2, []*cid.Cid{blocks[0].Cid()})
353
if err != nil {
354
t.Fatal(err)
355
}
@@ -396,9 +396,9 @@ func TestWantlistCleanup(t *testing.T) {
396
bswap := instances.Exchange
397
blocks := bg.Blocks(20)
398
399
- var keys []key.Key
399
+ var keys []*cid.Cid
400
for _, b := range blocks {
401
- keys = append(keys, b.Key())
401
+ keys = append(keys, b.Cid())
402
}
403
404
ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*50)
exchange/bitswap/decision/bench_test.go
+6
-2
@@ -1,12 +1,14 @@
1
package decision
2
3
import (
4
+ "fmt"
5
"math"
6
"testing"
7
8
"github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
9
"github.com/ipfs/go-ipfs/thirdparty/testutil"
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
10
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
11
+ u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
12
"gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
13
)
14
@@ -21,6 +23,8 @@ func BenchmarkTaskQueuePush(b *testing.B) {
23
}
24
b.ResetTimer()
25
for i := 0; i < b.N; i++ {
24
- q.Push(&wantlist.Entry{Key: key.Key(i), Priority: math.MaxInt32}, peers[i%len(peers)])
26
+ c := cid.NewCidV0(u.Hash([]byte(fmt.Sprint(i))))
27
+
28
+ q.Push(&wantlist.Entry{Cid: c, Priority: math.MaxInt32}, peers[i%len(peers)])
29
}
30
}
exchange/bitswap/decision/engine.go
+11
-10
@@ -169,8 +169,9 @@ func (e *Engine) nextEnvelope(ctx context.Context) (*Envelope, error) {
169
170
// with a task in hand, we're ready to prepare the envelope...
171
172
- block, err := e.bs.Get(nextTask.Entry.Key)
172
+ block, err := e.bs.Get(nextTask.Entry.Cid)
173
if err != nil {
174
+ log.Errorf("tried to execute a task and errored fetching block: %s", err)
175
// If we don't have the block, don't hold that against the peer
176
// make sure to update that the task has been 'completed'
177
nextTask.Done()
@@ -233,13 +234,13 @@ func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error {
234
235
for _, entry := range m.Wantlist() {
236
if entry.Cancel {
236
- log.Debugf("%s cancel %s", p, entry.Key)
237
- l.CancelWant(entry.Key)
238
- e.peerRequestQueue.Remove(entry.Key, p)
237
+ log.Debugf("%s cancel %s", p, entry.Cid)
238
+ l.CancelWant(entry.Cid)
239
+ e.peerRequestQueue.Remove(entry.Cid, p)
240
} else {
240
- log.Debugf("wants %s - %d", entry.Key, entry.Priority)
241
- l.Wants(entry.Key, entry.Priority)
242
- if exists, err := e.bs.Has(entry.Key); err == nil && exists {
241
+ log.Debugf("wants %s - %d", entry.Cid, entry.Priority)
242
+ l.Wants(entry.Cid, entry.Priority)
243
+ if exists, err := e.bs.Has(entry.Cid); err == nil && exists {
244
e.peerRequestQueue.Push(entry.Entry, p)
245
newWorkExists = true
246
}
@@ -258,7 +259,7 @@ func (e *Engine) addBlock(block blocks.Block) {
259
260
for _, l := range e.ledgerMap {
261
l.lk.Lock()
261
- if entry, ok := l.WantListContains(block.Key()); ok {
262
+ if entry, ok := l.WantListContains(block.Cid()); ok {
263
e.peerRequestQueue.Push(entry, l.Partner)
264
work = true
265
}
@@ -287,8 +288,8 @@ func (e *Engine) MessageSent(p peer.ID, m bsmsg.BitSwapMessage) error {
288
l := e.findOrCreate(p)
289
for _, block := range m.Blocks() {
290
l.SentBytes(len(block.RawData()))
290
- l.wantList.Remove(block.Key())
291
- e.peerRequestQueue.Remove(block.Key(), p)
291
+ l.wantList.Remove(block.Cid())
292
+ e.peerRequestQueue.Remove(block.Cid(), p)
293
}
294
295
return nil
exchange/bitswap/decision/engine_test.go
+3
-3
@@ -167,7 +167,7 @@ func partnerWants(e *Engine, keys []string, partner peer.ID) {
167
add := message.New(false)
168
for i, letter := range keys {
169
block := blocks.NewBlock([]byte(letter))
170
- add.AddEntry(block.Key(), math.MaxInt32-i)
170
+ add.AddEntry(block.Cid(), math.MaxInt32-i)
171
}
172
e.MessageReceived(partner, add)
173
}
@@ -176,7 +176,7 @@ func partnerCancels(e *Engine, keys []string, partner peer.ID) {
176
cancels := message.New(false)
177
for _, k := range keys {
178
block := blocks.NewBlock([]byte(k))
179
- cancels.Cancel(block.Key())
179
+ cancels.Cancel(block.Cid())
180
}
181
e.MessageReceived(partner, cancels)
182
}
@@ -187,7 +187,7 @@ func checkHandledInOrder(t *testing.T, e *Engine, keys []string) error {
187
envelope := <-next
188
received := envelope.Block
189
expected := blocks.NewBlock([]byte(k))
190
- if received.Key() != expected.Key() {
190
+ if !received.Cid().Equals(expected.Cid()) {
191
return errors.New(fmt.Sprintln("received", string(received.RawData()), "expected", string(expected.RawData())))
192
}
193
}
exchange/bitswap/decision/ledger.go
+7
-10
@@ -5,19 +5,16 @@ import (
5
"time"
6
7
wl "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
8
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
8
+
9
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
10
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
11
)
12
12
-// keySet is just a convenient alias for maps of keys, where we only care
13
-// access/lookups.
14
-type keySet map[key.Key]struct{}
15
-
13
func newLedger(p peer.ID) *ledger {
14
return &ledger{
15
wantList: wl.New(),
16
Partner: p,
20
- sentToPeer: make(map[key.Key]time.Time),
17
+ sentToPeer: make(map[string]time.Time),
18
}
19
}
20
@@ -44,7 +41,7 @@ type ledger struct {
41
42
// sentToPeer is a set of keys to ensure we dont send duplicate blocks
43
// to a given peer
47
- sentToPeer map[key.Key]time.Time
44
+ sentToPeer map[string]time.Time
45
46
lk sync.Mutex
47
}
@@ -78,16 +75,16 @@ func (l *ledger) ReceivedBytes(n int) {
75
l.Accounting.BytesRecv += uint64(n)
76
}
77
81
-func (l *ledger) Wants(k key.Key, priority int) {
78
+func (l *ledger) Wants(k *cid.Cid, priority int) {
79
log.Debugf("peer %s wants %s", l.Partner, k)
80
l.wantList.Add(k, priority)
81
}
82
86
-func (l *ledger) CancelWant(k key.Key) {
83
+func (l *ledger) CancelWant(k *cid.Cid) {
84
l.wantList.Remove(k)
85
}
86
90
-func (l *ledger) WantListContains(k key.Key) (*wl.Entry, bool) {
87
+func (l *ledger) WantListContains(k *cid.Cid) (*wl.Entry, bool) {
88
return l.wantList.Contains(k)
89
}
90
exchange/bitswap/decision/peer_request_queue.go
+17
-17
@@ -6,7 +6,8 @@ import (
6
7
wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
8
pq "github.com/ipfs/go-ipfs/thirdparty/pq"
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
+
10
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
11
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
12
)
13
@@ -14,7 +15,7 @@ type peerRequestQueue interface {
15
// Pop returns the next peerRequestTask. Returns nil if the peerRequestQueue is empty.
16
Pop() *peerRequestTask
17
Push(entry *wantlist.Entry, to peer.ID)
17
- Remove(k key.Key, p peer.ID)
18
+ Remove(k *cid.Cid, p peer.ID)
19
20
// NB: cannot expose simply expose taskQueue.Len because trashed elements
21
// may exist. These trashed elements should not contribute to the count.
@@ -57,12 +58,11 @@ func (tl *prq) Push(entry *wantlist.Entry, to peer.ID) {
58
59
partner.activelk.Lock()
60
defer partner.activelk.Unlock()
60
- _, ok = partner.activeBlocks[entry.Key]
61
- if ok {
61
+ if partner.activeBlocks.Has(entry.Cid) {
62
return
63
}
64
65
- if task, ok := tl.taskMap[taskKey(to, entry.Key)]; ok {
65
+ if task, ok := tl.taskMap[taskKey(to, entry.Cid)]; ok {
66
task.Entry.Priority = entry.Priority
67
partner.taskQueue.Update(task.index)
68
return
@@ -74,7 +74,7 @@ func (tl *prq) Push(entry *wantlist.Entry, to peer.ID) {
74
created: time.Now(),
75
Done: func() {
76
tl.lock.Lock()
77
- partner.TaskDone(entry.Key)
77
+ partner.TaskDone(entry.Cid)
78
tl.pQueue.Update(partner.Index())
79
tl.lock.Unlock()
80
},
@@ -104,7 +104,7 @@ func (tl *prq) Pop() *peerRequestTask {
104
continue // discarding tasks that have been removed
105
}
106
107
- partner.StartTask(out.Entry.Key)
107
+ partner.StartTask(out.Entry.Cid)
108
partner.requests--
109
break // and return |out|
110
}
@@ -114,7 +114,7 @@ func (tl *prq) Pop() *peerRequestTask {
114
}
115
116
// Remove removes a task from the queue
117
-func (tl *prq) Remove(k key.Key, p peer.ID) {
117
+func (tl *prq) Remove(k *cid.Cid, p peer.ID) {
118
tl.lock.Lock()
119
t, ok := tl.taskMap[taskKey(p, k)]
120
if ok {
@@ -181,7 +181,7 @@ type peerRequestTask struct {
181
182
// Key uniquely identifies a task.
183
func (t *peerRequestTask) Key() string {
184
- return taskKey(t.Target, t.Entry.Key)
184
+ return taskKey(t.Target, t.Entry.Cid)
185
}
186
187
// Index implements pq.Elem
@@ -195,8 +195,8 @@ func (t *peerRequestTask) SetIndex(i int) {
195
}
196
197
// taskKey returns a key that uniquely identifies a task.
198
-func taskKey(p peer.ID, k key.Key) string {
199
- return string(p) + string(k)
198
+func taskKey(p peer.ID, k *cid.Cid) string {
199
+ return string(p) + k.KeyString()
200
}
201
202
// FIFO is a basic task comparator that returns tasks in the order created.
@@ -226,7 +226,7 @@ type activePartner struct {
226
activelk sync.Mutex
227
active int
228
229
- activeBlocks map[key.Key]struct{}
229
+ activeBlocks *cid.Set
230
231
// requests is the number of blocks this peer is currently requesting
232
// request need not be locked around as it will only be modified under
@@ -245,7 +245,7 @@ type activePartner struct {
245
func newActivePartner() *activePartner {
246
return &activePartner{
247
taskQueue: pq.New(wrapCmp(V1)),
248
- activeBlocks: make(map[key.Key]struct{}),
248
+ activeBlocks: cid.NewSet(),
249
}
250
}
251
@@ -281,17 +281,17 @@ func partnerCompare(a, b pq.Elem) bool {
281
}
282
283
// StartTask signals that a task was started for this partner
284
-func (p *activePartner) StartTask(k key.Key) {
284
+func (p *activePartner) StartTask(k *cid.Cid) {
285
p.activelk.Lock()
286
- p.activeBlocks[k] = struct{}{}
286
+ p.activeBlocks.Add(k)
287
p.active++
288
p.activelk.Unlock()
289
}
290
291
// TaskDone signals that a task was completed for this partner
292
-func (p *activePartner) TaskDone(k key.Key) {
292
+func (p *activePartner) TaskDone(k *cid.Cid) {
293
p.activelk.Lock()
294
- delete(p.activeBlocks, k)
294
+ p.activeBlocks.Remove(k)
295
p.active--
296
if p.active < 0 {
297
panic("more tasks finished than started!")
exchange/bitswap/decision/peer_request_queue_test.go
+16
-9
@@ -1,6 +1,7 @@
1
package decision
2
3
import (
4
+ "fmt"
5
"math"
6
"math/rand"
7
"sort"
@@ -9,7 +10,8 @@ import (
10
11
"github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
12
"github.com/ipfs/go-ipfs/thirdparty/testutil"
12
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
13
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
14
+ u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
15
)
16
17
func TestPushPop(t *testing.T) {
@@ -41,10 +43,13 @@ func TestPushPop(t *testing.T) {
43
for _, index := range rand.Perm(len(alphabet)) { // add blocks for all letters
44
letter := alphabet[index]
45
t.Log(partner.String())
44
- prq.Push(&wantlist.Entry{Key: key.Key(letter), Priority: math.MaxInt32 - index}, partner)
46
+
47
+ c := cid.NewCidV0(u.Hash([]byte(letter)))
48
+ prq.Push(&wantlist.Entry{Cid: c, Priority: math.MaxInt32 - index}, partner)
49
}
50
for _, consonant := range consonants {
47
- prq.Remove(key.Key(consonant), partner)
51
+ c := cid.NewCidV0(u.Hash([]byte(consonant)))
52
+ prq.Remove(c, partner)
53
}
54
55
prq.fullThaw()
@@ -56,12 +61,13 @@ func TestPushPop(t *testing.T) {
61
break
62
}
63
59
- out = append(out, string(received.Entry.Key))
64
+ out = append(out, received.Entry.Cid.String())
65
}
66
67
// Entries popped should already be in correct order
68
for i, expected := range vowels {
64
- if out[i] != expected {
69
+ exp := cid.NewCidV0(u.Hash([]byte(expected))).String()
70
+ if out[i] != exp {
71
t.Fatal("received", out[i], "expected", expected)
72
}
73
}
@@ -78,10 +84,11 @@ func TestPeerRepeats(t *testing.T) {
84
// Have each push some blocks
85
86
for i := 0; i < 5; i++ {
81
- prq.Push(&wantlist.Entry{Key: key.Key(i)}, a)
82
- prq.Push(&wantlist.Entry{Key: key.Key(i)}, b)
83
- prq.Push(&wantlist.Entry{Key: key.Key(i)}, c)
84
- prq.Push(&wantlist.Entry{Key: key.Key(i)}, d)
87
+ elcid := cid.NewCidV0(u.Hash([]byte(fmt.Sprint(i))))
88
+ prq.Push(&wantlist.Entry{Cid: elcid}, a)
89
+ prq.Push(&wantlist.Entry{Cid: elcid}, b)
90
+ prq.Push(&wantlist.Entry{Cid: elcid}, c)
91
+ prq.Push(&wantlist.Entry{Cid: elcid}, d)
92
}
93
94
// now, pop off four entries, there should be one from each
exchange/bitswap/message/message.go
+26
-21
@@ -1,16 +1,17 @@
1
package message
2
3
import (
4
+ "fmt"
5
"io"
6
7
blocks "github.com/ipfs/go-ipfs/blocks"
8
pb "github.com/ipfs/go-ipfs/exchange/bitswap/message/pb"
9
wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
10
- inet "gx/ipfs/QmdXimY9QHaasZmw6hWojWnCJvfgxETjZQfg9g6ZrA9wMX/go-libp2p-net"
10
11
ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
12
proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
13
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
14
+ inet "gx/ipfs/QmdXimY9QHaasZmw6hWojWnCJvfgxETjZQfg9g6ZrA9wMX/go-libp2p-net"
15
)
16
17
// TODO move message.go into the bitswap package
@@ -25,9 +26,9 @@ type BitSwapMessage interface {
26
Blocks() []blocks.Block
27
28
// AddEntry adds an entry to the Wantlist.
28
- AddEntry(key key.Key, priority int)
29
+ AddEntry(key *cid.Cid, priority int)
30
30
- Cancel(key key.Key)
31
+ Cancel(key *cid.Cid)
32
33
Empty() bool
34
@@ -47,8 +48,8 @@ type Exportable interface {
48
49
type impl struct {
50
full bool
50
- wantlist map[key.Key]Entry
51
- blocks map[key.Key]blocks.Block
51
+ wantlist map[string]Entry
52
+ blocks map[string]blocks.Block
53
}
54
55
func New(full bool) BitSwapMessage {
@@ -57,8 +58,8 @@ func New(full bool) BitSwapMessage {
58
59
func newMsg(full bool) *impl {
60
return &impl{
60
- blocks: make(map[key.Key]blocks.Block),
61
- wantlist: make(map[key.Key]Entry),
61
+ blocks: make(map[string]blocks.Block),
62
+ wantlist: make(map[string]Entry),
63
full: full,
64
}
65
}
@@ -68,16 +69,20 @@ type Entry struct {
69
Cancel bool
70
}
71
71
-func newMessageFromProto(pbm pb.Message) BitSwapMessage {
72
+func newMessageFromProto(pbm pb.Message) (BitSwapMessage, error) {
73
m := newMsg(pbm.GetWantlist().GetFull())
74
for _, e := range pbm.GetWantlist().GetEntries() {
74
- m.addEntry(key.Key(e.GetBlock()), int(e.GetPriority()), e.GetCancel())
75
+ c, err := cid.Cast([]byte(e.GetBlock()))
76
+ if err != nil {
77
+ return nil, fmt.Errorf("incorrectly formatted cid in wantlist: %s", err)
78
+ }
79
+ m.addEntry(c, int(e.GetPriority()), e.GetCancel())
80
}
81
for _, d := range pbm.GetBlocks() {
82
b := blocks.NewBlock(d)
83
m.AddBlock(b)
84
}
80
- return m
85
+ return m, nil
86
}
87
88
func (m *impl) Full() bool {
@@ -104,16 +109,17 @@ func (m *impl) Blocks() []blocks.Block {
109
return bs
110
}
111
107
-func (m *impl) Cancel(k key.Key) {
108
- delete(m.wantlist, k)
112
+func (m *impl) Cancel(k *cid.Cid) {
113
+ delete(m.wantlist, k.KeyString())
114
m.addEntry(k, 0, true)
115
}
116
112
-func (m *impl) AddEntry(k key.Key, priority int) {
117
+func (m *impl) AddEntry(k *cid.Cid, priority int) {
118
m.addEntry(k, priority, false)
119
}
120
116
-func (m *impl) addEntry(k key.Key, priority int, cancel bool) {
121
+func (m *impl) addEntry(c *cid.Cid, priority int, cancel bool) {
122
+ k := c.KeyString()
123
e, exists := m.wantlist[k]
124
if exists {
125
e.Priority = priority
@@ -121,7 +127,7 @@ func (m *impl) addEntry(k key.Key, priority int, cancel bool) {
127
} else {
128
m.wantlist[k] = Entry{
129
Entry: &wantlist.Entry{
124
- Key: k,
130
+ Cid: c,
131
Priority: priority,
132
},
133
Cancel: cancel,
@@ -130,7 +136,7 @@ func (m *impl) addEntry(k key.Key, priority int, cancel bool) {
136
}
137
138
func (m *impl) AddBlock(b blocks.Block) {
133
- m.blocks[b.Key()] = b
139
+ m.blocks[b.Cid().KeyString()] = b
140
}
141
142
func FromNet(r io.Reader) (BitSwapMessage, error) {
@@ -144,8 +150,7 @@ func FromPBReader(pbr ggio.Reader) (BitSwapMessage, error) {
150
return nil, err
151
}
152
147
- m := newMessageFromProto(*pb)
148
- return m, nil
153
+ return newMessageFromProto(*pb)
154
}
155
156
func (m *impl) ToProto() *pb.Message {
@@ -153,7 +158,7 @@ func (m *impl) ToProto() *pb.Message {
158
pbm.Wantlist = new(pb.Message_Wantlist)
159
for _, e := range m.wantlist {
160
pbm.Wantlist.Entries = append(pbm.Wantlist.Entries, &pb.Message_Wantlist_Entry{
156
- Block: proto.String(string(e.Key)),
161
+ Block: proto.String(e.Cid.KeyString()),
162
Priority: proto.Int32(int32(e.Priority)),
163
Cancel: proto.Bool(e.Cancel),
164
})
@@ -176,7 +181,7 @@ func (m *impl) ToNet(w io.Writer) error {
181
func (m *impl) Loggable() map[string]interface{} {
182
var blocks []string
183
for _, v := range m.blocks {
179
- blocks = append(blocks, v.Key().B58String())
184
+ blocks = append(blocks, v.Cid().String())
185
}
186
return map[string]interface{}{
187
"blocks": blocks,
exchange/bitswap/message/message_test.go
+36
-27
@@ -8,13 +8,18 @@ import (
8
9
blocks "github.com/ipfs/go-ipfs/blocks"
10
pb "github.com/ipfs/go-ipfs/exchange/bitswap/message/pb"
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
11
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
+ u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
13
)
14
15
+func mkFakeCid(s string) *cid.Cid {
16
+ return cid.NewCidV0(u.Hash([]byte(s)))
17
+}
18
+
19
func TestAppendWanted(t *testing.T) {
15
- const str = "foo"
20
+ str := mkFakeCid("foo")
21
m := New(true)
17
- m.AddEntry(key.Key(str), 1)
22
+ m.AddEntry(str, 1)
23
24
if !wantlistContains(m.ToProto().GetWantlist(), str) {
25
t.Fail()
@@ -23,16 +28,20 @@ func TestAppendWanted(t *testing.T) {
28
}
29
30
func TestNewMessageFromProto(t *testing.T) {
26
- const str = "a_key"
31
+ str := mkFakeCid("a_key")
32
protoMessage := new(pb.Message)
33
protoMessage.Wantlist = new(pb.Message_Wantlist)
34
protoMessage.Wantlist.Entries = []*pb.Message_Wantlist_Entry{
30
- {Block: proto.String(str)},
35
+ {Block: proto.String(str.KeyString())},
36
}
37
if !wantlistContains(protoMessage.Wantlist, str) {
38
t.Fail()
39
}
35
- m := newMessageFromProto(*protoMessage)
40
+ m, err := newMessageFromProto(*protoMessage)
41
+ if err != nil {
42
+ t.Fatal(err)
43
+ }
44
+
45
if !wantlistContains(m.ToProto().GetWantlist(), str) {
46
t.Fail()
47
}
@@ -60,10 +69,10 @@ func TestAppendBlock(t *testing.T) {
69
}
70
71
func TestWantlist(t *testing.T) {
63
- keystrs := []string{"foo", "bar", "baz", "bat"}
72
+ keystrs := []*cid.Cid{mkFakeCid("foo"), mkFakeCid("bar"), mkFakeCid("baz"), mkFakeCid("bat")}
73
m := New(true)
74
for _, s := range keystrs {
66
- m.AddEntry(key.Key(s), 1)
75
+ m.AddEntry(s, 1)
76
}
77
exported := m.Wantlist()
78
@@ -71,22 +80,22 @@ func TestWantlist(t *testing.T) {
80
present := false
81
for _, s := range keystrs {
82
74
- if s == string(k.Key) {
83
+ if s.Equals(k.Cid) {
84
present = true
85
}
86
}
87
if !present {
79
- t.Logf("%v isn't in original list", k.Key)
88
+ t.Logf("%v isn't in original list", k.Cid)
89
t.Fail()
90
}
91
}
92
}
93
94
func TestCopyProtoByValue(t *testing.T) {
86
- const str = "foo"
95
+ str := mkFakeCid("foo")
96
m := New(true)
97
protoBeforeAppend := m.ToProto()
89
- m.AddEntry(key.Key(str), 1)
98
+ m.AddEntry(str, 1)
99
if wantlistContains(protoBeforeAppend.GetWantlist(), str) {
100
t.Fail()
101
}
@@ -94,11 +103,11 @@ func TestCopyProtoByValue(t *testing.T) {
103
104
func TestToNetFromNetPreservesWantList(t *testing.T) {
105
original := New(true)
97
- original.AddEntry(key.Key("M"), 1)
98
- original.AddEntry(key.Key("B"), 1)
99
- original.AddEntry(key.Key("D"), 1)
100
- original.AddEntry(key.Key("T"), 1)
101
- original.AddEntry(key.Key("F"), 1)
106
+ original.AddEntry(mkFakeCid("M"), 1)
107
+ original.AddEntry(mkFakeCid("B"), 1)
108
+ original.AddEntry(mkFakeCid("D"), 1)
109
+ original.AddEntry(mkFakeCid("T"), 1)
110
+ original.AddEntry(mkFakeCid("F"), 1)
111
112
buf := new(bytes.Buffer)
113
if err := original.ToNet(buf); err != nil {
@@ -110,13 +119,13 @@ func TestToNetFromNetPreservesWantList(t *testing.T) {
119
t.Fatal(err)
120
}
121
113
- keys := make(map[key.Key]bool)
122
+ keys := make(map[string]bool)
123
for _, k := range copied.Wantlist() {
115
- keys[k.Key] = true
124
+ keys[k.Cid.KeyString()] = true
125
}
126
127
for _, k := range original.Wantlist() {
119
- if _, ok := keys[k.Key]; !ok {
128
+ if _, ok := keys[k.Cid.KeyString()]; !ok {
129
t.Fatalf("Key Missing: \"%v\"", k)
130
}
131
}
@@ -140,21 +149,21 @@ func TestToAndFromNetMessage(t *testing.T) {
149
t.Fatal(err)
150
}
151
143
- keys := make(map[key.Key]bool)
152
+ keys := make(map[string]bool)
153
for _, b := range m2.Blocks() {
145
- keys[b.Key()] = true
154
+ keys[b.Cid().KeyString()] = true
155
}
156
157
for _, b := range original.Blocks() {
149
- if _, ok := keys[b.Key()]; !ok {
158
+ if _, ok := keys[b.Cid().KeyString()]; !ok {
159
t.Fail()
160
}
161
}
162
}
163
155
-func wantlistContains(wantlist *pb.Message_Wantlist, x string) bool {
164
+func wantlistContains(wantlist *pb.Message_Wantlist, c *cid.Cid) bool {
165
for _, e := range wantlist.GetEntries() {
157
- if e.GetBlock() == x {
166
+ if e.GetBlock() == c.KeyString() {
167
return true
168
}
169
}
@@ -174,8 +183,8 @@ func TestDuplicates(t *testing.T) {
183
b := blocks.NewBlock([]byte("foo"))
184
msg := New(true)
185
177
- msg.AddEntry(b.Key(), 1)
178
- msg.AddEntry(b.Key(), 1)
186
+ msg.AddEntry(b.Cid(), 1)
187
+ msg.AddEntry(b.Cid(), 1)
188
if len(msg.Wantlist()) != 1 {
189
t.Fatal("Duplicate in BitSwapMessage")
190
}
exchange/bitswap/network/interface.go
+5
-4
@@ -1,10 +1,11 @@
1
package network
2
3
import (
4
- context "context"
4
+ "context"
5
+
6
bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
6
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
7
protocol "gx/ipfs/QmZNkThpqfVXs9GNbexPrfBbXSLNYeKrE7jwFM2oqHbyqN/go-libp2p-protocol"
8
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
9
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
10
)
11
@@ -52,8 +53,8 @@ type Receiver interface {
53
54
type Routing interface {
55
// FindProvidersAsync returns a channel of providers for the given key
55
- FindProvidersAsync(context.Context, key.Key, int) <-chan peer.ID
56
+ FindProvidersAsync(context.Context, *cid.Cid, int) <-chan peer.ID
57
58
// Provide provides the key to the network
58
- Provide(context.Context, key.Key) error
59
+ Provide(context.Context, *cid.Cid) error
60
}
exchange/bitswap/network/ipfs_impl.go
+4
-9
@@ -10,7 +10,6 @@ import (
10
ma "gx/ipfs/QmUAQaWbKxGCUTuoQVvvicbQNZ9APF5pDGWyAZSe93AtKH/go-multiaddr"
11
routing "gx/ipfs/QmXKuGUzLcgoQvp8M6ZEJzupWUNmx8NoqXEbYLMDjL4rjj/go-libp2p-routing"
12
pstore "gx/ipfs/QmXXCcQ7CLg5a81Ui9TTR35QcR4y7ZyihxwfjqaHfUVcVo/go-libp2p-peerstore"
13
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
13
ggio "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/io"
14
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
15
host "gx/ipfs/QmdML3R42PRSwnt46jSuEts9bHSqLctVYEjJqMR3UYV8ki/go-libp2p-host"
@@ -130,7 +129,7 @@ func (bsnet *impl) ConnectTo(ctx context.Context, p peer.ID) error {
129
}
130
131
// FindProvidersAsync returns a channel of providers for the given key
133
-func (bsnet *impl) FindProvidersAsync(ctx context.Context, k key.Key, max int) <-chan peer.ID {
132
+func (bsnet *impl) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {
133
134
// Since routing queries are expensive, give bitswap the peers to which we
135
// have open connections. Note that this may cause issues if bitswap starts
@@ -147,12 +146,9 @@ func (bsnet *impl) FindProvidersAsync(ctx context.Context, k key.Key, max int) <
146
out <- id
147
}
148
150
- // TEMPORARY SHIM UNTIL CID GETS PROPAGATED
151
- c := cid.NewCidV0(k.ToMultihash())
152
-
149
go func() {
150
defer close(out)
155
- providers := bsnet.routing.FindProvidersAsync(ctx, c, max)
151
+ providers := bsnet.routing.FindProvidersAsync(ctx, k, max)
152
for info := range providers {
153
if info.ID == bsnet.host.ID() {
154
continue // ignore self as provider
@@ -169,9 +165,8 @@ func (bsnet *impl) FindProvidersAsync(ctx context.Context, k key.Key, max int) <
165
}
166
167
// Provide provides the key to the network
172
-func (bsnet *impl) Provide(ctx context.Context, k key.Key) error {
173
- c := cid.NewCidV0(k.ToMultihash())
174
- return bsnet.routing.Provide(ctx, c)
168
+func (bsnet *impl) Provide(ctx context.Context, k *cid.Cid) error {
169
+ return bsnet.routing.Provide(ctx, k)
170
}
171
172
// handleNewStream receives a new stream from the network.
exchange/bitswap/notifications/notifications.go
+10
-9
@@ -1,17 +1,19 @@
1
package notifications
2
3
import (
4
- context "context"
5
- pubsub "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/briantigerchow/pubsub"
4
+ "context"
5
+
6
blocks "github.com/ipfs/go-ipfs/blocks"
7
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
7
+
8
+ pubsub "github.com/ipfs/go-ipfs/Godeps/_workspace/src/github.com/briantigerchow/pubsub"
9
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
10
)
11
12
const bufferSize = 16
13
14
type PubSub interface {
15
Publish(block blocks.Block)
14
- Subscribe(ctx context.Context, keys ...key.Key) <-chan blocks.Block
16
+ Subscribe(ctx context.Context, keys ...*cid.Cid) <-chan blocks.Block
17
Shutdown()
18
}
19
@@ -24,8 +26,7 @@ type impl struct {
26
}
27
28
func (ps *impl) Publish(block blocks.Block) {
27
- topic := string(block.Key())
28
- ps.wrapped.Pub(block, topic)
29
+ ps.wrapped.Pub(block, block.Cid().KeyString())
30
}
31
32
func (ps *impl) Shutdown() {
@@ -35,7 +36,7 @@ func (ps *impl) Shutdown() {
36
// Subscribe returns a channel of blocks for the given |keys|. |blockChannel|
37
// is closed if the |ctx| times out or is cancelled, or after sending len(keys)
38
// blocks.
38
-func (ps *impl) Subscribe(ctx context.Context, keys ...key.Key) <-chan blocks.Block {
39
+func (ps *impl) Subscribe(ctx context.Context, keys ...*cid.Cid) <-chan blocks.Block {
40
41
blocksCh := make(chan blocks.Block, len(keys))
42
valuesCh := make(chan interface{}, len(keys)) // provide our own channel to control buffer, prevent blocking
@@ -71,10 +72,10 @@ func (ps *impl) Subscribe(ctx context.Context, keys ...key.Key) <-chan blocks.Bl
72
return blocksCh
73
}
74
74
-func toStrings(keys []key.Key) []string {
75
+func toStrings(keys []*cid.Cid) []string {
76
strs := make([]string, 0)
77
for _, key := range keys {
77
- strs = append(strs, string(key))
78
+ strs = append(strs, key.KeyString())
79
}
80
return strs
81
}
exchange/bitswap/notifications/notifications_test.go
+12
-12
@@ -2,13 +2,13 @@ package notifications
2
3
import (
4
"bytes"
5
+ "context"
6
"testing"
7
"time"
8
8
- context "context"
9
blocks "github.com/ipfs/go-ipfs/blocks"
10
blocksutil "github.com/ipfs/go-ipfs/blocks/blocksutil"
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
11
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
)
13
14
func TestDuplicates(t *testing.T) {
@@ -17,7 +17,7 @@ func TestDuplicates(t *testing.T) {
17
18
n := New()
19
defer n.Shutdown()
20
- ch := n.Subscribe(context.Background(), b1.Key(), b2.Key())
20
+ ch := n.Subscribe(context.Background(), b1.Cid(), b2.Cid())
21
22
n.Publish(b1)
23
blockRecvd, ok := <-ch
@@ -41,7 +41,7 @@ func TestPublishSubscribe(t *testing.T) {
41
42
n := New()
43
defer n.Shutdown()
44
- ch := n.Subscribe(context.Background(), blockSent.Key())
44
+ ch := n.Subscribe(context.Background(), blockSent.Cid())
45
46
n.Publish(blockSent)
47
blockRecvd, ok := <-ch
@@ -59,7 +59,7 @@ func TestSubscribeMany(t *testing.T) {
59
60
n := New()
61
defer n.Shutdown()
62
- ch := n.Subscribe(context.Background(), e1.Key(), e2.Key())
62
+ ch := n.Subscribe(context.Background(), e1.Cid(), e2.Cid())
63
64
n.Publish(e1)
65
r1, ok := <-ch
@@ -83,8 +83,8 @@ func TestDuplicateSubscribe(t *testing.T) {
83
84
n := New()
85
defer n.Shutdown()
86
- ch1 := n.Subscribe(context.Background(), e1.Key())
87
- ch2 := n.Subscribe(context.Background(), e1.Key())
86
+ ch1 := n.Subscribe(context.Background(), e1.Cid())
87
+ ch2 := n.Subscribe(context.Background(), e1.Cid())
88
89
n.Publish(e1)
90
r1, ok := <-ch1
@@ -118,7 +118,7 @@ func TestCarryOnWhenDeadlineExpires(t *testing.T) {
118
n := New()
119
defer n.Shutdown()
120
block := blocks.NewBlock([]byte("A Missed Connection"))
121
- blockChannel := n.Subscribe(fastExpiringCtx, block.Key())
121
+ blockChannel := n.Subscribe(fastExpiringCtx, block.Cid())
122
123
assertBlockChannelNil(t, blockChannel)
124
}
@@ -132,10 +132,10 @@ func TestDoesNotDeadLockIfContextCancelledBeforePublish(t *testing.T) {
132
133
t.Log("generate a large number of blocks. exceed default buffer")
134
bs := g.Blocks(1000)
135
- ks := func() []key.Key {
136
- var keys []key.Key
135
+ ks := func() []*cid.Cid {
136
+ var keys []*cid.Cid
137
for _, b := range bs {
138
- keys = append(keys, b.Key())
138
+ keys = append(keys, b.Cid())
139
}
140
return keys
141
}()
@@ -162,7 +162,7 @@ func assertBlocksEqual(t *testing.T, a, b blocks.Block) {
162
if !bytes.Equal(a.RawData(), b.RawData()) {
163
t.Fatal("blocks aren't equal")
164
}
165
- if a.Key() != b.Key() {
165
+ if a.Cid() != b.Cid() {
166
t.Fatal("block keys aren't equal")
167
}
168
}
exchange/bitswap/stat.go
+3
-2
@@ -1,13 +1,14 @@
1
package bitswap
2
3
import (
4
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
4
"sort"
5
+
6
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
7
)
8
9
type Stat struct {
10
ProvideBufLen int
10
- Wantlist []key.Key
11
+ Wantlist []*cid.Cid
12
Peers []string
13
BlocksReceived int
14
DupBlksReceived int
exchange/bitswap/testnet/virtual.go
+4
-7
@@ -10,7 +10,6 @@ import (
10
delay "github.com/ipfs/go-ipfs/thirdparty/delay"
11
testutil "github.com/ipfs/go-ipfs/thirdparty/testutil"
12
routing "gx/ipfs/QmXKuGUzLcgoQvp8M6ZEJzupWUNmx8NoqXEbYLMDjL4rjj/go-libp2p-routing"
13
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
13
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
14
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
15
)
@@ -92,18 +91,17 @@ func (nc *networkClient) SendMessage(
91
}
92
93
// FindProvidersAsync returns a channel of providers for the given key
95
-func (nc *networkClient) FindProvidersAsync(ctx context.Context, k key.Key, max int) <-chan peer.ID {
94
+func (nc *networkClient) FindProvidersAsync(ctx context.Context, k *cid.Cid, max int) <-chan peer.ID {
95
96
// NB: this function duplicates the PeerInfo -> ID transformation in the
97
// bitswap network adapter. Not to worry. This network client will be
98
// deprecated once the ipfsnet.Mock is added. The code below is only
99
// temporary.
100
102
- c := cid.NewCidV0(k.ToMultihash())
101
out := make(chan peer.ID)
102
go func() {
103
defer close(out)
106
- providers := nc.routing.FindProvidersAsync(ctx, c, max)
104
+ providers := nc.routing.FindProvidersAsync(ctx, k, max)
105
for info := range providers {
106
select {
107
case <-ctx.Done():
@@ -139,9 +137,8 @@ func (n *networkClient) NewMessageSender(ctx context.Context, p peer.ID) (bsnet.
137
}
138
139
// Provide provides the key to the network
142
-func (nc *networkClient) Provide(ctx context.Context, k key.Key) error {
143
- c := cid.NewCidV0(k.ToMultihash())
144
- return nc.routing.Provide(ctx, c)
140
+func (nc *networkClient) Provide(ctx context.Context, k *cid.Cid) error {
141
+ return nc.routing.Provide(ctx, k)
142
}
143
144
func (nc *networkClient) SetDelegate(r bsnet.Receiver) {
exchange/bitswap/wantlist/wantlist.go
+17
-14
@@ -6,7 +6,7 @@ import (
6
"sort"
7
"sync"
8
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
10
)
11
12
type ThreadSafe struct {
@@ -16,11 +16,11 @@ type ThreadSafe struct {
16
17
// not threadsafe
18
type Wantlist struct {
19
- set map[key.Key]*Entry
19
+ set map[string]*Entry
20
}
21
22
type Entry struct {
23
- Key key.Key
23
+ Cid *cid.Cid
24
Priority int
25
26
RefCnt int
@@ -40,11 +40,11 @@ func NewThreadSafe() *ThreadSafe {
40
41
func New() *Wantlist {
42
return &Wantlist{
43
- set: make(map[key.Key]*Entry),
43
+ set: make(map[string]*Entry),
44
}
45
}
46
47
-func (w *ThreadSafe) Add(k key.Key, priority int) bool {
47
+func (w *ThreadSafe) Add(k *cid.Cid, priority int) bool {
48
w.lk.Lock()
49
defer w.lk.Unlock()
50
return w.Wantlist.Add(k, priority)
@@ -56,13 +56,13 @@ func (w *ThreadSafe) AddEntry(e *Entry) bool {
56
return w.Wantlist.AddEntry(e)
57
}
58
59
-func (w *ThreadSafe) Remove(k key.Key) bool {
59
+func (w *ThreadSafe) Remove(k *cid.Cid) bool {
60
w.lk.Lock()
61
defer w.lk.Unlock()
62
return w.Wantlist.Remove(k)
63
}
64
65
-func (w *ThreadSafe) Contains(k key.Key) (*Entry, bool) {
65
+func (w *ThreadSafe) Contains(k *cid.Cid) (*Entry, bool) {
66
w.lk.RLock()
67
defer w.lk.RUnlock()
68
return w.Wantlist.Contains(k)
@@ -90,14 +90,15 @@ func (w *Wantlist) Len() int {
90
return len(w.set)
91
}
92
93
-func (w *Wantlist) Add(k key.Key, priority int) bool {
93
+func (w *Wantlist) Add(c *cid.Cid, priority int) bool {
94
+ k := c.KeyString()
95
if e, ok := w.set[k]; ok {
96
e.RefCnt++
97
return false
98
}
99
100
w.set[k] = &Entry{
100
- Key: k,
101
+ Cid: c,
102
Priority: priority,
103
RefCnt: 1,
104
}
@@ -106,15 +107,17 @@ func (w *Wantlist) Add(k key.Key, priority int) bool {
107
}
108
109
func (w *Wantlist) AddEntry(e *Entry) bool {
109
- if ex, ok := w.set[e.Key]; ok {
110
+ k := e.Cid.KeyString()
111
+ if ex, ok := w.set[k]; ok {
112
ex.RefCnt++
113
return false
114
}
113
- w.set[e.Key] = e
115
+ w.set[k] = e
116
return true
117
}
118
117
-func (w *Wantlist) Remove(k key.Key) bool {
119
+func (w *Wantlist) Remove(c *cid.Cid) bool {
120
+ k := c.KeyString()
121
e, ok := w.set[k]
122
if !ok {
123
return false
@@ -128,8 +131,8 @@ func (w *Wantlist) Remove(k key.Key) bool {
131
return false
132
}
133
131
-func (w *Wantlist) Contains(k key.Key) (*Entry, bool) {
132
- e, ok := w.set[k]
134
+func (w *Wantlist) Contains(k *cid.Cid) (*Entry, bool) {
135
+ e, ok := w.set[k.KeyString()]
136
return e, ok
137
}
138
exchange/bitswap/wantmanager.go
+11
-11
@@ -1,15 +1,15 @@
1
package bitswap
2
3
import (
4
+ "context"
5
"sync"
6
"time"
7
7
- context "context"
8
engine "github.com/ipfs/go-ipfs/exchange/bitswap/decision"
9
bsmsg "github.com/ipfs/go-ipfs/exchange/bitswap/message"
10
bsnet "github.com/ipfs/go-ipfs/exchange/bitswap/network"
11
wantlist "github.com/ipfs/go-ipfs/exchange/bitswap/wantlist"
12
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
12
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
13
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
14
)
15
@@ -51,7 +51,7 @@ type msgPair struct {
51
52
type cancellation struct {
53
who peer.ID
54
- blk key.Key
54
+ blk *cid.Cid
55
}
56
57
type msgQueue struct {
@@ -69,23 +69,23 @@ type msgQueue struct {
69
done chan struct{}
70
}
71
72
-func (pm *WantManager) WantBlocks(ctx context.Context, ks []key.Key) {
72
+func (pm *WantManager) WantBlocks(ctx context.Context, ks []*cid.Cid) {
73
log.Infof("want blocks: %s", ks)
74
pm.addEntries(ctx, ks, false)
75
}
76
77
-func (pm *WantManager) CancelWants(ks []key.Key) {
77
+func (pm *WantManager) CancelWants(ks []*cid.Cid) {
78
log.Infof("cancel wants: %s", ks)
79
pm.addEntries(context.TODO(), ks, true)
80
}
81
82
-func (pm *WantManager) addEntries(ctx context.Context, ks []key.Key, cancel bool) {
82
+func (pm *WantManager) addEntries(ctx context.Context, ks []*cid.Cid, cancel bool) {
83
var entries []*bsmsg.Entry
84
for i, k := range ks {
85
entries = append(entries, &bsmsg.Entry{
86
Cancel: cancel,
87
Entry: &wantlist.Entry{
88
- Key: k,
88
+ Cid: k,
89
Priority: kMaxPriority - i,
90
RefCnt: 1,
91
},
@@ -130,7 +130,7 @@ func (pm *WantManager) startPeerHandler(p peer.ID) *msgQueue {
130
// new peer, we will want to give them our full wantlist
131
fullwantlist := bsmsg.New(true)
132
for _, e := range pm.wl.Entries() {
133
- fullwantlist.AddEntry(e.Key, e.Priority)
133
+ fullwantlist.AddEntry(e.Cid, e.Priority)
134
}
135
mq.out = fullwantlist
136
mq.work <- struct{}{}
@@ -246,7 +246,7 @@ func (pm *WantManager) Run() {
246
var filtered []*bsmsg.Entry
247
for _, e := range entries {
248
if e.Cancel {
249
- if pm.wl.Remove(e.Key) {
249
+ if pm.wl.Remove(e.Cid) {
250
filtered = append(filtered, e)
251
}
252
} else {
@@ -323,9 +323,9 @@ func (mq *msgQueue) addMessage(entries []*bsmsg.Entry) {
323
// one passed in
324
for _, e := range entries {
325
if e.Cancel {
326
- mq.out.Cancel(e.Key)
326
+ mq.out.Cancel(e.Cid)
327
} else {
328
- mq.out.AddEntry(e.Key, e.Priority)
328
+ mq.out.AddEntry(e.Cid, e.Priority)
329
}
330
}
331
}
exchange/bitswap/workers.go
+13
-13
@@ -1,15 +1,15 @@
1
package bitswap
2
3
import (
4
+ "context"
5
"math/rand"
6
"sync"
7
"time"
8
8
- context "context"
9
process "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess"
10
procctx "gx/ipfs/QmSF8fPo3jgVBAy8fpdjjYqgG87dkJgUprRBHRd2tmfgpP/goprocess/context"
11
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
12
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
12
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
13
peer "gx/ipfs/QmfMmLGoKzCHDN7cGgk64PJr4iipzidDRME8HABSJqvmhC/go-libp2p-peer"
14
)
15
@@ -77,7 +77,7 @@ func (bs *Bitswap) provideWorker(px process.Process) {
77
78
limit := make(chan struct{}, provideWorkerMax)
79
80
- limitedGoProvide := func(k key.Key, wid int) {
80
+ limitedGoProvide := func(k *cid.Cid, wid int) {
81
defer func() {
82
// replace token when done
83
<-limit
@@ -85,7 +85,7 @@ func (bs *Bitswap) provideWorker(px process.Process) {
85
ev := logging.LoggableMap{"ID": wid}
86
87
ctx := procctx.OnClosingContext(px) // derive ctx from px
88
- defer log.EventBegin(ctx, "Bitswap.ProvideWorker.Work", ev, &k).Done()
88
+ defer log.EventBegin(ctx, "Bitswap.ProvideWorker.Work", ev, k).Done()
89
90
ctx, cancel := context.WithTimeout(ctx, provideTimeout) // timeout ctx
91
defer cancel()
@@ -121,9 +121,9 @@ func (bs *Bitswap) provideWorker(px process.Process) {
121
122
func (bs *Bitswap) provideCollector(ctx context.Context) {
123
defer close(bs.provideKeys)
124
- var toProvide []key.Key
125
- var nextKey key.Key
126
- var keysOut chan key.Key
124
+ var toProvide []*cid.Cid
125
+ var nextKey *cid.Cid
126
+ var keysOut chan *cid.Cid
127
128
for {
129
select {
@@ -181,7 +181,7 @@ func (bs *Bitswap) rebroadcastWorker(parent context.Context) {
181
// for new providers for blocks.
182
i := rand.Intn(len(entries))
183
bs.findKeys <- &blockRequest{
184
- Key: entries[i].Key,
184
+ Cid: entries[i].Cid,
185
Ctx: ctx,
186
}
187
case <-parent.Done():
@@ -192,23 +192,23 @@ func (bs *Bitswap) rebroadcastWorker(parent context.Context) {
192
193
func (bs *Bitswap) providerQueryManager(ctx context.Context) {
194
var activeLk sync.Mutex
195
- kset := key.NewKeySet()
195
+ kset := cid.NewSet()
196
197
for {
198
select {
199
case e := <-bs.findKeys:
200
activeLk.Lock()
201
- if kset.Has(e.Key) {
201
+ if kset.Has(e.Cid) {
202
activeLk.Unlock()
203
continue
204
}
205
- kset.Add(e.Key)
205
+ kset.Add(e.Cid)
206
activeLk.Unlock()
207
208
go func(e *blockRequest) {
209
child, cancel := context.WithTimeout(e.Ctx, providerRequestTimeout)
210
defer cancel()
211
- providers := bs.network.FindProvidersAsync(child, e.Key, maxProvidersPerRequest)
211
+ providers := bs.network.FindProvidersAsync(child, e.Cid, maxProvidersPerRequest)
212
wg := &sync.WaitGroup{}
213
for p := range providers {
214
wg.Add(1)
@@ -222,7 +222,7 @@ func (bs *Bitswap) providerQueryManager(ctx context.Context) {
222
}
223
wg.Wait()
224
activeLk.Lock()
225
- kset.Remove(e.Key)
225
+ kset.Remove(e.Cid)
226
activeLk.Unlock()
227
}(e)
228
exchange/interface.go
+4
-4
@@ -2,21 +2,21 @@
2
package exchange
3
4
import (
5
+ "context"
6
"io"
7
8
blocks "github.com/ipfs/go-ipfs/blocks"
8
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
10
- context "context"
10
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
11
)
12
13
// Any type that implements exchange.Interface may be used as an IPFS block
14
// exchange protocol.
15
type Interface interface { // type Exchanger interface
16
// GetBlock returns the block associated with a given key.
17
- GetBlock(context.Context, key.Key) (blocks.Block, error)
17
+ GetBlock(context.Context, *cid.Cid) (blocks.Block, error)
18
19
- GetBlocks(context.Context, []key.Key) (<-chan blocks.Block, error)
19
+ GetBlocks(context.Context, []*cid.Cid) (<-chan blocks.Block, error)
20
21
// TODO Should callers be concerned with whether the block was made
22
// available on the network?
exchange/offline/offline.go
+6
-5
@@ -3,12 +3,13 @@
3
package offline
4
5
import (
6
+ "context"
7
+
8
blocks "github.com/ipfs/go-ipfs/blocks"
9
"github.com/ipfs/go-ipfs/blocks/blockstore"
10
exchange "github.com/ipfs/go-ipfs/exchange"
9
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
11
11
- context "context"
12
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
13
)
14
15
func Exchange(bs blockstore.Blockstore) exchange.Interface {
@@ -24,7 +25,7 @@ type offlineExchange struct {
25
// GetBlock returns nil to signal that a block could not be retrieved for the
26
// given key.
27
// NB: This function may return before the timeout expires.
27
-func (e *offlineExchange) GetBlock(_ context.Context, k key.Key) (blocks.Block, error) {
28
+func (e *offlineExchange) GetBlock(_ context.Context, k *cid.Cid) (blocks.Block, error) {
29
return e.bs.Get(k)
30
}
31
@@ -40,11 +41,11 @@ func (_ *offlineExchange) Close() error {
41
return nil
42
}
43
43
-func (e *offlineExchange) GetBlocks(ctx context.Context, ks []key.Key) (<-chan blocks.Block, error) {
44
+func (e *offlineExchange) GetBlocks(ctx context.Context, ks []*cid.Cid) (<-chan blocks.Block, error) {
45
out := make(chan blocks.Block, 0)
46
go func() {
47
defer close(out)
47
- var misses []key.Key
48
+ var misses []*cid.Cid
49
for _, k := range ks {
50
hit, err := e.bs.Get(k)
51
if err != nil {
exchange/offline/offline_test.go
+10
-7
@@ -1,20 +1,23 @@
1
package offline
2
3
import (
4
+ "context"
5
"testing"
6
6
- context "context"
7
blocks "github.com/ipfs/go-ipfs/blocks"
8
"github.com/ipfs/go-ipfs/blocks/blockstore"
9
"github.com/ipfs/go-ipfs/blocks/blocksutil"
10
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
10
+
11
+ cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
+ u "gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
13
ds "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore"
14
ds_sync "gx/ipfs/QmbzuUusHqaLLoNTDEVLcSF6vZDHZDLPC7p4bztRvvkXxU/go-datastore/sync"
15
)
16
17
func TestBlockReturnsErr(t *testing.T) {
18
off := Exchange(bstore())
17
- _, err := off.GetBlock(context.Background(), key.Key("foo"))
19
+ c := cid.NewCidV0(u.Hash([]byte("foo")))
20
+ _, err := off.GetBlock(context.Background(), c)
21
if err != nil {
22
return // as desired
23
}
@@ -31,7 +34,7 @@ func TestHasBlockReturnsNil(t *testing.T) {
34
t.Fail()
35
}
36
34
- if _, err := store.Get(block.Key()); err != nil {
37
+ if _, err := store.Get(block.Cid()); err != nil {
38
t.Fatal(err)
39
}
40
}
@@ -49,11 +52,11 @@ func TestGetBlocks(t *testing.T) {
52
}
53
}
54
52
- request := func() []key.Key {
53
- var ks []key.Key
55
+ request := func() []*cid.Cid {
56
+ var ks []*cid.Cid
57
58
for _, b := range expected {
56
- ks = append(ks, b.Key())
59
+ ks = append(ks, b.Cid())
60
}
61
return ks
62
}()
exchange/reprovide/reprovide.go
+1
-3
@@ -9,7 +9,6 @@ import (
9
backoff "gx/ipfs/QmPJUtEJsm5YLUWhF6imvyCH8KZXRJa9Wup7FDMwTy5Ufz/backoff"
10
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
11
routing "gx/ipfs/QmXKuGUzLcgoQvp8M6ZEJzupWUNmx8NoqXEbYLMDjL4rjj/go-libp2p-routing"
12
- cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
)
13
14
var log = logging.Logger("reprovider")
@@ -53,8 +52,7 @@ func (rp *Reprovider) Reprovide(ctx context.Context) error {
52
if err != nil {
53
return fmt.Errorf("Failed to get key chan from blockstore: %s", err)
54
}
56
- for k := range keychan {
57
- c := cid.NewCidV0(k.ToMultihash())
55
+ for c := range keychan {
56
op := func() error {
57
err := rp.rsys.Provide(ctx, c)
58
if err != nil {
importer/chunk/rabin_test.go
+3
-4
@@ -4,7 +4,6 @@ import (
4
"bytes"
5
"fmt"
6
"github.com/ipfs/go-ipfs/blocks"
7
- "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
7
"gx/ipfs/Qmb912gdngC1UWwTkhuW8knyRbcWeu5kqkxBpveLmW8bSr/go-ipfs-util"
8
"io"
9
"testing"
@@ -39,10 +38,10 @@ func TestRabinChunking(t *testing.T) {
38
}
39
}
40
42
-func chunkData(t *testing.T, data []byte) map[key.Key]blocks.Block {
41
+func chunkData(t *testing.T, data []byte) map[string]blocks.Block {
42
r := NewRabin(bytes.NewReader(data), 1024*256)
43
45
- blkmap := make(map[key.Key]blocks.Block)
44
+ blkmap := make(map[string]blocks.Block)
45
46
for {
47
blk, err := r.NextBytes()
@@ -54,7 +53,7 @@ func chunkData(t *testing.T, data []byte) map[key.Key]blocks.Block {
53
}
54
55
b := blocks.NewBlock(blk)
57
- blkmap[b.Key()] = b
56
+ blkmap[b.Cid().KeyString()] = b
57
}
58
59
return blkmap
merkledag/merkledag.go
+9
-25
@@ -2,15 +2,15 @@
2
package merkledag
3
4
import (
5
+ "context"
6
"fmt"
7
"strings"
8
"sync"
9
10
+ blocks "github.com/ipfs/go-ipfs/blocks"
11
bserv "github.com/ipfs/go-ipfs/blockservice"
12
offline "github.com/ipfs/go-ipfs/exchange/offline"
11
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
13
13
- "context"
14
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
15
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
16
)
@@ -60,7 +60,7 @@ func (n *dagService) Add(nd *Node) (*cid.Cid, error) {
60
return nil, fmt.Errorf("dagService is nil")
61
}
62
63
- return n.Blocks.AddObject(nd)
63
+ return n.Blocks.AddBlock(nd)
64
}
65
66
func (n *dagService) Batch() *Batch {
@@ -122,7 +122,7 @@ func (n *dagService) GetOfflineLinkService() LinkService {
122
}
123
124
func (n *dagService) Remove(nd *Node) error {
125
- return n.Blocks.DeleteObject(nd)
125
+ return n.Blocks.DeleteBlock(nd)
126
}
127
128
// FetchGraph fetches all nodes that are children of the given node
@@ -147,27 +147,11 @@ type NodeOption struct {
147
Err error
148
}
149
150
-// TODO: this is a mid-term hack to get around the fact that blocks don't
151
-// have full CIDs and potentially (though we don't know of any such scenario)
152
-// may have the same block with multiple different encodings.
153
-// We have discussed the possiblity of using CIDs as datastore keys
154
-// in the future. This would be a much larger changeset than i want to make
155
-// right now.
156
-func cidsToKeyMapping(cids []*cid.Cid) map[key.Key]*cid.Cid {
157
- mapping := make(map[key.Key]*cid.Cid)
158
- for _, c := range cids {
159
- mapping[key.Key(c.Hash())] = c
160
- }
161
- return mapping
162
-}
163
-
150
func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *NodeOption {
151
out := make(chan *NodeOption, len(keys))
152
blocks := ds.Blocks.GetBlocks(ctx, keys)
153
var count int
154
169
- mapping := cidsToKeyMapping(keys)
170
-
155
go func() {
156
defer close(out)
157
for {
@@ -180,7 +164,7 @@ func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *Node
164
return
165
}
166
183
- c := mapping[b.Key()]
167
+ c := b.Cid()
168
169
var nd *Node
170
switch c.Type() {
@@ -361,7 +345,7 @@ func (np *nodePromise) Get(ctx context.Context) (*Node, error) {
345
type Batch struct {
346
ds *dagService
347
364
- objects []bserv.Object
348
+ blocks []blocks.Block
349
size int
350
MaxSize int
351
}
@@ -372,7 +356,7 @@ func (t *Batch) Add(nd *Node) (*cid.Cid, error) {
356
return nil, err
357
}
358
375
- t.objects = append(t.objects, nd)
359
+ t.blocks = append(t.blocks, nd)
360
t.size += len(d)
361
if t.size > t.MaxSize {
362
return nd.Cid(), t.Commit()
@@ -381,8 +365,8 @@ func (t *Batch) Add(nd *Node) (*cid.Cid, error) {
365
}
366
367
func (t *Batch) Commit() error {
384
- _, err := t.ds.Blocks.AddObjects(t.objects)
385
- t.objects = nil
368
+ _, err := t.ds.Blocks.AddBlocks(t.blocks)
369
+ t.blocks = nil
370
t.size = 0
371
return err
372
}
mfs/file.go
+1
-2
@@ -1,6 +1,7 @@
1
package mfs
2
3
import (
4
+ "context"
5
"fmt"
6
"sync"
7
@@ -8,8 +9,6 @@ import (
9
dag "github.com/ipfs/go-ipfs/merkledag"
10
ft "github.com/ipfs/go-ipfs/unixfs"
11
mod "github.com/ipfs/go-ipfs/unixfs/mod"
11
-
12
- context "context"
12
)
13
14
type File struct {
pin/gc/gc.go
+10
-18
@@ -1,12 +1,12 @@
1
package gc
2
3
import (
4
+ "context"
5
+
6
bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
7
dag "github.com/ipfs/go-ipfs/merkledag"
8
pin "github.com/ipfs/go-ipfs/pin"
7
- key "gx/ipfs/QmYEoKZXHoAToWfhGF3vryhMn3WWhE1o2MasQ8uzY5iDi9/go-key"
9
9
- context "context"
10
logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
11
cid "gx/ipfs/QmakyCk6Vnn16WEKjbkxieZmM2YLTzkFWizbmGowoYPjro/go-cid"
12
)
@@ -22,7 +22,7 @@ var log = logging.Logger("gc")
22
//
23
// The routine then iterates over every block in the blockstore and
24
// deletes any block that is not found in the marked set.
25
-func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan key.Key, error) {
25
+func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan *cid.Cid, error) {
26
unlocker := bs.GCLock()
27
28
ls = ls.GetOfflineLinkService()
@@ -37,7 +37,7 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.
37
return nil, err
38
}
39
40
- output := make(chan key.Key)
40
+ output := make(chan *cid.Cid)
41
go func() {
42
defer close(output)
43
defer unlocker.Unlock()
@@ -68,20 +68,12 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, ls dag.LinkService, pn pin.
68
return output, nil
69
}
70
71
-func Descendants(ctx context.Context, ls dag.LinkService, set key.KeySet, roots []*cid.Cid, bestEffort bool) error {
71
+func Descendants(ctx context.Context, ls dag.LinkService, set *cid.Set, roots []*cid.Cid, bestEffort bool) error {
72
for _, c := range roots {
73
- set.Add(key.Key(c.Hash()))
73
+ set.Add(c)
74
75
// EnumerateChildren recursively walks the dag and adds the keys to the given set
76
- err := dag.EnumerateChildren(ctx, ls, c, func(c *cid.Cid) bool {
77
- k := key.Key(c.Hash())
78
- seen := set.Has(k)
79
- if seen {
80
- return false
81
- }
82
- set.Add(k)
83
- return true
84
- }, bestEffort)
76
+ err := dag.EnumerateChildren(ctx, ls, c, set.Visit, bestEffort)
77
if err != nil {
78
return err
79
}
@@ -90,10 +82,10 @@ func Descendants(ctx context.Context, ls dag.LinkService, set key.KeySet, roots
82
return nil
83
}
84
93
-func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffortRoots []*cid.Cid) (key.KeySet, error) {
85
+func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffortRoots []*cid.Cid) (*cid.Set, error) {
86
// KeySet currently implemented in memory, in the future, may be bloom filter or
87
// disk backed to conserve memory.
96
- gcs := key.NewKeySet()
88
+ gcs := cid.NewSet()
89
err := Descendants(ctx, ls, gcs, pn.RecursiveKeys(), false)
90
if err != nil {
91
return nil, err
@@ -105,7 +97,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ls dag.LinkService, bestEffo
97
}
98
99
for _, k := range pn.DirectKeys() {
108
- gcs.Add(key.Key(k.Hash()))
100
+ gcs.Add(k)
101
}
102
103
err = Descendants(ctx, ls, gcs, pn.InternalPins(), false)
test/integration/bitswap_wo_routing_test.go
+2
-2
@@ -76,7 +76,7 @@ func TestBitswapWithoutRouting(t *testing.T) {
76
} else if !bytes.Equal(b.RawData(), block0.RawData()) {
77
t.Error("byte comparison fail")
78
} else {
79
- log.Debug("got block: %s", b.Key())
79
+ log.Debug("got block: %s", b.Cid())
80
}
81
}
82
@@ -93,7 +93,7 @@ func TestBitswapWithoutRouting(t *testing.T) {
93
} else if !bytes.Equal(b.RawData(), block1.RawData()) {
94
t.Error("byte comparison fail")
95
} else {
96
- log.Debug("got block: %s", b.Key())
96
+ log.Debug("got block: %s", b.Cid())
97
}
98
}
99
}
thirdparty/ds-help/key.go
+4
@@ -9,3 +9,7 @@ import (
9
func NewKeyFromBinary(s string) ds.Key {
10
return ds.NewKey(base32.RawStdEncoding.EncodeToString([]byte(s)))
11
}
12
+
13
+func BinaryFromDsKey(k ds.Key) ([]byte, error) {
14
+ return base32.RawStdEncoding.DecodeString(k.String()[1:])
15
+}