@cryptotaxi247 / kubo / commits / c8fe49593

integrate CIDv0

License: MIT Signed-off-by: Jeromy <why@ipfs.io>

Jeromy committed Sep 1, 2016 at 07:50 UTC c8fe49593448927ea4114f27805b3bccab9eab22
70 files changed +879 -1002
assets/assets.go
+12 -8
@@ -8,10 +8,10 @@ import (
8 "fmt"
9 "path/filepath"
10
11 - "github.com/ipfs/go-ipfs/blocks/key"
11 "github.com/ipfs/go-ipfs/core"
12 "github.com/ipfs/go-ipfs/core/coreunix"
13 uio "github.com/ipfs/go-ipfs/unixfs/io"
14 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
15 )
16
17 // initDocPaths lists the paths for the docs we want to seed during --init
@@ -25,7 +25,7 @@ var initDocPaths = []string{
25 }
26
27 // SeedInitDocs adds the list of embedded init documentation to the passed node, pins it and returns the root key
28 -func SeedInitDocs(nd *core.IpfsNode) (*key.Key, error) {
28 +func SeedInitDocs(nd *core.IpfsNode) (*cid.Cid, error) {
29 return addAssetList(nd, initDocPaths)
30 }
31
@@ -34,11 +34,11 @@ var initDirIndex = []string{
34 filepath.Join("..", "vendor", "dir-index-html-v1.0.0", "dir-index.html"),
35 }
36
37 -func SeedInitDirIndex(nd *core.IpfsNode) (*key.Key, error) {
37 +func SeedInitDirIndex(nd *core.IpfsNode) (*cid.Cid, error) {
38 return addAssetList(nd, initDirIndex)
39 }
40
41 -func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
41 +func addAssetList(nd *core.IpfsNode, l []string) (*cid.Cid, error) {
42 dirb := uio.NewDirectory(nd.DAG)
43
44 for _, p := range l {
@@ -53,14 +53,18 @@ func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
53 }
54
55 fname := filepath.Base(p)
56 - k := key.B58KeyDecode(s)
57 - if err := dirb.AddChild(nd.Context(), fname, k); err != nil {
56 + c, err := cid.Decode(s)
57 + if err != nil {
58 + return nil, err
59 + }
60 +
61 + if err := dirb.AddChild(nd.Context(), fname, c); err != nil {
62 return nil, fmt.Errorf("assets: could not add '%s' as a child: %s", fname, err)
63 }
64 }
65
66 dir := dirb.GetNode()
63 - dkey, err := nd.DAG.Add(dir)
67 + dcid, err := nd.DAG.Add(dir)
68 if err != nil {
69 return nil, fmt.Errorf("assets: DAG.Add(dir) failed: %s", err)
70 }
@@ -73,5 +77,5 @@ func addAssetList(nd *core.IpfsNode, l []string) (*key.Key, error) {
77 return nil, fmt.Errorf("assets: Pinning flush failed: %s", err)
78 }
79
76 - return &dkey, nil
80 + return dcid, nil
81 }
blocks/blocks.go
+8 -2
@@ -7,15 +7,17 @@ import (
7 "fmt"
8
9 key "github.com/ipfs/go-ipfs/blocks/key"
10 +
11 mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
12 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
13 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
14 )
15
16 var ErrWrongHash = errors.New("data did not match given hash!")
17
18 type Block interface {
19 Multihash() mh.Multihash
18 - Data() []byte
20 + RawData() []byte
21 Key() key.Key
22 String() string
23 Loggable() map[string]interface{}
@@ -49,10 +51,14 @@ func (b *BasicBlock) Multihash() mh.Multihash {
51 return b.multihash
52 }
53
52 -func (b *BasicBlock) Data() []byte {
54 +func (b *BasicBlock) RawData() []byte {
55 return b.data
56 }
57
58 +func (b *BasicBlock) Cid() *cid.Cid {
59 + return cid.NewCidV0(b.multihash)
60 +}
61 +
62 // Key returns the block's Multihash as a Key value.
63 func (b *BasicBlock) Key() key.Key {
64 return key.Key(b.multihash)
blocks/blocks_test.go
+1 -1
@@ -25,7 +25,7 @@ func TestData(t *testing.T) {
25 data := []byte("some data")
26 block := NewBlock(data)
27
28 - if !bytes.Equal(block.Data(), data) {
28 + if !bytes.Equal(block.RawData(), data) {
29 t.Error("data is wrong")
30 }
31 }
blocks/blockstore/blockstore.go
+2 -2
@@ -117,7 +117,7 @@ func (bs *blockstore) Put(block blocks.Block) error {
117 if err == nil && exists {
118 return nil // already stored.
119 }
120 - return bs.datastore.Put(k, block.Data())
120 + return bs.datastore.Put(k, block.RawData())
121 }
122
123 func (bs *blockstore) PutMany(blocks []blocks.Block) error {
@@ -132,7 +132,7 @@ func (bs *blockstore) PutMany(blocks []blocks.Block) error {
132 continue
133 }
134
135 - err = t.Put(k, b.Data())
135 + err = t.Put(k, b.RawData())
136 if err != nil {
137 return err
138 }
blocks/blockstore/blockstore_test.go
+1 -1
@@ -48,7 +48,7 @@ func TestPutThenGetBlock(t *testing.T) {
48 if err != nil {
49 t.Fatal(err)
50 }
51 - if !bytes.Equal(block.Data(), blockFromBlockstore.Data()) {
51 + if !bytes.Equal(block.RawData(), blockFromBlockstore.RawData()) {
52 t.Fail()
53 }
54 }
blockservice/blockservice.go
+58 -29
@@ -5,13 +5,16 @@ package blockservice
5
6 import (
7 "errors"
8 + "fmt"
9
10 blocks "github.com/ipfs/go-ipfs/blocks"
11 "github.com/ipfs/go-ipfs/blocks/blockstore"
12 key "github.com/ipfs/go-ipfs/blocks/key"
13 exchange "github.com/ipfs/go-ipfs/exchange"
14 +
15 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
16 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
17 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
18 )
19
20 var log = logging.Logger("blockservice")
@@ -27,6 +30,12 @@ type BlockService struct {
30 Exchange exchange.Interface
31 }
32
33 +// an Object is simply a typed block
34 +type Object interface {
35 + Cid() *cid.Cid
36 + blocks.Block
37 +}
38 +
39 // NewBlockService creates a BlockService with given datastore instance.
40 func New(bs blockstore.Blockstore, rem exchange.Interface) *BlockService {
41 if rem == nil {
@@ -41,30 +50,41 @@ func New(bs blockstore.Blockstore, rem exchange.Interface) *BlockService {
50
51 // AddBlock adds a particular block to the service, Putting it into the datastore.
52 // TODO pass a context into this if the remote.HasBlock is going to remain here.
44 -func (s *BlockService) AddBlock(b blocks.Block) (key.Key, error) {
45 - k := b.Key()
46 - has, err := s.Blockstore.Has(k)
53 +func (s *BlockService) AddObject(o Object) (*cid.Cid, error) {
54 + // TODO: while this is a great optimization, we should think about the
55 + // possibility of streaming writes directly to disk. If we can pass this object
56 + // all the way down to the datastore without having to 'buffer' its data,
57 + // we could implement a `WriteTo` method on it that could do a streaming write
58 + // of the content, saving us (probably) considerable memory.
59 + c := o.Cid()
60 + has, err := s.Blockstore.Has(key.Key(c.Hash()))
61 if err != nil {
48 - return k, err
62 + return nil, err
63 }
64 +
65 if has {
51 - return k, nil
66 + return c, nil
67 }
68
54 - err = s.Blockstore.Put(b)
69 + err = s.Blockstore.Put(o)
70 if err != nil {
56 - return k, err
71 + return nil, err
72 }
58 - if err := s.Exchange.HasBlock(b); err != nil {
59 - return "", errors.New("blockservice is closed")
73 +
74 + if err := s.Exchange.HasBlock(o); err != nil {
75 + return nil, errors.New("blockservice is closed")
76 }
61 - return k, nil
77 +
78 + return c, nil
79 }
80
64 -func (s *BlockService) AddBlocks(bs []blocks.Block) ([]key.Key, error) {
81 +func (s *BlockService) AddObjects(bs []Object) ([]*cid.Cid, error) {
82 var toput []blocks.Block
83 + var toputcids []*cid.Cid
84 for _, b := range bs {
67 - has, err := s.Blockstore.Has(b.Key())
85 + c := b.Cid()
86 +
87 + has, err := s.Blockstore.Has(key.Key(c.Hash()))
88 if err != nil {
89 return nil, err
90 }
@@ -74,6 +94,7 @@ func (s *BlockService) AddBlocks(bs []blocks.Block) ([]key.Key, error) {
94 }
95
96 toput = append(toput, b)
97 + toputcids = append(toputcids, c)
98 }
99
100 err := s.Blockstore.PutMany(toput)
@@ -81,26 +102,25 @@ func (s *BlockService) AddBlocks(bs []blocks.Block) ([]key.Key, error) {
102 return nil, err
103 }
104
84 - var ks []key.Key
85 - for _, b := range toput {
86 - if err := s.Exchange.HasBlock(b); err != nil {
87 - return nil, errors.New("blockservice is closed")
105 + var ks []*cid.Cid
106 + for _, o := range toput {
107 + if err := s.Exchange.HasBlock(o); err != nil {
108 + return nil, fmt.Errorf("blockservice is closed (%s)", err)
109 }
89 - ks = append(ks, b.Key())
110 +
111 + c := o.(Object).Cid() // cast is safe, we created these
112 + ks = append(ks, c)
113 }
114 return ks, nil
115 }
116
117 // GetBlock retrieves a particular block from the service,
118 // Getting it from the datastore using the key (hash).
96 -func (s *BlockService) GetBlock(ctx context.Context, k key.Key) (blocks.Block, error) {
97 - if k == "" {
98 - log.Debug("BlockService GetBlock: Nil Key")
99 - return nil, ErrNotFound
100 - }
119 +func (s *BlockService) GetBlock(ctx context.Context, c *cid.Cid) (blocks.Block, error) {
120 + log.Debugf("BlockService GetBlock: '%s'", c)
121
102 - log.Debugf("BlockService GetBlock: '%s'", k)
103 - block, err := s.Blockstore.Get(k)
122 + // TODO: blockstore shouldnt care about Cids, need an easier way to strip the abstraction
123 + block, err := s.Blockstore.Get(key.Key(c.Hash()))
124 if err == nil {
125 return block, nil
126 }
@@ -109,7 +129,7 @@ func (s *BlockService) GetBlock(ctx context.Context, k key.Key) (blocks.Block, e
129 // TODO be careful checking ErrNotFound. If the underlying
130 // implementation changes, this will break.
131 log.Debug("Blockservice: Searching bitswap")
112 - blk, err := s.Exchange.GetBlock(ctx, k)
132 + blk, err := s.Exchange.GetBlock(ctx, key.Key(c.Hash()))
133 if err != nil {
134 if err == blockstore.ErrNotFound {
135 return nil, ErrNotFound
@@ -130,12 +150,13 @@ func (s *BlockService) GetBlock(ctx context.Context, k key.Key) (blocks.Block, e
150 // GetBlocks gets a list of blocks asynchronously and returns through
151 // the returned channel.
152 // NB: No guarantees are made about order.
133 -func (s *BlockService) GetBlocks(ctx context.Context, ks []key.Key) <-chan blocks.Block {
153 +func (s *BlockService) GetBlocks(ctx context.Context, ks []*cid.Cid) <-chan blocks.Block {
154 out := make(chan blocks.Block, 0)
155 go func() {
156 defer close(out)
157 var misses []key.Key
138 - for _, k := range ks {
158 + for _, c := range ks {
159 + k := key.Key(c.Hash())
160 hit, err := s.Blockstore.Get(k)
161 if err != nil {
162 misses = append(misses, k)
@@ -171,11 +192,19 @@ func (s *BlockService) GetBlocks(ctx context.Context, ks []key.Key) <-chan block
192 }
193
194 // DeleteBlock deletes a block in the blockservice from the datastore
174 -func (s *BlockService) DeleteBlock(k key.Key) error {
175 - return s.Blockstore.DeleteBlock(k)
195 +func (s *BlockService) DeleteObject(o Object) error {
196 + return s.Blockstore.DeleteBlock(o.Key())
197 }
198
199 func (s *BlockService) Close() error {
200 log.Debug("blockservice is shutting down...")
201 return s.Exchange.Close()
202 }
203 +
204 +type RawBlockObject struct {
205 + blocks.Block
206 +}
207 +
208 +func (rob *RawBlockObject) Cid() *cid.Cid {
209 + return cid.NewCidV0(rob.Block.Multihash())
210 +}
blockservice/test/blocks_test.go
+41 -23
@@ -2,80 +2,98 @@ package bstest
2
3 import (
4 "bytes"
5 + "fmt"
6 "testing"
7 "time"
8
9 blocks "github.com/ipfs/go-ipfs/blocks"
10 blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
10 - blocksutil "github.com/ipfs/go-ipfs/blocks/blocksutil"
11 key "github.com/ipfs/go-ipfs/blocks/key"
12 . "github.com/ipfs/go-ipfs/blockservice"
13 offline "github.com/ipfs/go-ipfs/exchange/offline"
14 +
15 ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
16 dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
17 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
18 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
19 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
20 )
21
22 +func newObject(data []byte) *testObject {
23 + return &testObject{
24 + Block: blocks.NewBlock(data),
25 + }
26 +}
27 +
28 +type testObject struct {
29 + blocks.Block
30 +}
31 +
32 +func (o *testObject) Cid() *cid.Cid {
33 + return cid.NewCidV0(o.Block.Multihash())
34 +}
35 +
36 func TestBlocks(t *testing.T) {
37 bstore := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
38 bs := New(bstore, offline.Exchange(bstore))
39 defer bs.Close()
40
25 - _, err := bs.GetBlock(context.Background(), key.Key(""))
26 - if err != ErrNotFound {
27 - t.Error("Empty String Key should error", err)
28 - }
29 -
30 - b := blocks.NewBlock([]byte("beep boop"))
41 + o := newObject([]byte("beep boop"))
42 h := u.Hash([]byte("beep boop"))
32 - if !bytes.Equal(b.Multihash(), h) {
43 + if !bytes.Equal(o.Multihash(), h) {
44 t.Error("Block Multihash and data multihash not equal")
45 }
46
36 - if b.Key() != key.Key(h) {
47 + if o.Key() != key.Key(h) {
48 t.Error("Block key and data multihash key not equal")
49 }
50
40 - k, err := bs.AddBlock(b)
51 + k, err := bs.AddObject(o)
52 if err != nil {
53 t.Error("failed to add block to BlockService", err)
54 return
55 }
56
46 - if k != b.Key() {
57 + if !k.Equals(o.Cid()) {
58 t.Error("returned key is not equal to block key", err)
59 }
60
61 ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
62 defer cancel()
52 - b2, err := bs.GetBlock(ctx, b.Key())
63 + b2, err := bs.GetBlock(ctx, o.Cid())
64 if err != nil {
65 t.Error("failed to retrieve block from BlockService", err)
66 return
67 }
68
58 - if b.Key() != b2.Key() {
69 + if o.Key() != b2.Key() {
70 t.Error("Block keys not equal.")
71 }
72
62 - if !bytes.Equal(b.Data(), b2.Data()) {
73 + if !bytes.Equal(o.RawData(), b2.RawData()) {
74 t.Error("Block data is not equal.")
75 }
76 }
77
78 +func makeObjects(n int) []*testObject {
79 + var out []*testObject
80 + for i := 0; i < n; i++ {
81 + out = append(out, newObject([]byte(fmt.Sprintf("object %d", i))))
82 + }
83 + return out
84 +}
85 +
86 func TestGetBlocksSequential(t *testing.T) {
87 var servs = Mocks(4)
88 for _, s := range servs {
89 defer s.Close()
90 }
72 - bg := blocksutil.NewBlockGenerator()
73 - blks := bg.Blocks(50)
91 + objs := makeObjects(50)
92
75 - var keys []key.Key
76 - for _, blk := range blks {
77 - keys = append(keys, blk.Key())
78 - servs[0].AddBlock(blk)
93 + var cids []*cid.Cid
94 + for _, o := range objs {
95 + cids = append(cids, o.Cid())
96 + servs[0].AddObject(o)
97 }
98
99 t.Log("one instance at a time, get blocks concurrently")
@@ -83,7 +101,7 @@ func TestGetBlocksSequential(t *testing.T) {
101 for i := 1; i < len(servs); i++ {
102 ctx, cancel := context.WithTimeout(context.Background(), time.Second*50)
103 defer cancel()
86 - out := servs[i].GetBlocks(ctx, keys)
104 + out := servs[i].GetBlocks(ctx, cids)
105 gotten := make(map[key.Key]blocks.Block)
106 for blk := range out {
107 if _, ok := gotten[blk.Key()]; ok {
@@ -91,8 +109,8 @@ func TestGetBlocksSequential(t *testing.T) {
109 }
110 gotten[blk.Key()] = blk
111 }
94 - if len(gotten) != len(blks) {
95 - t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(blks))
112 + if len(gotten) != len(objs) {
113 + t.Fatalf("Didnt get enough blocks back: %d/%d", len(gotten), len(objs))
114 }
115 }
116 }
core/commands/bitswap.go
+5 -4
@@ -13,6 +13,7 @@ import (
13 "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14 peer "gx/ipfs/QmWtbQU15LaB5B1JC2F7TV9P4K88vD3PpA4AJrwfCjhML8/go-libp2p-peer"
15 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
17 )
18
19 var BitswapCmd = &cmds.Command{
@@ -55,13 +56,13 @@ var unwantCmd = &cmds.Command{
56
57 var ks []key.Key
58 for _, arg := range req.Arguments() {
58 - dec := key.B58KeyDecode(arg)
59 - if dec == "" {
60 - res.SetError(fmt.Errorf("Incorrectly formatted key: %s", arg), cmds.ErrNormal)
59 + c, err := cid.Decode(arg)
60 + if err != nil {
61 + res.SetError(err, cmds.ErrNormal)
62 return
63 }
64
64 - ks = append(ks, dec)
65 + ks = append(ks, key.Key(c.Hash()))
66 }
67
68 bs.CancelWants(ks)
core/commands/block.go
+24 -20
@@ -14,8 +14,8 @@ import (
14 cmds "github.com/ipfs/go-ipfs/commands"
15 "github.com/ipfs/go-ipfs/pin"
16 ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
17 - mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
17 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
18 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
19 )
20
21 type BlockStat struct {
@@ -70,7 +70,7 @@ on raw ipfs blocks. It outputs the following to stdout:
70
71 res.SetOutput(&BlockStat{
72 Key: b.Key().B58String(),
73 - Size: len(b.Data()),
73 + Size: len(b.RawData()),
74 })
75 },
76 Type: BlockStat{},
@@ -101,7 +101,7 @@ It outputs to stdout, and <key> is a base58 encoded multihash.
101 return
102 }
103
104 - res.SetOutput(bytes.NewReader(b.Data()))
104 + res.SetOutput(bytes.NewReader(b.RawData()))
105 },
106 }
107
@@ -145,7 +145,7 @@ It reads from stdin, and <key> is a base58 encoded multihash.
145 b := blocks.NewBlock(data)
146 log.Debugf("BlockPut key: '%q'", b.Key())
147
148 - k, err := n.Blocks.AddBlock(b)
148 + k, err := n.Blocks.AddObject(b)
149 if err != nil {
150 res.SetError(err, cmds.ErrNormal)
151 return
@@ -175,13 +175,12 @@ func getBlockForKey(req cmds.Request, skey string) (blocks.Block, error) {
175 return nil, errors.New("Not a valid hash")
176 }
177
178 - h, err := mh.FromB58String(skey)
178 + c, err := cid.Decode(skey)
179 if err != nil {
180 return nil, err
181 }
182
183 - k := key.Key(h)
184 - b, err := n.Blocks.GetBlock(req.Context(), k)
183 + b, err := n.Blocks.GetBlock(req.Context(), c)
184 if err != nil {
185 return nil, err
186 }
@@ -214,17 +213,22 @@ It takes a list of base58 encoded multihashs to remove.
213 hashes := req.Arguments()
214 force, _, _ := req.Option("force").Bool()
215 quiet, _, _ := req.Option("quiet").Bool()
217 - keys := make([]key.Key, 0, len(hashes))
216 + cids := make([]*cid.Cid, 0, len(hashes))
217 for _, hash := range hashes {
219 - k := key.B58KeyDecode(hash)
220 - keys = append(keys, k)
218 + c, err := cid.Decode(hash)
219 + if err != nil {
220 + res.SetError(fmt.Errorf("invalid content id: %s (%s)", hash, err), cmds.ErrNormal)
221 + return
222 + }
223 +
224 + cids = append(cids, c)
225 }
226 outChan := make(chan interface{})
227 res.SetOutput((<-chan interface{})(outChan))
228 go func() {
229 defer close(outChan)
230 pinning := n.Pinning
227 - err := rmBlocks(n.Blockstore, pinning, outChan, keys, rmBlocksOpts{
231 + err := rmBlocks(n.Blockstore, pinning, outChan, cids, rmBlocksOpts{
232 quiet: quiet,
233 force: force,
234 })
@@ -275,31 +279,31 @@ type rmBlocksOpts struct {
279 force bool
280 }
281
278 -func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, keys []key.Key, opts rmBlocksOpts) error {
282 +func rmBlocks(blocks bs.GCBlockstore, pins pin.Pinner, out chan<- interface{}, cids []*cid.Cid, opts rmBlocksOpts) error {
283 unlocker := blocks.GCLock()
284 defer unlocker.Unlock()
285
282 - stillOkay, err := checkIfPinned(pins, keys, out)
286 + stillOkay, err := checkIfPinned(pins, cids, out)
287 if err != nil {
288 return fmt.Errorf("pin check failed: %s", err)
289 }
290
287 - for _, k := range stillOkay {
288 - err := blocks.DeleteBlock(k)
291 + for _, c := range stillOkay {
292 + err := blocks.DeleteBlock(key.Key(c.Hash()))
293 if err != nil && opts.force && (err == bs.ErrNotFound || err == ds.ErrNotFound) {
294 // ignore non-existent blocks
295 } else if err != nil {
292 - out <- &RemovedBlock{Hash: k.String(), Error: err.Error()}
296 + out <- &RemovedBlock{Hash: c.String(), Error: err.Error()}
297 } else if !opts.quiet {
294 - out <- &RemovedBlock{Hash: k.String()}
298 + out <- &RemovedBlock{Hash: c.String()}
299 }
300 }
301 return nil
302 }
303
300 -func checkIfPinned(pins pin.Pinner, keys []key.Key, out chan<- interface{}) ([]key.Key, error) {
301 - stillOkay := make([]key.Key, 0, len(keys))
302 - res, err := pins.CheckIfPinned(keys...)
304 +func checkIfPinned(pins pin.Pinner, cids []*cid.Cid, out chan<- interface{}) ([]*cid.Cid, error) {
305 + stillOkay := make([]*cid.Cid, 0, len(cids))
306 + res, err := pins.CheckIfPinned(cids...)
307 if err != nil {
308 return nil, err
309 }
core/commands/dht.go
+25 -16
@@ -14,10 +14,12 @@ import (
14 path "github.com/ipfs/go-ipfs/path"
15 routing "github.com/ipfs/go-ipfs/routing"
16 ipdht "github.com/ipfs/go-ipfs/routing/dht"
17 +
18 pstore "gx/ipfs/QmSZi9ygLohBUGyHMqE5N6eToPwqcg7bZQTULeVLFu7Q6d/go-libp2p-peerstore"
19 peer "gx/ipfs/QmWtbQU15LaB5B1JC2F7TV9P4K88vD3PpA4AJrwfCjhML8/go-libp2p-peer"
20 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
21 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
22 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
23 )
24
25 var ErrNotDHT = errors.New("routing service is not a DHT")
@@ -257,26 +259,26 @@ var provideRefDhtCmd = &cmds.Command{
259
260 rec, _, _ := req.Option("recursive").Bool()
261
260 - var keys []key.Key
262 + var cids []*cid.Cid
263 for _, arg := range req.Arguments() {
262 - k := key.B58KeyDecode(arg)
263 - if k == "" {
264 - res.SetError(fmt.Errorf("incorrectly formatted key: ", arg), cmds.ErrNormal)
264 + c, err := cid.Decode(arg)
265 + if err != nil {
266 + res.SetError(err, cmds.ErrNormal)
267 return
268 }
269
268 - has, err := n.Blockstore.Has(k)
270 + has, err := n.Blockstore.Has(key.Key(c.Hash()))
271 if err != nil {
272 res.SetError(err, cmds.ErrNormal)
273 return
274 }
275
276 if !has {
275 - res.SetError(fmt.Errorf("block %s not found locally, cannot provide", k), cmds.ErrNormal)
277 + res.SetError(fmt.Errorf("block %s not found locally, cannot provide", c), cmds.ErrNormal)
278 return
279 }
280
279 - keys = append(keys, k)
281 + cids = append(cids, c)
282 }
283
284 outChan := make(chan interface{})
@@ -296,9 +298,9 @@ var provideRefDhtCmd = &cmds.Command{
298 defer close(events)
299 var err error
300 if rec {
299 - err = provideKeysRec(ctx, n.Routing, n.DAG, keys)
301 + err = provideKeysRec(ctx, n.Routing, n.DAG, cids)
302 } else {
301 - err = provideKeys(ctx, n.Routing, keys)
303 + err = provideKeys(ctx, n.Routing, cids)
304 }
305 if err != nil {
306 notif.PublishQueryEvent(ctx, &notif.QueryEvent{
@@ -345,9 +347,9 @@ var provideRefDhtCmd = &cmds.Command{
347 Type: notif.QueryEvent{},
348 }
349
348 -func provideKeys(ctx context.Context, r routing.IpfsRouting, keys []key.Key) error {
349 - for _, k := range keys {
350 - err := r.Provide(ctx, k)
350 +func provideKeys(ctx context.Context, r routing.IpfsRouting, cids []*cid.Cid) error {
351 + for _, c := range cids {
352 + err := r.Provide(ctx, key.Key(c.Hash()))
353 if err != nil {
354 return err
355 }
@@ -355,16 +357,23 @@ func provideKeys(ctx context.Context, r routing.IpfsRouting, keys []key.Key) err
357 return nil
358 }
359
358 -func provideKeysRec(ctx context.Context, r routing.IpfsRouting, dserv dag.DAGService, keys []key.Key) error {
360 +func provideKeysRec(ctx context.Context, r routing.IpfsRouting, dserv dag.DAGService, cids []*cid.Cid) error {
361 provided := make(map[key.Key]struct{})
360 - for _, k := range keys {
362 + for _, c := range cids {
363 kset := key.NewKeySet()
362 - node, err := dserv.Get(ctx, k)
364 + node, err := dserv.Get(ctx, c)
365 if err != nil {
366 return err
367 }
368
367 - err = dag.EnumerateChildrenAsync(ctx, dserv, node, kset)
369 + err = dag.EnumerateChildrenAsync(ctx, dserv, node, func(c *cid.Cid) bool {
370 + k := key.Key(c.Hash())
371 + if kset.Has(k) {
372 + kset.Add(k)
373 + return true
374 + }
375 + return false
376 + })
377 if err != nil {
378 return err
379 }
core/commands/files/files.go
+2 -5
@@ -158,10 +158,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
158 return nil, err
159 }
160
161 - k, err := nd.Key()
162 - if err != nil {
163 - return nil, err
164 - }
161 + c := nd.Cid()
162
163 d, err := ft.FromBytes(nd.Data())
164 if err != nil {
@@ -184,7 +181,7 @@ func statNode(ds dag.DAGService, fsn mfs.FSNode) (*Object, error) {
181 }
182
183 return &Object{
187 - Hash: k.B58String(),
184 + Hash: c.String(),
185 Blocks: len(nd.Links),
186 Size: d.GetFilesize(),
187 CumulativeSize: cumulsize,
core/commands/ls.go
+1 -1
@@ -97,7 +97,7 @@ The JSON output contains type information.
97 res.SetError(err, cmds.ErrNormal)
98 return
99 }
100 - linkNode, err = merkledag.DecodeProtobuf(b.Data())
100 + linkNode, err = merkledag.DecodeProtobuf(b.RawData())
101 if err != nil {
102 res.SetError(err, cmds.ErrNormal)
103 return
core/commands/object/object.go
+9 -8
@@ -87,7 +87,12 @@ is the raw data of the object.
87 return
88 }
89
90 - fpath := path.Path(req.Arguments()[0])
90 + fpath, err := path.ParsePath(req.Arguments()[0])
91 + if err != nil {
92 + res.SetError(err, cmds.ErrNormal)
93 + return
94 + }
95 +
96 node, err := core.Resolve(req.Context(), n, fpath)
97 if err != nil {
98 res.SetError(err, cmds.ErrNormal)
@@ -424,7 +429,7 @@ Available templates:
429 res.SetError(err, cmds.ErrNormal)
430 return
431 }
427 - res.SetOutput(&Object{Hash: k.B58String()})
432 + res.SetOutput(&Object{Hash: k.String()})
433 },
434 Marshalers: cmds.MarshalerMap{
435 cmds.Text: func(res cmds.Response) (io.Reader, error) {
@@ -538,13 +543,9 @@ func getObjectEnc(o interface{}) objectEncoding {
543 }
544
545 func getOutput(dagnode *dag.Node) (*Object, error) {
541 - key, err := dagnode.Key()
542 - if err != nil {
543 - return nil, err
544 - }
545 -
546 + c := dagnode.Cid()
547 output := &Object{
547 - Hash: key.B58String(),
548 + Hash: c.String(),
549 Links: make([]Link, len(dagnode.Links)),
550 }
551
core/commands/object/patch.go
+6 -14
@@ -99,7 +99,7 @@ the limit will not be respected by the network.
99 return
100 }
101
102 - res.SetOutput(&Object{Hash: newkey.B58String()})
102 + res.SetOutput(&Object{Hash: newkey.String()})
103 },
104 Type: Object{},
105 Marshalers: cmds.MarshalerMap{
@@ -161,7 +161,7 @@ Example:
161 return
162 }
163
164 - res.SetOutput(&Object{Hash: newkey.B58String()})
164 + res.SetOutput(&Object{Hash: newkey.String()})
165 },
166 Type: Object{},
167 Marshalers: cmds.MarshalerMap{
@@ -215,13 +215,9 @@ Removes a link by the given name from root.
215 return
216 }
217
218 - nk, err := nnode.Key()
219 - if err != nil {
220 - res.SetError(err, cmds.ErrNormal)
221 - return
222 - }
218 + nc := nnode.Cid()
219
224 - res.SetOutput(&Object{Hash: nk.B58String()})
220 + res.SetOutput(&Object{Hash: nc.String()})
221 },
222 Type: Object{},
223 Marshalers: cmds.MarshalerMap{
@@ -310,13 +306,9 @@ to a file containing 'bar', and returns the hash of the new object.
306 return
307 }
308
313 - nk, err := nnode.Key()
314 - if err != nil {
315 - res.SetError(err, cmds.ErrNormal)
316 - return
317 - }
309 + nc := nnode.Cid()
310
319 - res.SetOutput(&Object{Hash: nk.B58String()})
311 + res.SetOutput(&Object{Hash: nc.String()})
312 },
313 Type: Object{},
314 Marshalers: cmds.MarshalerMap{
core/commands/pin.go
+15 -12
@@ -5,15 +5,16 @@ import (
5 "fmt"
6 "io"
7
8 - key "github.com/ipfs/go-ipfs/blocks/key"
8 cmds "github.com/ipfs/go-ipfs/commands"
9 core "github.com/ipfs/go-ipfs/core"
10 corerepo "github.com/ipfs/go-ipfs/core/corerepo"
11 dag "github.com/ipfs/go-ipfs/merkledag"
12 path "github.com/ipfs/go-ipfs/path"
13 pin "github.com/ipfs/go-ipfs/pin"
14 +
15 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
17 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
18 )
19
20 var PinCmd = &cmds.Command{
@@ -29,7 +30,7 @@ var PinCmd = &cmds.Command{
30 }
31
32 type PinOutput struct {
32 - Pins []key.Key
33 + Pins []*cid.Cid
34 }
35
36 var addPinCmd = &cmds.Command{
@@ -271,12 +272,12 @@ func pinLsKeys(args []string, typeStr string, ctx context.Context, n *core.IpfsN
272 keys := make(map[string]RefKeyObject)
273
274 for _, p := range args {
274 - dagNode, err := core.Resolve(ctx, n, path.Path(p))
275 + pth, err := path.ParsePath(p)
276 if err != nil {
277 return nil, err
278 }
279
279 - k, err := dagNode.Key()
280 + dagNode, err := core.Resolve(ctx, n, pth)
281 if err != nil {
282 return nil, err
283 }
@@ -286,7 +287,8 @@ func pinLsKeys(args []string, typeStr string, ctx context.Context, n *core.IpfsN
287 return nil, fmt.Errorf("Invalid pin mode '%s'", typeStr)
288 }
289
289 - pinType, pinned, err := n.Pinning.IsPinnedWithType(k, mode)
290 + c := dagNode.Cid()
291 + pinType, pinned, err := n.Pinning.IsPinnedWithType(c, mode)
292 if err != nil {
293 return nil, err
294 }
@@ -300,7 +302,7 @@ func pinLsKeys(args []string, typeStr string, ctx context.Context, n *core.IpfsN
302 default:
303 pinType = "indirect through " + pinType
304 }
303 - keys[k.B58String()] = RefKeyObject{
305 + keys[c.String()] = RefKeyObject{
306 Type: pinType,
307 }
308 }
@@ -312,9 +314,9 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string
314
315 keys := make(map[string]RefKeyObject)
316
315 - AddToResultKeys := func(keyList []key.Key, typeStr string) {
316 - for _, k := range keyList {
317 - keys[k.B58String()] = RefKeyObject{
317 + AddToResultKeys := func(keyList []*cid.Cid, typeStr string) {
318 + for _, c := range keyList {
319 + keys[c.String()] = RefKeyObject{
320 Type: typeStr,
321 }
322 }
@@ -324,18 +326,19 @@ func pinLsAll(typeStr string, ctx context.Context, n *core.IpfsNode) (map[string
326 AddToResultKeys(n.Pinning.DirectKeys(), "direct")
327 }
328 if typeStr == "indirect" || typeStr == "all" {
327 - ks := key.NewKeySet()
329 + set := cid.NewSet()
330 for _, k := range n.Pinning.RecursiveKeys() {
331 nd, err := n.DAG.Get(ctx, k)
332 if err != nil {
333 return nil, err
334 }
333 - err = dag.EnumerateChildren(n.Context(), n.DAG, nd, ks, false)
335 +
336 + err = dag.EnumerateChildren(n.Context(), n.DAG, nd, set.Visit, false)
337 if err != nil {
338 return nil, err
339 }
340 }
338 - AddToResultKeys(ks.Keys(), "indirect")
341 + AddToResultKeys(set.Keys(), "indirect")
342 }
343 if typeStr == "recursive" || typeStr == "all" {
344 AddToResultKeys(n.Pinning.RecursiveKeys(), "recursive")
core/commands/refs.go
+23 -27
@@ -11,8 +11,10 @@ import (
11 "github.com/ipfs/go-ipfs/core"
12 dag "github.com/ipfs/go-ipfs/merkledag"
13 path "github.com/ipfs/go-ipfs/path"
14 +
15 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
17 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
18 )
19
20 // KeyList is a general type for outputting lists of keys
@@ -220,7 +222,7 @@ type RefWriter struct {
222 Recursive bool
223 PrintFmt string
224
223 - seen map[key.Key]struct{}
225 + seen *cid.Set
226 }
227
228 // WriteRefs writes refs of the given object to the underlying writer.
@@ -232,19 +234,16 @@ func (rw *RefWriter) WriteRefs(n *dag.Node) (int, error) {
234 }
235
236 func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) {
235 - nkey, err := n.Key()
236 - if err != nil {
237 - return 0, err
238 - }
237 + nc := n.Cid()
238
239 var count int
240 for i, ng := range dag.GetDAG(rw.Ctx, rw.DAG, n) {
242 - lk := key.Key(n.Links[i].Hash)
243 - if rw.skip(lk) {
241 + lc := cid.NewCidV0(n.Links[i].Hash)
242 + if rw.skip(lc) {
243 continue
244 }
245
247 - if err := rw.WriteEdge(nkey, lk, n.Links[i].Name); err != nil {
246 + if err := rw.WriteEdge(nc, lc, n.Links[i].Name); err != nil {
247 return count, err
248 }
249
@@ -263,24 +262,21 @@ func (rw *RefWriter) writeRefsRecursive(n *dag.Node) (int, error) {
262 }
263
264 func (rw *RefWriter) writeRefsSingle(n *dag.Node) (int, error) {
266 - nkey, err := n.Key()
267 - if err != nil {
268 - return 0, err
269 - }
265 + c := n.Cid()
266
271 - if rw.skip(nkey) {
267 + if rw.skip(c) {
268 return 0, nil
269 }
270
271 count := 0
272 for _, l := range n.Links {
277 - lk := key.Key(l.Hash)
273 + lc := cid.NewCidV0(l.Hash)
274
279 - if rw.skip(lk) {
275 + if rw.skip(lc) {
276 continue
277 }
278
283 - if err := rw.WriteEdge(nkey, lk, l.Name); err != nil {
279 + if err := rw.WriteEdge(c, lc, l.Name); err != nil {
280 return count, err
281 }
282 count++
@@ -288,25 +284,25 @@ func (rw *RefWriter) writeRefsSingle(n *dag.Node) (int, error) {
284 return count, nil
285 }
286
291 -// skip returns whether to skip a key
292 -func (rw *RefWriter) skip(k key.Key) bool {
287 +// skip returns whether to skip a cid
288 +func (rw *RefWriter) skip(c *cid.Cid) bool {
289 if !rw.Unique {
290 return false
291 }
292
293 if rw.seen == nil {
298 - rw.seen = make(map[key.Key]struct{})
294 + rw.seen = cid.NewSet()
295 }
296
301 - _, found := rw.seen[k]
302 - if !found {
303 - rw.seen[k] = struct{}{}
297 + has := rw.seen.Has(c)
298 + if !has {
299 + rw.seen.Add(c)
300 }
305 - return found
301 + return has
302 }
303
304 // Write one edge
309 -func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error {
305 +func (rw *RefWriter) WriteEdge(from, to *cid.Cid, linkname string) error {
306 if rw.Ctx != nil {
307 select {
308 case <-rw.Ctx.Done(): // just in case.
@@ -319,11 +315,11 @@ func (rw *RefWriter) WriteEdge(from, to key.Key, linkname string) error {
315 switch {
316 case rw.PrintFmt != "":
317 s = rw.PrintFmt
322 - s = strings.Replace(s, "<src>", from.B58String(), -1)
323 - s = strings.Replace(s, "<dst>", to.B58String(), -1)
318 + s = strings.Replace(s, "<src>", from.String(), -1)
319 + s = strings.Replace(s, "<dst>", to.String(), -1)
320 s = strings.Replace(s, "<linkname>", linkname, -1)
321 default:
326 - s += to.B58String()
322 + s += to.String()
323 }
324
325 rw.out <- &RefWrapper{Ref: s}
core/commands/resolve.go
+2 -6
@@ -105,13 +105,9 @@ Resolve the value of an IPFS DAG path:
105 return
106 }
107
108 - key, err := node.Key()
109 - if err != nil {
110 - res.SetError(err, cmds.ErrNormal)
111 - return
112 - }
108 + c := node.Cid()
109
114 - res.SetOutput(&ResolvedPath{path.FromKey(key)})
110 + res.SetOutput(&ResolvedPath{path.FromCid(c)})
111 },
112 Marshalers: cmds.MarshalerMap{
113 cmds.Text: func(res cmds.Response) (io.Reader, error) {
core/commands/tar.go
+2 -6
@@ -53,16 +53,12 @@ represent it.
53 return
54 }
55
56 - k, err := node.Key()
57 - if err != nil {
58 - res.SetError(err, cmds.ErrNormal)
59 - return
60 - }
56 + c := node.Cid()
57
58 fi.FileName()
59 res.SetOutput(&coreunix.AddedObject{
60 Name: fi.FileName(),
65 - Hash: k.B58String(),
61 + Hash: c.String(),
62 })
63 },
64 Type: coreunix.AddedObject{},
core/commands/unixfs/ls.go
+3 -7
@@ -93,13 +93,9 @@ possible, please use 'ipfs ls' instead.
93 return
94 }
95
96 - key, err := merkleNode.Key()
97 - if err != nil {
98 - res.SetError(err, cmds.ErrNormal)
99 - return
100 - }
96 + c := merkleNode.Cid()
97
102 - hash := key.B58String()
98 + hash := c.String()
99 output.Arguments[fpath] = hash
100
101 if _, ok := output.Objects[hash]; ok {
@@ -116,7 +112,7 @@ possible, please use 'ipfs ls' instead.
112 t := unixFSNode.GetType()
113
114 output.Objects[hash] = &LsObject{
119 - Hash: key.String(),
115 + Hash: c.String(),
116 Type: t.String(),
117 Size: unixFSNode.GetFilesize(),
118 }
core/core.go
+9 -5
@@ -35,6 +35,7 @@ import (
35 swarm "gx/ipfs/Qmf4ETeAWXuThBfWwonVyFqGFSgTWepUDEr1txcctvpTXS/go-libp2p/p2p/net/swarm"
36 addrutil "gx/ipfs/Qmf4ETeAWXuThBfWwonVyFqGFSgTWepUDEr1txcctvpTXS/go-libp2p/p2p/net/swarm/addr"
37 ping "gx/ipfs/Qmf4ETeAWXuThBfWwonVyFqGFSgTWepUDEr1txcctvpTXS/go-libp2p/p2p/protocol/ping"
38 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
39
40 routing "github.com/ipfs/go-ipfs/routing"
41 dht "github.com/ipfs/go-ipfs/routing/dht"
@@ -42,7 +43,6 @@ import (
43 offroute "github.com/ipfs/go-ipfs/routing/offline"
44
45 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
45 - key "github.com/ipfs/go-ipfs/blocks/key"
46 bserv "github.com/ipfs/go-ipfs/blockservice"
47 exchange "github.com/ipfs/go-ipfs/exchange"
48 bitswap "github.com/ipfs/go-ipfs/exchange/bitswap"
@@ -487,8 +487,8 @@ func (n *IpfsNode) loadBootstrapPeers() ([]pstore.PeerInfo, error) {
487
488 func (n *IpfsNode) loadFilesRoot() error {
489 dsk := ds.NewKey("/local/filesroot")
490 - pf := func(ctx context.Context, k key.Key) error {
491 - return n.Repo.Datastore().Put(dsk, []byte(k))
490 + pf := func(ctx context.Context, c *cid.Cid) error {
491 + return n.Repo.Datastore().Put(dsk, c.Bytes())
492 }
493
494 var nd *merkledag.Node
@@ -502,8 +502,12 @@ func (n *IpfsNode) loadFilesRoot() error {
502 return fmt.Errorf("failure writing to dagstore: %s", err)
503 }
504 case err == nil:
505 - k := key.Key(val.([]byte))
506 - nd, err = n.DAG.Get(n.Context(), k)
505 + c, err := cid.Cast(val.([]byte))
506 + if err != nil {
507 + return err
508 + }
509 +
510 + nd, err = n.DAG.Get(n.Context(), c)
511 if err != nil {
512 return fmt.Errorf("error loading filesroot from DAG: %s", err)
513 }
core/corehttp/gateway_handler.go
+23 -32
@@ -12,8 +12,8 @@ import (
12
13 humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
15 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
16
16 - key "github.com/ipfs/go-ipfs/blocks/key"
17 core "github.com/ipfs/go-ipfs/core"
18 "github.com/ipfs/go-ipfs/importer"
19 chunk "github.com/ipfs/go-ipfs/importer/chunk"
@@ -357,14 +357,20 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
357 newPath = path.Join(rsegs[2:])
358 }
359
360 - var newkey key.Key
360 + var newcid *cid.Cid
361 rnode, err := core.Resolve(ctx, i.node, rootPath)
362 switch ev := err.(type) {
363 case path.ErrNoLink:
364 // ev.Node < node where resolve failed
365 // ev.Name < new link
366 // but we need to patch from the root
367 - rnode, err := i.node.DAG.Get(ctx, key.B58KeyDecode(rsegs[1]))
367 + c, err := cid.Decode(rsegs[1])
368 + if err != nil {
369 + webError(w, "putHandler: bad input path", err, http.StatusBadRequest)
370 + return
371 + }
372 +
373 + rnode, err := i.node.DAG.Get(ctx, c)
374 if err != nil {
375 webError(w, "putHandler: Could not create DAG from request", err, http.StatusInternalServerError)
376 return
@@ -383,21 +389,17 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
389 return
390 }
391
386 - newkey, err = nnode.Key()
387 - if err != nil {
388 - webError(w, "putHandler: could not get key of edited node", err, http.StatusInternalServerError)
389 - return
390 - }
392 + newcid = nnode.Cid()
393
394 case nil:
395 // object set-data case
396 rnode.SetData(newnode.Data())
397
396 - newkey, err = i.node.DAG.Add(rnode)
398 + newcid, err = i.node.DAG.Add(rnode)
399 if err != nil {
398 - nnk, _ := newnode.Key()
399 - rk, _ := rnode.Key()
400 - webError(w, fmt.Sprintf("putHandler: Could not add newnode(%q) to root(%q)", nnk.B58String(), rk.B58String()), err, http.StatusInternalServerError)
400 + nnk := newnode.Cid()
401 + rk := rnode.Cid()
402 + webError(w, fmt.Sprintf("putHandler: Could not add newnode(%q) to root(%q)", nnk.String(), rk.String()), err, http.StatusInternalServerError)
403 return
404 }
405 default:
@@ -407,8 +409,8 @@ func (i *gatewayHandler) putHandler(w http.ResponseWriter, r *http.Request) {
409 }
410
411 i.addUserHeaders(w) // ok, _now_ write user's headers.
410 - w.Header().Set("IPFS-Hash", newkey.String())
411 - http.Redirect(w, r, gopath.Join(ipfsPathPrefix, newkey.String(), newPath), http.StatusCreated)
412 + w.Header().Set("IPFS-Hash", newcid.String())
413 + http.Redirect(w, r, gopath.Join(ipfsPathPrefix, newcid.String(), newPath), http.StatusCreated)
414 }
415
416 func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
@@ -416,20 +418,13 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
418 ctx, cancel := context.WithCancel(i.node.Context())
419 defer cancel()
420
419 - ipfsNode, err := core.Resolve(ctx, i.node, path.Path(urlPath))
421 + p, err := path.ParsePath(urlPath)
422 if err != nil {
421 - // FIXME HTTP error code
422 - webError(w, "Could not resolve name", err, http.StatusInternalServerError)
423 + webError(w, "failed to parse path", err, http.StatusBadRequest)
424 return
425 }
426
426 - k, err := ipfsNode.Key()
427 - if err != nil {
428 - webError(w, "Could not get key from resolved node", err, http.StatusInternalServerError)
429 - return
430 - }
431 -
432 - h, components, err := path.SplitAbsPath(path.FromKey(k))
427 + c, components, err := path.SplitAbsPath(p)
428 if err != nil {
429 webError(w, "Could not split path", err, http.StatusInternalServerError)
430 return
@@ -437,7 +432,7 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
432
433 tctx, cancel := context.WithTimeout(ctx, time.Minute)
434 defer cancel()
440 - rootnd, err := i.node.Resolver.DAG.Get(tctx, key.Key(h))
435 + rootnd, err := i.node.Resolver.DAG.Get(tctx, c)
436 if err != nil {
437 webError(w, "Could not resolve root object", err, http.StatusBadRequest)
438 return
@@ -475,15 +470,11 @@ func (i *gatewayHandler) deleteHandler(w http.ResponseWriter, r *http.Request) {
470 }
471
472 // Redirect to new path
478 - key, err := newnode.Key()
479 - if err != nil {
480 - webError(w, "Could not get key of new node", err, http.StatusInternalServerError)
481 - return
482 - }
473 + ncid := newnode.Cid()
474
475 i.addUserHeaders(w) // ok, _now_ write user's headers.
485 - w.Header().Set("IPFS-Hash", key.String())
486 - http.Redirect(w, r, gopath.Join(ipfsPathPrefix+key.String(), path.Join(components[:len(components)-1])), http.StatusCreated)
476 + w.Header().Set("IPFS-Hash", ncid.String())
477 + http.Redirect(w, r, gopath.Join(ipfsPathPrefix+ncid.String(), path.Join(components[:len(components)-1])), http.StatusCreated)
478 }
479
480 func (i *gatewayHandler) addUserHeaders(w http.ResponseWriter) {
core/corehttp/gateway_test.go
+2 -8
@@ -197,10 +197,7 @@ func TestIPNSHostnameRedirect(t *testing.T) {
197 t.Fatal(err)
198 }
199
200 - k, err := dagn1.Key()
201 - if err != nil {
202 - t.Fatal(err)
203 - }
200 + k := dagn1.Key()
201 t.Logf("k: %s\n", k)
202 ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
203
@@ -290,10 +287,7 @@ func TestIPNSHostnameBacklinks(t *testing.T) {
287 t.Fatal(err)
288 }
289
293 - k, err := dagn1.Key()
294 - if err != nil {
295 - t.Fatal(err)
296 - }
290 + k := dagn1.Key()
291 t.Logf("k: %s\n", k)
292 ns["/ipns/example.net"] = path.FromString("/ipfs/" + k.String())
293
core/corerepo/gc.go
+5 -6
@@ -9,9 +9,11 @@ import (
9 mfs "github.com/ipfs/go-ipfs/mfs"
10 gc "github.com/ipfs/go-ipfs/pin/gc"
11 repo "github.com/ipfs/go-ipfs/repo"
12 +
13 humanize "gx/ipfs/QmPSBJL4momYnE7DcUyk2DVhD6rH488ZmHBGLbxNdhU44K/go-humanize"
14 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
15 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
16 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
17 )
18
19 var log = logging.Logger("corerepo")
@@ -72,16 +74,13 @@ func NewGC(n *core.IpfsNode) (*GC, error) {
74 }, nil
75 }
76
75 -func BestEffortRoots(filesRoot *mfs.Root) ([]key.Key, error) {
77 +func BestEffortRoots(filesRoot *mfs.Root) ([]*cid.Cid, error) {
78 rootDag, err := filesRoot.GetValue().GetNode()
79 if err != nil {
80 return nil, err
81 }
80 - rootKey, err := rootDag.Key()
81 - if err != nil {
82 - return nil, err
83 - }
84 - return []key.Key{rootKey}, nil
82 +
83 + return []*cid.Cid{rootDag.Cid()}, nil
84 }
85
86 func GarbageCollect(n *core.IpfsNode, ctx context.Context) error {
core/corerepo/pinning.go
+11 -14
@@ -16,15 +16,15 @@ package corerepo
16 import (
17 "fmt"
18
19 - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
20 -
21 - key "github.com/ipfs/go-ipfs/blocks/key"
19 "github.com/ipfs/go-ipfs/core"
20 "github.com/ipfs/go-ipfs/merkledag"
21 path "github.com/ipfs/go-ipfs/path"
22 +
23 + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
24 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
25 )
26
27 -func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
27 +func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) {
28 dagnodes := make([]*merkledag.Node, 0)
29 for _, fpath := range paths {
30 dagnode, err := core.Resolve(ctx, n, path.Path(fpath))
@@ -34,20 +34,17 @@ func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool)
34 dagnodes = append(dagnodes, dagnode)
35 }
36
37 - var out []key.Key
37 + var out []*cid.Cid
38 for _, dagnode := range dagnodes {
39 - k, err := dagnode.Key()
40 - if err != nil {
41 - return nil, err
42 - }
39 + c := dagnode.Cid()
40
41 ctx, cancel := context.WithCancel(ctx)
42 defer cancel()
46 - err = n.Pinning.Pin(ctx, dagnode, recursive)
43 + err := n.Pinning.Pin(ctx, dagnode, recursive)
44 if err != nil {
45 return nil, fmt.Errorf("pin: %s", err)
46 }
50 - out = append(out, k)
47 + out = append(out, c)
48 }
49
50 err := n.Pinning.Flush()
@@ -58,16 +55,16 @@ func Pin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool)
55 return out, nil
56 }
57
61 -func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]key.Key, error) {
58 +func Unpin(n *core.IpfsNode, ctx context.Context, paths []string, recursive bool) ([]*cid.Cid, error) {
59
63 - var unpinned []key.Key
60 + var unpinned []*cid.Cid
61 for _, p := range paths {
62 p, err := path.ParsePath(p)
63 if err != nil {
64 return nil, err
65 }
66
70 - k, err := core.ResolveToKey(ctx, n, p)
67 + k, err := core.ResolveToCid(ctx, n, p)
68 if err != nil {
69 return nil, err
70 }
core/coreunix/add.go
+17 -38
@@ -7,24 +7,24 @@ import (
7 "os"
8 gopath "path"
9
10 + bs "github.com/ipfs/go-ipfs/blocks/blockstore"
11 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
11 - key "github.com/ipfs/go-ipfs/blocks/key"
12 bserv "github.com/ipfs/go-ipfs/blockservice"
13 + "github.com/ipfs/go-ipfs/commands/files"
14 + core "github.com/ipfs/go-ipfs/core"
15 "github.com/ipfs/go-ipfs/exchange/offline"
16 importer "github.com/ipfs/go-ipfs/importer"
17 "github.com/ipfs/go-ipfs/importer/chunk"
18 + dag "github.com/ipfs/go-ipfs/merkledag"
19 mfs "github.com/ipfs/go-ipfs/mfs"
20 "github.com/ipfs/go-ipfs/pin"
21 + unixfs "github.com/ipfs/go-ipfs/unixfs"
22 +
23 ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
24 syncds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
20 - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
21 -
22 - bs "github.com/ipfs/go-ipfs/blocks/blockstore"
23 - "github.com/ipfs/go-ipfs/commands/files"
24 - core "github.com/ipfs/go-ipfs/core"
25 - dag "github.com/ipfs/go-ipfs/merkledag"
26 - unixfs "github.com/ipfs/go-ipfs/unixfs"
25 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
26 + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
27 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
28 )
29
30 var log = logging.Logger("coreunix")
@@ -103,7 +103,7 @@ type Adder struct {
103 root *dag.Node
104 mr *mfs.Root
105 unlocker bs.Unlocker
106 - tempRoot key.Key
106 + tempRoot *cid.Cid
107 }
108
109 func (adder *Adder) SetMfsRoot(r *mfs.Root) {
@@ -166,7 +166,7 @@ func (adder *Adder) PinRoot() error {
166 return err
167 }
168
169 - if adder.tempRoot != "" {
169 + if adder.tempRoot != nil {
170 err := adder.pinning.Unpin(adder.ctx, adder.tempRoot, true)
171 if err != nil {
172 return err
@@ -259,12 +259,8 @@ func Add(n *core.IpfsNode, r io.Reader) (string, error) {
259 if err != nil {
260 return "", err
261 }
262 - k, err := node.Key()
263 - if err != nil {
264 - return "", err
265 - }
262
267 - return k.String(), nil
263 + return node.Cid().String(), nil
264 }
265
266 // AddR recursively adds files in |path|.
@@ -297,12 +293,7 @@ func AddR(n *core.IpfsNode, root string) (key string, err error) {
293 return "", err
294 }
295
300 - k, err := nd.Key()
301 - if err != nil {
302 - return "", err
303 - }
304 -
305 - return k.String(), nil
296 + return nd.String(), nil
297 }
298
299 // AddWrapped adds data from a reader, and wraps it with a directory object
@@ -329,23 +320,14 @@ func AddWrapped(n *core.IpfsNode, r io.Reader, filename string) (string, *dag.No
320 return "", nil, err
321 }
322
332 - k, err := dagnode.Key()
333 - if err != nil {
334 - return "", nil, err
335 - }
336 -
337 - return gopath.Join(k.String(), filename), dagnode, nil
323 + c := dagnode.Cid()
324 + return gopath.Join(c.String(), filename), dagnode, nil
325 }
326
327 func (adder *Adder) addNode(node *dag.Node, path string) error {
328 // patch it into the root
329 if path == "" {
343 - key, err := node.Key()
344 - if err != nil {
345 - return err
346 - }
347 -
348 - path = key.B58String()
330 + path = node.Cid().String()
331 }
332
333 dir := gopath.Dir(path)
@@ -490,13 +472,10 @@ func NewMemoryDagService() dag.DAGService {
472
473 // from core/commands/object.go
474 func getOutput(dagnode *dag.Node) (*Object, error) {
493 - key, err := dagnode.Key()
494 - if err != nil {
495 - return nil, err
496 - }
475 + c := dagnode.Cid()
476
477 output := &Object{
499 - Hash: key.B58String(),
478 + Hash: c.String(),
479 Links: make([]Link, len(dagnode.Links)),
480 }
481
core/coreunix/add_test.go
+10 -3
@@ -15,7 +15,9 @@ import (
15 "github.com/ipfs/go-ipfs/repo"
16 "github.com/ipfs/go-ipfs/repo/config"
17 "github.com/ipfs/go-ipfs/thirdparty/testutil"
18 +
19 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
20 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
21 )
22
23 func TestAddRecursive(t *testing.T) {
@@ -142,10 +144,14 @@ func TestAddGCLive(t *testing.T) {
144 }
145 }
146
145 - var last key.Key
147 + var last *cid.Cid
148 for a := range out {
149 // wait for it to finish
148 - last = key.B58KeyDecode(a.(*AddedObject).Hash)
150 + c, err := cid.Decode(a.(*AddedObject).Hash)
151 + if err != nil {
152 + t.Fatal(err)
153 + }
154 + last = c
155 }
156
157 ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
@@ -155,7 +161,8 @@ func TestAddGCLive(t *testing.T) {
161 t.Fatal(err)
162 }
163
158 - err = dag.EnumerateChildren(ctx, node.DAG, root, key.NewKeySet(), false)
164 + set := cid.NewSet()
165 + err = dag.EnumerateChildren(ctx, node.DAG, root, set.Visit, false)
166 if err != nil {
167 t.Fatal(err)
168 }
core/coreunix/metadata.go
+12 -6
@@ -1,16 +1,19 @@
1 package coreunix
2
3 import (
4 - key "github.com/ipfs/go-ipfs/blocks/key"
4 core "github.com/ipfs/go-ipfs/core"
5 dag "github.com/ipfs/go-ipfs/merkledag"
6 ft "github.com/ipfs/go-ipfs/unixfs"
7 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
8 )
9
10 func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error) {
11 - ukey := key.B58KeyDecode(skey)
11 + c, err := cid.Decode(skey)
12 + if err != nil {
13 + return "", err
14 + }
15
13 - nd, err := n.DAG.Get(n.Context(), ukey)
16 + nd, err := n.DAG.Get(n.Context(), c)
17 if err != nil {
18 return "", err
19 }
@@ -31,13 +34,16 @@ func AddMetadataTo(n *core.IpfsNode, skey string, m *ft.Metadata) (string, error
34 return "", err
35 }
36
34 - return nk.B58String(), nil
37 + return nk.String(), nil
38 }
39
40 func Metadata(n *core.IpfsNode, skey string) (*ft.Metadata, error) {
38 - ukey := key.B58KeyDecode(skey)
41 + c, err := cid.Decode(skey)
42 + if err != nil {
43 + return nil, err
44 + }
45
40 - nd, err := n.DAG.Get(n.Context(), ukey)
46 + nd, err := n.DAG.Get(n.Context(), c)
47 if err != nil {
48 return nil, err
49 }
core/coreunix/metadata_test.go
+13 -11
@@ -5,12 +5,7 @@ import (
5 "io/ioutil"
6 "testing"
7
8 - ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
9 - dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
10 - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
11 -
8 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
13 - key "github.com/ipfs/go-ipfs/blocks/key"
9 bserv "github.com/ipfs/go-ipfs/blockservice"
10 core "github.com/ipfs/go-ipfs/core"
11 offline "github.com/ipfs/go-ipfs/exchange/offline"
@@ -19,7 +14,12 @@ import (
14 merkledag "github.com/ipfs/go-ipfs/merkledag"
15 ft "github.com/ipfs/go-ipfs/unixfs"
16 uio "github.com/ipfs/go-ipfs/unixfs/io"
17 +
18 + ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
19 + dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
20 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
21 + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
22 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
23 )
24
25 func getDagserv(t *testing.T) merkledag.DAGService {
@@ -41,10 +41,7 @@ func TestMetadata(t *testing.T) {
41 t.Fatal(err)
42 }
43
44 - k, err := nd.Key()
45 - if err != nil {
46 - t.Fatal(err)
47 - }
44 + c := nd.Cid()
45
46 m := new(ft.Metadata)
47 m.MimeType = "THIS IS A TEST"
@@ -52,7 +49,7 @@ func TestMetadata(t *testing.T) {
49 // Such effort, many compromise
50 ipfsnode := &core.IpfsNode{DAG: ds}
51
55 - mdk, err := AddMetadataTo(ipfsnode, k.B58String(), m)
52 + mdk, err := AddMetadataTo(ipfsnode, c.String(), m)
53 if err != nil {
54 t.Fatal(err)
55 }
@@ -65,7 +62,12 @@ func TestMetadata(t *testing.T) {
62 t.Fatalf("something went wrong in conversion: '%s' != '%s'", rec.MimeType, m.MimeType)
63 }
64
68 - retnode, err := ds.Get(ctx, key.B58KeyDecode(mdk))
65 + cdk, err := cid.Decode(mdk)
66 + if err != nil {
67 + t.Fatal(err)
68 + }
69 +
70 + retnode, err := ds.Get(ctx, cdk)
71 if err != nil {
72 t.Fatal(err)
73 }
core/pathresolver.go
+7 -7
@@ -6,9 +6,9 @@ import (
6
7 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
8
9 - key "github.com/ipfs/go-ipfs/blocks/key"
9 merkledag "github.com/ipfs/go-ipfs/merkledag"
10 path "github.com/ipfs/go-ipfs/path"
11 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
12 )
13
14 // ErrNoNamesys is an explicit error for when an IPFS node doesn't
@@ -61,31 +61,31 @@ func Resolve(ctx context.Context, n *IpfsNode, p path.Path) (*merkledag.Node, er
61 // It first checks if the path is already in the form of just a key (<key> or
62 // /ipfs/<key>) and returns immediately if so. Otherwise, it falls back onto
63 // Resolve to perform resolution of the dagnode being referenced.
64 -func ResolveToKey(ctx context.Context, n *IpfsNode, p path.Path) (key.Key, error) {
64 +func ResolveToCid(ctx context.Context, n *IpfsNode, p path.Path) (*cid.Cid, error) {
65
66 // If the path is simply a key, parse and return it. Parsed paths are already
67 // normalized (read: prepended with /ipfs/ if needed), so segment[1] should
68 // always be the key.
69 if p.IsJustAKey() {
70 - return key.B58KeyDecode(p.Segments()[1]), nil
70 + return cid.Decode(p.Segments()[1])
71 }
72
73 // Fall back onto regular dagnode resolution. Retrieve the second-to-last
74 // segment of the path and resolve its link to the last segment.
75 head, tail, err := p.PopLastSegment()
76 if err != nil {
77 - return key.Key(""), err
77 + return nil, err
78 }
79 dagnode, err := Resolve(ctx, n, head)
80 if err != nil {
81 - return key.Key(""), err
81 + return nil, err
82 }
83
84 // Extract and return the key of the link to the target dag node.
85 link, err := dagnode.GetNodeLink(tail)
86 if err != nil {
87 - return key.Key(""), err
87 + return nil, err
88 }
89
90 - return key.Key(link.Hash), nil
90 + return cid.NewCidV0(link.Hash), nil
91 }
exchange/bitswap/bitswap.go
+9 -9
@@ -8,12 +8,6 @@ import (
8 "sync"
9 "time"
10
11 - process "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
12 - procctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context"
13 - logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
14 - peer "gx/ipfs/QmWtbQU15LaB5B1JC2F7TV9P4K88vD3PpA4AJrwfCjhML8/go-libp2p-peer"
15 - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
16 -
11 blocks "github.com/ipfs/go-ipfs/blocks"
12 blockstore "github.com/ipfs/go-ipfs/blocks/blockstore"
13 key "github.com/ipfs/go-ipfs/blocks/key"
@@ -26,6 +20,12 @@ import (
20 flags "github.com/ipfs/go-ipfs/flags"
21 "github.com/ipfs/go-ipfs/thirdparty/delay"
22 loggables "github.com/ipfs/go-ipfs/thirdparty/loggables"
23 +
24 + process "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess"
25 + procctx "gx/ipfs/QmQopLATEYMNg7dVqZRNDfeE2S1yKy8zrRh5xnYiuqeZBn/goprocess/context"
26 + logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
27 + peer "gx/ipfs/QmWtbQU15LaB5B1JC2F7TV9P4K88vD3PpA4AJrwfCjhML8/go-libp2p-peer"
28 + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
29 )
30
31 var log = logging.Logger("bitswap")
@@ -252,8 +252,8 @@ func (bs *Bitswap) GetBlocks(ctx context.Context, keys []key.Key) (<-chan blocks
252 }
253
254 // CancelWant removes a given key from the wantlist
255 -func (bs *Bitswap) CancelWants(ks []key.Key) {
256 - bs.wm.CancelWants(ks)
255 +func (bs *Bitswap) CancelWants(keys []key.Key) {
256 + bs.wm.CancelWants(keys)
257 }
258
259 // HasBlock announces the existance of a block to this bitswap service. The
@@ -343,7 +343,7 @@ func (bs *Bitswap) updateReceiveCounters(b blocks.Block) error {
343 }
344 if err == nil && has {
345 bs.dupBlocksRecvd++
346 - bs.dupDataRecvd += uint64(len(b.Data()))
346 + bs.dupDataRecvd += uint64(len(b.RawData()))
347 }
348
349 if has {
exchange/bitswap/bitswap_test.go
+5 -2
@@ -90,7 +90,7 @@ func TestGetBlockFromPeerAfterPeerAnnounces(t *testing.T) {
90 t.Fatal("Expected to succeed")
91 }
92
93 - if !bytes.Equal(block.Data(), received.Data()) {
93 + if !bytes.Equal(block.RawData(), received.RawData()) {
94 t.Fatal("Data doesn't match")
95 }
96 }
@@ -289,7 +289,10 @@ func TestEmptyKey(t *testing.T) {
289 defer sg.Close()
290 bs := sg.Instances(1)[0].Exchange
291
292 - _, err := bs.GetBlock(context.Background(), key.Key(""))
292 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5)
293 + defer cancel()
294 +
295 + _, err := bs.GetBlock(ctx, key.Key(""))
296 if err != blockstore.ErrNotFound {
297 t.Error("empty str key should return ErrNotFound")
298 }
exchange/bitswap/decision/engine.go
+3 -3
@@ -247,8 +247,8 @@ func (e *Engine) MessageReceived(p peer.ID, m bsmsg.BitSwapMessage) error {
247 }
248
249 for _, block := range m.Blocks() {
250 - log.Debugf("got block %s %d bytes", block.Key(), len(block.Data()))
251 - l.ReceivedBytes(len(block.Data()))
250 + log.Debugf("got block %s %d bytes", block, len(block.RawData()))
251 + l.ReceivedBytes(len(block.RawData()))
252 }
253 return nil
254 }
@@ -286,7 +286,7 @@ func (e *Engine) AddBlock(block blocks.Block) {
286 func (e *Engine) MessageSent(p peer.ID, m bsmsg.BitSwapMessage) error {
287 l := e.findOrCreate(p)
288 for _, block := range m.Blocks() {
289 - l.SentBytes(len(block.Data()))
289 + l.SentBytes(len(block.RawData()))
290 l.wantList.Remove(block.Key())
291 e.peerRequestQueue.Remove(block.Key(), p)
292 }
exchange/bitswap/decision/engine_test.go
+1 -1
@@ -188,7 +188,7 @@ func checkHandledInOrder(t *testing.T, e *Engine, keys []string) error {
188 received := envelope.Block
189 expected := blocks.NewBlock([]byte(k))
190 if received.Key() != expected.Key() {
191 - return errors.New(fmt.Sprintln("received", string(received.Data()), "expected", string(expected.Data())))
191 + return errors.New(fmt.Sprintln("received", string(received.RawData()), "expected", string(expected.RawData())))
192 }
193 }
194 return nil
exchange/bitswap/message/message.go
+1 -1
@@ -159,7 +159,7 @@ func (m *impl) ToProto() *pb.Message {
159 })
160 }
161 for _, b := range m.Blocks() {
162 - pbm.Blocks = append(pbm.Blocks, b.Data())
162 + pbm.Blocks = append(pbm.Blocks, b.RawData())
163 }
164 return pbm
165 }
exchange/bitswap/notifications/notifications_test.go
+1 -1
@@ -159,7 +159,7 @@ func assertBlockChannelNil(t *testing.T, blockChannel <-chan blocks.Block) {
159 }
160
161 func assertBlocksEqual(t *testing.T, a, b blocks.Block) {
162 - if !bytes.Equal(a.Data(), b.Data()) {
162 + if !bytes.Equal(a.RawData(), b.RawData()) {
163 t.Fatal("blocks aren't equal")
164 }
165 if a.Key() != b.Key() {
exchange/bitswap/testnet/network_test.go
+1 -1
@@ -44,7 +44,7 @@ func TestSendMessageAsyncButWaitForResponse(t *testing.T) {
44 // TODO assert that this came from the correct peer and that the message contents are as expected
45 ok := false
46 for _, b := range msgFromResponder.Blocks() {
47 - if string(b.Data()) == expectedStr {
47 + if string(b.RawData()) == expectedStr {
48 wg.Done()
49 ok = true
50 }
exchange/interface.go
+1
@@ -6,6 +6,7 @@ import (
6
7 blocks "github.com/ipfs/go-ipfs/blocks"
8 key "github.com/ipfs/go-ipfs/blocks/key"
9 +
10 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
11 )
12
exchange/offline/offline.go
+1
@@ -7,6 +7,7 @@ import (
7 "github.com/ipfs/go-ipfs/blocks/blockstore"
8 key "github.com/ipfs/go-ipfs/blocks/key"
9 exchange "github.com/ipfs/go-ipfs/exchange"
10 +
11 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
12 )
13
fuse/ipns/common.go
+1 -1
@@ -33,7 +33,7 @@ func InitializeKeyspace(n *core.IpfsNode, key ci.PrivKey) error {
33 }
34
35 pub := nsys.NewRoutingPublisher(n.Routing, n.Repo.Datastore())
36 - if err := pub.Publish(ctx, key, path.FromKey(nodek)); err != nil {
36 + if err := pub.Publish(ctx, key, path.FromCid(nodek)); err != nil {
37 return err
38 }
39
fuse/ipns/ipns_unix.go
+3 -2
@@ -21,6 +21,7 @@ import (
21 path "github.com/ipfs/go-ipfs/path"
22 ft "github.com/ipfs/go-ipfs/unixfs"
23 ci "gx/ipfs/QmVoi5es8D5fNHZDqoW6DgDAEPEV5hQp8GBz161vZXiwpQ/go-libp2p-crypto"
24 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
25 )
26
27 func init() {
@@ -81,8 +82,8 @@ type Root struct {
82 }
83
84 func ipnsPubFunc(ipfs *core.IpfsNode, k ci.PrivKey) mfs.PubFunc {
84 - return func(ctx context.Context, key key.Key) error {
85 - return ipfs.Namesys.Publish(ctx, k, path.FromKey(key))
85 + return func(ctx context.Context, c *cid.Cid) error {
86 + return ipfs.Namesys.Publish(ctx, k, path.FromCid(c))
87 }
88 }
89
fuse/readonly/ipfs_test.go
+11 -25
@@ -13,9 +13,6 @@ import (
13 "sync"
14 "testing"
15
16 - fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
17 -
18 - key "github.com/ipfs/go-ipfs/blocks/key"
16 core "github.com/ipfs/go-ipfs/core"
17 coreunix "github.com/ipfs/go-ipfs/core/coreunix"
18 coremock "github.com/ipfs/go-ipfs/core/mock"
@@ -24,7 +21,10 @@ import (
21 dag "github.com/ipfs/go-ipfs/merkledag"
22 ci "github.com/ipfs/go-ipfs/thirdparty/testutil/ci"
23 uio "github.com/ipfs/go-ipfs/unixfs/io"
24 +
25 + fstest "github.com/ipfs/go-ipfs/Godeps/_workspace/src/bazil.org/fuse/fs/fstestutil"
26 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
27 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
28 )
29
30 func maybeSkipFuseTests(t *testing.T) {
@@ -74,11 +74,7 @@ func TestIpfsBasicRead(t *testing.T) {
74 defer mnt.Close()
75
76 fi, data := randObj(t, nd, 10000)
77 - k, err := fi.Key()
78 - if err != nil {
79 - t.Fatal(err)
80 - }
81 -
77 + k := fi.Key()
78 fname := path.Join(mnt.Dir, k.String())
79 rbuf, err := ioutil.ReadFile(fname)
80 if err != nil {
@@ -114,7 +110,7 @@ func TestIpfsStressRead(t *testing.T) {
110 nd, mnt := setupIpfsTest(t, nil)
111 defer mnt.Close()
112
117 - var ks []key.Key
113 + var ks []*cid.Cid
114 var paths []string
115
116 nobj := 50
@@ -123,13 +119,9 @@ func TestIpfsStressRead(t *testing.T) {
119 // Make a bunch of objects
120 for i := 0; i < nobj; i++ {
121 fi, _ := randObj(t, nd, rand.Int63n(50000))
126 - k, err := fi.Key()
127 - if err != nil {
128 - t.Fatal(err)
129 - }
130 -
131 - ks = append(ks, k)
132 - paths = append(paths, k.String())
122 + c := fi.Cid()
123 + ks = append(ks, c)
124 + paths = append(paths, c.String())
125 }
126
127 // Now make a bunch of dirs
@@ -209,14 +201,11 @@ func TestIpfsBasicDirRead(t *testing.T) {
201
202 // Make a 'file'
203 fi, data := randObj(t, nd, 10000)
212 - k, err := fi.Key()
213 - if err != nil {
214 - t.Fatal(err)
215 - }
204 + k := fi.Cid()
205
206 // Make a directory and put that file in it
207 db := uio.NewDirectory(nd.DAG)
219 - err = db.AddChild(nd.Context(), "actual", k)
208 + err := db.AddChild(nd.Context(), "actual", k)
209 if err != nil {
210 t.Fatal(err)
211 }
@@ -259,10 +248,7 @@ func TestFileSizeReporting(t *testing.T) {
248 defer mnt.Close()
249
250 fi, data := randObj(t, nd, 10000)
262 - k, err := fi.Key()
263 - if err != nil {
264 - t.Fatal(err)
265 - }
251 + k := fi.Key()
252
253 fname := path.Join(mnt.Dir, k.String())
254
fuse/readonly/readonly_unix.go
+2 -5
@@ -163,15 +163,12 @@ func (s *Node) Readlink(ctx context.Context, req *fuse.ReadlinkRequest) (string,
163
164 func (s *Node) Read(ctx context.Context, req *fuse.ReadRequest, resp *fuse.ReadResponse) error {
165
166 - k, err := s.Nd.Key()
167 - if err != nil {
168 - return err
169 - }
166 + c := s.Nd.Cid()
167
168 // setup our logging event
169 lm := make(lgbl.DeferredMap)
170 lm["fs"] = "ipfs"
174 - lm["key"] = func() interface{} { return k.B58String() }
171 + lm["key"] = func() interface{} { return c.String() }
172 lm["req_offset"] = req.Offset
173 lm["req_size"] = req.Size
174 defer log.EventBegin(ctx, "fuseRead", lm).Done()
importer/balanced/balanced_test.go
+25
@@ -15,6 +15,7 @@ import (
15 mdtest "github.com/ipfs/go-ipfs/merkledag/test"
16 pin "github.com/ipfs/go-ipfs/pin"
17 uio "github.com/ipfs/go-ipfs/unixfs/io"
18 +
19 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
20 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
21 )
@@ -78,6 +79,30 @@ func TestBuilderConsistency(t *testing.T) {
79 testFileConsistency(t, 100000, chunk.DefaultBlockSize)
80 }
81
82 +func TestNoChunking(t *testing.T) {
83 + ds := mdtest.Mock()
84 +
85 + nd, should := getTestDag(t, ds, 1000, 2000)
86 + r, err := uio.NewDagReader(context.Background(), nd, ds)
87 + if err != nil {
88 + t.Fatal(err)
89 + }
90 +
91 + dagrArrComp(t, r, should)
92 +}
93 +
94 +func TestTwoChunks(t *testing.T) {
95 + ds := mdtest.Mock()
96 +
97 + nd, should := getTestDag(t, ds, 2000, 1000)
98 + r, err := uio.NewDagReader(context.Background(), nd, ds)
99 + if err != nil {
100 + t.Fatal(err)
101 + }
102 +
103 + dagrArrComp(t, r, should)
104 +}
105 +
106 func arrComp(a, b []byte) error {
107 if len(a) != len(b) {
108 return fmt.Errorf("Arrays differ in length. %d != %d", len(a), len(b))
merkledag/coding.go
+4 -3
@@ -4,10 +4,11 @@ import (
4 "fmt"
5 "sort"
6
7 - mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
8 -
7 pb "github.com/ipfs/go-ipfs/merkledag/pb"
8 +
9 + mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
10 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
11 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
12 )
13
14 // for now, we use a PBNode intermediate thing.
@@ -83,7 +84,7 @@ func (n *Node) EncodeProtobuf(force bool) ([]byte, error) {
84 }
85
86 if n.cached == nil {
86 - n.cached = u.Hash(n.encoded)
87 + n.cached = cid.NewCidV0(u.Hash(n.encoded))
88 }
89
90 return n.encoded, nil
merkledag/merkledag.go
+94 -101
@@ -6,11 +6,12 @@ import (
6 "strings"
7 "sync"
8
9 - blocks "github.com/ipfs/go-ipfs/blocks"
9 key "github.com/ipfs/go-ipfs/blocks/key"
10 bserv "github.com/ipfs/go-ipfs/blockservice"
11 +
12 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
13 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
14 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
15 )
16
17 var log = logging.Logger("merkledag")
@@ -18,13 +19,13 @@ var ErrNotFound = fmt.Errorf("merkledag: not found")
19
20 // DAGService is an IPFS Merkle DAG service.
21 type DAGService interface {
21 - Add(*Node) (key.Key, error)
22 - Get(context.Context, key.Key) (*Node, error)
22 + Add(*Node) (*cid.Cid, error)
23 + Get(context.Context, *cid.Cid) (*Node, error)
24 Remove(*Node) error
25
26 // GetDAG returns, in order, all the single leve child
27 // nodes of the passed in node.
27 - GetMany(context.Context, []key.Key) <-chan *NodeOption
28 + GetMany(context.Context, []*cid.Cid) <-chan *NodeOption
29
30 Batch() *Batch
31 }
@@ -43,24 +44,12 @@ type dagService struct {
44 }
45
46 // Add adds a node to the dagService, storing the block in the BlockService
46 -func (n *dagService) Add(nd *Node) (key.Key, error) {
47 +func (n *dagService) Add(nd *Node) (*cid.Cid, error) {
48 if n == nil { // FIXME remove this assertion. protect with constructor invariant
48 - return "", fmt.Errorf("dagService is nil")
49 - }
50 -
51 - d, err := nd.EncodeProtobuf(false)
52 - if err != nil {
53 - return "", err
54 - }
55 -
56 - mh, err := nd.Multihash()
57 - if err != nil {
58 - return "", err
49 + return nil, fmt.Errorf("dagService is nil")
50 }
51
61 - b, _ := blocks.NewBlockWithHash(d, mh)
62 -
63 - return n.Blocks.AddBlock(b)
52 + return n.Blocks.AddObject(nd)
53 }
54
55 func (n *dagService) Batch() *Batch {
@@ -68,56 +57,57 @@ func (n *dagService) Batch() *Batch {
57 }
58
59 // Get retrieves a node from the dagService, fetching the block in the BlockService
71 -func (n *dagService) Get(ctx context.Context, k key.Key) (*Node, error) {
72 - if k == "" {
73 - return nil, ErrNotFound
74 - }
60 +func (n *dagService) Get(ctx context.Context, c *cid.Cid) (*Node, error) {
61 if n == nil {
62 return nil, fmt.Errorf("dagService is nil")
63 }
64 +
65 ctx, cancel := context.WithCancel(ctx)
66 defer cancel()
67
81 - b, err := n.Blocks.GetBlock(ctx, k)
68 + b, err := n.Blocks.GetBlock(ctx, c)
69 if err != nil {
70 if err == bserv.ErrNotFound {
71 return nil, ErrNotFound
72 }
86 - return nil, fmt.Errorf("Failed to get block for %s: %v", k.B58String(), err)
73 + return nil, fmt.Errorf("Failed to get block for %s: %v", c, err)
74 }
75
89 - res, err := DecodeProtobuf(b.Data())
90 - if err != nil {
91 - if strings.Contains(err.Error(), "Unmarshal failed") {
92 - return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", k)
76 + var res *Node
77 + switch c.Type() {
78 + case cid.Protobuf:
79 + out, err := DecodeProtobuf(b.RawData())
80 + if err != nil {
81 + if strings.Contains(err.Error(), "Unmarshal failed") {
82 + return nil, fmt.Errorf("The block referred to by '%s' was not a valid merkledag node", c)
83 + }
84 + return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
85 }
94 - return nil, fmt.Errorf("Failed to decode Protocol Buffers: %v", err)
86 + res = out
87 + default:
88 + return nil, fmt.Errorf("unrecognized formatting type")
89 }
90
97 - res.cached = k.ToMultihash()
91 + res.cached = c
92
93 return res, nil
94 }
95
96 func (n *dagService) Remove(nd *Node) error {
103 - k, err := nd.Key()
104 - if err != nil {
105 - return err
106 - }
107 - return n.Blocks.DeleteBlock(k)
97 + return n.Blocks.DeleteObject(nd)
98 }
99
100 // FetchGraph fetches all nodes that are children of the given node
101 func FetchGraph(ctx context.Context, root *Node, serv DAGService) error {
112 - return EnumerateChildrenAsync(ctx, serv, root, key.NewKeySet())
102 + return EnumerateChildrenAsync(ctx, serv, root, cid.NewSet().Visit)
103 }
104
105 // FindLinks searches this nodes links for the given key,
106 // returns the indexes of any links pointing to it
117 -func FindLinks(links []key.Key, k key.Key, start int) []int {
107 +func FindLinks(links []*cid.Cid, c *cid.Cid, start int) []int {
108 var out []int
119 - for i, lnk_k := range links[start:] {
120 - if k == lnk_k {
109 + for i, lnk_c := range links[start:] {
110 + if c.Equals(lnk_c) {
111 out = append(out, i+start)
112 }
113 }
@@ -129,11 +119,21 @@ type NodeOption struct {
119 Err error
120 }
121
132 -func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeOption {
122 +func cidsToKeyMapping(cids []*cid.Cid) map[key.Key]*cid.Cid {
123 + mapping := make(map[key.Key]*cid.Cid)
124 + for _, c := range cids {
125 + mapping[key.Key(c.Hash())] = c
126 + }
127 + return mapping
128 +}
129 +
130 +func (ds *dagService) GetMany(ctx context.Context, keys []*cid.Cid) <-chan *NodeOption {
131 out := make(chan *NodeOption, len(keys))
132 blocks := ds.Blocks.GetBlocks(ctx, keys)
133 var count int
134
135 + mapping := cidsToKeyMapping(keys)
136 +
137 go func() {
138 defer close(out)
139 for {
@@ -145,12 +145,23 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
145 }
146 return
147 }
148 - nd, err := DecodeProtobuf(b.Data())
149 - if err != nil {
150 - out <- &NodeOption{Err: err}
148 +
149 + c := mapping[b.Key()]
150 +
151 + var nd *Node
152 + switch c.Type() {
153 + case cid.Protobuf:
154 + decnd, err := DecodeProtobuf(b.RawData())
155 + if err != nil {
156 + out <- &NodeOption{Err: err}
157 + return
158 + }
159 + decnd.cached = cid.NewCidV0(b.Multihash())
160 + nd = decnd
161 + default:
162 + out <- &NodeOption{Err: fmt.Errorf("unrecognized object type: %s", c.Type())}
163 return
164 }
153 - nd.cached = b.Key().ToMultihash()
165
166 // buffered, no need to select
167 out <- &NodeOption{Node: nd}
@@ -169,17 +180,17 @@ func (ds *dagService) GetMany(ctx context.Context, keys []key.Key) <-chan *NodeO
180 // It returns a channel of nodes, which the caller can receive
181 // all the child nodes of 'root' on, in proper order.
182 func GetDAG(ctx context.Context, ds DAGService, root *Node) []NodeGetter {
172 - var keys []key.Key
183 + var cids []*cid.Cid
184 for _, lnk := range root.Links {
174 - keys = append(keys, key.Key(lnk.Hash))
185 + cids = append(cids, cid.NewCidV0(lnk.Hash))
186 }
187
177 - return GetNodes(ctx, ds, keys)
188 + return GetNodes(ctx, ds, cids)
189 }
190
191 // GetNodes returns an array of 'NodeGetter' promises, with each corresponding
192 // to the key with the same index as the passed in keys
182 -func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
193 +func GetNodes(ctx context.Context, ds DAGService, keys []*cid.Cid) []NodeGetter {
194
195 // Early out if no work to do
196 if len(keys) == 0 {
@@ -216,14 +227,7 @@ func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
227 }
228
229 nd := opt.Node
219 -
220 - k, err := nd.Key()
221 - if err != nil {
222 - log.Error("Failed to get node key: ", err)
223 - continue
224 - }
225 -
226 - is := FindLinks(keys, k, 0)
230 + is := FindLinks(keys, nd.Cid(), 0)
231 for _, i := range is {
232 count++
233 promises[i].Send(nd)
@@ -237,16 +241,12 @@ func GetNodes(ctx context.Context, ds DAGService, keys []key.Key) []NodeGetter {
241 }
242
243 // Remove duplicates from a list of keys
240 -func dedupeKeys(ks []key.Key) []key.Key {
241 - kmap := make(map[key.Key]struct{})
242 - var out []key.Key
243 - for _, k := range ks {
244 - if _, ok := kmap[k]; !ok {
245 - kmap[k] = struct{}{}
246 - out = append(out, k)
247 - }
244 +func dedupeKeys(cids []*cid.Cid) []*cid.Cid {
245 + set := cid.NewSet()
246 + for _, c := range cids {
247 + set.Add(c)
248 }
249 - return out
249 + return set.Keys()
250 }
251
252 func newNodePromise(ctx context.Context) NodeGetter {
@@ -327,50 +327,44 @@ func (np *nodePromise) Get(ctx context.Context) (*Node, error) {
327 type Batch struct {
328 ds *dagService
329
330 - blocks []blocks.Block
330 + objects []bserv.Object
331 size int
332 MaxSize int
333 }
334
335 -func (t *Batch) Add(nd *Node) (key.Key, error) {
335 +func (t *Batch) Add(nd *Node) (*cid.Cid, error) {
336 d, err := nd.EncodeProtobuf(false)
337 if err != nil {
338 - return "", err
339 - }
340 -
341 - mh, err := nd.Multihash()
342 - if err != nil {
343 - return "", err
338 + return nil, err
339 }
340
346 - b, _ := blocks.NewBlockWithHash(d, mh)
347 -
348 - k := key.Key(mh)
349 -
350 - t.blocks = append(t.blocks, b)
351 - t.size += len(b.Data())
341 + t.objects = append(t.objects, nd)
342 + t.size += len(d)
343 if t.size > t.MaxSize {
353 - return k, t.Commit()
344 + return nd.Cid(), t.Commit()
345 }
355 - return k, nil
346 + return nd.Cid(), nil
347 }
348
349 func (t *Batch) Commit() error {
359 - _, err := t.ds.Blocks.AddBlocks(t.blocks)
360 - t.blocks = nil
350 + _, err := t.ds.Blocks.AddObjects(t.objects)
351 + t.objects = nil
352 t.size = 0
353 return err
354 }
355
356 +func legacyCidFromLink(lnk *Link) *cid.Cid {
357 + return cid.NewCidV0(lnk.Hash)
358 +}
359 +
360 // EnumerateChildren will walk the dag below the given root node and add all
361 // unseen children to the passed in set.
362 // TODO: parallelize to avoid disk latency perf hits?
368 -func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.KeySet, bestEffort bool) error {
363 +func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, visit func(*cid.Cid) bool, bestEffort bool) error {
364 for _, lnk := range root.Links {
370 - k := key.Key(lnk.Hash)
371 - if !set.Has(k) {
372 - set.Add(k)
373 - child, err := ds.Get(ctx, k)
365 + c := legacyCidFromLink(lnk)
366 + if visit(c) {
367 + child, err := ds.Get(ctx, c)
368 if err != nil {
369 if bestEffort && err == ErrNotFound {
370 continue
@@ -378,7 +372,7 @@ func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.K
372 return err
373 }
374 }
381 - err = EnumerateChildren(ctx, ds, child, set, bestEffort)
375 + err = EnumerateChildren(ctx, ds, child, visit, bestEffort)
376 if err != nil {
377 return err
378 }
@@ -387,8 +381,8 @@ func EnumerateChildren(ctx context.Context, ds DAGService, root *Node, set key.K
381 return nil
382 }
383
390 -func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set key.KeySet) error {
391 - toprocess := make(chan []key.Key, 8)
384 +func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, visit func(*cid.Cid) bool) error {
385 + toprocess := make(chan []*cid.Cid, 8)
386 nodes := make(chan *NodeOption, 8)
387
388 ctx, cancel := context.WithCancel(ctx)
@@ -416,13 +410,12 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set
410 // a node has been fetched
411 live--
412
419 - var keys []key.Key
413 + var cids []*cid.Cid
414 for _, lnk := range nd.Links {
421 - k := key.Key(lnk.Hash)
422 - if !set.Has(k) {
423 - set.Add(k)
415 + c := legacyCidFromLink(lnk)
416 + if visit(c) {
417 live++
425 - keys = append(keys, k)
418 + cids = append(cids, c)
419 }
420 }
421
@@ -430,9 +423,9 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set
423 return nil
424 }
425
433 - if len(keys) > 0 {
426 + if len(cids) > 0 {
427 select {
435 - case toprocess <- keys:
428 + case toprocess <- cids:
429 case <-ctx.Done():
430 return ctx.Err()
431 }
@@ -443,7 +436,7 @@ func EnumerateChildrenAsync(ctx context.Context, ds DAGService, root *Node, set
436 }
437 }
438
446 -func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out chan<- *NodeOption) {
439 +func fetchNodes(ctx context.Context, ds DAGService, in <-chan []*cid.Cid, out chan<- *NodeOption) {
440 var wg sync.WaitGroup
441 defer func() {
442 // wait for all 'get' calls to complete so we don't accidentally send
@@ -452,7 +445,7 @@ func fetchNodes(ctx context.Context, ds DAGService, in <-chan []key.Key, out cha
445 close(out)
446 }()
447
455 - get := func(ks []key.Key) {
448 + get := func(ks []*cid.Cid) {
449 defer wg.Done()
450 nodes := ds.GetMany(ctx, ks)
451 for opt := range nodes {
merkledag/merkledag_test.go
+36 -48
@@ -20,8 +20,10 @@ import (
20 mdpb "github.com/ipfs/go-ipfs/merkledag/pb"
21 dstest "github.com/ipfs/go-ipfs/merkledag/test"
22 uio "github.com/ipfs/go-ipfs/unixfs/io"
23 +
24 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
25 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
26 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
27 )
28
29 func TestNode(t *testing.T) {
@@ -52,17 +54,9 @@ func TestNode(t *testing.T) {
54 fmt.Println("encoded:", e)
55 }
56
55 - h, err := n.Multihash()
56 - if err != nil {
57 - t.Error(err)
58 - } else {
59 - fmt.Println("hash:", h)
60 - }
61 -
62 - k, err := n.Key()
63 - if err != nil {
64 - t.Error(err)
65 - } else if k != key.Key(h) {
57 + h := n.Multihash()
58 + k := n.Key()
59 + if k != key.Key(h) {
60 t.Error("Key is not equivalent to multihash")
61 } else {
62 fmt.Println("key: ", k)
@@ -89,11 +83,7 @@ func SubtestNodeStat(t *testing.T, n *Node) {
83 return
84 }
85
92 - k, err := n.Key()
93 - if err != nil {
94 - t.Error("n.Key() failed")
95 - return
96 - }
86 + k := n.Key()
87
88 expected := NodeStat{
89 NumLinks: len(n.Links),
@@ -169,10 +159,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
159
160 t.Log("Added file to first node.")
161
172 - k, err := root.Key()
173 - if err != nil {
174 - t.Fatal(err)
175 - }
162 + c := root.Cid()
163
164 wg := sync.WaitGroup{}
165 errs := make(chan error)
@@ -181,7 +168,7 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
168 wg.Add(1)
169 go func(i int) {
170 defer wg.Done()
184 - first, err := dagservs[i].Get(ctx, k)
171 + first, err := dagservs[i].Get(ctx, c)
172 if err != nil {
173 errs <- err
174 }
@@ -215,34 +202,17 @@ func runBatchFetchTest(t *testing.T, read io.Reader) {
202 }
203
204 func assertCanGet(t *testing.T, ds DAGService, n *Node) {
218 - k, err := n.Key()
219 - if err != nil {
220 - t.Fatal(err)
221 - }
222 -
223 - if _, err := ds.Get(context.Background(), k); err != nil {
205 + if _, err := ds.Get(context.Background(), n.Cid()); err != nil {
206 t.Fatal(err)
207 }
208 }
209
228 -func TestEmptyKey(t *testing.T) {
229 - ds := dstest.Mock()
230 - _, err := ds.Get(context.Background(), key.Key(""))
231 - if err != ErrNotFound {
232 - t.Error("dag service should error when key is nil", err)
233 - }
234 -}
235 -
210 func TestCantGet(t *testing.T) {
211 ds := dstest.Mock()
212 a := NodeWithData([]byte("A"))
213
240 - k, err := a.Key()
241 - if err != nil {
242 - t.Fatal(err)
243 - }
244 -
245 - _, err = ds.Get(context.Background(), k)
214 + c := a.Cid()
215 + _, err := ds.Get(context.Background(), c)
216 if !strings.Contains(err.Error(), "not found") {
217 t.Fatal("expected err not found, got: ", err)
218 }
@@ -270,9 +240,8 @@ func TestFetchGraph(t *testing.T) {
240 bs := bserv.New(bsis[1].Blockstore, offline.Exchange(bsis[1].Blockstore))
241
242 offline_ds := NewDAGService(bs)
273 - ks := key.NewKeySet()
243
275 - err = EnumerateChildren(context.Background(), offline_ds, root, ks, false)
244 + err = EnumerateChildren(context.Background(), offline_ds, root, func(_ *cid.Cid) bool { return true }, false)
245 if err != nil {
246 t.Fatal(err)
247 }
@@ -288,8 +257,8 @@ func TestEnumerateChildren(t *testing.T) {
257 t.Fatal(err)
258 }
259
291 - ks := key.NewKeySet()
292 - err = EnumerateChildren(context.Background(), ds, root, ks, false)
260 + set := cid.NewSet()
261 + err = EnumerateChildren(context.Background(), ds, root, set.Visit, false)
262 if err != nil {
263 t.Fatal(err)
264 }
@@ -298,11 +267,11 @@ func TestEnumerateChildren(t *testing.T) {
267 traverse = func(n *Node) {
268 // traverse dag and check
269 for _, lnk := range n.Links {
301 - k := key.Key(lnk.Hash)
302 - if !ks.Has(k) {
270 + c := cid.NewCidV0(lnk.Hash)
271 + if !set.Has(c) {
272 t.Fatal("missing key in set!")
273 }
305 - child, err := ds.Get(context.Background(), k)
274 + child, err := ds.Get(context.Background(), c)
275 if err != nil {
276 t.Fatal(err)
277 }
@@ -379,3 +348,22 @@ func TestUnmarshalFailure(t *testing.T) {
348 n := &Node{}
349 n.Marshal()
350 }
351 +
352 +func TestBasicAddGet(t *testing.T) {
353 + ds := dstest.Mock()
354 + nd := new(Node)
355 +
356 + c, err := ds.Add(nd)
357 + if err != nil {
358 + t.Fatal(err)
359 + }
360 +
361 + out, err := ds.Get(context.Background(), c)
362 + if err != nil {
363 + t.Fatal(err)
364 + }
365 +
366 + if !nd.Cid().Equals(out.Cid()) {
367 + t.Fatal("output didnt match input")
368 + }
369 +}
merkledag/node.go
+35 -21
@@ -7,6 +7,7 @@ import (
7
8 key "github.com/ipfs/go-ipfs/blocks/key"
9 mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
10 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
11 )
12
13 var ErrLinkNotFound = fmt.Errorf("no link by that name")
@@ -20,7 +21,7 @@ type Node struct {
21 // cache encoded/marshaled value
22 encoded []byte
23
23 - cached mh.Multihash
24 + cached *cid.Cid
25 }
26
27 // NodeStat is a statistics object for a Node. Mostly sizes.
@@ -63,10 +64,8 @@ func MakeLink(n *Node) (*Link, error) {
64 return nil, err
65 }
66
66 - h, err := n.Multihash()
67 - if err != nil {
68 - return nil, err
69 - }
67 + h := n.Multihash()
68 +
69 return &Link{
70 Size: s,
71 Hash: h,
@@ -75,7 +74,7 @@ func MakeLink(n *Node) (*Link, error) {
74
75 // GetNode returns the MDAG Node that this link points to
76 func (l *Link) GetNode(ctx context.Context, serv DAGService) (*Node, error) {
78 - return serv.Get(ctx, key.Key(l.Hash))
77 + return serv.Get(ctx, legacyCidFromLink(l))
78 }
79
80 func NodeWithData(d []byte) *Node {
@@ -184,6 +183,11 @@ func (n *Node) Copy() *Node {
183 return nnode
184 }
185
186 +func (n *Node) RawData() []byte {
187 + out, _ := n.EncodeProtobuf(false)
188 + return out
189 +}
190 +
191 func (n *Node) Data() []byte {
192 return n.data
193 }
@@ -231,13 +235,8 @@ func (n *Node) Stat() (*NodeStat, error) {
235 return nil, err
236 }
237
234 - key, err := n.Key()
235 - if err != nil {
236 - return nil, err
237 - }
238 -
238 return &NodeStat{
240 - Hash: key.B58String(),
239 + Hash: n.Key().B58String(),
240 NumLinks: len(n.Links),
241 BlockSize: len(enc),
242 LinksSize: len(enc) - len(n.data), // includes framing.
@@ -246,19 +245,34 @@ func (n *Node) Stat() (*NodeStat, error) {
245 }, nil
246 }
247
248 +func (n *Node) Key() key.Key {
249 + return key.Key(n.Multihash())
250 +}
251 +
252 +func (n *Node) Loggable() map[string]interface{} {
253 + return map[string]interface{}{
254 + "node": n.String(),
255 + }
256 +}
257 +
258 +func (n *Node) Cid() *cid.Cid {
259 + h := n.Multihash()
260 +
261 + return cid.NewCidV0(h)
262 +}
263 +
264 +func (n *Node) String() string {
265 + return n.Cid().String()
266 +}
267 +
268 // Multihash hashes the encoded data of this node.
250 -func (n *Node) Multihash() (mh.Multihash, error) {
269 +func (n *Node) Multihash() mh.Multihash {
270 // NOTE: EncodeProtobuf generates the hash and puts it in n.cached.
271 _, err := n.EncodeProtobuf(false)
272 if err != nil {
254 - return nil, err
273 + // Note: no possibility exists for an error to be returned through here
274 + panic(err)
275 }
276
257 - return n.cached, nil
258 -}
259 -
260 -// Key returns the Multihash as a key, for maps.
261 -func (n *Node) Key() (key.Key, error) {
262 - h, err := n.Multihash()
263 - return key.Key(h), err
277 + return n.cached.Hash()
278 }
merkledag/node_test.go
+4 -4
@@ -67,9 +67,9 @@ func TestFindLink(t *testing.T) {
67
68 nd := &Node{
69 Links: []*Link{
70 - &Link{Name: "a", Hash: k.ToMultihash()},
71 - &Link{Name: "c", Hash: k.ToMultihash()},
72 - &Link{Name: "b", Hash: k.ToMultihash()},
70 + &Link{Name: "a", Hash: k.Hash()},
71 + &Link{Name: "c", Hash: k.Hash()},
72 + &Link{Name: "b", Hash: k.Hash()},
73 },
74 }
75
@@ -107,7 +107,7 @@ func TestFindLink(t *testing.T) {
107 t.Fatal(err)
108 }
109
110 - if olnk.Hash.B58String() == k.B58String() {
110 + if olnk.Hash.B58String() == k.String() {
111 t.Fatal("new link should have different hash")
112 }
113 }
merkledag/traverse/traverse.go
+1 -5
@@ -41,11 +41,7 @@ type traversal struct {
41
42 func (t *traversal) shouldSkip(n *mdag.Node) (bool, error) {
43 if t.opts.SkipDuplicates {
44 - k, err := n.Key()
45 - if err != nil {
46 - return true, err
47 - }
48 -
44 + k := n.Key()
45 if _, found := t.seen[string(k)]; found {
46 return true, nil
47 }
merkledag/utils/diff.go
+11 -20
@@ -5,9 +5,10 @@ import (
5 "fmt"
6 "path"
7
8 - key "github.com/ipfs/go-ipfs/blocks/key"
8 dag "github.com/ipfs/go-ipfs/merkledag"
9 +
10 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
11 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
12 )
13
14 const (
@@ -19,18 +20,18 @@ const (
20 type Change struct {
21 Type int
22 Path string
22 - Before key.Key
23 - After key.Key
23 + Before *cid.Cid
24 + After *cid.Cid
25 }
26
27 func (c *Change) String() string {
28 switch c.Type {
29 case Add:
29 - return fmt.Sprintf("Added %s at %s", c.After.B58String()[:6], c.Path)
30 + return fmt.Sprintf("Added %s at %s", c.After.String(), c.Path)
31 case Remove:
31 - return fmt.Sprintf("Removed %s from %s", c.Before.B58String()[:6], c.Path)
32 + return fmt.Sprintf("Removed %s from %s", c.Before.String(), c.Path)
33 case Mod:
33 - return fmt.Sprintf("Changed %s to %s at %s", c.Before.B58String()[:6], c.After.B58String()[:6], c.Path)
34 + return fmt.Sprintf("Changed %s to %s at %s", c.Before.String(), c.After.String(), c.Path)
35 default:
36 panic("nope")
37 }
@@ -77,21 +78,11 @@ func ApplyChange(ctx context.Context, ds dag.DAGService, nd *dag.Node, cs []*Cha
78
79 func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.Node) ([]*Change, error) {
80 if len(a.Links) == 0 && len(b.Links) == 0 {
80 - ak, err := a.Key()
81 - if err != nil {
82 - return nil, err
83 - }
84 -
85 - bk, err := b.Key()
86 - if err != nil {
87 - return nil, err
88 - }
89 -
81 return []*Change{
82 &Change{
83 Type: Mod,
93 - Before: ak,
94 - After: bk,
84 + Before: a.Cid(),
85 + After: b.Cid(),
86 },
87 }, nil
88 }
@@ -136,14 +127,14 @@ func Diff(ctx context.Context, ds dag.DAGService, a, b *dag.Node) ([]*Change, er
127 out = append(out, &Change{
128 Type: Remove,
129 Path: lnk.Name,
139 - Before: key.Key(lnk.Hash),
130 + Before: cid.NewCidV0(lnk.Hash),
131 })
132 }
133 for _, lnk := range clean_b.Links {
134 out = append(out, &Change{
135 Type: Add,
136 Path: lnk.Name,
146 - After: key.Key(lnk.Hash),
137 + After: cid.NewCidV0(lnk.Hash),
138 })
139 }
140
merkledag/utils/utils_test.go
+9 -20
@@ -3,12 +3,12 @@ package dagutils
3 import (
4 "testing"
5
6 - key "github.com/ipfs/go-ipfs/blocks/key"
6 dag "github.com/ipfs/go-ipfs/merkledag"
7 mdtest "github.com/ipfs/go-ipfs/merkledag/test"
8 path "github.com/ipfs/go-ipfs/path"
9
10 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
11 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
12 )
13
14 func TestAddLink(t *testing.T) {
@@ -31,17 +31,13 @@ func TestAddLink(t *testing.T) {
31 t.Fatal(err)
32 }
33
34 - fnpkey, err := fnprime.Key()
35 - if err != nil {
36 - t.Fatal(err)
37 - }
38 -
39 - if fnpkey != fk {
34 + fnpkey := fnprime.Cid()
35 + if !fnpkey.Equals(fk) {
36 t.Fatal("wrong child node found!")
37 }
38 }
39
44 -func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, pth string, exp key.Key) {
40 +func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, pth string, exp *cid.Cid) {
41 parts := path.SplitList(pth)
42 cur := root
43 for _, e := range parts {
@@ -53,12 +49,8 @@ func assertNodeAtPath(t *testing.T, ds dag.DAGService, root *dag.Node, pth strin
49 cur = nxt
50 }
51
56 - curk, err := cur.Key()
57 - if err != nil {
58 - t.Fatal(err)
59 - }
60 -
61 - if curk != exp {
52 + curc := cur.Cid()
53 + if !curc.Equals(exp) {
54 t.Fatal("node not as expected at end of path")
55 }
56 }
@@ -77,13 +69,10 @@ func TestInsertNode(t *testing.T) {
69 testInsert(t, e, "", "bar", true, "cannot create link with no name!")
70 testInsert(t, e, "////", "slashes", true, "cannot create link with no name!")
71
80 - k, err := e.GetNode().Key()
81 - if err != nil {
82 - t.Fatal(err)
83 - }
72 + c := e.GetNode().Cid()
73
85 - if k.B58String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" {
86 - t.Fatal("output was different than expected: ", k)
74 + if c.String() != "QmZ8yeT9uD6ouJPNAYt62XffYuXBT6b4mP4obRSE9cJrSt" {
75 + t.Fatal("output was different than expected: ", c)
76 }
77 }
78
mfs/dir.go
+1 -6
@@ -246,12 +246,7 @@ func (d *Directory) List() ([]NodeListing, error) {
246 return nil, err
247 }
248
249 - k, err := nd.Key()
250 - if err != nil {
251 - return nil, err
252 - }
253 -
254 - child.Hash = k.B58String()
249 + child.Hash = nd.Key().B58String()
250
251 out = append(out, child)
252 }
mfs/mfs_test.go
+6 -16
@@ -14,7 +14,6 @@ import (
14 "time"
15
16 bstore "github.com/ipfs/go-ipfs/blocks/blockstore"
17 - key "github.com/ipfs/go-ipfs/blocks/key"
17 bserv "github.com/ipfs/go-ipfs/blockservice"
18 offline "github.com/ipfs/go-ipfs/exchange/offline"
19 importer "github.com/ipfs/go-ipfs/importer"
@@ -28,6 +27,7 @@ import (
27 dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
28 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
29 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
30 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
31 )
32
33 func emptyDirNode() *dag.Node {
@@ -187,8 +187,8 @@ func setupRoot(ctx context.Context, t *testing.T) (dag.DAGService, *Root) {
187 ds := getDagserv(t)
188
189 root := emptyDirNode()
190 - rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, k key.Key) error {
191 - fmt.Println("PUBLISHED: ", k)
190 + rt, err := NewRoot(ctx, ds, root, func(ctx context.Context, c *cid.Cid) error {
191 + fmt.Println("PUBLISHED: ", c)
192 return nil
193 })
194
@@ -280,10 +280,7 @@ func TestDirectoryLoadFromDag(t *testing.T) {
280 t.Fatal(err)
281 }
282
283 - fihash, err := nd.Multihash()
284 - if err != nil {
285 - t.Fatal(err)
286 - }
283 + fihash := nd.Multihash()
284
285 dir := emptyDirNode()
286 _, err = ds.Add(dir)
@@ -291,10 +288,7 @@ func TestDirectoryLoadFromDag(t *testing.T) {
288 t.Fatal(err)
289 }
290
294 - dirhash, err := dir.Multihash()
295 - if err != nil {
296 - t.Fatal(err)
297 - }
291 + dirhash := dir.Multihash()
292
293 top := emptyDirNode()
294 top.Links = []*dag.Link{
@@ -803,11 +797,7 @@ func TestFlushing(t *testing.T) {
797 t.Fatal("root wasnt a directory")
798 }
799
806 - rnk, err := rnd.Key()
807 - if err != nil {
808 - t.Fatal(err)
809 - }
810 -
800 + rnk := rnd.Key()
801 exp := "QmWMVyhTuyxUrXX3ynz171jq76yY3PktfY9Bxiph7b9ikr"
802 if rnk.B58String() != exp {
803 t.Fatalf("dag looks wrong, expected %s, but got %s", exp, rnk.B58String())
mfs/repub_test.go
+4 -4
@@ -4,10 +4,10 @@ import (
4 "testing"
5 "time"
6
7 - key "github.com/ipfs/go-ipfs/blocks/key"
7 ci "github.com/ipfs/go-ipfs/thirdparty/testutil/ci"
8
9 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
10 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
11 )
12
13 func TestRepublisher(t *testing.T) {
@@ -19,7 +19,7 @@ func TestRepublisher(t *testing.T) {
19
20 pub := make(chan struct{})
21
22 - pf := func(ctx context.Context, k key.Key) error {
22 + pf := func(ctx context.Context, c *cid.Cid) error {
23 pub <- struct{}{}
24 return nil
25 }
@@ -30,7 +30,7 @@ func TestRepublisher(t *testing.T) {
30 rp := NewRepublisher(ctx, pf, tshort, tlong)
31 go rp.Run()
32
33 - rp.Update("test")
33 + rp.Update(nil)
34
35 // should hit short timeout
36 select {
@@ -43,7 +43,7 @@ func TestRepublisher(t *testing.T) {
43
44 go func() {
45 for {
46 - rp.Update("a")
46 + rp.Update(nil)
47 time.Sleep(time.Millisecond * 10)
48 select {
49 case <-cctx.Done():
mfs/system.go
+16 -30
@@ -14,12 +14,12 @@ import (
14 "sync"
15 "time"
16
17 - key "github.com/ipfs/go-ipfs/blocks/key"
17 dag "github.com/ipfs/go-ipfs/merkledag"
18 ft "github.com/ipfs/go-ipfs/unixfs"
19
20 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
21 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
22 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
23 )
24
25 var ErrNotExist = errors.New("no such rootfs")
@@ -61,19 +61,15 @@ type Root struct {
61 Type string
62 }
63
64 -type PubFunc func(context.Context, key.Key) error
64 +type PubFunc func(context.Context, *cid.Cid) error
65
66 // newRoot creates a new Root and starts up a republisher routine for it
67 func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFunc) (*Root, error) {
68 - ndk, err := node.Key()
69 - if err != nil {
70 - return nil, err
71 - }
68
69 var repub *Republisher
70 if pf != nil {
71 repub = NewRepublisher(parent, pf, time.Millisecond*300, time.Second*3)
76 - repub.setVal(ndk)
72 + repub.setVal(node.Cid())
73 go repub.Run()
74 }
75
@@ -91,9 +87,9 @@ func NewRoot(parent context.Context, ds dag.DAGService, node *dag.Node, pf PubFu
87
88 switch pbn.GetType() {
89 case ft.TDirectory:
94 - root.val = NewDirectory(parent, ndk.String(), node, root, ds)
90 + root.val = NewDirectory(parent, node.String(), node, root, ds)
91 case ft.TFile, ft.TMetadata, ft.TRaw:
96 - fi, err := NewFile(ndk.String(), node, root, ds)
92 + fi, err := NewFile(node.String(), node, root, ds)
93 if err != nil {
94 return nil, err
95 }
@@ -114,13 +110,8 @@ func (kr *Root) Flush() error {
110 return err
111 }
112
117 - k, err := nd.Key()
118 - if err != nil {
119 - return err
120 - }
121 -
113 if kr.repub != nil {
123 - kr.repub.Update(k)
114 + kr.repub.Update(nd.Cid())
115 }
116 return nil
117 }
@@ -128,13 +119,13 @@ func (kr *Root) Flush() error {
119 // closeChild implements the childCloser interface, and signals to the publisher that
120 // there are changes ready to be published
121 func (kr *Root) closeChild(name string, nd *dag.Node, sync bool) error {
131 - k, err := kr.dserv.Add(nd)
122 + c, err := kr.dserv.Add(nd)
123 if err != nil {
124 return err
125 }
126
127 if kr.repub != nil {
137 - kr.repub.Update(k)
128 + kr.repub.Update(c)
129 }
130 return nil
131 }
@@ -145,13 +136,8 @@ func (kr *Root) Close() error {
136 return err
137 }
138
148 - k, err := nd.Key()
149 - if err != nil {
150 - return err
151 - }
152 -
139 if kr.repub != nil {
154 - kr.repub.Update(k)
140 + kr.repub.Update(nd.Cid())
141 return kr.repub.Close()
142 }
143
@@ -170,11 +156,11 @@ type Republisher struct {
156 cancel func()
157
158 lk sync.Mutex
173 - val key.Key
174 - lastpub key.Key
159 + val *cid.Cid
160 + lastpub *cid.Cid
161 }
162
177 -func (rp *Republisher) getVal() key.Key {
163 +func (rp *Republisher) getVal() *cid.Cid {
164 rp.lk.Lock()
165 defer rp.lk.Unlock()
166 return rp.val
@@ -195,10 +181,10 @@ func NewRepublisher(ctx context.Context, pf PubFunc, tshort, tlong time.Duration
181 }
182 }
183
198 -func (p *Republisher) setVal(k key.Key) {
184 +func (p *Republisher) setVal(c *cid.Cid) {
185 p.lk.Lock()
186 defer p.lk.Unlock()
201 - p.val = k
187 + p.val = c
188 }
189
190 func (p *Republisher) pubNow() {
@@ -230,8 +216,8 @@ func (p *Republisher) Close() error {
216 // Touch signals that an update has occurred since the last publish.
217 // Multiple consecutive touches may extend the time period before
218 // the next Publish occurs in order to more efficiently batch updates
233 -func (np *Republisher) Update(k key.Key) {
234 - np.setVal(k)
219 +func (np *Republisher) Update(c *cid.Cid) {
220 + np.setVal(c)
221 select {
222 case np.Publish <- struct{}{}:
223 default:
namesys/dns.go
+1 -1
@@ -114,7 +114,7 @@ func workDomain(r *DNSResolver, name string, res chan lookupRes) {
114 }
115
116 func parseEntry(txt string) (path.Path, error) {
117 - p, err := path.ParseKeyToPath(txt) // bare IPFS multihashes
117 + p, err := path.ParseCidToPath(txt) // bare IPFS multihashes
118 if err == nil {
119 return p, nil
120 }
namesys/publisher.go
+1 -1
@@ -348,7 +348,7 @@ func InitializeKeyspace(ctx context.Context, ds dag.DAGService, pub Publisher, p
348 return err
349 }
350
351 - err = pub.Publish(ctx, key, path.FromKey(nodek))
351 + err = pub.Publish(ctx, key, path.FromCid(nodek))
352 if err != nil {
353 return err
354 }
namesys/routing.go
+3 -1
@@ -14,9 +14,11 @@ import (
14 pb "github.com/ipfs/go-ipfs/namesys/pb"
15 path "github.com/ipfs/go-ipfs/path"
16 routing "github.com/ipfs/go-ipfs/routing"
17 +
18 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
19 ci "gx/ipfs/QmVoi5es8D5fNHZDqoW6DgDAEPEV5hQp8GBz161vZXiwpQ/go-libp2p-crypto"
20 u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
21 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
22 )
23
24 var log = logging.Logger("namesys")
@@ -196,7 +198,7 @@ func (r *routingResolver) resolveOnce(ctx context.Context, name string) (path.Pa
198 } else {
199 // Its an old style multihash record
200 log.Warning("Detected old style multihash record")
199 - p := path.FromKey(key.Key(valh))
201 + p := path.FromCid(cid.NewCidV0(valh))
202 r.cacheSet(name, p, entry)
203 return p, nil
204 }
package.json
+6
@@ -203,6 +203,12 @@
203 "hash": "Qmf4ETeAWXuThBfWwonVyFqGFSgTWepUDEr1txcctvpTXS",
204 "name": "go-libp2p",
205 "version": "3.4.1"
206 + },
207 + {
208 + "author": "whyrusleeping",
209 + "hash": "QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp",
210 + "name": "go-cid",
211 + "version": "0.1.0"
212 }
213 ],
214 "gxVersion": "0.4.0",
path/path.go
+12 -18
@@ -5,10 +5,7 @@ import (
5 "path"
6 "strings"
7
8 - key "github.com/ipfs/go-ipfs/blocks/key"
9 -
10 - b58 "gx/ipfs/QmT8rehPR3F6bmwL6zjUN8XpiDBFFpMP2myPdC6ApsWfJf/go-base58"
11 - mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
8 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
9 )
10
11 // ErrBadPath is returned when a given path is incorrectly formatted
@@ -23,9 +20,9 @@ func FromString(s string) Path {
20 return Path(s)
21 }
22
26 -// FromKey safely converts a Key type to a Path type
27 -func FromKey(k key.Key) Path {
28 - return Path("/ipfs/" + k.String())
23 +// FromCid safely converts a cid.Cid type to a Path type
24 +func FromCid(c *cid.Cid) Path {
25 + return Path("/ipfs/" + c.String())
26 }
27
28 func (p Path) Segments() []string {
@@ -75,7 +72,7 @@ func FromSegments(prefix string, seg ...string) (Path, error) {
72 func ParsePath(txt string) (Path, error) {
73 parts := strings.Split(txt, "/")
74 if len(parts) == 1 {
78 - kp, err := ParseKeyToPath(txt)
75 + kp, err := ParseCidToPath(txt)
76 if err == nil {
77 return kp, nil
78 }
@@ -84,7 +81,7 @@ func ParsePath(txt string) (Path, error) {
81 // if the path doesnt being with a '/'
82 // we expect this to start with a hash, and be an 'ipfs' path
83 if parts[0] != "" {
87 - if _, err := ParseKeyToPath(parts[0]); err != nil {
84 + if _, err := ParseCidToPath(parts[0]); err != nil {
85 return "", ErrBadPath
86 }
87 // The case when the path starts with hash without a protocol prefix
@@ -96,7 +93,7 @@ func ParsePath(txt string) (Path, error) {
93 }
94
95 if parts[1] == "ipfs" {
99 - if _, err := ParseKeyToPath(parts[2]); err != nil {
96 + if _, err := ParseCidToPath(parts[2]); err != nil {
97 return "", err
98 }
99 } else if parts[1] != "ipns" {
@@ -106,20 +103,17 @@ func ParsePath(txt string) (Path, error) {
103 return Path(txt), nil
104 }
105
109 -func ParseKeyToPath(txt string) (Path, error) {
106 +func ParseCidToPath(txt string) (Path, error) {
107 if txt == "" {
108 return "", ErrNoComponents
109 }
110
114 - chk := b58.Decode(txt)
115 - if len(chk) == 0 {
116 - return "", errors.New("not a key")
117 - }
118 -
119 - if _, err := mh.Cast(chk); err != nil {
111 + c, err := cid.Decode(txt)
112 + if err != nil {
113 return "", err
114 }
122 - return FromKey(key.Key(chk)), nil
115 +
116 + return FromCid(c), nil
117 }
118
119 func (p *Path) IsValid() error {
path/resolver.go
+6 -8
@@ -9,9 +9,9 @@ import (
9 mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
10 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
11
12 - key "github.com/ipfs/go-ipfs/blocks/key"
12 merkledag "github.com/ipfs/go-ipfs/merkledag"
13 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
14 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
15 )
16
17 var log = logging.Logger("path")
@@ -38,7 +38,7 @@ type Resolver struct {
38
39 // SplitAbsPath clean up and split fpath. It extracts the first component (which
40 // must be a Multihash) and return it separately.
41 -func SplitAbsPath(fpath Path) (mh.Multihash, []string, error) {
41 +func SplitAbsPath(fpath Path) (*cid.Cid, []string, error) {
42
43 log.Debugf("Resolve: '%s'", fpath)
44
@@ -52,14 +52,12 @@ func SplitAbsPath(fpath Path) (mh.Multihash, []string, error) {
52 return nil, nil, ErrNoComponents
53 }
54
55 - // first element in the path is a b58 hash (for now)
56 - h, err := mh.FromB58String(parts[0])
55 + c, err := cid.Decode(parts[0])
56 if err != nil {
58 - log.Debug("given path element is not a base58 string.\n")
57 return nil, nil, err
58 }
59
62 - return h, parts[1:], nil
60 + return c, parts[1:], nil
61 }
62
63 // ResolvePath fetches the node for given path. It returns the last item
@@ -87,7 +85,7 @@ func (s *Resolver) ResolvePathComponents(ctx context.Context, fpath Path) ([]*me
85 }
86
87 log.Debug("resolve dag get")
90 - nd, err := s.DAG.Get(ctx, key.Key(h))
88 + nd, err := s.DAG.Get(ctx, h)
89 if err != nil {
90 return nil, err
91 }
@@ -117,7 +115,7 @@ func (s *Resolver) ResolveLinks(ctx context.Context, ndd *merkledag.Node, names
115
116 nextnode, err := nd.GetLinkedNode(ctx, s.DAG, name)
117 if err == merkledag.ErrLinkNotFound {
120 - n, _ := nd.Multihash()
118 + n := nd.Multihash()
119 return result, ErrNoLink{Name: name, Node: n}
120 } else if err != nil {
121 return append(result, nextnode), err
path/resolver_test.go
+3 -9
@@ -17,7 +17,7 @@ func randNode() (*merkledag.Node, key.Key) {
17 node := new(merkledag.Node)
18 node.SetData(make([]byte, 32))
19 util.NewTimeSeededRand().Read(node.Data())
20 - k, _ := node.Key()
20 + k := node.Key()
21 return node, k
22 }
23
@@ -46,10 +46,7 @@ func TestRecurivePathResolution(t *testing.T) {
46 }
47 }
48
49 - aKey, err := a.Key()
50 - if err != nil {
51 - t.Fatal(err)
52 - }
49 + aKey := a.Key()
50
51 segments := []string{aKey.String(), "child", "grandchild"}
52 p, err := path.FromSegments("/ipfs/", segments...)
@@ -63,10 +60,7 @@ func TestRecurivePathResolution(t *testing.T) {
60 t.Fatal(err)
61 }
62
66 - key, err := node.Key()
67 - if err != nil {
68 - t.Fatal(err)
69 - }
63 + key := node.Key()
64 if key.String() != cKey.String() {
65 t.Fatal(fmt.Errorf(
66 "recursive path resolution failed for %s: %s != %s",
pin/gc/gc.go
+17 -8
@@ -10,6 +10,7 @@ import (
10
11 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
12 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
13 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
14 )
15
16 var log = logging.Logger("gc")
@@ -23,7 +24,7 @@ var log = logging.Logger("gc")
24 //
25 // The routine then iterates over every block in the blockstore and
26 // deletes any block that is not found in the marked set.
26 -func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRoots []key.Key) (<-chan key.Key, error) {
27 +func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRoots []*cid.Cid) (<-chan key.Key, error) {
28 unlocker := bs.GCLock()
29
30 bsrv := bserv.New(bs, offline.Exchange(bs))
@@ -70,16 +71,24 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, pn pin.Pinner, bestEffortRo
71 return output, nil
72 }
73
73 -func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots []key.Key, bestEffort bool) error {
74 - for _, k := range roots {
75 - set.Add(k)
76 - nd, err := ds.Get(ctx, k)
74 +func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots []*cid.Cid, bestEffort bool) error {
75 + for _, c := range roots {
76 + set.Add(key.Key(c.Hash()))
77 + nd, err := ds.Get(ctx, c)
78 if err != nil {
79 return err
80 }
81
82 // EnumerateChildren recursively walks the dag and adds the keys to the given set
82 - err = dag.EnumerateChildren(ctx, ds, nd, set, bestEffort)
83 + err = dag.EnumerateChildren(ctx, ds, nd, func(c *cid.Cid) bool {
84 + k := key.Key(c.Hash())
85 + seen := set.Has(k)
86 + if seen {
87 + return false
88 + }
89 + set.Add(k)
90 + return true
91 + }, bestEffort)
92 if err != nil {
93 return err
94 }
@@ -88,7 +97,7 @@ func Descendants(ctx context.Context, ds dag.DAGService, set key.KeySet, roots [
97 return nil
98 }
99
91 -func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService, bestEffortRoots []key.Key) (key.KeySet, error) {
100 +func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService, bestEffortRoots []*cid.Cid) (key.KeySet, error) {
101 // KeySet currently implemented in memory, in the future, may be bloom filter or
102 // disk backed to conserve memory.
103 gcs := key.NewKeySet()
@@ -103,7 +112,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ds dag.DAGService, bestEffor
112 }
113
114 for _, k := range pn.DirectKeys() {
106 - gcs.Add(k)
115 + gcs.Add(key.Key(k.Hash()))
116 }
117
118 err = Descendants(ctx, ds, gcs, pn.InternalPins(), false)
pin/pin.go
+130 -113
@@ -4,22 +4,33 @@ package pin
4
5 import (
6 "fmt"
7 + "os"
8 "sync"
9 "time"
10
11 key "github.com/ipfs/go-ipfs/blocks/key"
11 - "github.com/ipfs/go-ipfs/blocks/set"
12 mdag "github.com/ipfs/go-ipfs/merkledag"
13 +
14 ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
15 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
16 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
17 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
18 )
19
20 var log = logging.Logger("pin")
21
22 var pinDatastoreKey = ds.NewKey("/local/pins")
23
22 -var emptyKey = key.B58KeyDecode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")
24 +var emptyKey *cid.Cid
25 +
26 +func init() {
27 + e, err := cid.Decode("QmdfTbBqBPQ7VNxZEYEj14VmRuZBkqFbiwReogJgS1zR1n")
28 + if err != nil {
29 + log.Error("failed to decode empty key constant")
30 + os.Exit(1)
31 + }
32 + emptyKey = e
33 +}
34
35 const (
36 linkRecursive = "recursive"
@@ -70,45 +81,45 @@ func StringToPinMode(s string) (PinMode, bool) {
81 }
82
83 type Pinner interface {
73 - IsPinned(key.Key) (string, bool, error)
74 - IsPinnedWithType(key.Key, PinMode) (string, bool, error)
84 + IsPinned(*cid.Cid) (string, bool, error)
85 + IsPinnedWithType(*cid.Cid, PinMode) (string, bool, error)
86 Pin(context.Context, *mdag.Node, bool) error
76 - Unpin(context.Context, key.Key, bool) error
87 + Unpin(context.Context, *cid.Cid, bool) error
88
89 // Check if a set of keys are pinned, more efficient than
90 // calling IsPinned for each key
80 - CheckIfPinned(keys ...key.Key) ([]Pinned, error)
91 + CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error)
92
93 // PinWithMode is for manually editing the pin structure. Use with
94 // care! If used improperly, garbage collection may not be
95 // successful.
85 - PinWithMode(key.Key, PinMode)
96 + PinWithMode(*cid.Cid, PinMode)
97 // RemovePinWithMode is for manually editing the pin structure.
98 // Use with care! If used improperly, garbage collection may not
99 // be successful.
89 - RemovePinWithMode(key.Key, PinMode)
100 + RemovePinWithMode(*cid.Cid, PinMode)
101
102 Flush() error
92 - DirectKeys() []key.Key
93 - RecursiveKeys() []key.Key
94 - InternalPins() []key.Key
103 + DirectKeys() []*cid.Cid
104 + RecursiveKeys() []*cid.Cid
105 + InternalPins() []*cid.Cid
106 }
107
108 type Pinned struct {
98 - Key key.Key
109 + Key *cid.Cid
110 Mode PinMode
100 - Via key.Key
111 + Via *cid.Cid
112 }
113
114 // pinner implements the Pinner interface
115 type pinner struct {
116 lock sync.RWMutex
106 - recursePin set.BlockSet
107 - directPin set.BlockSet
117 + recursePin *cid.Set
118 + directPin *cid.Set
119
120 // Track the keys used for storing the pinning state, so gc does
121 // not delete them.
111 - internalPin map[key.Key]struct{}
122 + internalPin *cid.Set
123 dserv mdag.DAGService
124 internal mdag.DAGService // dagservice used to store internal objects
125 dstore ds.Datastore
@@ -117,15 +128,16 @@ type pinner struct {
128 // NewPinner creates a new pinner using the given datastore as a backend
129 func NewPinner(dstore ds.Datastore, serv, internal mdag.DAGService) Pinner {
130
120 - rcset := set.NewSimpleBlockSet()
121 - dirset := set.NewSimpleBlockSet()
131 + rcset := cid.NewSet()
132 + dirset := cid.NewSet()
133
134 return &pinner{
124 - recursePin: rcset,
125 - directPin: dirset,
126 - dserv: serv,
127 - dstore: dstore,
128 - internal: internal,
135 + recursePin: rcset,
136 + directPin: dirset,
137 + dserv: serv,
138 + dstore: dstore,
139 + internal: internal,
140 + internalPin: cid.NewSet(),
141 }
142 }
143
@@ -133,18 +145,16 @@ func NewPinner(dstore ds.Datastore, serv, internal mdag.DAGService) Pinner {
145 func (p *pinner) Pin(ctx context.Context, node *mdag.Node, recurse bool) error {
146 p.lock.Lock()
147 defer p.lock.Unlock()
136 - k, err := node.Key()
137 - if err != nil {
138 - return err
139 - }
148 + c := node.Cid()
149 + k := key.Key(c.Hash())
150
151 if recurse {
142 - if p.recursePin.HasKey(k) {
152 + if p.recursePin.Has(c) {
153 return nil
154 }
155
146 - if p.directPin.HasKey(k) {
147 - p.directPin.RemoveBlock(k)
156 + if p.directPin.Has(c) {
157 + p.directPin.Remove(c)
158 }
159
160 // fetch entire graph
@@ -153,17 +163,17 @@ func (p *pinner) Pin(ctx context.Context, node *mdag.Node, recurse bool) error {
163 return err
164 }
165
156 - p.recursePin.AddBlock(k)
166 + p.recursePin.Add(c)
167 } else {
158 - if _, err := p.dserv.Get(ctx, k); err != nil {
168 + if _, err := p.dserv.Get(ctx, c); err != nil {
169 return err
170 }
171
162 - if p.recursePin.HasKey(k) {
172 + if p.recursePin.Has(c) {
173 return fmt.Errorf("%s already pinned recursively", k.B58String())
174 }
175
166 - p.directPin.AddBlock(k)
176 + p.directPin.Add(c)
177 }
178 return nil
179 }
@@ -171,10 +181,10 @@ func (p *pinner) Pin(ctx context.Context, node *mdag.Node, recurse bool) error {
181 var ErrNotPinned = fmt.Errorf("not pinned")
182
183 // Unpin a given key
174 -func (p *pinner) Unpin(ctx context.Context, k key.Key, recursive bool) error {
184 +func (p *pinner) Unpin(ctx context.Context, c *cid.Cid, recursive bool) error {
185 p.lock.Lock()
186 defer p.lock.Unlock()
177 - reason, pinned, err := p.isPinnedWithType(k, Any)
187 + reason, pinned, err := p.isPinnedWithType(c, Any)
188 if err != nil {
189 return err
190 }
@@ -184,41 +194,41 @@ func (p *pinner) Unpin(ctx context.Context, k key.Key, recursive bool) error {
194 switch reason {
195 case "recursive":
196 if recursive {
187 - p.recursePin.RemoveBlock(k)
197 + p.recursePin.Remove(c)
198 return nil
199 } else {
190 - return fmt.Errorf("%s is pinned recursively", k)
200 + return fmt.Errorf("%s is pinned recursively", c)
201 }
202 case "direct":
193 - p.directPin.RemoveBlock(k)
203 + p.directPin.Remove(c)
204 return nil
205 default:
196 - return fmt.Errorf("%s is pinned indirectly under %s", k, reason)
206 + return fmt.Errorf("%s is pinned indirectly under %s", c, reason)
207 }
208 }
209
200 -func (p *pinner) isInternalPin(key key.Key) bool {
201 - _, ok := p.internalPin[key]
202 - return ok
210 +func (p *pinner) isInternalPin(c *cid.Cid) bool {
211 + return p.internalPin.Has(c)
212 }
213
214 // IsPinned returns whether or not the given key is pinned
215 // and an explanation of why its pinned
207 -func (p *pinner) IsPinned(k key.Key) (string, bool, error) {
216 +func (p *pinner) IsPinned(c *cid.Cid) (string, bool, error) {
217 p.lock.RLock()
218 defer p.lock.RUnlock()
210 - return p.isPinnedWithType(k, Any)
219 + return p.isPinnedWithType(c, Any)
220 }
221
213 -func (p *pinner) IsPinnedWithType(k key.Key, mode PinMode) (string, bool, error) {
222 +func (p *pinner) IsPinnedWithType(c *cid.Cid, mode PinMode) (string, bool, error) {
223 p.lock.RLock()
224 defer p.lock.RUnlock()
216 - return p.isPinnedWithType(k, mode)
225 + return p.isPinnedWithType(c, mode)
226 }
227
228 // isPinnedWithType is the implementation of IsPinnedWithType that does not lock.
229 // intended for use by other pinned methods that already take locks
221 -func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error) {
230 +func (p *pinner) isPinnedWithType(c *cid.Cid, mode PinMode) (string, bool, error) {
231 + k := key.Key(c.Hash())
232 switch mode {
233 case Any, Direct, Indirect, Recursive, Internal:
234 default:
@@ -226,21 +236,21 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
236 mode, Direct, Indirect, Recursive, Internal, Any)
237 return "", false, err
238 }
229 - if (mode == Recursive || mode == Any) && p.recursePin.HasKey(k) {
239 + if (mode == Recursive || mode == Any) && p.recursePin.Has(c) {
240 return linkRecursive, true, nil
241 }
242 if mode == Recursive {
243 return "", false, nil
244 }
245
236 - if (mode == Direct || mode == Any) && p.directPin.HasKey(k) {
246 + if (mode == Direct || mode == Any) && p.directPin.Has(c) {
247 return linkDirect, true, nil
248 }
249 if mode == Direct {
250 return "", false, nil
251 }
252
243 - if (mode == Internal || mode == Any) && p.isInternalPin(k) {
253 + if (mode == Internal || mode == Any) && p.isInternalPin(c) {
254 return linkInternal, true, nil
255 }
256 if mode == Internal {
@@ -248,8 +258,8 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
258 }
259
260 // Default is Indirect
251 - for _, rk := range p.recursePin.GetKeys() {
252 - rnd, err := p.dserv.Get(context.Background(), rk)
261 + for _, rc := range p.recursePin.Keys() {
262 + rnd, err := p.dserv.Get(context.Background(), rc)
263 if err != nil {
264 return "", false, err
265 }
@@ -259,90 +269,99 @@ func (p *pinner) isPinnedWithType(k key.Key, mode PinMode) (string, bool, error)
269 return "", false, err
270 }
271 if has {
262 - return rk.B58String(), true, nil
272 + return rc.String(), true, nil
273 }
274 }
275 return "", false, nil
276 }
277
268 -func (p *pinner) CheckIfPinned(keys ...key.Key) ([]Pinned, error) {
278 +func (p *pinner) CheckIfPinned(cids ...*cid.Cid) ([]Pinned, error) {
279 p.lock.RLock()
280 defer p.lock.RUnlock()
271 - pinned := make([]Pinned, 0, len(keys))
272 - toCheck := make(map[key.Key]struct{})
281 + pinned := make([]Pinned, 0, len(cids))
282 + toCheck := cid.NewSet()
283
284 // First check for non-Indirect pins directly
275 - for _, k := range keys {
276 - if p.recursePin.HasKey(k) {
277 - pinned = append(pinned, Pinned{Key: k, Mode: Recursive})
278 - } else if p.directPin.HasKey(k) {
279 - pinned = append(pinned, Pinned{Key: k, Mode: Direct})
280 - } else if p.isInternalPin(k) {
281 - pinned = append(pinned, Pinned{Key: k, Mode: Internal})
285 + for _, c := range cids {
286 + if p.recursePin.Has(c) {
287 + pinned = append(pinned, Pinned{Key: c, Mode: Recursive})
288 + } else if p.directPin.Has(c) {
289 + pinned = append(pinned, Pinned{Key: c, Mode: Direct})
290 + } else if p.isInternalPin(c) {
291 + pinned = append(pinned, Pinned{Key: c, Mode: Internal})
292 } else {
283 - toCheck[k] = struct{}{}
293 + toCheck.Add(c)
294 }
295 }
296
297 // Now walk all recursive pins to check for indirect pins
288 - var checkChildren func(key.Key, key.Key) error
289 - checkChildren = func(rk key.Key, parentKey key.Key) error {
298 + var checkChildren func(*cid.Cid, *cid.Cid) error
299 + checkChildren = func(rk, parentKey *cid.Cid) error {
300 parent, err := p.dserv.Get(context.Background(), parentKey)
301 if err != nil {
302 return err
303 }
304 for _, lnk := range parent.Links {
295 - k := key.Key(lnk.Hash)
305 + c := cid.NewCidV0(lnk.Hash)
306
297 - if _, found := toCheck[k]; found {
307 + if toCheck.Has(c) {
308 pinned = append(pinned,
299 - Pinned{Key: k, Mode: Indirect, Via: rk})
300 - delete(toCheck, k)
309 + Pinned{Key: c, Mode: Indirect, Via: rk})
310 + toCheck.Remove(c)
311 }
312
303 - err := checkChildren(rk, k)
313 + err := checkChildren(rk, c)
314 if err != nil {
315 return err
316 }
317
308 - if len(toCheck) == 0 {
318 + if toCheck.Len() == 0 {
319 return nil
320 }
321 }
322 return nil
323 }
314 - for _, rk := range p.recursePin.GetKeys() {
324 +
325 + for _, rk := range p.recursePin.Keys() {
326 err := checkChildren(rk, rk)
327 if err != nil {
328 return nil, err
329 }
319 - if len(toCheck) == 0 {
330 + if toCheck.Len() == 0 {
331 break
332 }
333 }
334
335 // Anything left in toCheck is not pinned
325 - for k, _ := range toCheck {
336 + for _, k := range toCheck.Keys() {
337 pinned = append(pinned, Pinned{Key: k, Mode: NotPinned})
338 }
339
340 return pinned, nil
341 }
342
332 -func (p *pinner) RemovePinWithMode(key key.Key, mode PinMode) {
343 +func (p *pinner) RemovePinWithMode(c *cid.Cid, mode PinMode) {
344 p.lock.Lock()
345 defer p.lock.Unlock()
346 switch mode {
347 case Direct:
337 - p.directPin.RemoveBlock(key)
348 + p.directPin.Remove(c)
349 case Recursive:
339 - p.recursePin.RemoveBlock(key)
350 + p.recursePin.Remove(c)
351 default:
352 // programmer error, panic OK
353 panic("unrecognized pin type")
354 }
355 }
356
357 +func cidSetWithValues(cids []*cid.Cid) *cid.Set {
358 + out := cid.NewSet()
359 + for _, c := range cids {
360 + out.Add(c)
361 + }
362 + return out
363 +}
364 +
365 // LoadPinner loads a pinner and its keysets from the given datastore
366 func LoadPinner(d ds.Datastore, dserv, internal mdag.DAGService) (Pinner, error) {
367 p := new(pinner)
@@ -356,29 +375,29 @@ func LoadPinner(d ds.Datastore, dserv, internal mdag.DAGService) (Pinner, error)
375 return nil, fmt.Errorf("cannot load pin state: %s was not bytes", pinDatastoreKey)
376 }
377
359 - rootKey := key.Key(rootKeyBytes)
378 + rootCid, err := cid.Cast(rootKeyBytes)
379 + if err != nil {
380 + return nil, err
381 + }
382
383 ctx, cancel := context.WithTimeout(context.TODO(), time.Second*5)
384 defer cancel()
385
364 - root, err := internal.Get(ctx, rootKey)
386 + root, err := internal.Get(ctx, rootCid)
387 if err != nil {
388 return nil, fmt.Errorf("cannot find pinning root object: %v", err)
389 }
390
369 - internalPin := map[key.Key]struct{}{
370 - rootKey: struct{}{},
371 - }
372 - recordInternal := func(k key.Key) {
373 - internalPin[k] = struct{}{}
374 - }
391 + internalset := cid.NewSet()
392 + internalset.Add(rootCid)
393 + recordInternal := internalset.Add
394
395 { // load recursive set
396 recurseKeys, err := loadSet(ctx, internal, root, linkRecursive, recordInternal)
397 if err != nil {
398 return nil, fmt.Errorf("cannot load recursive pins: %v", err)
399 }
381 - p.recursePin = set.SimpleSetFromKeys(recurseKeys)
400 + p.recursePin = cidSetWithValues(recurseKeys)
401 }
402
403 { // load direct set
@@ -386,10 +405,10 @@ func LoadPinner(d ds.Datastore, dserv, internal mdag.DAGService) (Pinner, error)
405 if err != nil {
406 return nil, fmt.Errorf("cannot load direct pins: %v", err)
407 }
389 - p.directPin = set.SimpleSetFromKeys(directKeys)
408 + p.directPin = cidSetWithValues(directKeys)
409 }
410
392 - p.internalPin = internalPin
411 + p.internalPin = internalset
412
413 // assign services
414 p.dserv = dserv
@@ -400,13 +419,13 @@ func LoadPinner(d ds.Datastore, dserv, internal mdag.DAGService) (Pinner, error)
419 }
420
421 // DirectKeys returns a slice containing the directly pinned keys
403 -func (p *pinner) DirectKeys() []key.Key {
404 - return p.directPin.GetKeys()
422 +func (p *pinner) DirectKeys() []*cid.Cid {
423 + return p.directPin.Keys()
424 }
425
426 // RecursiveKeys returns a slice containing the recursively pinned keys
408 -func (p *pinner) RecursiveKeys() []key.Key {
409 - return p.recursePin.GetKeys()
427 +func (p *pinner) RecursiveKeys() []*cid.Cid {
428 + return p.recursePin.Keys()
429 }
430
431 // Flush encodes and writes pinner keysets to the datastore
@@ -416,14 +435,12 @@ func (p *pinner) Flush() error {
435
436 ctx := context.TODO()
437
419 - internalPin := make(map[key.Key]struct{})
420 - recordInternal := func(k key.Key) {
421 - internalPin[k] = struct{}{}
422 - }
438 + internalset := cid.NewSet()
439 + recordInternal := internalset.Add
440
441 root := &mdag.Node{}
442 {
426 - n, err := storeSet(ctx, p.internal, p.directPin.GetKeys(), recordInternal)
443 + n, err := storeSet(ctx, p.internal, p.directPin.Keys(), recordInternal)
444 if err != nil {
445 return err
446 }
@@ -433,7 +450,7 @@ func (p *pinner) Flush() error {
450 }
451
452 {
436 - n, err := storeSet(ctx, p.internal, p.recursePin.GetKeys(), recordInternal)
453 + n, err := storeSet(ctx, p.internal, p.recursePin.Keys(), recordInternal)
454 if err != nil {
455 return err
456 }
@@ -453,45 +470,45 @@ func (p *pinner) Flush() error {
470 return err
471 }
472
456 - internalPin[k] = struct{}{}
457 - if err := p.dstore.Put(pinDatastoreKey, []byte(k)); err != nil {
473 + internalset.Add(k)
474 + if err := p.dstore.Put(pinDatastoreKey, k.Bytes()); err != nil {
475 return fmt.Errorf("cannot store pin state: %v", err)
476 }
460 - p.internalPin = internalPin
477 + p.internalPin = internalset
478 return nil
479 }
480
464 -func (p *pinner) InternalPins() []key.Key {
481 +func (p *pinner) InternalPins() []*cid.Cid {
482 p.lock.Lock()
483 defer p.lock.Unlock()
467 - var out []key.Key
468 - for k, _ := range p.internalPin {
469 - out = append(out, k)
484 + var out []*cid.Cid
485 + for _, c := range p.internalPin.Keys() {
486 + out = append(out, c)
487 }
488 return out
489 }
490
491 // PinWithMode allows the user to have fine grained control over pin
492 // counts
476 -func (p *pinner) PinWithMode(k key.Key, mode PinMode) {
493 +func (p *pinner) PinWithMode(c *cid.Cid, mode PinMode) {
494 p.lock.Lock()
495 defer p.lock.Unlock()
496 switch mode {
497 case Recursive:
481 - p.recursePin.AddBlock(k)
498 + p.recursePin.Add(c)
499 case Direct:
483 - p.directPin.AddBlock(k)
500 + p.directPin.Add(c)
501 }
502 }
503
504 func hasChild(ds mdag.DAGService, root *mdag.Node, child key.Key) (bool, error) {
505 for _, lnk := range root.Links {
489 - k := key.Key(lnk.Hash)
490 - if k == child {
506 + c := cid.NewCidV0(lnk.Hash)
507 + if key.Key(c.Hash()) == child {
508 return true, nil
509 }
510
494 - nd, err := ds.Get(context.Background(), k)
511 + nd, err := ds.Get(context.Background(), c)
512 if err != nil {
513 return false, err
514 }
pin/pin_test.go
+9 -9
@@ -4,28 +4,28 @@ import (
4 "testing"
5 "time"
6
7 - context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
8 -
7 "github.com/ipfs/go-ipfs/blocks/blockstore"
10 - key "github.com/ipfs/go-ipfs/blocks/key"
8 bs "github.com/ipfs/go-ipfs/blockservice"
9 "github.com/ipfs/go-ipfs/exchange/offline"
10 mdag "github.com/ipfs/go-ipfs/merkledag"
11 +
12 ds "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
13 dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
14 "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
15 + context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
16 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
17 )
18
19 -func randNode() (*mdag.Node, key.Key) {
19 +func randNode() (*mdag.Node, *cid.Cid) {
20 nd := new(mdag.Node)
21 nd.SetData(make([]byte, 32))
22 util.NewTimeSeededRand().Read(nd.Data())
23 - k, _ := nd.Key()
23 + k := nd.Cid()
24 return nd, k
25 }
26
27 -func assertPinned(t *testing.T, p Pinner, k key.Key, failmsg string) {
28 - _, pinned, err := p.IsPinned(k)
27 +func assertPinned(t *testing.T, p Pinner, c *cid.Cid, failmsg string) {
28 + _, pinned, err := p.IsPinned(c)
29 if err != nil {
30 t.Fatal(err)
31 }
@@ -93,7 +93,7 @@ func TestPinnerBasic(t *testing.T) {
93
94 assertPinned(t, p, ck, "child of recursively pinned node not found")
95
96 - bk, _ := b.Key()
96 + bk := b.Cid()
97 assertPinned(t, p, bk, "Recursively pinned node not found..")
98
99 d, _ := randNode()
@@ -119,7 +119,7 @@ func TestPinnerBasic(t *testing.T) {
119 t.Fatal(err)
120 }
121
122 - dk, _ := d.Key()
122 + dk := d.Cid()
123 assertPinned(t, p, dk, "pinned node not found.")
124
125 // Test recursive unpin
pin/set.go
+33 -71
@@ -7,7 +7,6 @@ import (
7 "errors"
8 "fmt"
9 "hash/fnv"
10 - "io"
10 "sort"
11 "unsafe"
12
@@ -16,6 +15,7 @@ import (
15 "github.com/ipfs/go-ipfs/pin/internal/pb"
16 "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
17 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
18 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
19 )
20
21 const (
@@ -31,18 +31,18 @@ func randomSeed() (uint32, error) {
31 return binary.LittleEndian.Uint32(buf[:]), nil
32 }
33
34 -func hash(seed uint32, k key.Key) uint32 {
34 +func hash(seed uint32, c *cid.Cid) uint32 {
35 var buf [4]byte
36 binary.LittleEndian.PutUint32(buf[:], seed)
37 h := fnv.New32a()
38 _, _ = h.Write(buf[:])
39 - _, _ = io.WriteString(h, string(k))
39 + _, _ = h.Write(c.Bytes())
40 return h.Sum32()
41 }
42
43 -type itemIterator func() (k key.Key, data []byte, ok bool)
43 +type itemIterator func() (c *cid.Cid, data []byte, ok bool)
44
45 -type keyObserver func(key.Key)
45 +type keyObserver func(*cid.Cid)
46
47 // refcount is the marshaled format of refcounts. It may change
48 // between versions; this is valid for version 1. Changing it may
@@ -100,7 +100,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
100 Links: make([]*merkledag.Link, 0, defaultFanout+maxItems),
101 }
102 for i := 0; i < defaultFanout; i++ {
103 - n.Links = append(n.Links, &merkledag.Link{Hash: emptyKey.ToMultihash()})
103 + n.Links = append(n.Links, &merkledag.Link{Hash: emptyKey.Hash()})
104 }
105 internalKeys(emptyKey)
106 hdr := &pb.Set{
@@ -121,7 +121,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
121 // all done
122 break
123 }
124 - n.Links = append(n.Links, &merkledag.Link{Hash: k.ToMultihash()})
124 + n.Links = append(n.Links, &merkledag.Link{Hash: k.Hash()})
125 n.SetData(append(n.Data(), data...))
126 }
127 // sort by hash, also swap item Data
@@ -134,7 +134,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
134
135 // wasteful but simple
136 type item struct {
137 - k key.Key
137 + c *cid.Cid
138 data []byte
139 }
140 hashed := make(map[uint32][]item)
@@ -147,13 +147,13 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
147 hashed[h] = append(hashed[h], item{k, data})
148 }
149 for h, items := range hashed {
150 - childIter := func() (k key.Key, data []byte, ok bool) {
150 + childIter := func() (c *cid.Cid, data []byte, ok bool) {
151 if len(items) == 0 {
152 - return "", nil, false
152 + return nil, nil, false
153 }
154 first := items[0]
155 items = items[1:]
156 - return first.k, first.data, true
156 + return first.c, first.data, true
157 }
158 child, err := storeItems(ctx, dag, uint64(len(items)), childIter, internalKeys)
159 if err != nil {
@@ -170,7 +170,7 @@ func storeItems(ctx context.Context, dag merkledag.DAGService, estimatedLen uint
170 internalKeys(childKey)
171 l := &merkledag.Link{
172 Name: "",
173 - Hash: childKey.ToMultihash(),
173 + Hash: childKey.Hash(),
174 Size: size,
175 }
176 n.Links[int(h%defaultFanout)] = l
@@ -231,8 +231,9 @@ func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.Node,
231 }
232 }
233 for _, l := range n.Links[:fanout] {
234 - children(key.Key(l.Hash))
235 - if key.Key(l.Hash) == emptyKey {
234 + c := cid.NewCidV0(l.Hash)
235 + children(c)
236 + if c.Equals(emptyKey) {
237 continue
238 }
239 subtree, err := l.GetNode(ctx, dag)
@@ -246,20 +247,23 @@ func walkItems(ctx context.Context, dag merkledag.DAGService, n *merkledag.Node,
247 return nil
248 }
249
249 -func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) ([]key.Key, error) {
250 +func loadSet(ctx context.Context, dag merkledag.DAGService, root *merkledag.Node, name string, internalKeys keyObserver) ([]*cid.Cid, error) {
251 l, err := root.GetNodeLink(name)
252 if err != nil {
253 return nil, err
254 }
254 - internalKeys(key.Key(l.Hash))
255 +
256 + lnkc := cid.NewCidV0(l.Hash)
257 + internalKeys(lnkc)
258 +
259 n, err := l.GetNode(ctx, dag)
260 if err != nil {
261 return nil, err
262 }
263
260 - var res []key.Key
264 + var res []*cid.Cid
265 walk := func(buf []byte, idx int, link *merkledag.Link) error {
262 - res = append(res, key.Key(link.Hash))
266 + res = append(res, cid.NewCidV0(link.Hash))
267 return nil
268 }
269 if err := walkItems(ctx, dag, n, walk, internalKeys); err != nil {
@@ -273,7 +277,8 @@ func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag
277 if err != nil {
278 return nil, fmt.Errorf("Failed to get link %s: %v", name, err)
279 }
276 - internalKeys(key.Key(l.Hash))
280 + c := cid.NewCidV0(l.Hash)
281 + internalKeys(c)
282 n, err := l.GetNode(ctx, dag)
283 if err != nil {
284 return nil, fmt.Errorf("Failed to get node from link %s: %v", name, err)
@@ -292,24 +297,24 @@ func loadMultiset(ctx context.Context, dag merkledag.DAGService, root *merkledag
297 return refcounts, nil
298 }
299
295 -func storeSet(ctx context.Context, dag merkledag.DAGService, keys []key.Key, internalKeys keyObserver) (*merkledag.Node, error) {
296 - iter := func() (k key.Key, data []byte, ok bool) {
297 - if len(keys) == 0 {
298 - return "", nil, false
300 +func storeSet(ctx context.Context, dag merkledag.DAGService, cids []*cid.Cid, internalKeys keyObserver) (*merkledag.Node, error) {
301 + iter := func() (c *cid.Cid, data []byte, ok bool) {
302 + if len(cids) == 0 {
303 + return nil, nil, false
304 }
300 - first := keys[0]
301 - keys = keys[1:]
305 + first := cids[0]
306 + cids = cids[1:]
307 return first, nil, true
308 }
304 - n, err := storeItems(ctx, dag, uint64(len(keys)), iter, internalKeys)
309 + n, err := storeItems(ctx, dag, uint64(len(cids)), iter, internalKeys)
310 if err != nil {
311 return nil, err
312 }
308 - k, err := dag.Add(n)
313 + c, err := dag.Add(n)
314 if err != nil {
315 return nil, err
316 }
312 - internalKeys(k)
317 + internalKeys(c)
318 return n, nil
319 }
320
@@ -320,46 +325,3 @@ func copyRefcounts(orig map[key.Key]uint64) map[key.Key]uint64 {
325 }
326 return r
327 }
323 -
324 -func storeMultiset(ctx context.Context, dag merkledag.DAGService, refcounts map[key.Key]uint64, internalKeys keyObserver) (*merkledag.Node, error) {
325 - // make a working copy of the refcounts
326 - refcounts = copyRefcounts(refcounts)
327 -
328 - iter := func() (k key.Key, data []byte, ok bool) {
329 - // Every call of this function returns the next refcount item.
330 - //
331 - // This function splits out the uint64 reference counts as
332 - // smaller increments, as fits in type refcount. Most of the
333 - // time the refcount will fit inside just one, so this saves
334 - // space.
335 - //
336 - // We use range here to pick an arbitrary item in the map, but
337 - // not really iterate the map.
338 - for k, refs := range refcounts {
339 - // Max value a single multiset item can store
340 - num := ^refcount(0)
341 - if refs <= uint64(num) {
342 - // Remaining count fits in a single item; remove the
343 - // key from the map.
344 - num = refcount(refs)
345 - delete(refcounts, k)
346 - } else {
347 - // Count is too large to fit in one item, the key will
348 - // repeat in some later call.
349 - refcounts[k] -= uint64(num)
350 - }
351 - return k, num.Bytes(), true
352 - }
353 - return "", nil, false
354 - }
355 - n, err := storeItems(ctx, dag, uint64(len(refcounts)), iter, internalKeys)
356 - if err != nil {
357 - return nil, err
358 - }
359 - k, err := dag.Add(n)
360 - if err != nil {
361 - return nil, err
362 - }
363 - internalKeys(k)
364 - return n, nil
365 -}
pin/set_test.go
+1 -91
@@ -1,20 +1,6 @@
1 package pin
2
3 -import (
4 - "testing"
5 - "testing/quick"
6 -
7 - "github.com/ipfs/go-ipfs/blocks/blockstore"
8 - "github.com/ipfs/go-ipfs/blocks/key"
9 - "github.com/ipfs/go-ipfs/blockservice"
10 - "github.com/ipfs/go-ipfs/exchange/offline"
11 - "github.com/ipfs/go-ipfs/merkledag"
12 - "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore"
13 - dssync "gx/ipfs/QmNgqJarToRiq2GBaPJhkmW4B5BxS5B74E1rkGvv2JoaTp/go-datastore/sync"
14 - mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
15 - u "gx/ipfs/QmZNVWh8LLjAavuQ2JXuFmuYH3C11xo988vSgp7UQrTRj1/go-ipfs-util"
16 - "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
17 -)
3 +import "github.com/ipfs/go-ipfs/blocks/key"
4
5 func ignoreKeys(key.Key) {}
6
@@ -25,79 +11,3 @@ func copyMap(m map[key.Key]uint16) map[key.Key]uint64 {
11 }
12 return c
13 }
28 -
29 -func TestMultisetRoundtrip(t *testing.T) {
30 - dstore := dssync.MutexWrap(datastore.NewMapDatastore())
31 - bstore := blockstore.NewBlockstore(dstore)
32 - bserv := blockservice.New(bstore, offline.Exchange(bstore))
33 - dag := merkledag.NewDAGService(bserv)
34 -
35 - fn := func(m map[key.Key]uint16) bool {
36 - // Convert invalid multihash from input to valid ones
37 - for k, v := range m {
38 - if _, err := mh.Cast([]byte(k)); err != nil {
39 - delete(m, k)
40 - m[key.Key(u.Hash([]byte(k)))] = v
41 - }
42 - }
43 -
44 - // Generate a smaller range for refcounts than full uint64, as
45 - // otherwise this just becomes overly cpu heavy, splitting it
46 - // out into too many items. That means we need to convert to
47 - // the right kind of map. As storeMultiset mutates the map as
48 - // part of its bookkeeping, this is actually good.
49 - refcounts := copyMap(m)
50 -
51 - ctx := context.Background()
52 - n, err := storeMultiset(ctx, dag, refcounts, ignoreKeys)
53 - if err != nil {
54 - t.Fatalf("storing multiset: %v", err)
55 - }
56 -
57 - // Check that the node n is in the DAG
58 - k, err := n.Key()
59 - if err != nil {
60 - t.Fatalf("Could not get key: %v", err)
61 - }
62 - _, err = dag.Get(ctx, k)
63 - if err != nil {
64 - t.Fatalf("Could not get node: %v", err)
65 - }
66 -
67 - root := &merkledag.Node{}
68 - const linkName = "dummylink"
69 - if err := root.AddNodeLink(linkName, n); err != nil {
70 - t.Fatalf("adding link to root node: %v", err)
71 - }
72 -
73 - roundtrip, err := loadMultiset(ctx, dag, root, linkName, ignoreKeys)
74 - if err != nil {
75 - t.Fatalf("loading multiset: %v", err)
76 - }
77 -
78 - orig := copyMap(m)
79 - success := true
80 - for k, want := range orig {
81 - if got, ok := roundtrip[k]; ok {
82 - if got != want {
83 - success = false
84 - t.Logf("refcount changed: %v -> %v for %q", want, got, k)
85 - }
86 - delete(orig, k)
87 - delete(roundtrip, k)
88 - }
89 - }
90 - for k, v := range orig {
91 - success = false
92 - t.Logf("refcount missing: %v for %q", v, k)
93 - }
94 - for k, v := range roundtrip {
95 - success = false
96 - t.Logf("refcount extra: %v for %q", v, k)
97 - }
98 - return success
99 - }
100 - if err := quick.Check(fn, nil); err != nil {
101 - t.Fatal(err)
102 - }
103 -}
test/integration/bitswap_wo_routing_test.go
+6 -4
@@ -7,8 +7,10 @@ import (
7 "github.com/ipfs/go-ipfs/blocks"
8 "github.com/ipfs/go-ipfs/core"
9 "github.com/ipfs/go-ipfs/core/mock"
10 +
11 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
12 mocknet "gx/ipfs/Qmf4ETeAWXuThBfWwonVyFqGFSgTWepUDEr1txcctvpTXS/go-libp2p/p2p/net/mock"
13 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
14 )
15
16 func TestBitswapWithoutRouting(t *testing.T) {
@@ -68,10 +70,10 @@ func TestBitswapWithoutRouting(t *testing.T) {
70 }
71
72 log.Debugf("%d %s get block.", i, n.Identity)
71 - b, err := n.Blocks.GetBlock(ctx, block0.Key())
73 + b, err := n.Blocks.GetBlock(ctx, cid.NewCidV0(block0.Multihash()))
74 if err != nil {
75 t.Error(err)
74 - } else if !bytes.Equal(b.Data(), block0.Data()) {
76 + } else if !bytes.Equal(b.RawData(), block0.RawData()) {
77 t.Error("byte comparison fail")
78 } else {
79 log.Debug("got block: %s", b.Key())
@@ -85,10 +87,10 @@ func TestBitswapWithoutRouting(t *testing.T) {
87
88 // get it out.
89 for _, n := range nodes {
88 - b, err := n.Blocks.GetBlock(ctx, block1.Key())
90 + b, err := n.Blocks.GetBlock(ctx, cid.NewCidV0(block1.Multihash()))
91 if err != nil {
92 t.Error(err)
91 - } else if !bytes.Equal(b.Data(), block1.Data()) {
93 + } else if !bytes.Equal(b.RawData(), block1.RawData()) {
94 t.Error("byte comparison fail")
95 } else {
96 log.Debug("got block: %s", b.Key())
unixfs/io/dirbuilder.go
+3 -3
@@ -3,9 +3,9 @@ package io
3 import (
4 "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
5
6 - key "github.com/ipfs/go-ipfs/blocks/key"
6 mdag "github.com/ipfs/go-ipfs/merkledag"
7 format "github.com/ipfs/go-ipfs/unixfs"
8 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
9 )
10
11 type directoryBuilder struct {
@@ -29,8 +29,8 @@ func NewDirectory(dserv mdag.DAGService) *directoryBuilder {
29 }
30
31 // AddChild adds a (name, key)-pair to the root node.
32 -func (d *directoryBuilder) AddChild(ctx context.Context, name string, k key.Key) error {
33 - cnode, err := d.dserv.Get(ctx, k)
32 +func (d *directoryBuilder) AddChild(ctx context.Context, name string, c *cid.Cid) error {
33 + cnode, err := d.dserv.Get(ctx, c)
34 if err != nil {
35 return err
36 }
unixfs/mod/dagmodifier.go
+13 -14
@@ -6,7 +6,6 @@ import (
6 "io"
7 "os"
8
9 - key "github.com/ipfs/go-ipfs/blocks/key"
9 chunk "github.com/ipfs/go-ipfs/importer/chunk"
10 help "github.com/ipfs/go-ipfs/importer/helpers"
11 trickle "github.com/ipfs/go-ipfs/importer/trickle"
@@ -15,9 +14,9 @@ import (
14 uio "github.com/ipfs/go-ipfs/unixfs/io"
15
16 logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
18 - mh "gx/ipfs/QmYf7ng2hG5XBtJA3tN34DQ2GUN5HNksEw1rLDkmr6vGku/go-multihash"
17 proto "gx/ipfs/QmZ4Qi3GaRbjcx28Sme5eMH7RQjGkt8wHxt2a65oLaeFEV/gogo-protobuf/proto"
18 context "gx/ipfs/QmZy2y8t9zQH2a1b8q2ZSLKp17ATuJoCNxxyMFG5qFExpt/go-net/context"
19 + cid "gx/ipfs/QmfSc2xehWmWLnwwYR91Y8QF4xdASypTFVknutoKQS3GHp/go-cid"
20 )
21
22 var ErrSeekFail = errors.New("failed to seek properly")
@@ -169,12 +168,12 @@ func (dm *DagModifier) Sync() error {
168 buflen := dm.wrBuf.Len()
169
170 // overwrite existing dag nodes
172 - thisk, done, err := dm.modifyDag(dm.curNode, dm.writeStart, dm.wrBuf)
171 + thisc, done, err := dm.modifyDag(dm.curNode, dm.writeStart, dm.wrBuf)
172 if err != nil {
173 return err
174 }
175
177 - nd, err := dm.dagserv.Get(dm.ctx, thisk)
176 + nd, err := dm.dagserv.Get(dm.ctx, thisc)
177 if err != nil {
178 return err
179 }
@@ -188,7 +187,7 @@ func (dm *DagModifier) Sync() error {
187 return err
188 }
189
191 - thisk, err = dm.dagserv.Add(nd)
190 + _, err = dm.dagserv.Add(nd)
191 if err != nil {
192 return err
193 }
@@ -205,30 +204,30 @@ func (dm *DagModifier) Sync() error {
204 // modifyDag writes the data in 'data' over the data in 'node' starting at 'offset'
205 // returns the new key of the passed in node and whether or not all the data in the reader
206 // has been consumed.
208 -func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (key.Key, bool, error) {
207 +func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader) (*cid.Cid, bool, error) {
208 f, err := ft.FromBytes(node.Data())
209 if err != nil {
211 - return "", false, err
210 + return nil, false, err
211 }
212
213 // If we've reached a leaf node.
214 if len(node.Links) == 0 {
215 n, err := data.Read(f.Data[offset:])
216 if err != nil && err != io.EOF {
218 - return "", false, err
217 + return nil, false, err
218 }
219
220 // Update newly written node..
221 b, err := proto.Marshal(f)
222 if err != nil {
224 - return "", false, err
223 + return nil, false, err
224 }
225
226 nd := new(mdag.Node)
227 nd.SetData(b)
228 k, err := dm.dagserv.Add(nd)
229 if err != nil {
231 - return "", false, err
230 + return nil, false, err
231 }
232
233 // Hey look! we're done!
@@ -247,20 +246,20 @@ func (dm *DagModifier) modifyDag(node *mdag.Node, offset uint64, data io.Reader)
246 if cur+bs > offset {
247 child, err := node.Links[i].GetNode(dm.ctx, dm.dagserv)
248 if err != nil {
250 - return "", false, err
249 + return nil, false, err
250 }
251 k, sdone, err := dm.modifyDag(child, offset-cur, data)
252 if err != nil {
254 - return "", false, err
253 + return nil, false, err
254 }
255
256 offset += bs
258 - node.Links[i].Hash = mh.Multihash(k)
257 + node.Links[i].Hash = k.Hash()
258
259 // Recache serialized node
260 _, err = node.EncodeProtobuf(true)
261 if err != nil {
263 - return "", false, err
262 + return nil, false, err
263 }
264
265 if sdone {