feat: switch to raw multihashes for blocks
Part of: https://github.com/ipfs/go-ipfs/issues/6815
Steven Allen committed
Jan 7, 2020 at 09:43 UTC
7f27dbfd1570e187b8a76035f13bc31cf6b86e10
4 files changed
+34
-94
core/commands/refs.go
+1
-1
@@ -136,7 +136,7 @@ var RefsLocalCmd = &cmds.Command{
136
Helptext: cmds.HelpText{
137
Tagline: "List all local references.",
138
ShortDescription: `
139
-Displays the hashes of all local objects.
139
+Displays the hashes of all local objects. NOTE: This treats all local objects as "raw blocks" and returns CIDv1-Raw CIDs.
140
`,
141
},
142
core/node/storage.go
-2
@@ -9,7 +9,6 @@ import (
9
"github.com/ipfs/go-filestore"
10
"github.com/ipfs/go-ipfs/core/node/helpers"
11
"github.com/ipfs/go-ipfs/repo"
12
- "github.com/ipfs/go-ipfs/thirdparty/cidv0v1"
12
"github.com/ipfs/go-ipfs/thirdparty/verifbs"
13
)
14
@@ -41,7 +40,6 @@ func BaseBlockstoreCtor(cacheOpts blockstore.CacheOpts, nilRepo bool, hashOnRead
40
}
41
42
bs = blockstore.NewIdStore(bs)
44
- bs = cidv0v1.NewBlockstore(bs)
43
44
if hashOnRead { // TODO: review: this is how it was done originally, is there a reason we can't just pass this directly?
45
bs.HashOnRead(true)
gc/gc.go
+33
-2
@@ -28,6 +28,16 @@ type Result struct {
28
Error error
29
}
30
31
+// converts a set of CIDs with different codecs to a set of CIDs with the raw codec.
32
+func toRawCids(set *cid.Set) *cid.Set {
33
+ newSet := cid.NewSet()
34
+ set.ForEach(func(c cid.Cid) error {
35
+ newSet.Add(cid.NewCidV1(cid.Raw, c.Hash()))
36
+ return nil
37
+ })
38
+ return newSet
39
+}
40
+
41
// GC performs a mark and sweep garbage collection of the blocks in the blockstore
42
// first, it creates a 'marked' set and adds to it the following:
43
// - all recursively pinned blocks, plus all of their descendants (recursively)
@@ -60,6 +70,15 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
70
}
71
return
72
}
73
+
74
+ // The blockstore reports raw blocks. We need to remove the codecs from the CIDs.
75
+ gcs = toRawCids(gcs)
76
+ emark.Append(logging.LoggableMap{
77
+ "blackSetSize": fmt.Sprintf("%d", gcs.Len()),
78
+ })
79
+ emark.Done()
80
+ esweep := log.EventBegin(ctx, "GC.sweep")
81
+
82
keychan, err := bs.AllKeysChan(ctx)
83
if err != nil {
84
select {
@@ -79,6 +98,8 @@ func GC(ctx context.Context, bs bstore.GCBlockstore, dstor dstore.Datastore, pn
98
if !ok {
99
break loop
100
}
101
+ // NOTE: assumes that all CIDs returned by the keychan are _raw_ CIDv1 CIDs.
102
+ // This means we keep the block as long as we want it somewhere (CIDv1, CIDv0, Raw, other...).
103
if !gcs.Has(k) {
104
err := bs.DeleteBlock(ctx, k)
105
removed++
@@ -154,7 +175,9 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
175
176
for _, c := range roots {
177
// Walk recursively walks the dag and adds the keys to the given set
157
- err := dag.Walk(ctx, verifyGetLinks, c, set.Visit, dag.Concurrent())
178
+ err := dag.Walk(ctx, verifyGetLinks, c, func(k cid.Cid) bool {
179
+ return set.Visit(toCidV1(k))
180
+ }, dag.Concurrent())
181
182
if err != nil {
183
err = verboseCidError(err)
@@ -165,6 +188,14 @@ func Descendants(ctx context.Context, getLinks dag.GetLinks, set *cid.Set, roots
188
return nil
189
}
190
191
+// toCidV1 converts any CIDv0s to CIDv1s.
192
+func toCidV1(c cid.Cid) cid.Cid {
193
+ if c.Version() == 0 {
194
+ return cid.NewCidV1(c.Type(), c.Hash())
195
+ }
196
+ return c
197
+}
198
+
199
// ColoredSet computes the set of nodes in the graph that are pinned by the
200
// pins in the given pinner.
201
func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffortRoots []cid.Cid, output chan<- Result) (*cid.Set, error) {
@@ -225,7 +256,7 @@ func ColoredSet(ctx context.Context, pn pin.Pinner, ng ipld.NodeGetter, bestEffo
256
return nil, err
257
}
258
for _, k := range dkeys {
228
- gcs.Add(k)
259
+ gcs.Add(toCidV1(k))
260
}
261
262
ikeys, err := pn.InternalPins(ctx)
thirdparty/cidv0v1/blockstore.go
deleted
-89
@@ -1,89 +0,0 @@
1
-package cidv0v1
2
-
3
-import (
4
- "context"
5
-
6
- blocks "github.com/ipfs/go-block-format"
7
- cid "github.com/ipfs/go-cid"
8
- bs "github.com/ipfs/go-ipfs-blockstore"
9
- mh "github.com/multiformats/go-multihash"
10
-)
11
-
12
-type blockstore struct {
13
- bs.Blockstore
14
-}
15
-
16
-func NewBlockstore(b bs.Blockstore) bs.Blockstore {
17
- return &blockstore{b}
18
-}
19
-
20
-func (b *blockstore) Has(ctx context.Context, c cid.Cid) (bool, error) {
21
- have, err := b.Blockstore.Has(ctx, c)
22
- if have || err != nil {
23
- return have, err
24
- }
25
- c1 := tryOtherCidVersion(c)
26
- if !c1.Defined() {
27
- return false, nil
28
- }
29
- return b.Blockstore.Has(ctx, c1)
30
-}
31
-
32
-func (b *blockstore) Get(ctx context.Context, c cid.Cid) (blocks.Block, error) {
33
- block, err := b.Blockstore.Get(ctx, c)
34
- if err == nil {
35
- return block, nil
36
- }
37
- if err != bs.ErrNotFound {
38
- return nil, err
39
- }
40
- c1 := tryOtherCidVersion(c)
41
- if !c1.Defined() {
42
- return nil, bs.ErrNotFound
43
- }
44
- block, err = b.Blockstore.Get(ctx, c1)
45
- if err != nil {
46
- return nil, err
47
- }
48
- // modify block so it has the original CID
49
- block, err = blocks.NewBlockWithCid(block.RawData(), c)
50
- if err != nil {
51
- return nil, err
52
- }
53
- // insert the block with the original CID to avoid problems
54
- // with pinning
55
- err = b.Blockstore.Put(ctx, block)
56
- if err != nil {
57
- return nil, err
58
- }
59
- return block, nil
60
-}
61
-
62
-func (b *blockstore) GetSize(ctx context.Context, c cid.Cid) (int, error) {
63
- size, err := b.Blockstore.GetSize(ctx, c)
64
- if err == nil {
65
- return size, nil
66
- }
67
- if err != bs.ErrNotFound {
68
- return -1, err
69
- }
70
- c1 := tryOtherCidVersion(c)
71
- if !c1.Defined() {
72
- return -1, bs.ErrNotFound
73
- }
74
- return b.Blockstore.GetSize(ctx, c1)
75
-}
76
-
77
-func tryOtherCidVersion(c cid.Cid) cid.Cid {
78
- prefix := c.Prefix()
79
- if prefix.Codec != cid.DagProtobuf || prefix.MhType != mh.SHA2_256 || prefix.MhLength != 32 {
80
- return cid.Undef
81
- }
82
- var c1 cid.Cid
83
- if prefix.Version == 0 {
84
- c1 = cid.NewCidV1(cid.DagProtobuf, c.Hash())
85
- } else {
86
- c1 = cid.NewCidV0(c.Hash())
87
- }
88
- return c1
89
-}