refactor(block): CIDv1 and BlockPutSettings CidPrefix (#80)
* feat(block options): add store codec * refactor: BlockPutSettings.CidPrefix Removes duplicated fields and replaces them with cid.Prefix Codec, MhType and MhLength were already in prefix, and we already return prefix. A lot of duplicated values and code responsible for syncing them did not really need to exist. * test: CIDv1 raw and dag-pb cases * chore: release 0.7.0 Co-authored-by: Marcin Rataj <lidel@lidel.org> This commit was moved from ipfs/interface-go-ipfs-core@a3374d99028d96a1ef262b81acb385690eb36f97 This commit was moved from ipfs/boxo@aca3a1839f42a590b5fa7ce30e743232c9336023
Lucas Molas committed
Apr 21, 2022 at 12:41 UTC
caa42b563414150604234fbe9b7588b240187aaa
2 files changed
+181
-56
core/coreiface/options/block.go
+86
-48
@@ -2,15 +2,15 @@ package options
2
3
import (
4
"fmt"
5
+
6
cid "github.com/ipfs/go-cid"
7
+ mc "github.com/multiformats/go-multicodec"
8
mh "github.com/multiformats/go-multihash"
9
)
10
11
type BlockPutSettings struct {
10
- Codec string
11
- MhType uint64
12
- MhLength int
13
- Pin bool
12
+ CidPrefix cid.Prefix
13
+ Pin bool
14
}
15
16
type BlockRmSettings struct {
@@ -20,53 +20,29 @@ type BlockRmSettings struct {
20
type BlockPutOption func(*BlockPutSettings) error
21
type BlockRmOption func(*BlockRmSettings) error
22
23
-func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, cid.Prefix, error) {
23
+func BlockPutOptions(opts ...BlockPutOption) (*BlockPutSettings, error) {
24
+ var cidPrefix cid.Prefix
25
+
26
+ // Baseline is CIDv1 raw sha2-255-32 (can be tweaked later via opts)
27
+ cidPrefix.Version = 1
28
+ cidPrefix.Codec = uint64(mc.Raw)
29
+ cidPrefix.MhType = mh.SHA2_256
30
+ cidPrefix.MhLength = -1 // -1 means len is to be calculated during mh.Sum()
31
+
32
options := &BlockPutSettings{
25
- Codec: "",
26
- MhType: mh.SHA2_256,
27
- MhLength: -1,
28
- Pin: false,
33
+ CidPrefix: cidPrefix,
34
+ Pin: false,
35
}
36
37
+ // Apply any overrides
38
for _, opt := range opts {
39
err := opt(options)
40
if err != nil {
34
- return nil, cid.Prefix{}, err
35
- }
36
- }
37
-
38
- var pref cid.Prefix
39
- pref.Version = 1
40
-
41
- if options.Codec == "" {
42
- if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
43
- options.Codec = "protobuf"
44
- } else {
45
- options.Codec = "v0"
46
- }
47
- }
48
-
49
- if options.Codec == "v0" && options.MhType == mh.SHA2_256 {
50
- pref.Version = 0
51
- }
52
-
53
- formatval, ok := cid.Codecs[options.Codec]
54
- if !ok {
55
- return nil, cid.Prefix{}, fmt.Errorf("unrecognized format: %s", options.Codec)
56
- }
57
-
58
- if options.Codec == "v0" {
59
- if options.MhType != mh.SHA2_256 || (options.MhLength != -1 && options.MhLength != 32) {
60
- return nil, cid.Prefix{}, fmt.Errorf("only sha2-255-32 is allowed with CIDv0")
41
+ return nil, err
42
}
43
}
44
64
- pref.Codec = formatval
65
-
66
- pref.MhType = options.MhType
67
- pref.MhLength = options.MhLength
68
-
69
- return options, pref, nil
45
+ return options, nil
46
}
47
48
func BlockRmOptions(opts ...BlockRmOption) (*BlockRmSettings, error) {
@@ -87,13 +63,75 @@ type blockOpts struct{}
63
64
var Block blockOpts
65
90
-// Format is an option for Block.Put which specifies the multicodec to use to
91
-// serialize the object. Default is "v0"
92
-func (blockOpts) Format(codec string) BlockPutOption {
66
+// CidCodec is the modern option for Block.Put which specifies the multicodec to use
67
+// in the CID returned by the Block.Put operation.
68
+// It uses correct codes from go-multicodec and replaces the old Format now with CIDv1 as the default.
69
+func (blockOpts) CidCodec(codecName string) BlockPutOption {
70
+ return func(settings *BlockPutSettings) error {
71
+ if codecName == "" {
72
+ return nil
73
+ }
74
+ code, err := codeFromName(codecName)
75
+ if err != nil {
76
+ return err
77
+ }
78
+ settings.CidPrefix.Codec = uint64(code)
79
+ return nil
80
+ }
81
+}
82
+
83
+// Map string to code from go-multicodec
84
+func codeFromName(codecName string) (mc.Code, error) {
85
+ var cidCodec mc.Code
86
+ err := cidCodec.Set(codecName)
87
+ return cidCodec, err
88
+}
89
+
90
+// Format is a legacy option for Block.Put which specifies the multicodec to
91
+// use to serialize the object.
92
+// Provided for backward-compatibility only. Use CidCodec instead.
93
+func (blockOpts) Format(format string) BlockPutOption {
94
return func(settings *BlockPutSettings) error {
94
- settings.Codec = codec
95
+ if format == "" {
96
+ return nil
97
+ }
98
+ // Opt-in CIDv0 support for backward-compatibility
99
+ if format == "v0" {
100
+ settings.CidPrefix.Version = 0
101
+ }
102
+
103
+ // Fixup a legacy (invalid) names for dag-pb (0x70)
104
+ if format == "v0" || format == "protobuf" {
105
+ format = "dag-pb"
106
+ }
107
+
108
+ // Fixup invalid name for dag-cbor (0x71)
109
+ if format == "cbor" {
110
+ format = "dag-cbor"
111
+ }
112
+
113
+ // Set code based on name passed as "format"
114
+ code, err := codeFromName(format)
115
+ if err != nil {
116
+ return err
117
+ }
118
+ settings.CidPrefix.Codec = uint64(code)
119
+
120
+ // If CIDv0, ensure all parameters are compatible
121
+ // (in theory go-cid would validate this anyway, but we want to provide better errors)
122
+ pref := settings.CidPrefix
123
+ if pref.Version == 0 {
124
+ if pref.Codec != uint64(mc.DagPb) {
125
+ return fmt.Errorf("only dag-pb is allowed with CIDv0")
126
+ }
127
+ if pref.MhType != mh.SHA2_256 || (pref.MhLength != -1 && pref.MhLength != 32) {
128
+ return fmt.Errorf("only sha2-255-32 is allowed with CIDv0")
129
+ }
130
+ }
131
+
132
return nil
133
}
134
+
135
}
136
137
// Hash is an option for Block.Put which specifies the multihash settings to use
@@ -101,8 +139,8 @@ func (blockOpts) Format(codec string) BlockPutOption {
139
// If mhLen is set to -1, default length for the hash will be used
140
func (blockOpts) Hash(mhType uint64, mhLen int) BlockPutOption {
141
return func(settings *BlockPutSettings) error {
104
- settings.MhType = mhType
105
- settings.MhLength = mhLen
142
+ settings.CidPrefix.MhType = mhType
143
+ settings.CidPrefix.MhLength = mhLen
144
return nil
145
}
146
}
core/coreiface/tests/block.go
+95
-8
@@ -17,15 +17,19 @@ import (
17
)
18
19
var (
20
- pbCid = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN"
21
- cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm"
22
- cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw"
20
+ pbCidV0 = "QmZULkCELmmk5XNfCgTnCyFgAVxBRBXyDHGGMVoLFLiXEN" // dag-pb
21
+ pbCid = "bafybeiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // dag-pb
22
+ rawCid = "bafkreiffndsajwhk3lwjewwdxqntmjm4b5wxaaanokonsggenkbw6slwk4" // raw bytes
23
+ cborCid = "bafyreicnga62zhxnmnlt6ymq5hcbsg7gdhqdu6z4ehu3wpjhvqnflfy6nm" // dag-cbor
24
+ cborKCid = "bafyr2qgsohbwdlk7ajmmbb4lhoytmest4wdbe5xnexfvtxeatuyqqmwv3fgxp3pmhpc27gwey2cct56gloqefoqwcf3yqiqzsaqb7p4jefhcw" // dag-cbor keccak-512
25
)
26
27
+// dag-pb
28
func pbBlock() io.Reader {
29
return bytes.NewReader([]byte{10, 12, 8, 2, 18, 6, 104, 101, 108, 108, 111, 10, 24, 6})
30
}
31
32
+// dag-cbor
33
func cborBlock() io.Reader {
34
return bytes.NewReader([]byte{101, 72, 101, 108, 108, 111})
35
}
@@ -38,8 +42,12 @@ func (tp *TestSuite) TestBlock(t *testing.T) {
42
return nil
43
})
44
41
- t.Run("TestBlockPut", tp.TestBlockPut)
42
- t.Run("TestBlockPutFormat", tp.TestBlockPutFormat)
45
+ t.Run("TestBlockPut (get raw CIDv1)", tp.TestBlockPut)
46
+ t.Run("TestBlockPutCidCodec: dag-pb", tp.TestBlockPutCidCodecDagPb)
47
+ t.Run("TestBlockPutCidCodec: dag-cbor", tp.TestBlockPutCidCodecDagCbor)
48
+ t.Run("TestBlockPutFormat (legacy): cbor → dag-cbor", tp.TestBlockPutFormatDagCbor)
49
+ t.Run("TestBlockPutFormat (legacy): protobuf → dag-pb", tp.TestBlockPutFormatDagPb)
50
+ t.Run("TestBlockPutFormat (legacy): v0 → CIDv0", tp.TestBlockPutFormatV0)
51
t.Run("TestBlockPutHash", tp.TestBlockPutHash)
52
t.Run("TestBlockGet", tp.TestBlockGet)
53
t.Run("TestBlockRm", tp.TestBlockRm)
@@ -47,6 +55,7 @@ func (tp *TestSuite) TestBlock(t *testing.T) {
55
t.Run("TestBlockPin", tp.TestBlockPin)
56
}
57
58
+// when no opts are passed, produced CID has 'raw' codec
59
func (tp *TestSuite) TestBlockPut(t *testing.T) {
60
ctx, cancel := context.WithCancel(context.Background())
61
defer cancel()
@@ -60,12 +69,14 @@ func (tp *TestSuite) TestBlockPut(t *testing.T) {
69
t.Fatal(err)
70
}
71
63
- if res.Path().Cid().String() != pbCid {
72
+ if res.Path().Cid().String() != rawCid {
73
t.Errorf("got wrong cid: %s", res.Path().Cid().String())
74
}
75
}
76
68
-func (tp *TestSuite) TestBlockPutFormat(t *testing.T) {
77
+// Format is deprecated, it used invalid codec names.
78
+// Confirm 'cbor' gets fixed to 'dag-cbor'
79
+func (tp *TestSuite) TestBlockPutFormatDagCbor(t *testing.T) {
80
ctx, cancel := context.WithCancel(context.Background())
81
defer cancel()
82
api, err := tp.makeAPI(ctx)
@@ -83,6 +94,82 @@ func (tp *TestSuite) TestBlockPutFormat(t *testing.T) {
94
}
95
}
96
97
+// Format is deprecated, it used invalid codec names.
98
+// Confirm 'protobuf' got fixed to 'dag-pb'
99
+func (tp *TestSuite) TestBlockPutFormatDagPb(t *testing.T) {
100
+ ctx, cancel := context.WithCancel(context.Background())
101
+ defer cancel()
102
+ api, err := tp.makeAPI(ctx)
103
+ if err != nil {
104
+ t.Fatal(err)
105
+ }
106
+
107
+ res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("protobuf"))
108
+ if err != nil {
109
+ t.Fatal(err)
110
+ }
111
+
112
+ if res.Path().Cid().String() != pbCid {
113
+ t.Errorf("got wrong cid: %s", res.Path().Cid().String())
114
+ }
115
+}
116
+
117
+// Format is deprecated, it used invalid codec names.
118
+// Confirm fake codec 'v0' got fixed to CIDv0 (with implicit dag-pb codec)
119
+func (tp *TestSuite) TestBlockPutFormatV0(t *testing.T) {
120
+ ctx, cancel := context.WithCancel(context.Background())
121
+ defer cancel()
122
+ api, err := tp.makeAPI(ctx)
123
+ if err != nil {
124
+ t.Fatal(err)
125
+ }
126
+
127
+ res, err := api.Block().Put(ctx, pbBlock(), opt.Block.Format("v0"))
128
+ if err != nil {
129
+ t.Fatal(err)
130
+ }
131
+
132
+ if res.Path().Cid().String() != pbCidV0 {
133
+ t.Errorf("got wrong cid: %s", res.Path().Cid().String())
134
+ }
135
+}
136
+
137
+func (tp *TestSuite) TestBlockPutCidCodecDagCbor(t *testing.T) {
138
+ ctx, cancel := context.WithCancel(context.Background())
139
+ defer cancel()
140
+ api, err := tp.makeAPI(ctx)
141
+ if err != nil {
142
+ t.Fatal(err)
143
+ }
144
+
145
+ res, err := api.Block().Put(ctx, cborBlock(), opt.Block.CidCodec("dag-cbor"))
146
+ if err != nil {
147
+ t.Fatal(err)
148
+ }
149
+
150
+ if res.Path().Cid().String() != cborCid {
151
+ t.Errorf("got wrong cid: %s", res.Path().Cid().String())
152
+ }
153
+}
154
+
155
+func (tp *TestSuite) TestBlockPutCidCodecDagPb(t *testing.T) {
156
+ ctx, cancel := context.WithCancel(context.Background())
157
+ defer cancel()
158
+ api, err := tp.makeAPI(ctx)
159
+ if err != nil {
160
+ t.Fatal(err)
161
+ }
162
+
163
+ res, err := api.Block().Put(ctx, pbBlock(), opt.Block.CidCodec("dag-pb"))
164
+ if err != nil {
165
+ t.Fatal(err)
166
+ }
167
+
168
+ if res.Path().Cid().String() != pbCid {
169
+ t.Errorf("got wrong cid: %s", res.Path().Cid().String())
170
+ }
171
+}
172
+
173
func (tp *TestSuite) TestBlockPutHash(t *testing.T) {
174
ctx, cancel := context.WithCancel(context.Background())
175
defer cancel()
@@ -95,7 +182,7 @@ func (tp *TestSuite) TestBlockPutHash(t *testing.T) {
182
ctx,
183
cborBlock(),
184
opt.Block.Hash(mh.KECCAK_512, -1),
98
- opt.Block.Format("cbor"),
185
+ opt.Block.CidCodec("dag-cbor"),
186
)
187
if err != nil {
188
t.Fatal(err)