feat(cmds): extend block size check for dag|block put (#8751)
* feat(cmds): extend block size check for dag|block put * feat(cmds): block size check for dag import * style: dag-pb → UnixFS, 1MB → 1MiB Co-authored-by: Marcin Rataj <lidel@lidel.org>
Lucas Molas committed
Mar 11, 2022 at 19:23 UTC
04e7e9502e09959171c31bb880255ddda17ed848
11 files changed
+120
-39
core/commands/block.go
+6
@@ -10,6 +10,7 @@ import (
10
11
util "github.com/ipfs/go-ipfs/blocks/blockstoreutil"
12
cmdenv "github.com/ipfs/go-ipfs/core/commands/cmdenv"
13
+ "github.com/ipfs/go-ipfs/core/commands/cmdutils"
14
15
cmds "github.com/ipfs/go-ipfs-cmds"
16
options "github.com/ipfs/interface-go-ipfs-core/options"
@@ -138,6 +139,7 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
139
cmds.StringOption(mhtypeOptionName, "multihash hash function").WithDefault("sha2-256"),
140
cmds.IntOption(mhlenOptionName, "multihash hash length").WithDefault(-1),
141
cmds.BoolOption(pinOptionName, "pin added blocks recursively").WithDefault(false),
142
+ cmdutils.AllowBigBlockOption,
143
},
144
Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
145
api, err := cmdenv.GetApi(env, req)
@@ -182,6 +184,10 @@ other than 'sha2-256' or format to anything other than 'v0' will result in CIDv1
184
return err
185
}
186
187
+ if err := cmdutils.CheckBlockSize(req, uint64(p.Size())); err != nil {
188
+ return err
189
+ }
190
+
191
err = res.Emit(&BlockStat{
192
Key: p.Path().Cid().String(),
193
Size: p.Size(),
core/commands/cmdutils/utils.go
new
+51
@@ -0,0 +1,51 @@
1
+package cmdutils
2
+
3
+import (
4
+ "fmt"
5
+
6
+ cmds "github.com/ipfs/go-ipfs-cmds"
7
+
8
+ "github.com/ipfs/go-cid"
9
+ coreiface "github.com/ipfs/interface-go-ipfs-core"
10
+)
11
+
12
+const (
13
+ AllowBigBlockOptionName = "allow-big-block"
14
+ SoftBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499
15
+)
16
+
17
+var AllowBigBlockOption cmds.Option
18
+
19
+func init() {
20
+ AllowBigBlockOption = cmds.BoolOption(AllowBigBlockOptionName, "Disable block size check and allow creation of blocks bigger than 1MiB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false)
21
+}
22
+
23
+func CheckCIDSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error {
24
+ n, err := dagAPI.Get(req.Context, c)
25
+ if err != nil {
26
+ return fmt.Errorf("CheckCIDSize: getting dag: %w", err)
27
+ }
28
+
29
+ nodeSize, err := n.Size()
30
+ if err != nil {
31
+ return fmt.Errorf("CheckCIDSize: getting node size: %w", err)
32
+ }
33
+
34
+ return CheckBlockSize(req, nodeSize)
35
+}
36
+
37
+func CheckBlockSize(req *cmds.Request, size uint64) error {
38
+ allowAnyBlockSize, _ := req.Options[AllowBigBlockOptionName].(bool)
39
+ if allowAnyBlockSize {
40
+ return nil
41
+ }
42
+
43
+ // We do not allow producing blocks bigger than 1 MiB to avoid errors
44
+ // when transmitting them over BitSwap. The 1 MiB constant is an
45
+ // unenforced and undeclared rule of thumb hard-coded here.
46
+ if size > SoftBlockLimit {
47
+ return fmt.Errorf("produced block is over 1MiB: big blocks can't be exchanged with other peers. consider using UnixFS for automatic chunking of bigger files, or pass --allow-big-block to override")
48
+ }
49
+ return nil
50
+
51
+}
core/commands/dag/dag.go
+3
@@ -5,6 +5,7 @@ import (
5
"io"
6
7
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
8
+ "github.com/ipfs/go-ipfs/core/commands/cmdutils"
9
10
cid "github.com/ipfs/go-cid"
11
cidenc "github.com/ipfs/go-cidutil/cidenc"
@@ -88,6 +89,7 @@ into an object of the specified format.
89
cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),
90
cmds.BoolOption("pin", "Pin this object when adding."),
91
cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),
92
+ cmdutils.AllowBigBlockOption,
93
},
94
Run: dagPut,
95
Type: OutputObject{},
@@ -205,6 +207,7 @@ Maximum supported CAR version: 1
207
cmds.BoolOption(pinRootsOptionName, "Pin optional roots listed in the .car headers after importing.").WithDefault(true),
208
cmds.BoolOption(silentOptionName, "No output."),
209
cmds.BoolOption(statsOptionName, "Output stats."),
210
+ cmdutils.AllowBigBlockOption,
211
},
212
Type: CarImportOutput{},
213
Run: dagImport,
core/commands/dag/import.go
+4
@@ -8,6 +8,7 @@ import (
8
cid "github.com/ipfs/go-cid"
9
files "github.com/ipfs/go-ipfs-files"
10
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
11
+ "github.com/ipfs/go-ipfs/core/commands/cmdutils"
12
ipld "github.com/ipfs/go-ipld-format"
13
iface "github.com/ipfs/interface-go-ipfs-core"
14
"github.com/ipfs/interface-go-ipfs-core/options"
@@ -180,6 +181,9 @@ func importWorker(req *cmds.Request, re cmds.ResponseEmitter, api iface.CoreAPI,
181
} else if block == nil {
182
break
183
}
184
+ if err := cmdutils.CheckBlockSize(req, uint64(len(block.RawData()))); err != nil {
185
+ return err
186
+ }
187
188
// the double-decode is suboptimal, but we need it for batching
189
nd, err := ipld.Decode(block)
core/commands/dag/put.go
+5
@@ -7,6 +7,7 @@ import (
7
blocks "github.com/ipfs/go-block-format"
8
"github.com/ipfs/go-cid"
9
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
10
+ "github.com/ipfs/go-ipfs/core/commands/cmdutils"
11
ipldlegacy "github.com/ipfs/go-ipld-legacy"
12
"github.com/ipld/go-ipld-prime/multicodec"
13
basicnode "github.com/ipld/go-ipld-prime/node/basic"
@@ -102,6 +103,10 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e
103
Node: n,
104
}
105
106
+ if err := cmdutils.CheckBlockSize(req, uint64(bd.Len())); err != nil {
107
+ return err
108
+ }
109
+
110
if err := b.Add(req.Context, &ln); err != nil {
111
return err
112
}
core/commands/get.go
+1
-1
@@ -257,7 +257,7 @@ func getCompressOptions(req *cmds.Request) (int, error) {
257
return cmplvl, nil
258
}
259
260
-// DefaultBufSize is the buffer size for gets. for now, 1MB, which is ~4 blocks.
260
+// DefaultBufSize is the buffer size for gets. for now, 1MiB, which is ~4 blocks.
261
// TODO: does this need to be configurable?
262
var DefaultBufSize = 1048576
263
core/commands/object/patch.go
+7
-36
@@ -4,20 +4,14 @@ import (
4
"fmt"
5
"io"
6
7
- "github.com/ipfs/go-cid"
7
cmds "github.com/ipfs/go-ipfs-cmds"
8
"github.com/ipfs/go-ipfs/core/commands/cmdenv"
10
- coreiface "github.com/ipfs/interface-go-ipfs-core"
9
+ "github.com/ipfs/go-ipfs/core/commands/cmdutils"
10
11
"github.com/ipfs/interface-go-ipfs-core/options"
12
"github.com/ipfs/interface-go-ipfs-core/path"
13
)
14
16
-const (
17
- softBlockLimit = 1024 * 1024 // https://github.com/ipfs/go-ipfs/issues/7421#issuecomment-910833499
18
- allowBigBlock = "allow-big-block"
19
-)
20
-
15
var ObjectPatchCmd = &cmds.Command{
16
Helptext: cmds.HelpText{
17
Tagline: "Deprecated way to create a new merkledag object based on an existing one. Use MFS with 'files cp|rm' instead.",
@@ -49,7 +43,7 @@ For modern use cases, use MFS with 'files' commands: 'ipfs files --help'.
43
"set-data": patchSetDataCmd,
44
},
45
Options: []cmds.Option{
52
- cmds.BoolOption(allowBigBlock, "Disable block size check and allow creation of blocks bigger than 1MB. WARNING: such blocks won't be transferable over the standard bitswap.").WithDefault(false),
46
+ cmdutils.AllowBigBlockOption,
47
},
48
}
49
@@ -64,7 +58,7 @@ Example:
58
$ echo "hello" | ipfs object patch $HASH append-data
59
60
NOTE: This does not append data to a file - it modifies the actual raw
67
-data within a dag-pb object. Blocks have a max size of 1MB and objects larger than
61
+data within a dag-pb object. Blocks have a max size of 1MiB and objects larger than
62
the limit will not be respected by the network.
63
64
DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' instead.
@@ -92,7 +86,7 @@ DEPRECATED and provided for legacy reasons. Use 'ipfs add' or 'ipfs files' inste
86
return err
87
}
88
95
- if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
89
+ if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
90
return err
91
}
92
@@ -142,7 +136,7 @@ DEPRECATED and provided for legacy reasons. Use 'files cp' and 'dag put' instead
136
return err
137
}
138
145
- if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
139
+ if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
140
return err
141
}
142
@@ -184,7 +178,7 @@ DEPRECATED and provided for legacy reasons. Use 'files rm' instead.
178
return err
179
}
180
187
- if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
181
+ if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
182
return err
183
}
184
@@ -254,7 +248,7 @@ Use MFS and 'files' commands instead:
248
return err
249
}
250
257
- if err := checkBlockSize(req, p.Cid(), api.Dag()); err != nil {
251
+ if err := cmdutils.CheckCIDSize(req, p.Cid(), api.Dag()); err != nil {
252
return err
253
}
254
@@ -268,26 +262,3 @@ Use MFS and 'files' commands instead:
262
}),
263
},
264
}
271
-
272
-func checkBlockSize(req *cmds.Request, c cid.Cid, dagAPI coreiface.APIDagService) error {
273
- allowAnyBlockSize, _ := req.Options[allowBigBlock].(bool)
274
- if allowAnyBlockSize {
275
- return nil
276
- }
277
-
278
- // We do not allow producing blocks bigger than 1 MiB to avoid errors
279
- // when transmitting them over BitSwap. The 1 MiB constant is an
280
- // unenforced and undeclared rule of thumb hard-coded here.
281
- modifiedNode, err := dagAPI.Get(req.Context, c)
282
- if err != nil {
283
- return err
284
- }
285
- modifiedNodeSize, err := modifiedNode.Size()
286
- if err != nil {
287
- return err
288
- }
289
- if modifiedNodeSize > softBlockLimit {
290
- return fmt.Errorf("produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)")
291
- }
292
- return nil
293
-}
test/sharness/t0050-block.sh
+14
@@ -248,4 +248,18 @@ test_expect_success "put with sha3 and cidv0 fails" '
248
echo "foooo" | test_must_fail ipfs block put --mhtype=sha3 --mhlen=20 --format=v0
249
'
250
251
+test_expect_success "'ipfs block put' check block size" '
252
+ dd if=/dev/zero bs=2MB count=1 > 2-MB-file &&
253
+ test_expect_code 1 ipfs block put 2-MB-file >block_put_out 2>&1
254
+ '
255
+
256
+ test_expect_success "ipfs block put output has the correct error" '
257
+ grep "produced block is over 1MiB" block_put_out
258
+ '
259
+
260
+ test_expect_success "ipfs block put --allow-big-block=true works" '
261
+ test_expect_code 0 ipfs block put 2-MB-file --allow-big-block=true &&
262
+ rm 2-MB-file
263
+ '
264
+
265
test_done
test/sharness/t0051-object.sh
+2
-2
@@ -229,12 +229,12 @@ test_object_cmd() {
229
do
230
DIR=$(ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR")
231
done
232
- # Fail when new block goes over the BS limit of 1MB, but allow manual override
232
+ # Fail when new block goes over the BS limit of 1MiB, but allow manual override
233
test_expect_code 1 ipfs object patch "$DIR" add-link "$DIR.jpg" "$DIR" >patch_out 2>&1
234
'
235
236
test_expect_success "ipfs object patch add-link output has the correct error" '
237
- grep "produced block is over 1MB, object API is deprecated and does not support HAMT-sharding: to create big directories, please use the files API (MFS)" patch_out
237
+ grep "produced block is over 1MiB" patch_out
238
'
239
240
test_expect_success "ipfs object patch --allow-big-block=true add-link works" '
test/sharness/t0053-dag.sh
+14
@@ -44,6 +44,20 @@ test_dag_cmd() {
44
test $EXPHASH = $IPLDHASH
45
'
46
47
+test_expect_success "'ipfs dag put' check block size" '
48
+ dd if=/dev/zero bs=2MB count=1 > 2-MB-file &&
49
+ test_expect_code 1 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file >dag_put_out 2>&1
50
+ '
51
+
52
+ test_expect_success "ipfs dag put output has the correct error" '
53
+ grep "produced block is over 1MiB" dag_put_out
54
+ '
55
+
56
+ test_expect_success "ipfs dag put --allow-big-block=true works" '
57
+ test_expect_code 0 ipfs dag put --input-codec=raw --store-codec=raw 2-MB-file --allow-big-block=true &&
58
+ rm 2-MB-file
59
+ '
60
+
61
test_expect_success "can add an ipld object using dag-json to dag-json" '
62
IPLDHASH=$(cat ipld_object | ipfs dag put --input-codec dag-json --store-codec dag-json)
63
'
test/sharness/t0054-dag-car-import-export.sh
+13
@@ -233,4 +233,17 @@ test_expect_success "naked root import expected output" '
233
test_cmp_sorted naked_root_import_json_expected naked_root_import_json_actual
234
'
235
236
+test_expect_success "'ipfs dag import' check block size" '
237
+ BIG_CID=$(dd if=/dev/zero bs=2MB count=1 | ipfs dag put --input-codec=raw --store-codec=raw --allow-big-block) &&
238
+ ipfs dag export $BIG_CID > 2-MB-block.car &&
239
+ test_expect_code 1 ipfs dag import 2-MB-block.car >dag_import_out 2>&1
240
+'
241
+test_expect_success "ipfs dag import output has the correct error" '
242
+ grep "block is over 1MiB" dag_import_out
243
+'
244
+
245
+test_expect_success "ipfs dag import --allow-big-block works" '
246
+ test_expect_code 0 ipfs dag import --allow-big-block 2-MB-block.car
247
+'
248
+
249
test_done