@cryptotaxi247 / kubo / commits / 8022e13a6

config: introduce Import section (#10421)

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Henrique Dias committed May 14, 2024 at 16:17 UTC 8022e13a6b28d58a209e3d4eaefeb102b9e840e0

11 files changed +305 -8

config/config.go

@@ -36,6 +36,7 @@ type Config struct {

36 Experimental Experiments

37 Plugins Plugins

38 Pinning Pinning

39 + Import Import

41 Internal Internal // experimental/unstable options

42 }

config/import.go new

+17

@@ -0,0 +1,17 @@

1 +package config

2 +

3 +const (

4 + DefaultCidVersion = 0

5 + DefaultUnixFSRawLeaves = false

6 + DefaultUnixFSChunker = "size-262144"

7 + DefaultHashFunction = "sha2-256"

8 +)

9 +

10 +// Import configures the default options for ingesting data. This affects commands

11 +// that ingest data, such as 'ipfs add', 'ipfs dag put, 'ipfs block put', 'ipfs files write'.

12 +type Import struct {

13 + CidVersion OptionalInteger

14 + UnixFSRawLeaves Flag

15 + UnixFSChunker OptionalString

16 + HashFunction OptionalString

17 +}

config/profile.go

+22

@@ -204,6 +204,28 @@ fetching may be degraded.

204 return nil

205 },

206 },

207 + "legacy-cid-v0": {

208 + Description: `Makes UnixFS import produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.`,

209 +

210 + Transform: func(c *Config) error {

211 + c.Import.CidVersion = *NewOptionalInteger(0)

212 + c.Import.UnixFSRawLeaves = False

213 + c.Import.UnixFSChunker = *NewOptionalString("size-262144")

214 + c.Import.HashFunction = *NewOptionalString("sha2-256")

215 + return nil

216 + },

217 + },

218 + "test-cid-v1": {

219 + Description: `Makes UnixFS import produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.`,

220 +

221 + Transform: func(c *Config) error {

222 + c.Import.CidVersion = *NewOptionalInteger(1)

223 + c.Import.UnixFSRawLeaves = True

224 + c.Import.UnixFSChunker = *NewOptionalString("size-1048576")

225 + c.Import.HashFunction = *NewOptionalString("sha2-256")

226 + return nil

227 + },

228 + },

229 }

230

231 func getAvailablePort() (port int, err error) {

core/commands/add.go

+31 -2

@@ -8,6 +8,7 @@ import (

8 gopath "path"

9 "strings"

11 + "github.com/ipfs/kubo/config"

12 "github.com/ipfs/kubo/core/commands/cmdenv"

14 "github.com/cheggaaa/pb"

@@ -155,12 +156,12 @@ See 'dag export' and 'dag import' for more information.

156 cmds.BoolOption(trickleOptionName, "t", "Use trickle-dag format for dag generation."),

157 cmds.BoolOption(onlyHashOptionName, "n", "Only chunk and hash - do not write to disk."),

158 cmds.BoolOption(wrapOptionName, "w", "Wrap files with a directory object."),

158 - cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash").WithDefault("size-262144"),

159 + cmds.StringOption(chunkerOptionName, "s", "Chunking algorithm, size-[bytes], rabin-[min]-[avg]-[max] or buzhash"),

160 cmds.BoolOption(rawLeavesOptionName, "Use raw blocks for leaf nodes."),

161 cmds.BoolOption(noCopyOptionName, "Add the file using filestore. Implies raw-leaves. (experimental)"),

162 cmds.BoolOption(fstoreCacheOptionName, "Check the filestore for pre-existing blocks. (experimental)"),

163 cmds.IntOption(cidVersionOptionName, "CID version. Defaults to 0 unless an option that depends on CIDv1 is passed. Passing version 1 will cause the raw-leaves option to default to true."),

163 - cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)").WithDefault("sha2-256"),

164 + cmds.StringOption(hashOptionName, "Hash function to use. Implies CIDv1 if not sha2-256. (experimental)"),

165 cmds.BoolOption(inlineOptionName, "Inline small blocks into CIDs. (experimental)"),

166 cmds.IntOption(inlineLimitOptionName, "Maximum block size to inline. (experimental)").WithDefault(32),

167 cmds.BoolOption(pinOptionName, "Pin locally to protect added files from garbage collection.").WithDefault(true),

@@ -191,6 +192,16 @@ See 'dag export' and 'dag import' for more information.

192 return err

193 }

194

195 + nd, err := cmdenv.GetNode(env)

196 + if err != nil {

197 + return err

198 + }

199 +

200 + cfg, err := nd.Repo.Config()

201 + if err != nil {

202 + return err

203 + }

204 +

205 progress, _ := req.Options[progressOptionName].(bool)

206 trickle, _ := req.Options[trickleOptionName].(bool)

207 wrap, _ := req.Options[wrapOptionName].(bool)

@@ -207,6 +218,24 @@ See 'dag export' and 'dag import' for more information.

218 inlineLimit, _ := req.Options[inlineLimitOptionName].(int)

219 toFilesStr, toFilesSet := req.Options[toFilesOptionName].(string)

220

221 + if chunker == "" {

222 + chunker = cfg.Import.UnixFSChunker.WithDefault(config.DefaultUnixFSChunker)

223 + }

224 +

225 + if hashFunStr == "" {

226 + hashFunStr = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)

227 + }

228 +

229 + if !cidVerSet && !cfg.Import.CidVersion.IsDefault() {

230 + cidVerSet = true

231 + cidVer = int(cfg.Import.CidVersion.WithDefault(config.DefaultCidVersion))

232 + }

233 +

234 + if !rbset && cfg.Import.UnixFSRawLeaves != config.Default {

235 + rbset = true

236 + rawblks = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)

237 + }

238 +

239 if onlyHash && toFilesSet {

240 return fmt.Errorf("%s and %s options are not compatible", onlyHashOptionName, toFilesOptionName)

241 }

core/commands/block.go

+16 -1

@@ -8,6 +8,7 @@ import (

9 "github.com/ipfs/boxo/files"

11 + "github.com/ipfs/kubo/config"

12 cmdenv "github.com/ipfs/kubo/core/commands/cmdenv"

13 "github.com/ipfs/kubo/core/commands/cmdutils"

@@ -153,7 +154,7 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).

154 },

155 Options: []cmds.Option{

156 cmds.StringOption(blockCidCodecOptionName, "Multicodec to use in returned CID").WithDefault("raw"),

156 - cmds.StringOption(mhtypeOptionName, "Multihash hash function").WithDefault("sha2-256"),

157 + cmds.StringOption(mhtypeOptionName, "Multihash hash function"),

158 cmds.IntOption(mhlenOptionName, "Multihash hash length").WithDefault(-1),

159 cmds.BoolOption(pinOptionName, "Pin added blocks recursively").WithDefault(false),

160 cmdutils.AllowBigBlockOption,

@@ -165,7 +166,21 @@ only for backward compatibility when a legacy CIDv0 is required (--format=v0).

166 return err

167 }

168

169 + nd, err := cmdenv.GetNode(env)

170 + if err != nil {

171 + return err

172 + }

173 +

174 + cfg, err := nd.Repo.Config()

175 + if err != nil {

176 + return err

177 + }

178 +

179 mhtype, _ := req.Options[mhtypeOptionName].(string)

180 + if mhtype == "" {

181 + mhtype = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)

182 + }

183 +

184 mhtval, ok := mh.Names[mhtype]

185 if !ok {

186 return fmt.Errorf("unrecognized multihash function: %s", mhtype)

core/commands/dag/dag.go

+1 -1

@@ -87,7 +87,7 @@ into an object of the specified format.

87 cmds.StringOption("store-codec", "Codec that the stored object will be encoded with").WithDefault("dag-cbor"),

88 cmds.StringOption("input-codec", "Codec that the input object is encoded in").WithDefault("dag-json"),

89 cmds.BoolOption("pin", "Pin this object when adding."),

90 - cmds.StringOption("hash", "Hash function to use").WithDefault("sha2-256"),

90 + cmds.StringOption("hash", "Hash function to use"),

91 cmdutils.AllowBigBlockOption,

92 },

93 Run: dagPut,

core/commands/dag/put.go

+15

@@ -7,6 +7,7 @@ import (

7 blocks "github.com/ipfs/go-block-format"

8 "github.com/ipfs/go-cid"

9 ipldlegacy "github.com/ipfs/go-ipld-legacy"

10 + "github.com/ipfs/kubo/config"

11 "github.com/ipfs/kubo/core/commands/cmdenv"

12 "github.com/ipfs/kubo/core/commands/cmdutils"

13 "github.com/ipld/go-ipld-prime/multicodec"

@@ -32,11 +33,25 @@ func dagPut(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) e

33 return err

34 }

36 + nd, err := cmdenv.GetNode(env)

37 + if err != nil {

38 + return err

39 + }

40 +

41 + cfg, err := nd.Repo.Config()

42 + if err != nil {

43 + return err

44 + }

45 +

46 inputCodec, _ := req.Options["input-codec"].(string)

47 storeCodec, _ := req.Options["store-codec"].(string)

48 hash, _ := req.Options["hash"].(string)

49 dopin, _ := req.Options["pin"].(bool)

51 + if hash == "" {

52 + hash = cfg.Import.HashFunction.WithDefault(config.DefaultHashFunction)

53 + }

54 +

55 var icodec mc.Code

56 if err := icodec.Set(inputCodec); err != nil {

57 return err

core/commands/files.go

+15 -4

@@ -11,6 +11,7 @@ import (

11 "strings"

13 humanize "github.com/dustin/go-humanize"

14 + "github.com/ipfs/kubo/config"

15 "github.com/ipfs/kubo/core"

16 "github.com/ipfs/kubo/core/commands/cmdenv"

@@ -802,18 +803,28 @@ See '--to-files' in 'ipfs add --help' for more information.

803 return err

804 }

805

806 + nd, err := cmdenv.GetNode(env)

807 + if err != nil {

808 + return err

809 + }

810 +

811 + cfg, err := nd.Repo.Config()

812 + if err != nil {

813 + return err

814 + }

815 +

816 create, _ := req.Options[filesCreateOptionName].(bool)

817 mkParents, _ := req.Options[filesParentsOptionName].(bool)

818 trunc, _ := req.Options[filesTruncateOptionName].(bool)

819 flush, _ := req.Options[filesFlushOptionName].(bool)

820 rawLeaves, rawLeavesDef := req.Options[filesRawLeavesOptionName].(bool)

821

811 - prefix, err := getPrefixNew(req)

812 - if err != nil {

813 - return err

822 + if !rawLeavesDef && cfg.Import.UnixFSRawLeaves != config.Default {

823 + rawLeavesDef = true

824 + rawLeaves = cfg.Import.UnixFSRawLeaves.WithDefault(config.DefaultUnixFSRawLeaves)

825 }

826

816 - nd, err := cmdenv.GetNode(env)

827 + prefix, err := getPrefixNew(req)

828 if err != nil {

829 return err

830 }

docs/changelogs/v0.29.md

+11

@@ -7,6 +7,7 @@

7 - [Overview](#overview)

8 - [🔦 Highlights](#-highlights)

9 - [Add search functionality for pin names](#add-search-functionality-for-pin-names)

10 + - [Customizing `ipfs add` defaults](#customizing-ipfs-add-defaults)

11 - [📝 Changelog](#-changelog)

12 - [👨‍👩‍👧‍👦 Contributors](#-contributors)

@@ -18,6 +19,16 @@

20 It is now possible to search for pins by name. To do so, use `ipfs pin ls --name "SomeName"`. The search is case-sensitive and will return all pins having a name which contains the exact word provided.

22 +#### Customizing `ipfs add` defaults

23 +

24 +This release supports overriding global data ingestion defaults used by commands like `ipfs add` via user-defined [`Import.*` configuration options](../config.md#import).

25 +The hash function, CID version, or UnixFS raw leaves and chunker behaviors can be set once, and used as the new implicit default for `ipfs add`.

26 +

27 +> [!TIP]

28 +> As a convenience, two CID [profiles](../config.md#profile) are provided: `legacy-cid-v0` and `test-cid-v1`.

29 +> A test profile that defaults to modern CIDv1 can be applied via `ipfs config profile apply test-cid-v1`.

30 +> We encourage users to try it and report any issues.

31 +

32 ### 📝 Changelog

34 ### 👨‍👩‍👧‍👦 Contributors

docs/config.md

+58

@@ -175,6 +175,11 @@ config file at runtime.

175 - [`DNS`](#dns)

176 - [`DNS.Resolvers`](#dnsresolvers)

177 - [`DNS.MaxCacheTTL`](#dnsmaxcachettl)

178 + - [`Import`](#import)

179 + - [`Import.CidVersion`](#importcidversion)

180 + - [`Import.UnixFSRawLeaves`](#importunixfsrawleaves)

181 + - [`Import.UnixFSChunker`](#importunixfschunker)

182 + - [`Import.HashFunction`](#importhashfunction)

183

184 ## Profiles

185

@@ -265,6 +270,21 @@ documented in `ipfs config profile --help`.

270

271 Use this profile with caution.

272

273 +- `legacy-cid-v0`

274 +

275 + Makes UnixFS import (`ipfs add`) produce legacy CIDv0 with no raw leaves, sha2-256 and 256 KiB chunks.

276 +

277 + > [!WARNING]

278 + > This profile is provided for legacy users and should not be used for new projects.

279 +

280 +- `test-cid-v1`

281 +

282 + Makes UnixFS import (`ipfs add`) produce modern CIDv1 with raw leaves, sha2-256 and 1 MiB chunks.

283 +

284 + > [!NOTE]

285 + > This profile will become the new implicit default, provided for testing purposes.

286 + > Follow [kubo#4143](https://github.com/ipfs/kubo/issues/4143) for more details.

287 +

288 ## Types

289

290 This document refers to the standard JSON types (e.g., `null`, `string`,

@@ -2377,3 +2397,41 @@ Note: this does NOT work with Go's default DNS resolver. To make this a global s

2397 Default: Respect DNS Response TTL

2398

2399 Type: `optionalDuration`

2400 +

2401 +## `Import`

2402 +

2403 +Options to configure the default options used for ingesting data, in commands such as `ipfs add` or `ipfs block put`. All affected commands are detailed per option.

2404 +

2405 +Note that using flags will override the options defined here.

2406 +

2407 +### `Import.CidVersion`

2408 +

2409 +The default CID version. Commands affected: `ipfs add`.

2410 +

2411 +Default: `0`

2412 +

2413 +Type: `optionalInteger`

2414 +

2415 +### `Import.UnixFSRawLeaves`

2416 +

2417 +The default UnixFS raw leaves option. Commands affected: `ipfs add`, `ipfs files write`.

2418 +

2419 +Default: `false` if `CidVersion=0`; `true` if `CidVersion=1`

2420 +

2421 +Type: `flag`

2422 +

2423 +### `Import.UnixFSChunker`

2424 +

2425 +The default UnixFS chunker. Commands affected: `ipfs add`.

2426 +

2427 +Default: `size-262144`

2428 +

2429 +Type: `optionalString`

2430 +

2431 +### `Import.HashFunction`

2432 +

2433 +The default hash function. Commands affected: `ipfs add`, `ipfs block put`, `ipfs dag put`.

2434 +

2435 +Default: `sha2-256`

2436 +

2437 +Type: `optionalString`

test/cli/add_test.go new

+118

@@ -0,0 +1,118 @@

1 +package cli

2 +

3 +import (

4 + "testing"

5 +

6 + "github.com/ipfs/kubo/config"

7 + "github.com/ipfs/kubo/test/cli/harness"

8 + "github.com/stretchr/testify/require"

9 +)

10 +

11 +func TestAdd(t *testing.T) {

12 + t.Parallel()

13 +

14 + var (

15 + shortString = "hello world"

16 + shortStringCidV0 = "Qmf412jQZiuVUtdgnB36FXFX7xg5V6KEbSJ4dpQuhkLyfD" // cidv0 - dag-pb - sha2-256

17 + shortStringCidV1 = "bafkreifzjut3te2nhyekklss27nh3k72ysco7y32koao5eei66wof36n5e" // cidv1 - raw - sha2-256

18 + shortStringCidV1NoRawLeaves = "bafybeihykld7uyxzogax6vgyvag42y7464eywpf55gxi5qpoisibh3c5wa" // cidv1 - dag-pb - sha2-256

19 + shortStringCidV1Sha512 = "bafkrgqbqt3gerhas23vuzrapkdeqf4vu2dwxp3srdj6hvg6nhsug2tgyn6mj3u23yx7utftq3i2ckw2fwdh5qmhid5qf3t35yvkc5e5ottlw6"

20 + )

21 +

22 + t.Run("produced cid version: implicit default (CIDv0)", func(t *testing.T) {

23 + t.Parallel()

24 + node := harness.NewT(t).NewNode().Init().StartDaemon()

25 + defer node.StopDaemon()

26 +

27 + cidStr := node.IPFSAddStr(shortString)

28 + require.Equal(t, shortStringCidV0, cidStr)

29 + })

30 +

31 + t.Run("produced cid version: follows user-set configuration Import.CidVersion=0", func(t *testing.T) {

32 + t.Parallel()

33 + node := harness.NewT(t).NewNode().Init()

34 + node.UpdateConfig(func(cfg *config.Config) {

35 + cfg.Import.CidVersion = *config.NewOptionalInteger(0)

36 + })

37 + node.StartDaemon()

38 + defer node.StopDaemon()

39 +

40 + cidStr := node.IPFSAddStr(shortString)

41 + require.Equal(t, shortStringCidV0, cidStr)

42 + })

43 +

44 + t.Run("produced cid multihash: follows user-set configuration in Import.HashFunction", func(t *testing.T) {

45 + t.Parallel()

46 + node := harness.NewT(t).NewNode().Init()

47 + node.UpdateConfig(func(cfg *config.Config) {

48 + cfg.Import.HashFunction = *config.NewOptionalString("sha2-512")

49 + })

50 + node.StartDaemon()

51 + defer node.StopDaemon()

52 +

53 + cidStr := node.IPFSAddStr(shortString)

54 + require.Equal(t, shortStringCidV1Sha512, cidStr)

55 + })

56 +

57 + t.Run("produced cid version: follows user-set configuration Import.CidVersion=1", func(t *testing.T) {

58 + t.Parallel()

59 + node := harness.NewT(t).NewNode().Init()

60 + node.UpdateConfig(func(cfg *config.Config) {

61 + cfg.Import.CidVersion = *config.NewOptionalInteger(1)

62 + })

63 + node.StartDaemon()

64 + defer node.StopDaemon()

65 +

66 + cidStr := node.IPFSAddStr(shortString)

67 + require.Equal(t, shortStringCidV1, cidStr)

68 + })

69 +

70 + t.Run("produced cid version: command flag overrides configuration in Import.CidVersion", func(t *testing.T) {

71 + t.Parallel()

72 + node := harness.NewT(t).NewNode().Init()

73 + node.UpdateConfig(func(cfg *config.Config) {

74 + cfg.Import.CidVersion = *config.NewOptionalInteger(1)

75 + })

76 + node.StartDaemon()

77 + defer node.StopDaemon()

78 +

79 + cidStr := node.IPFSAddStr(shortString, "--cid-version", "0")

80 + require.Equal(t, shortStringCidV0, cidStr)

81 + })

82 +

83 + t.Run("produced unixfs raw leaves: follows user-set configuration Import.UnixFSRawLeaves", func(t *testing.T) {

84 + t.Parallel()

85 + node := harness.NewT(t).NewNode().Init()

86 + node.UpdateConfig(func(cfg *config.Config) {

87 + // CIDv1 defaults to raw-leaves=true

88 + cfg.Import.CidVersion = *config.NewOptionalInteger(1)

89 + // disable manually

90 + cfg.Import.UnixFSRawLeaves = config.False

91 + })

92 + node.StartDaemon()

93 + defer node.StopDaemon()

94 +

95 + cidStr := node.IPFSAddStr(shortString)

96 + require.Equal(t, shortStringCidV1NoRawLeaves, cidStr)

97 + })

98 +

99 + t.Run("ipfs init --profile=legacy-cid-v0 sets config that produces legacy CIDv0", func(t *testing.T) {

100 + t.Parallel()

101 + node := harness.NewT(t).NewNode().Init("--profile=legacy-cid-v0")

102 + node.StartDaemon()

103 + defer node.StopDaemon()

104 +

105 + cidStr := node.IPFSAddStr(shortString)

106 + require.Equal(t, shortStringCidV0, cidStr)

107 + })

108 +

109 + t.Run("ipfs init --profile=test-cid-v1 produces modern CIDv1", func(t *testing.T) {

110 + t.Parallel()

111 + node := harness.NewT(t).NewNode().Init("--profile=test-cid-v1")

112 + node.StartDaemon()

113 + defer node.StopDaemon()

114 +

115 + cidStr := node.IPFSAddStr(shortString)

116 + require.Equal(t, shortStringCidV1, cidStr)

117 + })

118 +}