@cryptotaxi247 / kubo / commits / 52b00624c

feat: pebbleds profile and plugin (#10530)

* include pebble as built-in plugin Pebble provides a high-performance alternative to leveldb as the datastore, and will serve as a replacement for badger1. There are a number of tuning parameters available for tuning pebble's performance to your specific needs. Default values are used for any that are not configured or are set to the parameter's zero-value. Requires https://github.com/ipfs/go-ds-pebble/pull/39 Closes #10347 * docs: remove mention of ipfs-ds-convert. Rationale: https://github.com/ipfs/ipfs-ds-convert/issues/50 * docs: pebbleds profile * test: meaningful t0025-datastores.sh * Update config/init.go * Update docs/config.md * Do not hard-code zero values into pebble config

Andrew Gillis committed Oct 3, 2024 at 13:58 UTC 52b00624cde178e04e406560af2252c85376fab6
16 files changed +476 -28
config/init.go
+11
@@ -138,6 +138,17 @@ func DefaultDatastoreConfig() Datastore {
138 }
139 }
140
141 +func pebbleSpec() map[string]interface{} {
142 + return map[string]interface{}{
143 + "type": "measure",
144 + "prefix": "pebble.datastore",
145 + "child": map[string]interface{}{
146 + "type": "pebbleds",
147 + "path": "pebbleds",
148 + },
149 + }
150 +}
151 +
152 func badgerSpec() map[string]interface{} {
153 return map[string]interface{}{
154 "type": "measure",
config/profile.go
+37 -2
@@ -135,7 +135,11 @@ You should use this datastore if:
135 * You want to minimize memory usage.
136 * You are ok with the default speed of data import, or prefer to use --nocopy.
137
138 -This profile may only be applied when first initializing the node.
138 +See configuration documentation at:
139 +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#flatfs
140 +
141 +NOTE: This profile may only be applied when first initializing node at IPFS_PATH
142 + via 'ipfs init --profile flatfs'
143 `,
144
145 InitOnly: true,
@@ -144,6 +148,32 @@ This profile may only be applied when first initializing the node.
148 return nil
149 },
150 },
151 + "pebbleds": {
152 + Description: `Configures the node to use the pebble high-performance datastore.
153 +
154 +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance
155 +and internal usage by CockroachDB.
156 +You should use this datastore if:
157 +
158 +- You need a datastore that is focused on performance.
159 +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
160 +- This datastore is good for multi-terabyte data sets.
161 +- May benefit from tuning depending on read/write patterns and throughput.
162 +- Performance is helped significantly by running on a system with plenty of memory.
163 +
164 +See configuration documentation at:
165 +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds
166 +
167 +NOTE: This profile may only be applied when first initializing node at IPFS_PATH
168 + via 'ipfs init --profile pebbleds'
169 +`,
170 +
171 + InitOnly: true,
172 + Transform: func(c *Config) error {
173 + c.Datastore.Spec = pebbleSpec()
174 + return nil
175 + },
176 + },
177 "badgerds": {
178 Description: `Configures the node to use the legacy badgerv1 datastore.
179
@@ -160,7 +190,12 @@ Other caveats:
190 * Good for medium-size datastores, but may run into performance issues
191 if your dataset is bigger than a terabyte.
192
163 -This profile may only be applied when first initializing the node.`,
193 +See configuration documentation at:
194 +https://github.com/ipfs/kubo/blob/master/docs/datastores.md#badgerds
195 +
196 +NOTE: This profile may only be applied when first initializing node at IPFS_PATH
197 + via 'ipfs init --profile badgerds'
198 +`,
199
200 InitOnly: true,
201 Transform: func(c *Config) error {
docs/changelogs/v0.31.md
+10
@@ -6,6 +6,7 @@
6
7 - [Overview](#overview)
8 - [🔦 Highlights](#-highlights)
9 + - [Experimental Pebble Datastore](#experimental-pebble-datastore)
10 - [New metrics](#new-metrics)
11 - [`lowpower` profile no longer breaks DHT announcements](#lowpower-profile-no-longer-breaks-dht-announcements)
12 - [📝 Changelog](#-changelog)
@@ -15,6 +16,15 @@
16
17 ### 🔦 Highlights
18
19 +#### Experimental Pebble Datastore
20 +
21 +[Pebble](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) provides a high-performance alternative to leveldb as the datastore, and provides a modern replacement for [legacy badgerv1](https://github.com/ipfs/kubo/blob/master/docs/config.md#badgerds-profile).
22 +
23 +A fresh Kubo node can be initialized with [`pebbleds` profile](https://github.com/ipfs/kubo/blob/master/docs/config.md#pebbleds-profile) via `ipfs init --profile pebbleds`.
24 +
25 +There are a number of parameters available for tuning pebble's performance to your specific needs. Default values are used for any parameters that are not configured or are set to their zero-value.
26 +For a description of the available tuning parameters, see [kubo/docs/datastores.md#pebbleds](https://github.com/ipfs/kubo/blob/master/docs/datastores.md#pebbleds).
27 +
28 #### New metrics
29
30 - Added 3 new go metrics: `go_gc_gogc_percent`, `go_gc_gomemlimit_bytes` and `go_sched_gomaxprocs_threads` as those are [recommended by the Go team](https://github.com/prometheus/client_golang/pull/1559)
docs/config.md
+34 -11
@@ -181,6 +181,7 @@ config file at runtime.
181 - [`local-discovery` profile](#local-discovery-profile)
182 - [`default-networking` profile](#default-networking-profile)
183 - [`flatfs` profile](#flatfs-profile)
184 + - [`pebbleds` profile](#pebbleds-profile)
185 - [`badgerds` profile](#badgerds-profile)
186 - [`lowpower` profile](#lowpower-profile)
187 - [`announce-off` profile](#announce-off-profile)
@@ -524,13 +525,8 @@ Spec defines the structure of the ipfs datastore. It is a composable structure,
525 where each datastore is represented by a json object. Datastores can wrap other
526 datastores to provide extra functionality (eg metrics, logging, or caching).
527
527 -This can be changed manually, however, if you make any changes that require a
528 -different on-disk structure, you will need to run the [ipfs-ds-convert
529 -tool](https://github.com/ipfs/ipfs-ds-convert) to migrate data into the new
530 -structures.
531 -
532 -For more information on possible values for this configuration option, see
533 -[docs/datastores.md](datastores.md)
528 +> [!NOTE]
529 +> For more information on possible values for this configuration option, see [`kubo/docs/datastores.md`](datastores.md)
530
531 Default:
532 ```
@@ -2403,9 +2399,9 @@ Inverse profile of the test profile.
2399
2400 ### `flatfs` profile
2401
2406 -Configures the node to use the flatfs datastore. Flatfs is the default datastore.
2402 +Configures the node to use the flatfs datastore.
2403 +Flatfs is the default, most battle-tested and reliable datastore.
2404
2408 -This is the most battle-tested and reliable datastore.
2405 You should use this datastore if:
2406
2407 - You need a very simple and very reliable datastore, and you trust your
@@ -2416,7 +2412,30 @@ You should use this datastore if:
2412 - You want to minimize memory usage.
2413 - You are ok with the default speed of data import, or prefer to use `--nocopy`.
2414
2419 -This profile may only be applied when first initializing the node.
2415 +> [!WARNING]
2416 +> This profile may only be applied when first initializing the node via `ipfs init --profile flatfs`
2417 +
2418 +> [!NOTE]
2419 +> See caveats and configuration options at [`datastores.md#flatfs`](datastores.md#flatfs)
2420 +
2421 +### `pebbleds` profile
2422 +
2423 +Configures the node to use the pebble high-performance datastore.
2424 +
2425 +Pebble is a LevelDB/RocksDB inspired key-value store focused on performance and internal usage by CockroachDB.
2426 +You should use this datastore if:
2427 +
2428 +- You need a datastore that is focused on performance.
2429 +- You need reliability by default, but may choose to disable WAL for maximum performance when reliability is not critical.
2430 +- This datastore is good for multi-terrabyte data sets.
2431 +- May benefit from tuning depending on read/write patterns and throughput.
2432 +- Performance is helped significantly by running on a system with plenty of memory.
2433 +
2434 +> [!WARNING]
2435 +> This profile may only be applied when first initializing the node via `ipfs init --profile pebbleds`
2436 +
2437 +> [!NOTE]
2438 +> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
2439
2440 ### `badgerds` profile
2441
@@ -2437,7 +2456,11 @@ Also, be aware that:
2456 - Good for medium-size datastores, but may run into performance issues if your dataset is bigger than a terabyte.
2457 - The current implementation is based on old badger 1.x which is no longer supported by the upstream team.
2458
2440 -This profile may only be applied when first initializing the node.
2459 +> [!WARNING]
2460 +> This profile may only be applied when first initializing the node via `ipfs init --profile badgerds`
2461 +
2462 +> [!NOTE]
2463 +> See other caveats and configuration options at [`datastores.md#pebbleds`](datastores.md#pebbleds)
2464
2465 ### `lowpower` profile
2466
docs/datastores.md
+36
@@ -3,6 +3,13 @@
3 This document describes the different possible values for the `Datastore.Spec`
4 field in the ipfs configuration file.
5
6 +- [flatfs](#flatfs)
7 +- [levelds](#levelds)
8 +- [pebbleds](#pebbleds)
9 +- [badgerds](#badgerds)
10 +- [mount](#mount)
11 +- [measure](#measure)
12 +
13 ## flatfs
14
15 Stores each key value pair as a file on the filesystem.
@@ -35,6 +42,35 @@ Uses a leveldb database to store key value pairs.
42 }
43 ```
44
45 +## pebbleds
46 +
47 +Uses [pebble](https://github.com/cockroachdb/pebble) as a key value store.
48 +
49 +```json
50 +{
51 + "type": "pebbleds",
52 + "path": "<location of pebble inside repo>",
53 +}
54 +```
55 +
56 +The following options are availble for tuning pebble.
57 +If they are not configured (or assigned their zero-valued), then default values are used.
58 +
59 +* `bytesPerSync`: int, Sync sstables periodically in order to smooth out writes to disk. (default: 512KB)
60 +* `bisableWAL`: true|false, Disable the write-ahead log (WAL) at expense of prohibiting crash recovery. (default: false)
61 +* `cacheSize`: Size of pebble's shared block cache. (default: 8MB)
62 +* `l0CompactionThreshold`: int, Count of L0 files necessary to trigger an L0 compaction.
63 +* `l0StopWritesThreshold`: int, Limit on L0 read-amplification, computed as the number of L0 sublevels.
64 +* `lBaseMaxBytes`: int, Maximum number of bytes for LBase. The base level is the level which L0 is compacted into.
65 +* `maxConcurrentCompactions`: int, Maximum number of concurrent compactions. (default: 1)
66 +* `memTableSize`: int, Size of a MemTable in steady state. The actual MemTable size starts at min(256KB, MemTableSize) and doubles for each subsequent MemTable up to MemTableSize (default: 4MB)
67 +* `memTableStopWritesThreshold`: int, Limit on the number of queued of MemTables. (default: 2)
68 +* `walBytesPerSync`: int: Sets the number of bytes to write to a WAL before calling Sync on it in the background. (default: 0, no background syncing)
69 +* `walMinSyncSeconds`: int: Sets the minimum duration between syncs of the WAL. (default: 0)
70 +
71 +> [!TIP]
72 +> Start using pebble with only default values and configure tuning items are needed for your needs. For a more complete description of these values, see: `https://pkg.go.dev/github.com/cockroachdb/pebble@vA.B.C#Options` (where `A.B.C` is pebble version from Kubo's `go.mod`).
73 +
74 ## badgerds
75
76 Uses [badger](https://github.com/dgraph-io/badger) as a key value store.
docs/examples/kubo-as-a-library/go.mod
+12
@@ -16,6 +16,7 @@ require (
16 require (
17 bazil.org/fuse v0.0.0-20200117225306-7b5117fecadc // indirect
18 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
19 + github.com/DataDog/zstd v1.4.5 // indirect
20 github.com/Jorropo/jsync v1.0.1 // indirect
21 github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
22 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
@@ -25,6 +26,12 @@ require (
26 github.com/cenkalti/backoff/v4 v4.3.0 // indirect
27 github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
28 github.com/cespare/xxhash/v2 v2.3.0 // indirect
29 + github.com/cockroachdb/errors v1.11.3 // indirect
30 + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
31 + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
32 + github.com/cockroachdb/pebble v1.1.2 // indirect
33 + github.com/cockroachdb/redact v1.1.5 // indirect
34 + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
35 github.com/containerd/cgroups v1.1.0 // indirect
36 github.com/coreos/go-systemd/v22 v22.5.0 // indirect
37 github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
@@ -43,6 +50,7 @@ require (
50 github.com/francoispqt/gojay v1.2.13 // indirect
51 github.com/fsnotify/fsnotify v1.7.0 // indirect
52 github.com/gabriel-vasile/mimetype v1.4.4 // indirect
53 + github.com/getsentry/sentry-go v0.27.0 // indirect
54 github.com/go-logr/logr v1.4.2 // indirect
55 github.com/go-logr/stdr v1.2.2 // indirect
56 github.com/go-task/slim-sprig/v3 v3.0.0 // indirect
@@ -74,6 +82,7 @@ require (
82 github.com/ipfs/go-ds-flatfs v0.5.1 // indirect
83 github.com/ipfs/go-ds-leveldb v0.5.0 // indirect
84 github.com/ipfs/go-ds-measure v0.2.0 // indirect
85 + github.com/ipfs/go-ds-pebble v0.4.0 // indirect
86 github.com/ipfs/go-fs-lock v0.0.7 // indirect
87 github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect
88 github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
@@ -103,6 +112,8 @@ require (
112 github.com/klauspost/compress v1.17.9 // indirect
113 github.com/klauspost/cpuid/v2 v2.2.8 // indirect
114 github.com/koron/go-ssdp v0.0.4 // indirect
115 + github.com/kr/pretty v0.3.1 // indirect
116 + github.com/kr/text v0.2.0 // indirect
117 github.com/libp2p/go-buffer-pool v0.1.0 // indirect
118 github.com/libp2p/go-cidranger v1.1.0 // indirect
119 github.com/libp2p/go-doh-resolver v0.4.0 // indirect
@@ -172,6 +183,7 @@ require (
183 github.com/quic-go/quic-go v0.45.2 // indirect
184 github.com/quic-go/webtransport-go v0.8.0 // indirect
185 github.com/raulk/go-watchdog v1.3.0 // indirect
186 + github.com/rogpeppe/go-internal v1.12.0 // indirect
187 github.com/samber/lo v1.46.0 // indirect
188 github.com/spaolacci/murmur3 v1.1.0 // indirect
189 github.com/stretchr/testify v1.9.0 // indirect
docs/examples/kubo-as-a-library/go.sum
+26
@@ -29,6 +29,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
29 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
30 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
31 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
32 +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
33 +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
34 github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
35 github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
36 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
@@ -74,6 +76,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
76 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
77 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
78 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
79 +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
80 +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
81 +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
82 +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
83 +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
84 +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
85 +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
86 +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
87 +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
88 +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
89 +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
90 +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
91 +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
92 +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
93 github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
94 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
95 github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
@@ -143,10 +159,14 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
159 github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
160 github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
161 github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
162 +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
163 +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
164 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
165 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
166 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
167 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
168 +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
169 +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
170 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
171 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
172 github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
@@ -301,6 +321,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
321 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
322 github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
323 github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
324 +github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ=
325 +github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM=
326 github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
327 github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
328 github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
@@ -585,6 +607,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
607 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
608 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
609 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
610 +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
611 +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
612 github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
613 github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
614 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
@@ -627,6 +651,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
651 github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
652 github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
653 github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
654 +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
655 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
656 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
657 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -658,6 +683,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
683 github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
684 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
685 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
686 +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
687 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
688 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
689 github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
go.mod
+13 -1
@@ -10,6 +10,7 @@ require (
10 github.com/cenkalti/backoff/v4 v4.3.0
11 github.com/ceramicnetwork/go-dag-jose v0.1.0
12 github.com/cheggaaa/pb v1.0.29
13 + github.com/cockroachdb/pebble v1.1.2
14 github.com/coreos/go-systemd/v22 v22.5.0
15 github.com/dustin/go-humanize v1.0.1
16 github.com/elgris/jsondiff v0.0.0-20160530203242-765b5c24c302
@@ -30,6 +31,7 @@ require (
31 github.com/ipfs/go-ds-flatfs v0.5.1
32 github.com/ipfs/go-ds-leveldb v0.5.0
33 github.com/ipfs/go-ds-measure v0.2.0
34 + github.com/ipfs/go-ds-pebble v0.4.0
35 github.com/ipfs/go-fs-lock v0.0.7
36 github.com/ipfs/go-ipfs-cmds v0.13.0
37 github.com/ipfs/go-ipld-cbor v0.2.0
@@ -95,12 +97,18 @@ require (
97
98 require (
99 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
100 + github.com/DataDog/zstd v1.4.5 // indirect
101 github.com/Jorropo/jsync v1.0.1 // indirect
102 github.com/alecthomas/units v0.0.0-20240626203959-61d1e3462e30 // indirect
103 github.com/alexbrainman/goissue34681 v0.0.0-20191006012335-3fc7a47baff5 // indirect
104 github.com/beorn7/perks v1.0.1 // indirect
105 github.com/cespare/xxhash v1.1.0 // indirect
106 github.com/cespare/xxhash/v2 v2.3.0 // indirect
107 + github.com/cockroachdb/errors v1.11.3 // indirect
108 + github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce // indirect
109 + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
110 + github.com/cockroachdb/redact v1.1.5 // indirect
111 + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
112 github.com/containerd/cgroups v1.1.0 // indirect
113 github.com/crackcomm/go-gitignore v0.0.0-20231225121904-e25f5bc08668 // indirect
114 github.com/cskr/pubsub v1.0.2 // indirect
@@ -115,6 +123,7 @@ require (
123 github.com/flynn/noise v1.1.0 // indirect
124 github.com/francoispqt/gojay v1.2.13 // indirect
125 github.com/gabriel-vasile/mimetype v1.4.4 // indirect
126 + github.com/getsentry/sentry-go v0.27.0 // indirect
127 github.com/go-kit/log v0.2.1 // indirect
128 github.com/go-logfmt/logfmt v0.5.1 // indirect
129 github.com/go-logr/logr v1.4.2 // indirect
@@ -151,6 +160,8 @@ require (
160 github.com/klauspost/compress v1.17.9 // indirect
161 github.com/klauspost/cpuid/v2 v2.2.8 // indirect
162 github.com/koron/go-ssdp v0.0.4 // indirect
163 + github.com/kr/pretty v0.3.1 // indirect
164 + github.com/kr/text v0.2.0 // indirect
165 github.com/libp2p/go-buffer-pool v0.1.0 // indirect
166 github.com/libp2p/go-cidranger v1.1.0 // indirect
167 github.com/libp2p/go-flow-metrics v0.1.0 // indirect
@@ -164,7 +175,7 @@ require (
175 github.com/libp2p/go-yamux/v4 v4.0.1 // indirect
176 github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
177 github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
167 - github.com/mattn/go-colorable v0.1.6 // indirect
178 + github.com/mattn/go-colorable v0.1.13 // indirect
179 github.com/mattn/go-isatty v0.0.20 // indirect
180 github.com/mattn/go-runewidth v0.0.4 // indirect
181 github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b // indirect
@@ -210,6 +221,7 @@ require (
221 github.com/quic-go/quic-go v0.45.2 // indirect
222 github.com/quic-go/webtransport-go v0.8.0 // indirect
223 github.com/raulk/go-watchdog v1.3.0 // indirect
224 + github.com/rogpeppe/go-internal v1.12.0 // indirect
225 github.com/rs/cors v1.10.1 // indirect
226 github.com/samber/lo v1.46.0 // indirect
227 github.com/spaolacci/murmur3 v1.1.0 // indirect
go.sum
+30 -3
@@ -47,6 +47,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIo
47 github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
48 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
49 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
50 +github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
51 +github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
52 github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU=
53 github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ=
54 github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
@@ -104,6 +106,20 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn
106 github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs=
107 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
108 github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
109 +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4=
110 +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU=
111 +github.com/cockroachdb/errors v1.11.3 h1:5bA+k2Y6r+oz/6Z/RFlNeVCesGARKuC6YymtcDrbC/I=
112 +github.com/cockroachdb/errors v1.11.3/go.mod h1:m4UIW4CDjx+R5cybPsNrRbreomiFqt8o1h1wUVazSd8=
113 +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce h1:giXvy4KSc/6g/esnpM7Geqxka4WSqI1SZc7sMJFd3y4=
114 +github.com/cockroachdb/fifo v0.0.0-20240606204812-0bbfbd93a7ce/go.mod h1:9/y3cnZ5GKakj/H4y9r9GTjCvAFta7KLgSHPJJYc52M=
115 +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE=
116 +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs=
117 +github.com/cockroachdb/pebble v1.1.2 h1:CUh2IPtR4swHlEj48Rhfzw6l/d0qA31fItcIszQVIsA=
118 +github.com/cockroachdb/pebble v1.1.2/go.mod h1:4exszw1r40423ZsmkG/09AFEG83I0uDgfujJdbL6kYU=
119 +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30=
120 +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg=
121 +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo=
122 +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ=
123 github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE=
124 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
125 github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw=
@@ -177,10 +193,14 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4
193 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw=
194 github.com/gabriel-vasile/mimetype v1.4.4 h1:QjV6pZ7/XZ7ryI2KuyeEDE8wnh7fHP9YnQy+R0LnH8I=
195 github.com/gabriel-vasile/mimetype v1.4.4/go.mod h1:JwLei5XPtWdGiMFB5Pjle1oEeoSeEuJfJE+TtfvdB/s=
196 +github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
197 +github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
198 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
199 github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0=
200 github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98=
201 github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
202 +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA=
203 +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og=
204 github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU=
205 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
206 github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
@@ -365,6 +385,8 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN
385 github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q=
386 github.com/ipfs/go-ds-measure v0.2.0 h1:sG4goQe0KDTccHMyT45CY1XyUbxe5VwTKpg2LjApYyQ=
387 github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9UvVh8V3JxE=
388 +github.com/ipfs/go-ds-pebble v0.4.0 h1:88lgFAs2ck8jCQ8lMYRBtksEg18r9BlvTxIMnNJkZaQ=
389 +github.com/ipfs/go-ds-pebble v0.4.0/go.mod h1:ZyYU+weIni+4NG/Yjva+cPkU3ghlsU1HA2R/VLHJ9sM=
390 github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U=
391 github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc=
392 github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ=
@@ -560,15 +582,15 @@ github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs
582 github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
583 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
584 github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
563 -github.com/mattn/go-colorable v0.1.6 h1:6Su7aK7lXmJ/U79bYtBjLNaha4Fs1Rg9plHpcH+vvnE=
564 -github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
585 +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
586 +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
587 github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
588 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
589 github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
590 github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
569 -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
591 github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
592 github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
593 +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
594 github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
595 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
596 github.com/mattn/go-runewidth v0.0.4 h1:2BvfKmzob6Bmd4YsL0zygOqfdFnK7GR4QL06Do4/p7Y=
@@ -685,6 +707,8 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhM
707 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
708 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk=
709 github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw=
710 +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4=
711 +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8=
712 github.com/pion/datachannel v1.5.8 h1:ph1P1NsGkazkjrvyMfhRBUAWMxugJjq2HfQifaOoSNo=
713 github.com/pion/datachannel v1.5.8/go.mod h1:PgmdpoaNBLX9HNzNClmdki4DYW5JtI7Yibu8QzbL3tI=
714 github.com/pion/dtls/v2 v2.2.7/go.mod h1:8WiMkebSHFD0T+dIU+UeBaoV7kDhOW5oDCzZ7WZ/F9s=
@@ -727,6 +751,7 @@ github.com/pion/turn/v2 v2.1.6 h1:Xr2niVsiPTB0FPtt+yAWKFUkU1eotQbGgpTIld4x1Gc=
751 github.com/pion/turn/v2 v2.1.6/go.mod h1:huEpByKKHix2/b9kmTAM3YoX6MKP+/D//0ClgUYR2fY=
752 github.com/pion/webrtc/v3 v3.3.0 h1:Rf4u6n6U5t5sUxhYPQk/samzU/oDv7jk6BA5hyO2F9I=
753 github.com/pion/webrtc/v3 v3.3.0/go.mod h1:hVmrDJvwhEertRWObeb1xzulzHGeVUoPlWvxdGzcfU0=
754 +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=
755 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
756 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
757 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
@@ -785,6 +810,7 @@ github.com/raulk/go-watchdog v1.3.0 h1:oUmdlHxdkXRJlwfG0O9omj8ukerm8MEQavSiDTEtB
810 github.com/raulk/go-watchdog v1.3.0/go.mod h1:fIvOnLbF0b0ZwkB9YU4mOW9Did//4vPZtDqv66NfsMU=
811 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
812 github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
813 +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs=
814 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
815 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
816 github.com/rs/cors v1.10.1 h1:L0uuZVXIKlI1SShY2nhFfo44TYvDPQ1w4oFkUJNfhyo=
@@ -1200,6 +1226,7 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc
1226 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1227 golang.org/x/sys v0.0.0-20220708085239-5a0f0661e09d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1228 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1229 +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1230 golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1231 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
1232 golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
plugin/loader/preload.go
+2
@@ -8,6 +8,7 @@ import (
8 pluginipldgit "github.com/ipfs/kubo/plugin/plugins/git"
9 pluginlevelds "github.com/ipfs/kubo/plugin/plugins/levelds"
10 pluginnopfs "github.com/ipfs/kubo/plugin/plugins/nopfs"
11 + pluginpebbleds "github.com/ipfs/kubo/plugin/plugins/pebbleds"
12 pluginpeerlog "github.com/ipfs/kubo/plugin/plugins/peerlog"
13 )
14
@@ -21,6 +22,7 @@ func init() {
22 Preload(pluginbadgerds.Plugins...)
23 Preload(pluginflatfs.Plugins...)
24 Preload(pluginlevelds.Plugins...)
25 + Preload(pluginpebbleds.Plugins...)
26 Preload(pluginpeerlog.Plugins...)
27 Preload(pluginfxtest.Plugins...)
28 Preload(pluginnopfs.Plugins...)
plugin/loader/preload_list
+2 -1
@@ -9,6 +9,7 @@ iplddagjose github.com/ipfs/kubo/plugin/plugins/dagjose *
9 badgerds github.com/ipfs/kubo/plugin/plugins/badgerds *
10 flatfs github.com/ipfs/kubo/plugin/plugins/flatfs *
11 levelds github.com/ipfs/kubo/plugin/plugins/levelds *
12 +pebbleds github.com/ipfs/kubo/plugin/plugins/pebbleds *
13 peerlog github.com/ipfs/kubo/plugin/plugins/peerlog *
14 fxtest github.com/ipfs/kubo/plugin/plugins/fxtest *
14 -nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
\ No newline at end of file
15 +nopfs github.com/ipfs/kubo/plugin/plugins/nopfs *
plugin/plugins/flatfs/flatfs.go
+1 -1
@@ -42,7 +42,7 @@ type datastoreConfig struct {
42 syncField bool
43 }
44
45 -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
45 +// BadgerdsDatastoreConfig returns a configuration stub for a flatfs datastore
46 // from the given parameters.
47 func (*flatfsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48 return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
plugin/plugins/levelds/levelds.go
+1 -1
@@ -42,7 +42,7 @@ type datastoreConfig struct {
42 compression ldbopts.Compression
43 }
44
45 -// BadgerdsDatastoreConfig returns a configuration stub for a badger datastore
45 +// DatastoreConfigParser returns a configuration stub for a badger datastore
46 // from the given parameters.
47 func (*leveldsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
48 return func(params map[string]interface{}) (fsrepo.DatastoreConfig, error) {
plugin/plugins/pebbleds/pebbleds.go new
+246
@@ -0,0 +1,246 @@
1 +package pebbleds
2 +
3 +import (
4 + "errors"
5 + "fmt"
6 + "io/fs"
7 + "os"
8 + "path/filepath"
9 + "time"
10 +
11 + "github.com/cockroachdb/pebble"
12 + pebbleds "github.com/ipfs/go-ds-pebble"
13 + "github.com/ipfs/kubo/plugin"
14 + "github.com/ipfs/kubo/repo"
15 + "github.com/ipfs/kubo/repo/fsrepo"
16 +)
17 +
18 +// Plugins is exported list of plugins that will be loaded.
19 +var Plugins = []plugin.Plugin{
20 + &pebbledsPlugin{},
21 +}
22 +
23 +type pebbledsPlugin struct{}
24 +
25 +var _ plugin.PluginDatastore = (*pebbledsPlugin)(nil)
26 +
27 +func (*pebbledsPlugin) Name() string {
28 + return "ds-pebble"
29 +}
30 +
31 +func (*pebbledsPlugin) Version() string {
32 + return "0.1.0"
33 +}
34 +
35 +func (*pebbledsPlugin) Init(_ *plugin.Environment) error {
36 + return nil
37 +}
38 +
39 +func (*pebbledsPlugin) DatastoreTypeName() string {
40 + return "pebbleds"
41 +}
42 +
43 +type datastoreConfig struct {
44 + path string
45 + cacheSize int64
46 +
47 + // Documentation of these values: https://pkg.go.dev/github.com/cockroachdb/pebble@v1.1.2#Options
48 + pebbleOpts *pebble.Options
49 +}
50 +
51 +// PebbleDatastoreConfig returns a configuration stub for a pebble datastore
52 +// from the given parameters.
53 +func (*pebbledsPlugin) DatastoreConfigParser() fsrepo.ConfigFromMap {
54 + return func(params map[string]any) (fsrepo.DatastoreConfig, error) {
55 + var c datastoreConfig
56 + var ok bool
57 +
58 + c.path, ok = params["path"].(string)
59 + if !ok {
60 + return nil, fmt.Errorf("'path' field is missing or not string")
61 + }
62 +
63 + cacheSize, err := getConfigInt("cacheSize", params)
64 + if err != nil {
65 + return nil, err
66 + }
67 + c.cacheSize = int64(cacheSize)
68 +
69 + bytesPerSync, err := getConfigInt("bytesPerSync", params)
70 + if err != nil {
71 + return nil, err
72 + }
73 + disableWAL, err := getConfigBool("disableWAL", params)
74 + if err != nil {
75 + return nil, err
76 + }
77 + l0CompactionThreshold, err := getConfigInt("l0CompactionThreshold", params)
78 + if err != nil {
79 + return nil, err
80 + }
81 + l0StopWritesThreshold, err := getConfigInt("l0StopWritesThreshold", params)
82 + if err != nil {
83 + return nil, err
84 + }
85 + lBaseMaxBytes, err := getConfigInt("lBaseMaxBytes", params)
86 + if err != nil {
87 + return nil, err
88 + }
89 + maxConcurrentCompactions, err := getConfigInt("maxConcurrentCompactions", params)
90 + if err != nil {
91 + return nil, err
92 + }
93 + memTableSize, err := getConfigInt("memTableSize", params)
94 + if err != nil {
95 + return nil, err
96 + }
97 + memTableStopWritesThreshold, err := getConfigInt("memTableStopWritesThreshold", params)
98 + if err != nil {
99 + return nil, err
100 + }
101 + walBytesPerSync, err := getConfigInt("walBytesPerSync", params)
102 + if err != nil {
103 + return nil, err
104 + }
105 + walMinSyncSec, err := getConfigInt("walMinSyncIntervalSeconds", params)
106 + if err != nil {
107 + return nil, err
108 + }
109 +
110 + if bytesPerSync != 0 || disableWAL || l0CompactionThreshold != 0 || l0StopWritesThreshold != 0 || lBaseMaxBytes != 0 || maxConcurrentCompactions != 0 || memTableSize != 0 || memTableStopWritesThreshold != 0 || walBytesPerSync != 0 || walMinSyncSec != 0 {
111 + c.pebbleOpts = &pebble.Options{
112 + BytesPerSync: bytesPerSync,
113 + DisableWAL: disableWAL,
114 + L0CompactionThreshold: l0CompactionThreshold,
115 + L0StopWritesThreshold: l0StopWritesThreshold,
116 + LBaseMaxBytes: int64(lBaseMaxBytes),
117 + MemTableSize: uint64(memTableSize),
118 + MemTableStopWritesThreshold: memTableStopWritesThreshold,
119 + WALBytesPerSync: walBytesPerSync,
120 + }
121 + if maxConcurrentCompactions != 0 {
122 + c.pebbleOpts.MaxConcurrentCompactions = func() int { return maxConcurrentCompactions }
123 + }
124 + if walMinSyncSec != 0 {
125 + c.pebbleOpts.WALMinSyncInterval = func() time.Duration { return time.Duration(walMinSyncSec) * time.Second }
126 + }
127 + }
128 +
129 + return &c, nil
130 + }
131 +}
132 +
133 +func getConfigBool(name string, params map[string]any) (bool, error) {
134 + val, ok := params[name]
135 + if ok {
136 + bval, ok := val.(bool)
137 + if !ok {
138 + return false, fmt.Errorf("%q field was not a bool", name)
139 + }
140 + return bval, nil
141 + }
142 + return false, nil
143 +}
144 +
145 +func getConfigInt(name string, params map[string]any) (int, error) {
146 + val, ok := params[name]
147 + if ok {
148 + // TODO: see why val may be an int or a float64.
149 + ival, ok := val.(int)
150 + if !ok {
151 + fval, ok := val.(float64)
152 + if !ok {
153 + return 0, fmt.Errorf("%q field was not an integer or a float64", name)
154 + }
155 + return int(fval), nil
156 + }
157 + return ival, nil
158 + }
159 + return 0, nil
160 +}
161 +
162 +func (c *datastoreConfig) DiskSpec() fsrepo.DiskSpec {
163 + return map[string]interface{}{
164 + "type": "pebbleds",
165 + "path": c.path,
166 + }
167 +}
168 +
169 +func (c *datastoreConfig) Create(path string) (repo.Datastore, error) {
170 + p := c.path
171 + if !filepath.IsAbs(p) {
172 + p = filepath.Join(path, p)
173 + }
174 +
175 + if err := dirWritable(p); err != nil {
176 + return nil, err
177 + }
178 +
179 + return pebbleds.NewDatastore(p, pebbleds.WithCacheSize(c.cacheSize), pebbleds.WithPebbleOpts(c.pebbleOpts))
180 +}
181 +
182 +// dirWritable checks if a directory is writable. If the directory does
183 +// not exist it is created with writable permission.
184 +func dirWritable(dir string) error {
185 + if dir == "" {
186 + return errors.New("directory not specified")
187 + }
188 + var err error
189 + dir, err = expandHome(dir)
190 + if err != nil {
191 + return err
192 + }
193 +
194 + fi, err := os.Stat(dir)
195 + if err != nil {
196 + if errors.Is(err, fs.ErrNotExist) {
197 + // Directory does not exist, so create it.
198 + err = os.Mkdir(dir, 0775)
199 + if err == nil {
200 + return nil
201 + }
202 + }
203 + if errors.Is(err, fs.ErrPermission) {
204 + err = fs.ErrPermission
205 + }
206 + return fmt.Errorf("directory not writable: %s: %w", dir, err)
207 + }
208 + if !fi.IsDir() {
209 + return fmt.Errorf("not a directory: %s", dir)
210 + }
211 +
212 + // Directory exists, check that a file can be written.
213 + file, err := os.CreateTemp(dir, "writetest")
214 + if err != nil {
215 + if errors.Is(err, fs.ErrPermission) {
216 + err = fs.ErrPermission
217 + }
218 + return fmt.Errorf("directory not writable: %s: %w", dir, err)
219 + }
220 + file.Close()
221 + return os.Remove(file.Name())
222 +}
223 +
224 +// expandHome expands the path to include the home directory if the path is
225 +// prefixed with `~`. If it isn't prefixed with `~`, the path is returned
226 +// as-is.
227 +func expandHome(path string) (string, error) {
228 + if path == "" {
229 + return path, nil
230 + }
231 +
232 + if path[0] != '~' {
233 + return path, nil
234 + }
235 +
236 + if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
237 + return "", errors.New("cannot expand user-specific home dir")
238 + }
239 +
240 + dir, err := os.UserHomeDir()
241 + if err != nil {
242 + return "", err
243 + }
244 +
245 + return filepath.Join(dir, path[1:]), nil
246 +}
test/sharness/t0021-config.sh
+1 -1
@@ -281,7 +281,7 @@ test_config_cmd() {
281
282 # won't work as it changes datastore definition, which makes ipfs not launch
283 # without converting first
284 - # test_profile_apply_revert badgerds
284 + # test_profile_apply_revert pebbleds
285
286 test_expect_success "cleanup config backups" '
287 find "$IPFS_PATH" -name "config-*" -exec rm {} \;
test/sharness/t0025-datastores.sh
+14 -7
@@ -4,13 +4,20 @@ test_description="Test non-standard datastores"
4
5 . lib/test-lib.sh
6
7 -test_expect_success "'ipfs init --empty-repo=false --profile=badgerds' succeeds" '
8 - BITS="2048" &&
9 - ipfs init --empty-repo=false --profile=badgerds
10 -'
7 +profiles=("flatfs" "pebbleds" "badgerds")
8 +proot="$(mktemp -d "${TMPDIR:-/tmp}/t0025.XXXXXX")"
9
12 -test_expect_success "'ipfs pin ls' works" '
13 - ipfs pin ls | wc -l | grep 9
14 -'
10 +for profile in "${profiles[@]}"; do
11 + test_expect_success "'ipfs init --empty-repo=false --profile=$profile' succeeds" '
12 + BITS="2048" &&
13 + IPFS_PATH="$proot/$profile" &&
14 + ipfs init --empty-repo=false --profile=$profile
15 + '
16 + test_expect_success "'ipfs pin add' and 'pin ls' works with $profile" '
17 + export IPFS_PATH="$proot/$profile" &&
18 + echo -n "hello_$profile" | ipfs block put --pin=true > hello_cid &&
19 + ipfs pin ls -t recursive "$(cat hello_cid)"
20 + '
21 +done
22
23 test_done