bitswap/client: configurable broadcast reduction (#10825)
* Configure bitswap braodcast reduction Add new config items to `Internal.Bitswap` to allow configuration of bitswap broadcast reduction behavior. Broadcast reduction behavior is enabled by default, and uses settings that should be suitable for most installations of kubo. * update sharness metrics test * Explicit defaults for broadcast reduction configuration * Update docs/config.md
Andrew Gillis committed
Jun 17, 2025 at 04:35 UTC
0cf1b22536d3ae103120285a0ab5e0a36055aeb9
5 files changed
+188
-2
config/internal.go
+47
@@ -16,4 +16,51 @@ type InternalBitswap struct {
16
ProviderSearchDelay OptionalDuration
17
ProviderSearchMaxResults OptionalInteger
18
WantHaveReplaceSize OptionalInteger
19
+ BroadcastControl *BitswapBroadcastControl
20
}
21
+
22
+type BitswapBroadcastControl struct {
23
+ // EnableEnables or disables broadcast control functionality. Setting this
24
+ // to false disables broadcast control functionality and restores the
25
+ // previous broadcast behavior of sending broadcasts to all peers. When
26
+ // disabled, all other BroadcastControl configuration items are ignored.
27
+ // Default is [DefaultBroadcastControlEnable].
28
+ Enable Flag `json:",omitempty"`
29
+ // MaxPeers sets a hard limit on the number of peers to send broadcasts to.
30
+ // A value of 0 means no broadcasts are sent. A value of -1 means there is
31
+ // no limit. Default is [DefaultBroadcastControlMaxPeers].
32
+ MaxPeers OptionalInteger `json:",omitempty"`
33
+ // LocalPeers enables or disables broadcast control for peers on the local
34
+ // network. If false, than always broadcast to peers on the local network.
35
+ // If true, apply broadcast control to local peers. Default is
36
+ // [DefaultBroadcastControlLocalPeers].
37
+ LocalPeers Flag `json:",omitempty"`
38
+ // PeeredPeers enables or disables broadcast reduction for peers configured
39
+ // for peering. If false, than always broadcast to peers configured for
40
+ // peering. If true, apply broadcast reduction to peered peers. Default is
41
+ // [DefaultBroadcastControlPeeredPeers].
42
+ PeeredPeers Flag `json:",omitempty"`
43
+ // MaxRandomPeers is the number of peers to broadcast to anyway, even
44
+ // though broadcast reduction logic has determined that they are not
45
+ // broadcast targets. Setting this to a non-zero value ensures at least
46
+ // this number of random peers receives a broadcast. This may be helpful in
47
+ // cases where peers that are not receiving broadcasts my have wanted
48
+ // blocks. Default is [DefaultBroadcastControlMaxRandomPeers].
49
+ MaxRandomPeers OptionalInteger `json:",omitempty"`
50
+ // SendToPendingPeers enables or disables sending broadcasts to any peers
51
+ // to which there is a pending message to send. When enabled, this sends
52
+ // broadcasts to many more peers, but does so in a way that does not
53
+ // increase the number of separate broadcast messages. There is still the
54
+ // increased cost of the recipients having to process and respond to the
55
+ // broadcasts. Default is [DefaultBroadcastControlSendToPendingPeers].
56
+ SendToPendingPeers Flag `json:",omitempty"`
57
+}
58
+
59
+const (
60
+ DefaultBroadcastControlEnable = true // Enabled
61
+ DefaultBroadcastControlMaxPeers = -1 // Unlimited
62
+ DefaultBroadcastControlLocalPeers = false // No control of local
63
+ DefaultBroadcastControlPeeredPeers = false // No control of peered
64
+ DefaultBroadcastControlMaxRandomPeers = 0 // No randoms
65
+ DefaultBroadcastControlSendToPendingPeers = false // Disabled
66
+)
core/node/bitswap.go
+43
@@ -117,9 +117,52 @@ func Bitswap(serverEnabled, libp2pEnabled, httpEnabled bool) interface{} {
117
// Kubo uses own, customized ProviderQueryManager
118
in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
119
var maxProviders int = DefaultMaxProviders
120
+
121
+ var bcDisposition string
122
if in.Cfg.Internal.Bitswap != nil {
123
maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
124
+ if in.Cfg.Internal.Bitswap.BroadcastControl != nil {
125
+ bcCfg := in.Cfg.Internal.Bitswap.BroadcastControl
126
+ bcEnable := bcCfg.Enable.WithDefault(config.DefaultBroadcastControlEnable)
127
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlEnable(bcEnable)))
128
+ if bcEnable {
129
+ bcDisposition = "enabled"
130
+ bcMaxPeers := int(bcCfg.MaxPeers.WithDefault(config.DefaultBroadcastControlMaxPeers))
131
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxPeers(bcMaxPeers)))
132
+
133
+ bcLocalPeers := bcCfg.LocalPeers.WithDefault(config.DefaultBroadcastControlLocalPeers)
134
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlLocalPeers(bcLocalPeers)))
135
+
136
+ bcPeeredPeers := bcCfg.PeeredPeers.WithDefault(config.DefaultBroadcastControlPeeredPeers)
137
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlPeeredPeers(bcPeeredPeers)))
138
+
139
+ bcMaxRandomPeers := int(bcCfg.MaxRandomPeers.WithDefault(config.DefaultBroadcastControlMaxRandomPeers))
140
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxRandomPeers(bcMaxRandomPeers)))
141
+
142
+ bcSendToPendingPeers := bcCfg.SendToPendingPeers.WithDefault(config.DefaultBroadcastControlSendToPendingPeers)
143
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlSendToPendingPeers(bcSendToPendingPeers)))
144
+ } else {
145
+ bcDisposition = "disabled"
146
+ }
147
+ }
148
}
149
+
150
+ // If broadcast control is not configured, then configure with defaults.
151
+ if bcDisposition == "" {
152
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlEnable(config.DefaultBroadcastControlEnable)))
153
+ if config.DefaultBroadcastControlEnable {
154
+ bcDisposition = "enabled"
155
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxPeers(config.DefaultBroadcastControlMaxPeers)))
156
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlLocalPeers(config.DefaultBroadcastControlLocalPeers)))
157
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlPeeredPeers(config.DefaultBroadcastControlPeeredPeers)))
158
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlMaxRandomPeers(config.DefaultBroadcastControlMaxRandomPeers)))
159
+ in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.BroadcastControlSendToPendingPeers(config.DefaultBroadcastControlSendToPendingPeers)))
160
+ } else {
161
+ bcDisposition = "enabled"
162
+ }
163
+ }
164
+ logger.Infof("bitswap client broadcast control %s", bcDisposition)
165
+
166
ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders))
167
for _, str := range in.Cfg.Routing.IgnoreProviders {
168
pid, err := peer.Decode(str)
docs/changelogs/v0.36.md
+34
-2
@@ -8,7 +8,6 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
8
9
## v0.36.0
10
11
-
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
- [HTTP Retrieval client enabled by default](#http-retrieval-client-enabled-by-default)
@@ -16,6 +15,11 @@ This release was brought to you by the [Shipyard](http://ipshipyard.com/) team.
15
- [AutoNATv2 Client](#autonatv2-client)
16
- [Smarter AutoTLS registration](#smarter-autotls-registration)
17
- [Overwrite option for files cp command](#overwrite-option-for-files-cp-command)
18
+ - [Update go-log to v2](#update-go-log-to-v2)
19
+ - [Bitswap Broadcast Reduction](#bitswap-broadcast-reduction)
20
+ - [Bitswap Broadcast Reduction](#bitswap-broadcast-reduction)
21
+ - [Update go-log to v2](#update-go-log-to-v2)
22
+ - [Overwrite option for files cp command](#overwrite-option-for-files-cp-command)
23
- [Option for filestore command to remove bad blocks](#option-for-filestore-command-to-remove-bad-blocks)
24
- [📦️ Important dependency updates](#-important-dependency-updates)
25
- [📝 Changelog](#-changelog)
@@ -31,9 +35,27 @@ This release promotes the HTTP Retrieval client from an experimental feature to
35
36
See [`HTTPRetrieval`](https://github.com/ipfs/kubo/blob/master/docs/config.md#httpretrieval) for more details.
37
38
+### Bitswap Broadcast Reduction
39
+
40
+The Bitswap client now supports broadcast reduction logic, which is enabled by default. This feature significantly reduces the number of broadcast messages sent to peers, resulting in lower bandwidth usage during load spikes.
41
+
42
+The overall logic works by sending to non-local peers only if those peers have previously replied that they have wanted data blocks. To minimize impact on existing workloads, by default, broadcasts are still always sent to peers on the local network, or the ones defined in `Peering.Peers`.
43
+
44
+We've performed A/B testing on our internal Kubo staging gateway with organic CID requests to `ipfs.io`. While this may not translate 1:1 to your workload, the benefits were significant enough to enable this feature by default. Here are the key findings:
45
+
46
+- **Dramatic Resource Usage Reduction:** Internal testing demonstrated reduction in Bitswap broadcast messages by 80-98% and network bandwidth savings of 50-95%, with the greatest improvements occurring during high traffic and peer spikes. These efficiency gains lower operational costs of running Kubo under high load and improve the IPFS Mainnet (which is >80% Kubo-based) by reducing ambient traffic for all connected peers.
47
+- **Improved Memory Stability:** Memory stays stable even during major CID request spikes that increase peer count, preventing the out-of-memory (OOM) issues found in earlier Kubo versions.
48
+- **Data Retrieval Performance Remains Strong:** Our tests suggest that Kubo gateway hosts with broadcast reduction enabled achieve similar or better HTTP 200 success rates compared to version 0.35, while maintaining equivalent or higher want-have responses and unique blocks received.
49
+
50
+For more information about our A/B tests, see [kubo#10825](https://github.com/ipfs/kubo/pull/10825).
51
+
52
+To revert to the previous behavior for your own A/B testing, set `Internal.Bitswap.BroadcastControl.Enable` to `false` and monitor relevant metrics (`ipfs_bitswap_bcast_skips_total`, `ipfs_bitswap_haves_received`, `ipfs_bitswap_unique_blocks_received`, `ipfs_bitswap_wanthaves_broadcast`, HTTP 200 success rate).
53
+
54
+For a description of the configuration items, see the documentation of [`Internal.Bitswap.BroadcastControl`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapbroadcastcontrol).
55
+
56
#### Update go-log to v2
57
36
-go-log v2 has been out for quite a while now and it is time to deprecate v1.
58
+go-log v2 has been out for quite a while now and it is time to deprecate v1.
59
60
- Replace all use of `go-log` with `go-log/v2`
61
- Makes `/api/v0/log/tail` useful over HTTP
@@ -52,6 +74,16 @@ This update to libp2p and [AutoTLS](https://github.com/ipfs/kubo/blob/master/doc
74
75
The `ipfs files cp` command has a `--force` option to allow it to overwrite existing files. Attempting to overwrite an existing directory results in an error.
76
77
+#### Bitswap Broadcast Reduction
78
+
79
+The bitswap client now supports logic to reduce the amount of bitswap broadcast messages that are send to peers. This logic is enabled by default.
80
+
81
+The bitswap broadcast control logic can be enabled and disabled and configured in the `Internal.Bitswap.BroadcastControl` section of the ipfs config file. For a description of the configuration items, see the documentation of [`Internal.Bitswap.BroadcastControl`](https://github.com/ipfs/kubo/blob/master/docs/config.md#internalbitswapbroadcastcontrol).
82
+
83
+The overall logic works by sending to non-local peers only if those peers have previously replied that they have wanted data blocks. By default, broadcasts are always sent to peers on the local network.
84
+
85
+To revert to the previous behavior for A/B testing, set `Internal.Bitswap.BroadcastReductionEnabled` to `false` and monitor relevant metrics (`ipfs_bitswap_bcast_skips_total`, `ipfs_bitswap_haves_received`, `ipfs_bitswap_unique_blocks_received`, `ipfs_bitswap_wanthaves_broadcast`). Depending on your workload, the number of broadcasts should decrease, while the block receipt success rate should remain acceptable.
86
+
87
#### Option for filestore command to remove bad blocks
88
89
The `filestore` command has a new option, `--remove-bad-blocks`, to verify objects in the filestore and remove those that fail verification.
docs/config.md
+63
@@ -84,6 +84,13 @@ config file at runtime.
84
- [`Internal.Bitswap.MaxOutstandingBytesPerPeer`](#internalbitswapmaxoutstandingbytesperpeer)
85
- [`Internal.Bitswap.ProviderSearchDelay`](#internalbitswapprovidersearchdelay)
86
- [`Internal.Bitswap.ProviderSearchMaxResults`](#internalbitswapprovidersearchmaxresults)
87
+ - [`Internal.Bitswap.BroadcastControl`](#internalbitswapbroadcastcontrol)
88
+ - [`Internal.Bitswap.BroadcastControl.Enable`](#internalbitswapbroadcastcontrolenable)
89
+ - [`Internal.Bitswap.BroadcastControl.MaxPeers`](#internalbitswapbroadcastcontrolmaxpeers)
90
+ - [`Internal.Bitswap.BroadcastControl.LocalPeers`](#internalbitswapbroadcastcontrollocalpeers)
91
+ - [`Internal.Bitswap.BroadcastControl.PeeredPeers`](#internalbitswapbroadcastcontrolpeeredpeers)
92
+ - [`Internal.Bitswap.BroadcastControl.MaxRandomPeers`](#internalbitswapbroadcastcontrolmaxrandompeers)
93
+ - [`Internal.Bitswap.BroadcastControl.SendToPendingPeers`](#internalbitswapbroadcastcontrolsendtopendingpeers)
94
- [`Internal.UnixFSShardingSizeThreshold`](#internalunixfsshardingsizethreshold)
95
- [`Ipns`](#ipns)
96
- [`Ipns.RepublishPeriod`](#ipnsrepublishperiod)
@@ -1282,6 +1289,62 @@ Setting to 0 means unlimited.
1289
1290
Type: `optionalInteger` (`null` means default which is 10)
1291
1292
+#### `Internal.Bitswap.BroadcastControl`
1293
+
1294
+`Internal.Bitswap.BroadcastControl` contains settings for the bitswap client's broadcast control functionality.
1295
+
1296
+Broadcast control tries to reduce the number of bitswap broadcast messages sent to peers by choosing a subset of of the peers to send to. Peers are chosen based on whether they have previously responded indicating they have wanted blocks, as well as other configurable criteria. The settings here change how peers are selected as broadcast targets. Broadcast control can also be completely disabled to return bitswap to its previous behavior before broadcast control was introduced.
1297
+
1298
+Enabling broadcast control should generally reduce the number of broadcasts significantly without significantly degrading the ability to discover which peers have wanted blocks. However, if block discovery on your network relies sufficiently on broadcasts to discover peers that have wanted blocks, then adjusting the broadcast control configuration or disabling it altogether, may be helpful.
1299
+
1300
+##### `Internal.Bitswap.BroadcastControl.Enable`
1301
+
1302
+Enables or disables broadcast control functionality. Setting this to `false` disables broadcast reduction logic and restores the previous (Kubo < 0.36) broadcast behavior of sending broadcasts to all peers. When disabled, all other `Bitswap.BroadcastControl` configuration items are ignored.
1303
+
1304
+Default: `true` (Enabled)
1305
+
1306
+Type: `flag`
1307
+
1308
+##### `Internal.Bitswap.BroadcastControl.MaxPeers`
1309
+
1310
+Sets a hard limit on the number of peers to send broadcasts to. A value of `0` means no broadcasts are sent. A value of `-1` means there is no limit.
1311
+
1312
+Default: `0` (no limit)
1313
+
1314
+Type: `optionalInteger` (non-negative, 0 means no limit)
1315
+
1316
+##### `Internal.Bitswap.BroadcastControl.LocalPeers`
1317
+
1318
+Enables or disables broadcast control for peers on the local network. Peers that have private or loopback addresses are considered to be on the local network. If this setting is `false`, than always broadcast to peers on the local network. If `true`, apply broadcast control to local peers.
1319
+
1320
+Default: `false` (Always broadcast to peers on local network)
1321
+
1322
+Type: `flag`
1323
+
1324
+##### `Internal.Bitswap.BroadcastControl.PeeredPeers`
1325
+
1326
+Enables or disables broadcast reduction for peers configured for peering. If `false`, than always broadcast to peers configured for peering. If `true`, apply broadcast reduction to peered peers.
1327
+
1328
+Default: `false` (Always broadcast to peers configured for peering)
1329
+
1330
+Type: `flag`
1331
+
1332
+##### `Internal.Bitswap.BroadcastControl.MaxRandomPeers`
1333
+
1334
+Sets the number of peers to broadcast to anyway, even though broadcast control logic has determined that they are not broadcast targets. Setting this to a non-zero value ensures at least this number of random peers receives a broadcast. This may be helpful in cases where peers that are not receiving broadcasts my have wanted blocks.
1335
+
1336
+Default: `0` (do not send broadcasts to peers not already targeted broadcast control)
1337
+
1338
+Type: `optionalInteger` (non-negative, 0 means do not broadcast to any random peers)
1339
+
1340
+##### `Internal.Bitswap.BroadcastControl.SendToPendingPeers`
1341
+
1342
+Enables or disables sending broadcasts to any peers to which there is a pending message to send. When enabled, this sends broadcasts to many more peers, but does so in a way that does not increase the number of separate broadcast messages. There is still the increased cost of the recipients having to process and respond to the broadcasts.
1343
+
1344
+Default: `false` (Do not send broadcasts to all peers for which there are pending messages)
1345
+
1346
+Type: `flag`
1347
+
1348
### `Internal.UnixFSShardingSizeThreshold`
1349
1350
**MOVED:** see [`Import.UnixFSHAMTDirectorySizeThreshold`](#importunixfshamtdirectorysizethreshold)
test/sharness/t0119-prometheus-data/prometheus_metrics
+1
@@ -56,6 +56,7 @@ go_sched_gomaxprocs_threads
56
go_threads
57
ipfs_bitswap_active_block_tasks
58
ipfs_bitswap_active_tasks
59
+ipfs_bitswap_bcast_skips_total
60
ipfs_bitswap_blocks_received
61
ipfs_bitswap_haves_received
62
ipfs_bitswap_pending_block_tasks