Adjust inbound connection limits depending on memory.
Antonio Navarro Perez committed
Jan 27, 2023 at 16:47 UTC
633c497f63e377602adebdaaaaea7dc0d6d0beef
4 files changed
+56
-57
core/node/libp2p/rcmgr_defaults.go
+30
-41
@@ -44,17 +44,18 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{
44
// This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled
45
46
// createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager.
47
-// The defaults follow the documentation in docs/config.md.
47
+// The defaults follow the documentation in docs/libp2p-resource-management.md.
48
// Any changes in the logic here should be reflected there.
49
func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) {
50
- maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 4)
50
+ maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 2)
51
maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString)
52
maxMemory, err := humanize.ParseBytes(maxMemoryString)
53
if err != nil {
54
return rcmgr.LimitConfig{}, err
55
}
56
57
- numFD := cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2)
57
+ maxMemoryMB := maxMemory / (1024 * 1024)
58
+ maxFD := int(cfg.ResourceMgr.MaxFileDescriptors.WithDefault(int64(fd.GetNumFDs()) / 2))
59
60
// We want to see this message on startup, that's why we are using fmt instead of log.
61
fmt.Printf(`
@@ -65,65 +66,53 @@ Computing default go-libp2p Resource Manager limits based on:
66
Applying any user-supplied overrides on top.
67
Run 'ipfs swarm limit all' to see the resulting limits.
68
68
-`, maxMemoryString, numFD)
69
+`, maxMemoryString, maxFD)
70
+
71
+ // At least as of 2023-01-25, it's possible to open a connection that
72
+ // doesn't ask for any memory usage with the libp2p Resource Manager/Accountant
73
+ // (see https://github.com/libp2p/go-libp2p/issues/2010#issuecomment-1404280736).
74
+ // As a result, we can't curretly rely on Memory limits to full protect us.
75
+ // Until https://github.com/libp2p/go-libp2p/issues/2010 is addressed,
76
+ // we take a proxy now of restricting to 1 inbound connection per MB.
77
+ // Note: this is more generous than go-libp2p's default autoscaled limits which do
78
+ // 64 connections per 1GB
79
+ // (see https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L357 ).
80
+ systemConnsInbound := int(1 * maxMemoryMB)
81
82
scalingLimitConfig := rcmgr.ScalingLimitConfig{
83
SystemBaseLimit: rcmgr.BaseLimit{
84
Memory: int64(maxMemory),
73
- FD: int(numFD),
85
+ FD: maxFD,
86
87
// By default, we just limit connections on the inbound side.
88
Conns: bigEnough,
77
- ConnsInbound: rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default
89
+ ConnsInbound: systemConnsInbound,
90
ConnsOutbound: bigEnough,
91
80
- // We limit streams since they not only take up memory and CPU.
81
- // The Memory limit protects us on the memory side,
82
- // but a StreamsInbound limit helps protect against unbound CPU consumption from stream processing.
92
Streams: bigEnough,
84
- StreamsInbound: rcmgr.DefaultLimits.SystemBaseLimit.StreamsInbound,
93
+ StreamsInbound: bigEnough,
94
StreamsOutbound: bigEnough,
95
},
87
- // Most limits don't see an increase because they're already infinite/bigEnough or at their max value.
88
- // The values that should scale based on the amount of memory allocated to libp2p need to increase accordingly.
89
- SystemLimitIncrease: rcmgr.BaseLimitIncrease{
90
- Memory: 0,
91
- FDFraction: 0,
92
-
93
- Conns: 0,
94
- ConnsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.ConnsInbound,
95
- ConnsOutbound: 0,
96
-
97
- Streams: 0,
98
- StreamsInbound: rcmgr.DefaultLimits.SystemLimitIncrease.StreamsInbound,
99
- StreamsOutbound: 0,
100
- },
96
+ SystemLimitIncrease: noLimitIncrease,
97
98
+ // Transient connections won't cause any memory to accounted for by the resource manager.
99
+ // Only established connections do.
100
+ // As a result, we can't rely on System.Memory to protect us from a bunch of transient connection being opened.
101
+ // We limit the same values as the System scope, but only allow the Transient scope to take 25% of what is allowed for the System scope.
102
TransientBaseLimit: rcmgr.BaseLimit{
103
- Memory: rcmgr.DefaultLimits.TransientBaseLimit.Memory,
104
- FD: rcmgr.DefaultLimits.TransientBaseLimit.FD,
103
+ Memory: int64(maxMemory / 4),
104
+ FD: maxFD / 4,
105
106
Conns: bigEnough,
107
- ConnsInbound: rcmgr.DefaultLimits.TransientBaseLimit.ConnsInbound,
107
+ ConnsInbound: systemConnsInbound / 4,
108
ConnsOutbound: bigEnough,
109
110
Streams: bigEnough,
111
- StreamsInbound: rcmgr.DefaultLimits.TransientBaseLimit.StreamsInbound,
111
+ StreamsInbound: bigEnough,
112
StreamsOutbound: bigEnough,
113
},
114
115
- TransientLimitIncrease: rcmgr.BaseLimitIncrease{
116
- Memory: rcmgr.DefaultLimits.TransientLimitIncrease.Memory,
117
- FDFraction: rcmgr.DefaultLimits.TransientLimitIncrease.FDFraction,
118
-
119
- Conns: 0,
120
- ConnsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.ConnsInbound,
121
- ConnsOutbound: 0,
122
-
123
- Streams: 0,
124
- StreamsInbound: rcmgr.DefaultLimits.TransientLimitIncrease.StreamsInbound,
125
- StreamsOutbound: 0,
126
- },
115
+ TransientLimitIncrease: noLimitIncrease,
116
117
// Lets get out of the way of the allow list functionality.
118
// If someone specified "Swarm.ResourceMgr.Allowlist" we should let it go through.
@@ -184,7 +173,7 @@ Run 'ipfs swarm limit all' to see the resulting limits.
173
// Whatever limits libp2p has specifically tuned for its protocols/services we'll apply.
174
libp2p.SetDefaultServiceLimits(&scalingLimitConfig)
175
187
- defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), int(numFD))
176
+ defaultLimitConfig := scalingLimitConfig.Scale(int64(maxMemory), maxFD)
177
178
// Simple checks to overide autoscaling ensuring limits make sense versus the connmgr values.
179
// There are ways to break this, but this should catch most problems already.
docs/changelogs/v0.18.md
+24
-12
@@ -2,14 +2,14 @@
2
3
## v0.18.1
4
5
-This release includes improvements around Pubsub message deduplication, and more.
6
-
5
+This release includes improvements around Pubsub message deduplication, libp2p resource management, and more.
6
7
<!-- TOC depthfrom:3 -->
8
9
- [Overview](#overview)
10
- [🔦 Highlights](#-highlights)
11
- [New default Pubsub.SeenMessagesStrategy](#new-default-pubsubseenmessagesstrategy)
12
+ - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
13
- [📝 Changelog](#-changelog)
14
- [👨👩👧👦 Contributors](#-contributors)
15
@@ -33,11 +33,24 @@ If you prefer the old behavior, which calculates the TTL countdown based on the
33
first time a message is seen, you can set `Pubsub.SeenMessagesStrategy` to
34
`first-seen`.
35
36
+#### Improving libp2p resource management integration
37
+
38
+This builds on the default protection nodes get against DoS (resource exhaustion) and eclipse attacks
39
+with the [go-libp2p Network Resource Manager/Accountant](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md)
40
+that was fine-tuned in [Kubo 0.18](https://github.com/ipfs/kubo/blob/biglep/resource-manager-example-of-what-want/docs/changelogs/v0.18.md#improving-libp2p-resource-management-integration).
41
+
42
+Adding default hard-limits from the Resource Manager/Accountant after the fact is tricky,
43
+and some additional improvements have been made to improve the [computed defaults](https://github.com/ipfs/kubo/blob/master/docs/libp2p-resource-management.md#computed-default-limits).
44
+As much as possible, the aim is for a user to only think about how much memory they want to bound libp2p to,
45
+and not need to think about translating that to hard numbers for connections, streams, etc.
46
+More updates are likely in future Kubo releases, but with this release:
47
+1. ``System.StreamsInbound`` is no longer bounded directly
48
+2. ``System.ConnsInbound``, ``Transient.Memory``, ``Transiet.ConnsInbound`` have higher default computed values.
49
+
50
### 📝 Changelog
51
52
### 👨👩👧👦 Contributors
53
40
-
54
## v0.18.0
55
56
### Overview
@@ -46,22 +59,21 @@ Below is an outline of all that is in this release, so you get a sense of all th
59
60
<!-- TOC depthfrom:3 -->
61
49
-- [Overview](#overview)
50
-- [🔦 Highlights](#-highlights)
51
- - [Content routing](#content-routing)
62
+ - [🔦 Highlights](#-highlights)
63
+ - [Content routing](#content-routing)
64
- [Default InterPlanetary Network Indexer](#default-interplanetary-network-indexer)
65
- [Increase provider record republish interval and expiration](#increase-provider-record-republish-interval-and-expiration)
54
- - [Gateways](#gateways)
55
- - [DAG-JSON and DAG-CBOR response formats](#dag-json-and-dag-cbor-response-formats)
66
+ - [Gateways](#gateways)
67
+ - [(DAG-)JSON and (DAG-)CBOR response formats](#dag-json-and-dag-cbor-response-formats)
68
- [🐎 Fast directory listings with DAG sizes](#-fast-directory-listings-with-dag-sizes)
57
- - [QUIC and WebTransport](#quic-and-webtransport)
69
+ - [QUIC and WebTransport](#quic-and-webtransport)
70
- [WebTransport enabled by default](#webtransport-enabled-by-default)
71
- [QUIC and WebTransport share a single port](#quic-and-webtransport-share-a-single-port)
72
- [Differentiating QUIC versions](#differentiating-quic-versions)
73
- [QUICv1 and WebTransport config migration](#quicv1-and-webtransport-config-migration)
62
- - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
63
-- [📝 Changelog](#-changelog)
64
-- [👨👩👧👦 Contributors](#-contributors)
74
+ - [Improving libp2p resource management integration](#improving-libp2p-resource-management-integration)
75
+ - [📝 Changelog](#-changelog)
76
+ - [👨👩👧👦 Contributors](#-contributors)
77
78
<!-- /TOC -->
79
docs/config.md
+1
-1
@@ -1843,7 +1843,7 @@ This value is also used to scale the limit on various resources at various scope
1843
when the default limits (discussed in [libp2p resource management](./libp2p-resource-management.md)) are used.
1844
For example, increasing this value will increase the default limit for incoming connections.
1845
1846
-Default: `[TOTAL_SYSTEM_MEMORY]/4`
1846
+Default: `[TOTAL_SYSTEM_MEMORY]/2`
1847
Type: `optionalBytes`
1848
1849
#### `Swarm.ResourceMgr.MaxFileDescriptors`
docs/libp2p-resource-management.md
+1
-3
@@ -70,8 +70,7 @@ The reason these scopes are chosen is because:
70
71
Within these scopes, limits are just set on
72
[memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory),
73
-[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections),
74
-and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams).
73
+[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), and [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections).
74
Limits are set based on the `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` inputs above.
75
76
There are also some special cases where minimum values are enforced.
@@ -89,7 +88,6 @@ These become the [active limits](#how-does-one-see-the-active-limits).
88
89
While `Swarm.ResourceMgr.Limits` can be edited directly, it is also possible to use `ipfs swarm limit` command to inspect and tweak specific limits at runtime.
90
92
-
91
To see all resources that are close to hitting their respective limit:
92
93
```console