Doc improvements and changelog for resource manager (#9413)
Co-authored-by: Antonio Navarro Perez <antnavper@gmail.com>
Steve Loeppky committed
Nov 16, 2022 at 10:26 UTC
83034d840cd3ecfed5084835490a3088fc3e230b
4 files changed
+115
-72
core/node/libp2p/rcmgr.go
+8
-4
@@ -52,18 +52,22 @@ func ResourceManager(cfg config.SwarmConfig) interface{} {
52
return nil, opts, fmt.Errorf("opening IPFS_PATH: %w", err)
53
}
54
55
- limits, err := createDefaultLimitConfig(cfg)
55
+ limitConfig, err := createDefaultLimitConfig(cfg)
56
if err != nil {
57
return nil, opts, err
58
}
59
60
+ // The logic for defaults and overriding with specified SwarmConfig.ResourceMgr.Limits
61
+ // is documented in docs/config.md.
62
+ // Any changes here should be reflected there.
63
if cfg.ResourceMgr.Limits != nil {
64
l := *cfg.ResourceMgr.Limits
62
- l.Apply(limits)
63
- limits = l
65
+ // This effectively overrides the computed default LimitConfig with any vlues from cfg.ResourceMgr.Limits
66
+ l.Apply(limitConfig)
67
+ limitConfig = l
68
}
69
66
- limiter := rcmgr.NewFixedLimiter(limits)
70
+ limiter := rcmgr.NewFixedLimiter(limitConfig)
71
72
str, err := rcmgrObs.NewStatsTraceReporter()
73
if err != nil {
core/node/libp2p/rcmgr_defaults.go
+2
-54
@@ -44,59 +44,8 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{
44
// This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled
45
46
// createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager.
47
-// libp2p's resource manager provides tremendous flexibility but also adds a lot of complexity.
48
-// The intent of the default config here is to provide good defaults,
49
-// and where the defaults aren't good enough,
50
-// to expose a good set of higher-level "knobs" to users to satisfy most use cases
51
-// without requiring users to wade into all the intricacies of libp2p's resource manager.
52
-//
53
-// The inputs one can specify in SwarmConfig are:
54
-// - cfg.ResourceMgr.MaxMemory: This is the max amount of memory in bytes to allow libp2p to use.
55
-// libp2p's resource manager will prevent additional resource creation while this limit is hit.
56
-// If this value isn't specified, 1/8th of the total system memory is used.
57
-// - cfg.ResourceMgr.MaxFileDescriptors: This is the maximum number of file descriptors to allow libp2p to use.
58
-// libp2p's resource manager will prevent additional file descriptor consumption while this limit is hit.
59
-// If this value isn't specified, the maximum between 1/2 of system FD limit and 4096 is used.
60
-//
61
-// With these inputs defined, limits are created at the system, transient, and peer scopes.
62
-// Other scopes are ignored (by being set to infinity).
63
-// The reason these scopes are chosen is because:
64
-// - system - This gives us the coarse-grained control we want so we can reason about the system as a whole.
65
-// It is the backstop, and allows us to reason about resource consumption more easily
66
-// since don't have think about the interaction of many other scopes.
67
-// - transient - Limiting connections that are in process of being established provides backpressure so not too much work queues up.
68
-// - peer - The peer scope doesn't protect us against intentional DoS attacks.
69
-// It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers.
70
-// We are reliant on the system scope for protection here in the malicious case.
71
-// The reason for having a peer scope is to protect against unintentional DoS attacks
72
-// (e.g., bug in a peer which is causing it to "misbehave").
73
-// In the unintional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary.
74
-//
75
-// Within these scopes, limits are just set on memory, FD, and inbound connections/streams.
76
-// Limits are set based on the inputs above.
77
-// We trust this node to behave properly and thus ignore outbound connection/stream limits.
78
-// We apply any limits that libp2p has for its protocols/services
79
-// since we assume libp2p knows best here.
80
-//
81
-// This leaves 3 levels of resource management protection:
82
-// 1. The user who does nothing and uses defaults - In this case they get some sane defaults
83
-// based on the amount of memory and file descriptors their system has.
84
-// This should protect the node from many attacks.
85
-// 2. Slightly more advanced user - They can tweak the above by passing in config on
86
-// maxMemory, maxFD, or maxConns with Swarm.HighWater.ConnMgr.
87
-// 3. Power user - They specify all the limits they want set via Swarm.ResourceMgr.Limits
88
-// and we don't do any defaults/overrides. We pass that config blindly into libp2p resource manager.
89
-//
90
-// Note that within libp2p. Swarm.ConnMgr settings have no impact on libp2p's resource manager limits.
91
-// See https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/README.md#connmanager-vs-resource-manager
92
-// and https://github.com/libp2p/go-libp2p/issues/1640
93
-// We also don't layer on extra logic in this function because SystemBaseLimit.Conns is already "bigEnough".
94
-// There is headroom for the connection manager to apply any Swarm.ConnMgr.HighWater mark.
95
-// We're keeping things simple by avoiding any interaction between libp2p's resource manager and connection manager.
96
-// For example we don't set SystemBaseLimit.Conns to be related to Swarm.ConnMgr.HighWater.
97
-// SystemBaseLimit.Conns is "bigEnough" and won't won't limit total connections.
98
-// (We will limit SystemBaseLimit.ConnsInbound though.)
99
-// The Swarm.ConnMgr can manage connections based on Swarm.ConnMgr.HighWater.
47
+// The defaults follow the documentation in docs/config.md.
48
+// Any changes in the logic here should be reflected there.
49
func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) {
50
maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 8)
51
maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString)
@@ -113,7 +62,6 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error)
62
FD: int(numFD),
63
64
// By default, we just limit connections on the inbound side.
116
- // Note that the limit gets adjusted below if "cfg.ConnMgr.HighWater" is set.
65
Conns: bigEnough,
66
ConnsInbound: rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default
67
ConnsOutbound: bigEnough,
docs/changelogs/v0.17.md
+23
@@ -10,6 +10,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
10
- [v0.17.0](#v0170)
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
+ - [libp2p resource management enabled by default](#libp2p-resource-management-enabled-by-default)
14
- [Implicit connection manager limits](#implicit-connection-manager-limits)
15
- [TAR Response Format on Gateways](#tar-response-format-on-gateways)
16
- [Dialling `/wss` peer behind a reverse proxy](#dialling-wss-peer-behind-a-reverse-proxy)
@@ -20,6 +21,28 @@ Below is an outline of all that is in this release, so you get a sense of all th
21
22
<!-- TODO -->
23
24
+#### libp2p resource management enabled by default
25
+
26
+To help protect nodes from DoS (resource exhaustion) and eclipse attacks,
27
+go-libp2p released a [Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager) with a host of improvements throughout 2022.
28
+
29
+Kubo first [exposed this functionality in Kubo 0.13](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.13.md#-libp2p-network-resource-manager-swarmresourcemgr),
30
+but it was disabled by default.
31
+
32
+The resource manager is now enabled by default to protect nodes.
33
+The defaults balance providing protection from various attacks while still enabling normal usecases to work as expected.
34
+
35
+If you want to adjust the defaults, then you can:
36
+1. bound the amount of memory and file descriptors that libp2p will use with [Swarm.ResourceMgr.MaxMemory](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxmemory)
37
+and [Swarm.ResourceMgr.MaxFileDescriptors](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxfiledescriptors) and/or
38
+2. override any specific resource scopes/limits with [Swarm.ResourceMgr.Limits](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrlimits)
39
+
40
+See [Swarm.ResourceMgr](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgr) for
41
+1. what limits are set by default,
42
+2. example override configuration,
43
+3. how to access prometheus metrics and view grafana dashboards of resource usage, and
44
+4. how to set explicit "allow lists" to protect against eclipse attacks.
45
+
46
#### Implicit connection manager limits
47
48
Starting with this release, `ipfs init` will no longer store the default
docs/config.md
+82
-14
@@ -1773,30 +1773,78 @@ Type: `optionalDuration`
1773
1774
### `Swarm.ResourceMgr`
1775
1776
-The [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p-resource-manager#readme) allows setting limits per a scope,
1776
+The [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme) allows setting limits per [Resource Scope](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#resource-scopes),
1777
and tracking recource usage over time.
1778
1779
-#### `Swarm.ResourceMgr.Enabled`
1779
+##### Levels of Configuration
1780
+
1781
+libp2p's resource manager provides tremendous flexibility but also adds a lot of complexity.
1782
+There are these levels of limit configuration for resource management protection:
1783
+1. "The user who does nothing" - In this case they get some sane defaults discussed below
1784
+ based on the amount of memory and file descriptors their system has.
1785
+ This should protect the node from many attacks.
1786
+2. "Slightly more advanced user" - They can tweak the default limits discussed below.
1787
+ Where the defaults aren't good enough, a good set of higher-level "knobs" are exposed to satisfy most use cases
1788
+ without requiring users to wade into all the intricacies of libp2p's resource manager.
1789
+ The "knobs"/inputs are `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` as described below.
1790
+3. "Power user" - They specify all the default limits from below they want override via `Swarm.ResourceMgr.Limits`;
1791
+
1792
+##### Default Limits
1793
+
1794
+With these inputs defined, [resource manager limits](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#limits) are created at the
1795
+[system](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-system-scope),
1796
+[transient](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-transient-scope),
1797
+and [peer](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#peer-scopes) scopes.
1798
+Other scopes are ignored (by being set to "~infinity".
1799
+
1800
+The reason these scopes are chosen is because:
1801
+- system - This gives us the coarse-grained control we want so we can reason about the system as a whole.
1802
+ It is the backstop, and allows us to reason about resource consumption more easily
1803
+ since don't have think about the interaction of many other scopes.
1804
+- transient - Limiting connections that are in process of being established provides backpressure so not too much work queues up.
1805
+- peer - The peer scope doesn't protect us against intentional DoS attacks.
1806
+ It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers.
1807
+ We are reliant on the system scope for protection here in the malicious case.
1808
+ The reason for having a peer scope is to protect against unintentional DoS attacks
1809
+ (e.g., bug in a peer which is causing it to "misbehave").
1810
+ In the unintional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary.
1811
+
1812
+Within these scopes, limits are just set on
1813
+[memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory),
1814
+[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections),
1815
+and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams).
1816
+Limits are set based on the inputs above.
1817
+We trust this node to behave properly and thus don't limit *outbound* connection/stream limits.
1818
+We apply any limits that libp2p has for its protocols/services
1819
+since we assume libp2p knows best here.
1820
+
1821
+** libp2p resource monitoring **
1822
+For [monitoring libp2p resource usage](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#monitoring),
1823
+various `*rcmgr_*` metrics can be accessed as the prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`).
1824
+There are also [pre-built Grafana dashboards](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager/obs/grafana-dashboards) that can be added to a Grafana instance.
1825
1781
-Enables the libp2p Network Resource Manager and auguments the default limits
1782
-using user-defined ones in `Swarm.ResourceMgr.Limits` (if present).
1826
+#### `Swarm.ResourceMgr.Enabled`
1827
1784
-Various `*rcmgr_*` metrics can be accessed as the prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`)
1828
+Enables the libp2p Resource Manager using limits based on the defaults and/or other configuration as discussed above.
1829
1830
Default: `true`
1787
-
1831
Type: `flag`
1832
1833
#### `Swarm.ResourceMgr.MaxMemory`
1834
1792
-The maximum amount of memory that the libp2p resource manager will allow.
1835
+This is the max amount of memory to allow libp2p to use.
1836
+libp2p's resource manager will prevent additional resource creation while this limit is reached.
1837
+This value is also used to scale the limit on various resources at various scopes
1838
+when the default limits (discuseed above) are used.
1839
+For example, increasing this value will increase the default limit for incoming connections.
1840
1841
Default: `[TOTAL_SYSTEM_MEMORY]/8`
1842
Type: `optionalBytes`
1843
1844
#### `Swarm.ResourceMgr.MaxFileDescriptors`
1845
1799
-Define the maximum number of file descriptors that libp2p can use.
1846
+This is the maximum number of file descriptors to allow libp2p to use.
1847
+libp2p's resource manager will prevent additional file descriptor consumption while this limit is reached.
1848
1849
This param is ignored on Windows.
1850
@@ -1807,21 +1855,26 @@ Type: `optionalInteger`
1855
1856
Map of resource limits [per scope](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#resource-scopes).
1857
1810
-The map supports fields from [`ScalingLimitConfig`](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L21-L59)
1811
-struct from [go-libp2p-resource-manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme).
1858
+The map supports fields from the [`LimitConfig` struct](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L111).
1859
1860
+[`BaseLimit`s](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit.go#L89) can be set for any scope, and within the `BaseLimit`, all limit <key,value>s are optional.
1861
+
1862
+The `Swarm.ResourceMgr.Limits` override the default limits described above.
1863
+Any override `BaseLimits` or limit <key,value>s from `Swarm.ResourceMgr.Limits`
1864
+that aren't specified will use the default limits.
1865
+
1866
+Example #1: setting limits for a specific scope
1867
```json
1868
{
1869
"Swarm": {
1870
"ResourceMgr": {
1817
- "Enabled": true,
1871
"Limits": {
1872
"System": {
1873
+ "Memory": 1073741824,
1874
+ "FD": 512,
1875
"Conns": 1024,
1876
"ConnsInbound": 256,
1877
"ConnsOutbound": 1024,
1823
- "FD": 512,
1824
- "Memory": 1073741824,
1878
"Streams": 16384,
1879
"StreamsInbound": 4096,
1880
"StreamsOutbound": 16384
@@ -1832,13 +1885,28 @@ struct from [go-libp2p-resource-manager](https://github.com/libp2p/go-libp2p/tre
1885
}
1886
```
1887
1888
+Example #2: setting a specific <key,value> limit
1889
+```json
1890
+{
1891
+ "Swarm": {
1892
+ "ResourceMgr": {
1893
+ "Limits": {
1894
+ "Transient": {
1895
+ "ConnsOutbound": 256,
1896
+ }
1897
+ }
1898
+ }
1899
+ }
1900
+}
1901
+```
1902
+
1903
Current resource usage and a list of services, protocols, and peers can be
1904
obtained via `ipfs swarm stats --help`
1905
1906
It is also possible to adjust some runtime limits via `ipfs swarm limit --help`.
1907
Changes made via `ipfs swarm limit` are persisted in `Swarm.ResourceMgr.Limits`.
1908
1841
-Default: `{}` (use the safe implicit defaults)
1909
+Default: `{}` (use the safe implicit defaults described above)
1910
1911
Type: `object[string->object]`
1912