Doc improvements and changelog for resource manager (#9413)
Co-authored-by: Antonio Navarro Perez <antnavper@gmail.com>
Steve Loeppky committed
Nov 16, 2022 at 10:26 UTC
6f730dc268c3712d65efb5b0ab251a714cb85ec4
4 files changed
+115
-64
core/node/libp2p/rcmgr.go
+8
-4
@@ -52,18 +52,22 @@ func ResourceManager(cfg config.SwarmConfig) interface{} {
52
return nil, opts, fmt.Errorf("opening IPFS_PATH: %w", err)
53
}
54
55
- limits, err := createDefaultLimitConfig(cfg)
55
+ limitConfig, err := createDefaultLimitConfig(cfg)
56
if err != nil {
57
return nil, opts, err
58
}
59
60
+ // The logic for defaults and overriding with specified SwarmConfig.ResourceMgr.Limits
61
+ // is documented in docs/config.md.
62
+ // Any changes here should be reflected there.
63
if cfg.ResourceMgr.Limits != nil {
64
l := *cfg.ResourceMgr.Limits
62
- l.Apply(limits)
63
- limits = l
65
+ // This effectively overrides the computed default LimitConfig with any vlues from cfg.ResourceMgr.Limits
66
+ l.Apply(limitConfig)
67
+ limitConfig = l
68
}
69
66
- limiter := rcmgr.NewFixedLimiter(limits)
70
+ limiter := rcmgr.NewFixedLimiter(limitConfig)
71
72
str, err := rcmgrObs.NewStatsTraceReporter()
73
if err != nil {
core/node/libp2p/rcmgr_defaults.go
+2
-46
@@ -44,51 +44,8 @@ var noLimitIncrease = rcmgr.BaseLimitIncrease{
44
// This file defines implicit limit defaults used when Swarm.ResourceMgr.Enabled
45
46
// createDefaultLimitConfig creates LimitConfig to pass to libp2p's resource manager.
47
-// libp2p's resource manager provides tremendous flexibility but also adds a lot of complexity.
48
-// The intent of the default config here is to provide good defaults,
49
-// and where the defaults aren't good enough,
50
-// to expose a good set of higher-level "knobs" to users to satisfy most use cases
51
-// without requiring users to wade into all the intricacies of libp2p's resource manager.
52
-//
53
-// The inputs one can specify in SwarmConfig are:
54
-// - cfg.ResourceMgr.MaxMemory: This is the max amount of memory in bytes to allow libp2p to use.
55
-// libp2p's resource manager will prevent additional resource creation while this limit is hit.
56
-// If this value isn't specified, 1/8th of the total system memory is used.
57
-// - cfg.ResourceMgr.MaxFileDescriptors: This is the maximum number of file descriptors to allow libp2p to use.
58
-// libp2p's resource manager will prevent additional file descriptor consumption while this limit is hit.
59
-// If this value isn't specified, the maximum between 1/2 of system FD limit and 4096 is used.
60
-// - Swarm.ConnMgr.HighWater: If a connection manager is specified, libp2p's resource manager
61
-// will allow 2x more connections than the HighWater mark
62
-// so the connection manager has "space and time" to close "least useful" connections.
63
-//
64
-// With these inputs defined, limits are created at the system, transient, and peer scopes.
65
-// Other scopes are ignored (by being set to infinity).
66
-// The reason these scopes are chosen is because:
67
-// - system - This gives us the coarse-grained control we want so we can reason about the system as a whole.
68
-// It is the backstop, and allows us to reason about resource consumption more easily
69
-// since don't have think about the interaction of many other scopes.
70
-// - transient - Limiting connections that are in process of being established provides backpressure so not too much work queues up.
71
-// - peer - The peer scope doesn't protect us against intentional DoS attacks.
72
-// It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers.
73
-// We are reliant on the system scope for protection here in the malicious case.
74
-// The reason for having a peer scope is to protect against unintentional DoS attacks
75
-// (e.g., bug in a peer which is causing it to "misbehave").
76
-// In the unintional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary.
77
-//
78
-// Within these scopes, limits are just set on memory, FD, and inbound connections/streams.
79
-// Limits are set based on the inputs above.
80
-// We trust this node to behave properly and thus ignore outbound connection/stream limits.
81
-// We apply any limits that libp2p has for its protocols/services
82
-// since we assume libp2p knows best here.
83
-//
84
-// This leaves 3 levels of resource management protection:
85
-// 1. The user who does nothing and uses defaults - In this case they get some sane defaults
86
-// based on the amount of memory and file descriptors their system has.
87
-// This should protect the node from many attacks.
88
-// 2. Slightly more advanced user - They can tweak the above by passing in config on
89
-// maxMemory, maxFD, or maxConns with Swarm.HighWater.ConnMgr.
90
-// 3. Power user - They specify all the limits they want set via Swarm.ResourceMgr.Limits
91
-// and we don't do any defaults/overrides. We pass that config blindly into libp2p resource manager.
47
+// The defaults follow the documentation in docs/config.md.
48
+// Any changes in the logic here should be reflected there.
49
func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error) {
50
maxMemoryDefaultString := humanize.Bytes(uint64(memory.TotalMemory()) / 8)
51
maxMemoryString := cfg.ResourceMgr.MaxMemory.WithDefault(maxMemoryDefaultString)
@@ -105,7 +62,6 @@ func createDefaultLimitConfig(cfg config.SwarmConfig) (rcmgr.LimitConfig, error)
62
FD: int(numFD),
63
64
// By default, we just limit connections on the inbound side.
108
- // Note that the limit gets adjusted below if "cfg.ConnMgr.HighWater" is set.
65
Conns: bigEnough,
66
ConnsInbound: rcmgr.DefaultLimits.SystemBaseLimit.ConnsInbound, // same as libp2p default
67
ConnsOutbound: bigEnough,
docs/changelogs/v0.17.md
+23
@@ -10,6 +10,7 @@ Below is an outline of all that is in this release, so you get a sense of all th
10
- [v0.17.0](#v0170)
11
- [Overview](#overview)
12
- [🔦 Highlights](#-highlights)
13
+ - [libp2p resource management enabled by default](#libp2p-resource-management-enabled-by-default)
14
- [Implicit connection manager limits](#implicit-connection-manager-limits)
15
- [TAR Response Format on Gateways](#tar-response-format-on-gateways)
16
- [Dialling `/wss` peer behind a reverse proxy](#dialling-wss-peer-behind-a-reverse-proxy)
@@ -20,6 +21,28 @@ Below is an outline of all that is in this release, so you get a sense of all th
21
22
<!-- TODO -->
23
24
+#### libp2p resource management enabled by default
25
+
26
+To help protect nodes from DoS (resource exhaustion) and eclipse attacks,
27
+go-libp2p released a [Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager) with a host of improvements throughout 2022.
28
+
29
+Kubo first [exposed this functionality in Kubo 0.13](https://github.com/ipfs/kubo/blob/master/docs/changelogs/v0.13.md#-libp2p-network-resource-manager-swarmresourcemgr),
30
+but it was disabled by default.
31
+
32
+The resource manager is now enabled by default to protect nodes.
33
+The defaults balance providing protection from various attacks while still enabling normal usecases to work as expected.
34
+
35
+If you want to adjust the defaults, then you can:
36
+1. bound the amount of memory and file descriptors that libp2p will use with [Swarm.ResourceMgr.MaxMemory](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxmemory)
37
+and [Swarm.ResourceMgr.MaxFileDescriptors](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrmaxfiledescriptors) and/or
38
+2. override any specific resource scopes/limits with [Swarm.ResourceMgr.Limits](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgrlimits)
39
+
40
+See [Swarm.ResourceMgr](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#swarmresourcemgr) for
41
+1. what limits are set by default,
42
+2. example override configuration,
43
+3. how to access prometheus metrics and view grafana dashboards of resource usage, and
44
+4. how to set explicit "allow lists" to protect against eclipse attacks.
45
+
46
#### Implicit connection manager limits
47
48
Starting with this release, `ipfs init` will no longer store the default
docs/config.md
+82
-14
@@ -1808,30 +1808,78 @@ Type: `optionalDuration`
1808
1809
### `Swarm.ResourceMgr`
1810
1811
-The [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p-resource-manager#readme) allows setting limits per a scope,
1811
+The [libp2p Network Resource Manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme) allows setting limits per [Resource Scope](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#resource-scopes),
1812
and tracking recource usage over time.
1813
1814
-#### `Swarm.ResourceMgr.Enabled`
1814
+##### Levels of Configuration
1815
+
1816
+libp2p's resource manager provides tremendous flexibility but also adds a lot of complexity.
1817
+There are these levels of limit configuration for resource management protection:
1818
+1. "The user who does nothing" - In this case they get some sane defaults discussed below
1819
+ based on the amount of memory and file descriptors their system has.
1820
+ This should protect the node from many attacks.
1821
+2. "Slightly more advanced user" - They can tweak the default limits discussed below.
1822
+ Where the defaults aren't good enough, a good set of higher-level "knobs" are exposed to satisfy most use cases
1823
+ without requiring users to wade into all the intricacies of libp2p's resource manager.
1824
+ The "knobs"/inputs are `Swarm.ResourceMgr.MaxMemory` and `Swarm.ResourceMgr.MaxFileDescriptors` as described below.
1825
+3. "Power user" - They specify all the default limits from below they want override via `Swarm.ResourceMgr.Limits`;
1826
+
1827
+##### Default Limits
1828
+
1829
+With these inputs defined, [resource manager limits](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#limits) are created at the
1830
+[system](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-system-scope),
1831
+[transient](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#the-transient-scope),
1832
+and [peer](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#peer-scopes) scopes.
1833
+Other scopes are ignored (by being set to "~infinity".
1834
+
1835
+The reason these scopes are chosen is because:
1836
+- system - This gives us the coarse-grained control we want so we can reason about the system as a whole.
1837
+ It is the backstop, and allows us to reason about resource consumption more easily
1838
+ since don't have think about the interaction of many other scopes.
1839
+- transient - Limiting connections that are in process of being established provides backpressure so not too much work queues up.
1840
+- peer - The peer scope doesn't protect us against intentional DoS attacks.
1841
+ It's just as easy for an attacker to send 100 requests/second with 1 peerId vs. 10 requests/second with 10 peers.
1842
+ We are reliant on the system scope for protection here in the malicious case.
1843
+ The reason for having a peer scope is to protect against unintentional DoS attacks
1844
+ (e.g., bug in a peer which is causing it to "misbehave").
1845
+ In the unintional case, we want to make sure a "misbehaving" node doesn't consume more resources than necessary.
1846
+
1847
+Within these scopes, limits are just set on
1848
+[memory](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#memory),
1849
+[file descriptors (FD)](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#file-descriptors), [*inbound* connections](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#connections),
1850
+and [*inbound* streams](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#streams).
1851
+Limits are set based on the inputs above.
1852
+We trust this node to behave properly and thus don't limit *outbound* connection/stream limits.
1853
+We apply any limits that libp2p has for its protocols/services
1854
+since we assume libp2p knows best here.
1855
+
1856
+** libp2p resource monitoring **
1857
+For [monitoring libp2p resource usage](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#monitoring),
1858
+various `*rcmgr_*` metrics can be accessed as the prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`).
1859
+There are also [pre-built Grafana dashboards](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager/obs/grafana-dashboards) that can be added to a Grafana instance.
1860
1816
-Enables the libp2p Network Resource Manager and auguments the default limits
1817
-using user-defined ones in `Swarm.ResourceMgr.Limits` (if present).
1861
+#### `Swarm.ResourceMgr.Enabled`
1862
1819
-Various `*rcmgr_*` metrics can be accessed as the prometheus endpoint at `{Addresses.API}/debug/metrics/prometheus` (default: `http://127.0.0.1:5001/debug/metrics/prometheus`)
1863
+Enables the libp2p Resource Manager using limits based on the defaults and/or other configuration as discussed above.
1864
1865
Default: `true`
1822
-
1866
Type: `flag`
1867
1868
#### `Swarm.ResourceMgr.MaxMemory`
1869
1827
-The maximum amount of memory that the libp2p resource manager will allow.
1870
+This is the max amount of memory to allow libp2p to use.
1871
+libp2p's resource manager will prevent additional resource creation while this limit is reached.
1872
+This value is also used to scale the limit on various resources at various scopes
1873
+when the default limits (discuseed above) are used.
1874
+For example, increasing this value will increase the default limit for incoming connections.
1875
1876
Default: `[TOTAL_SYSTEM_MEMORY]/8`
1877
Type: `optionalBytes`
1878
1879
#### `Swarm.ResourceMgr.MaxFileDescriptors`
1880
1834
-Define the maximum number of file descriptors that libp2p can use.
1881
+This is the maximum number of file descriptors to allow libp2p to use.
1882
+libp2p's resource manager will prevent additional file descriptor consumption while this limit is reached.
1883
1884
This param is ignored on Windows.
1885
@@ -1842,21 +1890,26 @@ Type: `optionalInteger`
1890
1891
Map of resource limits [per scope](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#resource-scopes).
1892
1845
-The map supports fields from [`ScalingLimitConfig`](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L21-L59)
1846
-struct from [go-libp2p-resource-manager](https://github.com/libp2p/go-libp2p/tree/master/p2p/host/resource-manager#readme).
1893
+The map supports fields from the [`LimitConfig` struct](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit_defaults.go#L111).
1894
1895
+[`BaseLimit`s](https://github.com/libp2p/go-libp2p/blob/master/p2p/host/resource-manager/limit.go#L89) can be set for any scope, and within the `BaseLimit`, all limit <key,value>s are optional.
1896
+
1897
+The `Swarm.ResourceMgr.Limits` override the default limits described above.
1898
+Any override `BaseLimits` or limit <key,value>s from `Swarm.ResourceMgr.Limits`
1899
+that aren't specified will use the default limits.
1900
+
1901
+Example #1: setting limits for a specific scope
1902
```json
1903
{
1904
"Swarm": {
1905
"ResourceMgr": {
1852
- "Enabled": true,
1906
"Limits": {
1907
"System": {
1908
+ "Memory": 1073741824,
1909
+ "FD": 512,
1910
"Conns": 1024,
1911
"ConnsInbound": 256,
1912
"ConnsOutbound": 1024,
1858
- "FD": 512,
1859
- "Memory": 1073741824,
1913
"Streams": 16384,
1914
"StreamsInbound": 4096,
1915
"StreamsOutbound": 16384
@@ -1867,13 +1920,28 @@ struct from [go-libp2p-resource-manager](https://github.com/libp2p/go-libp2p/tre
1920
}
1921
```
1922
1923
+Example #2: setting a specific <key,value> limit
1924
+```json
1925
+{
1926
+ "Swarm": {
1927
+ "ResourceMgr": {
1928
+ "Limits": {
1929
+ "Transient": {
1930
+ "ConnsOutbound": 256,
1931
+ }
1932
+ }
1933
+ }
1934
+ }
1935
+}
1936
+```
1937
+
1938
Current resource usage and a list of services, protocols, and peers can be
1939
obtained via `ipfs swarm stats --help`
1940
1941
It is also possible to adjust some runtime limits via `ipfs swarm limit --help`.
1942
Changes made via `ipfs swarm limit` are persisted in `Swarm.ResourceMgr.Limits`.
1943
1876
-Default: `{}` (use the safe implicit defaults)
1944
+Default: `{}` (use the safe implicit defaults described above)
1945
1946
Type: `object[string->object]`
1947