This patch contains more safe checking in syscall rlimit and unit tests
The soft limit is the value that the kernel enforces for the corresponding resource The hard limit acts as a ceiling for the soft limit an unprivileged process may only set its soft limit to a value in the range from 0 up to the hard limit. So in order to make the change in fds count without any error we should inform the user to make the process have CAP_SYS_RESOURCE capability in order to set the hard limit. License: MIT Signed-off-by: hoenirvili <hoenirvili@gmail.com>
hoenirvili committed
Mar 21, 2017 at 23:52 UTC
f16ada6159f12532b291fac7e47ab78ca6f684ed
9 files changed
+243
-112
cmd/ipfs/daemon.go
+3
-5
@@ -11,6 +11,7 @@ import (
11
"sort"
12
"sync"
13
14
+ utilmain "github.com/ipfs/go-ipfs/cmd/ipfs/util"
15
"github.com/ipfs/go-ipfs/core"
16
commands "github.com/ipfs/go-ipfs/core/commands"
17
corehttp "github.com/ipfs/go-ipfs/core/corehttp"
@@ -180,8 +181,6 @@ func defaultMux(path string) corehttp.ServeOption {
181
}
182
}
183
183
-var fileDescriptorCheck = func() error { return nil }
184
-
184
func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
185
// Inject metrics before we do anything
186
@@ -193,9 +192,8 @@ func daemonFunc(req cmds.Request, re cmds.ResponseEmitter) {
192
// let the user know we're going.
193
fmt.Printf("Initializing daemon...\n")
194
196
- managefd, _, _ := req.Option(adjustFDLimitKwd).Bool()
197
- if managefd {
198
- if err := fileDescriptorCheck(); err != nil {
195
+ if managed, _, _ := req.Option(adjustFDLimitKwd).Bool(); managed {
196
+ if err := utilmain.ManageFdLimit(); err != nil {
197
log.Errorf("setting file descriptor limit: %s", err)
198
}
199
}
cmd/ipfs/ulimit.go
deleted
-19
@@ -1,19 +0,0 @@
1
-package main
2
-
3
-import (
4
- "os"
5
- "strconv"
6
-)
7
-
8
-var ipfsFileDescNum = uint64(2048)
9
-
10
-func init() {
11
- if val := os.Getenv("IPFS_FD_MAX"); val != "" {
12
- n, err := strconv.Atoi(val)
13
- if err != nil {
14
- log.Errorf("bad value for IPFS_FD_MAX: %s", err)
15
- } else {
16
- ipfsFileDescNum = uint64(n)
17
- }
18
- }
19
-}
cmd/ipfs/ulimit_freebsd.go
deleted
-45
@@ -1,45 +0,0 @@
1
-// +build freebsd
2
-
3
-package main
4
-
5
-import (
6
- "fmt"
7
-
8
- unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
9
-)
10
-
11
-func init() {
12
- fileDescriptorCheck = checkAndSetUlimit
13
-}
14
-
15
-func checkAndSetUlimit() error {
16
- var rLimit unix.Rlimit
17
- err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
18
- if err != nil {
19
- return fmt.Errorf("error getting rlimit: %s", err)
20
- }
21
-
22
- ipfsFileDescNum := int64(ipfsFileDescNum)
23
-
24
- var setting bool
25
- if rLimit.Cur < ipfsFileDescNum {
26
- if rLimit.Max < ipfsFileDescNum {
27
- log.Error("adjusting max")
28
- rLimit.Max = ipfsFileDescNum
29
- }
30
- fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
31
- rLimit.Cur = ipfsFileDescNum
32
- setting = true
33
- }
34
-
35
- err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
36
- if err != nil {
37
- return fmt.Errorf("error setting ulimit: %s", err)
38
- }
39
-
40
- if setting {
41
- fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
42
- }
43
-
44
- return nil
45
-}
cmd/ipfs/ulimit_unix.go
deleted
-43
@@ -1,43 +0,0 @@
1
-// +build darwin linux netbsd openbsd
2
-
3
-package main
4
-
5
-import (
6
- "fmt"
7
-
8
- unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
9
-)
10
-
11
-func init() {
12
- fileDescriptorCheck = checkAndSetUlimit
13
-}
14
-
15
-func checkAndSetUlimit() error {
16
- var rLimit unix.Rlimit
17
- err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rLimit)
18
- if err != nil {
19
- return fmt.Errorf("error getting rlimit: %s", err)
20
- }
21
-
22
- var setting bool
23
- if rLimit.Cur < ipfsFileDescNum {
24
- if rLimit.Max < ipfsFileDescNum {
25
- log.Error("adjusting max")
26
- rLimit.Max = ipfsFileDescNum
27
- }
28
- fmt.Printf("Adjusting current ulimit to %d...\n", ipfsFileDescNum)
29
- rLimit.Cur = ipfsFileDescNum
30
- setting = true
31
- }
32
-
33
- err = unix.Setrlimit(unix.RLIMIT_NOFILE, &rLimit)
34
- if err != nil {
35
- return fmt.Errorf("error setting ulimit: %s", err)
36
- }
37
-
38
- if setting {
39
- fmt.Printf("Successfully raised file descriptor limit to %d.\n", ipfsFileDescNum)
40
- }
41
-
42
- return nil
43
-}
cmd/ipfs/util/ulimit.go
new
+91
@@ -0,0 +1,91 @@
1
+package util
2
+
3
+import (
4
+ "errors"
5
+ "fmt"
6
+ "os"
7
+ "strconv"
8
+ "syscall"
9
+
10
+ logging "gx/ipfs/QmSpJByNKFX1sCsHBEp3R73FL4NF6FnQTEGyNAXHm2GS52/go-log"
11
+)
12
+
13
+var log = logging.Logger("ulimit")
14
+
15
+var (
16
+ supportsFDManagement = false
17
+
18
+ // getlimit returns the soft and hard limits of file descriptors counts
19
+ getLimit func() (int64, int64, error)
20
+ // set limit sets the soft and hard limits of file descriptors counts
21
+ setLimit func(int64, int64) error
22
+)
23
+
24
+// maxFds is the maximum number of file descriptors that go-ipfs
25
+// can use. The default value is 1024. This can be overwritten by the
26
+// IPFS_FD_MAX env variable
27
+var maxFds = uint64(2048)
28
+
29
+// setMaxFds sets the maxFds value from IPFS_FD_MAX
30
+// env variable if it's present on the system
31
+func setMaxFds() {
32
+ // check if the IPFS_FD_MAX is set up and if it does
33
+ // not have a valid fds number notify the user
34
+ if val := os.Getenv("IPFS_FD_MAX"); val != "" {
35
+
36
+ fds, err := strconv.ParseUint(val, 10, 64)
37
+ if err != nil {
38
+ log.Errorf("bad value for IPFS_FD_MAX: %s", err)
39
+ return
40
+ }
41
+
42
+ maxFds = fds
43
+ }
44
+}
45
+
46
+// ManageFdLimit raise the current max file descriptor count
47
+// of the process based on the IPFS_FD_MAX value
48
+func ManageFdLimit() error {
49
+ if !supportsFDManagement {
50
+ return nil
51
+ }
52
+
53
+ setMaxFds()
54
+ soft, hard, err := getLimit()
55
+ if err != nil {
56
+ return err
57
+ }
58
+
59
+ max := int64(maxFds)
60
+
61
+ if max <= soft {
62
+ return nil
63
+ }
64
+
65
+ // the soft limit is the value that the kernel enforces for the
66
+ // corresponding resource
67
+ // the hard limit acts as a ceiling for the soft limit
68
+ // an unprivileged process may only set it's soft limit to a
69
+ // alue in the range from 0 up to the hard limit
70
+ if err = setLimit(max, max); err != nil {
71
+ if err != syscall.EPERM {
72
+ return fmt.Errorf("error setting: ulimit: %s", err)
73
+ }
74
+
75
+ // the process does not have permission so we should only
76
+ // set the soft value
77
+ if max > hard {
78
+ return errors.New(
79
+ "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit",
80
+ )
81
+ }
82
+
83
+ if err = setLimit(max, hard); err != nil {
84
+ return fmt.Errorf("error setting ulimit wihout hard limit: %s", err)
85
+ }
86
+ }
87
+
88
+ fmt.Printf("Successfully raised file descriptor limit to %d.\n", max)
89
+
90
+ return nil
91
+}
cmd/ipfs/util/ulimit_freebsd.go
new
+27
@@ -0,0 +1,27 @@
1
+// +build freebsd
2
+
3
+package util
4
+
5
+import (
6
+ unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
7
+)
8
+
9
+func init() {
10
+ supportsFDManagement = true
11
+ getLimit = freebsdGetLimit
12
+ setLimit = freebdsSetLimit
13
+}
14
+
15
+func freebsdGetLimit() (int64, int64, error) {
16
+ rlimit := unix.Rlimit{}
17
+ err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
18
+ return rlimit.Cur, rlimit.Max, err
19
+}
20
+
21
+func freebdsSetLimit(soft int64, max int64) error {
22
+ rlimit := unix.Rlimit{
23
+ Cur: soft,
24
+ Max: max,
25
+ }
26
+ return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
27
+}
cmd/ipfs/util/ulimit_test.go
new
+88
@@ -0,0 +1,88 @@
1
+package util
2
+
3
+import (
4
+ "fmt"
5
+ "os"
6
+ "strings"
7
+ "syscall"
8
+ "testing"
9
+)
10
+
11
+func TestManageFdLimit(t *testing.T) {
12
+ t.Log("Testing file descriptor count")
13
+ if err := ManageFdLimit(); err != nil {
14
+ t.Errorf("Cannot manage file descriptors")
15
+ }
16
+
17
+ if maxFds != uint64(2048) {
18
+ t.Errorf("Maximum file descriptors default value changed")
19
+ }
20
+}
21
+
22
+func TestManageInvalidNFds(t *testing.T) {
23
+ t.Logf("Testing file descriptor invalidity")
24
+ var err error
25
+ if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
26
+ t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
27
+ }
28
+
29
+ rlimit := syscall.Rlimit{}
30
+ if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
31
+ t.Fatal("Cannot get the file descriptor count")
32
+ }
33
+
34
+ value := rlimit.Max + rlimit.Cur
35
+ if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
36
+ t.Fatal("Cannot set the IPFS_FD_MAX env variable")
37
+ }
38
+
39
+ // call to check and set the maximum file descriptor from the env
40
+ setMaxFds()
41
+
42
+ if err = ManageFdLimit(); err == nil {
43
+ t.Errorf("ManageFdLimit should return an error")
44
+ } else if err != nil {
45
+ flag := strings.Contains(err.Error(),
46
+ "cannot set rlimit, IPFS_FD_MAX is larger than the hard limit")
47
+ if !flag {
48
+ t.Errorf("ManageFdLimit returned unexpected error")
49
+ }
50
+ }
51
+
52
+ // unset all previous operations
53
+ if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
54
+ t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
55
+ }
56
+}
57
+
58
+func TestManageFdLimitWithEnvSet(t *testing.T) {
59
+ t.Logf("Testing file descriptor manager with IPFS_FD_MAX set")
60
+ var err error
61
+ if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
62
+ t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
63
+ }
64
+
65
+ rlimit := syscall.Rlimit{}
66
+ if err = syscall.Getrlimit(syscall.RLIMIT_NOFILE, &rlimit); err != nil {
67
+ t.Fatal("Cannot get the file descriptor count")
68
+ }
69
+
70
+ value := rlimit.Max - rlimit.Cur + 1
71
+ if err = os.Setenv("IPFS_FD_MAX", fmt.Sprintf("%d", value)); err != nil {
72
+ t.Fatal("Cannot set the IPFS_FD_MAX env variable")
73
+ }
74
+
75
+ setMaxFds()
76
+ if maxFds != value {
77
+ t.Errorf("The maxfds is not set from IPFS_FD_MAX")
78
+ }
79
+
80
+ if err = ManageFdLimit(); err != nil {
81
+ t.Errorf("Cannot manage file descriptor count")
82
+ }
83
+
84
+ // unset all previous operations
85
+ if err = os.Unsetenv("IPFS_FD_MAX"); err != nil {
86
+ t.Fatal("Cannot unset the IPFS_FD_MAX env variable")
87
+ }
88
+}
cmd/ipfs/util/ulimit_unix.go
new
+27
@@ -0,0 +1,27 @@
1
+// +build darwin linux netbsd openbsd
2
+
3
+package util
4
+
5
+import (
6
+ unix "gx/ipfs/QmPXvegq26x982cQjSfbTvSzZXn7GiaMwhhVPHkeTEhrPT/sys/unix"
7
+)
8
+
9
+func init() {
10
+ supportsFDManagement = true
11
+ getLimit = unixGetLimit
12
+ setLimit = unixSetLimit
13
+}
14
+
15
+func unixGetLimit() (int64, int64, error) {
16
+ rlimit := unix.Rlimit{}
17
+ err := unix.Getrlimit(unix.RLIMIT_NOFILE, &rlimit)
18
+ return int64(rlimit.Cur), int64(rlimit.Max), err
19
+}
20
+
21
+func unixSetLimit(soft int64, max int64) error {
22
+ rlimit := unix.Rlimit{
23
+ Cur: uint64(soft),
24
+ Max: uint64(max),
25
+ }
26
+ return unix.Setrlimit(unix.RLIMIT_NOFILE, &rlimit)
27
+}
cmd/ipfs/util/ulimit_windows.go
new
+7
@@ -0,0 +1,7 @@
1
+// +build windows
2
+
3
+package util
4
+
5
+func init() {
6
+ supportsFDManagement = false
7
+}