feat: add full goroutine stack dump (#8790)
Gus Eggert committed
Mar 16, 2022 at 09:42 UTC
5e1b2248c576a5ff1ad8b1d681ccdb347aa63556
5 files changed
+52
bin/collect-profiles.sh
+2
@@ -24,6 +24,8 @@ fi
24
echo Collecting goroutine stacks
25
curl -s -o goroutines.stacks "$SOURCE_URL"'/debug/pprof/goroutine?debug=2'
26
27
+curl -s -o goroutines.stacks.full "$SOURCE_URL"'/debug/stack'
28
+
29
echo Collecting goroutine profile
30
go tool pprof -symbolize=remote -svg -output goroutine.svg "$SOURCE_URL/debug/pprof/goroutine"
31
cmd/ipfs/daemon.go
+1
@@ -660,6 +660,7 @@ func serveHTTPApi(req *cmds.Request, cctx *oldcmds.Context) (<-chan error, error
660
corehttp.VersionOption(),
661
defaultMux("/debug/vars"),
662
defaultMux("/debug/pprof/"),
663
+ defaultMux("/debug/stack"),
664
corehttp.MutexFractionOption("/debug/pprof-mutex/"),
665
corehttp.BlockProfileRateOption("/debug/pprof-block/"),
666
corehttp.MetricsScrapingOption("/debug/metrics/prometheus"),
cmd/ipfs/debug.go
new
+15
@@ -0,0 +1,15 @@
1
+package main
2
+
3
+import (
4
+ "net/http"
5
+
6
+ "github.com/ipfs/go-ipfs/core/commands"
7
+)
8
+
9
+func init() {
10
+ http.HandleFunc("/debug/stack",
11
+ func(w http.ResponseWriter, _ *http.Request) {
12
+ _ = commands.WriteAllGoroutineStacks(w)
13
+ },
14
+ )
15
+}
core/commands/profile.go
+30
@@ -121,6 +121,25 @@ However, it could reveal:
121
},
122
}
123
124
+func WriteAllGoroutineStacks(w io.Writer) error {
125
+ // this is based on pprof.writeGoroutineStacks, and removes the 64 MB limit
126
+ buf := make([]byte, 1<<20)
127
+ for i := 0; ; i++ {
128
+ n := runtime.Stack(buf, true)
129
+ if n < len(buf) {
130
+ buf = buf[:n]
131
+ break
132
+ }
133
+ // if len(buf) >= 64<<20 {
134
+ // // Filled 64 MB - stop there.
135
+ // break
136
+ // }
137
+ buf = make([]byte, 2*len(buf))
138
+ }
139
+ _, err := w.Write(buf)
140
+ return err
141
+}
142
+
143
func writeProfiles(ctx context.Context, cpuProfileTime time.Duration, w io.Writer) error {
144
archive := zip.NewWriter(w)
145
@@ -143,6 +162,17 @@ func writeProfiles(ctx context.Context, cpuProfileTime time.Duration, w io.Write
162
file: "heap.pprof",
163
}}
164
165
+ {
166
+ out, err := archive.Create("goroutines-all.stacks")
167
+ if err != nil {
168
+ return err
169
+ }
170
+ err = WriteAllGoroutineStacks(out)
171
+ if err != nil {
172
+ return err
173
+ }
174
+ }
175
+
176
for _, profile := range profiles {
177
prof := pprof.Lookup(profile.name)
178
out, err := archive.Create(profile.file)
test/sharness/t0152-profile.sh
+4
@@ -61,4 +61,8 @@ test_expect_success "goroutines stacktrace is valid" '
61
grep -q "goroutine" "profiles/goroutines.stacks"
62
'
63
64
+test_expect_success "full goroutines stacktrace is valid" '
65
+ grep -q "goroutine" "profiles/goroutines-all.stacks"
66
+'
67
+
68
test_done