@cryptotaxi247 / kubo / commits / bb68a6852

feat: port collect-profiles.sh to 'ipfs diag profile' (#8786)

* feat: add block profiling to collect-profiles.sh * feat: add more profiles to 'ipfs diag profile' This adds mutex and block profiles, and brings the command up-to-par with 'collect-profiles.sh', so that we can remove it. Profiles are also now collected concurrently, which improves the runtime from (profile_time * num_profiles) to just (profile_time). Note that this has a backwards-incompatible change, removing --cpu-profile-time in favor of the more general --profile-time, which covers all sampling profiles. * docs(cli): ipfs diag profile * add CLI flag to select specific diag collectors Co-authored-by: Marcin Rataj <lidel@lidel.org>

Gus Eggert committed Apr 12, 2022 at 11:58 UTC bb68a685253f77fdf49475cb963ee1995c93657f
10 files changed +574 -248
bin/collect-profiles.sh deleted
-53
@@ -1,53 +0,0 @@
1 -#!/usr/bin/env bash
2 -
3 -# collect-profiles.sh
4 -#
5 -# Collects go profile information from a running `ipfs` daemon.
6 -# Creates an archive including the profiles, profile graph svgs,
7 -# ...and where available, a copy of the `ipfs` binary on the PATH.
8 -#
9 -# Please run this script and attach the profile archive it creates
10 -# when reporting bugs at https://github.com/ipfs/go-ipfs/issues
11 -
12 -set -euo pipefail
13 -IFS=$'\n\t'
14 -
15 -SOURCE_URL="${1:-http://127.0.0.1:5001}"
16 -tmpdir=$(mktemp -d)
17 -export PPROF_TMPDIR="$tmpdir"
18 -pushd "$tmpdir" > /dev/null
19 -
20 -if command -v ipfs > /dev/null 2>&1; then
21 - cp "$(command -v ipfs)" ipfs
22 -fi
23 -
24 -echo Collecting goroutine stacks
25 -curl -s -o goroutines.stacks "$SOURCE_URL"'/debug/pprof/goroutine?debug=2'
26 -
27 -curl -s -o goroutines.stacks.full "$SOURCE_URL"'/debug/stack'
28 -
29 -echo Collecting goroutine profile
30 -go tool pprof -symbolize=remote -svg -output goroutine.svg "$SOURCE_URL/debug/pprof/goroutine"
31 -
32 -echo Collecting heap profile
33 -go tool pprof -symbolize=remote -svg -output heap.svg "$SOURCE_URL/debug/pprof/heap"
34 -
35 -echo "Collecting cpu profile (~30s)"
36 -go tool pprof -symbolize=remote -svg -output cpu.svg "$SOURCE_URL/debug/pprof/profile"
37 -
38 -echo "Enabling mutex profiling"
39 -curl -X POST "$SOURCE_URL"'/debug/pprof-mutex/?fraction=4'
40 -
41 -echo "Waiting for mutex data to be updated (30s)"
42 -sleep 30
43 -curl -s -o mutex.txt "$SOURCE_URL"'/debug/pprof/mutex?debug=2'
44 -go tool pprof -symbolize=remote -svg -output mutex.svg "$SOURCE_URL/debug/pprof/mutex"
45 -
46 -echo "Disabling mutex profiling"
47 -curl -X POST "$SOURCE_URL"'/debug/pprof-mutex/?fraction=0'
48 -
49 -OUTPUT_NAME=ipfs-profile-$(uname -n)-$(date +'%Y-%m-%dT%H:%M:%S%z').tar.gz
50 -echo "Creating $OUTPUT_NAME"
51 -popd > /dev/null
52 -tar czf "./$OUTPUT_NAME" -C "$tmpdir" .
53 -rm -rf "$tmpdir"
cmd/ipfs/debug.go
+2 -2
@@ -3,13 +3,13 @@ package main
3 import (
4 "net/http"
5
6 - "github.com/ipfs/go-ipfs/core/commands"
6 + "github.com/ipfs/go-ipfs/profile"
7 )
8
9 func init() {
10 http.HandleFunc("/debug/stack",
11 func(w http.ResponseWriter, _ *http.Request) {
12 - _ = commands.WriteAllGoroutineStacks(w)
12 + _ = profile.WriteAllGoroutineStacks(w)
13 },
14 )
15 }
core/commands/profile.go
+54 -163
@@ -2,18 +2,15 @@ package commands
2
3 import (
4 "archive/zip"
5 - "context"
6 - "encoding/json"
5 "fmt"
6 "io"
7 "os"
10 - "runtime"
11 - "runtime/pprof"
8 "strings"
9 "time"
10
11 cmds "github.com/ipfs/go-ipfs-cmds"
12 "github.com/ipfs/go-ipfs/core/commands/e"
13 + "github.com/ipfs/go-ipfs/profile"
14 )
15
16 // time format that works in filenames on windows.
@@ -23,22 +20,27 @@ type profileResult struct {
20 File string
21 }
22
26 -const cpuProfileTimeOption = "cpu-profile-time"
23 +const (
24 + collectorsOptionName = "collectors"
25 + profileTimeOption = "profile-time"
26 + mutexProfileFractionOption = "mutex-profile-fraction"
27 + blockProfileRateOption = "block-profile-rate"
28 +)
29
30 var sysProfileCmd = &cmds.Command{
31 Helptext: cmds.HelpText{
32 Tagline: "Collect a performance profile for debugging.",
33 ShortDescription: `
32 -Collects cpu, heap, and goroutine profiles from a running go-ipfs daemon
33 -into a single zip file. To aid in debugging, this command also attempts to
34 -include a copy of the running go-ipfs binary.
34 +Collects profiles from a running go-ipfs daemon into a single zip file.
35 +To aid in debugging, this command also attempts to include a copy of
36 +the running go-ipfs binary.
37 `,
38 LongDescription: `
37 -Collects cpu, heap, and goroutine profiles from a running go-ipfs daemon
38 -into a single zipfile. To aid in debugging, this command also attempts to
39 -include a copy of the running go-ipfs binary.
39 +Collects profiles from a running go-ipfs daemon into a single zipfile.
40 +To aid in debugging, this command also attempts to include a copy of
41 +the running go-ipfs binary.
42
41 -Profile's can be examined using 'go tool pprof', some tips can be found at
43 +Profiles can be examined using 'go tool pprof', some tips can be found at
44 https://github.com/ipfs/go-ipfs/blob/master/docs/debug-guide.md.
45
46 Privacy Notice:
@@ -48,6 +50,8 @@ The output file includes:
50 - A list of running goroutines.
51 - A CPU profile.
52 - A heap profile.
53 +- A mutex profile.
54 +- A block profile.
55 - Your copy of go-ipfs.
56 - The output of 'ipfs version --all'.
57
@@ -68,19 +72,51 @@ However, it could reveal:
72 },
73 NoLocal: true,
74 Options: []cmds.Option{
71 - cmds.StringOption(outputOptionName, "o", "The path where the output should be stored."),
72 - cmds.StringOption(cpuProfileTimeOption, "The amount of time spent profiling CPU usage.").WithDefault("30s"),
75 + cmds.StringOption(outputOptionName, "o", "The path where the output .zip should be stored. Default: ./ipfs-profile-[timestamp].zip"),
76 + cmds.DelimitedStringsOption(",", collectorsOptionName, "The list of collectors to use for collecting diagnostic data.").
77 + WithDefault([]string{
78 + profile.CollectorGoroutinesStack,
79 + profile.CollectorGoroutinesPprof,
80 + profile.CollectorVersion,
81 + profile.CollectorHeap,
82 + profile.CollectorBin,
83 + profile.CollectorCPU,
84 + profile.CollectorMutex,
85 + profile.CollectorBlock,
86 + }),
87 + cmds.StringOption(profileTimeOption, "The amount of time spent profiling. If this is set to 0, then sampling profiles are skipped.").WithDefault("30s"),
88 + cmds.IntOption(mutexProfileFractionOption, "The fraction 1/n of mutex contention events that are reported in the mutex profile.").WithDefault(4),
89 + cmds.StringOption(blockProfileRateOption, "The duration to wait between sampling goroutine-blocking events for the blocking profile.").WithDefault("1ms"),
90 },
91 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
75 - cpuProfileTimeStr, _ := req.Options[cpuProfileTimeOption].(string)
76 - cpuProfileTime, err := time.ParseDuration(cpuProfileTimeStr)
92 + collectors := req.Options[collectorsOptionName].([]string)
93 +
94 + profileTimeStr, _ := req.Options[profileTimeOption].(string)
95 + profileTime, err := time.ParseDuration(profileTimeStr)
96 if err != nil {
78 - return fmt.Errorf("failed to parse CPU profile duration %q: %w", cpuProfileTimeStr, err)
97 + return fmt.Errorf("failed to parse profile duration %q: %w", profileTimeStr, err)
98 }
99
100 + blockProfileRateStr, _ := req.Options[blockProfileRateOption].(string)
101 + blockProfileRate, err := time.ParseDuration(blockProfileRateStr)
102 + if err != nil {
103 + return fmt.Errorf("failed to parse block profile rate %q: %w", blockProfileRateStr, err)
104 + }
105 +
106 + mutexProfileFraction, _ := req.Options[mutexProfileFractionOption].(int)
107 +
108 r, w := io.Pipe()
109 +
110 go func() {
83 - _ = w.CloseWithError(writeProfiles(req.Context, cpuProfileTime, w))
111 + archive := zip.NewWriter(w)
112 + err = profile.WriteProfiles(req.Context, archive, profile.Options{
113 + Collectors: collectors,
114 + ProfileDuration: profileTime,
115 + MutexProfileFraction: mutexProfileFraction,
116 + BlockProfileRate: blockProfileRate,
117 + })
118 + archive.Close()
119 + _ = w.CloseWithError(err)
120 }()
121 return res.Emit(r)
122 },
@@ -120,148 +156,3 @@ However, it could reveal:
156 }),
157 },
158 }
123 -
124 -func WriteAllGoroutineStacks(w io.Writer) error {
125 - // this is based on pprof.writeGoroutineStacks, and removes the 64 MB limit
126 - buf := make([]byte, 1<<20)
127 - for i := 0; ; i++ {
128 - n := runtime.Stack(buf, true)
129 - if n < len(buf) {
130 - buf = buf[:n]
131 - break
132 - }
133 - // if len(buf) >= 64<<20 {
134 - // // Filled 64 MB - stop there.
135 - // break
136 - // }
137 - buf = make([]byte, 2*len(buf))
138 - }
139 - _, err := w.Write(buf)
140 - return err
141 -}
142 -
143 -func writeProfiles(ctx context.Context, cpuProfileTime time.Duration, w io.Writer) error {
144 - archive := zip.NewWriter(w)
145 -
146 - // Take some profiles.
147 - type profile struct {
148 - name string
149 - file string
150 - debug int
151 - }
152 -
153 - profiles := []profile{{
154 - name: "goroutine",
155 - file: "goroutines.stacks",
156 - debug: 2,
157 - }, {
158 - name: "goroutine",
159 - file: "goroutines.pprof",
160 - }, {
161 - name: "heap",
162 - file: "heap.pprof",
163 - }}
164 -
165 - {
166 - out, err := archive.Create("goroutines-all.stacks")
167 - if err != nil {
168 - return err
169 - }
170 - err = WriteAllGoroutineStacks(out)
171 - if err != nil {
172 - return err
173 - }
174 - }
175 -
176 - for _, profile := range profiles {
177 - prof := pprof.Lookup(profile.name)
178 - out, err := archive.Create(profile.file)
179 - if err != nil {
180 - return err
181 - }
182 - err = prof.WriteTo(out, profile.debug)
183 - if err != nil {
184 - return err
185 - }
186 - }
187 -
188 - // Take a CPU profile.
189 - if cpuProfileTime != 0 {
190 - out, err := archive.Create("cpu.pprof")
191 - if err != nil {
192 - return err
193 - }
194 -
195 - err = writeCPUProfile(ctx, cpuProfileTime, out)
196 - if err != nil {
197 - return err
198 - }
199 - }
200 -
201 - // Collect version info
202 - // I'd use diag sysinfo, but that includes some more sensitive information
203 - // (GOPATH, etc.).
204 - {
205 - out, err := archive.Create("version.json")
206 - if err != nil {
207 - return err
208 - }
209 -
210 - err = json.NewEncoder(out).Encode(getVersionInfo())
211 - if err != nil {
212 - return err
213 - }
214 - }
215 -
216 - // Collect binary
217 - if fi, err := openIPFSBinary(); err == nil {
218 - fname := "ipfs"
219 - if runtime.GOOS == "windows" {
220 - fname += ".exe"
221 - }
222 -
223 - out, err := archive.Create(fname)
224 - if err != nil {
225 - return err
226 - }
227 -
228 - _, err = io.Copy(out, fi)
229 - _ = fi.Close()
230 - if err != nil {
231 - return err
232 - }
233 - }
234 - return archive.Close()
235 -}
236 -
237 -func writeCPUProfile(ctx context.Context, d time.Duration, w io.Writer) error {
238 - if err := pprof.StartCPUProfile(w); err != nil {
239 - return err
240 - }
241 - defer pprof.StopCPUProfile()
242 -
243 - timer := time.NewTimer(d)
244 - defer timer.Stop()
245 -
246 - select {
247 - case <-timer.C:
248 - case <-ctx.Done():
249 - return ctx.Err()
250 - }
251 - return nil
252 -}
253 -
254 -func openIPFSBinary() (*os.File, error) {
255 - if runtime.GOOS == "linux" {
256 - pid := os.Getpid()
257 - fi, err := os.Open(fmt.Sprintf("/proc/%d/exe", pid))
258 - if err == nil {
259 - return fi, nil
260 - }
261 - }
262 - path, err := os.Executable()
263 - if err != nil {
264 - return nil, err
265 - }
266 - return os.Open(path)
267 -}
core/commands/root.go
+1 -1
@@ -67,13 +67,13 @@ NETWORK COMMANDS
67 swarm Manage connections to the p2p network
68 dht Query the DHT for values or peers
69 ping Measure the latency of a connection
70 - diag Print diagnostics
70 bitswap Inspect bitswap state
71 pubsub Send and receive messages via pubsub
72
73 TOOL COMMANDS
74 config Manage configuration
75 version Show IPFS version information
76 + diag Generate diagnostic reports
77 update Download and apply go-ipfs updates
78 commands List all available commands
79 log Manage and show logs of running daemon
core/commands/version.go
+3 -23
@@ -4,23 +4,13 @@ import (
4 "errors"
5 "fmt"
6 "io"
7 - "runtime"
7 "runtime/debug"
8
9 version "github.com/ipfs/go-ipfs"
11 - fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
10
11 cmds "github.com/ipfs/go-ipfs-cmds"
12 )
13
16 -type VersionOutput struct {
17 - Version string
18 - Commit string
19 - Repo string
20 - System string
21 - Golang string
22 -}
23 -
14 const (
15 versionNumberOptionName = "number"
16 versionCommitOptionName = "commit"
@@ -28,16 +18,6 @@ const (
18 versionAllOptionName = "all"
19 )
20
31 -func getVersionInfo() *VersionOutput {
32 - return &VersionOutput{
33 - Version: version.CurrentVersionNumber,
34 - Commit: version.CurrentCommit,
35 - Repo: fmt.Sprint(fsrepo.RepoVersion),
36 - System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here
37 - Golang: runtime.Version(),
38 - }
39 -}
40 -
21 var VersionCmd = &cmds.Command{
22 Helptext: cmds.HelpText{
23 Tagline: "Show IPFS version information.",
@@ -56,10 +36,10 @@ var VersionCmd = &cmds.Command{
36 // must be permitted to run before init
37 Extra: CreateCmdExtras(SetDoesNotUseRepo(true), SetDoesNotUseConfigAsInput(true)),
38 Run: func(req *cmds.Request, res cmds.ResponseEmitter, env cmds.Environment) error {
59 - return cmds.EmitOnce(res, getVersionInfo())
39 + return cmds.EmitOnce(res, version.GetVersionInfo())
40 },
41 Encoders: cmds.EncoderMap{
62 - cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, version *VersionOutput) error {
42 + cmds.Text: cmds.MakeTypedEncoder(func(req *cmds.Request, w io.Writer, version *version.VersionInfo) error {
43 all, _ := req.Options[versionAllOptionName].(bool)
44 if all {
45 ver := version.Version
@@ -95,7 +75,7 @@ var VersionCmd = &cmds.Command{
75 return nil
76 }),
77 },
98 - Type: VersionOutput{},
78 + Type: version.VersionInfo{},
79 }
80
81 type Dependency struct {
profile/goroutines.go new
+27
@@ -0,0 +1,27 @@
1 +package profile
2 +
3 +import (
4 + "io"
5 + "runtime"
6 +)
7 +
8 +// WriteAllGoroutineStacks writes a stack trace to the given writer.
9 +// This is distinct from the Go-provided method because it does not truncate after 64 MB.
10 +func WriteAllGoroutineStacks(w io.Writer) error {
11 + // this is based on pprof.writeGoroutineStacks, and removes the 64 MB limit
12 + buf := make([]byte, 1<<20)
13 + for i := 0; ; i++ {
14 + n := runtime.Stack(buf, true)
15 + if n < len(buf) {
16 + buf = buf[:n]
17 + break
18 + }
19 + // if len(buf) >= 64<<20 {
20 + // // Filled 64 MB - stop there.
21 + // break
22 + // }
23 + buf = make([]byte, 2*len(buf))
24 + }
25 + _, err := w.Write(buf)
26 + return err
27 +}
profile/profile.go new
+268
@@ -0,0 +1,268 @@
1 +package profile
2 +
3 +import (
4 + "archive/zip"
5 + "bytes"
6 + "context"
7 + "encoding/json"
8 + "fmt"
9 + "io"
10 + "os"
11 + "runtime"
12 + "runtime/pprof"
13 + "sync"
14 + "time"
15 +
16 + version "github.com/ipfs/go-ipfs"
17 + "github.com/ipfs/go-log"
18 +)
19 +
20 +const (
21 + CollectorGoroutinesStack = "goroutines-stack"
22 + CollectorGoroutinesPprof = "goroutines-pprof"
23 + CollectorVersion = "version"
24 + CollectorHeap = "heap"
25 + CollectorBin = "bin"
26 + CollectorCPU = "cpu"
27 + CollectorMutex = "mutex"
28 + CollectorBlock = "block"
29 +)
30 +
31 +var (
32 + logger = log.Logger("profile")
33 + goos = runtime.GOOS
34 +)
35 +
36 +type collector struct {
37 + outputFile string
38 + isExecutable bool
39 + collectFunc func(ctx context.Context, opts Options, writer io.Writer) error
40 + enabledFunc func(opts Options) bool
41 +}
42 +
43 +func (p *collector) outputFileName() string {
44 + fName := p.outputFile
45 + if p.isExecutable {
46 + if goos == "windows" {
47 + fName += ".exe"
48 + }
49 + }
50 + return fName
51 +}
52 +
53 +var collectors = map[string]collector{
54 + CollectorGoroutinesStack: {
55 + outputFile: "goroutines.stacks",
56 + collectFunc: goroutineStacksText,
57 + enabledFunc: func(opts Options) bool { return true },
58 + },
59 + CollectorGoroutinesPprof: {
60 + outputFile: "goroutines.pprof",
61 + collectFunc: goroutineStacksProto,
62 + enabledFunc: func(opts Options) bool { return true },
63 + },
64 + CollectorVersion: {
65 + outputFile: "version.json",
66 + collectFunc: versionInfo,
67 + enabledFunc: func(opts Options) bool { return true },
68 + },
69 + CollectorHeap: {
70 + outputFile: "heap.pprof",
71 + collectFunc: heapProfile,
72 + enabledFunc: func(opts Options) bool { return true },
73 + },
74 + CollectorBin: {
75 + outputFile: "ipfs",
76 + isExecutable: true,
77 + collectFunc: binary,
78 + enabledFunc: func(opts Options) bool { return true },
79 + },
80 + CollectorCPU: {
81 + outputFile: "cpu.pprof",
82 + collectFunc: profileCPU,
83 + enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 },
84 + },
85 + CollectorMutex: {
86 + outputFile: "mutex.pprof",
87 + collectFunc: mutexProfile,
88 + enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 && opts.MutexProfileFraction > 0 },
89 + },
90 + CollectorBlock: {
91 + outputFile: "block.pprof",
92 + collectFunc: blockProfile,
93 + enabledFunc: func(opts Options) bool { return opts.ProfileDuration > 0 && opts.BlockProfileRate > 0 },
94 + },
95 +}
96 +
97 +type Options struct {
98 + Collectors []string
99 + ProfileDuration time.Duration
100 + MutexProfileFraction int
101 + BlockProfileRate time.Duration
102 +}
103 +
104 +func WriteProfiles(ctx context.Context, archive *zip.Writer, opts Options) error {
105 + p := profiler{
106 + archive: archive,
107 + opts: opts,
108 + }
109 + return p.runProfile(ctx)
110 +}
111 +
112 +// profiler runs the collectors concurrently and writes the results to the zip archive.
113 +type profiler struct {
114 + archive *zip.Writer
115 + opts Options
116 +}
117 +
118 +func (p *profiler) runProfile(ctx context.Context) error {
119 + type profileResult struct {
120 + fName string
121 + buf *bytes.Buffer
122 + err error
123 + }
124 +
125 + ctx, cancelFn := context.WithCancel(ctx)
126 + defer cancelFn()
127 +
128 + var collectorsToRun []collector
129 + for _, name := range p.opts.Collectors {
130 + c, ok := collectors[name]
131 + if !ok {
132 + return fmt.Errorf("unknown collector '%s'", name)
133 + }
134 + collectorsToRun = append(collectorsToRun, c)
135 + }
136 +
137 + results := make(chan profileResult, len(p.opts.Collectors))
138 + wg := sync.WaitGroup{}
139 + for _, c := range collectorsToRun {
140 + if !c.enabledFunc(p.opts) {
141 + continue
142 + }
143 +
144 + fName := c.outputFileName()
145 +
146 + wg.Add(1)
147 + go func(c collector) {
148 + defer wg.Done()
149 + logger.Infow("collecting profile", "File", fName)
150 + defer logger.Infow("profile done", "File", fName)
151 + b := bytes.Buffer{}
152 + err := c.collectFunc(ctx, p.opts, &b)
153 + if err != nil {
154 + select {
155 + case results <- profileResult{err: fmt.Errorf("generating profile data for %q: %w", fName, err)}:
156 + case <-ctx.Done():
157 + return
158 + }
159 + }
160 + select {
161 + case results <- profileResult{buf: &b, fName: fName}:
162 + case <-ctx.Done():
163 + }
164 + }(c)
165 + }
166 + go func() {
167 + wg.Wait()
168 + close(results)
169 + }()
170 +
171 + for res := range results {
172 + if res.err != nil {
173 + return res.err
174 + }
175 + out, err := p.archive.Create(res.fName)
176 + if err != nil {
177 + return fmt.Errorf("creating output file %q: %w", res.fName, err)
178 + }
179 + _, err = io.Copy(out, res.buf)
180 + if err != nil {
181 + return fmt.Errorf("compressing result %q: %w", res.fName, err)
182 + }
183 + }
184 +
185 + return nil
186 +}
187 +
188 +func goroutineStacksText(ctx context.Context, _ Options, w io.Writer) error {
189 + return WriteAllGoroutineStacks(w)
190 +}
191 +
192 +func goroutineStacksProto(ctx context.Context, _ Options, w io.Writer) error {
193 + return pprof.Lookup("goroutine").WriteTo(w, 0)
194 +}
195 +
196 +func heapProfile(ctx context.Context, _ Options, w io.Writer) error {
197 + return pprof.Lookup("heap").WriteTo(w, 0)
198 +}
199 +
200 +func versionInfo(ctx context.Context, _ Options, w io.Writer) error {
201 + return json.NewEncoder(w).Encode(version.GetVersionInfo())
202 +}
203 +
204 +func binary(ctx context.Context, _ Options, w io.Writer) error {
205 + var (
206 + path string
207 + err error
208 + )
209 + if goos == "linux" {
210 + pid := os.Getpid()
211 + path = fmt.Sprintf("/proc/%d/exe", pid)
212 + } else {
213 + path, err = os.Executable()
214 + if err != nil {
215 + return fmt.Errorf("finding binary path: %w", err)
216 + }
217 + }
218 + fi, err := os.Open(path)
219 + if err != nil {
220 + return fmt.Errorf("opening binary %q: %w", path, err)
221 + }
222 + _, err = io.Copy(w, fi)
223 + _ = fi.Close()
224 + if err != nil {
225 + return fmt.Errorf("copying binary %q: %w", path, err)
226 + }
227 + return nil
228 +}
229 +
230 +func mutexProfile(ctx context.Context, opts Options, w io.Writer) error {
231 + prev := runtime.SetMutexProfileFraction(opts.MutexProfileFraction)
232 + defer runtime.SetMutexProfileFraction(prev)
233 + err := waitOrCancel(ctx, opts.ProfileDuration)
234 + if err != nil {
235 + return err
236 + }
237 + return pprof.Lookup("mutex").WriteTo(w, 2)
238 +}
239 +
240 +func blockProfile(ctx context.Context, opts Options, w io.Writer) error {
241 + runtime.SetBlockProfileRate(int(opts.BlockProfileRate.Nanoseconds()))
242 + defer runtime.SetBlockProfileRate(0)
243 + err := waitOrCancel(ctx, opts.ProfileDuration)
244 + if err != nil {
245 + return err
246 + }
247 + return pprof.Lookup("block").WriteTo(w, 2)
248 +}
249 +
250 +func profileCPU(ctx context.Context, opts Options, w io.Writer) error {
251 + err := pprof.StartCPUProfile(w)
252 + if err != nil {
253 + return err
254 + }
255 + defer pprof.StopCPUProfile()
256 + return waitOrCancel(ctx, opts.ProfileDuration)
257 +}
258 +
259 +func waitOrCancel(ctx context.Context, d time.Duration) error {
260 + timer := time.NewTimer(d)
261 + defer timer.Stop()
262 + select {
263 + case <-timer.C:
264 + return nil
265 + case <-ctx.Done():
266 + return ctx.Err()
267 + }
268 +}
profile/profile_test.go new
+172
@@ -0,0 +1,172 @@
1 +package profile
2 +
3 +import (
4 + "archive/zip"
5 + "bytes"
6 + "context"
7 + "testing"
8 + "time"
9 +
10 + "github.com/stretchr/testify/assert"
11 + "github.com/stretchr/testify/require"
12 +)
13 +
14 +func TestProfiler(t *testing.T) {
15 + allCollectors := []string{
16 + CollectorGoroutinesStack,
17 + CollectorGoroutinesPprof,
18 + CollectorVersion,
19 + CollectorHeap,
20 + CollectorBin,
21 + CollectorCPU,
22 + CollectorMutex,
23 + CollectorBlock,
24 + }
25 +
26 + cases := []struct {
27 + name string
28 + opts Options
29 + goos string
30 +
31 + expectFiles []string
32 + }{
33 + {
34 + name: "happy case",
35 + opts: Options{
36 + Collectors: allCollectors,
37 + ProfileDuration: 1 * time.Millisecond,
38 + MutexProfileFraction: 4,
39 + BlockProfileRate: 50 * time.Nanosecond,
40 + },
41 + expectFiles: []string{
42 + "goroutines.stacks",
43 + "goroutines.pprof",
44 + "version.json",
45 + "heap.pprof",
46 + "ipfs",
47 + "cpu.pprof",
48 + "mutex.pprof",
49 + "block.pprof",
50 + },
51 + },
52 + {
53 + name: "windows",
54 + opts: Options{
55 + Collectors: allCollectors,
56 + ProfileDuration: 1 * time.Millisecond,
57 + MutexProfileFraction: 4,
58 + BlockProfileRate: 50 * time.Nanosecond,
59 + },
60 + goos: "windows",
61 + expectFiles: []string{
62 + "goroutines.stacks",
63 + "goroutines.pprof",
64 + "version.json",
65 + "heap.pprof",
66 + "ipfs.exe",
67 + "cpu.pprof",
68 + "mutex.pprof",
69 + "block.pprof",
70 + },
71 + },
72 + {
73 + name: "sampling profiling disabled",
74 + opts: Options{
75 + Collectors: allCollectors,
76 + MutexProfileFraction: 4,
77 + BlockProfileRate: 50 * time.Nanosecond,
78 + },
79 + expectFiles: []string{
80 + "goroutines.stacks",
81 + "goroutines.pprof",
82 + "version.json",
83 + "heap.pprof",
84 + "ipfs",
85 + },
86 + },
87 + {
88 + name: "Mutex profiling disabled",
89 + opts: Options{
90 + Collectors: allCollectors,
91 + ProfileDuration: 1 * time.Millisecond,
92 + BlockProfileRate: 50 * time.Nanosecond,
93 + },
94 + expectFiles: []string{
95 + "goroutines.stacks",
96 + "goroutines.pprof",
97 + "version.json",
98 + "heap.pprof",
99 + "ipfs",
100 + "cpu.pprof",
101 + "block.pprof",
102 + },
103 + },
104 + {
105 + name: "block profiling disabled",
106 + opts: Options{
107 + Collectors: allCollectors,
108 + ProfileDuration: 1 * time.Millisecond,
109 + MutexProfileFraction: 4,
110 + BlockProfileRate: 0,
111 + },
112 + expectFiles: []string{
113 + "goroutines.stacks",
114 + "goroutines.pprof",
115 + "version.json",
116 + "heap.pprof",
117 + "ipfs",
118 + "cpu.pprof",
119 + "mutex.pprof",
120 + },
121 + },
122 + {
123 + name: "single collector",
124 + opts: Options{
125 + Collectors: []string{CollectorVersion},
126 + ProfileDuration: 1 * time.Millisecond,
127 + MutexProfileFraction: 4,
128 + BlockProfileRate: 0,
129 + },
130 + expectFiles: []string{
131 + "version.json",
132 + },
133 + },
134 + }
135 + for _, c := range cases {
136 + t.Run(c.name, func(t *testing.T) {
137 + if c.goos != "" {
138 + oldGOOS := goos
139 + goos = c.goos
140 + defer func() { goos = oldGOOS }()
141 + }
142 +
143 + buf := &bytes.Buffer{}
144 + archive := zip.NewWriter(buf)
145 + err := WriteProfiles(context.Background(), archive, c.opts)
146 + require.NoError(t, err)
147 +
148 + err = archive.Close()
149 + require.NoError(t, err)
150 +
151 + zr, err := zip.NewReader(bytes.NewReader(buf.Bytes()), int64(buf.Len()))
152 + require.NoError(t, err)
153 +
154 + for _, f := range zr.File {
155 + logger.Info("zip file: ", f.Name)
156 + }
157 +
158 + require.Equal(t, len(c.expectFiles), len(zr.File))
159 +
160 + for _, expectedFile := range c.expectFiles {
161 + func() {
162 + f, err := zr.Open(expectedFile)
163 + require.NoError(t, err)
164 + defer f.Close()
165 + fi, err := f.Stat()
166 + require.NoError(t, err)
167 + assert.NotZero(t, fi.Size())
168 + }()
169 + }
170 + })
171 + }
172 +}
test/sharness/t0152-profile.sh
+22 -6
@@ -16,8 +16,8 @@ test_expect_success "profiling requires a running daemon" '
16
17 test_launch_ipfs_daemon
18
19 -test_expect_success "test profiling (without CPU)" '
20 - ipfs diag profile --cpu-profile-time=0 > cmd_out
19 +test_expect_success "test profiling (without sampling)" '
20 + ipfs diag profile --profile-time=0 > cmd_out
21 '
22
23 test_expect_success "filename shows up in output" '
@@ -29,12 +29,17 @@ test_expect_success "profile file created" '
29 '
30
31 test_expect_success "test profiling with -o" '
32 - ipfs diag profile --cpu-profile-time=1s -o test-profile.zip
32 + ipfs diag profile --profile-time=1s -o test-profile.zip
33 '
34
35 test_expect_success "test that test-profile.zip exists" '
36 test -e test-profile.zip
37 '
38 +
39 +test_expect_success "test profiling with specific collectors" '
40 + ipfs diag profile --collectors version,goroutines-stack -o test-profile-small.zip
41 +'
42 +
43 test_kill_ipfs_daemon
44
45 if ! test_have_prereq UNZIP; then
@@ -42,7 +47,8 @@ if ! test_have_prereq UNZIP; then
47 fi
48
49 test_expect_success "unpack profiles" '
45 - unzip -d profiles test-profile.zip
50 + unzip -d profiles test-profile.zip &&
51 + unzip -d profiles-small test-profile-small.zip
52 '
53
54 test_expect_success "cpu profile is valid" '
@@ -57,12 +63,22 @@ test_expect_success "goroutines profile is valid" '
63 go tool pprof -top profiles/ipfs "profiles/goroutines.pprof" | grep -q "Type: goroutine"
64 '
65
66 +test_expect_success "mutex profile is valid" '
67 + go tool pprof -top profiles/ipfs "profiles/mutex.pprof" | grep -q "Type: delay"
68 +'
69 +
70 +test_expect_success "block profile is valid" '
71 + go tool pprof -top profiles/ipfs "profiles/block.pprof" | grep -q "Type: delay"
72 +'
73 +
74 test_expect_success "goroutines stacktrace is valid" '
75 grep -q "goroutine" "profiles/goroutines.stacks"
76 '
77
64 -test_expect_success "full goroutines stacktrace is valid" '
65 - grep -q "goroutine" "profiles/goroutines-all.stacks"
78 +test_expect_success "the small profile only contains the requested data" '
79 + find profiles-small -type f | sort > actual &&
80 + echo -e "profiles-small/goroutines.stacks\nprofiles-small/version.json" > expected &&
81 + test_cmp expected actual
82 '
83
84 test_done
version.go
+25
@@ -1,5 +1,12 @@
1 package ipfs
2
3 +import (
4 + "fmt"
5 + "runtime"
6 +
7 + "github.com/ipfs/go-ipfs/repo/fsrepo"
8 +)
9 +
10 // CurrentCommit is the current git commit, this is set as a ldflag in the Makefile
11 var CurrentCommit string
12
@@ -27,3 +34,21 @@ var userAgentSuffix string
34 func SetUserAgentSuffix(suffix string) {
35 userAgentSuffix = suffix
36 }
37 +
38 +type VersionInfo struct {
39 + Version string
40 + Commit string
41 + Repo string
42 + System string
43 + Golang string
44 +}
45 +
46 +func GetVersionInfo() *VersionInfo {
47 + return &VersionInfo{
48 + Version: CurrentVersionNumber,
49 + Commit: CurrentCommit,
50 + Repo: fmt.Sprint(fsrepo.RepoVersion),
51 + System: runtime.GOARCH + "/" + runtime.GOOS, //TODO: Precise version here
52 + Golang: runtime.Version(),
53 + }
54 +}