fix prometheus concurrent map write bug
fixes #4132 License: MIT Signed-off-by: Steven Allen <steven@stebalien.com>
Steven Allen committed
Oct 30, 2018 at 15:38 UTC
4674f199133c029189db72cfcea1e72adcb91af6
3 files changed
+76
-9
cmd/ipfs/daemon.go
+2
-2
@@ -23,11 +23,11 @@ import (
23
fsrepo "github.com/ipfs/go-ipfs/repo/fsrepo"
24
migrate "github.com/ipfs/go-ipfs/repo/fsrepo/migrations"
25
26
+ mprome "gx/ipfs/QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T/go-metrics-prometheus"
27
cmds "gx/ipfs/QmSXUokcP4TJpFfqozT69AVAYRtzXVMUjzQVkYX41R9Svs/go-ipfs-cmds"
28
ma "gx/ipfs/QmT4U94DnD8FRfqr21obWY32HLM5VExccPKMjQHofeYqr9/go-multiaddr"
28
- "gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus"
29
+ "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus"
30
"gx/ipfs/Qmaabb1tJZ2CX5cp6MuuiGgns71NYoxdgQP6Xdid1dVceC/go-multiaddr-net"
30
- mprome "gx/ipfs/QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL/go-metrics-prometheus"
31
"gx/ipfs/Qmde5VP1qUkyQXKCfmEUA7bP64V2HAptbJ7phuPp7jXWwg/go-ipfs-cmdkit"
32
)
33
core/corehttp/metrics.go
+70
-3
@@ -6,13 +6,14 @@ import (
6
7
core "github.com/ipfs/go-ipfs/core"
8
9
- prometheus "gx/ipfs/QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH/client_golang/prometheus"
9
+ prometheus "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus"
10
+ promhttp "gx/ipfs/QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS/client_golang/prometheus/promhttp"
11
)
12
13
// This adds the scraping endpoint which Prometheus uses to fetch metrics.
14
func MetricsScrapingOption(path string) ServeOption {
15
return func(n *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
15
- mux.Handle(path, prometheus.UninstrumentedHandler())
16
+ mux.Handle(path, promhttp.HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{}))
17
return mux, nil
18
}
19
}
@@ -20,8 +21,74 @@ func MetricsScrapingOption(path string) ServeOption {
21
// This adds collection of net/http-related metrics
22
func MetricsCollectionOption(handlerName string) ServeOption {
23
return func(_ *core.IpfsNode, _ net.Listener, mux *http.ServeMux) (*http.ServeMux, error) {
24
+ // Adapted from github.com/prometheus/client_golang/prometheus/http.go
25
+ // Work around https://github.com/prometheus/client_golang/pull/311
26
+ opts := prometheus.SummaryOpts{
27
+ Subsystem: "http",
28
+ ConstLabels: prometheus.Labels{"handler": handlerName},
29
+ Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001},
30
+ }
31
+
32
+ reqCnt := prometheus.NewCounterVec(
33
+ prometheus.CounterOpts{
34
+ Namespace: opts.Namespace,
35
+ Subsystem: opts.Subsystem,
36
+ Name: "requests_total",
37
+ Help: "Total number of HTTP requests made.",
38
+ ConstLabels: opts.ConstLabels,
39
+ },
40
+ []string{"method", "code"},
41
+ )
42
+ if err := prometheus.Register(reqCnt); err != nil {
43
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
44
+ reqCnt = are.ExistingCollector.(*prometheus.CounterVec)
45
+ } else {
46
+ return nil, err
47
+ }
48
+ }
49
+
50
+ opts.Name = "request_duration_microseconds"
51
+ opts.Help = "The HTTP request latencies in microseconds."
52
+ reqDur := prometheus.NewSummaryVec(opts, nil)
53
+ if err := prometheus.Register(reqDur); err != nil {
54
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
55
+ reqDur = are.ExistingCollector.(*prometheus.SummaryVec)
56
+ } else {
57
+ return nil, err
58
+ }
59
+ }
60
+
61
+ opts.Name = "request_size_bytes"
62
+ opts.Help = "The HTTP request sizes in bytes."
63
+ reqSz := prometheus.NewSummaryVec(opts, nil)
64
+ if err := prometheus.Register(reqSz); err != nil {
65
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
66
+ reqSz = are.ExistingCollector.(*prometheus.SummaryVec)
67
+ } else {
68
+ return nil, err
69
+ }
70
+ }
71
+
72
+ opts.Name = "response_size_bytes"
73
+ opts.Help = "The HTTP response sizes in bytes."
74
+ resSz := prometheus.NewSummaryVec(opts, nil)
75
+ if err := prometheus.Register(resSz); err != nil {
76
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
77
+ resSz = are.ExistingCollector.(*prometheus.SummaryVec)
78
+ } else {
79
+ return nil, err
80
+ }
81
+ }
82
+
83
+ // Construct the mux
84
childMux := http.NewServeMux()
24
- mux.HandleFunc("/", prometheus.InstrumentHandler(handlerName, childMux))
85
+ var promMux http.Handler = childMux
86
+ promMux = promhttp.InstrumentHandlerResponseSize(resSz, promMux)
87
+ promMux = promhttp.InstrumentHandlerRequestSize(reqSz, promMux)
88
+ promMux = promhttp.InstrumentHandlerDuration(reqDur, promMux)
89
+ promMux = promhttp.InstrumentHandlerCounter(reqCnt, promMux)
90
+ mux.Handle("/", promMux)
91
+
92
return childMux, nil
93
}
94
}
package.json
+4
-4
@@ -42,9 +42,9 @@
42
"version": "0.0.0"
43
},
44
{
45
- "hash": "QmYYv3QFnfQbiwmi1tpkgKF8o4xFnZoBrvpupTiGJwL9nH",
45
+ "hash": "QmTQuFQWHAWy4wMH6ZyPfGiawA5u9T8rs79FENoV8yXaoS",
46
"name": "client_golang",
47
- "version": "0.1.3"
47
+ "version": "0.1.4"
48
},
49
{
50
"hash": "QmV1DPm5F46LvQMxCVPhu35zHgZEeMvyVtpxjb5TwfGiua",
@@ -160,9 +160,9 @@
160
},
161
{
162
"author": "ipfs",
163
- "hash": "QmbqQP7GMwJCePfEvbMSZmyPifAz1kKZifo8vwVnVHt1wL",
163
+ "hash": "QmQXBfkuwgMaPx334WuL9NmyrKnbZ5udaWnHTHEsts2x3T",
164
"name": "go-metrics-prometheus",
165
- "version": "0.3.9"
165
+ "version": "0.3.10"
166
},
167
{
168
"author": "ipfs",