feat: add gateway histogram metrics (#8443)
* feat(gw): response type histogram metrics - response-type agnostic firstContentBlockGetMetric which counts the latency til the first content block. - car/block/file/gen-dir-index duration histogram metrics that show how long each response type takes * docs: improve metrics descriptions * feat: more gw histogram buckets 0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60 secs as suggested in reviews at https://github.com/ipfs/go-ipfs/pull/8443 Co-authored-by: Marcin Rataj <lidel@lidel.org> Co-authored-by: Gus Eggert <gus@gus.dev>
Adin Schmahmann committed
Mar 21, 2022 at 10:57 UTC
beaa8fc29b472214283b9aab884ed92f03908d13
6 files changed
+116
-24
core/corehttp/gateway_handler.go
+91
-16
@@ -62,7 +62,15 @@ type gatewayHandler struct {
62
config GatewayConfig
63
api coreiface.CoreAPI
64
65
- unixfsGetMetric *prometheus.SummaryVec
65
+ // generic metrics
66
+ firstContentBlockGetMetric *prometheus.HistogramVec
67
+ unixfsGetMetric *prometheus.SummaryVec // deprecated, use firstContentBlockGetMetric
68
+
69
+ // response type metrics
70
+ unixfsFileGetMetric *prometheus.HistogramVec
71
+ unixfsGenDirGetMetric *prometheus.HistogramVec
72
+ carStreamGetMetric *prometheus.HistogramVec
73
+ rawBlockGetMetric *prometheus.HistogramVec
74
}
75
76
// StatusResponseWriter enables us to override HTTP Status Code passed to
@@ -85,29 +93,93 @@ func (sw *statusResponseWriter) WriteHeader(code int) {
93
sw.ResponseWriter.WriteHeader(code)
94
}
95
88
-func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
89
- unixfsGetMetric := prometheus.NewSummaryVec(
90
- // TODO: deprecate and switch to content type agnostic metrics: https://github.com/ipfs/go-ipfs/issues/8441
96
+func newGatewaySummaryMetric(name string, help string) *prometheus.SummaryVec {
97
+ summaryMetric := prometheus.NewSummaryVec(
98
prometheus.SummaryOpts{
99
Namespace: "ipfs",
100
Subsystem: "http",
94
- Name: "unixfs_get_latency_seconds",
95
- Help: "The time till the first block is received when 'getting' a file from the gateway.",
101
+ Name: name,
102
+ Help: help,
103
+ },
104
+ []string{"gateway"},
105
+ )
106
+ if err := prometheus.Register(summaryMetric); err != nil {
107
+ if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
108
+ summaryMetric = are.ExistingCollector.(*prometheus.SummaryVec)
109
+ } else {
110
+ log.Errorf("failed to register ipfs_http_%s: %v", name, err)
111
+ }
112
+ }
113
+ return summaryMetric
114
+}
115
+
116
+func newGatewayHistogramMetric(name string, help string) *prometheus.HistogramVec {
117
+ // We can add buckets as a parameter in the future, but for now using static defaults
118
+ // suggested in https://github.com/ipfs/go-ipfs/issues/8441
119
+ defaultBuckets := []float64{0.05, 0.1, 0.25, 0.5, 1, 2, 5, 10, 30, 60}
120
+ histogramMetric := prometheus.NewHistogramVec(
121
+ prometheus.HistogramOpts{
122
+ Namespace: "ipfs",
123
+ Subsystem: "http",
124
+ Name: name,
125
+ Help: help,
126
+ Buckets: defaultBuckets,
127
},
128
[]string{"gateway"},
129
)
99
- if err := prometheus.Register(unixfsGetMetric); err != nil {
130
+ if err := prometheus.Register(histogramMetric); err != nil {
131
if are, ok := err.(prometheus.AlreadyRegisteredError); ok {
101
- unixfsGetMetric = are.ExistingCollector.(*prometheus.SummaryVec)
132
+ histogramMetric = are.ExistingCollector.(*prometheus.HistogramVec)
133
} else {
103
- log.Errorf("failed to register unixfsGetMetric: %v", err)
134
+ log.Errorf("failed to register ipfs_http_%s: %v", name, err)
135
}
136
}
137
+ return histogramMetric
138
+}
139
140
+func newGatewayHandler(c GatewayConfig, api coreiface.CoreAPI) *gatewayHandler {
141
i := &gatewayHandler{
108
- config: c,
109
- api: api,
110
- unixfsGetMetric: unixfsGetMetric,
142
+ config: c,
143
+ api: api,
144
+ // Improved Metrics
145
+ // ----------------------------
146
+ // Time till the first content block (bar in /ipfs/cid/foo/bar)
147
+ // (format-agnostic, across all response types)
148
+ firstContentBlockGetMetric: newGatewayHistogramMetric(
149
+ "gw_first_content_block_get_latency_seconds",
150
+ "The time till the first content block is received on GET from the gateway.",
151
+ ),
152
+
153
+ // Response-type specific metrics
154
+ // ----------------------------
155
+ // UnixFS: time it takes to return a file
156
+ unixfsFileGetMetric: newGatewayHistogramMetric(
157
+ "gw_unixfs_file_get_duration_seconds",
158
+ "The time to serve an entire UnixFS file from the gateway.",
159
+ ),
160
+ // UnixFS: time it takes to generate static HTML with directory listing
161
+ unixfsGenDirGetMetric: newGatewayHistogramMetric(
162
+ "gw_unixfs_gen_dir_listing_get_duration_seconds",
163
+ "The time to serve a generated UnixFS HTML directory listing from the gateway.",
164
+ ),
165
+ // CAR: time it takes to return requested CAR stream
166
+ carStreamGetMetric: newGatewayHistogramMetric(
167
+ "gw_car_stream_get_duration_seconds",
168
+ "The time to GET an entire CAR stream from the gateway.",
169
+ ),
170
+ // Block: time it takes to return requested Block
171
+ rawBlockGetMetric: newGatewayHistogramMetric(
172
+ "gw_raw_block_get_duration_seconds",
173
+ "The time to GET an entire raw Block from the gateway.",
174
+ ),
175
+
176
+ // Legacy Metrics
177
+ // ----------------------------
178
+ unixfsGetMetric: newGatewaySummaryMetric( // TODO: remove?
179
+ // (deprecated, use firstContentBlockGetMetric instead)
180
+ "unixfs_get_latency_seconds",
181
+ "The time to receive the first UnixFS node on a GET from the gateway.",
182
+ ),
183
}
184
return i
185
}
@@ -291,7 +363,10 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
363
webError(w, "ipfs block get "+resolvedPath.Cid().String(), err, http.StatusInternalServerError)
364
return
365
}
294
- i.unixfsGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
366
+ ns := contentPath.Namespace()
367
+ timeToGetFirstContentBlock := time.Since(begin).Seconds()
368
+ i.unixfsGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock) // deprecated, use firstContentBlockGetMetric instead
369
+ i.firstContentBlockGetMetric.WithLabelValues(ns).Observe(timeToGetFirstContentBlock)
370
371
// HTTP Headers
372
i.addUserHeaders(w) // ok, _now_ write user's headers.
@@ -308,15 +383,15 @@ func (i *gatewayHandler) getOrHeadHandler(w http.ResponseWriter, r *http.Request
383
switch responseFormat {
384
case "": // The implicit response format is UnixFS
385
logger.Debugw("serving unixfs", "path", contentPath)
311
- i.serveUnixFs(w, r, resolvedPath, contentPath, logger)
386
+ i.serveUnixFs(w, r, resolvedPath, contentPath, begin, logger)
387
return
388
case "application/vnd.ipld.raw":
389
logger.Debugw("serving raw block", "path", contentPath)
315
- i.serveRawBlock(w, r, resolvedPath.Cid(), contentPath)
390
+ i.serveRawBlock(w, r, resolvedPath.Cid(), contentPath, begin)
391
return
392
case "application/vnd.ipld.car", "application/vnd.ipld.car; version=1":
393
logger.Debugw("serving car stream", "path", contentPath)
319
- i.serveCar(w, r, resolvedPath.Cid(), contentPath)
394
+ i.serveCar(w, r, resolvedPath.Cid(), contentPath, begin)
395
return
396
default: // catch-all for unsuported application/vnd.*
397
err := fmt.Errorf("unsupported format %q", responseFormat)
core/corehttp/gateway_handler_block.go
+5
-1
@@ -4,13 +4,14 @@ import (
4
"bytes"
5
"io/ioutil"
6
"net/http"
7
+ "time"
8
9
cid "github.com/ipfs/go-cid"
10
ipath "github.com/ipfs/interface-go-ipfs-core/path"
11
)
12
13
// serveRawBlock returns bytes behind a raw block
13
-func (i *gatewayHandler) serveRawBlock(w http.ResponseWriter, r *http.Request, blockCid cid.Cid, contentPath ipath.Path) {
14
+func (i *gatewayHandler) serveRawBlock(w http.ResponseWriter, r *http.Request, blockCid cid.Cid, contentPath ipath.Path, begin time.Time) {
15
blockReader, err := i.api.Block().Get(r.Context(), contentPath)
16
if err != nil {
17
webError(w, "ipfs block get "+blockCid.String(), err, http.StatusInternalServerError)
@@ -35,4 +36,7 @@ func (i *gatewayHandler) serveRawBlock(w http.ResponseWriter, r *http.Request, b
36
// Done: http.ServeContent will take care of
37
// If-None-Match+Etag, Content-Length and range requests
38
http.ServeContent(w, r, name, modtime, content)
39
+
40
+ // Update metrics
41
+ i.rawBlockGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
42
}
core/corehttp/gateway_handler_car.go
+5
-1
@@ -3,6 +3,7 @@ package corehttp
3
import (
4
"context"
5
"net/http"
6
+ "time"
7
8
blocks "github.com/ipfs/go-block-format"
9
cid "github.com/ipfs/go-cid"
@@ -13,7 +14,7 @@ import (
14
)
15
16
// serveCar returns a CAR stream for specific DAG+selector
16
-func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCid cid.Cid, contentPath ipath.Path) {
17
+func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCid cid.Cid, contentPath ipath.Path, begin time.Time) {
18
ctx, cancel := context.WithCancel(r.Context())
19
defer cancel()
20
@@ -59,6 +60,9 @@ func (i *gatewayHandler) serveCar(w http.ResponseWriter, r *http.Request, rootCi
60
w.Header().Set("X-Stream-Error", err.Error())
61
return
62
}
63
+
64
+ // Update metrics
65
+ i.carStreamGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
66
}
67
68
type dagStore struct {
core/corehttp/gateway_handler_unixfs.go
+4
-3
@@ -4,13 +4,14 @@ import (
4
"fmt"
5
"html"
6
"net/http"
7
+ "time"
8
9
files "github.com/ipfs/go-ipfs-files"
10
ipath "github.com/ipfs/interface-go-ipfs-core/path"
11
"go.uber.org/zap"
12
)
13
13
-func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, logger *zap.SugaredLogger) {
14
+func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, begin time.Time, logger *zap.SugaredLogger) {
15
// Handling UnixFS
16
dr, err := i.api.Unixfs().Get(r.Context(), resolvedPath)
17
if err != nil {
@@ -22,7 +23,7 @@ func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, res
23
// Handling Unixfs file
24
if f, ok := dr.(files.File); ok {
25
logger.Debugw("serving unixfs file", "path", contentPath)
25
- i.serveFile(w, r, contentPath, resolvedPath.Cid(), f)
26
+ i.serveFile(w, r, contentPath, resolvedPath.Cid(), f, begin)
27
return
28
}
29
@@ -33,5 +34,5 @@ func (i *gatewayHandler) serveUnixFs(w http.ResponseWriter, r *http.Request, res
34
return
35
}
36
logger.Debugw("serving unixfs directory", "path", contentPath)
36
- i.serveDirectory(w, r, resolvedPath, contentPath, dir, logger)
37
+ i.serveDirectory(w, r, resolvedPath, contentPath, dir, begin, logger)
38
}
core/corehttp/gateway_handler_unixfs_dir.go
+6
-2
@@ -5,6 +5,7 @@ import (
5
"net/url"
6
gopath "path"
7
"strings"
8
+ "time"
9
10
"github.com/dustin/go-humanize"
11
files "github.com/ipfs/go-ipfs-files"
@@ -18,7 +19,7 @@ import (
19
// serveDirectory returns the best representation of UnixFS directory
20
//
21
// It will return index.html if present, or generate directory listing otherwise.
21
-func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, logger *zap.SugaredLogger) {
22
+func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request, resolvedPath ipath.Resolved, contentPath ipath.Path, dir files.Directory, begin time.Time, logger *zap.SugaredLogger) {
23
24
// HostnameOption might have constructed an IPNS/IPFS path using the Host header.
25
// In this case, we need the original path for constructing redirects
@@ -62,7 +63,7 @@ func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request,
63
64
logger.Debugw("serving index.html file", "path", idxPath)
65
// write to request
65
- i.serveFile(w, r, idxPath, resolvedPath.Cid(), f)
66
+ i.serveFile(w, r, idxPath, resolvedPath.Cid(), f, begin)
67
return
68
case resolver.ErrNoLink:
69
logger.Debugw("no index.html; noop", "path", idxPath)
@@ -194,4 +195,7 @@ func (i *gatewayHandler) serveDirectory(w http.ResponseWriter, r *http.Request,
195
internalWebError(w, err)
196
return
197
}
198
+
199
+ // Update metrics
200
+ i.unixfsGenDirGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
201
}
core/corehttp/gateway_handler_unixfs_file.go
+5
-1
@@ -7,6 +7,7 @@ import (
7
"net/http"
8
gopath "path"
9
"strings"
10
+ "time"
11
12
"github.com/gabriel-vasile/mimetype"
13
cid "github.com/ipfs/go-cid"
@@ -16,7 +17,7 @@ import (
17
18
// serveFile returns data behind a file along with HTTP headers based on
19
// the file itself, its CID and the contentPath used for accessing it.
19
-func (i *gatewayHandler) serveFile(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid, file files.File) {
20
+func (i *gatewayHandler) serveFile(w http.ResponseWriter, r *http.Request, contentPath ipath.Path, fileCid cid.Cid, file files.File, begin time.Time) {
21
22
// Set Cache-Control and read optional Last-Modified time
23
modtime := addCacheControlHeaders(w, r, contentPath, fileCid)
@@ -80,4 +81,7 @@ func (i *gatewayHandler) serveFile(w http.ResponseWriter, r *http.Request, conte
81
// Done: http.ServeContent will take care of
82
// If-None-Match+Etag, Content-Length and range requests
83
http.ServeContent(w, r, name, modtime, content)
84
+
85
+ // Update metrics
86
+ i.unixfsFileGetMetric.WithLabelValues(contentPath.Namespace()).Observe(time.Since(begin).Seconds())
87
}