Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 22:00 UTC
reverseproxy: Add more debug logs (#5793)
* reverseproxy: Add more debug logs This makes debug logging very noisy when reverse proxying, but I guess that's the point. This has shown to be useful in troubleshooting infrastructure issues. * Update modules/caddyhttp/reverseproxy/streaming.go Co-authored-by: Francis Lavoie <lavofr@gmail.com> * Update modules/caddyhttp/reverseproxy/streaming.go Co-authored-by: Francis Lavoie <lavofr@gmail.com> * Add opt-in `trace_logs` option * Rename to VerboseLogs --------- Co-authored-by: Francis Lavoie <lavofr@gmail.com>
Matt Holt committed
Oct 11, 2023 at 13:36 UTC
3a3182fba3eb20c45dce3efd9eb2856288a5ae04
4 files changed
+62
-6
modules/caddyhttp/reverseproxy/caddyfile.go
+7
@@ -90,6 +90,7 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
90
// max_buffer_size <size>
91
// stream_timeout <duration>
92
// stream_close_delay <duration>
93
+// trace_logs
94
//
95
// # request manipulation
96
// trusted_proxies [private_ranges] <ranges...>
@@ -782,6 +783,12 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
783
responseHandler,
784
)
785
786
+ case "verbose_logs":
787
+ if h.VerboseLogs {
788
+ return d.Err("verbose_logs already specified")
789
+ }
790
+ h.VerboseLogs = true
791
+
792
default:
793
return d.Errf("unrecognized subdirective %s", d.Val())
794
}
modules/caddyhttp/reverseproxy/reverseproxy.go
+19
-2
@@ -191,6 +191,13 @@ type Handler struct {
191
// - `{http.reverse_proxy.header.*}` The headers from the response
192
HandleResponse []caddyhttp.ResponseHandler `json:"handle_response,omitempty"`
193
194
+ // If set, the proxy will write very detailed logs about its
195
+ // inner workings. Enable this only when debugging, as it
196
+ // will produce a lot of output.
197
+ //
198
+ // EXPERIMENTAL: This feature is subject to change or removal.
199
+ VerboseLogs bool `json:"verbose_logs,omitempty"`
200
+
201
Transport http.RoundTripper `json:"-"`
202
CB CircuitBreaker `json:"-"`
203
DynamicUpstreams UpstreamSource `json:"-"`
@@ -943,9 +950,15 @@ func (h *Handler) finalizeResponse(
950
}
951
952
rw.WriteHeader(res.StatusCode)
953
+ if h.VerboseLogs {
954
+ logger.Debug("wrote header")
955
+ }
956
947
- err := h.copyResponse(rw, res.Body, h.flushInterval(req, res))
948
- res.Body.Close() // close now, instead of defer, to populate res.Trailer
957
+ err := h.copyResponse(rw, res.Body, h.flushInterval(req, res), logger)
958
+ errClose := res.Body.Close() // close now, instead of defer, to populate res.Trailer
959
+ if h.VerboseLogs || errClose != nil {
960
+ logger.Debug("closed response body from upstream", zap.Error(errClose))
961
+ }
962
if err != nil {
963
// we're streaming the response and we've already written headers, so
964
// there's nothing an error handler can do to recover at this point;
@@ -979,6 +992,10 @@ func (h *Handler) finalizeResponse(
992
}
993
}
994
995
+ if h.VerboseLogs {
996
+ logger.Debug("response finalized")
997
+ }
998
+
999
return nil
1000
}
1001
modules/caddyhttp/reverseproxy/streaming.go
+33
-3
@@ -184,15 +184,22 @@ func (h Handler) isBidirectionalStream(req *http.Request, res *http.Response) bo
184
(ae == "identity" || ae == "")
185
}
186
187
-func (h Handler) copyResponse(dst http.ResponseWriter, src io.Reader, flushInterval time.Duration) error {
187
+func (h Handler) copyResponse(dst http.ResponseWriter, src io.Reader, flushInterval time.Duration, logger *zap.Logger) error {
188
var w io.Writer = dst
189
190
if flushInterval != 0 {
191
+ var mlwLogger *zap.Logger
192
+ if h.VerboseLogs {
193
+ mlwLogger = logger.Named("max_latency_writer")
194
+ } else {
195
+ mlwLogger = zap.NewNop()
196
+ }
197
mlw := &maxLatencyWriter{
198
dst: dst,
199
//nolint:bodyclose
200
flush: http.NewResponseController(dst).Flush,
201
latency: flushInterval,
202
+ logger: mlwLogger,
203
}
204
defer mlw.stop()
205
@@ -205,19 +212,30 @@ func (h Handler) copyResponse(dst http.ResponseWriter, src io.Reader, flushInter
212
213
buf := streamingBufPool.Get().(*[]byte)
214
defer streamingBufPool.Put(buf)
208
- _, err := h.copyBuffer(w, src, *buf)
215
+
216
+ var copyLogger *zap.Logger
217
+ if h.VerboseLogs {
218
+ copyLogger = logger
219
+ } else {
220
+ copyLogger = zap.NewNop()
221
+ }
222
+
223
+ _, err := h.copyBuffer(w, src, *buf, copyLogger)
224
return err
225
}
226
227
// copyBuffer returns any write errors or non-EOF read errors, and the amount
228
// of bytes written.
214
-func (h Handler) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (int64, error) {
229
+func (h Handler) copyBuffer(dst io.Writer, src io.Reader, buf []byte, logger *zap.Logger) (int64, error) {
230
if len(buf) == 0 {
231
buf = make([]byte, defaultBufferSize)
232
}
233
var written int64
234
for {
235
+ logger.Debug("waiting to read from upstream")
236
nr, rerr := src.Read(buf)
237
+ logger := logger.With(zap.Int("read", nr))
238
+ logger.Debug("read from upstream", zap.Error(rerr))
239
if rerr != nil && rerr != io.EOF && rerr != context.Canceled {
240
// TODO: this could be useful to know (indeed, it revealed an error in our
241
// fastcgi PoC earlier; but it's this single error report here that necessitates
@@ -229,10 +247,15 @@ func (h Handler) copyBuffer(dst io.Writer, src io.Reader, buf []byte) (int64, er
247
h.logger.Error("reading from backend", zap.Error(rerr))
248
}
249
if nr > 0 {
250
+ logger.Debug("writing to downstream")
251
nw, werr := dst.Write(buf[:nr])
252
if nw > 0 {
253
written += int64(nw)
254
}
255
+ logger.Debug("wrote to downstream",
256
+ zap.Int("written", nw),
257
+ zap.Int64("written_total", written),
258
+ zap.Error(werr))
259
if werr != nil {
260
return written, fmt.Errorf("writing: %w", werr)
261
}
@@ -452,18 +475,22 @@ type maxLatencyWriter struct {
475
mu sync.Mutex // protects t, flushPending, and dst.Flush
476
t *time.Timer
477
flushPending bool
478
+ logger *zap.Logger
479
}
480
481
func (m *maxLatencyWriter) Write(p []byte) (n int, err error) {
482
m.mu.Lock()
483
defer m.mu.Unlock()
484
n, err = m.dst.Write(p)
485
+ m.logger.Debug("wrote bytes", zap.Int("n", n), zap.Error(err))
486
if m.latency < 0 {
487
+ m.logger.Debug("flushing immediately")
488
//nolint:errcheck
489
m.flush()
490
return
491
}
492
if m.flushPending {
493
+ m.logger.Debug("delayed flush already pending")
494
return
495
}
496
if m.t == nil {
@@ -471,6 +498,7 @@ func (m *maxLatencyWriter) Write(p []byte) (n int, err error) {
498
} else {
499
m.t.Reset(m.latency)
500
}
501
+ m.logger.Debug("timer set for delayed flush", zap.Duration("duration", m.latency))
502
m.flushPending = true
503
return
504
}
@@ -479,8 +507,10 @@ func (m *maxLatencyWriter) delayedFlush() {
507
m.mu.Lock()
508
defer m.mu.Unlock()
509
if !m.flushPending { // if stop was called but AfterFunc already started this goroutine
510
+ m.logger.Debug("delayed flush is not pending")
511
return
512
}
513
+ m.logger.Debug("delayed flush")
514
//nolint:errcheck
515
m.flush()
516
m.flushPending = false
modules/caddyhttp/reverseproxy/streaming_test.go
+3
-1
@@ -5,6 +5,8 @@ import (
5
"net/http/httptest"
6
"strings"
7
"testing"
8
+
9
+ "github.com/caddyserver/caddy/v2"
10
)
11
12
func TestHandlerCopyResponse(t *testing.T) {
@@ -22,7 +24,7 @@ func TestHandlerCopyResponse(t *testing.T) {
24
for _, d := range testdata {
25
src := bytes.NewBuffer([]byte(d))
26
dst.Reset()
25
- err := h.copyResponse(recorder, src, 0)
27
+ err := h.copyResponse(recorder, src, 0, caddy.Log())
28
if err != nil {
29
t.Errorf("failed with error: %v", err)
30
}