Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 03:30 UTC
reverseproxy: emit OpenTelemetry client spans (#8177)
* reverseproxy: emit OpenTelemetry client spans for upstream requests When the request carries a valid span context (set by the tracing handler), each upstream round trip, including retries, is recorded as a client span and that span's context is propagated to the upstream. Requests that are not traced use the transport directly. The transport is wrapped at the call site rather than replacing h.Transport, so transport interface checks and health checks are unaffected. The tracer provider comes from the span in the request context, so no global provider is needed. Fixes #7223 * reverseproxy: assert client span URL, upstream address and status Use a provisioned HTTPTransport so the test fails if roundTrip stops setting the scheme before the traced transport records url.full. * reverseproxy: add license header to tracing test
a committed
Oct 23, 2026 at 01:28 UTC
2b47a1849f86389825623d6d36ee528da8b9e5fb
2 files changed
+163
-1
modules/caddyhttp/reverseproxy/reverseproxy.go
+35
-1
@@ -39,6 +39,9 @@ import (
39
40
"github.com/dunglas/httpsfv"
41
"github.com/quic-go/quic-go/http3"
42
+ "go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp"
43
+ "go.opentelemetry.io/contrib/propagators/autoprop"
44
+ oteltrace "go.opentelemetry.io/otel/trace"
45
"go.uber.org/zap"
46
"go.uber.org/zap/zapcore"
47
"golang.org/x/net/http/httpguts"
@@ -281,6 +284,10 @@ type Handler struct {
284
// closed when their upstream is removed from the config.
285
tunnelTracker *tunnelTracker
286
287
+ // tracedTransport wraps Transport to emit OpenTelemetry client spans
288
+ // for upstream round trips when the request is being traced.
289
+ tracedTransport http.RoundTripper
290
+
291
ctx caddy.Context
292
logger *zap.Logger
293
events *caddyevents.App
@@ -418,6 +425,8 @@ func (h *Handler) Provision(ctx caddy.Context) error {
425
h.Transport = t
426
}
427
428
+ h.tracedTransport = newTracedTransport(h.Transport)
429
+
430
// If the transport can provide header ops, cache them now so we don't
431
// have to compute them per-request. Provision the HeaderOps if present
432
// so any runtime artifacts (like precompiled regex) are prepared.
@@ -1235,7 +1244,7 @@ func (h *Handler) reverseProxy(rw http.ResponseWriter, req *http.Request, origRe
1244
1245
// do the round-trip
1246
start := time.Now()
1238
- res, err := h.Transport.RoundTrip(req)
1247
+ res, err := h.roundTrip(req)
1248
duration := time.Since(start)
1249
1250
// record that the round trip is done for the 1xx response handler
@@ -1611,6 +1620,31 @@ func (lb LoadBalancing) tryAgain(ctx caddy.Context, start time.Time, retries int
1620
}
1621
}
1622
1623
+// newTracedTransport wraps rt so that round trips are recorded as
1624
+// OpenTelemetry client spans. The tracer provider is taken from the span
1625
+// in the request context (set by the tracing handler), so the wrapper does
1626
+// not depend on any global tracer provider.
1627
+func newTracedTransport(rt http.RoundTripper) http.RoundTripper {
1628
+ return otelhttp.NewTransport(rt,
1629
+ otelhttp.WithPropagators(autoprop.NewTextMapPropagator()),
1630
+ )
1631
+}
1632
+
1633
+// roundTrip sends req to the upstream. If req carries a valid span context,
1634
+// the round trip is recorded as an OpenTelemetry client span and that span's
1635
+// context is propagated to the upstream.
1636
+func (h *Handler) roundTrip(req *http.Request) (*http.Response, error) {
1637
+ if h.tracedTransport == nil || !oteltrace.SpanContextFromContext(req.Context()).IsValid() {
1638
+ return h.Transport.RoundTrip(req)
1639
+ }
1640
+ // the traced transport clones req, so set the scheme beforehand
1641
+ // to keep it visible to the span attributes and to req itself
1642
+ if ss, ok := h.Transport.(interface{ SetScheme(*http.Request) }); ok {
1643
+ ss.SetScheme(req)
1644
+ }
1645
+ return h.tracedTransport.RoundTrip(req)
1646
+}
1647
+
1648
// directRequest modifies only req.URL so that it points to the upstream
1649
// in the given DialInfo. It must modify ONLY the request URL.
1650
func (h *Handler) directRequest(req *http.Request, di DialInfo) {
modules/caddyhttp/reverseproxy/tracing_test.go
new
+128
@@ -0,0 +1,128 @@
1
+// Copyright 2015 Matthew Holt and The Caddy Authors
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package reverseproxy
16
+
17
+import (
18
+ "context"
19
+ "net"
20
+ "net/http"
21
+ "net/http/httptest"
22
+ "strings"
23
+ "testing"
24
+
25
+ "go.opentelemetry.io/otel/attribute"
26
+ "go.opentelemetry.io/otel/codes"
27
+ sdktrace "go.opentelemetry.io/otel/sdk/trace"
28
+ "go.opentelemetry.io/otel/sdk/trace/tracetest"
29
+ "go.opentelemetry.io/otel/trace"
30
+
31
+ "github.com/caddyserver/caddy/v2"
32
+)
33
+
34
+func TestUpstreamClientSpans(t *testing.T) {
35
+ var gotTraceparent []string
36
+ upstream := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
37
+ gotTraceparent = append(gotTraceparent, r.Header.Get("Traceparent"))
38
+ w.WriteHeader(http.StatusOK)
39
+ }))
40
+ t.Cleanup(upstream.Close)
41
+
42
+ t.Run("traced request", func(t *testing.T) {
43
+ gotTraceparent = nil
44
+ recorder := tracetest.NewSpanRecorder()
45
+ tp := sdktrace.NewTracerProvider(sdktrace.WithSpanProcessor(recorder))
46
+ t.Cleanup(func() { _ = tp.Shutdown(context.Background()) })
47
+
48
+ // round robin selects index 1 first, which refuses connections, so
49
+ // the retry produces a second client span under the same server span
50
+ h := minimalHandler(1,
51
+ &Upstream{Host: new(Host), Dial: upstream.Listener.Addr().String()},
52
+ &Upstream{Host: new(Host), Dial: deadUpstreamAddr(t)},
53
+ )
54
+ // HTTPTransport sets the URL scheme itself, so the span only records
55
+ // it if roundTrip calls SetScheme before the traced transport runs
56
+ caddyCtx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
57
+ t.Cleanup(cancel)
58
+ ht := new(HTTPTransport)
59
+ if err := ht.Provision(caddyCtx); err != nil {
60
+ t.Fatalf("provisioning transport: %v", err)
61
+ }
62
+ h.Transport = ht
63
+ h.tracedTransport = newTracedTransport(h.Transport)
64
+
65
+ ctx, serverSpan := tp.Tracer("test").Start(context.Background(), "server",
66
+ trace.WithSpanKind(trace.SpanKindServer))
67
+ req := httptest.NewRequest(http.MethodGet, "http://example.com/", nil).WithContext(ctx)
68
+ req = prepareTestRequest(req)
69
+ rec := httptest.NewRecorder()
70
+ if err := h.ServeHTTP(rec, req, nil); err != nil {
71
+ t.Fatalf("ServeHTTP: %v", err)
72
+ }
73
+ serverSpan.End()
74
+
75
+ var clientSpans []sdktrace.ReadOnlySpan
76
+ for _, s := range recorder.Ended() {
77
+ if s.SpanKind() == trace.SpanKindClient {
78
+ clientSpans = append(clientSpans, s)
79
+ }
80
+ }
81
+ if len(clientSpans) != 2 {
82
+ t.Fatalf("expected 2 client spans, got %d", len(clientSpans))
83
+ }
84
+ for i, s := range clientSpans {
85
+ if s.Parent().SpanID() != serverSpan.SpanContext().SpanID() {
86
+ t.Errorf("client span %d: parent is not the server span", i)
87
+ }
88
+ }
89
+ if clientSpans[0].Status().Code != codes.Error {
90
+ t.Errorf("expected failed attempt to have error status, got %v", clientSpans[0].Status().Code)
91
+ }
92
+
93
+ upstreamHost, upstreamPort, _ := net.SplitHostPort(upstream.Listener.Addr().String())
94
+ attrs := attribute.NewSet(clientSpans[1].Attributes()...)
95
+ for key, want := range map[attribute.Key]string{
96
+ "url.full": "http://" + upstream.Listener.Addr().String() + "/",
97
+ "server.address": upstreamHost,
98
+ "server.port": upstreamPort,
99
+ "http.response.status_code": "200",
100
+ } {
101
+ if got, _ := attrs.Value(key); got.Emit() != want {
102
+ t.Errorf("client span attribute %s: expected %q, got %q", key, want, got.Emit())
103
+ }
104
+ }
105
+
106
+ if len(gotTraceparent) != 1 {
107
+ t.Fatalf("expected 1 upstream request, got %d", len(gotTraceparent))
108
+ }
109
+ want := clientSpans[1].SpanContext().SpanID().String()
110
+ if !strings.Contains(gotTraceparent[0], want) {
111
+ t.Errorf("upstream traceparent %q does not reference client span %s", gotTraceparent[0], want)
112
+ }
113
+ })
114
+
115
+ t.Run("untraced request", func(t *testing.T) {
116
+ gotTraceparent = nil
117
+ h := minimalHandler(0, &Upstream{Host: new(Host), Dial: upstream.Listener.Addr().String()})
118
+ h.tracedTransport = newTracedTransport(h.Transport)
119
+
120
+ req := prepareTestRequest(httptest.NewRequest(http.MethodGet, "http://example.com/", nil))
121
+ if err := h.ServeHTTP(httptest.NewRecorder(), req, nil); err != nil {
122
+ t.Fatalf("ServeHTTP: %v", err)
123
+ }
124
+ if len(gotTraceparent) != 1 || gotTraceparent[0] != "" {
125
+ t.Errorf("expected no traceparent upstream, got %q", gotTraceparent)
126
+ }
127
+ })
128
+}