@marcel91200 / caddy Mirror · read-only
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 11, 01:00 UTC

master
go 977 lines 37 KB
Raw
1 // Copyright 2015 Matthew Holt and The Caddy Authors
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 package caddyhttp
16
17 import (
18 "cmp"
19 "context"
20 "crypto/tls"
21 "errors"
22 "fmt"
23 "log"
24 "net"
25 "net/http"
26 "strconv"
27 "strings"
28 "sync"
29 "time"
30
31 "go.uber.org/zap"
32 "golang.org/x/net/http2"
33
34 "github.com/caddyserver/caddy/v2"
35 "github.com/caddyserver/caddy/v2/modules/caddyevents"
36 "github.com/caddyserver/caddy/v2/modules/caddytls"
37 )
38
39 func init() {
40 caddy.RegisterModule(App{})
41 }
42
43 // App is a robust, production-ready HTTP server.
44 //
45 // HTTPS is enabled by default if host matchers with qualifying names are used
46 // in any of routes; certificates are automatically provisioned and renewed.
47 // Additionally, automatic HTTPS will also enable HTTPS for servers that listen
48 // only on the HTTPS port but which do not have any TLS connection policies
49 // defined by adding a good, default TLS connection policy.
50 //
51 // Similar to how other popular web servers work, incoming request header fields
52 // with underscores or dots are ignored/dropped implicitly to mitigate security
53 // risks. Specific headers to allow can be explicitly configured using
54 // `expected_underscore_headers` and `expected_dot_headers`.
55 //
56 // ### Placeholders
57 //
58 // In HTTP routes, additional placeholders are available (replace any `*`):
59 //
60 // Placeholder | Description
61 // ------------|---------------
62 // `{http.request.body}` | The request body (⚠️ inefficient; use only for debugging); if reading it exceeds a `request_body` `max_size` limit, templates and the `vars` and `vars_regexp` matchers fail the request with HTTP 413 instead of silently returning a truncated body
63 // `{http.request.body_base64}` | The request body, base64-encoded (⚠️ for debugging); same 413 behavior on `max_size` limits as `{http.request.body}` in templates and the `vars` matchers
64 // `{http.request.cookie.*}` | HTTP request cookie
65 // `{http.request.duration}` | Time up to now spent handling the request (after decoding headers from client)
66 // `{http.request.duration_ms}` | Same as 'duration', but in milliseconds.
67 // `{http.request.uuid}` | The request unique identifier
68 // `{http.request.header.*}` | Specific request header field
69 // `{http.request.host}` | The host part of the request's Host header
70 // `{http.request.host.labels.*}` | Request host labels (0-based from right); e.g. for foo.example.com: 0=com, 1=example, 2=foo
71 // `{http.request.hostport}` | The host and port from the request's Host header
72 // `{http.request.method}` | The request method
73 // `{http.request.orig_method}` | The request's original method
74 // `{http.request.orig_uri}` | The request's original URI
75 // `{http.request.orig_uri.path}` | The request's original path
76 // `{http.request.orig_uri.path.*}` | Parts of the original path, split by `/` (0-based from left)
77 // `{http.request.orig_uri.path.dir}` | The request's original directory
78 // `{http.request.orig_uri.path.file}` | The request's original filename
79 // `{http.request.orig_uri.path.file_escaped}` | The request's original filename, percent-encoded as in the request's escaped path
80 // `{http.request.orig_uri.query}` | The request's original query string (without `?`)
81 // `{http.request.orig_uri.prefixed_query}` | The request's original query string with a `?` prefix, if non-empty
82 // `{http.request.port}` | The port part of the request's Host header
83 // `{http.request.proto}` | The raw protocol of the request as returned by Go (e.g., HTTP/2.0 or HTTP/3.0)
84 // `{http.request.proto_name}` | The spec-defined protocol of the request (e.g., HTTP/2 or HTTP/3)
85 // `{http.request.local.host}` | The host (IP) part of the local address the connection arrived on
86 // `{http.request.local.port}` | The port part of the local address the connection arrived on
87 // `{http.request.local}` | The local address the connection arrived on
88 // `{http.request.remote.host}` | The host (IP) part of the remote client's address, if available (not known with HTTP/3 early data)
89 // `{http.request.remote.port}` | The port part of the remote client's address
90 // `{http.request.remote}` | The address of the remote client
91 // `{http.request.scheme}` | The request scheme, typically `http` or `https`
92 // `{http.request.tls.version}` | The TLS version name
93 // `{http.request.tls.cipher_suite}` | The TLS cipher suite
94 // `{http.request.tls.resumed}` | The TLS connection resumed a previous connection
95 // `{http.request.tls.proto}` | The negotiated next protocol
96 // `{http.request.tls.proto_mutual}` | The negotiated next protocol was advertised by the server
97 // `{http.request.tls.server_name}` | The server name requested by the client, if any
98 // `{http.request.tls.ech}` | Whether ECH was offered by the client and accepted by the server
99 // `{http.request.tls.client.fingerprint}` | The SHA256 checksum of the client certificate
100 // `{http.request.tls.client.public_key}` | The public key of the client certificate.
101 // `{http.request.tls.client.public_key_sha256}` | The SHA256 checksum of the client's public key.
102 // `{http.request.tls.client.certificate_pem}` | The PEM-encoded value of the certificate.
103 // `{http.request.tls.client.certificate_der_base64}` | The base64-encoded value of the certificate.
104 // `{http.request.tls.client.issuer}` | The issuer DN of the client certificate
105 // `{http.request.tls.client.serial}` | The serial number of the client certificate
106 // `{http.request.tls.client.subject}` | The subject DN of the client certificate
107 // `{http.request.tls.client.san.dns_names.*}` | SAN DNS names(index optional)
108 // `{http.request.tls.client.san.emails.*}` | SAN email addresses (index optional)
109 // `{http.request.tls.client.san.ips.*}` | SAN IP addresses (index optional)
110 // `{http.request.tls.client.san.uris.*}` | SAN URIs (index optional)
111 // `{http.request.uri}` | The full request URI
112 // `{http.request.uri_escaped}` | The full request URI with query-style URL encoding applied (using url.QueryEscape)
113 // `{http.request.uri.path}` | The path component of the request URI
114 // `{http.request.uri.path_escaped}` | The path component of the request URI with query-style URL encoding applied (using url.QueryEscape)
115 // `{http.request.uri.path.*}` | Parts of the path, split by `/` (0-based from left)
116 // `{http.request.uri.path.dir}` | The directory, excluding leaf filename
117 // `{http.request.uri.path.file}` | The filename of the path, excluding directory
118 // `{http.request.uri.query}` | The query string (without `?`)
119 // `{http.request.uri.query_escaped}` | The query string with query-style URL encoding applied (using url.QueryEscape)
120 // `{http.request.uri.prefixed_query}` | The query string with a `?` prefix, if non-empty
121 // `{http.request.uri.query.*}` | Individual query string value
122 // `{http.response.header.*}` | Specific response header field
123 // `{http.vars.*}` | Custom variables in the HTTP handler chain
124 // `{http.shutting_down}` | True if the HTTP app is shutting down
125 // `{http.time_until_shutdown}` | Time until HTTP server shutdown, if scheduled
126 type App struct {
127 // HTTPPort specifies the port to use for HTTP (as opposed to HTTPS),
128 // which is used when setting up HTTP->HTTPS redirects or ACME HTTP
129 // challenge solvers. Default: 80.
130 HTTPPort int `json:"http_port,omitempty"`
131
132 // HTTPSPort specifies the port to use for HTTPS, which is used when
133 // solving the ACME TLS-ALPN challenges, or whenever HTTPS is needed
134 // but no specific port number is given. Default: 443.
135 HTTPSPort int `json:"https_port,omitempty"`
136
137 // GracePeriod is how long to wait for active connections when shutting
138 // down the servers. During the grace period, no new connections are
139 // accepted, idle connections are closed, and active connections will
140 // be given the full length of time to become idle and close.
141 // Once the grace period is over, connections will be forcefully closed.
142 // If zero, the grace period is eternal. Default: 0.
143 GracePeriod caddy.Duration `json:"grace_period,omitempty"`
144
145 // ShutdownDelay is how long to wait before initiating the grace
146 // period. When this app is stopping (e.g. during a config reload or
147 // process exit), all servers will be shut down. Normally this immediately
148 // initiates the grace period. However, if this delay is configured, servers
149 // will not be shut down until the delay is over. During this time, servers
150 // continue to function normally and allow new connections. At the end, the
151 // grace period will begin. This can be useful to allow downstream load
152 // balancers time to move this instance out of the rotation without hiccups.
153 //
154 // When shutdown has been scheduled, placeholders {http.shutting_down} (bool)
155 // and {http.time_until_shutdown} (duration) may be useful for health checks.
156 ShutdownDelay caddy.Duration `json:"shutdown_delay,omitempty"`
157
158 // Servers is the list of servers, keyed by arbitrary names chosen
159 // at your discretion for your own convenience; the keys do not
160 // affect functionality.
161 Servers map[string]*Server `json:"servers,omitempty"`
162
163 // If set, metrics observations will be enabled.
164 // This setting is EXPERIMENTAL and subject to change.
165 Metrics *Metrics `json:"metrics,omitempty"`
166
167 ctx caddy.Context
168 logger *zap.Logger
169 tlsApp *caddytls.TLS
170
171 // stopped indicates whether the app has stopped
172 // It can only happen if it has started successfully in the first place.
173 // Otherwise, Cleanup will call Stop to clean up resources.
174 stopped bool
175
176 // used temporarily between phases 1 and 2 of auto HTTPS
177 allCertDomains map[string]struct{}
178 }
179
180 // CaddyModule returns the Caddy module information.
181 func (App) CaddyModule() caddy.ModuleInfo {
182 return caddy.ModuleInfo{
183 ID: "http",
184 New: func() caddy.Module { return new(App) },
185 }
186 }
187
188 // Provision sets up the app.
189 func (app *App) Provision(ctx caddy.Context) error {
190 // store some references
191 app.logger = ctx.Logger()
192 app.ctx = ctx
193
194 // provision TLS and events apps
195 tlsAppIface, err := ctx.App("tls")
196 if err != nil {
197 return fmt.Errorf("getting tls app: %v", err)
198 }
199 app.tlsApp = tlsAppIface.(*caddytls.TLS)
200
201 eventsAppIface, err := ctx.App("events")
202 if err != nil {
203 return fmt.Errorf("getting events app: %v", err)
204 }
205
206 repl := caddy.NewReplacer()
207
208 // this provisions the matchers for each route,
209 // and prepares auto HTTP->HTTPS redirects, and
210 // is required before we provision each server
211 err = app.automaticHTTPSPhase1(ctx, repl)
212 if err != nil {
213 return err
214 }
215
216 if app.Metrics != nil {
217 app.Metrics.init = sync.Once{}
218 app.Metrics.httpMetrics = &httpMetrics{}
219 // Scan config for allowed hosts to prevent cardinality explosion
220 app.Metrics.scanConfigForHosts(app)
221 if err := app.Metrics.provisionOTLP(ctx); err != nil {
222 return err
223 }
224 }
225 // prepare each server
226 oldContext := ctx.Context
227 for srvName, srv := range app.Servers {
228 ctx.Context = context.WithValue(oldContext, ServerCtxKey, srv)
229 srv.name = srvName
230 srv.tlsApp = app.tlsApp
231 srv.events = eventsAppIface.(*caddyevents.App)
232 srv.ctx = ctx
233 srv.logger = app.logger.Named("log")
234 srv.errorLogger = app.logger.Named("log.error")
235 if srv.Metrics != nil {
236 srv.logger.Warn("per-server 'metrics' is deprecated; use 'metrics' in the root 'http' app instead")
237 app.Metrics = cmp.Or(app.Metrics, &Metrics{
238 init: sync.Once{},
239 httpMetrics: &httpMetrics{},
240 })
241 app.Metrics.PerHost = app.Metrics.PerHost || srv.Metrics.PerHost
242 }
243
244 // only enable access logs if configured
245 if srv.Logs != nil {
246 srv.Logs.normalizeLoggerNames()
247 srv.accessLogger = app.logger.Named("log.access")
248 if srv.Logs.Trace {
249 srv.traceLogger = app.logger.Named("log.trace")
250 }
251 }
252
253 // if no protocols configured explicitly, enable all except h2c
254 if len(srv.Protocols) == 0 {
255 srv.Protocols = srv.protocolsWithDefaults()
256 }
257
258 if srv.ListenProtocols != nil {
259 if len(srv.ListenProtocols) != len(srv.Listen) {
260 return fmt.Errorf("server %s: listener protocols count does not match address count: %d != %d",
261 srvName, len(srv.ListenProtocols), len(srv.Listen))
262 }
263
264 for i, lnProtocols := range srv.ListenProtocols {
265 if lnProtocols != nil {
266 srv.ListenProtocols[i] = srv.listenerProtocolsWithDefaults(lnProtocols)
267 }
268 }
269 }
270
271 // limit max header bytes to a more reasonable default than 1MB from Go std lib
272 // (see https://github.com/php/frankenphp/issues/2459#issuecomment-4655612909)
273 if srv.MaxHeaderBytes <= 0 {
274 srv.MaxHeaderBytes = 16 * 1024
275 }
276
277 // if not explicitly configured by the user, disallow TLS
278 // client auth bypass (domain fronting) which could
279 // otherwise be exploited by sending an unprotected SNI
280 // value during a TLS handshake, then putting a protected
281 // domain in the Host header after establishing connection;
282 // this is a safe default, but we allow users to override
283 // it for example in the case of running a proxy where
284 // domain fronting is desired and access is not restricted
285 // based on hostname
286 if srv.StrictSNIHost == nil && srv.hasTLSClientAuth() {
287 app.logger.Warn("enabling strict SNI-Host enforcement because TLS client auth is configured",
288 zap.String("server_id", srvName))
289 trueBool := true
290 srv.StrictSNIHost = &trueBool
291 }
292
293 // set up the trusted proxies source
294 for srv.TrustedProxiesRaw != nil {
295 val, err := ctx.LoadModule(srv, "TrustedProxiesRaw")
296 if err != nil {
297 return fmt.Errorf("loading trusted proxies modules: %v", err)
298 }
299 srv.trustedProxies = val.(IPRangeSource)
300 }
301
302 // set the default client IP header to read from
303 if srv.ClientIPHeaders == nil {
304 srv.ClientIPHeaders = []string{"X-Forwarded-For"}
305 }
306
307 // precompute underscore and dot header allowlist rules
308 if err := srv.provisionUnderscoreHeaders(); err != nil {
309 return fmt.Errorf("server %s: %v", srvName, err)
310 }
311 if err := srv.provisionDotHeaders(); err != nil {
312 return fmt.Errorf("server %s: %v", srvName, err)
313 }
314
315 // process each listener address
316 for i := range srv.Listen {
317 lnOut, err := repl.ReplaceOrErr(srv.Listen[i], true, true)
318 if err != nil {
319 return fmt.Errorf("server %s, listener %d: %v", srvName, i, err)
320 }
321 srv.Listen[i] = lnOut
322 }
323
324 // set up each listener modifier
325 if srv.ListenerWrappersRaw != nil {
326 vals, err := ctx.LoadModule(srv, "ListenerWrappersRaw")
327 if err != nil {
328 return fmt.Errorf("loading listener wrapper modules: %v", err)
329 }
330 var hasTLSPlaceholder bool
331 for i, val := range vals.([]any) {
332 if _, ok := val.(*tlsPlaceholderWrapper); ok {
333 if i == 0 {
334 // putting the tls placeholder wrapper first is nonsensical because
335 // that is the default, implicit setting: without it, all wrappers
336 // will go after the TLS listener anyway
337 return fmt.Errorf("it is unnecessary to specify the TLS listener wrapper in the first position because that is the default")
338 }
339 if hasTLSPlaceholder {
340 return fmt.Errorf("TLS listener wrapper can only be specified once")
341 }
342 hasTLSPlaceholder = true
343 }
344 srv.listenerWrappers = append(srv.listenerWrappers, val.(caddy.ListenerWrapper))
345 }
346 // if any wrappers were configured but the TLS placeholder wrapper is
347 // absent, prepend it so all defined wrappers come after the TLS
348 // handshake; this simplifies logic when starting the server, since we
349 // can simply assume the TLS placeholder will always be there
350 if !hasTLSPlaceholder && len(srv.listenerWrappers) > 0 {
351 srv.listenerWrappers = append([]caddy.ListenerWrapper{new(tlsPlaceholderWrapper)}, srv.listenerWrappers...)
352 }
353 }
354
355 // set up each packet conn modifier
356 if srv.PacketConnWrappersRaw != nil {
357 vals, err := ctx.LoadModule(srv, "PacketConnWrappersRaw")
358 if err != nil {
359 return fmt.Errorf("loading packet conn wrapper modules: %v", err)
360 }
361 // if any wrappers were configured, they come before the QUIC handshake;
362 // unlike TLS above, there is no QUIC placeholder
363 for _, val := range vals.([]any) {
364 srv.packetConnWrappers = append(srv.packetConnWrappers, val.(caddy.PacketConnWrapper))
365 }
366 }
367
368 // pre-compile the primary handler chain, and be sure to wrap it in our
369 // route handler so that important security checks are done, etc.
370 primaryRoute := emptyHandler
371 if srv.Routes != nil {
372 err := srv.Routes.ProvisionHandlers(ctx, app.Metrics)
373 if err != nil {
374 return fmt.Errorf("server %s: setting up route handlers: %v", srvName, err)
375 }
376 primaryRoute = srv.Routes.Compile(emptyHandler)
377 }
378 srv.primaryHandlerChain = srv.wrapPrimaryRoute(primaryRoute)
379
380 // pre-compile the error handler chain
381 if srv.Errors != nil {
382 err := srv.Errors.Routes.Provision(ctx)
383 if err != nil {
384 return fmt.Errorf("server %s: setting up error handling routes: %v", srvName, err)
385 }
386 srv.errorHandlerChain = srv.Errors.Routes.Compile(errorEmptyHandler)
387 }
388
389 // provision the named routes (they get compiled at runtime)
390 for name, route := range srv.NamedRoutes {
391 err := route.Provision(ctx, app.Metrics)
392 if err != nil {
393 return fmt.Errorf("server %s: setting up named route '%s' handlers: %v", name, srvName, err)
394 }
395 }
396
397 // prepare the TLS connection policies
398 err = srv.TLSConnPolicies.Provision(ctx)
399 if err != nil {
400 return fmt.Errorf("server %s: setting up TLS connection policies: %v", srvName, err)
401 }
402
403 // if there is no idle timeout, set a sane default; users have complained
404 // before that aggressive CDNs leave connections open until the server
405 // closes them, so if we don't close them it leads to resource exhaustion
406 if srv.IdleTimeout == 0 {
407 srv.IdleTimeout = defaultIdleTimeout
408 }
409 if srv.ReadHeaderTimeout == 0 {
410 srv.ReadHeaderTimeout = defaultReadHeaderTimeout // see #6663
411 }
412 if srv.ReadIdleTimeout == 0 {
413 srv.ReadIdleTimeout = defaultReadIdleTimeout
414 }
415 if srv.WriteIdleTimeout == 0 {
416 srv.WriteIdleTimeout = defaultWriteIdleTimeout
417 }
418 if srv.MaxWriteChunk == 0 {
419 srv.MaxWriteChunk = DefaultMaxWriteChunk
420 }
421 }
422 ctx.Context = oldContext
423 return nil
424 }
425
426 // Validate ensures the app's configuration is valid.
427 func (app *App) Validate() error {
428 lnAddrs := make(map[string]string)
429
430 for srvName, srv := range app.Servers {
431 // each server must use distinct listener addresses
432 for _, addr := range srv.Listen {
433 listenAddr, err := caddy.ParseNetworkAddress(addr)
434 if err != nil {
435 return fmt.Errorf("invalid listener address '%s': %v", addr, err)
436 }
437 // check that every address in the port range is unique to this server;
438 // we do not use <= here because PortRangeSize() adds 1 to EndPort for us
439 for i := uint(0); i < listenAddr.PortRangeSize(); i++ {
440 addr := caddy.JoinNetworkAddress(listenAddr.Network, listenAddr.Host, strconv.FormatUint(uint64(listenAddr.StartPort+i), 10))
441 if sn, ok := lnAddrs[addr]; ok {
442 return fmt.Errorf("server %s: listener address repeated: %s (already claimed by server '%s')", srvName, addr, sn)
443 }
444 lnAddrs[addr] = srvName
445 }
446 }
447
448 // logger names must not have ports
449 if srv.Logs != nil {
450 for host := range srv.Logs.LoggerNames {
451 if _, _, err := net.SplitHostPort(host); err == nil {
452 return fmt.Errorf("server %s: logger name must not have a port: %s", srvName, host)
453 }
454 }
455 }
456 }
457 return nil
458 }
459
460 func removeTLSALPN(srv *Server, target string) {
461 for _, cp := range srv.TLSConnPolicies {
462 // the TLSConfig was already provisioned, so... manually remove it
463 for i, np := range cp.TLSConfig.NextProtos {
464 if np == target {
465 cp.TLSConfig.NextProtos = append(cp.TLSConfig.NextProtos[:i], cp.TLSConfig.NextProtos[i+1:]...)
466 break
467 }
468 }
469 // remove it from the parent connection policy too, just to keep things tidy
470 for i, alpn := range cp.ALPN {
471 if alpn == target {
472 cp.ALPN = append(cp.ALPN[:i], cp.ALPN[i+1:]...)
473 break
474 }
475 }
476 }
477 }
478
479 // Start runs the app. It finishes automatic HTTPS if enabled,
480 // including management of certificates.
481 func (app *App) Start() error {
482 // get a logger compatible with http.Server; recovered handler panics are
483 // routed to ERROR so they remain visible at the default log level (#7923)
484 serverLogger := serverErrorLogger(app.logger.Named("stdlib"))
485
486 for srvName, srv := range app.Servers {
487 srv.server = &http.Server{
488 ReadTimeout: time.Duration(srv.ReadTimeout),
489 ReadHeaderTimeout: time.Duration(srv.ReadHeaderTimeout),
490 WriteTimeout: time.Duration(srv.WriteTimeout),
491 IdleTimeout: time.Duration(srv.IdleTimeout),
492 MaxHeaderBytes: srv.MaxHeaderBytes,
493 Handler: srv,
494 ErrorLog: serverLogger,
495 Protocols: new(http.Protocols),
496 ConnContext: func(ctx context.Context, c net.Conn) context.Context {
497 if nc, ok := c.(interface{ tlsNetConn() net.Conn }); ok {
498 getTlsConStateFunc := sync.OnceValue(func() *tls.ConnectionState {
499 tlsConnState := nc.tlsNetConn().(connectionStater).ConnectionState()
500 return &tlsConnState
501 })
502 ctx = context.WithValue(ctx, tlsConnectionStateFuncCtxKey, getTlsConStateFunc)
503 }
504 return ctx
505 },
506 }
507
508 // disable HTTP/2, which we enabled by default during provisioning
509 if !srv.protocol("h2") {
510 srv.server.TLSNextProto = make(map[string]func(*http.Server, *tls.Conn, http.Handler))
511 removeTLSALPN(srv, "h2")
512 }
513 if !srv.protocol("h1") {
514 removeTLSALPN(srv, "http/1.1")
515 }
516
517 // configure the http versions the server will serve
518 if srv.protocol("h1") {
519 srv.server.Protocols.SetHTTP1(true)
520 }
521
522 if srv.protocol("h2") || srv.protocol("h2c") {
523 // skip setting h2 because if NextProtos is present, it's list of alpn versions will take precedence.
524 // it will always be present because http2.ConfigureServer will populate that field
525 // enabling h2c because some listener wrapper will wrap the connection that is no longer *tls.Conn
526 // However, we need to handle the case that if the connection is h2c but h2c is not enabled. We identify
527 // this type of connection by checking if it's behind a TLS listener wrapper or if it implements tls.ConnectionState.
528 srv.server.Protocols.SetUnencryptedHTTP2(true)
529 // when h2c is enabled but h2 disabled, we already removed h2 from NextProtos
530 // the handshake will never succeed with h2
531 // http2.ConfigureServer will enable the server to handle both h2 and h2c
532 // TODO: Migrate to http.Server.Protocols once listener-wrapper behaviour is covered.
533 //nolint:errcheck,staticcheck
534 http2.ConfigureServer(srv.server, new(http2.Server))
535 }
536
537 // this TLS config is used by the std lib to choose the actual TLS config for connections
538 // by looking through the connection policies to find the first one that matches
539 tlsCfg := srv.TLSConnPolicies.TLSConfig(app.ctx)
540 srv.configureServer(srv.server)
541
542 for lnIndex, lnAddr := range srv.Listen {
543 listenAddr, err := caddy.ParseNetworkAddress(lnAddr)
544 if err != nil {
545 return fmt.Errorf("%s: parsing listen address '%s': %v", srvName, lnAddr, err)
546 }
547
548 srv.addresses = append(srv.addresses, listenAddr)
549
550 protocols := srv.Protocols
551 if srv.ListenProtocols != nil && srv.ListenProtocols[lnIndex] != nil {
552 protocols = srv.ListenProtocols[lnIndex]
553 }
554
555 protocolsUnique := map[string]struct{}{}
556 for _, protocol := range protocols {
557 protocolsUnique[protocol] = struct{}{}
558 }
559 _, h1ok := protocolsUnique["h1"]
560 _, h2ok := protocolsUnique["h2"]
561 _, h2cok := protocolsUnique["h2c"]
562 _, h3ok := protocolsUnique["h3"]
563
564 for portOffset := uint(0); portOffset < listenAddr.PortRangeSize(); portOffset++ {
565 hostport := listenAddr.JoinHostPort(portOffset)
566
567 // enable TLS if there is a policy and if this is not the HTTP port
568 useTLS := len(srv.TLSConnPolicies) > 0 && int(listenAddr.StartPort+portOffset) != app.httpPort()
569
570 if h1ok || h2ok && useTLS || h2cok {
571 // create the listener for this socket
572 lnAny, err := listenAddr.Listen(app.ctx, portOffset, net.ListenConfig{
573 KeepAliveConfig: net.KeepAliveConfig{
574 Enable: srv.KeepAliveInterval >= 0,
575 Interval: time.Duration(srv.KeepAliveInterval),
576 Idle: time.Duration(srv.KeepAliveIdle),
577 Count: srv.KeepAliveCount,
578 },
579 })
580 if err != nil {
581 return fmt.Errorf("listening on %s: %v", listenAddr.At(portOffset), err)
582 }
583 ln, ok := lnAny.(net.Listener)
584 if !ok {
585 return fmt.Errorf("network '%s' cannot handle HTTP/1 or HTTP/2 connections", listenAddr.Network)
586 }
587
588 // wrap listener before TLS (up to the TLS placeholder wrapper)
589 var lnWrapperIdx int
590 for i, lnWrapper := range srv.listenerWrappers {
591 if _, ok := lnWrapper.(*tlsPlaceholderWrapper); ok {
592 lnWrapperIdx = i + 1 // mark the next wrapper's spot
593 break
594 }
595 ln = lnWrapper.WrapListener(ln)
596 }
597
598 if useTLS {
599 // create TLS listener - this enables and terminates TLS
600 ln = tls.NewListener(ln, tlsCfg)
601 }
602
603 // finish wrapping listener where we left off before TLS
604 for i := lnWrapperIdx; i < len(srv.listenerWrappers); i++ {
605 ln = srv.listenerWrappers[i].WrapListener(ln)
606 }
607
608 // check if the connection is h2c
609 ln = &http2Listener{
610 useTLS: useTLS,
611 useH1: h1ok,
612 useH2: h2ok || h2cok,
613 Listener: ln,
614 logger: app.logger,
615 }
616
617 // if binding to port 0, the OS chooses a port for us;
618 // but the user won't know the port unless we print it
619 if !listenAddr.IsUnixNetwork() && !listenAddr.IsFdNetwork() && listenAddr.StartPort == 0 && listenAddr.EndPort == 0 {
620 app.logger.Info("port 0 listener",
621 zap.String("input_address", lnAddr),
622 zap.String("actual_address", ln.Addr().String()))
623 }
624
625 app.logger.Debug("starting server loop",
626 zap.String("address", ln.Addr().String()),
627 zap.Bool("tls", useTLS),
628 zap.Bool("http3", srv.h3server != nil))
629
630 srv.listeners = append(srv.listeners, ln)
631
632 //nolint:errcheck
633 go srv.server.Serve(ln)
634 }
635
636 if h2ok && !useTLS {
637 // Can only serve h2 with TLS enabled
638 app.logger.Warn("HTTP/2 skipped because it requires TLS",
639 zap.String("network", listenAddr.Network),
640 zap.String("addr", hostport))
641 }
642
643 if h3ok {
644 // Can't serve HTTP/3 on the same socket as HTTP/1 and 2 because it uses
645 // a different transport mechanism... which is fine, but the OS doesn't
646 // differentiate between a SOCK_STREAM file and a SOCK_DGRAM file; they
647 // are still one file on the system. So even though "unixpacket" and
648 // "unixgram" are different network types just as "tcp" and "udp" are,
649 // the OS will not let us use the same file as both STREAM and DGRAM.
650 if listenAddr.IsUnixNetwork() {
651 app.logger.Warn("HTTP/3 disabled because Unix can't multiplex STREAM and DGRAM on same socket",
652 zap.String("file", hostport))
653 continue
654 }
655
656 if useTLS {
657 // enable HTTP/3 if configured
658 app.logger.Info("enabling HTTP/3 listener", zap.String("addr", hostport))
659 if err := srv.serveHTTP3(listenAddr.At(portOffset), tlsCfg); err != nil {
660 return err
661 }
662 } else {
663 // Can only serve h3 with TLS enabled
664 app.logger.Warn("HTTP/3 skipped because it requires TLS",
665 zap.String("network", listenAddr.Network),
666 zap.String("addr", hostport))
667 }
668 }
669 }
670 }
671
672 srv.logger.Info("server running",
673 zap.String("name", srvName),
674 zap.Strings("protocols", srv.Protocols))
675 }
676
677 // finish automatic HTTPS by finally beginning
678 // certificate management
679 err := app.automaticHTTPSPhase2()
680 if err != nil {
681 return fmt.Errorf("finalizing automatic HTTPS: %v", err)
682 }
683
684 return nil
685 }
686
687 // stdlibLogPrefixPanic is the prefix Go's net/http server uses when it writes a
688 // recovered handler panic (and its stack trace) to http.Server.ErrorLog.
689 // See the deferred recover in net/http.(*conn).serve.
690 const stdlibLogPrefixPanic = "http: panic serving"
691
692 // serverErrorLogger returns a *log.Logger suitable for http.Server.ErrorLog
693 // that forwards the standard library's server messages to Caddy's structured
694 // logger. Most of these messages are low-signal and logged at DEBUG, but
695 // recovered handler panics indicate a real server-side error, so they are
696 // logged at ERROR to stay visible at the default log level.
697 func serverErrorLogger(logger *zap.Logger) *log.Logger {
698 return log.New(stdlibLogRouter{logger: logger}, "", 0)
699 }
700
701 // stdlibLogRouter is an io.Writer that routes standard library http.Server
702 // error-log messages to the appropriate level on the wrapped structured logger.
703 type stdlibLogRouter struct {
704 logger *zap.Logger
705 }
706
707 func (w stdlibLogRouter) Write(p []byte) (int, error) {
708 // log.Logger always appends a trailing newline; trim it so the structured
709 // message doesn't carry a dangling blank line.
710 msg := strings.TrimSuffix(string(p), "\n")
711 if strings.HasPrefix(msg, stdlibLogPrefixPanic) {
712 w.logger.Error(msg)
713 } else {
714 w.logger.Debug(msg)
715 }
716 return len(p), nil
717 }
718
719 // Stop gracefully shuts down the HTTP server.
720 func (app *App) Stop() error {
721 return app.stop(caddy.Exiting())
722 }
723
724 func (app *App) stop(exiting bool) error {
725 ctx := context.Background()
726
727 // see if any listeners in our config will be closing or if they are continuing
728 // through a reload; because if any are closing, we will enforce shutdown delay
729 var delay bool
730 scheduledTime := time.Now().Add(time.Duration(app.ShutdownDelay))
731 if app.ShutdownDelay > 0 {
732 for _, server := range app.Servers {
733 for _, na := range server.addresses {
734 for _, addr := range na.Expand() {
735 if caddy.ListenerUsage(addr.Network, addr.JoinHostPort(0)) < 2 {
736 app.logger.Debug("listener closing and shutdown delay is configured", zap.String("address", addr.String()))
737 server.shutdownAt.Store(&scheduledTime)
738 delay = true
739 } else {
740 app.logger.Debug("shutdown delay configured but listener will remain open", zap.String("address", addr.String()))
741 }
742 }
743 }
744 }
745 }
746
747 // honor scheduled/delayed shutdown time
748 if delay {
749 app.logger.Info("shutdown scheduled",
750 zap.Duration("delay_duration", time.Duration(app.ShutdownDelay)),
751 zap.Time("time", scheduledTime))
752 time.Sleep(time.Duration(app.ShutdownDelay))
753 }
754
755 // enforce grace period if configured
756 var finishedShutdown sync.WaitGroup
757 if app.GracePeriod > 0 {
758 var cancel context.CancelFunc
759 timeout := time.Duration(app.GracePeriod)
760 ctx, cancel = context.WithTimeoutCause(ctx, timeout, fmt.Errorf("server graceful shutdown %ds timeout", int(timeout.Seconds())))
761 defer func() {
762 // A reload must leave the grace period alive while its requests finish.
763 go func() {
764 finishedShutdown.Wait()
765 cancel()
766 }()
767 }()
768 app.logger.Info("servers shutting down; grace period initiated", zap.Duration("duration", timeout))
769 } else {
770 app.logger.Info("servers shutting down with eternal grace period")
771 }
772
773 // goroutines aren't guaranteed to be scheduled right away,
774 // so we'll use one WaitGroup to wait for all the goroutines
775 // to start their server shutdowns, and another to wait for
776 // them to finish; we'll always block for them to start so
777 // that when we return the caller can be confident* that the
778 // old servers are no longer accepting new connections
779 // (* the scheduler might still pause them right before
780 // calling Shutdown(), but it's unlikely)
781 var startedShutdown sync.WaitGroup
782
783 // these will run in goroutines
784 stopServer := func(server *Server) {
785 defer finishedShutdown.Done()
786 startedShutdown.Done()
787
788 // possible if server failed to Start
789 if server.server == nil {
790 return
791 }
792
793 if err := server.server.Shutdown(ctx); err != nil {
794 if cause := context.Cause(ctx); cause != nil && errors.Is(err, context.DeadlineExceeded) {
795 err = cause
796 }
797 app.logger.Error("server shutdown",
798 zap.Error(err),
799 zap.Strings("addresses", server.Listen))
800 }
801 }
802 stopH3Server := func(server *Server) {
803 defer finishedShutdown.Done()
804 startedShutdown.Done()
805
806 if server.h3server == nil {
807 return
808 }
809
810 // closing quic listeners won't affect accepted connections now
811 // so like stdlib, close listeners first, but keep the net.PacketConns open
812 for _, h3ln := range server.quicListeners {
813 if err := h3ln.Close(); err != nil {
814 app.logger.Error("http3 listener close",
815 zap.Error(err))
816 }
817 }
818
819 // ServeQUICConn connections are not in the HTTP/3 server's set, so
820 // Shutdown does not wait for them and, with none tracked, it closes
821 // the server immediately. webtransport.Server.Close would do that
822 // too. waitWebTransportConns keeps the sockets up until the accept
823 // loops finish. GracePeriod 0 is eternal: the wait ends when the
824 // peer leaves, and the session is not force-closed. A finite grace
825 // period closes sessions at the deadline, while the sockets are
826 // still up, so the close capsule can reach the client.
827 // The close below is for a session whose loop already returned.
828 // Fallback HTTP/3 servers are drained at the same time so an
829 // active request on a reused non-WebTransport listener gets this
830 // grace period instead of dying when the packet conn closes.
831 fbErr := make(chan error, 1)
832 go func() {
833 fbErr <- server.shutdownFallbackHTTP3(ctx)
834 }()
835 server.waitWebTransportConns(ctx)
836 server.closeWebTransportSessions()
837
838 if err := server.h3server.Shutdown(ctx); err != nil {
839 if cause := context.Cause(ctx); cause != nil && errors.Is(err, context.DeadlineExceeded) {
840 err = cause
841 }
842 app.logger.Error("HTTP/3 server shutdown",
843 zap.Error(err),
844 zap.Strings("addresses", server.Listen))
845 }
846 if err := <-fbErr; err != nil {
847 app.logger.Error("HTTP/3 fallback shutdown",
848 zap.Error(err),
849 zap.Strings("addresses", server.Listen))
850 }
851
852 // close the underlying net.PacketConns now
853 // see the comment for ListenQUIC
854 for _, h3ln := range server.quicListeners {
855 if err := h3ln.Close(); err != nil {
856 app.logger.Error("http3 listener close socket",
857 zap.Error(err))
858 }
859 }
860 }
861
862 for _, server := range app.Servers {
863 startedShutdown.Add(2)
864 finishedShutdown.Add(2)
865 go stopServer(server)
866 go stopH3Server(server)
867 }
868
869 shutdownDone := make(chan struct{})
870 pendingServerShutdowns.Store(shutdownDone, struct{}{})
871 go func() {
872 finishedShutdown.Wait()
873 close(shutdownDone)
874 pendingServerShutdowns.Delete(shutdownDone)
875 }()
876
877 // block until all the goroutines have been run by the scheduler;
878 // this means that they have likely called Shutdown() by now
879 startedShutdown.Wait()
880
881 // if the process is exiting, we need to block here and wait
882 // for the grace periods to complete, otherwise the process will
883 // terminate before the servers are finished shutting down; but
884 // we don't really need to wait for the grace period to finish
885 // if the process isn't exiting (but note that frequent config
886 // reloads with long grace periods for a sustained length of time
887 // may deplete resources)
888 if exiting {
889 finishedShutdown.Wait()
890
891 // Responses from earlier configurations must finish before the process exits.
892 pendingServerShutdowns.Range(func(done, _ any) bool {
893 select {
894 case <-done.(chan struct{}):
895 return true
896 case <-ctx.Done():
897 app.logger.Error("waiting for previous server shutdowns", zap.Error(context.Cause(ctx)))
898 return false
899 }
900 })
901 }
902
903 // run stop callbacks now that the server shutdowns are complete
904 for name, s := range app.Servers {
905 for _, stopHook := range s.onStopFuncs {
906 if err := stopHook(ctx); err != nil {
907 app.logger.Error("server stop hook", zap.String("server", name), zap.Error(err))
908 }
909 }
910 }
911
912 // flush and shut down the OTLP metrics exporter (if configured) so any
913 // last data point reaches the collector before the process exits
914 if err := app.Metrics.shutdown(ctx); err != nil {
915 app.logger.Error("shutting down OTLP metrics", zap.Error(err))
916 }
917
918 app.stopped = true
919 return nil
920 }
921
922 // Cleanup will close remaining listeners if they still remain
923 // because some of the servers fail to start.
924 // It simply calls Stop because Stop won't be called when Start fails.
925 func (app *App) Cleanup() error {
926 if app.stopped {
927 return nil
928 }
929 return app.Stop()
930 }
931
932 func (app *App) httpPort() int {
933 if app.HTTPPort == 0 {
934 return DefaultHTTPPort
935 }
936 return app.HTTPPort
937 }
938
939 func (app *App) httpsPort() int {
940 if app.HTTPSPort == 0 {
941 return DefaultHTTPSPort
942 }
943 return app.HTTPSPort
944 }
945
946 const (
947 // defaultIdleTimeout is the default HTTP server timeout
948 // for closing idle connections; useful to avoid resource
949 // exhaustion behind hungry CDNs, for example (we've had
950 // several complaints without this).
951 defaultIdleTimeout = caddy.Duration(5 * time.Minute)
952
953 // defaultReadHeaderTimeout is the default timeout for
954 // reading HTTP headers from clients. Headers are generally
955 // small, often less than 1 KB, so it shouldn't take a
956 // long time even on legitimately slow connections or
957 // busy servers to read it.
958 defaultReadHeaderTimeout = caddy.Duration(time.Minute)
959
960 // defaultReadIdleTimeout and defaultWriteIdleTimeout mitigate
961 // slowloris-style attacks on the request body and response write.
962 // Unlike a hard deadline, these are safe defaults even for large
963 // payloads because the deadline is reset on every successful
964 // read/write; only a stalled connection is affected.
965 defaultReadIdleTimeout = caddy.Duration(time.Minute)
966 defaultWriteIdleTimeout = caddy.Duration(time.Minute)
967 )
968
969 // Reloads outlive their app, so termination must track shutdowns across configurations.
970 var pendingServerShutdowns sync.Map
971
972 // Interface guards
973 var (
974 _ caddy.App = (*App)(nil)
975 _ caddy.Provisioner = (*App)(nil)
976 _ caddy.Validator = (*App)(nil)
977 )