Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 17:15 UTC
caddytls: ECH key rotation (#7356)
* caddytls: ECH key rotation * Stop rotation goroutine on config unload * Publish ECH keys after rotating
Matt Holt committed
Dec 10, 2025 at 11:50 UTC
3c9c67e804eb3db9a3ac6532ddd2434246058cd5
3 files changed
+233
-72
modules/caddytls/connpolicy.go
+4
-14
@@ -168,21 +168,11 @@ func (cp ConnectionPolicies) TLSConfig(ctx caddy.Context) *tls.Config {
168
tlsApp.RegisterServerNames(echNames)
169
}
170
171
- // TODO: Ideally, ECH keys should be rotated. However, as of Go 1.24, the std lib implementation
172
- // does not support safely modifying the tls.Config's EncryptedClientHelloKeys field.
173
- // So, we implement static ECH keys temporarily. See https://github.com/golang/go/issues/71920.
174
- // Revisit this after Go 1.25 is released and implement key rotation.
175
- var stdECHKeys []tls.EncryptedClientHelloKey
176
- for _, echConfigs := range tlsApp.EncryptedClientHello.configs {
177
- for _, c := range echConfigs {
178
- stdECHKeys = append(stdECHKeys, tls.EncryptedClientHelloKey{
179
- Config: c.configBin,
180
- PrivateKey: c.privKeyBin,
181
- SendAsRetry: c.sendAsRetry,
182
- })
183
- }
171
+ tlsCfg.GetEncryptedClientHelloKeys = func(chi *tls.ClientHelloInfo) ([]tls.EncryptedClientHelloKey, error) {
172
+ tlsApp.EncryptedClientHello.configsMu.RLock()
173
+ defer tlsApp.EncryptedClientHello.configsMu.RUnlock()
174
+ return tlsApp.EncryptedClientHello.stdlibReady, nil
175
}
185
- tlsCfg.EncryptedClientHelloKeys = stdECHKeys
176
}
177
}
178
modules/caddytls/ech.go
+200
-53
@@ -2,6 +2,7 @@ package caddytls
2
3
import (
4
"context"
5
+ "crypto/tls"
6
"encoding/base64"
7
"encoding/json"
8
"errors"
@@ -11,6 +12,7 @@ import (
12
"path"
13
"strconv"
14
"strings"
15
+ "sync"
16
"time"
17
18
"github.com/caddyserver/certmagic"
@@ -73,14 +75,17 @@ type ECH struct {
75
// DNS RRs. (This also typically requires that they use DoH or DoT.)
76
Publication []*ECHPublication `json:"publication,omitempty"`
77
76
- // map of public_name to list of configs
77
- configs map[string][]echConfig
78
+ configsMu *sync.RWMutex // protects both configs and the list of configs/keys the standard library uses
79
+ configs map[string][]echConfig // map of public_name to list of configs
80
+ stdlibReady []tls.EncryptedClientHelloKey // ECH configs+keys in a format the standard library can use
81
}
82
83
// Provision loads or creates ECH configs and returns outer names (for certificate
84
// management), but does not publish any ECH configs. The DNS module is used as
85
// a default for later publishing if needed.
86
func (ech *ECH) Provision(ctx caddy.Context) ([]string, error) {
87
+ ech.configsMu = new(sync.RWMutex)
88
+
89
logger := ctx.Logger().Named("ech")
90
91
// set up publication modules before we need to obtain a lock in storage,
@@ -98,17 +103,57 @@ func (ech *ECH) Provision(ctx caddy.Context) ([]string, error) {
103
// the rest of provisioning needs an exclusive lock so that instances aren't
104
// stepping on each other when setting up ECH configs
105
storage := ctx.Storage()
101
- const echLockName = "ech_provision"
102
- if err := storage.Lock(ctx, echLockName); err != nil {
106
+ if err := storage.Lock(ctx, echStorageLockName); err != nil {
107
return nil, err
108
}
109
defer func() {
106
- if err := storage.Unlock(ctx, echLockName); err != nil {
110
+ if err := storage.Unlock(ctx, echStorageLockName); err != nil {
111
logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
112
}
113
}()
114
111
- var outerNames []string //nolint:prealloc // (FALSE POSITIVE - see https://github.com/alexkohler/prealloc/issues/30)
115
+ ech.configsMu.Lock()
116
+ defer ech.configsMu.Unlock()
117
+
118
+ outerNames, err := ech.setConfigsFromStorage(ctx, logger)
119
+ if err != nil {
120
+ return nil, fmt.Errorf("loading configs from storage: %w", err)
121
+ }
122
+
123
+ // see if we need to make any new ones based on the input configuration
124
+ for _, cfg := range ech.Configs {
125
+ publicName := strings.ToLower(strings.TrimSpace(cfg.PublicName))
126
+
127
+ if list, ok := ech.configs[publicName]; !ok || len(list) == 0 {
128
+ // no config with this public name was loaded, so create one
129
+ echCfg, err := generateAndStoreECHConfig(ctx, publicName)
130
+ if err != nil {
131
+ return nil, err
132
+ }
133
+ logger.Debug("generated new ECH config",
134
+ zap.String("public_name", echCfg.RawPublicName),
135
+ zap.Uint8("id", echCfg.ConfigID))
136
+ ech.configs[publicName] = append(ech.configs[publicName], echCfg)
137
+ outerNames = append(outerNames, publicName)
138
+ }
139
+ }
140
+
141
+ // ensure old keys are rotated out
142
+ if err = ech.rotateECHKeys(ctx, logger, true); err != nil {
143
+ return nil, fmt.Errorf("rotating ECH configs: %w", err)
144
+ }
145
+
146
+ return outerNames, nil
147
+}
148
+
149
+// setConfigsFromStorage sets the ECH configs in memory to those in storage.
150
+// It must be called in a write lock on ech.configsMu.
151
+func (ech *ECH) setConfigsFromStorage(ctx caddy.Context, logger *zap.Logger) ([]string, error) {
152
+ storage := ctx.Storage()
153
+
154
+ ech.configs = make(map[string][]echConfig)
155
+
156
+ var outerNames []string
157
158
// start by loading all the existing configs (even the older ones on the way out,
159
// since some clients may still be using them if they haven't yet picked up on the
@@ -131,48 +176,143 @@ func (ech *ECH) Provision(ctx caddy.Context) ([]string, error) {
176
logger.Debug("loaded ECH config",
177
zap.String("public_name", cfg.RawPublicName),
178
zap.Uint8("id", cfg.ConfigID))
179
+ if _, seen := ech.configs[cfg.RawPublicName]; !seen {
180
+ outerNames = append(outerNames, cfg.RawPublicName)
181
+ }
182
ech.configs[cfg.RawPublicName] = append(ech.configs[cfg.RawPublicName], cfg)
135
- outerNames = append(outerNames, cfg.RawPublicName)
183
}
184
138
- // all existing configs are now loaded; see if we need to make any new ones
139
- // based on the input configuration, and also mark the most recent one(s) as
140
- // current/active, so they can be used for ECH retries
141
- for _, cfg := range ech.Configs {
142
- publicName := strings.ToLower(strings.TrimSpace(cfg.PublicName))
185
+ return outerNames, nil
186
+}
187
144
- if list, ok := ech.configs[publicName]; ok && len(list) > 0 {
145
- // at least one config with this public name was loaded, so find the
146
- // most recent one and mark it as active to be used with retries
147
- var mostRecentDate time.Time
148
- var mostRecentIdx int
149
- for i, c := range list {
150
- if mostRecentDate.IsZero() || c.meta.Created.After(mostRecentDate) {
151
- mostRecentDate = c.meta.Created
152
- mostRecentIdx = i
153
- }
188
+// rotateECHKeys updates the ECH keys/configs that are outdated. It should be called
189
+// in a write lock on ech.configsMu. If a lock is already obtained in storage, then
190
+// pass true for storageSynced.
191
+func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSynced bool) error {
192
+ storage := ctx.Storage()
193
+
194
+ // all existing configs are now loaded; rotate keys "regularly" as recommended by the spec
195
+ // (also: "Rotating too frequently limits the client anonymity set." - but the more server
196
+ // names, the more frequently rotation can be done safely)
197
+ const (
198
+ rotationInterval = 24 * time.Hour * 30
199
+ deleteAfter = 24 * time.Hour * 90
200
+ )
201
+
202
+ if !ech.rotationNeeded(rotationInterval, deleteAfter) {
203
+ return nil
204
+ }
205
+
206
+ // sync this operation across cluster if not already
207
+ if !storageSynced {
208
+ if err := storage.Lock(ctx, echStorageLockName); err != nil {
209
+ return err
210
+ }
211
+ defer func() {
212
+ if err := storage.Unlock(ctx, echStorageLockName); err != nil {
213
+ logger.Error("unable to unlock ECH rotation in storage", zap.Error(err))
214
}
155
- list[mostRecentIdx].sendAsRetry = true
156
- } else {
157
- // no config with this public name was loaded, so create one
158
- echCfg, err := generateAndStoreECHConfig(ctx, publicName)
159
- if err != nil {
160
- return nil, err
215
+ }()
216
+ }
217
+
218
+ // update what storage has, in case another instance already updated things
219
+ if _, err := ech.setConfigsFromStorage(ctx, logger); err != nil {
220
+ return fmt.Errorf("updating ECH keys from storage: %v", err)
221
+ }
222
+
223
+ // iterate the updated list and do any updates as needed
224
+ for publicName := range ech.configs {
225
+ for i := 0; i < len(ech.configs[publicName]); i++ {
226
+ cfg := ech.configs[publicName][i]
227
+ if time.Since(cfg.meta.Created) >= rotationInterval && cfg.meta.Replaced.IsZero() {
228
+ // key is due for rotation and it hasn't been replaced yet; do that now
229
+ logger.Debug("ECH config is due for rotation",
230
+ zap.String("public_name", cfg.RawPublicName),
231
+ zap.Uint8("id", cfg.ConfigID),
232
+ zap.Time("created", cfg.meta.Created),
233
+ zap.Duration("age", time.Since(cfg.meta.Created)),
234
+ zap.Duration("rotation_interval", rotationInterval))
235
+
236
+ // start by generating and storing the replacement ECH config
237
+ newCfg, err := generateAndStoreECHConfig(ctx, publicName)
238
+ if err != nil {
239
+ return fmt.Errorf("generating and storing new replacement ECH config: %w", err)
240
+ }
241
+
242
+ // mark the key as replaced so we don't rotate it again, and instead delete it later
243
+ ech.configs[publicName][i].meta.Replaced = time.Now()
244
+
245
+ // persist the updated metadata
246
+ metaBytes, err := json.Marshal(ech.configs[publicName][i].meta)
247
+ if err != nil {
248
+ return fmt.Errorf("marshaling updated ECH config metadata: %v", err)
249
+ }
250
+ if err := storage.Store(ctx, echMetaKey(cfg.ConfigID), metaBytes); err != nil {
251
+ return fmt.Errorf("storing updated ECH config metadata: %v", err)
252
+ }
253
+
254
+ ech.configs[publicName] = append(ech.configs[publicName], newCfg)
255
+
256
+ logger.Debug("rotated ECH key",
257
+ zap.String("public_name", cfg.RawPublicName),
258
+ zap.Uint8("old_id", cfg.ConfigID),
259
+ zap.Uint8("new_id", newCfg.ConfigID))
260
+ } else if time.Since(cfg.meta.Created) >= deleteAfter && !cfg.meta.Replaced.IsZero() {
261
+ // key has expired and is no longer supported; delete it from storage and memory
262
+ cfgIDKey := path.Join(echConfigsKey, strconv.Itoa(int(cfg.ConfigID)))
263
+ if err := storage.Delete(ctx, cfgIDKey); err != nil {
264
+ return fmt.Errorf("deleting expired ECH config: %v", err)
265
+ }
266
+
267
+ ech.configs[publicName] = append(ech.configs[publicName][:i], ech.configs[publicName][i+1:]...)
268
+ i--
269
+
270
+ logger.Debug("deleted expired ECH key",
271
+ zap.String("public_name", cfg.RawPublicName),
272
+ zap.Uint8("id", cfg.ConfigID),
273
+ zap.Duration("age", time.Since(cfg.meta.Created)))
274
}
162
- logger.Debug("generated new ECH config",
163
- zap.String("public_name", echCfg.RawPublicName),
164
- zap.Uint8("id", echCfg.ConfigID))
165
- ech.configs[publicName] = append(ech.configs[publicName], echCfg)
166
- outerNames = append(outerNames, publicName)
275
}
276
}
277
170
- return outerNames, nil
278
+ ech.updateKeyList()
279
+
280
+ return nil
281
}
282
173
-func (t *TLS) publishECHConfigs() error {
174
- logger := t.logger.Named("ech")
283
+// rotationNeeded returns true if any ECH key needs to be replaced, or deleted.
284
+// It must be called inside a read or write lock of ech.configsMu (probably a
285
+// write lock, so that the rotation can occur correctly in the same lock).)
286
+func (ech *ECH) rotationNeeded(rotationInterval, deleteAfter time.Duration) bool {
287
+ for publicName := range ech.configs {
288
+ for i := 0; i < len(ech.configs[publicName]); i++ {
289
+ cfg := ech.configs[publicName][i]
290
+ if (time.Since(cfg.meta.Created) >= rotationInterval && cfg.meta.Replaced.IsZero()) ||
291
+ (time.Since(cfg.meta.Created) >= deleteAfter && !cfg.meta.Replaced.IsZero()) {
292
+ return true
293
+ }
294
+ }
295
+ }
296
+ return false
297
+}
298
299
+// updateKeyList updates the list of ECH keys the std lib uses to serve ECH.
300
+// It must be called inside a write lock on ech.configsMu.
301
+func (ech *ECH) updateKeyList() {
302
+ ech.stdlibReady = []tls.EncryptedClientHelloKey{}
303
+ for _, cfgs := range ech.configs {
304
+ for _, cfg := range cfgs {
305
+ ech.stdlibReady = append(ech.stdlibReady, tls.EncryptedClientHelloKey{
306
+ Config: cfg.configBin,
307
+ PrivateKey: cfg.privKeyBin,
308
+ SendAsRetry: cfg.meta.Replaced.IsZero(), // only send during retries if key has not been rotated out
309
+ })
310
+ }
311
+ }
312
+}
313
+
314
+// publishECHConfigs publishes any configs that are configured for publication and which haven't been published already.
315
+func (t *TLS) publishECHConfigs(logger *zap.Logger) error {
316
// make publication exclusive, since we don't need to repeat this unnecessarily
317
storage := t.ctx.Storage()
318
const echLockName = "ech_publish"
@@ -197,7 +337,7 @@ func (t *TLS) publishECHConfigs() error {
337
publishers: []ECHPublisher{
338
&ECHDNSPublisher{
339
provider: dnsProv,
200
- logger: t.logger,
340
+ logger: logger,
341
},
342
},
343
},
@@ -209,6 +349,7 @@ func (t *TLS) publishECHConfigs() error {
349
// publish with it, and figure out which inner names to publish
350
// to/for, then publish
351
for _, publication := range publicationList {
352
+ t.EncryptedClientHello.configsMu.RLock()
353
// this publication is either configured for specific ECH configs,
354
// or we just use an implied default of all ECH configs
355
var echCfgList echConfigList
@@ -231,6 +372,7 @@ func (t *TLS) publishECHConfigs() error {
372
}
373
}
374
}
375
+ t.EncryptedClientHello.configsMu.RUnlock()
376
377
// marshal the ECH config list as binary for publication
378
echCfgListBin, err := echCfgList.MarshalBinary()
@@ -304,7 +446,7 @@ func (t *TLS) publishECHConfigs() error {
446
// at least a partial failure, maybe a complete failure, but we can
447
// log each error by domain
448
for innerName, domainErr := range publishErrs {
307
- t.logger.Error("failed to publish ECH configuration list",
449
+ logger.Error("failed to publish ECH configuration list",
450
zap.String("publisher", publisherKey),
451
zap.String("domain", innerName),
452
zap.Uint8s("config_ids", configIDs),
@@ -312,7 +454,7 @@ func (t *TLS) publishECHConfigs() error {
454
}
455
} else if err != nil {
456
// generic error; assume the entire thing failed, I guess
315
- t.logger.Error("failed publishing ECH configuration list",
457
+ logger.Error("failed publishing ECH configuration list",
458
zap.String("publisher", publisherKey),
459
zap.Strings("domains", dnsNamesToPublish),
460
zap.Uint8s("config_ids", configIDs),
@@ -334,7 +476,7 @@ func (t *TLS) publishECHConfigs() error {
476
successNames = append(successNames, name)
477
}
478
}
337
- t.logger.Info("successfully published ECH configuration list for "+someAll+" domains",
479
+ logger.Info("successfully published ECH configuration list for "+someAll+" domains",
480
zap.String("publisher", publisherKey),
481
zap.Strings("domains", successNames),
482
zap.Uint8s("config_ids", configIDs))
@@ -353,13 +495,12 @@ func (t *TLS) publishECHConfigs() error {
495
if err != nil {
496
return fmt.Errorf("marshaling ECH config metadata: %v", err)
497
}
356
- metaKey := path.Join(echConfigsKey, strconv.Itoa(int(cfg.ConfigID)), "meta.json")
357
- if err := t.ctx.Storage().Store(t.ctx, metaKey, metaBytes); err != nil {
498
+ if err := t.ctx.Storage().Store(t.ctx, echMetaKey(cfg.ConfigID), metaBytes); err != nil {
499
return fmt.Errorf("storing updated ECH config metadata: %v", err)
500
}
501
}
502
} else {
362
- t.logger.Error("all domains failed to publish ECH configuration list (see earlier errors)",
503
+ logger.Error("all domains failed to publish ECH configuration list (see earlier errors)",
504
zap.String("publisher", publisherKey),
505
zap.Strings("domains", dnsNamesToPublish),
506
zap.Uint8s("config_ids", configIDs))
@@ -489,7 +630,7 @@ func generateAndStoreECHConfig(ctx caddy.Context, publicName string) (echConfig,
630
631
echCfg := echConfig{
632
PublicKey: publicKey,
492
- Version: draftTLSESNI22,
633
+ Version: draftTLSESNI25,
634
ConfigID: configID,
635
RawPublicName: publicName,
636
KEMID: kemChoice,
@@ -507,7 +648,6 @@ func generateAndStoreECHConfig(ctx caddy.Context, publicName string) (echConfig,
648
AEADID: hpke.AEAD_ChaCha20Poly1305,
649
},
650
},
510
- sendAsRetry: true,
651
}
652
meta := echConfigMeta{
653
Created: time.Now(),
@@ -786,10 +926,9 @@ type echConfig struct {
926
927
// these fields are not part of the spec, but are here for
928
// our use when setting up TLS servers or maintenance
789
- configBin []byte
790
- privKeyBin []byte
791
- meta echConfigMeta
792
- sendAsRetry bool
929
+ configBin []byte
930
+ privKeyBin []byte
931
+ meta echConfigMeta
932
}
933
934
func (echCfg echConfig) MarshalBinary() ([]byte, error) {
@@ -811,8 +950,8 @@ func (echCfg *echConfig) UnmarshalBinary(data []byte) error {
950
if !b.ReadUint16(&echCfg.Version) {
951
return errInvalidLen
952
}
814
- if echCfg.Version != draftTLSESNI22 {
815
- return fmt.Errorf("supported version must be %d: got %d", draftTLSESNI22, echCfg.Version)
953
+ if echCfg.Version != draftTLSESNI25 {
954
+ return fmt.Errorf("supported version must be %d: got %d", draftTLSESNI25, echCfg.Version)
955
}
956
957
if !b.ReadUint16LengthPrefixed(&content) || !b.Empty() {
@@ -1022,19 +1161,27 @@ func (p PublishECHConfigListErrors) Error() string {
1161
1162
type echConfigMeta struct {
1163
Created time.Time `json:"created"`
1164
+ Replaced time.Time `json:"replaced,omitzero"`
1165
Publications publicationHistory `json:"publications"`
1166
}
1167
1168
+func echMetaKey(configID uint8) string {
1169
+ return path.Join(echConfigsKey, strconv.Itoa(int(configID)), "meta.json")
1170
+}
1171
+
1172
// publicationHistory is a map of publisher key to
1173
// map of inner name to timestamp
1174
type publicationHistory map[string]map[string]time.Time
1175
1176
+// echStorageLockName is the name of the storage lock to sync ECH updates.
1177
+const echStorageLockName = "ech_rotation"
1178
+
1179
// The key prefix when putting ECH configs in storage. After this
1180
// comes the config ID.
1181
const echConfigsKey = "ech/configs"
1182
1036
-// https://www.ietf.org/archive/id/draft-ietf-tls-esni-22.html
1037
-const draftTLSESNI22 = 0xfe0d
1183
+// https://www.ietf.org/archive/id/draft-ietf-tls-esni-25.html
1184
+const draftTLSESNI25 = 0xfe0d
1185
1186
// Interface guard
1187
var _ ECHPublisher = (*ECHDNSPublisher)(nil)
modules/caddytls/tls.go
+29
-5
@@ -335,7 +335,6 @@ func (t *TLS) Provision(ctx caddy.Context) error {
335
336
// ECH (Encrypted ClientHello) initialization
337
if t.EncryptedClientHello != nil {
338
- t.EncryptedClientHello.configs = make(map[string][]echConfig)
338
outerNames, err := t.EncryptedClientHello.Provision(ctx)
339
if err != nil {
340
return fmt.Errorf("provisioning Encrypted ClientHello components: %v", err)
@@ -411,12 +410,37 @@ func (t *TLS) Start() error {
410
return fmt.Errorf("automate: managing %v: %v", t.automateNames, err)
411
}
412
414
- // publish ECH configs in the background; does not need to block
415
- // server startup, as it could take a while
413
if t.EncryptedClientHello != nil {
414
+ echLogger := t.logger.Named("ech")
415
+
416
+ // publish ECH configs in the background; does not need to block
417
+ // server startup, as it could take a while; then keep keys rotated
418
go func() {
418
- if err := t.publishECHConfigs(); err != nil {
419
- t.logger.Named("ech").Error("publication(s) failed", zap.Error(err))
419
+ // publish immediately first
420
+ if err := t.publishECHConfigs(echLogger); err != nil {
421
+ echLogger.Error("publication(s) failed", zap.Error(err))
422
+ }
423
+
424
+ // then every so often, rotate and publish if needed
425
+ // (both of these functions only do something if needed)
426
+ for {
427
+ select {
428
+ case <-time.After(1 * time.Hour):
429
+ // ensure old keys are rotated out
430
+ t.EncryptedClientHello.configsMu.Lock()
431
+ err = t.EncryptedClientHello.rotateECHKeys(t.ctx, echLogger, false)
432
+ t.EncryptedClientHello.configsMu.Unlock()
433
+ if err != nil {
434
+ echLogger.Error("rotating ECH configs failed", zap.Error(err))
435
+ return
436
+ }
437
+ err := t.publishECHConfigs(echLogger)
438
+ if err != nil {
439
+ echLogger.Error("publication(s) failed", zap.Error(err))
440
+ }
441
+ case <-t.ctx.Done():
442
+ return
443
+ }
444
}
445
}()
446
}