@marcel91200 / caddy Mirror · read-only / commits / 8626fa37
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 11, 06:00 UTC

caddytls: synchronize storage cleaner with TLS.Stop via context and WaitGroup (#7954)

* caddytls: synchronize storage cleaner with TLS.Stop via context and WaitGroup * Unify the storage cleanup synchronization to both tls and ech. * Cannot embed the sync.WaitGroup directly as the TLS struct is copied. * caddytls: propagate cancellable context to ECH rotation and add sync tests Pass cancellable context to ECH key rotation so in-flight storage locks and operations unblock when TLS.Stop is invoked. Add comprehensive tests verifying TLS.Stop cleanly unblocks and waits for storage cleaner and ECH workers. --------- Co-authored-by: Zen Dodd <mail@steadytao.com>

Jens-Uwe Mager committed Sep 5, 2026 at 15:37 UTC 8626fa370382d9b37c9732cb15730ed29d5f7168
3 files changed +621 -37
modules/caddytls/ech.go
+39 -9
@@ -101,8 +101,10 @@ func (ech *ECH) Provision(ctx caddy.Context) ([]string, error) {
101 return nil, err
102 }
103 defer func() {
104 - if err := storage.Unlock(ctx, echStorageLockName); err != nil {
105 - logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
104 + if err := storage.Unlock(context.WithoutCancel(ctx), echStorageLockName); err != nil {
105 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
106 + logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
107 + }
108 }
109 }()
110
@@ -188,7 +190,16 @@ func (ech *ECH) setConfigsFromStorage(ctx caddy.Context, logger *zap.Logger) ([]
190 //
191 // This function sets/updates the stdlib-ready key list only if a rotation occurs.
192 func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSynced bool) error {
193 + if err := ctx.Err(); err != nil {
194 + return err
195 + }
196 + if ctx.Context == nil {
197 + return nil
198 + }
199 storage := ctx.Storage()
200 + if storage == nil {
201 + return nil
202 + }
203
204 // all existing configs are now loaded; rotate keys "regularly" as recommended by the spec
205 // (also: "Rotating too frequently limits the client anonymity set." - but the more server
@@ -208,8 +219,10 @@ func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSync
219 return err
220 }
221 defer func() {
211 - if err := storage.Unlock(ctx, echStorageLockName); err != nil {
212 - logger.Error("unable to unlock ECH rotation in storage", zap.Error(err))
222 + if err := storage.Unlock(context.WithoutCancel(ctx), echStorageLockName); err != nil {
223 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
224 + logger.Error("unable to unlock ECH rotation in storage", zap.Error(err))
225 + }
226 }
227 }()
228 }
@@ -221,7 +234,13 @@ func (ech *ECH) rotateECHKeys(ctx caddy.Context, logger *zap.Logger, storageSync
234
235 // iterate the updated list and do any updates as needed
236 for publicName := range ech.configs {
237 + if err := ctx.Err(); err != nil {
238 + return err
239 + }
240 for i := 0; i < len(ech.configs[publicName]); i++ {
241 + if err := ctx.Err(); err != nil {
242 + return err
243 + }
244 cfg := ech.configs[publicName][i]
245 if time.Since(cfg.meta.Created) >= rotationInterval && cfg.meta.Replaced.IsZero() {
246 // key is due for rotation and it hasn't been replaced yet; do that now
@@ -311,16 +330,27 @@ func (ech *ECH) updateKeyList() {
330 }
331
332 // publishECHConfigs publishes any configs that are configured for publication and which haven't been published already.
314 -func (t *TLS) publishECHConfigs(logger *zap.Logger) error {
333 +func (t *TLS) publishECHConfigs(ctx context.Context, logger *zap.Logger) error {
334 + if err := ctx.Err(); err != nil {
335 + return err
336 + }
337 + if t.ctx.Context == nil {
338 + return nil
339 + }
340 // make publication exclusive, since we don't need to repeat this unnecessarily
341 storage := t.ctx.Storage()
342 + if storage == nil {
343 + return nil
344 + }
345 const echLockName = "ech_publish"
318 - if err := storage.Lock(t.ctx, echLockName); err != nil {
346 + if err := storage.Lock(ctx, echLockName); err != nil {
347 return err
348 }
349 defer func() {
322 - if err := storage.Unlock(t.ctx, echLockName); err != nil {
323 - logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
350 + if err := storage.Unlock(context.WithoutCancel(ctx), echLockName); err != nil {
351 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
352 + logger.Error("unable to unlock ECH provisioning in storage", zap.Error(err))
353 + }
354 }
355 }()
356
@@ -446,7 +476,7 @@ func (t *TLS) publishECHConfigs(logger *zap.Logger) error {
476
477 // publish this ECH config list with this publisher
478 pubTime := time.Now()
449 - err := publisher.PublishECHConfigList(t.ctx, dnsNamesToPublish, echCfgListBin)
479 + err := publisher.PublishECHConfigList(ctx, dnsNamesToPublish, echCfgListBin)
480
481 var publishErrs PublishECHConfigListErrors
482 if errors.As(err, &publishErrs) {
modules/caddytls/storage_clean_test.go new
+471
@@ -0,0 +1,471 @@
1 +// Copyright 2015 Matthew Holt and The Caddy Authors
2 +//
3 +// Licensed under the Apache License, Version 2.0 (the "License");
4 +// you may not use this file except in compliance with the License.
5 +// You may obtain a copy of the License at
6 +//
7 +// http://www.apache.org/licenses/LICENSE-2.0
8 +//
9 +// Unless required by applicable law or agreed to in writing, software
10 +// distributed under the License is distributed on an "AS IS" BASIS,
11 +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 +// See the License for the specific language governing permissions and
13 +// limitations under the License.
14 +
15 +package caddytls
16 +
17 +import (
18 + "context"
19 + "io/fs"
20 + "sync"
21 + "testing"
22 + "time"
23 +
24 + "github.com/caddyserver/caddy/v2"
25 + "github.com/caddyserver/certmagic"
26 + "github.com/cloudflare/circl/hpke"
27 + "go.uber.org/zap"
28 +)
29 +
30 +var (
31 + testStorageMu sync.Mutex
32 + activeTestStorage certmagic.Storage
33 +)
34 +
35 +type testStorageModule struct{}
36 +
37 +func (testStorageModule) CaddyModule() caddy.ModuleInfo {
38 + return caddy.ModuleInfo{
39 + ID: "caddy.storage.test_blocking",
40 + New: func() caddy.Module { return new(testStorageModule) },
41 + }
42 +}
43 +
44 +func (testStorageModule) CertMagicStorage() (certmagic.Storage, error) {
45 + testStorageMu.Lock()
46 + defer testStorageMu.Unlock()
47 + return activeTestStorage, nil
48 +}
49 +
50 +func init() {
51 + caddy.RegisterModule(testStorageModule{})
52 +}
53 +
54 +func newTestContextWithStorage(t *testing.T, storage certmagic.Storage) (caddy.Context, context.CancelFunc) {
55 + testStorageMu.Lock()
56 + activeTestStorage = storage
57 + testStorageMu.Unlock()
58 +
59 + cfg := &caddy.Config{
60 + StorageRaw: []byte(`{"module": "test_blocking"}`),
61 + }
62 + ctx, err := caddy.ProvisionContext(cfg)
63 + if err != nil {
64 + t.Fatalf("caddy.ProvisionContext failed: %v", err)
65 + }
66 + return ctx, func() {
67 + testStorageMu.Lock()
68 + activeTestStorage = nil
69 + testStorageMu.Unlock()
70 + }
71 +}
72 +
73 +type blockingStorage struct {
74 + lockFunc func(ctx context.Context, key string) error
75 + unlockFunc func(ctx context.Context, key string) error
76 + storeFunc func(ctx context.Context, key string, value []byte) error
77 + loadFunc func(ctx context.Context, key string) ([]byte, error)
78 + deleteFunc func(ctx context.Context, key string) error
79 + existsFunc func(ctx context.Context, key string) bool
80 + listFunc func(ctx context.Context, path string, recursive bool) ([]string, error)
81 + statFunc func(ctx context.Context, key string) (certmagic.KeyInfo, error)
82 +}
83 +
84 +func (s *blockingStorage) Lock(ctx context.Context, key string) error {
85 + if s.lockFunc != nil {
86 + return s.lockFunc(ctx, key)
87 + }
88 + return nil
89 +}
90 +
91 +func (s *blockingStorage) Unlock(ctx context.Context, key string) error {
92 + if s.unlockFunc != nil {
93 + return s.unlockFunc(ctx, key)
94 + }
95 + return nil
96 +}
97 +
98 +func (s *blockingStorage) Store(ctx context.Context, key string, value []byte) error {
99 + if s.storeFunc != nil {
100 + return s.storeFunc(ctx, key, value)
101 + }
102 + return nil
103 +}
104 +
105 +func (s *blockingStorage) Load(ctx context.Context, key string) ([]byte, error) {
106 + if s.loadFunc != nil {
107 + return s.loadFunc(ctx, key)
108 + }
109 + return nil, fs.ErrNotExist
110 +}
111 +
112 +func (s *blockingStorage) Delete(ctx context.Context, key string) error {
113 + if s.deleteFunc != nil {
114 + return s.deleteFunc(ctx, key)
115 + }
116 + return nil
117 +}
118 +
119 +func (s *blockingStorage) Exists(ctx context.Context, key string) bool {
120 + if s.existsFunc != nil {
121 + return s.existsFunc(ctx, key)
122 + }
123 + return false
124 +}
125 +
126 +func (s *blockingStorage) List(ctx context.Context, path string, recursive bool) ([]string, error) {
127 + if s.listFunc != nil {
128 + return s.listFunc(ctx, path, recursive)
129 + }
130 + return nil, nil
131 +}
132 +
133 +func (s *blockingStorage) Stat(ctx context.Context, key string) (certmagic.KeyInfo, error) {
134 + if s.statFunc != nil {
135 + return s.statFunc(ctx, key)
136 + }
137 + return certmagic.KeyInfo{}, fs.ErrNotExist
138 +}
139 +
140 +type blockingPublisher struct {
141 + publishFunc func(ctx context.Context, innerNames []string, echConfigList []byte) error
142 +}
143 +
144 +func (p *blockingPublisher) PublisherKey() string {
145 + return "blocking_test_publisher"
146 +}
147 +
148 +func (p *blockingPublisher) PublishECHConfigList(ctx context.Context, innerNames []string, echConfigList []byte) error {
149 + if p.publishFunc != nil {
150 + return p.publishFunc(ctx, innerNames, echConfigList)
151 + }
152 + return nil
153 +}
154 +
155 +func TestTLSStorageCleanStopSynchronization(t *testing.T) {
156 + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
157 + defer cancel()
158 +
159 + tlsApp := &TLS{
160 + ctx: ctx,
161 + logger: zap.NewNop(),
162 + Automation: &AutomationConfig{
163 + StorageCleanInterval: caddy.Duration(1 * time.Hour),
164 + },
165 + }
166 +
167 + // Start storage cleaner
168 + tlsApp.keepStorageClean()
169 +
170 + // Stop must cancel cleaner context and wait for completion cleanly
171 + err := tlsApp.Stop()
172 + if err != nil {
173 + t.Fatalf("TLS.Stop failed: %v", err)
174 + }
175 +}
176 +
177 +func TestTLSStorageCleanUnitsCanceledContext(t *testing.T) {
178 + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
179 + cancel() // canceled immediately
180 +
181 + tlsApp := &TLS{
182 + ctx: ctx,
183 + logger: zap.NewNop(),
184 + }
185 +
186 + // cleanStorageUnits with canceled context should return immediately
187 + tlsApp.cleanStorageUnits(ctx.Context)
188 +}
189 +
190 +func TestTLSECHStopSynchronization(t *testing.T) {
191 + ctx, cancel := caddy.NewContext(caddy.Context{Context: context.Background()})
192 + defer cancel()
193 +
194 + tlsApp := &TLS{
195 + ctx: ctx,
196 + logger: zap.NewNop(),
197 + Automation: &AutomationConfig{},
198 + EncryptedClientHello: &ECH{},
199 + DisableStorageClean: true,
200 + }
201 +
202 + err := tlsApp.Start()
203 + if err != nil {
204 + t.Fatalf("TLS.Start failed: %v", err)
205 + }
206 +
207 + // Stop must cancel ECH context and wait for background ECH worker to exit
208 + err = tlsApp.Stop()
209 + if err != nil {
210 + t.Fatalf("TLS.Stop failed: %v", err)
211 + }
212 +}
213 +
214 +func TestTLSStorageCleanBlockingStorageStopSynchronization(t *testing.T) {
215 + startedLock := make(chan struct{})
216 + storage := &blockingStorage{
217 + lockFunc: func(ctx context.Context, key string) error {
218 + if key == "storage_clean" {
219 + close(startedLock)
220 + <-ctx.Done()
221 + return ctx.Err()
222 + }
223 + return nil
224 + },
225 + }
226 +
227 + ctx, cleanup := newTestContextWithStorage(t, storage)
228 + defer cleanup()
229 +
230 + tlsApp := &TLS{
231 + ctx: ctx,
232 + logger: zap.NewNop(),
233 + Automation: &AutomationConfig{
234 + StorageCleanInterval: caddy.Duration(1 * time.Hour),
235 + },
236 + }
237 +
238 + // Reset storageClean timestamp so cleaning runs
239 + storageCleanMu.Lock()
240 + storageClean = time.Time{}
241 + storageCleanMu.Unlock()
242 +
243 + tlsApp.keepStorageClean()
244 +
245 + select {
246 + case <-startedLock:
247 + case <-time.After(5 * time.Second):
248 + t.Fatal("timed out waiting for storage lock to be attempted")
249 + }
250 +
251 + stopped := make(chan error, 1)
252 + go func() {
253 + stopped <- tlsApp.Stop()
254 + }()
255 +
256 + select {
257 + case err := <-stopped:
258 + if err != nil {
259 + t.Fatalf("TLS.Stop failed: %v", err)
260 + }
261 + case <-time.After(5 * time.Second):
262 + t.Fatal("TLS.Stop hung while storage cleaner was blocked on storage lock")
263 + }
264 +}
265 +
266 +func TestTLSECHPublishBlockingStorageStopSynchronization(t *testing.T) {
267 + startedLock := make(chan struct{})
268 + storage := &blockingStorage{
269 + lockFunc: func(ctx context.Context, key string) error {
270 + if key == "ech_publish" {
271 + close(startedLock)
272 + <-ctx.Done()
273 + return ctx.Err()
274 + }
275 + return nil
276 + },
277 + }
278 +
279 + ctx, cleanup := newTestContextWithStorage(t, storage)
280 + defer cleanup()
281 +
282 + ech := &ECH{
283 + configsMu: new(sync.RWMutex),
284 + Configs: []ECHConfiguration{{PublicName: "example.com"}},
285 + }
286 + tlsApp := &TLS{
287 + ctx: ctx,
288 + logger: zap.NewNop(),
289 + Automation: &AutomationConfig{},
290 + EncryptedClientHello: ech,
291 + DisableStorageClean: true,
292 + }
293 +
294 + err := tlsApp.Start()
295 + if err != nil {
296 + t.Fatalf("TLS.Start failed: %v", err)
297 + }
298 +
299 + select {
300 + case <-startedLock:
301 + case <-time.After(5 * time.Second):
302 + t.Fatal("timed out waiting for ech_publish storage lock to be attempted")
303 + }
304 +
305 + stopped := make(chan error, 1)
306 + go func() {
307 + stopped <- tlsApp.Stop()
308 + }()
309 +
310 + select {
311 + case err := <-stopped:
312 + if err != nil {
313 + t.Fatalf("TLS.Stop failed: %v", err)
314 + }
315 + case <-time.After(5 * time.Second):
316 + t.Fatal("TLS.Stop hung while ECH publisher was blocked on ech_publish storage lock")
317 + }
318 +}
319 +
320 +func TestTLSECHPublishBlockingPublisherStopSynchronization(t *testing.T) {
321 + startedPublish := make(chan struct{})
322 + publisher := &blockingPublisher{
323 + publishFunc: func(ctx context.Context, innerNames []string, echConfigList []byte) error {
324 + close(startedPublish)
325 + <-ctx.Done()
326 + return ctx.Err()
327 + },
328 + }
329 +
330 + storage := &blockingStorage{}
331 + ctx, cleanup := newTestContextWithStorage(t, storage)
332 + defer cleanup()
333 +
334 + publicKey, _, err := hpke.KEM_X25519_HKDF_SHA256.Scheme().GenerateKeyPair()
335 + if err != nil {
336 + t.Fatalf("GenerateKeyPair failed: %v", err)
337 + }
338 +
339 + ech := &ECH{
340 + configsMu: new(sync.RWMutex),
341 + Configs: []ECHConfiguration{{PublicName: "example.com"}},
342 + Publication: []*ECHPublication{
343 + {
344 + Domains: []string{"example.com"},
345 + publishers: []ECHPublisher{publisher},
346 + },
347 + },
348 + configs: map[string][]echConfig{
349 + "example.com": {
350 + {
351 + PublicKey: publicKey,
352 + Version: draftTLSESNI25,
353 + ConfigID: 1,
354 + RawPublicName: "example.com",
355 + KEMID: hpke.KEM_X25519_HKDF_SHA256,
356 + CipherSuites: []hpkeSymmetricCipherSuite{
357 + {
358 + KDFID: hpke.KDF_HKDF_SHA256,
359 + AEADID: hpke.AEAD_AES128GCM,
360 + },
361 + },
362 + meta: echConfigMeta{
363 + Publications: make(publicationHistory),
364 + },
365 + },
366 + },
367 + },
368 + }
369 + tlsApp := &TLS{
370 + ctx: ctx,
371 + logger: zap.NewNop(),
372 + Automation: &AutomationConfig{},
373 + EncryptedClientHello: ech,
374 + DisableStorageClean: true,
375 + serverNames: map[string]serverNameRegistration{
376 + "example.com": {},
377 + },
378 + }
379 +
380 + err = tlsApp.Start()
381 + if err != nil {
382 + t.Fatalf("TLS.Start failed: %v", err)
383 + }
384 +
385 + select {
386 + case <-startedPublish:
387 + case <-time.After(5 * time.Second):
388 + t.Fatal("timed out waiting for ECH publication to be attempted")
389 + }
390 +
391 + stopped := make(chan error, 1)
392 + go func() {
393 + stopped <- tlsApp.Stop()
394 + }()
395 +
396 + select {
397 + case err := <-stopped:
398 + if err != nil {
399 + t.Fatalf("TLS.Stop failed: %v", err)
400 + }
401 + case <-time.After(5 * time.Second):
402 + t.Fatal("TLS.Stop hung while ECH publisher was blocked inside PublishECHConfigList")
403 + }
404 +}
405 +
406 +func TestTLSECHRotateBlockingStorageStopSynchronization(t *testing.T) {
407 + startedLock := make(chan struct{})
408 + storage := &blockingStorage{
409 + lockFunc: func(ctx context.Context, key string) error {
410 + if key == echStorageLockName {
411 + close(startedLock)
412 + <-ctx.Done()
413 + return ctx.Err()
414 + }
415 + return nil
416 + },
417 + }
418 +
419 + ctx, cleanup := newTestContextWithStorage(t, storage)
420 + defer cleanup()
421 +
422 + ech := &ECH{
423 + configsMu: new(sync.RWMutex),
424 + Configs: []ECHConfiguration{{PublicName: "example.com"}},
425 + configs: map[string][]echConfig{
426 + "example.com": {
427 + {
428 + ConfigID: 1,
429 + RawPublicName: "example.com",
430 + meta: echConfigMeta{
431 + Created: time.Now().Add(-24 * time.Hour * 31), // > 30 days old so rotationNeeded returns true
432 + Publications: make(publicationHistory),
433 + },
434 + },
435 + },
436 + },
437 + }
438 + tlsApp := &TLS{
439 + ctx: ctx,
440 + logger: zap.NewNop(),
441 + Automation: &AutomationConfig{},
442 + EncryptedClientHello: ech,
443 + DisableStorageClean: true,
444 + echRotateInterval: 5 * time.Millisecond,
445 + }
446 +
447 + err := tlsApp.Start()
448 + if err != nil {
449 + t.Fatalf("TLS.Start failed: %v", err)
450 + }
451 +
452 + select {
453 + case <-startedLock:
454 + case <-time.After(5 * time.Second):
455 + t.Fatal("timed out waiting for echStorageLockName storage lock to be attempted during rotation")
456 + }
457 +
458 + stopped := make(chan error, 1)
459 + go func() {
460 + stopped <- tlsApp.Stop()
461 + }()
462 +
463 + select {
464 + case err := <-stopped:
465 + if err != nil {
466 + t.Fatalf("TLS.Stop failed: %v", err)
467 + }
468 + case <-time.After(5 * time.Second):
469 + t.Fatal("TLS.Stop hung while ECH key rotation was blocked on storage lock (context was likely ignored)")
470 + }
471 +}
modules/caddytls/tls.go
+111 -28
@@ -18,6 +18,7 @@ import (
18 "context"
19 "crypto/tls"
20 "encoding/json"
21 + "errors"
22 "fmt"
23 "log"
24 "net"
@@ -136,8 +137,11 @@ type TLS struct {
137 certificateLoaders []CertificateLoader
138 automateNames map[string]struct{}
139 ctx caddy.Context
140 + bgCtx context.Context
141 + bgCancel context.CancelFunc
142 + bgWg *sync.WaitGroup
143 storageCleanTicker *time.Ticker
140 - storageCleanStop chan struct{}
144 + echRotateInterval time.Duration
145 logger *zap.Logger
146 events *caddyevents.App
147
@@ -411,6 +415,17 @@ func (t *TLS) Start() error {
415 }
416 }
417
418 + if t.bgCtx == nil {
419 + parentCtx := t.ctx.Context
420 + if parentCtx == nil {
421 + parentCtx = context.Background()
422 + }
423 + t.bgCtx, t.bgCancel = context.WithCancel(parentCtx)
424 + }
425 + if t.bgWg == nil {
426 + t.bgWg = new(sync.WaitGroup)
427 + }
428 +
429 // now that we are running, and all manual certificates have
430 // been loaded, time to load the automated/managed certificates
431 err := t.Manage(t.automateNames)
@@ -423,34 +438,57 @@ func (t *TLS) Start() error {
438
439 // publish ECH configs in the background; does not need to block
440 // server startup, as it could take a while; then keep keys rotated
426 - go func() {
441 + t.bgWg.Add(1)
442 + go func(ctx context.Context) {
443 + defer t.bgWg.Done()
444 + defer func() {
445 + if err := recover(); err != nil {
446 + log.Printf("[PANIC] tls ech publisher: %v\n%s", err, debug.Stack())
447 + }
448 + }()
449 +
450 // publish immediately first
428 - if err := t.publishECHConfigs(echLogger); err != nil {
429 - echLogger.Error("publication(s) failed", zap.Error(err))
451 + if err := t.publishECHConfigs(ctx, echLogger); err != nil {
452 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
453 + echLogger.Error("publication(s) failed", zap.Error(err))
454 + }
455 }
456
457 + ticker := time.NewTicker(t.echRotationInterval())
458 + defer ticker.Stop()
459 +
460 // then every so often, rotate and publish if needed
461 // (both of these functions only do something if needed)
462 for {
463 select {
436 - case <-time.After(1 * time.Hour):
464 + case <-ticker.C:
465 // ensure old keys are rotated out
466 t.EncryptedClientHello.configsMu.Lock()
439 - err = t.EncryptedClientHello.rotateECHKeys(t.ctx, echLogger, false)
467 + err := t.EncryptedClientHello.rotateECHKeys(t.caddyContext(ctx), echLogger, false)
468 t.EncryptedClientHello.configsMu.Unlock()
469 if err != nil {
442 - echLogger.Error("rotating ECH configs failed", zap.Error(err))
470 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
471 + echLogger.Error("rotating ECH configs failed", zap.Error(err))
472 + }
473 + if ctx.Err() != nil {
474 + return
475 + }
476 continue
477 }
445 - err := t.publishECHConfigs(echLogger)
478 + if ctx.Err() != nil {
479 + return
480 + }
481 + err = t.publishECHConfigs(ctx, echLogger)
482 if err != nil {
447 - echLogger.Error("publication(s) failed", zap.Error(err))
483 + if !errors.Is(err, context.Canceled) && ctx.Err() == nil {
484 + echLogger.Error("publication(s) failed", zap.Error(err))
485 + }
486 }
449 - case <-t.ctx.Done():
487 + case <-ctx.Done():
488 return
489 }
490 }
453 - }()
491 + }(t.bgCtx)
492 }
493
494 if !t.DisableStorageClean {
@@ -464,13 +502,18 @@ func (t *TLS) Start() error {
502
503 // Stop stops the TLS module and cleans up any allocations.
504 func (t *TLS) Stop() error {
467 - // stop the storage cleaner goroutine and ticker
468 - if t.storageCleanStop != nil {
469 - close(t.storageCleanStop)
505 + // cancel all background goroutines (storage cleaner, ECH rotation/publication, etc.)
506 + if t.bgCancel != nil {
507 + t.bgCancel()
508 }
509 if t.storageCleanTicker != nil {
510 t.storageCleanTicker.Stop()
511 }
512 + // wait for all background goroutines to finish before returning,
513 + // ensuring no background storage users are active when module Cleanup() runs
514 + if t.bgWg != nil {
515 + t.bgWg.Wait()
516 + }
517 return nil
518 }
519
@@ -907,30 +950,45 @@ func (t *TLS) HasCertificateForSubject(subject string) bool {
950 // known storage units if it was not recently done, and then runs the
951 // operation at every tick from t.storageCleanTicker.
952 func (t *TLS) keepStorageClean() {
953 + if t.bgCtx == nil {
954 + parentCtx := t.ctx.Context
955 + if parentCtx == nil {
956 + parentCtx = context.Background()
957 + }
958 + t.bgCtx, t.bgCancel = context.WithCancel(parentCtx)
959 + }
960 + if t.bgWg == nil {
961 + t.bgWg = new(sync.WaitGroup)
962 + }
963 t.storageCleanTicker = time.NewTicker(t.storageCleanInterval())
911 - t.storageCleanStop = make(chan struct{})
912 - go func() {
964 + t.bgWg.Add(1)
965 + go func(ctx context.Context) {
966 + defer t.bgWg.Done()
967 defer func() {
968 if err := recover(); err != nil {
969 log.Printf("[PANIC] storage cleaner: %v\n%s", err, debug.Stack())
970 }
971 }()
918 - t.cleanStorageUnits()
972 + t.cleanStorageUnits(ctx)
973 for {
974 select {
921 - case <-t.storageCleanStop:
975 + case <-ctx.Done():
976 return
977 case <-t.storageCleanTicker.C:
924 - t.cleanStorageUnits()
978 + t.cleanStorageUnits(ctx)
979 }
980 }
927 - }()
981 + }(t.bgCtx)
982 }
983
930 -func (t *TLS) cleanStorageUnits() {
984 +func (t *TLS) cleanStorageUnits(ctx context.Context) {
985 storageCleanMu.Lock()
986 defer storageCleanMu.Unlock()
987
988 + if err := ctx.Err(); err != nil {
989 + return
990 + }
991 +
992 // TODO: This check might not be needed anymore now that CertMagic syncs
993 // and throttles storage cleaning globally across the cluster.
994 // The original comment below might be outdated:
@@ -963,12 +1021,17 @@ func (t *TLS) cleanStorageUnits() {
1021 }
1022
1023 // start with the default/global storage
966 - err = certmagic.CleanStorage(t.ctx, t.ctx.Storage(), options)
967 - if err != nil {
968 - // probably don't want to return early, since we should still
969 - // see if any other storages can get cleaned up
970 - if c := t.logger.Check(zapcore.ErrorLevel, "could not clean default/global storage"); c != nil {
971 - c.Write(zap.Error(err))
1024 + if storage := t.ctx.Storage(); storage != nil {
1025 + err = certmagic.CleanStorage(ctx, storage, options)
1026 + if err != nil {
1027 + if errors.Is(err, context.Canceled) || ctx.Err() != nil {
1028 + return
1029 + }
1030 + // probably don't want to return early, since we should still
1031 + // see if any other storages can get cleaned up
1032 + if c := t.logger.Check(zapcore.ErrorLevel, "could not clean default/global storage"); c != nil {
1033 + c.Write(zap.Error(err))
1034 + }
1035 }
1036 }
1037
@@ -978,7 +1041,10 @@ func (t *TLS) cleanStorageUnits() {
1041 if ap.storage == nil {
1042 continue
1043 }
981 - if err := certmagic.CleanStorage(t.ctx, ap.storage, options); err != nil {
1044 + if err := certmagic.CleanStorage(ctx, ap.storage, options); err != nil {
1045 + if errors.Is(err, context.Canceled) || ctx.Err() != nil {
1046 + return
1047 + }
1048 if c := t.logger.Check(zapcore.ErrorLevel, "could not clean storage configured in automation policy"); c != nil {
1049 c.Write(zap.Error(err))
1050 }
@@ -986,6 +1052,10 @@ func (t *TLS) cleanStorageUnits() {
1052 }
1053 }
1054
1055 + if err := ctx.Err(); err != nil {
1056 + return
1057 + }
1058 +
1059 // remember last time storage was finished cleaning
1060 storageClean = time.Now()
1061
@@ -999,6 +1069,19 @@ func (t *TLS) storageCleanInterval() time.Duration {
1069 return defaultStorageCleanInterval
1070 }
1071
1072 +func (t *TLS) echRotationInterval() time.Duration {
1073 + if t.echRotateInterval > 0 {
1074 + return t.echRotateInterval
1075 + }
1076 + return 1 * time.Hour
1077 +}
1078 +
1079 +func (t *TLS) caddyContext(ctx context.Context) caddy.Context {
1080 + caddyCtx := t.ctx
1081 + caddyCtx.Context = ctx
1082 + return caddyCtx
1083 +}
1084 +
1085 // onEvent translates CertMagic events into Caddy events then dispatches them.
1086 func (t *TLS) onEvent(ctx context.Context, eventName string, data map[string]any) error {
1087 evt := t.events.Emit(t.ctx, eventName, data)