Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
httpcaddyfile: Revise automation policy generation (#3824)
* httpcaddyfile: Revise automation policy generation This should fix a frustrating edge case where wildcard subjects are used, which potentially get shadowed by more specific versions of themselves; see the new tests for an example. This change is motivated by an actual customer requirement. Although all the tests pass, this logic is incredibly complex and nuanced, and I'm worried it is not correct. But it took me about 4 days to get this far on a solution. I did my best. * Fix typo
Matt Holt committed
Oct 28, 2020 at 20:36 UTC
db4f1c02772dfd1f50bd745b322be1b60c72ac41
3 files changed
+301
-130
caddyconfig/httpcaddyfile/tlsapp.go
+165
-130
@@ -76,30 +76,35 @@ func (st ServerType) buildTLSApp(
76
}
77
}
78
79
+ // a catch-all automation policy is used as a "default" for all subjects that
80
+ // don't have custom configuration explicitly associated with them; this
81
+ // is only to add if the global settings or defaults are non-empty
82
catchAllAP, err := newBaseAutomationPolicy(options, warnings, false)
83
if err != nil {
84
return nil, warnings, err
85
}
86
+ if catchAllAP != nil {
87
+ if tlsApp.Automation == nil {
88
+ tlsApp.Automation = new(caddytls.AutomationConfig)
89
+ }
90
+ tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, catchAllAP)
91
+ }
92
93
for _, p := range pairings {
94
for _, sblock := range p.serverBlocks {
95
// get values that populate an automation policy for this block
87
- var ap *caddytls.AutomationPolicy
96
+ ap, err := newBaseAutomationPolicy(options, warnings, true)
97
+ if err != nil {
98
+ return nil, warnings, err
99
+ }
100
101
sblockHosts := sblock.hostsFromKeys(false)
90
- if len(sblockHosts) == 0 {
102
+ if len(sblockHosts) == 0 && catchAllAP != nil {
103
ap = catchAllAP
104
}
105
106
// on-demand tls
107
if _, ok := sblock.pile["tls.on_demand"]; ok {
96
- if ap == nil {
97
- var err error
98
- ap, err = newBaseAutomationPolicy(options, warnings, true)
99
- if err != nil {
100
- return nil, warnings, err
101
- }
102
- }
108
ap.OnDemand = true
109
}
110
@@ -107,13 +112,6 @@ func (st ServerType) buildTLSApp(
112
if issuerVals, ok := sblock.pile["tls.cert_issuer"]; ok {
113
for _, issuerVal := range issuerVals {
114
issuer := issuerVal.Value.(certmagic.Issuer)
110
- if ap == nil {
111
- var err error
112
- ap, err = newBaseAutomationPolicy(options, warnings, true)
113
- if err != nil {
114
- return nil, warnings, err
115
- }
116
- }
115
if ap == catchAllAP && !reflect.DeepEqual(ap.Issuer, issuer) {
116
return nil, warnings, fmt.Errorf("automation policy from site block is also default/catch-all policy because of key without hostname, and the two are in conflict: %#v != %#v", ap.Issuer, issuer)
117
}
@@ -123,15 +121,6 @@ func (st ServerType) buildTLSApp(
121
122
// custom bind host
123
for _, cfgVal := range sblock.pile["bind"] {
126
- // either an existing issuer is already configured (and thus, ap is not
127
- // nil), or we need to configure an issuer, so we need ap to be non-nil
128
- if ap == nil {
129
- ap, err = newBaseAutomationPolicy(options, warnings, true)
130
- if err != nil {
131
- return nil, warnings, err
132
- }
133
- }
134
-
124
// if an issuer was already configured and it is NOT an ACME
125
// issuer, skip, since we intend to adjust only ACME issuers
126
var acmeIssuer *caddytls.ACMEIssuer
@@ -164,87 +153,74 @@ func (st ServerType) buildTLSApp(
153
ap.Issuer = acmeIssuer // we'll encode it later
154
}
155
167
- if ap != nil {
168
- if ap.Issuer != nil {
169
- // encode issuer now that it's all set up
170
- issuerName := ap.Issuer.(caddy.Module).CaddyModule().ID.Name()
171
- ap.IssuerRaw = caddyconfig.JSONModuleObject(ap.Issuer, "module", issuerName, &warnings)
156
+ // first make sure this block is allowed to create an automation policy;
157
+ // doing so is forbidden if it has a key with no host (i.e. ":443")
158
+ // and if there is a different server block that also has a key with no
159
+ // host -- since a key with no host matches any host, we need its
160
+ // associated automation policy to have an empty Subjects list, i.e. no
161
+ // host filter, which is indistinguishable between the two server blocks
162
+ // because automation is not done in the context of a particular server...
163
+ // this is an example of a poor mapping from Caddyfile to JSON but that's
164
+ // the least-leaky abstraction I could figure out
165
+ if len(sblockHosts) == 0 {
166
+ if serverBlocksWithTLSHostlessKey > 1 {
167
+ // this server block and at least one other has a key with no host,
168
+ // making the two indistinguishable; it is misleading to define such
169
+ // a policy within one server block since it actually will apply to
170
+ // others as well
171
+ return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other TLS-enabled server block addresses lacking a host")
172
}
173
-
174
- // first make sure this block is allowed to create an automation policy;
175
- // doing so is forbidden if it has a key with no host (i.e. ":443")
176
- // and if there is a different server block that also has a key with no
177
- // host -- since a key with no host matches any host, we need its
178
- // associated automation policy to have an empty Subjects list, i.e. no
179
- // host filter, which is indistinguishable between the two server blocks
180
- // because automation is not done in the context of a particular server...
181
- // this is an example of a poor mapping from Caddyfile to JSON but that's
182
- // the least-leaky abstraction I could figure out
183
- if len(sblockHosts) == 0 {
184
- if serverBlocksWithTLSHostlessKey > 1 {
185
- // this server block and at least one other has a key with no host,
186
- // making the two indistinguishable; it is misleading to define such
187
- // a policy within one server block since it actually will apply to
188
- // others as well
189
- return nil, warnings, fmt.Errorf("cannot make a TLS automation policy from a server block that has a host-less address when there are other TLS-enabled server block addresses lacking a host")
190
- }
191
- if catchAllAP == nil {
192
- // this server block has a key with no hosts, but there is not yet
193
- // a catch-all automation policy (probably because no global options
194
- // were set), so this one becomes it
195
- catchAllAP = ap
196
- }
173
+ if catchAllAP == nil {
174
+ // this server block has a key with no hosts, but there is not yet
175
+ // a catch-all automation policy (probably because no global options
176
+ // were set), so this one becomes it
177
+ catchAllAP = ap
178
}
179
+ }
180
199
- // associate our new automation policy with this server block's hosts,
200
- // unless, of course, the server block has a key with no hosts, in which
201
- // case its automation policy becomes or blends with the default/global
202
- // automation policy because, of necessity, it applies to all hostnames
203
- // (i.e. it has no Subjects filter) -- in that case, we'll append it last
204
- if ap != catchAllAP {
205
- ap.Subjects = sblockHosts
206
-
207
- // if a combination of public and internal names were given
208
- // for this same server block and no issuer was specified, we
209
- // need to separate them out in the automation policies so
210
- // that the internal names can use the internal issuer and
211
- // the other names can use the default/public/ACME issuer
212
- var ap2 *caddytls.AutomationPolicy
213
- if ap.Issuer == nil {
214
- var internal, external []string
215
- for _, s := range ap.Subjects {
216
- if !certmagic.SubjectQualifiesForCert(s) {
217
- return nil, warnings, fmt.Errorf("subject does not qualify for certificate: '%s'", s)
218
- }
219
- // we don't use certmagic.SubjectQualifiesForPublicCert() because of one nuance:
220
- // names like *.*.tld that may not qualify for a public certificate are actually
221
- // fine when used with OnDemand, since OnDemand (currently) does not obtain
222
- // wildcards (if it ever does, there will be a separate config option to enable
223
- // it that we would need to check here) since the hostname is known at handshake;
224
- // and it is unexpected to switch to internal issuer when the user wants to get
225
- // regular certificates on-demand for a class of certs like *.*.tld.
226
- if !certmagic.SubjectIsIP(s) && !certmagic.SubjectIsInternal(s) && (strings.Count(s, "*.") < 2 || ap.OnDemand) {
227
- external = append(external, s)
228
- } else {
229
- internal = append(internal, s)
230
- }
231
- }
232
- if len(external) > 0 && len(internal) > 0 {
233
- ap.Subjects = external
234
- apCopy := *ap
235
- ap2 = &apCopy
236
- ap2.Subjects = internal
237
- ap2.IssuerRaw = caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)
238
- }
181
+ // associate our new automation policy with this server block's hosts
182
+ ap.Subjects = sblockHosts
183
+ sort.Strings(ap.Subjects) // solely for deterministic test results
184
+
185
+ // if a combination of public and internal names were given
186
+ // for this same server block and no issuer was specified, we
187
+ // need to separate them out in the automation policies so
188
+ // that the internal names can use the internal issuer and
189
+ // the other names can use the default/public/ACME issuer
190
+ var ap2 *caddytls.AutomationPolicy
191
+ if ap.Issuer == nil {
192
+ var internal, external []string
193
+ for _, s := range ap.Subjects {
194
+ if !certmagic.SubjectQualifiesForCert(s) {
195
+ return nil, warnings, fmt.Errorf("subject does not qualify for certificate: '%s'", s)
196
}
240
- if tlsApp.Automation == nil {
241
- tlsApp.Automation = new(caddytls.AutomationConfig)
242
- }
243
- tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap)
244
- if ap2 != nil {
245
- tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap2)
197
+ // we don't use certmagic.SubjectQualifiesForPublicCert() because of one nuance:
198
+ // names like *.*.tld that may not qualify for a public certificate are actually
199
+ // fine when used with OnDemand, since OnDemand (currently) does not obtain
200
+ // wildcards (if it ever does, there will be a separate config option to enable
201
+ // it that we would need to check here) since the hostname is known at handshake;
202
+ // and it is unexpected to switch to internal issuer when the user wants to get
203
+ // regular certificates on-demand for a class of certs like *.*.tld.
204
+ if !certmagic.SubjectIsIP(s) && !certmagic.SubjectIsInternal(s) && (strings.Count(s, "*.") < 2 || ap.OnDemand) {
205
+ external = append(external, s)
206
+ } else {
207
+ internal = append(internal, s)
208
}
209
}
210
+ if len(external) > 0 && len(internal) > 0 {
211
+ ap.Subjects = external
212
+ apCopy := *ap
213
+ ap2 = &apCopy
214
+ ap2.Subjects = internal
215
+ ap2.IssuerRaw = caddyconfig.JSONModuleObject(caddytls.InternalIssuer{}, "module", "internal", &warnings)
216
+ }
217
+ }
218
+ if tlsApp.Automation == nil {
219
+ tlsApp.Automation = new(caddytls.AutomationConfig)
220
+ }
221
+ tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap)
222
+ if ap2 != nil {
223
+ tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, ap2)
224
}
225
226
// certificate loaders
@@ -319,23 +295,17 @@ func (st ServerType) buildTLSApp(
295
tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, internalAP)
296
}
297
322
- // if there is a global/catch-all automation policy, ensure it goes last
323
- if catchAllAP != nil {
324
- // first, encode its issuer, if there is one
325
- if catchAllAP.Issuer != nil {
326
- issuerName := catchAllAP.Issuer.(caddy.Module).CaddyModule().ID.Name()
327
- catchAllAP.IssuerRaw = caddyconfig.JSONModuleObject(catchAllAP.Issuer, "module", issuerName, &warnings)
328
- }
329
-
330
- // then append it to the end of the policies list
331
- if tlsApp.Automation == nil {
332
- tlsApp.Automation = new(caddytls.AutomationConfig)
298
+ // finalize and verify policies; do cleanup
299
+ if tlsApp.Automation != nil {
300
+ // encode any issuer values we created, so they will be rendered in the output
301
+ for _, ap := range tlsApp.Automation.Policies {
302
+ if ap.Issuer != nil && ap.IssuerRaw == nil {
303
+ // encode issuer now that it's all set up
304
+ issuerName := ap.Issuer.(caddy.Module).CaddyModule().ID.Name()
305
+ ap.IssuerRaw = caddyconfig.JSONModuleObject(ap.Issuer, "module", issuerName, &warnings)
306
+ }
307
}
334
- tlsApp.Automation.Policies = append(tlsApp.Automation.Policies, catchAllAP)
335
- }
308
337
- // do a little verification & cleanup
338
- if tlsApp.Automation != nil {
309
// consolidate automation policies that are the exact same
310
tlsApp.Automation.Policies = consolidateAutomationPolicies(tlsApp.Automation.Policies)
311
@@ -351,6 +321,14 @@ func (st ServerType) buildTLSApp(
321
automationHostSet[s] = struct{}{}
322
}
323
}
324
+
325
+ // if nothing remains, remove any excess values to clean up the resulting config
326
+ if len(tlsApp.Automation.Policies) == 0 {
327
+ tlsApp.Automation.Policies = nil
328
+ }
329
+ if reflect.DeepEqual(tlsApp.Automation, new(caddytls.AutomationConfig)) {
330
+ tlsApp.Automation = nil
331
+ }
332
}
333
334
return tlsApp, warnings, nil
@@ -448,12 +426,30 @@ func disambiguateACMEIssuer(acmeIssuer *caddytls.ACMEIssuer) certmagic.Issuer {
426
// consolidateAutomationPolicies combines automation policies that are the same,
427
// for a cleaner overall output.
428
func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls.AutomationPolicy {
429
+ // sort from most specific to least specific; we depend on this ordering
430
+ sort.SliceStable(aps, func(i, j int) bool {
431
+ if automationPolicyIsSubset(aps[i], aps[j]) {
432
+ return true
433
+ }
434
+ if automationPolicyIsSubset(aps[j], aps[i]) {
435
+ return false
436
+ }
437
+ return len(aps[i].Subjects) > len(aps[j].Subjects)
438
+ })
439
+
440
+ // remove any empty policies (except subjects, of course)
441
+ emptyAP := new(caddytls.AutomationPolicy)
442
for i := 0; i < len(aps); i++ {
452
- for j := 0; j < len(aps); j++ {
453
- if j == i {
454
- continue
455
- }
443
+ emptyAP.Subjects = aps[i].Subjects
444
+ if reflect.DeepEqual(aps[i], emptyAP) {
445
+ aps = append(aps[:i], aps[i+1:]...)
446
+ i--
447
+ }
448
+ }
449
450
+ // remove or combine duplicate policies
451
+ for i := 0; i < len(aps); i++ {
452
+ for j := i + 1; j < len(aps); j++ {
453
// if they're exactly equal in every way, just keep one of them
454
if reflect.DeepEqual(aps[i], aps[j]) {
455
aps = append(aps[:j], aps[j+1:]...)
@@ -473,10 +469,17 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
469
aps[i].KeyType == aps[j].KeyType &&
470
aps[i].OnDemand == aps[j].OnDemand &&
471
aps[i].RenewalWindowRatio == aps[j].RenewalWindowRatio {
476
- if len(aps[i].Subjects) == 0 && len(aps[j].Subjects) > 0 {
477
- aps = append(aps[:j], aps[j+1:]...)
478
- } else if len(aps[i].Subjects) > 0 && len(aps[j].Subjects) == 0 {
479
- aps = append(aps[:i], aps[i+1:]...)
472
+ if len(aps[i].Subjects) > 0 && len(aps[j].Subjects) == 0 {
473
+ // later policy (at j) has no subjects ("catch-all"), so we can
474
+ // remove the identical-but-more-specific policy that comes first
475
+ // AS LONG AS it is not shadowed by another policy before it; e.g.
476
+ // if policy i is for example.com, policy i+1 is '*.com', and policy
477
+ // j is catch-all, we cannot remove policy i because that would
478
+ // cause example.com to be served by the less specific policy for
479
+ // '*.com', which might be different (yes we've seen this happen)
480
+ if automationPolicyShadows(i, aps) >= j {
481
+ aps = append(aps[:i], aps[i+1:]...)
482
+ }
483
} else {
484
// avoid repeated subjects
485
for _, subj := range aps[j].Subjects {
@@ -486,16 +489,48 @@ func consolidateAutomationPolicies(aps []*caddytls.AutomationPolicy) []*caddytls
489
}
490
aps = append(aps[:j], aps[j+1:]...)
491
}
489
- i--
490
- break
492
}
493
}
494
}
495
495
- // ensure any catch-all policies go last
496
- sort.SliceStable(aps, func(i, j int) bool {
497
- return len(aps[i].Subjects) > len(aps[j].Subjects)
498
- })
499
-
496
return aps
497
}
498
+
499
+// automationPolicyIsSubset returns true if a's subjects are a subset
500
+// of b's subjects.
501
+func automationPolicyIsSubset(a, b *caddytls.AutomationPolicy) bool {
502
+ if len(b.Subjects) == 0 {
503
+ return true
504
+ }
505
+ if len(a.Subjects) == 0 {
506
+ return false
507
+ }
508
+ for _, aSubj := range a.Subjects {
509
+ var inSuperset bool
510
+ for _, bSubj := range b.Subjects {
511
+ if certmagic.MatchWildcard(aSubj, bSubj) {
512
+ inSuperset = true
513
+ break
514
+ }
515
+ }
516
+ if !inSuperset {
517
+ return false
518
+ }
519
+ }
520
+ return true
521
+}
522
+
523
+// automationPolicyShadows returns the index of a policy that aps[i] shadows;
524
+// in other words, for all policies after position i, if that policy covers
525
+// the same subjects but is less specific, that policy's position is returned,
526
+// or -1 if no shadowing is found. For example, if policy i is for
527
+// "foo.example.com" and policy i+2 is for "*.example.com", then i+2 will be
528
+// returned, since that policy is shadowed by i, which is in front.
529
+func automationPolicyShadows(i int, aps []*caddytls.AutomationPolicy) int {
530
+ for j := i + 1; j < len(aps); j++ {
531
+ if automationPolicyIsSubset(aps[i], aps[j]) {
532
+ return j
533
+ }
534
+ }
535
+ return -1
536
+}
caddyconfig/httpcaddyfile/tlsapp_test.go
new
+56
@@ -0,0 +1,56 @@
1
+package httpcaddyfile
2
+
3
+import (
4
+ "testing"
5
+
6
+ "github.com/caddyserver/caddy/v2/modules/caddytls"
7
+)
8
+
9
+func TestAutomationPolicyIsSubset(t *testing.T) {
10
+ for i, test := range []struct {
11
+ a, b []string
12
+ expect bool
13
+ }{
14
+ {
15
+ a: []string{"example.com"},
16
+ b: []string{},
17
+ expect: true,
18
+ },
19
+ {
20
+ a: []string{},
21
+ b: []string{"example.com"},
22
+ expect: false,
23
+ },
24
+ {
25
+ a: []string{"foo.example.com"},
26
+ b: []string{"*.example.com"},
27
+ expect: true,
28
+ },
29
+ {
30
+ a: []string{"foo.example.com"},
31
+ b: []string{"foo.example.com"},
32
+ expect: true,
33
+ },
34
+ {
35
+ a: []string{"foo.example.com"},
36
+ b: []string{"example.com"},
37
+ expect: false,
38
+ },
39
+ {
40
+ a: []string{"example.com", "foo.example.com"},
41
+ b: []string{"*.com", "*.*.com"},
42
+ expect: true,
43
+ },
44
+ {
45
+ a: []string{"example.com", "foo.example.com"},
46
+ b: []string{"*.com"},
47
+ expect: false,
48
+ },
49
+ } {
50
+ apA := &caddytls.AutomationPolicy{Subjects: test.a}
51
+ apB := &caddytls.AutomationPolicy{Subjects: test.b}
52
+ if actual := automationPolicyIsSubset(apA, apB); actual != test.expect {
53
+ t.Errorf("Test %d: Expected %t but got %t (A: %v B: %v)", i, test.expect, actual, test.a, test.b)
54
+ }
55
+ }
56
+}
caddytest/integration/caddyfile_adapt/tls_automation_policies.txt
new
+80
@@ -0,0 +1,80 @@
1
+{
2
+ local_certs
3
+}
4
+
5
+*.tld, *.*.tld {
6
+ tls {
7
+ on_demand
8
+ }
9
+}
10
+
11
+foo.tld, www.foo.tld {
12
+}
13
+----------
14
+{
15
+ "apps": {
16
+ "http": {
17
+ "servers": {
18
+ "srv0": {
19
+ "listen": [
20
+ ":443"
21
+ ],
22
+ "routes": [
23
+ {
24
+ "match": [
25
+ {
26
+ "host": [
27
+ "foo.tld",
28
+ "www.foo.tld"
29
+ ]
30
+ }
31
+ ],
32
+ "terminal": true
33
+ },
34
+ {
35
+ "match": [
36
+ {
37
+ "host": [
38
+ "*.tld",
39
+ "*.*.tld"
40
+ ]
41
+ }
42
+ ],
43
+ "terminal": true
44
+ }
45
+ ]
46
+ }
47
+ }
48
+ },
49
+ "tls": {
50
+ "automation": {
51
+ "policies": [
52
+ {
53
+ "subjects": [
54
+ "foo.tld",
55
+ "www.foo.tld"
56
+ ],
57
+ "issuer": {
58
+ "module": "internal"
59
+ }
60
+ },
61
+ {
62
+ "subjects": [
63
+ "*.*.tld",
64
+ "*.tld"
65
+ ],
66
+ "issuer": {
67
+ "module": "internal"
68
+ },
69
+ "on_demand": true
70
+ },
71
+ {
72
+ "issuer": {
73
+ "module": "internal"
74
+ }
75
+ }
76
+ ]
77
+ }
78
+ }
79
+ }
80
+}
\ No newline at end of file