@marcel91200 / caddy Mirror · read-only / commits / a479943a
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 13, 15:30 UTC

caddyhttp: Smarter path matching and rewriting (#4948)

Co-authored-by: RussellLuo <luopeng.he@gmail.com>

Matt Holt committed Aug 16, 2022 at 08:48 UTC a479943acd70068c4b80d3a8f4b8dd7ab93ca2ba
7 files changed +614 -112
modules/caddyhttp/caddyhttp.go
+35
@@ -20,6 +20,7 @@ import (
20 "io"
21 "net"
22 "net/http"
23 + "path"
24 "path/filepath"
25 "strconv"
26 "strings"
@@ -244,6 +245,40 @@ func SanitizedPathJoin(root, reqPath string) string {
245 return path
246 }
247
248 +// CleanPath cleans path p according to path.Clean(), but only
249 +// merges repeated slashes if collapseSlashes is true, and always
250 +// preserves trailing slashes.
251 +func CleanPath(p string, collapseSlashes bool) string {
252 + if collapseSlashes {
253 + return cleanPath(p)
254 + }
255 +
256 + // insert an invalid/impossible URI character into each two consecutive
257 + // slashes to expand empty path segments; then clean the path as usual,
258 + // and then remove the remaining temporary characters.
259 + const tmpCh = 0xff
260 + var sb strings.Builder
261 + for i, ch := range p {
262 + if ch == '/' && i > 0 && p[i-1] == '/' {
263 + sb.WriteByte(tmpCh)
264 + }
265 + sb.WriteRune(ch)
266 + }
267 + halfCleaned := cleanPath(sb.String())
268 + halfCleaned = strings.ReplaceAll(halfCleaned, string([]byte{tmpCh}), "")
269 +
270 + return halfCleaned
271 +}
272 +
273 +// cleanPath does path.Clean(p) but preserves any trailing slash.
274 +func cleanPath(p string) string {
275 + cleaned := path.Clean(p)
276 + if cleaned != "/" && strings.HasSuffix(p, "/") {
277 + cleaned = cleaned + "/"
278 + }
279 + return cleaned
280 +}
281 +
282 // tlsPlaceholderWrapper is a no-op listener wrapper that marks
283 // where the TLS listener should be in a chain of listener wrappers.
284 // It should only be used if another listener wrapper must be placed
modules/caddyhttp/caddyhttp_test.go
+57
@@ -92,3 +92,60 @@ func TestSanitizedPathJoin(t *testing.T) {
92 }
93 }
94 }
95 +
96 +func TestCleanPath(t *testing.T) {
97 + for i, tc := range []struct {
98 + input string
99 + mergeSlashes bool
100 + expect string
101 + }{
102 + {
103 + input: "/foo",
104 + expect: "/foo",
105 + },
106 + {
107 + input: "/foo/",
108 + expect: "/foo/",
109 + },
110 + {
111 + input: "//foo",
112 + expect: "//foo",
113 + },
114 + {
115 + input: "//foo",
116 + mergeSlashes: true,
117 + expect: "/foo",
118 + },
119 + {
120 + input: "/foo//bar/",
121 + mergeSlashes: true,
122 + expect: "/foo/bar/",
123 + },
124 + {
125 + input: "/foo/./.././bar",
126 + expect: "/bar",
127 + },
128 + {
129 + input: "/foo//./..//./bar",
130 + expect: "/foo//bar",
131 + },
132 + {
133 + input: "/foo///./..//./bar",
134 + expect: "/foo///bar",
135 + },
136 + {
137 + input: "/foo///./..//.",
138 + expect: "/foo//",
139 + },
140 + {
141 + input: "/foo//./bar",
142 + expect: "/foo//bar",
143 + },
144 + } {
145 + actual := CleanPath(tc.input, tc.mergeSlashes)
146 + if actual != tc.expect {
147 + t.Errorf("Test %d [input='%s' mergeSlashes=%t]: Got '%s', expected '%s'",
148 + i, tc.input, tc.mergeSlashes, actual, tc.expect)
149 + }
150 + }
151 +}
modules/caddyhttp/fileserver/staticfiles.go
+1 -11
@@ -23,7 +23,6 @@ import (
23 weakrand "math/rand"
24 "mime"
25 "net/http"
26 - "net/url"
26 "os"
27 "path"
28 "path/filepath"
@@ -236,16 +235,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
235 filesToHide := fsrv.transformHidePaths(repl)
236
237 root := repl.ReplaceAll(fsrv.Root, ".")
239 - // PathUnescape returns an error if the escapes aren't well-formed,
240 - // meaning the count % matches the RFC. Return early if the escape is
241 - // improper.
242 - if _, err := url.PathUnescape(r.URL.Path); err != nil {
243 - fsrv.logger.Debug("improper path escape",
244 - zap.String("site_root", root),
245 - zap.String("request_path", r.URL.Path),
246 - zap.Error(err))
247 - return err
248 - }
238 +
239 filename := caddyhttp.SanitizedPathJoin(root, r.URL.Path)
240
241 fsrv.logger.Debug("sanitized path join",
modules/caddyhttp/matchers.go
+241 -77
@@ -23,7 +23,6 @@ import (
23 "net/textproto"
24 "net/url"
25 "path"
26 - "path/filepath"
26 "reflect"
27 "regexp"
28 "sort"
@@ -63,20 +62,51 @@ type (
62 // Duplicate entries will return an error.
63 MatchHost []string
64
66 - // MatchPath matches requests by the URI's path (case-insensitive). Path
67 - // matches are exact, but wildcards may be used:
65 + // MatchPath case-insensitively matches requests by the URI's path. Path
66 + // matching is exact, not prefix-based, giving you more control and clarity
67 + // over matching. Wildcards (`*`) may be used:
68 //
69 - // - At the end, for a prefix match (`/prefix/*`)
70 - // - At the beginning, for a suffix match (`*.suffix`)
71 - // - On both sides, for a substring match (`*/contains/*`)
69 + // - At the end only, for a prefix match (`/prefix/*`)
70 + // - At the beginning only, for a suffix match (`*.suffix`)
71 + // - On both sides only, for a substring match (`*/contains/*`)
72 // - In the middle, for a globular match (`/accounts/*/info`)
73 //
74 + // Slashes are significant; i.e. `/foo*` matches `/foo`, `/foo/`, `/foo/bar`,
75 + // and `/foobar`; but `/foo/*` does not match `/foo` or `/foobar`. Valid
76 + // paths start with a slash `/`.
77 + //
78 + // Because there are, in general, multiple possible escaped forms of any
79 + // path, path matchers operate in unescaped space; that is, path matchers
80 + // should be written in their unescaped form to prevent ambiguities and
81 + // possible security issues, as all request paths will be normalized to
82 + // their unescaped forms before matcher evaluation.
83 + //
84 + // However, escape sequences in a match pattern are supported; they are
85 + // compared with the request's raw/escaped path for those bytes only.
86 + // In other words, a matcher of `/foo%2Fbar` will match a request path
87 + // of precisely `/foo%2Fbar`, but not `/foo/bar`. It follows that matching
88 + // the literal percent sign (%) in normalized space can be done using the
89 + // escaped form, `%25`.
90 + //
91 + // Even though wildcards (`*`) operate in the normalized space, the special
92 + // escaped wildcard (`%*`), which is not a valid escape sequence, may be
93 + // used in place of a span that should NOT be decoded; that is, `/bands/%*`
94 + // will match `/bands/AC%2fDC` whereas `/bands/*` will not.
95 + //
96 + // Even though path matching is done in normalized space, the special
97 + // wildcard `%*` may be used in place of a span that should NOT be decoded;
98 + // that is, `/bands/%*/` will match `/bands/AC%2fDC/` whereas `/bands/*/`
99 + // will not.
100 + //
101 // This matcher is fast, so it does not support regular expressions or
102 // capture groups. For slower but more powerful matching, use the
76 - // path_regexp matcher.
103 + // path_regexp matcher. (Note that due to the special treatment of
104 + // escape sequences in matcher patterns, they may perform slightly slower
105 + // in high-traffic environments.)
106 MatchPath []string
107
108 // MatchPathRE matches requests by a regular expression on the URI's path.
109 + // Path matching is performed in the unescaped (decoded) form of the path.
110 //
111 // Upon a match, it adds placeholders to the request: `{http.regexp.name.capture_group}`
112 // where `name` is the regular expression's name, and `capture_group` is either
@@ -303,7 +333,8 @@ outer:
333 // expression matchers.
334 //
335 // Example:
306 -// expression host('localhost')
336 +//
337 +// expression host('localhost')
338 func (MatchHost) CELLibrary(ctx caddy.Context) (cel.Library, error) {
339 return CELMatcherImpl(
340 "host",
@@ -342,6 +373,7 @@ func (MatchPath) CaddyModule() caddy.ModuleInfo {
373 // Provision lower-cases the paths in m to ensure case-insensitive matching.
374 func (m MatchPath) Provision(_ caddy.Context) error {
375 for i := range m {
376 + // TODO: if m[i] == "*", put it first and delete all others (will always match)
377 m[i] = strings.ToLower(m[i])
378 }
379 return nil
@@ -349,77 +381,108 @@ func (m MatchPath) Provision(_ caddy.Context) error {
381
382 // Match returns true if r matches m.
383 func (m MatchPath) Match(r *http.Request) bool {
352 - // PathUnescape returns an error if the escapes aren't
353 - // well-formed, meaning the count % matches the RFC.
354 - // Return early if the escape is improper.
355 - unescapedPath, err := url.PathUnescape(r.URL.Path)
356 - if err != nil {
357 - return false
358 - }
359 -
360 - lowerPath := strings.ToLower(unescapedPath)
361 -
362 - // Clean the path, merges doubled slashes, etc.
363 - // This ensures maliciously crafted requests can't bypass
364 - // the path matcher. See #4407
365 - lowerPath = path.Clean(lowerPath)
366 -
367 - // see #2917; Windows ignores trailing dots and spaces
384 + // Even though RFC 9110 says that path matching is case-sensitive
385 + // (https://www.rfc-editor.org/rfc/rfc9110.html#section-4.2.3),
386 + // we do case-insensitive matching to mitigate security issues
387 + // related to differences between operating systems, applications,
388 + // etc; if case-sensitive matching is needed, the regex matcher
389 + // can be used instead.
390 + reqPath := strings.ToLower(r.URL.Path)
391 +
392 + // See #2917; Windows ignores trailing dots and spaces
393 // when accessing files (sigh), potentially causing a
394 // security risk (cry) if PHP files end up being served
395 // as static files, exposing the source code, instead of
371 - // being matched by *.php to be treated as PHP scripts
372 - lowerPath = strings.TrimRight(lowerPath, ". ")
373 -
374 - // Cleaning may remove the trailing slash, but we want to keep it
375 - if lowerPath != "/" && strings.HasSuffix(r.URL.Path, "/") {
376 - lowerPath = lowerPath + "/"
377 - }
396 + // being matched by *.php to be treated as PHP scripts.
397 + reqPath = strings.TrimRight(reqPath, ". ")
398
399 repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
400
381 - for _, matchPath := range m {
382 - matchPath = repl.ReplaceAll(matchPath, "")
401 + for _, matchPattern := range m {
402 + matchPattern = repl.ReplaceAll(matchPattern, "")
403
404 // special case: whole path is wildcard; this is unnecessary
405 // as it matches all requests, which is the same as no matcher
386 - if matchPath == "*" {
406 + if matchPattern == "*" {
407 return true
408 }
409
390 - // special case: first and last characters are wildcard,
391 - // treat it as a fast substring match
392 - if len(matchPath) > 1 &&
393 - strings.HasPrefix(matchPath, "*") &&
394 - strings.HasSuffix(matchPath, "*") {
395 - if strings.Contains(lowerPath, matchPath[1:len(matchPath)-1]) {
410 + // Clean the path, merge doubled slashes, etc.
411 + // This ensures maliciously crafted requests can't bypass
412 + // the path matcher. See #4407. Good security posture
413 + // requires that we should do all we can to reduce any
414 + // funny-looking paths into "normalized" forms such that
415 + // weird variants can't sneak by.
416 + //
417 + // How we clean the path depends on the kind of pattern:
418 + // we either merge slashes or we don't. If the pattern
419 + // has double slashes, we preserve them in the path.
420 + //
421 + // TODO: Despite the fact that the *vast* majority of path
422 + // matchers have only 1 pattern, a possible optimization is
423 + // to remember the cleaned form of the path for future
424 + // iterations; it's just that the way we clean depends on
425 + // the kind of pattern.
426 +
427 + mergeSlashes := !strings.Contains(matchPattern, "//")
428 +
429 + // if '%' appears in the match pattern, we interpret that to mean
430 + // the intent is to compare that part of the path in raw/escaped
431 + // space; i.e. "%40"=="%40", not "@", and "%2F"=="%2F", not "/"
432 + if strings.Contains(matchPattern, "%") {
433 + reqPathForPattern := CleanPath(r.URL.EscapedPath(), mergeSlashes)
434 + if m.matchPatternWithEscapeSequence(reqPathForPattern, matchPattern) {
435 return true
436 }
437 +
438 + // doing prefix/suffix/substring matches doesn't make sense
439 continue
440 }
441
401 - // special case: first character is a wildcard,
402 - // treat it as a fast suffix match
403 - if strings.HasPrefix(matchPath, "*") {
404 - if strings.HasSuffix(lowerPath, matchPath[1:]) {
442 + reqPathForPattern := CleanPath(reqPath, mergeSlashes)
443 +
444 + // for substring, prefix, and suffix matching, only perform those
445 + // special, fast matches if they are the only wildcards in the pattern;
446 + // otherwise we assume a globular match if any * appears in the middle
447 +
448 + // special case: first and last characters are wildcard,
449 + // treat it as a fast substring match
450 + if strings.Count(matchPattern, "*") == 2 &&
451 + strings.HasPrefix(matchPattern, "*") &&
452 + strings.HasSuffix(matchPattern, "*") &&
453 + strings.Count(matchPattern, "*") == 2 {
454 + if strings.Contains(reqPathForPattern, matchPattern[1:len(matchPattern)-1]) {
455 return true
456 }
457 continue
458 }
459
410 - // special case: last character is a wildcard,
411 - // treat it as a fast prefix match
412 - if strings.HasSuffix(matchPath, "*") {
413 - if strings.HasPrefix(lowerPath, matchPath[:len(matchPath)-1]) {
414 - return true
460 + // only perform prefix/suffix match if it is the only wildcard...
461 + // I think that is more correct most of the time
462 + if strings.Count(matchPattern, "*") == 1 {
463 + // special case: first character is a wildcard,
464 + // treat it as a fast suffix match
465 + if strings.HasPrefix(matchPattern, "*") {
466 + if strings.HasSuffix(reqPathForPattern, matchPattern[1:]) {
467 + return true
468 + }
469 + continue
470 + }
471 +
472 + // special case: last character is a wildcard,
473 + // treat it as a fast prefix match
474 + if strings.HasSuffix(matchPattern, "*") {
475 + if strings.HasPrefix(reqPathForPattern, matchPattern[:len(matchPattern)-1]) {
476 + return true
477 + }
478 + continue
479 }
416 - continue
480 }
481
419 - // for everything else, try globular matching, which also
420 - // is exact matching if there are no glob/wildcard chars;
421 - // can ignore error here because we can't handle it anyway
422 - matches, _ := filepath.Match(matchPath, lowerPath)
482 + // at last, use globular matching, which also is exact matching
483 + // if there are no glob/wildcard chars; we ignore the error here
484 + // because we can't handle it anyway
485 + matches, _ := path.Match(matchPattern, reqPathForPattern)
486 if matches {
487 return true
488 }
@@ -427,11 +490,118 @@ func (m MatchPath) Match(r *http.Request) bool {
490 return false
491 }
492
493 +func (MatchPath) matchPatternWithEscapeSequence(escapedPath, matchPath string) bool {
494 + // We would just compare the pattern against r.URL.Path,
495 + // but the pattern contains %, indicating that we should
496 + // compare at least some part of the path in raw/escaped
497 + // space, not normalized space; so we build the string we
498 + // will compare against by adding the normalized parts
499 + // of the path, then switching to the escaped parts where
500 + // the pattern hints to us wherever % is present.
501 + var sb strings.Builder
502 +
503 + // iterate the pattern and escaped path in lock-step;
504 + // increment iPattern every time we consume a char from the pattern,
505 + // increment iPath every time we consume a char from the path;
506 + // iPattern and iPath are our cursors/iterator positions for each string
507 + var iPattern, iPath int
508 + for {
509 + if iPattern >= len(matchPath) || iPath >= len(escapedPath) {
510 + break
511 + }
512 +
513 + // get the next character from the request path
514 +
515 + pathCh := string(escapedPath[iPath])
516 + var escapedPathCh string
517 +
518 + // normalize (decode) escape sequences
519 + if pathCh == "%" && len(escapedPath) >= iPath+3 {
520 + // hold onto this in case we find out the intent is to match in escaped space here;
521 + // we lowercase it even though technically the spec says: "For consistency, URI
522 + // producers and normalizers should use uppercase hexadecimal digits for all percent-
523 + // encodings" (RFC 3986 section 2.1) - we lowercased the matcher pattern earlier in
524 + // provisioning so we do the same here to gain case-insensitivity in equivalence;
525 + // besides, this string is never shown visibly
526 + escapedPathCh = strings.ToLower(escapedPath[iPath : iPath+3])
527 +
528 + var err error
529 + pathCh, err = url.PathUnescape(escapedPathCh)
530 + if err != nil {
531 + // should be impossible unless EscapedPath() is giving us an invalid sequence!
532 + return false
533 + }
534 + iPath += 2 // escape sequence is 2 bytes longer than normal char
535 + }
536 +
537 + // now get the next character from the pattern
538 +
539 + normalize := true
540 + switch matchPath[iPattern] {
541 + case '%':
542 + // escape sequence
543 +
544 + // if not a wildcard ("%*"), compare literally; consume next two bytes of pattern
545 + if len(matchPath) >= iPattern+3 && matchPath[iPattern+1] != '*' {
546 + sb.WriteString(escapedPathCh)
547 + iPath++
548 + iPattern += 2
549 + break
550 + }
551 +
552 + // escaped wildcard sequence; consume next byte only ('*')
553 + iPattern++
554 + normalize = false
555 +
556 + fallthrough
557 + case '*':
558 + // wildcard, so consume until next matching character
559 + remaining := escapedPath[iPath:]
560 + until := len(escapedPath) - iPath // go until end of string...
561 + if iPattern < len(matchPath)-1 { // ...unless the * is not at the end
562 + nextCh := matchPath[iPattern+1]
563 + until = strings.IndexByte(remaining, nextCh)
564 + if until == -1 {
565 + // terminating char of wildcard span not found, so definitely no match
566 + return false
567 + }
568 + }
569 + if until == 0 {
570 + // empty span; nothing to add on this iteration
571 + break
572 + }
573 + next := remaining[:until]
574 + if normalize {
575 + var err error
576 + next, err = url.PathUnescape(next)
577 + if err != nil {
578 + return false // should be impossible anyway
579 + }
580 + }
581 + sb.WriteString(next)
582 + iPath += until
583 + default:
584 + sb.WriteString(pathCh)
585 + iPath++
586 + }
587 +
588 + iPattern++
589 + }
590 +
591 + // we can now treat rawpath globs (%*) as regular globs (*)
592 + matchPath = strings.ReplaceAll(matchPath, "%*", "*")
593 +
594 + // ignore error here because we can't handle it anyway=
595 + matches, _ := path.Match(matchPath, sb.String())
596 + return matches
597 +}
598 +
599 // CELLibrary produces options that expose this matcher for use in CEL
600 // expression matchers.
601 //
602 // Example:
434 -// expression path('*substring*', '*suffix')
603 +//
604 +// expression path('*substring*', '*suffix')
605 func (MatchPath) CELLibrary(ctx caddy.Context) (cel.Library, error) {
606 return CELMatcherImpl(
607 // name of the macro, this is the function name that users see when writing expressions.
@@ -477,23 +647,10 @@ func (MatchPathRE) CaddyModule() caddy.ModuleInfo {
647 func (m MatchPathRE) Match(r *http.Request) bool {
648 repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
649
480 - // PathUnescape returns an error if the escapes aren't
481 - // well-formed, meaning the count % matches the RFC.
482 - // Return early if the escape is improper.
483 - unescapedPath, err := url.PathUnescape(r.URL.Path)
484 - if err != nil {
485 - return false
486 - }
487 -
650 // Clean the path, merges doubled slashes, etc.
651 // This ensures maliciously crafted requests can't bypass
652 // the path matcher. See #4407
491 - cleanedPath := path.Clean(unescapedPath)
492 -
493 - // Cleaning may remove the trailing slash, but we want to keep it
494 - if cleanedPath != "/" && strings.HasSuffix(r.URL.Path, "/") {
495 - cleanedPath = cleanedPath + "/"
496 - }
653 + cleanedPath := cleanPath(r.URL.Path)
654
655 return m.MatchRegexp.Match(cleanedPath, repl)
656 }
@@ -502,7 +659,8 @@ func (m MatchPathRE) Match(r *http.Request) bool {
659 // expression matchers.
660 //
661 // Example:
505 -// expression path_regexp('^/bar')
662 +//
663 +// expression path_regexp('^/bar')
664 func (MatchPathRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
665 unnamedPattern, err := CELMatcherImpl(
666 "path_regexp",
@@ -575,7 +733,8 @@ func (m MatchMethod) Match(r *http.Request) bool {
733 // expression matchers.
734 //
735 // Example:
578 -// expression method('PUT', 'POST')
736 +//
737 +// expression method('PUT', 'POST')
738 func (MatchMethod) CELLibrary(_ caddy.Context) (cel.Library, error) {
739 return CELMatcherImpl(
740 "method",
@@ -661,7 +820,8 @@ func (m MatchQuery) Match(r *http.Request) bool {
820 // expression matchers.
821 //
822 // Example:
664 -// expression query({'sort': 'asc'}) || query({'foo': ['*bar*', 'baz']})
823 +//
824 +// expression query({'sort': 'asc'}) || query({'foo': ['*bar*', 'baz']})
825 func (MatchQuery) CELLibrary(_ caddy.Context) (cel.Library, error) {
826 return CELMatcherImpl(
827 "query",
@@ -736,8 +896,9 @@ func (m MatchHeader) Match(r *http.Request) bool {
896 // expression matchers.
897 //
898 // Example:
739 -// expression header({'content-type': 'image/png'})
740 -// expression header({'foo': ['bar', 'baz']}) // match bar or baz
899 +//
900 +// expression header({'content-type': 'image/png'})
901 +// expression header({'foo': ['bar', 'baz']}) // match bar or baz
902 func (MatchHeader) CELLibrary(_ caddy.Context) (cel.Library, error) {
903 return CELMatcherImpl(
904 "header",
@@ -894,7 +1055,8 @@ func (m MatchHeaderRE) Validate() error {
1055 // expression matchers.
1056 //
1057 // Example:
897 -// expression header_regexp('foo', 'Field', 'fo+')
1058 +//
1059 +// expression header_regexp('foo', 'Field', 'fo+')
1060 func (MatchHeaderRE) CELLibrary(ctx caddy.Context) (cel.Library, error) {
1061 unnamedPattern, err := CELMatcherImpl(
1062 "header_regexp",
@@ -978,7 +1140,8 @@ func (m *MatchProtocol) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1140 // expression matchers.
1141 //
1142 // Example:
981 -// expression protocol('https')
1143 +//
1144 +// expression protocol('https')
1145 func (MatchProtocol) CELLibrary(_ caddy.Context) (cel.Library, error) {
1146 return CELMatcherImpl(
1147 "protocol",
@@ -1097,7 +1260,8 @@ func (m *MatchRemoteIP) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
1260 // expression matchers.
1261 //
1262 // Example:
1100 -// expression remote_ip('forwarded', '192.168.0.0/16', '172.16.0.0/12', '10.0.0.0/8')
1263 +//
1264 +// expression remote_ip('forwarded', '192.168.0.0/16', '172.16.0.0/12', '10.0.0.0/8')
1265 func (MatchRemoteIP) CELLibrary(ctx caddy.Context) (cel.Library, error) {
1266 return CELMatcherImpl(
1267 // name of the macro, this is the function name that users see when writing expressions.
modules/caddyhttp/matchers_test.go
+147 -6
@@ -158,9 +158,10 @@ func TestHostMatcher(t *testing.T) {
158
159 func TestPathMatcher(t *testing.T) {
160 for i, tc := range []struct {
161 - match MatchPath
162 - input string
163 - expect bool
161 + match MatchPath // not URI-encoded because not parsing from a URI
162 + input string // should be valid URI encoding (escaped) since it will become part of a request
163 + expect bool
164 + provisionErr bool
165 }{
166 {
167 match: MatchPath{},
@@ -252,6 +253,11 @@ func TestPathMatcher(t *testing.T) {
253 input: "/FOOOO",
254 expect: true,
255 },
256 + {
257 + match: MatchPath{"*.php"},
258 + input: "/foo/index.php. .",
259 + expect: true,
260 + },
261 {
262 match: MatchPath{"/foo/bar.txt"},
263 input: "/foo/BAR.txt",
@@ -263,10 +269,60 @@ func TestPathMatcher(t *testing.T) {
269 expect: true,
270 },
271 {
266 - match: MatchPath{"/foo*"},
272 + match: MatchPath{"/foo"},
273 input: "//foo",
274 expect: true,
275 },
276 + {
277 + match: MatchPath{"//foo"},
278 + input: "/foo",
279 + expect: false,
280 + },
281 + {
282 + match: MatchPath{"//foo"},
283 + input: "//foo",
284 + expect: true,
285 + },
286 + {
287 + match: MatchPath{"/foo//*"},
288 + input: "/foo//bar",
289 + expect: true,
290 + },
291 + {
292 + match: MatchPath{"/foo//*"},
293 + input: "/foo/%2Fbar",
294 + expect: true,
295 + },
296 + {
297 + match: MatchPath{"/foo/%2F*"},
298 + input: "/foo/%2Fbar",
299 + expect: true,
300 + },
301 + {
302 + match: MatchPath{"/foo/%2F*"},
303 + input: "/foo//bar",
304 + expect: false,
305 + },
306 + {
307 + match: MatchPath{"/foo//bar"},
308 + input: "/foo//bar",
309 + expect: true,
310 + },
311 + {
312 + match: MatchPath{"/foo/*//bar"},
313 + input: "/foo///bar",
314 + expect: true,
315 + },
316 + {
317 + match: MatchPath{"/foo/%*//bar"},
318 + input: "/foo///bar",
319 + expect: true,
320 + },
321 + {
322 + match: MatchPath{"/foo/%*//bar"},
323 + input: "/foo//%2Fbar",
324 + expect: true,
325 + },
326 {
327 match: MatchPath{"/foo*"},
328 input: "/%2F/foo",
@@ -292,8 +348,79 @@ func TestPathMatcher(t *testing.T) {
348 input: "/foo/bar",
349 expect: true,
350 },
351 + // notice these next three test cases are the same normalized path but are written differently
352 + {
353 + match: MatchPath{"/%25@.txt"},
354 + input: "/%25@.txt",
355 + expect: true,
356 + },
357 + {
358 + match: MatchPath{"/%25@.txt"},
359 + input: "/%25%40.txt",
360 + expect: true,
361 + },
362 + {
363 + match: MatchPath{"/%25%40.txt"},
364 + input: "/%25%40.txt",
365 + expect: true,
366 + },
367 + {
368 + match: MatchPath{"/bands/*/*"},
369 + input: "/bands/AC%2FDC/T.N.T",
370 + expect: false, // because * operates in normalized space
371 + },
372 + {
373 + match: MatchPath{"/bands/%*/%*"},
374 + input: "/bands/AC%2FDC/T.N.T",
375 + expect: true,
376 + },
377 + {
378 + match: MatchPath{"/bands/%*/%*"},
379 + input: "/bands/AC/DC/T.N.T",
380 + expect: false,
381 + },
382 + {
383 + match: MatchPath{"/bands/%*"},
384 + input: "/bands/AC/DC",
385 + expect: false, // not a suffix match
386 + },
387 + {
388 + match: MatchPath{"/bands/%*"},
389 + input: "/bands/AC%2FDC",
390 + expect: true,
391 + },
392 + {
393 + match: MatchPath{"/foo%2fbar/baz"},
394 + input: "/foo%2Fbar/baz",
395 + expect: true,
396 + },
397 + {
398 + match: MatchPath{"/foo%2fbar/baz"},
399 + input: "/foo/bar/baz",
400 + expect: false,
401 + },
402 + {
403 + match: MatchPath{"/foo/bar/baz"},
404 + input: "/foo%2fbar/baz",
405 + expect: true,
406 + },
407 } {
296 - req := &http.Request{URL: &url.URL{Path: tc.input}}
408 + err := tc.match.Provision(caddy.Context{})
409 + if err == nil && tc.provisionErr {
410 + t.Errorf("Test %d %v: Expected error provisioning, but there was no error", i, tc.match)
411 + }
412 + if err != nil && !tc.provisionErr {
413 + t.Errorf("Test %d %v: Expected no error provisioning, but there was an error: %v", i, tc.match, err)
414 + }
415 + if tc.provisionErr {
416 + continue // if it's not supposed to provision properly, pointless to test it
417 + }
418 +
419 + u, err := url.ParseRequestURI(tc.input)
420 + if err != nil {
421 + t.Fatalf("Test %d (%v): Invalid request URI (should be rejected by Go's HTTP server): %v", i, tc.input, err)
422 + }
423 + req := &http.Request{URL: u}
424 repl := caddy.NewReplacer()
425 ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
426 req = req.WithContext(ctx)
@@ -387,6 +514,16 @@ func TestPathREMatcher(t *testing.T) {
514 expect: true,
515 expectRepl: map[string]string{"name.myparam": "bar"},
516 },
517 + {
518 + match: MatchPathRE{MatchRegexp{Pattern: "^/%@.txt"}},
519 + input: "/%25@.txt",
520 + expect: true,
521 + },
522 + {
523 + match: MatchPathRE{MatchRegexp{Pattern: "^/%25@.txt"}},
524 + input: "/%25@.txt",
525 + expect: false,
526 + },
527 } {
528 // compile the regexp and validate its name
529 err := tc.match.Provision(caddy.Context{})
@@ -401,7 +538,11 @@ func TestPathREMatcher(t *testing.T) {
538 }
539
540 // set up the fake request and its Replacer
404 - req := &http.Request{URL: &url.URL{Path: tc.input}}
541 + u, err := url.ParseRequestURI(tc.input)
542 + if err != nil {
543 + t.Fatalf("Test %d: Bad input URI: %v", i, err)
544 + }
545 + req := &http.Request{URL: u}
546 repl := caddy.NewReplacer()
547 ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl)
548 req = req.WithContext(ctx)
modules/caddyhttp/rewrite/rewrite.go
+92 -18
@@ -31,13 +31,25 @@ func init() {
31 caddy.RegisterModule(Rewrite{})
32 }
33
34 -// Rewrite is a middleware which can rewrite HTTP requests.
34 +// Rewrite is a middleware which can rewrite/mutate HTTP requests.
35 //
36 -// The Method and URI properties are "setters": the request URI
37 -// will be set to the given values. Other properties are "modifiers":
38 -// they modify existing files but do not explicitly specify what the
39 -// result will be. It is atypical to combine the use of setters and
36 +// The Method and URI properties are "setters" (the request URI
37 +// will be overwritten with the given values). Other properties are
38 +// "modifiers" (they modify existing values in a differentiable
39 +// way). It is atypical to combine the use of setters and
40 // modifiers in a single rewrite.
41 +//
42 +// To ensure consistent behavior, prefix and suffix stripping is
43 +// performed in the URL-decoded (unescaped, normalized) space by
44 +// default except for the specific bytes where an escape sequence
45 +// is used in the prefix or suffix pattern.
46 +//
47 +// For all modifiers, paths are cleaned before being modified so that
48 +// multiple, consecutive slashes are collapsed into a single slash,
49 +// and dot elements are resolved and removed. In the special case
50 +// of a prefix, suffix, or substring containing "//" (repeated slashes),
51 +// slashes will not be merged while cleaning the path so that
52 +// the rewrite can be interpreted literally.
53 type Rewrite struct {
54 // Changes the request's HTTP verb.
55 Method string `json:"method,omitempty"`
@@ -59,9 +71,15 @@ type Rewrite struct {
71 URI string `json:"uri,omitempty"`
72
73 // Strips the given prefix from the beginning of the URI path.
74 + // The prefix should be written in normalized (unescaped) form,
75 + // but if an escaping (`%xx`) is used, the path will be required
76 + // to have that same escape at that position in order to match.
77 StripPathPrefix string `json:"strip_path_prefix,omitempty"`
78
79 // Strips the given suffix from the end of the URI path.
80 + // The suffix should be written in normalized (unescaped) form,
81 + // but if an escaping (`%xx`) is used, the path will be required
82 + // to have that same escape at that position in order to match.
83 StripPathSuffix string `json:"strip_path_suffix,omitempty"`
84
85 // Performs substring replacements on the URI.
@@ -227,17 +245,18 @@ func (rewr Rewrite) Rewrite(r *http.Request, repl *caddy.Replacer) bool {
245 // strip path prefix or suffix
246 if rewr.StripPathPrefix != "" {
247 prefix := repl.ReplaceAll(rewr.StripPathPrefix, "")
230 - r.URL.RawPath = strings.TrimPrefix(r.URL.RawPath, prefix)
231 - if p, err := url.PathUnescape(r.URL.RawPath); err == nil && p != "" {
232 - r.URL.Path = p
233 - } else {
234 - r.URL.Path = strings.TrimPrefix(r.URL.Path, prefix)
235 - }
248 + mergeSlashes := !strings.Contains(prefix, "//")
249 + changePath(r, func(escapedPath string) string {
250 + escapedPath = caddyhttp.CleanPath(escapedPath, mergeSlashes)
251 + return trimPathPrefix(escapedPath, prefix)
252 + })
253 }
254 if rewr.StripPathSuffix != "" {
255 suffix := repl.ReplaceAll(rewr.StripPathSuffix, "")
239 - changePath(r, func(pathOrRawPath string) string {
240 - return strings.TrimSuffix(pathOrRawPath, suffix)
256 + mergeSlashes := !strings.Contains(suffix, "//")
257 + changePath(r, func(escapedPath string) string {
258 + escapedPath = caddyhttp.CleanPath(escapedPath, mergeSlashes)
259 + return reverse(trimPathPrefix(reverse(escapedPath), reverse(suffix)))
260 })
261 }
262
@@ -324,6 +343,58 @@ func buildQueryString(qs string, repl *caddy.Replacer) string {
343 return sb.String()
344 }
345
346 +// trimPathPrefix is like strings.TrimPrefix, but customized for advanced URI
347 +// path prefix matching. The string prefix will be trimmed from the beginning
348 +// of escapedPath if escapedPath starts with prefix. Rather than a naive 1:1
349 +// comparison of each byte to determine if escapedPath starts with prefix,
350 +// both strings are iterated in lock-step, and if prefix has a '%' encoding
351 +// at a particular position, escapedPath must also have the same encoding
352 +// representation for that character. In other words, if the prefix string
353 +// uses the escaped form for a character, escapedPath must literally use the
354 +// same escape at that position. Otherwise, all character comparisons are
355 +// performed in normalized/unescaped space.
356 +func trimPathPrefix(escapedPath, prefix string) string {
357 + var iPath, iPrefix int
358 + for {
359 + if iPath >= len(escapedPath) || iPrefix >= len(prefix) {
360 + break
361 + }
362 +
363 + prefixCh := prefix[iPrefix]
364 + ch := string(escapedPath[iPath])
365 +
366 + if ch == "%" && prefixCh != '%' && len(escapedPath) >= iPath+3 {
367 + var err error
368 + ch, err = url.PathUnescape(escapedPath[iPath : iPath+3])
369 + if err != nil {
370 + // should be impossible unless EscapedPath() is returning invalid values!
371 + return escapedPath
372 + }
373 + iPath += 2
374 + }
375 +
376 + // prefix comparisons are case-insensitive to consistency with
377 + // path matcher, which is case-insensitive for good reasons
378 + if !strings.EqualFold(ch, string(prefixCh)) {
379 + return escapedPath
380 + }
381 +
382 + iPath++
383 + iPrefix++
384 + }
385 +
386 + // found matching prefix, trim it
387 + return escapedPath[iPath:]
388 +}
389 +
390 +func reverse(s string) string {
391 + r := []rune(s)
392 + for i, j := 0, len(r)-1; i < len(r)/2; i, j = i+1, j-1 {
393 + r[i], r[j] = r[j], r[i]
394 + }
395 + return string(r)
396 +}
397 +
398 // substrReplacer describes either a simple and fast substring replacement.
399 type substrReplacer struct {
400 // A substring to find. Supports placeholders.
@@ -351,8 +422,10 @@ func (rep substrReplacer) do(r *http.Request, repl *caddy.Replacer) {
422 find := repl.ReplaceAll(rep.Find, "")
423 replace := repl.ReplaceAll(rep.Replace, "")
424
425 + mergeSlashes := !strings.Contains(rep.Find, "//")
426 +
427 changePath(r, func(pathOrRawPath string) string {
355 - return strings.Replace(pathOrRawPath, find, replace, lim)
428 + return strings.Replace(caddyhttp.CleanPath(pathOrRawPath, mergeSlashes), find, replace, lim)
429 })
430
431 r.URL.RawQuery = strings.Replace(r.URL.RawQuery, find, replace, lim)
@@ -380,16 +453,17 @@ func (rep regexReplacer) do(r *http.Request, repl *caddy.Replacer) {
453 })
454 }
455
383 -// changePath updates the path on the request URL. It first executes newVal on
384 -// req.URL.RawPath, and if the result is a valid escaping, it will be copied
385 -// into req.URL.Path; otherwise newVal is evaluated only on req.URL.Path.
456 func changePath(req *http.Request, newVal func(pathOrRawPath string) string) {
387 - req.URL.RawPath = newVal(req.URL.RawPath)
457 + req.URL.RawPath = newVal(req.URL.EscapedPath())
458 if p, err := url.PathUnescape(req.URL.RawPath); err == nil && p != "" {
459 req.URL.Path = p
460 } else {
461 req.URL.Path = newVal(req.URL.Path)
462 }
463 + // RawPath is only set if it's different from the normalized Path (std lib)
464 + if req.URL.RawPath == req.URL.Path {
465 + req.URL.RawPath = ""
466 + }
467 }
468
469 // Interface guard
modules/caddyhttp/rewrite/rewrite_test.go
+41
@@ -235,6 +235,42 @@ func TestRewrite(t *testing.T) {
235 input: newRequest(t, "GET", "/foo/prefix/bar"),
236 expect: newRequest(t, "GET", "/foo/prefix/bar"),
237 },
238 + {
239 + rule: Rewrite{StripPathPrefix: "//prefix"},
240 + // scheme and host needed for URL parser to succeed in setting up test
241 + input: newRequest(t, "GET", "http://host//prefix/foo/bar"),
242 + expect: newRequest(t, "GET", "http://host/foo/bar"),
243 + },
244 + {
245 + rule: Rewrite{StripPathPrefix: "//prefix"},
246 + input: newRequest(t, "GET", "/prefix/foo/bar"),
247 + expect: newRequest(t, "GET", "/prefix/foo/bar"),
248 + },
249 + {
250 + rule: Rewrite{StripPathPrefix: "/a%2Fb/c"},
251 + input: newRequest(t, "GET", "/a%2Fb/c/d"),
252 + expect: newRequest(t, "GET", "/d"),
253 + },
254 + {
255 + rule: Rewrite{StripPathPrefix: "/a%2Fb/c"},
256 + input: newRequest(t, "GET", "/a%2fb/c/d"),
257 + expect: newRequest(t, "GET", "/d"),
258 + },
259 + {
260 + rule: Rewrite{StripPathPrefix: "/a/b/c"},
261 + input: newRequest(t, "GET", "/a%2Fb/c/d"),
262 + expect: newRequest(t, "GET", "/d"),
263 + },
264 + {
265 + rule: Rewrite{StripPathPrefix: "/a%2Fb/c"},
266 + input: newRequest(t, "GET", "/a/b/c/d"),
267 + expect: newRequest(t, "GET", "/a/b/c/d"),
268 + },
269 + {
270 + rule: Rewrite{StripPathPrefix: "//a%2Fb/c"},
271 + input: newRequest(t, "GET", "/a/b/c/d"),
272 + expect: newRequest(t, "GET", "/a/b/c/d"),
273 + },
274
275 {
276 rule: Rewrite{StripPathSuffix: "/suffix"},
@@ -251,6 +287,11 @@ func TestRewrite(t *testing.T) {
287 input: newRequest(t, "GET", "/foo%2Fbar/suffix"),
288 expect: newRequest(t, "GET", "/foo%2Fbar/"),
289 },
290 + {
291 + rule: Rewrite{StripPathSuffix: "%2fsuffix"},
292 + input: newRequest(t, "GET", "/foo%2Fbar%2fsuffix"),
293 + expect: newRequest(t, "GET", "/foo%2Fbar"),
294 + },
295 {
296 rule: Rewrite{StripPathSuffix: "/suffix"},
297 input: newRequest(t, "GET", "/foo/suffix/bar"),