@marcel91200 / caddy Mirror · read-only / commits / f35a7fa4
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 14, 05:15 UTC

encode,staticfiles: Content negotiation, precompressed files (#4045)

* encode: implement prefer setting * encode: minimum_length configurable via caddyfile * encode: configurable content-types which to encode * file_server: support precompressed files * encode: use ReponseMatcher for conditional encoding of content * linting error & documentation of encode.PrecompressedOrder * encode: allow just one response matcher also change the namespace of the encoders back, I accidently changed to precompressed >.> default matchers include a * to match to any charset, that may be appended * rounding of the PR * added integration tests for new caddyfile directives * improved various doc strings (punctuation and typos) * added json tag for file_server precompress order and encode matcher * file_server: add vary header, remove accept-ranges when serving precompressed files * encode: move Suffix implementation to precompressed modules

Steffen Brüheim committed Mar 30, 2021 at 02:47 UTC f35a7fa466ffb06c38dcb3216e30c13aa8e14ce5
12 files changed +768 -49
caddytest/integration/caddyfile_adapt/encode_options.txt new
+66
@@ -0,0 +1,66 @@
1 +:80
2 +
3 +encode gzip zstd {
4 + minimum_length 256
5 + prefer zstd gzip
6 + match {
7 + status 2xx 4xx 500
8 + header Content-Type text/*
9 + header Content-Type application/json*
10 + header Content-Type application/javascript*
11 + header Content-Type application/xhtml+xml*
12 + header Content-Type application/atom+xml*
13 + header Content-Type application/rss+xml*
14 + header Content-Type image/svg+xml*
15 + }
16 +}
17 +----------
18 +{
19 + "apps": {
20 + "http": {
21 + "servers": {
22 + "srv0": {
23 + "listen": [
24 + ":80"
25 + ],
26 + "routes": [
27 + {
28 + "handle": [
29 + {
30 + "encodings": {
31 + "gzip": {},
32 + "zstd": {}
33 + },
34 + "handler": "encode",
35 + "match": {
36 + "headers": {
37 + "Content-Type": [
38 + "text/*",
39 + "application/json*",
40 + "application/javascript*",
41 + "application/xhtml+xml*",
42 + "application/atom+xml*",
43 + "application/rss+xml*",
44 + "image/svg+xml*"
45 + ]
46 + },
47 + "status_code": [
48 + 2,
49 + 4,
50 + 500
51 + ]
52 + },
53 + "minimum_length": 256,
54 + "prefer": [
55 + "zstd",
56 + "gzip"
57 + ]
58 + }
59 + ]
60 + }
61 + ]
62 + }
63 + }
64 + }
65 + }
66 +}
caddytest/integration/caddyfile_adapt/file_server_precompressed.txt new
+41
@@ -0,0 +1,41 @@
1 +:80
2 +
3 +file_server {
4 + precompressed zstd br gzip
5 +}
6 +----------
7 +{
8 + "apps": {
9 + "http": {
10 + "servers": {
11 + "srv0": {
12 + "listen": [
13 + ":80"
14 + ],
15 + "routes": [
16 + {
17 + "handle": [
18 + {
19 + "handler": "file_server",
20 + "hide": [
21 + "./Caddyfile"
22 + ],
23 + "precompressed": {
24 + "br": {},
25 + "gzip": {},
26 + "zstd": {}
27 + },
28 + "precompressed_order": [
29 + "zstd",
30 + "br",
31 + "gzip"
32 + ]
33 + }
34 + ]
35 + }
36 + ]
37 + }
38 + }
39 + }
40 + }
41 +}
caddytest/integration/caddyfile_adapt_test.go
+7
@@ -1,7 +1,10 @@
1 package integration
2
3 import (
4 + jsonMod "encoding/json"
5 + "fmt"
6 "io/ioutil"
7 + "path/filepath"
8 "regexp"
9 "strings"
10 "testing"
@@ -39,6 +42,10 @@ func TestCaddyfileAdaptToJSON(t *testing.T) {
42 // replace windows newlines in the json with unix newlines
43 json = winNewlines.ReplaceAllString(json, "\n")
44
45 + // replace os-specific default path for file_server's hide field
46 + replacePath, _ := jsonMod.Marshal(fmt.Sprint(".", string(filepath.Separator), "Caddyfile"))
47 + json = strings.ReplaceAll(json, `"./Caddyfile"`, string(replacePath))
48 +
49 // run the test
50 ok := caddytest.CompareAdapt(t, filename, caddyfile, "caddyfile", json)
51 if !ok {
modules/caddyhttp/encode/brotli/brotli_precompressed.go new
+31
@@ -0,0 +1,31 @@
1 +package caddybrotli
2 +
3 +import (
4 + "github.com/caddyserver/caddy/v2"
5 + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
6 +)
7 +
8 +func init() {
9 + caddy.RegisterModule(BrotliPrecompressed{})
10 +}
11 +
12 +// BrotliPrecompressed provides the file extension for files precompressed with brotli encoding.
13 +type BrotliPrecompressed struct{}
14 +
15 +// CaddyModule returns the Caddy module information.
16 +func (BrotliPrecompressed) CaddyModule() caddy.ModuleInfo {
17 + return caddy.ModuleInfo{
18 + ID: "http.precompressed.br",
19 + New: func() caddy.Module { return new(BrotliPrecompressed) },
20 + }
21 +}
22 +
23 +// AcceptEncoding returns the name of the encoding as
24 +// used in the Accept-Encoding request headers.
25 +func (BrotliPrecompressed) AcceptEncoding() string { return "br" }
26 +
27 +// Suffix returns the filename suffix of precompressed files.
28 +func (BrotliPrecompressed) Suffix() string { return ".br" }
29 +
30 +// Interface guards
31 +var _ encode.Precompressed = (*BrotliPrecompressed)(nil)
modules/caddyhttp/encode/caddyfile.go
+122 -17
@@ -15,7 +15,9 @@
15 package encode
16
17 import (
18 - "fmt"
18 + "net/http"
19 + "strconv"
20 + "strings"
21
22 "github.com/caddyserver/caddy/v2"
23 "github.com/caddyserver/caddy/v2/caddyconfig"
@@ -40,21 +42,31 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
42 // UnmarshalCaddyfile sets up the handler from Caddyfile tokens. Syntax:
43 //
44 // encode [<matcher>] <formats...> {
43 -// gzip [<level>]
45 +// gzip [<level>]
46 // zstd
47 +// minimum_length <length>
48 +// prefer <formats...>
49 +// # response matcher block
50 +// match {
51 +// status <code...>
52 +// header <field> [<value>]
53 +// }
54 +// # or response matcher single line syntax
55 +// match [header <field> [<value>]] | [status <code...>]
56 // }
57 //
58 // Specifying the formats on the first line will use those formats' defaults.
59 func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
60 + responseMatchers := make(map[string]caddyhttp.ResponseMatcher)
61 for d.Next() {
62 for _, arg := range d.RemainingArgs() {
63 mod, err := caddy.GetModule("http.encoders." + arg)
64 if err != nil {
53 - return fmt.Errorf("finding encoder module '%s': %v", mod, err)
65 + return d.Errf("finding encoder module '%s': %v", mod, err)
66 }
67 encoding, ok := mod.New().(Encoding)
68 if !ok {
57 - return fmt.Errorf("module %s is not an HTTP encoding", mod)
69 + return d.Errf("module %s is not an HTTP encoding", mod)
70 }
71 if enc.EncodingsRaw == nil {
72 enc.EncodingsRaw = make(caddy.ModuleMap)
@@ -63,25 +75,118 @@ func (enc *Encode) UnmarshalCaddyfile(d *caddyfile.Dispenser) error {
75 }
76
77 for d.NextBlock(0) {
66 - name := d.Val()
67 - modID := "http.encoders." + name
68 - unm, err := caddyfile.UnmarshalModule(d, modID)
69 - if err != nil {
70 - return err
71 - }
72 - encoding, ok := unm.(Encoding)
73 - if !ok {
74 - return fmt.Errorf("module %s is not an HTTP encoding; is %T", modID, unm)
78 + switch d.Val() {
79 + case "minimum_length":
80 + if !d.NextArg() {
81 + return d.ArgErr()
82 + }
83 + minLength, err := strconv.Atoi(d.Val())
84 + if err != nil {
85 + return err
86 + }
87 + enc.MinLength = minLength
88 + case "prefer":
89 + var encs []string
90 + for d.NextArg() {
91 + encs = append(encs, d.Val())
92 + }
93 + if len(encs) == 0 {
94 + return d.ArgErr()
95 + }
96 + enc.Prefer = encs
97 + case "match":
98 + err := enc.parseNamedResponseMatcher(d.NewFromNextSegment(), responseMatchers)
99 + if err != nil {
100 + return err
101 + }
102 + matcher := responseMatchers["match"]
103 + enc.Matcher = &matcher
104 + default:
105 + name := d.Val()
106 + modID := "http.encoders." + name
107 + unm, err := caddyfile.UnmarshalModule(d, modID)
108 + if err != nil {
109 + return err
110 + }
111 + encoding, ok := unm.(Encoding)
112 + if !ok {
113 + return d.Errf("module %s is not an HTTP encoding; is %T", modID, unm)
114 + }
115 + if enc.EncodingsRaw == nil {
116 + enc.EncodingsRaw = make(caddy.ModuleMap)
117 + }
118 + enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
119 }
76 - if enc.EncodingsRaw == nil {
77 - enc.EncodingsRaw = make(caddy.ModuleMap)
78 - }
79 - enc.EncodingsRaw[name] = caddyconfig.JSON(encoding, nil)
120 }
121 }
122
123 return nil
124 }
125
126 +// Parse the tokens of a named response matcher.
127 +//
128 +// match {
129 +// header <field> [<value>]
130 +// status <code...>
131 +// }
132 +//
133 +// Or, single line syntax:
134 +//
135 +// match [header <field> [<value>]] | [status <code...>]
136 +//
137 +func (enc *Encode) parseNamedResponseMatcher(d *caddyfile.Dispenser, matchers map[string]caddyhttp.ResponseMatcher) error {
138 + for d.Next() {
139 + definitionName := d.Val()
140 +
141 + if _, ok := matchers[definitionName]; ok {
142 + return d.Errf("matcher is defined more than once: %s", definitionName)
143 + }
144 +
145 + matcher := caddyhttp.ResponseMatcher{}
146 + for nesting := d.Nesting(); d.NextArg() || d.NextBlock(nesting); {
147 + switch d.Val() {
148 + case "header":
149 + if matcher.Headers == nil {
150 + matcher.Headers = http.Header{}
151 + }
152 +
153 + // reuse the header request matcher's unmarshaler
154 + headerMatcher := caddyhttp.MatchHeader(matcher.Headers)
155 + err := headerMatcher.UnmarshalCaddyfile(d.NewFromNextSegment())
156 + if err != nil {
157 + return err
158 + }
159 +
160 + matcher.Headers = http.Header(headerMatcher)
161 + case "status":
162 + if matcher.StatusCode == nil {
163 + matcher.StatusCode = []int{}
164 + }
165 +
166 + args := d.RemainingArgs()
167 + if len(args) == 0 {
168 + return d.ArgErr()
169 + }
170 +
171 + for _, arg := range args {
172 + if len(arg) == 3 && strings.HasSuffix(arg, "xx") {
173 + arg = arg[:1]
174 + }
175 + statusNum, err := strconv.Atoi(arg)
176 + if err != nil {
177 + return d.Errf("bad status value '%s': %v", arg, err)
178 + }
179 + matcher.StatusCode = append(matcher.StatusCode, statusNum)
180 + }
181 + default:
182 + return d.Errf("unrecognized response matcher %s", d.Val())
183 + }
184 + }
185 +
186 + matchers[definitionName] = matcher
187 + }
188 + return nil
189 +}
190 +
191 // Interface guard
192 var _ caddyfile.Unmarshaler = (*Encode)(nil)
modules/caddyhttp/encode/encode.go
+87 -17
@@ -23,6 +23,7 @@ import (
23 "bytes"
24 "fmt"
25 "io"
26 + "math"
27 "net/http"
28 "sort"
29 "strconv"
@@ -43,12 +44,16 @@ type Encode struct {
44 // will be chosen based on the client's Accept-Encoding header.
45 EncodingsRaw caddy.ModuleMap `json:"encodings,omitempty" caddy:"namespace=http.encoders"`
46
46 - // If the client has no strong preference, choose this encoding. TODO: Not yet implemented
47 - // Prefer []string `json:"prefer,omitempty"`
47 + // If the client has no strong preference, choose these encodings in order.
48 + Prefer []string `json:"prefer,omitempty"`
49
50 // Only encode responses that are at least this many bytes long.
51 MinLength int `json:"minimum_length,omitempty"`
52
53 + // Only encode responses that match against this ResponseMmatcher.
54 + // The default is a collection of text-based Content-Type headers.
55 + Matcher *caddyhttp.ResponseMatcher `json:"match,omitempty"`
56 +
57 writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
58 }
59
@@ -75,11 +80,46 @@ func (enc *Encode) Provision(ctx caddy.Context) error {
80 if enc.MinLength == 0 {
81 enc.MinLength = defaultMinLength
82 }
83 +
84 + if enc.Matcher == nil {
85 + // common text-based content types
86 + enc.Matcher = &caddyhttp.ResponseMatcher{
87 + Headers: http.Header{
88 + "Content-Type": []string{
89 + "text/*",
90 + "application/json*",
91 + "application/javascript*",
92 + "application/xhtml+xml*",
93 + "application/atom+xml*",
94 + "application/rss+xml*",
95 + "image/svg+xml*",
96 + },
97 + },
98 + }
99 + }
100 +
101 + return nil
102 +}
103 +
104 +// Validate ensures that enc's configuration is valid.
105 +func (enc *Encode) Validate() error {
106 + check := make(map[string]bool)
107 + for _, encName := range enc.Prefer {
108 + if _, ok := enc.writerPools[encName]; !ok {
109 + return fmt.Errorf("encoding %s not enabled", encName)
110 + }
111 +
112 + if _, ok := check[encName]; ok {
113 + return fmt.Errorf("encoding %s is duplicated in prefer", encName)
114 + }
115 + check[encName] = true
116 + }
117 +
118 return nil
119 }
120
121 func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
82 - for _, encName := range acceptedEncodings(r) {
122 + for _, encName := range AcceptedEncodings(r, enc.Prefer) {
123 if _, ok := enc.writerPools[encName]; !ok {
124 continue // encoding not offered
125 }
@@ -150,6 +190,11 @@ func (rw *responseWriter) WriteHeader(status int) {
190 rw.statusCode = status
191 }
192
193 +// Match determines, if encoding should be done based on the ResponseMatcher.
194 +func (enc *Encode) Match(rw *responseWriter) bool {
195 + return enc.Matcher.Match(rw.statusCode, rw.Header())
196 +}
197 +
198 // Write writes to the response. If the response qualifies,
199 // it is encoded using the encoder, which is initialized
200 // if not done so already.
@@ -240,7 +285,10 @@ func (rw *responseWriter) Close() error {
285
286 // init should be called before we write a response, if rw.buf has contents.
287 func (rw *responseWriter) init() {
243 - if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength {
288 + if rw.Header().Get("Content-Encoding") == "" &&
289 + rw.buf.Len() >= rw.config.MinLength &&
290 + rw.config.Match(rw) {
291 +
292 rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
293 rw.w.Reset(rw.ResponseWriter)
294 rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
@@ -250,12 +298,14 @@ func (rw *responseWriter) init() {
298 rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
299 }
300
253 -// acceptedEncodings returns the list of encodings that the
254 -// client supports, in descending order of preference. If
301 +// AcceptedEncodings returns the list of encodings that the
302 +// client supports, in descending order of preference.
303 +// The client preference via q-factor and the server
304 +// preference via Prefer setting are taken into account. If
305 // the Sec-WebSocket-Key header is present then non-identity
306 // encodings are not considered. See
307 // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
258 -func acceptedEncodings(r *http.Request) []string {
308 +func AcceptedEncodings(r *http.Request, preferredOrder []string) []string {
309 acceptEncHeader := r.Header.Get("Accept-Encoding")
310 websocketKey := r.Header.Get("Sec-WebSocket-Key")
311 if acceptEncHeader == "" {
@@ -292,18 +342,29 @@ func acceptedEncodings(r *http.Request) []string {
342 continue
343 }
344
345 + // set server preference
346 + prefOrder := -1
347 + for i, p := range preferredOrder {
348 + if encName == p {
349 + prefOrder = len(preferredOrder) - i
350 + break
351 + }
352 + }
353 +
354 prefs = append(prefs, encodingPreference{
296 - encoding: encName,
297 - q: qFactor,
355 + encoding: encName,
356 + q: qFactor,
357 + preferOrder: prefOrder,
358 })
359 }
360
301 - // sort preferences by descending q-factor
302 - sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q })
303 -
304 - // TODO: If no preference, or same pref for all encodings,
305 - // and not websocket, use default encoding ordering (enc.Prefer)
306 - // for those which are accepted by the client
361 + // sort preferences by descending q-factor first, then by preferOrder
362 + sort.Slice(prefs, func(i, j int) bool {
363 + if math.Abs(prefs[i].q-prefs[j].q) < 0.00001 {
364 + return prefs[i].preferOrder > prefs[j].preferOrder
365 + }
366 + return prefs[i].q > prefs[j].q
367 + })
368
369 prefEncNames := make([]string, len(prefs))
370 for i := range prefs {
@@ -315,8 +376,9 @@ func acceptedEncodings(r *http.Request) []string {
376
377 // encodingPreference pairs an encoding with its q-factor.
378 type encodingPreference struct {
318 - encoding string
319 - q float64
379 + encoding string
380 + q float64
381 + preferOrder int
382 }
383
384 // Encoder is a type which can encode a stream of data.
@@ -332,6 +394,13 @@ type Encoding interface {
394 NewEncoder() Encoder
395 }
396
397 +// Precompressed is a type which returns filename suffix of precompressed
398 +// file and Accept-Encoding header to use when serving this file.
399 +type Precompressed interface {
400 + AcceptEncoding() string
401 + Suffix() string
402 +}
403 +
404 var bufPool = sync.Pool{
405 New: func() interface{} {
406 return new(bytes.Buffer)
@@ -344,6 +413,7 @@ const defaultMinLength = 512
413 // Interface guards
414 var (
415 _ caddy.Provisioner = (*Encode)(nil)
416 + _ caddy.Validator = (*Encode)(nil)
417 _ caddyhttp.MiddlewareHandler = (*Encode)(nil)
418 _ caddyhttp.HTTPInterfaces = (*responseWriter)(nil)
419 )
modules/caddyhttp/encode/encode_test.go
+251
@@ -1,6 +1,8 @@
1 package encode
2
3 import (
4 + "net/http"
5 + "sync"
6 "testing"
7 )
8
@@ -10,3 +12,252 @@ func BenchmarkOpenResponseWriter(b *testing.B) {
12 enc.openResponseWriter("test", nil)
13 }
14 }
15 +
16 +func TestPreferOrder(t *testing.T) {
17 + testCases := []struct {
18 + name string
19 + accept string
20 + prefer []string
21 + expected []string
22 + }{
23 + {
24 + name: "PreferOrder(): 4 accept, 3 prefer",
25 + accept: "deflate, gzip, br, zstd",
26 + prefer: []string{"zstd", "br", "gzip"},
27 + expected: []string{"zstd", "br", "gzip", "deflate"},
28 + },
29 + {
30 + name: "PreferOrder(): 2 accept, 3 prefer",
31 + accept: "deflate, zstd",
32 + prefer: []string{"zstd", "br", "gzip"},
33 + expected: []string{"zstd", "deflate"},
34 + },
35 + {
36 + name: "PreferOrder(): 2 accept (1 empty), 3 prefer",
37 + accept: "gzip,,zstd",
38 + prefer: []string{"zstd", "br", "gzip"},
39 + expected: []string{"zstd", "gzip", ""},
40 + },
41 + {
42 + name: "PreferOrder(): 1 accept, 2 prefer",
43 + accept: "gzip",
44 + prefer: []string{"zstd", "gzip"},
45 + expected: []string{"gzip"},
46 + },
47 + {
48 + name: "PreferOrder(): 4 accept (1 duplicate), 1 prefer",
49 + accept: "deflate, gzip, br, br",
50 + prefer: []string{"br"},
51 + expected: []string{"br", "br", "deflate", "gzip"},
52 + },
53 + {
54 + name: "PreferOrder(): empty accept, 0 prefer",
55 + accept: "",
56 + prefer: []string{},
57 + expected: []string{},
58 + },
59 + {
60 + name: "PreferOrder(): empty accept, 1 prefer",
61 + accept: "",
62 + prefer: []string{"gzip"},
63 + expected: []string{},
64 + },
65 + {
66 + name: "PreferOrder(): with q-factor",
67 + accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
68 + prefer: []string{"gzip"},
69 + expected: []string{"zstd", "deflate", "gzip", "br"},
70 + },
71 + {
72 + name: "PreferOrder(): with q-factor, no prefer",
73 + accept: "deflate;q=0.8, gzip;q=0.4, br;q=0.2, zstd",
74 + prefer: []string{},
75 + expected: []string{"zstd", "deflate", "gzip", "br"},
76 + },
77 + {
78 + name: "PreferOrder(): q-factor=0 filtered out",
79 + accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
80 + prefer: []string{"gzip"},
81 + expected: []string{"br", "gzip", "deflate"},
82 + },
83 + {
84 + name: "PreferOrder(): q-factor=0 filtered out, no prefer",
85 + accept: "deflate;q=0.1, gzip;q=0.4, br;q=0.5, zstd;q=0",
86 + prefer: []string{},
87 + expected: []string{"br", "gzip", "deflate"},
88 + },
89 + {
90 + name: "PreferOrder(): with invalid q-factor",
91 + accept: "br, deflate, gzip;q=2, zstd;q=0.1",
92 + prefer: []string{"zstd", "gzip"},
93 + expected: []string{"gzip", "br", "deflate", "zstd"},
94 + },
95 + {
96 + name: "PreferOrder(): with invalid q-factor, no prefer",
97 + accept: "br, deflate, gzip;q=2, zstd;q=0.1",
98 + prefer: []string{},
99 + expected: []string{"br", "deflate", "gzip", "zstd"},
100 + },
101 + }
102 +
103 + enc := new(Encode)
104 + r, _ := http.NewRequest("", "", nil)
105 +
106 + for _, test := range testCases {
107 + t.Run(test.name, func(t *testing.T) {
108 +
109 + if test.accept == "" {
110 + r.Header.Del("Accept-Encoding")
111 + } else {
112 + r.Header.Set("Accept-Encoding", test.accept)
113 + }
114 + enc.Prefer = test.prefer
115 + result := AcceptedEncodings(r, enc.Prefer)
116 + if !sliceEqual(result, test.expected) {
117 + t.Errorf("AcceptedEncodings() actual: %s expected: %s",
118 + result,
119 + test.expected)
120 + }
121 + })
122 + }
123 +}
124 +
125 +func sliceEqual(a, b []string) bool {
126 + if len(a) != len(b) {
127 + return false
128 + }
129 + for i := range a {
130 + if a[i] != b[i] {
131 + return false
132 + }
133 + }
134 + return true
135 +}
136 +
137 +func TestValidate(t *testing.T) {
138 + type testCase struct {
139 + name string
140 + prefer []string
141 + wantErr bool
142 + }
143 +
144 + var err error
145 + var testCases []testCase
146 + enc := new(Encode)
147 +
148 + enc.writerPools = map[string]*sync.Pool{
149 + "zstd": nil,
150 + "gzip": nil,
151 + "br": nil,
152 + }
153 + testCases = []testCase{
154 + {
155 + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with all encoder",
156 + prefer: []string{"zstd", "br", "gzip"},
157 + wantErr: false,
158 + },
159 + {
160 + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 2 out of 3 encoders",
161 + prefer: []string{"br", "gzip"},
162 + wantErr: false,
163 + },
164 + {
165 + name: "ValidatePrefer (zstd, gzip & br enabled): valid order with 1 out of 3 encoders",
166 + prefer: []string{"gzip"},
167 + wantErr: false,
168 + },
169 + {
170 + name: "ValidatePrefer (zstd, gzip & br enabled): 1 duplicated (once) encoder",
171 + prefer: []string{"gzip", "zstd", "gzip"},
172 + wantErr: true,
173 + },
174 + {
175 + name: "ValidatePrefer (zstd, gzip & br enabled): 1 not enabled encoder in prefer list",
176 + prefer: []string{"br", "zstd", "gzip", "deflate"},
177 + wantErr: true,
178 + },
179 + {
180 + name: "ValidatePrefer (zstd, gzip & br enabled): no prefer list",
181 + prefer: []string{},
182 + wantErr: false,
183 + },
184 + }
185 +
186 + for _, test := range testCases {
187 + t.Run(test.name, func(t *testing.T) {
188 + enc.Prefer = test.prefer
189 + err = enc.Validate()
190 + if (err != nil) != test.wantErr {
191 + t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
192 + }
193 + })
194 + }
195 +
196 + enc.writerPools = map[string]*sync.Pool{
197 + "zstd": nil,
198 + "gzip": nil,
199 + }
200 + testCases = []testCase{
201 + {
202 + name: "ValidatePrefer (zstd & gzip enabled): 1 not enabled encoder in prefer list",
203 + prefer: []string{"zstd", "br", "gzip"},
204 + wantErr: true,
205 + },
206 + {
207 + name: "ValidatePrefer (zstd & gzip enabled): 2 not enabled encoder in prefer list",
208 + prefer: []string{"br", "zstd", "gzip", "deflate"},
209 + wantErr: true,
210 + },
211 + {
212 + name: "ValidatePrefer (zstd & gzip enabled): only not enabled encoder in prefer list",
213 + prefer: []string{"deflate", "br", "gzip"},
214 + wantErr: true,
215 + },
216 + {
217 + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (once) encoder in prefer list",
218 + prefer: []string{"gzip", "zstd", "gzip"},
219 + wantErr: true,
220 + },
221 + {
222 + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated (twice) encoder in prefer list",
223 + prefer: []string{"gzip", "zstd", "gzip", "gzip"},
224 + wantErr: true,
225 + },
226 + {
227 + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated encoder in prefer list",
228 + prefer: []string{"zstd", "zstd", "gzip", "gzip"},
229 + wantErr: true,
230 + },
231 + {
232 + name: "ValidatePrefer (zstd & gzip enabled): 1 duplicated not enabled encoder in prefer list",
233 + prefer: []string{"br", "br", "gzip"},
234 + wantErr: true,
235 + },
236 + {
237 + name: "ValidatePrefer (zstd & gzip enabled): 2 duplicated not enabled encoder in prefer list",
238 + prefer: []string{"br", "deflate", "br", "deflate"},
239 + wantErr: true,
240 + },
241 + {
242 + name: "ValidatePrefer (zstd & gzip enabled): valid order zstd first",
243 + prefer: []string{"zstd", "gzip"},
244 + wantErr: false,
245 + },
246 + {
247 + name: "ValidatePrefer (zstd & gzip enabled): valid order gzip first",
248 + prefer: []string{"gzip", "zstd"},
249 + wantErr: false,
250 + },
251 + }
252 +
253 + for _, test := range testCases {
254 + t.Run(test.name, func(t *testing.T) {
255 + enc.Prefer = test.prefer
256 + err = enc.Validate()
257 + if (err != nil) != test.wantErr {
258 + t.Errorf("Validate() error = %v, wantErr = %v", err, test.wantErr)
259 + }
260 + })
261 +
262 + }
263 +}
modules/caddyhttp/encode/gzip/gzip_precompressed.go new
+28
@@ -0,0 +1,28 @@
1 +package caddygzip
2 +
3 +import (
4 + "github.com/caddyserver/caddy/v2"
5 + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
6 +)
7 +
8 +func init() {
9 + caddy.RegisterModule(GzipPrecompressed{})
10 +}
11 +
12 +// GzipPrecompressed provides the file extension for files precompressed with gzip encoding.
13 +type GzipPrecompressed struct {
14 + Gzip
15 +}
16 +
17 +// CaddyModule returns the Caddy module information.
18 +func (GzipPrecompressed) CaddyModule() caddy.ModuleInfo {
19 + return caddy.ModuleInfo{
20 + ID: "http.precompressed.gzip",
21 + New: func() caddy.Module { return new(GzipPrecompressed) },
22 + }
23 +}
24 +
25 +// Suffix returns the filename suffix of precompressed files.
26 +func (GzipPrecompressed) Suffix() string { return ".gz" }
27 +
28 +var _ encode.Precompressed = (*GzipPrecompressed)(nil)
modules/caddyhttp/encode/zstd/zstd_precompressed.go new
+28
@@ -0,0 +1,28 @@
1 +package caddyzstd
2 +
3 +import (
4 + "github.com/caddyserver/caddy/v2"
5 + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
6 +)
7 +
8 +func init() {
9 + caddy.RegisterModule(ZstdPrecompressed{})
10 +}
11 +
12 +// ZstdPrecompressed provides the file extension for files precompressed with zstandard encoding.
13 +type ZstdPrecompressed struct {
14 + Zstd
15 +}
16 +
17 +// CaddyModule returns the Caddy module information.
18 +func (ZstdPrecompressed) CaddyModule() caddy.ModuleInfo {
19 + return caddy.ModuleInfo{
20 + ID: "http.precompressed.zstd",
21 + New: func() caddy.Module { return new(ZstdPrecompressed) },
22 + }
23 +}
24 +
25 +// Suffix returns the filename suffix of precompressed files.
26 +func (ZstdPrecompressed) Suffix() string { return ".zst" }
27 +
28 +var _ encode.Precompressed = (*ZstdPrecompressed)(nil)
modules/caddyhttp/fileserver/caddyfile.go
+27 -4
@@ -19,8 +19,10 @@ import (
19 "strings"
20
21 "github.com/caddyserver/caddy/v2"
22 + "github.com/caddyserver/caddy/v2/caddyconfig"
23 "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
24 "github.com/caddyserver/caddy/v2/modules/caddyhttp"
25 + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
26 "github.com/caddyserver/caddy/v2/modules/caddyhttp/rewrite"
27 )
28
@@ -33,10 +35,11 @@ func init() {
35 // server and configures it with this syntax:
36 //
37 // file_server [<matcher>] [browse] {
36 -// root <path>
37 -// hide <files...>
38 -// index <files...>
39 -// browse [<template_file>]
38 +// root <path>
39 +// hide <files...>
40 +// index <files...>
41 +// browse [<template_file>]
42 +// precompressed <formats...>
43 // }
44 //
45 func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
@@ -77,6 +80,26 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error)
80 }
81 fsrv.Browse = new(Browse)
82 h.Args(&fsrv.Browse.TemplateFile)
83 + case "precompressed":
84 + var order []string
85 + for h.NextArg() {
86 + modID := "http.precompressed." + h.Val()
87 + mod, err := caddy.GetModule(modID)
88 + if err != nil {
89 + return nil, h.Errf("getting module named '%s': %v", modID, err)
90 + }
91 + inst := mod.New()
92 + precompress, ok := inst.(encode.Precompressed)
93 + if !ok {
94 + return nil, h.Errf("module %s is not a precompressor; is %T", modID, inst)
95 + }
96 + if fsrv.PrecompressedRaw == nil {
97 + fsrv.PrecompressedRaw = make(caddy.ModuleMap)
98 + }
99 + fsrv.PrecompressedRaw[h.Val()] = caddyconfig.JSON(precompress, nil)
100 + order = append(order, h.Val())
101 + }
102 + fsrv.PrecompressedOrder = order
103 default:
104 return nil, h.Errf("unknown subdirective '%s'", h.Val())
105 }
modules/caddyhttp/fileserver/staticfiles.go
+79 -11
@@ -31,6 +31,7 @@ import (
31
32 "github.com/caddyserver/caddy/v2"
33 "github.com/caddyserver/caddy/v2/modules/caddyhttp"
34 + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
35 "go.uber.org/zap"
36 )
37
@@ -79,6 +80,16 @@ type FileServer struct {
80 // a 404 error. By default, this is false (disabled).
81 PassThru bool `json:"pass_thru,omitempty"`
82
83 + // Selection of encoders to use to check for precompressed files.
84 + PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"`
85 +
86 + // If the client has no strong preference (q-factor), choose these encodings in order.
87 + // If no order specified here, the first encoding from the Accept-Encoding header
88 + // that both client and server support is used
89 + PrecompressedOrder []string `json:"precompressed_order,omitempty"`
90 +
91 + precompressors map[string]encode.Precompressed
92 +
93 logger *zap.Logger
94 }
95
@@ -129,6 +140,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
140 }
141 }
142
143 + mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw")
144 + if err != nil {
145 + return fmt.Errorf("loading encoder modules: %v", err)
146 + }
147 + for modName, modIface := range mods.(map[string]interface{}) {
148 + p, ok := modIface.(encode.Precompressed)
149 + if !ok {
150 + return fmt.Errorf("module %s is not precompressor", modName)
151 + }
152 + ae := p.AcceptEncoding()
153 + if ae == "" {
154 + return fmt.Errorf("precompressor does not specify an Accept-Encoding value")
155 + }
156 + suffix := p.Suffix()
157 + if suffix == "" {
158 + return fmt.Errorf("precompressor does not specify a Suffix value")
159 + }
160 + if _, ok := fsrv.precompressors[ae]; ok {
161 + return fmt.Errorf("precompressor already added: %s", ae)
162 + }
163 + if fsrv.precompressors == nil {
164 + fsrv.precompressors = make(map[string]encode.Precompressed)
165 + }
166 + fsrv.precompressors[ae] = p
167 + }
168 +
169 return nil
170 }
171
@@ -205,8 +242,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
242 return fsrv.notFound(w, r, next)
243 }
244
208 - // TODO: content negotiation (brotli sidecar files, etc...)
209 -
245 // one last check to ensure the file isn't hidden (we might
246 // have changed the filename from when we last checked)
247 if fileHidden(filename, filesToHide) {
@@ -230,18 +265,51 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
265 }
266 }
267
233 - fsrv.logger.Debug("opening file", zap.String("filename", filename))
268 + var file *os.File
269
235 - // open the file
236 - file, err := fsrv.openFile(filename, w)
237 - if err != nil {
238 - if herr, ok := err.(caddyhttp.HandlerError); ok &&
239 - herr.StatusCode == http.StatusNotFound {
240 - return fsrv.notFound(w, r, next)
270 + // check for precompressed files
271 + for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {
272 + precompress, ok := fsrv.precompressors[ae]
273 + if !ok {
274 + continue
275 + }
276 + compressedFilename := filename + precompress.Suffix()
277 + compressedInfo, err := os.Stat(compressedFilename)
278 + if err != nil || compressedInfo.IsDir() {
279 + fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
280 + continue
281 + }
282 + fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err))
283 + file, err = fsrv.openFile(compressedFilename, w)
284 + if err != nil {
285 + fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err))
286 + if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable {
287 + return err
288 + }
289 + continue
290 + }
291 + defer file.Close()
292 + w.Header().Set("Content-Encoding", ae)
293 + w.Header().Del("Accept-Ranges")
294 + w.Header().Add("Vary", "Accept-Encoding")
295 + break
296 + }
297 +
298 + // no precompressed file found, use the actual file
299 + if file == nil {
300 + fsrv.logger.Debug("opening file", zap.String("filename", filename))
301 +
302 + // open the file
303 + file, err = fsrv.openFile(filename, w)
304 + if err != nil {
305 + if herr, ok := err.(caddyhttp.HandlerError); ok &&
306 + herr.StatusCode == http.StatusNotFound {
307 + return fsrv.notFound(w, r, next)
308 + }
309 + return err // error is already structured
310 }
242 - return err // error is already structured
311 + defer file.Close()
312 }
244 - defer file.Close()
313
314 // set the ETag - note that a conditional If-None-Match request is handled
315 // by http.ServeContent below, which checks against this ETag value
modules/caddyhttp/standard/imports.go
+1
@@ -5,6 +5,7 @@ import (
5 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp"
6 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/caddyauth"
7 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode"
8 + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/brotli"
9 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/gzip"
10 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode/zstd"
11 _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"