@marcel91200 / caddy Mirror · read-only / commits / f5b4f268
Fork 0

Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS

Mirrored from https://github.com/Slazee/caddy.git

Synced Nov 14, 05:15 UTC

Implement encode middleware (#2)

* Implement encode middleware * Add missing break; and add missing JSON struct field tag

Matt Holt committed Jun 10, 2019 at 10:21 UTC f5b4f268dc73078467f19faf6068cc8047cabc7f
4 files changed +347
cmd/caddy2/main.go
+2
@@ -6,6 +6,8 @@ import (
6 // this is where modules get plugged in
7 _ "github.com/caddyserver/caddy2/modules/caddyhttp"
8 _ "github.com/caddyserver/caddy2/modules/caddyhttp/caddylog"
9 + _ "github.com/caddyserver/caddy2/modules/caddyhttp/encode"
10 + _ "github.com/caddyserver/caddy2/modules/caddyhttp/encode/gzip"
11 _ "github.com/caddyserver/caddy2/modules/caddyhttp/fileserver"
12 _ "github.com/caddyserver/caddy2/modules/caddyhttp/headers"
13 _ "github.com/caddyserver/caddy2/modules/caddyhttp/markdown"
modules/caddyhttp/encode/encode.go new
+286
@@ -0,0 +1,286 @@
1 +// Package encode implements an encoder middleware for Caddy. The initial
2 +// enhancements related to Accept-Encoding, minimum content length, and
3 +// buffer/writer pools were adapted from https://github.com/xi2/httpgzip
4 +// then modified heavily to accommodate modular encoders. Code borrowed
5 +// from that repository is Copyright (c) 2015 The Httpgzip Authors.
6 +package encode
7 +
8 +import (
9 + "bytes"
10 + "encoding/json"
11 + "fmt"
12 + "io"
13 + "net/http"
14 + "sort"
15 + "strconv"
16 + "strings"
17 + "sync"
18 +
19 + "github.com/caddyserver/caddy2"
20 + "github.com/caddyserver/caddy2/modules/caddyhttp"
21 +)
22 +
23 +func init() {
24 + caddy2.RegisterModule(caddy2.Module{
25 + Name: "http.middleware.encode",
26 + New: func() interface{} { return new(Encode) },
27 + })
28 +}
29 +
30 +// Encode is a middleware which can encode responses.
31 +type Encode struct {
32 + EncodingsRaw map[string]json.RawMessage `json:"encodings,omitempty"`
33 + Prefer []string `json:"prefer,omitempty"`
34 + MinLength int `json:"minimum_length,omitempty"`
35 +
36 + Encodings map[string]Encoding `json:"-"`
37 +
38 + writerPools map[string]*sync.Pool // TODO: these pools do not get reused through config reloads...
39 +}
40 +
41 +// Provision provisions enc.
42 +func (enc *Encode) Provision(ctx caddy2.Context) error {
43 + enc.Encodings = make(map[string]Encoding)
44 + enc.writerPools = make(map[string]*sync.Pool)
45 +
46 + for modName, rawMsg := range enc.EncodingsRaw {
47 + val, err := ctx.LoadModule("http.encoders."+modName, rawMsg)
48 + if err != nil {
49 + return fmt.Errorf("loading encoder module '%s': %v", modName, err)
50 + }
51 + encoder := val.(Encoding)
52 + enc.Encodings[modName] = encoder
53 +
54 + enc.writerPools[modName] = &sync.Pool{
55 + New: func() interface{} {
56 + return encoder.NewEncoder()
57 + },
58 + }
59 + }
60 + enc.EncodingsRaw = nil // allow GC to deallocate - TODO: Does this help?
61 +
62 + if enc.MinLength == 0 {
63 + enc.MinLength = defaultMinLength
64 + }
65 +
66 + return nil
67 +}
68 +
69 +func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
70 + for _, encName := range acceptedEncodings(r) {
71 + if _, ok := enc.writerPools[encName]; !ok {
72 + continue // encoding not offered
73 + }
74 + w = enc.openResponseWriter(encName, w)
75 + defer w.(*responseWriter).Close()
76 + break
77 + }
78 +
79 + return next.ServeHTTP(w, r)
80 +}
81 +
82 +// openResponseWriter creates a new response writer that may (or may not)
83 +// encode the response with encodingName. The returned response writer MUST
84 +// be closed after the handler completes.
85 +func (enc *Encode) openResponseWriter(encodingName string, w http.ResponseWriter) *responseWriter {
86 + buf := bufPool.Get().(*bytes.Buffer)
87 + buf.Reset()
88 + return &responseWriter{
89 + ResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w},
90 + encodingName: encodingName,
91 + buf: buf,
92 + config: enc,
93 + }
94 +}
95 +
96 +// responseWriter writes to an underlying response writer
97 +// using the encoding represented by encodingName and
98 +// configured by config.
99 +type responseWriter struct {
100 + *caddyhttp.ResponseWriterWrapper
101 + encodingName string
102 + w Encoder
103 + buf *bytes.Buffer
104 + config *Encode
105 + statusCode int
106 +}
107 +
108 +// WriteHeader stores the status to write when the time comes
109 +// to actually write the header.
110 +func (rw *responseWriter) WriteHeader(status int) {
111 + rw.statusCode = status
112 +}
113 +
114 +// Write writes to the response. If the response qualifies,
115 +// it is encoded using the encoder, which is initialized
116 +// if not done so already.
117 +func (rw *responseWriter) Write(p []byte) (int, error) {
118 + var n, written int
119 + var err error
120 +
121 + if rw.buf != nil && rw.config.MinLength > 0 {
122 + written = rw.buf.Len()
123 + _, err := rw.buf.Write(p)
124 + if err != nil {
125 + return 0, err
126 + }
127 + if rw.buf.Len() < rw.config.MinLength {
128 + return len(p), nil
129 + }
130 + rw.init()
131 + p = rw.buf.Bytes()
132 + defer func() {
133 + bufPool.Put(rw.buf)
134 + rw.buf = nil
135 + }()
136 + }
137 +
138 + switch {
139 + case rw.w != nil:
140 + n, err = rw.w.Write(p)
141 + default:
142 + n, err = rw.ResponseWriter.Write(p)
143 + }
144 + n -= written
145 + if n < 0 {
146 + n = 0
147 + }
148 + return n, err
149 +}
150 +
151 +// init should be called once we know we are writing an encoded response.
152 +func (rw *responseWriter) init() {
153 + if rw.Header().Get("Content-Encoding") == "" && rw.buf.Len() >= rw.config.MinLength {
154 + rw.w = rw.config.writerPools[rw.encodingName].Get().(Encoder)
155 + rw.w.Reset(rw.ResponseWriter)
156 + rw.Header().Del("Content-Length") // https://github.com/golang/go/issues/14975
157 + rw.Header().Set("Content-Encoding", rw.encodingName)
158 + }
159 + rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
160 + rw.ResponseWriter.WriteHeader(rw.statusCode)
161 +}
162 +
163 +// Close writes any remaining buffered response and
164 +// deallocates any active resources.
165 +func (rw *responseWriter) Close() error {
166 + var err error
167 + if rw.buf != nil {
168 + rw.init()
169 + p := rw.buf.Bytes()
170 + defer func() {
171 + bufPool.Put(rw.buf)
172 + rw.buf = nil
173 + }()
174 + switch {
175 + case rw.w != nil:
176 + _, err = rw.w.Write(p)
177 + default:
178 + _, err = rw.ResponseWriter.Write(p)
179 + }
180 + }
181 + if rw.w != nil {
182 + err2 := rw.w.Close()
183 + if err2 != nil && err == nil {
184 + err = err2
185 + }
186 + rw.config.writerPools[rw.encodingName].Put(rw.w)
187 + rw.w = nil
188 + }
189 + return err
190 +}
191 +
192 +// acceptedEncodings returns the list of encodings that the
193 +// client supports, in descending order of preference. If
194 +// the Sec-WebSocket-Key header is present then non-identity
195 +// encodings are not considered. See
196 +// http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html.
197 +func acceptedEncodings(r *http.Request) []string {
198 + acceptEncHeader := r.Header.Get("Accept-Encoding")
199 + websocketKey := r.Header.Get("Sec-WebSocket-Key")
200 + if acceptEncHeader == "" {
201 + return []string{}
202 + }
203 +
204 + var prefs []encodingPreference
205 +
206 + for _, accepted := range strings.Split(acceptEncHeader, ",") {
207 + parts := strings.Split(accepted, ";")
208 + encName := strings.ToLower(strings.TrimSpace(parts[0]))
209 +
210 + // determine q-factor
211 + qFactor := 1.0
212 + if len(parts) > 1 {
213 + qFactorStr := strings.ToLower(strings.TrimSpace(parts[1]))
214 + if strings.HasPrefix(qFactorStr, "q=") {
215 + if qFactorFloat, err := strconv.ParseFloat(qFactorStr[2:], 32); err == nil {
216 + if qFactorFloat >= 0 && qFactorFloat <= 1 {
217 + qFactor = qFactorFloat
218 + }
219 + }
220 + }
221 + }
222 +
223 + // encodings with q-factor of 0 are not accepted;
224 + // use a small theshold to account for float precision
225 + if qFactor < 0.00001 {
226 + continue
227 + }
228 +
229 + // don't encode WebSocket handshakes
230 + if websocketKey != "" && encName != "identity" {
231 + continue
232 + }
233 +
234 + prefs = append(prefs, encodingPreference{
235 + encoding: encName,
236 + q: qFactor,
237 + })
238 + }
239 +
240 + // sort preferences by descending q-factor
241 + sort.Slice(prefs, func(i, j int) bool { return prefs[i].q > prefs[j].q })
242 +
243 + // TODO: If no preference, or same pref for all encodings,
244 + // and not websocket, use default encoding ordering (enc.Prefer)
245 + // for those which are accepted by the client
246 +
247 + prefEncNames := make([]string, len(prefs))
248 + for i := range prefs {
249 + prefEncNames[i] = prefs[i].encoding
250 + }
251 +
252 + return prefEncNames
253 +}
254 +
255 +// encodingPreference pairs an encoding with its q-factor.
256 +type encodingPreference struct {
257 + encoding string
258 + q float64
259 +}
260 +
261 +// Encoder is a type which can encode a stream of data.
262 +type Encoder interface {
263 + io.WriteCloser
264 + Reset(io.Writer)
265 +}
266 +
267 +// Encoding is a type which can create encoders of its kind.
268 +type Encoding interface {
269 + NewEncoder() Encoder
270 +}
271 +
272 +var bufPool = sync.Pool{
273 + New: func() interface{} {
274 + return new(bytes.Buffer)
275 + },
276 +}
277 +
278 +// defaultMinLength is the minimum length at which to compress content.
279 +const defaultMinLength = 512
280 +
281 +// Interface guards
282 +var (
283 + _ caddy2.Provisioner = (*Encode)(nil)
284 + _ caddyhttp.MiddlewareHandler = (*Encode)(nil)
285 + _ caddyhttp.HTTPInterfaces = (*responseWriter)(nil)
286 +)
modules/caddyhttp/encode/gzip/gzip.go new
+36
@@ -0,0 +1,36 @@
1 +package caddygzip
2 +
3 +import (
4 + "compress/flate"
5 + "compress/gzip" // TODO: consider using https://github.com/klauspost/compress/gzip
6 +
7 + "github.com/caddyserver/caddy2"
8 + "github.com/caddyserver/caddy2/modules/caddyhttp/encode"
9 +)
10 +
11 +func init() {
12 + caddy2.RegisterModule(caddy2.Module{
13 + Name: "http.encoders.gzip",
14 + New: func() interface{} { return new(Gzip) },
15 + })
16 +}
17 +
18 +// Gzip can create gzip encoders.
19 +type Gzip struct {
20 + Level int `json:"level,omitempty"`
21 +}
22 +
23 +// NewEncoder returns a new gzip writer.
24 +func (g Gzip) NewEncoder() encode.Encoder {
25 + if g.Level <= flate.NoCompression {
26 + g.Level = defaultGzipLevel
27 + }
28 + if g.Level > flate.BestCompression {
29 + g.Level = flate.BestCompression
30 + }
31 + writer, _ := gzip.NewWriterLevel(nil, g.Level)
32 + return writer
33 +}
34 +
35 +// Informed from http://blog.klauspost.com/gzip-performance-for-go-webservers/
36 +var defaultGzipLevel = 5
modules/caddyhttp/encode/zstd/zstd.go new
+23
@@ -0,0 +1,23 @@
1 +package caddystd
2 +
3 +import (
4 + "github.com/caddyserver/caddy2"
5 + "github.com/caddyserver/caddy2/modules/caddyhttp/encode"
6 + "github.com/klauspost/compress/zstd"
7 +)
8 +
9 +func init() {
10 + caddy2.RegisterModule(caddy2.Module{
11 + Name: "http.encoders.zstd",
12 + New: func() interface{} { return new(Zstd) },
13 + })
14 +}
15 +
16 +// Zstd can create zstd encoders.
17 +type Zstd struct{}
18 +
19 +// NewEncoder returns a new gzip writer.
20 +func (z Zstd) NewEncoder() encode.Encoder {
21 + writer, _ := zstd.NewWriter(nil)
22 + return writer
23 +}