Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
caddyhttp: Improve ResponseRecorder to buffer headers
Matthew Holt committed
Oct 15, 2019 at 14:07 UTC
abf5ab340ed76792214ae80c62df7abe0ad1b8a8
6 files changed
+116
-47
modules/caddyhttp/caddyhttp.go
+13
@@ -593,6 +593,19 @@ func (ws WeakString) String() string {
593
return string(ws)
594
}
595
596
+// CopyHeader copies HTTP headers by completely
597
+// replacing dest with src. (This allows deletions
598
+// to be propagated, assuming src started as a
599
+// consistent copy of dest.)
600
+func CopyHeader(dest, src http.Header) {
601
+ for field := range dest {
602
+ delete(dest, field)
603
+ }
604
+ for field, val := range src {
605
+ dest[field] = val
606
+ }
607
+}
608
+
609
// StatusCodeMatches returns true if a real HTTP status code matches
610
// the configured status code, which may be either a real HTTP status
611
// code or an integer representing a class of codes (e.g. 4 for all
modules/caddyhttp/httpcache/httpcache.go
+2
-3
@@ -130,8 +130,7 @@ func (c *Cache) getter(ctx groupcache.Context, key string, dest groupcache.Sink)
130
131
// we need to record the response if we are to cache it; only cache if
132
// request is successful (TODO: there's probably much more nuance needed here)
133
- var rr caddyhttp.ResponseRecorder
134
- rr = caddyhttp.NewResponseRecorder(combo.rw, buf, func(status int) bool {
133
+ rr := caddyhttp.NewResponseRecorder(combo.rw, buf, func(status int, header http.Header) bool {
134
shouldBuf := status < 300
135
136
if shouldBuf {
@@ -141,7 +140,7 @@ func (c *Cache) getter(ctx groupcache.Context, key string, dest groupcache.Sink)
140
// the rest will be the body, which will be written
141
// implicitly for us by the recorder
142
err := gob.NewEncoder(buf).Encode(headerAndStatus{
144
- Header: rr.Header(),
143
+ Header: header,
144
Status: status,
145
})
146
if err != nil {
modules/caddyhttp/markdown/markdown.go
+4
-2
@@ -48,8 +48,8 @@ func (m Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
48
buf.Reset()
49
defer bufPool.Put(buf)
50
51
- shouldBuf := func(status int) bool {
52
- return strings.HasPrefix(w.Header().Get("Content-Type"), "text/")
51
+ shouldBuf := func(status int, header http.Header) bool {
52
+ return strings.HasPrefix(header.Get("Content-Type"), "text/")
53
}
54
55
rec := caddyhttp.NewResponseRecorder(w, buf, shouldBuf)
@@ -62,6 +62,8 @@ func (m Markdown) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyht
62
return nil
63
}
64
65
+ caddyhttp.CopyHeader(w.Header(), rec.Header())
66
+
67
output := blackfriday.Run(buf.Bytes())
68
69
w.Header().Set("Content-Length", strconv.Itoa(len(output)))
modules/caddyhttp/responsewriter.go
+88
-31
@@ -18,6 +18,7 @@ import (
18
"bufio"
19
"bytes"
20
"fmt"
21
+ "io"
22
"net"
23
"net/http"
24
)
@@ -78,52 +79,89 @@ type responseRecorder struct {
79
wroteHeader bool
80
statusCode int
81
buf *bytes.Buffer
81
- shouldBuffer func(status int) bool
82
+ shouldBuffer ShouldBufferFunc
83
stream bool
84
size int
85
+ header http.Header
86
}
87
88
// NewResponseRecorder returns a new ResponseRecorder that can be
87
-// used instead of a real http.ResponseWriter. The recorder is useful
88
-// for middlewares which need to buffer a responder's response and
89
-// process it in its entirety before actually allowing the response to
90
-// be written. Of course, this has a performance overhead, but
91
-// sometimes there is no way to avoid buffering the whole response.
92
-// Still, if at all practical, middlewares should strive to stream
89
+// used instead of a standard http.ResponseWriter. The recorder is
90
+// useful for middlewares which need to buffer a response and
91
+// potentially process its entire body before actually writing the
92
+// response to the underlying writer. Of course, buffering the entire
93
+// body has a memory overhead, but sometimes there is no way to avoid
94
+// buffering the whole response, hence the existence of this type.
95
+// Still, if at all practical, handlers should strive to stream
96
// responses by wrapping Write and WriteHeader methods instead of
97
// buffering whole response bodies.
98
//
96
-// Recorders optionally buffer the response. When the headers are
97
-// to be written, shouldBuffer will be called with the status
98
-// code that is being written. The rest of the headers can be read
99
-// from w.Header(). If shouldBuffer returns true, the response
100
-// will be buffered. You can know the response was buffered if
101
-// the Buffered() method returns true. If the response was not
102
-// buffered, Buffered() will return false and that means the
103
-// response bypassed the recorder and was written directly to the
104
-// underlying writer. If shouldBuffer is nil, the response will
105
-// never be buffered (it will always be streamed directly), and
106
-// buf can also safely be nil.
99
+// Buffering is actually optional. The shouldBuffer function will
100
+// be called just before the headers are written. If it returns
101
+// true, the headers and body will be buffered by this recorder
102
+// and not written to the underlying writer; if false, the headers
103
+// will be written immediately and the body will be streamed out
104
+// directly to the underlying writer. If shouldBuffer is nil,
105
+// the response will never be buffered and will always be streamed
106
+// directly to the writer.
107
//
108
-// Before calling this function in a middleware handler, make a
109
-// new buffer or obtain one from a pool (use the sync.Pool) type.
110
-// Using a pool is generally recommended for performance gains;
111
-// do profiling to ensure this is the case. If using a pool, be
112
-// sure to reset the buffer before using it.
108
+// You can know if shouldBuffer returned true by calling Buffered().
109
//
114
-// The returned recorder can be used in place of w when calling
115
-// the next handler in the chain. When that handler returns, you
116
-// can read the status code from the recorder's Status() method.
117
-// The response body fills buf if it was buffered, and the headers
118
-// are available via w.Header().
119
-func NewResponseRecorder(w http.ResponseWriter, buf *bytes.Buffer, shouldBuffer func(status int) bool) ResponseRecorder {
110
+// The provided buffer buf should be obtained from a pool for best
111
+// performance (see the sync.Pool type).
112
+//
113
+// Proper usage of a recorder looks like this:
114
+//
115
+// rec := caddyhttp.NewResponseRecorder(w, buf, shouldBuffer)
116
+// err := next.ServeHTTP(rec, req)
117
+// if err != nil {
118
+// return err
119
+// }
120
+// if !rec.Buffered() {
121
+// return nil
122
+// }
123
+// // process the buffered response here
124
+//
125
+// After a response has been buffered, remember that any upstream header
126
+// manipulations are only manifest in the recorder's Header(), not the
127
+// Header() of the underlying ResponseWriter. Thus if you wish to inspect
128
+// or change response headers, you either need to use rec.Header(), or
129
+// copy rec.Header() into w.Header() first (see caddyhttp.CopyHeader).
130
+//
131
+// Once you are ready to write the response, there are two ways you can do
132
+// it. The easier way is to have the recorder do it:
133
+//
134
+// rec.WriteResponse()
135
+//
136
+// This writes the recorded response headers as well as the buffered body.
137
+// Or, you may wish to do it yourself, especially if you manipulated the
138
+// buffered body. First you will need to copy the recorded headers, then
139
+// write the headers with the recorded status code, then write the body
140
+// (this example writes the recorder's body buffer, but you might have
141
+// your own body to write instead):
142
+//
143
+// caddyhttp.CopyHeader(w.Header(), rec.Header())
144
+// w.WriteHeader(rec.Status())
145
+// io.Copy(w, rec.Buffer())
146
+//
147
+func NewResponseRecorder(w http.ResponseWriter, buf *bytes.Buffer, shouldBuffer ShouldBufferFunc) ResponseRecorder {
148
+ // copy the current response header into this buffer so
149
+ // that any header manipulations on the buffered header
150
+ // are consistent with what would be written out
151
+ hdr := make(http.Header)
152
+ CopyHeader(hdr, w.Header())
153
return &responseRecorder{
154
ResponseWriterWrapper: &ResponseWriterWrapper{ResponseWriter: w},
155
buf: buf,
156
shouldBuffer: shouldBuffer,
157
+ header: hdr,
158
}
159
}
160
161
+func (rr *responseRecorder) Header() http.Header {
162
+ return rr.header
163
+}
164
+
165
func (rr *responseRecorder) WriteHeader(statusCode int) {
166
if rr.wroteHeader {
167
return
@@ -135,9 +173,12 @@ func (rr *responseRecorder) WriteHeader(statusCode int) {
173
if rr.shouldBuffer == nil {
174
rr.stream = true
175
} else {
138
- rr.stream = !rr.shouldBuffer(rr.statusCode)
176
+ rr.stream = !rr.shouldBuffer(rr.statusCode, rr.header)
177
}
178
+
179
+ // if not buffered, immediately write header
180
if rr.stream {
181
+ CopyHeader(rr.ResponseWriterWrapper.Header(), rr.header)
182
rr.ResponseWriterWrapper.WriteHeader(rr.statusCode)
183
}
184
}
@@ -179,16 +220,32 @@ func (rr *responseRecorder) Buffered() bool {
220
return !rr.stream
221
}
222
223
+func (rr *responseRecorder) WriteResponse() error {
224
+ if rr.stream {
225
+ return nil
226
+ }
227
+ CopyHeader(rr.ResponseWriterWrapper.Header(), rr.header)
228
+ rr.ResponseWriterWrapper.WriteHeader(rr.statusCode)
229
+ _, err := io.Copy(rr.ResponseWriterWrapper, rr.buf)
230
+ return err
231
+}
232
// ResponseRecorder is a http.ResponseWriter that records
183
-// responses instead of writing them to the client.
233
+// responses instead of writing them to the client. See
234
+// docs for NewResponseRecorder for proper usage.
235
type ResponseRecorder interface {
236
HTTPInterfaces
237
Status() int
238
Buffer() *bytes.Buffer
239
Buffered() bool
240
Size() int
241
+ WriteResponse() error
242
}
243
244
+// ShouldBufferFunc is a function that returns true if the
245
+// response should be buffered, given the pending HTTP status
246
+// code and response headers.
247
+type ShouldBufferFunc func(status int, header http.Header) bool
248
+
249
// Interface guards
250
var (
251
_ HTTPInterfaces = (*ResponseWriterWrapper)(nil)
modules/caddyhttp/templates/templates.go
+8
-10
@@ -17,7 +17,6 @@ package templates
17
import (
18
"bytes"
19
"fmt"
20
- "io"
20
"net/http"
21
"strconv"
22
"strings"
@@ -71,8 +70,8 @@ func (t *Templates) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy
70
71
// shouldBuf determines whether to execute templates on this response,
72
// since generally we will not want to execute for images or CSS, etc.
74
- shouldBuf := func(status int) bool {
75
- ct := w.Header().Get("Content-Type")
73
+ shouldBuf := func(status int, header http.Header) bool {
74
+ ct := header.Get("Content-Type")
75
for _, mt := range t.MIMETypes {
76
if strings.Contains(ct, mt) {
77
return true
@@ -96,18 +95,17 @@ func (t *Templates) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy
95
return err
96
}
97
99
- w.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
100
- w.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-created content
101
- w.Header().Del("Last-Modified") // useless for dynamic content since it's always changing
98
+ rec.Header().Set("Content-Length", strconv.Itoa(buf.Len()))
99
+ rec.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-created content
100
+ rec.Header().Del("Last-Modified") // useless for dynamic content since it's always changing
101
102
// we don't know a way to guickly generate etag for dynamic content,
103
// but we can convert this to a weak etag to kind of indicate that
105
- if etag := w.Header().Get("ETag"); etag != "" {
106
- w.Header().Set("ETag", "W/"+etag)
104
+ if etag := rec.Header().Get("Etag"); etag != "" {
105
+ rec.Header().Set("Etag", "W/"+etag)
106
}
107
109
- w.WriteHeader(rec.Status())
110
- io.Copy(w, buf)
108
+ rec.WriteResponse()
109
110
return nil
111
}
modules/caddyhttp/templates/tplcontext.go
+1
-1
@@ -80,7 +80,7 @@ func (c templateContext) Include(filename string, args ...interface{}) (template
80
// If it is not trusted, be sure to use escaping functions yourself.
81
func (c templateContext) HTTPInclude(uri string) (template.HTML, error) {
82
if c.Req.Header.Get(recursionPreventionHeader) == "1" {
83
- return "", fmt.Errorf("virtual include cycle")
83
+ return "", fmt.Errorf("virtual request cycle")
84
}
85
86
buf := bufPool.Get().(*bytes.Buffer)