Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 12, 14:30 UTC
encode: Improve Etag handling (fix #5849)
We also improve Last-Modified handling in the file server. Both changes should be more compliant with RFC 9110.
Matthew Holt committed
Apr 17, 2024 at 19:12 UTC
3067074d9cf94dba8b27c31408b71c9f75defb40
4 files changed
+80
-11
modules/caddyhttp/encode/encode.go
+32
@@ -156,6 +156,18 @@ func (enc *Encode) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyh
156
}
157
w = enc.openResponseWriter(encName, w)
158
defer w.(*responseWriter).Close()
159
+
160
+ // to comply with RFC 9110 section 8.8.3(.3), we modify the Etag when encoding
161
+ // by appending a hyphen and the encoder name; the problem is, the client will
162
+ // send back that Etag in a If-None-Match header, but upstream handlers that set
163
+ // the Etag in the first place don't know that we appended to their Etag! so here
164
+ // we have to strip our addition so the upstream handlers can still honor client
165
+ // caches without knowing about our changes...
166
+ if etag := r.Header.Get("If-None-Match"); etag != "" && !strings.HasPrefix(etag, "W/") {
167
+ etag = strings.TrimSuffix(etag, "-"+encName+`"`) + `"`
168
+ r.Header.Set("If-None-Match", etag)
169
+ }
170
+
171
break
172
}
173
}
@@ -220,6 +232,14 @@ type responseWriter struct {
232
func (rw *responseWriter) WriteHeader(status int) {
233
rw.statusCode = status
234
235
+ // See #5849 and RFC 9110 section 15.4.5 (https://www.rfc-editor.org/rfc/rfc9110.html#section-15.4.5) - 304
236
+ // Not Modified must have certain headers set as if it was a 200 response, and according to the issue
237
+ // we would miss the Vary header in this case when compression was also enabled; note that we set this
238
+ // header in the responseWriter.init() method but that is only called if we are writing a response body
239
+ if status == http.StatusNotModified {
240
+ rw.Header().Add("Vary", "Accept-Encoding")
241
+ }
242
+
243
// write status immediately when status code is informational
244
// see: https://caddy.community/t/disappear-103-early-hints-response-with-encode-enable-caddy-v2-7-6/23081/5
245
if 100 <= status && status <= 199 {
@@ -334,6 +354,18 @@ func (rw *responseWriter) init() {
354
rw.Header().Set("Content-Encoding", rw.encodingName)
355
rw.Header().Add("Vary", "Accept-Encoding")
356
rw.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-encoded content
357
+
358
+ // strong ETags need to be distinct depending on the encoding ("selected representation")
359
+ // see RFC 9110 section 8.8.3.3:
360
+ // https://www.rfc-editor.org/rfc/rfc9110.html#name-example-entity-tags-varying
361
+ // I don't know a great way to do this... how about appending? That's a neat trick!
362
+ // (We have to strip the value we append from If-None-Match headers before
363
+ // sending subsequent requests back upstream, however, since upstream handlers
364
+ // don't know about our appending to their Etag since they've already done their work)
365
+ if etag := rw.Header().Get("Etag"); etag != "" && !strings.HasPrefix(etag, "W/") {
366
+ etag = fmt.Sprintf(`%s-%s"`, strings.TrimSuffix(etag, `"`), rw.encodingName)
367
+ rw.Header().Set("Etag", etag)
368
+ }
369
}
370
}
371
modules/caddyhttp/fileserver/browse.go
+14
@@ -30,6 +30,7 @@ import (
30
"sync"
31
"text/tabwriter"
32
"text/template"
33
+ "time"
34
35
"go.uber.org/zap"
36
@@ -104,6 +105,18 @@ func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w ht
105
return caddyhttp.Error(http.StatusInternalServerError, err)
106
}
107
108
+ w.Header().Add("Vary", "Accept")
109
+
110
+ // speed up browser/client experience and caching by supporting If-Modified-Since
111
+ if ifModSinceStr := r.Header.Get("If-Modified-Since"); ifModSinceStr != "" {
112
+ ifModSince, err := time.ParseInLocation(http.TimeFormat, ifModSinceStr, time.Local)
113
+ lastModTrunc := listing.lastModified.Truncate(time.Second)
114
+ if err == nil && (lastModTrunc.Equal(ifModSince) || lastModTrunc.Before(ifModSince)) {
115
+ w.WriteHeader(http.StatusNotModified)
116
+ return nil
117
+ }
118
+ }
119
+
120
fsrv.browseApplyQueryParams(w, r, listing)
121
122
buf := bufPool.Get().(*bytes.Buffer)
@@ -111,6 +124,7 @@ func (fsrv *FileServer) serveBrowse(fileSystem fs.FS, root, dirPath string, w ht
124
defer bufPool.Put(buf)
125
126
acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ","))
127
+ w.Header().Set("Last-Modified", listing.lastModified.Format(http.TimeFormat))
128
129
switch {
130
case strings.Contains(acceptHeader, "application/json"):
modules/caddyhttp/fileserver/browsetplcontext.go
+11
-1
@@ -63,6 +63,12 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
63
continue
64
}
65
66
+ // keep track of the most recently modified item in the listing
67
+ modTime := info.ModTime()
68
+ if tplCtx.lastModified.IsZero() || modTime.After(tplCtx.lastModified) {
69
+ tplCtx.lastModified = modTime
70
+ }
71
+
72
isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(fileSystem, info, root, urlPath)
73
74
// add the slash after the escape of path to avoid escaping the slash as well
@@ -108,7 +114,7 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
114
Name: name,
115
Size: size,
116
URL: u.String(),
111
- ModTime: info.ModTime().UTC(),
117
+ ModTime: modTime.UTC(),
118
Mode: info.Mode(),
119
Tpl: tplCtx, // a reference up to the template context is useful
120
SymlinkPath: symlinkPath,
@@ -155,6 +161,10 @@ type browseTemplateContext struct {
161
162
// Display format (list or grid)
163
Layout string `json:"layout,omitempty"`
164
+
165
+ // The most recent file modification date in the listing.
166
+ // Used for HTTP header purposes.
167
+ lastModified time.Time
168
}
169
170
// Breadcrumbs returns l.Path where every element maps
modules/caddyhttp/fileserver/staticfiles.go
+23
-10
@@ -644,19 +644,32 @@ func (fsrv *FileServer) notFound(w http.ResponseWriter, r *http.Request, next ca
644
return caddyhttp.Error(http.StatusNotFound, nil)
645
}
646
647
-// calculateEtag produces a strong etag by default, although, for
648
-// efficiency reasons, it does not actually consume the contents
649
-// of the file to make a hash of all the bytes. ¯\_(ツ)_/¯
650
-// Prefix the etag with "W/" to convert it into a weak etag.
651
-// See: https://tools.ietf.org/html/rfc7232#section-2.3
647
+// calculateEtag computes an entity tag using a strong validator
648
+// without consuming the contents of the file. It requires the
649
+// file info contain the correct size and modification time.
650
+// It strives to implement the semantics regarding ETags as defined
651
+// by RFC 9110 section 8.8.3 and 8.8.1. See
652
+// https://www.rfc-editor.org/rfc/rfc9110.html#section-8.8.3.
653
+//
654
+// As our implementation uses file modification timestamp and size,
655
+// note the following from RFC 9110 section 8.8.1: "A representation's
656
+// modification time, if defined with only one-second resolution,
657
+// might be a weak validator if it is possible for the representation to
658
+// be modified twice during a single second and retrieved between those
659
+// modifications." The ext4 file system, which underpins the vast majority
660
+// of Caddy deployments, stores mod times with millisecond precision,
661
+// which we consider precise enough to qualify as a strong validator.
662
func calculateEtag(d os.FileInfo) string {
653
- mtime := d.ModTime().Unix()
654
- if mtime == 0 || mtime == 1 {
663
+ mtime := d.ModTime()
664
+ if mtimeUnix := mtime.Unix(); mtimeUnix == 0 || mtimeUnix == 1 {
665
return "" // not useful anyway; see issue #5548
666
}
657
- t := strconv.FormatInt(mtime, 36)
658
- s := strconv.FormatInt(d.Size(), 36)
659
- return `"` + t + s + `"`
667
+ var sb strings.Builder
668
+ sb.WriteRune('"')
669
+ sb.WriteString(strconv.FormatInt(mtime.UnixNano(), 36))
670
+ sb.WriteString(strconv.FormatInt(d.Size(), 36))
671
+ sb.WriteRune('"')
672
+ return sb.String()
673
}
674
675
// Finds the first corresponding etag file for a given file in the file system and return its content