Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 15:30 UTC
fileserver: Support virtual file systems (#4909)
* fileserver: Support virtual file systems (close #3720) This change replaces the hard-coded use of os.Open() and os.Stat() with the use of the new (Go 1.16) io/fs APIs, enabling virtual file systems. It introduces a new module namespace, caddy.fs, for such file systems. Also improve documentation for the file server. I realized it was one of the first modules written for Caddy 2, and the docs hadn't really been updated since! * Virtualize FS for file matcher; minor tweaks * Fix tests and rename dirFS -> osFS (Since we do not use a root directory, it is dynamic.)
Matt Holt committed
Jul 30, 2022 at 13:07 UTC
6668271661857b2cc43143ff65edf1071013e67e
5 files changed
+166
-55
modules/caddyhttp/fileserver/browse.go
+11
-10
@@ -19,6 +19,7 @@ import (
19
_ "embed"
20
"encoding/json"
21
"fmt"
22
+ "io/fs"
23
"net/http"
24
"os"
25
"path"
@@ -80,7 +81,7 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
81
repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
82
83
// calling path.Clean here prevents weird breadcrumbs when URL paths are sketchy like /%2e%2e%2f
83
- listing, err := fsrv.loadDirectoryContents(dir, root, path.Clean(r.URL.Path), repl)
84
+ listing, err := fsrv.loadDirectoryContents(dir.(fs.ReadDirFile), root, path.Clean(r.URL.Path), repl)
85
switch {
86
case os.IsPermission(err):
87
return caddyhttp.Error(http.StatusForbidden, err)
@@ -133,8 +134,8 @@ func (fsrv *FileServer) serveBrowse(root, dirPath string, w http.ResponseWriter,
134
return nil
135
}
136
136
-func (fsrv *FileServer) loadDirectoryContents(dir *os.File, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
137
- files, err := dir.Readdir(-1)
137
+func (fsrv *FileServer) loadDirectoryContents(dir fs.ReadDirFile, root, urlPath string, repl *caddy.Replacer) (browseTemplateContext, error) {
138
+ files, err := dir.ReadDir(10000) // TODO: this limit should probably be configurable
139
if err != nil {
140
return browseTemplateContext{}, err
141
}
@@ -201,25 +202,25 @@ func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.T
202
return tpl, nil
203
}
204
204
-// isSymlink return true if f is a symbolic link
205
-func isSymlink(f os.FileInfo) bool {
206
- return f.Mode()&os.ModeSymlink != 0
207
-}
208
-
205
// isSymlinkTargetDir returns true if f's symbolic link target
206
// is a directory.
211
-func isSymlinkTargetDir(f os.FileInfo, root, urlPath string) bool {
207
+func (fsrv *FileServer) isSymlinkTargetDir(f fs.FileInfo, root, urlPath string) bool {
208
if !isSymlink(f) {
209
return false
210
}
211
target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
216
- targetInfo, err := os.Stat(target)
212
+ targetInfo, err := fsrv.fileSystem.Stat(target)
213
if err != nil {
214
return false
215
}
216
return targetInfo.IsDir()
217
}
218
219
+// isSymlink return true if f is a symbolic link.
220
+func isSymlink(f fs.FileInfo) bool {
221
+ return f.Mode()&os.ModeSymlink != 0
222
+}
223
+
224
// templateContext powers the context used when evaluating the browse template.
225
// It combines browse-specific features with the standard templates handler
226
// features.
modules/caddyhttp/fileserver/browsetplcontext.go
+20
-10
@@ -15,6 +15,7 @@
15
package fileserver
16
17
import (
18
+ "io/fs"
19
"net/url"
20
"os"
21
"path"
@@ -26,22 +27,31 @@ import (
27
"github.com/caddyserver/caddy/v2"
28
"github.com/caddyserver/caddy/v2/modules/caddyhttp"
29
"github.com/dustin/go-humanize"
30
+ "go.uber.org/zap"
31
)
32
31
-func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
33
+func (fsrv *FileServer) directoryListing(entries []fs.DirEntry, canGoUp bool, root, urlPath string, repl *caddy.Replacer) browseTemplateContext {
34
filesToHide := fsrv.transformHidePaths(repl)
35
36
var dirCount, fileCount int
37
fileInfos := []fileInfo{}
38
37
- for _, f := range files {
38
- name := f.Name()
39
+ for _, entry := range entries {
40
+ name := entry.Name()
41
42
if fileHidden(name, filesToHide) {
43
continue
44
}
45
44
- isDir := f.IsDir() || isSymlinkTargetDir(f, root, urlPath)
46
+ info, err := entry.Info()
47
+ if err != nil {
48
+ fsrv.logger.Error("could not get info about directory entry",
49
+ zap.String("name", entry.Name()),
50
+ zap.String("root", root))
51
+ continue
52
+ }
53
+
54
+ isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(info, root, urlPath)
55
56
// add the slash after the escape of path to avoid escaping the slash as well
57
if isDir {
@@ -51,11 +61,11 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
61
fileCount++
62
}
63
54
- size := f.Size()
55
- fileIsSymlink := isSymlink(f)
64
+ size := info.Size()
65
+ fileIsSymlink := isSymlink(info)
66
if fileIsSymlink {
57
- path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
58
- fileInfo, err := os.Stat(path)
67
+ path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
68
+ fileInfo, err := fsrv.fileSystem.Stat(path)
69
if err == nil {
70
size = fileInfo.Size()
71
}
@@ -73,8 +83,8 @@ func (fsrv *FileServer) directoryListing(files []os.FileInfo, canGoUp bool, root
83
Name: name,
84
Size: size,
85
URL: u.String(),
76
- ModTime: f.ModTime().UTC(),
77
- Mode: f.Mode(),
86
+ ModTime: info.ModTime().UTC(),
87
+ Mode: info.Mode(),
88
})
89
}
90
name, _ := url.PathUnescape(urlPath)
modules/caddyhttp/fileserver/matcher.go
+27
-7
@@ -15,7 +15,9 @@
15
package fileserver
16
17
import (
18
+ "encoding/json"
19
"fmt"
20
+ "io/fs"
21
"net/http"
22
"os"
23
"path"
@@ -54,6 +56,11 @@ func init() {
56
// - `{http.matchers.file.remainder}` Set to the remainder
57
// of the path if the path was split by `split_path`.
58
type MatchFile struct {
59
+ // The file system implementation to use. By default, the
60
+ // local disk file system will be used.
61
+ FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
62
+ fileSystem fs.StatFS
63
+
64
// The root directory, used for creating absolute
65
// file paths, and required when working with
66
// relative paths; if not specified, `{http.vars.root}`
@@ -241,10 +248,23 @@ func celFileMatcherMacroExpander() parser.MacroExpander {
248
}
249
250
// Provision sets up m's defaults.
244
-func (m *MatchFile) Provision(_ caddy.Context) error {
251
+func (m *MatchFile) Provision(ctx caddy.Context) error {
252
+ // establish the file system to use
253
+ if len(m.FileSystemRaw) > 0 {
254
+ mod, err := ctx.LoadModule(m, "FileSystemRaw")
255
+ if err != nil {
256
+ return fmt.Errorf("loading file system module: %v", err)
257
+ }
258
+ m.fileSystem = mod.(fs.StatFS)
259
+ }
260
+ if m.fileSystem == nil {
261
+ m.fileSystem = osFS{}
262
+ }
263
+
264
if m.Root == "" {
265
m.Root = "{http.vars.root}"
266
}
267
+
268
// if list of files to try was omitted entirely, assume URL path
269
// (use placeholder instead of r.URL.Path; see issue #4146)
270
if m.TryFiles == nil {
@@ -316,7 +336,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
336
return
337
}
338
suffix, fullpath, remainder := prepareFilePath(f)
319
- if info, exists := strictFileExists(fullpath); exists {
339
+ if info, exists := m.strictFileExists(fullpath); exists {
340
setPlaceholders(info, suffix, fullpath, remainder)
341
return true
342
}
@@ -330,7 +350,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
350
var info os.FileInfo
351
for _, f := range m.TryFiles {
352
suffix, fullpath, splitRemainder := prepareFilePath(f)
333
- info, err := os.Stat(fullpath)
353
+ info, err := m.fileSystem.Stat(fullpath)
354
if err == nil && info.Size() > largestSize {
355
largestSize = info.Size()
356
largestFilename = fullpath
@@ -349,7 +369,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
369
var info os.FileInfo
370
for _, f := range m.TryFiles {
371
suffix, fullpath, splitRemainder := prepareFilePath(f)
352
- info, err := os.Stat(fullpath)
372
+ info, err := m.fileSystem.Stat(fullpath)
373
if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
374
smallestSize = info.Size()
375
smallestFilename = fullpath
@@ -368,7 +388,7 @@ func (m MatchFile) selectFile(r *http.Request) (matched bool) {
388
var info os.FileInfo
389
for _, f := range m.TryFiles {
390
suffix, fullpath, splitRemainder := prepareFilePath(f)
371
- info, err := os.Stat(fullpath)
391
+ info, err := m.fileSystem.Stat(fullpath)
392
if err == nil &&
393
(recentDate.IsZero() || info.ModTime().After(recentDate)) {
394
recentDate = info.ModTime()
@@ -404,8 +424,8 @@ func parseErrorCode(input string) error {
424
// the file must also be a directory; if it does
425
// NOT end in a forward slash, the file must NOT
426
// be a directory.
407
-func strictFileExists(file string) (os.FileInfo, bool) {
408
- stat, err := os.Stat(file)
427
+func (m MatchFile) strictFileExists(file string) (os.FileInfo, bool) {
428
+ stat, err := m.fileSystem.Stat(file)
429
if err != nil {
430
// in reality, this can be any error
431
// such as permission or even obscure
modules/caddyhttp/fileserver/matcher_test.go
+7
-5
@@ -111,8 +111,9 @@ func TestFileMatcher(t *testing.T) {
111
},
112
} {
113
m := &MatchFile{
114
- Root: "./testdata",
115
- TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
114
+ fileSystem: osFS{},
115
+ Root: "./testdata",
116
+ TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/"},
117
}
118
119
u, err := url.Parse(tc.path)
@@ -213,9 +214,10 @@ func TestPHPFileMatcher(t *testing.T) {
214
},
215
} {
216
m := &MatchFile{
216
- Root: "./testdata",
217
- TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
218
- SplitPath: []string{".php"},
217
+ fileSystem: osFS{},
218
+ Root: "./testdata",
219
+ TryFiles: []string{"{http.request.uri.path}", "{http.request.uri.path}/index.php"},
220
+ SplitPath: []string{".php"},
221
}
222
223
u, err := url.Parse(tc.path)
modules/caddyhttp/fileserver/staticfiles.go
+101
-23
@@ -15,7 +15,11 @@
15
package fileserver
16
17
import (
18
+ "encoding/json"
19
+ "errors"
20
"fmt"
21
+ "io"
22
+ "io/fs"
23
weakrand "math/rand"
24
"mime"
25
"net/http"
@@ -39,10 +43,63 @@ func init() {
43
caddy.RegisterModule(FileServer{})
44
}
45
42
-// FileServer implements a static file server responder for Caddy.
46
+// FileServer implements a handler that serves static files.
47
+//
48
+// The path of the file to serve is constructed by joining the site root
49
+// and the sanitized request path. Any and all files within the root and
50
+// links with targets outside the site root may therefore be accessed.
51
+// For example, with a site root of `/www`, requests to `/foo/bar.txt`
52
+// will serve the file at `/www/foo/bar.txt`.
53
+//
54
+// The request path is sanitized using the Go standard library's
55
+// path.Clean() function (https://pkg.go.dev/path#Clean) before being
56
+// joined to the root. Request paths must be valid and well-formed.
57
+//
58
+// For requests that access directories instead of regular files,
59
+// Caddy will attempt to serve an index file if present. For example,
60
+// a request to `/dir/` will attempt to serve `/dir/index.html` if
61
+// it exists. The index file names to try are configurable. If a
62
+// requested directory does not have an index file, Caddy writes a
63
+// 404 response. Alternatively, file browsing can be enabled with
64
+// the "browse" parameter which shows a list of files when directories
65
+// are requested if no index file is present.
66
+//
67
+// By default, this handler will canonicalize URIs so that requests to
68
+// directories end with a slash, but requests to regular files do not.
69
+// This is enforced with HTTP redirects automatically and can be disabled.
70
+// Canonicalization redirects are not issued, however, if a URI rewrite
71
+// modified the last component of the path (the filename).
72
+//
73
+// This handler sets the Etag and Last-Modified headers for static files.
74
+// It does not perform MIME sniffing to determine Content-Type based on
75
+// contents, but does use the extension (if known); see the Go docs for
76
+// details: https://pkg.go.dev/mime#TypeByExtension
77
+//
78
+// The file server properly handles requests with If-Match,
79
+// If-Unmodified-Since, If-Modified-Since, If-None-Match, Range, and
80
+// If-Range headers. It includes the file's modification time in the
81
+// Last-Modified header of the response.
82
type FileServer struct {
83
+ // The file system implementation to use. By default, Caddy uses the local
84
+ // disk file system.
85
+ //
86
+ // File system modules used here must adhere to the following requirements:
87
+ // - Implement fs.StatFS interface.
88
+ // - Support seeking on opened files; i.e.returned fs.File values must
89
+ // implement the io.Seeker interface. This is required for determining
90
+ // Content-Length and satisfying Range requests.
91
+ // - fs.File values that represent directories must implement the
92
+ // fs.ReadDirFile interface so that directory listings can be procured.
93
+ FileSystemRaw json.RawMessage `json:"file_system,omitempty" caddy:"namespace=caddy.fs inline_key=backend"`
94
+ fileSystem fs.StatFS
95
+
96
// The path to the root of the site. Default is `{http.vars.root}` if set,
45
- // or current working directory otherwise.
97
+ // or current working directory otherwise. This should be a trusted value.
98
+ //
99
+ // Note that a site root is not a sandbox. Although the file server does
100
+ // sanitize the request URI to prevent directory traversal, files (including
101
+ // links) within the site root may be directly accessed based on the request
102
+ // path. Files and folders within the root should be secure and trustworthy.
103
Root string `json:"root,omitempty"`
104
105
// A list of files or folders to hide; the file server will pretend as if
@@ -63,6 +120,7 @@ type FileServer struct {
120
Hide []string `json:"hide,omitempty"`
121
122
// The names of files to try as index files if a folder is requested.
123
+ // Default: index.html, index.txt.
124
IndexNames []string `json:"index_names,omitempty"`
125
126
// Enables file listings if a directory was requested and no index
@@ -95,8 +153,7 @@ type FileServer struct {
153
// If no order specified here, the first encoding from the Accept-Encoding header
154
// that both client and server support is used
155
PrecompressedOrder []string `json:"precompressed_order,omitempty"`
98
-
99
- precompressors map[string]encode.Precompressed
156
+ precompressors map[string]encode.Precompressed
157
158
logger *zap.Logger
159
}
@@ -113,6 +170,18 @@ func (FileServer) CaddyModule() caddy.ModuleInfo {
170
func (fsrv *FileServer) Provision(ctx caddy.Context) error {
171
fsrv.logger = ctx.Logger(fsrv)
172
173
+ // establish which file system (possibly a virtual one) we'll be using
174
+ if len(fsrv.FileSystemRaw) > 0 {
175
+ mod, err := ctx.LoadModule(fsrv, "FileSystemRaw")
176
+ if err != nil {
177
+ return fmt.Errorf("loading file system module: %v", err)
178
+ }
179
+ fsrv.fileSystem = mod.(fs.StatFS)
180
+ }
181
+ if fsrv.fileSystem == nil {
182
+ fsrv.fileSystem = osFS{}
183
+ }
184
+
185
if fsrv.Root == "" {
186
fsrv.Root = "{http.vars.root}"
187
}
@@ -131,6 +200,7 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error {
200
}
201
}
202
203
+ // support precompressed sidecar files
204
mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw")
205
if err != nil {
206
return fmt.Errorf("loading encoder modules: %v", err)
@@ -184,12 +254,12 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
254
zap.String("result", filename))
255
256
// get information about the file
187
- info, err := os.Stat(filename)
257
+ info, err := fsrv.fileSystem.Stat(filename)
258
if err != nil {
189
- err = mapDirOpenError(err, filename)
190
- if os.IsNotExist(err) {
259
+ err = fsrv.mapDirOpenError(err, filename)
260
+ if errors.Is(err, fs.ErrNotExist) {
261
return fsrv.notFound(w, r, next)
192
- } else if os.IsPermission(err) {
262
+ } else if errors.Is(err, fs.ErrPermission) {
263
return caddyhttp.Error(http.StatusForbidden, err)
264
}
265
return caddyhttp.Error(http.StatusInternalServerError, err)
@@ -210,7 +280,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
280
continue
281
}
282
213
- indexInfo, err := os.Stat(indexPath)
283
+ indexInfo, err := fsrv.fileSystem.Stat(indexPath)
284
if err != nil {
285
continue
286
}
@@ -280,7 +350,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
350
}
351
}
352
283
- var file *os.File
353
+ var file fs.File
354
355
// check for precompressed files
356
for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) {
@@ -289,7 +359,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
359
continue
360
}
361
compressedFilename := filename + precompress.Suffix()
292
- compressedInfo, err := os.Stat(compressedFilename)
362
+ compressedInfo, err := fsrv.fileSystem.Stat(compressedFilename)
363
if err != nil || compressedInfo.IsDir() {
364
fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err))
365
continue
@@ -328,14 +398,12 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
398
399
// set the ETag - note that a conditional If-None-Match request is handled
400
// by http.ServeContent below, which checks against this ETag value
331
- w.Header().Set("ETag", calculateEtag(info))
401
+ w.Header().Set("Etag", calculateEtag(info))
402
403
if w.Header().Get("Content-Type") == "" {
404
mtyp := mime.TypeByExtension(filepath.Ext(filename))
405
if mtyp == "" {
336
- // do not allow Go to sniff the content-type; see
337
- // https://www.youtube.com/watch?v=8t8JYpt0egE
338
- // TODO: If we want a Content-Type, consider writing a default of application/octet-stream - this is secure but violates spec
406
+ // do not allow Go to sniff the content-type; see https://www.youtube.com/watch?v=8t8JYpt0egE
407
w.Header()["Content-Type"] = nil
408
} else {
409
w.Header().Set("Content-Type", mtyp)
@@ -375,7 +443,7 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
443
// that errors generated by ServeContent are written immediately
444
// to the response, so we cannot handle them (but errors there
445
// are rare)
378
- http.ServeContent(w, r, info.Name(), info.ModTime(), file)
446
+ http.ServeContent(w, r, info.Name(), info.ModTime(), file.(io.ReadSeeker))
447
448
return nil
449
}
@@ -384,10 +452,10 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c
452
// the response is configured to inform the client how to best handle it
453
// and a well-described handler error is returned (do not wrap the
454
// returned error value).
387
-func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (*os.File, error) {
388
- file, err := os.Open(filename)
455
+func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (fs.File, error) {
456
+ file, err := fsrv.fileSystem.Open(filename)
457
if err != nil {
390
- err = mapDirOpenError(err, filename)
458
+ err = fsrv.mapDirOpenError(err, filename)
459
if os.IsNotExist(err) {
460
fsrv.logger.Debug("file not found", zap.String("filename", filename), zap.Error(err))
461
return nil, caddyhttp.Error(http.StatusNotFound, err)
@@ -412,8 +480,8 @@ func (fsrv *FileServer) openFile(filename string, w http.ResponseWriter) (*os.Fi
480
// Adapted from the Go standard library; originally written by Nathaniel Caza.
481
// https://go-review.googlesource.com/c/go/+/36635/
482
// https://go-review.googlesource.com/c/go/+/36804/
415
-func mapDirOpenError(originalErr error, name string) error {
416
- if os.IsNotExist(originalErr) || os.IsPermission(originalErr) {
483
+func (fsrv *FileServer) mapDirOpenError(originalErr error, name string) error {
484
+ if errors.Is(originalErr, fs.ErrNotExist) || errors.Is(originalErr, fs.ErrPermission) {
485
return originalErr
486
}
487
@@ -422,12 +490,12 @@ func mapDirOpenError(originalErr error, name string) error {
490
if parts[i] == "" {
491
continue
492
}
425
- fi, err := os.Stat(strings.Join(parts[:i+1], separator))
493
+ fi, err := fsrv.fileSystem.Stat(strings.Join(parts[:i+1], separator))
494
if err != nil {
495
return originalErr
496
}
497
if !fi.IsDir() {
430
- return os.ErrNotExist
498
+ return fs.ErrNotExist
499
}
500
}
501
@@ -545,6 +613,16 @@ func (wr statusOverrideResponseWriter) WriteHeader(int) {
613
wr.ResponseWriter.WriteHeader(wr.code)
614
}
615
616
+// osFS is a simple fs.StatFS implementation that uses the local
617
+// file system. (We do not use os.DirFS because we do our own
618
+// rooting or path prefixing without being constrained to a single
619
+// root folder. The standard os.DirFS implementation is problematic
620
+// since roots can be dynamic in our application.)
621
+type osFS struct{}
622
+
623
+func (osFS) Open(name string) (fs.File, error) { return os.Open(name) }
624
+func (osFS) Stat(name string) (fs.FileInfo, error) { return os.Stat(name) }
625
+
626
var defaultIndexNames = []string{"index.html", "index.txt"}
627
628
const (