Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 08:30 UTC
fileserver: speed up directory browsing for large directories (#7903)
* improve fileserver performance and allocations * skip instead of fatal * fix comment * implement more suggestions * use filesystems.OsFS instead of os.DirFS * fmt * remove outdated benchmarks
Christian Mehlmauer committed
Jul 31, 2026 at 16:38 UTC
2adc763020bd187415bde8ca1e5e529d9caf1c1a
5 files changed
+283
-51
modules/caddyhttp/fileserver/browse.go
-14
@@ -333,20 +333,6 @@ func (fsrv *FileServer) makeBrowseTemplate(tplCtx *templateContext) (*template.T
333
return tpl, nil
334
}
335
336
-// isSymlinkTargetDir returns true if f's symbolic link target
337
-// is a directory.
338
-func (fsrv *FileServer) isSymlinkTargetDir(fileSystem fs.FS, f fs.FileInfo, root, urlPath string) bool {
339
- if !isSymlink(f) {
340
- return false
341
- }
342
- target := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, f.Name()))
343
- targetInfo, err := fs.Stat(fileSystem, target)
344
- if err != nil {
345
- return false
346
- }
347
- return targetInfo.IsDir()
348
-}
349
-
336
// isSymlink return true if f is a symbolic link.
337
func isSymlink(f fs.FileInfo) bool {
338
return f.Mode()&os.ModeSymlink != 0
modules/caddyhttp/fileserver/browsetplcontext.go
+40
-18
@@ -44,6 +44,9 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
44
Path: urlPath,
45
CanGoUp: canGoUp,
46
lastModified: parentModTime,
47
+ // preallocate, since we know the upper bound on the number of
48
+ // items; avoids repeated slice growth/copy for large directories
49
+ Items: make([]fileInfo, 0, len(entries)),
50
}
51
52
for _, entry := range entries {
@@ -71,7 +74,23 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
74
tplCtx.lastModified = modTime
75
}
76
74
- isDir := entry.IsDir() || fsrv.isSymlinkTargetDir(fileSystem, info, root, urlPath)
77
+ fileIsSymlink := isSymlink(info)
78
+ size := info.Size()
79
+
80
+ // for a symlink, a single stat of its target tells us both whether
81
+ // the target is a directory and what its size is
82
+ var targetInfo fs.FileInfo
83
+ var targetPath string
84
+ if fileIsSymlink {
85
+ targetPath = caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
86
+ // An error most likely means the symlink target doesn't exist,
87
+ // which isn't entirely unusual and shouldn't fail the listing.
88
+ // In this case, targetInfo stays nil and we fall back to
89
+ // treating it as a non-directory, using the symlink's own size.
90
+ targetInfo, _ = fs.Stat(fileSystem, targetPath)
91
+ }
92
+
93
+ isDir := entry.IsDir() || (targetInfo != nil && targetInfo.IsDir())
94
95
// add the slash after the escape of path to avoid escaping the slash as well
96
if isDir {
@@ -81,34 +100,24 @@ func (fsrv *FileServer) directoryListing(ctx context.Context, fileSystem fs.FS,
100
tplCtx.NumFiles++
101
}
102
84
- size := info.Size()
85
-
103
if !isDir {
104
// increase the total by the symlink's size, not the target's size,
105
// by incrementing before we follow the symlink
106
tplCtx.TotalFileSize += size
107
}
108
92
- fileIsSymlink := isSymlink(info)
109
symlinkPath := ""
110
if fileIsSymlink {
95
- path := caddyhttp.SanitizedPathJoin(root, path.Join(urlPath, info.Name()))
96
- fileInfo, err := fs.Stat(fileSystem, path)
97
- if err == nil {
98
- size = fileInfo.Size()
111
+ if targetInfo != nil {
112
+ size = targetInfo.Size()
113
}
114
115
if fsrv.Browse.RevealSymlinks {
102
- symLinkTarget, err := os.Readlink(path)
116
+ symLinkTarget, err := os.Readlink(targetPath)
117
if err == nil {
118
symlinkPath = symLinkTarget
119
}
120
}
107
-
108
- // An error most likely means the symlink target doesn't exist,
109
- // which isn't entirely unusual and shouldn't fail the listing.
110
- // In this case, just use the size of the symlink itself, which
111
- // was already set above.
121
}
122
123
if !isDir {
@@ -220,6 +229,14 @@ func (l *browseTemplateContext) applySortAndLimit(sortParam, orderParam, limitPa
229
l.Sort = sortParam
230
l.Order = orderParam
231
232
+ // Only compute lowercase names when the selected sort comparator needs it;
233
+ // avoid O(n) work when sorting by time or when no sorting is requested.
234
+ if l.Sort == sortByName || l.Sort == sortByNameDirFirst || l.Sort == sortBySize {
235
+ for i := range l.Items {
236
+ l.Items[i].nameLower = strings.ToLower(l.Items[i].Name)
237
+ }
238
+ }
239
+
240
if l.Order == "desc" {
241
switch l.Sort {
242
case sortByName:
@@ -282,6 +299,11 @@ type fileInfo struct {
299
300
// a pointer to the template context is useful inside nested templates
301
Tpl *browseTemplateContext `json:"-"`
302
+
303
+ // cached lowercase Name, precomputed once by applySortAndLimit so that
304
+ // byName/byNameDirFirst/bySize don't each recompute it on every one of
305
+ // the O(n log n) comparisons sort.Sort makes; not serialized.
306
+ nameLower string
307
}
308
309
// HasExt returns true if the filename has any of the given suffixes, case-insensitive.
@@ -328,7 +350,7 @@ func (l byName) Len() int { return len(l.Items) }
350
func (l byName) Swap(i, j int) { l.Items[i], l.Items[j] = l.Items[j], l.Items[i] }
351
352
func (l byName) Less(i, j int) bool {
331
- return strings.ToLower(l.Items[i].Name) < strings.ToLower(l.Items[j].Name)
353
+ return l.Items[i].nameLower < l.Items[j].nameLower
354
}
355
356
func (l byNameDirFirst) Len() int { return len(l.Items) }
@@ -337,7 +359,7 @@ func (l byNameDirFirst) Swap(i, j int) { l.Items[i], l.Items[j] = l.Items[j], l.
359
func (l byNameDirFirst) Less(i, j int) bool {
360
// sort by name if both are dir or file
361
if l.Items[i].IsDir == l.Items[j].IsDir {
340
- return strings.ToLower(l.Items[i].Name) < strings.ToLower(l.Items[j].Name)
362
+ return l.Items[i].nameLower < l.Items[j].nameLower
363
}
364
// sort dir ahead of file
365
return l.Items[i].IsDir
@@ -347,7 +369,7 @@ func (l bySize) Len() int { return len(l.Items) }
369
func (l bySize) Swap(i, j int) { l.Items[i], l.Items[j] = l.Items[j], l.Items[i] }
370
371
func (l bySize) Less(i, j int) bool {
350
- const directoryOffset = -1 << 31 // = -math.MinInt32
372
+ const directoryOffset = -1 << 31 // = -2147483648 (min int32)
373
374
iSize, jSize := l.Items[i].Size, l.Items[j].Size
375
@@ -361,7 +383,7 @@ func (l bySize) Less(i, j int) bool {
383
jSize = directoryOffset
384
}
385
if l.Items[i].IsDir && l.Items[j].IsDir {
364
- return strings.ToLower(l.Items[i].Name) < strings.ToLower(l.Items[j].Name)
386
+ return l.Items[i].nameLower < l.Items[j].nameLower
387
}
388
389
return iSize < jSize
modules/caddyhttp/fileserver/browsetplcontext_bench_test.go
new
+224
@@ -0,0 +1,224 @@
1
+// Copyright 2015 Matthew Holt and The Caddy Authors
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package fileserver
16
+
17
+import (
18
+ "context"
19
+ "fmt"
20
+ "io/fs"
21
+ "os"
22
+ "path/filepath"
23
+ "sort"
24
+ "strings"
25
+ "testing"
26
+ "time"
27
+
28
+ "go.uber.org/zap"
29
+
30
+ "github.com/caddyserver/caddy/v2"
31
+ "github.com/caddyserver/caddy/v2/internal/filesystems"
32
+)
33
+
34
+// makeBenchDir populates dir with n regular files and a handful of
35
+// symlinks (both to files and to a subdirectory), mirroring what a
36
+// real, large, browsable directory tends to contain.
37
+func makeBenchDir(tb testing.TB, dir string, n int) {
38
+ tb.Helper()
39
+
40
+ subdir := filepath.Join(dir, "subdir")
41
+ if err := os.Mkdir(subdir, 0o755); err != nil {
42
+ tb.Fatal(err)
43
+ }
44
+
45
+ for i := range n {
46
+ name := filepath.Join(dir, fmt.Sprintf("file-%d.txt", i))
47
+ if err := os.WriteFile(name, []byte("x"), 0o600); err != nil {
48
+ tb.Fatal(err)
49
+ }
50
+ }
51
+
52
+ target := filepath.Join(dir, "file-0.txt")
53
+ if err := os.Symlink(target, filepath.Join(dir, "symlink-to-file")); err != nil {
54
+ tb.Skipf("symlink not supported on this platform: %v", err)
55
+ }
56
+ if err := os.Symlink(subdir, filepath.Join(dir, "symlink-to-dir")); err != nil {
57
+ tb.Skipf("symlink not supported on this platform: %v", err)
58
+ }
59
+}
60
+
61
+// readBenchDirEntries reads all entries of dir once, up front, so that
62
+// benchmark iterations measure only the cost of directoryListing itself
63
+// (allocations, stat calls per entry) rather than the readdir syscall.
64
+//
65
+// It deliberately opens the directory and calls the ReadDir method on the
66
+// resulting file (like loadDirectoryContents does in production), not
67
+// fs.ReadDir/os.ReadDir - the latter sort entries by filename before
68
+// returning them, which would silently hand the sort benchmarks
69
+// already-sorted input and let sort.Sort's adaptive algorithm breeze
70
+// through with far fewer comparisons than a real, filesystem-ordered
71
+// directory listing requires.
72
+func readBenchDirEntries(tb testing.TB, dir string) (fs.FS, []fs.DirEntry) {
73
+ tb.Helper()
74
+
75
+ fileSystem := filesystems.OsFS{}
76
+ f, err := fileSystem.Open(dir)
77
+ if err != nil {
78
+ tb.Fatal(err)
79
+ }
80
+ defer f.Close()
81
+
82
+ entries, err := f.(fs.ReadDirFile).ReadDir(-1)
83
+ if err != nil {
84
+ tb.Fatal(err)
85
+ }
86
+ return fileSystem, entries
87
+}
88
+
89
+func benchFileServer() *FileServer {
90
+ return &FileServer{
91
+ Browse: &Browse{},
92
+ logger: zap.NewNop(),
93
+ }
94
+}
95
+
96
+func benchmarkDirectoryListing(b *testing.B, n int, listFn func(fsrv *FileServer, ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, root string) *browseTemplateContext) {
97
+ dir := b.TempDir()
98
+ makeBenchDir(b, dir, n)
99
+ fileSystem, entries := readBenchDirEntries(b, dir)
100
+ fsrv := benchFileServer()
101
+ ctx := context.Background()
102
+
103
+ b.ReportAllocs()
104
+
105
+ for b.Loop() {
106
+ listFn(fsrv, ctx, fileSystem, entries, dir)
107
+ }
108
+}
109
+
110
+func BenchmarkDirectoryListing(b *testing.B) {
111
+ for _, n := range []int{100, 1_000, 10_000, 50_000} {
112
+ b.Run(fmt.Sprintf("entries=%d", n), func(b *testing.B) {
113
+ benchmarkDirectoryListing(b, n, func(fsrv *FileServer, ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, root string) *browseTemplateContext {
114
+ return fsrv.directoryListing(ctx, fileSystem, time.Time{}, entries, true, root, "/", caddy.NewReplacer())
115
+ })
116
+ })
117
+ }
118
+}
119
+
120
+// makeBenchItems builds n synthetic fileInfo items (no disk I/O), mixing
121
+// upper/lower case names and marking roughly 1 in 10 as directories, to
122
+// exercise the sort comparators - including bySize's directory name
123
+// tie-break - the way a real, large, mixed listing would.
124
+func makeBenchItems(n int) []fileInfo {
125
+ items := make([]fileInfo, n)
126
+ for i := range n {
127
+ name := fmt.Sprintf("file_%d.txt", i)
128
+ if i%2 == 0 {
129
+ name = fmt.Sprintf("FILE_%d.TXT", i)
130
+ }
131
+ items[i] = fileInfo{
132
+ Name: name,
133
+ Size: int64(i),
134
+ IsDir: i%10 == 0,
135
+ }
136
+ }
137
+ return items
138
+}
139
+
140
+func fillNameLower(items []fileInfo) {
141
+ for i := range items {
142
+ items[i].nameLower = strings.ToLower(items[i].Name)
143
+ }
144
+}
145
+
146
+// benchmarkSort times sortFn against a fresh, unsorted copy of a synthetic
147
+// item set on every iteration. The copy, and the optional prepare step
148
+// (e.g. filling the nameLower cache), happen outside the timed portion so
149
+// the benchmark isolates the cost of the sort itself.
150
+func benchmarkSort(b *testing.B, n int, prepare func(items []fileInfo), sortFn func(items []fileInfo)) {
151
+ pristine := makeBenchItems(n)
152
+ items := make([]fileInfo, n)
153
+
154
+ for b.Loop() {
155
+ b.StopTimer()
156
+ copy(items, pristine)
157
+ if prepare != nil {
158
+ prepare(items)
159
+ }
160
+ b.StartTimer()
161
+ sortFn(items)
162
+ }
163
+}
164
+
165
+func BenchmarkSortByName(b *testing.B) {
166
+ for _, n := range []int{100, 1_000, 10_000, 50_000} {
167
+ b.Run(fmt.Sprintf("entries=%d", n), func(b *testing.B) {
168
+ benchmarkSort(b, n, fillNameLower, func(items []fileInfo) {
169
+ sort.Sort(byName(browseTemplateContext{Items: items}))
170
+ })
171
+ })
172
+ }
173
+}
174
+
175
+func BenchmarkSortByNameDirFirst(b *testing.B) {
176
+ for _, n := range []int{100, 1_000, 10_000, 50_000} {
177
+ b.Run(fmt.Sprintf("entries=%d", n), func(b *testing.B) {
178
+ benchmarkSort(b, n, fillNameLower, func(items []fileInfo) {
179
+ sort.Sort(byNameDirFirst(browseTemplateContext{Items: items}))
180
+ })
181
+ })
182
+ }
183
+}
184
+
185
+func BenchmarkSortBySize(b *testing.B) {
186
+ for _, n := range []int{100, 1_000, 10_000, 50_000} {
187
+ b.Run(fmt.Sprintf("entries=%d", n), func(b *testing.B) {
188
+ benchmarkSort(b, n, fillNameLower, func(items []fileInfo) {
189
+ sort.Sort(bySize(browseTemplateContext{Items: items}))
190
+ })
191
+ })
192
+ }
193
+}
194
+
195
+// ---------------------------------------------------------------------
196
+// Overall benchmark: the whole real-world browse-request path - reading
197
+// directory entries into a listing and then sorting it.
198
+// ---------------------------------------------------------------------
199
+
200
+func benchmarkDirectoryListingAndSort(b *testing.B, n int, run func(fsrv *FileServer, ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, root string) *browseTemplateContext) {
201
+ dir := b.TempDir()
202
+ makeBenchDir(b, dir, n)
203
+ fileSystem, entries := readBenchDirEntries(b, dir)
204
+ fsrv := benchFileServer()
205
+ ctx := context.Background()
206
+
207
+ b.ReportAllocs()
208
+
209
+ for b.Loop() {
210
+ run(fsrv, ctx, fileSystem, entries, dir)
211
+ }
212
+}
213
+
214
+func BenchmarkDirectoryListingAndSort(b *testing.B) {
215
+ for _, n := range []int{100, 1_000, 10_000, 50_000} {
216
+ b.Run(fmt.Sprintf("entries=%d", n), func(b *testing.B) {
217
+ benchmarkDirectoryListingAndSort(b, n, func(fsrv *FileServer, ctx context.Context, fileSystem fs.FS, entries []fs.DirEntry, root string) *browseTemplateContext {
218
+ listing := fsrv.directoryListing(ctx, fileSystem, time.Time{}, entries, true, root, "/", caddy.NewReplacer())
219
+ listing.applySortAndLimit(sortByNameDirFirst, sortOrderAsc, "", "")
220
+ return listing
221
+ })
222
+ })
223
+ }
224
+}
modules/caddyhttp/reverseproxy/selectionpolicies_test.go
+16
-16
@@ -133,36 +133,36 @@ func TestWeightedRoundRobinPolicy(t *testing.T) {
133
134
func TestWeightedRoundRobinSelection_Validate(t *testing.T) {
135
tests := []struct {
136
- name string
137
- weights []int
138
- wantErr bool
136
+ name string
137
+ weights []int
138
+ wantErr bool
139
}{
140
{
141
- name: "Valid 0 2 1 case",
142
- weights: []int{0, 2, 1},
143
- wantErr: false,
141
+ name: "Valid 0 2 1 case",
142
+ weights: []int{0, 2, 1},
143
+ wantErr: false,
144
},
145
{
146
- name: "Invalid 0 case (single)",
147
- weights: []int{0},
148
- wantErr: true,
146
+ name: "Invalid 0 case (single)",
147
+ weights: []int{0},
148
+ wantErr: true,
149
},
150
{
151
- name: "Invalid 0 0 case (multiple)",
152
- weights: []int{0, 0},
153
- wantErr: true,
151
+ name: "Invalid 0 0 case (multiple)",
152
+ weights: []int{0, 0},
153
+ wantErr: true,
154
},
155
{
156
- name: "Valid weights",
157
- weights: []int{1, 1, 1},
158
- wantErr: false,
156
+ name: "Valid weights",
157
+ weights: []int{1, 1, 1},
158
+ wantErr: false,
159
},
160
}
161
162
for _, tt := range tests {
163
t.Run(tt.name, func(t *testing.T) {
164
s := &WeightedRoundRobinSelection{
165
- Weights: tt.weights,
165
+ Weights: tt.weights,
166
}
167
_ = s.Provision(caddy.Context{})
168
err := s.Validate()
modules/caddyhttp/templates/tplcontext_test.go
+3
-3
@@ -424,9 +424,9 @@ func TestStripHTML(t *testing.T) {
424
expect: ``,
425
},
426
{
427
- // false start — second '<' increments depth, single '>' only closes one level
428
- input: `<h1<b>hi`,
429
- expect: ``,
427
+ // false start — second '<' increments depth, single '>' only closes one level
428
+ input: `<h1<b>hi`,
429
+ expect: ``,
430
},
431
{
432
// XSS bypass via double opening bracket