Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 13, 05:30 UTC
caddyfile: Implement variadics for import args placeholders (#5249)
* implement variadic placeholders imported snippets reflect actual lines in file * add import directive line number for imported snippets add tests for parsing * add realfile field to help debug import cycle detection. * use file field to reflect import chain * Switch syntax, deprecate old syntax, refactoring - Moved the import args handling to a separate file - Using {args[0:1]} syntax now - Deprecate {args.*} syntax - Use a replacer map for better control over the parsing - Add plenty of warnings when invalid placeholders are detected - Renaming variables, cleanup comments for readability - More tests to cover edgecases I could think of - Minor cleanup to snippet tracking in tokens, drop a redundant boolean field in tokens --------- Co-authored-by: Francis Lavoie <lavofr@gmail.com>
WeidiDeng committed
Feb 17, 2023 at 08:08 UTC
8bc05e598da0068358a2876c45f92f2c77455341
5 files changed
+313
-18
caddyconfig/caddyfile/importargs.go
new
+142
@@ -0,0 +1,142 @@
1
+// Copyright 2015 Matthew Holt and The Caddy Authors
2
+//
3
+// Licensed under the Apache License, Version 2.0 (the "License");
4
+// you may not use this file except in compliance with the License.
5
+// You may obtain a copy of the License at
6
+//
7
+// http://www.apache.org/licenses/LICENSE-2.0
8
+//
9
+// Unless required by applicable law or agreed to in writing, software
10
+// distributed under the License is distributed on an "AS IS" BASIS,
11
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12
+// See the License for the specific language governing permissions and
13
+// limitations under the License.
14
+
15
+package caddyfile
16
+
17
+import (
18
+ "regexp"
19
+ "strconv"
20
+ "strings"
21
+
22
+ "github.com/caddyserver/caddy/v2"
23
+ "go.uber.org/zap"
24
+)
25
+
26
+// parseVariadic determines if the token is a variadic placeholder,
27
+// and if so, determines the index range (start/end) of args to use.
28
+// Returns a boolean signaling whether a variadic placeholder was found,
29
+// and the start and end indices.
30
+func parseVariadic(token Token, argCount int) (bool, int, int) {
31
+ if !strings.HasPrefix(token.Text, "{args[") {
32
+ return false, 0, 0
33
+ }
34
+ if !strings.HasSuffix(token.Text, "]}") {
35
+ return false, 0, 0
36
+ }
37
+
38
+ argRange := strings.TrimSuffix(strings.TrimPrefix(token.Text, "{args["), "]}")
39
+ if argRange == "" {
40
+ caddy.Log().Named("caddyfile").Warn(
41
+ "Placeholder "+token.Text+" cannot have an empty index",
42
+ zap.String("file", token.File+":"+strconv.Itoa(token.Line)))
43
+ return false, 0, 0
44
+ }
45
+
46
+ start, end, found := strings.Cut(argRange, ":")
47
+
48
+ // If no ":" delimiter is found, this is not a variadic.
49
+ // The replacer will pick this up.
50
+ if !found {
51
+ return false, 0, 0
52
+ }
53
+
54
+ var (
55
+ startIndex = 0
56
+ endIndex = argCount
57
+ err error
58
+ )
59
+ if start != "" {
60
+ startIndex, err = strconv.Atoi(start)
61
+ if err != nil {
62
+ caddy.Log().Named("caddyfile").Warn(
63
+ "Variadic placeholder "+token.Text+" has an invalid start index",
64
+ zap.String("file", token.File+":"+strconv.Itoa(token.Line)))
65
+ return false, 0, 0
66
+ }
67
+ }
68
+ if end != "" {
69
+ endIndex, err = strconv.Atoi(end)
70
+ if err != nil {
71
+ caddy.Log().Named("caddyfile").Warn(
72
+ "Variadic placeholder "+token.Text+" has an invalid end index",
73
+ zap.String("file", token.File+":"+strconv.Itoa(token.Line)))
74
+ return false, 0, 0
75
+ }
76
+ }
77
+
78
+ // bound check
79
+ if startIndex < 0 || startIndex > endIndex || endIndex > argCount {
80
+ caddy.Log().Named("caddyfile").Warn(
81
+ "Variadic placeholder "+token.Text+" indices are out of bounds, only "+strconv.Itoa(argCount)+" argument(s) exist",
82
+ zap.String("file", token.File+":"+strconv.Itoa(token.Line)))
83
+ return false, 0, 0
84
+ }
85
+ return true, startIndex, endIndex
86
+}
87
+
88
+// makeArgsReplacer prepares a Replacer which can replace
89
+// non-variadic args placeholders in imported tokens.
90
+func makeArgsReplacer(args []string) *caddy.Replacer {
91
+ repl := caddy.NewEmptyReplacer()
92
+ repl.Map(func(key string) (any, bool) {
93
+ // TODO: Remove the deprecated {args.*} placeholder
94
+ // support at some point in the future
95
+ if matches := argsRegexpIndexDeprecated.FindStringSubmatch(key); len(matches) > 0 {
96
+ value, err := strconv.Atoi(matches[1])
97
+ if err != nil {
98
+ caddy.Log().Named("caddyfile").Warn(
99
+ "Placeholder {args." + matches[1] + "} has an invalid index")
100
+ return nil, false
101
+ }
102
+ if value >= len(args) {
103
+ caddy.Log().Named("caddyfile").Warn(
104
+ "Placeholder {args." + matches[1] + "} index is out of bounds, only " + strconv.Itoa(len(args)) + " argument(s) exist")
105
+ return nil, false
106
+ }
107
+ caddy.Log().Named("caddyfile").Warn(
108
+ "Placeholder {args." + matches[1] + "} deprecated, use {args[" + matches[1] + "]} instead")
109
+ return args[value], true
110
+ }
111
+
112
+ // Handle args[*] form
113
+ if matches := argsRegexpIndex.FindStringSubmatch(key); len(matches) > 0 {
114
+ if strings.Contains(matches[1], ":") {
115
+ caddy.Log().Named("caddyfile").Warn(
116
+ "Variadic placeholder {args[" + matches[1] + "]} must be a token on its own")
117
+ return nil, false
118
+ }
119
+ value, err := strconv.Atoi(matches[1])
120
+ if err != nil {
121
+ caddy.Log().Named("caddyfile").Warn(
122
+ "Placeholder {args[" + matches[1] + "]} has an invalid index")
123
+ return nil, false
124
+ }
125
+ if value >= len(args) {
126
+ caddy.Log().Named("caddyfile").Warn(
127
+ "Placeholder {args[" + matches[1] + "]} index is out of bounds, only " + strconv.Itoa(len(args)) + " argument(s) exist")
128
+ return nil, false
129
+ }
130
+ return args[value], true
131
+ }
132
+
133
+ // Not an args placeholder, ignore
134
+ return nil, false
135
+ })
136
+ return repl
137
+}
138
+
139
+var (
140
+ argsRegexpIndexDeprecated = regexp.MustCompile(`args\.(.+)`)
141
+ argsRegexpIndex = regexp.MustCompile(`args\[(.+)]`)
142
+)
caddyconfig/caddyfile/lexer.go
+18
-1
@@ -36,14 +36,31 @@ type (
36
// Token represents a single parsable unit.
37
Token struct {
38
File string
39
+ origFile string
40
Line int
41
Text string
42
wasQuoted rune // enclosing quote character, if any
42
- inSnippet bool
43
snippetName string
44
}
45
)
46
47
+// originalFile gets original filename before import modification.
48
+func (t Token) originalFile() string {
49
+ if t.origFile != "" {
50
+ return t.origFile
51
+ }
52
+ return t.File
53
+}
54
+
55
+// updateFile updates the token's source filename for error display
56
+// and remembers the original filename. Used during "import" processing.
57
+func (t *Token) updateFile(file string) {
58
+ if t.origFile == "" {
59
+ t.origFile = t.File
60
+ }
61
+ t.File = file
62
+}
63
+
64
// load prepares the lexer to scan an input for tokens.
65
// It discards any leading byte order mark.
66
func (l *lexer) load(input io.Reader) error {
caddyconfig/caddyfile/parse.go
+28
-17
@@ -20,7 +20,6 @@ import (
20
"io"
21
"os"
22
"path/filepath"
23
- "strconv"
23
"strings"
24
25
"github.com/caddyserver/caddy/v2"
@@ -173,11 +172,10 @@ func (p *parser) begin() error {
172
return err
173
}
174
// Just as we need to track which file the token comes from, we need to
176
- // keep track of which snippets do the tokens come from. This is helpful
177
- // in tracking import cycles across files/snippets by namespacing them. Without
178
- // this we end up with false-positives in cycle-detection.
175
+ // keep track of which snippet the token comes from. This is helpful
176
+ // in tracking import cycles across files/snippets by namespacing them.
177
+ // Without this, we end up with false-positives in cycle-detection.
178
for k, v := range tokens {
180
- v.inSnippet = true
179
v.snippetName = name
180
tokens[k] = v
181
}
@@ -337,11 +335,8 @@ func (p *parser) doImport() error {
335
// grab remaining args as placeholder replacements
336
args := p.RemainingArgs()
337
340
- // add args to the replacer
341
- repl := caddy.NewEmptyReplacer()
342
- for index, arg := range args {
343
- repl.Set("args."+strconv.Itoa(index), arg)
344
- }
338
+ // set up a replacer for non-variadic args replacement
339
+ repl := makeArgsReplacer(args)
340
341
// splice out the import directive and its arguments
342
// (2 tokens, plus the length of args)
@@ -416,8 +411,8 @@ func (p *parser) doImport() error {
411
nodes = matches
412
}
413
419
- nodeName := p.File()
420
- if p.Token().inSnippet {
414
+ nodeName := p.Token().originalFile()
415
+ if p.Token().snippetName != "" {
416
nodeName += fmt.Sprintf(":%s", p.Token().snippetName)
417
}
418
p.importGraph.addNode(nodeName)
@@ -428,13 +423,29 @@ func (p *parser) doImport() error {
423
}
424
425
// copy the tokens so we don't overwrite p.definedSnippets
431
- tokensCopy := make([]Token, len(importedTokens))
432
- copy(tokensCopy, importedTokens)
426
+ tokensCopy := make([]Token, 0, len(importedTokens))
427
428
// run the argument replacer on the tokens
435
- for index, token := range tokensCopy {
436
- token.Text = repl.ReplaceKnown(token.Text, "")
437
- tokensCopy[index] = token
429
+ // golang for range slice return a copy of value
430
+ // similarly, append also copy value
431
+ for _, token := range importedTokens {
432
+ // set the token's file to refer to import directive line number and snippet name
433
+ if token.snippetName != "" {
434
+ token.updateFile(fmt.Sprintf("%s:%d (import %s)", token.File, p.Line(), token.snippetName))
435
+ } else {
436
+ token.updateFile(fmt.Sprintf("%s:%d (import)", token.File, p.Line()))
437
+ }
438
+
439
+ foundVariadic, startIndex, endIndex := parseVariadic(token, len(args))
440
+ if foundVariadic {
441
+ for _, arg := range args[startIndex:endIndex] {
442
+ token.Text = arg
443
+ tokensCopy = append(tokensCopy, token)
444
+ }
445
+ } else {
446
+ token.Text = repl.ReplaceKnown(token.Text, "")
447
+ tokensCopy = append(tokensCopy, token)
448
+ }
449
}
450
451
// splice the imported tokens in the place of the import statement
caddyconfig/caddyfile/parse_test.go
+82
@@ -21,6 +21,88 @@ import (
21
"testing"
22
)
23
24
+func TestParseVariadic(t *testing.T) {
25
+ var args = make([]string, 10)
26
+ for i, tc := range []struct {
27
+ input string
28
+ result bool
29
+ }{
30
+ {
31
+ input: "",
32
+ result: false,
33
+ },
34
+ {
35
+ input: "{args[1",
36
+ result: false,
37
+ },
38
+ {
39
+ input: "1]}",
40
+ result: false,
41
+ },
42
+ {
43
+ input: "{args[:]}aaaaa",
44
+ result: false,
45
+ },
46
+ {
47
+ input: "aaaaa{args[:]}",
48
+ result: false,
49
+ },
50
+ {
51
+ input: "{args.}",
52
+ result: false,
53
+ },
54
+ {
55
+ input: "{args.1}",
56
+ result: false,
57
+ },
58
+ {
59
+ input: "{args[]}",
60
+ result: false,
61
+ },
62
+ {
63
+ input: "{args[:]}",
64
+ result: true,
65
+ },
66
+ {
67
+ input: "{args[:]}",
68
+ result: true,
69
+ },
70
+ {
71
+ input: "{args[0:]}",
72
+ result: true,
73
+ },
74
+ {
75
+ input: "{args[:0]}",
76
+ result: true,
77
+ },
78
+ {
79
+ input: "{args[-1:]}",
80
+ result: false,
81
+ },
82
+ {
83
+ input: "{args[:11]}",
84
+ result: false,
85
+ },
86
+ {
87
+ input: "{args[10:0]}",
88
+ result: false,
89
+ },
90
+ {
91
+ input: "{args[0:10]}",
92
+ result: true,
93
+ },
94
+ } {
95
+ token := Token{
96
+ File: "test",
97
+ Line: 1,
98
+ Text: tc.input,
99
+ }
100
+ if v, _, _ := parseVariadic(token, len(args)); v != tc.result {
101
+ t.Errorf("Test %d error expectation failed Expected: %t, got %t", i, tc.result, v)
102
+ }
103
+ }
104
+}
105
+
106
func TestAllTokens(t *testing.T) {
107
input := []byte("a b c\nd e")
108
expected := []string{"a", "b", "c", "d", "e"}
caddyconfig/httpcaddyfile/builtins_test.go
+43
@@ -1,6 +1,7 @@
1
package httpcaddyfile
2
3
import (
4
+ "strings"
5
"testing"
6
7
"github.com/caddyserver/caddy/v2/caddyconfig/caddyfile"
@@ -213,3 +214,45 @@ func TestRedirDirectiveSyntax(t *testing.T) {
214
}
215
}
216
}
217
+
218
+func TestImportErrorLine(t *testing.T) {
219
+ for i, tc := range []struct {
220
+ input string
221
+ errorFunc func(err error) bool
222
+ }{
223
+ {
224
+ input: `(t1) {
225
+ abort {args[:]}
226
+ }
227
+ :8080 {
228
+ import t1
229
+ import t1 true
230
+ }`,
231
+ errorFunc: func(err error) bool {
232
+ return err != nil && strings.Contains(err.Error(), "Caddyfile:6 (import t1):2")
233
+ },
234
+ },
235
+ {
236
+ input: `(t1) {
237
+ abort {args[:]}
238
+ }
239
+ :8080 {
240
+ import t1 true
241
+ }`,
242
+ errorFunc: func(err error) bool {
243
+ return err != nil && strings.Contains(err.Error(), "Caddyfile:5 (import t1):2")
244
+ },
245
+ },
246
+ } {
247
+ adapter := caddyfile.Adapter{
248
+ ServerType: ServerType{},
249
+ }
250
+
251
+ _, _, err := adapter.Adapt([]byte(tc.input), nil)
252
+
253
+ if !tc.errorFunc(err) {
254
+ t.Errorf("Test %d error expectation failed, got %s", i, err)
255
+ continue
256
+ }
257
+ }
258
+}