Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 14, 05:15 UTC
caddyfile: Export Tokenize function for lexing (#3549)
Francis Lavoie committed
Jul 20, 2020 at 15:55 UTC
fb9d874fa91c796fdb1c6cb32ab3246e63d38af9
3 files changed
+57
-54
caddyconfig/caddyfile/lexer.go
+19
@@ -16,6 +16,7 @@ package caddyfile
16
17
import (
18
"bufio"
19
+ "bytes"
20
"io"
21
"unicode"
22
)
@@ -168,3 +169,21 @@ func (l *lexer) next() bool {
169
val = append(val, ch)
170
}
171
}
172
+
173
+// Tokenize takes bytes as input and lexes it into
174
+// a list of tokens that can be parsed as a Caddyfile.
175
+// Also takes a filename to fill the token's File as
176
+// the source of the tokens, which is important to
177
+// determine relative paths for `import` directives.
178
+func Tokenize(input []byte, filename string) ([]Token, error) {
179
+ l := lexer{}
180
+ if err := l.load(bytes.NewReader(input)); err != nil {
181
+ return nil, err
182
+ }
183
+ var tokens []Token
184
+ for l.next() {
185
+ l.token.File = filename
186
+ tokens = append(tokens, l.token)
187
+ }
188
+ return tokens, nil
189
+}
caddyconfig/caddyfile/lexer_test.go
+37
-47
@@ -15,37 +15,35 @@
15
package caddyfile
16
17
import (
18
- "log"
19
- "strings"
18
"testing"
19
)
20
21
type lexerTestCase struct {
24
- input string
22
+ input []byte
23
expected []Token
24
}
25
26
func TestLexer(t *testing.T) {
27
testCases := []lexerTestCase{
28
{
31
- input: `host:123`,
29
+ input: []byte(`host:123`),
30
expected: []Token{
31
{Line: 1, Text: "host:123"},
32
},
33
},
34
{
37
- input: `host:123
35
+ input: []byte(`host:123
36
39
- directive`,
37
+ directive`),
38
expected: []Token{
39
{Line: 1, Text: "host:123"},
40
{Line: 3, Text: "directive"},
41
},
42
},
43
{
46
- input: `host:123 {
44
+ input: []byte(`host:123 {
45
directive
48
- }`,
46
+ }`),
47
expected: []Token{
48
{Line: 1, Text: "host:123"},
49
{Line: 1, Text: "{"},
@@ -54,7 +52,7 @@ func TestLexer(t *testing.T) {
52
},
53
},
54
{
57
- input: `host:123 { directive }`,
55
+ input: []byte(`host:123 { directive }`),
56
expected: []Token{
57
{Line: 1, Text: "host:123"},
58
{Line: 1, Text: "{"},
@@ -63,12 +61,12 @@ func TestLexer(t *testing.T) {
61
},
62
},
63
{
66
- input: `host:123 {
64
+ input: []byte(`host:123 {
65
#comment
66
directive
67
# comment
68
foobar # another comment
71
- }`,
69
+ }`),
70
expected: []Token{
71
{Line: 1, Text: "host:123"},
72
{Line: 1, Text: "{"},
@@ -78,10 +76,10 @@ func TestLexer(t *testing.T) {
76
},
77
},
78
{
81
- input: `host:123 {
79
+ input: []byte(`host:123 {
80
# hash inside string is not a comment
81
redir / /some/#/path
84
- }`,
82
+ }`),
83
expected: []Token{
84
{Line: 1, Text: "host:123"},
85
{Line: 1, Text: "{"},
@@ -92,14 +90,14 @@ func TestLexer(t *testing.T) {
90
},
91
},
92
{
95
- input: "# comment at beginning of file\n# comment at beginning of line\nhost:123",
93
+ input: []byte("# comment at beginning of file\n# comment at beginning of line\nhost:123"),
94
expected: []Token{
95
{Line: 3, Text: "host:123"},
96
},
97
},
98
{
101
- input: `a "quoted value" b
102
- foobar`,
99
+ input: []byte(`a "quoted value" b
100
+ foobar`),
101
expected: []Token{
102
{Line: 1, Text: "a"},
103
{Line: 1, Text: "quoted value"},
@@ -108,7 +106,7 @@ func TestLexer(t *testing.T) {
106
},
107
},
108
{
111
- input: `A "quoted \"value\" inside" B`,
109
+ input: []byte(`A "quoted \"value\" inside" B`),
110
expected: []Token{
111
{Line: 1, Text: "A"},
112
{Line: 1, Text: `quoted "value" inside`},
@@ -116,7 +114,7 @@ func TestLexer(t *testing.T) {
114
},
115
},
116
{
119
- input: "An escaped \"newline\\\ninside\" quotes",
117
+ input: []byte("An escaped \"newline\\\ninside\" quotes"),
118
expected: []Token{
119
{Line: 1, Text: "An"},
120
{Line: 1, Text: "escaped"},
@@ -125,7 +123,7 @@ func TestLexer(t *testing.T) {
123
},
124
},
125
{
128
- input: "An escaped newline\\\noutside quotes",
126
+ input: []byte("An escaped newline\\\noutside quotes"),
127
expected: []Token{
128
{Line: 1, Text: "An"},
129
{Line: 1, Text: "escaped"},
@@ -135,7 +133,7 @@ func TestLexer(t *testing.T) {
133
},
134
},
135
{
138
- input: "line1\\\nescaped\nline2\nline3",
136
+ input: []byte("line1\\\nescaped\nline2\nline3"),
137
expected: []Token{
138
{Line: 1, Text: "line1"},
139
{Line: 1, Text: "escaped"},
@@ -144,7 +142,7 @@ func TestLexer(t *testing.T) {
142
},
143
},
144
{
147
- input: "line1\\\nescaped1\\\nescaped2\nline4\nline5",
145
+ input: []byte("line1\\\nescaped1\\\nescaped2\nline4\nline5"),
146
expected: []Token{
147
{Line: 1, Text: "line1"},
148
{Line: 1, Text: "escaped1"},
@@ -154,34 +152,34 @@ func TestLexer(t *testing.T) {
152
},
153
},
154
{
157
- input: `"unescapable\ in quotes"`,
155
+ input: []byte(`"unescapable\ in quotes"`),
156
expected: []Token{
157
{Line: 1, Text: `unescapable\ in quotes`},
158
},
159
},
160
{
163
- input: `"don't\escape"`,
161
+ input: []byte(`"don't\escape"`),
162
expected: []Token{
163
{Line: 1, Text: `don't\escape`},
164
},
165
},
166
{
169
- input: `"don't\\escape"`,
167
+ input: []byte(`"don't\\escape"`),
168
expected: []Token{
169
{Line: 1, Text: `don't\\escape`},
170
},
171
},
172
{
175
- input: `un\escapable`,
173
+ input: []byte(`un\escapable`),
174
expected: []Token{
175
{Line: 1, Text: `un\escapable`},
176
},
177
},
178
{
181
- input: `A "quoted value with line
179
+ input: []byte(`A "quoted value with line
180
break inside" {
181
foobar
184
- }`,
182
+ }`),
183
expected: []Token{
184
{Line: 1, Text: "A"},
185
{Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"},
@@ -191,13 +189,13 @@ func TestLexer(t *testing.T) {
189
},
190
},
191
{
194
- input: `"C:\php\php-cgi.exe"`,
192
+ input: []byte(`"C:\php\php-cgi.exe"`),
193
expected: []Token{
194
{Line: 1, Text: `C:\php\php-cgi.exe`},
195
},
196
},
197
{
200
- input: `empty "" string`,
198
+ input: []byte(`empty "" string`),
199
expected: []Token{
200
{Line: 1, Text: `empty`},
201
{Line: 1, Text: ``},
@@ -205,7 +203,7 @@ func TestLexer(t *testing.T) {
203
},
204
},
205
{
208
- input: "skip those\r\nCR characters",
206
+ input: []byte("skip those\r\nCR characters"),
207
expected: []Token{
208
{Line: 1, Text: "skip"},
209
{Line: 1, Text: "those"},
@@ -214,13 +212,13 @@ func TestLexer(t *testing.T) {
212
},
213
},
214
{
217
- input: "\xEF\xBB\xBF:8080", // test with leading byte order mark
215
+ input: []byte("\xEF\xBB\xBF:8080"), // test with leading byte order mark
216
expected: []Token{
217
{Line: 1, Text: ":8080"},
218
},
219
},
220
{
223
- input: "simple `backtick quoted` string",
221
+ input: []byte("simple `backtick quoted` string"),
222
expected: []Token{
223
{Line: 1, Text: `simple`},
224
{Line: 1, Text: `backtick quoted`},
@@ -228,7 +226,7 @@ func TestLexer(t *testing.T) {
226
},
227
},
228
{
231
- input: "multiline `backtick\nquoted\n` string",
229
+ input: []byte("multiline `backtick\nquoted\n` string"),
230
expected: []Token{
231
{Line: 1, Text: `multiline`},
232
{Line: 1, Text: "backtick\nquoted\n"},
@@ -236,7 +234,7 @@ func TestLexer(t *testing.T) {
234
},
235
},
236
{
239
- input: "nested `\"quotes inside\" backticks` string",
237
+ input: []byte("nested `\"quotes inside\" backticks` string"),
238
expected: []Token{
239
{Line: 1, Text: `nested`},
240
{Line: 1, Text: `"quotes inside" backticks`},
@@ -244,7 +242,7 @@ func TestLexer(t *testing.T) {
242
},
243
},
244
{
247
- input: "reverse-nested \"`backticks` inside\" quotes",
245
+ input: []byte("reverse-nested \"`backticks` inside\" quotes"),
246
expected: []Token{
247
{Line: 1, Text: `reverse-nested`},
248
{Line: 1, Text: "`backticks` inside"},
@@ -254,22 +252,14 @@ func TestLexer(t *testing.T) {
252
}
253
254
for i, testCase := range testCases {
257
- actual := tokenize(testCase.input)
255
+ actual, err := Tokenize(testCase.input, "")
256
+ if err != nil {
257
+ t.Errorf("%v", err)
258
+ }
259
lexerCompare(t, i, testCase.expected, actual)
260
}
261
}
262
262
-func tokenize(input string) (tokens []Token) {
263
- l := lexer{}
264
- if err := l.load(strings.NewReader(input)); err != nil {
265
- log.Printf("[ERROR] load failed: %v", err)
266
- }
267
- for l.next() {
268
- tokens = append(tokens, l.token)
269
- }
270
- return
271
-}
272
-
263
func lexerCompare(t *testing.T, n int, expected, actual []Token) {
264
if len(expected) != len(actual) {
265
t.Errorf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual))
caddyconfig/caddyfile/parse.go
+1
-7
@@ -87,16 +87,10 @@ func allTokens(filename string, input []byte) ([]Token, error) {
87
if err != nil {
88
return nil, err
89
}
90
- l := new(lexer)
91
- err = l.load(bytes.NewReader(input))
90
+ tokens, err := Tokenize(input, filename)
91
if err != nil {
92
return nil, err
93
}
95
- var tokens []Token
96
- for l.next() {
97
- l.token.File = filename
98
- tokens = append(tokens, l.token)
99
- }
94
return tokens, nil
95
}
96