Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 01:00 UTC
| 1 | // Copyright 2015 Matthew Holt and The Caddy Authors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package caddyfile |
| 16 | |
| 17 | import ( |
| 18 | "reflect" |
| 19 | "strings" |
| 20 | "testing" |
| 21 | ) |
| 22 | |
| 23 | func TestLexer(t *testing.T) { |
| 24 | testCases := []struct { |
| 25 | input []byte |
| 26 | expected []Token |
| 27 | expectErr bool |
| 28 | errorMessage string |
| 29 | }{ |
| 30 | { |
| 31 | input: []byte(`host:123`), |
| 32 | expected: []Token{ |
| 33 | {Line: 1, Text: "host:123"}, |
| 34 | }, |
| 35 | }, |
| 36 | { |
| 37 | input: []byte(`host:123 |
| 38 | |
| 39 | directive`), |
| 40 | expected: []Token{ |
| 41 | {Line: 1, Text: "host:123"}, |
| 42 | {Line: 3, Text: "directive"}, |
| 43 | }, |
| 44 | }, |
| 45 | { |
| 46 | input: []byte(`host:123 { |
| 47 | directive |
| 48 | }`), |
| 49 | expected: []Token{ |
| 50 | {Line: 1, Text: "host:123"}, |
| 51 | {Line: 1, Text: "{"}, |
| 52 | {Line: 2, Text: "directive"}, |
| 53 | {Line: 3, Text: "}"}, |
| 54 | }, |
| 55 | }, |
| 56 | { |
| 57 | input: []byte(`host:123 { directive }`), |
| 58 | expected: []Token{ |
| 59 | {Line: 1, Text: "host:123"}, |
| 60 | {Line: 1, Text: "{"}, |
| 61 | {Line: 1, Text: "directive"}, |
| 62 | {Line: 1, Text: "}"}, |
| 63 | }, |
| 64 | }, |
| 65 | { |
| 66 | input: []byte(`host:123 { |
| 67 | #comment |
| 68 | directive |
| 69 | # comment |
| 70 | foobar # another comment |
| 71 | }`), |
| 72 | expected: []Token{ |
| 73 | {Line: 1, Text: "host:123"}, |
| 74 | {Line: 1, Text: "{"}, |
| 75 | {Line: 3, Text: "directive"}, |
| 76 | {Line: 5, Text: "foobar"}, |
| 77 | {Line: 6, Text: "}"}, |
| 78 | }, |
| 79 | }, |
| 80 | { |
| 81 | input: []byte(`host:123 { |
| 82 | # hash inside string is not a comment |
| 83 | redir / /some/#/path |
| 84 | }`), |
| 85 | expected: []Token{ |
| 86 | {Line: 1, Text: "host:123"}, |
| 87 | {Line: 1, Text: "{"}, |
| 88 | {Line: 3, Text: "redir"}, |
| 89 | {Line: 3, Text: "/"}, |
| 90 | {Line: 3, Text: "/some/#/path"}, |
| 91 | {Line: 4, Text: "}"}, |
| 92 | }, |
| 93 | }, |
| 94 | { |
| 95 | input: []byte("# comment at beginning of file\n# comment at beginning of line\nhost:123"), |
| 96 | expected: []Token{ |
| 97 | {Line: 3, Text: "host:123"}, |
| 98 | }, |
| 99 | }, |
| 100 | { |
| 101 | input: []byte(`a "quoted value" b |
| 102 | foobar`), |
| 103 | expected: []Token{ |
| 104 | {Line: 1, Text: "a"}, |
| 105 | {Line: 1, Text: "quoted value"}, |
| 106 | {Line: 1, Text: "b"}, |
| 107 | {Line: 2, Text: "foobar"}, |
| 108 | }, |
| 109 | }, |
| 110 | { |
| 111 | input: []byte(`A "quoted \"value\" inside" B`), |
| 112 | expected: []Token{ |
| 113 | {Line: 1, Text: "A"}, |
| 114 | {Line: 1, Text: `quoted "value" inside`}, |
| 115 | {Line: 1, Text: "B"}, |
| 116 | }, |
| 117 | }, |
| 118 | { |
| 119 | input: []byte("An escaped \"newline\\\ninside\" quotes"), |
| 120 | expected: []Token{ |
| 121 | {Line: 1, Text: "An"}, |
| 122 | {Line: 1, Text: "escaped"}, |
| 123 | {Line: 1, Text: "newline\\\ninside"}, |
| 124 | {Line: 2, Text: "quotes"}, |
| 125 | }, |
| 126 | }, |
| 127 | { |
| 128 | input: []byte("An escaped newline \\\noutside quotes"), |
| 129 | expected: []Token{ |
| 130 | {Line: 1, Text: "An"}, |
| 131 | {Line: 1, Text: "escaped"}, |
| 132 | {Line: 1, Text: "newline"}, |
| 133 | {Line: 1, Text: "outside"}, |
| 134 | {Line: 1, Text: "quotes"}, |
| 135 | }, |
| 136 | }, |
| 137 | { |
| 138 | input: []byte("line1 \\\nescaped\nline2\nline3"), |
| 139 | expected: []Token{ |
| 140 | {Line: 1, Text: "line1"}, |
| 141 | {Line: 1, Text: "escaped"}, |
| 142 | {Line: 3, Text: "line2"}, |
| 143 | {Line: 4, Text: "line3"}, |
| 144 | }, |
| 145 | }, |
| 146 | { |
| 147 | input: []byte("line1 \\\nescaped1 \\\nescaped2\nline4\nline5"), |
| 148 | expected: []Token{ |
| 149 | {Line: 1, Text: "line1"}, |
| 150 | {Line: 1, Text: "escaped1"}, |
| 151 | {Line: 1, Text: "escaped2"}, |
| 152 | {Line: 4, Text: "line4"}, |
| 153 | {Line: 5, Text: "line5"}, |
| 154 | }, |
| 155 | }, |
| 156 | { |
| 157 | input: []byte(`"unescapable\ in quotes"`), |
| 158 | expected: []Token{ |
| 159 | {Line: 1, Text: `unescapable\ in quotes`}, |
| 160 | }, |
| 161 | }, |
| 162 | { |
| 163 | input: []byte(`"don't\escape"`), |
| 164 | expected: []Token{ |
| 165 | {Line: 1, Text: `don't\escape`}, |
| 166 | }, |
| 167 | }, |
| 168 | { |
| 169 | input: []byte(`"don't\\escape"`), |
| 170 | expected: []Token{ |
| 171 | {Line: 1, Text: `don't\\escape`}, |
| 172 | }, |
| 173 | }, |
| 174 | { |
| 175 | input: []byte(`un\escapable`), |
| 176 | expected: []Token{ |
| 177 | {Line: 1, Text: `un\escapable`}, |
| 178 | }, |
| 179 | }, |
| 180 | { |
| 181 | input: []byte(`A "quoted value with line |
| 182 | break inside" { |
| 183 | foobar |
| 184 | }`), |
| 185 | expected: []Token{ |
| 186 | {Line: 1, Text: "A"}, |
| 187 | {Line: 1, Text: "quoted value with line\n\t\t\t\t\tbreak inside"}, |
| 188 | {Line: 2, Text: "{"}, |
| 189 | {Line: 3, Text: "foobar"}, |
| 190 | {Line: 4, Text: "}"}, |
| 191 | }, |
| 192 | }, |
| 193 | { |
| 194 | input: []byte(`"C:\php\php-cgi.exe"`), |
| 195 | expected: []Token{ |
| 196 | {Line: 1, Text: `C:\php\php-cgi.exe`}, |
| 197 | }, |
| 198 | }, |
| 199 | { |
| 200 | input: []byte(`empty "" string`), |
| 201 | expected: []Token{ |
| 202 | {Line: 1, Text: `empty`}, |
| 203 | {Line: 1, Text: ``}, |
| 204 | {Line: 1, Text: `string`}, |
| 205 | }, |
| 206 | }, |
| 207 | { |
| 208 | input: []byte("skip those\r\nCR characters"), |
| 209 | expected: []Token{ |
| 210 | {Line: 1, Text: "skip"}, |
| 211 | {Line: 1, Text: "those"}, |
| 212 | {Line: 2, Text: "CR"}, |
| 213 | {Line: 2, Text: "characters"}, |
| 214 | }, |
| 215 | }, |
| 216 | { |
| 217 | input: []byte("\xEF\xBB\xBF:8080"), // test with leading byte order mark |
| 218 | expected: []Token{ |
| 219 | {Line: 1, Text: ":8080"}, |
| 220 | }, |
| 221 | }, |
| 222 | { |
| 223 | input: []byte("simple `backtick quoted` string"), |
| 224 | expected: []Token{ |
| 225 | {Line: 1, Text: `simple`}, |
| 226 | {Line: 1, Text: `backtick quoted`}, |
| 227 | {Line: 1, Text: `string`}, |
| 228 | }, |
| 229 | }, |
| 230 | { |
| 231 | input: []byte("multiline `backtick\nquoted\n` string"), |
| 232 | expected: []Token{ |
| 233 | {Line: 1, Text: `multiline`}, |
| 234 | {Line: 1, Text: "backtick\nquoted\n"}, |
| 235 | {Line: 3, Text: `string`}, |
| 236 | }, |
| 237 | }, |
| 238 | { |
| 239 | input: []byte("nested `\"quotes inside\" backticks` string"), |
| 240 | expected: []Token{ |
| 241 | {Line: 1, Text: `nested`}, |
| 242 | {Line: 1, Text: `"quotes inside" backticks`}, |
| 243 | {Line: 1, Text: `string`}, |
| 244 | }, |
| 245 | }, |
| 246 | { |
| 247 | input: []byte("reverse-nested \"`backticks` inside\" quotes"), |
| 248 | expected: []Token{ |
| 249 | {Line: 1, Text: `reverse-nested`}, |
| 250 | {Line: 1, Text: "`backticks` inside"}, |
| 251 | {Line: 1, Text: `quotes`}, |
| 252 | }, |
| 253 | }, |
| 254 | { |
| 255 | input: []byte(`heredoc <<EOF |
| 256 | content |
| 257 | EOF same-line-arg |
| 258 | `), |
| 259 | expected: []Token{ |
| 260 | {Line: 1, Text: `heredoc`}, |
| 261 | {Line: 1, Text: "content"}, |
| 262 | {Line: 3, Text: `same-line-arg`}, |
| 263 | }, |
| 264 | }, |
| 265 | { |
| 266 | input: []byte(`heredoc <<VERY-LONG-MARKER |
| 267 | content |
| 268 | VERY-LONG-MARKER same-line-arg |
| 269 | `), |
| 270 | expected: []Token{ |
| 271 | {Line: 1, Text: `heredoc`}, |
| 272 | {Line: 1, Text: "content"}, |
| 273 | {Line: 3, Text: `same-line-arg`}, |
| 274 | }, |
| 275 | }, |
| 276 | { |
| 277 | input: []byte(`heredoc <<EOF |
| 278 | extra-newline |
| 279 | |
| 280 | EOF same-line-arg |
| 281 | `), |
| 282 | expected: []Token{ |
| 283 | {Line: 1, Text: `heredoc`}, |
| 284 | {Line: 1, Text: "extra-newline\n"}, |
| 285 | {Line: 4, Text: `same-line-arg`}, |
| 286 | }, |
| 287 | }, |
| 288 | { |
| 289 | input: []byte(`heredoc <<EOF |
| 290 | EOF |
| 291 | HERE same-line-arg |
| 292 | `), |
| 293 | expected: []Token{ |
| 294 | {Line: 1, Text: `heredoc`}, |
| 295 | {Line: 1, Text: ``}, |
| 296 | {Line: 3, Text: `HERE`}, |
| 297 | {Line: 3, Text: `same-line-arg`}, |
| 298 | }, |
| 299 | }, |
| 300 | { |
| 301 | input: []byte(`heredoc <<EOF |
| 302 | EOF same-line-arg |
| 303 | `), |
| 304 | expected: []Token{ |
| 305 | {Line: 1, Text: `heredoc`}, |
| 306 | {Line: 1, Text: ""}, |
| 307 | {Line: 2, Text: `same-line-arg`}, |
| 308 | }, |
| 309 | }, |
| 310 | { |
| 311 | input: []byte(`heredoc <<EOF |
| 312 | content |
| 313 | EOF same-line-arg |
| 314 | `), |
| 315 | expected: []Token{ |
| 316 | {Line: 1, Text: `heredoc`}, |
| 317 | {Line: 1, Text: "content"}, |
| 318 | {Line: 3, Text: `same-line-arg`}, |
| 319 | }, |
| 320 | }, |
| 321 | { |
| 322 | input: []byte(`prev-line |
| 323 | heredoc <<EOF |
| 324 | multi |
| 325 | line |
| 326 | content |
| 327 | EOF same-line-arg |
| 328 | next-line |
| 329 | `), |
| 330 | expected: []Token{ |
| 331 | {Line: 1, Text: `prev-line`}, |
| 332 | {Line: 2, Text: `heredoc`}, |
| 333 | {Line: 2, Text: "\tmulti\n\tline\n\tcontent"}, |
| 334 | {Line: 6, Text: `same-line-arg`}, |
| 335 | {Line: 7, Text: `next-line`}, |
| 336 | }, |
| 337 | }, |
| 338 | { |
| 339 | input: []byte(`escaped-heredoc \<< >>`), |
| 340 | expected: []Token{ |
| 341 | {Line: 1, Text: `escaped-heredoc`}, |
| 342 | {Line: 1, Text: `<<`}, |
| 343 | {Line: 1, Text: `>>`}, |
| 344 | }, |
| 345 | }, |
| 346 | { |
| 347 | input: []byte(`not-a-heredoc <EOF |
| 348 | content |
| 349 | `), |
| 350 | expected: []Token{ |
| 351 | {Line: 1, Text: `not-a-heredoc`}, |
| 352 | {Line: 1, Text: `<EOF`}, |
| 353 | {Line: 2, Text: `content`}, |
| 354 | }, |
| 355 | }, |
| 356 | { |
| 357 | input: []byte(`not-a-heredoc <<<EOF content`), |
| 358 | expected: []Token{ |
| 359 | {Line: 1, Text: `not-a-heredoc`}, |
| 360 | {Line: 1, Text: `<<<EOF`}, |
| 361 | {Line: 1, Text: `content`}, |
| 362 | }, |
| 363 | }, |
| 364 | { |
| 365 | input: []byte(`not-a-heredoc "<<" ">>"`), |
| 366 | expected: []Token{ |
| 367 | {Line: 1, Text: `not-a-heredoc`}, |
| 368 | {Line: 1, Text: `<<`}, |
| 369 | {Line: 1, Text: `>>`}, |
| 370 | }, |
| 371 | }, |
| 372 | { |
| 373 | input: []byte(`not-a-heredoc << >>`), |
| 374 | expected: []Token{ |
| 375 | {Line: 1, Text: `not-a-heredoc`}, |
| 376 | {Line: 1, Text: `<<`}, |
| 377 | {Line: 1, Text: `>>`}, |
| 378 | }, |
| 379 | }, |
| 380 | { |
| 381 | input: []byte(`not-a-heredoc <<HERE SAME LINE |
| 382 | content |
| 383 | HERE same-line-arg |
| 384 | `), |
| 385 | expected: []Token{ |
| 386 | {Line: 1, Text: `not-a-heredoc`}, |
| 387 | {Line: 1, Text: `<<HERE`}, |
| 388 | {Line: 1, Text: `SAME`}, |
| 389 | {Line: 1, Text: `LINE`}, |
| 390 | {Line: 2, Text: `content`}, |
| 391 | {Line: 3, Text: `HERE`}, |
| 392 | {Line: 3, Text: `same-line-arg`}, |
| 393 | }, |
| 394 | }, |
| 395 | { |
| 396 | input: []byte(`heredoc <<s |
| 397 | � |
| 398 | s |
| 399 | `), |
| 400 | expected: []Token{ |
| 401 | {Line: 1, Text: `heredoc`}, |
| 402 | {Line: 1, Text: "�"}, |
| 403 | }, |
| 404 | }, |
| 405 | { |
| 406 | input: []byte("\u000Aheredoc \u003C\u003C\u0073\u0073\u000A\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F\u000A\u0073\u0073\u000A\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F\u000A\u00BF\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F"), |
| 407 | expected: []Token{ |
| 408 | { |
| 409 | Line: 2, |
| 410 | Text: "heredoc", |
| 411 | }, |
| 412 | { |
| 413 | Line: 2, |
| 414 | Text: "\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F", |
| 415 | }, |
| 416 | { |
| 417 | Line: 5, |
| 418 | Text: "\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F", |
| 419 | }, |
| 420 | { |
| 421 | Line: 6, |
| 422 | Text: "\u00BF\u00BF\u0057\u0001\u0000\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u00FF\u003D\u001F", |
| 423 | }, |
| 424 | }, |
| 425 | }, |
| 426 | { |
| 427 | input: []byte("not-a-heredoc <<\n"), |
| 428 | expectErr: true, |
| 429 | errorMessage: "missing opening heredoc marker on line #1; must contain only alphanumeric characters, dashes and underscores; got empty string", |
| 430 | }, |
| 431 | { |
| 432 | input: []byte(`heredoc <<<EOF |
| 433 | content |
| 434 | EOF same-line-arg |
| 435 | `), |
| 436 | expectErr: true, |
| 437 | errorMessage: "too many '<' for heredoc on line #1; only use two, for example <<END", |
| 438 | }, |
| 439 | { |
| 440 | input: []byte(`heredoc <<EOF |
| 441 | content |
| 442 | `), |
| 443 | expectErr: true, |
| 444 | errorMessage: "incomplete heredoc <<EOF on line #3, expected ending marker EOF", |
| 445 | }, |
| 446 | { |
| 447 | input: []byte(`heredoc <<EOF |
| 448 | content |
| 449 | EOF |
| 450 | `), |
| 451 | expectErr: true, |
| 452 | errorMessage: "mismatched leading whitespace in heredoc <<EOF on line #2 [\tcontent], expected whitespace [\t\t] to match the closing marker", |
| 453 | }, |
| 454 | { |
| 455 | input: []byte(`heredoc <<EOF |
| 456 | content |
| 457 | EOF |
| 458 | `), |
| 459 | expectErr: true, |
| 460 | errorMessage: "mismatched leading whitespace in heredoc <<EOF on line #2 [ content], expected whitespace [\t\t] to match the closing marker", |
| 461 | }, |
| 462 | { |
| 463 | input: []byte(`heredoc <<EOF |
| 464 | The next line is a blank line |
| 465 | |
| 466 | The previous line is a blank line |
| 467 | EOF`), |
| 468 | expected: []Token{ |
| 469 | {Line: 1, Text: "heredoc"}, |
| 470 | {Line: 1, Text: "The next line is a blank line\n\nThe previous line is a blank line"}, |
| 471 | }, |
| 472 | }, |
| 473 | { |
| 474 | input: []byte(`heredoc <<EOF |
| 475 | One tab indented heredoc with blank next line |
| 476 | |
| 477 | One tab indented heredoc with blank previous line |
| 478 | EOF`), |
| 479 | expected: []Token{ |
| 480 | {Line: 1, Text: "heredoc"}, |
| 481 | {Line: 1, Text: "One tab indented heredoc with blank next line\n\nOne tab indented heredoc with blank previous line"}, |
| 482 | }, |
| 483 | }, |
| 484 | { |
| 485 | input: []byte(`heredoc <<EOF |
| 486 | The next line is a blank line with one tab |
| 487 | |
| 488 | The previous line is a blank line with one tab |
| 489 | EOF`), |
| 490 | expected: []Token{ |
| 491 | {Line: 1, Text: "heredoc"}, |
| 492 | {Line: 1, Text: "The next line is a blank line with one tab\n\t\nThe previous line is a blank line with one tab"}, |
| 493 | }, |
| 494 | }, |
| 495 | { |
| 496 | input: []byte(`heredoc <<EOF |
| 497 | The next line is a blank line with one tab less than the correct indentation |
| 498 | |
| 499 | The previous line is a blank line with one tab less than the correct indentation |
| 500 | EOF`), |
| 501 | expectErr: true, |
| 502 | errorMessage: "mismatched leading whitespace in heredoc <<EOF on line #3 [\t], expected whitespace [\t\t] to match the closing marker", |
| 503 | }, |
| 504 | } |
| 505 | |
| 506 | for i, testCase := range testCases { |
| 507 | actual, err := Tokenize(testCase.input, "") |
| 508 | if testCase.expectErr { |
| 509 | if err == nil { |
| 510 | t.Fatalf("expected error, got actual: %v", actual) |
| 511 | continue |
| 512 | } |
| 513 | if err.Error() != testCase.errorMessage { |
| 514 | t.Fatalf("expected error '%v', got: %v", testCase.errorMessage, err) |
| 515 | } |
| 516 | continue |
| 517 | } |
| 518 | |
| 519 | if err != nil { |
| 520 | t.Fatalf("%v", err) |
| 521 | } |
| 522 | lexerCompare(t, i, testCase.expected, actual) |
| 523 | } |
| 524 | } |
| 525 | |
| 526 | func TestTokenFormatModeAccessors(t *testing.T) { |
| 527 | tok := Token{Text: "hello", raw: `"hello"`, isComment: false} |
| 528 | if tok.Raw() != `"hello"` { |
| 529 | t.Errorf("Raw() = %q, want %q", tok.Raw(), `"hello"`) |
| 530 | } |
| 531 | // falls back to Text when raw not captured |
| 532 | if (Token{Text: "hi"}).Raw() != "hi" { |
| 533 | t.Errorf("Raw() fallback = %q, want %q", (Token{Text: "hi"}).Raw(), "hi") |
| 534 | } |
| 535 | c := Token{Text: "# note", raw: "# note", isComment: true} |
| 536 | if !c.IsComment() { |
| 537 | t.Error("IsComment() = false, want true") |
| 538 | } |
| 539 | // Clone copies the new fields |
| 540 | clone := c.Clone() |
| 541 | if clone.raw != c.raw || clone.isComment != c.isComment { |
| 542 | t.Errorf("Clone did not copy format-mode fields: %+v", clone) |
| 543 | } |
| 544 | } |
| 545 | |
| 546 | func TestLexRawCapture(t *testing.T) { |
| 547 | cases := []struct { |
| 548 | in string |
| 549 | text string // processed Text of the single token |
| 550 | wantRaw string // verbatim source of that token |
| 551 | }{ |
| 552 | {`hello`, `hello`, `hello`}, |
| 553 | {`"a \"b\" "`, `a "b" `, `"a \"b\" "`}, |
| 554 | {"`raw \"x\"`", `raw "x"`, "`raw \"x\"`"}, |
| 555 | {`\<<NOTHEREDOC`, `<<NOTHEREDOC`, `\<<NOTHEREDOC`}, |
| 556 | } |
| 557 | for _, c := range cases { |
| 558 | toks, err := Lex([]byte(c.in), "T", LexOptions{Raw: true}) |
| 559 | if err != nil { |
| 560 | t.Fatalf("%q: %v", c.in, err) |
| 561 | } |
| 562 | if len(toks) != 1 { |
| 563 | t.Fatalf("%q: got %d tokens, want 1", c.in, len(toks)) |
| 564 | } |
| 565 | if toks[0].Text != c.text { |
| 566 | t.Errorf("%q: Text = %q, want %q", c.in, toks[0].Text, c.text) |
| 567 | } |
| 568 | if toks[0].Raw() != c.wantRaw { |
| 569 | t.Errorf("%q: Raw() = %q, want %q", c.in, toks[0].Raw(), c.wantRaw) |
| 570 | } |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | func TestLexRawCaptureHeredoc(t *testing.T) { |
| 575 | in := "x <<END\n\thello\n\tEND" |
| 576 | toks, err := Lex([]byte(in), "T", LexOptions{Raw: true}) |
| 577 | if err != nil { |
| 578 | t.Fatal(err) |
| 579 | } |
| 580 | // token 0 is "x"; token 1 is the heredoc |
| 581 | if len(toks) != 2 { |
| 582 | t.Fatalf("got %d tokens, want 2", len(toks)) |
| 583 | } |
| 584 | // heredoc Text is the de-indented body; Raw is the full framing verbatim. |
| 585 | if toks[1].Raw() != "<<END\n\thello\n\tEND" { |
| 586 | t.Errorf("heredoc Raw() = %q, want %q", toks[1].Raw(), "<<END\n\thello\n\tEND") |
| 587 | } |
| 588 | } |
| 589 | |
| 590 | func lexerCompare(t *testing.T, n int, expected, actual []Token) { |
| 591 | if len(expected) != len(actual) { |
| 592 | t.Fatalf("Test case %d: expected %d token(s) but got %d", n, len(expected), len(actual)) |
| 593 | } |
| 594 | |
| 595 | for i := 0; i < len(actual) && i < len(expected); i++ { |
| 596 | if actual[i].Line != expected[i].Line { |
| 597 | t.Fatalf("Test case %d token %d ('%s'): expected line %d but was line %d", |
| 598 | n, i, expected[i].Text, expected[i].Line, actual[i].Line) |
| 599 | break |
| 600 | } |
| 601 | if actual[i].Text != expected[i].Text { |
| 602 | t.Fatalf("Test case %d token %d: expected text '%s' but was '%s'", |
| 603 | n, i, expected[i].Text, actual[i].Text) |
| 604 | break |
| 605 | } |
| 606 | } |
| 607 | } |
| 608 | |
| 609 | func TestLexLineContinuationRequiresSeparateBackslash(t *testing.T) { |
| 610 | for _, tc := range []struct { |
| 611 | name string |
| 612 | input string |
| 613 | want []Token |
| 614 | }{ |
| 615 | {"space", "foo \\\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 1, Text: "bar"}}}, |
| 616 | {"tab and CRLF", "foo\t\\\r\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 1, Text: "bar"}}}, |
| 617 | {"attached", "foo\\\nbar", []Token{{Line: 1, Text: `foo\`}, {Line: 2, Text: "bar"}}}, |
| 618 | {"after quoted token", "\"foo\"\\\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 1, Text: `\`}, {Line: 2, Text: "bar"}}}, |
| 619 | {"after backtick token", "`foo`\\\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 1, Text: `\`}, {Line: 2, Text: "bar"}}}, |
| 620 | {"after quoted token with space", "\"foo\" \\\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 1, Text: "bar"}}}, |
| 621 | {"start of file", "\\\nbar", []Token{{Line: 1, Text: `\`}, {Line: 2, Text: "bar"}}}, |
| 622 | {"indented continuation", "foo\n\t\\\nbar", []Token{{Line: 1, Text: "foo"}, {Line: 2, Text: "bar"}}}, |
| 623 | {"Windows CRLF", "root * C:\\site\\\r\nfile_server", []Token{{Line: 1, Text: "root"}, {Line: 1, Text: "*"}, {Line: 1, Text: `C:\site\`}, {Line: 2, Text: "file_server"}}}, |
| 624 | {"path before space", "C:\\site\\ next", []Token{{Line: 1, Text: `C:\site\`}, {Line: 1, Text: "next"}}}, |
| 625 | {"path at EOF", `C:\site\`, []Token{{Line: 1, Text: `C:\site\`}}}, |
| 626 | {"standalone backslash at EOF", `foo \`, []Token{{Line: 1, Text: "foo"}}}, |
| 627 | {"paired backslashes", "foo\\\\\nbar", []Token{{Line: 1, Text: `foo\\`}, {Line: 2, Text: "bar"}}}, |
| 628 | {"backtick path", "`C:\\site\\`\nbar", []Token{{Line: 1, Text: `C:\site\`}, {Line: 2, Text: "bar"}}}, |
| 629 | } { |
| 630 | t.Run(tc.name, func(t *testing.T) { |
| 631 | for _, opts := range []LexOptions{{}, {Raw: true}, {Comments: true}, {Raw: true, Comments: true}} { |
| 632 | tokens, err := Lex([]byte(tc.input), "", opts) |
| 633 | if err != nil { |
| 634 | t.Fatal(err) |
| 635 | } |
| 636 | if len(tokens) != len(tc.want) { |
| 637 | t.Fatalf("options %+v: got %d tokens, want %d", opts, len(tokens), len(tc.want)) |
| 638 | } |
| 639 | for i, want := range tc.want { |
| 640 | if tokens[i].Text != want.Text || tokens[i].Line != want.Line { |
| 641 | t.Errorf("options %+v, token %d: got %q on line %d, want %q on line %d", opts, i, tokens[i].Text, tokens[i].Line, want.Text, want.Line) |
| 642 | } |
| 643 | } |
| 644 | } |
| 645 | }) |
| 646 | } |
| 647 | } |
| 648 | |
| 649 | func TestLexComments(t *testing.T) { |
| 650 | toks, err := Lex([]byte("foo # hi there\nbar"), "T", LexOptions{Comments: true, Raw: true}) |
| 651 | if err != nil { |
| 652 | t.Fatal(err) |
| 653 | } |
| 654 | // foo | # hi there | bar |
| 655 | if len(toks) != 3 { |
| 656 | t.Fatalf("got %d tokens, want 3: %+v", len(toks), toks) |
| 657 | } |
| 658 | if !toks[1].IsComment() || toks[1].Text != "# hi there" { |
| 659 | t.Errorf("token 1 = %q (comment=%v), want comment %q", toks[1].Text, toks[1].IsComment(), "# hi there") |
| 660 | } |
| 661 | if toks[2].Text != "bar" { |
| 662 | t.Errorf("token 2 = %q, want bar", toks[2].Text) |
| 663 | } |
| 664 | } |
| 665 | |
| 666 | func TestLexHashMidTokenIsLiteral(t *testing.T) { |
| 667 | toks, err := Lex([]byte("e#f redir /a/#/b"), "T", LexOptions{Comments: true}) |
| 668 | if err != nil { |
| 669 | t.Fatal(err) |
| 670 | } |
| 671 | texts := []string{} |
| 672 | for _, tk := range toks { |
| 673 | texts = append(texts, tk.Text) |
| 674 | } |
| 675 | want := []string{"e#f", "redir", "/a/#/b"} |
| 676 | if !reflect.DeepEqual(texts, want) { |
| 677 | t.Errorf("texts = %v, want %v", texts, want) |
| 678 | } |
| 679 | for _, tk := range toks { |
| 680 | if tk.IsComment() { |
| 681 | t.Errorf("unexpected comment token %q", tk.Text) |
| 682 | } |
| 683 | } |
| 684 | } |
| 685 | |
| 686 | func TestLexCommentsDisabledDiscards(t *testing.T) { |
| 687 | toks, _ := Lex([]byte("foo # hi\nbar"), "T", LexOptions{}) |
| 688 | if len(toks) != 2 { |
| 689 | t.Fatalf("got %d tokens, want 2 (comment discarded)", len(toks)) |
| 690 | } |
| 691 | } |
| 692 | |
| 693 | func TestLexSeparatorKind(t *testing.T) { |
| 694 | // glued comment after a closing quote: "x"#c -> [ "x", "#c" ] glued |
| 695 | toks, _ := Lex([]byte(`"x"#c`), "T", LexOptions{Comments: true, Raw: true}) |
| 696 | if len(toks) != 2 || !toks[1].IsComment() { |
| 697 | t.Fatalf("got %+v, want string + comment", toks) |
| 698 | } |
| 699 | if toks[1].precededBySpace { |
| 700 | t.Error(`"x"#c: comment should be glued (precededBySpace=false)`) |
| 701 | } |
| 702 | |
| 703 | toks2, _ := Lex([]byte(`"x" #c`), "T", LexOptions{Comments: true, Raw: true}) |
| 704 | if !toks2[1].precededBySpace { |
| 705 | t.Error(`"x" #c: comment should be space-separated (precededBySpace=true)`) |
| 706 | } |
| 707 | |
| 708 | // line continuation: `foo bar \<nl>baz` -> baz has continuation=true |
| 709 | toks3, _ := Lex([]byte("foo bar \\\nbaz"), "T", LexOptions{Raw: true}) |
| 710 | last := toks3[len(toks3)-1] |
| 711 | if last.Text != "baz" || !last.continuation { |
| 712 | t.Errorf("continuation token = %q continuation=%v, want baz/true", last.Text, last.continuation) |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | func lexTexts(t *testing.T, in string, opts LexOptions) []string { |
| 717 | t.Helper() |
| 718 | toks, err := Lex([]byte(in), "T", opts) |
| 719 | if err != nil { |
| 720 | t.Fatal(err) |
| 721 | } |
| 722 | out := []string{} |
| 723 | for _, tk := range toks { |
| 724 | out = append(out, tk.Text) |
| 725 | } |
| 726 | return out |
| 727 | } |
| 728 | |
| 729 | func TestLexOptionsPreserveTokenBoundaries(t *testing.T) { |
| 730 | cases := []struct { |
| 731 | in string |
| 732 | want []string |
| 733 | }{ |
| 734 | {"example.com{", []string{"example.com{"}}, |
| 735 | {"{$A}{", []string{"{$A}{"}}, // placeholder tail: not split |
| 736 | {"foo{bar}", []string{"foo{bar}"}}, // placeholder: not split |
| 737 | {"route {}", []string{"route", "{}"}}, |
| 738 | {"route { }", []string{"route", "{", "}"}}, |
| 739 | {"a { b {} }", []string{"a", "{", "b", "{}", "}"}}, |
| 740 | {"site {\n\troot\n}", []string{"site", "{", "root", "}"}}, |
| 741 | } |
| 742 | for _, c := range cases { |
| 743 | for _, opts := range []LexOptions{{Raw: true}, {Comments: true}, {Raw: true, Comments: true}} { |
| 744 | got := lexTexts(t, c.in, opts) |
| 745 | if !reflect.DeepEqual(got, c.want) { |
| 746 | t.Errorf("%q with %+v: got %v, want %v", c.in, opts, got, c.want) |
| 747 | } |
| 748 | } |
| 749 | } |
| 750 | } |
| 751 | |
| 752 | func TestLexEqualsTokenizeWithZeroOptions(t *testing.T) { |
| 753 | in := []byte("site {\n\troot * /srv\n\tfile_server\n}\n") |
| 754 | a, err := Tokenize(in, "Caddyfile") |
| 755 | if err != nil { |
| 756 | t.Fatal(err) |
| 757 | } |
| 758 | b, err := Lex(in, "Caddyfile", LexOptions{}) |
| 759 | if err != nil { |
| 760 | t.Fatal(err) |
| 761 | } |
| 762 | if !reflect.DeepEqual(a, b) { |
| 763 | t.Errorf("Lex(zero opts) != Tokenize\n a=%+v\n b=%+v", a, b) |
| 764 | } |
| 765 | } |
| 766 | |
| 767 | // TestLexerLongHeredoc exercises long and overlapping marker prefixes. Checking |
| 768 | // the entire marker after every rune made these inputs take seconds to tokenize. |
| 769 | func TestLexerLongHeredoc(t *testing.T) { |
| 770 | for _, tc := range []struct { |
| 771 | name, marker, body string |
| 772 | incomplete bool |
| 773 | }{ |
| 774 | {name: "long marker", marker: strings.Repeat("a", 8000) + "b", body: strings.Repeat("a", 500000)}, |
| 775 | {name: "overlapping prefixes", marker: "ababaca", body: "ababababac"}, |
| 776 | {name: "unicode body", marker: "END", body: "你好\xffEN"}, |
| 777 | {name: "incomplete", marker: strings.Repeat("a", 8000) + "b", body: strings.Repeat("a", 500000), incomplete: true}, |
| 778 | } { |
| 779 | t.Run(tc.name, func(t *testing.T) { |
| 780 | input := "<<" + tc.marker + "\n" + tc.body |
| 781 | if !tc.incomplete { |
| 782 | input += "\n" + tc.marker |
| 783 | } |
| 784 | for _, opts := range []LexOptions{{}, {Comments: true, Raw: true}} { |
| 785 | tokens, err := Lex([]byte(input), "Caddyfile", opts) |
| 786 | if tc.incomplete { |
| 787 | if err == nil { |
| 788 | t.Fatal("expected incomplete heredoc error") |
| 789 | } |
| 790 | continue |
| 791 | } |
| 792 | if err != nil { |
| 793 | t.Fatal(err) |
| 794 | } |
| 795 | if len(tokens) != 1 || tokens[0].Text != string([]rune(tc.body)) { |
| 796 | t.Fatalf("unexpected heredoc contents: %v", tokens) |
| 797 | } |
| 798 | if opts.Raw && tokens[0].Raw() != input { |
| 799 | t.Fatal("heredoc raw source changed") |
| 800 | } |
| 801 | } |
| 802 | }) |
| 803 | } |
| 804 | } |
| 805 | |
| 806 | func BenchmarkLexerLongHeredoc(b *testing.B) { |
| 807 | marker := strings.Repeat("a", 8000) + "b" |
| 808 | input := []byte("<<" + marker + "\n" + strings.Repeat("a", 500000) + "\n" + marker) |
| 809 | b.SetBytes(int64(len(input))) |
| 810 | b.ReportAllocs() |
| 811 | for b.Loop() { |
| 812 | if _, err := Tokenize(input, "Caddyfile"); err != nil { |
| 813 | b.Fatal(err) |
| 814 | } |
| 815 | } |
| 816 | } |
| 817 | |
| 818 | // TestIsHeredocMarker covers the marker check shared by the lexer and the |
| 819 | // formatter, including a long marker like the one that made the formatter time |
| 820 | // out under libFuzzer when the check was a regexp. |
| 821 | func TestIsHeredocMarker(t *testing.T) { |
| 822 | long := strings.Repeat("a", 500000) |
| 823 | for i, tc := range []struct { |
| 824 | marker string |
| 825 | want bool |
| 826 | }{ |
| 827 | {marker: "END", want: true}, |
| 828 | {marker: "a-b_C9", want: true}, |
| 829 | {marker: "09AZaz", want: true}, |
| 830 | {marker: long, want: true}, |
| 831 | {marker: "", want: false}, |
| 832 | {marker: "EN D", want: false}, |
| 833 | {marker: "END\n", want: false}, |
| 834 | {marker: "<END", want: false}, |
| 835 | {marker: "/", want: false}, |
| 836 | {marker: ":", want: false}, |
| 837 | {marker: "@", want: false}, |
| 838 | {marker: "[", want: false}, |
| 839 | {marker: "`", want: false}, |
| 840 | {marker: "{", want: false}, |
| 841 | {marker: "ÉND", want: false}, |
| 842 | {marker: "END\xff", want: false}, |
| 843 | {marker: long + "<<<<", want: false}, |
| 844 | } { |
| 845 | if got := isHeredocMarker(tc.marker); got != tc.want { |
| 846 | t.Errorf("Test %d: isHeredocMarker(%.20q) = %v, want %v", i, tc.marker, got, tc.want) |
| 847 | } |
| 848 | } |
| 849 | } |