Fast and extensible multi-platform HTTP/1-2-3 web server with automatic HTTPS
Mirrored from https://github.com/Slazee/caddy.git
Synced Nov 11, 01:00 UTC
| 1 | // Copyright 2015 Matthew Holt and The Caddy Authors |
| 2 | // |
| 3 | // Licensed under the Apache License, Version 2.0 (the "License"); |
| 4 | // you may not use this file except in compliance with the License. |
| 5 | // You may obtain a copy of the License at |
| 6 | // |
| 7 | // http://www.apache.org/licenses/LICENSE-2.0 |
| 8 | // |
| 9 | // Unless required by applicable law or agreed to in writing, software |
| 10 | // distributed under the License is distributed on an "AS IS" BASIS, |
| 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. |
| 12 | // See the License for the specific language governing permissions and |
| 13 | // limitations under the License. |
| 14 | |
| 15 | package caddyfile |
| 16 | |
| 17 | import ( |
| 18 | "bufio" |
| 19 | "bytes" |
| 20 | "fmt" |
| 21 | "io" |
| 22 | "strings" |
| 23 | "unicode" |
| 24 | ) |
| 25 | |
| 26 | type ( |
| 27 | // lexer is a utility which can get values, token by |
| 28 | // token, from a Reader. A token is a word, and tokens |
| 29 | // are separated by whitespace. A word can be enclosed |
| 30 | // in quotes if it contains whitespace. |
| 31 | lexer struct { |
| 32 | reader *bufio.Reader |
| 33 | token Token |
| 34 | line int |
| 35 | skippedLines int |
| 36 | lastRune rune // preceding source character, for line continuations |
| 37 | |
| 38 | // format-mode configuration and state (Tasks 4-6) |
| 39 | opts LexOptions |
| 40 | src []byte // original input, for raw slicing |
| 41 | pos int // bytes consumed from src |
| 42 | lastSize int // byte size of the most recently read rune |
| 43 | tokenStart int // byte offset in src where the current token began |
| 44 | } |
| 45 | |
| 46 | // Token represents a single parsable unit. |
| 47 | Token struct { |
| 48 | File string |
| 49 | imports []string |
| 50 | Line int |
| 51 | Text string |
| 52 | wasQuoted rune // enclosing quote character, if any |
| 53 | heredocMarker string |
| 54 | snippetName string |
| 55 | // Immutable physical-file ancestry, populated only during import discovery. |
| 56 | importAncestry *importAncestor |
| 57 | |
| 58 | // format mode only (populated by Lex with LexOptions; zero on the parse path) |
| 59 | raw string // verbatim source bytes of the token |
| 60 | isComment bool // this token is a # comment spanning to end-of-line |
| 61 | precededBySpace bool // whitespace separated this token from the previous one on the same line |
| 62 | continuation bool // this token followed a '\'+newline line continuation |
| 63 | } |
| 64 | ) |
| 65 | |
| 66 | // LexOptions configures optional, non-default lexer behavior used for |
| 67 | // formatting. The zero value reproduces Tokenize exactly. |
| 68 | type LexOptions struct { |
| 69 | // Comments, when true, emits comment tokens (spanning '#' to end of line) |
| 70 | // instead of discarding comments. |
| 71 | Comments bool |
| 72 | // Raw, when true, records each token's verbatim source bytes in Token.raw. |
| 73 | Raw bool |
| 74 | } |
| 75 | |
| 76 | // Tokenize takes bytes as input and lexes it into a list of tokens that can be |
| 77 | // parsed as a Caddyfile. filename fills each token's File field. |
| 78 | func Tokenize(input []byte, filename string) ([]Token, error) { |
| 79 | return Lex(input, filename, LexOptions{}) |
| 80 | } |
| 81 | |
| 82 | // Lex tokenizes input like Tokenize, but with the given options. With the zero |
| 83 | // LexOptions it is identical to Tokenize. Format-mode options (Comments, Raw) |
| 84 | // populate additional Token state used by the formatter and are not needed for |
| 85 | // parsing. |
| 86 | func Lex(input []byte, filename string, opts LexOptions) ([]Token, error) { |
| 87 | l := lexer{opts: opts} |
| 88 | if opts.Raw { |
| 89 | l.src = input |
| 90 | } |
| 91 | if err := l.load(bytes.NewReader(input)); err != nil { |
| 92 | return nil, err |
| 93 | } |
| 94 | var tokens []Token |
| 95 | for { |
| 96 | found, err := l.next() |
| 97 | if err != nil { |
| 98 | return nil, err |
| 99 | } |
| 100 | if !found { |
| 101 | break |
| 102 | } |
| 103 | l.token.File = filename |
| 104 | tokens = append(tokens, l.token) |
| 105 | } |
| 106 | return tokens, nil |
| 107 | } |
| 108 | |
| 109 | // readRune reads the next rune and advances the byte position tracker. |
| 110 | func (l *lexer) readRune() (rune, error) { |
| 111 | ch, size, err := l.reader.ReadRune() |
| 112 | if err == nil { |
| 113 | l.lastSize = size |
| 114 | l.pos += size |
| 115 | } |
| 116 | return ch, err |
| 117 | } |
| 118 | |
| 119 | // unreadRune pushes back the last rune read by readRune, keeping pos in sync. |
| 120 | func (l *lexer) unreadRune() error { |
| 121 | if err := l.reader.UnreadRune(); err != nil { |
| 122 | return err |
| 123 | } |
| 124 | l.pos -= l.lastSize |
| 125 | l.lastSize = 0 |
| 126 | return nil |
| 127 | } |
| 128 | |
| 129 | // rawSlice returns the verbatim source of the current token. trimLast |
| 130 | // excludes the most recently read rune (used when a delimiter terminated |
| 131 | // the token and should not be part of its raw span). |
| 132 | func (l *lexer) rawSlice(trimLast bool) string { |
| 133 | end := l.pos |
| 134 | if trimLast { |
| 135 | end -= l.lastSize |
| 136 | } |
| 137 | return string(l.src[l.tokenStart:end]) |
| 138 | } |
| 139 | |
| 140 | // load prepares the lexer to scan an input for tokens. |
| 141 | // It discards any leading byte order mark. |
| 142 | func (l *lexer) load(input io.Reader) error { |
| 143 | l.reader = bufio.NewReader(input) |
| 144 | l.line = 1 |
| 145 | |
| 146 | // discard byte order mark, if present |
| 147 | firstCh, err := l.readRune() |
| 148 | if err != nil { |
| 149 | return err |
| 150 | } |
| 151 | if firstCh != 0xFEFF { |
| 152 | err := l.unreadRune() |
| 153 | if err != nil { |
| 154 | return err |
| 155 | } |
| 156 | } |
| 157 | |
| 158 | return nil |
| 159 | } |
| 160 | |
| 161 | // next loads the next token into the lexer. |
| 162 | // A token is delimited by whitespace, unless |
| 163 | // the token starts with a quotes character (") |
| 164 | // in which case the token goes until the closing |
| 165 | // quotes (the enclosing quotes are not included). |
| 166 | // Inside quoted strings, quotes may be escaped |
| 167 | // with a preceding \ character. No other chars |
| 168 | // may be escaped. The rest of the line is skipped |
| 169 | // if a "#" character is read in. Returns true if |
| 170 | // a token was loaded; false otherwise. |
| 171 | // Outside quotes, a backslash preceded by a space or tab can escape a newline |
| 172 | // to continue the logical line. A backslash attached to a token is literal. |
| 173 | func (l *lexer) next() (bool, error) { |
| 174 | var val []rune |
| 175 | var comment, quoted, btQuoted, inHeredoc, heredocEscaped, escaped, continueLine bool |
| 176 | var heredocMarker string |
| 177 | var heredocPrefix []int |
| 178 | var heredocMatched int |
| 179 | var tokenStartSet bool // whether l.tokenStart has been set for the current token |
| 180 | |
| 181 | // format-mode separator tracking: record how this token was separated from |
| 182 | // the previous one. Only maintained when opts.Comments || opts.Raw. |
| 183 | formatMode := l.opts.Comments || l.opts.Raw |
| 184 | var sawSpace, sawContinuation bool |
| 185 | |
| 186 | makeToken := func(quoted rune, trimLast bool) bool { |
| 187 | l.token.Text = string(val) |
| 188 | l.token.wasQuoted = quoted |
| 189 | l.token.heredocMarker = heredocMarker |
| 190 | if l.opts.Raw { |
| 191 | l.token.raw = l.rawSlice(trimLast) |
| 192 | } |
| 193 | tokenStartSet = false |
| 194 | return true |
| 195 | } |
| 196 | |
| 197 | for { |
| 198 | // Read a character in; if err then if we had |
| 199 | // read some characters, make a token. If we |
| 200 | // reached EOF, then no more tokens to read. |
| 201 | // If no EOF, then we had a problem. |
| 202 | ch, err := l.readRune() |
| 203 | if err != nil { |
| 204 | if escaped && !quoted && !comment && (len(val) > 0 || !continueLine) { |
| 205 | val = append(val, '\\') |
| 206 | } |
| 207 | if len(val) > 0 { |
| 208 | if inHeredoc { |
| 209 | return false, fmt.Errorf("incomplete heredoc <<%s on line #%d, expected ending marker %s", heredocMarker, l.line+l.skippedLines, heredocMarker) |
| 210 | } |
| 211 | |
| 212 | return makeToken(0, false), nil |
| 213 | } |
| 214 | // An empty-bodied heredoc that hits EOF with no closing marker is |
| 215 | // silently dropped on the parse path (existing behavior). In format |
| 216 | // mode this loses the token and breaks idempotency, so surface it as |
| 217 | // an error there; Format then falls back to preserving the input. |
| 218 | if inHeredoc && formatMode { |
| 219 | return false, fmt.Errorf("incomplete heredoc <<%s on line #%d, expected ending marker %s", heredocMarker, l.line+l.skippedLines, heredocMarker) |
| 220 | } |
| 221 | if err == io.EOF { |
| 222 | return false, nil |
| 223 | } |
| 224 | return false, err |
| 225 | } |
| 226 | previous := l.lastRune |
| 227 | l.lastRune = ch |
| 228 | |
| 229 | // detect whether we have the start of a heredoc |
| 230 | if (!quoted && !btQuoted) && (!inHeredoc && !heredocEscaped) && |
| 231 | len(val) > 1 && string(val[:2]) == "<<" { |
| 232 | // a space means it's just a regular token and not a heredoc |
| 233 | if ch == ' ' { |
| 234 | return makeToken(0, true), nil |
| 235 | } |
| 236 | |
| 237 | // skip CR, we only care about LF |
| 238 | if ch == '\r' { |
| 239 | continue |
| 240 | } |
| 241 | |
| 242 | // after hitting a newline, we know that the heredoc marker |
| 243 | // is the characters after the two << and the newline. |
| 244 | // we reset the val because the heredoc is syntax we don't |
| 245 | // want to keep. |
| 246 | if ch == '\n' { |
| 247 | if len(val) == 2 { |
| 248 | return false, fmt.Errorf("missing opening heredoc marker on line #%d; must contain only alphanumeric characters, dashes and underscores; got empty string", l.line) |
| 249 | } |
| 250 | |
| 251 | // check if there's too many < |
| 252 | if string(val[:3]) == "<<<" { |
| 253 | return false, fmt.Errorf("too many '<' for heredoc on line #%d; only use two, for example <<END", l.line) |
| 254 | } |
| 255 | |
| 256 | heredocMarker = string(val[2:]) |
| 257 | if !isHeredocMarker(heredocMarker) { |
| 258 | return false, fmt.Errorf("heredoc marker on line #%d must contain only alphanumeric characters, dashes and underscores; got '%s'", l.line, heredocMarker) |
| 259 | } |
| 260 | |
| 261 | // Build a prefix table for incremental marker matching (KMP). |
| 262 | // Comparing the whole suffix after every rune costs O(body * marker) |
| 263 | // and repeatedly converts large rune slices to strings. |
| 264 | heredocPrefix = make([]int, len(heredocMarker)) |
| 265 | for i, matched := 1, 0; i < len(heredocMarker); i++ { |
| 266 | for matched > 0 && heredocMarker[i] != heredocMarker[matched] { |
| 267 | matched = heredocPrefix[matched-1] |
| 268 | } |
| 269 | if heredocMarker[i] == heredocMarker[matched] { |
| 270 | matched++ |
| 271 | } |
| 272 | heredocPrefix[i] = matched |
| 273 | } |
| 274 | |
| 275 | inHeredoc = true |
| 276 | l.skippedLines++ |
| 277 | val = nil |
| 278 | continue |
| 279 | } |
| 280 | val = append(val, ch) |
| 281 | continue |
| 282 | } |
| 283 | |
| 284 | // if we're in a heredoc, all characters are read as-is |
| 285 | if inHeredoc { |
| 286 | val = append(val, ch) |
| 287 | |
| 288 | if ch == '\n' { |
| 289 | l.skippedLines++ |
| 290 | } |
| 291 | |
| 292 | // Markers contain only ASCII, so compare each rune directly with |
| 293 | // the next marker byte. Prefix fallback also handles overlapping |
| 294 | // partial matches without rescanning the body. |
| 295 | for heredocMatched > 0 && ch != rune(heredocMarker[heredocMatched]) { |
| 296 | heredocMatched = heredocPrefix[heredocMatched-1] |
| 297 | } |
| 298 | if ch == rune(heredocMarker[heredocMatched]) { |
| 299 | heredocMatched++ |
| 300 | } |
| 301 | if heredocMatched == len(heredocMarker) { |
| 302 | // set the final value |
| 303 | val, err = l.finalizeHeredoc(val, heredocMarker) |
| 304 | if err != nil { |
| 305 | return false, err |
| 306 | } |
| 307 | |
| 308 | // set the line counter, and make the token |
| 309 | l.line += l.skippedLines |
| 310 | l.skippedLines = 0 |
| 311 | return makeToken('<', false), nil |
| 312 | } |
| 313 | |
| 314 | // stay in the heredoc until we find the ending marker |
| 315 | continue |
| 316 | } |
| 317 | |
| 318 | // track whether we found an escape '\' for the next |
| 319 | // iteration to be contextually aware |
| 320 | if !escaped && !btQuoted && ch == '\\' { |
| 321 | // a leading backslash starts a new token (e.g. \<<marker) |
| 322 | if len(val) == 0 { |
| 323 | l.token = Token{Line: l.line} |
| 324 | if formatMode { |
| 325 | l.token.precededBySpace = sawSpace |
| 326 | l.token.continuation = sawContinuation |
| 327 | } |
| 328 | } |
| 329 | if l.opts.Raw && len(val) == 0 && !tokenStartSet { |
| 330 | l.tokenStart = l.pos - l.lastSize |
| 331 | tokenStartSet = true |
| 332 | } |
| 333 | escaped = true |
| 334 | continueLine = previous == ' ' || previous == '\t' |
| 335 | continue |
| 336 | } |
| 337 | |
| 338 | if quoted || btQuoted { |
| 339 | if quoted && escaped { |
| 340 | // all is literal in quoted area, |
| 341 | // so only escape quotes |
| 342 | if ch != '"' { |
| 343 | val = append(val, '\\') |
| 344 | } |
| 345 | escaped = false |
| 346 | } else { |
| 347 | if (quoted && ch == '"') || (btQuoted && ch == '`') { |
| 348 | return makeToken(ch, false), nil |
| 349 | } |
| 350 | } |
| 351 | // allow quoted text to wrap continue on multiple lines |
| 352 | if ch == '\n' { |
| 353 | l.line += 1 + l.skippedLines |
| 354 | l.skippedLines = 0 |
| 355 | } |
| 356 | // collect this character as part of the quoted token |
| 357 | val = append(val, ch) |
| 358 | continue |
| 359 | } |
| 360 | |
| 361 | if unicode.IsSpace(ch) { |
| 362 | // ignore CR altogether, we only actually care about LF (\n) |
| 363 | if ch == '\r' { |
| 364 | continue |
| 365 | } |
| 366 | // Only a backslash preceded by a space or tab continues a line. |
| 367 | // A backslash attached to an unquoted token is literal, including |
| 368 | // the trailing separator in a Windows path. Preserve it at any |
| 369 | // token boundary. |
| 370 | if escaped && !continueLine && !comment { |
| 371 | val = append(val, '\\') |
| 372 | escaped = false |
| 373 | } |
| 374 | // end of the line |
| 375 | if ch == '\n' { |
| 376 | // when emitting comment tokens, the newline is handled in the |
| 377 | // comment block below (unread + return); don't process it here. |
| 378 | if comment && l.opts.Comments { |
| 379 | // fall through to the comment block below |
| 380 | } else { |
| 381 | // newlines can be escaped to chain arguments |
| 382 | // onto multiple lines; else, increment the line count |
| 383 | if escaped { |
| 384 | l.skippedLines++ |
| 385 | escaped = false |
| 386 | if formatMode { |
| 387 | sawContinuation = true |
| 388 | } |
| 389 | } else { |
| 390 | l.line += 1 + l.skippedLines |
| 391 | l.skippedLines = 0 |
| 392 | } |
| 393 | // comments (#) are single-line only |
| 394 | comment = false |
| 395 | // any kind of space means we're at the end of this token |
| 396 | if len(val) > 0 { |
| 397 | return makeToken(0, true), nil |
| 398 | } |
| 399 | continue |
| 400 | } |
| 401 | } else { |
| 402 | // non-newline space: end token if we have one (unless in comment mode) |
| 403 | if comment && l.opts.Comments { |
| 404 | // spaces inside a comment are part of the comment text; fall through |
| 405 | } else { |
| 406 | if len(val) > 0 { |
| 407 | return makeToken(0, true), nil |
| 408 | } |
| 409 | if formatMode { |
| 410 | sawSpace = true |
| 411 | } |
| 412 | continue |
| 413 | } |
| 414 | } |
| 415 | } |
| 416 | |
| 417 | // comments must be at the start of a token, |
| 418 | // in other words, preceded by space or newline |
| 419 | if ch == '#' && len(val) == 0 { |
| 420 | comment = true |
| 421 | if l.opts.Comments { |
| 422 | // begin a comment token; record its start for raw capture |
| 423 | l.tokenStart = l.pos - l.lastSize |
| 424 | tokenStartSet = true |
| 425 | l.token = Token{Line: l.line, isComment: true} |
| 426 | if formatMode { |
| 427 | l.token.precededBySpace = sawSpace |
| 428 | l.token.continuation = sawContinuation |
| 429 | sawSpace, sawContinuation = false, false |
| 430 | } |
| 431 | } |
| 432 | } |
| 433 | if comment { |
| 434 | if l.opts.Comments { |
| 435 | // a newline ends the comment; don't consume it (leave line accounting |
| 436 | // to the normal whitespace path on the next iteration) |
| 437 | if ch == '\n' { |
| 438 | if err := l.unreadRune(); err != nil { |
| 439 | return false, err |
| 440 | } |
| 441 | l.token.isComment = true |
| 442 | return makeToken(0, false), nil |
| 443 | } |
| 444 | val = append(val, ch) |
| 445 | } |
| 446 | continue |
| 447 | } |
| 448 | |
| 449 | if len(val) == 0 { |
| 450 | l.token = Token{Line: l.line} |
| 451 | if formatMode { |
| 452 | l.token.precededBySpace = sawSpace |
| 453 | l.token.continuation = sawContinuation |
| 454 | sawSpace, sawContinuation = false, false |
| 455 | } |
| 456 | if ch == '"' { |
| 457 | if l.opts.Raw && !tokenStartSet { |
| 458 | l.tokenStart = l.pos - l.lastSize |
| 459 | tokenStartSet = true |
| 460 | } |
| 461 | quoted = true |
| 462 | continue |
| 463 | } |
| 464 | if ch == '`' { |
| 465 | if l.opts.Raw && !tokenStartSet { |
| 466 | l.tokenStart = l.pos - l.lastSize |
| 467 | tokenStartSet = true |
| 468 | } |
| 469 | btQuoted = true |
| 470 | continue |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | if escaped { |
| 475 | // allow escaping the first < to skip the heredoc syntax |
| 476 | if ch == '<' { |
| 477 | heredocEscaped = true |
| 478 | } else { |
| 479 | val = append(val, '\\') |
| 480 | } |
| 481 | escaped = false |
| 482 | } |
| 483 | |
| 484 | // first character of a new unquoted token (tokenStart may already be |
| 485 | // set by a leading backslash; only set it if not yet recorded) |
| 486 | if l.opts.Raw && len(val) == 0 && !tokenStartSet { |
| 487 | l.tokenStart = l.pos - l.lastSize |
| 488 | tokenStartSet = true |
| 489 | } |
| 490 | |
| 491 | val = append(val, ch) |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // finalizeHeredoc takes the runes read as the heredoc text and the marker, |
| 496 | // and processes the text to strip leading whitespace, returning the final |
| 497 | // value without the leading whitespace. |
| 498 | func (l *lexer) finalizeHeredoc(val []rune, marker string) ([]rune, error) { |
| 499 | stringVal := string(val) |
| 500 | |
| 501 | // find the last newline of the heredoc, which is where the contents end |
| 502 | lastNewline := strings.LastIndex(stringVal, "\n") |
| 503 | |
| 504 | // collapse the content, then split into separate lines |
| 505 | lines := strings.Split(stringVal[:lastNewline+1], "\n") |
| 506 | |
| 507 | // figure out how much whitespace we need to strip from the front of every line |
| 508 | // by getting the string that precedes the marker, on the last line |
| 509 | paddingToStrip := stringVal[lastNewline+1 : len(stringVal)-len(marker)] |
| 510 | |
| 511 | // iterate over each line and strip the whitespace from the front |
| 512 | var out strings.Builder |
| 513 | if lastNewline > 0 { |
| 514 | out.Grow(lastNewline) |
| 515 | } |
| 516 | for lineNum, lineText := range lines[:len(lines)-1] { |
| 517 | if lineText == "" || lineText == "\r" { |
| 518 | out.WriteByte('\n') |
| 519 | continue |
| 520 | } |
| 521 | |
| 522 | // find an exact match for the padding |
| 523 | index := strings.Index(lineText, paddingToStrip) |
| 524 | |
| 525 | // if the padding doesn't match exactly at the start then we can't safely strip |
| 526 | if index != 0 { |
| 527 | cleanLineText := strings.TrimRight(lineText, "\r\n") |
| 528 | return nil, fmt.Errorf("mismatched leading whitespace in heredoc <<%s on line #%d [%s], expected whitespace [%s] to match the closing marker", marker, l.line+lineNum+1, cleanLineText, paddingToStrip) |
| 529 | } |
| 530 | |
| 531 | // strip, then append the line, with the newline, to the output. |
| 532 | // also removes all "\r" because Windows. |
| 533 | for _, ch := range lineText[len(paddingToStrip):] { |
| 534 | if ch != '\r' { |
| 535 | out.WriteRune(ch) |
| 536 | } |
| 537 | } |
| 538 | out.WriteByte('\n') |
| 539 | } |
| 540 | |
| 541 | // Remove the trailing newline from the loop |
| 542 | result := out.String() |
| 543 | if len(result) > 0 && result[len(result)-1] == '\n' { |
| 544 | result = result[:len(result)-1] |
| 545 | } |
| 546 | |
| 547 | // return the final value |
| 548 | return []rune(result), nil |
| 549 | } |
| 550 | |
| 551 | // Quoted returns true if the token was enclosed in quotes |
| 552 | // (i.e. double quotes, backticks, or heredoc). |
| 553 | func (t Token) Quoted() bool { |
| 554 | return t.wasQuoted > 0 |
| 555 | } |
| 556 | |
| 557 | // Raw returns the verbatim source text of the token as it appeared in the |
| 558 | // input, including quotes, backticks, escapes, and heredoc framing. It is only |
| 559 | // populated when the token was produced by Lex in format mode; otherwise it |
| 560 | // falls back to the processed Text. |
| 561 | func (t Token) Raw() string { |
| 562 | if t.raw != "" { |
| 563 | return t.raw |
| 564 | } |
| 565 | return t.Text |
| 566 | } |
| 567 | |
| 568 | // IsComment returns true if the token is a comment. Comment tokens are only |
| 569 | // produced by Lex in format mode. |
| 570 | func (t Token) IsComment() bool { |
| 571 | return t.isComment |
| 572 | } |
| 573 | |
| 574 | // isOpenCurlyBrace returns true if the token is a structural (unquoted) open curly brace. |
| 575 | func isOpenCurlyBrace(t Token) bool { |
| 576 | return t.Text == "{" && t.wasQuoted == 0 |
| 577 | } |
| 578 | |
| 579 | // isCloseCurlyBrace returns true if the token is a structural (unquoted) close curly brace. |
| 580 | func isCloseCurlyBrace(t Token) bool { |
| 581 | return t.Text == "}" && t.wasQuoted == 0 |
| 582 | } |
| 583 | |
| 584 | // NumLineBreaks counts how many line breaks are in the token text. |
| 585 | func (t Token) NumLineBreaks() int { |
| 586 | lineBreaks := strings.Count(t.Text, "\n") |
| 587 | if t.wasQuoted == '<' { |
| 588 | // heredocs have an extra linebreak because the opening |
| 589 | // delimiter is on its own line and is not included in the |
| 590 | // token Text itself, and the trailing newline is removed. |
| 591 | lineBreaks += 2 |
| 592 | } |
| 593 | return lineBreaks |
| 594 | } |
| 595 | |
| 596 | // Clone returns a deep copy of the token. |
| 597 | func (t Token) Clone() Token { |
| 598 | return Token{ |
| 599 | File: t.File, |
| 600 | imports: append([]string{}, t.imports...), |
| 601 | Line: t.Line, |
| 602 | Text: t.Text, |
| 603 | wasQuoted: t.wasQuoted, |
| 604 | heredocMarker: t.heredocMarker, |
| 605 | snippetName: t.snippetName, |
| 606 | importAncestry: t.importAncestry, |
| 607 | raw: t.raw, |
| 608 | isComment: t.isComment, |
| 609 | precededBySpace: t.precededBySpace, |
| 610 | continuation: t.continuation, |
| 611 | } |
| 612 | } |
| 613 | |
| 614 | // isHeredocMarker reports whether marker is a valid heredoc marker, made of |
| 615 | // one or more ASCII letters, digits, dashes and underscores. It scans bytes |
| 616 | // instead of matching a regexp because regexp matching of a long string is |
| 617 | // quadratic under libFuzzer instrumentation. |
| 618 | func isHeredocMarker(marker string) bool { |
| 619 | if len(marker) == 0 { |
| 620 | return false |
| 621 | } |
| 622 | for i := range len(marker) { |
| 623 | c := marker[i] |
| 624 | isAlnum := 'a' <= c && c <= 'z' || 'A' <= c && c <= 'Z' || '0' <= c && c <= '9' |
| 625 | if !isAlnum && c != '_' && c != '-' { |
| 626 | return false |
| 627 | } |
| 628 | } |
| 629 | return true |
| 630 | } |
| 631 | |
| 632 | // isNextOnNewLine tests whether t2 is on a different line from t1 |
| 633 | func isNextOnNewLine(t1, t2 Token) bool { |
| 634 | // If the second token is from a different file, |
| 635 | // we can assume it's from a different line |
| 636 | if t1.File != t2.File { |
| 637 | return true |
| 638 | } |
| 639 | |
| 640 | // If the second token is from a different import chain, |
| 641 | // we can assume it's from a different line |
| 642 | if len(t1.imports) != len(t2.imports) { |
| 643 | return true |
| 644 | } |
| 645 | for i, im := range t1.imports { |
| 646 | if im != t2.imports[i] { |
| 647 | return true |
| 648 | } |
| 649 | } |
| 650 | |
| 651 | // If the first token (incl line breaks) ends |
| 652 | // on a line earlier than the next token, |
| 653 | // then the second token is on a new line |
| 654 | return t1.Line+t1.NumLineBreaks() < t2.Line |
| 655 | } |