The best way to control Claude Code, Codex and more from any computer or phone
| 1 | /* |
| 2 | * Adapted from graphql-js v16.14.2 (https://github.com/graphql/graphql-js, |
| 3 | * `src/language/lexer.ts`), distributed under the MIT License: |
| 4 | * |
| 5 | * Copyright (c) GraphQL Contributors |
| 6 | * |
| 7 | * Permission is hereby granted, free of charge, to any person obtaining a copy |
| 8 | * of this software and associated documentation files (the "Software"), to deal |
| 9 | * in the Software without restriction, including without limitation the rights |
| 10 | * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell |
| 11 | * copies of the Software, and to permit persons to whom the Software is |
| 12 | * furnished to do so, subject to the following conditions: |
| 13 | * |
| 14 | * The above copyright notice and this permission notice shall be included in all |
| 15 | * copies or substantial portions of the Software. |
| 16 | * |
| 17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR |
| 18 | * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, |
| 19 | * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE |
| 20 | * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER |
| 21 | * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, |
| 22 | * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE |
| 23 | * SOFTWARE. |
| 24 | */ |
| 25 | /** |
| 26 | * GraphQL lexer for the current spec edition (September 2025). |
| 27 | * |
| 28 | * Produces one token at a time over a {@link Source}. Ignored tokens (BOM, |
| 29 | * whitespace, line terminators, commas and comments) are skipped. String and |
| 30 | * block string tokens carry their decoded value; numbers carry their source |
| 31 | * text. Every character, comments included, must be a Unicode scalar value; |
| 32 | * an unpaired surrogate is an error. Errors are thrown as {@link Diagnostic} |
| 33 | * and caught by the parser. |
| 34 | * |
| 35 | * The structure and messages follow graphql-js so disputes can be settled |
| 36 | * against the reference implementation. |
| 37 | * |
| 38 | * @internal |
| 39 | */ |
| 40 | import { make, type Source } from "./Diagnostic.ts" |
| 41 | |
| 42 | const punctuatorKinds = ["!", "$", "&", "(", ")", "...", ":", "=", "@", "[", "]", "{", "|", "}"] as const |
| 43 | |
| 44 | type PunctuatorKind = (typeof punctuatorKinds)[number] |
| 45 | |
| 46 | export type TokenKind = "<SOF>" | "<EOF>" | PunctuatorKind | "Name" | "Int" | "Float" | "String" | "BlockString" |
| 47 | |
| 48 | export interface Token { |
| 49 | readonly kind: TokenKind |
| 50 | /** Code-unit offset of the first character. */ |
| 51 | readonly start: number |
| 52 | /** Code-unit offset one past the last character. */ |
| 53 | readonly end: number |
| 54 | /** Decoded value for `Name`, `Int`, `Float`, `String` and `BlockString`. */ |
| 55 | readonly value: string | undefined |
| 56 | } |
| 57 | |
| 58 | const punctuators: ReadonlySet<string> = new Set(punctuatorKinds) |
| 59 | |
| 60 | export const isPunctuatorKind = (kind: TokenKind): kind is PunctuatorKind => punctuators.has(kind) |
| 61 | |
| 62 | export class Lexer { |
| 63 | readonly source: Source |
| 64 | /** The most recently read token. Starts as `<SOF>`. */ |
| 65 | token: Token |
| 66 | /** The token before `token`. */ |
| 67 | lastToken: Token |
| 68 | private next: Token | undefined = undefined |
| 69 | |
| 70 | constructor(source: Source) { |
| 71 | this.source = source |
| 72 | this.token = { kind: "<SOF>", start: 0, end: 0, value: undefined } |
| 73 | this.lastToken = this.token |
| 74 | } |
| 75 | |
| 76 | /** Advances to the next token and returns it. */ |
| 77 | advance(): Token { |
| 78 | this.lastToken = this.token |
| 79 | this.token = this.lookahead() |
| 80 | this.next = undefined |
| 81 | return this.token |
| 82 | } |
| 83 | |
| 84 | /** Returns the token after the current one without consuming it. */ |
| 85 | lookahead(): Token { |
| 86 | if (this.next === undefined) { |
| 87 | this.next = this.token.kind === "<EOF>" ? this.token : readToken(this.source, this.token.end) |
| 88 | } |
| 89 | return this.next |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | const token = (kind: TokenKind, start: number, end: number, value?: string): Token => ({ kind, start, end, value }) |
| 94 | |
| 95 | const isDigit = (code: number): boolean => code >= 0x30 && code <= 0x39 |
| 96 | |
| 97 | const isNameStart = (code: number): boolean => |
| 98 | (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || code === 0x5f |
| 99 | |
| 100 | const isNameContinue = (code: number): boolean => isNameStart(code) || isDigit(code) |
| 101 | |
| 102 | const isUnicodeScalarValue = (code: number): boolean => |
| 103 | (code >= 0x0000 && code <= 0xd7ff) || (code >= 0xe000 && code <= 0x10ffff) |
| 104 | |
| 105 | const isLeadingSurrogate = (code: number): boolean => code >= 0xd800 && code <= 0xdbff |
| 106 | |
| 107 | const isTrailingSurrogate = (code: number): boolean => code >= 0xdc00 && code <= 0xdfff |
| 108 | |
| 109 | /** |
| 110 | * The code-unit length of the `SourceCharacter` at `position`: 1, 2 for a |
| 111 | * surrogate pair, or 0 when it is an unpaired surrogate. |
| 112 | */ |
| 113 | const sourceCharacterSize = (body: string, position: number): number => |
| 114 | isUnicodeScalarValue(body.charCodeAt(position)) |
| 115 | ? 1 |
| 116 | : isLeadingSurrogate(body.charCodeAt(position)) && isTrailingSurrogate(body.charCodeAt(position + 1)) |
| 117 | ? 2 |
| 118 | : 0 |
| 119 | |
| 120 | /** Describes the code point at `location` for messages: `"?"`, `U+00E9`, or `<EOF>`. */ |
| 121 | const printCodePointAt = (body: string, location: number): string => { |
| 122 | const code = body.codePointAt(location) |
| 123 | if (code === undefined) return "<EOF>" |
| 124 | if (code >= 0x0020 && code <= 0x007e) { |
| 125 | return code === 0x0022 ? "'\"'" : `"${String.fromCodePoint(code)}"` |
| 126 | } |
| 127 | return `U+${code.toString(16).toUpperCase().padStart(4, "0")}` |
| 128 | } |
| 129 | |
| 130 | const readToken = (source: Source, start: number): Token => { |
| 131 | const body = source.body |
| 132 | const bodyLength = body.length |
| 133 | let position = start |
| 134 | while (position < bodyLength) { |
| 135 | const code = body.charCodeAt(position) |
| 136 | switch (code) { |
| 137 | // Ignored: BOM, tab, space, comma, line feed |
| 138 | case 0xfeff: |
| 139 | case 0x0009: |
| 140 | case 0x0020: |
| 141 | case 0x002c: |
| 142 | case 0x000a: |
| 143 | position++ |
| 144 | continue |
| 145 | // Carriage return, optionally followed by a line feed |
| 146 | case 0x000d: |
| 147 | position += body.charCodeAt(position + 1) === 0x000a ? 2 : 1 |
| 148 | continue |
| 149 | // Comment |
| 150 | case 0x0023: |
| 151 | position = readComment(body, position) |
| 152 | continue |
| 153 | case 0x002e: |
| 154 | if (body.charCodeAt(position + 1) === 0x002e && body.charCodeAt(position + 2) === 0x002e) { |
| 155 | return token("...", position, position + 3) |
| 156 | } |
| 157 | break |
| 158 | case 0x0022: |
| 159 | if (body.charCodeAt(position + 1) === 0x0022 && body.charCodeAt(position + 2) === 0x0022) { |
| 160 | return readBlockString(source, position) |
| 161 | } |
| 162 | return readString(source, position) |
| 163 | } |
| 164 | const char = body[position]! |
| 165 | if (punctuators.has(char)) { |
| 166 | return token(char as PunctuatorKind, position, position + 1) |
| 167 | } |
| 168 | if (isDigit(code) || code === 0x002d) { |
| 169 | return readNumber(source, position, code) |
| 170 | } |
| 171 | if (isNameStart(code)) { |
| 172 | return readName(body, position) |
| 173 | } |
| 174 | throw make( |
| 175 | source, |
| 176 | position, |
| 177 | code === 0x0027 |
| 178 | ? "Unexpected single quote character ('), did you mean to use a double quote (\")?" |
| 179 | : sourceCharacterSize(body, position) > 0 |
| 180 | ? `Unexpected character: ${printCodePointAt(body, position)}.` |
| 181 | : `Invalid character: ${printCodePointAt(body, position)}.` |
| 182 | ) |
| 183 | } |
| 184 | return token("<EOF>", bodyLength, bodyLength) |
| 185 | } |
| 186 | |
| 187 | const readComment = (body: string, start: number): number => { |
| 188 | let position = start + 1 |
| 189 | while (position < body.length) { |
| 190 | const code = body.charCodeAt(position) |
| 191 | if (code === 0x000a || code === 0x000d) break |
| 192 | // Not a SourceCharacter; stop here so the main loop reports it. |
| 193 | const size = sourceCharacterSize(body, position) |
| 194 | if (size === 0) break |
| 195 | position += size |
| 196 | } |
| 197 | return position |
| 198 | } |
| 199 | |
| 200 | const readName = (body: string, start: number): Token => { |
| 201 | let position = start + 1 |
| 202 | while (position < body.length && isNameContinue(body.charCodeAt(position))) { |
| 203 | position++ |
| 204 | } |
| 205 | return token("Name", start, position, body.slice(start, position)) |
| 206 | } |
| 207 | |
| 208 | const readNumber = (source: Source, start: number, firstCode: number): Token => { |
| 209 | const body = source.body |
| 210 | let position = start |
| 211 | let code = firstCode |
| 212 | let isFloat = false |
| 213 | if (code === 0x002d) { |
| 214 | code = body.charCodeAt(++position) |
| 215 | } |
| 216 | if (code === 0x0030) { |
| 217 | code = body.charCodeAt(++position) |
| 218 | if (isDigit(code)) { |
| 219 | throw make(source, position, `Invalid number, unexpected digit after 0: ${printCodePointAt(body, position)}.`) |
| 220 | } |
| 221 | } else { |
| 222 | position = readDigits(source, position, code) |
| 223 | code = body.charCodeAt(position) |
| 224 | } |
| 225 | if (code === 0x002e) { |
| 226 | isFloat = true |
| 227 | code = body.charCodeAt(++position) |
| 228 | position = readDigits(source, position, code) |
| 229 | code = body.charCodeAt(position) |
| 230 | } |
| 231 | if (code === 0x0045 || code === 0x0065) { |
| 232 | isFloat = true |
| 233 | code = body.charCodeAt(++position) |
| 234 | if (code === 0x002b || code === 0x002d) { |
| 235 | code = body.charCodeAt(++position) |
| 236 | } |
| 237 | position = readDigits(source, position, code) |
| 238 | code = body.charCodeAt(position) |
| 239 | } |
| 240 | if (code === 0x002e || isNameStart(code)) { |
| 241 | throw make(source, position, `Invalid number, expected digit but got: ${printCodePointAt(body, position)}.`) |
| 242 | } |
| 243 | return token(isFloat ? "Float" : "Int", start, position, body.slice(start, position)) |
| 244 | } |
| 245 | |
| 246 | const readDigits = (source: Source, start: number, firstCode: number): number => { |
| 247 | if (!isDigit(firstCode)) { |
| 248 | throw make(source, start, `Invalid number, expected digit but got: ${printCodePointAt(source.body, start)}.`) |
| 249 | } |
| 250 | const body = source.body |
| 251 | let position = start + 1 |
| 252 | while (isDigit(body.charCodeAt(position))) { |
| 253 | position++ |
| 254 | } |
| 255 | return position |
| 256 | } |
| 257 | |
| 258 | const readString = (source: Source, start: number): Token => { |
| 259 | const body = source.body |
| 260 | const bodyLength = body.length |
| 261 | let position = start + 1 |
| 262 | let chunkStart = position |
| 263 | let value = "" |
| 264 | while (position < bodyLength) { |
| 265 | const code = body.charCodeAt(position) |
| 266 | if (code === 0x0022) { |
| 267 | value += body.slice(chunkStart, position) |
| 268 | return token("String", start, position + 1, value) |
| 269 | } |
| 270 | if (code === 0x005c) { |
| 271 | value += body.slice(chunkStart, position) |
| 272 | const escape = body.charCodeAt(position + 1) === 0x0075 |
| 273 | ? body.charCodeAt(position + 2) === 0x007b |
| 274 | ? readEscapedUnicodeVariableWidth(source, position) |
| 275 | : readEscapedUnicodeFixedWidth(source, position) |
| 276 | : readEscapedCharacter(source, position) |
| 277 | value += escape.value |
| 278 | position += escape.size |
| 279 | chunkStart = position |
| 280 | continue |
| 281 | } |
| 282 | if (code === 0x000a || code === 0x000d) break |
| 283 | const size = sourceCharacterSize(body, position) |
| 284 | if (size === 0) { |
| 285 | throw make(source, position, `Invalid character within String: ${printCodePointAt(body, position)}.`) |
| 286 | } |
| 287 | position += size |
| 288 | } |
| 289 | throw make(source, position, "Unterminated string.") |
| 290 | } |
| 291 | |
| 292 | interface EscapeSequence { |
| 293 | readonly value: string |
| 294 | readonly size: number |
| 295 | } |
| 296 | |
| 297 | const readEscapedUnicodeVariableWidth = (source: Source, position: number): EscapeSequence => { |
| 298 | const body = source.body |
| 299 | let point = 0 |
| 300 | let size = 3 |
| 301 | // Cannot be larger than 12 chars (\u{00000000}). |
| 302 | while (size < 12) { |
| 303 | const code = body.charCodeAt(position + size++) |
| 304 | if (code === 0x007d) { |
| 305 | // Must be at least 5 chars (\u{0}) and encode a Unicode scalar value. |
| 306 | if (size < 5 || !isUnicodeScalarValue(point)) break |
| 307 | return { value: String.fromCodePoint(point), size } |
| 308 | } |
| 309 | point = (point << 4) | readHexDigit(code) |
| 310 | if (point < 0) break |
| 311 | } |
| 312 | throw make(source, position, `Invalid Unicode escape sequence: "${body.slice(position, position + size)}".`) |
| 313 | } |
| 314 | |
| 315 | const readEscapedUnicodeFixedWidth = (source: Source, position: number): EscapeSequence => { |
| 316 | const body = source.body |
| 317 | const code = read16BitHexCode(body, position + 2) |
| 318 | if (isUnicodeScalarValue(code)) { |
| 319 | return { value: String.fromCodePoint(code), size: 6 } |
| 320 | } |
| 321 | // JSON-style surrogate pair escapes are allowed only when they form a pair. |
| 322 | if (isLeadingSurrogate(code)) { |
| 323 | if (body.charCodeAt(position + 6) === 0x005c && body.charCodeAt(position + 7) === 0x0075) { |
| 324 | const trailingCode = read16BitHexCode(body, position + 8) |
| 325 | if (isTrailingSurrogate(trailingCode)) { |
| 326 | return { value: String.fromCodePoint(code, trailingCode), size: 12 } |
| 327 | } |
| 328 | } |
| 329 | } |
| 330 | throw make(source, position, `Invalid Unicode escape sequence: "${body.slice(position, position + 6)}".`) |
| 331 | } |
| 332 | |
| 333 | const read16BitHexCode = (body: string, position: number): number => |
| 334 | (readHexDigit(body.charCodeAt(position)) << 12) | |
| 335 | (readHexDigit(body.charCodeAt(position + 1)) << 8) | |
| 336 | (readHexDigit(body.charCodeAt(position + 2)) << 4) | |
| 337 | readHexDigit(body.charCodeAt(position + 3)) |
| 338 | |
| 339 | /** The value of a hex digit, or -1 for anything else. */ |
| 340 | const readHexDigit = (code: number): number => |
| 341 | code >= 0x0030 && code <= 0x0039 |
| 342 | ? code - 0x0030 |
| 343 | : code >= 0x0041 && code <= 0x0046 |
| 344 | ? code - 0x0037 |
| 345 | : code >= 0x0061 && code <= 0x0066 |
| 346 | ? code - 0x0057 |
| 347 | : -1 |
| 348 | |
| 349 | const escapedCharacters: ReadonlyMap<string | undefined, string> = new Map([ |
| 350 | ["\"", "\""], |
| 351 | ["\\", "\\"], |
| 352 | ["/", "/"], |
| 353 | ["b", "\b"], |
| 354 | ["f", "\f"], |
| 355 | ["n", "\n"], |
| 356 | ["r", "\r"], |
| 357 | ["t", "\t"] |
| 358 | ]) |
| 359 | |
| 360 | const readEscapedCharacter = (source: Source, position: number): EscapeSequence => { |
| 361 | const body = source.body |
| 362 | const value = escapedCharacters.get(body[position + 1]) |
| 363 | if (value !== undefined) return { value, size: 2 } |
| 364 | throw make(source, position, `Invalid character escape sequence: "${body.slice(position, position + 2)}".`) |
| 365 | } |
| 366 | |
| 367 | const readBlockString = (source: Source, start: number): Token => { |
| 368 | const body = source.body |
| 369 | const bodyLength = body.length |
| 370 | let position = start + 3 |
| 371 | let chunkStart = position |
| 372 | let currentLine = "" |
| 373 | const blockLines: Array<string> = [] |
| 374 | while (position < bodyLength) { |
| 375 | const code = body.charCodeAt(position) |
| 376 | if (code === 0x0022 && body.charCodeAt(position + 1) === 0x0022 && body.charCodeAt(position + 2) === 0x0022) { |
| 377 | currentLine += body.slice(chunkStart, position) |
| 378 | blockLines.push(currentLine) |
| 379 | return token("BlockString", start, position + 3, dedentBlockStringLines(blockLines).join("\n")) |
| 380 | } |
| 381 | if ( |
| 382 | code === 0x005c && |
| 383 | body.charCodeAt(position + 1) === 0x0022 && |
| 384 | body.charCodeAt(position + 2) === 0x0022 && |
| 385 | body.charCodeAt(position + 3) === 0x0022 |
| 386 | ) { |
| 387 | currentLine += body.slice(chunkStart, position) |
| 388 | chunkStart = position + 1 |
| 389 | position += 4 |
| 390 | continue |
| 391 | } |
| 392 | if (code === 0x000a || code === 0x000d) { |
| 393 | currentLine += body.slice(chunkStart, position) |
| 394 | blockLines.push(currentLine) |
| 395 | position += code === 0x000d && body.charCodeAt(position + 1) === 0x000a ? 2 : 1 |
| 396 | currentLine = "" |
| 397 | chunkStart = position |
| 398 | continue |
| 399 | } |
| 400 | const size = sourceCharacterSize(body, position) |
| 401 | if (size === 0) { |
| 402 | throw make(source, position, `Invalid character within String: ${printCodePointAt(body, position)}.`) |
| 403 | } |
| 404 | position += size |
| 405 | } |
| 406 | throw make(source, position, "Unterminated string.") |
| 407 | } |
| 408 | |
| 409 | /** The spec's `BlockStringValue` algorithm over already split lines. */ |
| 410 | const dedentBlockStringLines = (lines: ReadonlyArray<string>): ReadonlyArray<string> => { |
| 411 | let commonIndent = Number.MAX_SAFE_INTEGER |
| 412 | let firstNonEmptyLine: number | undefined = undefined |
| 413 | let lastNonEmptyLine = -1 |
| 414 | for (let i = 0; i < lines.length; i++) { |
| 415 | const line = lines[i]! |
| 416 | const indent = leadingWhitespace(line) |
| 417 | if (indent === line.length) continue |
| 418 | firstNonEmptyLine ??= i |
| 419 | lastNonEmptyLine = i |
| 420 | if (i !== 0 && indent < commonIndent) { |
| 421 | commonIndent = indent |
| 422 | } |
| 423 | } |
| 424 | return lines |
| 425 | .map((line, i) => (i === 0 ? line : line.slice(commonIndent))) |
| 426 | .slice(firstNonEmptyLine ?? 0, lastNonEmptyLine + 1) |
| 427 | } |
| 428 | |
| 429 | const leadingWhitespace = (line: string): number => { |
| 430 | let i = 0 |
| 431 | while (i < line.length && (line[i] === " " || line[i] === "\t")) { |
| 432 | i++ |
| 433 | } |
| 434 | return i |
| 435 | } |