The best way to control Claude Code, Codex and more from any computer or phone

main
ts 435 lines 15.1 KB
Raw
1 /*
2 * Adapted from graphql-js v16.14.2 (https://github.com/graphql/graphql-js,
3 * `src/language/lexer.ts`), distributed under the MIT License:
4 *
5 * Copyright (c) GraphQL Contributors
6 *
7 * Permission is hereby granted, free of charge, to any person obtaining a copy
8 * of this software and associated documentation files (the "Software"), to deal
9 * in the Software without restriction, including without limitation the rights
10 * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
11 * copies of the Software, and to permit persons to whom the Software is
12 * furnished to do so, subject to the following conditions:
13 *
14 * The above copyright notice and this permission notice shall be included in all
15 * copies or substantial portions of the Software.
16 *
17 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
18 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
19 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
20 * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
21 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
22 * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
23 * SOFTWARE.
24 */
25 /**
26 * GraphQL lexer for the current spec edition (September 2025).
27 *
28 * Produces one token at a time over a {@link Source}. Ignored tokens (BOM,
29 * whitespace, line terminators, commas and comments) are skipped. String and
30 * block string tokens carry their decoded value; numbers carry their source
31 * text. Every character, comments included, must be a Unicode scalar value;
32 * an unpaired surrogate is an error. Errors are thrown as {@link Diagnostic}
33 * and caught by the parser.
34 *
35 * The structure and messages follow graphql-js so disputes can be settled
36 * against the reference implementation.
37 *
38 * @internal
39 */
40 import { make, type Source } from "./Diagnostic.ts"
41
42 const punctuatorKinds = ["!", "$", "&", "(", ")", "...", ":", "=", "@", "[", "]", "{", "|", "}"] as const
43
44 type PunctuatorKind = (typeof punctuatorKinds)[number]
45
46 export type TokenKind = "<SOF>" | "<EOF>" | PunctuatorKind | "Name" | "Int" | "Float" | "String" | "BlockString"
47
48 export interface Token {
49 readonly kind: TokenKind
50 /** Code-unit offset of the first character. */
51 readonly start: number
52 /** Code-unit offset one past the last character. */
53 readonly end: number
54 /** Decoded value for `Name`, `Int`, `Float`, `String` and `BlockString`. */
55 readonly value: string | undefined
56 }
57
58 const punctuators: ReadonlySet<string> = new Set(punctuatorKinds)
59
60 export const isPunctuatorKind = (kind: TokenKind): kind is PunctuatorKind => punctuators.has(kind)
61
62 export class Lexer {
63 readonly source: Source
64 /** The most recently read token. Starts as `<SOF>`. */
65 token: Token
66 /** The token before `token`. */
67 lastToken: Token
68 private next: Token | undefined = undefined
69
70 constructor(source: Source) {
71 this.source = source
72 this.token = { kind: "<SOF>", start: 0, end: 0, value: undefined }
73 this.lastToken = this.token
74 }
75
76 /** Advances to the next token and returns it. */
77 advance(): Token {
78 this.lastToken = this.token
79 this.token = this.lookahead()
80 this.next = undefined
81 return this.token
82 }
83
84 /** Returns the token after the current one without consuming it. */
85 lookahead(): Token {
86 if (this.next === undefined) {
87 this.next = this.token.kind === "<EOF>" ? this.token : readToken(this.source, this.token.end)
88 }
89 return this.next
90 }
91 }
92
93 const token = (kind: TokenKind, start: number, end: number, value?: string): Token => ({ kind, start, end, value })
94
95 const isDigit = (code: number): boolean => code >= 0x30 && code <= 0x39
96
97 const isNameStart = (code: number): boolean =>
98 (code >= 0x61 && code <= 0x7a) || (code >= 0x41 && code <= 0x5a) || code === 0x5f
99
100 const isNameContinue = (code: number): boolean => isNameStart(code) || isDigit(code)
101
102 const isUnicodeScalarValue = (code: number): boolean =>
103 (code >= 0x0000 && code <= 0xd7ff) || (code >= 0xe000 && code <= 0x10ffff)
104
105 const isLeadingSurrogate = (code: number): boolean => code >= 0xd800 && code <= 0xdbff
106
107 const isTrailingSurrogate = (code: number): boolean => code >= 0xdc00 && code <= 0xdfff
108
109 /**
110 * The code-unit length of the `SourceCharacter` at `position`: 1, 2 for a
111 * surrogate pair, or 0 when it is an unpaired surrogate.
112 */
113 const sourceCharacterSize = (body: string, position: number): number =>
114 isUnicodeScalarValue(body.charCodeAt(position))
115 ? 1
116 : isLeadingSurrogate(body.charCodeAt(position)) && isTrailingSurrogate(body.charCodeAt(position + 1))
117 ? 2
118 : 0
119
120 /** Describes the code point at `location` for messages: `"?"`, `U+00E9`, or `<EOF>`. */
121 const printCodePointAt = (body: string, location: number): string => {
122 const code = body.codePointAt(location)
123 if (code === undefined) return "<EOF>"
124 if (code >= 0x0020 && code <= 0x007e) {
125 return code === 0x0022 ? "'\"'" : `"${String.fromCodePoint(code)}"`
126 }
127 return `U+${code.toString(16).toUpperCase().padStart(4, "0")}`
128 }
129
130 const readToken = (source: Source, start: number): Token => {
131 const body = source.body
132 const bodyLength = body.length
133 let position = start
134 while (position < bodyLength) {
135 const code = body.charCodeAt(position)
136 switch (code) {
137 // Ignored: BOM, tab, space, comma, line feed
138 case 0xfeff:
139 case 0x0009:
140 case 0x0020:
141 case 0x002c:
142 case 0x000a:
143 position++
144 continue
145 // Carriage return, optionally followed by a line feed
146 case 0x000d:
147 position += body.charCodeAt(position + 1) === 0x000a ? 2 : 1
148 continue
149 // Comment
150 case 0x0023:
151 position = readComment(body, position)
152 continue
153 case 0x002e:
154 if (body.charCodeAt(position + 1) === 0x002e && body.charCodeAt(position + 2) === 0x002e) {
155 return token("...", position, position + 3)
156 }
157 break
158 case 0x0022:
159 if (body.charCodeAt(position + 1) === 0x0022 && body.charCodeAt(position + 2) === 0x0022) {
160 return readBlockString(source, position)
161 }
162 return readString(source, position)
163 }
164 const char = body[position]!
165 if (punctuators.has(char)) {
166 return token(char as PunctuatorKind, position, position + 1)
167 }
168 if (isDigit(code) || code === 0x002d) {
169 return readNumber(source, position, code)
170 }
171 if (isNameStart(code)) {
172 return readName(body, position)
173 }
174 throw make(
175 source,
176 position,
177 code === 0x0027
178 ? "Unexpected single quote character ('), did you mean to use a double quote (\")?"
179 : sourceCharacterSize(body, position) > 0
180 ? `Unexpected character: ${printCodePointAt(body, position)}.`
181 : `Invalid character: ${printCodePointAt(body, position)}.`
182 )
183 }
184 return token("<EOF>", bodyLength, bodyLength)
185 }
186
187 const readComment = (body: string, start: number): number => {
188 let position = start + 1
189 while (position < body.length) {
190 const code = body.charCodeAt(position)
191 if (code === 0x000a || code === 0x000d) break
192 // Not a SourceCharacter; stop here so the main loop reports it.
193 const size = sourceCharacterSize(body, position)
194 if (size === 0) break
195 position += size
196 }
197 return position
198 }
199
200 const readName = (body: string, start: number): Token => {
201 let position = start + 1
202 while (position < body.length && isNameContinue(body.charCodeAt(position))) {
203 position++
204 }
205 return token("Name", start, position, body.slice(start, position))
206 }
207
208 const readNumber = (source: Source, start: number, firstCode: number): Token => {
209 const body = source.body
210 let position = start
211 let code = firstCode
212 let isFloat = false
213 if (code === 0x002d) {
214 code = body.charCodeAt(++position)
215 }
216 if (code === 0x0030) {
217 code = body.charCodeAt(++position)
218 if (isDigit(code)) {
219 throw make(source, position, `Invalid number, unexpected digit after 0: ${printCodePointAt(body, position)}.`)
220 }
221 } else {
222 position = readDigits(source, position, code)
223 code = body.charCodeAt(position)
224 }
225 if (code === 0x002e) {
226 isFloat = true
227 code = body.charCodeAt(++position)
228 position = readDigits(source, position, code)
229 code = body.charCodeAt(position)
230 }
231 if (code === 0x0045 || code === 0x0065) {
232 isFloat = true
233 code = body.charCodeAt(++position)
234 if (code === 0x002b || code === 0x002d) {
235 code = body.charCodeAt(++position)
236 }
237 position = readDigits(source, position, code)
238 code = body.charCodeAt(position)
239 }
240 if (code === 0x002e || isNameStart(code)) {
241 throw make(source, position, `Invalid number, expected digit but got: ${printCodePointAt(body, position)}.`)
242 }
243 return token(isFloat ? "Float" : "Int", start, position, body.slice(start, position))
244 }
245
246 const readDigits = (source: Source, start: number, firstCode: number): number => {
247 if (!isDigit(firstCode)) {
248 throw make(source, start, `Invalid number, expected digit but got: ${printCodePointAt(source.body, start)}.`)
249 }
250 const body = source.body
251 let position = start + 1
252 while (isDigit(body.charCodeAt(position))) {
253 position++
254 }
255 return position
256 }
257
258 const readString = (source: Source, start: number): Token => {
259 const body = source.body
260 const bodyLength = body.length
261 let position = start + 1
262 let chunkStart = position
263 let value = ""
264 while (position < bodyLength) {
265 const code = body.charCodeAt(position)
266 if (code === 0x0022) {
267 value += body.slice(chunkStart, position)
268 return token("String", start, position + 1, value)
269 }
270 if (code === 0x005c) {
271 value += body.slice(chunkStart, position)
272 const escape = body.charCodeAt(position + 1) === 0x0075
273 ? body.charCodeAt(position + 2) === 0x007b
274 ? readEscapedUnicodeVariableWidth(source, position)
275 : readEscapedUnicodeFixedWidth(source, position)
276 : readEscapedCharacter(source, position)
277 value += escape.value
278 position += escape.size
279 chunkStart = position
280 continue
281 }
282 if (code === 0x000a || code === 0x000d) break
283 const size = sourceCharacterSize(body, position)
284 if (size === 0) {
285 throw make(source, position, `Invalid character within String: ${printCodePointAt(body, position)}.`)
286 }
287 position += size
288 }
289 throw make(source, position, "Unterminated string.")
290 }
291
292 interface EscapeSequence {
293 readonly value: string
294 readonly size: number
295 }
296
297 const readEscapedUnicodeVariableWidth = (source: Source, position: number): EscapeSequence => {
298 const body = source.body
299 let point = 0
300 let size = 3
301 // Cannot be larger than 12 chars (\u{00000000}).
302 while (size < 12) {
303 const code = body.charCodeAt(position + size++)
304 if (code === 0x007d) {
305 // Must be at least 5 chars (\u{0}) and encode a Unicode scalar value.
306 if (size < 5 || !isUnicodeScalarValue(point)) break
307 return { value: String.fromCodePoint(point), size }
308 }
309 point = (point << 4) | readHexDigit(code)
310 if (point < 0) break
311 }
312 throw make(source, position, `Invalid Unicode escape sequence: "${body.slice(position, position + size)}".`)
313 }
314
315 const readEscapedUnicodeFixedWidth = (source: Source, position: number): EscapeSequence => {
316 const body = source.body
317 const code = read16BitHexCode(body, position + 2)
318 if (isUnicodeScalarValue(code)) {
319 return { value: String.fromCodePoint(code), size: 6 }
320 }
321 // JSON-style surrogate pair escapes are allowed only when they form a pair.
322 if (isLeadingSurrogate(code)) {
323 if (body.charCodeAt(position + 6) === 0x005c && body.charCodeAt(position + 7) === 0x0075) {
324 const trailingCode = read16BitHexCode(body, position + 8)
325 if (isTrailingSurrogate(trailingCode)) {
326 return { value: String.fromCodePoint(code, trailingCode), size: 12 }
327 }
328 }
329 }
330 throw make(source, position, `Invalid Unicode escape sequence: "${body.slice(position, position + 6)}".`)
331 }
332
333 const read16BitHexCode = (body: string, position: number): number =>
334 (readHexDigit(body.charCodeAt(position)) << 12) |
335 (readHexDigit(body.charCodeAt(position + 1)) << 8) |
336 (readHexDigit(body.charCodeAt(position + 2)) << 4) |
337 readHexDigit(body.charCodeAt(position + 3))
338
339 /** The value of a hex digit, or -1 for anything else. */
340 const readHexDigit = (code: number): number =>
341 code >= 0x0030 && code <= 0x0039
342 ? code - 0x0030
343 : code >= 0x0041 && code <= 0x0046
344 ? code - 0x0037
345 : code >= 0x0061 && code <= 0x0066
346 ? code - 0x0057
347 : -1
348
349 const escapedCharacters: ReadonlyMap<string | undefined, string> = new Map([
350 ["\"", "\""],
351 ["\\", "\\"],
352 ["/", "/"],
353 ["b", "\b"],
354 ["f", "\f"],
355 ["n", "\n"],
356 ["r", "\r"],
357 ["t", "\t"]
358 ])
359
360 const readEscapedCharacter = (source: Source, position: number): EscapeSequence => {
361 const body = source.body
362 const value = escapedCharacters.get(body[position + 1])
363 if (value !== undefined) return { value, size: 2 }
364 throw make(source, position, `Invalid character escape sequence: "${body.slice(position, position + 2)}".`)
365 }
366
367 const readBlockString = (source: Source, start: number): Token => {
368 const body = source.body
369 const bodyLength = body.length
370 let position = start + 3
371 let chunkStart = position
372 let currentLine = ""
373 const blockLines: Array<string> = []
374 while (position < bodyLength) {
375 const code = body.charCodeAt(position)
376 if (code === 0x0022 && body.charCodeAt(position + 1) === 0x0022 && body.charCodeAt(position + 2) === 0x0022) {
377 currentLine += body.slice(chunkStart, position)
378 blockLines.push(currentLine)
379 return token("BlockString", start, position + 3, dedentBlockStringLines(blockLines).join("\n"))
380 }
381 if (
382 code === 0x005c &&
383 body.charCodeAt(position + 1) === 0x0022 &&
384 body.charCodeAt(position + 2) === 0x0022 &&
385 body.charCodeAt(position + 3) === 0x0022
386 ) {
387 currentLine += body.slice(chunkStart, position)
388 chunkStart = position + 1
389 position += 4
390 continue
391 }
392 if (code === 0x000a || code === 0x000d) {
393 currentLine += body.slice(chunkStart, position)
394 blockLines.push(currentLine)
395 position += code === 0x000d && body.charCodeAt(position + 1) === 0x000a ? 2 : 1
396 currentLine = ""
397 chunkStart = position
398 continue
399 }
400 const size = sourceCharacterSize(body, position)
401 if (size === 0) {
402 throw make(source, position, `Invalid character within String: ${printCodePointAt(body, position)}.`)
403 }
404 position += size
405 }
406 throw make(source, position, "Unterminated string.")
407 }
408
409 /** The spec's `BlockStringValue` algorithm over already split lines. */
410 const dedentBlockStringLines = (lines: ReadonlyArray<string>): ReadonlyArray<string> => {
411 let commonIndent = Number.MAX_SAFE_INTEGER
412 let firstNonEmptyLine: number | undefined = undefined
413 let lastNonEmptyLine = -1
414 for (let i = 0; i < lines.length; i++) {
415 const line = lines[i]!
416 const indent = leadingWhitespace(line)
417 if (indent === line.length) continue
418 firstNonEmptyLine ??= i
419 lastNonEmptyLine = i
420 if (i !== 0 && indent < commonIndent) {
421 commonIndent = indent
422 }
423 }
424 return lines
425 .map((line, i) => (i === 0 ? line : line.slice(commonIndent)))
426 .slice(firstNonEmptyLine ?? 0, lastNonEmptyLine + 1)
427 }
428
429 const leadingWhitespace = (line: string): number => {
430 let i = 0
431 while (i < line.length && (line[i] === " " || line[i] === "\t")) {
432 i++
433 }
434 return i
435 }