A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Core] choice-based structured output with xgrammar (#12632)
Russell Bryant committed
Feb 14, 2025 at 07:36 UTC
7734e9a2912762e027a507198772c94370dbeb95
3 files changed
+35
-7
requirements-common.txt
+1
-1
@@ -20,7 +20,7 @@ tiktoken >= 0.6.0 # Required for DBRX tokenizer
20
lm-format-enforcer >= 0.10.9, < 0.11
21
outlines == 0.1.11
22
lark == 1.2.2
23
-xgrammar >= 0.1.6; platform_machine == "x86_64"
23
+xgrammar >= 0.1.11; platform_machine == "x86_64"
24
typing_extensions >= 4.10
25
filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
26
partial-json-parser # used for parsing partial JSON outputs
vllm/model_executor/guided_decoding/__init__.py
+4
-5
@@ -49,11 +49,10 @@ def maybe_backend_fallback(
49
"Falling back to use outlines instead.")
50
guided_params.backend = "outlines"
51
52
- # xgrammar doesn't support regex or choice, fallback to outlines
53
- if guided_params.regex is not None or guided_params.choice is not None:
54
- logger.warning(
55
- "xgrammar only supports json or grammar guided decoding. "
56
- "Falling back to use outlines instead.")
52
+ # xgrammar doesn't support regex, fallback to outlines
53
+ if guided_params.regex is not None:
54
+ logger.warning("xgrammar does not support regex guided decoding. "
55
+ "Falling back to use outlines instead.")
56
guided_params.backend = "outlines"
57
58
# xgrammar doesn't support some JSON schema features
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+30
-1
@@ -5,8 +5,9 @@ from __future__ import annotations
5
6
import copy
7
import json
8
+import re
9
from dataclasses import dataclass, field
9
-from typing import TYPE_CHECKING, Any
10
+from typing import TYPE_CHECKING, Any, List
11
12
import torch
13
from transformers import PreTrainedTokenizerFast
@@ -228,11 +229,39 @@ class GrammarConfig:
229
max_threads=max_threads,
230
tokenizer_data=tokenizer_data,
231
)
232
+ elif guided_params.choice:
233
+ choice_str = GrammarConfig.choice_as_grammar(guided_params.choice)
234
+ try:
235
+ xgr.Grammar.from_ebnf(choice_str)
236
+ except RuntimeError as err:
237
+ raise ValueError(str(err)) from err
238
+
239
+ return cls(
240
+ grammar_str=choice_str,
241
+ vocab_size=model_config.hf_text_config.vocab_size,
242
+ tokenizer_hash=tokenizer_hash,
243
+ max_threads=max_threads,
244
+ tokenizer_data=tokenizer_data,
245
+ )
246
else:
247
raise ValueError(
248
"Currently only support JSON and EBNF grammar mode for xgrammar"
249
)
250
251
+ @staticmethod
252
+ def escape_ebnf_string(s: str) -> str:
253
+ """Escape special characters in a EBNF string."""
254
+ # Escape double quotes and backslashes
255
+ return re.sub(r'(["\\])', r'\\\1', s)
256
+
257
+ @staticmethod
258
+ def choice_as_grammar(choice: List[str] | None) -> str:
259
+ if choice is None:
260
+ raise ValueError("Choice is not set")
261
+ escaped_choices = (GrammarConfig.escape_ebnf_string(c) for c in choice)
262
+ grammar = ('root ::= ' + ' | '.join(f'"{c}"' for c in escaped_choices))
263
+ return grammar
264
+
265
266
@dataclass
267
class XGrammarLogitsProcessor: