@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7734e9a291
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Core] choice-based structured output with xgrammar (#12632)

Russell Bryant committed Feb 14, 2025 at 07:36 UTC 7734e9a2912762e027a507198772c94370dbeb95
3 files changed +35 -7
requirements-common.txt
+1 -1
@@ -20,7 +20,7 @@ tiktoken >= 0.6.0 # Required for DBRX tokenizer
20 lm-format-enforcer >= 0.10.9, < 0.11
21 outlines == 0.1.11
22 lark == 1.2.2
23 -xgrammar >= 0.1.6; platform_machine == "x86_64"
23 +xgrammar >= 0.1.11; platform_machine == "x86_64"
24 typing_extensions >= 4.10
25 filelock >= 3.16.1 # need to contain https://github.com/tox-dev/filelock/pull/317
26 partial-json-parser # used for parsing partial JSON outputs
vllm/model_executor/guided_decoding/__init__.py
+4 -5
@@ -49,11 +49,10 @@ def maybe_backend_fallback(
49 "Falling back to use outlines instead.")
50 guided_params.backend = "outlines"
51
52 - # xgrammar doesn't support regex or choice, fallback to outlines
53 - if guided_params.regex is not None or guided_params.choice is not None:
54 - logger.warning(
55 - "xgrammar only supports json or grammar guided decoding. "
56 - "Falling back to use outlines instead.")
52 + # xgrammar doesn't support regex, fallback to outlines
53 + if guided_params.regex is not None:
54 + logger.warning("xgrammar does not support regex guided decoding. "
55 + "Falling back to use outlines instead.")
56 guided_params.backend = "outlines"
57
58 # xgrammar doesn't support some JSON schema features
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+30 -1
@@ -5,8 +5,9 @@ from __future__ import annotations
5
6 import copy
7 import json
8 +import re
9 from dataclasses import dataclass, field
9 -from typing import TYPE_CHECKING, Any
10 +from typing import TYPE_CHECKING, Any, List
11
12 import torch
13 from transformers import PreTrainedTokenizerFast
@@ -228,11 +229,39 @@ class GrammarConfig:
229 max_threads=max_threads,
230 tokenizer_data=tokenizer_data,
231 )
232 + elif guided_params.choice:
233 + choice_str = GrammarConfig.choice_as_grammar(guided_params.choice)
234 + try:
235 + xgr.Grammar.from_ebnf(choice_str)
236 + except RuntimeError as err:
237 + raise ValueError(str(err)) from err
238 +
239 + return cls(
240 + grammar_str=choice_str,
241 + vocab_size=model_config.hf_text_config.vocab_size,
242 + tokenizer_hash=tokenizer_hash,
243 + max_threads=max_threads,
244 + tokenizer_data=tokenizer_data,
245 + )
246 else:
247 raise ValueError(
248 "Currently only support JSON and EBNF grammar mode for xgrammar"
249 )
250
251 + @staticmethod
252 + def escape_ebnf_string(s: str) -> str:
253 + """Escape special characters in a EBNF string."""
254 + # Escape double quotes and backslashes
255 + return re.sub(r'(["\\])', r'\\\1', s)
256 +
257 + @staticmethod
258 + def choice_as_grammar(choice: List[str] | None) -> str:
259 + if choice is None:
260 + raise ValueError("Choice is not set")
261 + escaped_choices = (GrammarConfig.escape_ebnf_string(c) for c in choice)
262 + grammar = ('root ::= ' + ' | '.join(f'"{c}"' for c in escaped_choices))
263 + return grammar
264 +
265
266 @dataclass
267 class XGrammarLogitsProcessor: