A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Bugfix] Guided decoding falls back to outlines when fails to import xgrammar (#12976)
Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>
Yuan Tang committed
Feb 11, 2025 at 13:17 UTC
14ecab5be21b2af4ab1bbc6309d558ec620badc6
2 files changed
+11
vllm/model_executor/guided_decoding/__init__.py
+9
@@ -40,6 +40,8 @@ def maybe_backend_fallback(
40
guided_params.backend = "outlines"
41
42
if guided_params.backend == "xgrammar":
43
+ from vllm.model_executor.guided_decoding.xgrammar_decoding import (
44
+ xgr_installed)
45
# xgrammar only has x86 wheels for linux, fallback to outlines
46
from vllm.platforms import current_platform
47
if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
@@ -77,6 +79,13 @@ def maybe_backend_fallback(
79
"Falling back to use outlines instead.")
80
guided_params.backend = "outlines"
81
82
+ # If the xgrammar module cannot be imported successfully,
83
+ # we should still allow users to use guided decoding with a fallback.
84
+ elif not xgr_installed:
85
+ logger.warning("xgrammar module cannot be imported successfully. "
86
+ "Falling back to use outlines instead.")
87
+ guided_params.backend = "outlines"
88
+
89
if (guided_params.backend == "outlines"
90
and guided_params.json_object is not None):
91
# outlines doesn't support json_object, fallback to xgrammar
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+2
@@ -14,7 +14,9 @@ from transformers import PreTrainedTokenizerFast
14
try:
15
import xgrammar as xgr
16
from xgrammar.base import _core as xgr_core
17
+ xgr_installed = True
18
except ImportError:
19
+ xgr_installed = False
20
pass
21
22
from vllm.model_executor.guided_decoding.utils import (convert_lark_to_gbnf,