@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 14ecab5be2
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Bugfix] Guided decoding falls back to outlines when fails to import xgrammar (#12976)

Signed-off-by: Yuan Tang <terrytangyuan@gmail.com>

Yuan Tang committed Feb 11, 2025 at 13:17 UTC 14ecab5be21b2af4ab1bbc6309d558ec620badc6
2 files changed +11
vllm/model_executor/guided_decoding/__init__.py
+9
@@ -40,6 +40,8 @@ def maybe_backend_fallback(
40 guided_params.backend = "outlines"
41
42 if guided_params.backend == "xgrammar":
43 + from vllm.model_executor.guided_decoding.xgrammar_decoding import (
44 + xgr_installed)
45 # xgrammar only has x86 wheels for linux, fallback to outlines
46 from vllm.platforms import current_platform
47 if current_platform.get_cpu_architecture() is not CpuArchEnum.X86:
@@ -77,6 +79,13 @@ def maybe_backend_fallback(
79 "Falling back to use outlines instead.")
80 guided_params.backend = "outlines"
81
82 + # If the xgrammar module cannot be imported successfully,
83 + # we should still allow users to use guided decoding with a fallback.
84 + elif not xgr_installed:
85 + logger.warning("xgrammar module cannot be imported successfully. "
86 + "Falling back to use outlines instead.")
87 + guided_params.backend = "outlines"
88 +
89 if (guided_params.backend == "outlines"
90 and guided_params.json_object is not None):
91 # outlines doesn't support json_object, fallback to xgrammar
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+2
@@ -14,7 +14,9 @@ from transformers import PreTrainedTokenizerFast
14 try:
15 import xgrammar as xgr
16 from xgrammar.base import _core as xgr_core
17 + xgr_installed = True
18 except ImportError:
19 + xgr_installed = False
20 pass
21
22 from vllm.model_executor.guided_decoding.utils import (convert_lark_to_gbnf,