A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[BugFix] Lazily import XgrammarBackend to avoid early cuda init (#15171)
Signed-off-by: Nick Hill <nhill@redhat.com>
Nick Hill committed
Mar 19, 2025 at 18:30 UTC
c47aafa37c7579c3f9b3188b05f43cb71d83dbb5
1 file changed
+3
-1
vllm/v1/structured_output/__init__.py
+3
-1
@@ -9,7 +9,6 @@ from vllm.config import VllmConfig
9
from vllm.logger import init_logger
10
from vllm.v1.structured_output.backend_types import (StructuredOutputBackend,
11
StructuredOutputGrammar)
12
-from vllm.v1.structured_output.backend_xgrammar import XgrammarBackend
12
13
if TYPE_CHECKING:
14
import numpy as np
@@ -47,6 +46,9 @@ class StructuredOutputManager:
46
if self.backend is None:
47
backend_name = request.sampling_params.guided_decoding.backend_name
48
if backend_name == "xgrammar":
49
+ from vllm.v1.structured_output.backend_xgrammar import (
50
+ XgrammarBackend)
51
+
52
self.backend = XgrammarBackend(self.vllm_config)
53
else:
54
raise ValueError(