A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 06:00 UTC
[V1] Resolve failed concurrent structured output requests (#19565)
Signed-off-by: Russell Bryant <rbryant@redhat.com>
Russell Bryant committed
Jun 12, 2025 at 19:30 UTC
c57bb199b3eff82b4bdd5ffb089502936a5b0c9a
1 file changed
+8
-1
vllm/v1/worker/gpu_model_runner.py
+8
-1
@@ -66,11 +66,15 @@ from .utils import (gather_mm_placeholders, initialize_kv_cache_for_kv_sharing,
66
67
if TYPE_CHECKING:
68
import xgrammar as xgr
69
+ import xgrammar.kernels.apply_token_bitmask_inplace_torch_compile as xgr_torch_compile # noqa: E501
70
71
from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
72
from vllm.v1.core.sched.output import SchedulerOutput
73
else:
74
xgr = LazyLoader("xgr", globals(), "xgrammar")
75
+ xgr_torch_compile = LazyLoader(
76
+ "xgr_torch_compile", globals(),
77
+ "xgrammar.kernels.apply_token_bitmask_inplace_torch_compile")
78
79
logger = init_logger(__name__)
80
@@ -1103,7 +1107,10 @@ class GPUModelRunner(LoRAModelRunnerMixin):
1107
# so we receive it in that format.
1108
grammar_bitmask = torch.from_numpy(grammar_bitmask)
1109
1106
- xgr.apply_token_bitmask_inplace(
1110
+ # Force use of the torch.compile implementation from xgrammar to work
1111
+ # around issues with the Triton kernel in concurrent structured output
1112
+ # scenarios. See PR #19565 and issues #19493, #18376 for details.
1113
+ xgr_torch_compile.apply_token_bitmask_inplace_torch_compile(
1114
logits,
1115
grammar_bitmask.to(self.device, non_blocking=True),
1116
indices=out_indices,