A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 17:00 UTC
[Bugfix] Fix xgrammar dtype mismatch on macOS CPU inference (#32384)
Signed-off-by: Karan Bansal <karanb192@gmail.com> Co-authored-by: Inokinoki <inoki@inoki.cc>
Karan Bansal committed
Mar 14, 2026 at 22:59 UTC
821fde2df470e732bb2061daf1e8ef9838d7cce6
1 file changed
+12
-1
vllm/v1/structured_output/utils.py
+12
-1
@@ -116,7 +116,18 @@ def apply_grammar_bitmask(
116
)
117
index_tensor = index_tensor.to(logits.device, non_blocking=True)
118
119
- xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
119
+ # Handle dtype conversion for CPU (older xgrammar CPU kernels require float32)
120
+ # See: https://github.com/vllm-project/vllm/issues/31901
121
+ if logits.device.type == "cpu" and logits.dtype != torch.float32:
122
+ # Convert to float32, apply bitmask, then convert back
123
+ logits_float32 = logits.to(torch.float32)
124
+ xgr.apply_token_bitmask_inplace(
125
+ logits_float32, grammar_bitmask, indices=index_tensor
126
+ )
127
+ # Copy the modified values back to the original tensor
128
+ logits.copy_(logits_float32.to(logits.dtype))
129
+ else:
130
+ xgr.apply_token_bitmask_inplace(logits, grammar_bitmask, indices=index_tensor)
131
132
133
class OutlinesVocabulary: