A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[BUGFIX] Move scores to float32 in case of running xgrammar on cpu (#12152)
Signed-off-by: Michal Adamczyk <madamczyk@habana.ai>
Michal Adamczyk committed
Jan 19, 2025 at 04:12 UTC
4e94951bb16282c36de6d12ef14a1500f25a3bdf
1 file changed
+5
-2
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+5
-2
@@ -298,8 +298,11 @@ class XGrammarLogitsProcessor:
298
# token_bitmask is a CPU tensor for use with accept_token and
299
# fill_next_token_bitmask so we move it to the device of scores
300
device_type = scores.device.type
301
+ dtype = scores.dtype
302
if device_type != "cuda":
302
- scores = scores.to("cpu").unsqueeze(0)
303
+ # xgrammar on cpu only supports float32 scores
304
+ # see: https://github.com/mlc-ai/xgrammar/blob/c1b64920cad24f44f235778c1c00bb52d57da01a/python/xgrammar/kernels/apply_token_bitmask_inplace_cpu.py#L22
305
+ scores = scores.to("cpu").float().unsqueeze(0)
306
307
# Note: In this method, if the tensors have different dimensions
308
# on CPU device fails, but on GPU it runs without error. Hence the
@@ -307,7 +310,7 @@ class XGrammarLogitsProcessor:
310
xgr.apply_token_bitmask_inplace(scores,
311
self.token_bitmask.to(scores.device))
312
if device_type != "cuda":
310
- scores = scores.to(device_type).squeeze()
313
+ scores = scores.to(dtype).to(device_type).squeeze()
314
315
return scores
316