@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 4e94951bb1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[BUGFIX] Move scores to float32 in case of running xgrammar on cpu (#12152)

Signed-off-by: Michal Adamczyk <madamczyk@habana.ai>

Michal Adamczyk committed Jan 19, 2025 at 04:12 UTC 4e94951bb16282c36de6d12ef14a1500f25a3bdf
1 file changed +5 -2
vllm/model_executor/guided_decoding/xgrammar_decoding.py
+5 -2
@@ -298,8 +298,11 @@ class XGrammarLogitsProcessor:
298 # token_bitmask is a CPU tensor for use with accept_token and
299 # fill_next_token_bitmask so we move it to the device of scores
300 device_type = scores.device.type
301 + dtype = scores.dtype
302 if device_type != "cuda":
302 - scores = scores.to("cpu").unsqueeze(0)
303 + # xgrammar on cpu only supports float32 scores
304 + # see: https://github.com/mlc-ai/xgrammar/blob/c1b64920cad24f44f235778c1c00bb52d57da01a/python/xgrammar/kernels/apply_token_bitmask_inplace_cpu.py#L22
305 + scores = scores.to("cpu").float().unsqueeze(0)
306
307 # Note: In this method, if the tensors have different dimensions
308 # on CPU device fails, but on GPU it runs without error. Hence the
@@ -307,7 +310,7 @@ class XGrammarLogitsProcessor:
310 xgr.apply_token_bitmask_inplace(scores,
311 self.token_bitmask.to(scores.device))
312 if device_type != "cuda":
310 - scores = scores.to(device_type).squeeze()
313 + scores = scores.to(dtype).to(device_type).squeeze()
314
315 return scores
316