@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c9e2d644e7
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 15, 23:30 UTC

[Hardware][Gaudi][Bugfix] Fix error for guided decoding (#12317)

Yu-Zhou committed Feb 14, 2025 at 20:36 UTC c9e2d644e728e8e93ef2871276ed7a6b39c1d0eb
1 file changed +10 -1
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+10 -1
@@ -32,6 +32,8 @@ from outlines_core.fsm.json_schema import build_regex_from_schema
32 from pydantic import BaseModel
33 from transformers import PreTrainedTokenizerBase
34
35 +from vllm.platforms import current_platform
36 +
37
38 class BaseLogitsProcessor:
39
@@ -91,7 +93,14 @@ class BaseLogitsProcessor:
93 allowed_tokens = allowed_tokens.masked_select(
94 allowed_tokens < scores.shape[-1])
95 mask.index_fill_(0, allowed_tokens, 0)
94 - scores.add_(mask)
96 + if current_platform.is_hpu():
97 + # Workaround for HPU bug where add_() raise RuntimeError:
98 + # synNodeCreateWithId failed for node: strided_insert
99 + # with synStatus 1 [Invalid argument], hopefully it will
100 + # be fixed in the future releases of the HPU runtime.
101 + scores = scores.add(mask)
102 + else:
103 + scores.add_(mask)
104 return scores
105
106