A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 23:30 UTC
[Hardware][Gaudi][Bugfix] Fix error for guided decoding (#12317)
Yu-Zhou committed
Feb 14, 2025 at 20:36 UTC
c9e2d644e728e8e93ef2871276ed7a6b39c1d0eb
1 file changed
+10
-1
vllm/model_executor/guided_decoding/outlines_logits_processors.py
+10
-1
@@ -32,6 +32,8 @@ from outlines_core.fsm.json_schema import build_regex_from_schema
32
from pydantic import BaseModel
33
from transformers import PreTrainedTokenizerBase
34
35
+from vllm.platforms import current_platform
36
+
37
38
class BaseLogitsProcessor:
39
@@ -91,7 +93,14 @@ class BaseLogitsProcessor:
93
allowed_tokens = allowed_tokens.masked_select(
94
allowed_tokens < scores.shape[-1])
95
mask.index_fill_(0, allowed_tokens, 0)
94
- scores.add_(mask)
96
+ if current_platform.is_hpu():
97
+ # Workaround for HPU bug where add_() raise RuntimeError:
98
+ # synNodeCreateWithId failed for node: strided_insert
99
+ # with synStatus 1 [Invalid argument], hopefully it will
100
+ # be fixed in the future releases of the HPU runtime.
101
+ scores = scores.add(mask)
102
+ else:
103
+ scores.add_(mask)
104
return scores
105
106