@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / feb92fbe4a
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

Fix beam search eos (#9627)

Robert Shaw committed Oct 28, 2024 at 02:59 UTC feb92fbe4ab6803527df48658a87ebd00b99969f
1 file changed +6 -1
vllm/engine/protocol.py
+6 -1
@@ -140,7 +140,12 @@ class EngineClient(ABC):
140 best_beams = sorted_completed[:beam_width]
141
142 for beam in best_beams:
143 - beam.text = tokenizer.decode(beam.tokens[tokenized_length:])
143 + if (beam.tokens[-1] == tokenizer.eos_token_id and not ignore_eos):
144 + # Skip the eos token in the text.
145 + tokens = beam.tokens[tokenized_length:-1]
146 + else:
147 + tokens = beam.tokens[tokenized_length:]
148 + beam.text = tokenizer.decode(tokens)
149
150 beam_search_output = RequestOutput(
151 request_id=request_id,