@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 54d3544784
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 04:15 UTC

Fix: Output text is always truncated in some models (#3016)

Sherry committed Mar 1, 2024 at 15:52 UTC 54d3544784ff20e7038abf72793eaf734e727269
1 file changed +4 -1
vllm/engine/llm_engine.py
+4 -1
@@ -980,7 +980,10 @@ class LLMEngine:
980 def _finalize_sequence(self, seq: Sequence,
981 sampling_params: SamplingParams,
982 stop_string: str) -> None:
983 - if not sampling_params.include_stop_str_in_output and stop_string:
983 + if sampling_params.include_stop_str_in_output:
984 + return
985 +
986 + if stop_string and seq.output_text.endswith(stop_string):
987 # Truncate the output text so that the stop string is
988 # not included in the output.
989 seq.output_text = seq.output_text[:-len(stop_string)]