A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 04:15 UTC
Fix: Output text is always truncated in some models (#3016)
Sherry committed
Mar 1, 2024 at 15:52 UTC
54d3544784ff20e7038abf72793eaf734e727269
1 file changed
+4
-1
vllm/engine/llm_engine.py
+4
-1
@@ -980,7 +980,10 @@ class LLMEngine:
980
def _finalize_sequence(self, seq: Sequence,
981
sampling_params: SamplingParams,
982
stop_string: str) -> None:
983
- if not sampling_params.include_stop_str_in_output and stop_string:
983
+ if sampling_params.include_stop_str_in_output:
984
+ return
985
+
986
+ if stop_string and seq.output_text.endswith(stop_string):
987
# Truncate the output text so that the stop string is
988
# not included in the output.
989
seq.output_text = seq.output_text[:-len(stop_string)]