A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Remove eos tokens from output by default (#2611)
Rex committed
Feb 4, 2024 at 14:32 UTC
5a6c81b0511da333b1fabf5ad612eb7874d5e88e
1 file changed
+12
-4
vllm/engine/llm_engine.py
+12
-4
@@ -908,13 +908,13 @@ class LLMEngine:
908
"""Stop the finished sequences."""
909
for stop_str in sampling_params.stop:
910
if seq.output_text.endswith(stop_str):
911
- if not sampling_params.include_stop_str_in_output:
912
- # Truncate the output text so that the stop string is
913
- # not included in the output.
914
- seq.output_text = seq.output_text[:-len(stop_str)]
911
+ self._finalize_sequence(seq, sampling_params, stop_str)
912
seq.status = SequenceStatus.FINISHED_STOPPED
913
return
914
if seq.get_last_token_id() in sampling_params.stop_token_ids:
915
+ stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
916
+ seq.get_last_token_id())
917
+ self._finalize_sequence(seq, sampling_params, stop_str)
918
seq.status = SequenceStatus.FINISHED_STOPPED
919
return
920
@@ -934,6 +934,14 @@ class LLMEngine:
934
seq.status = SequenceStatus.FINISHED_STOPPED
935
return
936
937
+ def _finalize_sequence(self, seq: Sequence,
938
+ sampling_params: SamplingParams,
939
+ stop_string: str) -> None:
940
+ if not sampling_params.include_stop_str_in_output and stop_string:
941
+ # Truncate the output text so that the stop string is
942
+ # not included in the output.
943
+ seq.output_text = seq.output_text[:-len(stop_string)]
944
+
945
def add_lora(self, lora_request: LoRARequest) -> bool:
946
assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
947
return self._run_workers(