@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5a6c81b051
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Remove eos tokens from output by default (#2611)

Rex committed Feb 4, 2024 at 14:32 UTC 5a6c81b0511da333b1fabf5ad612eb7874d5e88e
1 file changed +12 -4
vllm/engine/llm_engine.py
+12 -4
@@ -908,13 +908,13 @@ class LLMEngine:
908 """Stop the finished sequences."""
909 for stop_str in sampling_params.stop:
910 if seq.output_text.endswith(stop_str):
911 - if not sampling_params.include_stop_str_in_output:
912 - # Truncate the output text so that the stop string is
913 - # not included in the output.
914 - seq.output_text = seq.output_text[:-len(stop_str)]
911 + self._finalize_sequence(seq, sampling_params, stop_str)
912 seq.status = SequenceStatus.FINISHED_STOPPED
913 return
914 if seq.get_last_token_id() in sampling_params.stop_token_ids:
915 + stop_str = self.get_tokenizer_for_seq(seq).convert_ids_to_tokens(
916 + seq.get_last_token_id())
917 + self._finalize_sequence(seq, sampling_params, stop_str)
918 seq.status = SequenceStatus.FINISHED_STOPPED
919 return
920
@@ -934,6 +934,14 @@ class LLMEngine:
934 seq.status = SequenceStatus.FINISHED_STOPPED
935 return
936
937 + def _finalize_sequence(self, seq: Sequence,
938 + sampling_params: SamplingParams,
939 + stop_string: str) -> None:
940 + if not sampling_params.include_stop_str_in_output and stop_string:
941 + # Truncate the output text so that the stop string is
942 + # not included in the output.
943 + seq.output_text = seq.output_text[:-len(stop_string)]
944 +
945 def add_lora(self, lora_request: LoRARequest) -> bool:
946 assert lora_request.lora_int_id > 0, "lora_id must be greater than 0."
947 return self._run_workers(