@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c06170cc8e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 05:30 UTC

Add a flag to include stop string in output text (#1976)

Yunfeng Bai committed Dec 15, 2023 at 00:45 UTC c06170cc8e324f4fe6a0c26b57d09e8c958e11bc
2 files changed +32 -24
vllm/engine/llm_engine.py
+4 -3
@@ -682,9 +682,10 @@ class LLMEngine:
682 """Stop the finished sequences."""
683 for stop_str in sampling_params.stop:
684 if seq.output_text.endswith(stop_str):
685 - # Truncate the output text so that the stop string is
686 - # not included in the output.
687 - seq.output_text = seq.output_text[:-len(stop_str)]
685 + if not sampling_params.include_stop_str_in_output:
686 + # Truncate the output text so that the stop string is
687 + # not included in the output.
688 + seq.output_text = seq.output_text[:-len(stop_str)]
689 seq.status = SequenceStatus.FINISHED_STOPPED
690 return
691 if seq.get_last_token_id() in sampling_params.stop_token_ids:
vllm/sampling_params.py
+28 -21
@@ -2,6 +2,7 @@
2 from enum import IntEnum
3 from functools import cached_property
4 from typing import Callable, List, Optional, Union
5 +
6 import torch
7
8 _SAMPLING_EPS = 1e-5
@@ -70,6 +71,8 @@ class SamplingParams:
71 stop_token_ids: List of tokens that stop the generation when they are
72 generated. The returned output will contain the stop tokens unless
73 the stop tokens are special tokens.
74 + include_stop_str_in_output: Whether to include the stop strings in output
75 + text. Defaults to False.
76 ignore_eos: Whether to ignore the EOS token and continue generating
77 tokens after the EOS token is generated.
78 max_tokens: Maximum number of tokens to generate per output sequence.
@@ -103,6 +106,7 @@ class SamplingParams:
106 early_stopping: Union[bool, str] = False,
107 stop: Optional[Union[str, List[str]]] = None,
108 stop_token_ids: Optional[List[int]] = None,
109 + include_stop_str_in_output: bool = False,
110 ignore_eos: bool = False,
111 max_tokens: int = 16,
112 logprobs: Optional[int] = None,
@@ -140,6 +144,7 @@ class SamplingParams:
144 self.skip_special_tokens = skip_special_tokens
145 self.spaces_between_special_tokens = spaces_between_special_tokens
146 self.logits_processors = logits_processors
147 + self.include_stop_str_in_output = include_stop_str_in_output
148 self._verify_args()
149 if self.use_beam_search:
150 self._verify_beam_search()
@@ -227,24 +232,26 @@ class SamplingParams:
232 return SamplingType.RANDOM
233
234 def __repr__(self) -> str:
230 - return (f"SamplingParams(n={self.n}, "
231 - f"best_of={self.best_of}, "
232 - f"presence_penalty={self.presence_penalty}, "
233 - f"frequency_penalty={self.frequency_penalty}, "
234 - f"repetition_penalty={self.repetition_penalty}, "
235 - f"temperature={self.temperature}, "
236 - f"top_p={self.top_p}, "
237 - f"top_k={self.top_k}, "
238 - f"min_p={self.min_p}, "
239 - f"use_beam_search={self.use_beam_search}, "
240 - f"length_penalty={self.length_penalty}, "
241 - f"early_stopping={self.early_stopping}, "
242 - f"stop={self.stop}, "
243 - f"stop_token_ids={self.stop_token_ids}, "
244 - f"ignore_eos={self.ignore_eos}, "
245 - f"max_tokens={self.max_tokens}, "
246 - f"logprobs={self.logprobs}, "
247 - f"prompt_logprobs={self.prompt_logprobs}, "
248 - f"skip_special_tokens={self.skip_special_tokens}, "
249 - "spaces_between_special_tokens="
250 - f"{self.spaces_between_special_tokens})")
235 + return (
236 + f"SamplingParams(n={self.n}, "
237 + f"best_of={self.best_of}, "
238 + f"presence_penalty={self.presence_penalty}, "
239 + f"frequency_penalty={self.frequency_penalty}, "
240 + f"repetition_penalty={self.repetition_penalty}, "
241 + f"temperature={self.temperature}, "
242 + f"top_p={self.top_p}, "
243 + f"top_k={self.top_k}, "
244 + f"min_p={self.min_p}, "
245 + f"use_beam_search={self.use_beam_search}, "
246 + f"length_penalty={self.length_penalty}, "
247 + f"early_stopping={self.early_stopping}, "
248 + f"stop={self.stop}, "
249 + f"stop_token_ids={self.stop_token_ids}, "
250 + f"include_stop_str_in_output={self.include_stop_str_in_output}, "
251 + f"ignore_eos={self.ignore_eos}, "
252 + f"max_tokens={self.max_tokens}, "
253 + f"logprobs={self.logprobs}, "
254 + f"prompt_logprobs={self.prompt_logprobs}, "
255 + f"skip_special_tokens={self.skip_special_tokens}, "
256 + "spaces_between_special_tokens="
257 + f"{self.spaces_between_special_tokens})")