A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 05:30 UTC
Add a flag to include stop string in output text (#1976)
Yunfeng Bai committed
Dec 15, 2023 at 00:45 UTC
c06170cc8e324f4fe6a0c26b57d09e8c958e11bc
2 files changed
+32
-24
vllm/engine/llm_engine.py
+4
-3
@@ -682,9 +682,10 @@ class LLMEngine:
682
"""Stop the finished sequences."""
683
for stop_str in sampling_params.stop:
684
if seq.output_text.endswith(stop_str):
685
- # Truncate the output text so that the stop string is
686
- # not included in the output.
687
- seq.output_text = seq.output_text[:-len(stop_str)]
685
+ if not sampling_params.include_stop_str_in_output:
686
+ # Truncate the output text so that the stop string is
687
+ # not included in the output.
688
+ seq.output_text = seq.output_text[:-len(stop_str)]
689
seq.status = SequenceStatus.FINISHED_STOPPED
690
return
691
if seq.get_last_token_id() in sampling_params.stop_token_ids:
vllm/sampling_params.py
+28
-21
@@ -2,6 +2,7 @@
2
from enum import IntEnum
3
from functools import cached_property
4
from typing import Callable, List, Optional, Union
5
+
6
import torch
7
8
_SAMPLING_EPS = 1e-5
@@ -70,6 +71,8 @@ class SamplingParams:
71
stop_token_ids: List of tokens that stop the generation when they are
72
generated. The returned output will contain the stop tokens unless
73
the stop tokens are special tokens.
74
+ include_stop_str_in_output: Whether to include the stop strings in output
75
+ text. Defaults to False.
76
ignore_eos: Whether to ignore the EOS token and continue generating
77
tokens after the EOS token is generated.
78
max_tokens: Maximum number of tokens to generate per output sequence.
@@ -103,6 +106,7 @@ class SamplingParams:
106
early_stopping: Union[bool, str] = False,
107
stop: Optional[Union[str, List[str]]] = None,
108
stop_token_ids: Optional[List[int]] = None,
109
+ include_stop_str_in_output: bool = False,
110
ignore_eos: bool = False,
111
max_tokens: int = 16,
112
logprobs: Optional[int] = None,
@@ -140,6 +144,7 @@ class SamplingParams:
144
self.skip_special_tokens = skip_special_tokens
145
self.spaces_between_special_tokens = spaces_between_special_tokens
146
self.logits_processors = logits_processors
147
+ self.include_stop_str_in_output = include_stop_str_in_output
148
self._verify_args()
149
if self.use_beam_search:
150
self._verify_beam_search()
@@ -227,24 +232,26 @@ class SamplingParams:
232
return SamplingType.RANDOM
233
234
def __repr__(self) -> str:
230
- return (f"SamplingParams(n={self.n}, "
231
- f"best_of={self.best_of}, "
232
- f"presence_penalty={self.presence_penalty}, "
233
- f"frequency_penalty={self.frequency_penalty}, "
234
- f"repetition_penalty={self.repetition_penalty}, "
235
- f"temperature={self.temperature}, "
236
- f"top_p={self.top_p}, "
237
- f"top_k={self.top_k}, "
238
- f"min_p={self.min_p}, "
239
- f"use_beam_search={self.use_beam_search}, "
240
- f"length_penalty={self.length_penalty}, "
241
- f"early_stopping={self.early_stopping}, "
242
- f"stop={self.stop}, "
243
- f"stop_token_ids={self.stop_token_ids}, "
244
- f"ignore_eos={self.ignore_eos}, "
245
- f"max_tokens={self.max_tokens}, "
246
- f"logprobs={self.logprobs}, "
247
- f"prompt_logprobs={self.prompt_logprobs}, "
248
- f"skip_special_tokens={self.skip_special_tokens}, "
249
- "spaces_between_special_tokens="
250
- f"{self.spaces_between_special_tokens})")
235
+ return (
236
+ f"SamplingParams(n={self.n}, "
237
+ f"best_of={self.best_of}, "
238
+ f"presence_penalty={self.presence_penalty}, "
239
+ f"frequency_penalty={self.frequency_penalty}, "
240
+ f"repetition_penalty={self.repetition_penalty}, "
241
+ f"temperature={self.temperature}, "
242
+ f"top_p={self.top_p}, "
243
+ f"top_k={self.top_k}, "
244
+ f"min_p={self.min_p}, "
245
+ f"use_beam_search={self.use_beam_search}, "
246
+ f"length_penalty={self.length_penalty}, "
247
+ f"early_stopping={self.early_stopping}, "
248
+ f"stop={self.stop}, "
249
+ f"stop_token_ids={self.stop_token_ids}, "
250
+ f"include_stop_str_in_output={self.include_stop_str_in_output}, "
251
+ f"ignore_eos={self.ignore_eos}, "
252
+ f"max_tokens={self.max_tokens}, "
253
+ f"logprobs={self.logprobs}, "
254
+ f"prompt_logprobs={self.prompt_logprobs}, "
255
+ f"skip_special_tokens={self.skip_special_tokens}, "
256
+ "spaces_between_special_tokens="
257
+ f"{self.spaces_between_special_tokens})")