A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
feat: support stop_token_ids parameter. (#1097)
Ricardo Lu committed
Sep 22, 2023 at 06:34 UTC
f98b745a81df1613a9c5f1d5986456663f86c457
4 files changed
+15
vllm/engine/llm_engine.py
+3
@@ -650,6 +650,9 @@ class LLMEngine:
650
seq.output_text = seq.output_text[:-len(stop_str)]
651
seq.status = SequenceStatus.FINISHED_STOPPED
652
return
653
+ if seq.get_last_token_id() in sampling_params.stop_token_ids:
654
+ seq.status = SequenceStatus.FINISHED_STOPPED
655
+ return
656
657
# Check if the sequence has reached max_model_len.
658
if seq.get_len() > self.scheduler_config.max_model_len:
vllm/entrypoints/openai/api_server.py
+2
@@ -217,6 +217,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
217
temperature=request.temperature,
218
top_p=request.top_p,
219
stop=request.stop,
220
+ stop_token_ids=request.stop_token_ids,
221
max_tokens=request.max_tokens,
222
best_of=request.best_of,
223
top_k=request.top_k,
@@ -418,6 +419,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
419
top_p=request.top_p,
420
top_k=request.top_k,
421
stop=request.stop,
422
+ stop_token_ids=request.stop_token_ids,
423
ignore_eos=request.ignore_eos,
424
max_tokens=request.max_tokens,
425
logprobs=request.logprobs,
vllm/entrypoints/openai/protocol.py
+2
@@ -70,6 +70,7 @@ class ChatCompletionRequest(BaseModel):
70
top_k: Optional[int] = -1
71
ignore_eos: Optional[bool] = False
72
use_beam_search: Optional[bool] = False
73
+ stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74
75
76
class CompletionRequest(BaseModel):
@@ -94,6 +95,7 @@ class CompletionRequest(BaseModel):
95
top_k: Optional[int] = -1
96
ignore_eos: Optional[bool] = False
97
use_beam_search: Optional[bool] = False
98
+ stop_token_ids: Optional[List[int]] = Field(default_factory=list)
99
100
101
class LogProbs(BaseModel):
vllm/sampling_params.py
+8
@@ -45,6 +45,9 @@ class SamplingParams:
45
(canonical beam search algorithm).
46
stop: List of strings that stop the generation when they are generated.
47
The returned output will not contain the stop strings.
48
+ stop_token_ids: List of tokens that stop the generation when they are
49
+ generated. The returned output will contain the stop tokens unless
50
+ the stop tokens are sepcial tokens.
51
ignore_eos: Whether to ignore the EOS token and continue generating
52
tokens after the EOS token is generated.
53
max_tokens: Maximum number of tokens to generate per output sequence.
@@ -64,6 +67,7 @@ class SamplingParams:
67
length_penalty: float = 1.0,
68
early_stopping: Union[bool, str] = False,
69
stop: Union[None, str, List[str]] = None,
70
+ stop_token_ids: List[int] = None,
71
ignore_eos: bool = False,
72
max_tokens: int = 16,
73
logprobs: Optional[int] = None,
@@ -84,6 +88,10 @@ class SamplingParams:
88
self.stop = [stop]
89
else:
90
self.stop = list(stop)
91
+ if stop_token_ids is None:
92
+ self.stop_token_ids = []
93
+ else:
94
+ self.stop_token_ids = list(stop_token_ids)
95
self.ignore_eos = ignore_eos
96
self.max_tokens = max_tokens
97
self.logprobs = logprobs