@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f98b745a81
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

feat: support stop_token_ids parameter. (#1097)

Ricardo Lu committed Sep 22, 2023 at 06:34 UTC f98b745a81df1613a9c5f1d5986456663f86c457
4 files changed +15
vllm/engine/llm_engine.py
+3
@@ -650,6 +650,9 @@ class LLMEngine:
650 seq.output_text = seq.output_text[:-len(stop_str)]
651 seq.status = SequenceStatus.FINISHED_STOPPED
652 return
653 + if seq.get_last_token_id() in sampling_params.stop_token_ids:
654 + seq.status = SequenceStatus.FINISHED_STOPPED
655 + return
656
657 # Check if the sequence has reached max_model_len.
658 if seq.get_len() > self.scheduler_config.max_model_len:
vllm/entrypoints/openai/api_server.py
+2
@@ -217,6 +217,7 @@ async def create_chat_completion(request: ChatCompletionRequest,
217 temperature=request.temperature,
218 top_p=request.top_p,
219 stop=request.stop,
220 + stop_token_ids=request.stop_token_ids,
221 max_tokens=request.max_tokens,
222 best_of=request.best_of,
223 top_k=request.top_k,
@@ -418,6 +419,7 @@ async def create_completion(request: CompletionRequest, raw_request: Request):
419 top_p=request.top_p,
420 top_k=request.top_k,
421 stop=request.stop,
422 + stop_token_ids=request.stop_token_ids,
423 ignore_eos=request.ignore_eos,
424 max_tokens=request.max_tokens,
425 logprobs=request.logprobs,
vllm/entrypoints/openai/protocol.py
+2
@@ -70,6 +70,7 @@ class ChatCompletionRequest(BaseModel):
70 top_k: Optional[int] = -1
71 ignore_eos: Optional[bool] = False
72 use_beam_search: Optional[bool] = False
73 + stop_token_ids: Optional[List[int]] = Field(default_factory=list)
74
75
76 class CompletionRequest(BaseModel):
@@ -94,6 +95,7 @@ class CompletionRequest(BaseModel):
95 top_k: Optional[int] = -1
96 ignore_eos: Optional[bool] = False
97 use_beam_search: Optional[bool] = False
98 + stop_token_ids: Optional[List[int]] = Field(default_factory=list)
99
100
101 class LogProbs(BaseModel):
vllm/sampling_params.py
+8
@@ -45,6 +45,9 @@ class SamplingParams:
45 (canonical beam search algorithm).
46 stop: List of strings that stop the generation when they are generated.
47 The returned output will not contain the stop strings.
48 + stop_token_ids: List of tokens that stop the generation when they are
49 + generated. The returned output will contain the stop tokens unless
50 + the stop tokens are sepcial tokens.
51 ignore_eos: Whether to ignore the EOS token and continue generating
52 tokens after the EOS token is generated.
53 max_tokens: Maximum number of tokens to generate per output sequence.
@@ -64,6 +67,7 @@ class SamplingParams:
67 length_penalty: float = 1.0,
68 early_stopping: Union[bool, str] = False,
69 stop: Union[None, str, List[str]] = None,
70 + stop_token_ids: List[int] = None,
71 ignore_eos: bool = False,
72 max_tokens: int = 16,
73 logprobs: Optional[int] = None,
@@ -84,6 +88,10 @@ class SamplingParams:
88 self.stop = [stop]
89 else:
90 self.stop = list(stop)
91 + if stop_token_ids is None:
92 + self.stop_token_ids = []
93 + else:
94 + self.stop_token_ids = list(stop_token_ids)
95 self.ignore_eos = ignore_eos
96 self.max_tokens = max_tokens
97 self.logprobs = logprobs