A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 00:30 UTC
[Frontend] OpenAI API server: Add `add_special_tokens` to ChatCompletionRequest (default False) (#5278)
tomeras91 committed
Jun 5, 2024 at 19:32 UTC
f0a500545f97d026c3873e8dc0043e06e42ae61c
3 files changed
+20
-7
vllm/entrypoints/openai/protocol.py
+9
@@ -176,6 +176,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
176
"This is a parameter used by chat template in tokenizer config of the "
177
"model."),
178
)
179
+ add_special_tokens: Optional[bool] = Field(
180
+ default=False,
181
+ description=(
182
+ "If true, special tokens (e.g. BOS) will be added to the prompt "
183
+ "on top of what is added by the chat template. "
184
+ "For most models, the chat template takes care of adding the "
185
+ "special tokens so this should be set to False (as is the "
186
+ "default)."),
187
+ )
188
include_stop_str_in_output: Optional[bool] = Field(
189
default=False,
190
description=(
vllm/entrypoints/openai/serving_chat.py
+3
-1
@@ -163,7 +163,9 @@ class OpenAIServingChat(OpenAIServing):
163
try:
164
# Tokenize/detokenize depending on prompt format (string/token list)
165
prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
166
- request, prompt=prompt, add_special_tokens=False)
166
+ request,
167
+ prompt=prompt,
168
+ add_special_tokens=request.add_special_tokens)
169
sampling_params = request.to_sampling_params()
170
lora_request = self._maybe_get_lora(request)
171
decoding_config = await self.engine.get_decoding_config()
vllm/entrypoints/openai/serving_engine.py
+8
-6
@@ -131,7 +131,8 @@ class OpenAIServing:
131
prompt_ids: Optional[List[int]] = None,
132
truncate_prompt_tokens: Optional[Annotated[int,
133
Field(ge=1)]] = None,
134
- add_special_tokens: bool = True) -> Tuple[List[int], str]:
134
+ add_special_tokens: Optional[bool] = True
135
+ ) -> Tuple[List[int], str]:
136
if not (prompt or prompt_ids):
137
raise ValueError("Either prompt or prompt_ids should be provided.")
138
if (prompt and prompt_ids):
@@ -139,11 +140,12 @@ class OpenAIServing:
140
"Only one of prompt or prompt_ids should be provided.")
141
142
if prompt_ids is None:
142
- # When using OpenAIServingChat for chat completions, the
143
- # special tokens (e.g., BOS) have already been added by the
144
- # chat template. Therefore, we do not need to add them again.
145
- # Set add_special_tokens to False to avoid adding the BOS tokens
146
- # again.
143
+ # When using OpenAIServingChat for chat completions, for
144
+ # most models the special tokens (e.g., BOS) have already
145
+ # been added by the chat template. Therefore, we do not
146
+ # need to add them again.
147
+ # Set add_special_tokens to False (by default) to avoid
148
+ # adding the BOS tokens again.
149
tokenizer_kwargs: Dict[str, Any] = {
150
"add_special_tokens": add_special_tokens
151
}