@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f0a500545f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 00:30 UTC

[Frontend] OpenAI API server: Add `add_special_tokens` to ChatCompletionRequest (default False) (#5278)

tomeras91 committed Jun 5, 2024 at 19:32 UTC f0a500545f97d026c3873e8dc0043e06e42ae61c
3 files changed +20 -7
vllm/entrypoints/openai/protocol.py
+9
@@ -176,6 +176,15 @@ class ChatCompletionRequest(OpenAIBaseModel):
176 "This is a parameter used by chat template in tokenizer config of the "
177 "model."),
178 )
179 + add_special_tokens: Optional[bool] = Field(
180 + default=False,
181 + description=(
182 + "If true, special tokens (e.g. BOS) will be added to the prompt "
183 + "on top of what is added by the chat template. "
184 + "For most models, the chat template takes care of adding the "
185 + "special tokens so this should be set to False (as is the "
186 + "default)."),
187 + )
188 include_stop_str_in_output: Optional[bool] = Field(
189 default=False,
190 description=(
vllm/entrypoints/openai/serving_chat.py
+3 -1
@@ -163,7 +163,9 @@ class OpenAIServingChat(OpenAIServing):
163 try:
164 # Tokenize/detokenize depending on prompt format (string/token list)
165 prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
166 - request, prompt=prompt, add_special_tokens=False)
166 + request,
167 + prompt=prompt,
168 + add_special_tokens=request.add_special_tokens)
169 sampling_params = request.to_sampling_params()
170 lora_request = self._maybe_get_lora(request)
171 decoding_config = await self.engine.get_decoding_config()
vllm/entrypoints/openai/serving_engine.py
+8 -6
@@ -131,7 +131,8 @@ class OpenAIServing:
131 prompt_ids: Optional[List[int]] = None,
132 truncate_prompt_tokens: Optional[Annotated[int,
133 Field(ge=1)]] = None,
134 - add_special_tokens: bool = True) -> Tuple[List[int], str]:
134 + add_special_tokens: Optional[bool] = True
135 + ) -> Tuple[List[int], str]:
136 if not (prompt or prompt_ids):
137 raise ValueError("Either prompt or prompt_ids should be provided.")
138 if (prompt and prompt_ids):
@@ -139,11 +140,12 @@ class OpenAIServing:
140 "Only one of prompt or prompt_ids should be provided.")
141
142 if prompt_ids is None:
142 - # When using OpenAIServingChat for chat completions, the
143 - # special tokens (e.g., BOS) have already been added by the
144 - # chat template. Therefore, we do not need to add them again.
145 - # Set add_special_tokens to False to avoid adding the BOS tokens
146 - # again.
143 + # When using OpenAIServingChat for chat completions, for
144 + # most models the special tokens (e.g., BOS) have already
145 + # been added by the chat template. Therefore, we do not
146 + # need to add them again.
147 + # Set add_special_tokens to False (by default) to avoid
148 + # adding the BOS tokens again.
149 tokenizer_kwargs: Dict[str, Any] = {
150 "add_special_tokens": add_special_tokens
151 }