A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 01:45 UTC
[Frontend] OpenAI API server: Do not add bos token by default when encoding (#4688)
bofeng huang committed
May 17, 2024 at 03:47 UTC
0150a1063029f0238c25bc5a2ea0943b9650d522
2 files changed
+22
-12
vllm/entrypoints/openai/serving_chat.py
+1
-1
@@ -158,7 +158,7 @@ class OpenAIServingChat(OpenAIServing):
158
try:
159
# Tokenize/detokenize depending on prompt format (string/token list)
160
prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
161
- request, prompt=prompt)
161
+ request, prompt=prompt, add_special_tokens=False)
162
sampling_params = request.to_sampling_params()
163
lora_request = self._maybe_get_lora(request)
164
decoding_config = await self.engine.get_decoding_config()
vllm/entrypoints/openai/serving_engine.py
+21
-11
@@ -1,7 +1,7 @@
1
import json
2
from dataclasses import dataclass
3
from http import HTTPStatus
4
-from typing import Dict, List, Optional, Tuple, Union
4
+from typing import Any, Dict, List, Optional, Tuple, Union
5
6
from pydantic import Field
7
from typing_extensions import Annotated
@@ -165,13 +165,14 @@ class OpenAIServing:
165
raise ValueError(f"The model `{request.model}` does not exist.")
166
167
def _validate_prompt_and_tokenize(
168
- self,
169
- request: Union[ChatCompletionRequest, CompletionRequest,
170
- EmbeddingRequest],
171
- prompt: Optional[str] = None,
172
- prompt_ids: Optional[List[int]] = None,
173
- truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
174
- ) -> Tuple[List[int], str]:
168
+ self,
169
+ request: Union[ChatCompletionRequest, CompletionRequest,
170
+ EmbeddingRequest],
171
+ prompt: Optional[str] = None,
172
+ prompt_ids: Optional[List[int]] = None,
173
+ truncate_prompt_tokens: Optional[Annotated[int,
174
+ Field(ge=1)]] = None,
175
+ add_special_tokens: bool = True) -> Tuple[List[int], str]:
176
if not (prompt or prompt_ids):
177
raise ValueError("Either prompt or prompt_ids should be provided.")
178
if (prompt and prompt_ids):
@@ -179,10 +180,19 @@ class OpenAIServing:
180
"Only one of prompt or prompt_ids should be provided.")
181
182
if prompt_ids is None:
182
- tokenizer_kwargs = {} if truncate_prompt_tokens is None else {
183
- "truncation": True,
184
- "max_length": truncate_prompt_tokens,
183
+ # When using OpenAIServingChat for chat completions, the
184
+ # special tokens (e.g., BOS) have already been added by the
185
+ # chat template. Therefore, we do not need to add them again.
186
+ # Set add_special_tokens to False to avoid adding the BOS tokens
187
+ # again.
188
+ tokenizer_kwargs: Dict[str, Any] = {
189
+ "add_special_tokens": add_special_tokens
190
}
191
+ if truncate_prompt_tokens is not None:
192
+ tokenizer_kwargs.update({
193
+ "truncation": True,
194
+ "max_length": truncate_prompt_tokens,
195
+ })
196
input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
197
elif truncate_prompt_tokens is not None:
198
input_ids = prompt_ids[-truncate_prompt_tokens:]