@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0150a10630
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 01:45 UTC

[Frontend] OpenAI API server: Do not add bos token by default when encoding (#4688)

bofeng huang committed May 17, 2024 at 03:47 UTC 0150a1063029f0238c25bc5a2ea0943b9650d522
2 files changed +22 -12
vllm/entrypoints/openai/serving_chat.py
+1 -1
@@ -158,7 +158,7 @@ class OpenAIServingChat(OpenAIServing):
158 try:
159 # Tokenize/detokenize depending on prompt format (string/token list)
160 prompt_ids, prompt_text = self._validate_prompt_and_tokenize(
161 - request, prompt=prompt)
161 + request, prompt=prompt, add_special_tokens=False)
162 sampling_params = request.to_sampling_params()
163 lora_request = self._maybe_get_lora(request)
164 decoding_config = await self.engine.get_decoding_config()
vllm/entrypoints/openai/serving_engine.py
+21 -11
@@ -1,7 +1,7 @@
1 import json
2 from dataclasses import dataclass
3 from http import HTTPStatus
4 -from typing import Dict, List, Optional, Tuple, Union
4 +from typing import Any, Dict, List, Optional, Tuple, Union
5
6 from pydantic import Field
7 from typing_extensions import Annotated
@@ -165,13 +165,14 @@ class OpenAIServing:
165 raise ValueError(f"The model `{request.model}` does not exist.")
166
167 def _validate_prompt_and_tokenize(
168 - self,
169 - request: Union[ChatCompletionRequest, CompletionRequest,
170 - EmbeddingRequest],
171 - prompt: Optional[str] = None,
172 - prompt_ids: Optional[List[int]] = None,
173 - truncate_prompt_tokens: Optional[Annotated[int, Field(ge=1)]] = None
174 - ) -> Tuple[List[int], str]:
168 + self,
169 + request: Union[ChatCompletionRequest, CompletionRequest,
170 + EmbeddingRequest],
171 + prompt: Optional[str] = None,
172 + prompt_ids: Optional[List[int]] = None,
173 + truncate_prompt_tokens: Optional[Annotated[int,
174 + Field(ge=1)]] = None,
175 + add_special_tokens: bool = True) -> Tuple[List[int], str]:
176 if not (prompt or prompt_ids):
177 raise ValueError("Either prompt or prompt_ids should be provided.")
178 if (prompt and prompt_ids):
@@ -179,10 +180,19 @@ class OpenAIServing:
180 "Only one of prompt or prompt_ids should be provided.")
181
182 if prompt_ids is None:
182 - tokenizer_kwargs = {} if truncate_prompt_tokens is None else {
183 - "truncation": True,
184 - "max_length": truncate_prompt_tokens,
183 + # When using OpenAIServingChat for chat completions, the
184 + # special tokens (e.g., BOS) have already been added by the
185 + # chat template. Therefore, we do not need to add them again.
186 + # Set add_special_tokens to False to avoid adding the BOS tokens
187 + # again.
188 + tokenizer_kwargs: Dict[str, Any] = {
189 + "add_special_tokens": add_special_tokens
190 }
191 + if truncate_prompt_tokens is not None:
192 + tokenizer_kwargs.update({
193 + "truncation": True,
194 + "max_length": truncate_prompt_tokens,
195 + })
196 input_ids = self.tokenizer(prompt, **tokenizer_kwargs).input_ids
197 elif truncate_prompt_tokens is not None:
198 input_ids = prompt_ids[-truncate_prompt_tokens:]