A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Frontend] Add template related params to request (#5709)
danieljannai21 committed
Jul 2, 2024 at 09:01 UTC
2c37540aa6af89f0ece874d831dff3bf62edf486
3 files changed
+30
-1
requirements-common.txt
+1
-1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6
requests
7
tqdm
8
py-cpuinfo
9
-transformers >= 4.42.0 # Required for Gemma 2.
9
+transformers >= 4.42.0 # Required for Gemma 2 and for additional chat template parameters.
10
tokenizers >= 0.19.1 # Required for Llama 3.
11
fastapi
12
aiohttp
vllm/entrypoints/openai/protocol.py
+21
@@ -190,6 +190,27 @@ class ChatCompletionRequest(OpenAIBaseModel):
190
"special tokens so this should be set to False (as is the "
191
"default)."),
192
)
193
+ documents: Optional[List[Dict[str, str]]] = Field(
194
+ default=None,
195
+ description=
196
+ ("A list of dicts representing documents that will be accessible to "
197
+ "the model if it is performing RAG (retrieval-augmented generation)."
198
+ " If the template does not support RAG, this argument will have no "
199
+ "effect. We recommend that each document should be a dict containing "
200
+ "\"title\" and \"text\" keys."),
201
+ )
202
+ chat_template: Optional[str] = Field(
203
+ default=None,
204
+ description=(
205
+ "A Jinja template to use for this conversion. "
206
+ "If this is not passed, the model's default chat template will be "
207
+ "used instead."),
208
+ )
209
+ chat_template_kwargs: Optional[Dict[str, Any]] = Field(
210
+ default=None,
211
+ description=("Additional kwargs to pass to the template renderer. "
212
+ "Will be accessible by the chat template."),
213
+ )
214
include_stop_str_in_output: Optional[bool] = Field(
215
default=False,
216
description=(
vllm/entrypoints/openai/serving_chat.py
+8
@@ -218,10 +218,18 @@ class OpenAIServingChat(OpenAIServing):
218
conversation.extend(chat_parsed_result.messages)
219
image_futures.extend(chat_parsed_result.image_futures)
220
221
+ tool_dicts = None if request.tools is None else [
222
+ tool.model_dump() for tool in request.tools
223
+ ]
224
+
225
prompt = self.tokenizer.apply_chat_template(
226
conversation=conversation,
227
tokenize=False,
228
add_generation_prompt=request.add_generation_prompt,
229
+ tools=tool_dicts,
230
+ documents=request.documents,
231
+ chat_template=request.chat_template,
232
+ **(request.chat_template_kwargs or {}),
233
)
234
except Exception as e:
235
logger.error("Error in applying chat template from request: %s", e)