@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c37540aa6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Frontend] Add template related params to request (#5709)

danieljannai21 committed Jul 2, 2024 at 09:01 UTC 2c37540aa6af89f0ece874d831dff3bf62edf486
3 files changed +30 -1
requirements-common.txt
+1 -1
@@ -6,7 +6,7 @@ numpy < 2.0.0
6 requests
7 tqdm
8 py-cpuinfo
9 -transformers >= 4.42.0 # Required for Gemma 2.
9 +transformers >= 4.42.0 # Required for Gemma 2 and for additional chat template parameters.
10 tokenizers >= 0.19.1 # Required for Llama 3.
11 fastapi
12 aiohttp
vllm/entrypoints/openai/protocol.py
+21
@@ -190,6 +190,27 @@ class ChatCompletionRequest(OpenAIBaseModel):
190 "special tokens so this should be set to False (as is the "
191 "default)."),
192 )
193 + documents: Optional[List[Dict[str, str]]] = Field(
194 + default=None,
195 + description=
196 + ("A list of dicts representing documents that will be accessible to "
197 + "the model if it is performing RAG (retrieval-augmented generation)."
198 + " If the template does not support RAG, this argument will have no "
199 + "effect. We recommend that each document should be a dict containing "
200 + "\"title\" and \"text\" keys."),
201 + )
202 + chat_template: Optional[str] = Field(
203 + default=None,
204 + description=(
205 + "A Jinja template to use for this conversion. "
206 + "If this is not passed, the model's default chat template will be "
207 + "used instead."),
208 + )
209 + chat_template_kwargs: Optional[Dict[str, Any]] = Field(
210 + default=None,
211 + description=("Additional kwargs to pass to the template renderer. "
212 + "Will be accessible by the chat template."),
213 + )
214 include_stop_str_in_output: Optional[bool] = Field(
215 default=False,
216 description=(
vllm/entrypoints/openai/serving_chat.py
+8
@@ -218,10 +218,18 @@ class OpenAIServingChat(OpenAIServing):
218 conversation.extend(chat_parsed_result.messages)
219 image_futures.extend(chat_parsed_result.image_futures)
220
221 + tool_dicts = None if request.tools is None else [
222 + tool.model_dump() for tool in request.tools
223 + ]
224 +
225 prompt = self.tokenizer.apply_chat_template(
226 conversation=conversation,
227 tokenize=False,
228 add_generation_prompt=request.add_generation_prompt,
229 + tools=tool_dicts,
230 + documents=request.documents,
231 + chat_template=request.chat_template,
232 + **(request.chat_template_kwargs or {}),
233 )
234 except Exception as e:
235 logger.error("Error in applying chat template from request: %s", e)