A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 09:45 UTC
add tools into TokenizeChatRequest (#18187)
Signed-off-by: yangxia <yangxiast@gmail.com>
hustxiayang committed
May 15, 2025 at 07:01 UTC
451da4bcbdc2dcabf3e319b4a82b72674c33f4de
3 files changed
+84
tests/entrypoints/openai/test_tokenization.py
+77
@@ -145,6 +145,83 @@ async def test_tokenize_chat(
145
}
146
147
148
+@pytest.mark.asyncio
149
+@pytest.mark.parametrize(
150
+ "model_name,tokenizer_name",
151
+ [(MODEL_NAME, MODEL_NAME), ("zephyr-lora2", "zephyr-lora2")],
152
+ indirect=["tokenizer_name"],
153
+)
154
+async def test_tokenize_chat_with_tools(
155
+ server: RemoteOpenAIServer,
156
+ model_name: str,
157
+ tokenizer_name: str,
158
+):
159
+ tokenizer = get_tokenizer(tokenizer_name=tokenizer_name,
160
+ tokenizer_mode="fast")
161
+
162
+ for add_generation in [False, True]:
163
+ for add_special in [False, True]:
164
+ conversation = [{
165
+ "role":
166
+ "user",
167
+ "content":
168
+ "What's the weather like in Paris today?",
169
+ }]
170
+
171
+ tools = [{
172
+ "type": "function",
173
+ "function": {
174
+ "name": "get_weather",
175
+ "parameters": {
176
+ "type": "object",
177
+ "properties": {
178
+ "location": {
179
+ "type": "string"
180
+ }
181
+ },
182
+ },
183
+ },
184
+ }]
185
+
186
+ for continue_final in [False, True]:
187
+ if add_generation and continue_final:
188
+ continue
189
+ if continue_final:
190
+ conversation.append({
191
+ "role": "assistant",
192
+ "content": "Sure,"
193
+ })
194
+
195
+ prompt = tokenizer.apply_chat_template(
196
+ add_generation_prompt=add_generation,
197
+ continue_final_message=continue_final,
198
+ conversation=conversation,
199
+ tools=tools,
200
+ tokenize=False,
201
+ )
202
+ tokens = tokenizer.encode(prompt,
203
+ add_special_tokens=add_special)
204
+
205
+ response = requests.post(
206
+ server.url_for("tokenize"),
207
+ json={
208
+ "add_generation_prompt": add_generation,
209
+ "continue_final_message": continue_final,
210
+ "add_special_tokens": add_special,
211
+ "messages": conversation,
212
+ "model": model_name,
213
+ "tools": tools,
214
+ },
215
+ )
216
+ response.raise_for_status()
217
+
218
+ assert response.json() == {
219
+ "tokens": tokens,
220
+ "count": len(tokens),
221
+ "max_model_len": 8192,
222
+ }
223
+
224
+
225
@pytest.mark.asyncio
226
@pytest.mark.parametrize(
227
"model_name,tokenizer_name",
vllm/entrypoints/openai/protocol.py
+4
@@ -1593,6 +1593,10 @@ class TokenizeChatRequest(OpenAIBaseModel):
1593
default=None,
1594
description=("Additional kwargs to pass to the HF processor."),
1595
)
1596
+ tools: Optional[list[ChatCompletionToolsParam]] = Field(
1597
+ default=None,
1598
+ description=("A list of tools the model may call."),
1599
+ )
1600
1601
@model_validator(mode="before")
1602
@classmethod
vllm/entrypoints/openai/serving_tokenization.py
+3
@@ -65,6 +65,8 @@ class OpenAIServingTokenization(OpenAIServing):
65
tokenizer = await self.engine_client.get_tokenizer(lora_request)
66
67
if isinstance(request, TokenizeChatRequest):
68
+ tool_dicts = (None if request.tools is None else
69
+ [tool.model_dump() for tool in request.tools])
70
(
71
_,
72
request_prompts,
@@ -73,6 +75,7 @@ class OpenAIServingTokenization(OpenAIServing):
75
request,
76
tokenizer,
77
request.messages,
78
+ tool_dicts=tool_dicts,
79
chat_template=request.chat_template or self.chat_template,
80
chat_template_content_format=self.
81
chat_template_content_format,