A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 12:30 UTC
[Frontend] Add `defer_loading` and `tool_reference` support for Anthropic and OpenAI APIs (#40190)
Signed-off-by: JaredforReal <w13431838023@gmail.com> Signed-off-by: sfeng33 <4florafeng@gmail.com> Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com> Co-authored-by: sfeng33 <4florafeng@gmail.com> Co-authored-by: Chauncey <chaunceyjiang@gmail.com>
Jared Wen committed
Apr 29, 2026 at 19:35 UTC
11b69129e2221b64302fb672552c0bc04dddece5
5 files changed
+97
-6
vllm/entrypoints/anthropic/protocol.py
+4
@@ -39,6 +39,7 @@ class AnthropicContentBlock(BaseModel):
39
"image",
40
"tool_use",
41
"tool_result",
42
+ "tool_reference",
43
"thinking",
44
"redacted_thinking",
45
]
@@ -52,6 +53,8 @@ class AnthropicContentBlock(BaseModel):
53
input: dict[str, Any] | None = None
54
content: str | list[dict[str, Any]] | None = None
55
is_error: bool | None = None
56
+ # For tool_reference content
57
+ tool_name: str | None = None
58
# For thinking content
59
thinking: str | None = None
60
signature: str | None = None
@@ -72,6 +75,7 @@ class AnthropicTool(BaseModel):
75
name: str
76
description: str | None = None
77
input_schema: dict[str, Any]
78
+ defer_loading: bool | None = None
79
80
@field_validator("input_schema")
81
@classmethod
vllm/entrypoints/anthropic/serving.py
+21
@@ -237,6 +237,10 @@ class AnthropicServingMessages(OpenAIServingChat):
237
cls._convert_tool_use_block(block, tool_calls)
238
elif block.type == "tool_result":
239
cls._convert_tool_result_block(block, role, openai_messages, content_parts)
240
+ elif block.type == "tool_reference":
241
+ # Tool references are expanded during tool_result processing
242
+ # when they appear inside tool_result content.
243
+ pass
244
245
@classmethod
246
def _convert_tool_use_block(cls, block, tool_calls: list[dict[str, Any]]) -> None:
@@ -275,6 +279,7 @@ class AnthropicServingMessages(OpenAIServingChat):
279
"""Convert user tool_result with text and image support"""
280
tool_text = ""
281
tool_image_urls: list[str] = []
282
+ tool_reference: list[dict[str, Any]] = []
283
284
if isinstance(block.content, str):
285
tool_text = block.content
@@ -291,6 +296,12 @@ class AnthropicServingMessages(OpenAIServingChat):
296
url = cls._convert_image_source_to_url(source)
297
if url:
298
tool_image_urls.append(url)
299
+ elif item_type == "tool_reference":
300
+ ref_name = item.get("tool_name") or item.get("name")
301
+ if ref_name:
302
+ tool_reference.append(
303
+ {"type": "tool_reference", "name": ref_name}
304
+ )
305
tool_text = "\n".join(text_parts)
306
307
openai_messages.append(
@@ -312,6 +323,15 @@ class AnthropicServingMessages(OpenAIServingChat):
323
}
324
)
325
326
+ if tool_reference:
327
+ openai_messages.append(
328
+ {
329
+ "role": "tool",
330
+ "tool_call_id": block.tool_use_id or "",
331
+ "content": tool_reference, # type: ignore[dict-item]
332
+ }
333
+ )
334
+
335
@classmethod
336
def _build_base_request(
337
cls,
@@ -400,6 +420,7 @@ class AnthropicServingMessages(OpenAIServingChat):
420
"name": tool.name,
421
"description": tool.description,
422
"parameters": tool.input_schema,
423
+ "defer_loading": tool.defer_loading,
424
},
425
}
426
)
vllm/entrypoints/chat_utils.py
+48
-5
@@ -255,6 +255,23 @@ class CustomThinkCompletionContentParam(TypedDict, total=False):
255
"""The thinking type."""
256
257
258
+class CustomChatCompletionContentToolReferenceParam(TypedDict, total=False):
259
+ """A tool reference content param that only accepts a plain tool name.
260
+
261
+ Example:
262
+ {
263
+ "name": "get_weather",
264
+ "type": "tool_reference"
265
+ }
266
+ """
267
+
268
+ name: str
269
+ """The name of the tool being referenced."""
270
+
271
+ type: Literal["tool_reference"]
272
+ """The content type."""
273
+
274
+
275
ChatCompletionContentPartParam: TypeAlias = (
276
OpenAIChatCompletionContentPartParam
277
| ChatCompletionContentPartAudioParam
@@ -267,6 +284,7 @@ ChatCompletionContentPartParam: TypeAlias = (
284
| ChatCompletionContentPartAudioEmbedsParam
285
| CustomChatCompletionContentSimpleAudioParam
286
| CustomChatCompletionContentSimpleVideoParam
287
+ | CustomChatCompletionContentToolReferenceParam
288
| str
289
| CustomThinkCompletionContentParam
290
)
@@ -1281,6 +1299,9 @@ MM_PARSER_MAP: dict[
1299
"input_audio": lambda part: _InputAudioParser(part).get("input_audio", None),
1300
"refusal": lambda part: _RefusalParser(part).get("refusal", None),
1301
"video_url": lambda part: _VideoParser(part).get("video_url", {}).get("url", None),
1302
+ "tool_reference": lambda part: cast(
1303
+ CustomChatCompletionContentToolReferenceParam, part
1304
+ ).get("name", None),
1305
}
1306
1307
@@ -1372,6 +1393,12 @@ def _parse_chat_message_content_mm_part(
1393
# with url as a dict of {"url": url}
1394
video_url = video_url.get("url", None)
1395
return "video_url", video_url
1396
+ if "tool_reference" in part:
1397
+ tool_reference_params = cast(
1398
+ CustomChatCompletionContentToolReferenceParam, part
1399
+ )
1400
+ tool_reference = tool_reference_params.get("name", None)
1401
+ return "tool_reference", tool_reference
1402
# Raise an error if no 'type' or direct URL is found.
1403
raise ValueError("Missing 'type' field in multimodal part.")
1404
@@ -1501,6 +1528,12 @@ def _parse_chat_message_content_part(
1528
str_content = cast(str, content)
1529
mm_parser.parse_video(str_content, uuid)
1530
modality = "video"
1531
+ elif part_type == "tool_reference":
1532
+ # Tool references are not multimodal data — they reference deferred
1533
+ # tools and are passed through as-is for the chat template to expand.
1534
+ if wrap_dicts:
1535
+ return {"type": "tool_reference", "name": cast(str, content)}
1536
+ return cast(str, content)
1537
else:
1538
supported = sorted(MM_PARSER_MAP.keys() | set(PART_TYPES_TO_SKIP_NONE_CONTENT))
1539
raise VLLMValidationError(
@@ -1567,14 +1600,24 @@ def _parse_chat_message_content(
1600
# string. Clients like Claude Code / Cursor send tool results as
1601
# [{"type": "text", "text": "..."}], but most chat templates only
1602
# handle string content for tool messages.
1603
+ # However, tool_reference items must be preserved as structured
1604
+ # dicts for the chat template to expand them.
1605
msg_content = result_msg.get("content")
1606
if isinstance(msg_content, list):
1572
- texts = [
1573
- item.get("text", "")
1607
+ has_non_text = any(
1608
+ isinstance(item, dict) and item.get("type") != "text"
1609
for item in msg_content
1575
- if isinstance(item, dict) and item.get("type") == "text"
1576
- ]
1577
- result_msg["content"] = "\n".join(texts) if texts else ""
1610
+ )
1611
+ if has_non_text:
1612
+ # Keep structured content (e.g., tool_reference)
1613
+ result_msg["content"] = msg_content
1614
+ else:
1615
+ texts = [
1616
+ item.get("text", "")
1617
+ for item in msg_content
1618
+ if isinstance(item, dict) and item.get("type") == "text"
1619
+ ]
1620
+ result_msg["content"] = "\n".join(texts) if texts else ""
1621
1622
if "name" in message and isinstance(message["name"], str):
1623
result_msg["name"] = message["name"]
vllm/entrypoints/openai/chat_completion/protocol.py
+15
-1
@@ -11,7 +11,7 @@ from openai.types.chat.chat_completion_audio import (
11
ChatCompletionAudio as OpenAIChatCompletionAudio,
12
)
13
from openai.types.chat.chat_completion_message import Annotation as OpenAIAnnotation
14
-from pydantic import Field, PrivateAttr, model_validator
14
+from pydantic import Field, PrivateAttr, model_serializer, model_validator
15
16
from vllm.config import ModelConfig
17
from vllm.config.utils import replace
@@ -139,6 +139,20 @@ class ChatCompletionStreamResponse(OpenAIBaseModel):
139
class ChatCompletionToolsParam(OpenAIBaseModel):
140
type: Literal["function"] = "function"
141
function: FunctionDefinition
142
+ defer_loading: bool | None = None
143
+
144
+ @model_validator(mode="after")
145
+ def _propagate_defer_loading(self) -> "ChatCompletionToolsParam":
146
+ if self.defer_loading is not None and self.function.defer_loading is None:
147
+ self.function.defer_loading = self.defer_loading
148
+ return self
149
+
150
+ @model_serializer(mode="wrap")
151
+ def _serialize(self, handler):
152
+ data = handler(self)
153
+ if self.defer_loading is None:
154
+ data.pop("defer_loading", None)
155
+ return data
156
157
158
class ChatCompletionNamedFunction(OpenAIBaseModel):
vllm/entrypoints/openai/engine/protocol.py
+9
@@ -12,6 +12,7 @@ from pydantic import (
12
BaseModel,
13
ConfigDict,
14
Field,
15
+ model_serializer,
16
model_validator,
17
)
18
@@ -166,6 +167,14 @@ class FunctionDefinition(OpenAIBaseModel):
167
name: str
168
description: str | None = None
169
parameters: dict[str, Any] | None = None
170
+ defer_loading: bool | None = None
171
+
172
+ @model_serializer(mode="wrap")
173
+ def _serialize(self, handler):
174
+ data = handler(self)
175
+ if self.defer_loading is None:
176
+ data.pop("defer_loading", None)
177
+ return data
178
179
180
# extra="forbid" is a workaround to have kwargs as a field,