A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 09:30 UTC
[P/D] AnthropicMessages add kv_transfer_params for PD disaggregation (#37535)
Signed-off-by: chaunceyjiang <chaunceyjiang@gmail.com>
Chauncey committed
Mar 20, 2026 at 00:41 UTC
2f9f946b22cc39506ae9c0e8e3730376e652a87d
2 files changed
+14
-1
vllm/entrypoints/anthropic/protocol.py
+12
-1
@@ -5,7 +5,7 @@
5
import time
6
from typing import Any, Literal
7
8
-from pydantic import BaseModel, field_validator, model_validator
8
+from pydantic import BaseModel, Field, field_validator, model_validator
9
10
11
class AnthropicError(BaseModel):
@@ -112,6 +112,12 @@ class AnthropicMessagesRequest(BaseModel):
112
top_k: int | None = None
113
top_p: float | None = None
114
115
+ # vLLM-specific fields that are not in Anthropic spec
116
+ kv_transfer_params: dict[str, Any] | None = Field(
117
+ default=None,
118
+ description="KVTransfer parameters used for disaggregated serving.",
119
+ )
120
+
121
@field_validator("model")
122
@classmethod
123
def validate_model(cls, v):
@@ -181,6 +187,11 @@ class AnthropicMessagesResponse(BaseModel):
187
stop_sequence: str | None = None
188
usage: AnthropicUsage | None = None
189
190
+ # vLLM-specific fields that are not in Anthropic spec
191
+ kv_transfer_params: dict[str, Any] | None = Field(
192
+ default=None, description="KVTransfer parameters."
193
+ )
194
+
195
def model_post_init(self, __context):
196
if not self.id:
197
self.id = f"msg_{int(time.time() * 1000)}"
vllm/entrypoints/anthropic/serving.py
+2
@@ -331,6 +331,7 @@ class AnthropicServingMessages(OpenAIServingChat):
331
temperature=anthropic_request.temperature,
332
top_p=anthropic_request.top_p,
333
top_k=anthropic_request.top_k,
334
+ kv_transfer_params=anthropic_request.kv_transfer_params,
335
)
336
337
@classmethod
@@ -441,6 +442,7 @@ class AnthropicServingMessages(OpenAIServingChat):
442
input_tokens=generator.usage.prompt_tokens,
443
output_tokens=generator.usage.completion_tokens,
444
),
445
+ kv_transfer_params=generator.kv_transfer_params,
446
)
447
choice = generator.choices[0]
448
if choice.finish_reason == "stop":