Forward Anthropic vllm_xargs to sampling params (#53308)
Signed-off-by: Maroon Ayoub <mayoub@redhat.com>
Maroon Ayoub committed
Aug 22, 2026 at 01:23 UTC
a556f3fccb701e5618d84d547ff454c56a1bfdfb
4 files changed
+59
-1
docs/features/custom_arguments.md
+1
-1
@@ -19,7 +19,7 @@ This allows arguments which are not already part of `SamplingParams` to be passe
19
20
## Online Custom Arguments
21
22
-The vLLM REST API allows custom arguments to be passed to the vLLM server via `vllm_xargs`. The example below integrates custom arguments into a vLLM REST API request:
22
+The OpenAI-compatible REST API and Anthropic-compatible `/v1/messages` endpoint allow custom arguments to be passed to the vLLM server via `vllm_xargs`. The example below integrates custom arguments into a vLLM REST API request:
23
24
``` bash
25
curl http://localhost:8000/v1/completions \
tests/entrypoints/anthropic/test_anthropic_messages_conversion.py
+50
@@ -182,6 +182,56 @@ class TestImageContentBlocks:
182
}
183
184
185
+# ======================================================================
186
+# vllm_xargs pass-through
187
+# ======================================================================
188
+
189
+
190
+class TestVllmXargs:
191
+ def test_vllm_xargs_passed_through(self):
192
+ request = _make_request(
193
+ [{"role": "user", "content": "Hello"}],
194
+ vllm_xargs={
195
+ "kv_cache_report_mode": "full",
196
+ "existing_extension": 7,
197
+ },
198
+ )
199
+
200
+ result = _convert(request)
201
+
202
+ assert result.vllm_xargs == {
203
+ "kv_cache_report_mode": "full",
204
+ "existing_extension": 7,
205
+ }
206
+
207
+ def test_vllm_xargs_reaches_sampling_params_with_kv_transfer(self):
208
+ kv_transfer_params = {
209
+ "do_remote_decode": True,
210
+ "do_remote_prefill": False,
211
+ }
212
+ request = _make_request(
213
+ [{"role": "user", "content": "Hello"}],
214
+ vllm_xargs={
215
+ "kv_cache_report_mode": "full",
216
+ "existing_extension": "kept",
217
+ },
218
+ kv_transfer_params=kv_transfer_params,
219
+ )
220
+
221
+ converted = _convert(request)
222
+ sampling_params = converted.to_sampling_params(
223
+ max_tokens=converted.max_completion_tokens or 0,
224
+ default_sampling_params={},
225
+ )
226
+
227
+ assert converted.kv_transfer_params == kv_transfer_params
228
+ assert sampling_params.extra_args == {
229
+ "kv_cache_report_mode": "full",
230
+ "existing_extension": "kept",
231
+ "kv_transfer_params": kv_transfer_params,
232
+ }
233
+
234
+
235
# ======================================================================
236
# tool_result content handling
237
# ======================================================================
vllm/entrypoints/anthropic/protocol.py
+7
@@ -160,6 +160,13 @@ class AnthropicMessagesRequest(BaseModel):
160
"ECTransfer parameters used for encoder-cache disaggregated serving."
161
),
162
)
163
+ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
164
+ default=None,
165
+ description=(
166
+ "Additional request parameters with (list of) string or "
167
+ "numeric values, used by custom extensions."
168
+ ),
169
+ )
170
chat_template_kwargs: dict[str, Any] | None = Field(
171
default=None,
172
description=(
vllm/entrypoints/anthropic/serving.py
+1
@@ -489,6 +489,7 @@ class AnthropicServingMessages(OpenAIServingChat):
489
cache_salt=anthropic_request.cache_salt,
490
kv_transfer_params=anthropic_request.kv_transfer_params,
491
ec_transfer_params=anthropic_request.ec_transfer_params,
492
+ vllm_xargs=anthropic_request.vllm_xargs,
493
chat_template_kwargs=anthropic_request.chat_template_kwargs,
494
)
495