A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 9, 01:15 UTC
[Frontend] Add sampling parameters to Responses API (#32609)
Signed-off-by: Daniel Mescheder <dmesch@amazon.com> Co-authored-by: Daniel Mescheder <dmesch@amazon.com>
Daniel Mescheder committed
Feb 3, 2026 at 06:51 UTC
4c4b6f7a9764bac8bf9f2a0bfedf852d8e59c98e
3 files changed
+165
-2
tests/entrypoints/openai/responses/test_sampling_params.py
new
+113
@@ -0,0 +1,113 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+"""Unit tests for ResponsesRequest.to_sampling_params() parameter mapping."""
5
+
6
+import pytest
7
+
8
+from vllm.entrypoints.openai.responses.protocol import ResponsesRequest
9
+
10
+
11
+class TestResponsesRequestSamplingParams:
12
+ """Test that ResponsesRequest correctly maps parameters to SamplingParams."""
13
+
14
+ def test_basic_sampling_params(self):
15
+ """Test basic sampling parameters are correctly mapped."""
16
+ request = ResponsesRequest(
17
+ model="test-model",
18
+ input="test input",
19
+ temperature=0.8,
20
+ top_p=0.95,
21
+ top_k=50,
22
+ max_output_tokens=100,
23
+ )
24
+
25
+ sampling_params = request.to_sampling_params(default_max_tokens=1000)
26
+
27
+ assert sampling_params.temperature == 0.8
28
+ assert sampling_params.top_p == 0.95
29
+ assert sampling_params.top_k == 50
30
+ assert sampling_params.max_tokens == 100
31
+
32
+ def test_extra_sampling_params(self):
33
+ """Test extra sampling parameters are correctly mapped."""
34
+ request = ResponsesRequest(
35
+ model="test-model",
36
+ input="test input",
37
+ repetition_penalty=1.2,
38
+ seed=42,
39
+ stop=["END", "STOP"],
40
+ ignore_eos=True,
41
+ vllm_xargs={"custom": "value"},
42
+ )
43
+
44
+ sampling_params = request.to_sampling_params(default_max_tokens=1000)
45
+
46
+ assert sampling_params.repetition_penalty == 1.2
47
+ assert sampling_params.seed == 42
48
+ assert sampling_params.stop == ["END", "STOP"]
49
+ assert sampling_params.ignore_eos is True
50
+ assert sampling_params.extra_args == {"custom": "value"}
51
+
52
+ def test_stop_string_conversion(self):
53
+ """Test that single stop string is converted to list."""
54
+ request = ResponsesRequest(
55
+ model="test-model",
56
+ input="test input",
57
+ stop="STOP",
58
+ )
59
+
60
+ sampling_params = request.to_sampling_params(default_max_tokens=1000)
61
+
62
+ assert sampling_params.stop == ["STOP"]
63
+
64
+ def test_default_values(self):
65
+ """Test default values for optional parameters."""
66
+ request = ResponsesRequest(
67
+ model="test-model",
68
+ input="test input",
69
+ )
70
+
71
+ sampling_params = request.to_sampling_params(default_max_tokens=1000)
72
+
73
+ assert sampling_params.repetition_penalty == 1.0 # None → 1.0
74
+ assert sampling_params.stop == [] # Empty list
75
+ assert sampling_params.extra_args == {} # Empty dict
76
+
77
+ def test_seed_bounds_validation(self):
78
+ """Test that seed values outside torch.long bounds are rejected."""
79
+ import torch
80
+ from pydantic import ValidationError
81
+
82
+ # Test seed below minimum
83
+ with pytest.raises(ValidationError) as exc_info:
84
+ ResponsesRequest(
85
+ model="test-model",
86
+ input="test input",
87
+ seed=torch.iinfo(torch.long).min - 1,
88
+ )
89
+ assert "greater_than_equal" in str(exc_info.value).lower()
90
+
91
+ # Test seed above maximum
92
+ with pytest.raises(ValidationError) as exc_info:
93
+ ResponsesRequest(
94
+ model="test-model",
95
+ input="test input",
96
+ seed=torch.iinfo(torch.long).max + 1,
97
+ )
98
+ assert "less_than_equal" in str(exc_info.value).lower()
99
+
100
+ # Test valid seed at boundaries
101
+ request_min = ResponsesRequest(
102
+ model="test-model",
103
+ input="test input",
104
+ seed=torch.iinfo(torch.long).min,
105
+ )
106
+ assert request_min.seed == torch.iinfo(torch.long).min
107
+
108
+ request_max = ResponsesRequest(
109
+ model="test-model",
110
+ input="test input",
111
+ seed=torch.iinfo(torch.long).max,
112
+ )
113
+ assert request_max.seed == torch.iinfo(torch.long).max
tests/entrypoints/openai/responses/test_simple.py
+24
-1
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-
4
import pytest
5
import pytest_asyncio
6
from openai import OpenAI
@@ -147,3 +146,27 @@ async def test_max_tokens(client: OpenAI, model_name: str):
146
assert response is not None
147
assert response.status == "incomplete"
148
assert response.incomplete_details.reason == "max_output_tokens"
149
+
150
+
151
+@pytest.mark.asyncio
152
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
153
+async def test_extra_sampling_params(client: OpenAI, model_name: str):
154
+ """Test that extra sampling parameters are accepted and work."""
155
+ # Test with multiple sampling parameters - just verify they're accepted
156
+ response = await client.responses.create(
157
+ model=model_name,
158
+ input="Write a short sentence",
159
+ max_output_tokens=50,
160
+ temperature=0.7,
161
+ top_p=0.9,
162
+ extra_body={
163
+ "top_k": 40,
164
+ "repetition_penalty": 1.2,
165
+ "seed": 42,
166
+ },
167
+ )
168
+
169
+ # Verify request succeeded and parameters were accepted
170
+ assert response.status in ["completed", "incomplete"]
171
+ assert len(response.output) > 0
172
+ assert response.output[0].content[0].text # Has text output
vllm/entrypoints/openai/responses/protocol.py
+28
-1
@@ -6,6 +6,7 @@
6
import time
7
from typing import Any, Literal, TypeAlias
8
9
+import torch
10
from openai.types.responses import (
11
ResponseCodeInterpreterCallCodeDeltaEvent,
12
ResponseCodeInterpreterCallCodeDoneEvent,
@@ -77,6 +78,8 @@ from vllm.utils import random_uuid
78
79
logger = init_logger(__name__)
80
81
+_LONG_INFO = torch.iinfo(torch.long)
82
+
83
84
class InputTokensDetails(OpenAIBaseModel):
85
cached_tokens: int
@@ -230,6 +233,18 @@ class ResponsesRequest(OpenAIBaseModel):
233
# this cannot be used in conjunction with previous_response_id
234
# TODO: consider supporting non harmony messages as well
235
previous_input_messages: list[OpenAIHarmonyMessage | dict] | None = None
236
+
237
+ repetition_penalty: float | None = None
238
+ seed: int | None = Field(None, ge=_LONG_INFO.min, le=_LONG_INFO.max)
239
+ stop: str | list[str] | None = []
240
+ ignore_eos: bool = False
241
+ vllm_xargs: dict[str, str | int | float | list[str | int | float]] | None = Field(
242
+ default=None,
243
+ description=(
244
+ "Additional request parameters with (list of) string or "
245
+ "numeric values, used by custom extensions."
246
+ ),
247
+ )
248
# --8<-- [end:responses-extra-params]
249
250
def build_chat_params(
@@ -297,6 +312,10 @@ class ResponsesRequest(OpenAIBaseModel):
312
top_k = default_sampling_params.get(
313
"top_k", self._DEFAULT_SAMPLING_PARAMS["top_k"]
314
)
315
+
316
+ if (repetition_penalty := self.repetition_penalty) is None:
317
+ repetition_penalty = default_sampling_params.get("repetition_penalty", 1.0)
318
+
319
stop_token_ids = default_sampling_params.get("stop_token_ids")
320
321
# Structured output
@@ -313,7 +332,10 @@ class ResponsesRequest(OpenAIBaseModel):
332
elif response_format.type == "json_object":
333
raise NotImplementedError("json_object is not supported")
334
316
- # TODO: add more parameters
335
+ stop = self.stop if self.stop else []
336
+ if isinstance(stop, str):
337
+ stop = [stop]
338
+
339
return SamplingParams.from_optional(
340
temperature=temperature,
341
top_p=top_p,
@@ -321,11 +343,16 @@ class ResponsesRequest(OpenAIBaseModel):
343
max_tokens=max_tokens,
344
logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
345
stop_token_ids=stop_token_ids,
346
+ stop=stop,
347
+ repetition_penalty=repetition_penalty,
348
+ seed=self.seed,
349
+ ignore_eos=self.ignore_eos,
350
output_kind=(
351
RequestOutputKind.DELTA if self.stream else RequestOutputKind.FINAL_ONLY
352
),
353
structured_outputs=structured_outputs,
354
logit_bias=self.logit_bias,
355
+ extra_args=self.vllm_xargs or {},
356
skip_clone=True, # Created fresh per request, safe to skip clone
357
skip_special_tokens=self.skip_special_tokens,
358
include_stop_str_in_output=self.include_stop_str_in_output,