A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[responsesAPI] add better error messaging for long prompts (#25724)
Signed-off-by: Andrew Xia <axia@meta.com> Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: Andrew Xia <axia@fb.com>
Andrew Xia committed
Oct 3, 2025 at 14:33 UTC
831b1241510210f31140df7f58d9859aa39336cd
2 files changed
+84
-1
tests/entrypoints/openai/test_serving_responses.py
+62
-1
@@ -8,9 +8,10 @@ import pytest
8
import pytest_asyncio
9
10
from vllm.entrypoints.context import ConversationContext
11
-from vllm.entrypoints.openai.protocol import ResponsesRequest
11
+from vllm.entrypoints.openai.protocol import ErrorResponse, ResponsesRequest
12
from vllm.entrypoints.openai.serving_responses import OpenAIServingResponses
13
from vllm.entrypoints.tool_server import ToolServer
14
+from vllm.inputs.data import TokensPrompt as EngineTokensPrompt
15
16
17
class MockConversationContext(ConversationContext):
@@ -127,3 +128,63 @@ class TestInitializeToolSessions:
128
129
# Verify that init_tool_sessions was called
130
assert mock_context.init_tool_sessions_called
131
+
132
+
133
+class TestValidateGeneratorInput:
134
+ """Test class for _validate_generator_input method"""
135
+
136
+ @pytest_asyncio.fixture
137
+ async def serving_responses_instance(self):
138
+ """Create a real OpenAIServingResponses instance for testing"""
139
+ # Create minimal mocks for required dependencies
140
+ engine_client = MagicMock()
141
+ engine_client.get_model_config = AsyncMock()
142
+
143
+ model_config = MagicMock()
144
+ model_config.hf_config.model_type = "test"
145
+ model_config.get_diff_sampling_param.return_value = {}
146
+
147
+ models = MagicMock()
148
+
149
+ # Create the actual instance
150
+ instance = OpenAIServingResponses(
151
+ engine_client=engine_client,
152
+ model_config=model_config,
153
+ models=models,
154
+ request_logger=None,
155
+ chat_template=None,
156
+ chat_template_content_format="auto",
157
+ )
158
+
159
+ # Set max_model_len for testing
160
+ instance.max_model_len = 100
161
+
162
+ return instance
163
+
164
+ def test_validate_generator_input(self, serving_responses_instance):
165
+ """Test _validate_generator_input with valid prompt length"""
166
+ # Create an engine prompt with valid length (less than max_model_len)
167
+ valid_prompt_token_ids = list(range(5)) # 5 tokens < 100 max_model_len
168
+ engine_prompt = EngineTokensPrompt(
169
+ prompt_token_ids=valid_prompt_token_ids)
170
+
171
+ # Call the method
172
+ result = serving_responses_instance._validate_generator_input(
173
+ engine_prompt)
174
+
175
+ # Should return None for valid input
176
+ assert result is None
177
+
178
+ # create an invalid engine prompt
179
+ invalid_prompt_token_ids = list(
180
+ range(200)) # 100 tokens >= 100 max_model_len
181
+ engine_prompt = EngineTokensPrompt(
182
+ prompt_token_ids=invalid_prompt_token_ids)
183
+
184
+ # Call the method
185
+ result = serving_responses_instance._validate_generator_input(
186
+ engine_prompt)
187
+
188
+ # Should return an ErrorResponse
189
+ assert result is not None
190
+ assert isinstance(result, ErrorResponse)
vllm/entrypoints/openai/serving_responses.py
+22
@@ -192,6 +192,23 @@ class OpenAIServingResponses(OpenAIServing):
192
193
self.tool_server = tool_server
194
195
+ def _validate_generator_input(
196
+ self,
197
+ engine_prompt: EngineTokensPrompt) -> Optional[ErrorResponse]:
198
+ """Add validations to the input to the generator here."""
199
+ if self.max_model_len <= len(engine_prompt["prompt_token_ids"]):
200
+ error_message = (
201
+ "The engine prompt length"
202
+ f" {len(engine_prompt['prompt_token_ids'])} "
203
+ f"exceeds the max_model_len {self.max_model_len}. "
204
+ "Please reduce prompt.")
205
+ return self.create_error_response(
206
+ err_type="invalid_request_error",
207
+ message=error_message,
208
+ status_code=HTTPStatus.BAD_REQUEST,
209
+ )
210
+ return None
211
+
212
async def create_responses(
213
self,
214
request: ResponsesRequest,
@@ -287,8 +304,13 @@ class OpenAIServingResponses(OpenAIServing):
304
available_tools = []
305
try:
306
for i, engine_prompt in enumerate(engine_prompts):
307
+ maybe_error = self._validate_generator_input(engine_prompt)
308
+ if maybe_error is not None:
309
+ return maybe_error
310
+
311
default_max_tokens = self.max_model_len - len(
312
engine_prompt["prompt_token_ids"])
313
+
314
sampling_params = request.to_sampling_params(
315
default_max_tokens, self.default_sampling_params)
316