A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 05:15 UTC
[responseAPI] support partial message generation (#32100)
Signed-off-by: Andrew Xia <axia@fb.com> Signed-off-by: Andrew Xia <mitandrewxia@gmail.com> Signed-off-by: Lu Fang <30275821+houseroad@users.noreply.github.com> Co-authored-by: Andrew Xia <axia@fb.com> Co-authored-by: Lu Fang <30275821+houseroad@users.noreply.github.com>
Andrew Xia committed
Jan 13, 2026 at 13:41 UTC
af54d2e2d0f6d8a3bb3bd3789961810203a1fc40
3 files changed
+337
tests/entrypoints/test_responses_utils.py
+280
@@ -21,6 +21,7 @@ from vllm.entrypoints.responses_utils import (
21
_maybe_combine_reasoning_and_tool_call,
22
construct_chat_messages_with_tool_call,
23
convert_tool_responses_to_completions_format,
24
+ should_continue_final_message,
25
)
26
27
@@ -165,6 +166,285 @@ class TestResponsesUtils:
166
assert formatted_item["content"] == "dongyi"
167
168
169
+class TestShouldContinueFinalMessage:
170
+ """Tests for should_continue_final_message function.
171
+
172
+ This function enables Anthropic-style partial message completion, where
173
+ users can provide an incomplete assistant message and have the model
174
+ continue from where it left off.
175
+ """
176
+
177
+ def test_string_input_returns_false(self):
178
+ """String input is always a user message, so should not continue."""
179
+ assert should_continue_final_message("Hello, world!") is False
180
+
181
+ def test_empty_list_returns_false(self):
182
+ """Empty list should not continue."""
183
+ assert should_continue_final_message([]) is False
184
+
185
+ def test_completed_message_returns_false(self):
186
+ """Completed message should not be continued."""
187
+ output_item = ResponseOutputMessage(
188
+ id="msg_123",
189
+ content=[
190
+ ResponseOutputText(
191
+ annotations=[],
192
+ text="The answer is 42.",
193
+ type="output_text",
194
+ logprobs=None,
195
+ )
196
+ ],
197
+ role="assistant",
198
+ status="completed",
199
+ type="message",
200
+ )
201
+ assert should_continue_final_message([output_item]) is False
202
+
203
+ def test_in_progress_message_returns_true(self):
204
+ """In-progress message should be continued.
205
+
206
+ This is the key use case for partial message completion.
207
+ Example: The user provides "The best answer is (" and wants
208
+ the model to continue from there.
209
+ """
210
+ output_item = ResponseOutputMessage(
211
+ id="msg_123",
212
+ content=[
213
+ ResponseOutputText(
214
+ annotations=[],
215
+ text="The best answer is (",
216
+ type="output_text",
217
+ logprobs=None,
218
+ )
219
+ ],
220
+ role="assistant",
221
+ status="in_progress",
222
+ type="message",
223
+ )
224
+ assert should_continue_final_message([output_item]) is True
225
+
226
+ def test_incomplete_message_returns_true(self):
227
+ """Incomplete message should be continued."""
228
+ output_item = ResponseOutputMessage(
229
+ id="msg_123",
230
+ content=[
231
+ ResponseOutputText(
232
+ annotations=[],
233
+ text="The answer",
234
+ type="output_text",
235
+ logprobs=None,
236
+ )
237
+ ],
238
+ role="assistant",
239
+ status="incomplete",
240
+ type="message",
241
+ )
242
+ assert should_continue_final_message([output_item]) is True
243
+
244
+ def test_in_progress_reasoning_returns_true(self):
245
+ """In-progress reasoning should be continued."""
246
+ reasoning_item = ResponseReasoningItem(
247
+ id="reasoning_123",
248
+ summary=[],
249
+ type="reasoning",
250
+ content=[
251
+ Content(
252
+ text="Let me think about this...",
253
+ type="reasoning_text",
254
+ )
255
+ ],
256
+ encrypted_content=None,
257
+ status="in_progress",
258
+ )
259
+ assert should_continue_final_message([reasoning_item]) is True
260
+
261
+ def test_incomplete_reasoning_returns_true(self):
262
+ """Incomplete reasoning should be continued."""
263
+ reasoning_item = ResponseReasoningItem(
264
+ id="reasoning_123",
265
+ summary=[],
266
+ type="reasoning",
267
+ content=[
268
+ Content(
269
+ text="Let me think",
270
+ type="reasoning_text",
271
+ )
272
+ ],
273
+ encrypted_content=None,
274
+ status="incomplete",
275
+ )
276
+ assert should_continue_final_message([reasoning_item]) is True
277
+
278
+ reasoning_item = {
279
+ "id": "reasoning_123",
280
+ "summary": [],
281
+ "type": "reasoning",
282
+ "content": [],
283
+ "status": "incomplete",
284
+ }
285
+ assert should_continue_final_message([reasoning_item]) is True
286
+
287
+ def test_completed_reasoning_returns_false(self):
288
+ """Completed reasoning should not be continued."""
289
+ reasoning_item = ResponseReasoningItem(
290
+ id="reasoning_123",
291
+ summary=[],
292
+ type="reasoning",
293
+ content=[
294
+ Content(
295
+ text="I have thought about this.",
296
+ type="reasoning_text",
297
+ )
298
+ ],
299
+ encrypted_content=None,
300
+ status="completed",
301
+ )
302
+ assert should_continue_final_message([reasoning_item]) is False
303
+
304
+ def test_reasoning_with_none_status_returns_false(self):
305
+ """Reasoning with None status should not be continued."""
306
+ reasoning_item = ResponseReasoningItem(
307
+ id="reasoning_123",
308
+ summary=[],
309
+ type="reasoning",
310
+ content=[
311
+ Content(
312
+ text="Some reasoning",
313
+ type="reasoning_text",
314
+ )
315
+ ],
316
+ encrypted_content=None,
317
+ status=None,
318
+ )
319
+ assert should_continue_final_message([reasoning_item]) is False
320
+
321
+ def test_only_last_item_matters(self):
322
+ """Only the last item in the list determines continuation."""
323
+ completed_item = ResponseOutputMessage(
324
+ id="msg_1",
325
+ content=[
326
+ ResponseOutputText(
327
+ annotations=[],
328
+ text="Complete message.",
329
+ type="output_text",
330
+ logprobs=None,
331
+ )
332
+ ],
333
+ role="assistant",
334
+ status="completed",
335
+ type="message",
336
+ )
337
+ in_progress_item = ResponseOutputMessage(
338
+ id="msg_2",
339
+ content=[
340
+ ResponseOutputText(
341
+ annotations=[],
342
+ text="Partial message...",
343
+ type="output_text",
344
+ logprobs=None,
345
+ )
346
+ ],
347
+ role="assistant",
348
+ status="in_progress",
349
+ type="message",
350
+ )
351
+
352
+ # In-progress as last item -> should continue
353
+ assert should_continue_final_message([completed_item, in_progress_item]) is True
354
+
355
+ # Completed as last item -> should not continue
356
+ assert (
357
+ should_continue_final_message([in_progress_item, completed_item]) is False
358
+ )
359
+
360
+ def test_tool_call_returns_false(self):
361
+ """Tool calls should not trigger continuation."""
362
+ tool_call = ResponseFunctionToolCall(
363
+ id="fc_123",
364
+ call_id="call_123",
365
+ type="function_call",
366
+ status="in_progress",
367
+ name="get_weather",
368
+ arguments='{"location": "NYC"}',
369
+ )
370
+ assert should_continue_final_message([tool_call]) is False
371
+
372
+ tool_call = {
373
+ "id": "msg_123",
374
+ "call_id": "call_123",
375
+ "type": "function_call",
376
+ "status": "in_progress",
377
+ "name": "get_weather",
378
+ "arguments": '{"location": "NYC"}',
379
+ }
380
+ assert should_continue_final_message([tool_call]) is False
381
+
382
+ # Tests for dict inputs (e.g., from curl requests)
383
+ def test_dict_in_progress_message_returns_true(self):
384
+ """Dict with in_progress status should be continued (curl input)."""
385
+ dict_item = {
386
+ "id": "msg_123",
387
+ "type": "message",
388
+ "role": "assistant",
389
+ "status": "in_progress",
390
+ "content": [{"type": "output_text", "text": "The answer is ("}],
391
+ }
392
+ assert should_continue_final_message([dict_item]) is True
393
+
394
+ def test_dict_incomplete_message_returns_true(self):
395
+ """Dict with incomplete status should be continued (curl input)."""
396
+ dict_item = {
397
+ "id": "msg_123",
398
+ "type": "message",
399
+ "role": "assistant",
400
+ "status": "incomplete",
401
+ "content": [{"type": "output_text", "text": "Partial answer"}],
402
+ }
403
+ assert should_continue_final_message([dict_item]) is True
404
+
405
+ def test_dict_completed_message_returns_false(self):
406
+ """Dict with completed status should not be continued (curl input)."""
407
+ dict_item = {
408
+ "id": "msg_123",
409
+ "type": "message",
410
+ "role": "assistant",
411
+ "status": "completed",
412
+ "content": [{"type": "output_text", "text": "Complete answer."}],
413
+ }
414
+ assert should_continue_final_message([dict_item]) is False
415
+
416
+ def test_dict_reasoning_in_progress_returns_true(self):
417
+ """Dict reasoning item with in_progress status should be continued."""
418
+ dict_item = {
419
+ "id": "reasoning_123",
420
+ "type": "reasoning",
421
+ "status": "in_progress",
422
+ "content": [{"type": "reasoning_text", "text": "Let me think..."}],
423
+ }
424
+ assert should_continue_final_message([dict_item]) is True
425
+
426
+ def test_dict_without_status_returns_false(self):
427
+ """Dict without status field should not be continued."""
428
+ dict_item = {
429
+ "id": "msg_123",
430
+ "type": "message",
431
+ "role": "assistant",
432
+ "content": [{"type": "output_text", "text": "Some text"}],
433
+ }
434
+ assert should_continue_final_message([dict_item]) is False
435
+
436
+ def test_dict_with_none_status_returns_false(self):
437
+ """Dict with None status should not be continued."""
438
+ dict_item = {
439
+ "id": "msg_123",
440
+ "type": "message",
441
+ "role": "assistant",
442
+ "status": None,
443
+ "content": [{"type": "output_text", "text": "Some text"}],
444
+ }
445
+ assert should_continue_final_message([dict_item]) is False
446
+
447
+
448
class TestMaybeCombineReasoningAndToolCall:
449
"""Tests for _maybe_combine_reasoning_and_tool_call function."""
450
vllm/entrypoints/openai/serving_responses.py
+10
@@ -111,6 +111,7 @@ from vllm.entrypoints.responses_utils import (
111
construct_input_messages,
112
construct_tool_dicts,
113
extract_tool_types,
114
+ should_continue_final_message,
115
)
116
from vllm.entrypoints.tool_server import ToolServer
117
from vllm.inputs.data import TokensPrompt
@@ -590,6 +591,10 @@ class OpenAIServingResponses(OpenAIServing):
591
prev_response_output=prev_response.output if prev_response else None,
592
)
593
594
+ # Check if we should continue the final message (partial completion)
595
+ # This enables Anthropic-style partial message completion where the
596
+ # user provides an incomplete assistant message to continue from.
597
+ continue_final = should_continue_final_message(request.input)
598
chat_template_kwargs = dict(
599
reasoning_effort=None
600
if request.reasoning is None
@@ -604,6 +609,11 @@ class OpenAIServingResponses(OpenAIServing):
609
tool_parser=self.tool_parser,
610
chat_template=self.chat_template,
611
chat_template_content_format=self.chat_template_content_format,
612
+ # When continuing a partial message, we set continue_final_message=True
613
+ # and add_generation_prompt=False so the model continues the message
614
+ # rather than starting a new one.
615
+ add_generation_prompt=not continue_final,
616
+ continue_final_message=continue_final,
617
chat_template_kwargs=chat_template_kwargs,
618
)
619
return messages, engine_prompts
vllm/entrypoints/responses_utils.py
+47
@@ -28,6 +28,53 @@ from vllm.entrypoints.openai.engine.protocol import (
28
)
29
30
31
+def should_continue_final_message(
32
+ request_input: str | list[ResponseInputOutputItem],
33
+) -> bool:
34
+ """
35
+ Determine if the last input message is a partial assistant message
36
+ that should be continued rather than starting a new generation.
37
+
38
+ This enables partial message completion similar to Anthropic's Messages API,
39
+ where users can provide an incomplete assistant message and have the model
40
+ continue from where it left off.
41
+
42
+ A message is considered partial if:
43
+ 1. It's a ResponseOutputMessage or ResponseReasoningItem
44
+ 2. Its status is "in_progress" or "incomplete"
45
+
46
+ Args:
47
+ request_input: The input to the Responses API request
48
+
49
+ Returns:
50
+ True if the final message should be continued, False otherwise
51
+ """
52
+ if isinstance(request_input, str):
53
+ # Simple string input is always a user message
54
+ return False
55
+
56
+ if not request_input:
57
+ return False
58
+
59
+ last_item = request_input[-1]
60
+
61
+ # Check if the last item is a partial assistant message
62
+ if isinstance(last_item, ResponseOutputMessage):
63
+ return last_item.status in ("in_progress", "incomplete")
64
+
65
+ # Check if the last item is a partial reasoning item
66
+ if isinstance(last_item, ResponseReasoningItem):
67
+ return last_item.status in ("in_progress", "incomplete")
68
+
69
+ if isinstance(last_item, dict):
70
+ # only support partial completion for messages for now
71
+ if last_item.get("type", "message") not in ("message", "reasoning"):
72
+ return False
73
+ return last_item.get("status") in ("in_progress", "incomplete")
74
+
75
+ return False
76
+
77
+
78
def construct_input_messages(
79
*,
80
request_instructions: str | None = None,