A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[gpt-oss][1b] streaming add item id, content id (#24788)
Signed-off-by: Andrew Xia <axia@meta.com>
Andrew Xia committed
Sep 16, 2025 at 11:41 UTC
f4d6eb95cfdcee2c47bb408890346003bb5a5c20
2 files changed
+31
-2
tests/entrypoints/openai/test_response_api_with_harmony.py
+23
@@ -318,6 +318,9 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
318
background=background,
319
)
320
321
+ current_item_id = ""
322
+ current_content_index = -1
323
+
324
events = []
325
current_event_mode = None
326
resp_id = None
@@ -329,6 +332,26 @@ async def test_streaming(client: OpenAI, model_name: str, background: bool):
332
current_event_mode = event.type
333
print(f"\n[{event.type}] ", end="", flush=True)
334
335
+ # verify current_item_id is correct
336
+ if event.type == "response.output_item.added":
337
+ assert event.item.id != current_item_id
338
+ current_item_id = event.item.id
339
+ elif event.type in [
340
+ "response.output_text.delta",
341
+ "response.reasoning_text.delta"
342
+ ]:
343
+ assert event.item_id == current_item_id
344
+
345
+ # verify content_index_id is correct
346
+ if event.type == "response.content_part.added":
347
+ assert event.content_index != current_content_index
348
+ current_content_index = event.content_index
349
+ elif event.type in [
350
+ "response.output_text.delta",
351
+ "response.reasoning_text.delta"
352
+ ]:
353
+ assert event.content_index == current_content_index
354
+
355
if "text.delta" in event.type:
356
print(event.delta, end="", flush=True)
357
elif "reasoning_text.delta" in event.type:
vllm/entrypoints/openai/serving_responses.py
+8
-2
@@ -1260,9 +1260,9 @@ class OpenAIServingResponses(OpenAIServing):
1260
_increment_sequence_number_and_return: Callable[[BaseModel],
1261
BaseModel],
1262
) -> AsyncGenerator[BaseModel, None]:
1263
- current_content_index = 0 # FIXME: this number is never changed
1263
+ current_content_index = -1
1264
current_output_index = 0
1265
- current_item_id = "" # FIXME: this number is never changed
1265
+ current_item_id: str = ""
1266
sent_output_item_added = False
1267
1268
async for ctx in result_generator:
@@ -1353,6 +1353,7 @@ class OpenAIServingResponses(OpenAIServing):
1353
and ctx.parser.current_recipient is None):
1354
if not sent_output_item_added:
1355
sent_output_item_added = True
1356
+ current_item_id = f"msg_{random_uuid()}"
1357
yield _increment_sequence_number_and_return(
1358
openai_responses_types.
1359
ResponseOutputItemAddedEvent(
@@ -1368,6 +1369,7 @@ class OpenAIServingResponses(OpenAIServing):
1369
status="in_progress",
1370
),
1371
))
1372
+ current_content_index += 1
1373
yield _increment_sequence_number_and_return(
1374
openai_responses_types.
1375
ResponseContentPartAddedEvent(
@@ -1398,6 +1400,7 @@ class OpenAIServingResponses(OpenAIServing):
1400
and ctx.parser.current_recipient is None):
1401
if not sent_output_item_added:
1402
sent_output_item_added = True
1403
+ current_item_id = f"msg_{random_uuid()}"
1404
yield _increment_sequence_number_and_return(
1405
openai_responses_types.
1406
ResponseOutputItemAddedEvent(
@@ -1412,6 +1415,7 @@ class OpenAIServingResponses(OpenAIServing):
1415
status="in_progress",
1416
),
1417
))
1418
+ current_content_index += 1
1419
yield _increment_sequence_number_and_return(
1420
openai_responses_types.
1421
ResponseContentPartAddedEvent(
@@ -1444,6 +1448,7 @@ class OpenAIServingResponses(OpenAIServing):
1448
) and ctx.parser.current_recipient == "python":
1449
if not sent_output_item_added:
1450
sent_output_item_added = True
1451
+ current_item_id = f"tool_{random_uuid()}"
1452
yield _increment_sequence_number_and_return(
1453
openai_responses_types.
1454
ResponseOutputItemAddedEvent(
@@ -1516,6 +1521,7 @@ class OpenAIServingResponses(OpenAIServing):
1521
raise ValueError(
1522
f"Unknown function name: {function_name}")
1523
1524
+ current_item_id = f"tool_{random_uuid()}"
1525
yield _increment_sequence_number_and_return(
1526
openai_responses_types.ResponseOutputItemAddedEvent(
1527
type="response.output_item.added",