A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 8, 11:15 UTC
Responses harmony system message structured (#34268)
Signed-off-by: Adam Binford <adamq43@gmail.com>
Adam Binford committed
Feb 11, 2026 at 08:14 UTC
1b8756562e1cc50bade1335e52aa36547d62e477
2 files changed
+43
-6
tests/entrypoints/openai/responses/test_harmony.py
+29
-4
@@ -1302,16 +1302,17 @@ async def test_system_prompt_override(client: OpenAI, model_name: str):
1302
# Message structure may vary, skip this specific check
1303
pass
1304
1305
+ custom_system_prompt_2 = (
1306
+ "You are a helpful assistant that always responds in exactly 5 words."
1307
+ )
1308
+
1309
# Test 3: Test with different custom system prompt
1310
response_2 = await client.responses.create(
1311
model=model_name,
1312
input=[
1313
{
1314
"role": "system",
1311
- "content": (
1312
- "You are a helpful assistant that always "
1313
- "responds in exactly 5 words."
1314
- ),
1315
+ "content": custom_system_prompt_2,
1316
},
1317
{"role": "user", "content": "What is the weather like?"},
1318
],
@@ -1328,3 +1329,27 @@ async def test_system_prompt_override(client: OpenAI, model_name: str):
1329
assert 3 <= word_count <= 8, (
1330
f"Expected around 5 words, got {word_count} words: {response_2.output_text}"
1331
)
1332
+
1333
+ # Test 4: Test with structured content
1334
+ response_3 = await client.responses.create(
1335
+ model=model_name,
1336
+ input=[
1337
+ {
1338
+ "role": "system",
1339
+ "content": [{"type": "input_text", "text": custom_system_prompt_2}],
1340
+ },
1341
+ {"role": "user", "content": "What is the weather like?"},
1342
+ ],
1343
+ temperature=0.0,
1344
+ )
1345
+
1346
+ assert response_3 is not None
1347
+ assert response_3.status == "completed"
1348
+ assert response_3.output_text is not None
1349
+
1350
+ # Count words in response (approximately, allowing for punctuation)
1351
+ word_count = len(response_3.output_text.split())
1352
+ # Allow some flexibility (4-7 words) since the model might not be perfectly precise
1353
+ assert 3 <= word_count <= 8, (
1354
+ f"Expected around 5 words, got {word_count} words: {response_3.output_text}"
1355
+ )
vllm/entrypoints/openai/responses/serving.py
+14
-2
@@ -980,7 +980,9 @@ class OpenAIServingResponses(OpenAIServing):
980
output_items.extend(last_items)
981
return output_items
982
983
- def _extract_system_message_from_request(self, request) -> str | None:
983
+ def _extract_system_message_from_request(
984
+ self, request: ResponsesRequest
985
+ ) -> str | None:
986
system_msg = None
987
if not isinstance(request.input, str):
988
for response_msg in request.input:
@@ -988,7 +990,17 @@ class OpenAIServingResponses(OpenAIServing):
990
isinstance(response_msg, dict)
991
and response_msg.get("role") == "system"
992
):
991
- system_msg = response_msg.get("content")
993
+ content = response_msg.get("content")
994
+ if isinstance(content, str):
995
+ system_msg = content
996
+ elif isinstance(content, list):
997
+ for param in content:
998
+ if (
999
+ isinstance(param, dict)
1000
+ and param.get("type") == "input_text"
1001
+ ):
1002
+ system_msg = param.get("text")
1003
+ break
1004
break
1005
return system_msg
1006