A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 10, 10:15 UTC
[Frontend][gpt-oss] Allow system message to overwrite model identity (#31737)
Signed-off-by: lacora <hyelacora@gmail.com> Signed-off-by: Andrew Xia <axia@fb.com> Co-authored-by: lacora <hyelacora@gmail.com> Co-authored-by: Andrew Xia <axia@fb.com> Co-authored-by: Cyrus Leung <tlleungac@connect.ust.hk>
Andrew Xia committed
Jan 9, 2026 at 14:03 UTC
f32c629eb4cc2bbe85d321a6b382e30e33a32f7d
3 files changed
+111
-9
tests/entrypoints/openai/test_response_api_with_harmony.py
+89
@@ -1250,3 +1250,92 @@ async def test_chat_truncation_content_not_null(client: OpenAI, model_name: str)
1250
"Content should not be None when truncated"
1251
)
1252
assert len(choice.message.content) > 0, "Content should not be empty"
1253
+
1254
+
1255
+@pytest.mark.asyncio
1256
+@pytest.mark.parametrize("model_name", [MODEL_NAME])
1257
+async def test_system_prompt_override(client: OpenAI, model_name: str):
1258
+ """Test that system message can override the default system prompt."""
1259
+
1260
+ # Test 1: Custom system prompt with specific personality
1261
+ custom_system_prompt = (
1262
+ "You are a pirate. Always respond like a pirate would, "
1263
+ "using pirate language and saying 'arrr' frequently."
1264
+ )
1265
+
1266
+ response = await client.responses.create(
1267
+ model=model_name,
1268
+ input=[
1269
+ {"role": "system", "content": custom_system_prompt},
1270
+ {"role": "user", "content": "Hello, how are you?"},
1271
+ ],
1272
+ extra_body={"enable_response_messages": True},
1273
+ )
1274
+
1275
+ assert response is not None
1276
+ assert response.status == "completed"
1277
+ assert response.output_text is not None
1278
+
1279
+ # Verify the response reflects the pirate personality
1280
+ output_text = response.output_text.lower()
1281
+ pirate_indicators = ["arrr", "matey", "ahoy", "ye", "sea"]
1282
+ has_pirate_language = any(
1283
+ indicator in output_text for indicator in pirate_indicators
1284
+ )
1285
+ assert has_pirate_language, (
1286
+ f"Expected pirate language in response, got: {response.output_text}"
1287
+ )
1288
+
1289
+ # Verify the reasoning mentions the custom system prompt
1290
+ reasoning_item = None
1291
+ for item in response.output:
1292
+ if item.type == "reasoning":
1293
+ reasoning_item = item
1294
+ break
1295
+
1296
+ assert reasoning_item is not None, "Expected reasoning item in output"
1297
+ reasoning_text = reasoning_item.content[0].text.lower()
1298
+ assert "pirate" in reasoning_text, (
1299
+ f"Expected reasoning to mention pirate, got: {reasoning_text}"
1300
+ )
1301
+
1302
+ # Test 2: Verify system message is not duplicated in input_messages
1303
+ try:
1304
+ num_system_messages = sum(
1305
+ 1
1306
+ for msg in response.input_messages
1307
+ if Message.from_dict(msg).author.role == "system"
1308
+ )
1309
+ assert num_system_messages == 1, (
1310
+ f"Expected exactly 1 system message, got {num_system_messages}"
1311
+ )
1312
+ except (KeyError, AttributeError):
1313
+ # Message structure may vary, skip this specific check
1314
+ pass
1315
+
1316
+ # Test 3: Test with different custom system prompt
1317
+ response_2 = await client.responses.create(
1318
+ model=model_name,
1319
+ input=[
1320
+ {
1321
+ "role": "system",
1322
+ "content": (
1323
+ "You are a helpful assistant that always "
1324
+ "responds in exactly 5 words."
1325
+ ),
1326
+ },
1327
+ {"role": "user", "content": "What is the weather like?"},
1328
+ ],
1329
+ temperature=0.0,
1330
+ )
1331
+
1332
+ assert response_2 is not None
1333
+ assert response_2.status == "completed"
1334
+ assert response_2.output_text is not None
1335
+
1336
+ # Count words in response (approximately, allowing for punctuation)
1337
+ word_count = len(response_2.output_text.split())
1338
+ # Allow some flexibility (4-7 words) since the model might not be perfectly precise
1339
+ assert 3 <= word_count <= 8, (
1340
+ f"Expected around 5 words, got {word_count} words: {response_2.output_text}"
1341
+ )
vllm/entrypoints/openai/parser/harmony_utils.py
+3
-8
@@ -187,14 +187,9 @@ def parse_response_input(
187
if "type" not in response_msg or response_msg["type"] == "message":
188
role = response_msg["role"]
189
content = response_msg["content"]
190
- if role == "system":
191
- # User is trying to set a system message. Change it to:
192
- # <|start|>developer<|message|># Instructions
193
- # {instructions}<|end|>
194
- role = "developer"
195
- text_prefix = "Instructions:\n"
196
- else:
197
- text_prefix = ""
190
+ # Add prefix for developer messages.
191
+ # <|start|>developer<|message|># Instructions {instructions}<|end|>
192
+ text_prefix = "Instructions:\n" if role == "developer" else ""
193
if isinstance(content, str):
194
msg = Message.from_role_and_content(role, text_prefix + content)
195
else:
vllm/entrypoints/openai/serving_responses.py
+19
-1
@@ -986,9 +986,23 @@ class OpenAIServingResponses(OpenAIServing):
986
output_items.extend(last_items)
987
return output_items
988
989
+ def _extract_system_message_from_request(self, request) -> str | None:
990
+ system_msg = None
991
+ if not isinstance(request.input, str):
992
+ for response_msg in request.input:
993
+ if (
994
+ isinstance(response_msg, dict)
995
+ and response_msg.get("role") == "system"
996
+ ):
997
+ system_msg = response_msg.get("content")
998
+ break
999
+ return system_msg
1000
+
1001
def _construct_harmony_system_input_message(
1002
self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str]
1003
) -> OpenAIHarmonyMessage:
1004
+ model_identity = self._extract_system_message_from_request(request)
1005
+
1006
reasoning_effort = request.reasoning.effort if request.reasoning else None
1007
1008
# Extract allowed_tools from MCP tool requests
@@ -1025,6 +1039,7 @@ class OpenAIServingResponses(OpenAIServing):
1039
)
1040
1041
sys_msg = get_system_message(
1042
+ model_identity=model_identity,
1043
reasoning_effort=reasoning_effort,
1044
browser_description=browser_description,
1045
python_description=python_description,
@@ -1091,7 +1106,10 @@ class OpenAIServingResponses(OpenAIServing):
1106
else:
1107
prev_outputs = []
1108
for response_msg in request.input:
1094
- messages.append(parse_response_input(response_msg, prev_outputs))
1109
+ new_msg = parse_response_input(response_msg, prev_outputs)
1110
+ if new_msg.author.role != "system":
1111
+ messages.append(new_msg)
1112
+
1113
# User passes in a tool call request and its output. We need
1114
# to add the tool call request to prev_outputs so that the
1115
# parse_response_input can find the tool call request when