A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 1, 20:00 UTC
[Bugfix] GPT-OSS instruction rendering (#44330)
Signed-off-by: Yifan Zong <yzong@redhat.com>
yzong-rh committed
Jun 5, 2026 at 13:52 UTC
703fb17b132f784c397493bfe69bce58ec58a0a7
9 files changed
+275
-111
tests/entrypoints/openai/chat_completion/test_serving_chat.py
+51
@@ -1364,6 +1364,57 @@ class TestServingChatWithHarmony:
1364
],
1365
)
1366
1367
+ @pytest.mark.asyncio
1368
+ async def test_system_message_without_tools(self, serving_chat, stream):
1369
+ """Leading system message produces a developer message with
1370
+ DeveloperContent (# Instructions header)."""
1371
+ messages = [
1372
+ {"role": "system", "content": "You are a helpful assistant."},
1373
+ {"role": "user", "content": "Hello"},
1374
+ ]
1375
+ req = ChatCompletionRequest(model=MODEL_NAME, messages=messages)
1376
+ input_messages, _ = (
1377
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1378
+ )
1379
+ verify_harmony_messages(
1380
+ input_messages,
1381
+ [
1382
+ {"role": "system"},
1383
+ {
1384
+ "role": "developer",
1385
+ "instructions": "You are a helpful assistant.",
1386
+ },
1387
+ {"role": "user", "content": "Hello"},
1388
+ ],
1389
+ )
1390
+
1391
+ @pytest.mark.asyncio
1392
+ async def test_system_message_with_tools(self, serving_chat, stream, weather_tools):
1393
+ """Leading system message is folded into the developer message
1394
+ alongside tool definitions."""
1395
+ messages = [
1396
+ {"role": "system", "content": "You are a helpful assistant."},
1397
+ {"role": "user", "content": "What's the weather?"},
1398
+ ]
1399
+ req = ChatCompletionRequest(
1400
+ model=MODEL_NAME, messages=messages, tools=weather_tools
1401
+ )
1402
+ input_messages, _ = (
1403
+ serving_chat.openai_serving_render._make_request_with_harmony(req)
1404
+ )
1405
+ verify_harmony_messages(
1406
+ input_messages,
1407
+ [
1408
+ {"role": "system"},
1409
+ {
1410
+ "role": "developer",
1411
+ "instructions": "You are a helpful assistant.",
1412
+ "tool_definitions": ["get_weather"],
1413
+ },
1414
+ {"role": "user", "content": "What's the weather?"},
1415
+ ],
1416
+ )
1417
+
1418
@pytest.mark.asyncio
1419
async def test_tool_call_response_with_content(
1420
self, serving_chat, stream, weather_tools, weather_messages_start
tests/entrypoints/openai/parser/test_harmony_render_parity.py
+25
@@ -45,6 +45,31 @@ class TestResponseInputToHarmonyRenderParity:
45
# Single-message cases
46
# -----------------------------------------------------------------------
47
48
+ def test_developer_message(self):
49
+ """Both APIs must render developer messages identically using
50
+ DeveloperContent (with the '# Instructions' header)."""
51
+ chat_msgs = parse_chat_input_to_harmony_message(
52
+ {"role": "developer", "content": "Be concise."}
53
+ )
54
+ resp_msgs = [
55
+ response_input_to_harmony(
56
+ {
57
+ "type": "message",
58
+ "role": "developer",
59
+ "content": "Be concise.",
60
+ },
61
+ prev_responses=[],
62
+ )
63
+ ]
64
+
65
+ expected = [{"role": "developer", "instructions": "Be concise."}]
66
+ verify_harmony_messages(chat_msgs, expected)
67
+ verify_harmony_messages(resp_msgs, expected)
68
+
69
+ assert render_for_completion([_system()] + chat_msgs) == render_for_completion(
70
+ [_system()] + resp_msgs
71
+ )
72
+
73
def test_user_message(self):
74
chat_msgs = parse_chat_input_to_harmony_message(
75
{"role": "user", "content": "What's the weather in Paris?"}
tests/entrypoints/openai/parser/test_harmony_utils.py
+8
-5
@@ -2,7 +2,7 @@
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
import pytest
5
-from openai_harmony import Message, Role
5
+from openai_harmony import DeveloperContent, Message, Role
6
7
from tests.entrypoints.openai.utils import verify_harmony_messages
8
from vllm.entrypoints.openai.parser.harmony_utils import (
@@ -269,7 +269,8 @@ class TestCommonParseInputToHarmonyMessage:
269
assert messages[0].recipient == "functions.get_current_time"
270
271
def test_system_message(self, parse_function):
272
- """Test parsing system message."""
272
+ """Test parsing system messages, which are parsed into developer messages
273
+ with DeveloperContent."""
274
chat_msg = {
275
"role": "system",
276
"content": "You are a helpful assistant",
@@ -278,9 +279,9 @@ class TestCommonParseInputToHarmonyMessage:
279
messages = parse_function(chat_msg)
280
281
assert len(messages) == 1
281
- # System messages are converted using Message.from_dict
282
- # which should preserve the role
283
- assert messages[0].author.role == Role.SYSTEM
282
+ assert messages[0].author.role == Role.DEVELOPER
283
+ assert isinstance(messages[0].content[0], DeveloperContent)
284
+ assert messages[0].content[0].instructions == "You are a helpful assistant"
285
286
def test_developer_message(self, parse_function):
287
"""Test parsing developer message."""
@@ -293,6 +294,8 @@ class TestCommonParseInputToHarmonyMessage:
294
295
assert len(messages) == 1
296
assert messages[0].author.role == Role.DEVELOPER
297
+ assert isinstance(messages[0].content[0], DeveloperContent)
298
+ assert messages[0].content[0].instructions == "Use concise language"
299
300
def test_user_message_with_string_content(self, parse_function):
301
"""Test parsing user message with string content."""
tests/entrypoints/openai/responses/test_response_input_to_harmony.py
+16
-9
@@ -12,7 +12,7 @@ from openai.types.responses import ResponseFunctionToolCall, ResponseReasoningIt
12
from openai.types.responses.response_reasoning_item import (
13
Content as ReasoningTextContent,
14
)
15
-from openai_harmony import Role
15
+from openai_harmony import DeveloperContent, Role
16
17
from vllm.entrypoints.openai.responses.harmony import response_input_to_harmony
18
@@ -65,14 +65,16 @@ class TestResponseInputToHarmonyMessage:
65
assert msg.content[0].text == "Hello"
66
67
def test_system_message(self):
68
+ """System messages carry developer instructions and must be rendered
69
+ as developer messages with DeveloperContent."""
70
msg = response_input_to_harmony(
71
{"type": "message", "role": "system", "content": "Be helpful."},
72
prev_responses=[],
73
)
74
73
- assert msg.author.role == Role.SYSTEM
74
- assert msg.content[0].text == "Be helpful."
75
- assert msg.channel is None
75
+ assert msg.author.role == Role.DEVELOPER
76
+ assert isinstance(msg.content[0], DeveloperContent)
77
+ assert msg.content[0].instructions == "Be helpful."
78
79
def test_assistant_message_gets_final_channel(self):
80
msg = response_input_to_harmony(
@@ -85,14 +87,16 @@ class TestResponseInputToHarmonyMessage:
87
assert msg.content[0].text == "The answer is 42."
88
89
def test_developer_message_gets_instructions_prefix(self):
90
+ """Developer messages must use DeveloperContent which adds the
91
+ '# Instructions' header the model was trained on."""
92
msg = response_input_to_harmony(
93
{"type": "message", "role": "developer", "content": "Be concise."},
94
prev_responses=[],
95
)
96
97
assert msg.author.role == Role.DEVELOPER
94
- assert msg.content[0].text == "Instructions:\nBe concise."
95
- assert msg.channel is None
98
+ assert isinstance(msg.content[0], DeveloperContent)
99
+ assert msg.content[0].instructions == "Be concise."
100
101
def test_message_with_array_content(self):
102
msg = response_input_to_harmony(
@@ -112,7 +116,9 @@ class TestResponseInputToHarmonyMessage:
116
assert msg.content[0].text == "Part one. "
117
assert msg.content[1].text == "Part two."
118
115
- def test_developer_message_array_content_gets_prefix_on_each_part(self):
119
+ def test_developer_message_array_content_concatenated(self):
120
+ """Array content in developer messages is flattened and rendered
121
+ via DeveloperContent with the '# Instructions' header."""
122
msg = response_input_to_harmony(
123
{
124
"type": "message",
@@ -125,8 +131,9 @@ class TestResponseInputToHarmonyMessage:
131
prev_responses=[],
132
)
133
128
- assert msg.content[0].text == "Instructions:\nRule 1."
129
- assert msg.content[1].text == "Instructions:\nRule 2."
134
+ assert msg.author.role == Role.DEVELOPER
135
+ assert isinstance(msg.content[0], DeveloperContent)
136
+ assert msg.content[0].instructions == "Rule 1.Rule 2."
137
138
# -----------------------------------------------------------------------
139
# type="reasoning"
tests/entrypoints/openai/utils.py
+2
@@ -155,6 +155,8 @@ def verify_harmony_messages(
155
assert msg.content[0].text == expected["content"]
156
if "content_type" in expected:
157
assert msg.content_type == expected["content_type"]
158
+ if "instructions" in expected:
159
+ assert msg.content[0].instructions == expected["instructions"]
160
if "tool_definitions" in expected:
161
# Check that the tool definitions match the expected list of tool names
162
actual_tools = [t.name for t in msg.content[0].tools["functions"].tools]
vllm/entrypoints/openai/parser/harmony_utils.py
+102
-13
@@ -3,6 +3,7 @@
3
4
import datetime
5
from collections.abc import Iterable, Sequence
6
+from typing import Any
7
8
from openai.types.responses.tool import Tool
9
from openai_harmony import (
@@ -195,6 +196,12 @@ def get_user_message(content: str) -> Message:
196
return Message.from_role_and_content(Role.USER, content)
197
198
199
+def get_system_or_developer_message(role: str, instructions: str) -> Message:
200
+ if role == "system" and envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
201
+ return get_system_message(instructions=instructions)
202
+ return get_developer_message(instructions=instructions)
203
+
204
+
205
def parse_chat_inputs_to_harmony_messages(chat_msgs: list) -> list[Message]:
206
"""
207
Parse a list of messages from request.messages in the Chat Completion API to
@@ -249,18 +256,96 @@ def auto_drop_analysis_messages(msgs: list[Message]) -> list[Message]:
256
return cleaned_msgs
257
258
252
-def flatten_chat_text_content(content: str | list | None) -> str | None:
259
+def flatten_input_text_content(content: Any) -> str | None:
260
+ """
261
+ Extract text parts from a Chat Completion or Responses API content field and
262
+ flatten them into a single string. Returns None if no text content is found.
263
+ """
264
+ if content is None or isinstance(content, str):
265
+ return content
266
+ if not isinstance(content, list):
267
+ return None
268
+
269
+ texts: list[str] = []
270
+ for item in content:
271
+ if isinstance(item, str):
272
+ texts.append(item)
273
+ continue
274
+ if isinstance(item, dict):
275
+ text = item.get("text")
276
+ if text is not None:
277
+ texts.append(text)
278
+ return "".join(texts) if texts else None
279
+
280
+
281
+def extract_instructions_from_messages(
282
+ messages: Sequence[Any],
283
+) -> tuple[str | None, list[Any]]:
284
+ """
285
+ Peel a leading system/developer Chat Completion or Responses message and
286
+ flatten its instruction text.
287
+ """
288
+ remaining_messages = list(messages)
289
+ if not remaining_messages:
290
+ return None, remaining_messages
291
+
292
+ first_message = remaining_messages[0]
293
+ if not isinstance(first_message, dict):
294
+ if hasattr(first_message, "to_dict"):
295
+ # Handle OpenAI Harmony Message
296
+ first_message = first_message.to_dict()
297
+ elif hasattr(first_message, "model_dump"):
298
+ first_message = first_message.model_dump(exclude_none=True)
299
+ else:
300
+ raise ValueError(f"Unknown message type: {type(first_message)}")
301
+
302
+ if first_message.get("role") not in (
303
+ "system",
304
+ "developer",
305
+ ):
306
+ return None, remaining_messages
307
+
308
+ instructions = flatten_input_text_content(first_message.get("content"))
309
+ return instructions, remaining_messages[1:]
310
+
311
+
312
+def build_harmony_preamble(
313
+ *,
314
+ instructions: str | None = None,
315
+ tools: list[Tool | ChatCompletionToolsParam] | None = None,
316
+ reasoning_effort: str | None = None,
317
+ browser_description: str | None = None,
318
+ python_description: str | None = None,
319
+ container_description: str | None = None,
320
+ with_custom_tools: bool = False,
321
+) -> list[Message]:
322
"""
254
- Extract the text parts from a chat message content field and flatten them
255
- into a single string.
323
+ Build the standard Harmony system/developer prefix for a request.
324
"""
257
- if isinstance(content, list):
258
- return "".join(
259
- item.get("text", "")
260
- for item in content
261
- if isinstance(item, dict) and item.get("type") == "text"
325
+ developer_instructions = system_instructions = None
326
+ if envs.VLLM_GPT_OSS_HARMONY_SYSTEM_INSTRUCTIONS:
327
+ system_instructions = instructions
328
+ else:
329
+ developer_instructions = instructions
330
+
331
+ messages = [
332
+ get_system_message(
333
+ reasoning_effort=reasoning_effort,
334
+ browser_description=browser_description,
335
+ python_description=python_description,
336
+ container_description=container_description,
337
+ instructions=system_instructions,
338
+ with_custom_tools=with_custom_tools,
339
+ )
340
+ ]
341
+ if developer_instructions or tools:
342
+ messages.append(
343
+ get_developer_message(
344
+ instructions=developer_instructions,
345
+ tools=tools,
346
+ )
347
)
263
- return content
348
+ return messages
349
350
351
def parse_chat_input_to_harmony_message(
@@ -283,7 +368,7 @@ def parse_chat_input_to_harmony_message(
368
tool_calls = chat_msg.get("tool_calls", [])
369
370
if role == "assistant" and tool_calls:
286
- content = flatten_chat_text_content(chat_msg.get("content"))
371
+ content = flatten_input_text_content(chat_msg.get("content"))
372
if content:
373
commentary_msg = Message.from_role_and_content(Role.ASSISTANT, content)
374
commentary_msg = commentary_msg.with_channel("commentary")
@@ -313,8 +398,7 @@ def parse_chat_input_to_harmony_message(
398
if role == "tool":
399
tool_call_id = chat_msg.get("tool_call_id", "")
400
name = tool_id_names.get(tool_call_id, "")
316
- content = chat_msg.get("content", "") or ""
317
- content = flatten_chat_text_content(content)
401
+ content = flatten_input_text_content(chat_msg.get("content")) or ""
402
403
msg = (
404
Message.from_author_and_content(
@@ -349,7 +433,12 @@ def parse_chat_input_to_harmony_message(
433
# Send non-tool assistant messages to the final channel
434
msg = msg.with_channel("final")
435
msgs.append(msg)
352
- # For user/system/developer messages, add them directly even if no content.
436
+ elif role in ("system", "developer"):
437
+ instructions = flatten_input_text_content(chat_msg.get("content"))
438
+ if instructions is not None:
439
+ msg = get_system_or_developer_message(role, instructions)
440
+ msgs.append(msg)
441
+ # For user messages, add them directly even if no content.
442
elif role != "assistant":
443
msg = Message.from_role_and_contents(role, contents)
444
msgs.append(msg)
vllm/entrypoints/openai/responses/harmony.py
+22
-10
@@ -32,7 +32,8 @@ from openai_harmony import Author, Message, Role, StreamableParser, TextContent
32
from vllm.entrypoints.openai.parser.harmony_utils import (
33
BUILTIN_TOOL_TO_MCP_SERVER_LABEL,
34
extract_function_from_recipient,
35
- flatten_chat_text_content,
35
+ flatten_input_text_content,
36
+ get_system_or_developer_message,
37
is_function_recipient,
38
)
39
from vllm.entrypoints.openai.responses.protocol import (
@@ -109,8 +110,7 @@ def _parse_chat_format_message(chat_msg: dict) -> list[Message]:
110
name = chat_msg.get("name", "")
111
if name and not name.startswith("functions."):
112
name = f"functions.{name}"
112
- content = chat_msg.get("content", "") or ""
113
- content = flatten_chat_text_content(content)
113
+ content = flatten_input_text_content(chat_msg.get("content")) or ""
114
# NOTE: .with_recipient("assistant") is required on tool messages
115
# to match parse_chat_input_to_harmony_message behavior and ensure
116
# proper routing in the Harmony protocol.
@@ -121,7 +121,15 @@ def _parse_chat_format_message(chat_msg: dict) -> list[Message]:
121
)
122
return [msg]
123
124
- # Default: user/assistant/system messages
124
+ # System/developer messages into proper DeveloperContent
125
+ if role in ("system", "developer"):
126
+ text = flatten_input_text_content(chat_msg.get("content"))
127
+ if text:
128
+ msg = get_system_or_developer_message(role, text)
129
+ return [msg]
130
+ return []
131
+
132
+ # Default: user/assistant messages
133
content = chat_msg.get("content", "")
134
if isinstance(content, str):
135
contents = [TextContent(text=content)]
@@ -151,13 +159,17 @@ def response_input_to_harmony(
159
if "type" not in response_msg or response_msg["type"] == "message":
160
role = response_msg["role"]
161
content = response_msg["content"]
154
- # Add prefix for developer messages.
155
- # <|start|>developer<|message|># Instructions {instructions}<|end|>
156
- text_prefix = "Instructions:\n" if role == "developer" else ""
157
- if isinstance(content, str):
158
- msg = Message.from_role_and_content(role, text_prefix + content)
162
+ if role in ("system", "developer"):
163
+ text = flatten_input_text_content(content)
164
+ if text:
165
+ msg = get_system_or_developer_message(role, text)
166
+ else:
167
+ # Empty content — skip, no message emitted.
168
+ return None
169
+ elif isinstance(content, str):
170
+ msg = Message.from_role_and_content(role, content)
171
else:
160
- contents = [TextContent(text=text_prefix + c["text"]) for c in content]
172
+ contents = [TextContent(text=c.get("text", "")) for c in content]
173
msg = Message.from_role_and_contents(role, contents)
174
if role == "assistant":
175
msg = msg.with_channel("final")
vllm/entrypoints/openai/responses/serving.py
+34
-57
@@ -44,8 +44,8 @@ from vllm.entrypoints.openai.engine.serving import (
44
)
45
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
46
from vllm.entrypoints.openai.parser.harmony_utils import (
47
- get_developer_message,
48
- get_system_message,
47
+ build_harmony_preamble,
48
+ extract_instructions_from_messages,
49
get_user_message,
50
has_custom_tools,
51
render_for_completion,
@@ -1078,37 +1078,9 @@ class OpenAIServingResponses(OpenAIServing):
1078
output_items.extend(last_items)
1079
return output_items
1080
1081
- def _extract_system_message_from_request(
1082
- self, request: ResponsesRequest
1083
- ) -> str | None:
1084
- system_msg = None
1085
- if not isinstance(request.input, str):
1086
- for response_msg in request.input:
1087
- if (
1088
- isinstance(response_msg, dict)
1089
- and response_msg.get("role") == "system"
1090
- ):
1091
- content = response_msg.get("content")
1092
- if isinstance(content, str):
1093
- system_msg = content
1094
- elif isinstance(content, list):
1095
- for param in content:
1096
- if (
1097
- isinstance(param, dict)
1098
- and param.get("type") == "input_text"
1099
- ):
1100
- system_msg = param.get("text")
1101
- break
1102
- break
1103
- return system_msg
1104
-
1105
- def _construct_harmony_system_input_message(
1106
- self, request: ResponsesRequest, with_custom_tools: bool, tool_types: set[str]
1107
- ) -> OpenAIHarmonyMessage:
1108
- model_identity = self._extract_system_message_from_request(request)
1109
-
1110
- reasoning_effort = request.reasoning.effort if request.reasoning else None
1111
-
1081
+ def _get_harmony_builtin_tool_descriptions(
1082
+ self, request: ResponsesRequest, tool_types: set[str]
1083
+ ) -> dict[str, str | None]:
1084
# Extract allowed_tools from MCP tool requests
1085
allowed_tools_map = _extract_allowed_tools_from_mcp_requests(request.tools)
1086
@@ -1141,17 +1113,11 @@ class OpenAIServingResponses(OpenAIServing):
1113
and self.tool_server.has_tool("container")
1114
else None
1115
)
1144
-
1145
- sys_msg = get_system_message(
1146
- model_identity=model_identity,
1147
- reasoning_effort=reasoning_effort,
1148
- browser_description=browser_description,
1149
- python_description=python_description,
1150
- container_description=container_description,
1151
- instructions=request.instructions,
1152
- with_custom_tools=with_custom_tools,
1153
- )
1154
- return sys_msg
1116
+ return {
1117
+ "browser_description": browser_description,
1118
+ "python_description": python_description,
1119
+ "container_description": container_description,
1120
+ }
1121
1122
def _construct_input_messages_with_harmony(
1123
self,
@@ -1159,20 +1125,31 @@ class OpenAIServingResponses(OpenAIServing):
1125
prev_response: ResponsesResponse | None,
1126
) -> list[OpenAIHarmonyMessage]:
1127
messages: list[OpenAIHarmonyMessage] = []
1128
+ request_input = request.input
1129
if prev_response is None:
1130
# New conversation.
1131
tool_types = extract_tool_types(request.tools)
1132
with_custom_tools = has_custom_tools(tool_types)
1166
-
1167
- sys_msg = self._construct_harmony_system_input_message(
1168
- request, with_custom_tools, tool_types
1133
+ instructions = request.instructions
1134
+ if instructions is None and isinstance(request_input, list):
1135
+ instructions, request_input = extract_instructions_from_messages(
1136
+ request_input
1137
+ )
1138
+ tool_descriptions = self._get_harmony_builtin_tool_descriptions(
1139
+ request, tool_types
1140
)
1170
- messages.append(sys_msg)
1171
- if with_custom_tools:
1172
- dev_msg = get_developer_message(
1173
- instructions=request.instructions, tools=request.tools
1141
+ tools = request.tools if with_custom_tools else None
1142
+ messages.extend(
1143
+ build_harmony_preamble(
1144
+ instructions=instructions,
1145
+ tools=tools,
1146
+ reasoning_effort=(
1147
+ request.reasoning.effort if request.reasoning else None
1148
+ ),
1149
+ with_custom_tools=with_custom_tools,
1150
+ **tool_descriptions,
1151
)
1175
- messages.append(dev_msg)
1152
+ )
1153
messages += construct_harmony_previous_input_messages(request)
1154
1155
else:
@@ -1208,20 +1185,20 @@ class OpenAIServingResponses(OpenAIServing):
1185
messages.extend(prev_msgs)
1186
# Append the new input.
1187
# Responses API supports simple text inputs without chat format.
1211
- if isinstance(request.input, str):
1188
+ if isinstance(request_input, str):
1189
# Skip empty string input when previous_input_messages supplies
1190
# the full conversation history --- an empty trailing user message
1191
# confuses the model into thinking nothing was sent.
1215
- if request.input or not request.previous_input_messages:
1216
- messages.append(get_user_message(request.input))
1192
+ if request_input or not request.previous_input_messages:
1193
+ messages.append(get_user_message(request_input))
1194
else:
1195
if prev_response is not None:
1196
prev_outputs = copy(prev_response.output)
1197
else:
1198
prev_outputs = []
1222
- for response_msg in request.input:
1199
+ for response_msg in request_input:
1200
new_msg = response_input_to_harmony(response_msg, prev_outputs)
1224
- if new_msg is not None and new_msg.author.role != "system":
1201
+ if new_msg is not None:
1202
messages.append(new_msg)
1203
1204
# User passes in a tool call request and its output. We need
vllm/entrypoints/serve/render/serving.py
+15
-17
@@ -18,8 +18,8 @@ from vllm.entrypoints.openai.engine.protocol import (
18
)
19
from vllm.entrypoints.openai.models.serving import OpenAIModelRegistry
20
from vllm.entrypoints.openai.parser.harmony_utils import (
21
- get_developer_message,
22
- get_system_message,
21
+ build_harmony_preamble,
22
+ extract_instructions_from_messages,
23
parse_chat_inputs_to_harmony_messages,
24
render_for_completion,
25
)
@@ -405,6 +405,9 @@ class OpenAIServingRender:
405
# for more info: see comment in `maybe_serialize_tool_calls`
406
_mt.maybe_serialize_tool_calls(request) # type: ignore[arg-type]
407
408
+ chat_messages = list(request.messages)
409
+ instructions, chat_messages = extract_instructions_from_messages(chat_messages)
410
+
411
# Add system message.
412
# NOTE: In Chat Completion API, browsing is enabled by default
413
# if the model supports it. TODO: Support browsing.
@@ -412,23 +415,18 @@ class OpenAIServingRender:
415
assert not self.supports_code_interpreter
416
if (reasoning_effort := request.reasoning_effort) == "none":
417
raise ValueError(f"Harmony does not support {reasoning_effort=}")
415
- sys_msg = get_system_message(
416
- reasoning_effort=reasoning_effort,
417
- browser_description=None,
418
- python_description=None,
419
- with_custom_tools=should_include_tools,
420
- )
421
- messages.append(sys_msg)
422
-
423
- # Add developer message.
424
- if request.tools:
425
- dev_msg = get_developer_message(
426
- tools=request.tools if should_include_tools else None # type: ignore[arg-type]
418
+ tools = request.tools if should_include_tools else None
419
+ messages.extend(
420
+ build_harmony_preamble(
421
+ instructions=instructions,
422
+ tools=tools, # type: ignore[arg-type]
423
+ reasoning_effort=reasoning_effort,
424
+ with_custom_tools=should_include_tools,
425
)
428
- messages.append(dev_msg)
426
+ )
427
430
- # Add user message.
431
- messages.extend(parse_chat_inputs_to_harmony_messages(request.messages))
428
+ # Add remaining conversation messages.
429
+ messages.extend(parse_chat_inputs_to_harmony_messages(chat_messages))
430
431
# Render prompt token ids.
432
prompt_token_ids = render_for_completion(messages)