A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 09:45 UTC
[Fronten] Clean up stop_token_ids override for Harmony (#44009)
Signed-off-by: Yifan Zong <yzong@redhat.com>
yzong-rh committed
May 29, 2026 at 16:28 UTC
46409fd2a101ccef1988a250179eefea6f1845d7
4 files changed
-23
vllm/entrypoints/openai/chat_completion/serving.py
-8
@@ -57,7 +57,6 @@ from vllm.entrypoints.openai.engine.serving import (
57
)
58
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
59
from vllm.entrypoints.openai.parser.harmony_utils import (
60
- get_stop_tokens_for_assistant_actions,
60
get_streamable_parser_for_assistant,
61
parse_chat_output,
62
)
@@ -158,13 +157,6 @@ class OpenAIServingChat(OpenAIServing):
157
else getattr(mc, "override_generation_config", {}).get("max_new_tokens")
158
)
159
self.use_harmony = self.model_config.hf_config.model_type == "gpt_oss"
161
- if self.use_harmony:
162
- if "stop_token_ids" not in self.default_sampling_params:
163
- self.default_sampling_params["stop_token_ids"] = []
164
- self.default_sampling_params["stop_token_ids"].extend(
165
- get_stop_tokens_for_assistant_actions()
166
- )
167
-
160
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
161
162
# NOTE(woosuk): While OpenAI's chat completion API supports browsing
vllm/entrypoints/openai/parser/harmony_utils.py
-4
@@ -365,10 +365,6 @@ def render_for_completion(messages: list[Message]) -> list[int]:
365
return token_ids
366
367
368
-def get_stop_tokens_for_assistant_actions() -> list[int]:
369
- return get_encoding().stop_tokens_for_assistant_actions()
370
-
371
-
368
def get_streamable_parser_for_assistant() -> StreamableParser:
369
return StreamableParser(get_encoding(), role=Role.ASSISTANT)
370
vllm/entrypoints/openai/responses/protocol.py
-3
@@ -372,8 +372,6 @@ class ResponsesRequest(OpenAIBaseModel):
372
if (frequency_penalty := self.frequency_penalty) is None:
373
frequency_penalty = default_sampling_params.get("frequency_penalty", 0.0)
374
375
- stop_token_ids = default_sampling_params.get("stop_token_ids")
376
-
375
# Structured output
376
structured_outputs = self.structured_outputs
377
@@ -409,7 +407,6 @@ class ResponsesRequest(OpenAIBaseModel):
407
top_k=top_k,
408
max_tokens=max_tokens,
409
logprobs=self.top_logprobs if self.is_include_output_logprobs() else None,
412
- stop_token_ids=stop_token_ids,
410
stop=stop,
411
frequency_penalty=frequency_penalty,
412
presence_penalty=presence_penalty,
vllm/entrypoints/openai/responses/serving.py
-8
@@ -46,7 +46,6 @@ from vllm.entrypoints.openai.engine.serving import (
46
from vllm.entrypoints.openai.models.serving import OpenAIServingModels
47
from vllm.entrypoints.openai.parser.harmony_utils import (
48
get_developer_message,
49
- get_stop_tokens_for_assistant_actions,
49
get_system_message,
50
get_user_message,
51
has_custom_tools,
@@ -222,13 +221,6 @@ class OpenAIServingResponses(OpenAIServing):
221
"For gpt-oss, we ignore --enable-auto-tool-choice "
222
"and always enable tool use."
223
)
225
- # OpenAI models have two EOS-like tokens: <|return|> and <|call|>.
226
- # We need to add them to the stop token ids.
227
- if "stop_token_ids" not in self.default_sampling_params:
228
- self.default_sampling_params["stop_token_ids"] = []
229
- self.default_sampling_params["stop_token_ids"].extend(
230
- get_stop_tokens_for_assistant_actions()
231
- )
224
225
self.tool_call_id_type = get_tool_call_id_type(self.model_config)
226