A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 2, 17:01 UTC
[BugFix] Fix blocked reasoning parsing with MRV2 (#43808)
Signed-off-by: Nick Hill <nickhill123@gmail.com>
Nick Hill committed
May 27, 2026 at 21:59 UTC
626fa9bba5663a5cf6a870debf031ee344ddb822
3 files changed
+36
-22
tests/entrypoints/openai/chat_completion/test_thinking_token_budget.py
+9
-4
@@ -72,7 +72,9 @@ def server():
72
"0.4",
73
"--no-async-scheduling",
74
]
75
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
75
+ # thinking_token_budget is not yet supported by the V2 model runner.
76
+ env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
77
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
78
yield remote_server
79
80
@@ -88,7 +90,9 @@ def server_with_auto_reasoning_config():
90
"0.4",
91
"--no-async-scheduling",
92
]
91
- with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
93
+ # thinking_token_budget is not yet supported by the V2 model runner.
94
+ env_dict = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
95
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=env_dict) as remote_server:
96
yield remote_server
97
98
@@ -122,11 +126,12 @@ def server_qwen35_fp8_mtp_tp2():
126
}
127
),
128
]
129
+ # thinking_token_budget is not yet supported by the V2 model runner.
130
+ env_dict: dict[str, str] = {"VLLM_USE_V2_MODEL_RUNNER": "0"}
131
# With 4+ GPUs, run TP=2 on physical devices 2,3 so module-scoped 0.6B servers
132
# on 0,1 do not exhaust memory on the same devices as this worker.
127
- env_dict = None
133
if current_platform.device_count() >= 4:
129
- env_dict = {"CUDA_VISIBLE_DEVICES": "2,3"}
134
+ env_dict["CUDA_VISIBLE_DEVICES"] = "2,3"
135
136
with RemoteOpenAIServer(
137
QWEN35_FP8_MTP_MODEL,
vllm/config/vllm.py
+10
-9
@@ -500,14 +500,14 @@ class VllmConfig:
500
501
if not HAS_TRITON:
502
logger.warning_once(
503
- "Model runner v2 requires Triton; using the v1 model runner instead."
503
+ "Model Runner V2 requires Triton; using the V1 model runner instead."
504
)
505
return False
506
507
unsupported = self._get_v2_model_runner_unsupported_features()
508
if unsupported:
509
logger.warning_once(
510
- "Model runner v2 does not yet support %s; using the v1 model "
510
+ "Model Runner V2 does not yet support %s; using the V1 model "
511
"runner instead.",
512
", ".join(unsupported),
513
)
@@ -1982,10 +1982,6 @@ class VllmConfig:
1982
):
1983
unsupported.append("EAGLE3 with pipeline parallelism")
1984
1985
- if self.reasoning_config is not None:
1986
- # TODO: add reasoning budget enforcement to ModelRunnerV2.
1987
- unsupported.append("reasoning budget enforcement")
1988
-
1985
if self.parallel_config.enable_dbo:
1986
unsupported.append("dual batch overlap")
1987
@@ -2027,13 +2023,18 @@ class VllmConfig:
2023
def _validate_v2_model_runner(self) -> None:
2024
"""Check for features not yet supported by the V2 model runner."""
2025
if not HAS_TRITON:
2030
- raise ValueError("VLLM_USE_V2_MODEL_RUNNER requires Triton.")
2026
+ raise ValueError("Model Runner V2 requires Triton.")
2027
2028
unsupported = self._get_v2_model_runner_unsupported_features()
2029
if unsupported:
2030
raise ValueError(
2035
- "VLLM_USE_V2_MODEL_RUNNER does not yet support: "
2036
- + ", ".join(unsupported)
2031
+ f"Model Runner V2 does not yet support: {', '.join(unsupported)}"
2032
+ )
2033
+
2034
+ if self.reasoning_config is not None:
2035
+ logger.warning_once(
2036
+ "Model Runner V2 does not yet support the thinking_token_budget "
2037
+ "request parameter. Set VLLM_USE_V2_MODEL_RUNNER=0 if this is required."
2038
)
2039
2040
def validate_block_size(self) -> None:
vllm/v1/engine/input_processor.py
+17
-9
@@ -49,6 +49,7 @@ class InputProcessor:
49
self.speculative_config = vllm_config.speculative_config
50
self.structured_outputs_config = vllm_config.structured_outputs_config
51
self.observability_config = vllm_config.observability_config
52
+ self.use_v2_model_runner = vllm_config.use_v2_model_runner
53
54
self.generation_config_fields = model_config.try_get_generation_config()
55
@@ -98,15 +99,22 @@ class InputProcessor:
99
self.tokenizer,
100
)
101
101
- if params.thinking_token_budget is not None and (
102
- self.vllm_config.reasoning_config is None
103
- or not self.vllm_config.reasoning_config.enabled
104
- ):
105
- raise ValueError(
106
- "thinking_token_budget is set but reasoning_config is "
107
- "not configured. Please set --reasoning-config to use "
108
- "thinking_token_budget."
109
- )
102
+ if params.thinking_token_budget is not None:
103
+ if (
104
+ self.vllm_config.reasoning_config is None
105
+ or not self.vllm_config.reasoning_config.enabled
106
+ ):
107
+ raise ValueError(
108
+ "thinking_token_budget is set but reasoning_config is "
109
+ "not configured. Please set --reasoning-parser "
110
+ "and/or --reasoning-config to use thinking_token_budget."
111
+ )
112
+ if self.use_v2_model_runner:
113
+ raise ValueError(
114
+ "thinking_token_budget is not yet supported by the V2 "
115
+ "model runner. Run vLLM with VLLM_USE_V2_MODEL_RUNNER=0 "
116
+ "to use thinking_token_budget."
117
+ )
118
elif isinstance(params, PoolingParams):
119
supported_pooling_tasks = [
120
task for task in supported_tasks if task in POOLING_TASKS