A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 3, 15:15 UTC
[Model Runner v2] Oracle for model runner v2 - qwen3 dense model by default [1/N] (#39337)
Signed-off-by: yewentao256 <zhyanwentao@126.com> Signed-off-by: Nick Hill <nickhill123@gmail.com> Signed-off-by: Wentao Ye <44945378+yewentao256@users.noreply.github.com> Co-authored-by: Nick Hill <nickhill123@gmail.com>
Wentao Ye committed
May 14, 2026 at 13:02 UTC
ae4f59f0ece88ca25ba1064ae6d3b512c3bfc606
6 files changed
+208
-20
tests/test_config.py
+99
@@ -12,6 +12,7 @@ import pytest
12
from pydantic import ValidationError
13
14
import vllm.config.vllm as vllm_config_module
15
+import vllm.envs as envs
16
from vllm.compilation.backends import VllmBackend
17
from vllm.config import (
18
CompilationConfig,
@@ -49,6 +50,104 @@ def test_compile_config_repr_succeeds():
50
assert "inductor_passes" in val
51
52
53
+@pytest.mark.parametrize(
54
+ ("env_value", "expected"),
55
+ [
56
+ (None, None),
57
+ ("0", False),
58
+ ("1", True),
59
+ ],
60
+)
61
+def test_v2_model_runner_env_tri_state(monkeypatch, env_value, expected):
62
+ if env_value is None:
63
+ monkeypatch.delenv("VLLM_USE_V2_MODEL_RUNNER", raising=False)
64
+ else:
65
+ monkeypatch.setenv("VLLM_USE_V2_MODEL_RUNNER", env_value)
66
+
67
+ assert envs.VLLM_USE_V2_MODEL_RUNNER is expected
68
+
69
+
70
+@pytest.mark.parametrize(
71
+ ("model_config", "expected"),
72
+ [
73
+ (
74
+ SimpleNamespace(
75
+ model="Qwen/Qwen3-1.7B-Base",
76
+ architectures=["Qwen3ForCausalLM"],
77
+ runner_type="generate",
78
+ is_moe=False,
79
+ is_quantized=False,
80
+ ),
81
+ True,
82
+ ),
83
+ (
84
+ SimpleNamespace(
85
+ model="Qwen/Qwen3-32B",
86
+ architectures=["Qwen3ForCausalLM"],
87
+ runner_type="generate",
88
+ is_moe=False,
89
+ is_quantized=False,
90
+ ),
91
+ True,
92
+ ),
93
+ (
94
+ SimpleNamespace(
95
+ model="facebook/opt-125m",
96
+ architectures=["OPTForCausalLM"],
97
+ runner_type="generate",
98
+ is_moe=False,
99
+ is_quantized=False,
100
+ ),
101
+ False,
102
+ ),
103
+ (
104
+ SimpleNamespace(
105
+ model="Qwen/Qwen3-30B-A3B",
106
+ architectures=["Qwen3MoeForCausalLM"],
107
+ runner_type="generate",
108
+ is_moe=True,
109
+ is_quantized=False,
110
+ ),
111
+ False,
112
+ ),
113
+ (
114
+ SimpleNamespace(
115
+ model="Qwen/Qwen3-1.7B-FP8",
116
+ architectures=["Qwen3ForCausalLM"],
117
+ runner_type="generate",
118
+ is_moe=False,
119
+ is_quantized=True,
120
+ ),
121
+ False,
122
+ ),
123
+ (
124
+ SimpleNamespace(
125
+ model="Qwen/Qwen3.5-4B",
126
+ architectures=["Qwen3_5ForConditionalGeneration"],
127
+ runner_type="generate",
128
+ is_moe=False,
129
+ is_quantized=False,
130
+ ),
131
+ False,
132
+ ),
133
+ (
134
+ SimpleNamespace(
135
+ model="Qwen/Qwen3-Embedding-0.6B",
136
+ architectures=["Qwen3ForCausalLM"],
137
+ runner_type="pooling",
138
+ is_moe=False,
139
+ is_quantized=False,
140
+ ),
141
+ False,
142
+ ),
143
+ ],
144
+)
145
+def test_is_default_v2_model_runner_model(model_config, expected):
146
+ config = SimpleNamespace(model_config=model_config)
147
+
148
+ assert VllmConfig._is_default_v2_model_runner_model(config) is expected
149
+
150
+
151
@pytest.mark.skip_global_cleanup
152
def test_with_hf_config_populates_missing_architectures_from_causal_lm_mapping(
153
monkeypatch,
vllm/config/vllm.py
+102
-12
@@ -24,6 +24,7 @@ from pydantic import ConfigDict, Field, model_validator
24
import vllm.envs as envs
25
from vllm.logger import enable_trace_function_call, init_logger
26
from vllm.transformers_utils.runai_utils import is_runai_obj_uri
27
+from vllm.triton_utils import HAS_TRITON
28
from vllm.utils import random_uuid
29
from vllm.utils.hashing import safe_hash
30
@@ -64,6 +65,8 @@ else:
65
66
logger = init_logger(__name__)
67
68
+DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES = frozenset({"Qwen3ForCausalLM"})
69
+
70
71
class OptimizationLevel(IntEnum):
72
"""Optimization level enum."""
@@ -489,6 +492,48 @@ class VllmConfig:
492
return self.speculative_config.num_speculative_tokens
493
return 0
494
495
+ @property
496
+ def use_v2_model_runner(self) -> bool:
497
+ use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
498
+ if use_v2_model_runner is not None:
499
+ return use_v2_model_runner
500
+
501
+ if not self._is_default_v2_model_runner_model():
502
+ return False
503
+
504
+ if not HAS_TRITON:
505
+ logger.warning_once(
506
+ "Model runner v2 requires Triton; using the v1 model runner instead."
507
+ )
508
+ return False
509
+
510
+ unsupported = self._get_v2_model_runner_unsupported_features()
511
+ if unsupported:
512
+ logger.warning_once(
513
+ "Model runner v2 does not yet support %s; using the v1 model "
514
+ "runner instead.",
515
+ ", ".join(unsupported),
516
+ )
517
+ return False
518
+
519
+ return True
520
+
521
+ def _is_default_v2_model_runner_model(self) -> bool:
522
+ model_config = self.model_config
523
+ if model_config is None:
524
+ return False
525
+
526
+ if model_config.runner_type != "generate":
527
+ return False
528
+
529
+ architectures = getattr(model_config, "architectures", [])
530
+ if not any(
531
+ arch in DEFAULT_V2_MODEL_RUNNER_ARCHITECTURES for arch in architectures
532
+ ):
533
+ return False
534
+
535
+ return not model_config.is_moe and not model_config.is_quantized
536
+
537
@property
538
def needs_dp_coordinator(self) -> bool:
539
"""
@@ -1233,7 +1278,7 @@ class VllmConfig:
1278
)
1279
current_platform.check_and_update_config(self)
1280
1236
- if envs.VLLM_USE_V2_MODEL_RUNNER:
1281
+ if self.use_v2_model_runner:
1282
self._validate_v2_model_runner()
1283
1284
# Re-compute compile ranges after platform-specific config updates
@@ -1908,35 +1953,72 @@ class VllmConfig:
1953
f"kernel_config={self.kernel_config!r}"
1954
)
1955
1911
- def _validate_v2_model_runner(self) -> None:
1912
- """Check for features not yet supported by the V2 model runner."""
1956
+ def _get_v2_model_runner_unsupported_features(self) -> list[str]:
1957
+ """Collect features not yet supported by the V2 model runner."""
1958
unsupported: list[str] = []
1959
+ model_config = self.model_config
1960
+ speculative_config = self.speculative_config
1961
1915
- if self.model_config is not None and self.model_config.has_inner_state:
1962
+ if model_config is not None and model_config.has_inner_state:
1963
unsupported.append("hybrid/mamba models")
1964
1965
if self.parallel_config.prefill_context_parallel_size > 1:
1966
unsupported.append("prefill context parallelism")
1967
1968
+ if self.compilation_config.mode == CompilationMode.STOCK_TORCH_COMPILE:
1969
+ unsupported.append("stock torch.compile")
1970
+
1971
if (
1922
- self.speculative_config is not None
1923
- and self.speculative_config.method not in ("eagle", "eagle3", "mtp")
1972
+ self.compilation_config.pass_config.enable_sp
1973
+ and self.parallel_config.tensor_parallel_size > 1
1974
):
1925
- unsupported.append(f"speculative method '{self.speculative_config.method}'")
1975
+ unsupported.append("sequence parallelism")
1976
+
1977
+ if speculative_config is not None:
1978
+ # TODO: ngram / ngram_gpu are not supported by the v2 model runner yet
1979
+ if speculative_config.method in ("ngram", "ngram_gpu"):
1980
+ unsupported.append("ngram/ngram_gpu speculative decoding")
1981
+ elif speculative_config.method not in ("eagle", "eagle3", "mtp"):
1982
+ unsupported.append(f"speculative method '{speculative_config.method}'")
1983
+
1984
+ if (
1985
+ speculative_config.method == "eagle3"
1986
+ and self.parallel_config.pipeline_parallel_size > 1
1987
+ ):
1988
+ unsupported.append("EAGLE3 with pipeline parallelism")
1989
+
1990
+ if self.reasoning_config is not None:
1991
+ # TODO: add reasoning budget enforcement to ModelRunnerV2.
1992
+ unsupported.append("reasoning budget enforcement")
1993
1994
if self.parallel_config.enable_dbo:
1995
unsupported.append("dual batch overlap")
1996
1930
- if (
1931
- self.model_config is not None
1932
- and self.model_config.enable_return_routed_experts
1933
- ):
1997
+ if model_config is not None and model_config.enable_return_routed_experts:
1998
# Will be added by https://github.com/vllm-project/vllm/pull/38163
1999
unsupported.append("routed experts capture")
2000
1937
- if self.model_config is not None and self.model_config.logits_processors:
2001
+ has_logitsproc_plugins = False
2002
+ if model_config is not None:
2003
+ from importlib.metadata import entry_points
2004
+
2005
+ has_logitsproc_plugins = bool(entry_points(group="vllm.logits_processors"))
2006
+
2007
+ if model_config is not None and (
2008
+ model_config.logits_processors or has_logitsproc_plugins
2009
+ ):
2010
unsupported.append("custom logits processors")
2011
2012
+ if model_config is not None and model_config.enable_prompt_embeds:
2013
+ unsupported.append("prompt embeds")
2014
+
2015
+ if (
2016
+ model_config is not None
2017
+ and model_config.runner_type == "generate"
2018
+ and model_config.logprobs_mode in ("raw_logits", "processed_logits")
2019
+ ):
2020
+ unsupported.append(f"logprobs mode '{model_config.logprobs_mode}'")
2021
+
2022
if self.cache_config.kv_sharing_fast_prefill:
2023
# Will be added by https://github.com/vllm-project/vllm/pull/35045
2024
unsupported.append("KV sharing fast prefill")
@@ -1945,6 +2027,14 @@ class VllmConfig:
2027
# Will be added by https://github.com/vllm-project/vllm/pull/38390
2028
unsupported.append("EC transfer")
2029
2030
+ return unsupported
2031
+
2032
+ def _validate_v2_model_runner(self) -> None:
2033
+ """Check for features not yet supported by the V2 model runner."""
2034
+ if not HAS_TRITON:
2035
+ raise ValueError("VLLM_USE_V2_MODEL_RUNNER requires Triton.")
2036
+
2037
+ unsupported = self._get_v2_model_runner_unsupported_features()
2038
if unsupported:
2039
raise ValueError(
2040
"VLLM_USE_V2_MODEL_RUNNER does not yet support: "
vllm/envs.py
+4
-4
@@ -248,7 +248,7 @@ if TYPE_CHECKING:
248
VLLM_SHARED_EXPERTS_STREAM_TOKEN_THRESHOLD: int = 256
249
VLLM_MULTI_STREAM_GEMM_TOKEN_THRESHOLD: int = 1024
250
VLLM_COMPILE_CACHE_SAVE_FORMAT: Literal["binary", "unpacked"] = "binary"
251
- VLLM_USE_V2_MODEL_RUNNER: bool = False
251
+ VLLM_USE_V2_MODEL_RUNNER: bool | None = None
252
VLLM_LOG_MODEL_INSPECTION: bool = False
253
VLLM_DEBUG_MFU_METRICS: bool = False
254
VLLM_WEIGHT_OFFLOADING_DISABLE_PIN_MEMORY: bool = False
@@ -1710,9 +1710,9 @@ environment_variables: dict[str, Callable[[], Any]] = {
1710
"VLLM_COMPILE_CACHE_SAVE_FORMAT": env_with_choices(
1711
"VLLM_COMPILE_CACHE_SAVE_FORMAT", "binary", ["binary", "unpacked"]
1712
),
1713
- # Flag to enable v2 model runner.
1714
- "VLLM_USE_V2_MODEL_RUNNER": lambda: bool(
1715
- int(os.getenv("VLLM_USE_V2_MODEL_RUNNER", "0"))
1713
+ # Flag to control the v2 model runner. If unset, use config defaults.
1714
+ "VLLM_USE_V2_MODEL_RUNNER": lambda: maybe_convert_bool(
1715
+ os.getenv("VLLM_USE_V2_MODEL_RUNNER", None)
1716
),
1717
# Log model inspection after loading.
1718
# If enabled, logs a transformers-style hierarchical view of the model
vllm/v1/attention/backends/flashinfer.py
+1
-1
@@ -681,7 +681,7 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
681
# reused CPU buffers to avoid a race condition between step N async copies to
682
# GPU and step N+1 buffer updates.
683
self.pin_memory = (
684
- not envs.VLLM_USE_V2_MODEL_RUNNER and is_pin_memory_available()
684
+ not vllm_config.use_v2_model_runner and is_pin_memory_available()
685
)
686
self.paged_kv_indptr = self._make_buffer(max_num_reqs + 1)
687
self.paged_kv_indptr_cpu_buffer = torch.zeros_like(
vllm/v1/core/sched/scheduler.py
+1
-2
@@ -7,7 +7,6 @@ from collections.abc import Iterable
7
from dataclasses import replace
8
from typing import Any
9
10
-from vllm import envs
10
from vllm.compilation.cuda_graph import CUDAGraphStat
11
from vllm.config import VllmConfig
12
from vllm.distributed.ec_transfer.ec_connector.base import (
@@ -242,7 +241,7 @@ class Scheduler(SchedulerInterface):
241
self.connector.bind_gpu_block_pool(self.kv_cache_manager.block_pool)
242
243
self.use_pp = self.parallel_config.pipeline_parallel_size > 1
245
- self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
244
+ self.use_v2_model_runner = vllm_config.use_v2_model_runner
245
self.scheduler_reserve_full_isl = (
246
self.scheduler_config.scheduler_reserve_full_isl
247
)
vllm/v1/worker/gpu_worker.py
+1
-1
@@ -153,7 +153,7 @@ class Worker(WorkerBase):
153
if self.profiler_config.profiler not in ("torch", "cuda", None):
154
raise ValueError(f"Unknown profiler type: {self.profiler_config.profiler}")
155
156
- self.use_v2_model_runner = envs.VLLM_USE_V2_MODEL_RUNNER
156
+ self.use_v2_model_runner = vllm_config.use_v2_model_runner
157
# pending non-blocking PP send work from the previous iteration
158
self._pp_send_work: list[Handle] = []
159