@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 03878d1c22
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Deprecations for v0.23 and v0.24 (#44992)

Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>

Harry Mellor committed Jun 11, 2026 at 15:35 UTC 03878d1c221b0eaeadc7fb6ffb82bd33df2f8555
35 files changed +102 -676
.buildkite/lm-eval-harness/configs/NVIDIA-Nemotron-3-Nano-30B-A3B-FP8.yaml
+1 -3
@@ -6,9 +6,7 @@ tasks:
6 value: 0.7142
7 - name: "exact_match,flexible-extract"
8 value: 0.4579
9 -env_vars:
10 - VLLM_USE_FLASHINFER_MOE_FP8: "1"
11 - VLLM_FLASHINFER_MOE_BACKEND: "throughput"
9 +moe_backend: "flashinfer_cutlass"
10 limit: 1319
11 num_fewshot: 5
12 max_model_len: 262144
.buildkite/lm-eval-harness/test_lm_eval_correctness.py
+4
@@ -68,6 +68,10 @@ def launch_lm_eval(eval_config, tp_size):
68 if current_platform.is_rocm() and "Nemotron-3" in eval_config["model_name"]:
69 model_args += "attention_backend=TRITON_ATTN"
70
71 + moe_backend = eval_config.get("moe_backend", None)
72 + if moe_backend is not None:
73 + model_args += f"moe_backend={moe_backend},"
74 +
75 env_vars = eval_config.get("env_vars", None)
76 with scoped_env_vars(env_vars):
77 results = lm_eval.simple_evaluate(
docs/design/moe_kernel_features.md
+1 -1
@@ -42,7 +42,7 @@ th {
42 1. All types: mxfp4, nvfp4, int4, int8, fp8
43 2. A,T quantization occurs after dispatch.
44 3. All quantization happens after dispatch.
45 - 4. Controlled by different env vars (`VLLM_FLASHINFER_MOE_BACKEND` "throughput" or "latency")
45 + 4. Controlled by `--moe-backend` (`flashinfer_cutlass` or `flashinfer_trtllm`)
46 5. This is a no-op dispatcher that can be used to pair with any modular experts to produce a modular kernel that runs without dispatch or combine. These cannot be selected via environment variable. These are generally use for testing or adapting an expert subclass to the `fused_experts` API.
47 6. This depends on the experts implementation.
48
docs/models/pooling_models/reward.md
+1 -1
@@ -143,4 +143,4 @@ More examples can be found here: [examples/pooling/reward](../../../examples/poo
143
144 ### `LLM.reward`
145
146 -`llm.reward` api is deprecated and will be removed in v0.23. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
146 +`llm.reward` API is deprecated and was removed in v0.24. Please use `LLM.encode` with `pooling_task="classify"` or `pooling_task="token_classify"` instead.
tests/compile/correctness_e2e/test_async_tp.py
+3 -3
@@ -102,7 +102,7 @@ def test_async_tp_pass_correctness(
102
103
104 @create_new_process_for_each_test()
105 -def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
105 +def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int):
106 if (
107 not current_platform.is_cuda()
108 or not current_platform.is_device_capability_family(100)
@@ -111,8 +111,6 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
111 if not has_flashinfer():
112 pytest.skip("FlashInfer is required for the NVFP4 AsyncTP path")
113
114 - monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", "flashinfer-cutlass")
115 -
114 tp_size = 2
115 if num_gpus_available < tp_size:
116 pytest.skip(f"Need at least {tp_size} GPUs")
@@ -126,6 +124,8 @@ def test_async_tp_pass_nvfp4_correctness(num_gpus_available: int, monkeypatch):
124 "8",
125 "--load-format",
126 "dummy",
127 + "--linear-backend",
128 + "flashinfer_cutlass",
129 "--hf-overrides",
130 json.dumps(NVFP4_HF_OVERRIDES),
131 ]
tests/conftest.py
-4
@@ -1226,10 +1226,6 @@ class VllmRunner:
1226 req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
1227 return [req_output.outputs.data for req_output in req_outputs]
1228
1229 - def reward(self, prompts: list[str]) -> list[list[float]]:
1230 - req_outputs = self.llm.encode(prompts, pooling_task="token_classify")
1231 - return [req_output.outputs.data for req_output in req_outputs]
1232 -
1229 def score(
1230 self,
1231 text_1: list[str] | str,
tests/distributed/test_eplb_fused_moe_layer_dep_nvfp4.py
+5 -6
@@ -37,6 +37,7 @@ class TestConfig:
37 hidden_size: int
38 intermediate_size: int
39 num_tokens: int
40 + moe_backend: str
41
42
43 def make_fused_moe_layer(
@@ -114,6 +115,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
115 vllm_config = VllmConfig()
116 vllm_config.parallel_config.data_parallel_size = world_size
117 vllm_config.parallel_config.enable_expert_parallel = True
118 + vllm_config.kernel_config.moe_backend = test_config.moe_backend
119
120 with set_current_vllm_config(vllm_config):
121 ensure_model_parallel_initialized(
@@ -250,7 +252,7 @@ def _test_eplb_fml(env, world_size: int, test_config: TestConfig):
252 @pytest.mark.parametrize("hidden_size", [256])
253 @pytest.mark.parametrize("intermediate_size", [256])
254 @pytest.mark.parametrize("num_tokens", [256])
253 -@pytest.mark.parametrize("backend", ["latency", "throughput"])
255 +@pytest.mark.parametrize("moe_backend", ["flashinfer_trtllm", "flashinfer_cutlass"])
256 def test_eplb_fml(
257 world_size: int,
258 num_layers: int,
@@ -258,12 +260,8 @@ def test_eplb_fml(
260 hidden_size: int,
261 intermediate_size: int,
262 num_tokens: int,
261 - backend: str,
262 - monkeypatch,
263 + moe_backend: str,
264 ):
264 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP4", "1")
265 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", backend)
266 -
265 if torch.accelerator.device_count() < world_size:
266 pytest.skip(f"Need at least {world_size} GPUs to run the test")
267
@@ -278,6 +276,7 @@ def test_eplb_fml(
276 hidden_size=hidden_size,
277 intermediate_size=intermediate_size,
278 num_tokens=num_tokens,
279 + moe_backend=moe_backend,
280 )
281
282 distributed_run(
tests/entrypoints/pooling/reward/test_token_reward_offline.py
+2 -1
@@ -45,9 +45,10 @@ def test_config(llm: LLM):
45
46 def test_pooling_params(llm: LLM):
47 def get_outputs(use_activation):
48 - outputs = llm.reward(
48 + outputs = llm.encode(
49 prompts,
50 pooling_params=PoolingParams(use_activation=use_activation),
51 + pooling_task="token_classify",
52 use_tqdm=False,
53 )
54 return torch.cat([x.outputs.data for x in outputs])
tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml renamed
+1 -3
@@ -3,6 +3,4 @@
3 model_name: "openai/gpt-oss-20b"
4 metric_threshold: 0.568
5 reasoning_effort: "low"
6 -server_args: "--tensor-parallel-size 2"
7 -env:
8 - VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: "1"
6 +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_cutlass"
tests/evals/gpt_oss/configs/gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml new
+6
@@ -0,0 +1,6 @@
1 +# SPDX-License-Identifier: Apache-2.0
2 +# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3 +model_name: "openai/gpt-oss-20b"
4 +metric_threshold: 0.568
5 +reasoning_effort: "low"
6 +server_args: "--tensor-parallel-size 2 --moe-backend flashinfer_trtllm"
tests/evals/gpt_oss/configs/gpt-oss-20b-marlin.yaml
+1 -3
@@ -3,6 +3,4 @@
3 model_name: "openai/gpt-oss-20b"
4 metric_threshold: 0.568
5 reasoning_effort: "low"
6 -server_args: "--tensor-parallel-size 2"
7 -env:
8 - VLLM_MXFP4_USE_MARLIN: "1"
6 +server_args: "--tensor-parallel-size 2 --moe-backend marlin --linear-backend marlin"
tests/evals/gpt_oss/configs/models-b200.txt
+1 -1
@@ -1,5 +1,5 @@
1 # B200 model configurations for GPQA evaluation
2 # Tests different environment variable combinations
3 -gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
3 +gpt-oss-20b-flashinfer-mxfp4-bf16-trtllm.yaml
4 gpt-oss-20b-flashinfer-mxfp4-mxfp8-cutlass.yaml
5 gpt-oss-20b-sm100-fi-mxfp4-mxfp8-trtllm.yaml
\ No newline at end of file
tests/evals/gpt_oss/configs/models-h100.txt
+1 -1
@@ -1,5 +1,5 @@
1 # H100 model configurations for GPQA evaluation
2 # Tests different environment variable combinations
3 gpt-oss-20b-baseline.yaml
4 -gpt-oss-20b-flashinfer-mxfp4-bf16.yaml
4 +gpt-oss-20b-flashinfer-mxfp4-bf16-cutlass.yaml
5 gpt-oss-20b-marlin.yaml
tests/evals/gsm8k/README.md
+2 -2
@@ -30,9 +30,9 @@ model_name: "Qwen/Qwen2.5-1.5B-Instruct"
30 accuracy_threshold: 0.54 # Minimum expected accuracy
31 num_questions: 1319 # Number of questions (default: full test set)
32 num_fewshot: 5 # Few-shot examples from train set
33 -server_args: "--max-model-len 4096 --tensor-parallel-size 2" # Server arguments
33 +server_args: "--max-model-len 4096 --tensor-parallel-size 2 --moe-backend flashinfer_cutlass" # Server arguments
34 env: # Environment variables (optional)
35 - VLLM_USE_FLASHINFER_MOE_FP4: "1"
35 + VLLM_LOGGING_LEVEL: "DEBUG"
36 ```
37
38 The `server_args` field accepts any arguments that can be passed to `vllm serve`.
tests/kernels/moe/test_moe.py
+1 -3
@@ -1585,7 +1585,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
1585 e: int,
1586 topk: int,
1587 dtype: torch.dtype,
1588 - monkeypatch,
1588 workspace_init,
1589 ):
1590 """
@@ -1593,8 +1592,6 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
1592 """
1593 set_random_seed(7)
1594
1596 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
1597 -
1595 from vllm.model_executor.layers.fused_moe.config import (
1596 FusedMoEConfig,
1597 FusedMoEParallelConfig,
@@ -1626,6 +1623,7 @@ def test_unquantized_bf16_flashinfer_trtllm_backend(
1623 in_dtype=dtype,
1624 routing_method=RoutingMethodType.Renormalize,
1625 max_num_tokens=next_power_of_2(m),
1626 + moe_backend="flashinfer_trtllm",
1627 )
1628
1629 with set_current_vllm_config(vllm_config):
tests/kernels/moe/test_moe_layer.py
+3 -6
@@ -1804,12 +1804,9 @@ def test_moe_layer(
1804 if os.environ.get("VLLM_LOGGING_LEVEL") is None:
1805 monkeypatch.setenv("VLLM_LOGGING_LEVEL", "ERROR")
1806
1807 - # TODO
1808 - # VLLM_FLASHINFER_MOE_BACKEND=latency
1809 - # VLLM_USE_FLASHINFER_MOE_FP16=1
1810 - # VLLM_USE_FLASHINFER_MOE_FP8
1811 - # VLLM_USE_FLASHINFER_MOE_FP4
1812 - # VLLM_USE_FLASHINFER_MOE_INT4
1807 + # TODO: cover FlashInfer MoE backends via moe_backend, e.g.
1808 + # moe_backend=flashinfer_trtllm / flashinfer_cutlass / flashinfer_cutedsl
1809 + # (BF16, FP8 and NVFP4 paths), and VLLM_USE_FLASHINFER_MOE_INT4=1.
1810
1811 parallel_config = ParallelConfig(
1812 pipeline_parallel_size=1,
tests/kernels/moe/test_unquantized_backend_selection.py
+4 -41
@@ -123,7 +123,7 @@ def test_select_rocm_aiter_backend(mock_aiter_enabled, mock_has_flashinfer):
123 @pytest.mark.skipif(
124 not current_platform.is_cuda(), reason="Only supported on NVIDIA platforms."
125 )
126 -def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeypatch):
126 +def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm):
127 """Test CUDA backend selection when FlashInfer TRTLLM is available and enabled."""
128 with (
129 patch.object(current_platform, "is_cuda", return_value=True),
@@ -134,9 +134,8 @@ def test_select_cuda_flashinfer_trtllm_backend(mock_is_supported_trtllm, monkeyp
134 patch.object(current_platform, "is_out_of_tree", return_value=False),
135 patch.object(current_platform, "has_device_capability", return_value=True),
136 ):
137 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
138 -
137 moe_config = make_dummy_moe_config()
138 + moe_config.moe_backend = "flashinfer_trtllm"
139 # TRTLLM requires EP and does not support DP
140 moe_config.moe_parallel_config.use_ep = True
141 moe_config.moe_parallel_config.use_dp = False
@@ -168,7 +167,6 @@ def test_select_cuda_flashinfer_cutlass_backend(
167 mock_has_flashinfer,
168 mock_is_supported_trtllm,
169 mock_is_supported_cutlass,
171 - monkeypatch,
170 ):
171 """Test CUDA backend selection when FlashInfer TRTLLM is not available
172 and FlashInfer CUTLASS is available."""
@@ -181,10 +179,9 @@ def test_select_cuda_flashinfer_cutlass_backend(
179 patch.object(current_platform, "is_out_of_tree", return_value=False),
180 patch.object(current_platform, "has_device_capability", return_value=True),
181 ):
184 - # Enable FlashInfer via env var
185 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
186 -
182 moe_config = make_dummy_moe_config()
183 + # Select FlashInfer CUTLASS explicitly
184 + moe_config.moe_backend = "flashinfer_cutlass"
185 # CUTLASS requires EP and does not support DP
186 moe_config.moe_parallel_config.use_ep = True
187 moe_config.moe_parallel_config.use_dp = False
@@ -241,37 +238,3 @@ def test_select_explicit_triton_backend(is_lora_enabled):
238
239 assert selected_backend == UnquantizedMoeBackend.TRITON
240 assert experts_cls is not None
244 -
245 -
246 -@skipif_not_cuda_rocm
247 -def test_select_explicit_triton_ignores_flashinfer_env(monkeypatch):
248 - """Explicit triton backend should override FlashInfer env selection."""
249 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
250 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
251 -
252 - moe_config = make_dummy_moe_config()
253 - moe_config.is_lora_enabled = False
254 - moe_config.moe_backend = "triton"
255 -
256 - selected_backend, experts_cls = select_unquantized_moe_backend(
257 - moe_config=moe_config
258 - )
259 -
260 - assert selected_backend == UnquantizedMoeBackend.TRITON
261 - assert experts_cls is not None
262 -
263 -
264 -@skipif_not_cuda_rocm
265 -def test_select_lora_ignores_flashinfer_env(monkeypatch):
266 - """LoRA path should still choose Triton even if FlashInfer env is on."""
267 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_FP16", "1")
268 - monkeypatch.setenv("VLLM_FLASHINFER_MOE_BACKEND", "throughput")
269 -
270 - moe_config = make_dummy_moe_config()
271 - moe_config.is_lora_enabled = True
272 - selected_backend, experts_cls = select_unquantized_moe_backend(
273 - moe_config=moe_config
274 - )
275 -
276 - assert selected_backend == UnquantizedMoeBackend.TRITON
277 - assert experts_cls is not None
tests/lora/test_gptoss_tp.py
+38 -40
@@ -83,57 +83,55 @@ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
83 @pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
84 @pytest.mark.parametrize("specialize_active_lora", [True, False])
85 def test_gpt_oss_lora(
86 - monkeypatch: pytest.MonkeyPatch,
86 gptoss20b_lora_files,
87 mxfp4_use_marlin,
88 specialize_active_lora,
89 ):
91 - with monkeypatch.context() as m:
92 - m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
93 - llm = vllm.LLM(
94 - MODEL_PATH,
95 - max_model_len=1024,
96 - enable_lora=True,
97 - max_loras=4,
98 - max_lora_rank=8,
99 - max_num_seqs=2,
100 - max_num_batched_tokens=2048,
101 - specialize_active_lora=specialize_active_lora,
102 - compilation_config=vllm.config.CompilationConfig( # Avoid OOM
103 - cudagraph_specialize_lora=False,
104 - ),
105 - )
106 -
107 - generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
108 - generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
90 + llm = vllm.LLM(
91 + MODEL_PATH,
92 + max_model_len=1024,
93 + enable_lora=True,
94 + max_loras=4,
95 + max_lora_rank=8,
96 + max_num_seqs=2,
97 + max_num_batched_tokens=2048,
98 + specialize_active_lora=specialize_active_lora,
99 + moe_backend="marlin" if mxfp4_use_marlin else "auto",
100 + linear_backend="marlin" if mxfp4_use_marlin else "auto",
101 + compilation_config=vllm.config.CompilationConfig( # Avoid OOM
102 + cudagraph_specialize_lora=False,
103 + ),
104 + )
105 +
106 + generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
107 + generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
108
109
110 @multi_gpu_test(num_gpus=2)
111 @pytest.mark.parametrize("fully_sharded_loras", [False, True])
112 @pytest.mark.parametrize("mxfp4_use_marlin", [True, False])
113 def test_gpt_oss_lora_tp2(
115 - monkeypatch: pytest.MonkeyPatch,
114 gptoss20b_lora_files,
115 fully_sharded_loras,
116 mxfp4_use_marlin,
117 ):
120 - with monkeypatch.context() as m:
121 - m.setenv("VLLM_MXFP4_USE_MARLIN", "1" if mxfp4_use_marlin else "0")
122 - llm = vllm.LLM(
123 - MODEL_PATH,
124 - max_model_len=1024,
125 - enable_lora=True,
126 - max_loras=2,
127 - max_num_seqs=2,
128 - max_num_batched_tokens=2048,
129 - tensor_parallel_size=2,
130 - gpu_memory_utilization=0.8,
131 - fully_sharded_loras=fully_sharded_loras,
132 - enable_expert_parallel=not fully_sharded_loras,
133 - compilation_config=vllm.config.CompilationConfig(
134 - cudagraph_specialize_lora=False,
135 - ),
136 - )
137 -
138 - generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
139 - generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
118 + llm = vllm.LLM(
119 + MODEL_PATH,
120 + max_model_len=1024,
121 + enable_lora=True,
122 + max_loras=2,
123 + max_num_seqs=2,
124 + max_num_batched_tokens=2048,
125 + tensor_parallel_size=2,
126 + gpu_memory_utilization=0.8,
127 + fully_sharded_loras=fully_sharded_loras,
128 + enable_expert_parallel=not fully_sharded_loras,
129 + moe_backend="marlin" if mxfp4_use_marlin else "auto",
130 + linear_backend="marlin" if mxfp4_use_marlin else "auto",
131 + compilation_config=vllm.config.CompilationConfig(
132 + cudagraph_specialize_lora=False,
133 + ),
134 + )
135 +
136 + generate_and_test(llm, gptoss20b_lora_files, lora_id=1)
137 + generate_and_test(llm, gptoss20b_lora_files, lora_id=2)
tests/models/language/pooling/test_pooler_config_init_behaviour.py
+2 -2
@@ -106,7 +106,7 @@ def test_reward_models_using_activation(
106 dtype=dtype,
107 pooler_config=PoolerConfig(use_activation=False),
108 ) as vllm_model:
109 - wo_activation = vllm_model.reward(example_prompts)
109 + wo_activation = vllm_model.token_classify(example_prompts)
110
111 with vllm_runner(
112 model,
@@ -114,7 +114,7 @@ def test_reward_models_using_activation(
114 dtype=dtype,
115 pooler_config=PoolerConfig(use_activation=True),
116 ) as vllm_model:
117 - w_activation = vllm_model.reward(example_prompts)
117 + w_activation = vllm_model.token_classify(example_prompts)
118
119 for wo, w in zip(wo_activation, w_activation):
120 wo = torch.tensor(wo)
tests/models/language/pooling/test_reward.py
+2 -2
@@ -107,7 +107,7 @@ def test_prm_models(
107 pytest.skip("CPU only supports V1")
108
109 with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
110 - vllm_outputs = vllm_model.reward(math_step_prompts)
110 + vllm_outputs = vllm_model.token_classify(math_step_prompts)
111
112 with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
113 hf_model = step_reward_patch_hf_model(hf_model)
@@ -146,7 +146,7 @@ def test_prm_models_with_golden_outputs(
146 pytest.skip(f"No available golden outputs for {model}.")
147
148 with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
149 - vllm_outputs = vllm_model.reward(math_step_prompts)
149 + vllm_outputs = vllm_model.token_classify(math_step_prompts)
150
151 golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
152
tests/models/quantization/test_nvfp4.py
+2 -2
@@ -133,11 +133,11 @@ def test_nvfp4(vllm_runner, model, eager, backend):
133 not current_platform.is_rocm(),
134 reason="NVFP4 MOE emulation is only useful on AMD Instinct MI3xx",
135 )
136 -def test_nvfp4_moe(vllm_runner, model, backend, monkeypatch):
137 - monkeypatch.setenv("VLLM_NVFP4_GEMM_BACKEND", backend)
136 +def test_nvfp4_moe(vllm_runner, model, backend):
137 with vllm_runner(
138 model,
139 moe_backend=backend,
140 + linear_backend=backend,
141 load_format="dummy",
142 hf_overrides={"num_hidden_layers": 2},
143 ) as llm:
tests/quantization/test_blackwell_moe.py
+5 -2
@@ -185,8 +185,11 @@ def test_deepseek_nvfp4_moe_flashinfer_trtllm(monkeypatch: pytest.MonkeyPatch):
185
186
187 def test_gptoss_mxfp4bf16_moe_flashinfer(monkeypatch: pytest.MonkeyPatch):
188 - monkeypatch.setenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "1")
189 - can_initialize("openai/gpt-oss-20b", hf_overrides=HF_OVERRIDE_TEXT)
188 + can_initialize(
189 + "openai/gpt-oss-20b",
190 + hf_overrides=HF_OVERRIDE_TEXT,
191 + extra_args=["--moe-backend=flashinfer_trtllm"],
192 + )
193
194
195 def test_gptoss_mxfp4mxfp8_moe_flashinfer_cutlass(monkeypatch: pytest.MonkeyPatch):
vllm/benchmarks/datasets/datasets.py
+1 -7
@@ -1617,7 +1617,6 @@ def add_dataset_parser(parser: FlexibleArgumentParser):
1617 "custom",
1618 "custom_audio",
1619 "custom_image",
1620 - "custom_mm",
1620 "prefix_repetition",
1621 "spec_bench",
1622 "speed_bench",
@@ -2106,12 +2105,7 @@ def get_samples(args, tokenizer: TokenizerLike) -> list[SampleRequest]:
2105 no_oversample=args.no_oversample,
2106 )
2107
2109 - elif args.dataset_name in ("custom_image", "custom_mm"):
2110 - if args.dataset_name == "custom_mm":
2111 - logger.warning(
2112 - "Dataset name 'custom_mm' is deprecated and will be removed in v0.24. "
2113 - "Use '--dataset-name custom_image' instead."
2114 - )
2108 + elif args.dataset_name == "custom_image":
2109 dataset = CustomImageDataset(
2110 dataset_path=args.dataset_path,
2111 disable_shuffle=args.disable_shuffle,
vllm/config/compilation.py
-11
@@ -134,10 +134,6 @@ class PassConfig:
134 """Enable async TP."""
135 fuse_allreduce_rms: bool = None # type: ignore[assignment]
136 """Enable flashinfer allreduce fusion."""
137 - fuse_minimax_qk_norm: bool = None # type: ignore[assignment]
138 - """Deprecated. The MiniMax QK norm fusion is now applied automatically at
139 - runtime (see `MiniMaxText01RMSNormTP.forward_qkv`). This flag is kept for
140 - backward compatibility and has no effect; it will be removed in v0.23."""
137 enable_qk_norm_rope_fusion: bool = None # type: ignore[assignment]
138 """Enable fused Q/K RMSNorm + RoPE pass."""
139 fuse_rope_kvcache_cat_mla: bool = None # type: ignore[assignment]
@@ -296,13 +292,6 @@ class PassConfig:
292 "current platform is not CUDA or ROCm. The fusion will be disabled."
293 )
294 self.fuse_rope_kvcache_cat_mla = False
299 - if self.fuse_minimax_qk_norm is not None:
300 - logger.warning_once(
301 - "`fuse_minimax_qk_norm` is deprecated and has no effect; "
302 - "the MiniMax QK norm fusion is now applied automatically at "
303 - "runtime when its conditions are met. This flag will be "
304 - "removed in v0.23."
305 - )
295
296 def log_enabled_passes(self) -> None:
297 """
vllm/config/kernel.py
+2
@@ -142,6 +142,7 @@ LinearBackend = Literal[
142 "flashinfer_cutlass",
143 "flashinfer_trtllm",
144 "flashinfer_cudnn",
145 + "flashinfer_b12x",
146 "marlin",
147 "triton",
148 "deep_gemm",
@@ -197,6 +198,7 @@ class KernelConfig:
198 - "flashinfer_cutlass": Use FlashInfer with CUTLASS kernels
199 - "flashinfer_trtllm": Use FlashInfer with TensorRT-LLM kernels
200 - "flashinfer_cudnn": Use FlashInfer with cuDNN kernels
201 + - "flashinfer_b12x": Use FlashInfer b12x CuteDSL NVFP4 GEMM (SM120+)
202 - "marlin": Use Marlin kernels
203 - "triton": Use Triton-based kernels
204 - "deep_gemm": Use DeepGEMM kernels
vllm/entrypoints/pooling/offline.py
-44
@@ -286,50 +286,6 @@ class PoolingOfflineMixin(OfflineInferenceMixin):
286
287 return [ClassificationRequestOutput.from_base(item) for item in items]
288
289 - def reward(
290 - self,
291 - prompts: PromptType | Sequence[PromptType],
292 - /,
293 - *,
294 - pooling_params: PoolingParams | Sequence[PoolingParams] | None = None,
295 - use_tqdm: bool | Callable[..., tqdm] = True,
296 - lora_request: list[LoRARequest] | LoRARequest | None = None,
297 - tokenization_kwargs: dict[str, Any] | None = None,
298 - ) -> list[PoolingRequestOutput]:
299 - """
300 - Generate rewards for each prompt.
301 -
302 - Args:
303 - prompts: The prompts to the LLM. You may pass a sequence of prompts
304 - for batch inference. See [PromptType][vllm.inputs.PromptType]
305 - for more details about the format of each prompt.
306 - pooling_params: The pooling parameters for pooling. If None, we
307 - use the default pooling parameters.
308 - use_tqdm: If `True`, shows a tqdm progress bar.
309 - If a callable (e.g., `functools.partial(tqdm, leave=False)`),
310 - it is used to create the progress bar.
311 - If `False`, no progress bar is created.
312 - lora_request: LoRA request to use for generation, if any.
313 - tokenization_kwargs: Overrides for `tokenizer.encode`.
314 -
315 - Returns:
316 - A list of `PoolingRequestOutput` objects containing the
317 - pooled hidden states in the same order as the input prompts.
318 - """
319 - logger.warning_once(
320 - "`llm.reward` api is deprecated and will be removed in v0.23. "
321 - 'Please use `LLM.encode` with `pooling_task="classify"` or '
322 - '`pooling_task="token_classify"` instead.'
323 - )
324 - return self.encode(
325 - prompts,
326 - use_tqdm=use_tqdm,
327 - lora_request=lora_request,
328 - pooling_params=pooling_params,
329 - pooling_task="token_classify",
330 - tokenization_kwargs=tokenization_kwargs,
331 - )
332 -
289 def score(
290 self,
291 data_1: ScoreInput | list[ScoreInput],
vllm/envs.py
-168
@@ -8,7 +8,6 @@ import os
8 import sys
9 import tempfile
10 import uuid
11 -import warnings
11 from collections.abc import Callable
12 from typing import TYPE_CHECKING, Any, Literal
13
@@ -167,7 +166,6 @@ if TYPE_CHECKING:
166 VLLM_HUMMING_INPUT_QUANT_CONFIG: dict[str, Any] | None = None
167 VLLM_HUMMING_USE_F16_ACCUM: bool = False
168 VLLM_HUMMING_MOE_GEMM_TYPE: Literal["indexed", "grouped", "auto"] | None = None
170 - VLLM_MXFP4_USE_MARLIN: bool | None = None
169 VLLM_DEEPEPLL_NVFP4_DISPATCH: bool = False
170 VLLM_V1_USE_OUTLINES_CACHE: bool = False
171 VLLM_TPU_BUCKET_PADDING_GAP: int = 0
@@ -184,13 +182,7 @@ if TYPE_CHECKING:
182 ] = "relax"
183 VLLM_USE_FUSED_MOE_GROUPED_TOPK: bool = True
184 VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER: bool = True
187 - VLLM_USE_FLASHINFER_MOE_FP16: bool = False
188 - VLLM_USE_FLASHINFER_MOE_FP8: bool = False
189 - VLLM_USE_FLASHINFER_MOE_FP4: bool = False
185 VLLM_USE_FLASHINFER_MOE_INT4: bool = False
191 - VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = (
192 - "latency"
193 - )
186 VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR: str | None = None
187 VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto"
188 VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024
@@ -212,7 +204,6 @@ if TYPE_CHECKING:
204 VLLM_KV_CACHE_LAYOUT: Literal["NHD", "HND"] | None = None
205 VLLM_SSM_CONV_STATE_LAYOUT: Literal["SD", "DS"] | None = None
206 VLLM_COMPUTE_NANS_IN_LOGITS: bool = False
215 - VLLM_USE_NVFP4_CT_EMULATIONS: bool = False
207 VLLM_ROCM_QUICK_REDUCE_QUANTIZATION: Literal[
208 "FP", "INT8", "INT6", "INT4", "NONE"
209 ] = "NONE"
@@ -225,12 +216,8 @@ if TYPE_CHECKING:
216 VLLM_LOOPBACK_IP: str = ""
217 VLLM_ALLOW_CHUNKED_LOCAL_ATTN_WITH_HYBRID_KV_CACHE: bool = True
218 VLLM_ENABLE_RESPONSES_API_STORE: bool = False
228 - VLLM_NVFP4_GEMM_BACKEND: str | None = None
219 VLLM_HAS_FLASHINFER_CUBIN: bool = False
230 - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: bool = False
231 - VLLM_USE_FLASHINFER_MOE_MXFP4_BF16: bool = False
220 VLLM_ROCM_FP8_MFMA_PAGE_ATTN: bool = False
233 - VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS: bool = False
221 VLLM_ALLREDUCE_USE_SYMM_MEM: bool = True
222 VLLM_ALLREDUCE_USE_FLASHINFER: bool = False
223 VLLM_TUNED_CONFIG_FOLDER: str | None = None
@@ -257,7 +244,6 @@ if TYPE_CHECKING:
244 VLLM_ENABLE_INDUCTOR_COORDINATE_DESCENT_TUNING: bool = True
245 VLLM_USE_NCCL_SYMM_MEM: bool = False
246 VLLM_NCCL_INCLUDE_PATH: str | None = None
260 - VLLM_USE_FBGEMM: bool = False
247 VLLM_GC_DEBUG: str = ""
248 VLLM_DEBUG_WORKSPACE: bool = False
249 VLLM_DISABLE_SHARED_EXPERTS_STREAM: bool = False
@@ -350,27 +336,6 @@ def use_mega_aot_artifact():
336 return os.environ.get("VLLM_USE_MEGA_AOT_ARTIFACT", default_value) == "1"
337
338
353 -def deprecated_env(
354 - env_name: str,
355 - removal_version: str,
356 - replacement: str,
357 - getter: Callable[[], Any],
358 -) -> Callable[[], Any]:
359 - """Wrap an env-var getter to emit a FutureWarning when the var is set."""
360 -
361 - def _read() -> Any:
362 - if env_name in os.environ:
363 - warnings.warn(
364 - f"{env_name} is deprecated and will be removed in "
365 - f"{removal_version}. {replacement}",
366 - FutureWarning,
367 - stacklevel=2,
368 - )
369 - return getter()
370 -
371 - return _read
372 -
373 -
339 def env_with_choices(
340 env_name: str,
341 default: str | None,
@@ -1371,15 +1336,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1336 "VLLM_MARLIN_USE_ATOMIC_ADD": lambda: (
1337 os.environ.get("VLLM_MARLIN_USE_ATOMIC_ADD", "0") == "1"
1338 ),
1374 - # Whether to use marlin kernel in mxfp4 quantization method
1375 - # Deprecated: use --moe-backend marlin (MoE) or --linear-backend marlin
1376 - # (linear) instead.
1377 - "VLLM_MXFP4_USE_MARLIN": deprecated_env(
1378 - "VLLM_MXFP4_USE_MARLIN",
1379 - "v0.23",
1380 - "Use --moe-backend marlin or --linear-backend marlin.",
1381 - lambda: maybe_convert_bool(os.environ.get("VLLM_MXFP4_USE_MARLIN", None)),
1382 - ),
1339 # The activation dtype for marlin kernel
1340 "VLLM_MARLIN_INPUT_DTYPE": env_with_choices(
1341 "VLLM_MARLIN_INPUT_DTYPE", None, ["int8", "fp8"]
@@ -1472,68 +1428,10 @@ environment_variables: dict[str, Callable[[], Any]] = {
1428 "VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER": lambda: bool(
1429 int(os.getenv("VLLM_BLOCKSCALE_FP8_GEMM_FLASHINFER", "1"))
1430 ),
1475 - # Allow use of FlashInfer BF16 MoE kernels for fused moe ops.
1476 - # Deprecated: use --moe-backend to select a kernel explicitly.
1477 - "VLLM_USE_FLASHINFER_MOE_FP16": deprecated_env(
1478 - "VLLM_USE_FLASHINFER_MOE_FP16",
1479 - "v0.23",
1480 - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
1481 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP16", "0"))),
1482 - ),
1483 - # Allow use of FlashInfer FP8 MoE kernels for fused moe ops.
1484 - # Deprecated: use --moe-backend to select a kernel explicitly.
1485 - "VLLM_USE_FLASHINFER_MOE_FP8": deprecated_env(
1486 - "VLLM_USE_FLASHINFER_MOE_FP8",
1487 - "v0.23",
1488 - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
1489 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP8", "0"))),
1490 - ),
1491 - # Allow use of FlashInfer NVFP4 MoE kernels for fused moe ops.
1492 - # Deprecated: use --moe-backend to select a kernel explicitly.
1493 - "VLLM_USE_FLASHINFER_MOE_FP4": deprecated_env(
1494 - "VLLM_USE_FLASHINFER_MOE_FP4",
1495 - "v0.23",
1496 - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass, "
1497 - "flashinfer_cutedsl).",
1498 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_FP4", "0"))),
1499 - ),
1431 # Allow use of FlashInfer MxInt4 MoE kernels for fused moe ops.
1432 "VLLM_USE_FLASHINFER_MOE_INT4": lambda: bool(
1433 int(os.getenv("VLLM_USE_FLASHINFER_MOE_INT4", "0"))
1434 ),
1504 - # If set to 1, use the FlashInfer
1505 - # MXFP8 (activation) x MXFP4 (weight) MoE backend.
1506 - # Deprecated: use --moe-backend flashinfer_trtllm combined with
1507 - # --quantization_config.moe.activation mxfp8.
1508 - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8": deprecated_env(
1509 - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8",
1510 - "v0.23",
1511 - "Use --moe-backend flashinfer_trtllm with "
1512 - "--quantization_config.moe.activation mxfp8.",
1513 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8", "0"))),
1514 - ),
1515 - # If set to 1, use the FlashInfer CUTLASS backend for
1516 - # MXFP8 (activation) x MXFP4 (weight) MoE.
1517 - # Deprecated: use --moe-backend flashinfer_cutlass combined with
1518 - # --quantization_config.moe.activation mxfp8.
1519 - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS": deprecated_env(
1520 - "VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS",
1521 - "v0.23",
1522 - "Use --moe-backend flashinfer_cutlass with "
1523 - "--quantization_config.moe.activation mxfp8.",
1524 - lambda: bool(
1525 - int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS", "0"))
1526 - ),
1527 - ),
1528 - # If set to 1, use the FlashInfer
1529 - # BF16 (activation) x MXFP4 (weight) MoE backend.
1530 - # Deprecated: use --moe-backend to select a kernel explicitly.
1531 - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16": deprecated_env(
1532 - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16",
1533 - "v0.23",
1534 - "Use --moe-backend (e.g. flashinfer_trtllm, flashinfer_cutlass).",
1535 - lambda: bool(int(os.getenv("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16", "0"))),
1536 - ),
1435 # Control the cache sized used by the xgrammar compiler. The default
1436 # of 512 MB should be enough for roughly 1000 JSON schemas.
1437 # It can be changed with this variable if needed for some reason.
@@ -1585,25 +1483,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1483 "MOONCAKE_REQUESTER_LOCAL_HOSTNAME": lambda: os.getenv(
1484 "MOONCAKE_REQUESTER_LOCAL_HOSTNAME"
1485 ),
1588 - # Flashinfer MoE backend for vLLM's fused Mixture-of-Experts support.
1589 - # Both require compute capability 10.0 or above.
1590 - # Available options:
1591 - # - "throughput": [default]
1592 - # Uses CUTLASS kernels optimized for high-throughput batch inference.
1593 - # - "latency":
1594 - # Uses TensorRT-LLM kernels optimized for low-latency inference.
1595 - # Deprecated: pass --moe-backend flashinfer_{trtllm,cutlass,cutedsl} directly.
1596 - "VLLM_FLASHINFER_MOE_BACKEND": deprecated_env(
1597 - "VLLM_FLASHINFER_MOE_BACKEND",
1598 - "v0.23",
1599 - "Use --moe-backend flashinfer_trtllm, flashinfer_cutlass, or "
1600 - "flashinfer_cutedsl.",
1601 - env_with_choices(
1602 - "VLLM_FLASHINFER_MOE_BACKEND",
1603 - "latency",
1604 - ["throughput", "latency", "masked_gemm"],
1605 - ),
1606 - ),
1486 # Override the directory for the FlashInfer autotune config cache.
1487 "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR": lambda: os.getenv(
1488 "VLLM_FLASHINFER_AUTOTUNE_CACHE_DIR", None
@@ -1681,16 +1560,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1560 "VLLM_COMPUTE_NANS_IN_LOGITS": lambda: bool(
1561 int(os.getenv("VLLM_COMPUTE_NANS_IN_LOGITS", "0"))
1562 ),
1684 - # Controls whether or not emulations are used for NVFP4
1685 - # generations on machines < 100 for compressed-tensors
1686 - # models
1687 - # Deprecated: use --linear-backend emulation instead.
1688 - "VLLM_USE_NVFP4_CT_EMULATIONS": deprecated_env(
1689 - "VLLM_USE_NVFP4_CT_EMULATIONS",
1690 - "v0.23",
1691 - "Use --linear-backend emulation.",
1692 - lambda: bool(int(os.getenv("VLLM_USE_NVFP4_CT_EMULATIONS", "0"))),
1693 - ),
1563 # Timeout (in seconds) for MooncakeConnector in PD disaggregated setup.
1564 "VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT": lambda: int(
1565 os.getenv("VLLM_MOONCAKE_ABORT_REQUEST_TIMEOUT", "480")
@@ -1700,35 +1569,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1569 "VLLM_HAS_FLASHINFER_CUBIN": lambda: bool(
1570 int(os.getenv("VLLM_HAS_FLASHINFER_CUBIN", "0"))
1571 ),
1703 - # Supported options:
1704 - # - "flashinfer-cudnn": use flashinfer cudnn GEMM backend
1705 - # - "flashinfer-trtllm": use flashinfer trtllm GEMM backend
1706 - # - "flashinfer-cutlass": use flashinfer cutlass GEMM backend
1707 - # - "marlin": use marlin GEMM backend (for GPUs without native FP4 support)
1708 - # - "emulation":
1709 - # use BF16/FP16 GEMM, dequantizing weights and running QDQ on activations.
1710 - # This is only meant for research purposes to run on devices where NVFP4
1711 - # GEMM kernels are not available.
1712 - # - <none>: automatically pick an available backend
1713 - # Deprecated: use --linear-backend instead.
1714 - "VLLM_NVFP4_GEMM_BACKEND": deprecated_env(
1715 - "VLLM_NVFP4_GEMM_BACKEND",
1716 - "v0.23",
1717 - "Use --linear-backend.",
1718 - env_with_choices(
1719 - "VLLM_NVFP4_GEMM_BACKEND",
1720 - None,
1721 - [
1722 - "flashinfer-b12x",
1723 - "flashinfer-cudnn",
1724 - "flashinfer-trtllm",
1725 - "flashinfer-cutlass",
1726 - "cutlass",
1727 - "marlin",
1728 - "emulation",
1729 - ],
1730 - ),
1731 - ),
1572 # Controls garbage collection during CUDA graph capture.
1573 # If set to 0 (default), enables GC freezing to speed up capture time.
1574 # If set to 1, allows GC to run during capture.
@@ -1892,14 +1732,6 @@ environment_variables: dict[str, Callable[[], Any]] = {
1732 ),
1733 # NCCL header path
1734 "VLLM_NCCL_INCLUDE_PATH": lambda: os.environ.get("VLLM_NCCL_INCLUDE_PATH", None),
1895 - # Flag to enable FBGemm kernels on model execution
1896 - # Deprecated: use --linear-backend fbgemm instead.
1897 - "VLLM_USE_FBGEMM": deprecated_env(
1898 - "VLLM_USE_FBGEMM",
1899 - "v0.23",
1900 - "Use --linear-backend fbgemm.",
1901 - lambda: bool(int(os.getenv("VLLM_USE_FBGEMM", "0"))),
1902 - ),
1735 # GC debug config
1736 # - VLLM_GC_DEBUG=0: disable GC debugger
1737 # - VLLM_GC_DEBUG=1: enable GC debugger with gc.collect elpased times
vllm/model_executor/kernels/linear/__init__.py
+8 -47
@@ -212,6 +212,9 @@ _LINEAR_BACKEND_KERNEL_MAP: dict[str, set[type]] = {
212 "flashinfer_cudnn": {
213 FlashInferCudnnNvFp4LinearKernel,
214 },
215 + "flashinfer_b12x": {
216 + FlashInferB12xNvFp4LinearKernel,
217 + },
218 "marlin": {
219 MarlinFP8ScaledMMLinearKernel,
220 MarlinLinearKernel,
@@ -392,7 +395,7 @@ _POSSIBLE_NVFP4_KERNELS: dict[PlatformEnum, list[type[NvFp4LinearKernel]]] = {
395 PlatformEnum.CUDA: [
396 # FlashInferB12xNvFp4LinearKernel excluded from auto-selection until
397 # upstream CUTLASS SM121 MMA op guard is resolved; use
395 - # VLLM_NVFP4_GEMM_BACKEND=flashinfer-b12x to opt in explicitly.
398 + # --linear-backend flashinfer_b12x to opt in explicitly.
399 FlashInferCutlassNvFp4LinearKernel,
400 CutlassNvFp4LinearKernel,
401 MarlinNvFp4LinearKernel,
@@ -752,20 +755,6 @@ def init_mxfp4_linear_kernel() -> MxFp4LinearKernel:
755 current platform."""
756 linear_backend = _get_linear_backend()
757
755 - force_kernel: type[MxFp4LinearKernel] | None = None
756 - if linear_backend == "auto" and envs.VLLM_MXFP4_USE_MARLIN:
757 - force_kernel = MarlinMxFp4LinearKernel
758 -
759 - if force_kernel is not None:
760 - is_supported, reason = force_kernel.is_supported()
761 - if not is_supported:
762 - raise ValueError(
763 - f"Forced MXFP4 kernel {force_kernel.__name__} is not "
764 - f"supported: {reason}"
765 - )
766 - logger.info_once("Using %s for MXFP4 GEMM", force_kernel.__name__)
767 - return force_kernel(MxFp4LinearLayerConfig())
768 -
758 platform = current_platform._enum
759 possible = list(_POSSIBLE_MXFP4_KERNELS.get(platform, []))
760
@@ -836,18 +825,6 @@ def init_wfp8_a16_linear_kernel(
825 )
826
827
839 -# Maps VLLM_NVFP4_GEMM_BACKEND env var values to kernel classes.
840 -_NVFP4_BACKEND_TO_KERNEL: dict[str, type[NvFp4LinearKernel]] = {
841 - "flashinfer-b12x": FlashInferB12xNvFp4LinearKernel,
842 - "flashinfer-cutlass": FlashInferCutlassNvFp4LinearKernel,
843 - "cutlass": CutlassNvFp4LinearKernel,
844 - "marlin": MarlinNvFp4LinearKernel,
845 - "flashinfer-trtllm": FlashInferTrtllmNvFp4LinearKernel,
846 - "flashinfer-cudnn": FlashInferCudnnNvFp4LinearKernel,
847 - "emulation": EmulationNvFp4LinearKernel,
848 -}
849 -
850 -
828 def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
829 """Select and instantiate the best NVFP4 linear kernel for the
830 current platform."""
@@ -855,8 +832,7 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
832
833 # VLLM_BATCH_INVARIANT forces deterministic execution. Prefer the
834 # batch-invariant CUTLASS implementation when available, otherwise fall
858 - # back to emulation. It overrides both --linear-backend and the deprecated
859 - # env vars below.
835 + # back to emulation. It overrides --linear-backend.
836 force_kernel: type[NvFp4LinearKernel] | None = None
837 linear_backend = _get_linear_backend()
838 if envs.VLLM_BATCH_INVARIANT:
@@ -888,24 +864,9 @@ def init_nvfp4_linear_kernel(use_a16: bool = False) -> NvFp4LinearKernel:
864 reason,
865 )
866 force_kernel = EmulationNvFp4LinearKernel
891 - elif linear_backend == "auto":
892 - # Deprecated env-var overrides — only honoured when --linear-backend
893 - # is "auto". Deprecation warnings are emitted from vllm/envs.py.
894 - if use_a16: # force a16 if running weight-only quantization
895 - force_kernel = MarlinNvFp4LinearKernel
896 - elif envs.VLLM_USE_FBGEMM:
897 - force_kernel = FbgemmNvFp4LinearKernel
898 - elif envs.VLLM_USE_NVFP4_CT_EMULATIONS:
899 - force_kernel = EmulationNvFp4LinearKernel
900 - elif envs.VLLM_NVFP4_GEMM_BACKEND is not None:
901 - backend_name = envs.VLLM_NVFP4_GEMM_BACKEND
902 - force_kernel = _NVFP4_BACKEND_TO_KERNEL.get(backend_name)
903 - if force_kernel is None:
904 - raise ValueError(
905 - f"Unknown VLLM_NVFP4_GEMM_BACKEND={backend_name!r}. "
906 - f"Valid choices: "
907 - f"{list(_NVFP4_BACKEND_TO_KERNEL.keys())}"
908 - )
867 + elif linear_backend == "auto" and use_a16:
868 + # Force a16 (Marlin) when running weight-only quantization.
869 + force_kernel = MarlinNvFp4LinearKernel
870
871 if force_kernel is not None:
872 is_supported, reason = force_kernel.is_supported()
vllm/model_executor/layers/fused_moe/oracle/fp8.py
-50
@@ -20,8 +20,6 @@ from vllm.model_executor.layers.fused_moe.config import (
20 )
21 from vllm.model_executor.layers.fused_moe.routed_experts import RoutedExperts
22 from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
23 - FlashinferMoeBackend,
24 - get_flashinfer_moe_backend,
23 prepare_fp8_moe_layer_for_fi,
24 )
25 from vllm.model_executor.layers.quantization.utils.fp8_utils import (
@@ -321,54 +319,6 @@ def select_fp8_moe_backend(
319 requested_backend, config, weight_key, activation_key, activation_format
320 )
321
324 - # Handle explicit FlashInfer FP8 configuration.
325 - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP8"):
326 - if not envs.VLLM_USE_FLASHINFER_MOE_FP8:
327 - # If the user rejects FlashInfer remove those backends.
328 - AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_TRTLLM)
329 - AVAILABLE_BACKENDS.remove(Fp8MoeBackend.FLASHINFER_CUTLASS)
330 -
331 - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
332 - # If user is explicit about backend, validate it.
333 - fi_backend = get_flashinfer_moe_backend()
334 - if fi_backend == FlashinferMoeBackend.CUTLASS:
335 - backend = Fp8MoeBackend.FLASHINFER_CUTLASS
336 - elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
337 - backend = Fp8MoeBackend.FLASHINFER_TRTLLM
338 - else:
339 - raise ValueError(
340 - f"FlashInfer MOE backend {fi_backend} does not support FP8 MoE."
341 - )
342 - k_cls = backend_to_kernel_cls(backend)[0]
343 - return _return_or_raise(
344 - backend, config, weight_key, activation_key, activation_format
345 - )
346 - else:
347 - # If the user is not explicit about the backend, try both.
348 - for backend in [
349 - Fp8MoeBackend.FLASHINFER_TRTLLM,
350 - Fp8MoeBackend.FLASHINFER_CUTLASS,
351 - ]:
352 - for k_cls in backend_to_kernel_cls(backend):
353 - supported, reason = k_cls.is_supported_config(
354 - k_cls,
355 - config,
356 - weight_key,
357 - activation_key,
358 - activation_format,
359 - )
360 -
361 - if supported:
362 - logger.info_once(_make_log_backend(backend))
363 - return backend, k_cls
364 - else:
365 - logger.debug_once(_make_log_unsupported(backend, reason))
366 -
367 - raise NotImplementedError(
368 - "Found VLLM_USE_FLASHINFER_MOE_FP8=1, but no "
369 - "FlashInfer FP8 MoE backend supports the configuration."
370 - )
371 -
322 # Handle explicit DeepGEMM FP8 configuration.
323 if envs.is_set("VLLM_USE_DEEP_GEMM") or envs.is_set("VLLM_MOE_USE_DEEP_GEMM"):
324 if not envs.VLLM_USE_DEEP_GEMM or not envs.VLLM_MOE_USE_DEEP_GEMM:
vllm/model_executor/layers/fused_moe/oracle/mxfp4.py
-69
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING, Literal, Union
6 import torch
7
8 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
9 -from vllm import envs
9 from vllm.config import get_current_vllm_config
10 from vllm.config.kernel import MoEBackend
11 from vllm.config.quantization import QuantizationConfigArgs
@@ -465,74 +464,6 @@ def select_mxfp4_moe_backend(
464 _get_priority_backends_for_gpt_oss(), requested_activation_key
465 )
466
468 - # Handle explicit FlashInfer MXFP4 BF16 configuration.
469 - if envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_BF16"):
470 - if not envs.VLLM_USE_FLASHINFER_MOE_MXFP4_BF16:
471 - for _b in (
472 - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
473 - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
474 - ):
475 - if _b in AVAILABLE_BACKENDS:
476 - AVAILABLE_BACKENDS.remove(_b)
477 - else:
478 - if current_platform.is_device_capability(90):
479 - return _return_or_raise(
480 - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_BF16,
481 - config,
482 - kMxfp4Static,
483 - None,
484 - activation_format,
485 - )
486 - if current_platform.is_device_capability_family(100):
487 - return _return_or_raise(
488 - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_BF16,
489 - config,
490 - kMxfp4Static,
491 - None,
492 - activation_format,
493 - )
494 - raise ValueError(
495 - "VLLM_USE_FLASHINFER_MOE_MXFP4_BF16=1 is set but the "
496 - "current device capability is not supported. "
497 - "Only SM90 (CUTLASS) and SM100+ (TRTLLM) are supported."
498 - )
499 -
500 - # Handle explicit FlashInfer MXFP4 MXFP8 TRTLLM configuration.
501 - if (
502 - envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8")
503 - and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8
504 - ):
505 - return _return_or_raise(
506 - Mxfp4MoeBackend.FLASHINFER_TRTLLM_MXFP4_MXFP8,
507 - config,
508 - kMxfp4Static,
509 - kMxfp8Dynamic,
510 - activation_format,
511 - )
512 -
513 - # Handle explicit FlashInfer MXFP4 MXFP8 CUTLASS configuration.
514 - if (
515 - envs.is_set("VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS")
516 - and envs.VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8_CUTLASS
517 - ):
518 - return _return_or_raise(
519 - Mxfp4MoeBackend.FLASHINFER_CUTLASS_MXFP4_MXFP8,
520 - config,
521 - kMxfp4Static,
522 - kMxfp8Dynamic,
523 - activation_format,
524 - )
525 -
526 - # Handle explicit Marlin MXFP4 configuration.
527 - if envs.is_set("VLLM_MXFP4_USE_MARLIN") and envs.VLLM_MXFP4_USE_MARLIN:
528 - return _return_or_raise(
529 - Mxfp4MoeBackend.MARLIN,
530 - config,
531 - kMxfp4Static,
532 - None,
533 - activation_format,
534 - )
535 -
467 for backend in AVAILABLE_BACKENDS:
468 # Use requested_activation_key if provided, otherwise use backend default
469 act_key = (
vllm/model_executor/layers/fused_moe/oracle/nvfp4.py
-59
@@ -22,10 +22,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_fp4_moe import (
22 prepare_nvfp4_moe_layer_for_fi_or_cutlass,
23 prepare_nvfp4_moe_layer_for_flashinfer_cutedsl,
24 )
25 -from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
26 - FlashinferMoeBackend,
27 - get_flashinfer_moe_backend,
28 -)
25 from vllm.model_executor.layers.quantization.utils.marlin_utils_fp4 import (
26 prepare_nvfp4_moe_layer_for_marlin,
27 )
@@ -58,12 +54,6 @@ FLASHINFER_NVFP4_MOE_BACKENDS = [
54 NvFp4MoeBackend.FLASHINFER_B12X,
55 ]
56
61 -fi_2_vllm_backend_map: dict[FlashinferMoeBackend, NvFp4MoeBackend] = {
62 - FlashinferMoeBackend.CUTLASS: NvFp4MoeBackend.FLASHINFER_CUTLASS,
63 - FlashinferMoeBackend.TENSORRT_LLM: NvFp4MoeBackend.FLASHINFER_TRTLLM,
64 - FlashinferMoeBackend.CUTEDSL: NvFp4MoeBackend.FLASHINFER_CUTEDSL,
65 -}
66 -
57
58 def is_global_sf_supported_for_nvfp4_backend(backend: NvFp4MoeBackend) -> bool:
59 # Checks whether `backend` supports quantizing with scaling factors
@@ -258,55 +248,6 @@ def select_nvfp4_moe_backend(
248 requested_backend, config, weight_key, activation_key, activation_format
249 )
250
261 - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP4"):
262 - if not envs.VLLM_USE_FLASHINFER_MOE_FP4:
263 - # If the user rejects FlashInfer remove those backends.
264 - for b in FLASHINFER_NVFP4_MOE_BACKENDS:
265 - if b in AVAILABLE_BACKENDS:
266 - AVAILABLE_BACKENDS.remove(b)
267 -
268 - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
269 - # If user is explicit about backend, validate it.
270 - backend = fi_2_vllm_backend_map[get_flashinfer_moe_backend()]
271 - if (
272 - config.swiglu_limit is not None
273 - and backend not in NVFP4_BACKENDS_WITH_CLAMP
274 - ):
275 - raise ValueError(
276 - f"Model sets swiglu_limit={config.swiglu_limit}, but the "
277 - f"FlashInfer backend selected via VLLM_FLASHINFER_MOE_BACKEND "
278 - f"({backend.value}) does not apply the SwiGLU clamp."
279 - )
280 - return _return_or_raise(
281 - backend, config, weight_key, activation_key, activation_format
282 - )
283 - else:
284 - # If the user is not explicit about the backend, try each.
285 - fi_backends = [
286 - b
287 - for b in FLASHINFER_NVFP4_MOE_BACKENDS
288 - if config.swiglu_limit is None or b in NVFP4_BACKENDS_WITH_CLAMP
289 - ]
290 - for backend in fi_backends:
291 - for k_cls in backend_to_kernel_cls(backend):
292 - supported, reason = k_cls.is_supported_config(
293 - k_cls,
294 - config,
295 - weight_key,
296 - activation_key,
297 - activation_format,
298 - )
299 - if supported:
300 - logger.info_once(_make_log_backend(backend))
301 - return backend, k_cls
302 - else:
303 - logger.debug_once(_make_log_unsupported(backend, reason))
304 -
305 - raise NotImplementedError(
306 - "Found VLLM_USE_FLASHINFER_MOE_FP4=1, but no "
307 - "FlashInfer NVFP4 MoE backend supports the configuration."
308 - )
309 -
251 if envs.VLLM_TEST_FORCE_FP8_MARLIN:
252 backend = NvFp4MoeBackend.MARLIN
253 return _return_or_raise(
vllm/model_executor/layers/fused_moe/oracle/unquantized.py
-45
@@ -19,9 +19,7 @@ from vllm.model_executor.layers.fused_moe.config import (
19 FusedMoEQuantConfig,
20 )
21 from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
22 - FlashinferMoeBackend,
22 convert_moe_weights_to_flashinfer_trtllm_block_layout,
24 - get_flashinfer_moe_backend,
23 swap_w13_to_w31,
24 )
25 from vllm.platforms import current_platform
@@ -230,49 +228,6 @@ def select_unquantized_moe_backend(
228
229 return _return_or_raise(requested_backend, moe_config, activation_format)
230
233 - # Handle explicit FlashInfer FP16 configuration.
234 - if envs.is_set("VLLM_USE_FLASHINFER_MOE_FP16"):
235 - if not envs.VLLM_USE_FLASHINFER_MOE_FP16:
236 - if UnquantizedMoeBackend.FLASHINFER_TRTLLM in AVAILABLE_BACKENDS:
237 - AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_TRTLLM)
238 - if UnquantizedMoeBackend.FLASHINFER_CUTLASS in AVAILABLE_BACKENDS:
239 - AVAILABLE_BACKENDS.remove(UnquantizedMoeBackend.FLASHINFER_CUTLASS)
240 -
241 - elif envs.is_set("VLLM_FLASHINFER_MOE_BACKEND"):
242 - # If user is explicit about backend, validate it.
243 - fi_backend = get_flashinfer_moe_backend()
244 - if fi_backend == FlashinferMoeBackend.CUTLASS:
245 - backend = UnquantizedMoeBackend.FLASHINFER_CUTLASS
246 - elif fi_backend == FlashinferMoeBackend.TENSORRT_LLM:
247 - backend = UnquantizedMoeBackend.FLASHINFER_TRTLLM
248 - else:
249 - raise ValueError(
250 - f"FlashInfer MOE backend {fi_backend} "
251 - "does not support unquantized MoE."
252 - )
253 - k_cls = backend_to_kernel_cls(backend)
254 - return _return_or_raise(backend, moe_config, activation_format)
255 - else:
256 - # If the user is not explicit about the backend, try both.
257 - for backend in [
258 - UnquantizedMoeBackend.FLASHINFER_TRTLLM,
259 - UnquantizedMoeBackend.FLASHINFER_CUTLASS,
260 - ]:
261 - k_cls = backend_to_kernel_cls(backend)
262 - supported, reason = k_cls.is_supported_config(
263 - k_cls, moe_config, None, None, activation_format
264 - )
265 - if supported:
266 - logger.info_once(_make_log_backend(backend))
267 - return backend, k_cls
268 - else:
269 - logger.debug_once(_make_log_unsupported(backend, reason))
270 -
271 - raise NotImplementedError(
272 - "Found VLLM_USE_FLASHINFER_MOE_FP16=1, but no "
273 - "FlashInfer unquantized MoE backend supports the configuration."
274 - )
275 -
231 # Handle explicit AITER FP8 configuration.
232 if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):
233 if not envs.VLLM_ROCM_USE_AITER or not envs.VLLM_ROCM_USE_AITER_MOE:
vllm/model_executor/layers/fused_moe/prepare_finalize/nixl_ep.py
+5 -4
@@ -6,7 +6,7 @@ import nixl_ep
6 import torch
7
8 import vllm.model_executor.layers.fused_moe.modular_kernel as mk
9 -from vllm import envs
9 +from vllm.config import get_current_vllm_config
10 from vllm.distributed import get_ep_group
11 from vllm.distributed.device_communicators.all2all import NixlEPAll2AllManager
12 from vllm.logger import init_logger
@@ -192,10 +192,11 @@ class NixlEPPrepareAndFinalize(mk.FusedMoEPrepareAndFinalizeModular):
192 x = x.view((-1, hidden_dim))
193 q_dtype = quant_config.quant_dtype
194
195 - if envs.VLLM_FLASHINFER_MOE_BACKEND == "masked_gemm":
195 + moe_backend = get_current_vllm_config().kernel_config.moe_backend
196 + if moe_backend == "flashinfer_cutedsl":
197 logger.info_once(
197 - "Skip quantization when using FlashInfer CUTEDSL(masked_gemm) "
198 - "for ModelOptNvFp4FusedMoE."
198 + "Skip quantization when using FlashInfer CUTEDSL "
199 + "(--moe-backend flashinfer_cutedsl) for ModelOptNvFp4FusedMoE."
200 )
201 q_dtype = None
202
vllm/model_executor/layers/quantization/utils/flashinfer_fp4_moe.py
-15
@@ -6,7 +6,6 @@ from typing import TYPE_CHECKING
6
7 import torch
8
9 -import vllm.envs as envs
9 from vllm.logger import init_logger
10 from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
11 align_fp4_moe_weights_for_fi,
@@ -15,10 +14,6 @@ from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (
14 from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
15 swizzle_blockscale,
16 )
18 -from vllm.platforms import current_platform
19 -from vllm.utils.flashinfer import (
20 - has_flashinfer_cutlass_fused_moe,
21 -)
17
18 if TYPE_CHECKING:
19 from vllm.model_executor.layers.fused_moe import RoutedExperts
@@ -34,16 +29,6 @@ __all__ = [
29 ]
30
31
37 -def is_flashinfer_fp4_cutlass_moe_available() -> bool:
38 - """Return `True` when FlashInfer CUTLASS NV-FP4 kernels can be used."""
39 - return (
40 - envs.VLLM_USE_FLASHINFER_MOE_FP4
41 - and has_flashinfer_cutlass_fused_moe()
42 - and current_platform.is_cuda()
43 - and current_platform.has_device_capability(100)
44 - )
45 -
46 -
32 def reorder_w1w3_to_w3w1(
33 weight: torch.Tensor, scale: torch.Tensor, dim: int = -2
34 ) -> tuple[torch.Tensor, torch.Tensor]:
vllm/model_executor/layers/quantization/utils/flashinfer_utils.py
-30
@@ -5,10 +5,8 @@ from typing import TYPE_CHECKING
5
6 import torch
7
8 -from vllm import envs
8 from vllm.logger import init_logger
9 from vllm.model_executor.layers.fused_moe.activation import MoEActivation
11 -from vllm.platforms import current_platform
10 from vllm.utils.math_utils import round_up
11
12 if TYPE_CHECKING:
@@ -95,34 +93,6 @@ def rotate_weights_for_fi_trtllm_fp8_per_tensor_moe(
93 )
94
95
98 -def get_flashinfer_moe_backend() -> FlashinferMoeBackend:
99 - backend_map = {
100 - "throughput": FlashinferMoeBackend.CUTLASS,
101 - "latency": FlashinferMoeBackend.TENSORRT_LLM,
102 - "masked_gemm": FlashinferMoeBackend.CUTEDSL,
103 - }
104 -
105 - flashinfer_moe_backend = envs.VLLM_FLASHINFER_MOE_BACKEND
106 - if flashinfer_moe_backend in backend_map:
107 - if (
108 - flashinfer_moe_backend == "latency"
109 - and not current_platform.is_device_capability_family(100)
110 - ):
111 - logger.info_once(
112 - "Flashinfer TRTLLM MOE backend is only supported on "
113 - "SM100 and later, using CUTLASS backend instead",
114 - )
115 - return FlashinferMoeBackend.CUTLASS
116 - return backend_map[flashinfer_moe_backend]
117 - elif current_platform.is_device_capability(90):
118 - return FlashinferMoeBackend.CUTLASS
119 -
120 - raise ValueError(
121 - f"Unknown flashinfer moe backend: {flashinfer_moe_backend!r}. "
122 - f"Expected one of {list(backend_map.keys())}."
123 - )
124 -
125 -
96 def is_flashinfer_supporting_global_sf(backend: FlashinferMoeBackend | None) -> bool:
97 # TODO(shuw@nvidia): Update when new backends are added.
98 backends_supporting_global_sf = (