@tduncan / Nvidia-Optimization-Stack-VLLM / commits / bca7bea240

Remove VLLM_TEST_FORCE_FP8_MARLIN to replace with linear_backend/moe_backend (#52182)

Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: OpenAI Codex <codex@openai.com>

Misha Goin committed Aug 18, 2026 at 12:58 UTC bca7bea2405127bd5291bb6fffa679bdcd8f6dd9

15 files changed +46 -66

tests/compile/passes/test_fusion.py

@@ -371,6 +371,10 @@ def test_fusion_rmsnorm_quant(

371 use_aiter_fusion=False,

372 use_aiter_quant=False,

373 )

374 + if any(

375 + type(layer.kernel) is not force_kernel for layer in model.fp8_linear_layers

376 + ):

377 + pytest.skip(f"{force_kernel.__name__} is not supported on this platform")

378

379 backend, _ = _run_fusion_test(

380 model, fusion_pass, vllm_config, dtype, hidden_size, num_tokens

tests/compile/passes/test_mla_attn_quant_fusion.py

@@ -481,6 +481,12 @@ def test_mla_attention_quant_pattern(

481 device=device,

482 vllm_config=vllm_config_unfused,

483 )

484 + if (

485 + model_class is TestMLAAttentionFp8GroupQuantPatternModel

486 + and type(model_unfused.block_fp8_linear.kernel)

487 + is not CutlassFp8BlockScaledMMKernel

488 + ):

489 + pytest.skip("CUTLASS FP8 block kernel is not supported on this platform")

490 model_unfused = model_unfused.to(device)

491 # HACK: See #131044

492 result_unfused_0 = model_unfused(q, kv_c_normed, k_pe) # noqa: F841

tests/evals/gsm8k/configs/moe-refactor/Llama-4-Scout-Fp8-ModelOpt-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "nvidia/Llama-4-Scout-17B-16E-Instruct-FP8"

2 accuracy_threshold: 0.92

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-AutoFp8-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "Qwen/Qwen3-Coder-30B-A3B-Instruct-FP8"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Block-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-block"

2 accuracy_threshold: 0.85

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-Fp8-CT-Channel-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-FP8-dynamic"

2 accuracy_threshold: 0.85

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-CT-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "RedHatAI/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/evals/gsm8k/configs/moe-refactor/Qwen3-30B-A3B-NvFp4-ModelOpt-marlin.yaml

+1 -3

@@ -2,6 +2,4 @@ model_name: "nvidia/Qwen3-30B-A3B-NVFP4"

2 accuracy_threshold: 0.88

3 num_questions: 1319

4 num_fewshot: 5

5 -server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2"

6 -env:

7 - VLLM_TEST_FORCE_FP8_MARLIN: "1"

5 +server_args: "--enforce-eager --max-model-len 8192 --tensor-parallel-size 2 --linear-backend marlin --moe-backend marlin"

tests/quantization/test_fp8.py

+20 -13

@@ -64,10 +64,12 @@ def test_model_load_and_run(

64 if use_rocm_aiter:

65 monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")

67 + kwargs = {}

68 if force_marlin:

68 - monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")

69 + kwargs["linear_backend"] = "marlin"

70 + kwargs["moe_backend"] = "marlin"

70 - with vllm_runner(model_id, enforce_eager=True) as llm:

72 + with vllm_runner(model_id, enforce_eager=True, **kwargs) as llm:

73 # note: this does not test accuracy, just that we can run through

74 # see lm-eval tests for accuracy

75 outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)

@@ -98,8 +100,10 @@ def test_online_quantization(

100 # `LLM.apply_model` requires pickling a function.

101 monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

102

103 + kwargs = {}

104 if force_marlin:

102 - monkeypatch.setenv("VLLM_TEST_FORCE_FP8_MARLIN", "1")

105 + kwargs["linear_backend"] = "marlin"

106 + kwargs["moe_backend"] = "marlin"

107

108 model_dtype = "auto"

109 if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):

@@ -112,6 +116,7 @@ def test_online_quantization(

116 dtype=model_dtype,

117 enforce_eager=True,

118 kv_cache_dtype=kv_cache_dtype,

119 + **kwargs,

120 ) as llm:

121

122 def check_model(model):

@@ -340,7 +345,7 @@ def test_scaled_fp8_quant(dtype) -> None:

345 @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])

346 # FP8 weight reloading does not support online quantization

347 @pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False

343 -@pytest.mark.parametrize("weight_block_size", [None, [1, 1]])

348 +@pytest.mark.parametrize("weight_block_size", [None, [128, 128]])

349 # any postprocessing that is applied to the weights such as padding and repacking

350 # (excluding device sharding) must also be applied to the reloaded weights

351 #

@@ -371,6 +376,8 @@ def test_fp8_reloading(

376

377 # Set model config as model_config.dtype is required in Fp8LinearMethod.

378 default_vllm_config.model_config = ModelConfig()

379 + default_vllm_config.kernel_config.moe_backend = "triton"

380 + layer_size = 128 if weight_block_size is not None else 1

381 with torch.device(f"{DEVICE_TYPE}:0"):

382 config = Fp8Config(

383 is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,

@@ -378,14 +385,14 @@ def test_fp8_reloading(

385 )

386

387 if method_cls is Fp8LinearMethod:

381 - layer = torch.nn.Linear(1, 1)

388 + layer = torch.nn.Linear(layer_size, layer_size)

389 method = method_cls(config)

390 method.create_weights(

391 layer=layer,

385 - input_size_per_partition=1,

386 - output_partition_sizes=[1],

387 - input_size=1,

388 - output_size=1,

392 + input_size_per_partition=layer_size,

393 + output_partition_sizes=[layer_size],

394 + input_size=layer_size,

395 + output_size=layer_size,

396 params_dtype=torch.bfloat16,

397 weight_loader=default_weight_loader,

398 )

@@ -395,16 +402,16 @@ def test_fp8_reloading(

402 layer = FusedMoEFactory(

403 num_experts=1,

404 top_k=1,

398 - hidden_size=1,

399 - intermediate_size=1,

405 + hidden_size=layer_size,

406 + intermediate_size=layer_size,

407 )

408 layer = layer.routed_experts

409 method = method_cls(config, layer)

410 method.create_weights(

411 layer=layer,

412 num_experts=1,

406 - hidden_size=1,

407 - intermediate_size_per_partition=1,

413 + hidden_size=layer_size,

414 + intermediate_size_per_partition=layer_size,

415 params_dtype=torch.bfloat16,

416 weight_loader=default_weight_loader,

417 )

tests/utils.py

+8 -3

@@ -2339,6 +2339,10 @@ class TestFP8Layer(torch.nn.Module):

2339 force_kernel: type[_KernelT] | None = None,

2340 ):

2341 super().__init__()

2342 + self.input_size_per_partition = weight_shape[1]

2343 + self.output_size_per_partition = weight_shape[0]

2344 + self.logical_widths = [self.output_size_per_partition]

2345 + self.orig_dtype = input_dtype

2346 act_scale_desc = activation_quant_key.scale

2347 weight_scale_desc = weight_quant_key.scale

2348 is_block_wise = act_scale_desc.group_shape.is_per_group()

@@ -2346,11 +2350,12 @@ class TestFP8Layer(torch.nn.Module):

2350 block_size = weight_scale_desc.group_shape.col

2351 weight_scale_shape = weight_shape[0] // block_size

2352 self.weight_scale_inv = torch.rand(

2349 - (weight_scale_shape, weight_scale_shape), dtype=torch.float32

2353 + (weight_scale_shape, weight_scale_shape),

2354 + dtype=torch.float32,

2355 + device=device,

2356 )

2351 - self.weight = torch.rand(weight_shape).to(dtype=FP8_DTYPE)

2357 + self.weight = torch.rand(weight_shape, device=device).to(dtype=FP8_DTYPE)

2358 self.input_scale = None

2353 - self.weight_scale = None

2359 self.weight_block_size = [block_size, block_size]

2360 if transpose_weights:

2361 self.weight = self.weight.t()

vllm/envs.py

-6

@@ -1099,12 +1099,6 @@ environment_variables: dict[str, Callable[[], Any]] = {

1099 os.environ.get("VLLM_ALLOW_LONG_MAX_MODEL_LEN", "0").strip().lower()

1100 in ("1", "true")

1101 ),

1102 - # If set, forces FP8 Marlin to be used for FP8 quantization regardless

1103 - # of the hardware support for FP8 compute.

1104 - "VLLM_TEST_FORCE_FP8_MARLIN": lambda: (

1105 - os.environ.get("VLLM_TEST_FORCE_FP8_MARLIN", "0").strip().lower()

1106 - in ("1", "true")

1107 - ),

1102 "VLLM_TEST_FORCE_LOAD_FORMAT": lambda: os.getenv(

1103 "VLLM_TEST_FORCE_LOAD_FORMAT", "dummy"

1104 ),

vllm/model_executor/kernels/linear/__init__.py

+2 -2

@@ -394,12 +394,12 @@ _POSSIBLE_INT8_KERNELS: dict[PlatformEnum, list[type[Int8ScaledMMLinearKernel]]]

394 # in priority/performance order (when available)

395 _POSSIBLE_FP8_KERNELS: dict[PlatformEnum, list[type[FP8ScaledMMLinearKernel]]] = {

396 PlatformEnum.CUDA: [

397 - MarlinFP8ScaledMMLinearKernel,

397 FlashInferFP8ScaledMMLinearKernel,

398 CutlassFP8ScaledMMLinearKernel,

399 B12xTensorFP8ScaledMMLinearKernel,

400 PerTensorTorchFP8ScaledMMLinearKernel,

401 ChannelWiseTorchFP8ScaledMMLinearKernel,

402 + MarlinFP8ScaledMMLinearKernel,

403 HummingFP8ScaledMMLinearKernel,

404 ],

405 PlatformEnum.ROCM: [

@@ -434,8 +434,8 @@ _POSSIBLE_FP8_BLOCK_KERNELS: dict[

434 CutlassFp8BlockScaledMMKernel,

435 B12xFp8BlockScaledMMKernel,

436 MarlinFP8ScaledMMLinearKernel,

437 - TritonFp8BlockScaledMMKernel,

437 HummingFP8ScaledMMLinearKernel,

438 + TritonFp8BlockScaledMMKernel,

439 BlockWiseTorchFP8ScaledMMLinearKernel,

440 ],

441 PlatformEnum.ROCM: [

vllm/model_executor/kernels/linear/scaled_mm/marlin.py

-10

@@ -43,16 +43,6 @@ class MarlinFP8ScaledMMLinearKernel(FP8ScaledMMLinearKernel):

43 return False, "FP8 Marlin requires compute capability 7.5 or higher"

44 if envs.VLLM_BATCH_INVARIANT:

45 return False, "FP8 Marlin not supported for batch invariant execution."

46 - if (

47 - compute_capability is not None

48 - and compute_capability >= 89

49 - and not envs.VLLM_TEST_FORCE_FP8_MARLIN

50 - ):

51 - return (

52 - False,

53 - "To apply FP8 Marlin on high-capability GPUs, please set "

54 - "VLLM_TEST_FORCE_FP8_MARLIN=1",

55 - )

46 return True, None

48 @classmethod

vllm/model_executor/layers/fused_moe/oracle/fp8.py

-7

@@ -370,13 +370,6 @@ def select_fp8_moe_backend(

370 backend, config, weight_key, activation_key, activation_format

371 )

372

373 - # Handle explicit MARLIN FP8 configuration.

374 - if envs.VLLM_TEST_FORCE_FP8_MARLIN:

375 - backend = Fp8MoeBackend.MARLIN

376 - return _return_or_raise(

377 - backend, config, weight_key, activation_key, activation_format

378 - )

379 -

373 # Handle explicit AITER FP8 configuration.

374 if envs.is_set("VLLM_ROCM_USE_AITER") or envs.is_set("VLLM_ROCM_USE_AITER_MOE"):

375 skip_aiter_moe = (

vllm/model_executor/layers/fused_moe/oracle/nvfp4.py

-7

@@ -4,7 +4,6 @@ from enum import Enum

5 import torch

7 -import vllm.envs as envs

7 import vllm.model_executor.layers.fused_moe.modular_kernel as mk

8 from vllm.config.kernel import MoEBackend

9 from vllm.logger import init_logger

@@ -271,12 +270,6 @@ def select_nvfp4_moe_backend(

270 requested_backend, config, weight_key, activation_key, activation_format

271 )

272

274 - if envs.VLLM_TEST_FORCE_FP8_MARLIN:

275 - backend = NvFp4MoeBackend.MARLIN

276 - return _return_or_raise(

277 - backend, config, weight_key, activation_key, activation_format

278 - )

279 -

273 # Select kernels in order of backend.

274 for backend in AVAILABLE_BACKENDS:

275 for k_cls in backend_to_kernel_cls(backend):