@tduncan / Nvidia-Optimization-Stack-VLLM / commits / 90a9515006

[Cleanup] Remove online quantization support in `fp8.py` in favor of online shorthands (#53585)

Signed-off-by: Felix Marty <Felix.Marty@amd.com> Signed-off-by: mgoin <mgoin64@gmail.com> Co-authored-by: mgoin <mgoin64@gmail.com> Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>

fxmarty-amd committed Sep 24, 2026 at 21:48 UTC 90a951500699b1aa499f350b3352450cdcc13340

15 files changed +240 -269

.buildkite/intel_jobs/test-intel.yaml

+3 -3

@@ -38,7 +38,7 @@ steps:

38 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE &&

39 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp &&

40 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN &&

41 - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8 &&

41 + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor &&

42 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8 &&

43 python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096 &&

44 python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192 &&

@@ -74,8 +74,8 @@ steps:

74 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096 &&

75 python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096 &&

76 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model Qwen/Qwen3-4B-Instruct-2507-FP8 --enforce-eager --max-model-len 4096 &&

77 - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096 &&

78 - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096

77 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096 &&

78 + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096

79 '

80 - label: "XPU V1 test"

81 key: xpu-v1-test

.buildkite/scripts/hardware_ci/run-intel-ci-test.sh

+3 -3

@@ -17,7 +17,7 @@ case "${test_suite}" in

17 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 -O3 -cc.cudagraph_mode=NONE

18 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager -tp 2 --distributed-executor-backend mp

19 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --attention-backend=TRITON_ATTN

20 - python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8

20 + python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --quantization fp8_per_tensor

21 python3 examples/basic/offline_inference/generate.py --model facebook/opt-125m --block-size 64 --enforce-eager --kv-cache-dtype fp8

22 python3 examples/basic/offline_inference/generate.py --model nvidia/Llama-3.1-8B-Instruct-FP8 --block-size 64 --enforce-eager --quantization modelopt --kv-cache-dtype fp8 --attention-backend TRITON_ATTN --max-model-len 4096

23 python3 examples/basic/offline_inference/generate.py --model superjob/Qwen3-4B-Instruct-2507-GPTQ-Int4 --block-size 64 --enforce-eager --max-model-len 8192

@@ -30,8 +30,8 @@ case "${test_suite}" in

30 python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096

31 python3 examples/basic/offline_inference/generate.py --linear-backend torch --model RedHatAI/Meta-Llama-3.1-8B-Instruct-FP8 --enforce-eager --max-model-len 4096

32 python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model neuralmagic/Llama-3.2-1B-Instruct-FP8-dynamic --enforce-eager --max-model-len 4096

33 - python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096

34 - python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8 --enforce-eager --max-model-len 4096

33 + python3 examples/basic/offline_inference/generate.py --linear-backend xpu --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096

34 + python3 examples/basic/offline_inference/generate.py --linear-backend torch --model meta-llama/Llama-3.2-1B-Instruct --quantization fp8_per_tensor --enforce-eager --max-model-len 4096

35 ;;

36 v1)

37 cd tests

docs/features/quantization/llm_compressor/fp8.md

+2 -2

@@ -143,14 +143,14 @@ If you encounter any issues or have feature requests, please open an issue on th

143

144 ## Online Dynamic Quantization

145

146 -Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8"` in the command line or setting `quantization="fp8"` in the LLM constructor.

146 +Dynamic quantization of an original precision BF16/FP16 model to FP8 can be achieved with vLLM without any calibration data required. You can enable the feature by specifying `--quantization="fp8_per_tensor"` in the command line or setting `quantization="fp8_per_tensor"` in the LLM constructor.

147

148 In this mode, all Linear modules (except for the final `lm_head`) have their weights quantized down to FP8_E4M3 precision with a per-tensor scale. Activations have their minimum and maximum values calculated during each forward pass to provide a dynamic per-tensor scale for high accuracy. As a result, latency improvements are limited in this mode.

149

150 ```python

151 from vllm import LLM

152

153 -llm = LLM("facebook/opt-125m", quantization="fp8")

153 +llm = LLM("facebook/opt-125m", quantization="fp8_per_tensor")

154 # INFO 06-10 17:55:42 model_runner.py:157] Loading model weights took 0.1550 GB

155 result = llm.generate("Hello, my name is")

156 print(result[0].outputs[0].text)

tests/compile/passes/test_mla_attn_quant_fusion.py

-1

@@ -271,7 +271,6 @@ class TestMLAAttentionFp8GroupQuantPatternModel(MLAAttentionQuantPatternModel):

271

272 quant_key = kFp8Dynamic128Sym

273 quant_config = Fp8Config(

274 - is_checkpoint_fp8_serialized=True,

274 weight_block_size=[128, 128],

275 )

276

tests/kernels/moe/test_moe_layer.py

+2 -2

@@ -796,11 +796,11 @@ def make_quant_config(

796 return None, QuantizedWeights(w13_weight=w1, w2_weight=w2)

797

798 if quantization == "fp8":

799 - return Fp8Config(True), _quantize_fp8_halves(w1, w2)

799 + return Fp8Config(), _quantize_fp8_halves(w1, w2)

800

801 if quantization == "fp8_blocked":

802 block_shape = [128, 128]

803 - return Fp8Config(True, weight_block_size=block_shape), _quantize_fp8_halves(

803 + return Fp8Config(weight_block_size=block_shape), _quantize_fp8_halves(

804 w1, w2, block_shape

805 )

806

tests/kernels/moe/utils.py

+1 -1

@@ -641,7 +641,7 @@ def make_shared_experts_with_weights(

641 if quant_dtype == torch.float8_e4m3fn:

642 from vllm.model_executor.layers.quantization.fp8 import Fp8Config

643

644 - quant_config = Fp8Config(True)

644 + quant_config = Fp8Config()

645 else:

646 quant_config = None

647

tests/model_executor/layers/test_fused_shared_expert.py

+1 -2

@@ -1227,7 +1227,6 @@ def test_non_quark_shared_expert_fse_is_incompatible() -> None:

1227

1228 def _fp8_config(**kwargs: Any) -> Fp8Config:

1229 return Fp8Config(

1230 - is_checkpoint_fp8_serialized=True,

1230 activation_scheme="dynamic",

1231 weight_block_size=[128, 128],

1232 **kwargs,

@@ -1248,7 +1247,7 @@ def test_block_fp8_shared_expert_fse_is_compatible() -> None:

1247 def test_per_tensor_fp8_shared_expert_fse_is_incompatible() -> None:

1248 """Per-tensor scales are 0-D, so the shared-expert chunker cannot slice them."""

1249 compatible, reason = is_shared_expert_quant_fse_compatible(

1251 - Fp8Config(is_checkpoint_fp8_serialized=True, activation_scheme="dynamic"),

1250 + Fp8Config(activation_scheme="dynamic"),

1251 "model.layers.0.mlp.experts",

1252 "model.layers.0.mlp.shared_experts",

1253 )

tests/model_executor/model_loader/test_reload.py

+3 -3

@@ -1158,13 +1158,13 @@ def test_kv_scale_reload(vllm_runner):

1158 "Qwen/Qwen3-0.6B",

1159 "inference-optimization/Qwen3-0.6B-debug-multiply",

1160 "inference-optimization/Qwen3-0.6B-debug-add",

1161 - "fp8",

1161 + "fp8_per_tensor",

1162 ),

1163 pytest.param(

1164 "inference-optimization/DeepSeek-V3-debug-empty",

1165 "inference-optimization/DeepSeek-V3-debug-multiply",

1166 "inference-optimization/DeepSeek-V3-debug-add",

1167 - "fp8",

1167 + "fp8_per_tensor",

1168 marks=[pytest.mark.slow_test],

1169 ),

1170 pytest.param(

@@ -1192,7 +1192,7 @@ def test_online_quantize_reload(

1192 if current_platform.device_count() < tp_size:

1193 pytest.skip(reason="Not enough GPU devices")

1194

1195 - if quantization == "fp8" and _fp8_reload_unsupported():

1195 + if quantization == "fp8_per_tensor" and _fp8_reload_unsupported():

1196 pytest.skip(reason="Requires FP8 support")

1197

1198 with vllm_runner(

tests/models/qwen4_exp/test_ple.py

+6 -6

@@ -14,11 +14,13 @@ from torch.nn import functional as F

14 import vllm.model_executor.layers.vocab_parallel_embedding as embedding_module

15 import vllm.model_executor.parameter as parameter_module

16 import vllm.models.qwen4_exp.nvidia.ngram_embedding as ngram_embedding_module

17 +from vllm.config.quantization import QuantizationConfigArgs

18 from vllm.model_executor.layers.quantization.fp8 import Fp8Config

19 from vllm.model_executor.layers.quantization.modelopt import (

20 ModelOptMixedPrecisionConfig,

21 ModelOptNvFp4Config,

22 )

23 +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig

24 from vllm.models.qwen4_exp.common.ple import (

25 PLEShardOverlap,

26 compute_ple_shard_overlap,

@@ -396,7 +398,6 @@ def test_ple_fp8_embedding_uses_int8_for_parallel_reduce(monkeypatch) -> None:

398 def test_ple_fp8_embedding_respects_checkpoint_shard_exclusions() -> None:

399 prefix = "model.layers.1.ple.ple_embedding.ngram_embedding"

400 quant_config = Fp8Config(

399 - is_checkpoint_fp8_serialized=True,

401 ignored_layers=[],

402 weight_block_size=[128, 128],

403 )

@@ -422,12 +423,11 @@ def test_ple_embedding_rejects_unsupported_quantization_configs() -> None:

423 with pytest.raises(NotImplementedError, match="ModelOptNvFp4Config"):

424 Qwen4ExpPLEEmbeddingMethod.from_quant_config(nvfp4_config, prefix)

425

425 - dynamic_fp8_config = Fp8Config(

426 - is_checkpoint_fp8_serialized=False,

427 - ignored_layers=[],

426 + online_fp8_config = OnlineQuantizationConfig(

427 + QuantizationConfigArgs(linear="fp8_per_tensor")

428 )

429 - with pytest.raises(NotImplementedError, match="serialized FP8"):

430 - Qwen4ExpPLEEmbeddingMethod.from_quant_config(dynamic_fp8_config, prefix)

429 + with pytest.raises(NotImplementedError, match="OnlineQuantizationConfig"):

430 + Qwen4ExpPLEEmbeddingMethod.from_quant_config(online_fp8_config, prefix)

431

432

433 def test_ple_embedding_respects_modelopt_exclusion() -> None:

tests/quantization/test_fp8.py

-196

@@ -5,11 +5,9 @@

5 Run `pytest tests/quantization/test_fp8.py --forked`.

6 """

8 -import logging

8 from types import SimpleNamespace

10 import pytest

12 -import regex as re

11 import torch

13 from tests.quantization.utils import (

@@ -22,9 +20,6 @@ from vllm.config.cache import CacheConfig

20 from vllm.config.kernel import KernelConfig

21 from vllm.config.model import ModelConfig

22 from vllm.forward_context import set_forward_context

25 -from vllm.model_executor.kernels.linear.scaled_mm import (

26 - MarlinFP8ScaledMMLinearKernel,

27 -)

23 from vllm.model_executor.layers.attention import Attention

24 from vllm.model_executor.layers.attention.attention import (

25 set_default_quant_scales,

@@ -32,15 +27,11 @@ from vllm.model_executor.layers.attention.attention import (

27 from vllm.model_executor.layers.fused_moe import FusedMoEFactory

28 from vllm.model_executor.layers.quantization.fp8 import (

29 Fp8Config,

35 - Fp8KVCacheMethod,

30 Fp8LinearMethod,

31 Fp8MoEMethod,

32 )

33 from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod

34 from vllm.model_executor.layers.quantization.modelopt import ModelOptLinearMethod

41 -from vllm.model_executor.layers.quantization.online.fp8 import (

42 - Fp8PerTensorOnlineLinearMethod,

43 -)

35 from vllm.model_executor.layers.quantization.utils import flashinfer_utils

36 from vllm.model_executor.layers.quantization.utils.flashinfer_utils import (

37 prepare_fp8_moe_layer_for_fi,

@@ -531,187 +522,6 @@ def test_model_load_and_run(

522 model(input_ids, positions, None)

523

524

534 -@pytest.mark.skipif(

535 - not is_quant_method_supported("fp8"),

536 - reason="FP8 is not supported on this GPU type.",

537 -)

538 -@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])

539 -@pytest.mark.parametrize(

540 - "force_marlin", [True, False] if current_platform.is_cuda() else [False]

541 -)

542 -@pytest.mark.parametrize(

543 - "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]

544 -)

545 -def test_online_quantization(

546 - vllm_runner,

547 - kv_cache_dtype: str,

548 - force_marlin: bool,

549 - use_rocm_aiter: bool,

550 - monkeypatch,

551 -) -> None:

552 - if use_rocm_aiter:

553 - monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")

554 -

555 - # `LLM.apply_model` requires pickling a function.

556 - monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")

557 -

558 - kwargs = {}

559 - if force_marlin:

560 - kwargs["linear_backend"] = "marlin"

561 - kwargs["moe_backend"] = "marlin"

562 -

563 - model_dtype = "auto"

564 - if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):

565 - # FA3 requires BF16 output when the query input is FP8.

566 - model_dtype = "bfloat16"

567 -

568 - with vllm_runner(

569 - "facebook/opt-125m",

570 - quantization="fp8",

571 - dtype=model_dtype,

572 - enforce_eager=True,

573 - kv_cache_dtype=kv_cache_dtype,

574 - **kwargs,

575 - ) as llm:

576 -

577 - def check_model(model):

578 - fc1 = model.model.decoder.layers[0].fc1

579 - assert isinstance(fc1.quant_method, Fp8PerTensorOnlineLinearMethod)

580 - if kv_cache_dtype == "fp8":

581 - attn = model.model.decoder.layers[0].self_attn.attn

582 - assert isinstance(attn.quant_method, Fp8KVCacheMethod)

583 - assert attn._k_scale == 1.0

584 - assert attn._v_scale == 1.0

585 -

586 - if current_platform.is_cuda() or current_platform.is_xpu():

587 - if current_platform.supports_fp8() and not force_marlin:

588 - # For GPUs with hardware support, we keep weights in fp8

589 - assert fc1.weight.dtype == torch.float8_e4m3fn

590 - assert not isinstance(

591 - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel

592 - )

593 - else:

594 - # For GPUs without hardware support, we pack the fp8 weights

595 - # for weight-only quantization using Marlin kernels

596 - assert fc1.weight.dtype == torch.int32

597 - assert isinstance(

598 - fc1.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel

599 - )

600 - elif current_platform.is_rocm():

601 - if current_platform.supports_fp8() and not force_marlin:

602 - # For GPUs with hardware support, we keep weights in fp8

603 - assert fc1.weight.dtype == current_platform.fp8_dtype()

604 - else: # unsupported ROCm platform

605 - pytest.skip(

606 - "Skip `test_load_fp16_model`. "

607 - "It only runs on ROCm platform with FP8 compute."

608 - " e.g. MI300X and above."

609 - )

610 - else: # unsupported platform

611 - pytest.skip(

612 - "Skip `test_load_fp16_model`. "

613 - "It only runs on CUDA and ROCm platform."

614 - )

615 -

616 - llm.apply_model(check_model)

617 -

618 - outputs = llm.generate_greedy(["Hello my name is"], max_tokens=4)

619 - print(outputs[0][1])

620 -

621 -

622 -@pytest.mark.skipif(

623 - not is_quant_method_supported("fp8"),

624 - reason="FP8 is not supported on this GPU type.",

625 -)

626 -def test_online_quant_peak_mem(

627 - vllm_runner,

628 - caplog_mp_spawn,

629 - monkeypatch,

630 -) -> None:

631 - # Note: `allenai/OLMoE-1B-7B-0125-Instruct` was selected because:

632 - # 1. it covers both Linear and MoE paths

633 - # 2. it is already used by other tests in CI, so adding it here

634 - # does not increase disk space for CI runners

635 - # I really wanted to use `ibm-granite/granite-3.0-1b-a400m-base`

636 - # which I think is the smallest MoE model in vLLM (2.5 GiB bf16,

637 - # 1.3 GiB fp8), but could not as adding one more model makes CI

638 - # run out of disk space.

639 - model_name = "allenai/OLMoE-1B-7B-0125-Instruct"

640 -

641 - # Force spawn to ensure caplog_mp_spawn works consistently

642 - # (it relies on VLLM_LOGGING_CONFIG_PATH which spawn reads but fork ignores)

643 - monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")

644 -

645 - with (

646 - caplog_mp_spawn(logging.DEBUG) as log_holder,

647 - vllm_runner(

648 - model_name,

649 - quantization="fp8",

650 - enforce_eager=True,

651 - ) as llm,

652 - ):

653 - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)

654 - print(outputs[0][1])

655 -

656 - log_text = log_holder.text

657 -

658 - # Parse memory usage from captured logs

659 - model_memory_gib = None

660 - peak_memory_gib = None

661 - for line in log_text.splitlines():

662 - if model_memory_gib is None:

663 - match = re.search(r"Model loading took ([\d.]+) GiB memory", line)

664 - if match:

665 - model_memory_gib = float(match.group(1))

666 - if peak_memory_gib is None:

667 - match = re.search(

668 - r"Peak GPU memory after loading weights: ([\d.]+) GiB", line

669 - )

670 - if match:

671 - peak_memory_gib = float(match.group(1))

672 -

673 - assert model_memory_gib is not None, "Could not find model loading memory log"

674 - assert peak_memory_gib is not None, "Could not find peak memory log"

675 - print(f"GPU memory used after loading weights: {model_memory_gib} GiB")

676 - print(f"Peak GPU memory usage while loading weights: {peak_memory_gib} GiB")

677 -

678 - # model specific, allenai/OLMoE-1B-7B-0125-Instruct fp8 online quant

679 - # uses 6.65 GiB for weight loading (bf16 checkpoint is ~12.89 GiB)

680 - expected_model_memory_gib = 6.7

681 -

682 - # for allenai/OLMoE-1B-7B-0125-Instruct the number we see today is 9.06

683 - # GiB, which is 1.36x above model_memory_gib. A slightly higher number is

684 - # expected as when we load and quantize weights in a streaming fashion we

685 - # need to have individual weights in bf16 + fp8 alive at the same time.

686 - expected_peak_memory_gib = expected_model_memory_gib * 1.4

687 -

688 - assert model_memory_gib < expected_model_memory_gib, (

689 - f"{model_memory_gib=} higher than {expected_model_memory_gib}"

690 - )

691 - assert peak_memory_gib < expected_peak_memory_gib, (

692 - f"{peak_memory_gib=} higher than {expected_peak_memory_gib}"

693 - )

694 -

695 -

696 -@pytest.mark.skipif(

697 - not is_quant_method_supported("fp8"),

698 - reason="FP8 is not supported on this GPU type.",

699 -)

700 -def test_online_quant_load_format_dummy(

701 - vllm_runner,

702 - monkeypatch,

703 - caplog,

704 -) -> None:

705 - with vllm_runner(

706 - "ibm-granite/granite-3.0-1b-a400m-base",

707 - quantization="fp8",

708 - enforce_eager=True,

709 - load_format="dummy",

710 - ) as llm:

711 - outputs = llm.generate_greedy(["The future of AI is"], max_tokens=4)

712 - print(outputs[0][1])

713 -

714 -

525 @pytest.mark.skipif(

526 not is_quant_method_supported("fp8"),

527 reason="FP8 is not supported on this GPU type.",

@@ -799,7 +609,6 @@ def test_scaled_fp8_quant(dtype) -> None:

609 )

610 @pytest.mark.parametrize("method_cls", [Fp8LinearMethod, Fp8MoEMethod])

611 # FP8 weight reloading does not support online quantization

802 -@pytest.mark.parametrize("is_checkpoint_fp8_serialized", [True]) # skip False

612 @pytest.mark.parametrize("weight_block_size", [None, [128, 128]])

613 # any postprocessing that is applied to the weights such as padding and repacking

614 # (excluding device sharding) must also be applied to the reloaded weights

@@ -809,7 +618,6 @@ def test_scaled_fp8_quant(dtype) -> None:

618 def test_fp8_reloading(

619 default_vllm_config,

620 method_cls,

812 - is_checkpoint_fp8_serialized,

621 weight_block_size,

622 use_marlin,

623 dist_init,

@@ -820,9 +628,6 @@ def test_fp8_reloading(

628 # we set fp8_backend to None, which sidestepped the issue.

629 monkeypatch.setenv("VLLM_USE_DEEP_GEMM", "0")

630

823 - if is_checkpoint_fp8_serialized is False:

824 - pytest.skip("FP8 weight reloading does not support online quantization")

825 -

631 if method_cls is Fp8MoEMethod and weight_block_size is None:

632 pytest.skip(

633 "FP8 Tensor weight reloading does not support fusing w13_weight_scale. "

@@ -835,7 +640,6 @@ def test_fp8_reloading(

640 layer_size = 128 if weight_block_size is not None else 1

641 with torch.device(f"{DEVICE_TYPE}:0"):

642 config = Fp8Config(

838 - is_checkpoint_fp8_serialized=is_checkpoint_fp8_serialized,

643 weight_block_size=weight_block_size,

644 )

645

tests/quantization/test_online.py

+180 -20

@@ -20,6 +20,8 @@ from tests.quantization.utils import (

20 from vllm import _custom_ops as ops

21 from vllm._aiter_ops import rocm_aiter_ops

22 from vllm._custom_ops import scaled_fp4_quant

23 +from vllm.config.cache import CacheConfig

24 +from vllm.config.kernel import KernelConfig

25 from vllm.config.load import LoadConfig

26 from vllm.config.model import ModelConfig

27 from vllm.config.quantization import (

@@ -34,6 +36,9 @@ from vllm.model_executor.kernels.linear.mxfp8.emulation import (

36 from vllm.model_executor.kernels.linear.mxfp8.marlin import (

37 MarlinMxfp8LinearKernel,

38 )

39 +from vllm.model_executor.kernels.linear.scaled_mm import (

40 + MarlinFP8ScaledMMLinearKernel,

41 +)

42 from vllm.model_executor.layers.attention import Attention

43 from vllm.model_executor.layers.fused_moe import FusedMoEFactory

44 from vllm.model_executor.layers.fused_moe.oracle import nvfp4 as nvfp4_oracle

@@ -125,6 +130,81 @@ PARTIALLY_PREQUANTIZED_MODEL_NAME = (

130 )

131

132

133 +@pytest.mark.parametrize(

134 + "overrides",

135 + [

136 + None,

137 + {"ignore": ["model.layers.0.self_attn.o_proj"]},

138 + {

139 + "linear": "fp8_per_block",

140 + "ignore": ["model.layers.0.self_attn.o_proj"],

141 + },

142 + ],

143 + ids=["defaults", "ignore", "linear-and-ignore"],

144 +)

145 +def test_legacy_fp8_online_quantization_uses_per_tensor_shorthand(

146 + tmp_path, caplog, disable_log_dedup, overrides

147 +) -> None:

148 + """The legacy alias preserves overrides and fills in per-tensor defaults."""

149 + _write_minimal_llama_config(tmp_path)

150 + model_config = ModelConfig(

151 + model=str(tmp_path), quantization="fp8", quantization_config=overrides

152 + )

153 +

154 + result = weight_utils.get_quant_config(model_config, LoadConfig())

155 +

156 + assert isinstance(result, OnlineQuantizationConfig)

157 + assert result.args == resolve_quantization_config("fp8_per_tensor", overrides)

158 + assert "--quantization fp8 is deprecated for online quantization" in caplog.text

159 + assert "--quantization fp8_per_tensor instead" in caplog.text

160 +

161 +

162 +@pytest.mark.parametrize(

163 + "shorthand,method_cls",

164 + [

165 + ("fp8_per_tensor", Fp8PerTensorOnlineLinearMethod),

166 + ("fp8_per_block", Fp8PerBlockOnlineLinearMethod),

167 + ("fp8_per_channel", Fp8PtpcOnlineLinearMethod),

168 + ],

169 +)

170 +def test_qianfan_online_fp8_keeps_vision_layers_unquantized(

171 + shorthand, method_cls

172 +) -> None:

173 + """Model exclusions preserve the language backbone and shared defaults."""

174 + from vllm.model_executor.models.qianfan_ocr import (

175 + QianfanOCRForConditionalGeneration,

176 + )

177 +

178 + args = resolve_quantization_config(shorthand, None)

179 + assert args is not None

180 + config = OnlineQuantizationConfig(args)

181 + other_config = OnlineQuantizationConfig(args)

182 + model = QianfanOCRForConditionalGeneration.__new__(

183 + QianfanOCRForConditionalGeneration

184 + )

185 + model._patch_quant_config(

186 + SimpleNamespace(vision_config=SimpleNamespace(num_hidden_layers=2)),

187 + config,

188 + )

189 +

190 + layer = Mock(spec=LinearBase)

191 + for prefix in (

192 + "vision_model.encoder.layers.0.attn.qkv",

193 + "vision_model.encoder.layers.1.mlp.fc2",

194 + "language_model.lm_head",

195 + "mlp1.1",

196 + "mlp1.3",

197 + ):

198 + assert config.resolve_quant_method_cls(layer, prefix) is None

199 +

200 + resolved = config.resolve_quant_method_cls(

201 + layer, "language_model.model.layers.0.self_attn.qkv_proj"

202 + )

203 + assert resolved is not None and resolved[-1] is method_cls

204 + assert other_config.ignored_layers == []

205 + assert args.ignore == []

206 +

207 +

208 def test_online_nvfp4_reuses_kernel_when_weights_are_reprocessed(

209 monkeypatch,

210 ) -> None:

@@ -579,33 +659,81 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp

659 reason="FP8 is not supported on this GPU type.",

660 )

661 @pytest.mark.parametrize(

582 - ("model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls"),

662 + (

663 + "model_name,quant_scheme,online_quant_args,expected_linear_cls,expected_moe_cls,extra_runner_kwargs"

664 + ),

665 [

666 # simple case - quantization='fp8_per_tensor'

585 - (

667 + pytest.param(

668 + GRANITE_MODEL_NAME,

669 + "fp8_per_tensor",

670 + None,

671 + Fp8PerTensorOnlineLinearMethod,

672 + Fp8PerTensorOnlineMoEMethod,

673 + {},

674 + id="fp8_per_tensor",

675 + ),

676 + # FP8 KV cache with the fp8_per_tensor shorthand.

677 + pytest.param(

678 + GRANITE_MODEL_NAME,

679 + "fp8_per_tensor",

680 + None,

681 + Fp8PerTensorOnlineLinearMethod,

682 + Fp8PerTensorOnlineMoEMethod,

683 + {"kv_cache_dtype": "fp8"},

684 + id="fp8_per_tensor-fp8-kv-cache",

685 + ),

686 + pytest.param(

687 + GRANITE_MODEL_NAME,

688 + "fp8_per_tensor",

689 + None,

690 + Fp8PerTensorOnlineLinearMethod,

691 + Fp8PerTensorOnlineMoEMethod,

692 + {"linear_backend": "marlin", "moe_backend": "marlin"},

693 + marks=pytest.mark.skipif(

694 + not current_platform.is_cuda(),

695 + reason="Marlin online FP8 coverage is CUDA-only.",

696 + ),

697 + id="fp8_per_tensor-marlin",

698 + ),

699 + pytest.param(

700 GRANITE_MODEL_NAME,

701 "fp8_per_tensor",

702 None,

703 Fp8PerTensorOnlineLinearMethod,

704 Fp8PerTensorOnlineMoEMethod,

705 + {

706 + "kv_cache_dtype": "fp8",

707 + "linear_backend": "marlin",

708 + "moe_backend": "marlin",

709 + },

710 + marks=pytest.mark.skipif(

711 + not current_platform.is_cuda(),

712 + reason="Marlin online FP8 coverage is CUDA-only.",

713 + ),

714 + id="fp8_per_tensor-fp8-kv-cache-marlin",

715 ),

716 # simple case - quantization='fp8_per_block'

593 - (

717 + pytest.param(

718 GRANITE_MODEL_NAME,

719 "fp8_per_block",

720 None,

721 Fp8PerBlockOnlineLinearMethod,

722 Fp8PerBlockOnlineMoEMethod,

723 + {},

724 + id="fp8_per_block",

725 ),

600 - (

726 + pytest.param(

727 GRANITE_MODEL_NAME,

728 "fp8_per_channel",

729 None,

730 Fp8PtpcOnlineLinearMethod,

731 Fp8PtpcOnlineMoEMethod,

732 + {},

733 + id="fp8_per_channel",

734 ),

735 # quantization='online' with per-layer-kind overrides

608 - (

736 + pytest.param(

737 GRANITE_MODEL_NAME,

738 "online",

739 {

@@ -614,9 +742,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp

742 },

743 Fp8PerBlockOnlineLinearMethod,

744 Fp8PerTensorOnlineMoEMethod,

745 + {},

746 + id="per_layer_kind_overrides",

747 ),

748 # quantization='online' with per-layer target patterns

619 - (

749 + pytest.param(

750 GRANITE_MODEL_NAME,

751 "online",

752 {

@@ -627,9 +757,11 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp

757 },

758 Fp8PerBlockOnlineLinearMethod,

759 Fp8PerTensorOnlineMoEMethod,

760 + {},

761 + id="targets",

762 ),

763 # ignore with direct layer name

632 - (

764 + pytest.param(

765 GRANITE_MODEL_NAME,

766 "fp8_per_tensor",

767 # qkv_proj is fused from q_proj/k_proj/v_proj. The shard regex

@@ -637,13 +769,17 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp

769 {"ignore": ["model.layers.1.self_attn.o_proj", "re:.*[qkv]_proj"]},

770 Fp8PerTensorOnlineLinearMethod,

771 Fp8PerTensorOnlineMoEMethod,

772 + {},

773 + id="ignore",

774 ),

641 - (

775 + pytest.param(

776 GRANITE_MODEL_NAME,

777 "mxfp4",

778 None,

779 Mxfp4OnlineLinearMethod,

780 Mxfp4OnlineMoEMethod,

781 + {},

782 + id="mxfp4",

783 ),

784 pytest.param(

785 PARTIALLY_PREQUANTIZED_MODEL_NAME,

@@ -651,19 +787,10 @@ def test_nvfp4_one_sided_rejects_unsupported_input_dtype(input_dtype: torch.dtyp

787 {"targets": {"model.layers.1.self_attn.o_proj": "mxfp8"}},

788 Mxfp8OnlineLinearMethod,

789 CompressedTensorsMoEMethod,

790 + {},

791 id="partially_prequantized_checkpoint",

792 ),

793 ],

657 - ids=[

658 - "fp8_per_tensor",

659 - "fp8_per_block",

660 - "fp8_per_channel",

661 - "per_layer_kind_overrides",

662 - "targets",

663 - "ignore",

664 - "mxfp4",

665 - "partially_prequantized_checkpoint",

666 - ],

794 )

795 @pytest.mark.parametrize(

796 "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]

@@ -674,6 +801,7 @@ def test_online_quantization(

801 online_quant_args: dict | None,

802 expected_linear_cls,

803 expected_moe_cls,

804 + extra_runner_kwargs: dict,

805 use_rocm_aiter: bool,

806 monkeypatch,

807 dist_init,

@@ -700,14 +828,31 @@ def test_online_quantization(

828 ):

829 pytest.skip(f"Skip test for online {quant_scheme} on XPU platform.")

830

831 + runner_kwargs = dict(extra_runner_kwargs)

832 + kv_cache_dtype = runner_kwargs.get("kv_cache_dtype", "auto")

833 + force_marlin = extra_runner_kwargs.get("linear_backend") == "marlin"

834 + if kv_cache_dtype == "fp8" and current_platform.is_device_capability_family(90):

835 + # FA3 requires BF16 output when the query input is FP8.

836 + runner_kwargs["dtype"] = "bfloat16"

837 +

838 + vllm_config_kwargs = {

839 + "cache_config": CacheConfig(cache_dtype=kv_cache_dtype),

840 + "kernel_config": KernelConfig(

841 + linear_backend=runner_kwargs.get("linear_backend", "auto"),

842 + moe_backend=runner_kwargs.get("moe_backend", "auto"),

843 + ),

844 + }

845 +

846 if model_name == PARTIALLY_PREQUANTIZED_MODEL_NAME:

847 model, vllm_config = load_model_without_vllm_runner(

848 model_name,

849 + dtype=runner_kwargs.get("dtype", "bfloat16"),

850 model_config_kwargs={

851 "quantization_config": resolve_quantization_config(

852 None, online_quant_args

853 )

854 },

855 + vllm_config_kwargs=vllm_config_kwargs,

856 model_loader_cls=DummyModelLoader,

857 )

858 assert isinstance(

@@ -725,7 +870,7 @@ def test_online_quantization(

870 else:

871 model, vllm_config = load_model_without_vllm_runner(

872 model_name,

728 - dtype="bfloat16",

873 + dtype=runner_kwargs.get("dtype", "bfloat16"),

874 quantization=quant_scheme,

875 model_config_kwargs={

876 "quantization_config": resolve_quantization_config(

@@ -743,6 +888,7 @@ def test_online_quantization(

888 "num_experts_per_tok": 2,

889 },

890 },

891 + vllm_config_kwargs=vllm_config_kwargs,

892 model_loader_cls=DummyModelLoader,

893 )

894

@@ -770,7 +916,16 @@ def test_online_quantization(

916 elif quant_scheme == "mxfp4":

917 assert o_proj.weight.dtype == torch.uint8

918 elif current_platform.is_cuda() or current_platform.is_xpu():

773 - assert o_proj.weight.dtype == torch.float8_e4m3fn

919 + if current_platform.supports_fp8() and not force_marlin:

920 + assert o_proj.weight.dtype == torch.float8_e4m3fn

921 + assert not isinstance(

922 + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel

923 + )

924 + else:

925 + assert o_proj.weight.dtype == torch.int32

926 + assert isinstance(

927 + o_proj.quant_method.fp8_linear, MarlinFP8ScaledMMLinearKernel

928 + )

929 elif current_platform.is_rocm():

930 assert o_proj.weight.dtype == current_platform.fp8_dtype()

931 else:

@@ -794,6 +949,11 @@ def test_online_quantization(

949 layer.self_attn.qkv_proj.quant_method, UnquantizedLinearMethod

950 )

951

952 + if kv_cache_dtype == "fp8":

953 + attn = model.model.layers[0].self_attn.attn

954 + assert attn._k_scale == 1.0

955 + assert attn._v_scale == 1.0

956 +

957 if (

958 model_name == GRANITE_MODEL_NAME

959 and isinstance(online_quant_args, dict)

tests/v1/spec_decode/test_speculators_correctness.py

+1 -1

@@ -37,7 +37,7 @@ DFLASH_CONFIG = SpeculatorTestConfig(

37 acceptance_len_rtol=0.15,

38 expected_per_pos_acceptance_rates=(0.795, 0.611, 0.429, 0.282),

39 per_pos_rtol=0.15,

40 - quantization="fp8",

40 + quantization="fp8_per_tensor",

41 )

43 PEAGLE_CONFIG = SpeculatorTestConfig(

vllm/model_executor/layers/quantization/fp8.py

+12 -27

@@ -93,7 +93,7 @@ class Fp8Config(QuantizationConfig):

94 def __init__(

95 self,

96 - is_checkpoint_fp8_serialized: bool = False,

96 + is_checkpoint_fp8_serialized: bool = True,

97 activation_scheme: str = "dynamic",

98 ignored_layers: list[str] | None = None,

99 weight_block_size: list[int] | None = None,

@@ -102,6 +102,13 @@ class Fp8Config(QuantizationConfig):

102 super().__init__()

103

104 self.is_checkpoint_fp8_serialized = is_checkpoint_fp8_serialized

105 + if not is_checkpoint_fp8_serialized:

106 + raise ValueError(

107 + "The `fp8` quantization method no longer supports online "

108 + "quantization. Please use `--quantization fp8_per_tensor` "

109 + "instead. See "

110 + "https://docs.vllm.ai/en/stable/features/quantization/online/"

111 + )

112

113 if activation_scheme not in ACTIVATION_SCHEMES:

114 raise ValueError(f"Unsupported activation scheme {activation_scheme}")

@@ -112,11 +119,6 @@ class Fp8Config(QuantizationConfig):

119 )

120 self.store_dtype = store_dtype

121 if weight_block_size is not None:

115 - if not is_checkpoint_fp8_serialized:

116 - raise ValueError(

117 - "The block-wise quantization only supports fp8-serialized "

118 - "checkpoint for now."

119 - )

122 if len(weight_block_size) != 2:

123 raise ValueError(

124 "The quantization block size of weight must have 2 "

@@ -182,18 +184,9 @@ class Fp8Config(QuantizationConfig):

184 match_mode=self.ignored_layers_match_mode,

185 ):

186 return UnquantizedLinearMethod()

185 - if not self.is_checkpoint_fp8_serialized:

186 - from vllm.model_executor.layers.quantization.online.fp8 import (

187 - Fp8PerTensorOnlineLinearMethod,

188 - )

189 -

190 - online_method = Fp8PerTensorOnlineLinearMethod()

191 - online_method.marlin_input_dtype = get_marlin_input_dtype(prefix)

192 - return online_method

193 - else:

194 - offline_method = Fp8LinearMethod(self)

195 - offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)

196 - return offline_method

187 + offline_method = Fp8LinearMethod(self)

188 + offline_method.marlin_input_dtype = get_marlin_input_dtype(prefix)

189 + return offline_method

190 elif isinstance(layer, RoutedExperts):

191 if is_layer_skipped(

192 prefix=prefix,

@@ -208,14 +201,7 @@ class Fp8Config(QuantizationConfig):

201 )

202

203 return Mxfp4MoEMethod(layer.moe_config)

211 - if self.is_checkpoint_fp8_serialized:

212 - return Fp8MoEMethod(self, layer)

213 - else:

214 - from vllm.model_executor.layers.quantization.online.fp8 import (

215 - Fp8PerTensorOnlineMoEMethod,

216 - )

217 -

218 - return Fp8PerTensorOnlineMoEMethod(moe=layer.moe_config)

204 + return Fp8MoEMethod(self, layer)

205 elif isinstance(layer, Attention):

206 return Fp8KVCacheMethod(self)

207 return None

@@ -548,7 +534,6 @@ class Fp8MoEMethod(FusedMoEMethodBase):

534 layer.orig_dtype = params_dtype

535 layer.weight_block_size = None

536

551 - assert self.quant_config.is_checkpoint_fp8_serialized

537 params_dtype = torch.float8_e4m3fn

538

539 if self.block_quant:

vllm/model_executor/model_loader/weight_utils.py

+13 -1

@@ -195,7 +195,11 @@ def get_quant_config(

195 if model_config.quantization is None:

196 raise ValueError("Model quantization method is not specified in the config.")

197 quant_cls = get_quantization_config(model_config.quantization)

198 - from vllm.config.quantization import _ONLINE_SHORTHANDS, QuantizationConfigArgs

198 + from vllm.config.quantization import (

199 + _ONLINE_SHORTHANDS,

200 + QuantizationConfigArgs,

201 + resolve_quantization_config,

202 + )

203 from vllm.model_executor.layers.quantization.online.base import (

204 OnlineQuantizationConfig,

205 )

@@ -335,6 +339,14 @@ def get_quant_config(

339

340 # If the quantization config is not found, use the default config.

341 if not possible_config_filenames:

342 + if model_config.quantization == "fp8":

343 + logger.warning_once(

344 + "--quantization fp8 is deprecated for online quantization; "

345 + "use --quantization fp8_per_tensor instead."

346 + )

347 + fp8_args = resolve_quantization_config("fp8_per_tensor", online_args)

348 + assert fp8_args is not None

349 + return OnlineQuantizationConfig(args=fp8_args)

350 if model_config.quantization in _ONLINE_SHORTHANDS:

351 args = online_args or _ONLINE_SHORTHANDS[model_config.quantization]

352 assert isinstance(args, QuantizationConfigArgs)

vllm/model_executor/models/qianfan_ocr.py

+13 -1

@@ -9,6 +9,12 @@ from transformers import PreTrainedConfig

10 from vllm.model_executor.layers.quantization import QuantizationConfig

11 from vllm.model_executor.layers.quantization.fp8 import Fp8Config

12 +from vllm.model_executor.layers.quantization.online.base import OnlineQuantizationConfig

13 +from vllm.model_executor.layers.quantization.utils.quant_utils import (

14 + kFp8Static128BlockSym,

15 + kFp8StaticChannelSym,

16 + kFp8StaticTensorSym,

17 +)

18 from vllm.multimodal import MULTIMODAL_REGISTRY

19 from vllm.transformers_utils.processors.internvl import (

20 InternVLImageProcessor,

@@ -71,7 +77,12 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):

77 ) -> None:

78 super()._patch_quant_config(config, quant_config)

79 # ignore vit layers to preserve model performance

74 - if isinstance(quant_config, Fp8Config):

80 + if isinstance(quant_config, Fp8Config) or (

81 + isinstance(quant_config, OnlineQuantizationConfig)

82 + and quant_config.args.linear is not None

83 + and quant_config.args.linear.weight

84 + in (kFp8StaticTensorSym, kFp8Static128BlockSym, kFp8StaticChannelSym)

85 + ):

86 _FP8_IGNORED_LAYERS = [

87 *(

88 layer

@@ -87,6 +98,7 @@ class QianfanOCRForConditionalGeneration(InternVLChatModel):

98 "mlp1.1",

99 "mlp1.3",

100 ]

101 + quant_config.ignored_layers = list(quant_config.ignored_layers)

102 for layer in _FP8_IGNORED_LAYERS:

103 if layer not in quant_config.ignored_layers:

104 quant_config.ignored_layers.append(layer)