A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 6, 05:45 UTC
[Quantization][Deprecation] Remove PTPC FP8 (#32700)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Mar 21, 2026 at 18:10 UTC
eeee5b262d9a730794ecfc82cb3d55ff030ca3e1
5 files changed
+1
-196
.buildkite/scripts/hardware_ci/run-amd-test.sh
+1
-2
@@ -327,8 +327,7 @@ apply_rocm_test_overrides() {
327
cmds="${cmds} \
328
--ignore=kernels/moe/test_moe.py \
329
--ignore=kernels/moe/test_cutlass_moe.py \
330
- --ignore=kernels/moe/test_triton_moe_ptpc_fp8.py"
331
- fi
330
+ fi
331
332
# --- Entrypoint ignores ---
333
if [[ $cmds == *" entrypoints/openai "* ]]; then
tests/quantization/test_ptpc_fp8.py
deleted
-57
@@ -1,57 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-"""Tests whether PTPC w8a8 FP8 computation is enabled correctly.
4
-
5
-Run `pytest tests/quantization/test_ptpc_fp8.py --forked`.
6
-"""
7
-
8
-import pytest
9
-
10
-from tests.quantization.utils import is_quant_method_supported
11
-from vllm.model_executor.layers.quantization.fp8 import Fp8KVCacheMethod
12
-from vllm.model_executor.layers.quantization.ptpc_fp8 import PTPCFp8LinearMethod
13
-from vllm.platforms import current_platform
14
-
15
-
16
-@pytest.fixture(scope="function", autouse=True)
17
-def enable_pickle(monkeypatch):
18
- """`LLM.apply_model` requires pickling a function."""
19
- monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
20
-
21
-
22
-@pytest.mark.skipif(
23
- not is_quant_method_supported("ptpc_fp8"),
24
- reason="PTPC FP8 is not supported on this GPU type.",
25
-)
26
-@pytest.mark.skipif(not current_platform.is_rocm(), reason="This test is for ROCm GPU.")
27
-@pytest.mark.parametrize("dtype", ["bfloat16"])
28
-@pytest.mark.parametrize("kv_cache_dtype", ["auto", "fp8"])
29
-def test_ptpc_fp8_rocm(vllm_runner, dtype: str, kv_cache_dtype: str) -> None:
30
- llm = vllm_runner(
31
- "facebook/opt-125m",
32
- dtype=dtype,
33
- quantization="ptpc_fp8",
34
- enforce_eager=True,
35
- kv_cache_dtype=kv_cache_dtype,
36
- allow_deprecated_quantization=True,
37
- )
38
-
39
- with llm:
40
-
41
- def check_model(model):
42
- fc1 = model.model.decoder.layers[0].fc1
43
- assert isinstance(fc1.quant_method, PTPCFp8LinearMethod)
44
- if kv_cache_dtype == "ptpc_fp8":
45
- attn = model.model.decoder.layers[0].self_attn.attn
46
- assert isinstance(attn.quant_method, Fp8KVCacheMethod)
47
- assert attn._k_scale == 1.0
48
- assert attn._v_scale == 1.0
49
-
50
- # For GPUs with hardware support, we keep weights in fp8
51
- if current_platform.has_device_capability(94):
52
- assert fc1.weight.dtype == current_platform.fp8_dtype()
53
-
54
- llm.apply_model(check_model)
55
-
56
- output = llm.generate_greedy("Hello my name is", max_tokens=4)
57
- assert output
vllm/model_executor/layers/quantization/__init__.py
-4
@@ -12,7 +12,6 @@ logger = init_logger(__name__)
12
QuantizationMethods = Literal[
13
"awq",
14
"fp8",
15
- "ptpc_fp8",
15
"fbgemm_fp8",
16
"fp_quant",
17
"modelopt",
@@ -39,7 +38,6 @@ QUANTIZATION_METHODS: list[str] = list(get_args(QuantizationMethods))
38
39
DEPRECATED_QUANTIZATION_METHODS = [
40
"tpu_int8",
42
- "ptpc_fp8",
41
"fbgemm_fp8",
42
"fp_quant",
43
"experts_int8",
@@ -132,7 +130,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
130
from .mxfp4 import Mxfp4Config
131
from .mxfp8 import Mxfp8Config
132
from .petit import PetitNvFp4Config
135
- from .ptpc_fp8 import PTPCFp8Config
133
from .torchao import TorchAOConfig
134
135
method_to_config: dict[str, type[QuantizationConfig]] = {
@@ -150,7 +147,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
147
"gptq": GPTQConfig,
148
"compressed-tensors": CompressedTensorsConfig,
149
"bitsandbytes": BitsAndBytesConfig,
153
- "ptpc_fp8": PTPCFp8Config,
150
"experts_int8": ExpertsInt8Config,
151
"quark": QuarkConfig,
152
"moe_wna16": MoeWNA16Config,
vllm/model_executor/layers/quantization/ptpc_fp8.py
deleted
-132
@@ -1,132 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-from typing import Any
5
-
6
-import torch
7
-from torch.nn.parameter import Parameter
8
-
9
-from vllm import _custom_ops as ops
10
-from vllm.model_executor.kernels.linear import (
11
- init_fp8_linear_kernel,
12
-)
13
-from vllm.model_executor.layers.attention import Attention
14
-from vllm.model_executor.layers.linear import LinearBase, UnquantizedLinearMethod
15
-from vllm.model_executor.layers.quantization import QuantizationMethods
16
-from vllm.model_executor.layers.quantization.base_config import QuantizeMethodBase
17
-from vllm.model_executor.layers.quantization.fp8 import (
18
- Fp8Config,
19
- Fp8KVCacheMethod,
20
- Fp8LinearMethod,
21
-)
22
-from vllm.model_executor.layers.quantization.utils.quant_utils import (
23
- is_layer_skipped,
24
- kFp8DynamicTokenSym,
25
-)
26
-from vllm.platforms import current_platform
27
-
28
-
29
-class PTPCFp8Config(Fp8Config):
30
- """Config class for Per-Token-Per-Channel Dynamic Quantization Fp8."""
31
-
32
- def __init__(
33
- self,
34
- activation_scheme: str = "dynamic",
35
- ignored_layers: list[str] | None = None,
36
- ) -> None:
37
- if not current_platform.is_rocm():
38
- raise ValueError("ptpc_fp8 quantization is supported only on ROCm.")
39
-
40
- if not current_platform.has_device_capability(94):
41
- raise ValueError(
42
- "ptpc_fp8 quantization is supported only on AMD Instinct MI300 GPUs and newer." # noqa: E501
43
- )
44
- if activation_scheme == "static":
45
- raise ValueError("ptpc_fp8 as of now only support dynamic quantization.")
46
-
47
- super().__init__(
48
- is_checkpoint_fp8_serialized=False,
49
- activation_scheme=activation_scheme,
50
- ignored_layers=ignored_layers,
51
- )
52
-
53
- @classmethod
54
- def get_name(cls) -> QuantizationMethods:
55
- return "ptpc_fp8"
56
-
57
- @classmethod
58
- def from_config(cls, config: dict[str, Any]) -> "PTPCFp8Config":
59
- activation_scheme = cls.get_from_keys(config, ["activation_scheme"])
60
- ignored_layers = cls.get_from_keys_or(config, ["ignored_layers"], None)
61
- return cls(activation_scheme=activation_scheme, ignored_layers=ignored_layers)
62
-
63
- def get_quant_method(
64
- self, layer: torch.nn.Module, prefix: str
65
- ) -> "QuantizeMethodBase | None":
66
- if isinstance(layer, LinearBase):
67
- if is_layer_skipped(prefix, self.ignored_layers):
68
- return UnquantizedLinearMethod()
69
- return PTPCFp8LinearMethod(self)
70
- elif isinstance(layer, Attention):
71
- return Fp8KVCacheMethod(self)
72
- return None
73
-
74
-
75
-class PTPCFp8LinearMethod(Fp8LinearMethod):
76
- """Linear method for Per-Token and Per-Channel FP8 Quantization.
77
- Only supports loading quantized BF16 model checkpoints with dynamic
78
- activation scaling. To load FP16 model checkpoints, user must specify
79
- to convert the FP16 model weight loading into BF16.
80
- The weight scaling factor will be initialized after
81
- the model weights are loaded.
82
-
83
- Limitations:
84
- 1. Only support float8_e4m3fnuz data type due to the limitation of
85
- torch._scaled_mm (https://github.com/ROCm/pytorch/blob/8c0504d7f3fb0ee4c278c096a5c3caedb01129fa/aten/src/ATen/native/cuda/Blas.cpp#L1041)
86
-
87
- Args:
88
- quant_config: The quantization config.
89
- """
90
-
91
- def __init__(self, quant_config: PTPCFp8Config):
92
- assert current_platform.is_rocm(), (
93
- "PTPCFp8LinearMethod is only supported on ROCm."
94
- )
95
- super().__init__(quant_config=quant_config)
96
- # Force weight quantization
97
- self.fp8_linear = init_fp8_linear_kernel(
98
- activation_quant_key=kFp8DynamicTokenSym,
99
- weight_quant_key=kFp8DynamicTokenSym,
100
- out_dtype=torch.get_default_dtype(),
101
- module_name=self.__class__.__name__,
102
- )
103
-
104
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
105
- assert layer.weight.data.dtype not in (torch.float16, torch.float32), (
106
- "Currently torch._scaled_mm (hipBLASLt) rowwise gemm only support "
107
- f"output dtype of bfloat16. {layer.weight.data.dtype} is specified."
108
- )
109
-
110
- if layer.weight.data.dtype == torch.bfloat16:
111
- # Quantize the weights.
112
- qweight, weight_scale = ops.scaled_fp8_quant(
113
- layer.weight, scale=None, use_per_token_if_dynamic=True
114
- )
115
-
116
- # Update the layer with the new values.
117
- layer.weight = Parameter(
118
- qweight.t(), requires_grad=False
119
- ) # Pretranspose the weight
120
- layer.weight_scale = Parameter(weight_scale, requires_grad=False)
121
- else:
122
- assert layer.weight.data.dtype == current_platform.fp8_dtype()
123
- assert getattr(layer, "weight_scale", None) is not None
124
- layer.input_scale = None
125
-
126
- def apply(
127
- self,
128
- layer: torch.nn.Module,
129
- x: torch.Tensor,
130
- bias: torch.Tensor | None = None,
131
- ) -> torch.Tensor:
132
- return self.fp8_linear.apply_weights(layer, x, bias)
vllm/platforms/rocm.py
-1
@@ -378,7 +378,6 @@ class RocmPlatform(Platform):
378
"fbgemm_fp8",
379
"gguf",
380
"quark",
381
- "ptpc_fp8",
381
"mxfp4",
382
"petit_nvfp4",
383
"torchao",