A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 13:30 UTC
[Quantization][Deprecation] Remove Petit NVFP4 (#32694)
Signed-off-by: Robert Shaw <robshaw@redhat.com> Co-authored-by: Robert Shaw <robshaw@redhat.com>
Robert Shaw committed
Apr 4, 2026 at 20:07 UTC
968ed02acedf60d9a8128f96cc69a350327a5143
7 files changed
-448
setup.py
-2
@@ -1060,8 +1060,6 @@ setup(
1060
], # Required for audio processing
1061
"video": [], # Kept for backwards compatibility
1062
"flashinfer": [], # Kept for backwards compatibility
1063
- # Optional deps for AMD FP4 quantization support
1064
- "petit-kernel": ["petit-kernel"],
1063
# Optional deps for Helion kernel development
1064
# NOTE: When updating helion version, also update CI files:
1065
# - .buildkite/test_areas/kernels.yaml
vllm/config/model.py
-1
@@ -948,7 +948,6 @@ class ModelConfig:
948
"modelopt_fp4",
949
"modelopt_mxfp8",
950
"modelopt_mixed",
951
- "petit_nvfp4",
951
# Ensure heavy backends are probed last to avoid unnecessary
952
# imports during override detection (e.g., MXFP4 imports Triton)
953
"mxfp4",
vllm/model_executor/layers/linear.py
-1
@@ -60,7 +60,6 @@ WEIGHT_LOADER_V2_SUPPORTED = [
60
"ModelOptFp8PbWoLinearMethod",
61
"QuarkLinearMethod",
62
"ModelOptNvFp4LinearMethod",
63
- "PetitNvFp4LinearMethod",
63
]
64
65
vllm/model_executor/layers/quantization/__init__.py
-4
@@ -31,7 +31,6 @@ QuantizationMethods = Literal[
31
"inc",
32
"mxfp4",
33
"mxfp8",
34
- "petit_nvfp4",
34
"cpu_awq",
35
"online",
36
# Below are values of the OnlineQuantScheme enum, specified as strings to
@@ -48,7 +47,6 @@ DEPRECATED_QUANTIZATION_METHODS = [
47
"fbgemm_fp8",
48
"fp_quant",
49
"experts_int8",
51
- "petit_nvfp4",
50
]
51
52
# The customized quantization methods which will be added to this dict.
@@ -138,7 +136,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
136
from .mxfp4 import Mxfp4Config
137
from .mxfp8 import Mxfp8Config
138
from .online.base import OnlineQuantizationConfig
141
- from .petit import PetitNvFp4Config
139
from .torchao import TorchAOConfig
140
141
method_to_config: dict[str, type[QuantizationConfig]] = {
@@ -164,7 +161,6 @@ def get_quantization_config(quantization: str) -> type[QuantizationConfig]:
161
"inc": INCConfig,
162
"mxfp4": Mxfp4Config,
163
"mxfp8": Mxfp8Config,
167
- "petit_nvfp4": PetitNvFp4Config,
164
"cpu_awq": CPUAWQConfig,
165
"online": OnlineQuantizationConfig,
166
}
vllm/model_executor/layers/quantization/petit.py
deleted
-319
@@ -1,319 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-# Adapted from https://github.com/vllm-project/vllm/blob/main/vllm/model_executor/layers/quantization/modelopt.py
4
-
5
-from typing import Any
6
-
7
-import regex as re
8
-import torch
9
-from torch.nn.parameter import Parameter
10
-
11
-from vllm.logger import init_logger
12
-from vllm.model_executor.layers.attention import Attention
13
-from vllm.model_executor.layers.linear import (
14
- LinearBase,
15
- LinearMethodBase,
16
- UnquantizedLinearMethod,
17
-)
18
-from vllm.model_executor.layers.quantization import QuantizationMethods
19
-from vllm.model_executor.layers.quantization.base_config import (
20
- QuantizationConfig,
21
- QuantizeMethodBase,
22
-)
23
-from vllm.model_executor.layers.quantization.kv_cache import BaseKVCacheMethod
24
-from vllm.model_executor.layers.quantization.utils.petit_utils import (
25
- apply_petit_nvfp4_linear,
26
- prepare_nvfp4_layer_for_petit,
27
- verify_petit_nvfp4_supported,
28
-)
29
-from vllm.model_executor.layers.quantization.utils.quant_utils import is_layer_skipped
30
-from vllm.model_executor.parameter import ModelWeightParameter, PerTensorScaleParameter
31
-from vllm.platforms import current_platform
32
-
33
-# Initialize logger for the module
34
-logger = init_logger(__name__)
35
-
36
-
37
-# Configuration class to support the NVFP4 quantized model
38
-# generated by the ModelOpt quantization tool
39
-class PetitNvFp4Config(QuantizationConfig):
40
- """Config class for Petit FP4."""
41
-
42
- def __init__(
43
- self,
44
- is_checkpoint_nvfp4_serialized: bool = False,
45
- kv_cache_quant_algo: str | None = None,
46
- group_size: int | None = None,
47
- exclude_modules: list[str] | None = None,
48
- ) -> None:
49
- self._check_hardware_support()
50
- self.is_checkpoint_nvfp4_serialized = is_checkpoint_nvfp4_serialized
51
- if is_checkpoint_nvfp4_serialized:
52
- logger.warning(
53
- "Detected nvfp4 checkpoint. Please note that the "
54
- "format is experimental and subject to change."
55
- )
56
- self.group_size = group_size
57
- self.kv_cache_quant_algo = kv_cache_quant_algo
58
- self.exclude_modules = exclude_modules
59
-
60
- def _check_hardware_support(self) -> None:
61
- """
62
- Verifies that the current hardware is supported by the Petit backend.
63
- This backend is specifically designed for AMD GPUs and is not
64
- supported on the CUDA platform.
65
- """
66
- # This check ensures the code is NOT running on an NVIDIA GPU.
67
- if current_platform.is_cuda():
68
- raise ValueError(
69
- "The 'petit' quantization backend is designed for AMD GPUs "
70
- "and is not supported on the CUDA platform. For NVIDIA GPUs, "
71
- "please use a different quantization method such as FP8, AWQ, "
72
- "or GPTQ."
73
- )
74
-
75
- @classmethod
76
- def get_name(cls) -> QuantizationMethods:
77
- return "petit_nvfp4"
78
-
79
- @classmethod
80
- def get_supported_act_dtypes(cls) -> list[torch.dtype]:
81
- return [torch.bfloat16, torch.half]
82
-
83
- @classmethod
84
- def get_min_capability(cls) -> int:
85
- # Petit supports the gfx90a and gfx942 GPUs
86
- return 90
87
-
88
- @classmethod
89
- def get_config_filenames(cls) -> list[str]:
90
- return ["hf_quant_config.json"]
91
-
92
- @classmethod
93
- def from_config(cls, config: dict[str, Any]) -> "PetitNvFp4Config":
94
- qc = cls.get_from_keys(config, ["quantization"])
95
-
96
- quant_method_raw = qc.get("quant_algo")
97
- if not isinstance(quant_method_raw, str) or not quant_method_raw:
98
- raise ValueError("Missing or invalid 'quant_algo' in quantization config.")
99
- quant_method = quant_method_raw.upper()
100
-
101
- group_size_raw = qc.get("group_size")
102
- if not isinstance(group_size_raw, int):
103
- raise ValueError(
104
- "Missing or invalid 'group_size' (int) in hf_quant_config.json."
105
- )
106
- group_size = group_size_raw
107
-
108
- verify_petit_nvfp4_supported(quant_method, group_size)
109
-
110
- kv_cache_quant_algo_raw = qc.get("kv_cache_quant_algo") or "auto"
111
- if not isinstance(kv_cache_quant_algo_raw, str):
112
- raise ValueError("'kv_cache_quant_algo' must be a string if provided.")
113
- kv_cache_quant_algo = kv_cache_quant_algo_raw
114
-
115
- exclude_raw = qc.get("exclude_modules", [])
116
- if exclude_raw is None:
117
- exclude_modules: list[str] = []
118
- elif isinstance(exclude_raw, list) and all(
119
- isinstance(x, str) for x in exclude_raw
120
- ):
121
- exclude_modules = exclude_raw
122
- else:
123
- raise ValueError("'exclude_modules' must be a list[str] (or omitted).")
124
-
125
- is_checkpoint_nvfp4_serialized = "NVFP4" in quant_method
126
-
127
- return cls(
128
- is_checkpoint_nvfp4_serialized=is_checkpoint_nvfp4_serialized,
129
- kv_cache_quant_algo=kv_cache_quant_algo,
130
- group_size=group_size,
131
- exclude_modules=exclude_modules,
132
- )
133
-
134
- @classmethod
135
- def override_quantization_method(
136
- cls, hf_quant_cfg, user_quant
137
- ) -> QuantizationMethods | None:
138
- if not current_platform.is_rocm():
139
- return None
140
-
141
- qc = hf_quant_cfg.get("quantization", hf_quant_cfg)
142
- algo = (qc.get("quant_algo") or qc.get("quant_method") or "").upper()
143
- if algo in ("NVFP4", "MODELOPT_FP4", "MODELOPT"):
144
- return cls.get_name() # "petit_nvfp4"
145
- return None
146
-
147
- @classmethod
148
- def is_petit_nvfp4_compatible(cls, quant_config: dict[str, Any]) -> bool:
149
- qc = quant_config.get("quantization", quant_config)
150
- algo = (qc.get("quant_algo") or qc.get("quant_method") or "").upper()
151
- return algo == "NVFP4"
152
-
153
- def is_layer_excluded(self, prefix: str, exclude_modules: list[str]) -> bool:
154
- for pattern in exclude_modules:
155
- regex_str = pattern.replace(".", r"\.").replace("*", r".*")
156
- if re.fullmatch(regex_str, prefix):
157
- return True
158
- return False
159
-
160
- def get_quant_method(
161
- self, layer: torch.nn.Module, prefix: str
162
- ) -> "QuantizeMethodBase | None":
163
- exclude = self.require_exclude_modules()
164
-
165
- if isinstance(layer, LinearBase):
166
- if is_layer_skipped(prefix, exclude) or self.is_layer_excluded(
167
- prefix, exclude
168
- ):
169
- return UnquantizedLinearMethod()
170
- return PetitNvFp4LinearMethod(self)
171
- elif isinstance(layer, Attention):
172
- return PetitFp8KVCacheMethod(self)
173
- return None
174
-
175
- def get_scaled_act_names(self) -> list[str]:
176
- return []
177
-
178
- def require_group_size(self) -> int:
179
- if self.group_size is None:
180
- logger.warning("group_size not set; defaulting to 16 for NVFP4.")
181
- return 16
182
- return self.group_size
183
-
184
- def require_kv_cache_quant_algo(self) -> str:
185
- return self.kv_cache_quant_algo or "auto"
186
-
187
- def require_exclude_modules(self) -> list[str]:
188
- return list(self.exclude_modules or [])
189
-
190
-
191
-class PetitFp8KVCacheMethod(BaseKVCacheMethod):
192
- """
193
- Supports loading kv-cache scaling factors from FP8 checkpoints.
194
- """
195
-
196
- def __init__(self, quant_config: PetitNvFp4Config):
197
- super().__init__(quant_config)
198
-
199
-
200
-class PetitNvFp4LinearMethod(LinearMethodBase):
201
- """Linear method for NVFP4.
202
- Supports loading NVFP4 checkpoints with the following structure:
203
-
204
- |Tensor Name | datatype | shape |
205
- |----------------------------------------------------|
206
- |input_scale | torch.float32 | scalar |
207
- |weight | NVFP4(SE2M1) | [1, X, y/2] |
208
- |weight_scale | FP8-E4M3 | [X, Y] |
209
- |weight_scale_2 | torch.float32 | scalar |
210
-
211
- The weights are quantized per block of 16 elements.
212
- Args: quant_config: The ModelOpt quantization config.
213
- """
214
-
215
- def __init__(self, quant_config: PetitNvFp4Config):
216
- self.quant_config = quant_config
217
-
218
- def create_weights(
219
- self,
220
- layer: torch.nn.Module,
221
- input_size_per_partition: int,
222
- output_partition_sizes: list[int],
223
- input_size: int,
224
- output_size: int,
225
- params_dtype: torch.dtype,
226
- **extra_weight_attrs,
227
- ):
228
- del input_size, output_size
229
- if not self.quant_config.is_checkpoint_nvfp4_serialized:
230
- raise ValueError(
231
- "NVFP4 quantization was selected, "
232
- " dynamic quantization is not supported."
233
- )
234
-
235
- output_size_per_partition = sum(output_partition_sizes)
236
- weight_loader = extra_weight_attrs.get("weight_loader")
237
-
238
- layer.logical_widths = output_partition_sizes
239
-
240
- layer.input_size_per_partition = input_size_per_partition
241
- layer.output_size_per_partition = output_size_per_partition
242
- if input_size_per_partition % 16 != 0:
243
- raise ValueError(
244
- "Unsupported model when in features size is not multiple of 16"
245
- )
246
-
247
- weight_dtype = (
248
- torch.float8_e4m3fn
249
- if self.quant_config.is_checkpoint_nvfp4_serialized
250
- else params_dtype
251
- )
252
-
253
- weight = ModelWeightParameter(
254
- data=torch.empty(
255
- # 2 fp4 data is packed in one uint8 in the input dimension
256
- output_size_per_partition,
257
- input_size_per_partition // 2,
258
- dtype=torch.uint8,
259
- ),
260
- input_dim=1,
261
- output_dim=0,
262
- weight_loader=weight_loader,
263
- )
264
- layer.register_parameter("weight", weight)
265
-
266
- input_scale = PerTensorScaleParameter(
267
- data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
268
- weight_loader=weight_loader,
269
- )
270
-
271
- layer.register_parameter("input_scale", input_scale)
272
-
273
- weight_scale_2 = PerTensorScaleParameter(
274
- data=torch.empty(len(output_partition_sizes), dtype=torch.float32),
275
- weight_loader=weight_loader,
276
- )
277
- layer.register_parameter("weight_scale_2", weight_scale_2)
278
-
279
- group_size = self.quant_config.require_group_size()
280
- weight_scale = ModelWeightParameter(
281
- data=torch.empty(
282
- output_size_per_partition,
283
- input_size_per_partition // group_size,
284
- dtype=weight_dtype,
285
- ),
286
- input_dim=1,
287
- output_dim=0,
288
- weight_loader=weight_loader,
289
- )
290
-
291
- layer.register_parameter("weight_scale", weight_scale)
292
-
293
- def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
294
- input_scale_2 = layer.input_scale.max().to(torch.float32)
295
- weight_scale_2 = layer.weight_scale_2.max().to(torch.float32)
296
- layer.input_scale = Parameter(input_scale_2, requires_grad=False)
297
- layer.weight_scale_2 = Parameter(weight_scale_2, requires_grad=False)
298
- layer.alpha = Parameter(
299
- layer.input_scale * layer.weight_scale_2, requires_grad=False
300
- )
301
-
302
- prepare_nvfp4_layer_for_petit(layer)
303
- del layer.input_scale
304
-
305
- def apply(
306
- self,
307
- layer: torch.nn.Module,
308
- x: torch.Tensor,
309
- bias: torch.Tensor | None = None,
310
- ) -> torch.Tensor:
311
- return apply_petit_nvfp4_linear(
312
- input=x,
313
- weight=layer.weight,
314
- weight_scale=layer.weight_scale,
315
- weight_scale_2=layer.weight_scale_2,
316
- size_n=layer.output_size_per_partition,
317
- size_k=layer.input_size_per_partition,
318
- bias=bias,
319
- )
vllm/model_executor/layers/quantization/utils/petit_utils.py
deleted
-120
@@ -1,120 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-from typing import TYPE_CHECKING
4
-
5
-import torch
6
-
7
-# TYPE_CHECKING is used for static type analysis to prevent circular imports.
8
-if TYPE_CHECKING:
9
- from types import ModuleType
10
-
11
-# 1. Create a global variable as a placeholder for the module
12
-_petit_kernel: "ModuleType | None" = None
13
-
14
-_PETIT_INSTALL_MSG = (
15
- "Petit is not installed. Please install it with `pip install petit-kernel`."
16
-)
17
-
18
-
19
-def _import_petit_kernel() -> "ModuleType":
20
- """
21
- A helper function to handle the lazy import.
22
- The first time this function is called, it will import the petit_kernel
23
- library and store it in the global _petit_kernel variable.
24
- Subsequent calls will return the already-loaded module directly.
25
- """
26
- global _petit_kernel
27
- if _petit_kernel is not None:
28
- return _petit_kernel
29
-
30
- try:
31
- import petit_kernel
32
-
33
- _petit_kernel = petit_kernel
34
- return _petit_kernel
35
- except ImportError:
36
- # The 'from None' syntax prevents chaining the original ImportError,
37
- # making the traceback cleaner.
38
- raise ImportError(_PETIT_INSTALL_MSG) from None
39
-
40
-
41
-def _check_petit_nvfp4_supported(
42
- quant_method: str, group_size: int | None
43
-) -> tuple[bool, str | None]:
44
- if quant_method != "NVFP4":
45
- return (
46
- False,
47
- (
48
- "Petit currently only supports: NVFP4 quantizations in sglang. "
49
- "Please check the `hf_quant_config.json` file for your model's "
50
- "quant configuration."
51
- ),
52
- )
53
- if group_size is not None and group_size != 16:
54
- return (
55
- False,
56
- "Petit currently only supports: group_size=16 quantizations.",
57
- )
58
- return (True, None)
59
-
60
-
61
-def verify_petit_nvfp4_supported(quant_method: str, group_size: int | None) -> None:
62
- supported, error_msg = _check_petit_nvfp4_supported(quant_method, group_size)
63
- if not supported:
64
- assert error_msg is not None
65
- raise ValueError(error_msg)
66
-
67
-
68
-def prepare_nvfp4_layer_for_petit(layer: torch.nn.Module) -> None:
69
- # 2. Call _import_petit_kernel() to trigger (or get) the import.
70
- petit_kernel = _import_petit_kernel()
71
-
72
- # Repack weights to petit format
73
- part_size_n = layer.output_size_per_partition
74
- part_size_k = layer.input_size_per_partition
75
- qweight = layer.weight.view(torch.int32).contiguous()
76
-
77
- # 3. Call functions through the imported module variable.
78
- petit_qweight = petit_kernel.repack_nvfp4(
79
- qweight, size_n=part_size_n, size_k=part_size_k
80
- )
81
- layer.weight = torch.nn.Parameter(petit_qweight, requires_grad=False)
82
-
83
- # Permute scales
84
- weight_scale = petit_kernel.process_nvfp4_scales(
85
- scales=layer.weight_scale, size_k=part_size_k, size_n=part_size_n
86
- )
87
- layer.weight_scale = torch.nn.Parameter(weight_scale, requires_grad=False)
88
-
89
-
90
-def apply_petit_nvfp4_linear(
91
- input: torch.Tensor,
92
- weight: torch.Tensor,
93
- weight_scale: torch.Tensor,
94
- weight_scale_2: torch.Tensor,
95
- size_n: int,
96
- size_k: int,
97
- bias: torch.Tensor | None = None,
98
-) -> torch.Tensor:
99
- # Trigger (or get) the import here as well.
100
- petit_kernel = _import_petit_kernel()
101
-
102
- reshaped_x = input.reshape(-1, input.shape[-1])
103
- out_shape = input.shape[:-1] + (size_n,)
104
-
105
- # TODO: Use auto-tuning to find the performant solution_id
106
- # Call the function via the module variable.
107
- output = petit_kernel.mul_nvfp4_a16(
108
- a=reshaped_x,
109
- b=weight,
110
- s=weight_scale,
111
- global_scale=weight_scale_2,
112
- size_m=reshaped_x.size(0),
113
- size_n=size_n,
114
- size_k=size_k,
115
- solution_id=-1,
116
- )
117
- if bias is not None:
118
- output.add_(bias) # In-place add
119
-
120
- return output.reshape(out_shape)
vllm/platforms/rocm.py
-1
@@ -402,7 +402,6 @@ class RocmPlatform(Platform):
402
"gguf",
403
"quark",
404
"mxfp4",
405
- "petit_nvfp4",
405
"torchao",
406
"bitsandbytes",
407
]