A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Quantization]: Support compressed-tensors mixed-precision model loading (#22468)
Signed-off-by: Dipika Sikka <dipikasikka1@gmail.com>
Dipika Sikka committed
Aug 14, 2025 at 17:32 UTC
4121de512ea44bd2ad4fae22c96c437598be6b62
1 file changed
+20
-5
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+20
-5
@@ -192,7 +192,15 @@ class CompressedTensorsConfig(QuantizationConfig):
192
quant_config.get("weights"))
193
194
target_scheme_map[target]["input_activations"] = None
195
- if is_activation_quantization_format(quant_format):
195
+ target_scheme_map[target]["format"] = quant_config.get(
196
+ "format")
197
+ format = target_scheme_map[target].get("format")
198
+ # If no per-config format defined, use global format in config
199
+ act_quant_format = is_activation_quantization_format(
200
+ format
201
+ ) if format is not None else is_activation_quantization_format(
202
+ quant_format)
203
+ if act_quant_format:
204
input_activations = quant_config.get("input_activations")
205
# The only case where we have activation quant supported
206
# but no input_activations provided in the config
@@ -389,8 +397,10 @@ class CompressedTensorsConfig(QuantizationConfig):
397
return (is_channel_group and input_quant_none and is_static)
398
399
def _get_scheme_from_parts(
392
- self, weight_quant: BaseModel,
393
- input_quant: BaseModel) -> "CompressedTensorsScheme":
400
+ self,
401
+ weight_quant: BaseModel,
402
+ input_quant: BaseModel,
403
+ format: Optional[str] = None) -> "CompressedTensorsScheme":
404
# Detect If Mixed Precision
405
if self._is_fp4a16_nvfp4(weight_quant, input_quant):
406
return CompressedTensorsW4A16Fp4()
@@ -412,7 +422,11 @@ class CompressedTensorsConfig(QuantizationConfig):
422
group_size=weight_quant.group_size,
423
actorder=weight_quant.actorder)
424
415
- if is_activation_quantization_format(self.quant_format):
425
+ act_quant_format = is_activation_quantization_format(
426
+ format
427
+ ) if format is not None else is_activation_quantization_format(
428
+ self.quant_format)
429
+ if act_quant_format:
430
if self._is_fp4a4_nvfp4(weight_quant, input_quant):
431
if cutlass_fp4_supported(
432
) or envs.VLLM_USE_NVFP4_CT_EMULATIONS:
@@ -507,6 +521,7 @@ class CompressedTensorsConfig(QuantizationConfig):
521
scheme_dict = self.target_scheme_map[matched_target]
522
weight_quant = scheme_dict.get("weights")
523
input_quant = scheme_dict.get("input_activations")
524
+ format = scheme_dict.get("format")
525
526
# Find the sparsity scheme of the layer
527
# assume that fused layers inerhit first component's sparsity scheme
@@ -547,7 +562,7 @@ class CompressedTensorsConfig(QuantizationConfig):
562
scheme = self._get_scheme_from_parts( # type: ignore
563
weight_quant=weight_quant,
564
input_quant=input_quant,
550
- )
565
+ format=format)
566
567
# Raise error if device does not support the scheme
568
# (e.g. fp8 needs ada lovelace)