A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 4, 03:45 UTC
[Compressed Tensors] Allow configs with non-explicit ignores (#41965)
Signed-off-by: Kyle Sayers <kylesayrs@gmail.com>
Kyle Sayers committed
May 7, 2026 at 17:03 UTC
c1819ca283021b65ac1ea71901d99f26b1103efe
3 files changed
+34
-19
tests/quantization/test_compressed_tensors.py
+21
@@ -32,6 +32,9 @@ from vllm.model_executor.layers.quantization.compressed_tensors.compressed_tenso
32
CompressedTensorsW8A16Fp8,
33
CompressedTensorsWNA16,
34
)
35
+from vllm.model_executor.layers.quantization.compressed_tensors.utils import (
36
+ find_matched_target,
37
+)
38
from vllm.model_executor.layers.quantization.input_quant_fp8 import QuantFP8
39
from vllm.model_executor.layers.quantization.utils.nvfp4_utils import (
40
cutlass_fp4_supported,
@@ -635,6 +638,24 @@ def test_get_quant_method_returns_none_for_unmatched_parallel_lm_head():
638
)
639
640
641
+def test_find_matched_target_returns_none_on_no_match():
642
+ result = find_matched_target(
643
+ layer_name="model.layers.0.self_attn.qkv_proj",
644
+ module=Mock(spec=torch.nn.Linear),
645
+ targets=["no_match_target"],
646
+ )
647
+ assert result is None
648
+
649
+
650
+def test_get_scheme_dict_returns_none_on_no_match():
651
+ config = _make_ct_config(target="matched_layer")
652
+ result = config.get_scheme_dict(
653
+ layer=Mock(spec=torch.nn.Linear),
654
+ layer_name="model.layers.0.unmatched_layer",
655
+ )
656
+ assert result is None
657
+
658
+
659
@pytest.mark.skipif(
660
not current_platform.is_cuda() or not current_platform.has_device_capability(75),
661
reason="MXFP8 requires Turing (sm_75+) or newer.",
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+12
-12
@@ -1,7 +1,6 @@
1
# SPDX-License-Identifier: Apache-2.0
2
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
4
-from contextlib import suppress
4
from functools import partial
5
from typing import TYPE_CHECKING, Any, Literal, cast
6
@@ -747,13 +746,13 @@ class CompressedTensorsConfig(QuantizationConfig):
746
self.sparsity_ignore_list
747
)
748
sparsity_scheme: SparsityCompressionConfig | None = None
750
- with suppress(ValueError):
751
- matched_target = find_matched_target(
752
- layer_name=layer_name,
753
- module=layer,
754
- targets=sparsity_targets,
755
- fused_mapping=self.packed_modules_mapping,
756
- )
749
+ matched_target = find_matched_target(
750
+ layer_name=layer_name,
751
+ module=layer,
752
+ targets=sparsity_targets,
753
+ fused_mapping=self.packed_modules_mapping,
754
+ )
755
+ if matched_target is not None:
756
sparsity_scheme = self.sparsity_scheme_map[matched_target]
757
758
if self.supports_cutlass_24(
@@ -821,10 +820,11 @@ class CompressedTensorsConfig(QuantizationConfig):
820
targets=self.target_scheme_map.keys(),
821
fused_mapping=self.packed_modules_mapping,
822
)
824
- scheme_dict = self.target_scheme_map[matched_target]
825
- if scheme_dict.get("format") is None:
826
- scheme_dict["format"] = self.quant_format
827
- return scheme_dict
823
+ if matched_target is not None:
824
+ scheme_dict = self.target_scheme_map[matched_target]
825
+ if scheme_dict.get("format") is None:
826
+ scheme_dict["format"] = self.quant_format
827
+ return scheme_dict
828
829
return None
830
vllm/model_executor/layers/quantization/compressed_tensors/utils.py
+1
-7
@@ -115,7 +115,7 @@ def find_matched_target(
115
module: Module,
116
targets: Iterable[str],
117
fused_mapping: Mapping[str, list[str]] = MappingProxyType({}),
118
-) -> str:
118
+) -> str | None:
119
"""
120
Helper function to look up which "target" in the compressed-tensors
121
config that a layer corresponds to.
@@ -150,12 +150,6 @@ def find_matched_target(
150
or _match_fused_layer(layer_name, targets, fused_mapping)
151
)
152
153
- if matched_target is None:
154
- raise ValueError(
155
- f"Unable to find matching target for {layer_name} in the "
156
- "compressed-tensors config."
157
- )
158
-
153
return matched_target
154
155