A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 20:45 UTC
[ Misc ] Improve Min Capability Checking in `compressed-tensors` (#6522)
Robert Shaw committed
Jul 18, 2024 at 10:39 UTC
58ca6632247cb738d069a585e1ec9a9d5e66da68
7 files changed
+41
-8
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+14
-8
@@ -37,7 +37,7 @@ class CompressedTensorsConfig(QuantizationConfig):
37
38
@classmethod
39
def get_min_capability(cls) -> int:
40
- return 75
40
+ return 70
41
42
def get_name(self) -> str:
43
return "compressed_tensors"
@@ -85,13 +85,14 @@ class CompressedTensorsConfig(QuantizationConfig):
85
def get_config_filenames(cls) -> List[str]:
86
return []
87
88
- def _check_gptq_and_marlin_can_run(self):
88
+ def _check_scheme_supported(self, min_capability: int):
89
capability = current_platform.get_device_capability()
90
capability = capability[0] * 10 + capability[1]
91
- if capability < 80:
92
- raise RuntimeError("The quantization config is not supported for ",
93
- "the current GPU. Minimum capability: 80. ",
94
- f"Current capability: {capability}.")
91
+ if capability < min_capability:
92
+ raise RuntimeError(
93
+ "Quantization scheme is not supported for ",
94
+ f"the current GPU. Min capability: {min_capability}. ",
95
+ f"Current capability: {capability}.")
96
97
def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
98
input_quant: BaseModel) -> bool:
@@ -171,7 +172,6 @@ class CompressedTensorsConfig(QuantizationConfig):
172
173
# Detect If Mixed Precision
174
if self._is_wNa16_group_channel(weight_quant, input_quant):
174
- self._check_gptq_and_marlin_can_run()
175
if (self.quant_format == CompressionFormat.marlin_24.value
176
and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
177
return CompressedTensorsW4A16Sparse24(
@@ -222,10 +222,16 @@ class CompressedTensorsConfig(QuantizationConfig):
222
raise ValueError(
223
f"Could not find quantization details for {layer}.")
224
225
- return self._get_schema(
225
+ scheme = self._get_schema(
226
weight_quant=layer_quant_details["weights"],
227
input_quant=layer_quant_details["input_activations"])
228
229
+ # Raise error if device does not support the scheme
230
+ # (e.g. fp8 needs ada lovelace)
231
+ self._check_scheme_supported(scheme.get_min_capability())
232
+
233
+ return scheme
234
+
235
236
class CompressedTensorsLinearMethod(LinearMethodBase):
237
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+7
@@ -12,6 +12,13 @@ class CompressedTensorsScheme(ABC):
12
of different quantization schemes supported by CompressedTensors.
13
"""
14
15
+ @abstractmethod
16
+ def get_min_capability(self) -> int:
17
+ """
18
+ Get minimum device capability.
19
+ """
20
+ raise NotImplementedError
21
+
22
@abstractmethod
23
def create_weights(self, *args, **kwargs):
24
"""
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+4
@@ -18,6 +18,10 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
18
in a linear transformation.
19
"""
20
21
+ def get_min_capability(self) -> int:
22
+ # volta and up
23
+ return 70
24
+
25
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
26
pass
27
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+4
@@ -29,6 +29,10 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
29
raise ValueError(
30
"group_size must be given when using strategy group")
31
32
+ def get_min_capability(self) -> int:
33
+ # ampere + up
34
+ return 80
35
+
36
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
37
pass
38
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+4
@@ -33,6 +33,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
33
"Consider quantizing with per tensor scales or upgrading "
34
"to Hopper.")
35
36
+ def get_min_capability(self) -> int:
37
+ # lovelace and up
38
+ return 89
39
+
40
def process_weights_after_loading(self, layer) -> None:
41
# If per tensor, when we have a fused module (e.g. QKV) with per
42
# tensor scales (thus N scales being passed to the kernel),
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+4
@@ -19,6 +19,10 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
19
self.strategy = strategy
20
self.is_static_input_scheme = is_static_input_scheme
21
22
+ def get_min_capability(self) -> int:
23
+ # turing and up
24
+ return 75
25
+
26
def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
27
# WEIGHT
28
# Cutlass kernels need transposed weight.
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+4
@@ -42,6 +42,10 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42
group_size=self.group_size,
43
is_sym=True)
44
45
+ def get_min_capability(self) -> int:
46
+ # ampere and up
47
+ return 80
48
+
49
def create_weights(self, layer: torch.nn.Module, input_size: int,
50
output_partition_sizes: List[int],
51
input_size_per_partition: int,