@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 58ca663224
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 20:45 UTC

[ Misc ] Improve Min Capability Checking in `compressed-tensors` (#6522)

Robert Shaw committed Jul 18, 2024 at 10:39 UTC 58ca6632247cb738d069a585e1ec9a9d5e66da68
7 files changed +41 -8
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+14 -8
@@ -37,7 +37,7 @@ class CompressedTensorsConfig(QuantizationConfig):
37
38 @classmethod
39 def get_min_capability(cls) -> int:
40 - return 75
40 + return 70
41
42 def get_name(self) -> str:
43 return "compressed_tensors"
@@ -85,13 +85,14 @@ class CompressedTensorsConfig(QuantizationConfig):
85 def get_config_filenames(cls) -> List[str]:
86 return []
87
88 - def _check_gptq_and_marlin_can_run(self):
88 + def _check_scheme_supported(self, min_capability: int):
89 capability = current_platform.get_device_capability()
90 capability = capability[0] * 10 + capability[1]
91 - if capability < 80:
92 - raise RuntimeError("The quantization config is not supported for ",
93 - "the current GPU. Minimum capability: 80. ",
94 - f"Current capability: {capability}.")
91 + if capability < min_capability:
92 + raise RuntimeError(
93 + "Quantization scheme is not supported for ",
94 + f"the current GPU. Min capability: {min_capability}. ",
95 + f"Current capability: {capability}.")
96
97 def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
98 input_quant: BaseModel) -> bool:
@@ -171,7 +172,6 @@ class CompressedTensorsConfig(QuantizationConfig):
172
173 # Detect If Mixed Precision
174 if self._is_wNa16_group_channel(weight_quant, input_quant):
174 - self._check_gptq_and_marlin_can_run()
175 if (self.quant_format == CompressionFormat.marlin_24.value
176 and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
177 return CompressedTensorsW4A16Sparse24(
@@ -222,10 +222,16 @@ class CompressedTensorsConfig(QuantizationConfig):
222 raise ValueError(
223 f"Could not find quantization details for {layer}.")
224
225 - return self._get_schema(
225 + scheme = self._get_schema(
226 weight_quant=layer_quant_details["weights"],
227 input_quant=layer_quant_details["input_activations"])
228
229 + # Raise error if device does not support the scheme
230 + # (e.g. fp8 needs ada lovelace)
231 + self._check_scheme_supported(scheme.get_min_capability())
232 +
233 + return scheme
234 +
235
236 class CompressedTensorsLinearMethod(LinearMethodBase):
237
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_scheme.py
+7
@@ -12,6 +12,13 @@ class CompressedTensorsScheme(ABC):
12 of different quantization schemes supported by CompressedTensors.
13 """
14
15 + @abstractmethod
16 + def get_min_capability(self) -> int:
17 + """
18 + Get minimum device capability.
19 + """
20 + raise NotImplementedError
21 +
22 @abstractmethod
23 def create_weights(self, *args, **kwargs):
24 """
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_unquantized.py
+4
@@ -18,6 +18,10 @@ class CompressedTensorsUnquantized(CompressedTensorsScheme):
18 in a linear transformation.
19 """
20
21 + def get_min_capability(self) -> int:
22 + # volta and up
23 + return 70
24 +
25 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
26 pass
27
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w4a16_24.py
+4
@@ -29,6 +29,10 @@ class CompressedTensorsW4A16Sparse24(CompressedTensorsScheme):
29 raise ValueError(
30 "group_size must be given when using strategy group")
31
32 + def get_min_capability(self) -> int:
33 + # ampere + up
34 + return 80
35 +
36 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
37 pass
38
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_fp8.py
+4
@@ -33,6 +33,10 @@ class CompressedTensorsW8A8Fp8(CompressedTensorsScheme):
33 "Consider quantizing with per tensor scales or upgrading "
34 "to Hopper.")
35
36 + def get_min_capability(self) -> int:
37 + # lovelace and up
38 + return 89
39 +
40 def process_weights_after_loading(self, layer) -> None:
41 # If per tensor, when we have a fused module (e.g. QKV) with per
42 # tensor scales (thus N scales being passed to the kernel),
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_w8a8_int8.py
+4
@@ -19,6 +19,10 @@ class CompressedTensorsW8A8Int8(CompressedTensorsScheme):
19 self.strategy = strategy
20 self.is_static_input_scheme = is_static_input_scheme
21
22 + def get_min_capability(self) -> int:
23 + # turing and up
24 + return 75
25 +
26 def process_weights_after_loading(self, layer: torch.nn.Module) -> None:
27 # WEIGHT
28 # Cutlass kernels need transposed weight.
vllm/model_executor/layers/quantization/compressed_tensors/schemes/compressed_tensors_wNa16.py
+4
@@ -42,6 +42,10 @@ class CompressedTensorsWNA16(CompressedTensorsScheme):
42 group_size=self.group_size,
43 is_sym=True)
44
45 + def get_min_capability(self) -> int:
46 + # ampere and up
47 + return 80
48 +
49 def create_weights(self, layer: torch.nn.Module, input_size: int,
50 output_partition_sizes: List[int],
51 input_size_per_partition: int,