A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Misc] Fix `get_min_capability` (#5971)
Dipika Sikka committed
Jun 30, 2024 at 16:15 UTC
7836fdcc11aef8c4494a4470522c685c2190eddc
5 files changed
+17
-6
vllm/model_executor/layers/quantization/awq.py
+2
-1
@@ -43,7 +43,8 @@ class AWQConfig(QuantizationConfig):
43
def get_supported_act_dtypes(self) -> List[torch.dtype]:
44
return [torch.half]
45
46
- def get_min_capability(self) -> int:
46
+ @classmethod
47
+ def get_min_capability(cls) -> int:
48
# The AWQ kernel only supports Turing or newer GPUs.
49
return 75
50
vllm/model_executor/layers/quantization/base_config.py
+2
-1
@@ -44,8 +44,9 @@ class QuantizationConfig(ABC):
44
"""List of supported activation dtypes."""
45
raise NotImplementedError
46
47
+ @classmethod
48
@abstractmethod
48
- def get_min_capability(self) -> int:
49
+ def get_min_capability(cls) -> int:
50
"""Minimum GPU capability to support the quantization method.
51
52
E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
vllm/model_executor/layers/quantization/bitsandbytes.py
+1
-1
@@ -38,7 +38,7 @@ class BitsAndBytesConfig(QuantizationConfig):
38
return [torch.float32, torch.float16, torch.bfloat16]
39
40
@classmethod
41
- def get_min_capability(self) -> int:
41
+ def get_min_capability(cls) -> int:
42
return 70
43
44
@staticmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+10
-2
@@ -33,10 +33,9 @@ class CompressedTensorsConfig(QuantizationConfig):
33
def get_supported_act_dtypes(cls) -> List[torch.dtype]:
34
return [torch.float16, torch.bfloat16]
35
36
- # Need to figure it out
36
@classmethod
37
def get_min_capability(cls) -> int:
39
- return 60
38
+ return 75
39
40
def get_name(self) -> str:
41
return "compressed_tensors"
@@ -84,6 +83,14 @@ class CompressedTensorsConfig(QuantizationConfig):
83
def get_config_filenames(cls) -> List[str]:
84
return []
85
86
+ def _check_gptq_and_marlin_can_run(self):
87
+ capability = torch.cuda.get_device_capability()
88
+ capability = capability[0] * 10 + capability[1]
89
+ if capability < 80:
90
+ raise RuntimeError("The quantization config is not supported for ",
91
+ "the current GPU. Minimum capability: 80. ",
92
+ f"Current capability: {capability}.")
93
+
94
def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
95
input_quant: BaseModel) -> bool:
96
is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
@@ -126,6 +133,7 @@ class CompressedTensorsConfig(QuantizationConfig):
133
input_quant: BaseModel) -> "CompressedTensorsScheme":
134
135
if self._is_wNa16_group_channel(weight_quant, input_quant):
136
+ self._check_gptq_and_marlin_can_run()
137
if (self.quant_format == CompressionFormat.marlin_24.value
138
and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
139
return CompressedTensorsW4A16Sparse24(
vllm/model_executor/layers/quantization/squeezellm.py
+2
-1
@@ -39,7 +39,8 @@ class SqueezeLLMConfig(QuantizationConfig):
39
def get_supported_act_dtypes(self) -> List[torch.dtype]:
40
return [torch.half]
41
42
- def get_min_capability(self) -> int:
42
+ @classmethod
43
+ def get_min_capability(cls) -> int:
44
return 70
45
46
@staticmethod