@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7836fdcc11
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Misc] Fix `get_min_capability` (#5971)

Dipika Sikka committed Jun 30, 2024 at 16:15 UTC 7836fdcc11aef8c4494a4470522c685c2190eddc
5 files changed +17 -6
vllm/model_executor/layers/quantization/awq.py
+2 -1
@@ -43,7 +43,8 @@ class AWQConfig(QuantizationConfig):
43 def get_supported_act_dtypes(self) -> List[torch.dtype]:
44 return [torch.half]
45
46 - def get_min_capability(self) -> int:
46 + @classmethod
47 + def get_min_capability(cls) -> int:
48 # The AWQ kernel only supports Turing or newer GPUs.
49 return 75
50
vllm/model_executor/layers/quantization/base_config.py
+2 -1
@@ -44,8 +44,9 @@ class QuantizationConfig(ABC):
44 """List of supported activation dtypes."""
45 raise NotImplementedError
46
47 + @classmethod
48 @abstractmethod
48 - def get_min_capability(self) -> int:
49 + def get_min_capability(cls) -> int:
50 """Minimum GPU capability to support the quantization method.
51
52 E.g., 70 for Volta, 75 for Turing, 80 for Ampere.
vllm/model_executor/layers/quantization/bitsandbytes.py
+1 -1
@@ -38,7 +38,7 @@ class BitsAndBytesConfig(QuantizationConfig):
38 return [torch.float32, torch.float16, torch.bfloat16]
39
40 @classmethod
41 - def get_min_capability(self) -> int:
41 + def get_min_capability(cls) -> int:
42 return 70
43
44 @staticmethod
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+10 -2
@@ -33,10 +33,9 @@ class CompressedTensorsConfig(QuantizationConfig):
33 def get_supported_act_dtypes(cls) -> List[torch.dtype]:
34 return [torch.float16, torch.bfloat16]
35
36 - # Need to figure it out
36 @classmethod
37 def get_min_capability(cls) -> int:
39 - return 60
38 + return 75
39
40 def get_name(self) -> str:
41 return "compressed_tensors"
@@ -84,6 +83,14 @@ class CompressedTensorsConfig(QuantizationConfig):
83 def get_config_filenames(cls) -> List[str]:
84 return []
85
86 + def _check_gptq_and_marlin_can_run(self):
87 + capability = torch.cuda.get_device_capability()
88 + capability = capability[0] * 10 + capability[1]
89 + if capability < 80:
90 + raise RuntimeError("The quantization config is not supported for ",
91 + "the current GPU. Minimum capability: 80. ",
92 + f"Current capability: {capability}.")
93 +
94 def _is_static_tensor_w8a8(self, weight_quant: BaseModel,
95 input_quant: BaseModel) -> bool:
96 is_8_bits = weight_quant.num_bits == input_quant.num_bits == 8
@@ -126,6 +133,7 @@ class CompressedTensorsConfig(QuantizationConfig):
133 input_quant: BaseModel) -> "CompressedTensorsScheme":
134
135 if self._is_wNa16_group_channel(weight_quant, input_quant):
136 + self._check_gptq_and_marlin_can_run()
137 if (self.quant_format == CompressionFormat.marlin_24.value
138 and weight_quant.num_bits in W4A16SPARSE24_SUPPORTED_BITS):
139 return CompressedTensorsW4A16Sparse24(
vllm/model_executor/layers/quantization/squeezellm.py
+2 -1
@@ -39,7 +39,8 @@ class SqueezeLLMConfig(QuantizationConfig):
39 def get_supported_act_dtypes(self) -> List[torch.dtype]:
40 return [torch.half]
41
42 - def get_min_capability(self) -> int:
42 + @classmethod
43 + def get_min_capability(cls) -> int:
44 return 70
45
46 @staticmethod