@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 311f743831
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Bugfix] Fix gptq failure on T4s (#7264)

Lucas Wilkinson committed Aug 7, 2024 at 16:05 UTC 311f743831f164f83361295b562f6cdb88ad3418
3 files changed +14 -15
vllm/model_executor/layers/quantization/awq_marlin.py
+1 -2
@@ -126,8 +126,7 @@ class AWQMarlinConfig(QuantizationConfig):
126
127 return check_marlin_supported(quant_type=cls.TYPE_MAP[num_bits],
128 group_size=group_size,
129 - has_zp=has_zp,
130 - min_capability=cls.get_min_capability())
129 + has_zp=has_zp)
130
131
132 class AWQMarlinLinearMethod(LinearMethodBase):
vllm/model_executor/layers/quantization/gptq_marlin.py
+1 -2
@@ -136,8 +136,7 @@ class GPTQMarlinConfig(QuantizationConfig):
136 return False
137
138 return check_marlin_supported(quant_type=cls.TYPE_MAP[(num_bits, sym)],
139 - group_size=group_size,
140 - min_capability=cls.get_min_capability())
139 + group_size=group_size)
140
141
142 class GPTQMarlinLinearMethod(LinearMethodBase):
vllm/model_executor/layers/quantization/utils/marlin_utils.py
+12 -11
@@ -26,12 +26,13 @@ USE_FP32_REDUCE_DEFAULT = True
26 # without runtime zero-point. We support common cases, i.e. AWQ and GPTQ.
27 # TODO: we may want to move this into the C++ so its closer to the actual impl
28 def query_marlin_supported_quant_types(has_zp: bool,
29 - min_capability: Optional[int] = None):
30 - if min_capability is None:
29 + device_capability: Optional[int] = None
30 + ):
31 + if device_capability is None:
32 major, minor = current_platform.get_device_capability()
32 - min_capability = major * 10 + minor
33 + device_capability = major * 10 + minor
34
34 - if min_capability < 80:
35 + if device_capability < 80:
36 return []
37
38 if has_zp:
@@ -48,20 +49,20 @@ def _check_marlin_supported(
49 quant_type: ScalarType,
50 group_size: Optional[int],
51 has_zp: bool,
51 - min_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]:
52 + device_capability: Optional[int] = None) -> Tuple[bool, Optional[str]]:
53
53 - if min_capability is None:
54 + if device_capability is None:
55 major, minor = current_platform.get_device_capability()
55 - min_capability = major * 10 + minor
56 + device_capability = major * 10 + minor
57
58 supported_types = query_marlin_supported_quant_types(
58 - has_zp, min_capability)
59 + has_zp, device_capability)
60
61 if quant_type not in supported_types:
62 return (False, f"Marlin does not support weight_bits = {quant_type}. "
63 f"Only types = {supported_types} "
64 f"are supported (for group_size = {group_size}, "
64 - f"min_capability = {min_capability}, zp = {has_zp}).")
65 + f"device_capability = {device_capability}, zp = {has_zp}).")
66 if (group_size is None or group_size not in MARLIN_SUPPORTED_GROUP_SIZES):
67 return (False, f"Marlin does not support group_size = {group_size}. "
68 f"Only group_sizes = {MARLIN_SUPPORTED_GROUP_SIZES} "
@@ -73,9 +74,9 @@ def _check_marlin_supported(
74 def check_marlin_supported(quant_type: ScalarType,
75 group_size: int,
76 has_zp: bool = False,
76 - min_capability: Optional[int] = None) -> bool:
77 + device_capability: Optional[int] = None) -> bool:
78 cond, _ = _check_marlin_supported(quant_type, group_size, has_zp,
78 - min_capability)
79 + device_capability)
80 return cond
81
82