A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[BugFix] Stop silent failures on compressed-tensors parsing (#9381)
Dipika Sikka committed
Oct 17, 2024 at 21:54 UTC
48138a8415f416df502e68a24f0b3025a425c04c
2 files changed
+23
-13
requirements-common.txt
+1
-1
@@ -31,4 +31,4 @@ pyyaml
31
six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32
setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33
einops # Required for Qwen2-VL.
34
-compressed-tensors == 0.6.0 # required for compressed-tensors
34
+compressed-tensors == 0.7.1 # required for compressed-tensors
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+22
-12
@@ -100,12 +100,21 @@ class CompressedTensorsConfig(QuantizationConfig):
100
target_scheme_map[target][
101
"weights"] = QuantizationArgs.parse_obj(
102
quant_config.get("weights"))
103
- try:
104
- target_scheme_map[target][
105
- "input_activations"] = QuantizationArgs.parse_obj(
106
- quant_config.get("input_activations"))
107
- except Exception:
108
- target_scheme_map[target]["input_activations"] = None
103
+
104
+ target_scheme_map[target]["input_activations"] = None
105
+ if is_activation_quantization_format(quant_format):
106
+ input_activations = quant_config.get("input_activations")
107
+ # The only case where we have activation quant supported
108
+ # but no input_activations provided in the config
109
+ # should be w8a16fp8 w8a16fp8 can also run for cases where
110
+ # there is an input_quant but it is ignored
111
+ if not input_activations:
112
+ assert target_scheme_map[target][
113
+ "weights"].type == QuantizationType.FLOAT
114
+ else:
115
+ target_scheme_map[target][
116
+ "input_activations"] = QuantizationArgs.parse_obj(
117
+ quant_config.get("input_activations"))
118
119
return cls(target_scheme_map=target_scheme_map,
120
ignore=ignore,
@@ -244,8 +253,6 @@ class CompressedTensorsConfig(QuantizationConfig):
253
group_size=weight_quant.group_size,
254
actorder=weight_quant.actorder)
255
247
- # Detect If Activation Quantization.
248
- # TODO @dsikka: clean-up conditions
256
if is_activation_quantization_format(self.quant_format):
257
if self._is_fp8_w8a8(weight_quant, input_quant):
258
is_fp8_w8a8_supported = self._check_scheme_supported(
@@ -256,16 +263,19 @@ class CompressedTensorsConfig(QuantizationConfig):
263
is_static_input_scheme=(input_quant
264
and not input_quant.dynamic))
265
else:
266
+ # note: input_quant will be present for converted models;
267
+ # will be ignored during inference post loading
268
return CompressedTensorsW8A16Fp8(
269
strategy=weight_quant.strategy,
261
- is_static_input_scheme=(input_quant
262
- and not input_quant.dynamic))
270
+ is_static_input_scheme=not input_quant.dynamic)
271
272
+ # note: input_quant can be None
273
if self._is_fp8_w8a16(weight_quant, input_quant):
274
+ is_static_input_scheme = (input_quant
275
+ and not input_quant.dynamic)
276
return CompressedTensorsW8A16Fp8(
277
strategy=weight_quant.strategy,
267
- is_static_input_scheme=(input_quant
268
- and not input_quant.dynamic))
278
+ is_static_input_scheme=is_static_input_scheme)
279
280
if self._is_static_tensor_w8a8(weight_quant, input_quant):
281
return CompressedTensorsW8A8Int8(