@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 48138a8415
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[BugFix] Stop silent failures on compressed-tensors parsing (#9381)

Dipika Sikka committed Oct 17, 2024 at 21:54 UTC 48138a8415f416df502e68a24f0b3025a425c04c
2 files changed +23 -13
requirements-common.txt
+1 -1
@@ -31,4 +31,4 @@ pyyaml
31 six>=1.16.0; python_version > '3.11' # transitive dependency of pandas that needs to be the latest version for python 3.12
32 setuptools>=74.1.1; python_version > '3.11' # Setuptools is used by triton, we need to ensure a modern version is installed for 3.12+ so that it does not try to import distutils, which was removed in 3.12
33 einops # Required for Qwen2-VL.
34 -compressed-tensors == 0.6.0 # required for compressed-tensors
34 +compressed-tensors == 0.7.1 # required for compressed-tensors
vllm/model_executor/layers/quantization/compressed_tensors/compressed_tensors.py
+22 -12
@@ -100,12 +100,21 @@ class CompressedTensorsConfig(QuantizationConfig):
100 target_scheme_map[target][
101 "weights"] = QuantizationArgs.parse_obj(
102 quant_config.get("weights"))
103 - try:
104 - target_scheme_map[target][
105 - "input_activations"] = QuantizationArgs.parse_obj(
106 - quant_config.get("input_activations"))
107 - except Exception:
108 - target_scheme_map[target]["input_activations"] = None
103 +
104 + target_scheme_map[target]["input_activations"] = None
105 + if is_activation_quantization_format(quant_format):
106 + input_activations = quant_config.get("input_activations")
107 + # The only case where we have activation quant supported
108 + # but no input_activations provided in the config
109 + # should be w8a16fp8 w8a16fp8 can also run for cases where
110 + # there is an input_quant but it is ignored
111 + if not input_activations:
112 + assert target_scheme_map[target][
113 + "weights"].type == QuantizationType.FLOAT
114 + else:
115 + target_scheme_map[target][
116 + "input_activations"] = QuantizationArgs.parse_obj(
117 + quant_config.get("input_activations"))
118
119 return cls(target_scheme_map=target_scheme_map,
120 ignore=ignore,
@@ -244,8 +253,6 @@ class CompressedTensorsConfig(QuantizationConfig):
253 group_size=weight_quant.group_size,
254 actorder=weight_quant.actorder)
255
247 - # Detect If Activation Quantization.
248 - # TODO @dsikka: clean-up conditions
256 if is_activation_quantization_format(self.quant_format):
257 if self._is_fp8_w8a8(weight_quant, input_quant):
258 is_fp8_w8a8_supported = self._check_scheme_supported(
@@ -256,16 +263,19 @@ class CompressedTensorsConfig(QuantizationConfig):
263 is_static_input_scheme=(input_quant
264 and not input_quant.dynamic))
265 else:
266 + # note: input_quant will be present for converted models;
267 + # will be ignored during inference post loading
268 return CompressedTensorsW8A16Fp8(
269 strategy=weight_quant.strategy,
261 - is_static_input_scheme=(input_quant
262 - and not input_quant.dynamic))
270 + is_static_input_scheme=not input_quant.dynamic)
271
272 + # note: input_quant can be None
273 if self._is_fp8_w8a16(weight_quant, input_quant):
274 + is_static_input_scheme = (input_quant
275 + and not input_quant.dynamic)
276 return CompressedTensorsW8A16Fp8(
277 strategy=weight_quant.strategy,
267 - is_static_input_scheme=(input_quant
268 - and not input_quant.dynamic))
278 + is_static_input_scheme=is_static_input_scheme)
279
280 if self._is_static_tensor_w8a8(weight_quant, input_quant):
281 return CompressedTensorsW8A8Int8(