@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f83feccd7f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 08:15 UTC

[Bugfix] Ignore GPTQ quantization of Qwen2-VL visual module (#10169)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 8, 2024 at 22:36 UTC f83feccd7f661d0a582f9c0cb0bc9f802f4d995e
1 file changed +12 -2
vllm/model_executor/models/qwen2_vl.py
+12 -2
@@ -51,7 +51,9 @@ from vllm.model_executor.layers.activation import QuickGELU
51 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52 RowParallelLinear)
53 from vllm.model_executor.layers.logits_processor import LogitsProcessor
54 -from vllm.model_executor.layers.quantization import QuantizationConfig
54 +from vllm.model_executor.layers.quantization import (GPTQConfig,
55 + GPTQMarlinConfig,
56 + QuantizationConfig)
57 from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58 from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
59 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -982,7 +984,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
984 self.visual = Qwen2VisionTransformer(
985 config.vision_config,
986 norm_eps=getattr(config, "rms_norm_eps", 1e-6),
985 - quant_config=quant_config,
987 + quant_config=self._maybe_ignore_quant_config(quant_config),
988 prefix="visual",
989 )
990
@@ -1008,6 +1010,14 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1010 make_empty_intermediate_tensors_factory(
1011 ["hidden_states", "residual"], config.hidden_size))
1012
1013 + def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1014 + # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
1015 + # seems to avoid vision encoder sections for some models.
1016 + # See: https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4
1017 + if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
1018 + return None
1019 + return quant_config
1020 +
1021 def _validate_and_reshape_mm_tensor(self,
1022 mm_input: Union[torch.Tensor,
1023 List[torch.Tensor]],