A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 08:15 UTC
[Bugfix] Ignore GPTQ quantization of Qwen2-VL visual module (#10169)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 8, 2024 at 22:36 UTC
f83feccd7f661d0a582f9c0cb0bc9f802f4d995e
1 file changed
+12
-2
vllm/model_executor/models/qwen2_vl.py
+12
-2
@@ -51,7 +51,9 @@ from vllm.model_executor.layers.activation import QuickGELU
51
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
52
RowParallelLinear)
53
from vllm.model_executor.layers.logits_processor import LogitsProcessor
54
-from vllm.model_executor.layers.quantization import QuantizationConfig
54
+from vllm.model_executor.layers.quantization import (GPTQConfig,
55
+ GPTQMarlinConfig,
56
+ QuantizationConfig)
57
from vllm.model_executor.layers.sampler import SamplerOutput, get_sampler
58
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
59
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
@@ -982,7 +984,7 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
984
self.visual = Qwen2VisionTransformer(
985
config.vision_config,
986
norm_eps=getattr(config, "rms_norm_eps", 1e-6),
985
- quant_config=quant_config,
987
+ quant_config=self._maybe_ignore_quant_config(quant_config),
988
prefix="visual",
989
)
990
@@ -1008,6 +1010,14 @@ class Qwen2VLForConditionalGeneration(nn.Module, SupportsMultiModal,
1010
make_empty_intermediate_tensors_factory(
1011
["hidden_states", "residual"], config.hidden_size))
1012
1013
+ def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
1014
+ # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
1015
+ # seems to avoid vision encoder sections for some models.
1016
+ # See: https://huggingface.co/Qwen/Qwen2-VL-2B-Instruct-GPTQ-Int4
1017
+ if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
1018
+ return None
1019
+ return quant_config
1020
+
1021
def _validate_and_reshape_mm_tensor(self,
1022
mm_input: Union[torch.Tensor,
1023
List[torch.Tensor]],