A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 12:15 UTC
[Bugfix] Fix Minicpm-O-int4 GPTQ model inference (#17397)
Signed-off-by: Isotr0py <2037008807@qq.com>
Isotr0py committed
Apr 30, 2025 at 02:21 UTC
2fa2a50bf950797cb59d48908d205c655ec02654
2 files changed
+36
-2
vllm/model_executor/models/minicpmo.py
+35
-1
@@ -28,12 +28,16 @@ from typing import (Any, Callable, Literal, Optional, Set, Tuple, TypedDict,
28
29
import torch
30
from torch import nn
31
-from transformers import BatchFeature
31
+from transformers import BatchFeature, PretrainedConfig
32
from transformers.modeling_outputs import BaseModelOutputWithPast
33
from transformers.models.whisper.modeling_whisper import (
34
ACT2FN, WHISPER_ATTENTION_CLASSES, WhisperConfig, WhisperEncoder)
35
36
from vllm.config import VllmConfig
37
+from vllm.model_executor.layers.quantization import QuantizationConfig
38
+from vllm.model_executor.layers.quantization.gptq import GPTQConfig
39
+from vllm.model_executor.layers.quantization.gptq_marlin import (
40
+ GPTQMarlinConfig)
41
from vllm.multimodal import MULTIMODAL_REGISTRY, MultiModalKwargs
42
from vllm.multimodal.inputs import (MultiModalDataDict, MultiModalFieldConfig,
43
NestedTensors)
@@ -512,6 +516,36 @@ class MiniCPMO(MiniCPMV2_6):
516
517
self.audio_token_id = None
518
519
+ def _maybe_ignore_quant_config(self, quant_config: QuantizationConfig):
520
+ # GPTQ configs do not have a list of ignored modules, however AutoGPTQ
521
+ # seems to avoid vision encoder sections for some models.
522
+ # See: https://huggingface.co/openbmb/MiniCPM-o-2_6-int4
523
+ if isinstance(quant_config, (GPTQConfig, GPTQMarlinConfig)):
524
+ return None
525
+ return quant_config
526
+
527
+ def init_vision_module(
528
+ self,
529
+ config: PretrainedConfig,
530
+ quant_config: Optional[QuantizationConfig] = None,
531
+ prefix: str = "",
532
+ ) -> nn.Module:
533
+ # MiniCPMO GPTQ model leave vpm unquantized.
534
+ quant_config = self._maybe_ignore_quant_config(quant_config)
535
+ return super().init_vision_module(config, quant_config, prefix)
536
+
537
+ def init_resampler(
538
+ self,
539
+ embed_dim: int,
540
+ vision_dim: int,
541
+ quant_config: Optional[QuantizationConfig] = None,
542
+ prefix: str = "",
543
+ ) -> nn.Module:
544
+ # MiniCPMO GPTQ model leave resampler unquantized.
545
+ quant_config = self._maybe_ignore_quant_config(quant_config)
546
+ return super().init_resampler(embed_dim, vision_dim, quant_config,
547
+ prefix)
548
+
549
def init_audio_module(self, *, vllm_config: VllmConfig, prefix: str = ""):
550
# Do not use parameters temporarily
551
audio_config = self.config.audio_config
vllm/model_executor/models/minicpmv.py
+1
-1
@@ -1181,7 +1181,7 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
1181
def init_vision_module(
1182
self,
1183
config: PretrainedConfig,
1184
- quant_config: Optional[QuantizationConfig],
1184
+ quant_config: Optional[QuantizationConfig] = None,
1185
prefix: str = "",
1186
) -> nn.Module:
1187
model = Idefics2VisionTransformer(config.vision_config,