@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / d746268e92
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 05:45 UTC

[Model] support bitsandbytes quantization with minicpm model (#10842)

Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>

zixuanzhang226 committed Dec 2, 2024 at 19:06 UTC d746268e92dc97d3a816c70637e20073eeac5103
1 file changed +10
vllm/model_executor/models/minicpm.py
+10
@@ -534,6 +534,16 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
534 }
535 embedding_padding_modules = ["lm_head"]
536
537 + # BitandBytes specific attributes
538 + bitsandbytes_stacked_params_mapping = {
539 + # shard_name, weight_name, index
540 + "q_proj": ("qkv_proj", 0),
541 + "k_proj": ("qkv_proj", 1),
542 + "v_proj": ("qkv_proj", 2),
543 + "gate_proj": ("gate_up_proj", 0),
544 + "up_proj": ("gate_up_proj", 1),
545 + }
546 +
547 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
548 super().__init__()
549 config = vllm_config.model_config.hf_config