A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 05:45 UTC
[Model] support bitsandbytes quantization with minicpm model (#10842)
Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>
zixuanzhang226 committed
Dec 2, 2024 at 19:06 UTC
d746268e92dc97d3a816c70637e20073eeac5103
1 file changed
+10
vllm/model_executor/models/minicpm.py
+10
@@ -534,6 +534,16 @@ class MiniCPMForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
534
}
535
embedding_padding_modules = ["lm_head"]
536
537
+ # BitandBytes specific attributes
538
+ bitsandbytes_stacked_params_mapping = {
539
+ # shard_name, weight_name, index
540
+ "q_proj": ("qkv_proj", 0),
541
+ "k_proj": ("qkv_proj", 1),
542
+ "v_proj": ("qkv_proj", 2),
543
+ "gate_proj": ("gate_up_proj", 0),
544
+ "up_proj": ("gate_up_proj", 1),
545
+ }
546
+
547
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
548
super().__init__()
549
config = vllm_config.model_config.hf_config