@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 93a76dd21d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Model] Support bitsandbytes for MiniCPMV (#9891)

Signed-off-by: mgoin <michael@neuralmagic.com>

Michael Goin committed Nov 1, 2024 at 01:31 UTC 93a76dd21dcec8977f1ffd0e21faa88fb515b9e4
1 file changed +43
vllm/model_executor/models/minicpmv.py
+43
@@ -810,6 +810,28 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
810 # resampler
811 "kv_proj",
812 ]
813 +
814 + # BitandBytes specific attributes
815 + default_bitsandbytes_target_modules = [
816 + ".gate_proj.",
817 + ".down_proj.",
818 + ".up_proj.",
819 + ".q_proj.",
820 + ".k_proj.",
821 + ".v_proj.",
822 + ".o_proj.",
823 + ]
824 + # in TP, these weights are partitioned along the column dimension (dim=-1)
825 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
826 + bitsandbytes_stacked_params_mapping = {
827 + # shard_name, weight_name, index
828 + "q_proj": ("qkv_proj", 0),
829 + "k_proj": ("qkv_proj", 1),
830 + "v_proj": ("qkv_proj", 2),
831 + "gate_proj": ("gate_up_proj", 0),
832 + "up_proj": ("gate_up_proj", 1),
833 + }
834 +
835 embedding_modules = {}
836 embedding_padding_modules = []
837
@@ -931,6 +953,27 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
953 "kv_proj",
954 ]
955
956 + # BitandBytes specific attributes
957 + default_bitsandbytes_target_modules = [
958 + ".gate_proj.",
959 + ".down_proj.",
960 + ".up_proj.",
961 + ".q_proj.",
962 + ".k_proj.",
963 + ".v_proj.",
964 + ".o_proj.",
965 + ]
966 + # in TP, these weights are partitioned along the column dimension (dim=-1)
967 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
968 + bitsandbytes_stacked_params_mapping = {
969 + # shard_name, weight_name, index
970 + "q_proj": ("qkv_proj", 0),
971 + "k_proj": ("qkv_proj", 1),
972 + "v_proj": ("qkv_proj", 2),
973 + "gate_proj": ("gate_up_proj", 0),
974 + "up_proj": ("gate_up_proj", 1),
975 + }
976 +
977 embedding_modules = {}
978 embedding_padding_modules = []
979