A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
[Model] Support bitsandbytes for MiniCPMV (#9891)
Signed-off-by: mgoin <michael@neuralmagic.com>
Michael Goin committed
Nov 1, 2024 at 01:31 UTC
93a76dd21dcec8977f1ffd0e21faa88fb515b9e4
1 file changed
+43
vllm/model_executor/models/minicpmv.py
+43
@@ -810,6 +810,28 @@ class MiniCPMV2_5(MiniCPMVBaseModel, SupportsLoRA):
810
# resampler
811
"kv_proj",
812
]
813
+
814
+ # BitandBytes specific attributes
815
+ default_bitsandbytes_target_modules = [
816
+ ".gate_proj.",
817
+ ".down_proj.",
818
+ ".up_proj.",
819
+ ".q_proj.",
820
+ ".k_proj.",
821
+ ".v_proj.",
822
+ ".o_proj.",
823
+ ]
824
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
825
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
826
+ bitsandbytes_stacked_params_mapping = {
827
+ # shard_name, weight_name, index
828
+ "q_proj": ("qkv_proj", 0),
829
+ "k_proj": ("qkv_proj", 1),
830
+ "v_proj": ("qkv_proj", 2),
831
+ "gate_proj": ("gate_up_proj", 0),
832
+ "up_proj": ("gate_up_proj", 1),
833
+ }
834
+
835
embedding_modules = {}
836
embedding_padding_modules = []
837
@@ -931,6 +953,27 @@ class MiniCPMV2_6(MiniCPMVBaseModel, SupportsLoRA):
953
"kv_proj",
954
]
955
956
+ # BitandBytes specific attributes
957
+ default_bitsandbytes_target_modules = [
958
+ ".gate_proj.",
959
+ ".down_proj.",
960
+ ".up_proj.",
961
+ ".q_proj.",
962
+ ".k_proj.",
963
+ ".v_proj.",
964
+ ".o_proj.",
965
+ ]
966
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
967
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
968
+ bitsandbytes_stacked_params_mapping = {
969
+ # shard_name, weight_name, index
970
+ "q_proj": ("qkv_proj", 0),
971
+ "k_proj": ("qkv_proj", 1),
972
+ "v_proj": ("qkv_proj", 2),
973
+ "gate_proj": ("gate_up_proj", 0),
974
+ "up_proj": ("gate_up_proj", 1),
975
+ }
976
+
977
embedding_modules = {}
978
embedding_padding_modules = []
979