A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
support bitsandbytes quantization with qwen model (#10549)
Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>
zixuanzhang226 committed
Nov 22, 2024 at 16:16 UTC
948c859571af9588e344079cc0e79bbf8597cb18
1 file changed
+12
vllm/model_executor/models/qwen.py
+12
@@ -1028,6 +1028,18 @@ class QWenLLM(QWenBaseModel):
1028
embedding_modules = {}
1029
embedding_padding_modules = []
1030
1031
+ default_bitsandbytes_target_modules = [
1032
+ ".c_attn.",
1033
+ ".c_proj.",
1034
+ ".w1.",
1035
+ ".w2.",
1036
+ ]
1037
+ bitsandbytes_stacked_params_mapping = {
1038
+ # shard_name, weight_name, index
1039
+ "w2": ("gate_up_proj", 0),
1040
+ "w1": ("gate_up_proj", 1),
1041
+ }
1042
+
1043
1044
class QWenVL(QWenBaseModel, SupportsMultiModal):
1045
packed_modules_mapping = {