@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 948c859571
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

support bitsandbytes quantization with qwen model (#10549)

Signed-off-by: Ubuntu <zixuanzhang@bytedance.com>

zixuanzhang226 committed Nov 22, 2024 at 16:16 UTC 948c859571af9588e344079cc0e79bbf8597cb18
1 file changed +12
vllm/model_executor/models/qwen.py
+12
@@ -1028,6 +1028,18 @@ class QWenLLM(QWenBaseModel):
1028 embedding_modules = {}
1029 embedding_padding_modules = []
1030
1031 + default_bitsandbytes_target_modules = [
1032 + ".c_attn.",
1033 + ".c_proj.",
1034 + ".w1.",
1035 + ".w2.",
1036 + ]
1037 + bitsandbytes_stacked_params_mapping = {
1038 + # shard_name, weight_name, index
1039 + "w2": ("gate_up_proj", 0),
1040 + "w1": ("gate_up_proj", 1),
1041 + }
1042 +
1043
1044 class QWenVL(QWenBaseModel, SupportsMultiModal):
1045 packed_modules_mapping = {