A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 09:30 UTC
support TP in qwen2 bnb (#9574)
chenqianfzh committed
Oct 22, 2024 at 00:13 UTC
0d02747f2ed5f65bd7100b6dcf1805cefb458f5d
1 file changed
+14
vllm/model_executor/models/qwen2.py
+14
@@ -364,6 +364,20 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
364
]
365
embedding_modules = {}
366
embedding_padding_modules = []
367
+
368
+ # BitandBytes specific attributes
369
+ default_bitsandbytes_target_modules = [
370
+ ".gate_proj.",
371
+ ".down_proj.",
372
+ ".up_proj.",
373
+ ".q_proj.",
374
+ ".k_proj.",
375
+ ".v_proj.",
376
+ ".o_proj.",
377
+ ]
378
+
379
+ # in TP, these weights are partitioned along the column dimension (dim=-1)
380
+ column_parallel_weights_modules = [".down_proj.", ".o_proj."]
381
bitsandbytes_stacked_params_mapping = {
382
# shard_name, weight_name, index
383
"q_proj": ("qkv_proj", 0),