@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 0d02747f2e
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

support TP in qwen2 bnb (#9574)

chenqianfzh committed Oct 22, 2024 at 00:13 UTC 0d02747f2ed5f65bd7100b6dcf1805cefb458f5d
1 file changed +14
vllm/model_executor/models/qwen2.py
+14
@@ -364,6 +364,20 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
364 ]
365 embedding_modules = {}
366 embedding_padding_modules = []
367 +
368 + # BitandBytes specific attributes
369 + default_bitsandbytes_target_modules = [
370 + ".gate_proj.",
371 + ".down_proj.",
372 + ".up_proj.",
373 + ".q_proj.",
374 + ".k_proj.",
375 + ".v_proj.",
376 + ".o_proj.",
377 + ]
378 +
379 + # in TP, these weights are partitioned along the column dimension (dim=-1)
380 + column_parallel_weights_modules = [".down_proj.", ".o_proj."]
381 bitsandbytes_stacked_params_mapping = {
382 # shard_name, weight_name, index
383 "q_proj": ("qkv_proj", 0),