@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 154a8ae880
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 09:30 UTC

[Qwen2.5] Support bnb quant for Qwen2.5 (#9467)

Haoyu Wang committed Oct 18, 2024 at 12:40 UTC 154a8ae880c800a8e6250b38a66fbf24c5d1be39
1 file changed +8
vllm/model_executor/models/qwen2.py
+8
@@ -364,6 +364,14 @@ class Qwen2ForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
364 ]
365 embedding_modules = {}
366 embedding_padding_modules = []
367 + bitsandbytes_stacked_params_mapping = {
368 + # shard_name, weight_name, index
369 + "q_proj": ("qkv_proj", 0),
370 + "k_proj": ("qkv_proj", 1),
371 + "v_proj": ("qkv_proj", 2),
372 + "gate_proj": ("gate_up_proj", 0),
373 + "up_proj": ("gate_up_proj", 1),
374 + }
375
376 def __init__(
377 self,