A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Gemma2] add bitsandbytes support for Gemma2 (#8338)
Blueyo0 committed
Sep 12, 2024 at 12:53 UTC
1bf2dd9df025feb82e27f90f534a3bf829ae75e9
1 file changed
+8
vllm/model_executor/models/gemma2.py
+8
@@ -312,6 +312,14 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
312
# Gemma does not apply LoRA to the embedding layer.
313
embedding_modules = {}
314
embedding_padding_modules = []
315
+ bitsandbytes_stacked_params_mapping = {
316
+ # shard_name, weight_name, index
317
+ "q_proj": ("qkv_proj", 0),
318
+ "k_proj": ("qkv_proj", 1),
319
+ "v_proj": ("qkv_proj", 2),
320
+ "gate_proj": ("gate_up_proj", 0),
321
+ "up_proj": ("gate_up_proj", 1),
322
+ }
323
324
def __init__(
325
self,