@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1bf2dd9df0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Gemma2] add bitsandbytes support for Gemma2 (#8338)

Blueyo0 committed Sep 12, 2024 at 12:53 UTC 1bf2dd9df025feb82e27f90f534a3bf829ae75e9
1 file changed +8
vllm/model_executor/models/gemma2.py
+8
@@ -312,6 +312,14 @@ class Gemma2ForCausalLM(nn.Module, SupportsLoRA):
312 # Gemma does not apply LoRA to the embedding layer.
313 embedding_modules = {}
314 embedding_padding_modules = []
315 + bitsandbytes_stacked_params_mapping = {
316 + # shard_name, weight_name, index
317 + "q_proj": ("qkv_proj", 0),
318 + "k_proj": ("qkv_proj", 1),
319 + "v_proj": ("qkv_proj", 2),
320 + "gate_proj": ("gate_up_proj", 0),
321 + "up_proj": ("gate_up_proj", 1),
322 + }
323
324 def __init__(
325 self,