@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / cfea9c04ef
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 07:00 UTC

[Model] Fix Baichuan BNB online quantization (#10572)

Signed-off-by: Chen Wu <cntryroa@gmail.com>

Chen Wu committed Nov 23, 2024 at 13:13 UTC cfea9c04ef43420be594f23fc1773009d1fe88c3
1 file changed +15
vllm/model_executor/models/baichuan.py
+15
@@ -350,6 +350,21 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350 embedding_modules = {}
351 embedding_padding_modules = []
352
353 + # BitandBytes specific attributes
354 + default_bitsandbytes_target_modules = [
355 + ".W_pack.",
356 + ".o_proj.",
357 + ".down_proj.",
358 + ".up_proj.",
359 + ".gate_proj.",
360 + ".up_proj.",
361 + ]
362 + bitsandbytes_stacked_params_mapping = {
363 + # shard_name, weight_name, index
364 + "gate_proj": ("gate_up_proj", 0),
365 + "up_proj": ("gate_up_proj", 1),
366 + }
367 +
368 def __init__(
369 self,
370 *,