A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 07:00 UTC
[Model] Fix Baichuan BNB online quantization (#10572)
Signed-off-by: Chen Wu <cntryroa@gmail.com>
Chen Wu committed
Nov 23, 2024 at 13:13 UTC
cfea9c04ef43420be594f23fc1773009d1fe88c3
1 file changed
+15
vllm/model_executor/models/baichuan.py
+15
@@ -350,6 +350,21 @@ class BaiChuanBaseForCausalLM(nn.Module, SupportsLoRA, SupportsPP):
350
embedding_modules = {}
351
embedding_padding_modules = []
352
353
+ # BitandBytes specific attributes
354
+ default_bitsandbytes_target_modules = [
355
+ ".W_pack.",
356
+ ".o_proj.",
357
+ ".down_proj.",
358
+ ".up_proj.",
359
+ ".gate_proj.",
360
+ ".up_proj.",
361
+ ]
362
+ bitsandbytes_stacked_params_mapping = {
363
+ # shard_name, weight_name, index
364
+ "gate_proj": ("gate_up_proj", 0),
365
+ "up_proj": ("gate_up_proj", 1),
366
+ }
367
+
368
def __init__(
369
self,
370
*,