@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2b981012a6
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 17, 06:45 UTC

Fix Baichuan2-7B-Chat (#1987)

firebook committed Dec 9, 2023 at 01:38 UTC 2b981012a6eb27d566f03cf61c06b1ef7a522f27
1 file changed +6 -2
vllm/model_executor/models/baichuan.py
+6 -2
@@ -366,12 +366,16 @@ class BaiChuanBaseForCausalLM(nn.Module):
366 weight_loader(param, loaded_weight)
367
368
369 -class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
369 +class BaichuanForCausalLM(BaiChuanBaseForCausalLM
370 + ): # baichuan 13b, baichuan2 13b, baichuan2 7b
371
372 def __init__(self,
373 config,
374 linear_method: Optional[LinearMethodBase] = None):
374 - super().__init__(config, "ALIBI", linear_method)
375 + if config.hidden_size == 4096: # baichuan2 7b
376 + super().__init__(config, "ROPE", linear_method)
377 + else: # baichuan 13b, baichuan2 13b
378 + super().__init__(config, "ALIBI", linear_method)
379
380
381 class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b