A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 17, 06:45 UTC
Fix Baichuan2-7B-Chat (#1987)
firebook committed
Dec 9, 2023 at 01:38 UTC
2b981012a6eb27d566f03cf61c06b1ef7a522f27
1 file changed
+6
-2
vllm/model_executor/models/baichuan.py
+6
-2
@@ -366,12 +366,16 @@ class BaiChuanBaseForCausalLM(nn.Module):
366
weight_loader(param, loaded_weight)
367
368
369
-class BaichuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 13b
369
+class BaichuanForCausalLM(BaiChuanBaseForCausalLM
370
+ ): # baichuan 13b, baichuan2 13b, baichuan2 7b
371
372
def __init__(self,
373
config,
374
linear_method: Optional[LinearMethodBase] = None):
374
- super().__init__(config, "ALIBI", linear_method)
375
+ if config.hidden_size == 4096: # baichuan2 7b
376
+ super().__init__(config, "ROPE", linear_method)
377
+ else: # baichuan 13b, baichuan2 13b
378
+ super().__init__(config, "ALIBI", linear_method)
379
380
381
class BaiChuanForCausalLM(BaiChuanBaseForCausalLM): # baichuan 7b