A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 03:15 UTC
[Bugfix] Fix TeleChat2ForCausalLM weights mapper (#11546)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Dec 27, 2024 at 18:39 UTC
2c9b8ea2b006e763b8268b8ab02181c9822cfe76
1 file changed
+13
-13
vllm/model_executor/models/telechat2.py
+13
-13
@@ -31,19 +31,6 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
31
32
class TeleChat2Model(LlamaModel):
33
34
- hf_to_vllm_mapper = WeightsMapper(
35
- orig_to_new_prefix={
36
- "transformer.": "model.",
37
- },
38
- orig_to_new_substr={
39
- ".h.": ".layers.",
40
- ".self_attention.": ".self_attn.",
41
- ".word_embeddings.": ".embed_tokens.",
42
- ".dense.": ".o_proj.",
43
- ".ln_f.": ".norm.",
44
- },
45
- )
46
-
34
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
35
# 1. Initialize the LlamaModel with bias
36
vllm_config.model_config.hf_config.bias = True
@@ -118,6 +105,19 @@ class TeleChat2Model(LlamaModel):
105
106
class TeleChat2ForCausalLM(LlamaForCausalLM):
107
108
+ hf_to_vllm_mapper = WeightsMapper(
109
+ orig_to_new_prefix={
110
+ "transformer.": "model.",
111
+ },
112
+ orig_to_new_substr={
113
+ ".h.": ".layers.",
114
+ ".self_attention.": ".self_attn.",
115
+ ".word_embeddings.": ".embed_tokens.",
116
+ ".dense.": ".o_proj.",
117
+ ".ln_f.": ".norm.",
118
+ },
119
+ )
120
+
121
def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
122
return TeleChat2Model(vllm_config=vllm_config, prefix=prefix)
123