@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 2c9b8ea2b0
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 03:15 UTC

[Bugfix] Fix TeleChat2ForCausalLM weights mapper (#11546)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>

Jee Jee Li committed Dec 27, 2024 at 18:39 UTC 2c9b8ea2b006e763b8268b8ab02181c9822cfe76
1 file changed +13 -13
vllm/model_executor/models/telechat2.py
+13 -13
@@ -31,19 +31,6 @@ from .utils import (AutoWeightsLoader, PPMissingLayer, WeightsMapper,
31
32 class TeleChat2Model(LlamaModel):
33
34 - hf_to_vllm_mapper = WeightsMapper(
35 - orig_to_new_prefix={
36 - "transformer.": "model.",
37 - },
38 - orig_to_new_substr={
39 - ".h.": ".layers.",
40 - ".self_attention.": ".self_attn.",
41 - ".word_embeddings.": ".embed_tokens.",
42 - ".dense.": ".o_proj.",
43 - ".ln_f.": ".norm.",
44 - },
45 - )
46 -
34 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
35 # 1. Initialize the LlamaModel with bias
36 vllm_config.model_config.hf_config.bias = True
@@ -118,6 +105,19 @@ class TeleChat2Model(LlamaModel):
105
106 class TeleChat2ForCausalLM(LlamaForCausalLM):
107
108 + hf_to_vllm_mapper = WeightsMapper(
109 + orig_to_new_prefix={
110 + "transformer.": "model.",
111 + },
112 + orig_to_new_substr={
113 + ".h.": ".layers.",
114 + ".self_attention.": ".self_attn.",
115 + ".word_embeddings.": ".embed_tokens.",
116 + ".dense.": ".o_proj.",
117 + ".ln_f.": ".norm.",
118 + },
119 + )
120 +
121 def _init_model(self, vllm_config: VllmConfig, prefix: str = ""):
122 return TeleChat2Model(vllm_config=vllm_config, prefix=prefix)
123