A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 5, 14:45 UTC
[New Model]: add support for telechat3 (#38510)
Signed-off-by: xiayongqiang <xiayq1@chinatelecom.cn> Co-authored-by: xiayongqiang <xiayq1@chinatelecom.cn>
1096125073 committed
Apr 3, 2026 at 08:26 UTC
71a9125c67b8f932bf19fe9faf84cfb545ccd14a
5 files changed
+93
docs/models/supported_models.md
+1
@@ -481,6 +481,7 @@ th {
481
| `Step3p5ForCausalLM` | Step-3.5-flash | `stepfun-ai/Step-3.5-Flash`, etc. | | ✅︎ |
482
| `TeleChatForCausalLM` | TeleChat | `chuhac/TeleChat2-35B`, etc. | ✅︎ | ✅︎ |
483
| `TeleChat2ForCausalLM` | TeleChat2 | `Tele-AI/TeleChat2-3B`, `Tele-AI/TeleChat2-7B`, `Tele-AI/TeleChat2-35B`, etc. | ✅︎ | ✅︎ |
484
+| `TeleChat3ForCausalLM` | TeleChat3 | `Tele-AI/TeleChat3-36B-Thinking`, `Tele-AI/TeleChat3-Coder-36B-Thinking`, etc. | ✅︎ | ✅︎ |
485
| `TeleFLMForCausalLM` | TeleFLM | `CofeAI/FLM-2-52B-Instruct-2407`, `CofeAI/Tele-FLM`, etc. | ✅︎ | ✅︎ |
486
| `XverseForCausalLM` | XVERSE | `xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc. | ✅︎ | ✅︎ |
487
| `MiniMaxM1ForCausalLM` | MiniMax-Text | `MiniMaxAI/MiniMax-M1-40k`, `MiniMaxAI/MiniMax-M1-80k`, etc. | | |
tests/models/registry.py
+3
@@ -537,6 +537,9 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
537
"TeleChat2ForCausalLM": _HfExamplesInfo(
538
"Tele-AI/TeleChat2-3B", trust_remote_code=True
539
),
540
+ "TeleChat3ForCausalLM": _HfExamplesInfo(
541
+ "Tele-AI/TeleChat3-36B-Thinking", trust_remote_code=True
542
+ ),
543
"TeleFLMForCausalLM": _HfExamplesInfo(
544
"CofeAI/FLM-2-52B-Instruct-2407", trust_remote_code=True
545
),
vllm/model_executor/layers/rotary_embedding/__init__.py
+31
@@ -20,6 +20,7 @@ from .mrope import MRotaryEmbedding
20
from .mrope_interleaved import MRotaryEmbeddingInterleaved
21
from .ntk_scaling_rope import NTKScalingRotaryEmbedding
22
from .phi3_long_rope_scaled_rope import Phi3LongRoPEScaledRotaryEmbedding
23
+from .telechat3_scaling_rope import TeleChat3RoPEScaledRotaryEmbedding
24
from .xdrope import XDRotaryEmbedding
25
from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
26
@@ -334,6 +335,36 @@ def get_rope(
335
)
336
else:
337
raise ValueError("Pangu mrope lacks necessary parameters.")
338
+ elif scaling_type == "telechat3-yarn":
339
+ scaling_factor = rope_parameters["factor"]
340
+ if "original_max_position_embeddings" in rope_parameters:
341
+ original_max_position = rope_parameters["original_max_position_embeddings"]
342
+ scaling_factor = max_position / original_max_position
343
+ else:
344
+ original_max_position = max_position
345
+ extra_kwargs = {
346
+ k: v
347
+ for k, v in rope_parameters.items()
348
+ if k
349
+ in (
350
+ "extrapolation_factor",
351
+ "attn_factor",
352
+ "beta_fast",
353
+ "beta_slow",
354
+ "mscale",
355
+ "mscale_all_dim",
356
+ )
357
+ }
358
+ rotary_emb = TeleChat3RoPEScaledRotaryEmbedding(
359
+ head_size,
360
+ rotary_dim,
361
+ original_max_position,
362
+ base,
363
+ is_neox_style,
364
+ scaling_factor,
365
+ dtype,
366
+ **extra_kwargs,
367
+ )
368
else:
369
raise ValueError(f"Unknown RoPE scaling type {scaling_type}")
370
_ROPE_DICT[key] = rotary_emb
vllm/model_executor/layers/rotary_embedding/telechat3_scaling_rope.py
new
+57
@@ -0,0 +1,57 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+import math
5
+
6
+import torch
7
+
8
+from .base import RotaryEmbedding
9
+from .yarn_scaling_rope import YaRNScalingRotaryEmbedding
10
+
11
+
12
+class TeleChat3RoPEScaledRotaryEmbedding(YaRNScalingRotaryEmbedding):
13
+ """TeleChat3 uses a variant of YaRN method.
14
+
15
+ To achieve code reuse as much as possible, we have rewritten the
16
+ `get_mscale` method in the initialization function
17
+ """
18
+
19
+ def __init__(
20
+ self,
21
+ head_size: int,
22
+ rotary_dim: int,
23
+ max_position_embeddings: int,
24
+ base: int,
25
+ is_neox_style: bool,
26
+ scaling_factor: float,
27
+ dtype: torch.dtype,
28
+ *,
29
+ extrapolation_factor: float = 1,
30
+ attn_factor: float = 1,
31
+ beta_fast: int = 32,
32
+ beta_slow: int = 1,
33
+ truncate: bool = True,
34
+ ) -> None:
35
+ self.scaling_factor = scaling_factor
36
+ self.extrapolation_factor = extrapolation_factor
37
+ self.attn_factor = attn_factor
38
+ self.beta_fast = beta_fast
39
+ self.beta_slow = beta_slow
40
+ self.truncate = truncate
41
+
42
+ def get_mscale(scale, mscale=1):
43
+ if scale <= 1:
44
+ return 1.0
45
+ return 0.07 * mscale * math.log(scale) + 1.0
46
+
47
+ self.mscale = float(get_mscale(self.scaling_factor) * attn_factor)
48
+ # Initialization must be performed after mscale, otherwise mscale is useless
49
+ RotaryEmbedding.__init__(
50
+ self,
51
+ head_size,
52
+ rotary_dim,
53
+ max_position_embeddings,
54
+ base,
55
+ is_neox_style,
56
+ dtype,
57
+ )
vllm/model_executor/models/registry.py
+1
@@ -206,6 +206,7 @@ _TEXT_GENERATION_MODELS = {
206
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
207
"TeleChatForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
208
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
209
+ "TeleChat3ForCausalLM": ("llama", "LlamaForCausalLM"),
210
"TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
211
"XverseForCausalLM": ("llama", "LlamaForCausalLM"),
212
"Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),