A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 15, 18:30 UTC
[Model] Support Tele-FLM Model (#15023)
Signed-off-by: Naitong Yu <ntyu@baai.ac.cn> Signed-off-by: jiangxin <horizon94@outlook.com> Co-authored-by: Jason Fang <jasonfang3900@gmail.com> Co-authored-by: jiangxin <horizon94@outlook.com>
Naitong Yu committed
Mar 22, 2025 at 17:04 UTC
2f4bd358f1d39846a38e8f61d11edfa446617dcd
5 files changed
+99
docs/source/models/supported_models.md
+5
@@ -472,6 +472,11 @@ See [this page](#generative-models) for more information on how to use generativ
472
* `Tele-AI/TeleChat2-3B`, `Tele-AI/TeleChat2-7B`, `Tele-AI/TeleChat2-35B`, etc.
473
* ✅︎
474
* ✅︎
475
+- * `TeleFLMForCausalLM`
476
+ * TeleFLM
477
+ * `CofeAI/FLM-2-52B-Instruct-2407`, `CofeAI/Tele-FLM`, etc.
478
+ * ✅︎
479
+ * ✅︎
480
- * `XverseForCausalLM`
481
* XVERSE
482
* `xverse/XVERSE-7B-Chat`, `xverse/XVERSE-13B-Chat`, `xverse/XVERSE-65B-Chat`, etc.
examples/template_teleflm.jinja
new
+12
@@ -0,0 +1,12 @@
1
+{%- for message in messages %}
2
+ {%- if message['role'] == 'user' %}
3
+ {{- '<_user>' + message['content']|trim }}
4
+ {%- elif message['role'] == 'system' %}
5
+ {{- '<_system>' + message['content']|trim }}
6
+ {%- elif message['role'] == 'assistant' %}
7
+ {{- '<_bot>' + message['content'] }}
8
+ {%- endif %}
9
+{%- endfor %}
10
+{%- if add_generation_prompt %}
11
+ {{- '<_bot>' }}
12
+{%- endif %}
tests/models/registry.py
+2
@@ -192,6 +192,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
192
"SolarForCausalLM": _HfExamplesInfo("upstage/solar-pro-preview-instruct"),
193
"TeleChat2ForCausalLM": _HfExamplesInfo("Tele-AI/TeleChat2-3B",
194
trust_remote_code=True),
195
+ "TeleFLMForCausalLM": _HfExamplesInfo("CofeAI/FLM-2-52B-Instruct-2407",
196
+ trust_remote_code=True),
197
"XverseForCausalLM": _HfExamplesInfo("xverse/XVERSE-7B-Chat",
198
is_available_online=False,
199
trust_remote_code=True),
vllm/model_executor/models/registry.py
+1
@@ -104,6 +104,7 @@ _TEXT_GENERATION_MODELS = {
104
"Starcoder2ForCausalLM": ("starcoder2", "Starcoder2ForCausalLM"),
105
"SolarForCausalLM": ("solar", "SolarForCausalLM"),
106
"TeleChat2ForCausalLM": ("telechat2", "TeleChat2ForCausalLM"),
107
+ "TeleFLMForCausalLM": ("teleflm", "TeleFLMForCausalLM"),
108
"XverseForCausalLM": ("llama", "LlamaForCausalLM"),
109
"Zamba2ForCausalLM": ("zamba2", "Zamba2ForCausalLM"),
110
# [Encoder-decoder]
vllm/model_executor/models/teleflm.py
new
+79
@@ -0,0 +1,79 @@
1
+# SPDX-License-Identifier: Apache-2.0
2
+
3
+# Adapted from
4
+# https://github.com/huggingface/transformers/blob/v4.28.0/src/transformers/models/llama/modeling_llama.py
5
+# Copyright 2023 The vLLM team.
6
+# Copyright 2022 EleutherAI and the HuggingFace Inc. team. All rights reserved.
7
+#
8
+# This code is based on EleutherAI's GPT-NeoX library and the GPT-NeoX
9
+# and OPT implementations in this library. It has been modified from its
10
+# original forms to accommodate minor architectural differences compared
11
+# to GPT-NeoX and OPT used by the Meta AI team that trained the model.
12
+#
13
+# Licensed under the Apache License, Version 2.0 (the "License");
14
+# you may not use this file except in compliance with the License.
15
+# You may obtain a copy of the License at
16
+#
17
+# http://www.apache.org/licenses/LICENSE-2.0
18
+#
19
+# Unless required by applicable law or agreed to in writing, software
20
+# distributed under the License is distributed on an "AS IS" BASIS,
21
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
22
+# See the License for the specific language governing permissions and
23
+# limitations under the License.
24
+
25
+from typing import Type
26
+
27
+import torch
28
+
29
+from vllm.config import VllmConfig
30
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
31
+from vllm.model_executor.models.llama import (LlamaDecoderLayer,
32
+ LlamaForCausalLM, LlamaModel)
33
+
34
+
35
+class TeleFLMModel(LlamaModel):
36
+
37
+ def __init__(
38
+ self,
39
+ *,
40
+ vllm_config: VllmConfig,
41
+ prefix: str = "",
42
+ layer_type: Type[LlamaDecoderLayer] = LlamaDecoderLayer,
43
+ ):
44
+ super().__init__(vllm_config=vllm_config,
45
+ prefix=prefix,
46
+ layer_type=layer_type)
47
+ """
48
+ This implementation is based on the µScaling paper presented at
49
+ the ICLR 2025 Workshop:
50
+ NanoLM: An Affordable LLM Study Benchmark \
51
+ via Accurate Loss Prediction across Scales
52
+ by Yiqun Yao et al.
53
+ Available at: https://openreview.net/forum?id=IwaPYg1SCA
54
+ arXiv preprint: https://arxiv.org/abs/2304.06875
55
+ """
56
+ self.use_mup = self.config.use_mup
57
+ if self.use_mup:
58
+ self.input_mult = self.config.input_mult
59
+
60
+ def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
61
+ embedding = self.embed_tokens(input_ids)
62
+ if self.use_mup:
63
+ embedding = embedding * self.input_mult
64
+ return embedding
65
+
66
+
67
+class TeleFLMForCausalLM(LlamaForCausalLM):
68
+
69
+ def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
70
+ super().__init__(vllm_config=vllm_config, prefix=prefix)
71
+ # mup
72
+ self.use_mup = self.config.use_mup
73
+ if self.use_mup:
74
+ self.mup_scale_factor = self.config.mup_scale_factor
75
+ self.output_mult = self.config.output_mult / self.mup_scale_factor
76
+ logit_scale = self.output_mult
77
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
78
+ self.config.vocab_size,
79
+ logit_scale)