A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 02:00 UTC
[Model]: Support internlm3 (#12037)
RunningLeon committed
Jan 15, 2025 at 19:35 UTC
97eb97b5a4fd64c3cbc97bb9d71a9bfd98348799
4 files changed
+28
-15
docs/source/models/supported_models.md
+5
@@ -216,6 +216,11 @@ See [this page](#generative-models) for more information on how to use generativ
216
- `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.
217
- ✅︎
218
- ✅︎
219
+* - `InternLM3ForCausalLM`
220
+ - InternLM3
221
+ - `internlm/internlm3-8b-instruct`, etc.
222
+ - ✅︎
223
+ - ✅︎
224
* - `JAISLMHeadModel`
225
- Jais
226
- `inceptionai/jais-13b`, `inceptionai/jais-13b-chat`, `inceptionai/jais-30b-v3`, `inceptionai/jais-30b-chat-v3`, etc.
tests/models/registry.py
+2
@@ -85,6 +85,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
85
trust_remote_code=True),
86
"InternLM2VEForCausalLM": _HfExamplesInfo("OpenGVLab/Mono-InternVL-2B",
87
trust_remote_code=True),
88
+ "InternLM3ForCausalLM": _HfExamplesInfo("internlm/internlm3-8b-instruct",
89
+ trust_remote_code=True),
90
"JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
91
"JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini"),
92
"LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
vllm/model_executor/models/llama.py
+20
-15
@@ -97,20 +97,19 @@ class LlamaMLP(nn.Module):
97
98
class LlamaAttention(nn.Module):
99
100
- def __init__(
101
- self,
102
- config: LlamaConfig,
103
- hidden_size: int,
104
- num_heads: int,
105
- num_kv_heads: int,
106
- rope_theta: float = 10000,
107
- rope_scaling: Optional[Dict[str, Any]] = None,
108
- max_position_embeddings: int = 8192,
109
- quant_config: Optional[QuantizationConfig] = None,
110
- bias: bool = False,
111
- cache_config: Optional[CacheConfig] = None,
112
- prefix: str = "",
113
- ) -> None:
100
+ def __init__(self,
101
+ config: LlamaConfig,
102
+ hidden_size: int,
103
+ num_heads: int,
104
+ num_kv_heads: int,
105
+ rope_theta: float = 10000,
106
+ rope_scaling: Optional[Dict[str, Any]] = None,
107
+ max_position_embeddings: int = 8192,
108
+ quant_config: Optional[QuantizationConfig] = None,
109
+ bias: bool = False,
110
+ cache_config: Optional[CacheConfig] = None,
111
+ prefix: str = "",
112
+ bias_o_proj: bool = False) -> None:
113
super().__init__()
114
layer_idx = extract_layer_index(prefix)
115
self.hidden_size = hidden_size
@@ -150,7 +149,7 @@ class LlamaAttention(nn.Module):
149
self.o_proj = RowParallelLinear(
150
input_size=self.total_num_heads * self.head_dim,
151
output_size=hidden_size,
153
- bias=bias,
152
+ bias=bias_o_proj,
153
quant_config=quant_config,
154
prefix=f"{prefix}.o_proj",
155
)
@@ -231,6 +230,11 @@ class LlamaDecoderLayer(nn.Module):
230
# Support internlm/internlm-7b with bias
231
attention_bias = getattr(config, "attention_bias", False) or getattr(
232
config, "bias", False)
233
+ bias_o_proj = attention_bias
234
+ # support internlm/internlm3-8b with qkv_bias
235
+ if hasattr(config, 'qkv_bias'):
236
+ attention_bias = config.qkv_bias
237
+
238
self.self_attn = LlamaAttention(
239
config=config,
240
hidden_size=self.hidden_size,
@@ -242,6 +246,7 @@ class LlamaDecoderLayer(nn.Module):
246
max_position_embeddings=max_position_embeddings,
247
quant_config=quant_config,
248
bias=attention_bias,
249
+ bias_o_proj=bias_o_proj,
250
cache_config=cache_config,
251
prefix=f"{prefix}.self_attn",
252
)
vllm/model_executor/models/registry.py
+1
@@ -60,6 +60,7 @@ _TEXT_GENERATION_MODELS = {
60
"InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
61
"InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
62
"InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
63
+ "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
64
"JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
65
"JambaForCausalLM": ("jamba", "JambaForCausalLM"),
66
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),