@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 97eb97b5a4
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 02:00 UTC

[Model]: Support internlm3 (#12037)

RunningLeon committed Jan 15, 2025 at 19:35 UTC 97eb97b5a4fd64c3cbc97bb9d71a9bfd98348799
4 files changed +28 -15
docs/source/models/supported_models.md
+5
@@ -216,6 +216,11 @@ See [this page](#generative-models) for more information on how to use generativ
216 - `internlm/internlm2-7b`, `internlm/internlm2-chat-7b`, etc.
217 - ✅︎
218 - ✅︎
219 +* - `InternLM3ForCausalLM`
220 + - InternLM3
221 + - `internlm/internlm3-8b-instruct`, etc.
222 + - ✅︎
223 + - ✅︎
224 * - `JAISLMHeadModel`
225 - Jais
226 - `inceptionai/jais-13b`, `inceptionai/jais-13b-chat`, `inceptionai/jais-30b-v3`, `inceptionai/jais-30b-chat-v3`, etc.
tests/models/registry.py
+2
@@ -85,6 +85,8 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
85 trust_remote_code=True),
86 "InternLM2VEForCausalLM": _HfExamplesInfo("OpenGVLab/Mono-InternVL-2B",
87 trust_remote_code=True),
88 + "InternLM3ForCausalLM": _HfExamplesInfo("internlm/internlm3-8b-instruct",
89 + trust_remote_code=True),
90 "JAISLMHeadModel": _HfExamplesInfo("inceptionai/jais-13b-chat"),
91 "JambaForCausalLM": _HfExamplesInfo("ai21labs/AI21-Jamba-1.5-Mini"),
92 "LlamaForCausalLM": _HfExamplesInfo("meta-llama/Meta-Llama-3-8B"),
vllm/model_executor/models/llama.py
+20 -15
@@ -97,20 +97,19 @@ class LlamaMLP(nn.Module):
97
98 class LlamaAttention(nn.Module):
99
100 - def __init__(
101 - self,
102 - config: LlamaConfig,
103 - hidden_size: int,
104 - num_heads: int,
105 - num_kv_heads: int,
106 - rope_theta: float = 10000,
107 - rope_scaling: Optional[Dict[str, Any]] = None,
108 - max_position_embeddings: int = 8192,
109 - quant_config: Optional[QuantizationConfig] = None,
110 - bias: bool = False,
111 - cache_config: Optional[CacheConfig] = None,
112 - prefix: str = "",
113 - ) -> None:
100 + def __init__(self,
101 + config: LlamaConfig,
102 + hidden_size: int,
103 + num_heads: int,
104 + num_kv_heads: int,
105 + rope_theta: float = 10000,
106 + rope_scaling: Optional[Dict[str, Any]] = None,
107 + max_position_embeddings: int = 8192,
108 + quant_config: Optional[QuantizationConfig] = None,
109 + bias: bool = False,
110 + cache_config: Optional[CacheConfig] = None,
111 + prefix: str = "",
112 + bias_o_proj: bool = False) -> None:
113 super().__init__()
114 layer_idx = extract_layer_index(prefix)
115 self.hidden_size = hidden_size
@@ -150,7 +149,7 @@ class LlamaAttention(nn.Module):
149 self.o_proj = RowParallelLinear(
150 input_size=self.total_num_heads * self.head_dim,
151 output_size=hidden_size,
153 - bias=bias,
152 + bias=bias_o_proj,
153 quant_config=quant_config,
154 prefix=f"{prefix}.o_proj",
155 )
@@ -231,6 +230,11 @@ class LlamaDecoderLayer(nn.Module):
230 # Support internlm/internlm-7b with bias
231 attention_bias = getattr(config, "attention_bias", False) or getattr(
232 config, "bias", False)
233 + bias_o_proj = attention_bias
234 + # support internlm/internlm3-8b with qkv_bias
235 + if hasattr(config, 'qkv_bias'):
236 + attention_bias = config.qkv_bias
237 +
238 self.self_attn = LlamaAttention(
239 config=config,
240 hidden_size=self.hidden_size,
@@ -242,6 +246,7 @@ class LlamaDecoderLayer(nn.Module):
246 max_position_embeddings=max_position_embeddings,
247 quant_config=quant_config,
248 bias=attention_bias,
249 + bias_o_proj=bias_o_proj,
250 cache_config=cache_config,
251 prefix=f"{prefix}.self_attn",
252 )
vllm/model_executor/models/registry.py
+1
@@ -60,6 +60,7 @@ _TEXT_GENERATION_MODELS = {
60 "InternLMForCausalLM": ("llama", "LlamaForCausalLM"),
61 "InternLM2ForCausalLM": ("internlm2", "InternLM2ForCausalLM"),
62 "InternLM2VEForCausalLM": ("internlm2_ve", "InternLM2VEForCausalLM"),
63 + "InternLM3ForCausalLM": ("llama", "LlamaForCausalLM"),
64 "JAISLMHeadModel": ("jais", "JAISLMHeadModel"),
65 "JambaForCausalLM": ("jamba", "JambaForCausalLM"),
66 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),