A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 14, 05:30 UTC
[Model] Remove MotifForCausalLM (#25866)
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
Jee Jee Li committed
Sep 30, 2025 at 00:36 UTC
e61eb5e09d644a57814038f48f658d4f8e3c1fee
5 files changed
+1
-355
docs/models/supported_models.md
-2
@@ -396,7 +396,6 @@ th {
396
| `MiniCPM3ForCausalLM` | MiniCPM3 | `openbmb/MiniCPM3-4B`, etc. | ✅︎ | ✅︎ | ✅︎ |
397
| `MistralForCausalLM` | Mistral, Mistral-Instruct | `mistralai/Mistral-7B-v0.1`, `mistralai/Mistral-7B-Instruct-v0.1`, etc. | ✅︎ | ✅︎ | ✅︎ |
398
| `MixtralForCausalLM` | Mixtral-8x7B, Mixtral-8x7B-Instruct | `mistralai/Mixtral-8x7B-v0.1`, `mistralai/Mixtral-8x7B-Instruct-v0.1`, `mistral-community/Mixtral-8x22B-v0.1`, etc. | ✅︎ | ✅︎ | ✅︎ |
399
-| `MotifForCausalLM` | Motif-1-Tiny | `Motif-Technologies/Motif-2.6B`, `Motif-Technologies/Motif-2.6b-v1.1-LC`, etc. | ✅︎ | ✅︎ | |
399
| `MPTForCausalLM` | MPT, MPT-Instruct, MPT-Chat, MPT-StoryWriter | `mosaicml/mpt-7b`, `mosaicml/mpt-7b-storywriter`, `mosaicml/mpt-30b`, etc. | | ✅︎ | ✅︎ |
400
| `NemotronForCausalLM` | Nemotron-3, Nemotron-4, Minitron | `nvidia/Minitron-8B-Base`, `mgoin/Nemotron-4-340B-Base-hf-FP8`, etc. | ✅︎ | ✅︎ | ✅︎ |
401
| `NemotronHForCausalLM` | Nemotron-H | `nvidia/Nemotron-H-8B-Base-8K`, `nvidia/Nemotron-H-47B-Base-8K`, `nvidia/Nemotron-H-56B-Base-8K`, etc. | ✅︎ | ✅︎ | ✅︎ |
@@ -409,7 +408,6 @@ th {
408
| `PhiForCausalLM` | Phi | `microsoft/phi-1_5`, `microsoft/phi-2`, etc. | ✅︎ | ✅︎ | ✅︎ |
409
| `Phi3ForCausalLM` | Phi-4, Phi-3 | `microsoft/Phi-4-mini-instruct`, `microsoft/Phi-4`, `microsoft/Phi-3-mini-4k-instruct`, `microsoft/Phi-3-mini-128k-instruct`, `microsoft/Phi-3-medium-128k-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
410
| `PhiMoEForCausalLM` | Phi-3.5-MoE | `microsoft/Phi-3.5-MoE-instruct`, etc. | ✅︎ | ✅︎ | ✅︎ |
412
-| `Phi4FlashForCausalLM` | Phi-4-mini-flash-reasoning | `microsoft/microsoft/Phi-4-mini-instruct`, etc. | | | |
411
| `PersimmonForCausalLM` | Persimmon | `adept/persimmon-8b-base`, `adept/persimmon-8b-chat`, etc. | | ✅︎ | ✅︎ |
412
| `Plamo2ForCausalLM` | PLaMo2 | `pfnet/plamo-2-1b`, `pfnet/plamo-2-8b`, etc. | | ✅︎ | ✅︎ |
413
| `QWenLMHeadModel` | Qwen | `Qwen/Qwen-7B`, `Qwen/Qwen-7B-Chat`, etc. | ✅︎ | ✅︎ | ✅︎ |
tests/models/registry.py
-3
@@ -298,9 +298,6 @@ _TEXT_GENERATION_EXAMPLE_MODELS = {
298
"MistralForCausalLM": _HfExamplesInfo("mistralai/Mistral-7B-Instruct-v0.1"),
299
"MixtralForCausalLM": _HfExamplesInfo("mistralai/Mixtral-8x7B-Instruct-v0.1", # noqa: E501
300
{"tiny": "TitanML/tiny-mixtral"}), # noqa: E501
301
- "MotifForCausalLM": _HfExamplesInfo("Motif-Technologies/Motif-2.6B",
302
- trust_remote_code=True,
303
- v0_only=True),
301
"MptForCausalLM": _HfExamplesInfo("mpt", is_available_online=False),
302
"MPTForCausalLM": _HfExamplesInfo("mosaicml/mpt-7b"),
303
"NemotronForCausalLM": _HfExamplesInfo("nvidia/Minitron-8B-Base"),
tests/models/test_initialization.py
-4
@@ -76,10 +76,6 @@ def can_initialize(model_arch: str, monkeypatch: pytest.MonkeyPatch,
76
if model_info.v0_only:
77
# NOTE(woosuk): skip the test for V0-only models
78
return
79
-
80
- if model_arch in ("Phi4FlashForCausalLM", "MotifForCausalLM"):
81
- pytest.skip(
82
- "Differential Flash Attention backend has been removed.")
79
if model_arch == "GptOssForCausalLM":
80
# FIXME: A hack to bypass FA3 assertion because our CI's L4 GPU
81
# has cc==8.9 which hasn't supported FA3 yet. Remove this hack when
vllm/model_executor/models/motif.py
deleted
-345
@@ -1,345 +0,0 @@
1
-# SPDX-License-Identifier: Apache-2.0
2
-# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
-
4
-# Adapted from
5
-# https://huggingface.co/Motif-Technologies/Motif-2.6B/blob/main/modeling_motif.py
6
-# Copyright (c) Alibaba Cloud.
7
-# LICENSE: https://huggingface.co/Motif-Technologies/Motif-2.6B/blob/main/LICENSE
8
-"""Inference-only Motif model compatible with HuggingFace weights."""
9
-import math
10
-from typing import Any, Optional
11
-
12
-import torch
13
-from torch import nn
14
-from transformers import PretrainedConfig
15
-
16
-from vllm.attention import Attention, AttentionType
17
-from vllm.attention.selector import _Backend
18
-from vllm.config import CacheConfig, VllmConfig
19
-from vllm.distributed import get_tensor_model_parallel_world_size
20
-from vllm.model_executor.layers.layernorm import PolyNorm, RMSNorm
21
-from vllm.model_executor.layers.linear import (MergedColumnParallelLinear,
22
- QKVParallelLinear,
23
- RowParallelLinear)
24
-from vllm.model_executor.layers.quantization import QuantizationConfig
25
-from vllm.model_executor.layers.rotary_embedding import get_rope
26
-from vllm.model_executor.models.llama import LlamaForCausalLM
27
-
28
-from .adapters import as_seq_cls_model
29
-from .interfaces import SupportsV0Only
30
-from .utils import extract_layer_index
31
-
32
-
33
-class MotifMLP(nn.Module):
34
- """MLP for the language component of the Motif model, which contains a
35
- MergedColumnParallelLinear merging 2 outputs via PolyNorm activation."""
36
-
37
- def __init__(
38
- self,
39
- hidden_size: int,
40
- intermediate_size: int,
41
- hidden_act: str = "poly_norm",
42
- quant_config: Optional[QuantizationConfig] = None,
43
- bias: bool = False,
44
- prefix: str = "",
45
- reduce_results: bool = True,
46
- ):
47
- super().__init__()
48
- self.gate_up_proj = MergedColumnParallelLinear(
49
- input_size=hidden_size,
50
- output_sizes=[intermediate_size] * 2,
51
- bias=bias,
52
- quant_config=quant_config,
53
- prefix=f"{prefix}.gate_up_proj",
54
- )
55
- self.down_proj = RowParallelLinear(
56
- input_size=intermediate_size,
57
- output_size=hidden_size,
58
- bias=bias,
59
- quant_config=quant_config,
60
- reduce_results=reduce_results,
61
- prefix=f"{prefix}.down_proj",
62
- )
63
- if hidden_act != "poly_norm":
64
- raise NotImplementedError(f"Unsupported activation: {hidden_act}. "
65
- "Only poly_norm is supported for now.")
66
- self.act_fn = PolyNorm()
67
- self.intermediate_size = intermediate_size
68
- tp_size = get_tensor_model_parallel_world_size()
69
- if hidden_act == "poly_norm" and tp_size > 1:
70
- raise NotImplementedError(
71
- "Tensor parallelism for poly_norm is not supported yet. "
72
- "Support will be added in the future.")
73
-
74
- def forward(self, x):
75
- x, _ = self.gate_up_proj(x)
76
- x = self.act_fn(
77
- x[..., :self.intermediate_size]) * x[..., self.intermediate_size:]
78
- x, _ = self.down_proj(x)
79
- return x
80
-
81
-
82
-class MotifAttention(nn.Module):
83
-
84
- def __init__(
85
- self,
86
- config: PretrainedConfig,
87
- hidden_size: int,
88
- num_heads: int,
89
- num_kv_heads: int,
90
- rope_theta: float = 10000,
91
- rope_scaling: Optional[dict[str, Any]] = None,
92
- max_position_embeddings: int = 8192,
93
- quant_config: Optional[QuantizationConfig] = None,
94
- bias: bool = False,
95
- bias_o_proj: bool = False,
96
- cache_config: Optional[CacheConfig] = None,
97
- prefix: str = "",
98
- attn_type: str = AttentionType.DECODER,
99
- ) -> None:
100
- super().__init__()
101
- layer_idx = extract_layer_index(prefix)
102
- self.hidden_size = hidden_size
103
- tp_size = get_tensor_model_parallel_world_size()
104
- self.total_num_heads = num_heads
105
- assert self.total_num_heads % tp_size == 0
106
- self.num_heads = self.total_num_heads // tp_size
107
- self.total_num_kv_heads = num_kv_heads
108
- if self.total_num_kv_heads >= tp_size:
109
- # Number of KV heads is greater than TP size, so we partition
110
- # the KV heads across multiple tensor parallel GPUs.
111
- assert self.total_num_kv_heads % tp_size == 0
112
- else:
113
- # Number of KV heads is less than TP size, so we replicate
114
- # the KV heads across multiple tensor parallel GPUs.
115
- assert tp_size % self.total_num_kv_heads == 0
116
- self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
117
- # MistralConfig has an optional head_dim introduced by Mistral-Nemo
118
- head_dim = getattr(config, "head_dim", None)
119
- if head_dim is None:
120
- head_dim = self.hidden_size // self.total_num_heads
121
- self.head_dim = head_dim
122
- # Phi models introduced a partial_rotary_factor parameter in the config
123
- self.partial_rotary_factor = getattr(config, "partial_rotary_factor",
124
- 1)
125
- self.q_size = self.num_heads * self.head_dim
126
- self.kv_size = self.num_kv_heads * self.head_dim
127
- self.scaling = self.head_dim**-0.5
128
- self.rope_theta = rope_theta
129
- self.max_position_embeddings = max_position_embeddings
130
-
131
- assert self.num_heads % 2 == 0, 'num_heads should be even'
132
- assert self.num_kv_heads % 2 == 0, 'num_heads should be even'
133
-
134
- self.qkv_proj = QKVParallelLinear(
135
- hidden_size=hidden_size,
136
- head_size=self.head_dim,
137
- total_num_heads=self.total_num_heads,
138
- total_num_kv_heads=self.total_num_kv_heads,
139
- bias=bias,
140
- quant_config=quant_config,
141
- prefix=f"{prefix}.qkv_proj",
142
- )
143
-
144
- self.o_proj = RowParallelLinear(
145
- input_size=self.total_num_heads * self.head_dim,
146
- output_size=hidden_size,
147
- bias=bias_o_proj,
148
- quant_config=quant_config,
149
- prefix=f"{prefix}.o_proj",
150
- )
151
-
152
- self._init_rotary_emb(config,
153
- rope_scaling=rope_scaling,
154
- quant_config=quant_config)
155
- sliding_window = None
156
-
157
- self.lambda_init = self.lambda_init_fn(layer_idx)
158
- self.lambda_q1 = nn.Parameter(
159
- torch.zeros(self.head_dim, dtype=torch.float32).normal_(mean=0,
160
- std=0.1))
161
- self.lambda_k1 = nn.Parameter(
162
- torch.zeros(self.head_dim, dtype=torch.float32).normal_(mean=0,
163
- std=0.1))
164
- self.lambda_q2 = nn.Parameter(
165
- torch.zeros(self.head_dim, dtype=torch.float32).normal_(mean=0,
166
- std=0.1))
167
- self.lambda_k2 = nn.Parameter(
168
- torch.zeros(self.head_dim, dtype=torch.float32).normal_(mean=0,
169
- std=0.1))
170
- self.subln = RMSNorm(2 * self.head_dim, eps=config.attn_rms_norm_eps)
171
-
172
- params = {
173
- 'differential_flash_attention_config': {
174
- 'lambda_init': self.lambda_init,
175
- 'lambda_q1': self.lambda_q1,
176
- 'lambda_k1': self.lambda_k1,
177
- 'lambda_q2': self.lambda_q2,
178
- 'lambda_k2': self.lambda_k2,
179
- "subln": self.subln,
180
- }
181
- }
182
-
183
- diff_attn_err_msg = (
184
- 'Set VLLM_ATTENTION_BACKEND="DIFFERENTIAL_FLASH_ATTN" '
185
- 'to enable Differential Flash Attention.')
186
- try:
187
- self.attn = Attention(
188
- self.num_heads,
189
- self.head_dim,
190
- self.scaling,
191
- num_kv_heads=self.num_kv_heads,
192
- cache_config=cache_config,
193
- quant_config=quant_config,
194
- per_layer_sliding_window=sliding_window,
195
- attn_type=attn_type,
196
- prefix=f"{prefix}.attn",
197
- **params,
198
- )
199
- except TypeError as e:
200
- raise ValueError(diff_attn_err_msg) from e
201
- assert (self.attn.backend == _Backend.DIFFERENTIAL_FLASH_ATTN
202
- ), diff_attn_err_msg
203
-
204
- def lambda_init_fn(self, depth):
205
- return 0.8 - 0.6 * math.exp(-0.3 * (depth - 1))
206
-
207
- def forward(
208
- self,
209
- positions: torch.Tensor,
210
- hidden_states: torch.Tensor,
211
- ) -> torch.Tensor:
212
- qkv, _ = self.qkv_proj(hidden_states)
213
- q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
214
- q, k = self.rotary_emb(positions, q, k)
215
- attn_output = self.attn(q, k, v)
216
- output, _ = self.o_proj(attn_output)
217
- return output
218
-
219
- def _init_rotary_emb(self, config: PretrainedConfig,
220
- rope_scaling: Optional[dict[str, Any]],
221
- quant_config: Optional[QuantizationConfig]) -> None:
222
- is_neox_style = True
223
- is_gguf = quant_config and quant_config.get_name() == "gguf"
224
- if is_gguf and config.model_type == "llama":
225
- is_neox_style = False
226
-
227
- self.rotary_emb = get_rope(
228
- self.head_dim,
229
- rotary_dim=self.head_dim,
230
- max_position=self.max_position_embeddings,
231
- base=self.rope_theta,
232
- rope_scaling=rope_scaling,
233
- is_neox_style=is_neox_style,
234
- partial_rotary_factor=self.partial_rotary_factor,
235
- )
236
-
237
-
238
-class MotifDecoderLayer(nn.Module):
239
-
240
- def __init__(
241
- self,
242
- config: PretrainedConfig,
243
- cache_config: Optional[CacheConfig] = None,
244
- quant_config: Optional[QuantizationConfig] = None,
245
- prefix: str = "",
246
- ) -> None:
247
- super().__init__()
248
- self.hidden_size = config.hidden_size
249
- rope_theta = getattr(config, "rope_theta", 10000)
250
- rope_scaling = getattr(config, "rope_scaling", None)
251
- if rope_scaling is not None and getattr(
252
- config, "original_max_position_embeddings", None):
253
- rope_scaling["original_max_position_embeddings"] = (
254
- config.original_max_position_embeddings)
255
- max_position_embeddings = getattr(config, "max_position_embeddings",
256
- 8192)
257
- attention_bias = getattr(config, "attention_bias", False) or getattr(
258
- config, "use_bias", False)
259
- bias_o_proj = attention_bias
260
- if hasattr(config, 'qkv_bias'):
261
- attention_bias = config.qkv_bias
262
-
263
- # By default, Motif uses causal attention as it is a decoder-only model.
264
- # You can override the HF config with `is_causal=False` to enable
265
- # bidirectional attention, which is used in some embedding models
266
- # (e.g. parasail-ai/GritLM-7B-vllm)
267
- if getattr(config, "is_causal", True):
268
- attn_type = AttentionType.DECODER
269
- else:
270
- attn_type = AttentionType.ENCODER_ONLY
271
-
272
- self.self_attn = MotifAttention(
273
- config=config,
274
- hidden_size=self.hidden_size,
275
- num_heads=config.num_attention_heads,
276
- num_kv_heads=getattr(config, "num_key_value_heads",
277
- config.num_attention_heads),
278
- rope_theta=rope_theta,
279
- rope_scaling=rope_scaling,
280
- max_position_embeddings=max_position_embeddings,
281
- quant_config=quant_config,
282
- bias=attention_bias,
283
- bias_o_proj=bias_o_proj,
284
- cache_config=cache_config,
285
- prefix=f"{prefix}.self_attn",
286
- attn_type=attn_type,
287
- )
288
- self.mlp = MotifMLP(
289
- hidden_size=self.hidden_size,
290
- intermediate_size=config.intermediate_size,
291
- hidden_act=config.hidden_act,
292
- quant_config=quant_config,
293
- bias=getattr(config, "use_bias", False),
294
- prefix=f"{prefix}.mlp",
295
- )
296
- self.input_layernorm = RMSNorm(config.hidden_size,
297
- eps=config.rms_norm_eps)
298
- self.post_attention_layernorm = RMSNorm(config.hidden_size,
299
- eps=config.rms_norm_eps)
300
-
301
- def forward(
302
- self,
303
- positions: torch.Tensor,
304
- hidden_states: torch.Tensor,
305
- residual: Optional[torch.Tensor],
306
- ) -> tuple[torch.Tensor, torch.Tensor]:
307
- # Self Attention
308
- if residual is None:
309
- residual = hidden_states
310
- hidden_states = self.input_layernorm(hidden_states)
311
- else:
312
- hidden_states, residual = self.input_layernorm(
313
- hidden_states, residual)
314
- hidden_states = self.self_attn(positions=positions,
315
- hidden_states=hidden_states)
316
-
317
- # Fully Connected
318
- hidden_states, residual = self.post_attention_layernorm(
319
- hidden_states, residual)
320
- hidden_states = self.mlp(hidden_states)
321
- return hidden_states, residual
322
-
323
-
324
-# Motif model uses differential attention
325
-# Only supported in v0 (no chunked prefill support)
326
-class MotifForCausalLM(LlamaForCausalLM, SupportsV0Only):
327
-
328
- def __init__(self,
329
- *,
330
- vllm_config: VllmConfig,
331
- prefix: str = "",
332
- layer_type: type[nn.Module] = MotifDecoderLayer):
333
-
334
- # Prefix caching and chunked prefill is not supported for this model.
335
- assert not vllm_config.cache_config.enable_prefix_caching, \
336
- "Motif currently does not support prefix caching"
337
- assert not vllm_config.scheduler_config.chunked_prefill_enabled, \
338
- "Motif currently does not support chunked prefill"
339
-
340
- super().__init__(vllm_config=vllm_config,
341
- prefix=prefix,
342
- layer_type=layer_type)
343
-
344
-
345
-MotifForSequenceClassification = as_seq_cls_model(MotifForCausalLM)
vllm/model_executor/models/registry.py
+1
-1
@@ -119,7 +119,6 @@ _TEXT_GENERATION_MODELS = {
119
"MiniCPM3ForCausalLM": ("minicpm3", "MiniCPM3ForCausalLM"),
120
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
121
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
122
- "MotifForCausalLM": ("motif", "MotifForCausalLM"),
122
# transformers's mpt class has lower case
123
"MptForCausalLM": ("mpt", "MPTForCausalLM"),
124
"MPTForCausalLM": ("mpt", "MPTForCausalLM"),
@@ -332,6 +331,7 @@ _SUBPROCESS_COMMAND = [
331
]
332
333
_PREVIOUSLY_SUPPORTED_MODELS = {
334
+ "MotifForCausalLM": "0.10.2",
335
"Phi3SmallForCausalLM": "0.9.2",
336
"Phi4FlashForCausalLM": "0.10.2",
337
# encoder-decoder models except whisper