A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Bugfix] Clean up some cruft in mamba.py (#9343)
Tyler Michael Smith committed
Oct 14, 2024 at 20:24 UTC
169b530607c0102fdb02ce1fd3323fd6085477b0
2 files changed
+11
-104
docs/source/models/supported_models.rst
+1
-1
@@ -155,7 +155,7 @@ Text Generation
155
* - :code:`MambaForCausalLM`
156
- Mamba
157
- :code:`state-spaces/mamba-130m-hf`, :code:`state-spaces/mamba-790m-hf`, :code:`state-spaces/mamba-2.8b-hf`, etc.
158
- - ✅︎
158
+ -
159
-
160
* - :code:`MiniCPMForCausalLM`
161
- MiniCPM
vllm/model_executor/models/mamba.py
+10
-103
@@ -1,6 +1,5 @@
1
# coding=utf-8
2
"""PyTorch MAMBA model."""
3
-from dataclasses import dataclass
3
from typing import Iterable, List, Optional, Tuple
4
5
import torch
@@ -10,7 +9,6 @@ from transformers import MambaConfig
9
from vllm.attention.backends.abstract import AttentionMetadata
10
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
13
-from vllm.model_executor.layers.activation import SiluAndMul
12
from vllm.model_executor.layers.layernorm import RMSNorm
13
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
14
MergedColumnParallelLinear,
@@ -39,13 +37,6 @@ from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
37
KVCache = Tuple[torch.Tensor, torch.Tensor]
38
39
42
-@dataclass
43
-class MambaCacheParams:
44
- is_prompt: bool = False
45
- conv_state: torch.Tensor = torch.Tensor()
46
- ssm_state: torch.Tensor = torch.Tensor()
47
-
48
-
40
# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
41
class MambaMixer(nn.Module):
42
"""
@@ -209,37 +200,6 @@ class MambaMixer(nn.Module):
200
return contextualized_states
201
202
212
-class MambaMLP(nn.Module):
213
-
214
- def __init__(
215
- self,
216
- config: MambaConfig,
217
- quant_config: Optional[QuantizationConfig] = None,
218
- ) -> None:
219
- super().__init__()
220
- hidden_size = config.hidden_size
221
- intermediate_size = config.intermediate_size
222
- hidden_act = config.hidden_act
223
- self.gate_up_proj = MergedColumnParallelLinear(
224
- hidden_size, [intermediate_size] * 2,
225
- bias=False,
226
- quant_config=quant_config)
227
- self.down_proj = RowParallelLinear(intermediate_size,
228
- hidden_size,
229
- bias=False,
230
- quant_config=quant_config)
231
- if hidden_act != "silu":
232
- raise ValueError(f"Unsupported activation: {hidden_act}. "
233
- "Only silu is supported for now.")
234
- self.act_fn = SiluAndMul()
235
-
236
- def forward(self, x):
237
- gate_up, _ = self.gate_up_proj(x)
238
- x = self.act_fn(gate_up)
239
- x, _ = self.down_proj(x)
240
- return x
241
-
242
-
203
class MambaDecoderLayer(nn.Module):
204
205
def __init__(self,
@@ -252,7 +212,6 @@ class MambaDecoderLayer(nn.Module):
212
self.config = config
213
self.mixer = MambaMixer(config, layer_idx)
214
255
- self.feed_forward = MambaMLP(config, quant_config=quant_config)
215
self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
216
self.pre_ff_layernorm = RMSNorm(config.hidden_size,
217
eps=config.layer_norm_epsilon)
@@ -274,10 +233,6 @@ class MambaDecoderLayer(nn.Module):
233
234
hidden_states = self.mixer(hidden_states, attn_metadata, conv_state,
235
ssm_state)
277
- # Fully Connected
278
- hidden_states, residual = self.pre_ff_layernorm(
279
- hidden_states, residual)
280
- hidden_states = self.feed_forward(hidden_states)
236
return hidden_states, residual
237
238
@@ -319,7 +274,6 @@ class MambaModel(nn.Module):
274
self,
275
input_ids: torch.Tensor,
276
positions: torch.Tensor,
322
- kv_caches: List[torch.Tensor],
277
attn_metadata: AttentionMetadata,
278
conv_state: torch.Tensor,
279
ssm_state: torch.Tensor,
@@ -346,26 +300,6 @@ class MambaModel(nn.Module):
300
301
302
class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
349
- packed_modules_mapping = {
350
- "qkv_proj": [
351
- "q_proj",
352
- "k_proj",
353
- "v_proj",
354
- ],
355
- }
356
-
357
- # LoRA specific attributes
358
- supported_lora_modules = [
359
- "qkv_proj",
360
- "o_proj",
361
- "embed_tokens",
362
- "lm_head",
363
- ]
364
- embedding_modules = {
365
- "embeddings": "input_embeddings",
366
- "lm_head": "output_embeddings",
367
- }
368
- embedding_padding_modules = ["lm_head"]
303
304
def __init__(
305
self,
@@ -416,8 +350,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
350
mamba_cache_tensors = self.mamba_cache.current_run_tensors(
351
input_ids, attn_metadata, **kwargs)
352
419
- hidden_states = self.backbone(input_ids, positions, kv_caches,
420
- attn_metadata, mamba_cache_tensors[0],
353
+ hidden_states = self.backbone(input_ids, positions, attn_metadata,
354
+ mamba_cache_tensors[0],
355
mamba_cache_tensors[1])
356
357
return hidden_states
@@ -457,43 +391,16 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
391
return next_tokens
392
393
def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
460
- stacked_params_mapping = [
461
- # (param_name, shard_name, shard_id)
462
- ("qkv_proj", "q_proj", "q"),
463
- ("qkv_proj", "k_proj", "k"),
464
- ("qkv_proj", "v_proj", "v"),
465
- ("gate_up_proj", "gate_proj", 0),
466
- ("gate_up_proj", "up_proj", 1),
467
- ]
468
-
394
params_dict = dict(self.named_parameters())
395
for name, loaded_weight in weights:
471
- if "rotary_emb.inv_freq" in name:
472
- continue
473
-
396
if "A_log" in name:
397
name = name.replace("A_log", "A")
398
477
- if ".self_attn." in name:
478
- name = name.replace(".self_attn", "")
479
-
480
- for param_name, weight_name, shard_id in stacked_params_mapping:
481
- if weight_name not in name:
482
- continue
483
- name = name.replace(weight_name, param_name)
484
- # Skip loading extra bias for GPTQ models.
485
- if name.endswith(".bias") and name not in params_dict:
486
- continue
487
- param = params_dict[name]
488
- weight_loader = param.weight_loader
489
- weight_loader(param, loaded_weight, shard_id)
490
- break
491
- else:
492
- # Skip loading extra bias for GPTQ models.
493
- if name.endswith(".bias") and name not in params_dict:
494
- continue
495
-
496
- param = params_dict[name]
497
- weight_loader = getattr(param, "weight_loader",
498
- default_weight_loader)
499
- weight_loader(param, loaded_weight)
399
+ # Skip loading extra bias for GPTQ models.
400
+ if name.endswith(".bias") and name not in params_dict:
401
+ continue
402
+
403
+ param = params_dict[name]
404
+ weight_loader = getattr(param, "weight_loader",
405
+ default_weight_loader)
406
+ weight_loader(param, loaded_weight)