A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 04:30 UTC
[Model] PP support for Mamba-like models (#10992)
Signed-off-by: mzusman <mor.zusmann@gmail.com>
Mor Zusman committed
Dec 11, 2024 at 04:53 UTC
ffa48c9146fda1e8810d1cfa159e1d70aadae6c6
11 files changed
+229
-81
docs/source/models/supported_models.rst
+3
-3
@@ -128,7 +128,7 @@ Text Generation
128
- FalconMamba
129
- :code:`tiiuae/falcon-mamba-7b`, :code:`tiiuae/falcon-mamba-7b-instruct`, etc.
130
- ✅︎
131
- -
131
+ - ✅︎
132
* - :code:`GemmaForCausalLM`
133
- Gemma
134
- :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
@@ -193,7 +193,7 @@ Text Generation
193
- Jamba
194
- :code:`ai21labs/AI21-Jamba-1.5-Large`, :code:`ai21labs/AI21-Jamba-1.5-Mini`, :code:`ai21labs/Jamba-v0.1`, etc.
195
- ✅︎
196
- -
196
+ - ✅︎
197
* - :code:`LlamaForCausalLM`
198
- Llama 3.1, Llama 3, Llama 2, LLaMA, Yi
199
- :code:`meta-llama/Meta-Llama-3.1-405B-Instruct`, :code:`meta-llama/Meta-Llama-3.1-70B`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-70b-hf`, :code:`01-ai/Yi-34B`, etc.
@@ -203,7 +203,7 @@ Text Generation
203
- Mamba
204
- :code:`state-spaces/mamba-130m-hf`, :code:`state-spaces/mamba-790m-hf`, :code:`state-spaces/mamba-2.8b-hf`, etc.
205
-
206
- -
206
+ - ✅︎
207
* - :code:`MiniCPMForCausalLM`
208
- MiniCPM
209
- :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, :code:`openbmb/MiniCPM-S-1B-sft`, etc.
tests/distributed/test_pipeline_parallel.py
+4
-2
@@ -156,13 +156,13 @@ TEXT_GENERATION_MODELS = {
156
# "internlm/internlm-chat-7b": PPTestSettings.fast(),
157
"internlm/internlm2-chat-7b": PPTestSettings.fast(trust_remote_code=True),
158
"inceptionai/jais-13b-chat": PPTestSettings.fast(),
159
- # TODO: Implement PP
160
- # "ai21labs/AI21-Jamba-1.5-Mini": PPTestSettings.fast(),
159
+ "ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
160
"meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
161
"openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
162
"openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
163
# Uses Llama
164
# "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
165
+ "state-spaces/mamba-130m-hf": PPTestSettings.fast(),
166
"mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(tp_base=4),
167
"mosaicml/mpt-7b": PPTestSettings.fast(),
168
"nvidia/Minitron-8B-Base": PPTestSettings.fast(),
@@ -234,6 +234,8 @@ TEST_MODELS = [
234
"OpenGVLab/InternVL2-1B",
235
"microsoft/Phi-3-vision-128k-instruct",
236
"fixie-ai/ultravox-v0_3",
237
+ # [LANGUAGE GENERATION - HYBRID ARCH]
238
+ "ai21labs/Jamba-tiny-dev",
239
]
240
241
vllm/config.py
+44
-14
@@ -27,8 +27,8 @@ from vllm.transformers_utils.config import (
27
ConfigFormat, get_config, get_hf_image_processor_config,
28
get_hf_text_config, get_pooling_config,
29
get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
30
-from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
31
- print_warning_once, random_uuid,
30
+from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
31
+ get_cpu_memory, print_warning_once, random_uuid,
32
resolve_obj_by_qualname)
33
34
if TYPE_CHECKING:
@@ -284,6 +284,7 @@ class ModelConfig:
284
self._verify_tokenizer_mode()
285
286
self.is_attention_free = self._init_attention_free()
287
+ self.is_hybrid = self._init_is_hybrid()
288
self.has_inner_state = self._init_has_inner_state()
289
290
if current_platform.is_neuron():
@@ -340,6 +341,10 @@ class ModelConfig:
341
architectures = getattr(self.hf_config, "architectures", [])
342
return ModelRegistry.is_attention_free_model(architectures)
343
344
+ def _init_is_hybrid(self) -> bool:
345
+ architectures = getattr(self.hf_config, "architectures", [])
346
+ return ModelRegistry.is_hybrid_model(architectures)
347
+
348
def _init_has_inner_state(self) -> bool:
349
architectures = getattr(self.hf_config, "architectures", [])
350
return ModelRegistry.model_has_inner_state(architectures)
@@ -669,26 +674,51 @@ class ModelConfig:
674
num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
675
return num_heads // parallel_config.tensor_parallel_size
676
672
- def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
677
+ def get_layers_start_end_indices(
678
+ self, parallel_config: "ParallelConfig") -> Tuple[int, int]:
679
from vllm.distributed.utils import get_pp_indices
680
total_num_hidden_layers = getattr(self.hf_text_config,
681
"num_hidden_layers", 0)
682
pp_rank = parallel_config.rank // parallel_config.tensor_parallel_size
683
pp_size = parallel_config.pipeline_parallel_size
684
start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
679
- return end - start
680
-
681
- def get_num_attention_layers(self,
682
- parallel_config: "ParallelConfig") -> int:
683
- if self.is_attention_free:
684
- return 0
685
+ return start, end
686
686
- num_layers = self.get_num_layers(parallel_config)
687
+ def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
688
+ start, end = self.get_layers_start_end_indices(parallel_config)
689
+ return end - start
690
688
- # Transformers supports layers_block_type @property
689
- layers = getattr(self.hf_config, "layers_block_type",
690
- ["attention"] * num_layers)
691
- return len([t for t in layers if t == "attention"])
691
+ def get_num_layers_by_block_type(
692
+ self,
693
+ parallel_config: "ParallelConfig",
694
+ block_type: LayerBlockType = LayerBlockType.attention,
695
+ ) -> int:
696
+ # This function relies on 'layers_block_type' in hf_config,
697
+ # for w/o this attribute, we will need to have workarounds like so
698
+ attn_block_type = block_type == LayerBlockType.attention
699
+ is_transformer = not self.is_hybrid and not self.is_attention_free
700
+ start, end = self.get_layers_start_end_indices(parallel_config)
701
+
702
+ if is_transformer:
703
+ # Handle the basic case first
704
+ return end - start if attn_block_type else 0
705
+ elif self.is_attention_free:
706
+ # Attention free
707
+ # Note that this code assumes there
708
+ # is only one type of attention-free block type.
709
+ return 0 if attn_block_type else end - start
710
+ else:
711
+ # Hybrid model
712
+ layers_block_type_value = getattr(self.hf_config,
713
+ "layers_block_type", None)
714
+ if layers_block_type_value is None:
715
+ raise ValueError("The model is an hybrid without a"
716
+ "layers_block_type in the hf_config,"
717
+ "cannot determine the num of "
718
+ f"{block_type.value} layers")
719
+
720
+ return sum(t == block_type.value
721
+ for t in layers_block_type_value[start:end])
722
723
def get_multimodal_config(self) -> "MultiModalConfig":
724
"""
vllm/model_executor/models/interfaces.py
+37
@@ -363,6 +363,43 @@ def is_attention_free(
363
return isinstance(model, IsAttentionFree)
364
365
366
+@runtime_checkable
367
+class IsHybrid(Protocol):
368
+ """The interface required for all models like Jamba that have both
369
+ attention and mamba blocks, indicates that
370
+ hf_config has 'layers_block_type'"""
371
+
372
+ is_hybrid: ClassVar[Literal[True]] = True
373
+ """
374
+ A flag that indicates this model has both mamba and attention blocks
375
+ , also indicates that the model's hf_config has
376
+ 'layers_block_type' """
377
+
378
+
379
+@runtime_checkable
380
+class _IsHybridType(Protocol):
381
+ is_hybrid: ClassVar[Literal[True]]
382
+
383
+
384
+@overload
385
+def is_hybrid(model: object) -> TypeIs[IsHybrid]:
386
+ ...
387
+
388
+
389
+@overload
390
+def is_hybrid(model: Type[object]) -> TypeIs[Type[IsHybrid]]:
391
+ ...
392
+
393
+
394
+def is_hybrid(
395
+ model: Union[Type[object], object]
396
+) -> Union[TypeIs[Type[IsHybrid]], TypeIs[IsHybrid]]:
397
+ if isinstance(model, type):
398
+ return isinstance(model, _IsHybridType)
399
+
400
+ return isinstance(model, IsHybrid)
401
+
402
+
403
@runtime_checkable
404
class SupportsCrossEncoding(Protocol):
405
"""The interface required for all models that support cross encoding."""
vllm/model_executor/models/jamba.py
+65
-28
@@ -9,6 +9,7 @@ from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.attention.layer import Attention
10
from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
11
from vllm.distributed import get_tensor_model_parallel_world_size
12
+from vllm.distributed.parallel_state import get_pp_group
13
from vllm.model_executor.layers.fused_moe import FusedMoE
14
from vllm.model_executor.layers.layernorm import RMSNorm
15
from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -25,9 +26,12 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
26
MambaCacheParams)
27
from vllm.model_executor.sampling_metadata import SamplingMetadata
28
from vllm.sequence import IntermediateTensors
29
+from vllm.utils import LayerBlockType
30
29
-from .interfaces import HasInnerState, SupportsLoRA
30
-from .utils import maybe_prefix
31
+from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
32
+from .utils import (is_pp_missing_parameter,
33
+ make_empty_intermediate_tensors_factory, make_layers,
34
+ maybe_prefix)
35
36
KVCache = Tuple[torch.Tensor, torch.Tensor]
37
@@ -281,16 +285,24 @@ class JambaModel(nn.Module):
285
org_num_embeddings=config.vocab_size,
286
)
287
284
- decoder_layers = []
285
- for i in range(config.num_hidden_layers):
286
- layer_class = ALL_DECODER_LAYER_TYPES[config.layers_block_type[i]]
287
- decoder_layers.append(
288
- layer_class(config,
289
- layer_idx=i,
290
- cache_config=cache_config,
291
- quant_config=quant_config,
292
- prefix=f"{prefix}.layers.{i}"))
293
- self.layers = nn.ModuleList(decoder_layers)
288
+ def get_layer(prefix: str):
289
+ layer_idx = int(prefix.rsplit(".", 1)[1])
290
+ layer_class = ALL_DECODER_LAYER_TYPES[
291
+ config.layers_block_type[layer_idx]]
292
+ return layer_class(
293
+ config,
294
+ layer_idx,
295
+ cache_config,
296
+ quant_config=quant_config,
297
+ prefix=prefix,
298
+ )
299
+
300
+ self.start_layer, self.end_layer, self.layers = make_layers(
301
+ config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers")
302
+ self.make_empty_intermediate_tensors = (
303
+ make_empty_intermediate_tensors_factory(
304
+ ["hidden_states", "residual"], config.hidden_size))
305
+
306
self.final_layernorm = RMSNorm(config.hidden_size,
307
eps=config.rms_norm_eps)
308
@@ -304,26 +316,34 @@ class JambaModel(nn.Module):
316
kv_caches: List[torch.Tensor],
317
attn_metadata: AttentionMetadata,
318
mamba_cache_params: MambaCacheParams,
319
+ intermediate_tensors: Optional[IntermediateTensors] = None,
320
inputs_embeds: Optional[torch.Tensor] = None,
321
) -> torch.Tensor:
309
- if inputs_embeds is not None:
310
- hidden_states = inputs_embeds
322
+ if get_pp_group().is_first_rank:
323
+ if inputs_embeds is not None:
324
+ hidden_states = inputs_embeds
325
+ else:
326
+ hidden_states = self.get_input_embeddings(input_ids)
327
+ residual = None
328
else:
312
- hidden_states = self.get_input_embeddings(input_ids)
313
- residual = None
314
- for i in range(len(self.layers)):
329
+ assert intermediate_tensors is not None
330
+ hidden_states = intermediate_tensors["hidden_states"]
331
+ residual = intermediate_tensors["residual"]
332
+
333
+ kv_cache_index = 0
334
+ mamba_cache_index = 0
335
+ for i in range(self.start_layer, self.end_layer):
336
layer = self.layers[i]
337
kv_cache = None
338
layer_mamba_cache_params = None
339
if isinstance(layer, JambaAttentionDecoderLayer):
319
- kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
320
- self.config.attn_layer_period]
340
+ kv_cache = kv_caches[kv_cache_index]
341
+ kv_cache_index += 1
342
if isinstance(layer, JambaMambaDecoderLayer):
322
- current_state_layer = i - (1 +
323
- (i - self.config.attn_layer_offset)
324
- // self.config.attn_layer_period)
343
+ current_state_layer = mamba_cache_index
344
layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
345
current_state_layer)
346
+ mamba_cache_index += 1
347
348
hidden_states, residual = layer(
349
positions=positions,
@@ -332,11 +352,17 @@ class JambaModel(nn.Module):
352
attn_metadata=attn_metadata,
353
residual=residual,
354
mamba_cache_params=layer_mamba_cache_params)
355
+ if not get_pp_group().is_last_rank:
356
+ return IntermediateTensors({
357
+ "hidden_states": hidden_states,
358
+ "residual": residual
359
+ })
360
hidden_states, _ = self.final_layernorm(hidden_states, residual)
361
return hidden_states
362
363
339
-class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
364
+class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
365
+ IsHybrid):
366
packed_modules_mapping = {
367
"qkv_proj": [
368
"q_proj",
@@ -368,6 +394,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
394
395
super().__init__()
396
self.config = config
397
+ self.vllm_config = vllm_config
398
+ self.model_config = vllm_config.model_config
399
self.scheduler_config = scheduler_config
400
self.model = JambaModel(vllm_config=vllm_config,
401
prefix=maybe_prefix(prefix, "model"))
@@ -390,6 +418,9 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
418
config.vocab_size)
419
self.sampler = get_sampler()
420
421
+ self.make_empty_intermediate_tensors = (
422
+ self.model.make_empty_intermediate_tensors)
423
+
424
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
425
return self.model.get_input_embeddings(input_ids)
426
@@ -406,10 +437,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
437
self.scheduler_config.max_num_seqs) if self.scheduler_config
438
else max(_BATCH_SIZES_TO_CAPTURE) + 2)
439
409
- layers_type = self.config.layers_block_type
410
- num_mamba_layers = sum(
411
- [layer_type == "mamba" for layer_type in layers_type])
412
-
440
+ num_mamba_layers = self.model_config.get_num_layers_by_block_type(
441
+ self.vllm_config.parallel_config, LayerBlockType.mamba)
442
self.mamba_cache = MambaCacheManager(
443
self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
444
*self._get_mamba_cache_shape())
@@ -423,7 +452,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
452
state_indices_tensor)
453
hidden_states = self.model(input_ids, positions, kv_caches,
454
attn_metadata, mamba_cache_params,
426
- inputs_embeds)
455
+ intermediate_tensors, inputs_embeds)
456
return hidden_states
457
458
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
@@ -504,8 +533,12 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
533
continue
534
name = name.replace(weight_name, param_name)
535
# Skip loading extra bias for GPTQ models.
536
+
537
if name.endswith(".bias") and name not in params_dict:
538
continue
539
+ # Skip layers on other devices.
540
+ if is_pp_missing_parameter(name, self):
541
+ continue
542
param = params_dict[name]
543
weight_loader = param.weight_loader
544
weight_loader(param, loaded_weight, shard_id)
@@ -520,6 +553,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
553
if weight_name not in name:
554
continue
555
556
+ if is_pp_missing_parameter(name, self):
557
+ continue
558
name = name.replace(weight_name, param_name)
559
param = params_dict[name]
560
weight_loader = param.weight_loader
@@ -533,6 +568,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
568
# Skip loading extra bias for GPTQ models.
569
if name.endswith(".bias") and name not in params_dict:
570
continue
571
+ if is_pp_missing_parameter(name, self):
572
+ continue
573
574
param = params_dict[name]
575
weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/mamba.py
+48
-20
@@ -8,6 +8,7 @@ from transformers import MambaConfig
8
from vllm.attention.backends.abstract import AttentionMetadata
9
from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10
from vllm.distributed import get_tensor_model_parallel_world_size
11
+from vllm.distributed.parallel_state import get_pp_group
12
from vllm.model_executor.layers.layernorm import RMSNorm
13
from vllm.model_executor.layers.logits_processor import LogitsProcessor
14
from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
@@ -18,13 +19,16 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
19
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
20
from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21
from vllm.model_executor.models.interfaces import (HasInnerState,
21
- IsAttentionFree)
22
+ IsAttentionFree, SupportsPP)
23
from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
24
MambaCacheParams)
25
from vllm.model_executor.sampling_metadata import SamplingMetadata
26
from vllm.sequence import IntermediateTensors
27
+from vllm.utils import LayerBlockType
28
27
-from .utils import maybe_prefix
29
+from .utils import (is_pp_missing_parameter,
30
+ make_empty_intermediate_tensors_factory, make_layers,
31
+ maybe_prefix)
32
33
KVCache = Tuple[torch.Tensor, torch.Tensor]
34
@@ -95,15 +99,17 @@ class MambaModel(nn.Module):
99
org_num_embeddings=config.vocab_size,
100
)
101
98
- decoder_layers = []
99
- for i in range(config.num_hidden_layers):
100
- decoder_layers.append(
101
- MambaDecoderLayer(config,
102
- cache_config=cache_config,
103
- quant_config=quant_config))
104
- self.layers = nn.ModuleList(decoder_layers)
102
+ self.start_layer, self.end_layer, self.layers = make_layers(
103
+ config.num_hidden_layers,
104
+ lambda prefix: MambaDecoderLayer(
105
+ config, cache_config=cache_config, quant_config=quant_config),
106
+ prefix=f"{prefix}.layers")
107
+
108
self.norm_f = RMSNorm(config.hidden_size,
109
eps=config.layer_norm_epsilon)
110
+ self.make_empty_intermediate_tensors = (
111
+ make_empty_intermediate_tensors_factory(
112
+ ["hidden_states", "residual"], config.hidden_size))
113
114
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
115
return self.embeddings(input_ids)
@@ -114,29 +120,40 @@ class MambaModel(nn.Module):
120
positions: torch.Tensor,
121
attn_metadata: AttentionMetadata,
122
mamba_cache_params: MambaCacheParams,
123
+ intermediate_tensors: Optional[IntermediateTensors] = None,
124
inputs_embeds: Optional[torch.Tensor] = None,
125
) -> torch.Tensor:
119
-
120
- if inputs_embeds is not None:
121
- hidden_states = inputs_embeds
126
+ if get_pp_group().is_first_rank:
127
+ if inputs_embeds is not None:
128
+ hidden_states = inputs_embeds
129
+ else:
130
+ hidden_states = self.get_input_embeddings(input_ids)
131
+ residual = None
132
else:
123
- hidden_states = self.get_input_embeddings(input_ids)
124
- residual = None
133
+ assert intermediate_tensors is not None
134
+ hidden_states = intermediate_tensors["hidden_states"]
135
+ residual = intermediate_tensors["residual"]
136
126
- for i in range(len(self.layers)):
137
+ for i in range(self.start_layer, self.end_layer):
138
layer = self.layers[i]
139
hidden_states, residual = layer(
140
positions=positions,
141
hidden_states=hidden_states,
142
attn_metadata=attn_metadata,
143
residual=residual,
133
- mamba_cache_params=mamba_cache_params.at_layer_idx(i))
144
+ mamba_cache_params=mamba_cache_params.at_layer_idx(
145
+ i - self.start_layer))
146
+ if not get_pp_group().is_last_rank:
147
+ return IntermediateTensors({
148
+ "hidden_states": hidden_states,
149
+ "residual": residual
150
+ })
151
hidden_states, _ = self.norm_f(hidden_states, residual)
152
153
return hidden_states
154
155
139
-class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
156
+class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
157
158
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
159
config = vllm_config.model_config.hf_config
@@ -148,7 +165,9 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
165
166
super().__init__()
167
self.config = config
168
+ self.vllm_config = vllm_config
169
self.scheduler_config = scheduler_config
170
+ self.model_config = vllm_config.model_config
171
self.backbone = MambaModel(vllm_config=vllm_config,
172
prefix=maybe_prefix(prefix, "backbone"))
173
self.unpadded_vocab_size = config.vocab_size
@@ -174,6 +193,9 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
193
config.vocab_size)
194
self.sampler = get_sampler()
195
196
+ self.make_empty_intermediate_tensors = (
197
+ self.backbone.make_empty_intermediate_tensors)
198
+
199
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
200
return self.backbone.get_input_embeddings(input_ids)
201
@@ -189,9 +211,12 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
211
max_batch_size = (VllmConfig.get_graph_batch_size(
212
self.scheduler_config.max_num_seqs) if self.scheduler_config
213
else max(_BATCH_SIZES_TO_CAPTURE) + 2)
214
+
215
+ num_mamba_layers = self.model_config.get_num_layers_by_block_type(
216
+ self.vllm_config.parallel_config, LayerBlockType.mamba)
217
self.mamba_cache = MambaCacheManager(
193
- self.lm_head.weight.dtype, self.config.num_hidden_layers,
194
- max_batch_size, *self._get_mamba_cache_shape())
218
+ self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
219
+ *self._get_mamba_cache_shape())
220
221
(
222
mamba_cache_tensors,
@@ -204,7 +229,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
229
state_indices_tensor)
230
231
hidden_states = self.backbone(input_ids, positions, attn_metadata,
207
- mamba_cache_params, inputs_embeds)
232
+ mamba_cache_params, intermediate_tensors,
233
+ inputs_embeds)
234
235
return hidden_states
236
@@ -252,6 +278,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
278
# Skip loading extra bias for GPTQ models.
279
if name.endswith(".bias") and name not in params_dict:
280
continue
281
+ if is_pp_missing_parameter(name, self):
282
+ continue
283
284
param = params_dict[name]
285
weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/registry.py
+10
-1
@@ -21,7 +21,7 @@ from vllm.logger import init_logger
21
from vllm.platforms import current_platform
22
23
from .adapters import as_embedding_model
24
-from .interfaces import (has_inner_state, is_attention_free,
24
+from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
25
supports_cross_encoding, supports_multimodal,
26
supports_pp)
27
from .interfaces_base import is_pooling_model, is_text_generation_model
@@ -218,6 +218,7 @@ class _ModelInfo:
218
supports_pp: bool
219
has_inner_state: bool
220
is_attention_free: bool
221
+ is_hybrid: bool
222
223
@staticmethod
224
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -239,6 +240,7 @@ class _ModelInfo:
240
supports_pp=supports_pp(model),
241
has_inner_state=has_inner_state(model),
242
is_attention_free=is_attention_free(model),
243
+ is_hybrid=is_hybrid(model),
244
)
245
246
@@ -484,6 +486,13 @@ class _ModelRegistry:
486
model_cls, _ = self.inspect_model_cls(architectures)
487
return model_cls.is_attention_free
488
489
+ def is_hybrid_model(
490
+ self,
491
+ architectures: Union[str, List[str]],
492
+ ) -> bool:
493
+ model_cls, _ = self.inspect_model_cls(architectures)
494
+ return model_cls.is_hybrid
495
+
496
497
ModelRegistry = _ModelRegistry({
498
model_arch: _LazyRegisteredModel(
vllm/utils.py
+5
@@ -170,6 +170,11 @@ class Device(enum.Enum):
170
CPU = enum.auto()
171
172
173
+class LayerBlockType(enum.Enum):
174
+ attention = "attention"
175
+ mamba = "mamba"
176
+
177
+
178
class Counter:
179
180
def __init__(self, start: int = 0) -> None:
vllm/v1/worker/gpu_model_runner.py
+4
-4
@@ -15,8 +15,8 @@ from vllm.logger import init_logger
15
from vllm.model_executor.model_loader import get_model
16
from vllm.multimodal import MultiModalKwargs
17
from vllm.sampling_params import SamplingType
18
-from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19
- is_pin_memory_available)
18
+from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19
+ LayerBlockType, cdiv, is_pin_memory_available)
20
from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21
FlashAttentionMetadata)
22
from vllm.v1.outputs import ModelRunnerOutput
@@ -68,8 +68,8 @@ class GPUModelRunner:
68
self.max_num_tokens = scheduler_config.max_num_batched_tokens
69
70
# Model-related.
71
- self.num_attn_layers = model_config.get_num_attention_layers(
72
- parallel_config)
71
+ self.num_attn_layers = model_config.get_num_layers_by_block_type(
72
+ parallel_config, LayerBlockType.attention)
73
self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
74
self.head_size = model_config.get_head_size()
75
self.hidden_size = model_config.get_hidden_size()
vllm/v1/worker/gpu_worker.py
+3
-3
@@ -14,7 +14,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
14
from vllm.logger import init_logger
15
from vllm.model_executor import set_random_seed
16
from vllm.platforms import current_platform
17
-from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
17
+from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType, get_dtype_size
18
from vllm.v1.core.scheduler import SchedulerOutput
19
from vllm.v1.outputs import ModelRunnerOutput
20
from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -260,8 +260,8 @@ def _get_cache_block_size(
260
) -> int:
261
head_size = model_config.get_head_size()
262
num_heads = model_config.get_num_kv_heads(parallel_config)
263
- num_attention_layers = model_config.get_num_attention_layers(
264
- parallel_config)
263
+ num_attention_layers = model_config.get_num_layers_by_block_type(
264
+ parallel_config, LayerBlockType.attention)
265
266
key_cache_block = cache_config.block_size * num_heads * head_size
267
value_cache_block = key_cache_block
vllm/worker/cache_engine.py
+6
-6
@@ -6,8 +6,8 @@ import torch
6
from vllm.attention import get_attn_backend
7
from vllm.config import CacheConfig, DeviceConfig, ModelConfig, ParallelConfig
8
from vllm.logger import init_logger
9
-from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size,
10
- is_pin_memory_available)
9
+from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType,
10
+ get_dtype_size, is_pin_memory_available)
11
12
logger = init_logger(__name__)
13
@@ -34,8 +34,8 @@ class CacheEngine:
34
35
self.head_size = model_config.get_head_size()
36
# Models like Jamba, have mixed typed layers, E.g Mamba
37
- self.num_attention_layers = model_config.get_num_attention_layers(
38
- parallel_config)
37
+ self.num_attention_layers = model_config.get_num_layers_by_block_type(
38
+ parallel_config, LayerBlockType.attention)
39
self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
40
41
self.block_size = cache_config.block_size
@@ -105,8 +105,8 @@ class CacheEngine:
105
) -> int:
106
head_size = model_config.get_head_size()
107
num_heads = model_config.get_num_kv_heads(parallel_config)
108
- num_attention_layers = model_config.get_num_attention_layers(
109
- parallel_config)
108
+ num_attention_layers = model_config.get_num_layers_by_block_type(
109
+ parallel_config, LayerBlockType.attention)
110
111
key_cache_block = cache_config.block_size * num_heads * head_size
112
value_cache_block = key_cache_block