@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ffa48c9146
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Model] PP support for Mamba-like models (#10992)

Signed-off-by: mzusman <mor.zusmann@gmail.com>

Mor Zusman committed Dec 11, 2024 at 04:53 UTC ffa48c9146fda1e8810d1cfa159e1d70aadae6c6
11 files changed +229 -81
docs/source/models/supported_models.rst
+3 -3
@@ -128,7 +128,7 @@ Text Generation
128 - FalconMamba
129 - :code:`tiiuae/falcon-mamba-7b`, :code:`tiiuae/falcon-mamba-7b-instruct`, etc.
130 - ✅︎
131 - -
131 + - ✅︎
132 * - :code:`GemmaForCausalLM`
133 - Gemma
134 - :code:`google/gemma-2b`, :code:`google/gemma-7b`, etc.
@@ -193,7 +193,7 @@ Text Generation
193 - Jamba
194 - :code:`ai21labs/AI21-Jamba-1.5-Large`, :code:`ai21labs/AI21-Jamba-1.5-Mini`, :code:`ai21labs/Jamba-v0.1`, etc.
195 - ✅︎
196 - -
196 + - ✅︎
197 * - :code:`LlamaForCausalLM`
198 - Llama 3.1, Llama 3, Llama 2, LLaMA, Yi
199 - :code:`meta-llama/Meta-Llama-3.1-405B-Instruct`, :code:`meta-llama/Meta-Llama-3.1-70B`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-70b-hf`, :code:`01-ai/Yi-34B`, etc.
@@ -203,7 +203,7 @@ Text Generation
203 - Mamba
204 - :code:`state-spaces/mamba-130m-hf`, :code:`state-spaces/mamba-790m-hf`, :code:`state-spaces/mamba-2.8b-hf`, etc.
205 -
206 - -
206 + - ✅︎
207 * - :code:`MiniCPMForCausalLM`
208 - MiniCPM
209 - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, :code:`openbmb/MiniCPM-S-1B-sft`, etc.
tests/distributed/test_pipeline_parallel.py
+4 -2
@@ -156,13 +156,13 @@ TEXT_GENERATION_MODELS = {
156 # "internlm/internlm-chat-7b": PPTestSettings.fast(),
157 "internlm/internlm2-chat-7b": PPTestSettings.fast(trust_remote_code=True),
158 "inceptionai/jais-13b-chat": PPTestSettings.fast(),
159 - # TODO: Implement PP
160 - # "ai21labs/AI21-Jamba-1.5-Mini": PPTestSettings.fast(),
159 + "ai21labs/Jamba-tiny-dev": PPTestSettings.fast(),
160 "meta-llama/Meta-Llama-3-8B": PPTestSettings.detailed(),
161 "openbmb/MiniCPM-2B-sft-bf16": PPTestSettings.fast(trust_remote_code=True),
162 "openbmb/MiniCPM3-4B": PPTestSettings.fast(trust_remote_code=True),
163 # Uses Llama
164 # "mistralai/Mistral-7B-Instruct-v0.1": PPTestSettings.fast(),
165 + "state-spaces/mamba-130m-hf": PPTestSettings.fast(),
166 "mistralai/Mixtral-8x7B-Instruct-v0.1": PPTestSettings.fast(tp_base=4),
167 "mosaicml/mpt-7b": PPTestSettings.fast(),
168 "nvidia/Minitron-8B-Base": PPTestSettings.fast(),
@@ -234,6 +234,8 @@ TEST_MODELS = [
234 "OpenGVLab/InternVL2-1B",
235 "microsoft/Phi-3-vision-128k-instruct",
236 "fixie-ai/ultravox-v0_3",
237 + # [LANGUAGE GENERATION - HYBRID ARCH]
238 + "ai21labs/Jamba-tiny-dev",
239 ]
240
241
vllm/config.py
+44 -14
@@ -27,8 +27,8 @@ from vllm.transformers_utils.config import (
27 ConfigFormat, get_config, get_hf_image_processor_config,
28 get_hf_text_config, get_pooling_config,
29 get_sentence_transformer_tokenizer_config, is_encoder_decoder, uses_mrope)
30 -from vllm.utils import (GiB_bytes, cuda_device_count_stateless, get_cpu_memory,
31 - print_warning_once, random_uuid,
30 +from vllm.utils import (GiB_bytes, LayerBlockType, cuda_device_count_stateless,
31 + get_cpu_memory, print_warning_once, random_uuid,
32 resolve_obj_by_qualname)
33
34 if TYPE_CHECKING:
@@ -284,6 +284,7 @@ class ModelConfig:
284 self._verify_tokenizer_mode()
285
286 self.is_attention_free = self._init_attention_free()
287 + self.is_hybrid = self._init_is_hybrid()
288 self.has_inner_state = self._init_has_inner_state()
289
290 if current_platform.is_neuron():
@@ -340,6 +341,10 @@ class ModelConfig:
341 architectures = getattr(self.hf_config, "architectures", [])
342 return ModelRegistry.is_attention_free_model(architectures)
343
344 + def _init_is_hybrid(self) -> bool:
345 + architectures = getattr(self.hf_config, "architectures", [])
346 + return ModelRegistry.is_hybrid_model(architectures)
347 +
348 def _init_has_inner_state(self) -> bool:
349 architectures = getattr(self.hf_config, "architectures", [])
350 return ModelRegistry.model_has_inner_state(architectures)
@@ -669,26 +674,51 @@ class ModelConfig:
674 num_heads = getattr(self.hf_text_config, "num_attention_heads", 0)
675 return num_heads // parallel_config.tensor_parallel_size
676
672 - def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
677 + def get_layers_start_end_indices(
678 + self, parallel_config: "ParallelConfig") -> Tuple[int, int]:
679 from vllm.distributed.utils import get_pp_indices
680 total_num_hidden_layers = getattr(self.hf_text_config,
681 "num_hidden_layers", 0)
682 pp_rank = parallel_config.rank // parallel_config.tensor_parallel_size
683 pp_size = parallel_config.pipeline_parallel_size
684 start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
679 - return end - start
680 -
681 - def get_num_attention_layers(self,
682 - parallel_config: "ParallelConfig") -> int:
683 - if self.is_attention_free:
684 - return 0
685 + return start, end
686
686 - num_layers = self.get_num_layers(parallel_config)
687 + def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
688 + start, end = self.get_layers_start_end_indices(parallel_config)
689 + return end - start
690
688 - # Transformers supports layers_block_type @property
689 - layers = getattr(self.hf_config, "layers_block_type",
690 - ["attention"] * num_layers)
691 - return len([t for t in layers if t == "attention"])
691 + def get_num_layers_by_block_type(
692 + self,
693 + parallel_config: "ParallelConfig",
694 + block_type: LayerBlockType = LayerBlockType.attention,
695 + ) -> int:
696 + # This function relies on 'layers_block_type' in hf_config,
697 + # for w/o this attribute, we will need to have workarounds like so
698 + attn_block_type = block_type == LayerBlockType.attention
699 + is_transformer = not self.is_hybrid and not self.is_attention_free
700 + start, end = self.get_layers_start_end_indices(parallel_config)
701 +
702 + if is_transformer:
703 + # Handle the basic case first
704 + return end - start if attn_block_type else 0
705 + elif self.is_attention_free:
706 + # Attention free
707 + # Note that this code assumes there
708 + # is only one type of attention-free block type.
709 + return 0 if attn_block_type else end - start
710 + else:
711 + # Hybrid model
712 + layers_block_type_value = getattr(self.hf_config,
713 + "layers_block_type", None)
714 + if layers_block_type_value is None:
715 + raise ValueError("The model is an hybrid without a"
716 + "layers_block_type in the hf_config,"
717 + "cannot determine the num of "
718 + f"{block_type.value} layers")
719 +
720 + return sum(t == block_type.value
721 + for t in layers_block_type_value[start:end])
722
723 def get_multimodal_config(self) -> "MultiModalConfig":
724 """
vllm/model_executor/models/interfaces.py
+37
@@ -363,6 +363,43 @@ def is_attention_free(
363 return isinstance(model, IsAttentionFree)
364
365
366 +@runtime_checkable
367 +class IsHybrid(Protocol):
368 + """The interface required for all models like Jamba that have both
369 + attention and mamba blocks, indicates that
370 + hf_config has 'layers_block_type'"""
371 +
372 + is_hybrid: ClassVar[Literal[True]] = True
373 + """
374 + A flag that indicates this model has both mamba and attention blocks
375 + , also indicates that the model's hf_config has
376 + 'layers_block_type' """
377 +
378 +
379 +@runtime_checkable
380 +class _IsHybridType(Protocol):
381 + is_hybrid: ClassVar[Literal[True]]
382 +
383 +
384 +@overload
385 +def is_hybrid(model: object) -> TypeIs[IsHybrid]:
386 + ...
387 +
388 +
389 +@overload
390 +def is_hybrid(model: Type[object]) -> TypeIs[Type[IsHybrid]]:
391 + ...
392 +
393 +
394 +def is_hybrid(
395 + model: Union[Type[object], object]
396 +) -> Union[TypeIs[Type[IsHybrid]], TypeIs[IsHybrid]]:
397 + if isinstance(model, type):
398 + return isinstance(model, _IsHybridType)
399 +
400 + return isinstance(model, IsHybrid)
401 +
402 +
403 @runtime_checkable
404 class SupportsCrossEncoding(Protocol):
405 """The interface required for all models that support cross encoding."""
vllm/model_executor/models/jamba.py
+65 -28
@@ -9,6 +9,7 @@ from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.attention.layer import Attention
10 from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
11 from vllm.distributed import get_tensor_model_parallel_world_size
12 +from vllm.distributed.parallel_state import get_pp_group
13 from vllm.model_executor.layers.fused_moe import FusedMoE
14 from vllm.model_executor.layers.layernorm import RMSNorm
15 from vllm.model_executor.layers.linear import (QKVParallelLinear,
@@ -25,9 +26,12 @@ from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
26 MambaCacheParams)
27 from vllm.model_executor.sampling_metadata import SamplingMetadata
28 from vllm.sequence import IntermediateTensors
29 +from vllm.utils import LayerBlockType
30
29 -from .interfaces import HasInnerState, SupportsLoRA
30 -from .utils import maybe_prefix
31 +from .interfaces import HasInnerState, IsHybrid, SupportsLoRA, SupportsPP
32 +from .utils import (is_pp_missing_parameter,
33 + make_empty_intermediate_tensors_factory, make_layers,
34 + maybe_prefix)
35
36 KVCache = Tuple[torch.Tensor, torch.Tensor]
37
@@ -281,16 +285,24 @@ class JambaModel(nn.Module):
285 org_num_embeddings=config.vocab_size,
286 )
287
284 - decoder_layers = []
285 - for i in range(config.num_hidden_layers):
286 - layer_class = ALL_DECODER_LAYER_TYPES[config.layers_block_type[i]]
287 - decoder_layers.append(
288 - layer_class(config,
289 - layer_idx=i,
290 - cache_config=cache_config,
291 - quant_config=quant_config,
292 - prefix=f"{prefix}.layers.{i}"))
293 - self.layers = nn.ModuleList(decoder_layers)
288 + def get_layer(prefix: str):
289 + layer_idx = int(prefix.rsplit(".", 1)[1])
290 + layer_class = ALL_DECODER_LAYER_TYPES[
291 + config.layers_block_type[layer_idx]]
292 + return layer_class(
293 + config,
294 + layer_idx,
295 + cache_config,
296 + quant_config=quant_config,
297 + prefix=prefix,
298 + )
299 +
300 + self.start_layer, self.end_layer, self.layers = make_layers(
301 + config.num_hidden_layers, get_layer, prefix=f"{prefix}.layers")
302 + self.make_empty_intermediate_tensors = (
303 + make_empty_intermediate_tensors_factory(
304 + ["hidden_states", "residual"], config.hidden_size))
305 +
306 self.final_layernorm = RMSNorm(config.hidden_size,
307 eps=config.rms_norm_eps)
308
@@ -304,26 +316,34 @@ class JambaModel(nn.Module):
316 kv_caches: List[torch.Tensor],
317 attn_metadata: AttentionMetadata,
318 mamba_cache_params: MambaCacheParams,
319 + intermediate_tensors: Optional[IntermediateTensors] = None,
320 inputs_embeds: Optional[torch.Tensor] = None,
321 ) -> torch.Tensor:
309 - if inputs_embeds is not None:
310 - hidden_states = inputs_embeds
322 + if get_pp_group().is_first_rank:
323 + if inputs_embeds is not None:
324 + hidden_states = inputs_embeds
325 + else:
326 + hidden_states = self.get_input_embeddings(input_ids)
327 + residual = None
328 else:
312 - hidden_states = self.get_input_embeddings(input_ids)
313 - residual = None
314 - for i in range(len(self.layers)):
329 + assert intermediate_tensors is not None
330 + hidden_states = intermediate_tensors["hidden_states"]
331 + residual = intermediate_tensors["residual"]
332 +
333 + kv_cache_index = 0
334 + mamba_cache_index = 0
335 + for i in range(self.start_layer, self.end_layer):
336 layer = self.layers[i]
337 kv_cache = None
338 layer_mamba_cache_params = None
339 if isinstance(layer, JambaAttentionDecoderLayer):
319 - kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
320 - self.config.attn_layer_period]
340 + kv_cache = kv_caches[kv_cache_index]
341 + kv_cache_index += 1
342 if isinstance(layer, JambaMambaDecoderLayer):
322 - current_state_layer = i - (1 +
323 - (i - self.config.attn_layer_offset)
324 - // self.config.attn_layer_period)
343 + current_state_layer = mamba_cache_index
344 layer_mamba_cache_params = mamba_cache_params.at_layer_idx(
345 current_state_layer)
346 + mamba_cache_index += 1
347
348 hidden_states, residual = layer(
349 positions=positions,
@@ -332,11 +352,17 @@ class JambaModel(nn.Module):
352 attn_metadata=attn_metadata,
353 residual=residual,
354 mamba_cache_params=layer_mamba_cache_params)
355 + if not get_pp_group().is_last_rank:
356 + return IntermediateTensors({
357 + "hidden_states": hidden_states,
358 + "residual": residual
359 + })
360 hidden_states, _ = self.final_layernorm(hidden_states, residual)
361 return hidden_states
362
363
339 -class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
364 +class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA, SupportsPP,
365 + IsHybrid):
366 packed_modules_mapping = {
367 "qkv_proj": [
368 "q_proj",
@@ -368,6 +394,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
394
395 super().__init__()
396 self.config = config
397 + self.vllm_config = vllm_config
398 + self.model_config = vllm_config.model_config
399 self.scheduler_config = scheduler_config
400 self.model = JambaModel(vllm_config=vllm_config,
401 prefix=maybe_prefix(prefix, "model"))
@@ -390,6 +418,9 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
418 config.vocab_size)
419 self.sampler = get_sampler()
420
421 + self.make_empty_intermediate_tensors = (
422 + self.model.make_empty_intermediate_tensors)
423 +
424 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
425 return self.model.get_input_embeddings(input_ids)
426
@@ -406,10 +437,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
437 self.scheduler_config.max_num_seqs) if self.scheduler_config
438 else max(_BATCH_SIZES_TO_CAPTURE) + 2)
439
409 - layers_type = self.config.layers_block_type
410 - num_mamba_layers = sum(
411 - [layer_type == "mamba" for layer_type in layers_type])
412 -
440 + num_mamba_layers = self.model_config.get_num_layers_by_block_type(
441 + self.vllm_config.parallel_config, LayerBlockType.mamba)
442 self.mamba_cache = MambaCacheManager(
443 self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
444 *self._get_mamba_cache_shape())
@@ -423,7 +452,7 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
452 state_indices_tensor)
453 hidden_states = self.model(input_ids, positions, kv_caches,
454 attn_metadata, mamba_cache_params,
426 - inputs_embeds)
455 + intermediate_tensors, inputs_embeds)
456 return hidden_states
457
458 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
@@ -504,8 +533,12 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
533 continue
534 name = name.replace(weight_name, param_name)
535 # Skip loading extra bias for GPTQ models.
536 +
537 if name.endswith(".bias") and name not in params_dict:
538 continue
539 + # Skip layers on other devices.
540 + if is_pp_missing_parameter(name, self):
541 + continue
542 param = params_dict[name]
543 weight_loader = param.weight_loader
544 weight_loader(param, loaded_weight, shard_id)
@@ -520,6 +553,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
553 if weight_name not in name:
554 continue
555
556 + if is_pp_missing_parameter(name, self):
557 + continue
558 name = name.replace(weight_name, param_name)
559 param = params_dict[name]
560 weight_loader = param.weight_loader
@@ -533,6 +568,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
568 # Skip loading extra bias for GPTQ models.
569 if name.endswith(".bias") and name not in params_dict:
570 continue
571 + if is_pp_missing_parameter(name, self):
572 + continue
573
574 param = params_dict[name]
575 weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/mamba.py
+48 -20
@@ -8,6 +8,7 @@ from transformers import MambaConfig
8 from vllm.attention.backends.abstract import AttentionMetadata
9 from vllm.config import _BATCH_SIZES_TO_CAPTURE, CacheConfig, VllmConfig
10 from vllm.distributed import get_tensor_model_parallel_world_size
11 +from vllm.distributed.parallel_state import get_pp_group
12 from vllm.model_executor.layers.layernorm import RMSNorm
13 from vllm.model_executor.layers.logits_processor import LogitsProcessor
14 from vllm.model_executor.layers.mamba.mamba_mixer import MambaMixer
@@ -18,13 +19,16 @@ from vllm.model_executor.layers.vocab_parallel_embedding import (
19 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
20 from vllm.model_executor.model_loader.weight_utils import default_weight_loader
21 from vllm.model_executor.models.interfaces import (HasInnerState,
21 - IsAttentionFree)
22 + IsAttentionFree, SupportsPP)
23 from vllm.model_executor.models.mamba_cache import (MambaCacheManager,
24 MambaCacheParams)
25 from vllm.model_executor.sampling_metadata import SamplingMetadata
26 from vllm.sequence import IntermediateTensors
27 +from vllm.utils import LayerBlockType
28
27 -from .utils import maybe_prefix
29 +from .utils import (is_pp_missing_parameter,
30 + make_empty_intermediate_tensors_factory, make_layers,
31 + maybe_prefix)
32
33 KVCache = Tuple[torch.Tensor, torch.Tensor]
34
@@ -95,15 +99,17 @@ class MambaModel(nn.Module):
99 org_num_embeddings=config.vocab_size,
100 )
101
98 - decoder_layers = []
99 - for i in range(config.num_hidden_layers):
100 - decoder_layers.append(
101 - MambaDecoderLayer(config,
102 - cache_config=cache_config,
103 - quant_config=quant_config))
104 - self.layers = nn.ModuleList(decoder_layers)
102 + self.start_layer, self.end_layer, self.layers = make_layers(
103 + config.num_hidden_layers,
104 + lambda prefix: MambaDecoderLayer(
105 + config, cache_config=cache_config, quant_config=quant_config),
106 + prefix=f"{prefix}.layers")
107 +
108 self.norm_f = RMSNorm(config.hidden_size,
109 eps=config.layer_norm_epsilon)
110 + self.make_empty_intermediate_tensors = (
111 + make_empty_intermediate_tensors_factory(
112 + ["hidden_states", "residual"], config.hidden_size))
113
114 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
115 return self.embeddings(input_ids)
@@ -114,29 +120,40 @@ class MambaModel(nn.Module):
120 positions: torch.Tensor,
121 attn_metadata: AttentionMetadata,
122 mamba_cache_params: MambaCacheParams,
123 + intermediate_tensors: Optional[IntermediateTensors] = None,
124 inputs_embeds: Optional[torch.Tensor] = None,
125 ) -> torch.Tensor:
119 -
120 - if inputs_embeds is not None:
121 - hidden_states = inputs_embeds
126 + if get_pp_group().is_first_rank:
127 + if inputs_embeds is not None:
128 + hidden_states = inputs_embeds
129 + else:
130 + hidden_states = self.get_input_embeddings(input_ids)
131 + residual = None
132 else:
123 - hidden_states = self.get_input_embeddings(input_ids)
124 - residual = None
133 + assert intermediate_tensors is not None
134 + hidden_states = intermediate_tensors["hidden_states"]
135 + residual = intermediate_tensors["residual"]
136
126 - for i in range(len(self.layers)):
137 + for i in range(self.start_layer, self.end_layer):
138 layer = self.layers[i]
139 hidden_states, residual = layer(
140 positions=positions,
141 hidden_states=hidden_states,
142 attn_metadata=attn_metadata,
143 residual=residual,
133 - mamba_cache_params=mamba_cache_params.at_layer_idx(i))
144 + mamba_cache_params=mamba_cache_params.at_layer_idx(
145 + i - self.start_layer))
146 + if not get_pp_group().is_last_rank:
147 + return IntermediateTensors({
148 + "hidden_states": hidden_states,
149 + "residual": residual
150 + })
151 hidden_states, _ = self.norm_f(hidden_states, residual)
152
153 return hidden_states
154
155
139 -class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
156 +class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree, SupportsPP):
157
158 def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
159 config = vllm_config.model_config.hf_config
@@ -148,7 +165,9 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
165
166 super().__init__()
167 self.config = config
168 + self.vllm_config = vllm_config
169 self.scheduler_config = scheduler_config
170 + self.model_config = vllm_config.model_config
171 self.backbone = MambaModel(vllm_config=vllm_config,
172 prefix=maybe_prefix(prefix, "backbone"))
173 self.unpadded_vocab_size = config.vocab_size
@@ -174,6 +193,9 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
193 config.vocab_size)
194 self.sampler = get_sampler()
195
196 + self.make_empty_intermediate_tensors = (
197 + self.backbone.make_empty_intermediate_tensors)
198 +
199 def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
200 return self.backbone.get_input_embeddings(input_ids)
201
@@ -189,9 +211,12 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
211 max_batch_size = (VllmConfig.get_graph_batch_size(
212 self.scheduler_config.max_num_seqs) if self.scheduler_config
213 else max(_BATCH_SIZES_TO_CAPTURE) + 2)
214 +
215 + num_mamba_layers = self.model_config.get_num_layers_by_block_type(
216 + self.vllm_config.parallel_config, LayerBlockType.mamba)
217 self.mamba_cache = MambaCacheManager(
193 - self.lm_head.weight.dtype, self.config.num_hidden_layers,
194 - max_batch_size, *self._get_mamba_cache_shape())
218 + self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
219 + *self._get_mamba_cache_shape())
220
221 (
222 mamba_cache_tensors,
@@ -204,7 +229,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
229 state_indices_tensor)
230
231 hidden_states = self.backbone(input_ids, positions, attn_metadata,
207 - mamba_cache_params, inputs_embeds)
232 + mamba_cache_params, intermediate_tensors,
233 + inputs_embeds)
234
235 return hidden_states
236
@@ -252,6 +278,8 @@ class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
278 # Skip loading extra bias for GPTQ models.
279 if name.endswith(".bias") and name not in params_dict:
280 continue
281 + if is_pp_missing_parameter(name, self):
282 + continue
283
284 param = params_dict[name]
285 weight_loader = getattr(param, "weight_loader",
vllm/model_executor/models/registry.py
+10 -1
@@ -21,7 +21,7 @@ from vllm.logger import init_logger
21 from vllm.platforms import current_platform
22
23 from .adapters import as_embedding_model
24 -from .interfaces import (has_inner_state, is_attention_free,
24 +from .interfaces import (has_inner_state, is_attention_free, is_hybrid,
25 supports_cross_encoding, supports_multimodal,
26 supports_pp)
27 from .interfaces_base import is_pooling_model, is_text_generation_model
@@ -218,6 +218,7 @@ class _ModelInfo:
218 supports_pp: bool
219 has_inner_state: bool
220 is_attention_free: bool
221 + is_hybrid: bool
222
223 @staticmethod
224 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -239,6 +240,7 @@ class _ModelInfo:
240 supports_pp=supports_pp(model),
241 has_inner_state=has_inner_state(model),
242 is_attention_free=is_attention_free(model),
243 + is_hybrid=is_hybrid(model),
244 )
245
246
@@ -484,6 +486,13 @@ class _ModelRegistry:
486 model_cls, _ = self.inspect_model_cls(architectures)
487 return model_cls.is_attention_free
488
489 + def is_hybrid_model(
490 + self,
491 + architectures: Union[str, List[str]],
492 + ) -> bool:
493 + model_cls, _ = self.inspect_model_cls(architectures)
494 + return model_cls.is_hybrid
495 +
496
497 ModelRegistry = _ModelRegistry({
498 model_arch: _LazyRegisteredModel(
vllm/utils.py
+5
@@ -170,6 +170,11 @@ class Device(enum.Enum):
170 CPU = enum.auto()
171
172
173 +class LayerBlockType(enum.Enum):
174 + attention = "attention"
175 + mamba = "mamba"
176 +
177 +
178 class Counter:
179
180 def __init__(self, start: int = 0) -> None:
vllm/v1/worker/gpu_model_runner.py
+4 -4
@@ -15,8 +15,8 @@ from vllm.logger import init_logger
15 from vllm.model_executor.model_loader import get_model
16 from vllm.multimodal import MultiModalKwargs
17 from vllm.sampling_params import SamplingType
18 -from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler, cdiv,
19 - is_pin_memory_available)
18 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, DeviceMemoryProfiler,
19 + LayerBlockType, cdiv, is_pin_memory_available)
20 from vllm.v1.attention.backends.flash_attn import (FlashAttentionBackend,
21 FlashAttentionMetadata)
22 from vllm.v1.outputs import ModelRunnerOutput
@@ -68,8 +68,8 @@ class GPUModelRunner:
68 self.max_num_tokens = scheduler_config.max_num_batched_tokens
69
70 # Model-related.
71 - self.num_attn_layers = model_config.get_num_attention_layers(
72 - parallel_config)
71 + self.num_attn_layers = model_config.get_num_layers_by_block_type(
72 + parallel_config, LayerBlockType.attention)
73 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
74 self.head_size = model_config.get_head_size()
75 self.hidden_size = model_config.get_hidden_size()
vllm/v1/worker/gpu_worker.py
+3 -3
@@ -14,7 +14,7 @@ from vllm.distributed import (ensure_model_parallel_initialized,
14 from vllm.logger import init_logger
15 from vllm.model_executor import set_random_seed
16 from vllm.platforms import current_platform
17 -from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size
17 +from vllm.utils import STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType, get_dtype_size
18 from vllm.v1.core.scheduler import SchedulerOutput
19 from vllm.v1.outputs import ModelRunnerOutput
20 from vllm.v1.worker.gpu_model_runner import GPUModelRunner
@@ -260,8 +260,8 @@ def _get_cache_block_size(
260 ) -> int:
261 head_size = model_config.get_head_size()
262 num_heads = model_config.get_num_kv_heads(parallel_config)
263 - num_attention_layers = model_config.get_num_attention_layers(
264 - parallel_config)
263 + num_attention_layers = model_config.get_num_layers_by_block_type(
264 + parallel_config, LayerBlockType.attention)
265
266 key_cache_block = cache_config.block_size * num_heads * head_size
267 value_cache_block = key_cache_block
vllm/worker/cache_engine.py
+6 -6
@@ -6,8 +6,8 @@ import torch
6 from vllm.attention import get_attn_backend
7 from vllm.config import CacheConfig, DeviceConfig, ModelConfig, ParallelConfig
8 from vllm.logger import init_logger
9 -from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, get_dtype_size,
10 - is_pin_memory_available)
9 +from vllm.utils import (STR_DTYPE_TO_TORCH_DTYPE, LayerBlockType,
10 + get_dtype_size, is_pin_memory_available)
11
12 logger = init_logger(__name__)
13
@@ -34,8 +34,8 @@ class CacheEngine:
34
35 self.head_size = model_config.get_head_size()
36 # Models like Jamba, have mixed typed layers, E.g Mamba
37 - self.num_attention_layers = model_config.get_num_attention_layers(
38 - parallel_config)
37 + self.num_attention_layers = model_config.get_num_layers_by_block_type(
38 + parallel_config, LayerBlockType.attention)
39 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
40
41 self.block_size = cache_config.block_size
@@ -105,8 +105,8 @@ class CacheEngine:
105 ) -> int:
106 head_size = model_config.get_head_size()
107 num_heads = model_config.get_num_kv_heads(parallel_config)
108 - num_attention_layers = model_config.get_num_attention_layers(
109 - parallel_config)
108 + num_attention_layers = model_config.get_num_layers_by_block_type(
109 + parallel_config, LayerBlockType.attention)
110
111 key_cache_block = cache_config.block_size * num_heads * head_size
112 value_cache_block = key_cache_block