@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9d6a8daa87
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Model] Jamba support (#4115)

Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai> Co-authored-by: Erez Schwartz <erezs@ai21.com> Co-authored-by: Mor Zusman <morz@ai21.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Tomer Asida <tomera@ai21.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>

Mor Zusman committed Jul 3, 2024 at 02:11 UTC 9d6a8daa87e2e0af3ff45d03d08ad5a94ec089a8
21 files changed +1192 -34
.buildkite/run-cpu-test.sh
+1 -1
@@ -23,4 +23,4 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23 docker exec cpu-test bash -c "cd tests;
24 pip install pytest Pillow protobuf
25 cd ../
26 - pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py"
26 + pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
Dockerfile
+23
@@ -43,6 +43,10 @@ COPY requirements-cuda.txt requirements-cuda.txt
43 RUN --mount=type=cache,target=/root/.cache/pip \
44 python3 -m pip install -r requirements-cuda.txt
45
46 +COPY requirements-mamba.txt requirements-mamba.txt
47 +RUN python3 -m pip install packaging
48 +RUN python3 -m pip install -r requirements-mamba.txt
49 +
50 # cuda arch list used by torch
51 # can be useful for both `dev` and `test`
52 # explicitly set the list to avoid issues with torch 2.2
@@ -123,6 +127,21 @@ RUN --mount=type=cache,target=/root/.cache/pip \
127 python3 -m pip install -r requirements-dev.txt
128
129 #################### DEV IMAGE ####################
130 +#################### MAMBA Build IMAGE ####################
131 +FROM dev as mamba-builder
132 +# max jobs used for build
133 +ARG max_jobs=2
134 +ENV MAX_JOBS=${max_jobs}
135 +
136 +WORKDIR /usr/src/mamba
137 +
138 +COPY requirements-mamba.txt requirements-mamba.txt
139 +
140 +# Download the wheel or build it if a pre-compiled release doesn't exist
141 +RUN pip --verbose wheel -r requirements-mamba.txt \
142 + --no-build-isolation --no-deps --no-cache-dir
143 +
144 +#################### MAMBA Build IMAGE ####################
145
146 #################### vLLM installation IMAGE ####################
147 # image with vLLM installed
@@ -143,6 +162,10 @@ RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
162 RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
163 --mount=type=cache,target=/root/.cache/pip \
164 python3 -m pip install dist/*.whl --verbose
165 +
166 +RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamba \
167 + --mount=type=cache,target=/root/.cache/pip \
168 + python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
169 #################### vLLM installation IMAGE ####################
170
171
docs/source/models/supported_models.rst
+4
@@ -87,6 +87,10 @@ Alongside each architecture, we include some popular models that use it.
87 - Jais
88 - :code:`core42/jais-13b`, :code:`core42/jais-13b-chat`, :code:`core42/jais-30b-v3`, :code:`core42/jais-30b-chat-v3`, etc.
89 -
90 + * - :code:`JambaForCausalLM`
91 + - Jamba
92 + - :code:`ai21labs/Jamba-v0.1`, etc.
93 + - ✅︎
94 * - :code:`LlamaForCausalLM`
95 - LLaMA, Llama 2, Meta Llama 3, Vicuna, Alpaca, Yi
96 - :code:`meta-llama/Meta-Llama-3-8B-Instruct`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
requirements-mamba.txt new
+3
@@ -0,0 +1,3 @@
1 +# Mamba dependencies
2 +mamba-ssm>=1.2.2
3 +causal-conv1d>=1.2.0
tests/models/test_jamba.py new
+65
@@ -0,0 +1,65 @@
1 +import pytest
2 +
3 +MODELS = ["ai21labs/Jamba-tiny-random"]
4 +
5 +
6 +@pytest.mark.parametrize("model", MODELS)
7 +@pytest.mark.parametrize("dtype", ["float"])
8 +@pytest.mark.parametrize("max_tokens", [20])
9 +def test_models(
10 + hf_runner,
11 + vllm_runner,
12 + example_prompts,
13 + model: str,
14 + dtype: str,
15 + max_tokens: int,
16 +) -> None:
17 + # To pass the small model tests, we need full precision.
18 + assert dtype == "float"
19 +
20 + with hf_runner(model, dtype=dtype) as hf_model:
21 + hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
22 +
23 + with vllm_runner(model, dtype=dtype) as vllm_model:
24 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
25 +
26 + for i in range(len(example_prompts)):
27 + hf_output_ids, hf_output_str = hf_outputs[i]
28 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
29 + assert hf_output_str == vllm_output_str, (
30 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
31 + assert hf_output_ids == vllm_output_ids, (
32 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
33 +
34 +
35 +@pytest.mark.parametrize("model", MODELS)
36 +@pytest.mark.parametrize("dtype", ["float"])
37 +def test_state_cleanup(
38 + vllm_runner,
39 + model: str,
40 + dtype: str,
41 + example_prompts,
42 +) -> None:
43 + # This test is for verifying that the Jamba state is cleaned up between
44 + # steps, If its not cleaned, an error would be expected.
45 + try:
46 + with vllm_runner(model, dtype=dtype) as vllm_model:
47 + for _ in range(10):
48 + vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
49 + except ValueError:
50 + pytest.fail("Jamba inner state wasn't cleaned up between states, "
51 + "could be related to finished_requests_ids")
52 +
53 +
54 +@pytest.mark.parametrize("model", MODELS)
55 +@pytest.mark.parametrize("dtype", ["float"])
56 +def test_model_print(
57 + vllm_runner,
58 + model: str,
59 + dtype: str,
60 +) -> None:
61 + with vllm_runner(model, dtype=dtype) as vllm_model:
62 + # This test is for verifying whether the model's extra_repr
63 + # can be printed correctly.
64 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
65 + model_runner.model)
vllm/config.py
+28 -1
@@ -386,9 +386,36 @@ class ModelConfig:
386 return num_heads // parallel_config.tensor_parallel_size
387
388 def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
389 - total_num_hidden_layers = self.hf_text_config.num_hidden_layers
389 + total_num_hidden_layers = getattr(self.hf_text_config,
390 + "num_hidden_layers", 0)
391 return total_num_hidden_layers // parallel_config.pipeline_parallel_size
392
393 + def contains_seqlen_agnostic_layers(
394 + self, parallel_config: "ParallelConfig") -> bool:
395 + """True for Mamba/SSM models (Jamba)"""
396 + return self._get_num_seqlen_agnostic_layers(parallel_config) > 0
397 +
398 + def get_layers_block_type(self,
399 + parallel_config: "ParallelConfig") -> List[str]:
400 + num_layers = self.get_num_layers(parallel_config)
401 + # Transformers supports layers_block_type @property
402 + return getattr(self.hf_config, "layers_block_type",
403 + ["attention"] * num_layers)
404 +
405 + def get_num_attention_layers(self,
406 + parallel_config: "ParallelConfig") -> int:
407 + return len([
408 + t for t in self.get_layers_block_type(parallel_config)
409 + if t == "attention"
410 + ])
411 +
412 + def _get_num_seqlen_agnostic_layers(
413 + self, parallel_config: "ParallelConfig") -> int:
414 + return len([
415 + t for t in self.get_layers_block_type(parallel_config)
416 + if t != "attention"
417 + ])
418 +
419
420 class CacheConfig:
421 """Configuration for the KV cache.
vllm/core/scheduler.py
+15 -1
@@ -299,7 +299,10 @@ class Scheduler:
299 # Sequence groups in the SWAPPED state.
300 # Contain decode requests that are swapped out.
301 self.swapped: Deque[SequenceGroup] = deque()
302 -
302 + # Sequence groups finished requests ids since last step iteration.
303 + # It lets the model know that any state associated with these requests
304 + # can and must be released after the current step.
305 + self._finished_requests_ids: List[str] = list()
306 # Time at previous scheduling step
307 self.prev_time = 0.0
308 # Did we schedule a prompt at previous step?
@@ -373,6 +376,12 @@ class Scheduler:
376 def get_num_unfinished_seq_groups(self) -> int:
377 return len(self.waiting) + len(self.running) + len(self.swapped)
378
379 + def get_and_reset_finished_requests_ids(self) -> List[str]:
380 + """Flushes the list of request ids of previously finished seq_groups."""
381 + finished_requests_ids = self._finished_requests_ids
382 + self._finished_requests_ids = list()
383 + return finished_requests_ids
384 +
385 def _schedule_running(
386 self,
387 running_queue: deque,
@@ -1036,6 +1045,11 @@ class Scheduler:
1045 self.block_manager.free(seq)
1046
1047 def free_finished_seq_groups(self) -> None:
1048 + for queue in [self.running, self.swapped, self.waiting]:
1049 + self._finished_requests_ids += [
1050 + seq_group.request_id for seq_group in queue
1051 + if seq_group.is_finished()
1052 + ]
1053 self.running = deque(seq_group for seq_group in self.running
1054 if not seq_group.is_finished())
1055
vllm/engine/async_llm_engine.py
+3 -1
@@ -224,6 +224,8 @@ class _AsyncLLMEngine(LLMEngine):
224 """
225 seq_group_metadata_list, scheduler_outputs = self.scheduler[
226 virtual_engine].schedule()
227 + finished_requests_ids = self.scheduler[
228 + virtual_engine].get_and_reset_finished_requests_ids()
229
230 if not scheduler_outputs.is_empty():
231 # Execute the model.
@@ -235,7 +237,7 @@ class _AsyncLLMEngine(LLMEngine):
237 virtual_engine=virtual_engine,
238 num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
239 running_queue_size=scheduler_outputs.running_queue_size,
238 - )
240 + finished_requests_ids=finished_requests_ids)
241 output = await self.model_executor.execute_model_async(
242 execute_model_req)
243 else:
vllm/engine/llm_engine.py
+3 -1
@@ -846,6 +846,8 @@ class LLMEngine:
846 "as performance will be severely degraded otherwise.")
847 seq_group_metadata_list, scheduler_outputs = self.scheduler[
848 0].schedule()
849 + finished_requests_ids = self.scheduler[
850 + 0].get_and_reset_finished_requests_ids()
851
852 if not scheduler_outputs.is_empty():
853 execute_model_req = ExecuteModelRequest(
@@ -855,7 +857,7 @@ class LLMEngine:
857 blocks_to_copy=scheduler_outputs.blocks_to_copy,
858 num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
859 running_queue_size=scheduler_outputs.running_queue_size,
858 - )
860 + finished_requests_ids=finished_requests_ids)
861 output = self.model_executor.execute_model(
862 execute_model_req=execute_model_req)
863 else:
vllm/model_executor/models/__init__.py
+1
@@ -63,6 +63,7 @@ _GENERATION_MODELS = {
63 "XverseForCausalLM": ("xverse", "XverseForCausalLM"),
64 "Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
65 "MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
66 + "JambaForCausalLM": ("jamba", "JambaForCausalLM")
67 }
68
69 _EMBEDDING_MODELS = {
vllm/model_executor/models/jamba.py new
+955
@@ -0,0 +1,955 @@
1 +# coding=utf-8
2 +"""Inference-only Jurassic model."""
3 +from dataclasses import dataclass
4 +from typing import Dict, Iterable, List, Optional, Tuple
5 +
6 +import torch
7 +from causal_conv1d import causal_conv1d_fn, causal_conv1d_update
8 +from mamba_ssm.ops.selective_scan_interface import selective_scan_fn
9 +from mamba_ssm.ops.triton.selective_state_update import selective_state_update
10 +from torch import nn
11 +from torch.nn.parameter import Parameter
12 +from transformers import JambaConfig
13 +
14 +from vllm.attention.backends.abstract import AttentionMetadata
15 +from vllm.attention.layer import Attention
16 +from vllm.config import CacheConfig, LoRAConfig
17 +from vllm.distributed import (get_tensor_model_parallel_rank,
18 + get_tensor_model_parallel_world_size,
19 + tensor_model_parallel_all_reduce)
20 +from vllm.model_executor.layers.activation import SiluAndMul
21 +from vllm.model_executor.layers.fused_moe import fused_moe
22 +from vllm.model_executor.layers.layernorm import RMSNorm
23 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24 + MergedColumnParallelLinear,
25 + QKVParallelLinear,
26 + ReplicatedLinear,
27 + RowParallelLinear)
28 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
29 +from vllm.model_executor.layers.quantization.base_config import (
30 + QuantizationConfig)
31 +from vllm.model_executor.layers.sampler import Sampler
32 +from vllm.model_executor.layers.vocab_parallel_embedding import (
33 + DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
34 +from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35 +from vllm.model_executor.sampling_metadata import SamplingMetadata
36 +from vllm.model_executor.utils import set_weight_attrs
37 +from vllm.sequence import IntermediateTensors, SamplerOutput
38 +from vllm.worker.model_runner import _BATCH_SIZES_TO_CAPTURE
39 +
40 +KVCache = Tuple[torch.Tensor, torch.Tensor]
41 +
42 +
43 +@dataclass
44 +class MambaCacheParams:
45 + is_prompt: bool = False
46 + conv_state: torch.Tensor = torch.Tensor()
47 + ssm_state: torch.Tensor = torch.Tensor()
48 +
49 +
50 +# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
51 +class JambaMambaMixer(nn.Module):
52 + """
53 + Compute ∆, A, B, C, and D the state space parameters and compute
54 + the `contextualized_states`. A, D are input independent
55 + (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
56 + for why A isn't selective) ∆, B, C are input-dependent
57 + (this is a key difference between Mamba and the linear time
58 + invariant S4, and is why Mamba is called
59 + **selective** state spaces)
60 + """
61 +
62 + def __init__(self, config: JambaConfig, layer_idx):
63 + super().__init__()
64 + self.config = config
65 + self.layer_idx = layer_idx
66 + self.hidden_size = config.hidden_size
67 + self.ssm_state_size = config.mamba_d_state
68 + self.conv_kernel_size = config.mamba_d_conv
69 + self.intermediate_size = config.mamba_expand * config.hidden_size
70 + self.time_step_rank = config.mamba_dt_rank
71 + self.use_conv_bias = config.mamba_conv_bias
72 + self.use_bias = config.mamba_proj_bias
73 + self.conv1d = ColumnParallelLinear(
74 + input_size=self.conv_kernel_size,
75 + output_size=self.intermediate_size,
76 + bias=self.use_conv_bias,
77 + )
78 + # unsqueeze to fit conv1d weights shape into the linear weights shape.
79 + # Can't do this in `weight_loader` since it already exists in
80 + # `ColumnParallelLinear` and `set_weight_attrs`
81 + # doesn't allow to override it
82 + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
83 +
84 + self.in_proj = MergedColumnParallelLinear(self.hidden_size,
85 + [self.intermediate_size] * 2,
86 + bias=self.use_bias)
87 + # selective projection used to make dt, B and C input dependent
88 + self.x_proj = RowParallelLinear(
89 + self.intermediate_size,
90 + self.time_step_rank + self.ssm_state_size * 2,
91 + bias=False,
92 + )
93 + # time step projection (discretization) -
94 + # In the forward we need to apply dt_proj without the bias,
95 + # as the bias is added in the selective scan kernel.
96 + self.dt_proj = ColumnParallelLinear(self.time_step_rank,
97 + self.intermediate_size,
98 + bias=True,
99 + skip_bias_add=True)
100 +
101 + def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
102 + tp_rank = get_tensor_model_parallel_rank()
103 + tp_size = get_tensor_model_parallel_world_size()
104 + param.data.copy_(
105 + loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
106 + dim=0)[tp_rank])
107 +
108 + def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
109 + weight_loader(param, -torch.exp(loaded_weight.float()))
110 +
111 + tp_size = get_tensor_model_parallel_world_size()
112 + self.A = nn.Parameter(
113 + torch.empty(
114 + self.intermediate_size // tp_size,
115 + self.ssm_state_size,
116 + dtype=torch.float32,
117 + ))
118 + self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
119 +
120 + set_weight_attrs(self.D, {"weight_loader": weight_loader})
121 + set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
122 +
123 + self.out_proj = RowParallelLinear(
124 + self.intermediate_size,
125 + self.hidden_size,
126 + bias=self.use_bias,
127 + input_is_parallel=True,
128 + )
129 + self.activation = config.hidden_act
130 +
131 + self.dt_layernorm = RMSNorm(self.time_step_rank,
132 + eps=config.rms_norm_eps)
133 + self.b_layernorm = RMSNorm(self.ssm_state_size,
134 + eps=config.rms_norm_eps)
135 + self.c_layernorm = RMSNorm(self.ssm_state_size,
136 + eps=config.rms_norm_eps)
137 +
138 + def mamba_forward(self,
139 + hidden_states: torch.Tensor,
140 + cache_params: MambaCacheParams = None):
141 + # 1. Gated MLP's linear projection
142 + projected_states = self.in_proj(hidden_states)[0].transpose(1, 2)
143 + hidden_states, gate = projected_states.chunk(2, dim=1)
144 +
145 + # 2. Convolution sequence transformation
146 + conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
147 + self.conv1d.weight.size(2))
148 + if cache_params is not None and not cache_params.is_prompt:
149 + hidden_states = causal_conv1d_update(
150 + hidden_states.squeeze(-1),
151 + cache_params.conv_state,
152 + conv_weights,
153 + self.conv1d.bias,
154 + self.activation,
155 + )
156 + hidden_states = hidden_states.unsqueeze(-1)
157 + else:
158 + if cache_params is not None:
159 + conv_states = nn.functional.pad(
160 + hidden_states,
161 + (self.conv_kernel_size - hidden_states.shape[-1], 0))
162 + cache_params.conv_state.copy_(conv_states)
163 +
164 + hidden_states = causal_conv1d_fn(
165 + hidden_states,
166 + conv_weights,
167 + self.conv1d.bias,
168 + activation=self.activation,
169 + )
170 +
171 + # 3. State Space Model sequence transformation
172 + # 3.a. input varying initialization of time_step, B and C
173 + ssm_parameters = self.x_proj(hidden_states.transpose(1, 2))[0]
174 +
175 + time_step, B, C = torch.split(
176 + ssm_parameters,
177 + [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
178 + dim=-1,
179 + )
180 + time_step = self.dt_layernorm(time_step.contiguous())
181 + B = self.b_layernorm(B.contiguous())
182 + C = self.c_layernorm(C.contiguous())
183 +
184 + discrete_time_step = self.dt_proj(time_step)[0].transpose(1, 2)
185 + # 3.c perform the recurrence y ← SSM(A, B, C)(x)
186 + time_proj_bias = (self.dt_proj.bias.float() if hasattr(
187 + self.dt_proj, "bias") else None)
188 + if cache_params is not None and not cache_params.is_prompt:
189 + scan_outputs = selective_state_update(
190 + cache_params.ssm_state,
191 + hidden_states[..., 0],
192 + discrete_time_step[..., 0],
193 + self.A,
194 + B[:, 0],
195 + C[:, 0],
196 + self.D,
197 + gate[..., 0],
198 + time_proj_bias,
199 + dt_softplus=True,
200 + ).unsqueeze(-1)
201 + else:
202 + scan_outputs, ssm_state = selective_scan_fn(
203 + hidden_states,
204 + discrete_time_step,
205 + self.A,
206 + B.transpose(1, 2),
207 + C.transpose(1, 2),
208 + self.D.float(),
209 + gate,
210 + time_proj_bias,
211 + delta_softplus=True,
212 + return_last_state=True,
213 + )
214 + if ssm_state is not None and cache_params is not None:
215 + cache_params.ssm_state.copy_(ssm_state)
216 +
217 + # 4. Final linear projection
218 + contextualized_states = self.out_proj(scan_outputs.transpose(1, 2))[0]
219 + return contextualized_states
220 +
221 + def forward(
222 + self,
223 + hidden_states: torch.Tensor,
224 + attn_metadata: AttentionMetadata,
225 + conv_state: torch.Tensor,
226 + ssm_state: torch.Tensor,
227 + ):
228 + if attn_metadata.prefill_metadata is not None:
229 + offset = 0
230 + for i, prompt_len in enumerate(
231 + attn_metadata.prefill_metadata.seq_lens):
232 + cache = MambaCacheParams(True,
233 + conv_state=conv_state[i].unsqueeze(0),
234 + ssm_state=ssm_state[i].unsqueeze(0))
235 + hidden_states[offset:offset + prompt_len].copy_(
236 + self.mamba_forward(hidden_states[offset:offset +
237 + prompt_len].unsqueeze(0),
238 + cache_params=cache)[0])
239 + offset += prompt_len
240 + else:
241 + cache = MambaCacheParams(False,
242 + conv_state=conv_state,
243 + ssm_state=ssm_state)
244 + hidden_states = self.mamba_forward(hidden_states.unsqueeze(1),
245 + cache_params=cache)
246 + hidden_states = hidden_states.squeeze(1)
247 +
248 + return hidden_states
249 +
250 +
251 +class JambaMLP(nn.Module):
252 +
253 + def __init__(
254 + self,
255 + config: JambaConfig,
256 + quant_config: Optional[QuantizationConfig] = None,
257 + ) -> None:
258 + super().__init__()
259 + hidden_size = config.hidden_size
260 + intermediate_size = config.intermediate_size
261 + hidden_act = config.hidden_act
262 + self.gate_up_proj = MergedColumnParallelLinear(
263 + hidden_size, [intermediate_size] * 2,
264 + bias=False,
265 + quant_config=quant_config)
266 + self.down_proj = RowParallelLinear(intermediate_size,
267 + hidden_size,
268 + bias=False,
269 + quant_config=quant_config)
270 + if hidden_act != "silu":
271 + raise ValueError(f"Unsupported activation: {hidden_act}. "
272 + "Only silu is supported for now.")
273 + self.act_fn = SiluAndMul()
274 +
275 + def forward(self, x):
276 + gate_up, _ = self.gate_up_proj(x)
277 + x = self.act_fn(gate_up)
278 + x, _ = self.down_proj(x)
279 + return x
280 +
281 +
282 +class JambaMoE(nn.Module):
283 + """A tensor-parallel MoE implementation for Mixtral that shards each expert
284 + across all ranks.
285 +
286 + Each expert's weights are sharded across all ranks and a fused MoE
287 + kernel is used for the forward pass, and finally we reduce the outputs
288 + across ranks.
289 + """
290 +
291 + def __init__(
292 + self,
293 + config: JambaConfig,
294 + params_dtype: Optional[torch.dtype] = None,
295 + tp_size: Optional[int] = None,
296 + quant_config: Optional[QuantizationConfig] = None,
297 + ):
298 + super().__init__()
299 + self.tp_size = tp_size or get_tensor_model_parallel_world_size()
300 + self.num_total_experts = config.num_experts
301 + self.top_k = config.num_experts_per_tok
302 + self.hidden_size = config.hidden_size
303 + self.intermediate_size = config.intermediate_size // self.tp_size
304 +
305 + if params_dtype is None:
306 + params_dtype = torch.get_default_dtype()
307 + self.params_dtype = params_dtype
308 +
309 + self.router = ReplicatedLinear(self.hidden_size,
310 + self.num_total_experts,
311 + bias=False,
312 + params_dtype=self.params_dtype)
313 +
314 + self.ws = nn.Parameter(
315 + torch.empty(
316 + self.num_total_experts,
317 + 2 * self.intermediate_size,
318 + self.hidden_size,
319 + device="cuda",
320 + dtype=self.params_dtype,
321 + ))
322 + self.w2s = nn.Parameter(
323 + torch.empty(
324 + self.num_total_experts,
325 + self.hidden_size,
326 + self.intermediate_size,
327 + device="cuda",
328 + dtype=self.params_dtype,
329 + ))
330 +
331 + set_weight_attrs(
332 + self.ws,
333 + {
334 + "weight_loader": self.weight_loader,
335 + },
336 + )
337 + set_weight_attrs(
338 + self.w2s,
339 + {
340 + "weight_loader": self.weight_loader,
341 + },
342 + )
343 +
344 + def weight_loader(
345 + self,
346 + param: nn.Parameter,
347 + loaded_weight: torch.Tensor,
348 + weight_name: str,
349 + expert_id: int,
350 + ):
351 + tp_rank = get_tensor_model_parallel_rank()
352 + param_data = param.data
353 + shard_size = self.intermediate_size
354 + shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
355 + if weight_name.endswith("gate_proj.weight"):
356 + param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
357 + if weight_name.endswith("up_proj.weight"):
358 + param_data[expert_id,
359 + shard_size:2 * shard_size, :] = loaded_weight[shard, :]
360 + if weight_name.endswith("down_proj.weight"):
361 + param_data[expert_id, :, :] = loaded_weight[:, shard]
362 +
363 + def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
364 + num_tokens, hidden_size = hidden_states.shape
365 + hidden_states = hidden_states.view(-1, self.hidden_size)
366 + # router_logits: (batch * sequence_length, n_experts)
367 + router_logits, _ = self.router(hidden_states)
368 +
369 + final_hidden_states = fused_moe(
370 + hidden_states,
371 + self.ws,
372 + self.w2s,
373 + router_logits,
374 + self.top_k,
375 + renormalize=
376 + False, # Mixtral normalize the expert probs to 1. We don't!
377 + inplace=True,
378 + )
379 +
380 + if self.tp_size > 1:
381 + final_hidden_states = tensor_model_parallel_all_reduce(
382 + final_hidden_states)
383 +
384 + return final_hidden_states.view(num_tokens, hidden_size)
385 +
386 +
387 +class JambaMambaDecoderLayer(nn.Module):
388 +
389 + def __init__(self,
390 + config: JambaConfig,
391 + layer_idx: int,
392 + cache_config: Optional[CacheConfig] = None,
393 + quant_config: Optional[QuantizationConfig] = None) -> None:
394 + super().__init__()
395 + self.layer_idx = layer_idx
396 + self.config = config
397 + self.mamba = JambaMambaMixer(config, layer_idx)
398 +
399 + num_experts = config.layers_num_experts[layer_idx]
400 + ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
401 + self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
402 + self.input_layernorm = RMSNorm(config.hidden_size,
403 + eps=config.rms_norm_eps)
404 + self.pre_ff_layernorm = RMSNorm(config.hidden_size,
405 + eps=config.rms_norm_eps)
406 +
407 + def forward(
408 + self,
409 + hidden_states: torch.Tensor,
410 + attn_metadata: AttentionMetadata,
411 + residual: Optional[torch.Tensor],
412 + conv_state: torch.Tensor,
413 + ssm_state: torch.Tensor,
414 + **kwargs,
415 + ):
416 + if residual is None:
417 + residual = hidden_states
418 + hidden_states = self.input_layernorm(hidden_states)
419 + else:
420 + hidden_states, residual = self.input_layernorm(
421 + hidden_states, residual)
422 +
423 + hidden_states = self.mamba(hidden_states, attn_metadata, conv_state,
424 + ssm_state)
425 + # Fully Connected
426 + hidden_states, residual = self.pre_ff_layernorm(
427 + hidden_states, residual)
428 + hidden_states = self.feed_forward(hidden_states)
429 + return hidden_states, residual
430 +
431 +
432 +class JambaAttentionDecoderLayer(nn.Module):
433 +
434 + def __init__(
435 + self,
436 + config: JambaConfig,
437 + layer_idx: int,
438 + cache_config: Optional[CacheConfig] = None,
439 + quant_config: Optional[QuantizationConfig] = None,
440 + ) -> None:
441 + super().__init__()
442 + self.hidden_size = config.hidden_size
443 + tp_size = get_tensor_model_parallel_world_size()
444 + self.total_num_heads = config.num_attention_heads
445 + assert self.total_num_heads % tp_size == 0
446 + self.num_heads = self.total_num_heads // tp_size
447 + self.total_num_kv_heads = config.num_key_value_heads
448 + if self.total_num_kv_heads >= tp_size:
449 + # Number of KV heads is greater than TP size, so we partition
450 + # the KV heads across multiple tensor parallel GPUs.
451 + assert self.total_num_kv_heads % tp_size == 0
452 + else:
453 + # Number of KV heads is less than TP size, so we replicate
454 + # the KV heads across multiple tensor parallel GPUs.
455 + assert tp_size % self.total_num_kv_heads == 0
456 + self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
457 + self.head_dim = config.hidden_size // self.total_num_heads
458 + self.q_size = self.num_heads * self.head_dim
459 + self.kv_size = self.num_kv_heads * self.head_dim
460 + self.scaling = self.head_dim**-0.5
461 +
462 + self.qkv_proj = QKVParallelLinear(
463 + config.hidden_size,
464 + self.head_dim,
465 + self.total_num_heads,
466 + self.total_num_kv_heads,
467 + bias=False,
468 + quant_config=quant_config,
469 + )
470 + self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
471 + config.hidden_size,
472 + bias=False,
473 + quant_config=quant_config)
474 +
475 + self.attn = Attention(
476 + self.num_heads,
477 + self.head_dim,
478 + self.scaling,
479 + num_kv_heads=self.num_kv_heads,
480 + cache_config=cache_config,
481 + )
482 +
483 + num_experts = config.layers_num_experts[layer_idx]
484 + ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
485 + self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
486 + self.input_layernorm = RMSNorm(config.hidden_size,
487 + eps=config.rms_norm_eps)
488 + self.pre_ff_layernorm = RMSNorm(config.hidden_size,
489 + eps=config.rms_norm_eps)
490 +
491 + def self_attention(
492 + self,
493 + positions: torch.Tensor,
494 + hidden_states: torch.Tensor,
495 + kv_cache: torch.Tensor,
496 + attn_metadata: AttentionMetadata,
497 + **kwargs,
498 + ) -> torch.Tensor:
499 + qkv, _ = self.qkv_proj(hidden_states)
500 + q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
501 + attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
502 + output, _ = self.o_proj(attn_output)
503 + return output
504 +
505 + def forward(
506 + self,
507 + positions: torch.Tensor,
508 + hidden_states: torch.Tensor,
509 + kv_cache: torch.Tensor,
510 + attn_metadata: AttentionMetadata,
511 + residual: Optional[torch.Tensor],
512 + **kwargs,
513 + ):
514 + if residual is None:
515 + residual = hidden_states
516 + hidden_states = self.input_layernorm(hidden_states)
517 + else:
518 + hidden_states, residual = self.input_layernorm(
519 + hidden_states, residual)
520 +
521 + hidden_states = self.self_attention(
522 + positions=positions,
523 + hidden_states=hidden_states,
524 + kv_cache=kv_cache,
525 + attn_metadata=attn_metadata,
526 + )
527 + # Fully Connected
528 + hidden_states, residual = self.pre_ff_layernorm(
529 + hidden_states, residual)
530 + hidden_states = self.feed_forward(hidden_states)
531 + return hidden_states, residual
532 +
533 +
534 +ALL_DECODER_LAYER_TYPES = {
535 + "attention": JambaAttentionDecoderLayer,
536 + "mamba": JambaMambaDecoderLayer
537 +}
538 +
539 +
540 +class JambaModel(nn.Module):
541 +
542 + def __init__(
543 + self,
544 + config: JambaConfig,
545 + quant_config: Optional[QuantizationConfig] = None,
546 + cache_config: Optional[CacheConfig] = None,
547 + lora_config: Optional[LoRAConfig] = None,
548 + ) -> None:
549 + super().__init__()
550 + self.config = config
551 + self.padding_idx = config.pad_token_id
552 + lora_vocab = ((lora_config.lora_extra_vocab_size *
553 + (lora_config.max_loras or 1)) if lora_config else 0)
554 + self.vocab_size = config.vocab_size + lora_vocab
555 + self.org_vocab_size = config.vocab_size
556 +
557 + self.embed_tokens = VocabParallelEmbedding(
558 + self.vocab_size,
559 + config.hidden_size,
560 + org_num_embeddings=config.vocab_size,
561 + )
562 +
563 + decoder_layers = []
564 + for i in range(config.num_hidden_layers):
565 + layer_class = ALL_DECODER_LAYER_TYPES[config.layers_block_type[i]]
566 + decoder_layers.append(
567 + layer_class(config,
568 + layer_idx=i,
569 + cache_config=cache_config,
570 + quant_config=quant_config))
571 + self.layers = nn.ModuleList(decoder_layers)
572 + self.final_layernorm = RMSNorm(config.hidden_size,
573 + eps=config.rms_norm_eps)
574 +
575 + def forward(
576 + self,
577 + input_ids: torch.Tensor,
578 + positions: torch.Tensor,
579 + kv_caches: List[torch.Tensor],
580 + attn_metadata: AttentionMetadata,
581 + conv_state: torch.Tensor,
582 + ssm_state: torch.Tensor,
583 + ) -> torch.Tensor:
584 + hidden_states = self.embed_tokens(input_ids)
585 + residual = None
586 +
587 + for i in range(len(self.layers)):
588 + layer = self.layers[i]
589 + kv_cache = None
590 + current_ssm_state = None
591 + current_conv_state = None
592 + if isinstance(layer, JambaAttentionDecoderLayer):
593 + kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
594 + self.config.attn_layer_period]
595 + if isinstance(layer, JambaMambaDecoderLayer):
596 + current_state_layer = i - (1 +
597 + (i - self.config.attn_layer_offset)
598 + // self.config.attn_layer_period)
599 + current_ssm_state = ssm_state[current_state_layer]
600 + current_conv_state = conv_state[current_state_layer]
601 +
602 + hidden_states, residual = layer(
603 + positions=positions,
604 + hidden_states=hidden_states,
605 + kv_cache=kv_cache,
606 + attn_metadata=attn_metadata,
607 + residual=residual,
608 + conv_state=current_conv_state,
609 + ssm_state=current_ssm_state,
610 + )
611 + hidden_states, _ = self.final_layernorm(hidden_states, residual)
612 + return hidden_states
613 +
614 +
615 +class JambaForCausalLM(nn.Module):
616 + packed_modules_mapping = {
617 + "qkv_proj": [
618 + "q_proj",
619 + "k_proj",
620 + "v_proj",
621 + ],
622 + }
623 +
624 + # LoRA specific attributes
625 + supported_lora_modules = [
626 + "qkv_proj",
627 + "o_proj",
628 + "embed_tokens",
629 + "lm_head",
630 + ]
631 + embedding_modules = {
632 + "embed_tokens": "input_embeddings",
633 + "lm_head": "output_embeddings",
634 + }
635 + embedding_padding_modules = ["lm_head"]
636 +
637 + def __init__(
638 + self,
639 + config: JambaConfig,
640 + cache_config: Optional[CacheConfig] = None,
641 + quant_config: Optional[QuantizationConfig] = None,
642 + lora_config: Optional[LoRAConfig] = None,
643 + ) -> None:
644 + super().__init__()
645 + self.config = config
646 + self.model = JambaModel(config,
647 + cache_config=cache_config,
648 + quant_config=quant_config,
649 + lora_config=lora_config)
650 + self.unpadded_vocab_size = config.vocab_size
651 + if lora_config:
652 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
653 + self.lm_head = ParallelLMHead(
654 + self.unpadded_vocab_size,
655 + config.hidden_size,
656 + org_num_embeddings=config.vocab_size,
657 + padding_size=DEFAULT_VOCAB_PADDING_SIZE
658 + # We need bigger padding if using lora for kernel
659 + # compatibility
660 + if not lora_config else lora_config.lora_vocab_padding_size,
661 + )
662 + # Current step used indices
663 + self.current_indices: List[int] = []
664 + # Used to track and store by the Mamba cache between steps.
665 + self.mamba_cache: Tuple[torch.Tensor, torch.Tensor] = tuple()
666 + # Used as an input_buffer for the CUDA graph runs.
667 + self.mamba_gc_cache_buffer: Tuple[torch.Tensor, torch.Tensor] = tuple()
668 + # Maps between the request id and a dict that maps between the seq_id
669 + # and its index inside the self.mamba_cache
670 + self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
671 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
672 + config.vocab_size)
673 + self.sampler = Sampler()
674 +
675 + def forward(self,
676 + input_ids: torch.Tensor,
677 + positions: torch.Tensor,
678 + kv_caches: List[KVCache],
679 + attn_metadata: AttentionMetadata,
680 + intermediate_tensors: Optional[IntermediateTensors] = None,
681 + **kwargs):
682 + if not self.mamba_cache:
683 + self._prepare_mamba_cache()
684 +
685 + if "seqlen_agnostic_capture_inputs" not in kwargs:
686 + # We get here only on Prefill/Eager mode runs
687 + assert all(
688 + key in kwargs
689 + for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
690 +
691 + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
692 + batch_size = input_ids.shape[0]
693 + if attn_metadata.prefill_metadata:
694 + batch_size = len(request_ids_to_seq_ids)
695 + (
696 + current_seqlen_agnostic_cache,
697 + indices,
698 + ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
699 + batch_size)
700 + finished_requests_ids = kwargs["finished_requests_ids"]
701 + self._release_mamba_cache(finished_requests_ids)
702 + else:
703 + # CUDA graph capturing runs
704 + current_seqlen_agnostic_cache, indices = (
705 + kwargs["seqlen_agnostic_capture_inputs"],
706 + [],
707 + )
708 + self.current_indices = indices
709 +
710 + hidden_states = self.model(input_ids, positions, kv_caches,
711 + attn_metadata,
712 + current_seqlen_agnostic_cache[0],
713 + current_seqlen_agnostic_cache[1])
714 +
715 + if "seqlen_agnostic_capture_inputs" not in kwargs:
716 + self._copy_mamba_cache_by_indices(self.current_indices,
717 + current_seqlen_agnostic_cache)
718 +
719 + return hidden_states
720 +
721 + def _copy_mamba_cache_by_indices(
722 + self, indices: List[int],
723 + current_seqlen_agnostic_cache: Tuple[torch.Tensor, torch.Tensor]):
724 + for i, offset in enumerate(indices):
725 + self._copy_mamba_cache(offset, i, current_seqlen_agnostic_cache)
726 +
727 + def _copy_mamba_cache(self, index_to: int, index_from: int,
728 + from_buffer: Tuple[torch.Tensor, torch.Tensor]):
729 + assert len(self.mamba_cache) > 0
730 + for (cache_t, from_buffer_t) in zip(self.mamba_cache, from_buffer):
731 + cache_t[:, index_to].copy_(from_buffer_t[:, index_from],
732 + non_blocking=True)
733 +
734 + def _assign_seq_id_to_mamba_cache(self, cur_rid: str,
735 + seqs_id: List[int]) -> List[int]:
736 + indices_for_current_run = []
737 + for seq_id in seqs_id:
738 + if cur_rid not in self.mamba_cache_indices_mapping:
739 + self.mamba_cache_indices_mapping[cur_rid] = {}
740 + first_free_index = self._first_free_index_in_mamba_cache()
741 + self.mamba_cache_indices_mapping[cur_rid][
742 + seq_id] = first_free_index
743 + index_for_current_run = first_free_index
744 + ## case of decoding n>1, copy prefill cache to decoding indices
745 + elif seq_id not in (seq_ids2indices :=
746 + self.mamba_cache_indices_mapping[cur_rid]):
747 + first_free_index = self._first_free_index_in_mamba_cache()
748 + index_exist = list(seq_ids2indices.values())[0]
749 + self._copy_mamba_cache(index_from=index_exist,
750 + index_to=first_free_index,
751 + from_buffer=self.mamba_cache)
752 + self.mamba_cache_indices_mapping[cur_rid][
753 + seq_id] = first_free_index
754 + index_for_current_run = first_free_index
755 + else:
756 + index_for_current_run = self.mamba_cache_indices_mapping[
757 + cur_rid][seq_id]
758 +
759 + indices_for_current_run.append(index_for_current_run)
760 + return indices_for_current_run
761 +
762 + def _prepare_current_run_mamba_cache(
763 + self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int
764 + ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], List[int]]:
765 + indices_for_current_run = []
766 + for request_id, seqs_id in request_ids_to_seq_ids.items():
767 + indices_for_current_run += self._assign_seq_id_to_mamba_cache(
768 + request_id, seqs_id)
769 + ## Pad the batch in case of running batch that was not captured via CG
770 + padded_indices = indices_for_current_run.copy()
771 + pad_index = self._first_free_index_in_mamba_cache()
772 +
773 + for _ in range(batch_size - len(indices_for_current_run)):
774 + padded_indices.append(pad_index)
775 +
776 + conv_state = self.mamba_cache[0][:, padded_indices]
777 + temporal_state = self.mamba_cache[1][:, padded_indices]
778 +
779 + return (conv_state, temporal_state), indices_for_current_run
780 +
781 + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
782 + """
783 + Copy the relevant Mamba cache into the CUDA graph input buffer
784 + that was provided during the capture runs
785 + (JambaForCausalLM.mamba_gc_cache_buffer).
786 + """
787 + assert all(
788 + key in kwargs
789 + for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
790 + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
791 + batch_size = len(request_ids_to_seq_ids)
792 + (
793 + current_mamba_cache,
794 + indices,
795 + ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796 + batch_size)
797 + self.current_indices = indices
798 + finished_requests_ids = kwargs["finished_requests_ids"]
799 + self._release_mamba_cache(finished_requests_ids)
800 +
801 + for input_buffer, current_cache_buffer in zip(
802 + input_buffers["seqlen_agnostic_capture_inputs"],
803 + current_mamba_cache):
804 + input_buffer.copy_(current_cache_buffer, non_blocking=True)
805 +
806 + def copy_outputs_after_cuda_graphs(self, input_buffers, **kwargs):
807 + """
808 + Copy the relevant Mamba cache from the CUDA graph input_buffers
809 + back to the JambaForCausalLM.mamba_cache after CUDA
810 + graph replay run is done.
811 + """
812 + self._copy_mamba_cache_by_indices(
813 + self.current_indices,
814 + input_buffers["seqlen_agnostic_capture_inputs"])
815 +
816 + def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
817 + """
818 + Provide the CUDA graph capture runs with a buffer in adjusted size.
819 + The buffer is used to maintain the Mamba Cache during the CUDA graph
820 + replay runs.
821 + """
822 + return tuple(buffer[:, :batch_size]
823 + for buffer in self.mamba_gc_cache_buffer)
824 +
825 + def _release_mamba_cache(self, finished_seq_groups_req_ids: List[str]):
826 + for req_id in finished_seq_groups_req_ids:
827 + if req_id in self.mamba_cache_indices_mapping:
828 + self.mamba_cache_indices_mapping.pop(req_id)
829 +
830 + def _first_free_index_in_mamba_cache(self) -> int:
831 + if self.mamba_cache:
832 + max_possible_batch_size = self.mamba_cache[0].shape[1]
833 + occupied = [
834 + id for seq_ids in self.mamba_cache_indices_mapping.values()
835 + for id in seq_ids.values()
836 + ]
837 + first_free_index = [
838 + i not in occupied for i in range(max_possible_batch_size)
839 + ].index(True)
840 + return first_free_index
841 + return 0
842 +
843 + def _get_mamba_cache_shape(
844 + self
845 + ) -> Tuple[Optional[Tuple[int, int]], Optional[Tuple[int, int]]]:
846 + world_size = get_tensor_model_parallel_world_size()
847 + hidden_size = self.config.hidden_size
848 + conv_state_shape = (
849 + self.config.mamba_expand * hidden_size // world_size,
850 + self.config.mamba_d_conv,
851 + )
852 + temporal_state_shape = (
853 + self.config.mamba_expand * self.config.hidden_size // world_size,
854 + self.config.mamba_d_state,
855 + )
856 + return conv_state_shape, temporal_state_shape
857 +
858 + def _prepare_mamba_cache(self):
859 + dtype = self.lm_head.weight.dtype
860 + layers_type = self.config.layers_block_type
861 + mamba_layers = sum(
862 + [layer_type == "mamba" for layer_type in layers_type])
863 + max_batch_size = _BATCH_SIZES_TO_CAPTURE[-1] + 10
864 + conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
865 + assert conv_state_shape is not None and temporal_state_shape is not None
866 + for buffername in ["mamba_cache", "mamba_gc_cache_buffer"]:
867 + buffer = (torch.empty(size=(mamba_layers, max_batch_size) +
868 + conv_state_shape,
869 + dtype=dtype,
870 + device="cuda"),
871 + torch.empty(size=(mamba_layers, max_batch_size) +
872 + temporal_state_shape,
873 + dtype=dtype,
874 + device="cuda"))
875 + setattr(self, buffername, buffer)
876 +
877 + def compute_logits(self, hidden_states: torch.Tensor,
878 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
879 + logits = self.logits_processor(self.lm_head.weight, hidden_states,
880 + sampling_metadata)
881 + return logits
882 +
883 + def sample(
884 + self,
885 + logits: Optional[torch.Tensor],
886 + sampling_metadata: SamplingMetadata,
887 + ) -> Optional[SamplerOutput]:
888 + next_tokens = self.sampler(logits, sampling_metadata)
889 + return next_tokens
890 +
891 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
892 + stacked_params_mapping = [
893 + # (param_name, shard_name, shard_id)
894 + ("qkv_proj", "q_proj", "q"),
895 + ("qkv_proj", "k_proj", "k"),
896 + ("qkv_proj", "v_proj", "v"),
897 + ("gate_up_proj", "gate_proj", 0),
898 + ("gate_up_proj", "up_proj", 1),
899 + ]
900 +
901 + expert_params_mapping = [
902 + # (param_name, weight_name, expert_id)
903 + (
904 + "ws" if weight_name in ["gate_proj", "up_proj"] else "w2s",
905 + f"experts.{expert_id}.{weight_name}.weight",
906 + expert_id,
907 + ) for expert_id in range(self.config.num_experts)
908 + for weight_name in ["down_proj", "up_proj", "gate_proj"]
909 + ]
910 +
911 + params_dict = dict(self.named_parameters())
912 + for name, loaded_weight in weights:
913 + if "rotary_emb.inv_freq" in name:
914 + continue
915 +
916 + if "A_log" in name:
917 + name = name.replace("A_log", "A")
918 +
919 + if ".self_attn." in name:
920 + name = name.replace(".self_attn", "")
921 +
922 + for param_name, weight_name, shard_id in stacked_params_mapping:
923 + if weight_name not in name:
924 + continue
925 + if 'experts' in name:
926 + continue
927 + name = name.replace(weight_name, param_name)
928 + # Skip loading extra bias for GPTQ models.
929 + if name.endswith(".bias") and name not in params_dict:
930 + continue
931 + param = params_dict[name]
932 + weight_loader = param.weight_loader
933 + weight_loader(param, loaded_weight, shard_id)
934 + break
935 + else:
936 + for param_name, weight_name, expert_id in expert_params_mapping:
937 + if weight_name not in name:
938 + continue
939 + name = name.replace(weight_name, param_name)
940 + param = params_dict[name]
941 + weight_loader = param.weight_loader
942 + weight_loader(param,
943 + loaded_weight,
944 + weight_name,
945 + expert_id=expert_id)
946 + break
947 + else:
948 + # Skip loading extra bias for GPTQ models.
949 + if name.endswith(".bias") and name not in params_dict:
950 + continue
951 +
952 + param = params_dict[name]
953 + weight_loader = getattr(param, "weight_loader",
954 + default_weight_loader)
955 + weight_loader(param, loaded_weight)
vllm/sequence.py
+3 -1
@@ -934,6 +934,8 @@ class ExecuteModelRequest:
934 previous_hidden_states: Optional[HiddenStates] = None
935 # The number of forward steps to run.
936 num_steps: int = 1
937 + # Finished request ids since last step.
938 + finished_requests_ids: List[str] = field(default_factory=list)
939
940 def clone(
941 self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -949,4 +951,4 @@ class ExecuteModelRequest:
951 running_queue_size=self.running_queue_size,
952 previous_hidden_states=self.previous_hidden_states,
953 num_steps=self.num_steps,
952 - )
954 + finished_requests_ids=self.finished_requests_ids)
vllm/spec_decode/draft_model_runner.py
+8 -4
@@ -75,15 +75,19 @@ class TP1DraftModelRunner(ModelRunner):
75 List[SequenceGroupMetadata]] = None
76
77 def prepare_model_input(
78 - self,
79 - seq_group_metadata_list: List[SequenceGroupMetadata],
80 - virtual_engine: int = 0) -> ModelInputForGPUWithSamplingMetadata:
78 + self,
79 + seq_group_metadata_list: List[SequenceGroupMetadata],
80 + virtual_engine: int = 0,
81 + finished_requests_ids: Optional[List[str]] = None
82 + ) -> ModelInputForGPUWithSamplingMetadata:
83 """A temporary solution that caches the seq_group_metadata_list
84 for multi-step execution.
85 TODO: In-place update model_input and remove this function.
86 """
87 self.cached_seq_group_metadata_list = seq_group_metadata_list
86 - return super().prepare_model_input(seq_group_metadata_list)
88 + return super().prepare_model_input(
89 + seq_group_metadata_list,
90 + finished_requests_ids=finished_requests_ids)
91
92 def update_model_input(
93 self, model_input: ModelInputForGPUWithSamplingMetadata,
vllm/worker/cache_engine.py
+9 -6
@@ -33,7 +33,9 @@ class CacheEngine:
33 self.device_config = device_config
34
35 self.head_size = model_config.get_head_size()
36 - self.num_layers = model_config.get_num_layers(parallel_config)
36 + # Models like Jamba, have mixed typed layers, E.g Mamba
37 + self.num_attention_layers = model_config.get_num_attention_layers(
38 + parallel_config)
39 self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
40
41 self.block_size = cache_config.block_size
@@ -75,7 +77,7 @@ class CacheEngine:
77 num_blocks, self.block_size, self.num_kv_heads, self.head_size)
78 pin_memory = is_pin_memory_available() if device == "cpu" else False
79 kv_cache: List[torch.Tensor] = []
78 - for _ in range(self.num_layers):
80 + for _ in range(self.num_attention_layers):
81 # null block in CpuGpuBlockAllocator requires at least that
82 # block to be zeroed-out.
83 # We zero-out everything for simplicity.
@@ -87,12 +89,12 @@ class CacheEngine:
89 return kv_cache
90
91 def swap_in(self, src_to_dst: torch.Tensor) -> None:
90 - for i in range(self.num_layers):
92 + for i in range(self.num_attention_layers):
93 self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
94 src_to_dst)
95
96 def swap_out(self, src_to_dst: torch.Tensor) -> None:
95 - for i in range(self.num_layers):
97 + for i in range(self.num_attention_layers):
98 self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
99 src_to_dst)
100
@@ -107,11 +109,12 @@ class CacheEngine:
109 ) -> int:
110 head_size = model_config.get_head_size()
111 num_heads = model_config.get_num_kv_heads(parallel_config)
110 - num_layers = model_config.get_num_layers(parallel_config)
112 + num_attention_layers = model_config.get_num_attention_layers(
113 + parallel_config)
114
115 key_cache_block = cache_config.block_size * num_heads * head_size
116 value_cache_block = key_cache_block
114 - total = num_layers * (key_cache_block + value_cache_block)
117 + total = num_attention_layers * (key_cache_block + value_cache_block)
118 if cache_config.cache_dtype == "auto":
119 dtype = model_config.dtype
120 else:
vllm/worker/cpu_model_runner.py
+4 -3
@@ -314,9 +314,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
314 )
315
316 def prepare_model_input(
317 - self,
318 - seq_group_metadata_list: List[SequenceGroupMetadata],
319 - virtual_engine: int = 0,
317 + self,
318 + seq_group_metadata_list: List[SequenceGroupMetadata],
319 + virtual_engine: int = 0,
320 + finished_requests_ids: Optional[List[str]] = None
321 ) -> CPUModelInput:
322 multi_modal_kwargs = None
323 # NOTE: We assume that all sequences in the group are all prompts or
vllm/worker/embedding_model_runner.py
+2 -1
@@ -120,10 +120,11 @@ class EmbeddingModelRunner(
120 self,
121 seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
122 virtual_engine: int = 0,
123 + finished_requests_ids: Optional[List[str]] = None
124 ) -> ModelInputForGPUWithPoolingMetadata:
125 assert seq_group_metadata_list is not None
126 model_input = self._prepare_model_input_tensors(
126 - seq_group_metadata_list)
127 + seq_group_metadata_list, finished_requests_ids)
128 # Prepare PoolingMetadata.
129 assert model_input.seq_lens is not None
130 pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
vllm/worker/model_runner.py
+57 -10
@@ -84,6 +84,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
84 lora_requests: Optional[Set[LoRARequest]] = None
85 attn_metadata: Optional["AttentionMetadata"] = None
86 multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
87 + request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
88 + finished_requests_ids: Optional[List[str]] = None
89 virtual_engine: int = 0
90
91 def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
@@ -94,6 +96,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
96 "lora_mapping": self.lora_mapping,
97 "multi_modal_kwargs": self.multi_modal_kwargs,
98 "virtual_engine": self.virtual_engine,
99 + "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
100 + "finished_requests_ids": self.finished_requests_ids,
101 }
102 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
103 return tensor_dict
@@ -128,6 +132,8 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
132 "lora_mapping": self.lora_mapping,
133 "multi_modal_kwargs": self.multi_modal_kwargs,
134 "virtual_engine": self.virtual_engine,
135 + "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
136 + "finished_requests_ids": self.finished_requests_ids,
137 }
138 _add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
139 _add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -191,6 +197,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
197 ]
198 self.graph_memory_pool: Optional[Tuple[
199 int, int]] = None # Set during graph capture.
200 +
201 + self.has_seqlen_agnostic = model_config.contains_seqlen_agnostic_layers(
202 + parallel_config)
203 +
204 # When using CUDA graph, the input block tables must be padded to
205 # max_seq_len_to_capture. However, creating the block table in
206 # Python can be expensive. To optimize this, we cache the block table
@@ -317,6 +327,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
327 def _prepare_model_input_tensors(
328 self,
329 seq_group_metadata_list: List[SequenceGroupMetadata],
330 + finished_requests_ids: Optional[List[str]] = None
331 ) -> TModelInputForGPU:
332 """Helper method to prepare the model input based on a given sequence
333 group. Prepares metadata needed for the base model forward pass but not
@@ -347,6 +358,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
358 block_tables: List[List[int]] = []
359 multi_modal_kwargs_list: Dict[str,
360 List[torch.Tensor]] = defaultdict(list)
361 + request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
362 decode_only = True
363 num_prefills = 0
364 num_prefill_tokens = 0
@@ -738,7 +750,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
750 k: torch.cat(v, dim=0).to(self.device)
751 for k, v in multi_modal_kwargs_list.items()
752 }
741 -
753 + request_ids_to_seq_ids = {
754 + seq_group_metadata.request_id:
755 + list(seq_group_metadata.seq_data.keys())
756 + for seq_group_metadata in seq_group_metadata_list
757 + }
758 return self._model_input_cls(
759 input_tokens=input_tokens_tensor,
760 input_positions=input_positions_tensor,
@@ -748,7 +764,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
764 lora_mapping=lora_mapping,
765 lora_requests=lora_requests,
766 multi_modal_kwargs=multi_modal_kwargs,
751 - )
767 + request_ids_to_seq_ids=request_ids_to_seq_ids,
768 + finished_requests_ids=finished_requests_ids)
769
770 @torch.inference_mode()
771 def profile_run(self) -> None:
@@ -821,7 +838,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
838 # Run the model with the dummy inputs.
839 num_layers = self.model_config.get_num_layers(self.parallel_config)
840 kv_caches = [None] * num_layers
824 - model_input = self.prepare_model_input(seqs)
841 + finished_requests_ids = [seq.request_id for seq in seqs]
842 + model_input = self.prepare_model_input(
843 + seqs, finished_requests_ids=finished_requests_ids)
844 intermediate_tensors = None
845 if not get_pp_group().is_first_rank:
846 intermediate_tensors = self.model.make_empty_intermediate_tensors(
@@ -1033,21 +1052,37 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1052 graph_runner.flashinfer_decode_wrapper = \
1053 decode_wrapper
1054
1036 - graph_runner.capture(
1055 + capture_inputs = {
1056 + "input_ids":
1057 input_tokens[:batch_size],
1058 + "positions":
1059 input_positions[:batch_size],
1060 + "hidden_or_intermediate_states":
1061 hidden_or_intermediate_states[
1062 virtual_engine] # type: ignore
1063 [:batch_size]
1064 if hidden_or_intermediate_states[virtual_engine]
1065 is not None else None,
1066 + "intermediate_inputs":
1067 intermediate_inputs[:batch_size]
1068 if intermediate_inputs is not None else None,
1069 + "kv_caches":
1070 kv_caches[virtual_engine],
1071 + "attn_metadata":
1072 attn_metadata,
1048 - memory_pool=self.graph_memory_pool,
1049 - stream=graph_capture_context.stream,
1050 - )
1073 + "memory_pool":
1074 + self.graph_memory_pool,
1075 + "stream":
1076 + graph_capture_context.stream
1077 + }
1078 + if self.has_seqlen_agnostic:
1079 + # Only used by Mamba-based models CUDA graph atm (Jamba)
1080 + capture_inputs.update({
1081 + "seqlen_agnostic_capture_inputs":
1082 + self.model.get_seqlen_agnostic_capture_inputs(
1083 + batch_size)
1084 + })
1085 + graph_runner.capture(**capture_inputs)
1086 self.graph_memory_pool = graph_runner.graph.pool()
1087 self.graph_runners[virtual_engine][batch_size] = (
1088 graph_runner)
@@ -1084,6 +1119,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1119 self,
1120 seq_group_metadata_list: List[SequenceGroupMetadata],
1121 virtual_engine: int = 0,
1122 + finished_requests_ids: Optional[List[str]] = None
1123 ) -> ModelInputForGPUWithSamplingMetadata:
1124 """Prepare the model input based on a given sequence group, including
1125 metadata for the sampling step.
@@ -1099,7 +1135,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1135 If cuda graph is required, this API automatically pads inputs.
1136 """
1137 model_input = self._prepare_model_input_tensors(
1102 - seq_group_metadata_list)
1138 + seq_group_metadata_list, finished_requests_ids)
1139 sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
1140 model_input.seq_lens,
1141 model_input.query_lens,
@@ -1175,6 +1211,10 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1211 model_executable = self.model
1212
1213 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
1214 + seqlen_agnostic_kwargs = {
1215 + "finished_requests_ids": model_input.finished_requests_ids,
1216 + "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1217 + } if self.has_seqlen_agnostic else {}
1218 hidden_or_intermediate_states = model_executable(
1219 input_ids=model_input.input_tokens,
1220 positions=model_input.input_positions,
@@ -1182,7 +1222,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1222 attn_metadata=model_input.attn_metadata,
1223 intermediate_tensors=intermediate_tensors,
1224 **multi_modal_kwargs,
1185 - )
1225 + **seqlen_agnostic_kwargs)
1226
1227 # Compute the logits in the last pipeline stage.
1228 if not get_pp_group().is_last_rank:
@@ -1305,6 +1345,7 @@ class CUDAGraphRunner:
1345 "positions": positions,
1346 "kv_caches": kv_caches,
1347 "slot_mapping": attn_metadata.slot_mapping,
1348 + **kwargs,
1349 }
1350 else:
1351 self.input_buffers = {
@@ -1315,6 +1356,7 @@ class CUDAGraphRunner:
1356 "seq_lens_tensor":
1357 attn_metadata.decode_metadata.seq_lens_tensor,
1358 "block_tables": attn_metadata.decode_metadata.block_tables,
1359 + **kwargs,
1360 }
1361 if intermediate_inputs is not None:
1362 self.input_buffers.update(intermediate_inputs.tensors)
@@ -1349,13 +1391,18 @@ class CUDAGraphRunner:
1391 non_blocking=True)
1392 self.input_buffers["block_tables"].copy_(
1393 attn_metadata.decode_metadata.block_tables, non_blocking=True)
1394 + if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1395 + self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1396 + **kwargs)
1397 if intermediate_tensors is not None:
1398 for key in intermediate_tensors.tensors:
1399 self.input_buffers[key].copy_(intermediate_tensors[key],
1400 non_blocking=True)
1401 # Run the graph.
1402 self.graph.replay()
1358 -
1403 + if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1404 + self.model.copy_outputs_after_cuda_graphs(self.input_buffers,
1405 + **kwargs)
1406 # Return the output tensor.
1407 if get_pp_group().is_last_rank:
1408 return self.output_buffers["hidden_states"]
vllm/worker/model_runner_base.py
+1
@@ -139,6 +139,7 @@ class ModelRunnerBase(ABC, Generic[T]):
139 self,
140 seq_group_metadata_list: List[SequenceGroupMetadata],
141 virtual_engine: int = 0,
142 + finished_requests_ids: Optional[List[str]] = None,
143 ) -> T:
144 """
145 Prepare the inputs to ModelRunnerBase.execute_model from an execution
vllm/worker/neuron_model_runner.py
+1
@@ -177,6 +177,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
177 self,
178 seq_group_metadata_list: List[SequenceGroupMetadata],
179 virtual_engine: int = 0,
180 + finished_requests_ids: Optional[List[str]] = None
181 ) -> ModelInputForNeuron:
182 # NOTE: We assume that all sequences in the group are all prompts or
183 # all decodes.
vllm/worker/worker_base.py
+2 -1
@@ -234,7 +234,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
234 model_input: ModelRunnerInputBase = (
235 self.model_runner.prepare_model_input(
236 execute_model_req.seq_group_metadata_list,
237 - execute_model_req.virtual_engine))
237 + execute_model_req.virtual_engine,
238 + execute_model_req.finished_requests_ids))
239 num_steps = execute_model_req.num_steps
240
241 if self.do_metadata_broadcast:
vllm/worker/xpu_model_runner.py
+4 -3
@@ -189,9 +189,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
189 ))
190
191 def prepare_model_input(
192 - self,
193 - seq_group_metadata_list: List[SequenceGroupMetadata],
194 - virtual_engine: int = 0,
192 + self,
193 + seq_group_metadata_list: List[SequenceGroupMetadata],
194 + virtual_engine: int = 0,
195 + finished_requests_ids: Optional[List[str]] = None
196 ) -> ModelInputForXPU:
197 multi_modal_input = None
198 if self.is_driver_worker: