A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Model] Jamba support (#4115)
Signed-off-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai> Co-authored-by: Erez Schwartz <erezs@ai21.com> Co-authored-by: Mor Zusman <morz@ai21.com> Co-authored-by: tomeras91 <57313761+tomeras91@users.noreply.github.com> Co-authored-by: Tomer Asida <tomera@ai21.com> Co-authored-by: Zhuohan Li <zhuohan123@gmail.com> Co-authored-by: Muralidhar Andoorveedu <muralidhar.andoorveedu@centml.ai>
Mor Zusman committed
Jul 3, 2024 at 02:11 UTC
9d6a8daa87e2e0af3ff45d03d08ad5a94ec089a8
21 files changed
+1192
-34
.buildkite/run-cpu-test.sh
+1
-1
@@ -23,4 +23,4 @@ docker exec cpu-test-avx2 bash -c "python3 examples/offline_inference.py"
23
docker exec cpu-test bash -c "cd tests;
24
pip install pytest Pillow protobuf
25
cd ../
26
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py"
26
+ pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py" # Mamba on CPU is not supported
Dockerfile
+23
@@ -43,6 +43,10 @@ COPY requirements-cuda.txt requirements-cuda.txt
43
RUN --mount=type=cache,target=/root/.cache/pip \
44
python3 -m pip install -r requirements-cuda.txt
45
46
+COPY requirements-mamba.txt requirements-mamba.txt
47
+RUN python3 -m pip install packaging
48
+RUN python3 -m pip install -r requirements-mamba.txt
49
+
50
# cuda arch list used by torch
51
# can be useful for both `dev` and `test`
52
# explicitly set the list to avoid issues with torch 2.2
@@ -123,6 +127,21 @@ RUN --mount=type=cache,target=/root/.cache/pip \
127
python3 -m pip install -r requirements-dev.txt
128
129
#################### DEV IMAGE ####################
130
+#################### MAMBA Build IMAGE ####################
131
+FROM dev as mamba-builder
132
+# max jobs used for build
133
+ARG max_jobs=2
134
+ENV MAX_JOBS=${max_jobs}
135
+
136
+WORKDIR /usr/src/mamba
137
+
138
+COPY requirements-mamba.txt requirements-mamba.txt
139
+
140
+# Download the wheel or build it if a pre-compiled release doesn't exist
141
+RUN pip --verbose wheel -r requirements-mamba.txt \
142
+ --no-build-isolation --no-deps --no-cache-dir
143
+
144
+#################### MAMBA Build IMAGE ####################
145
146
#################### vLLM installation IMAGE ####################
147
# image with vLLM installed
@@ -143,6 +162,10 @@ RUN ldconfig /usr/local/cuda-$(echo $CUDA_VERSION | cut -d. -f1,2)/compat/
162
RUN --mount=type=bind,from=build,src=/workspace/dist,target=/vllm-workspace/dist \
163
--mount=type=cache,target=/root/.cache/pip \
164
python3 -m pip install dist/*.whl --verbose
165
+
166
+RUN --mount=type=bind,from=mamba-builder,src=/usr/src/mamba,target=/usr/src/mamba \
167
+ --mount=type=cache,target=/root/.cache/pip \
168
+ python3 -m pip install /usr/src/mamba/*.whl --no-cache-dir
169
#################### vLLM installation IMAGE ####################
170
171
docs/source/models/supported_models.rst
+4
@@ -87,6 +87,10 @@ Alongside each architecture, we include some popular models that use it.
87
- Jais
88
- :code:`core42/jais-13b`, :code:`core42/jais-13b-chat`, :code:`core42/jais-30b-v3`, :code:`core42/jais-30b-chat-v3`, etc.
89
-
90
+ * - :code:`JambaForCausalLM`
91
+ - Jamba
92
+ - :code:`ai21labs/Jamba-v0.1`, etc.
93
+ - ✅︎
94
* - :code:`LlamaForCausalLM`
95
- LLaMA, Llama 2, Meta Llama 3, Vicuna, Alpaca, Yi
96
- :code:`meta-llama/Meta-Llama-3-8B-Instruct`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-13b-hf`, :code:`meta-llama/Llama-2-70b-hf`, :code:`openlm-research/open_llama_13b`, :code:`lmsys/vicuna-13b-v1.3`, :code:`01-ai/Yi-6B`, :code:`01-ai/Yi-34B`, etc.
requirements-mamba.txt
new
+3
@@ -0,0 +1,3 @@
1
+# Mamba dependencies
2
+mamba-ssm>=1.2.2
3
+causal-conv1d>=1.2.0
tests/models/test_jamba.py
new
+65
@@ -0,0 +1,65 @@
1
+import pytest
2
+
3
+MODELS = ["ai21labs/Jamba-tiny-random"]
4
+
5
+
6
+@pytest.mark.parametrize("model", MODELS)
7
+@pytest.mark.parametrize("dtype", ["float"])
8
+@pytest.mark.parametrize("max_tokens", [20])
9
+def test_models(
10
+ hf_runner,
11
+ vllm_runner,
12
+ example_prompts,
13
+ model: str,
14
+ dtype: str,
15
+ max_tokens: int,
16
+) -> None:
17
+ # To pass the small model tests, we need full precision.
18
+ assert dtype == "float"
19
+
20
+ with hf_runner(model, dtype=dtype) as hf_model:
21
+ hf_outputs = hf_model.generate_greedy(example_prompts, max_tokens)
22
+
23
+ with vllm_runner(model, dtype=dtype) as vllm_model:
24
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
25
+
26
+ for i in range(len(example_prompts)):
27
+ hf_output_ids, hf_output_str = hf_outputs[i]
28
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
29
+ assert hf_output_str == vllm_output_str, (
30
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
31
+ assert hf_output_ids == vllm_output_ids, (
32
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
33
+
34
+
35
+@pytest.mark.parametrize("model", MODELS)
36
+@pytest.mark.parametrize("dtype", ["float"])
37
+def test_state_cleanup(
38
+ vllm_runner,
39
+ model: str,
40
+ dtype: str,
41
+ example_prompts,
42
+) -> None:
43
+ # This test is for verifying that the Jamba state is cleaned up between
44
+ # steps, If its not cleaned, an error would be expected.
45
+ try:
46
+ with vllm_runner(model, dtype=dtype) as vllm_model:
47
+ for _ in range(10):
48
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
49
+ except ValueError:
50
+ pytest.fail("Jamba inner state wasn't cleaned up between states, "
51
+ "could be related to finished_requests_ids")
52
+
53
+
54
+@pytest.mark.parametrize("model", MODELS)
55
+@pytest.mark.parametrize("dtype", ["float"])
56
+def test_model_print(
57
+ vllm_runner,
58
+ model: str,
59
+ dtype: str,
60
+) -> None:
61
+ with vllm_runner(model, dtype=dtype) as vllm_model:
62
+ # This test is for verifying whether the model's extra_repr
63
+ # can be printed correctly.
64
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
65
+ model_runner.model)
vllm/config.py
+28
-1
@@ -386,9 +386,36 @@ class ModelConfig:
386
return num_heads // parallel_config.tensor_parallel_size
387
388
def get_num_layers(self, parallel_config: "ParallelConfig") -> int:
389
- total_num_hidden_layers = self.hf_text_config.num_hidden_layers
389
+ total_num_hidden_layers = getattr(self.hf_text_config,
390
+ "num_hidden_layers", 0)
391
return total_num_hidden_layers // parallel_config.pipeline_parallel_size
392
393
+ def contains_seqlen_agnostic_layers(
394
+ self, parallel_config: "ParallelConfig") -> bool:
395
+ """True for Mamba/SSM models (Jamba)"""
396
+ return self._get_num_seqlen_agnostic_layers(parallel_config) > 0
397
+
398
+ def get_layers_block_type(self,
399
+ parallel_config: "ParallelConfig") -> List[str]:
400
+ num_layers = self.get_num_layers(parallel_config)
401
+ # Transformers supports layers_block_type @property
402
+ return getattr(self.hf_config, "layers_block_type",
403
+ ["attention"] * num_layers)
404
+
405
+ def get_num_attention_layers(self,
406
+ parallel_config: "ParallelConfig") -> int:
407
+ return len([
408
+ t for t in self.get_layers_block_type(parallel_config)
409
+ if t == "attention"
410
+ ])
411
+
412
+ def _get_num_seqlen_agnostic_layers(
413
+ self, parallel_config: "ParallelConfig") -> int:
414
+ return len([
415
+ t for t in self.get_layers_block_type(parallel_config)
416
+ if t != "attention"
417
+ ])
418
+
419
420
class CacheConfig:
421
"""Configuration for the KV cache.
vllm/core/scheduler.py
+15
-1
@@ -299,7 +299,10 @@ class Scheduler:
299
# Sequence groups in the SWAPPED state.
300
# Contain decode requests that are swapped out.
301
self.swapped: Deque[SequenceGroup] = deque()
302
-
302
+ # Sequence groups finished requests ids since last step iteration.
303
+ # It lets the model know that any state associated with these requests
304
+ # can and must be released after the current step.
305
+ self._finished_requests_ids: List[str] = list()
306
# Time at previous scheduling step
307
self.prev_time = 0.0
308
# Did we schedule a prompt at previous step?
@@ -373,6 +376,12 @@ class Scheduler:
376
def get_num_unfinished_seq_groups(self) -> int:
377
return len(self.waiting) + len(self.running) + len(self.swapped)
378
379
+ def get_and_reset_finished_requests_ids(self) -> List[str]:
380
+ """Flushes the list of request ids of previously finished seq_groups."""
381
+ finished_requests_ids = self._finished_requests_ids
382
+ self._finished_requests_ids = list()
383
+ return finished_requests_ids
384
+
385
def _schedule_running(
386
self,
387
running_queue: deque,
@@ -1036,6 +1045,11 @@ class Scheduler:
1045
self.block_manager.free(seq)
1046
1047
def free_finished_seq_groups(self) -> None:
1048
+ for queue in [self.running, self.swapped, self.waiting]:
1049
+ self._finished_requests_ids += [
1050
+ seq_group.request_id for seq_group in queue
1051
+ if seq_group.is_finished()
1052
+ ]
1053
self.running = deque(seq_group for seq_group in self.running
1054
if not seq_group.is_finished())
1055
vllm/engine/async_llm_engine.py
+3
-1
@@ -224,6 +224,8 @@ class _AsyncLLMEngine(LLMEngine):
224
"""
225
seq_group_metadata_list, scheduler_outputs = self.scheduler[
226
virtual_engine].schedule()
227
+ finished_requests_ids = self.scheduler[
228
+ virtual_engine].get_and_reset_finished_requests_ids()
229
230
if not scheduler_outputs.is_empty():
231
# Execute the model.
@@ -235,7 +237,7 @@ class _AsyncLLMEngine(LLMEngine):
237
virtual_engine=virtual_engine,
238
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
239
running_queue_size=scheduler_outputs.running_queue_size,
238
- )
240
+ finished_requests_ids=finished_requests_ids)
241
output = await self.model_executor.execute_model_async(
242
execute_model_req)
243
else:
vllm/engine/llm_engine.py
+3
-1
@@ -846,6 +846,8 @@ class LLMEngine:
846
"as performance will be severely degraded otherwise.")
847
seq_group_metadata_list, scheduler_outputs = self.scheduler[
848
0].schedule()
849
+ finished_requests_ids = self.scheduler[
850
+ 0].get_and_reset_finished_requests_ids()
851
852
if not scheduler_outputs.is_empty():
853
execute_model_req = ExecuteModelRequest(
@@ -855,7 +857,7 @@ class LLMEngine:
857
blocks_to_copy=scheduler_outputs.blocks_to_copy,
858
num_lookahead_slots=scheduler_outputs.num_lookahead_slots,
859
running_queue_size=scheduler_outputs.running_queue_size,
858
- )
860
+ finished_requests_ids=finished_requests_ids)
861
output = self.model_executor.execute_model(
862
execute_model_req=execute_model_req)
863
else:
vllm/model_executor/models/__init__.py
+1
@@ -63,6 +63,7 @@ _GENERATION_MODELS = {
63
"XverseForCausalLM": ("xverse", "XverseForCausalLM"),
64
"Phi3SmallForCausalLM": ("phi3_small", "Phi3SmallForCausalLM"),
65
"MLPSpeculatorPreTrainedModel": ("mlp_speculator", "MLPSpeculator"),
66
+ "JambaForCausalLM": ("jamba", "JambaForCausalLM")
67
}
68
69
_EMBEDDING_MODELS = {
vllm/model_executor/models/jamba.py
new
+955
@@ -0,0 +1,955 @@
1
+# coding=utf-8
2
+"""Inference-only Jurassic model."""
3
+from dataclasses import dataclass
4
+from typing import Dict, Iterable, List, Optional, Tuple
5
+
6
+import torch
7
+from causal_conv1d import causal_conv1d_fn, causal_conv1d_update
8
+from mamba_ssm.ops.selective_scan_interface import selective_scan_fn
9
+from mamba_ssm.ops.triton.selective_state_update import selective_state_update
10
+from torch import nn
11
+from torch.nn.parameter import Parameter
12
+from transformers import JambaConfig
13
+
14
+from vllm.attention.backends.abstract import AttentionMetadata
15
+from vllm.attention.layer import Attention
16
+from vllm.config import CacheConfig, LoRAConfig
17
+from vllm.distributed import (get_tensor_model_parallel_rank,
18
+ get_tensor_model_parallel_world_size,
19
+ tensor_model_parallel_all_reduce)
20
+from vllm.model_executor.layers.activation import SiluAndMul
21
+from vllm.model_executor.layers.fused_moe import fused_moe
22
+from vllm.model_executor.layers.layernorm import RMSNorm
23
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
24
+ MergedColumnParallelLinear,
25
+ QKVParallelLinear,
26
+ ReplicatedLinear,
27
+ RowParallelLinear)
28
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
29
+from vllm.model_executor.layers.quantization.base_config import (
30
+ QuantizationConfig)
31
+from vllm.model_executor.layers.sampler import Sampler
32
+from vllm.model_executor.layers.vocab_parallel_embedding import (
33
+ DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
34
+from vllm.model_executor.model_loader.weight_utils import default_weight_loader
35
+from vllm.model_executor.sampling_metadata import SamplingMetadata
36
+from vllm.model_executor.utils import set_weight_attrs
37
+from vllm.sequence import IntermediateTensors, SamplerOutput
38
+from vllm.worker.model_runner import _BATCH_SIZES_TO_CAPTURE
39
+
40
+KVCache = Tuple[torch.Tensor, torch.Tensor]
41
+
42
+
43
+@dataclass
44
+class MambaCacheParams:
45
+ is_prompt: bool = False
46
+ conv_state: torch.Tensor = torch.Tensor()
47
+ ssm_state: torch.Tensor = torch.Tensor()
48
+
49
+
50
+# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
51
+class JambaMambaMixer(nn.Module):
52
+ """
53
+ Compute ∆, A, B, C, and D the state space parameters and compute
54
+ the `contextualized_states`. A, D are input independent
55
+ (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
56
+ for why A isn't selective) ∆, B, C are input-dependent
57
+ (this is a key difference between Mamba and the linear time
58
+ invariant S4, and is why Mamba is called
59
+ **selective** state spaces)
60
+ """
61
+
62
+ def __init__(self, config: JambaConfig, layer_idx):
63
+ super().__init__()
64
+ self.config = config
65
+ self.layer_idx = layer_idx
66
+ self.hidden_size = config.hidden_size
67
+ self.ssm_state_size = config.mamba_d_state
68
+ self.conv_kernel_size = config.mamba_d_conv
69
+ self.intermediate_size = config.mamba_expand * config.hidden_size
70
+ self.time_step_rank = config.mamba_dt_rank
71
+ self.use_conv_bias = config.mamba_conv_bias
72
+ self.use_bias = config.mamba_proj_bias
73
+ self.conv1d = ColumnParallelLinear(
74
+ input_size=self.conv_kernel_size,
75
+ output_size=self.intermediate_size,
76
+ bias=self.use_conv_bias,
77
+ )
78
+ # unsqueeze to fit conv1d weights shape into the linear weights shape.
79
+ # Can't do this in `weight_loader` since it already exists in
80
+ # `ColumnParallelLinear` and `set_weight_attrs`
81
+ # doesn't allow to override it
82
+ self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
83
+
84
+ self.in_proj = MergedColumnParallelLinear(self.hidden_size,
85
+ [self.intermediate_size] * 2,
86
+ bias=self.use_bias)
87
+ # selective projection used to make dt, B and C input dependent
88
+ self.x_proj = RowParallelLinear(
89
+ self.intermediate_size,
90
+ self.time_step_rank + self.ssm_state_size * 2,
91
+ bias=False,
92
+ )
93
+ # time step projection (discretization) -
94
+ # In the forward we need to apply dt_proj without the bias,
95
+ # as the bias is added in the selective scan kernel.
96
+ self.dt_proj = ColumnParallelLinear(self.time_step_rank,
97
+ self.intermediate_size,
98
+ bias=True,
99
+ skip_bias_add=True)
100
+
101
+ def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
102
+ tp_rank = get_tensor_model_parallel_rank()
103
+ tp_size = get_tensor_model_parallel_world_size()
104
+ param.data.copy_(
105
+ loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
106
+ dim=0)[tp_rank])
107
+
108
+ def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
109
+ weight_loader(param, -torch.exp(loaded_weight.float()))
110
+
111
+ tp_size = get_tensor_model_parallel_world_size()
112
+ self.A = nn.Parameter(
113
+ torch.empty(
114
+ self.intermediate_size // tp_size,
115
+ self.ssm_state_size,
116
+ dtype=torch.float32,
117
+ ))
118
+ self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
119
+
120
+ set_weight_attrs(self.D, {"weight_loader": weight_loader})
121
+ set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
122
+
123
+ self.out_proj = RowParallelLinear(
124
+ self.intermediate_size,
125
+ self.hidden_size,
126
+ bias=self.use_bias,
127
+ input_is_parallel=True,
128
+ )
129
+ self.activation = config.hidden_act
130
+
131
+ self.dt_layernorm = RMSNorm(self.time_step_rank,
132
+ eps=config.rms_norm_eps)
133
+ self.b_layernorm = RMSNorm(self.ssm_state_size,
134
+ eps=config.rms_norm_eps)
135
+ self.c_layernorm = RMSNorm(self.ssm_state_size,
136
+ eps=config.rms_norm_eps)
137
+
138
+ def mamba_forward(self,
139
+ hidden_states: torch.Tensor,
140
+ cache_params: MambaCacheParams = None):
141
+ # 1. Gated MLP's linear projection
142
+ projected_states = self.in_proj(hidden_states)[0].transpose(1, 2)
143
+ hidden_states, gate = projected_states.chunk(2, dim=1)
144
+
145
+ # 2. Convolution sequence transformation
146
+ conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
147
+ self.conv1d.weight.size(2))
148
+ if cache_params is not None and not cache_params.is_prompt:
149
+ hidden_states = causal_conv1d_update(
150
+ hidden_states.squeeze(-1),
151
+ cache_params.conv_state,
152
+ conv_weights,
153
+ self.conv1d.bias,
154
+ self.activation,
155
+ )
156
+ hidden_states = hidden_states.unsqueeze(-1)
157
+ else:
158
+ if cache_params is not None:
159
+ conv_states = nn.functional.pad(
160
+ hidden_states,
161
+ (self.conv_kernel_size - hidden_states.shape[-1], 0))
162
+ cache_params.conv_state.copy_(conv_states)
163
+
164
+ hidden_states = causal_conv1d_fn(
165
+ hidden_states,
166
+ conv_weights,
167
+ self.conv1d.bias,
168
+ activation=self.activation,
169
+ )
170
+
171
+ # 3. State Space Model sequence transformation
172
+ # 3.a. input varying initialization of time_step, B and C
173
+ ssm_parameters = self.x_proj(hidden_states.transpose(1, 2))[0]
174
+
175
+ time_step, B, C = torch.split(
176
+ ssm_parameters,
177
+ [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
178
+ dim=-1,
179
+ )
180
+ time_step = self.dt_layernorm(time_step.contiguous())
181
+ B = self.b_layernorm(B.contiguous())
182
+ C = self.c_layernorm(C.contiguous())
183
+
184
+ discrete_time_step = self.dt_proj(time_step)[0].transpose(1, 2)
185
+ # 3.c perform the recurrence y ← SSM(A, B, C)(x)
186
+ time_proj_bias = (self.dt_proj.bias.float() if hasattr(
187
+ self.dt_proj, "bias") else None)
188
+ if cache_params is not None and not cache_params.is_prompt:
189
+ scan_outputs = selective_state_update(
190
+ cache_params.ssm_state,
191
+ hidden_states[..., 0],
192
+ discrete_time_step[..., 0],
193
+ self.A,
194
+ B[:, 0],
195
+ C[:, 0],
196
+ self.D,
197
+ gate[..., 0],
198
+ time_proj_bias,
199
+ dt_softplus=True,
200
+ ).unsqueeze(-1)
201
+ else:
202
+ scan_outputs, ssm_state = selective_scan_fn(
203
+ hidden_states,
204
+ discrete_time_step,
205
+ self.A,
206
+ B.transpose(1, 2),
207
+ C.transpose(1, 2),
208
+ self.D.float(),
209
+ gate,
210
+ time_proj_bias,
211
+ delta_softplus=True,
212
+ return_last_state=True,
213
+ )
214
+ if ssm_state is not None and cache_params is not None:
215
+ cache_params.ssm_state.copy_(ssm_state)
216
+
217
+ # 4. Final linear projection
218
+ contextualized_states = self.out_proj(scan_outputs.transpose(1, 2))[0]
219
+ return contextualized_states
220
+
221
+ def forward(
222
+ self,
223
+ hidden_states: torch.Tensor,
224
+ attn_metadata: AttentionMetadata,
225
+ conv_state: torch.Tensor,
226
+ ssm_state: torch.Tensor,
227
+ ):
228
+ if attn_metadata.prefill_metadata is not None:
229
+ offset = 0
230
+ for i, prompt_len in enumerate(
231
+ attn_metadata.prefill_metadata.seq_lens):
232
+ cache = MambaCacheParams(True,
233
+ conv_state=conv_state[i].unsqueeze(0),
234
+ ssm_state=ssm_state[i].unsqueeze(0))
235
+ hidden_states[offset:offset + prompt_len].copy_(
236
+ self.mamba_forward(hidden_states[offset:offset +
237
+ prompt_len].unsqueeze(0),
238
+ cache_params=cache)[0])
239
+ offset += prompt_len
240
+ else:
241
+ cache = MambaCacheParams(False,
242
+ conv_state=conv_state,
243
+ ssm_state=ssm_state)
244
+ hidden_states = self.mamba_forward(hidden_states.unsqueeze(1),
245
+ cache_params=cache)
246
+ hidden_states = hidden_states.squeeze(1)
247
+
248
+ return hidden_states
249
+
250
+
251
+class JambaMLP(nn.Module):
252
+
253
+ def __init__(
254
+ self,
255
+ config: JambaConfig,
256
+ quant_config: Optional[QuantizationConfig] = None,
257
+ ) -> None:
258
+ super().__init__()
259
+ hidden_size = config.hidden_size
260
+ intermediate_size = config.intermediate_size
261
+ hidden_act = config.hidden_act
262
+ self.gate_up_proj = MergedColumnParallelLinear(
263
+ hidden_size, [intermediate_size] * 2,
264
+ bias=False,
265
+ quant_config=quant_config)
266
+ self.down_proj = RowParallelLinear(intermediate_size,
267
+ hidden_size,
268
+ bias=False,
269
+ quant_config=quant_config)
270
+ if hidden_act != "silu":
271
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
272
+ "Only silu is supported for now.")
273
+ self.act_fn = SiluAndMul()
274
+
275
+ def forward(self, x):
276
+ gate_up, _ = self.gate_up_proj(x)
277
+ x = self.act_fn(gate_up)
278
+ x, _ = self.down_proj(x)
279
+ return x
280
+
281
+
282
+class JambaMoE(nn.Module):
283
+ """A tensor-parallel MoE implementation for Mixtral that shards each expert
284
+ across all ranks.
285
+
286
+ Each expert's weights are sharded across all ranks and a fused MoE
287
+ kernel is used for the forward pass, and finally we reduce the outputs
288
+ across ranks.
289
+ """
290
+
291
+ def __init__(
292
+ self,
293
+ config: JambaConfig,
294
+ params_dtype: Optional[torch.dtype] = None,
295
+ tp_size: Optional[int] = None,
296
+ quant_config: Optional[QuantizationConfig] = None,
297
+ ):
298
+ super().__init__()
299
+ self.tp_size = tp_size or get_tensor_model_parallel_world_size()
300
+ self.num_total_experts = config.num_experts
301
+ self.top_k = config.num_experts_per_tok
302
+ self.hidden_size = config.hidden_size
303
+ self.intermediate_size = config.intermediate_size // self.tp_size
304
+
305
+ if params_dtype is None:
306
+ params_dtype = torch.get_default_dtype()
307
+ self.params_dtype = params_dtype
308
+
309
+ self.router = ReplicatedLinear(self.hidden_size,
310
+ self.num_total_experts,
311
+ bias=False,
312
+ params_dtype=self.params_dtype)
313
+
314
+ self.ws = nn.Parameter(
315
+ torch.empty(
316
+ self.num_total_experts,
317
+ 2 * self.intermediate_size,
318
+ self.hidden_size,
319
+ device="cuda",
320
+ dtype=self.params_dtype,
321
+ ))
322
+ self.w2s = nn.Parameter(
323
+ torch.empty(
324
+ self.num_total_experts,
325
+ self.hidden_size,
326
+ self.intermediate_size,
327
+ device="cuda",
328
+ dtype=self.params_dtype,
329
+ ))
330
+
331
+ set_weight_attrs(
332
+ self.ws,
333
+ {
334
+ "weight_loader": self.weight_loader,
335
+ },
336
+ )
337
+ set_weight_attrs(
338
+ self.w2s,
339
+ {
340
+ "weight_loader": self.weight_loader,
341
+ },
342
+ )
343
+
344
+ def weight_loader(
345
+ self,
346
+ param: nn.Parameter,
347
+ loaded_weight: torch.Tensor,
348
+ weight_name: str,
349
+ expert_id: int,
350
+ ):
351
+ tp_rank = get_tensor_model_parallel_rank()
352
+ param_data = param.data
353
+ shard_size = self.intermediate_size
354
+ shard = slice(tp_rank * shard_size, (tp_rank + 1) * shard_size)
355
+ if weight_name.endswith("gate_proj.weight"):
356
+ param_data[expert_id, 0:shard_size, :] = loaded_weight[shard, :]
357
+ if weight_name.endswith("up_proj.weight"):
358
+ param_data[expert_id,
359
+ shard_size:2 * shard_size, :] = loaded_weight[shard, :]
360
+ if weight_name.endswith("down_proj.weight"):
361
+ param_data[expert_id, :, :] = loaded_weight[:, shard]
362
+
363
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
364
+ num_tokens, hidden_size = hidden_states.shape
365
+ hidden_states = hidden_states.view(-1, self.hidden_size)
366
+ # router_logits: (batch * sequence_length, n_experts)
367
+ router_logits, _ = self.router(hidden_states)
368
+
369
+ final_hidden_states = fused_moe(
370
+ hidden_states,
371
+ self.ws,
372
+ self.w2s,
373
+ router_logits,
374
+ self.top_k,
375
+ renormalize=
376
+ False, # Mixtral normalize the expert probs to 1. We don't!
377
+ inplace=True,
378
+ )
379
+
380
+ if self.tp_size > 1:
381
+ final_hidden_states = tensor_model_parallel_all_reduce(
382
+ final_hidden_states)
383
+
384
+ return final_hidden_states.view(num_tokens, hidden_size)
385
+
386
+
387
+class JambaMambaDecoderLayer(nn.Module):
388
+
389
+ def __init__(self,
390
+ config: JambaConfig,
391
+ layer_idx: int,
392
+ cache_config: Optional[CacheConfig] = None,
393
+ quant_config: Optional[QuantizationConfig] = None) -> None:
394
+ super().__init__()
395
+ self.layer_idx = layer_idx
396
+ self.config = config
397
+ self.mamba = JambaMambaMixer(config, layer_idx)
398
+
399
+ num_experts = config.layers_num_experts[layer_idx]
400
+ ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
401
+ self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
402
+ self.input_layernorm = RMSNorm(config.hidden_size,
403
+ eps=config.rms_norm_eps)
404
+ self.pre_ff_layernorm = RMSNorm(config.hidden_size,
405
+ eps=config.rms_norm_eps)
406
+
407
+ def forward(
408
+ self,
409
+ hidden_states: torch.Tensor,
410
+ attn_metadata: AttentionMetadata,
411
+ residual: Optional[torch.Tensor],
412
+ conv_state: torch.Tensor,
413
+ ssm_state: torch.Tensor,
414
+ **kwargs,
415
+ ):
416
+ if residual is None:
417
+ residual = hidden_states
418
+ hidden_states = self.input_layernorm(hidden_states)
419
+ else:
420
+ hidden_states, residual = self.input_layernorm(
421
+ hidden_states, residual)
422
+
423
+ hidden_states = self.mamba(hidden_states, attn_metadata, conv_state,
424
+ ssm_state)
425
+ # Fully Connected
426
+ hidden_states, residual = self.pre_ff_layernorm(
427
+ hidden_states, residual)
428
+ hidden_states = self.feed_forward(hidden_states)
429
+ return hidden_states, residual
430
+
431
+
432
+class JambaAttentionDecoderLayer(nn.Module):
433
+
434
+ def __init__(
435
+ self,
436
+ config: JambaConfig,
437
+ layer_idx: int,
438
+ cache_config: Optional[CacheConfig] = None,
439
+ quant_config: Optional[QuantizationConfig] = None,
440
+ ) -> None:
441
+ super().__init__()
442
+ self.hidden_size = config.hidden_size
443
+ tp_size = get_tensor_model_parallel_world_size()
444
+ self.total_num_heads = config.num_attention_heads
445
+ assert self.total_num_heads % tp_size == 0
446
+ self.num_heads = self.total_num_heads // tp_size
447
+ self.total_num_kv_heads = config.num_key_value_heads
448
+ if self.total_num_kv_heads >= tp_size:
449
+ # Number of KV heads is greater than TP size, so we partition
450
+ # the KV heads across multiple tensor parallel GPUs.
451
+ assert self.total_num_kv_heads % tp_size == 0
452
+ else:
453
+ # Number of KV heads is less than TP size, so we replicate
454
+ # the KV heads across multiple tensor parallel GPUs.
455
+ assert tp_size % self.total_num_kv_heads == 0
456
+ self.num_kv_heads = max(1, self.total_num_kv_heads // tp_size)
457
+ self.head_dim = config.hidden_size // self.total_num_heads
458
+ self.q_size = self.num_heads * self.head_dim
459
+ self.kv_size = self.num_kv_heads * self.head_dim
460
+ self.scaling = self.head_dim**-0.5
461
+
462
+ self.qkv_proj = QKVParallelLinear(
463
+ config.hidden_size,
464
+ self.head_dim,
465
+ self.total_num_heads,
466
+ self.total_num_kv_heads,
467
+ bias=False,
468
+ quant_config=quant_config,
469
+ )
470
+ self.o_proj = RowParallelLinear(self.total_num_heads * self.head_dim,
471
+ config.hidden_size,
472
+ bias=False,
473
+ quant_config=quant_config)
474
+
475
+ self.attn = Attention(
476
+ self.num_heads,
477
+ self.head_dim,
478
+ self.scaling,
479
+ num_kv_heads=self.num_kv_heads,
480
+ cache_config=cache_config,
481
+ )
482
+
483
+ num_experts = config.layers_num_experts[layer_idx]
484
+ ffn_layer_class = JambaMoE if num_experts > 1 else JambaMLP
485
+ self.feed_forward = ffn_layer_class(config, quant_config=quant_config)
486
+ self.input_layernorm = RMSNorm(config.hidden_size,
487
+ eps=config.rms_norm_eps)
488
+ self.pre_ff_layernorm = RMSNorm(config.hidden_size,
489
+ eps=config.rms_norm_eps)
490
+
491
+ def self_attention(
492
+ self,
493
+ positions: torch.Tensor,
494
+ hidden_states: torch.Tensor,
495
+ kv_cache: torch.Tensor,
496
+ attn_metadata: AttentionMetadata,
497
+ **kwargs,
498
+ ) -> torch.Tensor:
499
+ qkv, _ = self.qkv_proj(hidden_states)
500
+ q, k, v = qkv.split([self.q_size, self.kv_size, self.kv_size], dim=-1)
501
+ attn_output = self.attn(q, k, v, kv_cache, attn_metadata)
502
+ output, _ = self.o_proj(attn_output)
503
+ return output
504
+
505
+ def forward(
506
+ self,
507
+ positions: torch.Tensor,
508
+ hidden_states: torch.Tensor,
509
+ kv_cache: torch.Tensor,
510
+ attn_metadata: AttentionMetadata,
511
+ residual: Optional[torch.Tensor],
512
+ **kwargs,
513
+ ):
514
+ if residual is None:
515
+ residual = hidden_states
516
+ hidden_states = self.input_layernorm(hidden_states)
517
+ else:
518
+ hidden_states, residual = self.input_layernorm(
519
+ hidden_states, residual)
520
+
521
+ hidden_states = self.self_attention(
522
+ positions=positions,
523
+ hidden_states=hidden_states,
524
+ kv_cache=kv_cache,
525
+ attn_metadata=attn_metadata,
526
+ )
527
+ # Fully Connected
528
+ hidden_states, residual = self.pre_ff_layernorm(
529
+ hidden_states, residual)
530
+ hidden_states = self.feed_forward(hidden_states)
531
+ return hidden_states, residual
532
+
533
+
534
+ALL_DECODER_LAYER_TYPES = {
535
+ "attention": JambaAttentionDecoderLayer,
536
+ "mamba": JambaMambaDecoderLayer
537
+}
538
+
539
+
540
+class JambaModel(nn.Module):
541
+
542
+ def __init__(
543
+ self,
544
+ config: JambaConfig,
545
+ quant_config: Optional[QuantizationConfig] = None,
546
+ cache_config: Optional[CacheConfig] = None,
547
+ lora_config: Optional[LoRAConfig] = None,
548
+ ) -> None:
549
+ super().__init__()
550
+ self.config = config
551
+ self.padding_idx = config.pad_token_id
552
+ lora_vocab = ((lora_config.lora_extra_vocab_size *
553
+ (lora_config.max_loras or 1)) if lora_config else 0)
554
+ self.vocab_size = config.vocab_size + lora_vocab
555
+ self.org_vocab_size = config.vocab_size
556
+
557
+ self.embed_tokens = VocabParallelEmbedding(
558
+ self.vocab_size,
559
+ config.hidden_size,
560
+ org_num_embeddings=config.vocab_size,
561
+ )
562
+
563
+ decoder_layers = []
564
+ for i in range(config.num_hidden_layers):
565
+ layer_class = ALL_DECODER_LAYER_TYPES[config.layers_block_type[i]]
566
+ decoder_layers.append(
567
+ layer_class(config,
568
+ layer_idx=i,
569
+ cache_config=cache_config,
570
+ quant_config=quant_config))
571
+ self.layers = nn.ModuleList(decoder_layers)
572
+ self.final_layernorm = RMSNorm(config.hidden_size,
573
+ eps=config.rms_norm_eps)
574
+
575
+ def forward(
576
+ self,
577
+ input_ids: torch.Tensor,
578
+ positions: torch.Tensor,
579
+ kv_caches: List[torch.Tensor],
580
+ attn_metadata: AttentionMetadata,
581
+ conv_state: torch.Tensor,
582
+ ssm_state: torch.Tensor,
583
+ ) -> torch.Tensor:
584
+ hidden_states = self.embed_tokens(input_ids)
585
+ residual = None
586
+
587
+ for i in range(len(self.layers)):
588
+ layer = self.layers[i]
589
+ kv_cache = None
590
+ current_ssm_state = None
591
+ current_conv_state = None
592
+ if isinstance(layer, JambaAttentionDecoderLayer):
593
+ kv_cache = kv_caches[(i - self.config.attn_layer_offset) //
594
+ self.config.attn_layer_period]
595
+ if isinstance(layer, JambaMambaDecoderLayer):
596
+ current_state_layer = i - (1 +
597
+ (i - self.config.attn_layer_offset)
598
+ // self.config.attn_layer_period)
599
+ current_ssm_state = ssm_state[current_state_layer]
600
+ current_conv_state = conv_state[current_state_layer]
601
+
602
+ hidden_states, residual = layer(
603
+ positions=positions,
604
+ hidden_states=hidden_states,
605
+ kv_cache=kv_cache,
606
+ attn_metadata=attn_metadata,
607
+ residual=residual,
608
+ conv_state=current_conv_state,
609
+ ssm_state=current_ssm_state,
610
+ )
611
+ hidden_states, _ = self.final_layernorm(hidden_states, residual)
612
+ return hidden_states
613
+
614
+
615
+class JambaForCausalLM(nn.Module):
616
+ packed_modules_mapping = {
617
+ "qkv_proj": [
618
+ "q_proj",
619
+ "k_proj",
620
+ "v_proj",
621
+ ],
622
+ }
623
+
624
+ # LoRA specific attributes
625
+ supported_lora_modules = [
626
+ "qkv_proj",
627
+ "o_proj",
628
+ "embed_tokens",
629
+ "lm_head",
630
+ ]
631
+ embedding_modules = {
632
+ "embed_tokens": "input_embeddings",
633
+ "lm_head": "output_embeddings",
634
+ }
635
+ embedding_padding_modules = ["lm_head"]
636
+
637
+ def __init__(
638
+ self,
639
+ config: JambaConfig,
640
+ cache_config: Optional[CacheConfig] = None,
641
+ quant_config: Optional[QuantizationConfig] = None,
642
+ lora_config: Optional[LoRAConfig] = None,
643
+ ) -> None:
644
+ super().__init__()
645
+ self.config = config
646
+ self.model = JambaModel(config,
647
+ cache_config=cache_config,
648
+ quant_config=quant_config,
649
+ lora_config=lora_config)
650
+ self.unpadded_vocab_size = config.vocab_size
651
+ if lora_config:
652
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
653
+ self.lm_head = ParallelLMHead(
654
+ self.unpadded_vocab_size,
655
+ config.hidden_size,
656
+ org_num_embeddings=config.vocab_size,
657
+ padding_size=DEFAULT_VOCAB_PADDING_SIZE
658
+ # We need bigger padding if using lora for kernel
659
+ # compatibility
660
+ if not lora_config else lora_config.lora_vocab_padding_size,
661
+ )
662
+ # Current step used indices
663
+ self.current_indices: List[int] = []
664
+ # Used to track and store by the Mamba cache between steps.
665
+ self.mamba_cache: Tuple[torch.Tensor, torch.Tensor] = tuple()
666
+ # Used as an input_buffer for the CUDA graph runs.
667
+ self.mamba_gc_cache_buffer: Tuple[torch.Tensor, torch.Tensor] = tuple()
668
+ # Maps between the request id and a dict that maps between the seq_id
669
+ # and its index inside the self.mamba_cache
670
+ self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
671
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
672
+ config.vocab_size)
673
+ self.sampler = Sampler()
674
+
675
+ def forward(self,
676
+ input_ids: torch.Tensor,
677
+ positions: torch.Tensor,
678
+ kv_caches: List[KVCache],
679
+ attn_metadata: AttentionMetadata,
680
+ intermediate_tensors: Optional[IntermediateTensors] = None,
681
+ **kwargs):
682
+ if not self.mamba_cache:
683
+ self._prepare_mamba_cache()
684
+
685
+ if "seqlen_agnostic_capture_inputs" not in kwargs:
686
+ # We get here only on Prefill/Eager mode runs
687
+ assert all(
688
+ key in kwargs
689
+ for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
690
+
691
+ request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
692
+ batch_size = input_ids.shape[0]
693
+ if attn_metadata.prefill_metadata:
694
+ batch_size = len(request_ids_to_seq_ids)
695
+ (
696
+ current_seqlen_agnostic_cache,
697
+ indices,
698
+ ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
699
+ batch_size)
700
+ finished_requests_ids = kwargs["finished_requests_ids"]
701
+ self._release_mamba_cache(finished_requests_ids)
702
+ else:
703
+ # CUDA graph capturing runs
704
+ current_seqlen_agnostic_cache, indices = (
705
+ kwargs["seqlen_agnostic_capture_inputs"],
706
+ [],
707
+ )
708
+ self.current_indices = indices
709
+
710
+ hidden_states = self.model(input_ids, positions, kv_caches,
711
+ attn_metadata,
712
+ current_seqlen_agnostic_cache[0],
713
+ current_seqlen_agnostic_cache[1])
714
+
715
+ if "seqlen_agnostic_capture_inputs" not in kwargs:
716
+ self._copy_mamba_cache_by_indices(self.current_indices,
717
+ current_seqlen_agnostic_cache)
718
+
719
+ return hidden_states
720
+
721
+ def _copy_mamba_cache_by_indices(
722
+ self, indices: List[int],
723
+ current_seqlen_agnostic_cache: Tuple[torch.Tensor, torch.Tensor]):
724
+ for i, offset in enumerate(indices):
725
+ self._copy_mamba_cache(offset, i, current_seqlen_agnostic_cache)
726
+
727
+ def _copy_mamba_cache(self, index_to: int, index_from: int,
728
+ from_buffer: Tuple[torch.Tensor, torch.Tensor]):
729
+ assert len(self.mamba_cache) > 0
730
+ for (cache_t, from_buffer_t) in zip(self.mamba_cache, from_buffer):
731
+ cache_t[:, index_to].copy_(from_buffer_t[:, index_from],
732
+ non_blocking=True)
733
+
734
+ def _assign_seq_id_to_mamba_cache(self, cur_rid: str,
735
+ seqs_id: List[int]) -> List[int]:
736
+ indices_for_current_run = []
737
+ for seq_id in seqs_id:
738
+ if cur_rid not in self.mamba_cache_indices_mapping:
739
+ self.mamba_cache_indices_mapping[cur_rid] = {}
740
+ first_free_index = self._first_free_index_in_mamba_cache()
741
+ self.mamba_cache_indices_mapping[cur_rid][
742
+ seq_id] = first_free_index
743
+ index_for_current_run = first_free_index
744
+ ## case of decoding n>1, copy prefill cache to decoding indices
745
+ elif seq_id not in (seq_ids2indices :=
746
+ self.mamba_cache_indices_mapping[cur_rid]):
747
+ first_free_index = self._first_free_index_in_mamba_cache()
748
+ index_exist = list(seq_ids2indices.values())[0]
749
+ self._copy_mamba_cache(index_from=index_exist,
750
+ index_to=first_free_index,
751
+ from_buffer=self.mamba_cache)
752
+ self.mamba_cache_indices_mapping[cur_rid][
753
+ seq_id] = first_free_index
754
+ index_for_current_run = first_free_index
755
+ else:
756
+ index_for_current_run = self.mamba_cache_indices_mapping[
757
+ cur_rid][seq_id]
758
+
759
+ indices_for_current_run.append(index_for_current_run)
760
+ return indices_for_current_run
761
+
762
+ def _prepare_current_run_mamba_cache(
763
+ self, request_ids_to_seq_ids: Dict[str, list[int]], batch_size: int
764
+ ) -> Tuple[Tuple[torch.Tensor, torch.Tensor], List[int]]:
765
+ indices_for_current_run = []
766
+ for request_id, seqs_id in request_ids_to_seq_ids.items():
767
+ indices_for_current_run += self._assign_seq_id_to_mamba_cache(
768
+ request_id, seqs_id)
769
+ ## Pad the batch in case of running batch that was not captured via CG
770
+ padded_indices = indices_for_current_run.copy()
771
+ pad_index = self._first_free_index_in_mamba_cache()
772
+
773
+ for _ in range(batch_size - len(indices_for_current_run)):
774
+ padded_indices.append(pad_index)
775
+
776
+ conv_state = self.mamba_cache[0][:, padded_indices]
777
+ temporal_state = self.mamba_cache[1][:, padded_indices]
778
+
779
+ return (conv_state, temporal_state), indices_for_current_run
780
+
781
+ def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
782
+ """
783
+ Copy the relevant Mamba cache into the CUDA graph input buffer
784
+ that was provided during the capture runs
785
+ (JambaForCausalLM.mamba_gc_cache_buffer).
786
+ """
787
+ assert all(
788
+ key in kwargs
789
+ for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
790
+ request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
791
+ batch_size = len(request_ids_to_seq_ids)
792
+ (
793
+ current_mamba_cache,
794
+ indices,
795
+ ) = self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
796
+ batch_size)
797
+ self.current_indices = indices
798
+ finished_requests_ids = kwargs["finished_requests_ids"]
799
+ self._release_mamba_cache(finished_requests_ids)
800
+
801
+ for input_buffer, current_cache_buffer in zip(
802
+ input_buffers["seqlen_agnostic_capture_inputs"],
803
+ current_mamba_cache):
804
+ input_buffer.copy_(current_cache_buffer, non_blocking=True)
805
+
806
+ def copy_outputs_after_cuda_graphs(self, input_buffers, **kwargs):
807
+ """
808
+ Copy the relevant Mamba cache from the CUDA graph input_buffers
809
+ back to the JambaForCausalLM.mamba_cache after CUDA
810
+ graph replay run is done.
811
+ """
812
+ self._copy_mamba_cache_by_indices(
813
+ self.current_indices,
814
+ input_buffers["seqlen_agnostic_capture_inputs"])
815
+
816
+ def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
817
+ """
818
+ Provide the CUDA graph capture runs with a buffer in adjusted size.
819
+ The buffer is used to maintain the Mamba Cache during the CUDA graph
820
+ replay runs.
821
+ """
822
+ return tuple(buffer[:, :batch_size]
823
+ for buffer in self.mamba_gc_cache_buffer)
824
+
825
+ def _release_mamba_cache(self, finished_seq_groups_req_ids: List[str]):
826
+ for req_id in finished_seq_groups_req_ids:
827
+ if req_id in self.mamba_cache_indices_mapping:
828
+ self.mamba_cache_indices_mapping.pop(req_id)
829
+
830
+ def _first_free_index_in_mamba_cache(self) -> int:
831
+ if self.mamba_cache:
832
+ max_possible_batch_size = self.mamba_cache[0].shape[1]
833
+ occupied = [
834
+ id for seq_ids in self.mamba_cache_indices_mapping.values()
835
+ for id in seq_ids.values()
836
+ ]
837
+ first_free_index = [
838
+ i not in occupied for i in range(max_possible_batch_size)
839
+ ].index(True)
840
+ return first_free_index
841
+ return 0
842
+
843
+ def _get_mamba_cache_shape(
844
+ self
845
+ ) -> Tuple[Optional[Tuple[int, int]], Optional[Tuple[int, int]]]:
846
+ world_size = get_tensor_model_parallel_world_size()
847
+ hidden_size = self.config.hidden_size
848
+ conv_state_shape = (
849
+ self.config.mamba_expand * hidden_size // world_size,
850
+ self.config.mamba_d_conv,
851
+ )
852
+ temporal_state_shape = (
853
+ self.config.mamba_expand * self.config.hidden_size // world_size,
854
+ self.config.mamba_d_state,
855
+ )
856
+ return conv_state_shape, temporal_state_shape
857
+
858
+ def _prepare_mamba_cache(self):
859
+ dtype = self.lm_head.weight.dtype
860
+ layers_type = self.config.layers_block_type
861
+ mamba_layers = sum(
862
+ [layer_type == "mamba" for layer_type in layers_type])
863
+ max_batch_size = _BATCH_SIZES_TO_CAPTURE[-1] + 10
864
+ conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
865
+ assert conv_state_shape is not None and temporal_state_shape is not None
866
+ for buffername in ["mamba_cache", "mamba_gc_cache_buffer"]:
867
+ buffer = (torch.empty(size=(mamba_layers, max_batch_size) +
868
+ conv_state_shape,
869
+ dtype=dtype,
870
+ device="cuda"),
871
+ torch.empty(size=(mamba_layers, max_batch_size) +
872
+ temporal_state_shape,
873
+ dtype=dtype,
874
+ device="cuda"))
875
+ setattr(self, buffername, buffer)
876
+
877
+ def compute_logits(self, hidden_states: torch.Tensor,
878
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
879
+ logits = self.logits_processor(self.lm_head.weight, hidden_states,
880
+ sampling_metadata)
881
+ return logits
882
+
883
+ def sample(
884
+ self,
885
+ logits: Optional[torch.Tensor],
886
+ sampling_metadata: SamplingMetadata,
887
+ ) -> Optional[SamplerOutput]:
888
+ next_tokens = self.sampler(logits, sampling_metadata)
889
+ return next_tokens
890
+
891
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
892
+ stacked_params_mapping = [
893
+ # (param_name, shard_name, shard_id)
894
+ ("qkv_proj", "q_proj", "q"),
895
+ ("qkv_proj", "k_proj", "k"),
896
+ ("qkv_proj", "v_proj", "v"),
897
+ ("gate_up_proj", "gate_proj", 0),
898
+ ("gate_up_proj", "up_proj", 1),
899
+ ]
900
+
901
+ expert_params_mapping = [
902
+ # (param_name, weight_name, expert_id)
903
+ (
904
+ "ws" if weight_name in ["gate_proj", "up_proj"] else "w2s",
905
+ f"experts.{expert_id}.{weight_name}.weight",
906
+ expert_id,
907
+ ) for expert_id in range(self.config.num_experts)
908
+ for weight_name in ["down_proj", "up_proj", "gate_proj"]
909
+ ]
910
+
911
+ params_dict = dict(self.named_parameters())
912
+ for name, loaded_weight in weights:
913
+ if "rotary_emb.inv_freq" in name:
914
+ continue
915
+
916
+ if "A_log" in name:
917
+ name = name.replace("A_log", "A")
918
+
919
+ if ".self_attn." in name:
920
+ name = name.replace(".self_attn", "")
921
+
922
+ for param_name, weight_name, shard_id in stacked_params_mapping:
923
+ if weight_name not in name:
924
+ continue
925
+ if 'experts' in name:
926
+ continue
927
+ name = name.replace(weight_name, param_name)
928
+ # Skip loading extra bias for GPTQ models.
929
+ if name.endswith(".bias") and name not in params_dict:
930
+ continue
931
+ param = params_dict[name]
932
+ weight_loader = param.weight_loader
933
+ weight_loader(param, loaded_weight, shard_id)
934
+ break
935
+ else:
936
+ for param_name, weight_name, expert_id in expert_params_mapping:
937
+ if weight_name not in name:
938
+ continue
939
+ name = name.replace(weight_name, param_name)
940
+ param = params_dict[name]
941
+ weight_loader = param.weight_loader
942
+ weight_loader(param,
943
+ loaded_weight,
944
+ weight_name,
945
+ expert_id=expert_id)
946
+ break
947
+ else:
948
+ # Skip loading extra bias for GPTQ models.
949
+ if name.endswith(".bias") and name not in params_dict:
950
+ continue
951
+
952
+ param = params_dict[name]
953
+ weight_loader = getattr(param, "weight_loader",
954
+ default_weight_loader)
955
+ weight_loader(param, loaded_weight)
vllm/sequence.py
+3
-1
@@ -934,6 +934,8 @@ class ExecuteModelRequest:
934
previous_hidden_states: Optional[HiddenStates] = None
935
# The number of forward steps to run.
936
num_steps: int = 1
937
+ # Finished request ids since last step.
938
+ finished_requests_ids: List[str] = field(default_factory=list)
939
940
def clone(
941
self, seq_group_metadata_list: List[SequenceGroupMetadata]
@@ -949,4 +951,4 @@ class ExecuteModelRequest:
951
running_queue_size=self.running_queue_size,
952
previous_hidden_states=self.previous_hidden_states,
953
num_steps=self.num_steps,
952
- )
954
+ finished_requests_ids=self.finished_requests_ids)
vllm/spec_decode/draft_model_runner.py
+8
-4
@@ -75,15 +75,19 @@ class TP1DraftModelRunner(ModelRunner):
75
List[SequenceGroupMetadata]] = None
76
77
def prepare_model_input(
78
- self,
79
- seq_group_metadata_list: List[SequenceGroupMetadata],
80
- virtual_engine: int = 0) -> ModelInputForGPUWithSamplingMetadata:
78
+ self,
79
+ seq_group_metadata_list: List[SequenceGroupMetadata],
80
+ virtual_engine: int = 0,
81
+ finished_requests_ids: Optional[List[str]] = None
82
+ ) -> ModelInputForGPUWithSamplingMetadata:
83
"""A temporary solution that caches the seq_group_metadata_list
84
for multi-step execution.
85
TODO: In-place update model_input and remove this function.
86
"""
87
self.cached_seq_group_metadata_list = seq_group_metadata_list
86
- return super().prepare_model_input(seq_group_metadata_list)
88
+ return super().prepare_model_input(
89
+ seq_group_metadata_list,
90
+ finished_requests_ids=finished_requests_ids)
91
92
def update_model_input(
93
self, model_input: ModelInputForGPUWithSamplingMetadata,
vllm/worker/cache_engine.py
+9
-6
@@ -33,7 +33,9 @@ class CacheEngine:
33
self.device_config = device_config
34
35
self.head_size = model_config.get_head_size()
36
- self.num_layers = model_config.get_num_layers(parallel_config)
36
+ # Models like Jamba, have mixed typed layers, E.g Mamba
37
+ self.num_attention_layers = model_config.get_num_attention_layers(
38
+ parallel_config)
39
self.num_kv_heads = model_config.get_num_kv_heads(parallel_config)
40
41
self.block_size = cache_config.block_size
@@ -75,7 +77,7 @@ class CacheEngine:
77
num_blocks, self.block_size, self.num_kv_heads, self.head_size)
78
pin_memory = is_pin_memory_available() if device == "cpu" else False
79
kv_cache: List[torch.Tensor] = []
78
- for _ in range(self.num_layers):
80
+ for _ in range(self.num_attention_layers):
81
# null block in CpuGpuBlockAllocator requires at least that
82
# block to be zeroed-out.
83
# We zero-out everything for simplicity.
@@ -87,12 +89,12 @@ class CacheEngine:
89
return kv_cache
90
91
def swap_in(self, src_to_dst: torch.Tensor) -> None:
90
- for i in range(self.num_layers):
92
+ for i in range(self.num_attention_layers):
93
self.attn_backend.swap_blocks(self.cpu_cache[i], self.gpu_cache[i],
94
src_to_dst)
95
96
def swap_out(self, src_to_dst: torch.Tensor) -> None:
95
- for i in range(self.num_layers):
97
+ for i in range(self.num_attention_layers):
98
self.attn_backend.swap_blocks(self.gpu_cache[i], self.cpu_cache[i],
99
src_to_dst)
100
@@ -107,11 +109,12 @@ class CacheEngine:
109
) -> int:
110
head_size = model_config.get_head_size()
111
num_heads = model_config.get_num_kv_heads(parallel_config)
110
- num_layers = model_config.get_num_layers(parallel_config)
112
+ num_attention_layers = model_config.get_num_attention_layers(
113
+ parallel_config)
114
115
key_cache_block = cache_config.block_size * num_heads * head_size
116
value_cache_block = key_cache_block
114
- total = num_layers * (key_cache_block + value_cache_block)
117
+ total = num_attention_layers * (key_cache_block + value_cache_block)
118
if cache_config.cache_dtype == "auto":
119
dtype = model_config.dtype
120
else:
vllm/worker/cpu_model_runner.py
+4
-3
@@ -314,9 +314,10 @@ class CPUModelRunner(ModelRunnerBase[CPUModelInput]):
314
)
315
316
def prepare_model_input(
317
- self,
318
- seq_group_metadata_list: List[SequenceGroupMetadata],
319
- virtual_engine: int = 0,
317
+ self,
318
+ seq_group_metadata_list: List[SequenceGroupMetadata],
319
+ virtual_engine: int = 0,
320
+ finished_requests_ids: Optional[List[str]] = None
321
) -> CPUModelInput:
322
multi_modal_kwargs = None
323
# NOTE: We assume that all sequences in the group are all prompts or
vllm/worker/embedding_model_runner.py
+2
-1
@@ -120,10 +120,11 @@ class EmbeddingModelRunner(
120
self,
121
seq_group_metadata_list: Optional[List[SequenceGroupMetadata]],
122
virtual_engine: int = 0,
123
+ finished_requests_ids: Optional[List[str]] = None
124
) -> ModelInputForGPUWithPoolingMetadata:
125
assert seq_group_metadata_list is not None
126
model_input = self._prepare_model_input_tensors(
126
- seq_group_metadata_list)
127
+ seq_group_metadata_list, finished_requests_ids)
128
# Prepare PoolingMetadata.
129
assert model_input.seq_lens is not None
130
pooling_metadata = self._prepare_pooling(seq_group_metadata_list,
vllm/worker/model_runner.py
+57
-10
@@ -84,6 +84,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
84
lora_requests: Optional[Set[LoRARequest]] = None
85
attn_metadata: Optional["AttentionMetadata"] = None
86
multi_modal_kwargs: Optional[Dict[str, torch.Tensor]] = None
87
+ request_ids_to_seq_ids: Optional[Dict[str, List[int]]] = None
88
+ finished_requests_ids: Optional[List[str]] = None
89
virtual_engine: int = 0
90
91
def as_broadcastable_tensor_dict(self) -> Dict[str, Any]:
@@ -94,6 +96,8 @@ class ModelInputForGPU(ModelRunnerInputBase):
96
"lora_mapping": self.lora_mapping,
97
"multi_modal_kwargs": self.multi_modal_kwargs,
98
"virtual_engine": self.virtual_engine,
99
+ "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
100
+ "finished_requests_ids": self.finished_requests_ids,
101
}
102
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
103
return tensor_dict
@@ -128,6 +132,8 @@ class ModelInputForGPUWithSamplingMetadata(ModelInputForGPU):
132
"lora_mapping": self.lora_mapping,
133
"multi_modal_kwargs": self.multi_modal_kwargs,
134
"virtual_engine": self.virtual_engine,
135
+ "request_ids_to_seq_ids": self.request_ids_to_seq_ids,
136
+ "finished_requests_ids": self.finished_requests_ids,
137
}
138
_add_attn_metadata_broadcastable_dict(tensor_dict, self.attn_metadata)
139
_add_sampling_metadata_broadcastable_dict(tensor_dict,
@@ -191,6 +197,10 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
197
]
198
self.graph_memory_pool: Optional[Tuple[
199
int, int]] = None # Set during graph capture.
200
+
201
+ self.has_seqlen_agnostic = model_config.contains_seqlen_agnostic_layers(
202
+ parallel_config)
203
+
204
# When using CUDA graph, the input block tables must be padded to
205
# max_seq_len_to_capture. However, creating the block table in
206
# Python can be expensive. To optimize this, we cache the block table
@@ -317,6 +327,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
327
def _prepare_model_input_tensors(
328
self,
329
seq_group_metadata_list: List[SequenceGroupMetadata],
330
+ finished_requests_ids: Optional[List[str]] = None
331
) -> TModelInputForGPU:
332
"""Helper method to prepare the model input based on a given sequence
333
group. Prepares metadata needed for the base model forward pass but not
@@ -347,6 +358,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
358
block_tables: List[List[int]] = []
359
multi_modal_kwargs_list: Dict[str,
360
List[torch.Tensor]] = defaultdict(list)
361
+ request_ids_to_seq_ids: Dict[str, List[int]] = defaultdict(list)
362
decode_only = True
363
num_prefills = 0
364
num_prefill_tokens = 0
@@ -738,7 +750,11 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
750
k: torch.cat(v, dim=0).to(self.device)
751
for k, v in multi_modal_kwargs_list.items()
752
}
741
-
753
+ request_ids_to_seq_ids = {
754
+ seq_group_metadata.request_id:
755
+ list(seq_group_metadata.seq_data.keys())
756
+ for seq_group_metadata in seq_group_metadata_list
757
+ }
758
return self._model_input_cls(
759
input_tokens=input_tokens_tensor,
760
input_positions=input_positions_tensor,
@@ -748,7 +764,8 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
764
lora_mapping=lora_mapping,
765
lora_requests=lora_requests,
766
multi_modal_kwargs=multi_modal_kwargs,
751
- )
767
+ request_ids_to_seq_ids=request_ids_to_seq_ids,
768
+ finished_requests_ids=finished_requests_ids)
769
770
@torch.inference_mode()
771
def profile_run(self) -> None:
@@ -821,7 +838,9 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
838
# Run the model with the dummy inputs.
839
num_layers = self.model_config.get_num_layers(self.parallel_config)
840
kv_caches = [None] * num_layers
824
- model_input = self.prepare_model_input(seqs)
841
+ finished_requests_ids = [seq.request_id for seq in seqs]
842
+ model_input = self.prepare_model_input(
843
+ seqs, finished_requests_ids=finished_requests_ids)
844
intermediate_tensors = None
845
if not get_pp_group().is_first_rank:
846
intermediate_tensors = self.model.make_empty_intermediate_tensors(
@@ -1033,21 +1052,37 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1052
graph_runner.flashinfer_decode_wrapper = \
1053
decode_wrapper
1054
1036
- graph_runner.capture(
1055
+ capture_inputs = {
1056
+ "input_ids":
1057
input_tokens[:batch_size],
1058
+ "positions":
1059
input_positions[:batch_size],
1060
+ "hidden_or_intermediate_states":
1061
hidden_or_intermediate_states[
1062
virtual_engine] # type: ignore
1063
[:batch_size]
1064
if hidden_or_intermediate_states[virtual_engine]
1065
is not None else None,
1066
+ "intermediate_inputs":
1067
intermediate_inputs[:batch_size]
1068
if intermediate_inputs is not None else None,
1069
+ "kv_caches":
1070
kv_caches[virtual_engine],
1071
+ "attn_metadata":
1072
attn_metadata,
1048
- memory_pool=self.graph_memory_pool,
1049
- stream=graph_capture_context.stream,
1050
- )
1073
+ "memory_pool":
1074
+ self.graph_memory_pool,
1075
+ "stream":
1076
+ graph_capture_context.stream
1077
+ }
1078
+ if self.has_seqlen_agnostic:
1079
+ # Only used by Mamba-based models CUDA graph atm (Jamba)
1080
+ capture_inputs.update({
1081
+ "seqlen_agnostic_capture_inputs":
1082
+ self.model.get_seqlen_agnostic_capture_inputs(
1083
+ batch_size)
1084
+ })
1085
+ graph_runner.capture(**capture_inputs)
1086
self.graph_memory_pool = graph_runner.graph.pool()
1087
self.graph_runners[virtual_engine][batch_size] = (
1088
graph_runner)
@@ -1084,6 +1119,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1119
self,
1120
seq_group_metadata_list: List[SequenceGroupMetadata],
1121
virtual_engine: int = 0,
1122
+ finished_requests_ids: Optional[List[str]] = None
1123
) -> ModelInputForGPUWithSamplingMetadata:
1124
"""Prepare the model input based on a given sequence group, including
1125
metadata for the sampling step.
@@ -1099,7 +1135,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1135
If cuda graph is required, this API automatically pads inputs.
1136
"""
1137
model_input = self._prepare_model_input_tensors(
1102
- seq_group_metadata_list)
1138
+ seq_group_metadata_list, finished_requests_ids)
1139
sampling_metadata = SamplingMetadata.prepare(seq_group_metadata_list,
1140
model_input.seq_lens,
1141
model_input.query_lens,
@@ -1175,6 +1211,10 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1211
model_executable = self.model
1212
1213
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
1214
+ seqlen_agnostic_kwargs = {
1215
+ "finished_requests_ids": model_input.finished_requests_ids,
1216
+ "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1217
+ } if self.has_seqlen_agnostic else {}
1218
hidden_or_intermediate_states = model_executable(
1219
input_ids=model_input.input_tokens,
1220
positions=model_input.input_positions,
@@ -1182,7 +1222,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1222
attn_metadata=model_input.attn_metadata,
1223
intermediate_tensors=intermediate_tensors,
1224
**multi_modal_kwargs,
1185
- )
1225
+ **seqlen_agnostic_kwargs)
1226
1227
# Compute the logits in the last pipeline stage.
1228
if not get_pp_group().is_last_rank:
@@ -1305,6 +1345,7 @@ class CUDAGraphRunner:
1345
"positions": positions,
1346
"kv_caches": kv_caches,
1347
"slot_mapping": attn_metadata.slot_mapping,
1348
+ **kwargs,
1349
}
1350
else:
1351
self.input_buffers = {
@@ -1315,6 +1356,7 @@ class CUDAGraphRunner:
1356
"seq_lens_tensor":
1357
attn_metadata.decode_metadata.seq_lens_tensor,
1358
"block_tables": attn_metadata.decode_metadata.block_tables,
1359
+ **kwargs,
1360
}
1361
if intermediate_inputs is not None:
1362
self.input_buffers.update(intermediate_inputs.tensors)
@@ -1349,13 +1391,18 @@ class CUDAGraphRunner:
1391
non_blocking=True)
1392
self.input_buffers["block_tables"].copy_(
1393
attn_metadata.decode_metadata.block_tables, non_blocking=True)
1394
+ if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1395
+ self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1396
+ **kwargs)
1397
if intermediate_tensors is not None:
1398
for key in intermediate_tensors.tensors:
1399
self.input_buffers[key].copy_(intermediate_tensors[key],
1400
non_blocking=True)
1401
# Run the graph.
1402
self.graph.replay()
1358
-
1403
+ if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1404
+ self.model.copy_outputs_after_cuda_graphs(self.input_buffers,
1405
+ **kwargs)
1406
# Return the output tensor.
1407
if get_pp_group().is_last_rank:
1408
return self.output_buffers["hidden_states"]
vllm/worker/model_runner_base.py
+1
@@ -139,6 +139,7 @@ class ModelRunnerBase(ABC, Generic[T]):
139
self,
140
seq_group_metadata_list: List[SequenceGroupMetadata],
141
virtual_engine: int = 0,
142
+ finished_requests_ids: Optional[List[str]] = None,
143
) -> T:
144
"""
145
Prepare the inputs to ModelRunnerBase.execute_model from an execution
vllm/worker/neuron_model_runner.py
+1
@@ -177,6 +177,7 @@ class NeuronModelRunner(ModelRunnerBase[ModelInputForNeuron]):
177
self,
178
seq_group_metadata_list: List[SequenceGroupMetadata],
179
virtual_engine: int = 0,
180
+ finished_requests_ids: Optional[List[str]] = None
181
) -> ModelInputForNeuron:
182
# NOTE: We assume that all sequences in the group are all prompts or
183
# all decodes.
vllm/worker/worker_base.py
+2
-1
@@ -234,7 +234,8 @@ class LocalOrDistributedWorkerBase(WorkerBase):
234
model_input: ModelRunnerInputBase = (
235
self.model_runner.prepare_model_input(
236
execute_model_req.seq_group_metadata_list,
237
- execute_model_req.virtual_engine))
237
+ execute_model_req.virtual_engine,
238
+ execute_model_req.finished_requests_ids))
239
num_steps = execute_model_req.num_steps
240
241
if self.do_metadata_broadcast:
vllm/worker/xpu_model_runner.py
+4
-3
@@ -189,9 +189,10 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPU]):
189
))
190
191
def prepare_model_input(
192
- self,
193
- seq_group_metadata_list: List[SequenceGroupMetadata],
194
- virtual_engine: int = 0,
192
+ self,
193
+ seq_group_metadata_list: List[SequenceGroupMetadata],
194
+ virtual_engine: int = 0,
195
+ finished_requests_ids: Optional[List[str]] = None
196
) -> ModelInputForXPU:
197
multi_modal_input = None
198
if self.is_driver_worker: