A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Model] Support Mamba (#6484)
Tyler Michael Smith committed
Oct 11, 2024 at 11:40 UTC
7342a7d7f87ea3f4e03ec0775093a0f1ce56e2a1
29 files changed
+1603
-343
.buildkite/run-cpu-test-ppc64le.sh
+7
-1
@@ -18,7 +18,13 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
18
# Run basic model test
19
docker exec cpu-test bash -c "
20
pip install pytest matplotlib einops transformers_stream_generator
21
- pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
21
+ pytest -v -s tests/models -m \"not vlm\" \
22
+ --ignore=tests/models/test_embedding.py \
23
+ --ignore=tests/models/test_oot_registration.py \
24
+ --ignore=tests/models/test_registry.py \
25
+ --ignore=tests/models/test_jamba.py \
26
+ --ignore=tests/models/test_mamba.py \
27
+ --ignore=tests/models/test_danube3_4b.py" # Mamba kernels and Danube3-4B on CPU is not supported
28
29
# online inference
30
docker exec cpu-test bash -c "
.buildkite/run-cpu-test.sh
+1
@@ -27,6 +27,7 @@ docker exec cpu-test bash -c "
27
pytest -v -s tests/models/decoder_only/language \
28
--ignore=tests/models/test_fp8.py \
29
--ignore=tests/models/decoder_only/language/test_jamba.py \
30
+ --ignore=tests/models/decoder_only/language/test_mamba.py \
31
--ignore=tests/models/decoder_only/language/test_granitemoe.py \
32
--ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
33
docs/source/models/supported_models.rst
+5
@@ -152,6 +152,11 @@ Text Generation
152
- :code:`meta-llama/Meta-Llama-3.1-405B-Instruct`, :code:`meta-llama/Meta-Llama-3.1-70B`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-70b-hf`, :code:`01-ai/Yi-34B`, etc.
153
- ✅︎
154
- ✅︎
155
+ * - :code:`MambaForCausalLM`
156
+ - Mamba
157
+ - :code:`state-spaces/mamba-130m-hf`, :code:`state-spaces/mamba-790m-hf`, :code:`state-spaces/mamba-2.8b-hf`, etc.
158
+ - ✅︎
159
+ -
160
* - :code:`MiniCPMForCausalLM`
161
- MiniCPM
162
- :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
tests/kernels/test_attention_selector.py
+21
-16
@@ -20,22 +20,22 @@ def test_env(name: str, device: str, monkeypatch):
20
21
if device == "cpu":
22
with patch("vllm.attention.selector.is_cpu", return_value=True):
23
- backend = which_attn_to_use(8, 16, 8, None, torch.float16,
24
- torch.float16, 16)
23
+ backend = which_attn_to_use(16, None, torch.float16, torch.float16,
24
+ 16, False)
25
assert backend.name == "TORCH_SDPA"
26
elif device == "hip":
27
with patch("vllm.attention.selector.is_hip", return_value=True):
28
- backend = which_attn_to_use(8, 16, 8, None, torch.float16,
29
- torch.float16, 16)
28
+ backend = which_attn_to_use(16, None, torch.float16, torch.float16,
29
+ 16, False)
30
assert backend.name == "ROCM_FLASH"
31
elif device == "openvino":
32
with patch("vllm.attention.selector.is_openvino", return_value=True):
33
- backend = which_attn_to_use(8, 16, 8, None, torch.float16,
34
- torch.float16, 16)
33
+ backend = which_attn_to_use(16, None, torch.float16, torch.float16,
34
+ 16, False)
35
assert backend.name == "OPENVINO"
36
else:
37
- backend = which_attn_to_use(8, 16, 8, None, torch.float16,
38
- torch.float16, 16)
37
+ backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
38
+ False)
39
assert backend.name == name
40
41
@@ -46,32 +46,37 @@ def test_flash_attn(monkeypatch):
46
47
# Unsupported CUDA arch
48
with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
49
- backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
49
+ backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
50
assert backend.name != STR_FLASH_ATTN_VAL
51
52
# Unsupported data type
53
- backend = which_attn_to_use(8, 16, 8, None, torch.float8_e4m3fn, None, 16)
53
+ backend = which_attn_to_use(16, None, torch.float8_e4m3fn, None, 16, False)
54
assert backend.name != STR_FLASH_ATTN_VAL
55
56
# Unsupported kv cache data type
57
- backend = which_attn_to_use(8, 16, 8, None, torch.float16, "fp8", 16)
57
+ backend = which_attn_to_use(16, None, torch.float16, "fp8", 16, False)
58
assert backend.name != STR_FLASH_ATTN_VAL
59
60
# Unsupported block size
61
- backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 8)
61
+ backend = which_attn_to_use(16, None, torch.float16, None, 8, False)
62
assert backend.name != STR_FLASH_ATTN_VAL
63
64
# Unsupported sliding window
65
- backend = which_attn_to_use(8, 16, 8, 1, torch.float16, None, 16)
65
+ backend = which_attn_to_use(16, 1, torch.float16, None, 16, False)
66
assert backend.name != STR_FLASH_ATTN_VAL
67
68
# flash-attn is not installed
69
with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70
- backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
70
+ backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
71
assert backend.name != STR_FLASH_ATTN_VAL
72
73
# Unsupported head size
74
- backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
74
+ backend = which_attn_to_use(17, None, torch.float16, None, 16, False)
75
+ assert backend.name != STR_FLASH_ATTN_VAL
76
+
77
+ # Attention-free models should bypass env and use PlaceholderAttention
78
+ backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
79
+ True)
80
assert backend.name != STR_FLASH_ATTN_VAL
81
82
@@ -79,4 +84,4 @@ def test_invalid_env(monkeypatch):
84
"""Throw an exception if the backend name is invalid."""
85
override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
86
with pytest.raises(ValueError):
82
- which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
87
+ which_attn_to_use(16, None, torch.float16, None, 16, False)
tests/models/decoder_only/language/test_mamba.py
new
+295
@@ -0,0 +1,295 @@
1
+"""Compare the outputs of HF and vLLM when using greedy sampling for Mamba.
2
+
3
+Run `pytest tests/models/test_mamba.py`.
4
+"""
5
+import pytest
6
+from transformers import AutoModelForCausalLM, AutoTokenizer
7
+
8
+from vllm.sampling_params import SamplingParams
9
+from vllm.worker.model_runner import _get_graph_batch_size
10
+
11
+from ...utils import check_outputs_equal
12
+
13
+MODELS = ["state-spaces/mamba-130m-hf"]
14
+
15
+
16
+# Use lower-level interfaces to create this greedy generator, as mamba will
17
+# choke on the model_kwarg 'attention_mask' if hf_model.generate_greedy is used.
18
+def generate_greedy(model_name, example_prompts, max_tokens):
19
+ # Create a text generation pipeline
20
+ tokenizer = AutoTokenizer.from_pretrained(model_name)
21
+ model = AutoModelForCausalLM.from_pretrained(model_name)
22
+
23
+ # Generate texts from the prompts
24
+ outputs = []
25
+ for prompt in example_prompts:
26
+ # Tokenize the input prompt with truncation
27
+ inputs = tokenizer(prompt, return_tensors="pt", truncation=True)
28
+ input_ids = inputs["input_ids"].to(model.device)
29
+
30
+ # Generate text using the model's generate method directly
31
+ generated_ids = model.generate(input_ids, max_new_tokens=max_tokens)
32
+ generated_text = tokenizer.decode(generated_ids[0],
33
+ skip_special_tokens=True)
34
+
35
+ outputs.append((generated_ids[0].tolist(), generated_text))
36
+
37
+ return outputs
38
+
39
+
40
+@pytest.mark.parametrize("model", MODELS)
41
+@pytest.mark.parametrize("dtype", ["float"])
42
+@pytest.mark.parametrize("max_tokens", [96])
43
+def test_models(
44
+ vllm_runner,
45
+ example_prompts,
46
+ model: str,
47
+ dtype: str,
48
+ max_tokens: int,
49
+) -> None:
50
+ hf_outputs = generate_greedy(model, example_prompts, max_tokens)
51
+
52
+ with vllm_runner(model, dtype=dtype) as vllm_model:
53
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54
+
55
+ for i in range(len(example_prompts)):
56
+ hf_output_ids, hf_output_str = hf_outputs[i]
57
+ vllm_output_ids, vllm_output_str = vllm_outputs[i]
58
+ assert hf_output_str == vllm_output_str, (
59
+ f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
60
+ assert hf_output_ids == vllm_output_ids, (
61
+ f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
62
+
63
+
64
+@pytest.mark.parametrize("model", MODELS)
65
+@pytest.mark.parametrize("dtype", ["float"])
66
+@pytest.mark.parametrize("max_tokens", [96])
67
+def test_batching(
68
+ vllm_runner,
69
+ example_prompts,
70
+ model: str,
71
+ dtype: str,
72
+ max_tokens: int,
73
+) -> None:
74
+ # To pass the small model tests, we need full precision.
75
+ for_loop_outputs = []
76
+ with vllm_runner(model, dtype=dtype) as vllm_model:
77
+ for prompt in example_prompts:
78
+ for_loop_outputs.append(
79
+ vllm_model.generate_greedy([prompt], max_tokens)[0])
80
+
81
+ batched_outputs = vllm_model.generate_greedy(example_prompts,
82
+ max_tokens)
83
+
84
+ check_outputs_equal(
85
+ outputs_0_lst=for_loop_outputs,
86
+ outputs_1_lst=batched_outputs,
87
+ name_0="for_loop_vllm",
88
+ name_1="batched_vllm",
89
+ )
90
+
91
+
92
+@pytest.mark.parametrize("model", MODELS)
93
+@pytest.mark.parametrize("dtype", ["float"])
94
+@pytest.mark.parametrize("max_tokens", [10])
95
+def test_chunked_prefill_with_parallel_sampling(vllm_runner, example_prompts,
96
+ model: str, dtype: str,
97
+ max_tokens: int) -> None:
98
+ # Tests chunked prefill in conjunction with n>1. In this case, prefill is
99
+ # populated with decoding tokens and we test that it doesn't fail.
100
+ # This test might fail if cache is not allocated correctly for n > 1
101
+ # decoding steps inside a chunked prefill forward pass (where we have both
102
+ # prefill and decode together )
103
+ sampling_params = SamplingParams(n=3,
104
+ temperature=1,
105
+ seed=0,
106
+ max_tokens=max_tokens)
107
+ with vllm_runner(
108
+ model,
109
+ dtype=dtype,
110
+ enable_chunked_prefill=True,
111
+ max_num_batched_tokens=30,
112
+ max_num_seqs=10 # forces prefill chunks with decoding
113
+ ) as vllm_model:
114
+ vllm_model.generate(example_prompts, sampling_params)
115
+
116
+
117
+@pytest.mark.parametrize("model", MODELS)
118
+@pytest.mark.parametrize("dtype", ["float"])
119
+@pytest.mark.parametrize("max_tokens", [32])
120
+@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
121
+def test_chunked_prefill(vllm_runner, example_prompts, model: str, dtype: str,
122
+ max_tokens: int,
123
+ chunked_prefill_token_size: int) -> None:
124
+ """
125
+ Checks exact match decode between huggingface model and vllm runner with
126
+ chunked prefill.
127
+ """
128
+ max_num_seqs = chunked_prefill_token_size
129
+ max_num_batched_tokens = chunked_prefill_token_size
130
+
131
+ non_chunked = generate_greedy(model, example_prompts, max_tokens)
132
+
133
+ with vllm_runner(model,
134
+ dtype=dtype,
135
+ enable_chunked_prefill=True,
136
+ max_num_batched_tokens=max_num_batched_tokens,
137
+ max_num_seqs=max_num_seqs) as vllm_model:
138
+ chunked = vllm_model.generate_greedy(example_prompts,
139
+ max_tokens=max_tokens)
140
+
141
+ check_outputs_equal(
142
+ outputs_0_lst=chunked,
143
+ outputs_1_lst=non_chunked,
144
+ name_0="chunked",
145
+ name_1="non_chunked",
146
+ )
147
+
148
+
149
+@pytest.mark.parametrize("model", MODELS)
150
+@pytest.mark.parametrize("dtype", ["float"])
151
+@pytest.mark.parametrize("max_tokens", [15])
152
+def test_parallel_sampling(
153
+ vllm_runner,
154
+ example_prompts,
155
+ model: str,
156
+ dtype: str,
157
+ max_tokens: int,
158
+) -> None:
159
+
160
+ with vllm_runner(model, dtype=dtype) as vllm_model:
161
+ for_loop_outputs = []
162
+ for _ in range(10):
163
+ for_loop_outputs.append(
164
+ # using example_prompts index 1 instead of 0 since with 0 the
165
+ # logprobs get really close and the test doesn't pass
166
+ vllm_model.generate_greedy([example_prompts[1]], max_tokens)
167
+ [0])
168
+ sampling_params = SamplingParams(n=10,
169
+ temperature=0.001,
170
+ seed=0,
171
+ max_tokens=max_tokens)
172
+ n_lt_1_outputs = vllm_model.generate([example_prompts[1]],
173
+ sampling_params)
174
+ token_ids, texts = n_lt_1_outputs[0]
175
+ n_lt_1_outputs = [(token_id, text)
176
+ for token_id, text in zip(token_ids, texts)]
177
+
178
+ check_outputs_equal(
179
+ outputs_0_lst=n_lt_1_outputs,
180
+ outputs_1_lst=for_loop_outputs,
181
+ name_0="vllm_n_lt_1_outputs",
182
+ name_1="vllm",
183
+ )
184
+
185
+
186
+@pytest.mark.parametrize("model", MODELS)
187
+@pytest.mark.parametrize("dtype", ["bfloat16"])
188
+@pytest.mark.parametrize("max_tokens", [20])
189
+def test_mamba_cache_cg_padding(
190
+ vllm_runner,
191
+ example_prompts,
192
+ model: str,
193
+ dtype: str,
194
+ max_tokens: int,
195
+) -> None:
196
+ # This test is for verifying that mamba cache is padded to CG captured
197
+ # batch size. If it's not, a torch RuntimeError will be raised because
198
+ # tensor dimensions aren't compatible
199
+ while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
200
+ example_prompts.append(example_prompts[0])
201
+
202
+ try:
203
+ with vllm_runner(model, dtype=dtype) as vllm_model:
204
+ vllm_model.generate_greedy(example_prompts, max_tokens)
205
+ except RuntimeError:
206
+ pytest.fail(
207
+ "Couldn't run batch size which is not equal to a Cuda Graph "
208
+ "captured batch size. "
209
+ "Could be related to mamba cache not padded correctly")
210
+
211
+
212
+@pytest.mark.parametrize("model", MODELS)
213
+@pytest.mark.parametrize("dtype", ["float"])
214
+@pytest.mark.parametrize("max_tokens", [20])
215
+def test_models_preemption_recompute(
216
+ vllm_runner,
217
+ example_prompts,
218
+ model: str,
219
+ dtype: str,
220
+ max_tokens: int,
221
+) -> None:
222
+ # Tests that outputs are identical with and w/o preemtions (recompute)
223
+ assert dtype == "float"
224
+
225
+ with vllm_runner(model, dtype=dtype) as vllm_model:
226
+ vllm_model.model.llm_engine.scheduler[
227
+ 0].ENABLE_ARTIFICIAL_PREEMPT = True
228
+ preempt_vllm_outputs = vllm_model.generate_greedy(
229
+ example_prompts, max_tokens)
230
+
231
+ vllm_model.model.llm_engine.scheduler[
232
+ 0].ENABLE_ARTIFICIAL_PREEMPT = False
233
+ vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
234
+
235
+ check_outputs_equal(
236
+ outputs_0_lst=preempt_vllm_outputs,
237
+ outputs_1_lst=vllm_outputs,
238
+ name_0="vllm_preepmtions",
239
+ name_1="vllm",
240
+ )
241
+
242
+
243
+@pytest.mark.parametrize("model", MODELS)
244
+@pytest.mark.parametrize("dtype", ["float"])
245
+def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
246
+ vllm_runner,
247
+ model: str,
248
+ dtype: str,
249
+ example_prompts,
250
+) -> None:
251
+ # This test is for verifying that the Mamba inner state management doesn't
252
+ # collapse in case where the number of incoming requests and
253
+ # finished_requests_ids is larger than the maximum Mamba block capacity.
254
+ # This could generally happen due to the fact that Mamba does support
255
+ # statelessness mechanism where it can cleanup new incoming requests in
256
+ # a single step.
257
+ try:
258
+ with vllm_runner(model, dtype=dtype, max_num_seqs=10) as vllm_model:
259
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
260
+ except ValueError:
261
+ pytest.fail("Mamba inner state wasn't cleaned up properly between"
262
+ "steps finished requests registered unnecessarily ")
263
+
264
+
265
+@pytest.mark.parametrize("model", MODELS)
266
+@pytest.mark.parametrize("dtype", ["float"])
267
+def test_state_cleanup(
268
+ vllm_runner,
269
+ model: str,
270
+ dtype: str,
271
+ example_prompts,
272
+) -> None:
273
+ # This test is for verifying that the Mamba state is cleaned up between
274
+ # steps, If its not cleaned, an error would be expected.
275
+ try:
276
+ with vllm_runner(model, dtype=dtype) as vllm_model:
277
+ for _ in range(10):
278
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
279
+ except ValueError:
280
+ pytest.fail("Mamba inner state wasn't cleaned up between states, "
281
+ "could be related to finished_requests_ids")
282
+
283
+
284
+@pytest.mark.parametrize("model", MODELS)
285
+@pytest.mark.parametrize("dtype", ["float"])
286
+def test_model_print(
287
+ vllm_runner,
288
+ model: str,
289
+ dtype: str,
290
+) -> None:
291
+ with vllm_runner(model, dtype=dtype) as vllm_model:
292
+ # This test is for verifying whether the model's extra_repr
293
+ # can be printed correctly.
294
+ print(vllm_model.model.llm_engine.model_executor.driver_worker.
295
+ model_runner.model)
vllm/attention/backends/placeholder_attn.py
new
+324
@@ -0,0 +1,324 @@
1
+from dataclasses import dataclass
2
+from typing import TYPE_CHECKING, List, Optional, Tuple, Type
3
+
4
+import torch
5
+
6
+from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7
+ AttentionMetadata,
8
+ AttentionMetadataBuilder)
9
+from vllm.attention.backends.utils import CommonAttentionState
10
+
11
+if TYPE_CHECKING:
12
+ from vllm.worker.model_runner import ModelInputForGPUBuilder
13
+
14
+# Placeholder attention backend for models like Mamba and embedding models that
15
+# lack attention.
16
+
17
+
18
+class PlaceholderAttentionBackend(AttentionBackend):
19
+ """Placeholder backend for when no attention is needed."""
20
+
21
+ @staticmethod
22
+ def get_name() -> str:
23
+ return "placeholder-attn"
24
+
25
+ @staticmethod
26
+ def get_impl_cls() -> Type["PlaceholderAttentionImpl"]:
27
+ return PlaceholderAttentionImpl
28
+
29
+ @staticmethod
30
+ def get_builder_cls() -> Type["PlaceholderAttentionMetadataBuilder"]:
31
+ return PlaceholderAttentionMetadataBuilder
32
+
33
+ @staticmethod
34
+ def get_metadata_cls() -> Type["PlaceholderAttentionMetadata"]:
35
+ return PlaceholderAttentionMetadata
36
+
37
+ @staticmethod
38
+ def get_state_cls() -> Type["CommonAttentionState"]:
39
+ return CommonAttentionState
40
+
41
+ @staticmethod
42
+ def get_kv_cache_shape(
43
+ num_blocks: int,
44
+ block_size: int,
45
+ num_kv_heads: int,
46
+ head_size: int,
47
+ ) -> Tuple[int, ...]:
48
+ return (1, 1, 1, 1, 1)
49
+
50
+ @staticmethod
51
+ def swap_blocks(
52
+ src_kv_cache: torch.Tensor,
53
+ dst_kv_cache: torch.Tensor,
54
+ src_to_dst: torch.Tensor,
55
+ ) -> None:
56
+ return
57
+
58
+ @staticmethod
59
+ def copy_blocks(
60
+ kv_caches: List[torch.Tensor],
61
+ src_to_dists: torch.Tensor,
62
+ ) -> None:
63
+ return
64
+
65
+
66
+@dataclass
67
+class PlaceholderAttentionMetadata(AttentionMetadata):
68
+ """Attention metadata for prefill and decode batched together."""
69
+ # (batch_size,). The sequence length per sequence. Sequence length means
70
+ # the computed tokens + new tokens None if it is a decoding.
71
+ seq_lens: Optional[List[int]]
72
+ # seq_lens stored as a tensor.
73
+ seq_lens_tensor: Optional[torch.Tensor]
74
+
75
+ # Maximum query length in the batch.
76
+ max_query_len: Optional[int]
77
+
78
+ # Number of query tokens for each request in the batch.
79
+ # Currently, we require that all requests have the same number of query
80
+ # tokens during the decoding phase. When speculavie decoding is enabled,
81
+ # decode_query_len might be greater than 1. In all other cases, it is 1.
82
+ decode_query_len: Optional[int]
83
+
84
+ # Maximum sequence length among prefill batch. 0 if there are decoding
85
+ # requests only.
86
+ max_prefill_seq_len: int
87
+ # Maximum sequence length among decode batch. 0 if there are prefill
88
+ # requests only.
89
+ max_decode_seq_len: int
90
+ # (batch_size + 1,). The cumulative subquery lengths of the sequences in
91
+ # the batch, used to index into subquery. E.g., if the subquery length
92
+ # is [4, 6], it is [0, 4, 10].
93
+ query_start_loc: Optional[torch.Tensor]
94
+ # (batch_size + 1,). The cumulative sequence lengths of the sequences in
95
+ # the batch, used to index into sequence. E.g., if the sequence length is
96
+ # [4, 6], it is [0, 4, 10].
97
+ seq_start_loc: Optional[torch.Tensor]
98
+ # (batch_size,) A tensor of context lengths (tokens that are computed
99
+ # so far).
100
+ context_lens_tensor: Optional[torch.Tensor]
101
+
102
+ # (batch_size, max_blocks_per_seq).
103
+ # Block addresses per sequence. (Seq id -> list of physical block)
104
+ # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
105
+ # in the kv cache. Each block can contain up to block_size tokens.
106
+ # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
107
+ # captured.
108
+ block_tables: Optional[torch.Tensor]
109
+
110
+ # Whether or not if cuda graph is enabled.
111
+ # Cuda-graph is currently enabled for decoding only.
112
+ # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
113
+ use_cuda_graph: bool
114
+
115
+ _cached_prefill_metadata: Optional["PlaceholderAttentionMetadata"] = None
116
+ _cached_decode_metadata: Optional["PlaceholderAttentionMetadata"] = None
117
+
118
+ @property
119
+ def prefill_metadata(self) -> Optional["PlaceholderAttentionMetadata"]:
120
+ if self.num_prefills == 0:
121
+ return None
122
+
123
+ if self._cached_prefill_metadata is not None:
124
+ return self._cached_prefill_metadata
125
+
126
+ assert self.seq_lens is not None
127
+ assert self.seq_lens_tensor is not None
128
+ assert self.query_start_loc is not None
129
+ assert self.context_lens_tensor is not None
130
+ assert self.seq_start_loc is not None
131
+
132
+ # Placeholders
133
+ slot_mapping = torch.empty(0)
134
+ block_tables = torch.empty(0)
135
+
136
+ self._cached_prefill_metadata = PlaceholderAttentionMetadata(
137
+ num_prefills=self.num_prefills,
138
+ num_prefill_tokens=self.num_prefill_tokens,
139
+ num_decode_tokens=0,
140
+ slot_mapping=slot_mapping,
141
+ seq_lens=self.seq_lens[:self.num_prefills],
142
+ seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
143
+ decode_query_len=0,
144
+ max_query_len=self.max_query_len,
145
+ max_prefill_seq_len=self.max_prefill_seq_len,
146
+ max_decode_seq_len=0,
147
+ query_start_loc=self.query_start_loc[:self.num_prefills + 1],
148
+ seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
149
+ context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
150
+ block_tables=block_tables,
151
+ use_cuda_graph=False,
152
+ )
153
+ return self._cached_prefill_metadata
154
+
155
+ @property
156
+ def decode_metadata(self) -> Optional["PlaceholderAttentionMetadata"]:
157
+ if self.num_decode_tokens == 0:
158
+ return None
159
+
160
+ if self._cached_decode_metadata is not None:
161
+ return self._cached_decode_metadata
162
+ assert self.seq_lens_tensor is not None
163
+
164
+ # Placeholders
165
+ slot_mapping = torch.empty(0)
166
+ block_tables = torch.empty(0)
167
+
168
+ self._cached_decode_metadata = PlaceholderAttentionMetadata(
169
+ num_prefills=0,
170
+ num_prefill_tokens=0,
171
+ num_decode_tokens=self.num_decode_tokens,
172
+ slot_mapping=slot_mapping,
173
+ seq_lens=None,
174
+ seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
175
+ decode_query_len=self.decode_query_len,
176
+ max_query_len=None,
177
+ max_prefill_seq_len=0,
178
+ max_decode_seq_len=self.max_decode_seq_len,
179
+ query_start_loc=None,
180
+ seq_start_loc=None,
181
+ context_lens_tensor=None,
182
+ block_tables=block_tables,
183
+ use_cuda_graph=self.use_cuda_graph,
184
+ )
185
+ return self._cached_decode_metadata
186
+
187
+
188
+class PlaceholderAttentionMetadataBuilder(
189
+ AttentionMetadataBuilder[PlaceholderAttentionMetadata]):
190
+
191
+ def __init__(self, input_builder: "ModelInputForGPUBuilder"):
192
+ self.prefill_seq_lens: List[int] = []
193
+ self.context_lens: List[int] = []
194
+ self.curr_seq_lens: List[int] = []
195
+ self.num_prefills = 0
196
+ self.num_prefill_tokens = 0
197
+ self.num_decode_tokens = 0
198
+
199
+ self.input_builder = input_builder
200
+ self.runner = input_builder.runner
201
+
202
+ def _add_seq_group(
203
+ self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
204
+ chunked_prefill_enabled: bool):
205
+ """Add a sequence group to the metadata. Specifically update/append
206
+ 1. context length.
207
+ """
208
+ is_prompt = inter_data.is_prompt
209
+
210
+ for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
211
+ curr_sliding_window_block) in zip(
212
+ inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
213
+ inter_data.orig_seq_lens, inter_data.seq_lens,
214
+ inter_data.query_lens, inter_data.context_lens,
215
+ inter_data.curr_sliding_window_blocks):
216
+ self.context_lens.append(context_len)
217
+
218
+ if is_prompt:
219
+ self.num_prefills += 1
220
+ self.num_prefill_tokens += token_len
221
+ self.prefill_seq_lens.append(seq_len)
222
+ else:
223
+ assert query_len == 1, (
224
+ "seq_len: {}, context_len: {}, query_len: {}".format(
225
+ seq_len, context_len, query_len))
226
+ self.num_decode_tokens += query_len
227
+ self.curr_seq_lens.append(curr_seq_len)
228
+
229
+ def build(self, seq_lens: List[int], query_lens: List[int],
230
+ cuda_graph_pad_size: int, batch_size: int):
231
+ """Build attention metadata with on-device tensors.
232
+
233
+ Args:
234
+ seq_lens: The maybe padded sequence lengths of the input sequences.
235
+ query_lens: The query lengths of the input sequences.
236
+ cuda_graph_pad_size: The padding size for cuda graph.
237
+ -1 if cuda graph is not used.
238
+ batch_size: The maybe padded batch size.
239
+ """
240
+ for inter_data in self.input_builder.inter_data_list:
241
+ self._add_seq_group(inter_data,
242
+ self.input_builder.chunked_prefill_enabled)
243
+
244
+ device = self.runner.device
245
+ use_captured_graph = cuda_graph_pad_size != -1
246
+
247
+ logits_soft_cap = getattr(self.runner.model_config.hf_config,
248
+ "attn_logit_softcapping", None)
249
+ if logits_soft_cap is not None:
250
+ raise ValueError(
251
+ "Please use Flashinfer backend for models with logits_soft_cap"
252
+ " (i.e., Gemma-2). Otherwise, the output might be wrong."
253
+ " Set Flashinfer backend by "
254
+ "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
255
+
256
+ max_query_len = max(query_lens)
257
+ decode_query_lens = query_lens[self.num_prefills:]
258
+ if len(decode_query_lens) > 0:
259
+ decode_query_len = max(decode_query_lens)
260
+ else:
261
+ decode_query_len = 1
262
+ max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
263
+ max_decode_seq_len = max(self.curr_seq_lens, default=0)
264
+ num_decode_tokens = self.num_decode_tokens
265
+
266
+ if use_captured_graph:
267
+ num_decode_tokens = batch_size
268
+
269
+ assert max_query_len > 0, ("query_lens: {}".format(query_lens))
270
+
271
+ context_lens_tensor = torch.tensor(self.context_lens,
272
+ dtype=torch.int,
273
+ device=device)
274
+ seq_lens_tensor = torch.tensor(seq_lens,
275
+ dtype=torch.int,
276
+ device=device)
277
+ query_lens_tensor = torch.tensor(query_lens,
278
+ dtype=torch.long,
279
+ device=device)
280
+ query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
281
+ dtype=torch.int32,
282
+ device=device)
283
+ seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
284
+ dtype=torch.int32,
285
+ device=device)
286
+ torch.cumsum(seq_lens_tensor,
287
+ dim=0,
288
+ dtype=seq_start_loc.dtype,
289
+ out=seq_start_loc[1:])
290
+ torch.cumsum(query_lens_tensor,
291
+ dim=0,
292
+ dtype=query_start_loc.dtype,
293
+ out=query_start_loc[1:])
294
+
295
+ # Placeholders
296
+ slot_mapping = torch.empty(0)
297
+ block_tables = torch.empty(0)
298
+
299
+ return PlaceholderAttentionMetadata(
300
+ num_prefills=self.num_prefills,
301
+ slot_mapping=slot_mapping,
302
+ num_prefill_tokens=self.num_prefill_tokens,
303
+ num_decode_tokens=num_decode_tokens,
304
+ seq_lens=seq_lens,
305
+ seq_lens_tensor=seq_lens_tensor,
306
+ max_query_len=max_query_len,
307
+ decode_query_len=decode_query_len,
308
+ max_prefill_seq_len=max_prefill_seq_len,
309
+ max_decode_seq_len=max_decode_seq_len,
310
+ query_start_loc=query_start_loc,
311
+ seq_start_loc=seq_start_loc,
312
+ context_lens_tensor=context_lens_tensor,
313
+ block_tables=block_tables,
314
+ use_cuda_graph=use_captured_graph,
315
+ )
316
+
317
+
318
+class PlaceholderAttentionImpl(AttentionImpl):
319
+
320
+ def __init__(self, *args, **kwargs) -> None:
321
+ return
322
+
323
+ def forward(self, *args, **kwargs) -> torch.Tensor:
324
+ raise NotImplementedError
vllm/attention/layer.py
+5
-3
@@ -42,10 +42,12 @@ class Attention(nn.Module):
42
kv_cache_dtype = cache_config.cache_dtype
43
block_size = cache_config.block_size
44
sliding_window = cache_config.sliding_window
45
+ is_attention_free = cache_config.is_attention_free
46
else:
47
kv_cache_dtype = "auto"
48
block_size = 16
49
sliding_window = None
50
+ is_attention_free = False
51
if num_kv_heads is None:
52
num_kv_heads = num_heads
53
@@ -76,9 +78,9 @@ class Attention(nn.Module):
78
# During model initialization, the default dtype is set as the model
79
# weight and activation dtype.
80
dtype = torch.get_default_dtype()
79
- attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
80
- sliding_window, dtype, kv_cache_dtype,
81
- block_size, blocksparse_params
81
+ attn_backend = get_attn_backend(head_size, sliding_window, dtype,
82
+ kv_cache_dtype, block_size,
83
+ is_attention_free, blocksparse_params
84
is not None)
85
impl_cls = attn_backend.get_impl_cls()
86
self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
vllm/attention/selector.py
+14
-7
@@ -24,6 +24,7 @@ class _Backend(enum.Enum):
24
FLASHINFER = enum.auto()
25
PALLAS = enum.auto()
26
IPEX = enum.auto()
27
+ NO_ATTENTION = enum.auto()
28
29
30
def backend_name_to_enum(backend_name: str) -> _Backend:
@@ -88,13 +89,12 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
89
90
@lru_cache(maxsize=None)
91
def get_attn_backend(
91
- num_heads: int,
92
head_size: int,
93
- num_kv_heads: int,
93
sliding_window: Optional[int],
94
dtype: torch.dtype,
95
kv_cache_dtype: Optional[str],
96
block_size: int,
97
+ is_attention_free: bool,
98
is_blocksparse: bool = False,
99
) -> Type[AttentionBackend]:
100
"""Selects which attention backend to use and lazily imports it."""
@@ -105,9 +105,8 @@ def get_attn_backend(
105
BlocksparseFlashAttentionBackend)
106
return BlocksparseFlashAttentionBackend
107
108
- backend = which_attn_to_use(num_heads, head_size, num_kv_heads,
109
- sliding_window, dtype, kv_cache_dtype,
110
- block_size)
108
+ backend = which_attn_to_use(head_size, sliding_window, dtype,
109
+ kv_cache_dtype, block_size, is_attention_free)
110
if backend == _Backend.FLASH_ATTN:
111
from vllm.attention.backends.flash_attn import ( # noqa: F401
112
FlashAttentionBackend)
@@ -146,23 +145,31 @@ def get_attn_backend(
145
logger.info("Using Pallas backend.")
146
from vllm.attention.backends.pallas import PallasAttentionBackend
147
return PallasAttentionBackend
148
+ elif backend == _Backend.NO_ATTENTION:
149
+ from vllm.attention.backends.placeholder_attn import (
150
+ PlaceholderAttentionBackend)
151
+ return PlaceholderAttentionBackend
152
else:
153
raise ValueError("Invalid attention backend.")
154
155
156
def which_attn_to_use(
154
- num_heads: int,
157
head_size: int,
156
- num_kv_heads: int,
158
sliding_window: Optional[int],
159
dtype: torch.dtype,
160
kv_cache_dtype: Optional[str],
161
block_size: int,
162
+ is_attention_free: bool,
163
) -> _Backend:
164
"""Returns which flash attention backend to use."""
165
# Default case.
166
selected_backend = _Backend.FLASH_ATTN
167
168
+ # If there are no attention layers (e.g. we are running Mamba),
169
+ # use the placeholder NO_ATTENTION
170
+ if is_attention_free:
171
+ return _Backend.NO_ATTENTION
172
+
173
# Check whether a particular choice of backend was
174
# previously forced.
175
#
vllm/config.py
+28
-22
@@ -196,6 +196,9 @@ class ModelConfig:
196
if not self.skip_tokenizer_init:
197
self._verify_tokenizer_mode()
198
199
+ self.is_attention_free = self._init_attention_free()
200
+ self.has_inner_state = self._init_has_inner_state()
201
+
202
self.override_neuron_config = override_neuron_config if is_neuron(
203
) else None
204
self._verify_embedding_mode()
@@ -216,6 +219,14 @@ class ModelConfig:
219
220
return None
221
222
+ def _init_attention_free(self) -> bool:
223
+ architectures = getattr(self.hf_config, "architectures", [])
224
+ return ModelRegistry.is_attention_free_model(architectures)
225
+
226
+ def _init_has_inner_state(self) -> bool:
227
+ architectures = getattr(self.hf_config, "architectures", [])
228
+ return ModelRegistry.model_has_inner_state(architectures)
229
+
230
def _verify_tokenizer_mode(self) -> None:
231
tokenizer_mode = self.tokenizer_mode.lower()
232
if tokenizer_mode not in ["auto", "slow", "mistral"]:
@@ -438,6 +449,10 @@ class ModelConfig:
449
# FlashAttention supports only head_size 32, 64, 128, 256,
450
# we need to pad head_size 192 to 256
451
return 256
452
+
453
+ if self.is_attention_free:
454
+ return 0
455
+
456
if hasattr(self.hf_text_config, "head_dim"):
457
return self.hf_text_config.head_dim
458
# FIXME(woosuk): This may not be true for all models.
@@ -469,6 +484,9 @@ class ModelConfig:
484
return getattr(self.hf_config.attn_config, "kv_n_heads",
485
self.hf_config.num_attention_heads)
486
487
+ if self.is_attention_free:
488
+ return 0
489
+
490
attributes = [
491
# For Falcon:
492
"n_head_kv",
@@ -511,31 +529,17 @@ class ModelConfig:
529
start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
530
return end - start
531
514
- def contains_seqlen_agnostic_layers(
515
- self, parallel_config: "ParallelConfig") -> bool:
516
- """True for Mamba/SSM models (Jamba)"""
517
- return self._get_num_seqlen_agnostic_layers(parallel_config) > 0
532
+ def get_num_attention_layers(self,
533
+ parallel_config: "ParallelConfig") -> int:
534
+ if self.is_attention_free:
535
+ return 0
536
519
- def get_layers_block_type(self,
520
- parallel_config: "ParallelConfig") -> List[str]:
537
num_layers = self.get_num_layers(parallel_config)
522
- # Transformers supports layers_block_type @property
523
- return getattr(self.hf_config, "layers_block_type",
524
- ["attention"] * num_layers)
538
526
- def get_num_attention_layers(self,
527
- parallel_config: "ParallelConfig") -> int:
528
- return len([
529
- t for t in self.get_layers_block_type(parallel_config)
530
- if t == "attention"
531
- ])
532
-
533
- def _get_num_seqlen_agnostic_layers(
534
- self, parallel_config: "ParallelConfig") -> int:
535
- return len([
536
- t for t in self.get_layers_block_type(parallel_config)
537
- if t != "attention"
538
- ])
539
+ # Transformers supports layers_block_type @property
540
+ layers = getattr(self.hf_config, "layers_block_type",
541
+ ["attention"] * num_layers)
542
+ return len([t for t in layers if t == "attention"])
543
544
def get_multimodal_config(self) -> "MultiModalConfig":
545
"""
@@ -585,6 +589,7 @@ class CacheConfig:
589
gpu_memory_utilization: float,
590
swap_space: float,
591
cache_dtype: str,
592
+ is_attention_free: bool = False,
593
num_gpu_blocks_override: Optional[int] = None,
594
sliding_window: Optional[int] = None,
595
enable_prefix_caching: bool = False,
@@ -595,6 +600,7 @@ class CacheConfig:
600
self.swap_space_bytes = swap_space * GiB_bytes
601
self.num_gpu_blocks_override = num_gpu_blocks_override
602
self.cache_dtype = cache_dtype
603
+ self.is_attention_free = is_attention_free
604
self.sliding_window = sliding_window
605
self.enable_prefix_caching = enable_prefix_caching
606
self.cpu_offload_gb = cpu_offload_gb
vllm/core/interfaces.py
+4
-4
@@ -36,10 +36,10 @@ class BlockSpaceManager(ABC):
36
from vllm.core.block_manager_v2 import BlockSpaceManagerV2
37
return BlockSpaceManagerV2
38
39
- if version == "embedding":
40
- from vllm.core.embedding_model_block_manager import (
41
- EmbeddingModelBlockSpaceManager)
42
- return EmbeddingModelBlockSpaceManager
39
+ if version == "placeholder":
40
+ from vllm.core.placeholder_block_space_manager import (
41
+ PlaceholderBlockSpaceManager)
42
+ return PlaceholderBlockSpaceManager
43
44
raise ValueError(f"Unknown version {version=}")
45
vllm/core/placeholder_block_space_manager.py
renamed
+5
-4
@@ -5,9 +5,10 @@ from vllm.sequence import Sequence, SequenceGroup
5
from vllm.utils import Device
6
7
8
-class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
9
- """An embedding version of BlockSpaceManager for use in environments
10
- with embedding models where block management is not required.
8
+class PlaceholderBlockSpaceManager(BlockSpaceManager):
9
+ """A version of BlockSpaceManager for use in environments
10
+ where block management is not required.
11
+ For example: embedding models or attention-free models like Mamba.
12
13
This class provides the same interface as BlockSpaceManager, but its
14
methods perform no actions or return simple values like True in specific
@@ -40,7 +41,7 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
41
seq: Sequence,
42
num_lookahead_slots: int,
43
) -> List[Tuple[int, int]]:
43
- return None # type: ignore
44
+ return []
45
46
def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
47
pass
vllm/core/scheduler.py
+3
-2
@@ -314,8 +314,9 @@ class Scheduler:
314
version = "v1"
315
if self.scheduler_config.use_v2_block_manager:
316
version = "v2"
317
- if self.scheduler_config.embedding_mode:
318
- version = "embedding"
317
+ if (self.scheduler_config.embedding_mode
318
+ or self.cache_config.is_attention_free):
319
+ version = "placeholder"
320
321
BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
322
version)
vllm/engine/arg_utils.py
+2
-5
@@ -912,6 +912,7 @@ class EngineArgs:
912
gpu_memory_utilization=self.gpu_memory_utilization,
913
swap_space=self.swap_space,
914
cache_dtype=self.kv_cache_dtype,
915
+ is_attention_free=model_config.is_attention_free,
916
num_gpu_blocks_override=self.num_gpu_blocks_override,
917
sliding_window=model_config.get_sliding_window(),
918
enable_prefix_caching=self.enable_prefix_caching,
@@ -945,13 +946,9 @@ class EngineArgs:
946
use_sliding_window = (model_config.get_sliding_window()
947
is not None)
948
use_spec_decode = self.speculative_model is not None
948
- has_seqlen_agnostic_layers = (
949
- model_config.contains_seqlen_agnostic_layers(
950
- parallel_config))
949
if (is_gpu and not use_sliding_window and not use_spec_decode
950
and not self.enable_lora
953
- and not self.enable_prompt_adapter
954
- and not has_seqlen_agnostic_layers):
951
+ and not self.enable_prompt_adapter):
952
self.enable_chunked_prefill = True
953
logger.warning(
954
"Chunked prefill is enabled by default for models with "
vllm/model_executor/model_loader/weight_utils.py
+34
-1
@@ -6,7 +6,8 @@ import json
6
import os
7
import tempfile
8
from collections import defaultdict
9
-from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union
9
+from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
10
+ Tuple, Union)
11
12
import filelock
13
import gguf
@@ -559,6 +560,38 @@ def row_parallel_weight_loader(param: torch.Tensor,
560
return default_weight_loader(param, loaded_weight)
561
562
563
+LoaderFunction = Callable[[torch.Tensor, torch.Tensor], torch.Tensor]
564
+
565
+
566
+def sharded_weight_loader(shard_axis: int) -> LoaderFunction:
567
+ """Create a weight loader that shards the weights along the given axis"""
568
+
569
+ def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
570
+ tp_rank = get_tensor_model_parallel_rank()
571
+
572
+ shard_size = param.data.shape[shard_axis]
573
+ start_idx = tp_rank * shard_size
574
+ loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
575
+
576
+ return default_weight_loader(param, loaded_weight)
577
+
578
+ return loader
579
+
580
+
581
+def composed_weight_loader(
582
+ loader: LoaderFunction, fn: Callable[[torch.Tensor],
583
+ torch.Tensor]) -> LoaderFunction:
584
+ """Create a weight loader that post-processes the weights after loading"""
585
+
586
+ def composed_loader(param: torch.Tensor,
587
+ loaded_weight: torch.Tensor) -> None:
588
+ loader(param, loaded_weight)
589
+ param.data.copy_(fn(param))
590
+ return
591
+
592
+ return composed_loader
593
+
594
+
595
def initialize_dummy_weights(
596
model: torch.nn.Module,
597
low: float = -1e-3,
vllm/model_executor/models/interfaces.py
+44
-1
@@ -271,7 +271,7 @@ class HasInnerState(Protocol):
271
"""
272
A flag that indicates this model has inner state.
273
Models that has inner state usually need access to the scheduler_config
274
- for max_num_seqs ,etc... (Currently only used by Jamba)
274
+ for max_num_seqs, etc. True for e.g. both Mamba and Jamba.
275
"""
276
277
def __init__(self,
@@ -307,3 +307,46 @@ def has_inner_state(
307
return isinstance(model, _HasInnerStateType)
308
309
return isinstance(model, HasInnerState)
310
+
311
+
312
+@runtime_checkable
313
+class IsAttentionFree(Protocol):
314
+ """The interface required for all models like Mamba that lack attention,
315
+ but do have state whose size is constant wrt the number of tokens."""
316
+
317
+ is_attention_free: ClassVar[Literal[True]] = True
318
+ """
319
+ A flag that indicates this model has no attention.
320
+ Used for block manager and attention backend selection.
321
+ True for Mamba but not Jamba.
322
+ """
323
+
324
+ def __init__(self) -> None:
325
+ ...
326
+
327
+
328
+@runtime_checkable
329
+class _IsAttentionFreeType(Protocol):
330
+ is_attention_free: ClassVar[Literal[True]]
331
+
332
+ def __init__(self) -> None:
333
+ ...
334
+
335
+
336
+@overload
337
+def is_attention_free(model: object) -> TypeIs[IsAttentionFree]:
338
+ ...
339
+
340
+
341
+@overload
342
+def is_attention_free(model: Type[object]) -> TypeIs[Type[IsAttentionFree]]:
343
+ ...
344
+
345
+
346
+def is_attention_free(
347
+ model: Union[Type[object], object]
348
+) -> Union[TypeIs[Type[IsAttentionFree]], TypeIs[IsAttentionFree]]:
349
+ if isinstance(model, type):
350
+ return isinstance(model, _IsAttentionFreeType)
351
+
352
+ return isinstance(model, IsAttentionFree)
vllm/model_executor/models/jamba.py
+32
-229
@@ -1,18 +1,16 @@
1
# coding=utf-8
2
"""Inference-only Jamba model."""
3
from dataclasses import dataclass
4
-from typing import Dict, Iterable, List, Optional, Tuple
4
+from typing import Iterable, List, Optional, Tuple
5
6
import torch
7
from torch import nn
8
-from torch.nn.parameter import Parameter
8
from transformers import JambaConfig
9
10
from vllm.attention.backends.abstract import AttentionMetadata
11
from vllm.attention.layer import Attention
12
from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
14
-from vllm.distributed import (get_tensor_model_parallel_rank,
15
- get_tensor_model_parallel_world_size)
13
+from vllm.distributed import get_tensor_model_parallel_world_size
14
from vllm.model_executor.layers.fused_moe import FusedMoE
15
from vllm.model_executor.layers.layernorm import RMSNorm
16
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -29,7 +27,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
27
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
28
from vllm.model_executor.layers.vocab_parallel_embedding import (
29
DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
32
-from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30
+from vllm.model_executor.model_loader.weight_utils import (
31
+ composed_weight_loader, default_weight_loader, sharded_weight_loader)
32
+from vllm.model_executor.models.mamba_cache import MambaCacheManager
33
from vllm.model_executor.sampling_metadata import SamplingMetadata
34
from vllm.model_executor.utils import set_weight_attrs
35
from vllm.sequence import IntermediateTensors
@@ -99,16 +99,6 @@ class JambaMambaMixer(nn.Module):
99
bias=True,
100
skip_bias_add=True)
101
102
- def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
103
- tp_rank = get_tensor_model_parallel_rank()
104
- tp_size = get_tensor_model_parallel_world_size()
105
- param.data.copy_(
106
- loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
107
- dim=0)[tp_rank])
108
-
109
- def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
110
- weight_loader(param, -torch.exp(loaded_weight.float()))
111
-
102
tp_size = get_tensor_model_parallel_world_size()
103
self.A = nn.Parameter(
104
torch.empty(
@@ -118,8 +108,10 @@ class JambaMambaMixer(nn.Module):
108
))
109
self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
110
121
- set_weight_attrs(self.D, {"weight_loader": weight_loader})
122
- set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
111
+ set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
112
+ a_weight_loader = composed_weight_loader(
113
+ sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
114
+ set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
115
116
self.out_proj = RowParallelLinear(
117
self.intermediate_size,
@@ -571,10 +563,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
563
if not lora_config else lora_config.lora_vocab_padding_size,
564
)
565
# Used to track and store by the Mamba cache between steps.
574
- self.mamba_cache: Tuple[torch.Tensor, torch.Tensor] = tuple()
575
- # Maps between the request id and a dict that maps between the seq_id
576
- # and its index inside the self.mamba_cache
577
- self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
566
+ self.mamba_cache: Optional[MambaCacheManager] = None
567
+
568
self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
569
config.vocab_size)
570
self.sampler = Sampler()
@@ -586,203 +576,36 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
576
attn_metadata: AttentionMetadata,
577
intermediate_tensors: Optional[IntermediateTensors] = None,
578
**kwargs):
589
- if not self.mamba_cache:
590
- self._prepare_mamba_cache()
591
-
592
- if "seqlen_agnostic_capture_inputs" not in kwargs:
593
- # We get here only on Prefill/Eager mode runs
594
- request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
595
- finished_requests_ids = kwargs["finished_requests_ids"]
596
- mamba_cache = self._release_finished_and_prepare_mamba_cache(
597
- finished_requests_ids, request_ids_to_seq_ids)
598
- else:
599
- # CUDA graph capturing runs
600
- mamba_cache = kwargs["seqlen_agnostic_capture_inputs"]
601
-
602
- hidden_states = self.model(input_ids, positions, kv_caches,
603
- attn_metadata, mamba_cache[0],
604
- mamba_cache[1])
605
- return hidden_states
606
-
607
- def _swap_mamba_cache(self, from_index: int, to_index: int):
608
- assert len(self.mamba_cache) > 0
609
- for cache_t in self.mamba_cache:
610
- cache_t[:, [to_index,from_index]] = \
611
- cache_t[:, [from_index,to_index]]
612
-
613
- def _copy_mamba_cache(self, from_index: int, to_index: int):
614
- assert len(self.mamba_cache) > 0
615
- for cache_t in self.mamba_cache:
616
- cache_t[:, to_index].copy_(cache_t[:, from_index],
617
- non_blocking=True)
618
-
619
- def _move_out_if_already_occupied(self, index: int,
620
- all_occupied_indices: List[int]):
621
- if index in all_occupied_indices:
622
- first_free_index = self._first_free_index_in_mamba_cache()
623
- # In case occupied, move the occupied to a new empty block
624
- self._move_cache_index_and_mappings(from_index=index,
625
- to_index=first_free_index)
626
-
627
- def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
628
- seq_id: int,
629
- destination_index: int):
630
- """
631
- Assign (req_id,seq_id) pair to a `destination_index` index, if
632
- already occupied, move the occupying index to a free index.
633
- """
634
- all_occupied_indices = self._get_all_occupied_indices()
635
- if cur_rid not in self.mamba_cache_indices_mapping:
636
- self._move_out_if_already_occupied(
637
- index=destination_index,
638
- all_occupied_indices=all_occupied_indices)
639
- self.mamba_cache_indices_mapping[cur_rid] = {
640
- seq_id: destination_index
641
- }
642
- elif seq_id not in (seq_ids2indices :=
643
- self.mamba_cache_indices_mapping[cur_rid]):
644
- # parallel sampling , where n > 1, assume prefill have
645
- # already happened now we only need to copy the already
646
- # existing cache into the siblings seq_ids caches
647
- self._move_out_if_already_occupied(
648
- index=destination_index,
649
- all_occupied_indices=all_occupied_indices)
650
- index_exists = list(seq_ids2indices.values())[0]
651
- # case of decoding n>1, copy prefill cache to decoding indices
652
- self._copy_mamba_cache(from_index=index_exists,
653
- to_index=destination_index)
654
- self.mamba_cache_indices_mapping[cur_rid][
655
- seq_id] = destination_index
656
- else:
657
- # already exists
658
- cache_index_already_exists = self.mamba_cache_indices_mapping[
659
- cur_rid][seq_id]
660
- if cache_index_already_exists != destination_index:
661
- # In case the seq id already exists but not in
662
- # the right destination, swap it with what's occupying it
663
- self._swap_pair_indices_and_mappings(
664
- from_index=cache_index_already_exists,
665
- to_index=destination_index)
666
-
667
- def _prepare_current_run_mamba_cache(
668
- self, request_ids_to_seq_ids: Dict[str, list[int]],
669
- finished_requests_ids: List[str]
670
- ) -> Tuple[torch.Tensor, torch.Tensor]:
671
- running_indices = []
672
- request_ids_to_seq_ids_flatten = [
673
- (req_id, seq_id)
674
- for req_id, seq_ids in request_ids_to_seq_ids.items()
675
- for seq_id in seq_ids
676
- ]
677
- batch_size = len(request_ids_to_seq_ids_flatten)
678
- for dest_index, (request_id,
679
- seq_id) in enumerate(request_ids_to_seq_ids_flatten):
680
- if request_id in finished_requests_ids:
681
- # Do not allocate cache index for requests that run
682
- # and finish right after
683
- continue
684
- self._assign_seq_id_to_mamba_cache_in_specific_dest(
685
- request_id, seq_id, dest_index)
686
- running_indices.append(dest_index)
579
+ if self.mamba_cache is None:
580
+ max_batch_size = (_get_graph_batch_size(
581
+ self.scheduler_config.max_num_seqs) if self.scheduler_config
582
+ else max(_BATCH_SIZES_TO_CAPTURE) + 2)
583
688
- self._clean_up_first_bs_blocks(batch_size, running_indices)
689
- conv_state = self.mamba_cache[0][:, :batch_size]
690
- temporal_state = self.mamba_cache[1][:, :batch_size]
584
+ layers_type = self.config.layers_block_type
585
+ num_mamba_layers = sum(
586
+ [layer_type == "mamba" for layer_type in layers_type])
587
692
- return (conv_state, temporal_state)
588
+ self.mamba_cache = MambaCacheManager(
589
+ self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
590
+ *self._get_mamba_cache_shape())
591
694
- def _get_all_occupied_indices(self):
695
- return [
696
- cache_idx
697
- for seq_ids2indices in self.mamba_cache_indices_mapping.values()
698
- for cache_idx in seq_ids2indices.values()
699
- ]
592
+ mamba_cache_tensors = self.mamba_cache.current_run_tensors(
593
+ input_ids, attn_metadata, **kwargs)
594
701
- def _clean_up_first_bs_blocks(self, batch_size: int,
702
- indices_for_current_run: List[int]):
703
- # move out all of the occupied but currently not running blocks
704
- # outside of the first n blocks
705
- destination_indices = range(batch_size)
706
- max_possible_batch_size = self.mamba_cache[0].shape[1]
707
- for destination_index in destination_indices:
708
- if destination_index in self._get_all_occupied_indices() and \
709
- destination_index not in indices_for_current_run:
710
- # move not running indices outside of the batch
711
- all_other_indices = list(
712
- range(batch_size, max_possible_batch_size))
713
- first_avail_index = self._first_free_index_in_mamba_cache(
714
- all_other_indices)
715
- self._swap_indices(from_index=destination_index,
716
- to_index=first_avail_index)
717
-
718
- def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
719
- self._copy_mamba_cache(from_index=from_index, to_index=to_index)
720
- self._update_mapping_index(from_index=from_index, to_index=to_index)
721
-
722
- def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
723
- self._swap_mamba_cache(from_index=from_index, to_index=to_index)
724
- self._swap_mapping_index(from_index=from_index, to_index=to_index)
725
-
726
- def _swap_mapping_index(self, from_index: int, to_index: int):
727
- for seq_ids2index in self.mamba_cache_indices_mapping.values():
728
- for seq_id, index in seq_ids2index.items():
729
- if from_index == index:
730
- seq_ids2index.update({seq_id: to_index})
731
- elif to_index == index:
732
- seq_ids2index.update({seq_id: from_index})
733
-
734
- def _update_mapping_index(self, from_index: int, to_index: int):
735
- for seq_ids2index in self.mamba_cache_indices_mapping.values():
736
- for seq_id, index in seq_ids2index.items():
737
- if from_index == index:
738
- seq_ids2index.update({seq_id: to_index})
739
- return
740
-
741
- def _release_finished_and_prepare_mamba_cache(
742
- self, finished_requests_ids,
743
- request_ids_to_seq_ids) -> Tuple[torch.Tensor, torch.Tensor]:
744
- self._release_mamba_cache(finished_requests_ids)
745
- return self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
746
- finished_requests_ids)
595
+ hidden_states = self.model(input_ids, positions, kv_caches,
596
+ attn_metadata, mamba_cache_tensors[0],
597
+ mamba_cache_tensors[1])
598
+ return hidden_states
599
600
def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
749
- """
750
- Copy the relevant Mamba cache into the CUDA graph input buffer
751
- that was provided during the capture runs
752
- (JambaForCausalLM.mamba_gc_cache_buffer).
753
- """
754
- self._release_finished_and_prepare_mamba_cache(
755
- kwargs["finished_requests_ids"], kwargs["request_ids_to_seq_ids"])
601
+ return self.mamba_cache.copy_inputs_before_cuda_graphs(
602
+ input_buffers, **kwargs)
603
604
def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
758
- """
759
- Provide the CUDA graph capture runs with a buffer in adjusted size.
760
- The buffer is used to maintain the Mamba Cache during the CUDA graph
761
- replay runs.
762
- """
763
- return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
764
-
765
- def _release_mamba_cache(self, finished_seq_groups_req_ids: List[str]):
766
- for req_id in finished_seq_groups_req_ids:
767
- if req_id in self.mamba_cache_indices_mapping:
768
- self.mamba_cache_indices_mapping.pop(req_id)
769
-
770
- def _first_free_index_in_mamba_cache(
771
- self, indices_range: Optional[List[int]] = None) -> int:
772
- assert self.mamba_cache is not None
773
- if indices_range is None:
774
- max_possible_batch_size = self.mamba_cache[0].shape[1]
775
- indices_range = list(range(max_possible_batch_size))
776
- all_occupied_indices = self._get_all_occupied_indices()
777
- for i in indices_range:
778
- if i not in all_occupied_indices:
779
- return i
780
- raise Exception("Couldn't find a free spot in the mamba cache! This"
781
- "should never happen")
605
+ return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size)
606
607
def _get_mamba_cache_shape(
784
- self
785
- ) -> Tuple[Optional[Tuple[int, int]], Optional[Tuple[int, int]]]:
608
+ self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
609
world_size = get_tensor_model_parallel_world_size()
610
hidden_size = self.config.hidden_size
611
conv_state_shape = (
@@ -790,31 +613,11 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
613
self.config.mamba_d_conv - 1,
614
)
615
temporal_state_shape = (
793
- self.config.mamba_expand * self.config.hidden_size // world_size,
616
+ self.config.mamba_expand * hidden_size // world_size,
617
self.config.mamba_d_state,
618
)
619
return conv_state_shape, temporal_state_shape
620
798
- def _prepare_mamba_cache(self):
799
- dtype = self.lm_head.weight.dtype
800
- layers_type = self.config.layers_block_type
801
- mamba_layers = sum(
802
- [layer_type == "mamba" for layer_type in layers_type])
803
- max_batch_size = (_get_graph_batch_size(
804
- self.scheduler_config.max_num_seqs) if self.scheduler_config else
805
- max(_BATCH_SIZES_TO_CAPTURE) + 2)
806
- conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
807
- assert conv_state_shape is not None and temporal_state_shape is not None
808
-
809
- self.mamba_cache = (torch.empty(size=(mamba_layers, max_batch_size) +
810
- conv_state_shape,
811
- dtype=dtype,
812
- device="cuda"),
813
- torch.empty(size=(mamba_layers, max_batch_size) +
814
- temporal_state_shape,
815
- dtype=dtype,
816
- device="cuda"))
817
-
621
def compute_logits(
622
self,
623
hidden_states: torch.Tensor,
vllm/model_executor/models/mamba.py
new
+499
@@ -0,0 +1,499 @@
1
+# coding=utf-8
2
+"""PyTorch MAMBA model."""
3
+from dataclasses import dataclass
4
+from typing import Iterable, List, Optional, Tuple
5
+
6
+import torch
7
+from torch import nn
8
+from transformers import MambaConfig
9
+
10
+from vllm.attention.backends.abstract import AttentionMetadata
11
+from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12
+from vllm.distributed import get_tensor_model_parallel_world_size
13
+from vllm.model_executor.layers.activation import SiluAndMul
14
+from vllm.model_executor.layers.layernorm import RMSNorm
15
+from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16
+ MergedColumnParallelLinear,
17
+ RowParallelLinear)
18
+from vllm.model_executor.layers.logits_processor import LogitsProcessor
19
+from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
20
+ causal_conv1d_fn, causal_conv1d_update)
21
+from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
22
+ selective_scan_fn, selective_state_update)
23
+from vllm.model_executor.layers.quantization.base_config import (
24
+ QuantizationConfig)
25
+from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26
+from vllm.model_executor.layers.vocab_parallel_embedding import (
27
+ VocabParallelEmbedding)
28
+from vllm.model_executor.model_loader.weight_utils import (
29
+ composed_weight_loader, default_weight_loader, sharded_weight_loader)
30
+from vllm.model_executor.models.interfaces import (HasInnerState,
31
+ IsAttentionFree)
32
+from vllm.model_executor.models.mamba_cache import MambaCacheManager
33
+from vllm.model_executor.sampling_metadata import SamplingMetadata
34
+from vllm.model_executor.utils import set_weight_attrs
35
+from vllm.sequence import IntermediateTensors
36
+from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
37
+ _get_graph_batch_size)
38
+
39
+KVCache = Tuple[torch.Tensor, torch.Tensor]
40
+
41
+
42
+@dataclass
43
+class MambaCacheParams:
44
+ is_prompt: bool = False
45
+ conv_state: torch.Tensor = torch.Tensor()
46
+ ssm_state: torch.Tensor = torch.Tensor()
47
+
48
+
49
+# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
50
+class MambaMixer(nn.Module):
51
+ """
52
+ Compute ∆, A, B, C, and D the state space parameters and compute
53
+ the `contextualized_states`. A, D are input independent
54
+ (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
55
+ for why A isn't selective) ∆, B, C are input-dependent
56
+ (this is a key difference between Mamba and the linear time
57
+ invariant S4, and is why Mamba is called
58
+ **selective** state spaces)
59
+ """
60
+
61
+ def __init__(self, config: MambaConfig, layer_idx):
62
+ super().__init__()
63
+ self.config = config
64
+ self.layer_idx = layer_idx
65
+ self.hidden_size = config.hidden_size
66
+ self.ssm_state_size = config.state_size
67
+ self.conv_kernel_size = config.conv_kernel
68
+ self.intermediate_size = config.intermediate_size
69
+ self.time_step_rank = int(config.time_step_rank)
70
+
71
+ self.conv1d = ColumnParallelLinear(
72
+ input_size=self.conv_kernel_size,
73
+ output_size=self.intermediate_size,
74
+ bias=config.use_conv_bias,
75
+ )
76
+ # unsqueeze to fit conv1d weights shape into the linear weights shape.
77
+ # Can't do this in `weight_loader` since it already exists in
78
+ # `ColumnParallelLinear` and `set_weight_attrs`
79
+ # doesn't allow to override it
80
+ self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
81
+
82
+ self.in_proj = MergedColumnParallelLinear(self.hidden_size,
83
+ [self.intermediate_size] * 2,
84
+ bias=config.use_bias)
85
+ # selective projection used to make dt, B and C input dependent
86
+ self.x_proj = RowParallelLinear(
87
+ self.intermediate_size,
88
+ self.time_step_rank + self.ssm_state_size * 2,
89
+ bias=False,
90
+ )
91
+ # time step projection (discretization) -
92
+ # In the forward we need to apply dt_proj without the bias,
93
+ # as the bias is added in the selective scan kernel.
94
+ self.dt_proj = ColumnParallelLinear(self.time_step_rank,
95
+ self.intermediate_size,
96
+ bias=True,
97
+ skip_bias_add=True)
98
+
99
+ tp_size = get_tensor_model_parallel_world_size()
100
+ self.A = nn.Parameter(
101
+ torch.empty(
102
+ self.intermediate_size // tp_size,
103
+ self.ssm_state_size,
104
+ dtype=torch.float32,
105
+ ))
106
+ self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
107
+
108
+ set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
109
+ a_weight_loader = composed_weight_loader(
110
+ sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
111
+ set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
112
+
113
+ self.out_proj = RowParallelLinear(
114
+ self.intermediate_size,
115
+ self.hidden_size,
116
+ bias=config.use_bias,
117
+ input_is_parallel=True,
118
+ )
119
+ self.activation = config.hidden_act
120
+
121
+ def forward(self, hidden_states: torch.Tensor,
122
+ attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
123
+ ssm_state: torch.Tensor):
124
+
125
+ # 1. Gated MLP's linear projection
126
+ projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
127
+ hidden_states, gate = projected_states.chunk(2, dim=-2)
128
+
129
+ # 2. Convolution sequence transformation
130
+ conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
131
+ self.conv1d.weight.size(2))
132
+
133
+ if attn_metadata.query_start_loc is not None \
134
+ and attn_metadata.context_lens_tensor is not None:
135
+ # |---------- N-1 iteration --------|
136
+ # |---------------- N iteration ---------------------|
137
+ # |- tokenA -|......................|-- newTokens ---|
138
+ # |---------- context_len ----------|
139
+ # |-------------------- seq_len ---------------------|
140
+ # |-- query_len ---|
141
+ hidden_states = causal_conv1d_fn(
142
+ hidden_states,
143
+ conv_weights,
144
+ self.conv1d.bias,
145
+ activation=self.activation,
146
+ conv_states=conv_state,
147
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
148
+ query_start_loc=attn_metadata.query_start_loc)
149
+ else:
150
+ hidden_states = causal_conv1d_update(
151
+ hidden_states.transpose(0, 1),
152
+ conv_state,
153
+ conv_weights,
154
+ self.conv1d.bias,
155
+ self.activation,
156
+ )
157
+ hidden_states = hidden_states.transpose(0, 1)
158
+
159
+ # 3. State Space Model sequence transformation
160
+ # 3.a. input varying initialization of time_step, B and C
161
+ ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
162
+
163
+ time_step, B, C = torch.split(
164
+ ssm_parameters,
165
+ [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166
+ dim=-1,
167
+ )
168
+
169
+ # Note that Jamba normalizes B, C, and time_step here but Mamba doesn't.
170
+
171
+ discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
172
+ # 3.c perform the recurrence y ← SSM(A, B, C)(x)
173
+ time_proj_bias = (self.dt_proj.bias.float() if hasattr(
174
+ self.dt_proj, "bias") else None)
175
+
176
+ if attn_metadata.query_start_loc is not None \
177
+ and attn_metadata.context_lens_tensor is not None:
178
+ scan_outputs = selective_scan_fn(
179
+ hidden_states,
180
+ ssm_state,
181
+ discrete_time_step,
182
+ self.A,
183
+ B.transpose(-2, -1),
184
+ C.transpose(-2, -1),
185
+ self.D.float(),
186
+ gate,
187
+ time_proj_bias,
188
+ delta_softplus=True,
189
+ has_initial_state=attn_metadata.context_lens_tensor > 0,
190
+ query_start_loc=attn_metadata.query_start_loc)
191
+ else:
192
+ scan_outputs = selective_state_update(
193
+ ssm_state,
194
+ hidden_states.transpose(0, 1),
195
+ discrete_time_step.transpose(0, 1),
196
+ self.A,
197
+ B,
198
+ C,
199
+ self.D,
200
+ gate.transpose(0, 1),
201
+ time_proj_bias,
202
+ dt_softplus=True,
203
+ )
204
+ scan_outputs = scan_outputs.transpose(0, 1)
205
+
206
+ # 4. Final linear projection
207
+ contextualized_states = self.out_proj(scan_outputs.transpose(-2,
208
+ -1))[0]
209
+ return contextualized_states
210
+
211
+
212
+class MambaMLP(nn.Module):
213
+
214
+ def __init__(
215
+ self,
216
+ config: MambaConfig,
217
+ quant_config: Optional[QuantizationConfig] = None,
218
+ ) -> None:
219
+ super().__init__()
220
+ hidden_size = config.hidden_size
221
+ intermediate_size = config.intermediate_size
222
+ hidden_act = config.hidden_act
223
+ self.gate_up_proj = MergedColumnParallelLinear(
224
+ hidden_size, [intermediate_size] * 2,
225
+ bias=False,
226
+ quant_config=quant_config)
227
+ self.down_proj = RowParallelLinear(intermediate_size,
228
+ hidden_size,
229
+ bias=False,
230
+ quant_config=quant_config)
231
+ if hidden_act != "silu":
232
+ raise ValueError(f"Unsupported activation: {hidden_act}. "
233
+ "Only silu is supported for now.")
234
+ self.act_fn = SiluAndMul()
235
+
236
+ def forward(self, x):
237
+ gate_up, _ = self.gate_up_proj(x)
238
+ x = self.act_fn(gate_up)
239
+ x, _ = self.down_proj(x)
240
+ return x
241
+
242
+
243
+class MambaDecoderLayer(nn.Module):
244
+
245
+ def __init__(self,
246
+ config: MambaConfig,
247
+ layer_idx: int,
248
+ cache_config: Optional[CacheConfig] = None,
249
+ quant_config: Optional[QuantizationConfig] = None) -> None:
250
+ super().__init__()
251
+ self.layer_idx = layer_idx
252
+ self.config = config
253
+ self.mixer = MambaMixer(config, layer_idx)
254
+
255
+ self.feed_forward = MambaMLP(config, quant_config=quant_config)
256
+ self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
257
+ self.pre_ff_layernorm = RMSNorm(config.hidden_size,
258
+ eps=config.layer_norm_epsilon)
259
+
260
+ def forward(
261
+ self,
262
+ hidden_states: torch.Tensor,
263
+ attn_metadata: AttentionMetadata,
264
+ residual: Optional[torch.Tensor],
265
+ conv_state: torch.Tensor,
266
+ ssm_state: torch.Tensor,
267
+ **kwargs,
268
+ ):
269
+ if residual is None:
270
+ residual = hidden_states
271
+ hidden_states = self.norm(hidden_states)
272
+ else:
273
+ hidden_states, residual = self.norm(hidden_states, residual)
274
+
275
+ hidden_states = self.mixer(hidden_states, attn_metadata, conv_state,
276
+ ssm_state)
277
+ # Fully Connected
278
+ hidden_states, residual = self.pre_ff_layernorm(
279
+ hidden_states, residual)
280
+ hidden_states = self.feed_forward(hidden_states)
281
+ return hidden_states, residual
282
+
283
+
284
+class MambaModel(nn.Module):
285
+
286
+ def __init__(
287
+ self,
288
+ config: MambaConfig,
289
+ quant_config: Optional[QuantizationConfig] = None,
290
+ cache_config: Optional[CacheConfig] = None,
291
+ lora_config: Optional[LoRAConfig] = None,
292
+ ) -> None:
293
+ super().__init__()
294
+ self.config = config
295
+ self.padding_idx = config.pad_token_id
296
+ lora_vocab = ((lora_config.lora_extra_vocab_size *
297
+ (lora_config.max_loras or 1)) if lora_config else 0)
298
+ self.vocab_size = config.vocab_size + lora_vocab
299
+ self.org_vocab_size = config.vocab_size
300
+
301
+ self.embeddings = VocabParallelEmbedding(
302
+ self.vocab_size,
303
+ config.hidden_size,
304
+ org_num_embeddings=config.vocab_size,
305
+ )
306
+
307
+ decoder_layers = []
308
+ for i in range(config.num_hidden_layers):
309
+ decoder_layers.append(
310
+ MambaDecoderLayer(config,
311
+ layer_idx=i,
312
+ cache_config=cache_config,
313
+ quant_config=quant_config))
314
+ self.layers = nn.ModuleList(decoder_layers)
315
+ self.norm_f = RMSNorm(config.hidden_size,
316
+ eps=config.layer_norm_epsilon)
317
+
318
+ def forward(
319
+ self,
320
+ input_ids: torch.Tensor,
321
+ positions: torch.Tensor,
322
+ kv_caches: List[torch.Tensor],
323
+ attn_metadata: AttentionMetadata,
324
+ conv_state: torch.Tensor,
325
+ ssm_state: torch.Tensor,
326
+ ) -> torch.Tensor:
327
+ hidden_states = self.embeddings(input_ids)
328
+ residual = None
329
+
330
+ for i in range(len(self.layers)):
331
+ layer = self.layers[i]
332
+ current_ssm_state = ssm_state[i]
333
+ current_conv_state = conv_state[i]
334
+
335
+ hidden_states, residual = layer(
336
+ positions=positions,
337
+ hidden_states=hidden_states,
338
+ attn_metadata=attn_metadata,
339
+ residual=residual,
340
+ conv_state=current_conv_state,
341
+ ssm_state=current_ssm_state,
342
+ )
343
+ hidden_states, _ = self.norm_f(hidden_states, residual)
344
+
345
+ return hidden_states
346
+
347
+
348
+class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
349
+ packed_modules_mapping = {
350
+ "qkv_proj": [
351
+ "q_proj",
352
+ "k_proj",
353
+ "v_proj",
354
+ ],
355
+ }
356
+
357
+ # LoRA specific attributes
358
+ supported_lora_modules = [
359
+ "qkv_proj",
360
+ "o_proj",
361
+ "embed_tokens",
362
+ "lm_head",
363
+ ]
364
+ embedding_modules = {
365
+ "embeddings": "input_embeddings",
366
+ "lm_head": "output_embeddings",
367
+ }
368
+ embedding_padding_modules = ["lm_head"]
369
+
370
+ def __init__(
371
+ self,
372
+ config: MambaConfig,
373
+ cache_config: Optional[CacheConfig] = None,
374
+ quant_config: Optional[QuantizationConfig] = None,
375
+ lora_config: Optional[LoRAConfig] = None,
376
+ scheduler_config: Optional[SchedulerConfig] = None,
377
+ ) -> None:
378
+ assert not cache_config.enable_prefix_caching, \
379
+ "Mamba does not support prefix caching"
380
+
381
+ super().__init__()
382
+ self.config = config
383
+ self.scheduler_config = scheduler_config
384
+ self.backbone = MambaModel(config,
385
+ cache_config=cache_config,
386
+ quant_config=quant_config,
387
+ lora_config=lora_config)
388
+ self.unpadded_vocab_size = config.vocab_size
389
+ if lora_config:
390
+ self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
391
+
392
+ self.lm_head = self.backbone.embeddings
393
+
394
+ # Used to track and store by the Mamba cache between steps.
395
+ self.mamba_cache: Optional[MambaCacheManager] = None
396
+
397
+ self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
398
+ config.vocab_size)
399
+ self.sampler = Sampler()
400
+
401
+ def forward(self,
402
+ input_ids: torch.Tensor,
403
+ positions: torch.Tensor,
404
+ kv_caches: List[KVCache],
405
+ attn_metadata: AttentionMetadata,
406
+ intermediate_tensors: Optional[IntermediateTensors] = None,
407
+ **kwargs):
408
+ if self.mamba_cache is None:
409
+ max_batch_size = (_get_graph_batch_size(
410
+ self.scheduler_config.max_num_seqs) if self.scheduler_config
411
+ else max(_BATCH_SIZES_TO_CAPTURE) + 2)
412
+ self.mamba_cache = MambaCacheManager(
413
+ self.lm_head.weight.dtype, self.config.num_hidden_layers,
414
+ max_batch_size, *self._get_mamba_cache_shape())
415
+
416
+ mamba_cache_tensors = self.mamba_cache.current_run_tensors(
417
+ input_ids, attn_metadata, **kwargs)
418
+
419
+ hidden_states = self.backbone(input_ids, positions, kv_caches,
420
+ attn_metadata, mamba_cache_tensors[0],
421
+ mamba_cache_tensors[1])
422
+
423
+ return hidden_states
424
+
425
+ def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
426
+ return self.mamba_cache.copy_inputs_before_cuda_graphs(
427
+ input_buffers, **kwargs)
428
+
429
+ def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
430
+ return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size)
431
+
432
+ def _get_mamba_cache_shape(
433
+ self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
434
+ world_size = get_tensor_model_parallel_world_size()
435
+ conv_state_shape = (
436
+ self.config.intermediate_size // world_size,
437
+ self.config.conv_kernel - 1,
438
+ )
439
+ temporal_state_shape = (
440
+ self.config.intermediate_size // world_size,
441
+ self.config.state_size,
442
+ )
443
+ return conv_state_shape, temporal_state_shape
444
+
445
+ def compute_logits(self, hidden_states: torch.Tensor,
446
+ sampling_metadata: SamplingMetadata) -> torch.Tensor:
447
+ logits = self.logits_processor(self.lm_head, hidden_states,
448
+ sampling_metadata)
449
+ return logits
450
+
451
+ def sample(
452
+ self,
453
+ logits: Optional[torch.Tensor],
454
+ sampling_metadata: SamplingMetadata,
455
+ ) -> Optional[SamplerOutput]:
456
+ next_tokens = self.sampler(logits, sampling_metadata)
457
+ return next_tokens
458
+
459
+ def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
460
+ stacked_params_mapping = [
461
+ # (param_name, shard_name, shard_id)
462
+ ("qkv_proj", "q_proj", "q"),
463
+ ("qkv_proj", "k_proj", "k"),
464
+ ("qkv_proj", "v_proj", "v"),
465
+ ("gate_up_proj", "gate_proj", 0),
466
+ ("gate_up_proj", "up_proj", 1),
467
+ ]
468
+
469
+ params_dict = dict(self.named_parameters())
470
+ for name, loaded_weight in weights:
471
+ if "rotary_emb.inv_freq" in name:
472
+ continue
473
+
474
+ if "A_log" in name:
475
+ name = name.replace("A_log", "A")
476
+
477
+ if ".self_attn." in name:
478
+ name = name.replace(".self_attn", "")
479
+
480
+ for param_name, weight_name, shard_id in stacked_params_mapping:
481
+ if weight_name not in name:
482
+ continue
483
+ name = name.replace(weight_name, param_name)
484
+ # Skip loading extra bias for GPTQ models.
485
+ if name.endswith(".bias") and name not in params_dict:
486
+ continue
487
+ param = params_dict[name]
488
+ weight_loader = param.weight_loader
489
+ weight_loader(param, loaded_weight, shard_id)
490
+ break
491
+ else:
492
+ # Skip loading extra bias for GPTQ models.
493
+ if name.endswith(".bias") and name not in params_dict:
494
+ continue
495
+
496
+ param = params_dict[name]
497
+ weight_loader = getattr(param, "weight_loader",
498
+ default_weight_loader)
499
+ weight_loader(param, loaded_weight)
vllm/model_executor/models/mamba_cache.py
new
+222
@@ -0,0 +1,222 @@
1
+from typing import Dict, List, Optional
2
+
3
+import torch
4
+
5
+from vllm.attention.backends.abstract import AttentionMetadata
6
+
7
+
8
+class MambaCacheManager:
9
+
10
+ def __init__(self, dtype, num_mamba_layers, max_batch_size,
11
+ conv_state_shape, temporal_state_shape):
12
+
13
+ conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
14
+ conv_state_shape,
15
+ dtype=dtype,
16
+ device="cuda")
17
+ temporal_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
18
+ temporal_state_shape,
19
+ dtype=dtype,
20
+ device="cuda")
21
+
22
+ self.mamba_cache = (conv_state, temporal_state)
23
+
24
+ # Maps between the request id and a dict that maps between the seq_id
25
+ # and its index inside the self.mamba_cache
26
+ self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
27
+
28
+ def current_run_tensors(self, input_ids: torch.Tensor,
29
+ attn_metadata: AttentionMetadata, **kwargs):
30
+ """
31
+ Return the tensors for the current run's conv and ssm state.
32
+ """
33
+ if "seqlen_agnostic_capture_inputs" not in kwargs:
34
+ # We get here only on Prefill/Eager mode runs
35
+ request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
36
+ finished_requests_ids = kwargs["finished_requests_ids"]
37
+
38
+ self._release_finished_requests(finished_requests_ids)
39
+ mamba_cache_tensors = self._prepare_current_run_mamba_cache(
40
+ request_ids_to_seq_ids, finished_requests_ids)
41
+
42
+ else:
43
+ # CUDA graph capturing runs
44
+ mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"]
45
+
46
+ return mamba_cache_tensors
47
+
48
+ def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
49
+ """
50
+ Copy the relevant Mamba cache into the CUDA graph input buffer
51
+ that was provided during the capture runs
52
+ (JambaForCausalLM.mamba_gc_cache_buffer).
53
+ """
54
+ assert all(
55
+ key in kwargs
56
+ for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
57
+ finished_requests_ids = kwargs["finished_requests_ids"]
58
+ request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
59
+
60
+ self._release_finished_requests(finished_requests_ids)
61
+ self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
62
+ finished_requests_ids)
63
+
64
+ def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
65
+ """
66
+ Provide the CUDA graph capture runs with a buffer in adjusted size.
67
+ The buffer is used to maintain the Mamba Cache during the CUDA graph
68
+ replay runs.
69
+ """
70
+ return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
71
+
72
+ def _swap_mamba_cache(self, from_index: int, to_index: int):
73
+ assert len(self.mamba_cache) > 0
74
+ for cache_t in self.mamba_cache:
75
+ cache_t[:, [to_index,from_index]] = \
76
+ cache_t[:, [from_index,to_index]]
77
+
78
+ def _copy_mamba_cache(self, from_index: int, to_index: int):
79
+ assert len(self.mamba_cache) > 0
80
+ for cache_t in self.mamba_cache:
81
+ cache_t[:, to_index].copy_(cache_t[:, from_index],
82
+ non_blocking=True)
83
+
84
+ def _move_out_if_already_occupied(self, index: int,
85
+ all_occupied_indices: List[int]):
86
+ if index in all_occupied_indices:
87
+ first_free_index = self._first_free_index_in_mamba_cache()
88
+ # In case occupied, move the occupied to a new empty block
89
+ self._move_cache_index_and_mappings(from_index=index,
90
+ to_index=first_free_index)
91
+
92
+ def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
93
+ seq_id: int,
94
+ destination_index: int):
95
+ """
96
+ Assign (req_id,seq_id) pair to a `destination_index` index, if
97
+ already occupied, move the occupying index to a free index.
98
+ """
99
+ all_occupied_indices = self._get_all_occupied_indices()
100
+ if cur_rid not in self.mamba_cache_indices_mapping:
101
+ self._move_out_if_already_occupied(
102
+ index=destination_index,
103
+ all_occupied_indices=all_occupied_indices)
104
+ self.mamba_cache_indices_mapping[cur_rid] = {
105
+ seq_id: destination_index
106
+ }
107
+ elif seq_id not in (seq_ids2indices :=
108
+ self.mamba_cache_indices_mapping[cur_rid]):
109
+ # parallel sampling , where n > 1, assume prefill have
110
+ # already happened now we only need to copy the already
111
+ # existing cache into the siblings seq_ids caches
112
+ self._move_out_if_already_occupied(
113
+ index=destination_index,
114
+ all_occupied_indices=all_occupied_indices)
115
+ index_exists = list(seq_ids2indices.values())[0]
116
+ # case of decoding n>1, copy prefill cache to decoding indices
117
+ self._copy_mamba_cache(from_index=index_exists,
118
+ to_index=destination_index)
119
+ self.mamba_cache_indices_mapping[cur_rid][
120
+ seq_id] = destination_index
121
+ else:
122
+ # already exists
123
+ cache_index_already_exists = self.mamba_cache_indices_mapping[
124
+ cur_rid][seq_id]
125
+ if cache_index_already_exists != destination_index:
126
+ # In case the seq id already exists but not in
127
+ # the right destination, swap it with what's occupying it
128
+ self._swap_pair_indices_and_mappings(
129
+ from_index=cache_index_already_exists,
130
+ to_index=destination_index)
131
+
132
+ def _prepare_current_run_mamba_cache(
133
+ self, request_ids_to_seq_ids: Dict[str, list[int]],
134
+ finished_requests_ids: List[str]):
135
+ running_indices = []
136
+ request_ids_to_seq_ids_flatten = [
137
+ (req_id, seq_id)
138
+ for req_id, seq_ids in request_ids_to_seq_ids.items()
139
+ for seq_id in seq_ids
140
+ ]
141
+ batch_size = len(request_ids_to_seq_ids_flatten)
142
+ for dest_index, (request_id,
143
+ seq_id) in enumerate(request_ids_to_seq_ids_flatten):
144
+ if request_id in finished_requests_ids:
145
+ # Do not allocate cache index for requests that run
146
+ # and finish right after
147
+ continue
148
+ self._assign_seq_id_to_mamba_cache_in_specific_dest(
149
+ request_id, seq_id, dest_index)
150
+ running_indices.append(dest_index)
151
+
152
+ self._clean_up_first_bs_blocks(batch_size, running_indices)
153
+ conv_state = self.mamba_cache[0][:, :batch_size]
154
+ temporal_state = self.mamba_cache[1][:, :batch_size]
155
+
156
+ return (conv_state, temporal_state)
157
+
158
+ def _get_all_occupied_indices(self):
159
+ return [
160
+ cache_idx
161
+ for seq_ids2indices in self.mamba_cache_indices_mapping.values()
162
+ for cache_idx in seq_ids2indices.values()
163
+ ]
164
+
165
+ def _clean_up_first_bs_blocks(self, batch_size: int,
166
+ indices_for_current_run: List[int]):
167
+ # move out all of the occupied but currently not running blocks
168
+ # outside of the first n blocks
169
+ destination_indices = range(batch_size)
170
+ max_possible_batch_size = self.mamba_cache[0].shape[1]
171
+ for destination_index in destination_indices:
172
+ if destination_index in self._get_all_occupied_indices() and \
173
+ destination_index not in indices_for_current_run:
174
+ # move not running indices outside of the batch
175
+ all_other_indices = list(
176
+ range(batch_size, max_possible_batch_size))
177
+ first_avail_index = self._first_free_index_in_mamba_cache(
178
+ all_other_indices)
179
+ self._swap_indices(from_index=destination_index,
180
+ to_index=first_avail_index)
181
+
182
+ def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
183
+ self._copy_mamba_cache(from_index=from_index, to_index=to_index)
184
+ self._update_mapping_index(from_index=from_index, to_index=to_index)
185
+
186
+ def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
187
+ self._swap_mamba_cache(from_index=from_index, to_index=to_index)
188
+ self._swap_mapping_index(from_index=from_index, to_index=to_index)
189
+
190
+ def _swap_mapping_index(self, from_index: int, to_index: int):
191
+ for seq_ids2index in self.mamba_cache_indices_mapping.values():
192
+ for seq_id, index in seq_ids2index.items():
193
+ if from_index == index:
194
+ seq_ids2index.update({seq_id: to_index})
195
+ elif to_index == index:
196
+ seq_ids2index.update({seq_id: from_index})
197
+
198
+ def _update_mapping_index(self, from_index: int, to_index: int):
199
+ for seq_ids2index in self.mamba_cache_indices_mapping.values():
200
+ for seq_id, index in seq_ids2index.items():
201
+ if from_index == index:
202
+ seq_ids2index.update({seq_id: to_index})
203
+ return
204
+
205
+ def _release_finished_requests(self,
206
+ finished_seq_groups_req_ids: List[str]):
207
+ for req_id in finished_seq_groups_req_ids:
208
+ if req_id in self.mamba_cache_indices_mapping:
209
+ self.mamba_cache_indices_mapping.pop(req_id)
210
+
211
+ def _first_free_index_in_mamba_cache(
212
+ self, indices_range: Optional[List[int]] = None) -> int:
213
+ assert self.mamba_cache is not None
214
+ if indices_range is None:
215
+ max_possible_batch_size = self.mamba_cache[0].shape[1]
216
+ indices_range = list(range(max_possible_batch_size))
217
+ all_occupied_indices = self._get_all_occupied_indices()
218
+ for i in indices_range:
219
+ if i not in all_occupied_indices:
220
+ return i
221
+ raise Exception("Couldn't find a free spot in the mamba cache! This"
222
+ "should never happen")
vllm/model_executor/models/registry.py
+15
-1
@@ -14,7 +14,8 @@ import torch.nn as nn
14
from vllm.logger import init_logger
15
from vllm.utils import is_hip
16
17
-from .interfaces import supports_multimodal, supports_pp
17
+from .interfaces import (has_inner_state, is_attention_free,
18
+ supports_multimodal, supports_pp)
19
from .interfaces_base import is_embedding_model, is_text_generation_model
20
21
logger = init_logger(__name__)
@@ -52,6 +53,7 @@ _TEXT_GENERATION_MODELS = {
53
"LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
54
# For decapoda-research/llama-*
55
"LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
56
+ "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
57
"MistralForCausalLM": ("llama", "LlamaForCausalLM"),
58
"MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
59
"QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
@@ -157,6 +159,8 @@ class _ModelInfo:
159
is_embedding_model: bool
160
supports_multimodal: bool
161
supports_pp: bool
162
+ has_inner_state: bool
163
+ is_attention_free: bool
164
165
@staticmethod
166
def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -165,6 +169,8 @@ class _ModelInfo:
169
is_embedding_model=is_embedding_model(model),
170
supports_multimodal=supports_multimodal(model),
171
supports_pp=supports_pp(model),
172
+ has_inner_state=has_inner_state(model),
173
+ is_attention_free=is_attention_free(model),
174
)
175
176
@@ -380,6 +386,14 @@ class _ModelRegistry:
386
) -> bool:
387
return self.inspect_model_cls(architectures).supports_pp
388
389
+ def model_has_inner_state(self, architectures: Union[str,
390
+ List[str]]) -> bool:
391
+ return self.inspect_model_cls(architectures).has_inner_state
392
+
393
+ def is_attention_free_model(self, architectures: Union[str,
394
+ List[str]]) -> bool:
395
+ return self.inspect_model_cls(architectures).is_attention_free
396
+
397
398
ModelRegistry = _ModelRegistry({
399
model_arch: _LazyRegisteredModel(
vllm/worker/cache_engine.py
+6
-9
@@ -52,15 +52,12 @@ class CacheEngine:
52
self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
53
54
# Get attention backend.
55
- self.attn_backend = get_attn_backend(
56
- model_config.get_num_attention_heads(parallel_config),
57
- self.head_size,
58
- self.num_kv_heads,
59
- model_config.get_sliding_window(),
60
- model_config.dtype,
61
- cache_config.cache_dtype,
62
- self.block_size,
63
- )
55
+ self.attn_backend = get_attn_backend(self.head_size,
56
+ model_config.get_sliding_window(),
57
+ model_config.dtype,
58
+ cache_config.cache_dtype,
59
+ self.block_size,
60
+ model_config.is_attention_free)
61
62
# Initialize the cache.
63
self.gpu_cache = self._allocate_kv_cache(
vllm/worker/cpu_model_runner.py
+1
-2
@@ -418,13 +418,12 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
418
self.sliding_window = model_config.get_sliding_window()
419
self.block_size = cache_config.block_size
420
self.attn_backend = get_attn_backend(
421
- self.model_config.get_num_attention_heads(self.parallel_config),
421
self.model_config.get_head_size(),
423
- self.model_config.get_num_kv_heads(self.parallel_config),
422
self.model_config.get_sliding_window(),
423
self.model_config.dtype,
424
self.kv_cache_dtype,
425
self.block_size,
426
+ self.model_config.is_attention_free,
427
)
428
429
# Multi-modal data support
vllm/worker/cpu_worker.py
+1
-2
@@ -56,13 +56,12 @@ class CPUCacheEngine:
56
57
# Get attention backend.
58
self.attn_backend = get_attn_backend(
59
- self.model_config.get_num_attention_heads(self.parallel_config),
59
self.model_config.get_head_size(),
61
- self.model_config.get_num_kv_heads(self.parallel_config),
60
self.model_config.get_sliding_window(),
61
self.model_config.dtype,
62
cache_config.cache_dtype,
63
self.block_size,
64
+ self.model_config.is_attention_free,
65
)
66
67
# Initialize the cache.
vllm/worker/enc_dec_model_runner.py
+1
-1
@@ -196,7 +196,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
196
seqlen_agnostic_kwargs = {
197
"finished_requests_ids": model_input.finished_requests_ids,
198
"request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
199
- } if self.has_seqlen_agnostic else {}
199
+ } if self.has_inner_state else {}
200
201
multi_modal_kwargs = model_input.multi_modal_kwargs or {}
202
with set_forward_context(model_input.attn_metadata):
vllm/worker/model_runner.py
+13
-17
@@ -17,7 +17,6 @@ import torch.nn as nn
17
import vllm.envs as envs
18
from vllm.attention import AttentionMetadata, get_attn_backend
19
from vllm.attention.backends.abstract import AttentionState
20
-from vllm.attention.backends.utils import CommonAttentionState
20
from vllm.compilation.compile_context import set_compile_context
21
from vllm.compilation.levels import CompilationLevel
22
from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
@@ -991,8 +990,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
990
self.graph_memory_pool: Optional[Tuple[
991
int, int]] = None # Set during graph capture.
992
994
- self.has_seqlen_agnostic = model_config.contains_seqlen_agnostic_layers(
995
- parallel_config)
993
+ self.has_inner_state = model_config.has_inner_state
994
995
# When using CUDA graph, the input block tables must be padded to
996
# max_seq_len_to_capture. However, creating the block table in
@@ -1003,22 +1001,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1001
self.graph_block_tables = np.zeros(
1002
(self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1003
dtype=np.int32)
1006
- num_attn_heads = self.model_config.get_num_attention_heads(
1007
- self.parallel_config)
1004
self.attn_backend = get_attn_backend(
1009
- num_attn_heads,
1005
self.model_config.get_head_size(),
1011
- self.model_config.get_num_kv_heads(self.parallel_config),
1006
self.model_config.get_sliding_window(),
1007
self.model_config.dtype,
1008
self.kv_cache_dtype,
1009
self.block_size,
1016
- ) if num_attn_heads else None
1017
- if self.attn_backend:
1018
- self.attn_state = self.attn_backend.get_state_cls()(
1019
- weakref.proxy(self))
1020
- else:
1021
- self.attn_state = CommonAttentionState(weakref.proxy(self))
1010
+ self.model_config.is_attention_free,
1011
+ )
1012
+ self.attn_state = self.attn_backend.get_state_cls()(
1013
+ weakref.proxy(self))
1014
1015
# Multi-modal data support
1016
self.input_registry = input_registry
@@ -1498,7 +1490,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1490
"previous_hidden_states"] = previous_hidden_states[:
1491
batch_size]
1492
1501
- if self.has_seqlen_agnostic:
1493
+ if self.has_inner_state:
1494
# Only used by Mamba-based models CUDA graph atm (Jamba)
1495
capture_inputs.update({
1496
"seqlen_agnostic_capture_inputs":
@@ -1647,7 +1639,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1639
seqlen_agnostic_kwargs = {
1640
"finished_requests_ids": model_input.finished_requests_ids,
1641
"request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1650
- } if self.has_seqlen_agnostic else {}
1642
+ } if self.has_inner_state else {}
1643
if (self.observability_config is not None
1644
and self.observability_config.collect_model_forward_time):
1645
model_forward_start = torch.cuda.Event(enable_timing=True)
@@ -1852,10 +1844,14 @@ class CUDAGraphRunner:
1844
# Copy the input tensors to the input buffers.
1845
self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1846
self.input_buffers["positions"].copy_(positions, non_blocking=True)
1855
- self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1856
- non_blocking=True)
1847
+
1848
+ if self.backend_name != "placeholder-attn":
1849
+ self.input_buffers["slot_mapping"].copy_(
1850
+ attn_metadata.slot_mapping, non_blocking=True)
1851
+
1852
self.attn_state.prepare_graph_input_buffers(
1853
self.input_buffers, attn_metadata, self._is_encoder_decoder_model)
1854
+
1855
if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1856
self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1857
**kwargs)
vllm/worker/openvino_model_runner.py
+1
-2
@@ -74,13 +74,12 @@ class OpenVINOModelRunner:
74
self.block_size = cache_config.block_size
75
76
self.attn_backend = get_attn_backend(
77
- self.model_config.get_num_attention_heads(self.parallel_config),
77
self.model_config.get_head_size(),
79
- self.model_config.get_num_kv_heads(self.parallel_config),
78
self.model_config.get_sliding_window(),
79
self.model_config.dtype,
80
self.kv_cache_dtype,
81
self.block_size,
82
+ self.model_config.is_attention_free,
83
)
84
85
# Multi-modal data support
vllm/worker/openvino_worker.py
+1
-2
@@ -70,13 +70,12 @@ class OpenVINOCacheEngine:
70
71
# Get attention backend.
72
self.attn_backend = get_attn_backend(
73
- self.model_config.get_num_attention_heads(self.parallel_config),
73
self.head_size,
75
- self.model_config.get_num_kv_heads(self.parallel_config),
74
self.model_config.get_sliding_window(),
75
self.model_config.dtype,
76
self.cache_config.cache_dtype,
77
self.block_size,
78
+ self.model_config.is_attention_free,
79
)
80
81
# Initialize the cache.
vllm/worker/tpu_model_runner.py
+1
-2
@@ -113,13 +113,12 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
113
(self.scheduler_config.max_num_seqs, self.max_num_blocks_per_seq),
114
dtype=np.int32)
115
self.attn_backend = get_attn_backend(
116
- self.model_config.get_num_attention_heads(self.parallel_config),
116
self.model_config.get_head_size(),
118
- self.model_config.get_num_kv_heads(self.parallel_config),
117
self.model_config.get_sliding_window(),
118
self.model_config.dtype,
119
self.cache_config.cache_dtype,
120
self.block_size,
121
+ self.model_config.is_attention_free,
122
False,
123
)
124
self.cached_step_outputs: List[torch.Tensor] = []
vllm/worker/worker.py
+17
-8
@@ -236,11 +236,15 @@ class Worker(LocalOrDistributedWorkerBase):
236
"not properly cleaned up before initializing the vLLM instance.")
237
238
cache_block_size = self.get_cache_block_size_bytes()
239
- num_gpu_blocks = int(
240
- (total_gpu_memory * self.cache_config.gpu_memory_utilization -
241
- peak_memory) // cache_block_size)
242
- num_cpu_blocks = int(self.cache_config.swap_space_bytes //
243
- cache_block_size)
239
+ if cache_block_size == 0:
240
+ num_gpu_blocks = 0
241
+ num_cpu_blocks = 0
242
+ else:
243
+ num_gpu_blocks = int(
244
+ (total_gpu_memory * self.cache_config.gpu_memory_utilization -
245
+ peak_memory) // cache_block_size)
246
+ num_cpu_blocks = int(self.cache_config.swap_space_bytes //
247
+ cache_block_size)
248
num_gpu_blocks = max(num_gpu_blocks, 0)
249
num_cpu_blocks = max(num_cpu_blocks, 0)
250
if self.model_runner.lora_manager:
@@ -257,6 +261,7 @@ class Worker(LocalOrDistributedWorkerBase):
261
"""
262
raise_if_cache_size_invalid(num_gpu_blocks,
263
self.cache_config.block_size,
264
+ self.cache_config.is_attention_free,
265
self.model_config.max_model_len)
266
267
self.cache_config.num_gpu_blocks = num_gpu_blocks
@@ -472,14 +477,18 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
477
"`dtype` flag in CLI, for example: --dtype=half.")
478
479
475
-def raise_if_cache_size_invalid(num_gpu_blocks, block_size,
480
+def raise_if_cache_size_invalid(num_gpu_blocks, block_size, is_attention_free,
481
max_model_len) -> None:
477
- if num_gpu_blocks <= 0:
482
+ if is_attention_free and num_gpu_blocks != 0:
483
+ raise ValueError("No memory should be allocated for the cache blocks "
484
+ f"for an attention-free model, but {num_gpu_blocks}"
485
+ "blocks are allocated.")
486
+ if not is_attention_free and num_gpu_blocks <= 0:
487
raise ValueError("No available memory for the cache blocks. "
488
"Try increasing `gpu_memory_utilization` when "
489
"initializing the engine.")
490
max_seq_len = block_size * num_gpu_blocks
482
- if max_model_len > max_seq_len:
491
+ if not is_attention_free and max_model_len > max_seq_len:
492
raise ValueError(
493
f"The model's max seq len ({max_model_len}) "
494
"is larger than the maximum number of tokens that can be "
vllm/worker/xpu_model_runner.py
+1
-2
@@ -372,13 +372,12 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
372
self.block_size = cache_config.block_size
373
374
self.attn_backend = get_attn_backend(
375
- self.model_config.get_num_attention_heads(self.parallel_config),
375
self.model_config.get_head_size(),
377
- self.model_config.get_num_kv_heads(self.parallel_config),
376
self.model_config.get_sliding_window(),
377
self.model_config.dtype,
378
self.kv_cache_dtype,
379
self.block_size,
380
+ self.model_config.is_attention_free,
381
)
382
383
# Multi-modal data support