@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 7342a7d7f8
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Model] Support Mamba (#6484)

Tyler Michael Smith committed Oct 11, 2024 at 11:40 UTC 7342a7d7f87ea3f4e03ec0775093a0f1ce56e2a1
29 files changed +1603 -343
.buildkite/run-cpu-test-ppc64le.sh
+7 -1
@@ -18,7 +18,13 @@ docker run -itd --entrypoint /bin/bash -v ~/.cache/huggingface:/root/.cache/hugg
18 # Run basic model test
19 docker exec cpu-test bash -c "
20 pip install pytest matplotlib einops transformers_stream_generator
21 - pytest -v -s tests/models -m \"not vlm\" --ignore=tests/models/test_embedding.py --ignore=tests/models/test_oot_registration.py --ignore=tests/models/test_registry.py --ignore=tests/models/test_jamba.py --ignore=tests/models/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
21 + pytest -v -s tests/models -m \"not vlm\" \
22 + --ignore=tests/models/test_embedding.py \
23 + --ignore=tests/models/test_oot_registration.py \
24 + --ignore=tests/models/test_registry.py \
25 + --ignore=tests/models/test_jamba.py \
26 + --ignore=tests/models/test_mamba.py \
27 + --ignore=tests/models/test_danube3_4b.py" # Mamba kernels and Danube3-4B on CPU is not supported
28
29 # online inference
30 docker exec cpu-test bash -c "
.buildkite/run-cpu-test.sh
+1
@@ -27,6 +27,7 @@ docker exec cpu-test bash -c "
27 pytest -v -s tests/models/decoder_only/language \
28 --ignore=tests/models/test_fp8.py \
29 --ignore=tests/models/decoder_only/language/test_jamba.py \
30 + --ignore=tests/models/decoder_only/language/test_mamba.py \
31 --ignore=tests/models/decoder_only/language/test_granitemoe.py \
32 --ignore=tests/models/decoder_only/language/test_danube3_4b.py" # Mamba and Danube3-4B on CPU is not supported
33
docs/source/models/supported_models.rst
+5
@@ -152,6 +152,11 @@ Text Generation
152 - :code:`meta-llama/Meta-Llama-3.1-405B-Instruct`, :code:`meta-llama/Meta-Llama-3.1-70B`, :code:`meta-llama/Meta-Llama-3-70B-Instruct`, :code:`meta-llama/Llama-2-70b-hf`, :code:`01-ai/Yi-34B`, etc.
153 - ✅︎
154 - ✅︎
155 + * - :code:`MambaForCausalLM`
156 + - Mamba
157 + - :code:`state-spaces/mamba-130m-hf`, :code:`state-spaces/mamba-790m-hf`, :code:`state-spaces/mamba-2.8b-hf`, etc.
158 + - ✅︎
159 + -
160 * - :code:`MiniCPMForCausalLM`
161 - MiniCPM
162 - :code:`openbmb/MiniCPM-2B-sft-bf16`, :code:`openbmb/MiniCPM-2B-dpo-bf16`, etc.
tests/kernels/test_attention_selector.py
+21 -16
@@ -20,22 +20,22 @@ def test_env(name: str, device: str, monkeypatch):
20
21 if device == "cpu":
22 with patch("vllm.attention.selector.is_cpu", return_value=True):
23 - backend = which_attn_to_use(8, 16, 8, None, torch.float16,
24 - torch.float16, 16)
23 + backend = which_attn_to_use(16, None, torch.float16, torch.float16,
24 + 16, False)
25 assert backend.name == "TORCH_SDPA"
26 elif device == "hip":
27 with patch("vllm.attention.selector.is_hip", return_value=True):
28 - backend = which_attn_to_use(8, 16, 8, None, torch.float16,
29 - torch.float16, 16)
28 + backend = which_attn_to_use(16, None, torch.float16, torch.float16,
29 + 16, False)
30 assert backend.name == "ROCM_FLASH"
31 elif device == "openvino":
32 with patch("vllm.attention.selector.is_openvino", return_value=True):
33 - backend = which_attn_to_use(8, 16, 8, None, torch.float16,
34 - torch.float16, 16)
33 + backend = which_attn_to_use(16, None, torch.float16, torch.float16,
34 + 16, False)
35 assert backend.name == "OPENVINO"
36 else:
37 - backend = which_attn_to_use(8, 16, 8, None, torch.float16,
38 - torch.float16, 16)
37 + backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
38 + False)
39 assert backend.name == name
40
41
@@ -46,32 +46,37 @@ def test_flash_attn(monkeypatch):
46
47 # Unsupported CUDA arch
48 with patch("torch.cuda.get_device_capability", return_value=(7, 5)):
49 - backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
49 + backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
50 assert backend.name != STR_FLASH_ATTN_VAL
51
52 # Unsupported data type
53 - backend = which_attn_to_use(8, 16, 8, None, torch.float8_e4m3fn, None, 16)
53 + backend = which_attn_to_use(16, None, torch.float8_e4m3fn, None, 16, False)
54 assert backend.name != STR_FLASH_ATTN_VAL
55
56 # Unsupported kv cache data type
57 - backend = which_attn_to_use(8, 16, 8, None, torch.float16, "fp8", 16)
57 + backend = which_attn_to_use(16, None, torch.float16, "fp8", 16, False)
58 assert backend.name != STR_FLASH_ATTN_VAL
59
60 # Unsupported block size
61 - backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 8)
61 + backend = which_attn_to_use(16, None, torch.float16, None, 8, False)
62 assert backend.name != STR_FLASH_ATTN_VAL
63
64 # Unsupported sliding window
65 - backend = which_attn_to_use(8, 16, 8, 1, torch.float16, None, 16)
65 + backend = which_attn_to_use(16, 1, torch.float16, None, 16, False)
66 assert backend.name != STR_FLASH_ATTN_VAL
67
68 # flash-attn is not installed
69 with patch.dict('sys.modules', {'vllm_flash_attn': None}):
70 - backend = which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
70 + backend = which_attn_to_use(16, None, torch.float16, None, 16, False)
71 assert backend.name != STR_FLASH_ATTN_VAL
72
73 # Unsupported head size
74 - backend = which_attn_to_use(8, 17, 8, None, torch.float16, None, 16)
74 + backend = which_attn_to_use(17, None, torch.float16, None, 16, False)
75 + assert backend.name != STR_FLASH_ATTN_VAL
76 +
77 + # Attention-free models should bypass env and use PlaceholderAttention
78 + backend = which_attn_to_use(16, None, torch.float16, torch.float16, 16,
79 + True)
80 assert backend.name != STR_FLASH_ATTN_VAL
81
82
@@ -79,4 +84,4 @@ def test_invalid_env(monkeypatch):
84 """Throw an exception if the backend name is invalid."""
85 override_backend_env_variable(monkeypatch, STR_INVALID_VAL)
86 with pytest.raises(ValueError):
82 - which_attn_to_use(8, 16, 8, None, torch.float16, None, 16)
87 + which_attn_to_use(16, None, torch.float16, None, 16, False)
tests/models/decoder_only/language/test_mamba.py new
+295
@@ -0,0 +1,295 @@
1 +"""Compare the outputs of HF and vLLM when using greedy sampling for Mamba.
2 +
3 +Run `pytest tests/models/test_mamba.py`.
4 +"""
5 +import pytest
6 +from transformers import AutoModelForCausalLM, AutoTokenizer
7 +
8 +from vllm.sampling_params import SamplingParams
9 +from vllm.worker.model_runner import _get_graph_batch_size
10 +
11 +from ...utils import check_outputs_equal
12 +
13 +MODELS = ["state-spaces/mamba-130m-hf"]
14 +
15 +
16 +# Use lower-level interfaces to create this greedy generator, as mamba will
17 +# choke on the model_kwarg 'attention_mask' if hf_model.generate_greedy is used.
18 +def generate_greedy(model_name, example_prompts, max_tokens):
19 + # Create a text generation pipeline
20 + tokenizer = AutoTokenizer.from_pretrained(model_name)
21 + model = AutoModelForCausalLM.from_pretrained(model_name)
22 +
23 + # Generate texts from the prompts
24 + outputs = []
25 + for prompt in example_prompts:
26 + # Tokenize the input prompt with truncation
27 + inputs = tokenizer(prompt, return_tensors="pt", truncation=True)
28 + input_ids = inputs["input_ids"].to(model.device)
29 +
30 + # Generate text using the model's generate method directly
31 + generated_ids = model.generate(input_ids, max_new_tokens=max_tokens)
32 + generated_text = tokenizer.decode(generated_ids[0],
33 + skip_special_tokens=True)
34 +
35 + outputs.append((generated_ids[0].tolist(), generated_text))
36 +
37 + return outputs
38 +
39 +
40 +@pytest.mark.parametrize("model", MODELS)
41 +@pytest.mark.parametrize("dtype", ["float"])
42 +@pytest.mark.parametrize("max_tokens", [96])
43 +def test_models(
44 + vllm_runner,
45 + example_prompts,
46 + model: str,
47 + dtype: str,
48 + max_tokens: int,
49 +) -> None:
50 + hf_outputs = generate_greedy(model, example_prompts, max_tokens)
51 +
52 + with vllm_runner(model, dtype=dtype) as vllm_model:
53 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
54 +
55 + for i in range(len(example_prompts)):
56 + hf_output_ids, hf_output_str = hf_outputs[i]
57 + vllm_output_ids, vllm_output_str = vllm_outputs[i]
58 + assert hf_output_str == vllm_output_str, (
59 + f"Test{i}:\nHF: {hf_output_str!r}\nvLLM: {vllm_output_str!r}")
60 + assert hf_output_ids == vllm_output_ids, (
61 + f"Test{i}:\nHF: {hf_output_ids}\nvLLM: {vllm_output_ids}")
62 +
63 +
64 +@pytest.mark.parametrize("model", MODELS)
65 +@pytest.mark.parametrize("dtype", ["float"])
66 +@pytest.mark.parametrize("max_tokens", [96])
67 +def test_batching(
68 + vllm_runner,
69 + example_prompts,
70 + model: str,
71 + dtype: str,
72 + max_tokens: int,
73 +) -> None:
74 + # To pass the small model tests, we need full precision.
75 + for_loop_outputs = []
76 + with vllm_runner(model, dtype=dtype) as vllm_model:
77 + for prompt in example_prompts:
78 + for_loop_outputs.append(
79 + vllm_model.generate_greedy([prompt], max_tokens)[0])
80 +
81 + batched_outputs = vllm_model.generate_greedy(example_prompts,
82 + max_tokens)
83 +
84 + check_outputs_equal(
85 + outputs_0_lst=for_loop_outputs,
86 + outputs_1_lst=batched_outputs,
87 + name_0="for_loop_vllm",
88 + name_1="batched_vllm",
89 + )
90 +
91 +
92 +@pytest.mark.parametrize("model", MODELS)
93 +@pytest.mark.parametrize("dtype", ["float"])
94 +@pytest.mark.parametrize("max_tokens", [10])
95 +def test_chunked_prefill_with_parallel_sampling(vllm_runner, example_prompts,
96 + model: str, dtype: str,
97 + max_tokens: int) -> None:
98 + # Tests chunked prefill in conjunction with n>1. In this case, prefill is
99 + # populated with decoding tokens and we test that it doesn't fail.
100 + # This test might fail if cache is not allocated correctly for n > 1
101 + # decoding steps inside a chunked prefill forward pass (where we have both
102 + # prefill and decode together )
103 + sampling_params = SamplingParams(n=3,
104 + temperature=1,
105 + seed=0,
106 + max_tokens=max_tokens)
107 + with vllm_runner(
108 + model,
109 + dtype=dtype,
110 + enable_chunked_prefill=True,
111 + max_num_batched_tokens=30,
112 + max_num_seqs=10 # forces prefill chunks with decoding
113 + ) as vllm_model:
114 + vllm_model.generate(example_prompts, sampling_params)
115 +
116 +
117 +@pytest.mark.parametrize("model", MODELS)
118 +@pytest.mark.parametrize("dtype", ["float"])
119 +@pytest.mark.parametrize("max_tokens", [32])
120 +@pytest.mark.parametrize("chunked_prefill_token_size", [1, 4, 16])
121 +def test_chunked_prefill(vllm_runner, example_prompts, model: str, dtype: str,
122 + max_tokens: int,
123 + chunked_prefill_token_size: int) -> None:
124 + """
125 + Checks exact match decode between huggingface model and vllm runner with
126 + chunked prefill.
127 + """
128 + max_num_seqs = chunked_prefill_token_size
129 + max_num_batched_tokens = chunked_prefill_token_size
130 +
131 + non_chunked = generate_greedy(model, example_prompts, max_tokens)
132 +
133 + with vllm_runner(model,
134 + dtype=dtype,
135 + enable_chunked_prefill=True,
136 + max_num_batched_tokens=max_num_batched_tokens,
137 + max_num_seqs=max_num_seqs) as vllm_model:
138 + chunked = vllm_model.generate_greedy(example_prompts,
139 + max_tokens=max_tokens)
140 +
141 + check_outputs_equal(
142 + outputs_0_lst=chunked,
143 + outputs_1_lst=non_chunked,
144 + name_0="chunked",
145 + name_1="non_chunked",
146 + )
147 +
148 +
149 +@pytest.mark.parametrize("model", MODELS)
150 +@pytest.mark.parametrize("dtype", ["float"])
151 +@pytest.mark.parametrize("max_tokens", [15])
152 +def test_parallel_sampling(
153 + vllm_runner,
154 + example_prompts,
155 + model: str,
156 + dtype: str,
157 + max_tokens: int,
158 +) -> None:
159 +
160 + with vllm_runner(model, dtype=dtype) as vllm_model:
161 + for_loop_outputs = []
162 + for _ in range(10):
163 + for_loop_outputs.append(
164 + # using example_prompts index 1 instead of 0 since with 0 the
165 + # logprobs get really close and the test doesn't pass
166 + vllm_model.generate_greedy([example_prompts[1]], max_tokens)
167 + [0])
168 + sampling_params = SamplingParams(n=10,
169 + temperature=0.001,
170 + seed=0,
171 + max_tokens=max_tokens)
172 + n_lt_1_outputs = vllm_model.generate([example_prompts[1]],
173 + sampling_params)
174 + token_ids, texts = n_lt_1_outputs[0]
175 + n_lt_1_outputs = [(token_id, text)
176 + for token_id, text in zip(token_ids, texts)]
177 +
178 + check_outputs_equal(
179 + outputs_0_lst=n_lt_1_outputs,
180 + outputs_1_lst=for_loop_outputs,
181 + name_0="vllm_n_lt_1_outputs",
182 + name_1="vllm",
183 + )
184 +
185 +
186 +@pytest.mark.parametrize("model", MODELS)
187 +@pytest.mark.parametrize("dtype", ["bfloat16"])
188 +@pytest.mark.parametrize("max_tokens", [20])
189 +def test_mamba_cache_cg_padding(
190 + vllm_runner,
191 + example_prompts,
192 + model: str,
193 + dtype: str,
194 + max_tokens: int,
195 +) -> None:
196 + # This test is for verifying that mamba cache is padded to CG captured
197 + # batch size. If it's not, a torch RuntimeError will be raised because
198 + # tensor dimensions aren't compatible
199 + while len(example_prompts) == _get_graph_batch_size(len(example_prompts)):
200 + example_prompts.append(example_prompts[0])
201 +
202 + try:
203 + with vllm_runner(model, dtype=dtype) as vllm_model:
204 + vllm_model.generate_greedy(example_prompts, max_tokens)
205 + except RuntimeError:
206 + pytest.fail(
207 + "Couldn't run batch size which is not equal to a Cuda Graph "
208 + "captured batch size. "
209 + "Could be related to mamba cache not padded correctly")
210 +
211 +
212 +@pytest.mark.parametrize("model", MODELS)
213 +@pytest.mark.parametrize("dtype", ["float"])
214 +@pytest.mark.parametrize("max_tokens", [20])
215 +def test_models_preemption_recompute(
216 + vllm_runner,
217 + example_prompts,
218 + model: str,
219 + dtype: str,
220 + max_tokens: int,
221 +) -> None:
222 + # Tests that outputs are identical with and w/o preemtions (recompute)
223 + assert dtype == "float"
224 +
225 + with vllm_runner(model, dtype=dtype) as vllm_model:
226 + vllm_model.model.llm_engine.scheduler[
227 + 0].ENABLE_ARTIFICIAL_PREEMPT = True
228 + preempt_vllm_outputs = vllm_model.generate_greedy(
229 + example_prompts, max_tokens)
230 +
231 + vllm_model.model.llm_engine.scheduler[
232 + 0].ENABLE_ARTIFICIAL_PREEMPT = False
233 + vllm_outputs = vllm_model.generate_greedy(example_prompts, max_tokens)
234 +
235 + check_outputs_equal(
236 + outputs_0_lst=preempt_vllm_outputs,
237 + outputs_1_lst=vllm_outputs,
238 + name_0="vllm_preepmtions",
239 + name_1="vllm",
240 + )
241 +
242 +
243 +@pytest.mark.parametrize("model", MODELS)
244 +@pytest.mark.parametrize("dtype", ["float"])
245 +def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
246 + vllm_runner,
247 + model: str,
248 + dtype: str,
249 + example_prompts,
250 +) -> None:
251 + # This test is for verifying that the Mamba inner state management doesn't
252 + # collapse in case where the number of incoming requests and
253 + # finished_requests_ids is larger than the maximum Mamba block capacity.
254 + # This could generally happen due to the fact that Mamba does support
255 + # statelessness mechanism where it can cleanup new incoming requests in
256 + # a single step.
257 + try:
258 + with vllm_runner(model, dtype=dtype, max_num_seqs=10) as vllm_model:
259 + vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
260 + except ValueError:
261 + pytest.fail("Mamba inner state wasn't cleaned up properly between"
262 + "steps finished requests registered unnecessarily ")
263 +
264 +
265 +@pytest.mark.parametrize("model", MODELS)
266 +@pytest.mark.parametrize("dtype", ["float"])
267 +def test_state_cleanup(
268 + vllm_runner,
269 + model: str,
270 + dtype: str,
271 + example_prompts,
272 +) -> None:
273 + # This test is for verifying that the Mamba state is cleaned up between
274 + # steps, If its not cleaned, an error would be expected.
275 + try:
276 + with vllm_runner(model, dtype=dtype) as vllm_model:
277 + for _ in range(10):
278 + vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
279 + except ValueError:
280 + pytest.fail("Mamba inner state wasn't cleaned up between states, "
281 + "could be related to finished_requests_ids")
282 +
283 +
284 +@pytest.mark.parametrize("model", MODELS)
285 +@pytest.mark.parametrize("dtype", ["float"])
286 +def test_model_print(
287 + vllm_runner,
288 + model: str,
289 + dtype: str,
290 +) -> None:
291 + with vllm_runner(model, dtype=dtype) as vllm_model:
292 + # This test is for verifying whether the model's extra_repr
293 + # can be printed correctly.
294 + print(vllm_model.model.llm_engine.model_executor.driver_worker.
295 + model_runner.model)
vllm/attention/backends/placeholder_attn.py new
+324
@@ -0,0 +1,324 @@
1 +from dataclasses import dataclass
2 +from typing import TYPE_CHECKING, List, Optional, Tuple, Type
3 +
4 +import torch
5 +
6 +from vllm.attention.backends.abstract import (AttentionBackend, AttentionImpl,
7 + AttentionMetadata,
8 + AttentionMetadataBuilder)
9 +from vllm.attention.backends.utils import CommonAttentionState
10 +
11 +if TYPE_CHECKING:
12 + from vllm.worker.model_runner import ModelInputForGPUBuilder
13 +
14 +# Placeholder attention backend for models like Mamba and embedding models that
15 +# lack attention.
16 +
17 +
18 +class PlaceholderAttentionBackend(AttentionBackend):
19 + """Placeholder backend for when no attention is needed."""
20 +
21 + @staticmethod
22 + def get_name() -> str:
23 + return "placeholder-attn"
24 +
25 + @staticmethod
26 + def get_impl_cls() -> Type["PlaceholderAttentionImpl"]:
27 + return PlaceholderAttentionImpl
28 +
29 + @staticmethod
30 + def get_builder_cls() -> Type["PlaceholderAttentionMetadataBuilder"]:
31 + return PlaceholderAttentionMetadataBuilder
32 +
33 + @staticmethod
34 + def get_metadata_cls() -> Type["PlaceholderAttentionMetadata"]:
35 + return PlaceholderAttentionMetadata
36 +
37 + @staticmethod
38 + def get_state_cls() -> Type["CommonAttentionState"]:
39 + return CommonAttentionState
40 +
41 + @staticmethod
42 + def get_kv_cache_shape(
43 + num_blocks: int,
44 + block_size: int,
45 + num_kv_heads: int,
46 + head_size: int,
47 + ) -> Tuple[int, ...]:
48 + return (1, 1, 1, 1, 1)
49 +
50 + @staticmethod
51 + def swap_blocks(
52 + src_kv_cache: torch.Tensor,
53 + dst_kv_cache: torch.Tensor,
54 + src_to_dst: torch.Tensor,
55 + ) -> None:
56 + return
57 +
58 + @staticmethod
59 + def copy_blocks(
60 + kv_caches: List[torch.Tensor],
61 + src_to_dists: torch.Tensor,
62 + ) -> None:
63 + return
64 +
65 +
66 +@dataclass
67 +class PlaceholderAttentionMetadata(AttentionMetadata):
68 + """Attention metadata for prefill and decode batched together."""
69 + # (batch_size,). The sequence length per sequence. Sequence length means
70 + # the computed tokens + new tokens None if it is a decoding.
71 + seq_lens: Optional[List[int]]
72 + # seq_lens stored as a tensor.
73 + seq_lens_tensor: Optional[torch.Tensor]
74 +
75 + # Maximum query length in the batch.
76 + max_query_len: Optional[int]
77 +
78 + # Number of query tokens for each request in the batch.
79 + # Currently, we require that all requests have the same number of query
80 + # tokens during the decoding phase. When speculavie decoding is enabled,
81 + # decode_query_len might be greater than 1. In all other cases, it is 1.
82 + decode_query_len: Optional[int]
83 +
84 + # Maximum sequence length among prefill batch. 0 if there are decoding
85 + # requests only.
86 + max_prefill_seq_len: int
87 + # Maximum sequence length among decode batch. 0 if there are prefill
88 + # requests only.
89 + max_decode_seq_len: int
90 + # (batch_size + 1,). The cumulative subquery lengths of the sequences in
91 + # the batch, used to index into subquery. E.g., if the subquery length
92 + # is [4, 6], it is [0, 4, 10].
93 + query_start_loc: Optional[torch.Tensor]
94 + # (batch_size + 1,). The cumulative sequence lengths of the sequences in
95 + # the batch, used to index into sequence. E.g., if the sequence length is
96 + # [4, 6], it is [0, 4, 10].
97 + seq_start_loc: Optional[torch.Tensor]
98 + # (batch_size,) A tensor of context lengths (tokens that are computed
99 + # so far).
100 + context_lens_tensor: Optional[torch.Tensor]
101 +
102 + # (batch_size, max_blocks_per_seq).
103 + # Block addresses per sequence. (Seq id -> list of physical block)
104 + # E.g., [0, 1, 2] means tokens are stored in 0th, 1st, and 2nd blocks
105 + # in the kv cache. Each block can contain up to block_size tokens.
106 + # 2nd dimensions are padded up to max_blocks_per_seq if it is cuda-graph
107 + # captured.
108 + block_tables: Optional[torch.Tensor]
109 +
110 + # Whether or not if cuda graph is enabled.
111 + # Cuda-graph is currently enabled for decoding only.
112 + # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
113 + use_cuda_graph: bool
114 +
115 + _cached_prefill_metadata: Optional["PlaceholderAttentionMetadata"] = None
116 + _cached_decode_metadata: Optional["PlaceholderAttentionMetadata"] = None
117 +
118 + @property
119 + def prefill_metadata(self) -> Optional["PlaceholderAttentionMetadata"]:
120 + if self.num_prefills == 0:
121 + return None
122 +
123 + if self._cached_prefill_metadata is not None:
124 + return self._cached_prefill_metadata
125 +
126 + assert self.seq_lens is not None
127 + assert self.seq_lens_tensor is not None
128 + assert self.query_start_loc is not None
129 + assert self.context_lens_tensor is not None
130 + assert self.seq_start_loc is not None
131 +
132 + # Placeholders
133 + slot_mapping = torch.empty(0)
134 + block_tables = torch.empty(0)
135 +
136 + self._cached_prefill_metadata = PlaceholderAttentionMetadata(
137 + num_prefills=self.num_prefills,
138 + num_prefill_tokens=self.num_prefill_tokens,
139 + num_decode_tokens=0,
140 + slot_mapping=slot_mapping,
141 + seq_lens=self.seq_lens[:self.num_prefills],
142 + seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
143 + decode_query_len=0,
144 + max_query_len=self.max_query_len,
145 + max_prefill_seq_len=self.max_prefill_seq_len,
146 + max_decode_seq_len=0,
147 + query_start_loc=self.query_start_loc[:self.num_prefills + 1],
148 + seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
149 + context_lens_tensor=self.context_lens_tensor[:self.num_prefills],
150 + block_tables=block_tables,
151 + use_cuda_graph=False,
152 + )
153 + return self._cached_prefill_metadata
154 +
155 + @property
156 + def decode_metadata(self) -> Optional["PlaceholderAttentionMetadata"]:
157 + if self.num_decode_tokens == 0:
158 + return None
159 +
160 + if self._cached_decode_metadata is not None:
161 + return self._cached_decode_metadata
162 + assert self.seq_lens_tensor is not None
163 +
164 + # Placeholders
165 + slot_mapping = torch.empty(0)
166 + block_tables = torch.empty(0)
167 +
168 + self._cached_decode_metadata = PlaceholderAttentionMetadata(
169 + num_prefills=0,
170 + num_prefill_tokens=0,
171 + num_decode_tokens=self.num_decode_tokens,
172 + slot_mapping=slot_mapping,
173 + seq_lens=None,
174 + seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
175 + decode_query_len=self.decode_query_len,
176 + max_query_len=None,
177 + max_prefill_seq_len=0,
178 + max_decode_seq_len=self.max_decode_seq_len,
179 + query_start_loc=None,
180 + seq_start_loc=None,
181 + context_lens_tensor=None,
182 + block_tables=block_tables,
183 + use_cuda_graph=self.use_cuda_graph,
184 + )
185 + return self._cached_decode_metadata
186 +
187 +
188 +class PlaceholderAttentionMetadataBuilder(
189 + AttentionMetadataBuilder[PlaceholderAttentionMetadata]):
190 +
191 + def __init__(self, input_builder: "ModelInputForGPUBuilder"):
192 + self.prefill_seq_lens: List[int] = []
193 + self.context_lens: List[int] = []
194 + self.curr_seq_lens: List[int] = []
195 + self.num_prefills = 0
196 + self.num_prefill_tokens = 0
197 + self.num_decode_tokens = 0
198 +
199 + self.input_builder = input_builder
200 + self.runner = input_builder.runner
201 +
202 + def _add_seq_group(
203 + self, inter_data: "ModelInputForGPUBuilder.InterDataForSeqGroup",
204 + chunked_prefill_enabled: bool):
205 + """Add a sequence group to the metadata. Specifically update/append
206 + 1. context length.
207 + """
208 + is_prompt = inter_data.is_prompt
209 +
210 + for (seq_id, token_len, seq_len, curr_seq_len, query_len, context_len,
211 + curr_sliding_window_block) in zip(
212 + inter_data.seq_ids, [len(t) for t in inter_data.input_tokens],
213 + inter_data.orig_seq_lens, inter_data.seq_lens,
214 + inter_data.query_lens, inter_data.context_lens,
215 + inter_data.curr_sliding_window_blocks):
216 + self.context_lens.append(context_len)
217 +
218 + if is_prompt:
219 + self.num_prefills += 1
220 + self.num_prefill_tokens += token_len
221 + self.prefill_seq_lens.append(seq_len)
222 + else:
223 + assert query_len == 1, (
224 + "seq_len: {}, context_len: {}, query_len: {}".format(
225 + seq_len, context_len, query_len))
226 + self.num_decode_tokens += query_len
227 + self.curr_seq_lens.append(curr_seq_len)
228 +
229 + def build(self, seq_lens: List[int], query_lens: List[int],
230 + cuda_graph_pad_size: int, batch_size: int):
231 + """Build attention metadata with on-device tensors.
232 +
233 + Args:
234 + seq_lens: The maybe padded sequence lengths of the input sequences.
235 + query_lens: The query lengths of the input sequences.
236 + cuda_graph_pad_size: The padding size for cuda graph.
237 + -1 if cuda graph is not used.
238 + batch_size: The maybe padded batch size.
239 + """
240 + for inter_data in self.input_builder.inter_data_list:
241 + self._add_seq_group(inter_data,
242 + self.input_builder.chunked_prefill_enabled)
243 +
244 + device = self.runner.device
245 + use_captured_graph = cuda_graph_pad_size != -1
246 +
247 + logits_soft_cap = getattr(self.runner.model_config.hf_config,
248 + "attn_logit_softcapping", None)
249 + if logits_soft_cap is not None:
250 + raise ValueError(
251 + "Please use Flashinfer backend for models with logits_soft_cap"
252 + " (i.e., Gemma-2). Otherwise, the output might be wrong."
253 + " Set Flashinfer backend by "
254 + "export VLLM_ATTENTION_BACKEND=FLASHINFER.")
255 +
256 + max_query_len = max(query_lens)
257 + decode_query_lens = query_lens[self.num_prefills:]
258 + if len(decode_query_lens) > 0:
259 + decode_query_len = max(decode_query_lens)
260 + else:
261 + decode_query_len = 1
262 + max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
263 + max_decode_seq_len = max(self.curr_seq_lens, default=0)
264 + num_decode_tokens = self.num_decode_tokens
265 +
266 + if use_captured_graph:
267 + num_decode_tokens = batch_size
268 +
269 + assert max_query_len > 0, ("query_lens: {}".format(query_lens))
270 +
271 + context_lens_tensor = torch.tensor(self.context_lens,
272 + dtype=torch.int,
273 + device=device)
274 + seq_lens_tensor = torch.tensor(seq_lens,
275 + dtype=torch.int,
276 + device=device)
277 + query_lens_tensor = torch.tensor(query_lens,
278 + dtype=torch.long,
279 + device=device)
280 + query_start_loc = torch.zeros(query_lens_tensor.shape[0] + 1,
281 + dtype=torch.int32,
282 + device=device)
283 + seq_start_loc = torch.zeros(seq_lens_tensor.shape[0] + 1,
284 + dtype=torch.int32,
285 + device=device)
286 + torch.cumsum(seq_lens_tensor,
287 + dim=0,
288 + dtype=seq_start_loc.dtype,
289 + out=seq_start_loc[1:])
290 + torch.cumsum(query_lens_tensor,
291 + dim=0,
292 + dtype=query_start_loc.dtype,
293 + out=query_start_loc[1:])
294 +
295 + # Placeholders
296 + slot_mapping = torch.empty(0)
297 + block_tables = torch.empty(0)
298 +
299 + return PlaceholderAttentionMetadata(
300 + num_prefills=self.num_prefills,
301 + slot_mapping=slot_mapping,
302 + num_prefill_tokens=self.num_prefill_tokens,
303 + num_decode_tokens=num_decode_tokens,
304 + seq_lens=seq_lens,
305 + seq_lens_tensor=seq_lens_tensor,
306 + max_query_len=max_query_len,
307 + decode_query_len=decode_query_len,
308 + max_prefill_seq_len=max_prefill_seq_len,
309 + max_decode_seq_len=max_decode_seq_len,
310 + query_start_loc=query_start_loc,
311 + seq_start_loc=seq_start_loc,
312 + context_lens_tensor=context_lens_tensor,
313 + block_tables=block_tables,
314 + use_cuda_graph=use_captured_graph,
315 + )
316 +
317 +
318 +class PlaceholderAttentionImpl(AttentionImpl):
319 +
320 + def __init__(self, *args, **kwargs) -> None:
321 + return
322 +
323 + def forward(self, *args, **kwargs) -> torch.Tensor:
324 + raise NotImplementedError
vllm/attention/layer.py
+5 -3
@@ -42,10 +42,12 @@ class Attention(nn.Module):
42 kv_cache_dtype = cache_config.cache_dtype
43 block_size = cache_config.block_size
44 sliding_window = cache_config.sliding_window
45 + is_attention_free = cache_config.is_attention_free
46 else:
47 kv_cache_dtype = "auto"
48 block_size = 16
49 sliding_window = None
50 + is_attention_free = False
51 if num_kv_heads is None:
52 num_kv_heads = num_heads
53
@@ -76,9 +78,9 @@ class Attention(nn.Module):
78 # During model initialization, the default dtype is set as the model
79 # weight and activation dtype.
80 dtype = torch.get_default_dtype()
79 - attn_backend = get_attn_backend(num_heads, head_size, num_kv_heads,
80 - sliding_window, dtype, kv_cache_dtype,
81 - block_size, blocksparse_params
81 + attn_backend = get_attn_backend(head_size, sliding_window, dtype,
82 + kv_cache_dtype, block_size,
83 + is_attention_free, blocksparse_params
84 is not None)
85 impl_cls = attn_backend.get_impl_cls()
86 self.impl = impl_cls(num_heads, head_size, scale, num_kv_heads,
vllm/attention/selector.py
+14 -7
@@ -24,6 +24,7 @@ class _Backend(enum.Enum):
24 FLASHINFER = enum.auto()
25 PALLAS = enum.auto()
26 IPEX = enum.auto()
27 + NO_ATTENTION = enum.auto()
28
29
30 def backend_name_to_enum(backend_name: str) -> _Backend:
@@ -88,13 +89,12 @@ def get_global_forced_attn_backend() -> Optional[_Backend]:
89
90 @lru_cache(maxsize=None)
91 def get_attn_backend(
91 - num_heads: int,
92 head_size: int,
93 - num_kv_heads: int,
93 sliding_window: Optional[int],
94 dtype: torch.dtype,
95 kv_cache_dtype: Optional[str],
96 block_size: int,
97 + is_attention_free: bool,
98 is_blocksparse: bool = False,
99 ) -> Type[AttentionBackend]:
100 """Selects which attention backend to use and lazily imports it."""
@@ -105,9 +105,8 @@ def get_attn_backend(
105 BlocksparseFlashAttentionBackend)
106 return BlocksparseFlashAttentionBackend
107
108 - backend = which_attn_to_use(num_heads, head_size, num_kv_heads,
109 - sliding_window, dtype, kv_cache_dtype,
110 - block_size)
108 + backend = which_attn_to_use(head_size, sliding_window, dtype,
109 + kv_cache_dtype, block_size, is_attention_free)
110 if backend == _Backend.FLASH_ATTN:
111 from vllm.attention.backends.flash_attn import ( # noqa: F401
112 FlashAttentionBackend)
@@ -146,23 +145,31 @@ def get_attn_backend(
145 logger.info("Using Pallas backend.")
146 from vllm.attention.backends.pallas import PallasAttentionBackend
147 return PallasAttentionBackend
148 + elif backend == _Backend.NO_ATTENTION:
149 + from vllm.attention.backends.placeholder_attn import (
150 + PlaceholderAttentionBackend)
151 + return PlaceholderAttentionBackend
152 else:
153 raise ValueError("Invalid attention backend.")
154
155
156 def which_attn_to_use(
154 - num_heads: int,
157 head_size: int,
156 - num_kv_heads: int,
158 sliding_window: Optional[int],
159 dtype: torch.dtype,
160 kv_cache_dtype: Optional[str],
161 block_size: int,
162 + is_attention_free: bool,
163 ) -> _Backend:
164 """Returns which flash attention backend to use."""
165 # Default case.
166 selected_backend = _Backend.FLASH_ATTN
167
168 + # If there are no attention layers (e.g. we are running Mamba),
169 + # use the placeholder NO_ATTENTION
170 + if is_attention_free:
171 + return _Backend.NO_ATTENTION
172 +
173 # Check whether a particular choice of backend was
174 # previously forced.
175 #
vllm/config.py
+28 -22
@@ -196,6 +196,9 @@ class ModelConfig:
196 if not self.skip_tokenizer_init:
197 self._verify_tokenizer_mode()
198
199 + self.is_attention_free = self._init_attention_free()
200 + self.has_inner_state = self._init_has_inner_state()
201 +
202 self.override_neuron_config = override_neuron_config if is_neuron(
203 ) else None
204 self._verify_embedding_mode()
@@ -216,6 +219,14 @@ class ModelConfig:
219
220 return None
221
222 + def _init_attention_free(self) -> bool:
223 + architectures = getattr(self.hf_config, "architectures", [])
224 + return ModelRegistry.is_attention_free_model(architectures)
225 +
226 + def _init_has_inner_state(self) -> bool:
227 + architectures = getattr(self.hf_config, "architectures", [])
228 + return ModelRegistry.model_has_inner_state(architectures)
229 +
230 def _verify_tokenizer_mode(self) -> None:
231 tokenizer_mode = self.tokenizer_mode.lower()
232 if tokenizer_mode not in ["auto", "slow", "mistral"]:
@@ -438,6 +449,10 @@ class ModelConfig:
449 # FlashAttention supports only head_size 32, 64, 128, 256,
450 # we need to pad head_size 192 to 256
451 return 256
452 +
453 + if self.is_attention_free:
454 + return 0
455 +
456 if hasattr(self.hf_text_config, "head_dim"):
457 return self.hf_text_config.head_dim
458 # FIXME(woosuk): This may not be true for all models.
@@ -469,6 +484,9 @@ class ModelConfig:
484 return getattr(self.hf_config.attn_config, "kv_n_heads",
485 self.hf_config.num_attention_heads)
486
487 + if self.is_attention_free:
488 + return 0
489 +
490 attributes = [
491 # For Falcon:
492 "n_head_kv",
@@ -511,31 +529,17 @@ class ModelConfig:
529 start, end = get_pp_indices(total_num_hidden_layers, pp_rank, pp_size)
530 return end - start
531
514 - def contains_seqlen_agnostic_layers(
515 - self, parallel_config: "ParallelConfig") -> bool:
516 - """True for Mamba/SSM models (Jamba)"""
517 - return self._get_num_seqlen_agnostic_layers(parallel_config) > 0
532 + def get_num_attention_layers(self,
533 + parallel_config: "ParallelConfig") -> int:
534 + if self.is_attention_free:
535 + return 0
536
519 - def get_layers_block_type(self,
520 - parallel_config: "ParallelConfig") -> List[str]:
537 num_layers = self.get_num_layers(parallel_config)
522 - # Transformers supports layers_block_type @property
523 - return getattr(self.hf_config, "layers_block_type",
524 - ["attention"] * num_layers)
538
526 - def get_num_attention_layers(self,
527 - parallel_config: "ParallelConfig") -> int:
528 - return len([
529 - t for t in self.get_layers_block_type(parallel_config)
530 - if t == "attention"
531 - ])
532 -
533 - def _get_num_seqlen_agnostic_layers(
534 - self, parallel_config: "ParallelConfig") -> int:
535 - return len([
536 - t for t in self.get_layers_block_type(parallel_config)
537 - if t != "attention"
538 - ])
539 + # Transformers supports layers_block_type @property
540 + layers = getattr(self.hf_config, "layers_block_type",
541 + ["attention"] * num_layers)
542 + return len([t for t in layers if t == "attention"])
543
544 def get_multimodal_config(self) -> "MultiModalConfig":
545 """
@@ -585,6 +589,7 @@ class CacheConfig:
589 gpu_memory_utilization: float,
590 swap_space: float,
591 cache_dtype: str,
592 + is_attention_free: bool = False,
593 num_gpu_blocks_override: Optional[int] = None,
594 sliding_window: Optional[int] = None,
595 enable_prefix_caching: bool = False,
@@ -595,6 +600,7 @@ class CacheConfig:
600 self.swap_space_bytes = swap_space * GiB_bytes
601 self.num_gpu_blocks_override = num_gpu_blocks_override
602 self.cache_dtype = cache_dtype
603 + self.is_attention_free = is_attention_free
604 self.sliding_window = sliding_window
605 self.enable_prefix_caching = enable_prefix_caching
606 self.cpu_offload_gb = cpu_offload_gb
vllm/core/interfaces.py
+4 -4
@@ -36,10 +36,10 @@ class BlockSpaceManager(ABC):
36 from vllm.core.block_manager_v2 import BlockSpaceManagerV2
37 return BlockSpaceManagerV2
38
39 - if version == "embedding":
40 - from vllm.core.embedding_model_block_manager import (
41 - EmbeddingModelBlockSpaceManager)
42 - return EmbeddingModelBlockSpaceManager
39 + if version == "placeholder":
40 + from vllm.core.placeholder_block_space_manager import (
41 + PlaceholderBlockSpaceManager)
42 + return PlaceholderBlockSpaceManager
43
44 raise ValueError(f"Unknown version {version=}")
45
vllm/core/placeholder_block_space_manager.py renamed
+5 -4
@@ -5,9 +5,10 @@ from vllm.sequence import Sequence, SequenceGroup
5 from vllm.utils import Device
6
7
8 -class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
9 - """An embedding version of BlockSpaceManager for use in environments
10 - with embedding models where block management is not required.
8 +class PlaceholderBlockSpaceManager(BlockSpaceManager):
9 + """A version of BlockSpaceManager for use in environments
10 + where block management is not required.
11 + For example: embedding models or attention-free models like Mamba.
12
13 This class provides the same interface as BlockSpaceManager, but its
14 methods perform no actions or return simple values like True in specific
@@ -40,7 +41,7 @@ class EmbeddingModelBlockSpaceManager(BlockSpaceManager):
41 seq: Sequence,
42 num_lookahead_slots: int,
43 ) -> List[Tuple[int, int]]:
43 - return None # type: ignore
44 + return []
45
46 def fork(self, parent_seq: Sequence, child_seq: Sequence) -> None:
47 pass
vllm/core/scheduler.py
+3 -2
@@ -314,8 +314,9 @@ class Scheduler:
314 version = "v1"
315 if self.scheduler_config.use_v2_block_manager:
316 version = "v2"
317 - if self.scheduler_config.embedding_mode:
318 - version = "embedding"
317 + if (self.scheduler_config.embedding_mode
318 + or self.cache_config.is_attention_free):
319 + version = "placeholder"
320
321 BlockSpaceManagerImpl = BlockSpaceManager.get_block_space_manager_class(
322 version)
vllm/engine/arg_utils.py
+2 -5
@@ -912,6 +912,7 @@ class EngineArgs:
912 gpu_memory_utilization=self.gpu_memory_utilization,
913 swap_space=self.swap_space,
914 cache_dtype=self.kv_cache_dtype,
915 + is_attention_free=model_config.is_attention_free,
916 num_gpu_blocks_override=self.num_gpu_blocks_override,
917 sliding_window=model_config.get_sliding_window(),
918 enable_prefix_caching=self.enable_prefix_caching,
@@ -945,13 +946,9 @@ class EngineArgs:
946 use_sliding_window = (model_config.get_sliding_window()
947 is not None)
948 use_spec_decode = self.speculative_model is not None
948 - has_seqlen_agnostic_layers = (
949 - model_config.contains_seqlen_agnostic_layers(
950 - parallel_config))
949 if (is_gpu and not use_sliding_window and not use_spec_decode
950 and not self.enable_lora
953 - and not self.enable_prompt_adapter
954 - and not has_seqlen_agnostic_layers):
951 + and not self.enable_prompt_adapter):
952 self.enable_chunked_prefill = True
953 logger.warning(
954 "Chunked prefill is enabled by default for models with "
vllm/model_executor/model_loader/weight_utils.py
+34 -1
@@ -6,7 +6,8 @@ import json
6 import os
7 import tempfile
8 from collections import defaultdict
9 -from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union
9 +from typing import (Any, Callable, Dict, Generator, Iterable, List, Optional,
10 + Tuple, Union)
11
12 import filelock
13 import gguf
@@ -559,6 +560,38 @@ def row_parallel_weight_loader(param: torch.Tensor,
560 return default_weight_loader(param, loaded_weight)
561
562
563 +LoaderFunction = Callable[[torch.Tensor, torch.Tensor], torch.Tensor]
564 +
565 +
566 +def sharded_weight_loader(shard_axis: int) -> LoaderFunction:
567 + """Create a weight loader that shards the weights along the given axis"""
568 +
569 + def loader(param: torch.Tensor, loaded_weight: torch.Tensor) -> None:
570 + tp_rank = get_tensor_model_parallel_rank()
571 +
572 + shard_size = param.data.shape[shard_axis]
573 + start_idx = tp_rank * shard_size
574 + loaded_weight = loaded_weight.narrow(shard_axis, start_idx, shard_size)
575 +
576 + return default_weight_loader(param, loaded_weight)
577 +
578 + return loader
579 +
580 +
581 +def composed_weight_loader(
582 + loader: LoaderFunction, fn: Callable[[torch.Tensor],
583 + torch.Tensor]) -> LoaderFunction:
584 + """Create a weight loader that post-processes the weights after loading"""
585 +
586 + def composed_loader(param: torch.Tensor,
587 + loaded_weight: torch.Tensor) -> None:
588 + loader(param, loaded_weight)
589 + param.data.copy_(fn(param))
590 + return
591 +
592 + return composed_loader
593 +
594 +
595 def initialize_dummy_weights(
596 model: torch.nn.Module,
597 low: float = -1e-3,
vllm/model_executor/models/interfaces.py
+44 -1
@@ -271,7 +271,7 @@ class HasInnerState(Protocol):
271 """
272 A flag that indicates this model has inner state.
273 Models that has inner state usually need access to the scheduler_config
274 - for max_num_seqs ,etc... (Currently only used by Jamba)
274 + for max_num_seqs, etc. True for e.g. both Mamba and Jamba.
275 """
276
277 def __init__(self,
@@ -307,3 +307,46 @@ def has_inner_state(
307 return isinstance(model, _HasInnerStateType)
308
309 return isinstance(model, HasInnerState)
310 +
311 +
312 +@runtime_checkable
313 +class IsAttentionFree(Protocol):
314 + """The interface required for all models like Mamba that lack attention,
315 + but do have state whose size is constant wrt the number of tokens."""
316 +
317 + is_attention_free: ClassVar[Literal[True]] = True
318 + """
319 + A flag that indicates this model has no attention.
320 + Used for block manager and attention backend selection.
321 + True for Mamba but not Jamba.
322 + """
323 +
324 + def __init__(self) -> None:
325 + ...
326 +
327 +
328 +@runtime_checkable
329 +class _IsAttentionFreeType(Protocol):
330 + is_attention_free: ClassVar[Literal[True]]
331 +
332 + def __init__(self) -> None:
333 + ...
334 +
335 +
336 +@overload
337 +def is_attention_free(model: object) -> TypeIs[IsAttentionFree]:
338 + ...
339 +
340 +
341 +@overload
342 +def is_attention_free(model: Type[object]) -> TypeIs[Type[IsAttentionFree]]:
343 + ...
344 +
345 +
346 +def is_attention_free(
347 + model: Union[Type[object], object]
348 +) -> Union[TypeIs[Type[IsAttentionFree]], TypeIs[IsAttentionFree]]:
349 + if isinstance(model, type):
350 + return isinstance(model, _IsAttentionFreeType)
351 +
352 + return isinstance(model, IsAttentionFree)
vllm/model_executor/models/jamba.py
+32 -229
@@ -1,18 +1,16 @@
1 # coding=utf-8
2 """Inference-only Jamba model."""
3 from dataclasses import dataclass
4 -from typing import Dict, Iterable, List, Optional, Tuple
4 +from typing import Iterable, List, Optional, Tuple
5
6 import torch
7 from torch import nn
8 -from torch.nn.parameter import Parameter
8 from transformers import JambaConfig
9
10 from vllm.attention.backends.abstract import AttentionMetadata
11 from vllm.attention.layer import Attention
12 from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
14 -from vllm.distributed import (get_tensor_model_parallel_rank,
15 - get_tensor_model_parallel_world_size)
13 +from vllm.distributed import get_tensor_model_parallel_world_size
14 from vllm.model_executor.layers.fused_moe import FusedMoE
15 from vllm.model_executor.layers.layernorm import RMSNorm
16 from vllm.model_executor.layers.linear import (ColumnParallelLinear,
@@ -29,7 +27,9 @@ from vllm.model_executor.layers.quantization import QuantizationConfig
27 from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
28 from vllm.model_executor.layers.vocab_parallel_embedding import (
29 DEFAULT_VOCAB_PADDING_SIZE, ParallelLMHead, VocabParallelEmbedding)
32 -from vllm.model_executor.model_loader.weight_utils import default_weight_loader
30 +from vllm.model_executor.model_loader.weight_utils import (
31 + composed_weight_loader, default_weight_loader, sharded_weight_loader)
32 +from vllm.model_executor.models.mamba_cache import MambaCacheManager
33 from vllm.model_executor.sampling_metadata import SamplingMetadata
34 from vllm.model_executor.utils import set_weight_attrs
35 from vllm.sequence import IntermediateTensors
@@ -99,16 +99,6 @@ class JambaMambaMixer(nn.Module):
99 bias=True,
100 skip_bias_add=True)
101
102 - def weight_loader(param: Parameter, loaded_weight: torch.Tensor):
103 - tp_rank = get_tensor_model_parallel_rank()
104 - tp_size = get_tensor_model_parallel_world_size()
105 - param.data.copy_(
106 - loaded_weight.data.split(loaded_weight.shape[0] // tp_size,
107 - dim=0)[tp_rank])
108 -
109 - def A_weight_loader(param: Parameter, loaded_weight: torch.Tensor):
110 - weight_loader(param, -torch.exp(loaded_weight.float()))
111 -
102 tp_size = get_tensor_model_parallel_world_size()
103 self.A = nn.Parameter(
104 torch.empty(
@@ -118,8 +108,10 @@ class JambaMambaMixer(nn.Module):
108 ))
109 self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
110
121 - set_weight_attrs(self.D, {"weight_loader": weight_loader})
122 - set_weight_attrs(self.A, {"weight_loader": A_weight_loader})
111 + set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
112 + a_weight_loader = composed_weight_loader(
113 + sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
114 + set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
115
116 self.out_proj = RowParallelLinear(
117 self.intermediate_size,
@@ -571,10 +563,8 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
563 if not lora_config else lora_config.lora_vocab_padding_size,
564 )
565 # Used to track and store by the Mamba cache between steps.
574 - self.mamba_cache: Tuple[torch.Tensor, torch.Tensor] = tuple()
575 - # Maps between the request id and a dict that maps between the seq_id
576 - # and its index inside the self.mamba_cache
577 - self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
566 + self.mamba_cache: Optional[MambaCacheManager] = None
567 +
568 self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
569 config.vocab_size)
570 self.sampler = Sampler()
@@ -586,203 +576,36 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
576 attn_metadata: AttentionMetadata,
577 intermediate_tensors: Optional[IntermediateTensors] = None,
578 **kwargs):
589 - if not self.mamba_cache:
590 - self._prepare_mamba_cache()
591 -
592 - if "seqlen_agnostic_capture_inputs" not in kwargs:
593 - # We get here only on Prefill/Eager mode runs
594 - request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
595 - finished_requests_ids = kwargs["finished_requests_ids"]
596 - mamba_cache = self._release_finished_and_prepare_mamba_cache(
597 - finished_requests_ids, request_ids_to_seq_ids)
598 - else:
599 - # CUDA graph capturing runs
600 - mamba_cache = kwargs["seqlen_agnostic_capture_inputs"]
601 -
602 - hidden_states = self.model(input_ids, positions, kv_caches,
603 - attn_metadata, mamba_cache[0],
604 - mamba_cache[1])
605 - return hidden_states
606 -
607 - def _swap_mamba_cache(self, from_index: int, to_index: int):
608 - assert len(self.mamba_cache) > 0
609 - for cache_t in self.mamba_cache:
610 - cache_t[:, [to_index,from_index]] = \
611 - cache_t[:, [from_index,to_index]]
612 -
613 - def _copy_mamba_cache(self, from_index: int, to_index: int):
614 - assert len(self.mamba_cache) > 0
615 - for cache_t in self.mamba_cache:
616 - cache_t[:, to_index].copy_(cache_t[:, from_index],
617 - non_blocking=True)
618 -
619 - def _move_out_if_already_occupied(self, index: int,
620 - all_occupied_indices: List[int]):
621 - if index in all_occupied_indices:
622 - first_free_index = self._first_free_index_in_mamba_cache()
623 - # In case occupied, move the occupied to a new empty block
624 - self._move_cache_index_and_mappings(from_index=index,
625 - to_index=first_free_index)
626 -
627 - def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
628 - seq_id: int,
629 - destination_index: int):
630 - """
631 - Assign (req_id,seq_id) pair to a `destination_index` index, if
632 - already occupied, move the occupying index to a free index.
633 - """
634 - all_occupied_indices = self._get_all_occupied_indices()
635 - if cur_rid not in self.mamba_cache_indices_mapping:
636 - self._move_out_if_already_occupied(
637 - index=destination_index,
638 - all_occupied_indices=all_occupied_indices)
639 - self.mamba_cache_indices_mapping[cur_rid] = {
640 - seq_id: destination_index
641 - }
642 - elif seq_id not in (seq_ids2indices :=
643 - self.mamba_cache_indices_mapping[cur_rid]):
644 - # parallel sampling , where n > 1, assume prefill have
645 - # already happened now we only need to copy the already
646 - # existing cache into the siblings seq_ids caches
647 - self._move_out_if_already_occupied(
648 - index=destination_index,
649 - all_occupied_indices=all_occupied_indices)
650 - index_exists = list(seq_ids2indices.values())[0]
651 - # case of decoding n>1, copy prefill cache to decoding indices
652 - self._copy_mamba_cache(from_index=index_exists,
653 - to_index=destination_index)
654 - self.mamba_cache_indices_mapping[cur_rid][
655 - seq_id] = destination_index
656 - else:
657 - # already exists
658 - cache_index_already_exists = self.mamba_cache_indices_mapping[
659 - cur_rid][seq_id]
660 - if cache_index_already_exists != destination_index:
661 - # In case the seq id already exists but not in
662 - # the right destination, swap it with what's occupying it
663 - self._swap_pair_indices_and_mappings(
664 - from_index=cache_index_already_exists,
665 - to_index=destination_index)
666 -
667 - def _prepare_current_run_mamba_cache(
668 - self, request_ids_to_seq_ids: Dict[str, list[int]],
669 - finished_requests_ids: List[str]
670 - ) -> Tuple[torch.Tensor, torch.Tensor]:
671 - running_indices = []
672 - request_ids_to_seq_ids_flatten = [
673 - (req_id, seq_id)
674 - for req_id, seq_ids in request_ids_to_seq_ids.items()
675 - for seq_id in seq_ids
676 - ]
677 - batch_size = len(request_ids_to_seq_ids_flatten)
678 - for dest_index, (request_id,
679 - seq_id) in enumerate(request_ids_to_seq_ids_flatten):
680 - if request_id in finished_requests_ids:
681 - # Do not allocate cache index for requests that run
682 - # and finish right after
683 - continue
684 - self._assign_seq_id_to_mamba_cache_in_specific_dest(
685 - request_id, seq_id, dest_index)
686 - running_indices.append(dest_index)
579 + if self.mamba_cache is None:
580 + max_batch_size = (_get_graph_batch_size(
581 + self.scheduler_config.max_num_seqs) if self.scheduler_config
582 + else max(_BATCH_SIZES_TO_CAPTURE) + 2)
583
688 - self._clean_up_first_bs_blocks(batch_size, running_indices)
689 - conv_state = self.mamba_cache[0][:, :batch_size]
690 - temporal_state = self.mamba_cache[1][:, :batch_size]
584 + layers_type = self.config.layers_block_type
585 + num_mamba_layers = sum(
586 + [layer_type == "mamba" for layer_type in layers_type])
587
692 - return (conv_state, temporal_state)
588 + self.mamba_cache = MambaCacheManager(
589 + self.lm_head.weight.dtype, num_mamba_layers, max_batch_size,
590 + *self._get_mamba_cache_shape())
591
694 - def _get_all_occupied_indices(self):
695 - return [
696 - cache_idx
697 - for seq_ids2indices in self.mamba_cache_indices_mapping.values()
698 - for cache_idx in seq_ids2indices.values()
699 - ]
592 + mamba_cache_tensors = self.mamba_cache.current_run_tensors(
593 + input_ids, attn_metadata, **kwargs)
594
701 - def _clean_up_first_bs_blocks(self, batch_size: int,
702 - indices_for_current_run: List[int]):
703 - # move out all of the occupied but currently not running blocks
704 - # outside of the first n blocks
705 - destination_indices = range(batch_size)
706 - max_possible_batch_size = self.mamba_cache[0].shape[1]
707 - for destination_index in destination_indices:
708 - if destination_index in self._get_all_occupied_indices() and \
709 - destination_index not in indices_for_current_run:
710 - # move not running indices outside of the batch
711 - all_other_indices = list(
712 - range(batch_size, max_possible_batch_size))
713 - first_avail_index = self._first_free_index_in_mamba_cache(
714 - all_other_indices)
715 - self._swap_indices(from_index=destination_index,
716 - to_index=first_avail_index)
717 -
718 - def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
719 - self._copy_mamba_cache(from_index=from_index, to_index=to_index)
720 - self._update_mapping_index(from_index=from_index, to_index=to_index)
721 -
722 - def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
723 - self._swap_mamba_cache(from_index=from_index, to_index=to_index)
724 - self._swap_mapping_index(from_index=from_index, to_index=to_index)
725 -
726 - def _swap_mapping_index(self, from_index: int, to_index: int):
727 - for seq_ids2index in self.mamba_cache_indices_mapping.values():
728 - for seq_id, index in seq_ids2index.items():
729 - if from_index == index:
730 - seq_ids2index.update({seq_id: to_index})
731 - elif to_index == index:
732 - seq_ids2index.update({seq_id: from_index})
733 -
734 - def _update_mapping_index(self, from_index: int, to_index: int):
735 - for seq_ids2index in self.mamba_cache_indices_mapping.values():
736 - for seq_id, index in seq_ids2index.items():
737 - if from_index == index:
738 - seq_ids2index.update({seq_id: to_index})
739 - return
740 -
741 - def _release_finished_and_prepare_mamba_cache(
742 - self, finished_requests_ids,
743 - request_ids_to_seq_ids) -> Tuple[torch.Tensor, torch.Tensor]:
744 - self._release_mamba_cache(finished_requests_ids)
745 - return self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
746 - finished_requests_ids)
595 + hidden_states = self.model(input_ids, positions, kv_caches,
596 + attn_metadata, mamba_cache_tensors[0],
597 + mamba_cache_tensors[1])
598 + return hidden_states
599
600 def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
749 - """
750 - Copy the relevant Mamba cache into the CUDA graph input buffer
751 - that was provided during the capture runs
752 - (JambaForCausalLM.mamba_gc_cache_buffer).
753 - """
754 - self._release_finished_and_prepare_mamba_cache(
755 - kwargs["finished_requests_ids"], kwargs["request_ids_to_seq_ids"])
601 + return self.mamba_cache.copy_inputs_before_cuda_graphs(
602 + input_buffers, **kwargs)
603
604 def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
758 - """
759 - Provide the CUDA graph capture runs with a buffer in adjusted size.
760 - The buffer is used to maintain the Mamba Cache during the CUDA graph
761 - replay runs.
762 - """
763 - return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
764 -
765 - def _release_mamba_cache(self, finished_seq_groups_req_ids: List[str]):
766 - for req_id in finished_seq_groups_req_ids:
767 - if req_id in self.mamba_cache_indices_mapping:
768 - self.mamba_cache_indices_mapping.pop(req_id)
769 -
770 - def _first_free_index_in_mamba_cache(
771 - self, indices_range: Optional[List[int]] = None) -> int:
772 - assert self.mamba_cache is not None
773 - if indices_range is None:
774 - max_possible_batch_size = self.mamba_cache[0].shape[1]
775 - indices_range = list(range(max_possible_batch_size))
776 - all_occupied_indices = self._get_all_occupied_indices()
777 - for i in indices_range:
778 - if i not in all_occupied_indices:
779 - return i
780 - raise Exception("Couldn't find a free spot in the mamba cache! This"
781 - "should never happen")
605 + return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size)
606
607 def _get_mamba_cache_shape(
784 - self
785 - ) -> Tuple[Optional[Tuple[int, int]], Optional[Tuple[int, int]]]:
608 + self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
609 world_size = get_tensor_model_parallel_world_size()
610 hidden_size = self.config.hidden_size
611 conv_state_shape = (
@@ -790,31 +613,11 @@ class JambaForCausalLM(nn.Module, HasInnerState, SupportsLoRA):
613 self.config.mamba_d_conv - 1,
614 )
615 temporal_state_shape = (
793 - self.config.mamba_expand * self.config.hidden_size // world_size,
616 + self.config.mamba_expand * hidden_size // world_size,
617 self.config.mamba_d_state,
618 )
619 return conv_state_shape, temporal_state_shape
620
798 - def _prepare_mamba_cache(self):
799 - dtype = self.lm_head.weight.dtype
800 - layers_type = self.config.layers_block_type
801 - mamba_layers = sum(
802 - [layer_type == "mamba" for layer_type in layers_type])
803 - max_batch_size = (_get_graph_batch_size(
804 - self.scheduler_config.max_num_seqs) if self.scheduler_config else
805 - max(_BATCH_SIZES_TO_CAPTURE) + 2)
806 - conv_state_shape, temporal_state_shape = self._get_mamba_cache_shape()
807 - assert conv_state_shape is not None and temporal_state_shape is not None
808 -
809 - self.mamba_cache = (torch.empty(size=(mamba_layers, max_batch_size) +
810 - conv_state_shape,
811 - dtype=dtype,
812 - device="cuda"),
813 - torch.empty(size=(mamba_layers, max_batch_size) +
814 - temporal_state_shape,
815 - dtype=dtype,
816 - device="cuda"))
817 -
621 def compute_logits(
622 self,
623 hidden_states: torch.Tensor,
vllm/model_executor/models/mamba.py new
+499
@@ -0,0 +1,499 @@
1 +# coding=utf-8
2 +"""PyTorch MAMBA model."""
3 +from dataclasses import dataclass
4 +from typing import Iterable, List, Optional, Tuple
5 +
6 +import torch
7 +from torch import nn
8 +from transformers import MambaConfig
9 +
10 +from vllm.attention.backends.abstract import AttentionMetadata
11 +from vllm.config import CacheConfig, LoRAConfig, SchedulerConfig
12 +from vllm.distributed import get_tensor_model_parallel_world_size
13 +from vllm.model_executor.layers.activation import SiluAndMul
14 +from vllm.model_executor.layers.layernorm import RMSNorm
15 +from vllm.model_executor.layers.linear import (ColumnParallelLinear,
16 + MergedColumnParallelLinear,
17 + RowParallelLinear)
18 +from vllm.model_executor.layers.logits_processor import LogitsProcessor
19 +from vllm.model_executor.layers.mamba.ops.causal_conv1d import (
20 + causal_conv1d_fn, causal_conv1d_update)
21 +from vllm.model_executor.layers.mamba.ops.mamba_ssm import (
22 + selective_scan_fn, selective_state_update)
23 +from vllm.model_executor.layers.quantization.base_config import (
24 + QuantizationConfig)
25 +from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
26 +from vllm.model_executor.layers.vocab_parallel_embedding import (
27 + VocabParallelEmbedding)
28 +from vllm.model_executor.model_loader.weight_utils import (
29 + composed_weight_loader, default_weight_loader, sharded_weight_loader)
30 +from vllm.model_executor.models.interfaces import (HasInnerState,
31 + IsAttentionFree)
32 +from vllm.model_executor.models.mamba_cache import MambaCacheManager
33 +from vllm.model_executor.sampling_metadata import SamplingMetadata
34 +from vllm.model_executor.utils import set_weight_attrs
35 +from vllm.sequence import IntermediateTensors
36 +from vllm.worker.model_runner import (_BATCH_SIZES_TO_CAPTURE,
37 + _get_graph_batch_size)
38 +
39 +KVCache = Tuple[torch.Tensor, torch.Tensor]
40 +
41 +
42 +@dataclass
43 +class MambaCacheParams:
44 + is_prompt: bool = False
45 + conv_state: torch.Tensor = torch.Tensor()
46 + ssm_state: torch.Tensor = torch.Tensor()
47 +
48 +
49 +# Adapted from transformers.models.mamba.modeling_mamba.MambaMixer
50 +class MambaMixer(nn.Module):
51 + """
52 + Compute ∆, A, B, C, and D the state space parameters and compute
53 + the `contextualized_states`. A, D are input independent
54 + (see Mamba paper [1] Section 3.5.2 "Interpretation of A"
55 + for why A isn't selective) ∆, B, C are input-dependent
56 + (this is a key difference between Mamba and the linear time
57 + invariant S4, and is why Mamba is called
58 + **selective** state spaces)
59 + """
60 +
61 + def __init__(self, config: MambaConfig, layer_idx):
62 + super().__init__()
63 + self.config = config
64 + self.layer_idx = layer_idx
65 + self.hidden_size = config.hidden_size
66 + self.ssm_state_size = config.state_size
67 + self.conv_kernel_size = config.conv_kernel
68 + self.intermediate_size = config.intermediate_size
69 + self.time_step_rank = int(config.time_step_rank)
70 +
71 + self.conv1d = ColumnParallelLinear(
72 + input_size=self.conv_kernel_size,
73 + output_size=self.intermediate_size,
74 + bias=config.use_conv_bias,
75 + )
76 + # unsqueeze to fit conv1d weights shape into the linear weights shape.
77 + # Can't do this in `weight_loader` since it already exists in
78 + # `ColumnParallelLinear` and `set_weight_attrs`
79 + # doesn't allow to override it
80 + self.conv1d.weight.data = self.conv1d.weight.data.unsqueeze(1)
81 +
82 + self.in_proj = MergedColumnParallelLinear(self.hidden_size,
83 + [self.intermediate_size] * 2,
84 + bias=config.use_bias)
85 + # selective projection used to make dt, B and C input dependent
86 + self.x_proj = RowParallelLinear(
87 + self.intermediate_size,
88 + self.time_step_rank + self.ssm_state_size * 2,
89 + bias=False,
90 + )
91 + # time step projection (discretization) -
92 + # In the forward we need to apply dt_proj without the bias,
93 + # as the bias is added in the selective scan kernel.
94 + self.dt_proj = ColumnParallelLinear(self.time_step_rank,
95 + self.intermediate_size,
96 + bias=True,
97 + skip_bias_add=True)
98 +
99 + tp_size = get_tensor_model_parallel_world_size()
100 + self.A = nn.Parameter(
101 + torch.empty(
102 + self.intermediate_size // tp_size,
103 + self.ssm_state_size,
104 + dtype=torch.float32,
105 + ))
106 + self.D = nn.Parameter(torch.ones(self.intermediate_size // tp_size))
107 +
108 + set_weight_attrs(self.D, {"weight_loader": sharded_weight_loader(0)})
109 + a_weight_loader = composed_weight_loader(
110 + sharded_weight_loader(0), lambda x: -torch.exp(x.float()))
111 + set_weight_attrs(self.A, {"weight_loader": a_weight_loader})
112 +
113 + self.out_proj = RowParallelLinear(
114 + self.intermediate_size,
115 + self.hidden_size,
116 + bias=config.use_bias,
117 + input_is_parallel=True,
118 + )
119 + self.activation = config.hidden_act
120 +
121 + def forward(self, hidden_states: torch.Tensor,
122 + attn_metadata: AttentionMetadata, conv_state: torch.Tensor,
123 + ssm_state: torch.Tensor):
124 +
125 + # 1. Gated MLP's linear projection
126 + projected_states = self.in_proj(hidden_states)[0].transpose(-2, -1)
127 + hidden_states, gate = projected_states.chunk(2, dim=-2)
128 +
129 + # 2. Convolution sequence transformation
130 + conv_weights = self.conv1d.weight.view(self.conv1d.weight.size(0),
131 + self.conv1d.weight.size(2))
132 +
133 + if attn_metadata.query_start_loc is not None \
134 + and attn_metadata.context_lens_tensor is not None:
135 + # |---------- N-1 iteration --------|
136 + # |---------------- N iteration ---------------------|
137 + # |- tokenA -|......................|-- newTokens ---|
138 + # |---------- context_len ----------|
139 + # |-------------------- seq_len ---------------------|
140 + # |-- query_len ---|
141 + hidden_states = causal_conv1d_fn(
142 + hidden_states,
143 + conv_weights,
144 + self.conv1d.bias,
145 + activation=self.activation,
146 + conv_states=conv_state,
147 + has_initial_state=attn_metadata.context_lens_tensor > 0,
148 + query_start_loc=attn_metadata.query_start_loc)
149 + else:
150 + hidden_states = causal_conv1d_update(
151 + hidden_states.transpose(0, 1),
152 + conv_state,
153 + conv_weights,
154 + self.conv1d.bias,
155 + self.activation,
156 + )
157 + hidden_states = hidden_states.transpose(0, 1)
158 +
159 + # 3. State Space Model sequence transformation
160 + # 3.a. input varying initialization of time_step, B and C
161 + ssm_parameters = self.x_proj(hidden_states.transpose(-2, -1))[0]
162 +
163 + time_step, B, C = torch.split(
164 + ssm_parameters,
165 + [self.time_step_rank, self.ssm_state_size, self.ssm_state_size],
166 + dim=-1,
167 + )
168 +
169 + # Note that Jamba normalizes B, C, and time_step here but Mamba doesn't.
170 +
171 + discrete_time_step = self.dt_proj(time_step)[0].transpose(-2, -1)
172 + # 3.c perform the recurrence y ← SSM(A, B, C)(x)
173 + time_proj_bias = (self.dt_proj.bias.float() if hasattr(
174 + self.dt_proj, "bias") else None)
175 +
176 + if attn_metadata.query_start_loc is not None \
177 + and attn_metadata.context_lens_tensor is not None:
178 + scan_outputs = selective_scan_fn(
179 + hidden_states,
180 + ssm_state,
181 + discrete_time_step,
182 + self.A,
183 + B.transpose(-2, -1),
184 + C.transpose(-2, -1),
185 + self.D.float(),
186 + gate,
187 + time_proj_bias,
188 + delta_softplus=True,
189 + has_initial_state=attn_metadata.context_lens_tensor > 0,
190 + query_start_loc=attn_metadata.query_start_loc)
191 + else:
192 + scan_outputs = selective_state_update(
193 + ssm_state,
194 + hidden_states.transpose(0, 1),
195 + discrete_time_step.transpose(0, 1),
196 + self.A,
197 + B,
198 + C,
199 + self.D,
200 + gate.transpose(0, 1),
201 + time_proj_bias,
202 + dt_softplus=True,
203 + )
204 + scan_outputs = scan_outputs.transpose(0, 1)
205 +
206 + # 4. Final linear projection
207 + contextualized_states = self.out_proj(scan_outputs.transpose(-2,
208 + -1))[0]
209 + return contextualized_states
210 +
211 +
212 +class MambaMLP(nn.Module):
213 +
214 + def __init__(
215 + self,
216 + config: MambaConfig,
217 + quant_config: Optional[QuantizationConfig] = None,
218 + ) -> None:
219 + super().__init__()
220 + hidden_size = config.hidden_size
221 + intermediate_size = config.intermediate_size
222 + hidden_act = config.hidden_act
223 + self.gate_up_proj = MergedColumnParallelLinear(
224 + hidden_size, [intermediate_size] * 2,
225 + bias=False,
226 + quant_config=quant_config)
227 + self.down_proj = RowParallelLinear(intermediate_size,
228 + hidden_size,
229 + bias=False,
230 + quant_config=quant_config)
231 + if hidden_act != "silu":
232 + raise ValueError(f"Unsupported activation: {hidden_act}. "
233 + "Only silu is supported for now.")
234 + self.act_fn = SiluAndMul()
235 +
236 + def forward(self, x):
237 + gate_up, _ = self.gate_up_proj(x)
238 + x = self.act_fn(gate_up)
239 + x, _ = self.down_proj(x)
240 + return x
241 +
242 +
243 +class MambaDecoderLayer(nn.Module):
244 +
245 + def __init__(self,
246 + config: MambaConfig,
247 + layer_idx: int,
248 + cache_config: Optional[CacheConfig] = None,
249 + quant_config: Optional[QuantizationConfig] = None) -> None:
250 + super().__init__()
251 + self.layer_idx = layer_idx
252 + self.config = config
253 + self.mixer = MambaMixer(config, layer_idx)
254 +
255 + self.feed_forward = MambaMLP(config, quant_config=quant_config)
256 + self.norm = RMSNorm(config.hidden_size, eps=config.layer_norm_epsilon)
257 + self.pre_ff_layernorm = RMSNorm(config.hidden_size,
258 + eps=config.layer_norm_epsilon)
259 +
260 + def forward(
261 + self,
262 + hidden_states: torch.Tensor,
263 + attn_metadata: AttentionMetadata,
264 + residual: Optional[torch.Tensor],
265 + conv_state: torch.Tensor,
266 + ssm_state: torch.Tensor,
267 + **kwargs,
268 + ):
269 + if residual is None:
270 + residual = hidden_states
271 + hidden_states = self.norm(hidden_states)
272 + else:
273 + hidden_states, residual = self.norm(hidden_states, residual)
274 +
275 + hidden_states = self.mixer(hidden_states, attn_metadata, conv_state,
276 + ssm_state)
277 + # Fully Connected
278 + hidden_states, residual = self.pre_ff_layernorm(
279 + hidden_states, residual)
280 + hidden_states = self.feed_forward(hidden_states)
281 + return hidden_states, residual
282 +
283 +
284 +class MambaModel(nn.Module):
285 +
286 + def __init__(
287 + self,
288 + config: MambaConfig,
289 + quant_config: Optional[QuantizationConfig] = None,
290 + cache_config: Optional[CacheConfig] = None,
291 + lora_config: Optional[LoRAConfig] = None,
292 + ) -> None:
293 + super().__init__()
294 + self.config = config
295 + self.padding_idx = config.pad_token_id
296 + lora_vocab = ((lora_config.lora_extra_vocab_size *
297 + (lora_config.max_loras or 1)) if lora_config else 0)
298 + self.vocab_size = config.vocab_size + lora_vocab
299 + self.org_vocab_size = config.vocab_size
300 +
301 + self.embeddings = VocabParallelEmbedding(
302 + self.vocab_size,
303 + config.hidden_size,
304 + org_num_embeddings=config.vocab_size,
305 + )
306 +
307 + decoder_layers = []
308 + for i in range(config.num_hidden_layers):
309 + decoder_layers.append(
310 + MambaDecoderLayer(config,
311 + layer_idx=i,
312 + cache_config=cache_config,
313 + quant_config=quant_config))
314 + self.layers = nn.ModuleList(decoder_layers)
315 + self.norm_f = RMSNorm(config.hidden_size,
316 + eps=config.layer_norm_epsilon)
317 +
318 + def forward(
319 + self,
320 + input_ids: torch.Tensor,
321 + positions: torch.Tensor,
322 + kv_caches: List[torch.Tensor],
323 + attn_metadata: AttentionMetadata,
324 + conv_state: torch.Tensor,
325 + ssm_state: torch.Tensor,
326 + ) -> torch.Tensor:
327 + hidden_states = self.embeddings(input_ids)
328 + residual = None
329 +
330 + for i in range(len(self.layers)):
331 + layer = self.layers[i]
332 + current_ssm_state = ssm_state[i]
333 + current_conv_state = conv_state[i]
334 +
335 + hidden_states, residual = layer(
336 + positions=positions,
337 + hidden_states=hidden_states,
338 + attn_metadata=attn_metadata,
339 + residual=residual,
340 + conv_state=current_conv_state,
341 + ssm_state=current_ssm_state,
342 + )
343 + hidden_states, _ = self.norm_f(hidden_states, residual)
344 +
345 + return hidden_states
346 +
347 +
348 +class MambaForCausalLM(nn.Module, HasInnerState, IsAttentionFree):
349 + packed_modules_mapping = {
350 + "qkv_proj": [
351 + "q_proj",
352 + "k_proj",
353 + "v_proj",
354 + ],
355 + }
356 +
357 + # LoRA specific attributes
358 + supported_lora_modules = [
359 + "qkv_proj",
360 + "o_proj",
361 + "embed_tokens",
362 + "lm_head",
363 + ]
364 + embedding_modules = {
365 + "embeddings": "input_embeddings",
366 + "lm_head": "output_embeddings",
367 + }
368 + embedding_padding_modules = ["lm_head"]
369 +
370 + def __init__(
371 + self,
372 + config: MambaConfig,
373 + cache_config: Optional[CacheConfig] = None,
374 + quant_config: Optional[QuantizationConfig] = None,
375 + lora_config: Optional[LoRAConfig] = None,
376 + scheduler_config: Optional[SchedulerConfig] = None,
377 + ) -> None:
378 + assert not cache_config.enable_prefix_caching, \
379 + "Mamba does not support prefix caching"
380 +
381 + super().__init__()
382 + self.config = config
383 + self.scheduler_config = scheduler_config
384 + self.backbone = MambaModel(config,
385 + cache_config=cache_config,
386 + quant_config=quant_config,
387 + lora_config=lora_config)
388 + self.unpadded_vocab_size = config.vocab_size
389 + if lora_config:
390 + self.unpadded_vocab_size += lora_config.lora_extra_vocab_size
391 +
392 + self.lm_head = self.backbone.embeddings
393 +
394 + # Used to track and store by the Mamba cache between steps.
395 + self.mamba_cache: Optional[MambaCacheManager] = None
396 +
397 + self.logits_processor = LogitsProcessor(self.unpadded_vocab_size,
398 + config.vocab_size)
399 + self.sampler = Sampler()
400 +
401 + def forward(self,
402 + input_ids: torch.Tensor,
403 + positions: torch.Tensor,
404 + kv_caches: List[KVCache],
405 + attn_metadata: AttentionMetadata,
406 + intermediate_tensors: Optional[IntermediateTensors] = None,
407 + **kwargs):
408 + if self.mamba_cache is None:
409 + max_batch_size = (_get_graph_batch_size(
410 + self.scheduler_config.max_num_seqs) if self.scheduler_config
411 + else max(_BATCH_SIZES_TO_CAPTURE) + 2)
412 + self.mamba_cache = MambaCacheManager(
413 + self.lm_head.weight.dtype, self.config.num_hidden_layers,
414 + max_batch_size, *self._get_mamba_cache_shape())
415 +
416 + mamba_cache_tensors = self.mamba_cache.current_run_tensors(
417 + input_ids, attn_metadata, **kwargs)
418 +
419 + hidden_states = self.backbone(input_ids, positions, kv_caches,
420 + attn_metadata, mamba_cache_tensors[0],
421 + mamba_cache_tensors[1])
422 +
423 + return hidden_states
424 +
425 + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
426 + return self.mamba_cache.copy_inputs_before_cuda_graphs(
427 + input_buffers, **kwargs)
428 +
429 + def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
430 + return self.mamba_cache.get_seqlen_agnostic_capture_inputs(batch_size)
431 +
432 + def _get_mamba_cache_shape(
433 + self) -> Tuple[Tuple[int, int], Tuple[int, int]]:
434 + world_size = get_tensor_model_parallel_world_size()
435 + conv_state_shape = (
436 + self.config.intermediate_size // world_size,
437 + self.config.conv_kernel - 1,
438 + )
439 + temporal_state_shape = (
440 + self.config.intermediate_size // world_size,
441 + self.config.state_size,
442 + )
443 + return conv_state_shape, temporal_state_shape
444 +
445 + def compute_logits(self, hidden_states: torch.Tensor,
446 + sampling_metadata: SamplingMetadata) -> torch.Tensor:
447 + logits = self.logits_processor(self.lm_head, hidden_states,
448 + sampling_metadata)
449 + return logits
450 +
451 + def sample(
452 + self,
453 + logits: Optional[torch.Tensor],
454 + sampling_metadata: SamplingMetadata,
455 + ) -> Optional[SamplerOutput]:
456 + next_tokens = self.sampler(logits, sampling_metadata)
457 + return next_tokens
458 +
459 + def load_weights(self, weights: Iterable[Tuple[str, torch.Tensor]]):
460 + stacked_params_mapping = [
461 + # (param_name, shard_name, shard_id)
462 + ("qkv_proj", "q_proj", "q"),
463 + ("qkv_proj", "k_proj", "k"),
464 + ("qkv_proj", "v_proj", "v"),
465 + ("gate_up_proj", "gate_proj", 0),
466 + ("gate_up_proj", "up_proj", 1),
467 + ]
468 +
469 + params_dict = dict(self.named_parameters())
470 + for name, loaded_weight in weights:
471 + if "rotary_emb.inv_freq" in name:
472 + continue
473 +
474 + if "A_log" in name:
475 + name = name.replace("A_log", "A")
476 +
477 + if ".self_attn." in name:
478 + name = name.replace(".self_attn", "")
479 +
480 + for param_name, weight_name, shard_id in stacked_params_mapping:
481 + if weight_name not in name:
482 + continue
483 + name = name.replace(weight_name, param_name)
484 + # Skip loading extra bias for GPTQ models.
485 + if name.endswith(".bias") and name not in params_dict:
486 + continue
487 + param = params_dict[name]
488 + weight_loader = param.weight_loader
489 + weight_loader(param, loaded_weight, shard_id)
490 + break
491 + else:
492 + # Skip loading extra bias for GPTQ models.
493 + if name.endswith(".bias") and name not in params_dict:
494 + continue
495 +
496 + param = params_dict[name]
497 + weight_loader = getattr(param, "weight_loader",
498 + default_weight_loader)
499 + weight_loader(param, loaded_weight)
vllm/model_executor/models/mamba_cache.py new
+222
@@ -0,0 +1,222 @@
1 +from typing import Dict, List, Optional
2 +
3 +import torch
4 +
5 +from vllm.attention.backends.abstract import AttentionMetadata
6 +
7 +
8 +class MambaCacheManager:
9 +
10 + def __init__(self, dtype, num_mamba_layers, max_batch_size,
11 + conv_state_shape, temporal_state_shape):
12 +
13 + conv_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
14 + conv_state_shape,
15 + dtype=dtype,
16 + device="cuda")
17 + temporal_state = torch.empty(size=(num_mamba_layers, max_batch_size) +
18 + temporal_state_shape,
19 + dtype=dtype,
20 + device="cuda")
21 +
22 + self.mamba_cache = (conv_state, temporal_state)
23 +
24 + # Maps between the request id and a dict that maps between the seq_id
25 + # and its index inside the self.mamba_cache
26 + self.mamba_cache_indices_mapping: Dict[str, Dict[int, int]] = {}
27 +
28 + def current_run_tensors(self, input_ids: torch.Tensor,
29 + attn_metadata: AttentionMetadata, **kwargs):
30 + """
31 + Return the tensors for the current run's conv and ssm state.
32 + """
33 + if "seqlen_agnostic_capture_inputs" not in kwargs:
34 + # We get here only on Prefill/Eager mode runs
35 + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
36 + finished_requests_ids = kwargs["finished_requests_ids"]
37 +
38 + self._release_finished_requests(finished_requests_ids)
39 + mamba_cache_tensors = self._prepare_current_run_mamba_cache(
40 + request_ids_to_seq_ids, finished_requests_ids)
41 +
42 + else:
43 + # CUDA graph capturing runs
44 + mamba_cache_tensors = kwargs["seqlen_agnostic_capture_inputs"]
45 +
46 + return mamba_cache_tensors
47 +
48 + def copy_inputs_before_cuda_graphs(self, input_buffers, **kwargs):
49 + """
50 + Copy the relevant Mamba cache into the CUDA graph input buffer
51 + that was provided during the capture runs
52 + (JambaForCausalLM.mamba_gc_cache_buffer).
53 + """
54 + assert all(
55 + key in kwargs
56 + for key in ["request_ids_to_seq_ids", "finished_requests_ids"])
57 + finished_requests_ids = kwargs["finished_requests_ids"]
58 + request_ids_to_seq_ids = kwargs["request_ids_to_seq_ids"]
59 +
60 + self._release_finished_requests(finished_requests_ids)
61 + self._prepare_current_run_mamba_cache(request_ids_to_seq_ids,
62 + finished_requests_ids)
63 +
64 + def get_seqlen_agnostic_capture_inputs(self, batch_size: int):
65 + """
66 + Provide the CUDA graph capture runs with a buffer in adjusted size.
67 + The buffer is used to maintain the Mamba Cache during the CUDA graph
68 + replay runs.
69 + """
70 + return tuple(buffer[:, :batch_size] for buffer in self.mamba_cache)
71 +
72 + def _swap_mamba_cache(self, from_index: int, to_index: int):
73 + assert len(self.mamba_cache) > 0
74 + for cache_t in self.mamba_cache:
75 + cache_t[:, [to_index,from_index]] = \
76 + cache_t[:, [from_index,to_index]]
77 +
78 + def _copy_mamba_cache(self, from_index: int, to_index: int):
79 + assert len(self.mamba_cache) > 0
80 + for cache_t in self.mamba_cache:
81 + cache_t[:, to_index].copy_(cache_t[:, from_index],
82 + non_blocking=True)
83 +
84 + def _move_out_if_already_occupied(self, index: int,
85 + all_occupied_indices: List[int]):
86 + if index in all_occupied_indices:
87 + first_free_index = self._first_free_index_in_mamba_cache()
88 + # In case occupied, move the occupied to a new empty block
89 + self._move_cache_index_and_mappings(from_index=index,
90 + to_index=first_free_index)
91 +
92 + def _assign_seq_id_to_mamba_cache_in_specific_dest(self, cur_rid: str,
93 + seq_id: int,
94 + destination_index: int):
95 + """
96 + Assign (req_id,seq_id) pair to a `destination_index` index, if
97 + already occupied, move the occupying index to a free index.
98 + """
99 + all_occupied_indices = self._get_all_occupied_indices()
100 + if cur_rid not in self.mamba_cache_indices_mapping:
101 + self._move_out_if_already_occupied(
102 + index=destination_index,
103 + all_occupied_indices=all_occupied_indices)
104 + self.mamba_cache_indices_mapping[cur_rid] = {
105 + seq_id: destination_index
106 + }
107 + elif seq_id not in (seq_ids2indices :=
108 + self.mamba_cache_indices_mapping[cur_rid]):
109 + # parallel sampling , where n > 1, assume prefill have
110 + # already happened now we only need to copy the already
111 + # existing cache into the siblings seq_ids caches
112 + self._move_out_if_already_occupied(
113 + index=destination_index,
114 + all_occupied_indices=all_occupied_indices)
115 + index_exists = list(seq_ids2indices.values())[0]
116 + # case of decoding n>1, copy prefill cache to decoding indices
117 + self._copy_mamba_cache(from_index=index_exists,
118 + to_index=destination_index)
119 + self.mamba_cache_indices_mapping[cur_rid][
120 + seq_id] = destination_index
121 + else:
122 + # already exists
123 + cache_index_already_exists = self.mamba_cache_indices_mapping[
124 + cur_rid][seq_id]
125 + if cache_index_already_exists != destination_index:
126 + # In case the seq id already exists but not in
127 + # the right destination, swap it with what's occupying it
128 + self._swap_pair_indices_and_mappings(
129 + from_index=cache_index_already_exists,
130 + to_index=destination_index)
131 +
132 + def _prepare_current_run_mamba_cache(
133 + self, request_ids_to_seq_ids: Dict[str, list[int]],
134 + finished_requests_ids: List[str]):
135 + running_indices = []
136 + request_ids_to_seq_ids_flatten = [
137 + (req_id, seq_id)
138 + for req_id, seq_ids in request_ids_to_seq_ids.items()
139 + for seq_id in seq_ids
140 + ]
141 + batch_size = len(request_ids_to_seq_ids_flatten)
142 + for dest_index, (request_id,
143 + seq_id) in enumerate(request_ids_to_seq_ids_flatten):
144 + if request_id in finished_requests_ids:
145 + # Do not allocate cache index for requests that run
146 + # and finish right after
147 + continue
148 + self._assign_seq_id_to_mamba_cache_in_specific_dest(
149 + request_id, seq_id, dest_index)
150 + running_indices.append(dest_index)
151 +
152 + self._clean_up_first_bs_blocks(batch_size, running_indices)
153 + conv_state = self.mamba_cache[0][:, :batch_size]
154 + temporal_state = self.mamba_cache[1][:, :batch_size]
155 +
156 + return (conv_state, temporal_state)
157 +
158 + def _get_all_occupied_indices(self):
159 + return [
160 + cache_idx
161 + for seq_ids2indices in self.mamba_cache_indices_mapping.values()
162 + for cache_idx in seq_ids2indices.values()
163 + ]
164 +
165 + def _clean_up_first_bs_blocks(self, batch_size: int,
166 + indices_for_current_run: List[int]):
167 + # move out all of the occupied but currently not running blocks
168 + # outside of the first n blocks
169 + destination_indices = range(batch_size)
170 + max_possible_batch_size = self.mamba_cache[0].shape[1]
171 + for destination_index in destination_indices:
172 + if destination_index in self._get_all_occupied_indices() and \
173 + destination_index not in indices_for_current_run:
174 + # move not running indices outside of the batch
175 + all_other_indices = list(
176 + range(batch_size, max_possible_batch_size))
177 + first_avail_index = self._first_free_index_in_mamba_cache(
178 + all_other_indices)
179 + self._swap_indices(from_index=destination_index,
180 + to_index=first_avail_index)
181 +
182 + def _move_cache_index_and_mappings(self, from_index: int, to_index: int):
183 + self._copy_mamba_cache(from_index=from_index, to_index=to_index)
184 + self._update_mapping_index(from_index=from_index, to_index=to_index)
185 +
186 + def _swap_pair_indices_and_mappings(self, from_index: int, to_index: int):
187 + self._swap_mamba_cache(from_index=from_index, to_index=to_index)
188 + self._swap_mapping_index(from_index=from_index, to_index=to_index)
189 +
190 + def _swap_mapping_index(self, from_index: int, to_index: int):
191 + for seq_ids2index in self.mamba_cache_indices_mapping.values():
192 + for seq_id, index in seq_ids2index.items():
193 + if from_index == index:
194 + seq_ids2index.update({seq_id: to_index})
195 + elif to_index == index:
196 + seq_ids2index.update({seq_id: from_index})
197 +
198 + def _update_mapping_index(self, from_index: int, to_index: int):
199 + for seq_ids2index in self.mamba_cache_indices_mapping.values():
200 + for seq_id, index in seq_ids2index.items():
201 + if from_index == index:
202 + seq_ids2index.update({seq_id: to_index})
203 + return
204 +
205 + def _release_finished_requests(self,
206 + finished_seq_groups_req_ids: List[str]):
207 + for req_id in finished_seq_groups_req_ids:
208 + if req_id in self.mamba_cache_indices_mapping:
209 + self.mamba_cache_indices_mapping.pop(req_id)
210 +
211 + def _first_free_index_in_mamba_cache(
212 + self, indices_range: Optional[List[int]] = None) -> int:
213 + assert self.mamba_cache is not None
214 + if indices_range is None:
215 + max_possible_batch_size = self.mamba_cache[0].shape[1]
216 + indices_range = list(range(max_possible_batch_size))
217 + all_occupied_indices = self._get_all_occupied_indices()
218 + for i in indices_range:
219 + if i not in all_occupied_indices:
220 + return i
221 + raise Exception("Couldn't find a free spot in the mamba cache! This"
222 + "should never happen")
vllm/model_executor/models/registry.py
+15 -1
@@ -14,7 +14,8 @@ import torch.nn as nn
14 from vllm.logger import init_logger
15 from vllm.utils import is_hip
16
17 -from .interfaces import supports_multimodal, supports_pp
17 +from .interfaces import (has_inner_state, is_attention_free,
18 + supports_multimodal, supports_pp)
19 from .interfaces_base import is_embedding_model, is_text_generation_model
20
21 logger = init_logger(__name__)
@@ -52,6 +53,7 @@ _TEXT_GENERATION_MODELS = {
53 "LlamaForCausalLM": ("llama", "LlamaForCausalLM"),
54 # For decapoda-research/llama-*
55 "LLaMAForCausalLM": ("llama", "LlamaForCausalLM"),
56 + "MambaForCausalLM": ("mamba", "MambaForCausalLM"),
57 "MistralForCausalLM": ("llama", "LlamaForCausalLM"),
58 "MixtralForCausalLM": ("mixtral", "MixtralForCausalLM"),
59 "QuantMixtralForCausalLM": ("mixtral_quant", "MixtralForCausalLM"),
@@ -157,6 +159,8 @@ class _ModelInfo:
159 is_embedding_model: bool
160 supports_multimodal: bool
161 supports_pp: bool
162 + has_inner_state: bool
163 + is_attention_free: bool
164
165 @staticmethod
166 def from_model_cls(model: Type[nn.Module]) -> "_ModelInfo":
@@ -165,6 +169,8 @@ class _ModelInfo:
169 is_embedding_model=is_embedding_model(model),
170 supports_multimodal=supports_multimodal(model),
171 supports_pp=supports_pp(model),
172 + has_inner_state=has_inner_state(model),
173 + is_attention_free=is_attention_free(model),
174 )
175
176
@@ -380,6 +386,14 @@ class _ModelRegistry:
386 ) -> bool:
387 return self.inspect_model_cls(architectures).supports_pp
388
389 + def model_has_inner_state(self, architectures: Union[str,
390 + List[str]]) -> bool:
391 + return self.inspect_model_cls(architectures).has_inner_state
392 +
393 + def is_attention_free_model(self, architectures: Union[str,
394 + List[str]]) -> bool:
395 + return self.inspect_model_cls(architectures).is_attention_free
396 +
397
398 ModelRegistry = _ModelRegistry({
399 model_arch: _LazyRegisteredModel(
vllm/worker/cache_engine.py
+6 -9
@@ -52,15 +52,12 @@ class CacheEngine:
52 self.dtype = STR_DTYPE_TO_TORCH_DTYPE[cache_config.cache_dtype]
53
54 # Get attention backend.
55 - self.attn_backend = get_attn_backend(
56 - model_config.get_num_attention_heads(parallel_config),
57 - self.head_size,
58 - self.num_kv_heads,
59 - model_config.get_sliding_window(),
60 - model_config.dtype,
61 - cache_config.cache_dtype,
62 - self.block_size,
63 - )
55 + self.attn_backend = get_attn_backend(self.head_size,
56 + model_config.get_sliding_window(),
57 + model_config.dtype,
58 + cache_config.cache_dtype,
59 + self.block_size,
60 + model_config.is_attention_free)
61
62 # Initialize the cache.
63 self.gpu_cache = self._allocate_kv_cache(
vllm/worker/cpu_model_runner.py
+1 -2
@@ -418,13 +418,12 @@ class CPUModelRunner(ModelRunnerBase[ModelInputForCPU]):
418 self.sliding_window = model_config.get_sliding_window()
419 self.block_size = cache_config.block_size
420 self.attn_backend = get_attn_backend(
421 - self.model_config.get_num_attention_heads(self.parallel_config),
421 self.model_config.get_head_size(),
423 - self.model_config.get_num_kv_heads(self.parallel_config),
422 self.model_config.get_sliding_window(),
423 self.model_config.dtype,
424 self.kv_cache_dtype,
425 self.block_size,
426 + self.model_config.is_attention_free,
427 )
428
429 # Multi-modal data support
vllm/worker/cpu_worker.py
+1 -2
@@ -56,13 +56,12 @@ class CPUCacheEngine:
56
57 # Get attention backend.
58 self.attn_backend = get_attn_backend(
59 - self.model_config.get_num_attention_heads(self.parallel_config),
59 self.model_config.get_head_size(),
61 - self.model_config.get_num_kv_heads(self.parallel_config),
60 self.model_config.get_sliding_window(),
61 self.model_config.dtype,
62 cache_config.cache_dtype,
63 self.block_size,
64 + self.model_config.is_attention_free,
65 )
66
67 # Initialize the cache.
vllm/worker/enc_dec_model_runner.py
+1 -1
@@ -196,7 +196,7 @@ class EncoderDecoderModelRunner(GPUModelRunnerBase[EncoderDecoderModelInput]):
196 seqlen_agnostic_kwargs = {
197 "finished_requests_ids": model_input.finished_requests_ids,
198 "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
199 - } if self.has_seqlen_agnostic else {}
199 + } if self.has_inner_state else {}
200
201 multi_modal_kwargs = model_input.multi_modal_kwargs or {}
202 with set_forward_context(model_input.attn_metadata):
vllm/worker/model_runner.py
+13 -17
@@ -17,7 +17,6 @@ import torch.nn as nn
17 import vllm.envs as envs
18 from vllm.attention import AttentionMetadata, get_attn_backend
19 from vllm.attention.backends.abstract import AttentionState
20 -from vllm.attention.backends.utils import CommonAttentionState
20 from vllm.compilation.compile_context import set_compile_context
21 from vllm.compilation.levels import CompilationLevel
22 from vllm.config import (CacheConfig, DeviceConfig, LoadConfig, LoRAConfig,
@@ -991,8 +990,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
990 self.graph_memory_pool: Optional[Tuple[
991 int, int]] = None # Set during graph capture.
992
994 - self.has_seqlen_agnostic = model_config.contains_seqlen_agnostic_layers(
995 - parallel_config)
993 + self.has_inner_state = model_config.has_inner_state
994
995 # When using CUDA graph, the input block tables must be padded to
996 # max_seq_len_to_capture. However, creating the block table in
@@ -1003,22 +1001,16 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1001 self.graph_block_tables = np.zeros(
1002 (self.max_batchsize_to_capture, self.get_max_block_per_batch()),
1003 dtype=np.int32)
1006 - num_attn_heads = self.model_config.get_num_attention_heads(
1007 - self.parallel_config)
1004 self.attn_backend = get_attn_backend(
1009 - num_attn_heads,
1005 self.model_config.get_head_size(),
1011 - self.model_config.get_num_kv_heads(self.parallel_config),
1006 self.model_config.get_sliding_window(),
1007 self.model_config.dtype,
1008 self.kv_cache_dtype,
1009 self.block_size,
1016 - ) if num_attn_heads else None
1017 - if self.attn_backend:
1018 - self.attn_state = self.attn_backend.get_state_cls()(
1019 - weakref.proxy(self))
1020 - else:
1021 - self.attn_state = CommonAttentionState(weakref.proxy(self))
1010 + self.model_config.is_attention_free,
1011 + )
1012 + self.attn_state = self.attn_backend.get_state_cls()(
1013 + weakref.proxy(self))
1014
1015 # Multi-modal data support
1016 self.input_registry = input_registry
@@ -1498,7 +1490,7 @@ class GPUModelRunnerBase(ModelRunnerBase[TModelInputForGPU]):
1490 "previous_hidden_states"] = previous_hidden_states[:
1491 batch_size]
1492
1501 - if self.has_seqlen_agnostic:
1493 + if self.has_inner_state:
1494 # Only used by Mamba-based models CUDA graph atm (Jamba)
1495 capture_inputs.update({
1496 "seqlen_agnostic_capture_inputs":
@@ -1647,7 +1639,7 @@ class ModelRunner(GPUModelRunnerBase[ModelInputForGPUWithSamplingMetadata]):
1639 seqlen_agnostic_kwargs = {
1640 "finished_requests_ids": model_input.finished_requests_ids,
1641 "request_ids_to_seq_ids": model_input.request_ids_to_seq_ids,
1650 - } if self.has_seqlen_agnostic else {}
1642 + } if self.has_inner_state else {}
1643 if (self.observability_config is not None
1644 and self.observability_config.collect_model_forward_time):
1645 model_forward_start = torch.cuda.Event(enable_timing=True)
@@ -1852,10 +1844,14 @@ class CUDAGraphRunner:
1844 # Copy the input tensors to the input buffers.
1845 self.input_buffers["input_ids"].copy_(input_ids, non_blocking=True)
1846 self.input_buffers["positions"].copy_(positions, non_blocking=True)
1855 - self.input_buffers["slot_mapping"].copy_(attn_metadata.slot_mapping,
1856 - non_blocking=True)
1847 +
1848 + if self.backend_name != "placeholder-attn":
1849 + self.input_buffers["slot_mapping"].copy_(
1850 + attn_metadata.slot_mapping, non_blocking=True)
1851 +
1852 self.attn_state.prepare_graph_input_buffers(
1853 self.input_buffers, attn_metadata, self._is_encoder_decoder_model)
1854 +
1855 if "seqlen_agnostic_capture_inputs" in self.input_buffers:
1856 self.model.copy_inputs_before_cuda_graphs(self.input_buffers,
1857 **kwargs)
vllm/worker/openvino_model_runner.py
+1 -2
@@ -74,13 +74,12 @@ class OpenVINOModelRunner:
74 self.block_size = cache_config.block_size
75
76 self.attn_backend = get_attn_backend(
77 - self.model_config.get_num_attention_heads(self.parallel_config),
77 self.model_config.get_head_size(),
79 - self.model_config.get_num_kv_heads(self.parallel_config),
78 self.model_config.get_sliding_window(),
79 self.model_config.dtype,
80 self.kv_cache_dtype,
81 self.block_size,
82 + self.model_config.is_attention_free,
83 )
84
85 # Multi-modal data support
vllm/worker/openvino_worker.py
+1 -2
@@ -70,13 +70,12 @@ class OpenVINOCacheEngine:
70
71 # Get attention backend.
72 self.attn_backend = get_attn_backend(
73 - self.model_config.get_num_attention_heads(self.parallel_config),
73 self.head_size,
75 - self.model_config.get_num_kv_heads(self.parallel_config),
74 self.model_config.get_sliding_window(),
75 self.model_config.dtype,
76 self.cache_config.cache_dtype,
77 self.block_size,
78 + self.model_config.is_attention_free,
79 )
80
81 # Initialize the cache.
vllm/worker/tpu_model_runner.py
+1 -2
@@ -113,13 +113,12 @@ class TPUModelRunner(ModelRunnerBase[ModelInputForTPU]):
113 (self.scheduler_config.max_num_seqs, self.max_num_blocks_per_seq),
114 dtype=np.int32)
115 self.attn_backend = get_attn_backend(
116 - self.model_config.get_num_attention_heads(self.parallel_config),
116 self.model_config.get_head_size(),
118 - self.model_config.get_num_kv_heads(self.parallel_config),
117 self.model_config.get_sliding_window(),
118 self.model_config.dtype,
119 self.cache_config.cache_dtype,
120 self.block_size,
121 + self.model_config.is_attention_free,
122 False,
123 )
124 self.cached_step_outputs: List[torch.Tensor] = []
vllm/worker/worker.py
+17 -8
@@ -236,11 +236,15 @@ class Worker(LocalOrDistributedWorkerBase):
236 "not properly cleaned up before initializing the vLLM instance.")
237
238 cache_block_size = self.get_cache_block_size_bytes()
239 - num_gpu_blocks = int(
240 - (total_gpu_memory * self.cache_config.gpu_memory_utilization -
241 - peak_memory) // cache_block_size)
242 - num_cpu_blocks = int(self.cache_config.swap_space_bytes //
243 - cache_block_size)
239 + if cache_block_size == 0:
240 + num_gpu_blocks = 0
241 + num_cpu_blocks = 0
242 + else:
243 + num_gpu_blocks = int(
244 + (total_gpu_memory * self.cache_config.gpu_memory_utilization -
245 + peak_memory) // cache_block_size)
246 + num_cpu_blocks = int(self.cache_config.swap_space_bytes //
247 + cache_block_size)
248 num_gpu_blocks = max(num_gpu_blocks, 0)
249 num_cpu_blocks = max(num_cpu_blocks, 0)
250 if self.model_runner.lora_manager:
@@ -257,6 +261,7 @@ class Worker(LocalOrDistributedWorkerBase):
261 """
262 raise_if_cache_size_invalid(num_gpu_blocks,
263 self.cache_config.block_size,
264 + self.cache_config.is_attention_free,
265 self.model_config.max_model_len)
266
267 self.cache_config.num_gpu_blocks = num_gpu_blocks
@@ -472,14 +477,18 @@ def _check_if_gpu_supports_dtype(torch_dtype: torch.dtype):
477 "`dtype` flag in CLI, for example: --dtype=half.")
478
479
475 -def raise_if_cache_size_invalid(num_gpu_blocks, block_size,
480 +def raise_if_cache_size_invalid(num_gpu_blocks, block_size, is_attention_free,
481 max_model_len) -> None:
477 - if num_gpu_blocks <= 0:
482 + if is_attention_free and num_gpu_blocks != 0:
483 + raise ValueError("No memory should be allocated for the cache blocks "
484 + f"for an attention-free model, but {num_gpu_blocks}"
485 + "blocks are allocated.")
486 + if not is_attention_free and num_gpu_blocks <= 0:
487 raise ValueError("No available memory for the cache blocks. "
488 "Try increasing `gpu_memory_utilization` when "
489 "initializing the engine.")
490 max_seq_len = block_size * num_gpu_blocks
482 - if max_model_len > max_seq_len:
491 + if not is_attention_free and max_model_len > max_seq_len:
492 raise ValueError(
493 f"The model's max seq len ({max_model_len}) "
494 "is larger than the maximum number of tokens that can be "
vllm/worker/xpu_model_runner.py
+1 -2
@@ -372,13 +372,12 @@ class XPUModelRunner(ModelRunnerBase[ModelInputForXPUWithSamplingMetadata]):
372 self.block_size = cache_config.block_size
373
374 self.attn_backend = get_attn_backend(
375 - self.model_config.get_num_attention_heads(self.parallel_config),
375 self.model_config.get_head_size(),
377 - self.model_config.get_num_kv_heads(self.parallel_config),
376 self.model_config.get_sliding_window(),
377 self.model_config.dtype,
378 self.kv_cache_dtype,
379 self.block_size,
380 + self.model_config.is_attention_free,
381 )
382
383 # Multi-modal data support