@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 9a93973708
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 04:30 UTC

[Bugfix] Fix Mamba multistep (#11071)

Signed-off-by: Tyler Michael Smith <tyler@neuralmagic.com>

Tyler Michael Smith committed Dec 10, 2024 at 19:16 UTC 9a93973708d7f52f1d1439f8f32b8c1514d18b86
2 files changed +66 -2
vllm/attention/backends/placeholder_attn.py
+63 -1
@@ -11,7 +11,8 @@ from vllm.attention.backends.utils import CommonAttentionState
11 from vllm.multimodal import MultiModalPlaceholderMap
12
13 if TYPE_CHECKING:
14 - from vllm.worker.model_runner import ModelInputForGPUBuilder
14 + from vllm.worker.model_runner import (ModelInputForGPUBuilder,
15 + ModelInputForGPUWithSamplingMetadata)
16
17 # Placeholder attention backend for models like Mamba and embedding models that
18 # lack attention.
@@ -186,6 +187,67 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
187 )
188 return self._cached_decode_metadata
189
190 + def advance_step(self,
191 + model_input: "ModelInputForGPUWithSamplingMetadata",
192 + sampled_token_ids: Optional[torch.Tensor],
193 + block_size: int,
194 + num_seqs: int,
195 + num_queries: int,
196 + turn_prefills_into_decodes: bool = False):
197 + """
198 + Update metadata in-place to advance one decode step.
199 + """
200 + # When using cudagraph, the num_seqs is padded to the next captured
201 + # batch sized, but num_queries tracks the actual number of requests in
202 + # the batch. For --enforce-eager mode, num_seqs == num_queries
203 + if num_seqs != num_queries:
204 + assert num_seqs > num_queries
205 + assert self.use_cuda_graph
206 +
207 + assert not turn_prefills_into_decodes, \
208 + ("Multi-Step + Chunked-Prefill is not supported for attention-free"
209 + "models. turn_prefills_into_decodes is a "
210 + "Multi-Step + Chunked-Prefill specific parameter.")
211 +
212 + assert self.seq_lens is not None
213 + assert self.max_decode_seq_len == max(self.seq_lens)
214 +
215 + assert self.num_prefills == 0
216 + assert self.num_prefill_tokens == 0
217 + assert self.num_decode_tokens == num_seqs
218 +
219 + assert self.seq_lens is not None
220 + assert len(self.seq_lens) == num_seqs
221 + assert self.seq_lens_tensor is not None
222 + assert self.seq_lens_tensor.shape == (num_seqs, )
223 + assert self.max_query_len == 1
224 + assert self.max_prefill_seq_len == 0
225 +
226 + assert self.query_start_loc is not None
227 + assert self.query_start_loc.shape == (num_queries + 1, )
228 + assert self.seq_start_loc is not None
229 + assert self.seq_start_loc.shape == (num_seqs + 1, )
230 +
231 + assert self.context_lens_tensor is not None
232 + assert self.context_lens_tensor.shape == (num_queries, )
233 +
234 + assert self.block_tables is not None
235 +
236 + # Update query lengths. Note that we update only queries and not seqs,
237 + # since tensors may be padded due to captured cuda graph batch size
238 + for i in range(num_queries):
239 + self.seq_lens[i] += 1
240 + self.max_decode_seq_len = max(self.seq_lens)
241 +
242 + # Update sequences, masking off entries greater than num_queries
243 + device = self.seq_lens_tensor.device
244 + mask = torch.arange(self.seq_lens_tensor.size(0),
245 + device=device) < num_queries
246 + self.seq_lens_tensor += mask.to(self.seq_lens_tensor.dtype)
247 + if sampled_token_ids is not None:
248 + model_input.input_tokens.masked_scatter_(
249 + mask, sampled_token_ids[:num_queries])
250 +
251
252 class PlaceholderAttentionMetadataBuilder(
253 AttentionMetadataBuilder[PlaceholderAttentionMetadata]):
vllm/worker/multi_step_model_runner.py
+3 -1
@@ -29,7 +29,9 @@ if TYPE_CHECKING:
29
30 logger = init_logger(__name__)
31
32 -MULTI_STEP_ATTENTION_BACKENDS = ["FLASH_ATTN", "ROCM_FLASH", "FLASHINFER"]
32 +MULTI_STEP_ATTENTION_BACKENDS = [
33 + "FLASH_ATTN", "ROCM_FLASH", "FLASHINFER", "NO_ATTENTION"
34 +]
35 MULTI_STEP_CHUNKED_PREFILL_ATTENTION_BACKENDS = ["FLASH_ATTN"]
36
37 def _get_supported_attention_backends(chunked_prefill_enabled: bool) \