@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 5faedf1b62
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 17:00 UTC

[Spec Decode] Move ops.advance_step to flash attn advance_step (#8224)

Kevin Lin committed Sep 10, 2024 at 15:18 UTC 5faedf1b6224f6e7348e9223f3e3107ec03954d3
3 files changed +23 -33
vllm/attention/backends/flash_attn.py
+15 -6
@@ -16,7 +16,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
16 from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17
18 if TYPE_CHECKING:
19 - from vllm.worker.model_runner import ModelInputForGPUBuilder
19 + from vllm.worker.model_runner import (ModelInputForGPUBuilder,
20 + ModelInputForGPUWithSamplingMetadata)
21
22 from vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
23 from vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache
@@ -302,14 +303,12 @@ class FlashAttentionMetadata(AttentionMetadata):
303 )
304 return self._cached_decode_metadata
305
305 - def advance_step(self, num_seqs: int, num_queries: int):
306 + def advance_step(self, model_input: "ModelInputForGPUWithSamplingMetadata",
307 + sampled_token_ids: Optional[torch.Tensor],
308 + block_size: int, num_seqs: int, num_queries: int):
309 """
310 Update metadata in-place to advance one decode step.
311 """
309 - # GPU in-place update is currently called separately through
310 - # custom_ops.advance_step(). See draft_model_runner. TODO(will): Move
311 - # this logic to the backend.
312 -
312 # When using cudagraph, the num_seqs is padded to the next captured
313 # batch sized, but num_queries tracks the actual number of requests in
314 # the batch. For --enforce-eager mode, num_seqs == num_queries
@@ -347,6 +346,16 @@ class FlashAttentionMetadata(AttentionMetadata):
346 self.seq_lens[i] += 1
347 self.max_decode_seq_len = max(self.seq_lens)
348
349 + ops.advance_step(num_seqs=num_seqs,
350 + num_queries=num_queries,
351 + block_size=block_size,
352 + input_tokens=model_input.input_tokens,
353 + sampled_token_ids=sampled_token_ids,
354 + input_positions=model_input.input_positions,
355 + seq_lens=self.seq_lens_tensor,
356 + slot_mapping=self.slot_mapping,
357 + block_tables=self.block_tables)
358 +
359
360 class FlashAttentionMetadataBuilder(
361 AttentionMetadataBuilder[FlashAttentionMetadata]):
vllm/spec_decode/draft_model_runner.py
+3 -13
@@ -2,7 +2,6 @@ from typing import List, Optional
2
3 import torch
4
5 -from vllm import _custom_ops as ops
5 from vllm.model_executor.layers.sampler import SamplerOutput
6
7 try:
@@ -116,18 +115,9 @@ class TP1DraftModelRunner(ModelRunner):
115 # Update attn_metadata
116 attn_metadata = model_input.attn_metadata
117 assert isinstance(attn_metadata, FlashAttentionMetadata)
119 - attn_metadata.advance_step(num_seqs, num_queries)
120 -
121 - # Update GPU tensors
122 - ops.advance_step(num_seqs=num_seqs,
123 - num_queries=num_queries,
124 - block_size=self.block_size,
125 - input_tokens=model_input.input_tokens,
126 - sampled_token_ids=sampled_token_ids,
127 - input_positions=model_input.input_positions,
128 - seq_lens=attn_metadata.seq_lens_tensor,
129 - slot_mapping=attn_metadata.slot_mapping,
130 - block_tables=attn_metadata.block_tables)
118 +
119 + attn_metadata.advance_step(model_input, sampled_token_ids,
120 + self.block_size, num_seqs, num_queries)
121
122 # Update sampling_metadata
123 sampling_metadata = model_input.sampling_metadata
vllm/worker/multi_step_model_runner.py
+5 -14
@@ -13,7 +13,6 @@ except ModuleNotFoundError:
13
14 import torch
15
16 -from vllm import _custom_ops as ops
16 from vllm.distributed import get_pp_group
17 from vllm.logger import init_logger
18 from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
@@ -499,19 +498,11 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
498
499 attn_metadata = frozen_model_input.attn_metadata
500 assert isinstance(attn_metadata, FlashAttentionMetadata)
502 - attn_metadata.advance_step(num_seqs, num_queries)
503 -
504 - # Update GPU tensors
505 - ops.advance_step(
506 - num_seqs=num_seqs,
507 - num_queries=num_queries,
508 - block_size=self.block_size,
509 - input_tokens=frozen_model_input.input_tokens,
510 - sampled_token_ids=model_input.cached_outputs[-1].sampled_token_ids,
511 - input_positions=frozen_model_input.input_positions,
512 - seq_lens=attn_metadata.seq_lens_tensor,
513 - slot_mapping=attn_metadata.slot_mapping,
514 - block_tables=attn_metadata.block_tables)
501 +
502 + attn_metadata.advance_step(
503 + frozen_model_input,
504 + model_input.cached_outputs[-1].sampled_token_ids, self.block_size,
505 + num_seqs, num_queries)
506
507 if frozen_model_input.seq_lens is not None:
508 for i in range(num_queries):