A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 17:00 UTC
[Spec Decode] Move ops.advance_step to flash attn advance_step (#8224)
Kevin Lin committed
Sep 10, 2024 at 15:18 UTC
5faedf1b6224f6e7348e9223f3e3107ec03954d3
3 files changed
+23
-33
vllm/attention/backends/flash_attn.py
+15
-6
@@ -16,7 +16,8 @@ from vllm.attention.backends.utils import (PAD_SLOT_ID, CommonAttentionState,
16
from vllm.utils import async_tensor_h2d, make_tensor_with_pad
17
18
if TYPE_CHECKING:
19
- from vllm.worker.model_runner import ModelInputForGPUBuilder
19
+ from vllm.worker.model_runner import (ModelInputForGPUBuilder,
20
+ ModelInputForGPUWithSamplingMetadata)
21
22
from vllm_flash_attn import flash_attn_varlen_func as _flash_attn_varlen_func
23
from vllm_flash_attn import flash_attn_with_kvcache as _flash_attn_with_kvcache
@@ -302,14 +303,12 @@ class FlashAttentionMetadata(AttentionMetadata):
303
)
304
return self._cached_decode_metadata
305
305
- def advance_step(self, num_seqs: int, num_queries: int):
306
+ def advance_step(self, model_input: "ModelInputForGPUWithSamplingMetadata",
307
+ sampled_token_ids: Optional[torch.Tensor],
308
+ block_size: int, num_seqs: int, num_queries: int):
309
"""
310
Update metadata in-place to advance one decode step.
311
"""
309
- # GPU in-place update is currently called separately through
310
- # custom_ops.advance_step(). See draft_model_runner. TODO(will): Move
311
- # this logic to the backend.
312
-
312
# When using cudagraph, the num_seqs is padded to the next captured
313
# batch sized, but num_queries tracks the actual number of requests in
314
# the batch. For --enforce-eager mode, num_seqs == num_queries
@@ -347,6 +346,16 @@ class FlashAttentionMetadata(AttentionMetadata):
346
self.seq_lens[i] += 1
347
self.max_decode_seq_len = max(self.seq_lens)
348
349
+ ops.advance_step(num_seqs=num_seqs,
350
+ num_queries=num_queries,
351
+ block_size=block_size,
352
+ input_tokens=model_input.input_tokens,
353
+ sampled_token_ids=sampled_token_ids,
354
+ input_positions=model_input.input_positions,
355
+ seq_lens=self.seq_lens_tensor,
356
+ slot_mapping=self.slot_mapping,
357
+ block_tables=self.block_tables)
358
+
359
360
class FlashAttentionMetadataBuilder(
361
AttentionMetadataBuilder[FlashAttentionMetadata]):
vllm/spec_decode/draft_model_runner.py
+3
-13
@@ -2,7 +2,6 @@ from typing import List, Optional
2
3
import torch
4
5
-from vllm import _custom_ops as ops
5
from vllm.model_executor.layers.sampler import SamplerOutput
6
7
try:
@@ -116,18 +115,9 @@ class TP1DraftModelRunner(ModelRunner):
115
# Update attn_metadata
116
attn_metadata = model_input.attn_metadata
117
assert isinstance(attn_metadata, FlashAttentionMetadata)
119
- attn_metadata.advance_step(num_seqs, num_queries)
120
-
121
- # Update GPU tensors
122
- ops.advance_step(num_seqs=num_seqs,
123
- num_queries=num_queries,
124
- block_size=self.block_size,
125
- input_tokens=model_input.input_tokens,
126
- sampled_token_ids=sampled_token_ids,
127
- input_positions=model_input.input_positions,
128
- seq_lens=attn_metadata.seq_lens_tensor,
129
- slot_mapping=attn_metadata.slot_mapping,
130
- block_tables=attn_metadata.block_tables)
118
+
119
+ attn_metadata.advance_step(model_input, sampled_token_ids,
120
+ self.block_size, num_seqs, num_queries)
121
122
# Update sampling_metadata
123
sampling_metadata = model_input.sampling_metadata
vllm/worker/multi_step_model_runner.py
+5
-14
@@ -13,7 +13,6 @@ except ModuleNotFoundError:
13
14
import torch
15
16
-from vllm import _custom_ops as ops
16
from vllm.distributed import get_pp_group
17
from vllm.logger import init_logger
18
from vllm.model_executor.layers.sampler import (PromptLogprobs, SampleLogprobs,
@@ -499,19 +498,11 @@ class MultiStepModelRunner(GPUModelRunnerBase[StatefulModelInput]):
498
499
attn_metadata = frozen_model_input.attn_metadata
500
assert isinstance(attn_metadata, FlashAttentionMetadata)
502
- attn_metadata.advance_step(num_seqs, num_queries)
503
-
504
- # Update GPU tensors
505
- ops.advance_step(
506
- num_seqs=num_seqs,
507
- num_queries=num_queries,
508
- block_size=self.block_size,
509
- input_tokens=frozen_model_input.input_tokens,
510
- sampled_token_ids=model_input.cached_outputs[-1].sampled_token_ids,
511
- input_positions=frozen_model_input.input_positions,
512
- seq_lens=attn_metadata.seq_lens_tensor,
513
- slot_mapping=attn_metadata.slot_mapping,
514
- block_tables=attn_metadata.block_tables)
501
+
502
+ attn_metadata.advance_step(
503
+ frozen_model_input,
504
+ model_input.cached_outputs[-1].sampled_token_ids, self.block_size,
505
+ num_seqs, num_queries)
506
507
if frozen_model_input.seq_lens is not None:
508
for i in range(num_queries):