A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Fix Flashinfer Correctness (#7284)
Lily Liu committed
Aug 7, 2024 at 16:26 UTC
e53dfd3eafd9bd5cc217c3796c7d1911a88ec893
1 file changed
+7
-3
vllm/attention/backends/flashinfer.py
+7
-3
@@ -127,6 +127,7 @@ class FlashInferMetadata(AttentionMetadata):
127
raise ValueError(
128
f"Only {supported_head_sizes} are supported for head_dim,",
129
f"received {self.head_dim}.")
130
+ self.is_profile_run = is_block_tables_empty(self.block_tables)
131
132
def begin_forward(self):
133
if self.num_prefill_tokens > 0:
@@ -140,11 +141,14 @@ class FlashInferMetadata(AttentionMetadata):
141
assert self.paged_kv_last_page_len is not None
142
batch_size = self.query_start_loc.shape[0] - 1
143
assert batch_size >= 0
143
- # The prefill stage does not read kv cache.
144
+ # The profile run does not read kv cache.
145
# Both paged_kv_indices and paged_kv_last_page_len are empty.
146
# paged_kv_indptr is a zero tensor with size batch_size + 1.
146
- self.paged_kv_indptr = torch.zeros(batch_size + 1,
147
- device=self.device)
147
+ if self.is_profile_run:
148
+ self.paged_kv_indptr = torch.zeros(batch_size + 1,
149
+ device=self.device)
150
+ else:
151
+ self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
152
self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
153
self.device)
154
self.paged_kv_indices = self.paged_kv_indices.to(self.device)