@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / e53dfd3eaf
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Fix Flashinfer Correctness (#7284)

Lily Liu committed Aug 7, 2024 at 16:26 UTC e53dfd3eafd9bd5cc217c3796c7d1911a88ec893
1 file changed +7 -3
vllm/attention/backends/flashinfer.py
+7 -3
@@ -127,6 +127,7 @@ class FlashInferMetadata(AttentionMetadata):
127 raise ValueError(
128 f"Only {supported_head_sizes} are supported for head_dim,",
129 f"received {self.head_dim}.")
130 + self.is_profile_run = is_block_tables_empty(self.block_tables)
131
132 def begin_forward(self):
133 if self.num_prefill_tokens > 0:
@@ -140,11 +141,14 @@ class FlashInferMetadata(AttentionMetadata):
141 assert self.paged_kv_last_page_len is not None
142 batch_size = self.query_start_loc.shape[0] - 1
143 assert batch_size >= 0
143 - # The prefill stage does not read kv cache.
144 + # The profile run does not read kv cache.
145 # Both paged_kv_indices and paged_kv_last_page_len are empty.
146 # paged_kv_indptr is a zero tensor with size batch_size + 1.
146 - self.paged_kv_indptr = torch.zeros(batch_size + 1,
147 - device=self.device)
147 + if self.is_profile_run:
148 + self.paged_kv_indptr = torch.zeros(batch_size + 1,
149 + device=self.device)
150 + else:
151 + self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
152 self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
153 self.device)
154 self.paged_kv_indices = self.paged_kv_indices.to(self.device)