@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 954f7305a1
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 19:30 UTC

[Kernel] Fix input for flashinfer prefill wrapper. (#7008)

Lily Liu committed Aug 1, 2024 at 18:44 UTC 954f7305a106058815bd7e47f5b9d585d8764c05
1 file changed +9 -2
vllm/attention/backends/flashinfer.py
+9 -2
@@ -133,13 +133,20 @@ class FlashInferMetadata(AttentionMetadata):
133 return
134
135 assert self.prefill_wrapper is not None
136 + assert self.query_start_loc is not None
137 assert self.paged_kv_indices is not None
138 assert self.paged_kv_indptr is not None
139 assert self.paged_kv_last_page_len is not None
139 - self.paged_kv_indices = self.paged_kv_indices.to(self.device)
140 - self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
140 + batch_size = self.query_start_loc.shape[0] - 1
141 + assert batch_size >= 0
142 + # The prefill stage does not read kv cache.
143 + # Both paged_kv_indices and paged_kv_last_page_len are empty.
144 + # paged_kv_indptr is a zero tensor with size batch_size + 1.
145 + self.paged_kv_indptr = torch.zeros(batch_size + 1,
146 + device=self.device)
147 self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
148 self.device)
149 + self.paged_kv_indices = self.paged_kv_indices.to(self.device)
150 self.prefill_wrapper.end_forward()
151 self.prefill_wrapper.begin_forward(
152 self.query_start_loc, self.paged_kv_indptr,