A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 19:30 UTC
[Kernel] Fix input for flashinfer prefill wrapper. (#7008)
Lily Liu committed
Aug 1, 2024 at 18:44 UTC
954f7305a106058815bd7e47f5b9d585d8764c05
1 file changed
+9
-2
vllm/attention/backends/flashinfer.py
+9
-2
@@ -133,13 +133,20 @@ class FlashInferMetadata(AttentionMetadata):
133
return
134
135
assert self.prefill_wrapper is not None
136
+ assert self.query_start_loc is not None
137
assert self.paged_kv_indices is not None
138
assert self.paged_kv_indptr is not None
139
assert self.paged_kv_last_page_len is not None
139
- self.paged_kv_indices = self.paged_kv_indices.to(self.device)
140
- self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
140
+ batch_size = self.query_start_loc.shape[0] - 1
141
+ assert batch_size >= 0
142
+ # The prefill stage does not read kv cache.
143
+ # Both paged_kv_indices and paged_kv_last_page_len are empty.
144
+ # paged_kv_indptr is a zero tensor with size batch_size + 1.
145
+ self.paged_kv_indptr = torch.zeros(batch_size + 1,
146
+ device=self.device)
147
self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
148
self.device)
149
+ self.paged_kv_indices = self.paged_kv_indices.to(self.device)
150
self.prefill_wrapper.end_forward()
151
self.prefill_wrapper.begin_forward(
152
self.query_start_loc, self.paged_kv_indptr,