A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Bugfix] Add explicit `end_forward` calls to flashinfer (#6044)
Antoni Baum committed
Jul 1, 2024 at 16:08 UTC
c4059ea54ff36e62b03f1a88baa41ca72dc695e4
1 file changed
+2
vllm/attention/backends/flashinfer.py
+2
@@ -126,6 +126,7 @@ class FlashInferMetadata(AttentionMetadata):
126
self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
127
self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
128
self.device)
129
+ self.prefill_wrapper.end_forward()
130
self.prefill_wrapper.begin_forward(
131
self.query_start_loc, self.paged_kv_indptr,
132
self.paged_kv_indices, self.paged_kv_last_page_len,
@@ -142,6 +143,7 @@ class FlashInferMetadata(AttentionMetadata):
143
self.device)
144
145
assert self.decode_wrapper is not None
146
+ self.decode_wrapper.end_forward()
147
self.decode_wrapper.begin_forward(
148
self.paged_kv_indptr,
149
self.paged_kv_indices,