@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / c4059ea54f
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Bugfix] Add explicit `end_forward` calls to flashinfer (#6044)

Antoni Baum committed Jul 1, 2024 at 16:08 UTC c4059ea54ff36e62b03f1a88baa41ca72dc695e4
1 file changed +2
vllm/attention/backends/flashinfer.py
+2
@@ -126,6 +126,7 @@ class FlashInferMetadata(AttentionMetadata):
126 self.paged_kv_indptr = self.paged_kv_indptr.to(self.device)
127 self.paged_kv_last_page_len = self.paged_kv_last_page_len.to(
128 self.device)
129 + self.prefill_wrapper.end_forward()
130 self.prefill_wrapper.begin_forward(
131 self.query_start_loc, self.paged_kv_indptr,
132 self.paged_kv_indices, self.paged_kv_last_page_len,
@@ -142,6 +143,7 @@ class FlashInferMetadata(AttentionMetadata):
143 self.device)
144
145 assert self.decode_wrapper is not None
146 + self.decode_wrapper.end_forward()
147 self.decode_wrapper.begin_forward(
148 self.paged_kv_indptr,
149 self.paged_kv_indices,