@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / f519902c52
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[CI] Fix merge conflict (#9317)

Lily Liu committed Oct 12, 2024 at 23:41 UTC f519902c52cfd61da9026ab714fad9d95502d2f1
1 file changed +7 -10
vllm/attention/backends/placeholder_attn.py
+7 -10
@@ -75,11 +75,8 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
75 # Maximum query length in the batch.
76 max_query_len: Optional[int]
77
78 - # Number of query tokens for each request in the batch.
79 - # Currently, we require that all requests have the same number of query
80 - # tokens during the decoding phase. When speculavie decoding is enabled,
81 - # decode_query_len might be greater than 1. In all other cases, it is 1.
82 - decode_query_len: Optional[int]
78 + # Max number of query tokens among request in the batch.
79 + max_decode_query_len: Optional[int]
80
81 # Maximum sequence length among prefill batch. 0 if there are decoding
82 # requests only.
@@ -140,7 +137,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
137 slot_mapping=slot_mapping,
138 seq_lens=self.seq_lens[:self.num_prefills],
139 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
143 - decode_query_len=0,
140 + max_decode_query_len=0,
141 max_query_len=self.max_query_len,
142 max_prefill_seq_len=self.max_prefill_seq_len,
143 max_decode_seq_len=0,
@@ -172,7 +169,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
169 slot_mapping=slot_mapping,
170 seq_lens=None,
171 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
175 - decode_query_len=self.decode_query_len,
172 + max_decode_query_len=self.max_decode_query_len,
173 max_query_len=None,
174 max_prefill_seq_len=0,
175 max_decode_seq_len=self.max_decode_seq_len,
@@ -256,9 +253,9 @@ class PlaceholderAttentionMetadataBuilder(
253 max_query_len = max(query_lens)
254 decode_query_lens = query_lens[self.num_prefills:]
255 if len(decode_query_lens) > 0:
259 - decode_query_len = max(decode_query_lens)
256 + max_decode_query_len = max(decode_query_lens)
257 else:
261 - decode_query_len = 1
258 + max_decode_query_len = 1
259 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
260 max_decode_seq_len = max(self.curr_seq_lens, default=0)
261 num_decode_tokens = self.num_decode_tokens
@@ -304,7 +301,7 @@ class PlaceholderAttentionMetadataBuilder(
301 seq_lens=seq_lens,
302 seq_lens_tensor=seq_lens_tensor,
303 max_query_len=max_query_len,
307 - decode_query_len=decode_query_len,
304 + max_decode_query_len=max_decode_query_len,
305 max_prefill_seq_len=max_prefill_seq_len,
306 max_decode_seq_len=max_decode_seq_len,
307 query_start_loc=query_start_loc,