A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[CI] Fix merge conflict (#9317)
Lily Liu committed
Oct 12, 2024 at 23:41 UTC
f519902c52cfd61da9026ab714fad9d95502d2f1
1 file changed
+7
-10
vllm/attention/backends/placeholder_attn.py
+7
-10
@@ -75,11 +75,8 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
75
# Maximum query length in the batch.
76
max_query_len: Optional[int]
77
78
- # Number of query tokens for each request in the batch.
79
- # Currently, we require that all requests have the same number of query
80
- # tokens during the decoding phase. When speculavie decoding is enabled,
81
- # decode_query_len might be greater than 1. In all other cases, it is 1.
82
- decode_query_len: Optional[int]
78
+ # Max number of query tokens among request in the batch.
79
+ max_decode_query_len: Optional[int]
80
81
# Maximum sequence length among prefill batch. 0 if there are decoding
82
# requests only.
@@ -140,7 +137,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
137
slot_mapping=slot_mapping,
138
seq_lens=self.seq_lens[:self.num_prefills],
139
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
143
- decode_query_len=0,
140
+ max_decode_query_len=0,
141
max_query_len=self.max_query_len,
142
max_prefill_seq_len=self.max_prefill_seq_len,
143
max_decode_seq_len=0,
@@ -172,7 +169,7 @@ class PlaceholderAttentionMetadata(AttentionMetadata):
169
slot_mapping=slot_mapping,
170
seq_lens=None,
171
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
175
- decode_query_len=self.decode_query_len,
172
+ max_decode_query_len=self.max_decode_query_len,
173
max_query_len=None,
174
max_prefill_seq_len=0,
175
max_decode_seq_len=self.max_decode_seq_len,
@@ -256,9 +253,9 @@ class PlaceholderAttentionMetadataBuilder(
253
max_query_len = max(query_lens)
254
decode_query_lens = query_lens[self.num_prefills:]
255
if len(decode_query_lens) > 0:
259
- decode_query_len = max(decode_query_lens)
256
+ max_decode_query_len = max(decode_query_lens)
257
else:
261
- decode_query_len = 1
258
+ max_decode_query_len = 1
259
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
260
max_decode_seq_len = max(self.curr_seq_lens, default=0)
261
num_decode_tokens = self.num_decode_tokens
@@ -304,7 +301,7 @@ class PlaceholderAttentionMetadataBuilder(
301
seq_lens=seq_lens,
302
seq_lens_tensor=seq_lens_tensor,
303
max_query_len=max_query_len,
307
- decode_query_len=decode_query_len,
304
+ max_decode_query_len=max_decode_query_len,
305
max_prefill_seq_len=max_prefill_seq_len,
306
max_decode_seq_len=max_decode_seq_len,
307
query_start_loc=query_start_loc,