A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[SpecDec] Remove Batch Expansion (2/3) (#9298)
Lily Liu committed
Oct 11, 2024 at 22:13 UTC
89feb4c84dc8938738ef5d7b613f0d351cc2dc11
8 files changed
+122
-70
tests/spec_decode/test_scorer.py
+39
-13
@@ -1,3 +1,6 @@
1
+import random
2
+from typing import List
3
+
4
import pytest
5
import torch
6
@@ -10,31 +13,45 @@ from vllm.worker.worker import Worker
13
from .utils import create_batch, create_worker
14
15
13
-def create_proposal(batch_size: int, propose_len: int, vocab_size: int,
16
+def create_proposal(propose_lens: List[int], vocab_size: int,
17
device: str) -> SpeculativeProposals:
15
- proposal_probs = torch.rand((batch_size, propose_len, vocab_size),
18
+ batch_size = len(propose_lens)
19
+ max_propose_len = max(propose_lens)
20
+ proposal_probs = torch.rand((batch_size, max_propose_len, vocab_size),
21
device=device)
17
- proposal_token_ids = torch.argmax(proposal_probs, dim=-1)
18
- proposal_lens = torch.tensor([propose_len] * batch_size, device=device)
22
+
23
+ proposal_token_ids = torch.full((batch_size, max_propose_len),
24
+ fill_value=-1,
25
+ device=device)
26
+ for i in range(batch_size):
27
+ proposal_token_ids[i][:propose_lens[i]] = torch.argmax(
28
+ proposal_probs[i][:propose_lens[i]], dim=-1)
29
+
30
+ propose_lens = torch.tensor(propose_lens, device=device)
31
return SpeculativeProposals(proposal_token_ids, proposal_probs,
20
- proposal_lens)
32
+ propose_lens)
33
34
35
def assert_score_equal(score1: SpeculativeScores,
36
score2: SpeculativeScores) -> None:
37
assert torch.allclose(score1.probs, score2.probs)
38
assert torch.allclose(score1.logprobs, score2.logprobs)
27
- assert torch.equal(score1.token_ids, score2.token_ids)
39
+ assert torch.equal(
40
+ score1.token_ids,
41
+ score2.token_ids), f"{score1.token_ids}, {score2.token_ids}"
42
43
44
@pytest.mark.parametrize('model_name', ['facebook/opt-125m'])
45
@pytest.mark.parametrize('batch_size', [1, 2, 4, 8, 16])
32
-@pytest.mark.parametrize('propose_len', [1, 3, 5])
46
+@pytest.mark.parametrize('max_propose_len', [1, 3, 5])
47
+@pytest.mark.parametrize('mixed_propose_len', [True])
48
@pytest.mark.parametrize('device', ['cuda'])
34
-def test_scoroer(model_name: str, batch_size: int, propose_len: int,
35
- device: str) -> None:
49
+def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
50
+ mixed_propose_len: bool, device: str) -> None:
51
"""
37
- Compare the batch expansion scorer and mqa scorer return the same score
52
+ Compare the batch expansion scorer and mqa scorer return the same score.
53
+ We test for both queries with the same propose length and different
54
+ propose length.
55
"""
56
seed = 0
57
block_size = 32
@@ -46,13 +63,22 @@ def test_scoroer(model_name: str, batch_size: int, propose_len: int,
63
should_modify_greedy_probs_inplace = True
64
65
vocab_size = scorer_worker.vocab_size
49
- proposals = create_proposal(batch_size, propose_len, vocab_size, device)
66
+
67
+ if not mixed_propose_len:
68
+ propose_lens = [max_propose_len] * batch_size
69
+ else:
70
+ non_zero_cnt = random.randint(0, batch_size)
71
+ propose_lens = [max_propose_len
72
+ ] * non_zero_cnt + [0] * (batch_size - non_zero_cnt)
73
+ random.shuffle(propose_lens)
74
+
75
+ proposals = create_proposal(propose_lens, vocab_size, device)
76
seq_group_metadatalist, _, _ = create_batch(batch_size,
51
- propose_len,
77
+ max_propose_len,
78
block_size=block_size,
79
num_gpu_blocks=num_gpu_blocks)
80
requests = ExecuteModelRequest(seq_group_metadatalist,
55
- num_lookahead_slots=propose_len)
81
+ num_lookahead_slots=max_propose_len)
82
83
batch_expansion_scorer = BatchExpansionTop1Scorer(scorer_worker, device,
84
vocab_size)
vllm/attention/backends/blocksparse_attn.py
+2
-5
@@ -186,11 +186,8 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
186
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
187
use_cuda_graph: bool
188
189
- # Number of query tokens for each request in the batch.
190
- # Currently, we require that all requests have the same number of query
191
- # tokens during the decoding phase. When speculavie decoding is enabled,
192
- # decode_query_len might be greater than 1. In all other cases, it is 1.
193
- decode_query_len: Optional[int] = None
189
+ # Max number of query tokens for among request in the batch.
190
+ max_decode_query_len: Optional[int] = None
191
192
_cached_prefill_metadata: Optional[
193
"BlocksparseFlashAttentionMetadata"] = None
vllm/attention/backends/flash_attn.py
+42
-27
@@ -111,11 +111,8 @@ class FlashAttentionMetadata(AttentionMetadata):
111
# Maximum query length in the batch.
112
max_query_len: Optional[int]
113
114
- # Number of query tokens for each request in the batch.
115
- # Currently, we require that all requests have the same number of query
116
- # tokens during the decoding phase. When speculavie decoding is enabled,
117
- # decode_query_len might be greater than 1. In all other cases, it is 1.
118
- decode_query_len: Optional[int]
114
+ # Max number of query tokens among request in the batch.
115
+ max_decode_query_len: Optional[int]
116
117
# Maximum sequence length among prefill batch. 0 if there are decoding
118
# requests only.
@@ -173,9 +170,9 @@ class FlashAttentionMetadata(AttentionMetadata):
170
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
171
seq_lens=self.seq_lens[:self.num_prefills],
172
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
176
- decode_query_len=0,
173
max_query_len=self.max_query_len,
174
max_prefill_seq_len=self.max_prefill_seq_len,
175
+ max_decode_query_len=0,
176
max_decode_seq_len=0,
177
query_start_loc=self.query_start_loc[:self.num_prefills + 1],
178
seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
@@ -202,12 +199,14 @@ class FlashAttentionMetadata(AttentionMetadata):
199
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
200
seq_lens=None,
201
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
205
- decode_query_len=self.decode_query_len,
202
+ max_decode_query_len=self.max_decode_query_len,
203
max_query_len=self.max_query_len,
204
max_prefill_seq_len=0,
205
max_decode_seq_len=self.max_decode_seq_len,
209
- query_start_loc=None,
210
- seq_start_loc=None,
206
+ query_start_loc=self.query_start_loc[self.num_prefills:]
207
+ if self.query_start_loc is not None else None,
208
+ seq_start_loc=self.seq_start_loc[self.num_prefills:]
209
+ if self.seq_start_loc is not None else None,
210
context_lens_tensor=None,
211
block_tables=self.block_tables[self.num_prefills:],
212
use_cuda_graph=self.use_cuda_graph,
@@ -413,9 +412,9 @@ class FlashAttentionMetadataBuilder(
412
max_query_len = max(query_lens)
413
decode_query_lens = query_lens[self.num_prefills:]
414
if len(decode_query_lens) > 0:
416
- decode_query_len = max(decode_query_lens)
415
+ max_decode_query_len = max(decode_query_lens)
416
else:
418
- decode_query_len = 1
417
+ max_decode_query_len = 1
418
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
419
max_decode_seq_len = max(self.curr_seq_lens, default=0)
420
num_decode_tokens = self.num_decode_tokens
@@ -468,7 +467,7 @@ class FlashAttentionMetadataBuilder(
467
seq_lens=seq_lens,
468
seq_lens_tensor=seq_lens_tensor,
469
max_query_len=max_query_len,
471
- decode_query_len=decode_query_len,
470
+ max_decode_query_len=max_decode_query_len,
471
max_prefill_seq_len=max_prefill_seq_len,
472
max_decode_seq_len=max_decode_seq_len,
473
query_start_loc=query_start_loc,
@@ -714,20 +713,37 @@ def unified_flash_attention(
713
714
if decode_meta := attn_metadata.decode_metadata:
715
# Decoding run.
717
- _, num_head, head_dim = decode_query.shape
718
- decode_query = decode_query.reshape(-1, decode_meta.decode_query_len,
719
- num_head, head_dim)
720
- decode_output = flash_attn_with_kvcache(
721
- q=decode_query,
722
- k_cache=key_cache,
723
- v_cache=value_cache,
724
- block_table=decode_meta.block_tables,
725
- cache_seqlens=decode_meta.seq_lens_tensor,
726
- softmax_scale=softmax_scale,
727
- causal=True,
728
- alibi_slopes=alibi_slopes,
729
- softcap=logits_soft_cap,
730
- ).squeeze(1)
716
+ # Use flash_attn_varlen_func kernel for speculative decoding
717
+ # because different queries might have different lengths.
718
+ assert decode_meta.max_decode_query_len is not None
719
+ if decode_meta.max_decode_query_len > 1:
720
+ decode_output = flash_attn_varlen_func(
721
+ q=decode_query,
722
+ k=key_cache,
723
+ v=value_cache,
724
+ cu_seqlens_q=decode_meta.query_start_loc,
725
+ max_seqlen_q=decode_meta.max_decode_query_len,
726
+ cu_seqlens_k=decode_meta.seq_start_loc,
727
+ max_seqlen_k=decode_meta.max_decode_seq_len,
728
+ softmax_scale=softmax_scale,
729
+ causal=True,
730
+ alibi_slopes=alibi_slopes,
731
+ softcap=logits_soft_cap,
732
+ block_table=decode_meta.block_tables,
733
+ )
734
+ else:
735
+ # Use flash_attn_with_kvcache for normal decoding.
736
+ decode_output = flash_attn_with_kvcache(
737
+ q=decode_query.unsqueeze(1),
738
+ k_cache=key_cache,
739
+ v_cache=value_cache,
740
+ block_table=decode_meta.block_tables,
741
+ cache_seqlens=decode_meta.seq_lens_tensor,
742
+ softmax_scale=softmax_scale,
743
+ causal=True,
744
+ alibi_slopes=alibi_slopes,
745
+ softcap=logits_soft_cap,
746
+ ).squeeze(1)
747
748
if prefill_output is None:
749
assert decode_output is not None
@@ -739,7 +755,6 @@ def unified_flash_attention(
755
# Chunked prefill does not work with speculative decoding.
756
# Therefore, the query length for decode should be 1 in chunked prefill.
757
assert decode_meta is not None
742
- assert decode_meta.decode_query_len == 1
758
decode_output = decode_output.squeeze(1)
759
output = torch.cat([prefill_output, decode_output], dim=0)
760
return output.view(num_tokens, hidden_size)
vllm/attention/backends/rocm_flash_attn.py
+2
-5
@@ -121,11 +121,8 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
121
# so far).
122
context_lens_tensor: Optional[torch.Tensor]
123
124
- # Number of query tokens for each request in the batch.
125
- # Currently, we require that all requests have the same number of query
126
- # tokens during the decoding phase. When speculavie decoding is enabled,
127
- # decode_query_len might be greater than 1. In all other cases, it is 1.
128
- decode_query_len: Optional[int] = None
124
+ # Max number of query tokens among request in the batch.
125
+ max_decode_query_len: Optional[int] = None
126
127
_cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
128
_cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
vllm/attention/backends/utils.py
+1
-1
@@ -313,7 +313,7 @@ class CommonAttentionState(AttentionState):
313
seq_lens=None,
314
seq_lens_tensor=self._graph_seq_lens[:batch_size],
315
max_query_len=1,
316
- decode_query_len=1,
316
+ max_decode_query_len=1,
317
max_prefill_seq_len=0,
318
max_decode_seq_len=self.runner.max_seq_len_to_capture,
319
query_start_loc=None,
vllm/attention/backends/xformers.py
+2
-5
@@ -118,11 +118,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
118
# Maximum query length in the batch. None for decoding.
119
max_query_len: Optional[int] = None
120
121
- # Number of query tokens for each request in the batch.
122
- # Currently, we require that all requests have the same number of query
123
- # tokens during the decoding phase. When speculavie decoding is enabled,
124
- # decode_query_len might be greater than 1. In all other cases, it is 1.
125
- decode_query_len: Optional[int] = None
121
+ # Max number of query tokens among request in the batch.
122
+ max_decode_query_len: Optional[int] = None
123
124
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
125
# the batch, used to index into subquery. E.g., if the subquery length
vllm/spec_decode/mqa_scorer.py
+34
-8
@@ -18,6 +18,7 @@ class MQAScorer(SpeculativeScorer):
18
target_seq_id_start = max(
19
get_all_seq_ids(execute_model_req.seq_group_metadata_list)) + 1
20
all_proposal_tokens = proposals.proposal_token_ids.tolist()
21
+ all_proposal_lengths = proposals.proposal_lens.tolist()
22
for i, seq_group_metadata in enumerate(
23
execute_model_req.seq_group_metadata_list):
24
seq_data_dict = seq_group_metadata.seq_data
@@ -27,7 +28,8 @@ class MQAScorer(SpeculativeScorer):
28
seq_data: SequenceData = seq_data_dict[seq_id]
29
prompt_token_ids = seq_data.get_prompt_token_ids()
30
output_token_ids = seq_data.get_output_token_ids()
30
- proposal_token_ids = all_proposal_tokens[i]
31
+ proposal_token_ids = all_proposal_tokens[
32
+ i][:all_proposal_lengths[i]]
33
new_output_token_ids = [*output_token_ids, *proposal_token_ids]
34
35
target_seq_id = target_seq_id_start + i
@@ -62,18 +64,42 @@ class MQAScorer(SpeculativeScorer):
64
65
target_sampler_output = target_sampler_output[0]
66
65
- bs, k = proposals.proposal_token_ids.shape
66
- all_tokens = target_sampler_output.sampled_token_ids.reshape(bs, k + 1)
67
-
68
- all_probs = target_sampler_output.sampled_token_probs.reshape(
69
- bs, k + 1, self._vocab_size)
70
- all_logprobs = target_sampler_output.logprobs.reshape(
71
- bs, k + 1, self._vocab_size)
67
+ k = execute_model_req.num_lookahead_slots
68
+ bs = len(execute_model_req.seq_group_metadata_list)
69
+ target_token_ids = target_sampler_output.sampled_token_ids
70
+ target_probs = target_sampler_output.sampled_token_probs
71
+ target_logprobs = target_sampler_output.logprobs
72
+ # If all requests have the same number of query tokens, we can avoid
73
+ # the for loop to build output for better performance.
74
+ if min(all_proposal_lengths) == k:
75
+ bs, _ = proposals.proposal_token_ids.shape
76
+ all_tokens = target_token_ids.reshape(bs, k + 1)
77
+ all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
78
+ all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size)
79
+ else:
80
+ all_tokens = target_token_ids.new_full(size=(bs, k + 1),
81
+ fill_value=-1)
82
+ all_probs = target_probs.new_zeros(*all_tokens.shape,
83
+ self._vocab_size)
84
+ all_logprobs = target_logprobs.new_full(size=all_probs.shape,
85
+ fill_value=-float("inf"))
86
+ target_token_ids = target_token_ids.flatten()
87
+ start_loc = 0
88
+ for i, proposed_len in enumerate(all_proposal_lengths):
89
+ output_len = proposed_len + 1
90
+ end_loc = start_loc + output_len
91
+ all_tokens[
92
+ i, :output_len] = target_token_ids[start_loc:end_loc]
93
+ all_probs[i, :output_len] = target_probs[start_loc:end_loc]
94
+ all_logprobs[
95
+ i, :output_len] = target_logprobs[start_loc:end_loc]
96
+ start_loc = end_loc
97
98
hidden_states = None
99
if target_sampler_output.hidden_states is not None:
100
hidden_states = target_sampler_output.hidden_states.reshape(
101
bs, (k + 1), -1)
102
+
103
return SpeculativeScores(probs=all_probs,
104
token_ids=all_tokens,
105
logprobs=all_logprobs,
vllm/spec_decode/spec_decode_worker.py
-6
@@ -190,12 +190,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
190
"[Speculative Decoding] Disabling MQA scorer as the "
191
"MQA is only available with flash attn backend.")
192
193
- if ngram_prompt_lookup_max > 0:
194
- disable_mqa_scorer = True
195
- logger.info(
196
- "[Speculative Decoding] Disabling MQA scorer as the "
197
- "NGramWorker does not support MQA scorer.")
198
-
193
if "model_config" in draft_worker_kwargs and \
194
draft_worker_kwargs["model_config"].max_model_len < \
195
scorer_worker.model_config.max_model_len: