@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 89feb4c84d
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[SpecDec] Remove Batch Expansion (2/3) (#9298)

Lily Liu committed Oct 11, 2024 at 22:13 UTC 89feb4c84dc8938738ef5d7b613f0d351cc2dc11
8 files changed +122 -70
tests/spec_decode/test_scorer.py
+39 -13
@@ -1,3 +1,6 @@
1 +import random
2 +from typing import List
3 +
4 import pytest
5 import torch
6
@@ -10,31 +13,45 @@ from vllm.worker.worker import Worker
13 from .utils import create_batch, create_worker
14
15
13 -def create_proposal(batch_size: int, propose_len: int, vocab_size: int,
16 +def create_proposal(propose_lens: List[int], vocab_size: int,
17 device: str) -> SpeculativeProposals:
15 - proposal_probs = torch.rand((batch_size, propose_len, vocab_size),
18 + batch_size = len(propose_lens)
19 + max_propose_len = max(propose_lens)
20 + proposal_probs = torch.rand((batch_size, max_propose_len, vocab_size),
21 device=device)
17 - proposal_token_ids = torch.argmax(proposal_probs, dim=-1)
18 - proposal_lens = torch.tensor([propose_len] * batch_size, device=device)
22 +
23 + proposal_token_ids = torch.full((batch_size, max_propose_len),
24 + fill_value=-1,
25 + device=device)
26 + for i in range(batch_size):
27 + proposal_token_ids[i][:propose_lens[i]] = torch.argmax(
28 + proposal_probs[i][:propose_lens[i]], dim=-1)
29 +
30 + propose_lens = torch.tensor(propose_lens, device=device)
31 return SpeculativeProposals(proposal_token_ids, proposal_probs,
20 - proposal_lens)
32 + propose_lens)
33
34
35 def assert_score_equal(score1: SpeculativeScores,
36 score2: SpeculativeScores) -> None:
37 assert torch.allclose(score1.probs, score2.probs)
38 assert torch.allclose(score1.logprobs, score2.logprobs)
27 - assert torch.equal(score1.token_ids, score2.token_ids)
39 + assert torch.equal(
40 + score1.token_ids,
41 + score2.token_ids), f"{score1.token_ids}, {score2.token_ids}"
42
43
44 @pytest.mark.parametrize('model_name', ['facebook/opt-125m'])
45 @pytest.mark.parametrize('batch_size', [1, 2, 4, 8, 16])
32 -@pytest.mark.parametrize('propose_len', [1, 3, 5])
46 +@pytest.mark.parametrize('max_propose_len', [1, 3, 5])
47 +@pytest.mark.parametrize('mixed_propose_len', [True])
48 @pytest.mark.parametrize('device', ['cuda'])
34 -def test_scoroer(model_name: str, batch_size: int, propose_len: int,
35 - device: str) -> None:
49 +def test_scorer(model_name: str, batch_size: int, max_propose_len: int,
50 + mixed_propose_len: bool, device: str) -> None:
51 """
37 - Compare the batch expansion scorer and mqa scorer return the same score
52 + Compare the batch expansion scorer and mqa scorer return the same score.
53 + We test for both queries with the same propose length and different
54 + propose length.
55 """
56 seed = 0
57 block_size = 32
@@ -46,13 +63,22 @@ def test_scoroer(model_name: str, batch_size: int, propose_len: int,
63 should_modify_greedy_probs_inplace = True
64
65 vocab_size = scorer_worker.vocab_size
49 - proposals = create_proposal(batch_size, propose_len, vocab_size, device)
66 +
67 + if not mixed_propose_len:
68 + propose_lens = [max_propose_len] * batch_size
69 + else:
70 + non_zero_cnt = random.randint(0, batch_size)
71 + propose_lens = [max_propose_len
72 + ] * non_zero_cnt + [0] * (batch_size - non_zero_cnt)
73 + random.shuffle(propose_lens)
74 +
75 + proposals = create_proposal(propose_lens, vocab_size, device)
76 seq_group_metadatalist, _, _ = create_batch(batch_size,
51 - propose_len,
77 + max_propose_len,
78 block_size=block_size,
79 num_gpu_blocks=num_gpu_blocks)
80 requests = ExecuteModelRequest(seq_group_metadatalist,
55 - num_lookahead_slots=propose_len)
81 + num_lookahead_slots=max_propose_len)
82
83 batch_expansion_scorer = BatchExpansionTop1Scorer(scorer_worker, device,
84 vocab_size)
vllm/attention/backends/blocksparse_attn.py
+2 -5
@@ -186,11 +186,8 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
186 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
187 use_cuda_graph: bool
188
189 - # Number of query tokens for each request in the batch.
190 - # Currently, we require that all requests have the same number of query
191 - # tokens during the decoding phase. When speculavie decoding is enabled,
192 - # decode_query_len might be greater than 1. In all other cases, it is 1.
193 - decode_query_len: Optional[int] = None
189 + # Max number of query tokens for among request in the batch.
190 + max_decode_query_len: Optional[int] = None
191
192 _cached_prefill_metadata: Optional[
193 "BlocksparseFlashAttentionMetadata"] = None
vllm/attention/backends/flash_attn.py
+42 -27
@@ -111,11 +111,8 @@ class FlashAttentionMetadata(AttentionMetadata):
111 # Maximum query length in the batch.
112 max_query_len: Optional[int]
113
114 - # Number of query tokens for each request in the batch.
115 - # Currently, we require that all requests have the same number of query
116 - # tokens during the decoding phase. When speculavie decoding is enabled,
117 - # decode_query_len might be greater than 1. In all other cases, it is 1.
118 - decode_query_len: Optional[int]
114 + # Max number of query tokens among request in the batch.
115 + max_decode_query_len: Optional[int]
116
117 # Maximum sequence length among prefill batch. 0 if there are decoding
118 # requests only.
@@ -173,9 +170,9 @@ class FlashAttentionMetadata(AttentionMetadata):
170 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
171 seq_lens=self.seq_lens[:self.num_prefills],
172 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
176 - decode_query_len=0,
173 max_query_len=self.max_query_len,
174 max_prefill_seq_len=self.max_prefill_seq_len,
175 + max_decode_query_len=0,
176 max_decode_seq_len=0,
177 query_start_loc=self.query_start_loc[:self.num_prefills + 1],
178 seq_start_loc=self.seq_start_loc[:self.num_prefills + 1],
@@ -202,12 +199,14 @@ class FlashAttentionMetadata(AttentionMetadata):
199 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
200 seq_lens=None,
201 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
205 - decode_query_len=self.decode_query_len,
202 + max_decode_query_len=self.max_decode_query_len,
203 max_query_len=self.max_query_len,
204 max_prefill_seq_len=0,
205 max_decode_seq_len=self.max_decode_seq_len,
209 - query_start_loc=None,
210 - seq_start_loc=None,
206 + query_start_loc=self.query_start_loc[self.num_prefills:]
207 + if self.query_start_loc is not None else None,
208 + seq_start_loc=self.seq_start_loc[self.num_prefills:]
209 + if self.seq_start_loc is not None else None,
210 context_lens_tensor=None,
211 block_tables=self.block_tables[self.num_prefills:],
212 use_cuda_graph=self.use_cuda_graph,
@@ -413,9 +412,9 @@ class FlashAttentionMetadataBuilder(
412 max_query_len = max(query_lens)
413 decode_query_lens = query_lens[self.num_prefills:]
414 if len(decode_query_lens) > 0:
416 - decode_query_len = max(decode_query_lens)
415 + max_decode_query_len = max(decode_query_lens)
416 else:
418 - decode_query_len = 1
417 + max_decode_query_len = 1
418 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
419 max_decode_seq_len = max(self.curr_seq_lens, default=0)
420 num_decode_tokens = self.num_decode_tokens
@@ -468,7 +467,7 @@ class FlashAttentionMetadataBuilder(
467 seq_lens=seq_lens,
468 seq_lens_tensor=seq_lens_tensor,
469 max_query_len=max_query_len,
471 - decode_query_len=decode_query_len,
470 + max_decode_query_len=max_decode_query_len,
471 max_prefill_seq_len=max_prefill_seq_len,
472 max_decode_seq_len=max_decode_seq_len,
473 query_start_loc=query_start_loc,
@@ -714,20 +713,37 @@ def unified_flash_attention(
713
714 if decode_meta := attn_metadata.decode_metadata:
715 # Decoding run.
717 - _, num_head, head_dim = decode_query.shape
718 - decode_query = decode_query.reshape(-1, decode_meta.decode_query_len,
719 - num_head, head_dim)
720 - decode_output = flash_attn_with_kvcache(
721 - q=decode_query,
722 - k_cache=key_cache,
723 - v_cache=value_cache,
724 - block_table=decode_meta.block_tables,
725 - cache_seqlens=decode_meta.seq_lens_tensor,
726 - softmax_scale=softmax_scale,
727 - causal=True,
728 - alibi_slopes=alibi_slopes,
729 - softcap=logits_soft_cap,
730 - ).squeeze(1)
716 + # Use flash_attn_varlen_func kernel for speculative decoding
717 + # because different queries might have different lengths.
718 + assert decode_meta.max_decode_query_len is not None
719 + if decode_meta.max_decode_query_len > 1:
720 + decode_output = flash_attn_varlen_func(
721 + q=decode_query,
722 + k=key_cache,
723 + v=value_cache,
724 + cu_seqlens_q=decode_meta.query_start_loc,
725 + max_seqlen_q=decode_meta.max_decode_query_len,
726 + cu_seqlens_k=decode_meta.seq_start_loc,
727 + max_seqlen_k=decode_meta.max_decode_seq_len,
728 + softmax_scale=softmax_scale,
729 + causal=True,
730 + alibi_slopes=alibi_slopes,
731 + softcap=logits_soft_cap,
732 + block_table=decode_meta.block_tables,
733 + )
734 + else:
735 + # Use flash_attn_with_kvcache for normal decoding.
736 + decode_output = flash_attn_with_kvcache(
737 + q=decode_query.unsqueeze(1),
738 + k_cache=key_cache,
739 + v_cache=value_cache,
740 + block_table=decode_meta.block_tables,
741 + cache_seqlens=decode_meta.seq_lens_tensor,
742 + softmax_scale=softmax_scale,
743 + causal=True,
744 + alibi_slopes=alibi_slopes,
745 + softcap=logits_soft_cap,
746 + ).squeeze(1)
747
748 if prefill_output is None:
749 assert decode_output is not None
@@ -739,7 +755,6 @@ def unified_flash_attention(
755 # Chunked prefill does not work with speculative decoding.
756 # Therefore, the query length for decode should be 1 in chunked prefill.
757 assert decode_meta is not None
742 - assert decode_meta.decode_query_len == 1
758 decode_output = decode_output.squeeze(1)
759 output = torch.cat([prefill_output, decode_output], dim=0)
760 return output.view(num_tokens, hidden_size)
vllm/attention/backends/rocm_flash_attn.py
+2 -5
@@ -121,11 +121,8 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
121 # so far).
122 context_lens_tensor: Optional[torch.Tensor]
123
124 - # Number of query tokens for each request in the batch.
125 - # Currently, we require that all requests have the same number of query
126 - # tokens during the decoding phase. When speculavie decoding is enabled,
127 - # decode_query_len might be greater than 1. In all other cases, it is 1.
128 - decode_query_len: Optional[int] = None
124 + # Max number of query tokens among request in the batch.
125 + max_decode_query_len: Optional[int] = None
126
127 _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
128 _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
vllm/attention/backends/utils.py
+1 -1
@@ -313,7 +313,7 @@ class CommonAttentionState(AttentionState):
313 seq_lens=None,
314 seq_lens_tensor=self._graph_seq_lens[:batch_size],
315 max_query_len=1,
316 - decode_query_len=1,
316 + max_decode_query_len=1,
317 max_prefill_seq_len=0,
318 max_decode_seq_len=self.runner.max_seq_len_to_capture,
319 query_start_loc=None,
vllm/attention/backends/xformers.py
+2 -5
@@ -118,11 +118,8 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
118 # Maximum query length in the batch. None for decoding.
119 max_query_len: Optional[int] = None
120
121 - # Number of query tokens for each request in the batch.
122 - # Currently, we require that all requests have the same number of query
123 - # tokens during the decoding phase. When speculavie decoding is enabled,
124 - # decode_query_len might be greater than 1. In all other cases, it is 1.
125 - decode_query_len: Optional[int] = None
121 + # Max number of query tokens among request in the batch.
122 + max_decode_query_len: Optional[int] = None
123
124 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
125 # the batch, used to index into subquery. E.g., if the subquery length
vllm/spec_decode/mqa_scorer.py
+34 -8
@@ -18,6 +18,7 @@ class MQAScorer(SpeculativeScorer):
18 target_seq_id_start = max(
19 get_all_seq_ids(execute_model_req.seq_group_metadata_list)) + 1
20 all_proposal_tokens = proposals.proposal_token_ids.tolist()
21 + all_proposal_lengths = proposals.proposal_lens.tolist()
22 for i, seq_group_metadata in enumerate(
23 execute_model_req.seq_group_metadata_list):
24 seq_data_dict = seq_group_metadata.seq_data
@@ -27,7 +28,8 @@ class MQAScorer(SpeculativeScorer):
28 seq_data: SequenceData = seq_data_dict[seq_id]
29 prompt_token_ids = seq_data.get_prompt_token_ids()
30 output_token_ids = seq_data.get_output_token_ids()
30 - proposal_token_ids = all_proposal_tokens[i]
31 + proposal_token_ids = all_proposal_tokens[
32 + i][:all_proposal_lengths[i]]
33 new_output_token_ids = [*output_token_ids, *proposal_token_ids]
34
35 target_seq_id = target_seq_id_start + i
@@ -62,18 +64,42 @@ class MQAScorer(SpeculativeScorer):
64
65 target_sampler_output = target_sampler_output[0]
66
65 - bs, k = proposals.proposal_token_ids.shape
66 - all_tokens = target_sampler_output.sampled_token_ids.reshape(bs, k + 1)
67 -
68 - all_probs = target_sampler_output.sampled_token_probs.reshape(
69 - bs, k + 1, self._vocab_size)
70 - all_logprobs = target_sampler_output.logprobs.reshape(
71 - bs, k + 1, self._vocab_size)
67 + k = execute_model_req.num_lookahead_slots
68 + bs = len(execute_model_req.seq_group_metadata_list)
69 + target_token_ids = target_sampler_output.sampled_token_ids
70 + target_probs = target_sampler_output.sampled_token_probs
71 + target_logprobs = target_sampler_output.logprobs
72 + # If all requests have the same number of query tokens, we can avoid
73 + # the for loop to build output for better performance.
74 + if min(all_proposal_lengths) == k:
75 + bs, _ = proposals.proposal_token_ids.shape
76 + all_tokens = target_token_ids.reshape(bs, k + 1)
77 + all_probs = target_probs.reshape(bs, k + 1, self._vocab_size)
78 + all_logprobs = target_logprobs.reshape(bs, k + 1, self._vocab_size)
79 + else:
80 + all_tokens = target_token_ids.new_full(size=(bs, k + 1),
81 + fill_value=-1)
82 + all_probs = target_probs.new_zeros(*all_tokens.shape,
83 + self._vocab_size)
84 + all_logprobs = target_logprobs.new_full(size=all_probs.shape,
85 + fill_value=-float("inf"))
86 + target_token_ids = target_token_ids.flatten()
87 + start_loc = 0
88 + for i, proposed_len in enumerate(all_proposal_lengths):
89 + output_len = proposed_len + 1
90 + end_loc = start_loc + output_len
91 + all_tokens[
92 + i, :output_len] = target_token_ids[start_loc:end_loc]
93 + all_probs[i, :output_len] = target_probs[start_loc:end_loc]
94 + all_logprobs[
95 + i, :output_len] = target_logprobs[start_loc:end_loc]
96 + start_loc = end_loc
97
98 hidden_states = None
99 if target_sampler_output.hidden_states is not None:
100 hidden_states = target_sampler_output.hidden_states.reshape(
101 bs, (k + 1), -1)
102 +
103 return SpeculativeScores(probs=all_probs,
104 token_ids=all_tokens,
105 logprobs=all_logprobs,
vllm/spec_decode/spec_decode_worker.py
-6
@@ -190,12 +190,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
190 "[Speculative Decoding] Disabling MQA scorer as the "
191 "MQA is only available with flash attn backend.")
192
193 - if ngram_prompt_lookup_max > 0:
194 - disable_mqa_scorer = True
195 - logger.info(
196 - "[Speculative Decoding] Disabling MQA scorer as the "
197 - "NGramWorker does not support MQA scorer.")
198 -
193 if "model_config" in draft_worker_kwargs and \
194 draft_worker_kwargs["model_config"].max_model_len < \
195 scorer_worker.model_config.max_model_len: