@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / 1570203864
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 10:45 UTC

[Spec Decode] (1/2) Remove batch expansion (#8839)

Lily Liu committed Oct 1, 2024 at 16:04 UTC 15702038642192002cd8973cf8948751b750fd07
29 files changed +531 -99
.buildkite/test-pipeline.yaml
+1 -1
@@ -208,7 +208,7 @@ steps:
208 - tests/spec_decode
209 commands:
210 - pytest -v -s spec_decode/e2e/test_multistep_correctness.py
211 - - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
211 + - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
212
213 - label: LoRA Test %N # 15min each
214 mirror_hardwares: [amd]
tests/samplers/test_sampler.py
+1 -1
@@ -434,7 +434,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
434 sampling_metadata = SamplingMetadata.prepare(
435 seq_group_metadata_list,
436 seq_lens=seq_lens if seq_lens else None,
437 - query_lens=seq_lens if seq_lens else None,
437 + query_lens=seq_lens if seq_lens else [1] * batch_size,
438 device=device,
439 pin_memory=is_pin_memory_available())
440 # the logits tensor is modified in-place by the sampler
tests/spec_decode/e2e/test_integration.py
+44
@@ -102,3 +102,47 @@ def test_speculative_model_quantization_config(vllm_runner, common_llm_kwargs,
102 max_output_len=32,
103 seed=seed,
104 temperature=0.0)
105 +
106 +
107 +@pytest.mark.parametrize(
108 + "common_llm_kwargs",
109 + [{
110 + "model_name": MAIN_MODEL,
111 +
112 + # Skip cuda graph recording for fast test.
113 + "enforce_eager": True,
114 +
115 + # Required for spec decode.
116 + "use_v2_block_manager": True,
117 + "speculative_model": "JackFram/llama-68m",
118 + "num_speculative_tokens": 3,
119 + }])
120 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
121 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
122 +@pytest.mark.parametrize("test_llm_kwargs",
123 + [{
124 + "speculative_disable_mqa_scorer": True,
125 + }])
126 +@pytest.mark.parametrize("batch_size", [1, 5])
127 +@pytest.mark.parametrize(
128 + "output_len",
129 + [
130 + # Use smaller output len for fast test.
131 + 32,
132 + ])
133 +@pytest.mark.parametrize("seed", [1])
134 +def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
135 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
136 + output_len: int, seed: int):
137 + """Verify that ngram speculative decoding generates the same output
138 + with batch expansion scorer and mqa scorer.
139 + """
140 + run_equality_correctness_test(vllm_runner,
141 + common_llm_kwargs,
142 + per_test_common_llm_kwargs,
143 + baseline_llm_kwargs,
144 + test_llm_kwargs,
145 + batch_size,
146 + max_output_len=output_len,
147 + seed=seed,
148 + temperature=0.0)
tests/spec_decode/e2e/test_medusa_correctness.py
+49
@@ -350,6 +350,55 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
350 temperature=0.0)
351
352
353 +@pytest.mark.parametrize(
354 + "common_llm_kwargs",
355 + [{
356 + # Skip cuda graph recording for fast test.
357 + "enforce_eager": True,
358 +
359 + # Required for spec decode.
360 + "use_v2_block_manager": True,
361 +
362 + # Precision
363 + "dtype": PRECISION,
364 +
365 + # Main model
366 + "model_name": MAIN_MODEL,
367 + "speculative_model": SPEC_MODEL,
368 + "num_speculative_tokens": MAX_SPEC_TOKENS,
369 + "speculative_disable_by_batch_size": 4
370 + }])
371 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
372 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
373 +@pytest.mark.parametrize("test_llm_kwargs",
374 + [{
375 + "speculative_disable_mqa_scorer": True,
376 + }])
377 +@pytest.mark.parametrize("batch_size", [1, 5])
378 +@pytest.mark.parametrize(
379 + "output_len",
380 + [
381 + # Use smaller output len for fast test.
382 + 32,
383 + ])
384 +@pytest.mark.parametrize("seed", [1])
385 +def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
386 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
387 + output_len: int, seed: int):
388 + """Verify that speculative decoding generates the same output
389 + with batch expansion scorer and mqa scorer.
390 + """
391 + run_equality_correctness_test(vllm_runner,
392 + common_llm_kwargs,
393 + per_test_common_llm_kwargs,
394 + baseline_llm_kwargs,
395 + test_llm_kwargs,
396 + batch_size,
397 + max_output_len=output_len,
398 + seed=seed,
399 + temperature=0.0)
400 +
401 +
402 if __name__ == "__main__":
403 import pytest
404 pytest.main([__file__])
tests/spec_decode/e2e/test_mlp_correctness.py
+43
@@ -460,3 +460,46 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
460 max_output_len=output_len,
461 seed=seed,
462 temperature=0.0)
463 +
464 +
465 +@pytest.mark.parametrize(
466 + "common_llm_kwargs",
467 + [{
468 + "model_name": MAIN_MODEL,
469 +
470 + # Skip cuda graph recording for fast test.
471 + "enforce_eager": True,
472 +
473 + # Required for spec decode.
474 + "use_v2_block_manager": True,
475 + "speculative_model": SPEC_MODEL,
476 + }])
477 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
478 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
479 +@pytest.mark.parametrize("test_llm_kwargs",
480 + [{
481 + "speculative_disable_mqa_scorer": True,
482 + }])
483 +@pytest.mark.parametrize("batch_size", [1, 5])
484 +@pytest.mark.parametrize(
485 + "output_len",
486 + [
487 + # Use smaller output len for fast test.
488 + 32,
489 + ])
490 +@pytest.mark.parametrize("seed", [1])
491 +def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
492 + baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
493 + output_len: int, seed: int):
494 + """Verify that speculative decoding generates the same output
495 + with batch expansion scorer and mqa scorer.
496 + """
497 + run_equality_correctness_test(vllm_runner,
498 + common_llm_kwargs,
499 + per_test_common_llm_kwargs,
500 + baseline_llm_kwargs,
501 + test_llm_kwargs,
502 + batch_size,
503 + max_output_len=output_len,
504 + seed=seed,
505 + temperature=0.0)
tests/spec_decode/e2e/test_ngram_correctness.py
+46
@@ -292,3 +292,49 @@ def test_ngram_disable_queue(vllm_runner, common_llm_kwargs,
292 max_output_len=output_len,
293 seed=seed,
294 temperature=0.0)
295 +
296 +
297 +@pytest.mark.parametrize(
298 + "common_llm_kwargs",
299 + [{
300 + "model_name": "JackFram/llama-68m",
301 +
302 + # Skip cuda graph recording for fast test.
303 + "enforce_eager": True,
304 +
305 + # Required for spec decode.
306 + "use_v2_block_manager": True,
307 + "speculative_model": "[ngram]",
308 + "num_speculative_tokens": 5,
309 + "ngram_prompt_lookup_max": 3,
310 + }])
311 +@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
312 +@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
313 +@pytest.mark.parametrize("test_llm_kwargs",
314 + [{
315 + "speculative_disable_mqa_scorer": True,
316 + }])
317 +@pytest.mark.parametrize("batch_size", [1, 5])
318 +@pytest.mark.parametrize(
319 + "output_len",
320 + [
321 + # Use smaller output len for fast test.
322 + 32,
323 + ])
324 +@pytest.mark.parametrize("seed", [1])
325 +def test_ngram_scorer(vllm_runner, common_llm_kwargs,
326 + per_test_common_llm_kwargs, baseline_llm_kwargs,
327 + test_llm_kwargs, batch_size: int, output_len: int,
328 + seed: int):
329 + """Verify that ngram speculative decoding generates the same output
330 + with batch expansion scorer and mqa scorer.
331 + """
332 + run_equality_correctness_test(vllm_runner,
333 + common_llm_kwargs,
334 + per_test_common_llm_kwargs,
335 + baseline_llm_kwargs,
336 + test_llm_kwargs,
337 + batch_size,
338 + max_output_len=output_len,
339 + seed=seed,
340 + temperature=0.0)
tests/spec_decode/test_multi_step_worker.py
-1
@@ -173,7 +173,6 @@ def test_same_output_for_multi_step():
173 block_size,
174 num_gpu_blocks,
175 seed,
176 - model_runner_cls=TP1DraftModelRunner,
176 )
177
178 worker = create_worker(
tests/spec_decode/test_scorer.py new
+65
@@ -0,0 +1,65 @@
1 +import pytest
2 +import torch
3 +
4 +from vllm.sequence import ExecuteModelRequest
5 +from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
6 +from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
7 +from vllm.spec_decode.mqa_scorer import MQAScorer
8 +from vllm.worker.worker import Worker
9 +
10 +from .utils import create_batch, create_worker
11 +
12 +
13 +def create_proposal(batch_size: int, propose_len: int, vocab_size: int,
14 + device: str) -> SpeculativeProposals:
15 + proposal_probs = torch.rand((batch_size, propose_len, vocab_size),
16 + device=device)
17 + proposal_token_ids = torch.argmax(proposal_probs, dim=-1)
18 + proposal_lens = torch.tensor([propose_len] * batch_size, device=device)
19 + return SpeculativeProposals(proposal_token_ids, proposal_probs,
20 + proposal_lens)
21 +
22 +
23 +def assert_score_equal(score1: SpeculativeScores,
24 + score2: SpeculativeScores) -> None:
25 + assert torch.allclose(score1.probs, score2.probs)
26 + assert torch.allclose(score1.logprobs, score2.logprobs)
27 + assert torch.equal(score1.token_ids, score2.token_ids)
28 +
29 +
30 +@pytest.mark.parametrize('model_name', ['facebook/opt-125m'])
31 +@pytest.mark.parametrize('batch_size', [1, 2, 4, 8, 16])
32 +@pytest.mark.parametrize('propose_len', [1, 3, 5])
33 +@pytest.mark.parametrize('device', ['cuda'])
34 +def test_scoroer(model_name: str, batch_size: int, propose_len: int,
35 + device: str) -> None:
36 + """
37 + Compare the batch expansion scorer and mqa scorer return the same score
38 + """
39 + seed = 0
40 + block_size = 32
41 + num_gpu_blocks = 2048 // block_size
42 + scorer_worker = create_worker(Worker, model_name, block_size,
43 + num_gpu_blocks, seed)
44 + scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor = True
45 + scorer_worker.model_runner.model.sampler.\
46 + should_modify_greedy_probs_inplace = True
47 +
48 + vocab_size = scorer_worker.vocab_size
49 + proposals = create_proposal(batch_size, propose_len, vocab_size, device)
50 + seq_group_metadatalist, _, _ = create_batch(batch_size,
51 + propose_len,
52 + block_size=block_size,
53 + num_gpu_blocks=num_gpu_blocks)
54 + requests = ExecuteModelRequest(seq_group_metadatalist,
55 + num_lookahead_slots=propose_len)
56 +
57 + batch_expansion_scorer = BatchExpansionTop1Scorer(scorer_worker, device,
58 + vocab_size)
59 + batch_expansion_score = batch_expansion_scorer.score_proposals(
60 + requests, proposals)
61 +
62 + mqa_scorer = MQAScorer(scorer_worker, device, vocab_size)
63 + mqa_score = mqa_scorer.score_proposals(requests, proposals)
64 +
65 + assert_score_equal(batch_expansion_score, mqa_score)
tests/spec_decode/test_spec_decode_worker.py
+5 -4
@@ -63,10 +63,10 @@ def test_correctly_calls_draft_model(k: int, batch_size: int,
63 @pytest.mark.parametrize("acceptance_sampler_method",
64 ["rejection_sampler", "typical_acceptance_sampler"])
65 @torch.inference_mode()
66 -def test_correctly_calls_target_model(k: int, batch_size: int,
67 - acceptance_sampler_method: str):
66 +def test_batch_expansion_correctly_calls_target_model(
67 + k: int, batch_size: int, acceptance_sampler_method: str):
68 """Verify SpecDecodeWorker calls the target model with correct
69 - inputs. Everything else is mocked out.
69 + inputs with batch expansion. Everything else is mocked out.
70 """
71 draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
72 target_worker = mock_worker(use_spec=False)
@@ -82,7 +82,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int,
82 target_worker,
83 mock_spec_decode_sampler(acceptance_sampler_method),
84 disable_logprobs=False,
85 - metrics_collector=metrics_collector)
85 + metrics_collector=metrics_collector,
86 + disable_mqa_scorer=True)
87 worker.init_device()
88
89 vocab_size = 32_000
tests/spec_decode/utils.py
+16 -13
@@ -131,19 +131,22 @@ def create_seq_group_metadata_from_prompts(
131 for i, final_len in enumerate(final_prompt_lens)
132 }
133
134 - return [
135 - SequenceGroupMetadata(
136 - request_id=str(i),
137 - is_prompt=len(cont_token_ids) == 0,
138 - seq_data={
139 - i: SequenceData.from_seqs(prompt_token_ids[:],
140 - cont_token_ids[:]),
141 - },
142 - sampling_params=SamplingParams(temperature=0.0, ),
143 - block_tables={i: block_allocations[i][:]},
144 - ) for i, (prompt_token_ids,
145 - cont_token_ids) in enumerate(zip(prompts, continuations))
146 - ]
134 + seq_grou_metadata_list = []
135 + for i, (prompt_token_ids,
136 + cont_token_ids) in enumerate(zip(prompts, continuations)):
137 + data = SequenceData.from_seqs(prompt_token_ids, cont_token_ids)
138 + data.update_num_computed_tokens(
139 + len(prompt_token_ids) + len(cont_token_ids) - 1)
140 + seq_data = {i: data}
141 + seq_grou_metadata_list.append(
142 + SequenceGroupMetadata(
143 + request_id=str(i),
144 + is_prompt=len(cont_token_ids) == 0,
145 + seq_data=seq_data,
146 + sampling_params=SamplingParams(temperature=0.0),
147 + block_tables={i: block_allocations[i][:]},
148 + ))
149 + return seq_grou_metadata_list
150
151
152 def assert_logprobs_dict_allclose(
vllm/attention/backends/blocksparse_attn.py
+6
@@ -186,6 +186,12 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
186 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
187 use_cuda_graph: bool
188
189 + # Number of query tokens for each request in the batch.
190 + # Currently, we require that all requests have the same number of query
191 + # tokens during the decoding phase. When speculavie decoding is enabled,
192 + # decode_query_len might be greater than 1. In all other cases, it is 1.
193 + decode_query_len: Optional[int] = None
194 +
195 _cached_prefill_metadata: Optional[
196 "BlocksparseFlashAttentionMetadata"] = None
197 _cached_decode_metadata: Optional[
vllm/attention/backends/flash_attn.py
+29 -7
@@ -245,8 +245,15 @@ class FlashAttentionMetadata(AttentionMetadata):
245 # |-------------------- seq_len ---------------------|
246 # |-- query_len ---|
247
248 - # Maximum query length in the batch. None for decoding.
248 + # Maximum query length in the batch.
249 max_query_len: Optional[int]
250 +
251 + # Number of query tokens for each request in the batch.
252 + # Currently, we require that all requests have the same number of query
253 + # tokens during the decoding phase. When speculavie decoding is enabled,
254 + # decode_query_len might be greater than 1. In all other cases, it is 1.
255 + decode_query_len: Optional[int]
256 +
257 # Maximum sequence length among prefill batch. 0 if there are decoding
258 # requests only.
259 max_prefill_seq_len: int
@@ -303,6 +310,7 @@ class FlashAttentionMetadata(AttentionMetadata):
310 slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
311 seq_lens=self.seq_lens[:self.num_prefills],
312 seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
313 + decode_query_len=0,
314 max_query_len=self.max_query_len,
315 max_prefill_seq_len=self.max_prefill_seq_len,
316 max_decode_seq_len=0,
@@ -331,7 +339,8 @@ class FlashAttentionMetadata(AttentionMetadata):
339 slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
340 seq_lens=None,
341 seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
334 - max_query_len=None,
342 + decode_query_len=self.decode_query_len,
343 + max_query_len=self.max_query_len,
344 max_prefill_seq_len=0,
345 max_decode_seq_len=self.max_decode_seq_len,
346 query_start_loc=None,
@@ -461,9 +470,6 @@ class FlashAttentionMetadataBuilder(
470 self.num_prefill_tokens += token_len
471 self.prefill_seq_lens.append(seq_len)
472 else:
464 - assert query_len == 1, (
465 - "seq_len: {}, context_len: {}, query_len: {}".format(
466 - seq_len, context_len, query_len))
473 self.num_decode_tokens += query_len
474 self.curr_seq_lens.append(curr_seq_len)
475
@@ -518,6 +524,11 @@ class FlashAttentionMetadataBuilder(
524 use_captured_graph = cuda_graph_pad_size != -1
525
526 max_query_len = max(query_lens)
527 + decode_query_lens = query_lens[self.num_prefills:]
528 + if len(decode_query_lens) > 0:
529 + decode_query_len = max(decode_query_lens)
530 + else:
531 + decode_query_len = 1
532 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
533 max_decode_seq_len = max(self.curr_seq_lens, default=0)
534 num_decode_tokens = self.num_decode_tokens
@@ -586,6 +597,7 @@ class FlashAttentionMetadataBuilder(
597 seq_lens=seq_lens,
598 seq_lens_tensor=seq_lens_tensor,
599 max_query_len=max_query_len,
600 + decode_query_len=decode_query_len,
601 max_prefill_seq_len=max_prefill_seq_len,
602 max_decode_seq_len=max_decode_seq_len,
603 query_start_loc=query_start_loc,
@@ -786,8 +798,12 @@ class FlashAttentionImpl(AttentionImpl):
798
799 if decode_meta := attn_metadata.decode_metadata:
800 # Decoding run.
801 + _, num_head, head_dim = decode_query.shape
802 + decode_query = decode_query.reshape(-1,
803 + decode_meta.decode_query_len,
804 + num_head, head_dim)
805 decode_output = torch.ops.vllm.flash_attn_with_kvcache(
790 - decode_query.unsqueeze(1),
806 + decode_query,
807 key_cache,
808 value_cache,
809 block_table=decode_meta.block_tables,
@@ -796,7 +812,7 @@ class FlashAttentionImpl(AttentionImpl):
812 causal=True,
813 alibi_slopes=self.alibi_slopes,
814 softcap=self.logits_soft_cap,
799 - ).squeeze(1)
815 + )
816
817 if prefill_output is None:
818 assert decode_output is not None
@@ -804,5 +820,11 @@ class FlashAttentionImpl(AttentionImpl):
820 if decode_output is None:
821 assert prefill_output is not None
822 return prefill_output.view(num_prefill_tokens, hidden_size)
823 +
824 + # Chunked prefill does not work with speculative decoding.
825 + # Therefore, the query length for decode should be 1 in chunked prefill.
826 + assert decode_meta is not None
827 + assert decode_meta.decode_query_len == 1
828 + decode_output = decode_output.squeeze(1)
829 output = torch.cat([prefill_output, decode_output], dim=0)
830 return output.view(num_tokens, hidden_size)
vllm/attention/backends/flashinfer.py
-2
@@ -595,7 +595,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
595 device = self.runner.device
596 use_captured_graph = cuda_graph_pad_size != -1
597
598 - max_query_len = max(query_lens)
598 max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
599 num_decode_tokens = self.num_decode_tokens
600
@@ -634,7 +633,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
633 dtype=torch.int,
634 device=device,
635 )
637 - assert max_query_len > 0, ("query_lens: {}".format(query_lens))
636
637 assert device is not None
638 seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
vllm/attention/backends/rocm_flash_attn.py
+8
@@ -116,9 +116,17 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
116 # Cuda-graph is currently enabled for decoding only.
117 # TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
118 use_cuda_graph: bool
119 +
120 # (batch_size,) A tensor of context lengths (tokens that are computed
121 # so far).
122 context_lens_tensor: Optional[torch.Tensor]
123 +
124 + # Number of query tokens for each request in the batch.
125 + # Currently, we require that all requests have the same number of query
126 + # tokens during the decoding phase. When speculavie decoding is enabled,
127 + # decode_query_len might be greater than 1. In all other cases, it is 1.
128 + decode_query_len: Optional[int] = None
129 +
130 _cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
131 _cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
132
vllm/attention/backends/utils.py
+2 -1
@@ -312,7 +312,8 @@ class CommonAttentionState(AttentionState):
312 slot_mapping=self._graph_slot_mapping[:batch_size],
313 seq_lens=None,
314 seq_lens_tensor=self._graph_seq_lens[:batch_size],
315 - max_query_len=None,
315 + max_query_len=1,
316 + decode_query_len=1,
317 max_prefill_seq_len=0,
318 max_decode_seq_len=self.runner.max_seq_len_to_capture,
319 query_start_loc=None,
vllm/attention/backends/xformers.py
+6
@@ -118,6 +118,12 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
118 # Maximum query length in the batch. None for decoding.
119 max_query_len: Optional[int] = None
120
121 + # Number of query tokens for each request in the batch.
122 + # Currently, we require that all requests have the same number of query
123 + # tokens during the decoding phase. When speculavie decoding is enabled,
124 + # decode_query_len might be greater than 1. In all other cases, it is 1.
125 + decode_query_len: Optional[int] = None
126 +
127 # (batch_size + 1,). The cumulative subquery lengths of the sequences in
128 # the batch, used to index into subquery. E.g., if the subquery length
129 # is [4, 6], it is [0, 4, 10].
vllm/config.py
+7
@@ -1116,6 +1116,7 @@ class SpeculativeConfig:
1116 speculative_model_quantization: Optional[str],
1117 speculative_draft_tensor_parallel_size: Optional[int],
1118 num_speculative_tokens: Optional[int],
1119 + speculative_disable_mqa_scorer: Optional[bool],
1120 speculative_max_model_len: Optional[int],
1121 enable_chunked_prefill: bool,
1122 use_v2_block_manager: bool,
@@ -1150,6 +1151,9 @@ class SpeculativeConfig:
1151 num_speculative_tokens (Optional[int]): The number of speculative
1152 tokens, if provided. Will default to the number in the draft
1153 model config if present, otherwise is required.
1154 + speculative_disable_mqa_scorer (Optional[bool]): Disable the MQA
1155 + scorer for the speculative model and fall back to batch
1156 + expansion for scoring.
1157 speculative_max_model_len (Optional[int]): The maximum model len of
1158 the speculative model. Used when testing the ability to skip
1159 speculation for some sequences.
@@ -1304,6 +1308,7 @@ class SpeculativeConfig:
1308 draft_model_config,
1309 draft_parallel_config,
1310 num_speculative_tokens,
1311 + speculative_disable_mqa_scorer,
1312 speculative_disable_by_batch_size,
1313 ngram_prompt_lookup_max,
1314 ngram_prompt_lookup_min,
@@ -1400,6 +1405,7 @@ class SpeculativeConfig:
1405 draft_model_config: ModelConfig,
1406 draft_parallel_config: ParallelConfig,
1407 num_speculative_tokens: int,
1408 + speculative_disable_mqa_scorer: Optional[bool],
1409 speculative_disable_by_batch_size: Optional[int],
1410 ngram_prompt_lookup_max: Optional[int],
1411 ngram_prompt_lookup_min: Optional[int],
@@ -1446,6 +1452,7 @@ class SpeculativeConfig:
1452 self.draft_model_config = draft_model_config
1453 self.draft_parallel_config = draft_parallel_config
1454 self.num_speculative_tokens = num_speculative_tokens
1455 + self.speculative_disable_mqa_scorer = speculative_disable_mqa_scorer
1456 self.speculative_disable_by_batch_size = \
1457 speculative_disable_by_batch_size
1458 self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0
vllm/engine/arg_utils.py
+8
@@ -162,6 +162,7 @@ class EngineArgs:
162 speculative_model_quantization: Optional[str] = None
163 speculative_draft_tensor_parallel_size: Optional[int] = None
164 num_speculative_tokens: Optional[int] = None
165 + speculative_disable_mqa_scorer: Optional[bool] = False
166 speculative_max_model_len: Optional[int] = None
167 speculative_disable_by_batch_size: Optional[int] = None
168 ngram_prompt_lookup_max: Optional[int] = None
@@ -640,6 +641,12 @@ class EngineArgs:
641 default=EngineArgs.num_speculative_tokens,
642 help='The number of speculative tokens to sample from '
643 'the draft model in speculative decoding.')
644 + parser.add_argument(
645 + '--speculative-disable-mqa-scorer',
646 + action='store_true',
647 + help=
648 + 'If set to True, the MQA scorer will be disabled in speculative '
649 + ' and fall back to batch expansion')
650 parser.add_argument(
651 '--speculative-draft-tensor-parallel-size',
652 '-spec-draft-tp',
@@ -970,6 +977,7 @@ class EngineArgs:
977 speculative_draft_tensor_parallel_size = \
978 self.speculative_draft_tensor_parallel_size,
979 num_speculative_tokens=self.num_speculative_tokens,
980 + speculative_disable_mqa_scorer=self.speculative_disable_mqa_scorer,
981 speculative_disable_by_batch_size=self.
982 speculative_disable_by_batch_size,
983 speculative_max_model_len=self.speculative_max_model_len,
vllm/engine/llm_engine.py
+14 -4
@@ -1110,6 +1110,8 @@ class LLMEngine:
1110 update_prefill_num_computed_tokens(seq_group, seq_group_meta,
1111 len(output),
1112 is_first_step_output)
1113 + elif not is_async:
1114 + seq_group.update_num_computed_tokens(1)
1115
1116 if outputs:
1117 for o in outputs:
@@ -1133,8 +1135,16 @@ class LLMEngine:
1135 else:
1136 self.output_processor.process_prompt_logprob(seq_group, output)
1137 if seq_group_meta.do_sample:
1136 - self.output_processor.process_outputs(
1138 + output_token_num = self.output_processor.process_outputs(
1139 seq_group, output, is_async)
1140 + if self.speculative_config:
1141 + # We -1 here because we always
1142 + # (w/o speculative decoding) add the number of
1143 + # computed tokens by one in the decoding phase.
1144 + # Therefore, we remove that one token that
1145 + # is already added.
1146 + seq_group.update_num_computed_tokens(output_token_num -
1147 + 1)
1148
1149 if seq_group.is_finished():
1150 finished_now.append(i)
@@ -1251,11 +1261,12 @@ class LLMEngine:
1261 # decodes after the very first step. Therefore,
1262 # we skip the update to the num_computed_tokens
1263 # here.
1254 - pass
1264 + seq_group.update_num_computed_tokens(1)
1265 else:
1266 seq_group.update_num_computed_tokens(
1267 seq_group_metadata.token_chunk_size)
1258 -
1268 + else:
1269 + seq_group.update_num_computed_tokens(1)
1270 if seq_group_metadata.do_sample:
1271 assert len(sequence_group_outputs.samples) == 1, (
1272 "Async output processor expects a single sample"
@@ -1266,7 +1277,6 @@ class LLMEngine:
1277 assert len(seq_group.seqs) == 1
1278 seq = seq_group.seqs[0]
1279 seq.append_token_id(sample.output_token, sample.logprobs)
1269 - seq_group.update_num_computed_tokens(1)
1280
1281 def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1282 """Performs one decoding iteration and returns newly generated results.
vllm/engine/output_processor/interfaces.py
+6 -2
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import Callable, List
2 +from typing import Callable, List, Optional
3
4 from vllm.config import SchedulerConfig
5 from vllm.core.scheduler import Scheduler
@@ -58,10 +58,14 @@ class SequenceGroupOutputProcessor(ABC):
58 @abstractmethod
59 def process_outputs(self, sequence_group: SequenceGroup,
60 outputs: List[SequenceGroupOutput],
61 - is_async: bool) -> None:
61 + is_async: bool) -> Optional[int]:
62 """Process new token ids for the sequence group. Handles logic such as
63 detokenization, stop checking, and freeing/forking sequences in the
64 scheduler.
65 +
66 + Return the number of new tokens generated in the sequence group.
67 + The returned value is optional because it is only used for
68 + speculative decoding mqa scorer.
69 """
70 pass
71
vllm/engine/output_processor/multi_step.py
+11 -8
@@ -1,5 +1,5 @@
1 import functools
2 -from typing import Callable, List
2 +from typing import Callable, List, Optional
3
4 from vllm.core.scheduler import Scheduler
5 from vllm.engine.output_processor.interfaces import (
@@ -69,7 +69,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
69 def process_outputs(self,
70 sequence_group: SequenceGroup,
71 outputs: List[SequenceGroupOutput],
72 - is_async: bool = False) -> None:
72 + is_async: bool = False) -> Optional[int]:
73 """Append new tokens in the outputs to sequences in the sequence group.
74
75 This only supports sequence groups of size 1. It supports greater than
@@ -84,6 +84,10 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
84 tokens from the previous step. If this is true, then
85 no tokens need to be appended since it is already done
86 externally (before the next schedule() call)
87 +
88 + Returns:
89 + The number of tokens appended to the sequence. This is optional
90 + because only speculative decode uses this return value.
91 """
92 # Sequences can be in RUNNING or FINISHED_ABORTED state
93 # once scheduled, as a sequence is moved to FINSIHED_ABORTED
@@ -106,6 +110,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
110 # was already appended, so we only need to do the rest of the
111 # postprocessor: Detokenization + stopping logic
112 self._process_decode_and_stop(seq, sequence_group.sampling_params)
113 + return None
114 else:
115 # Standard multi-step case
116
@@ -121,8 +126,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
126 ]
127 assert valid_samples
128
124 - self._process_seq_outputs(seq, valid_samples,
125 - sequence_group.sampling_params)
129 + return self._process_seq_outputs(seq, valid_samples,
130 + sequence_group.sampling_params)
131
132 def _process_decode_and_stop(self, seq: Sequence,
133 sampling_params: SamplingParams) -> None:
@@ -140,7 +145,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
145
146 def _process_seq_outputs(self, seq: Sequence,
147 valid_samples: List[SequenceOutput],
143 - sampling_params: SamplingParams) -> None:
148 + sampling_params: SamplingParams) -> int:
149 output_token_ids = [sample.output_token for sample in valid_samples]
150 output_logprobs = [sample.logprobs for sample in valid_samples]
151
@@ -148,7 +153,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
153 remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
154 len(output_token_ids))
155 if remaining_tokens < 0:
151 - valid_samples = valid_samples[:remaining_tokens]
156 output_token_ids = output_token_ids[:remaining_tokens]
157
158 # Truncate any tokens after EOS. This is required as spec decode
@@ -162,7 +166,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
166 for i in range(len(output_token_ids)):
167 if output_token_ids[i] == eos_token_id:
168 output_token_ids = output_token_ids[:i + 1]
165 - valid_samples = valid_samples[:i + 1]
169 break
170
171 # Incrementally append tokens to the sequence, as if we had only one new
@@ -173,9 +176,9 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
176 token_id=output_token_id,
177 logprobs=output_logprob,
178 )
176 - seq.data.update_num_computed_tokens(1)
179
180 self._process_decode_and_stop(seq, sampling_params)
181
182 if seq.is_finished():
183 break
184 + return len(output_token_ids)
vllm/model_executor/layers/sampler.py
+1 -1
@@ -912,7 +912,7 @@ def get_logprobs(
912 sampling_metadata: SamplingMetadata,
913 sample_results: SampleResultType,
914 ) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
915 - """Return sample lobprobs and prompt logprobs.
915 + """Return sample logprobs and prompt logprobs.
916
917 The logic consists of 3 parts.
918 - Select indices to compute logprob from, ranks of token ids, and
vllm/model_executor/sampling_metadata.py
+12 -11
@@ -146,7 +146,7 @@ class SamplingMetadata:
146 def prepare(
147 seq_group_metadata_list: List[SequenceGroupMetadata],
148 seq_lens: List[int],
149 - query_lens: Optional[List[int]],
149 + query_lens: List[int],
150 device: str,
151 pin_memory: bool,
152 generators: Optional[Dict[str, torch.Generator]] = None,
@@ -194,7 +194,7 @@ class SamplingMetadata:
194 def _prepare_seq_groups(
195 seq_group_metadata_list: List[SequenceGroupMetadata],
196 seq_lens: List[int],
197 - query_lens: Optional[List[int]],
197 + query_lens: List[int],
198 device: str,
199 generators: Optional[Dict[str, torch.Generator]] = None,
200 cache: Optional[SamplingMetadataCache] = None,
@@ -284,7 +284,8 @@ def _prepare_seq_groups(
284 else:
285 # Decode
286 prompt_logprob_len = 0
287 - sample_len = len(seq_ids) if do_sample else 0
287 + query_len = query_lens[i] if query_lens is not None else 1
288 + sample_len = len(seq_ids) * query_len if do_sample else 0
289
290 if sampling_params.seed is not None and generators is not None:
291 generator = generators.get(seq_group_metadata.request_id)
@@ -440,14 +441,14 @@ class SamplingTensors:
441
442 if seq_group.do_sample:
443 sample_lens = len(seq_group.sample_indices)
443 - assert sample_lens == len(seq_ids)
444 - temperatures += [temperature] * len(seq_ids)
445 - top_ps += [top_p] * len(seq_ids)
446 - top_ks += [top_k] * len(seq_ids)
447 - min_ps += [min_p] * len(seq_ids)
448 - presence_penalties += [p] * len(seq_ids)
449 - frequency_penalties += [f] * len(seq_ids)
450 - repetition_penalties += [r] * len(seq_ids)
444 + assert sample_lens >= len(seq_ids)
445 + temperatures += [temperature] * sample_lens
446 + top_ps += [top_p] * sample_lens
447 + top_ks += [top_k] * sample_lens
448 + min_ps += [min_p] * sample_lens
449 + presence_penalties += [p] * sample_lens
450 + frequency_penalties += [f] * sample_lens
451 + repetition_penalties += [r] * sample_lens
452
453 if do_penalties:
454 for seq_group in sampling_metadata.seq_groups:
vllm/spec_decode/batch_expansion.py
-7
@@ -12,7 +12,6 @@ from vllm.sequence import (VLLM_INVALID_TOKEN_ID, VLLM_TOKEN_ID_ARRAY_TYPE,
12 from vllm.spec_decode.interfaces import (SpeculativeProposals,
13 SpeculativeScorer, SpeculativeScores)
14 from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
15 -from vllm.worker.worker_base import WorkerBase
15
16 SeqId = int
17 TargetSeqId = int
@@ -36,12 +35,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
35 of topk/tree.
36 """
37
39 - def __init__(self, scorer_worker: WorkerBase, device: str,
40 - vocab_size: int):
41 - self._scorer_worker = scorer_worker
42 - self._device = device
43 - self._vocab_size = vocab_size
44 -
38 @nvtx_range("BatchExpansionTop1Scorer.score_proposals")
39 def score_proposals(
40 self,
vllm/spec_decode/draft_model_runner.py
-2
@@ -94,8 +94,6 @@ class TP1DraftModelRunner(ModelRunner):
94 assert seq_group.is_prompt is False # No prompt
95 assert seq_group.prompt_logprob_indices == [] # No prompt
96 assert seq_group.sample_indices == [i] # Simple
97 - assert seq_group.seq_len is None # Decode
98 - assert seq_group.query_len is None # Decode
97
98 def _gpu_advance_step(
99 self, model_input: ModelInputForGPUWithSamplingMetadata,
vllm/spec_decode/interfaces.py
+7
@@ -5,6 +5,7 @@ from typing import Optional, Set
5 import torch
6
7 from vllm.sequence import ExecuteModelRequest
8 +from vllm.worker.worker_base import WorkerBase
9
10
11 @dataclass
@@ -74,6 +75,12 @@ class SpeculativeProposer(ABC):
75
76 class SpeculativeScorer(ABC):
77
78 + def __init__(self, scorer_worker: WorkerBase, device: str,
79 + vocab_size: int):
80 + self._scorer_worker = scorer_worker
81 + self._device = device
82 + self._vocab_size = vocab_size
83 +
84 @abstractmethod
85 def score_proposals(
86 self,
vllm/spec_decode/mqa_scorer.py new
+80
@@ -0,0 +1,80 @@
1 +from vllm.sequence import (ExecuteModelRequest, SequenceData,
2 + SequenceGroupMetadata, get_all_seq_ids)
3 +from vllm.spec_decode.interfaces import (SpeculativeProposals,
4 + SpeculativeScorer, SpeculativeScores)
5 +
6 +SeqId = int
7 +TargetSeqId = int
8 +
9 +
10 +class MQAScorer(SpeculativeScorer):
11 +
12 + def score_proposals(
13 + self,
14 + execute_model_req: ExecuteModelRequest,
15 + proposals: SpeculativeProposals,
16 + ) -> SpeculativeScores:
17 + target_seq_group_metadata_list = []
18 + target_seq_id_start = max(
19 + get_all_seq_ids(execute_model_req.seq_group_metadata_list)) + 1
20 + all_proposal_tokens = proposals.proposal_token_ids.tolist()
21 + for i, seq_group_metadata in enumerate(
22 + execute_model_req.seq_group_metadata_list):
23 + seq_data_dict = seq_group_metadata.seq_data
24 + assert len(seq_data_dict) == 1
25 + seq_id = next(iter(seq_data_dict.keys()))
26 +
27 + seq_data: SequenceData = seq_data_dict[seq_id]
28 + prompt_token_ids = seq_data.get_prompt_token_ids()
29 + output_token_ids = seq_data.get_output_token_ids()
30 + proposal_token_ids = all_proposal_tokens[i]
31 + new_output_token_ids = [*output_token_ids, *proposal_token_ids]
32 +
33 + target_seq_id = target_seq_id_start + i
34 + new_seq_data = SequenceData.from_seqs(
35 + prompt_token_ids=prompt_token_ids,
36 + output_token_ids=new_output_token_ids,
37 + )
38 + new_seq_data.update_num_computed_tokens(
39 + len(prompt_token_ids) + len(output_token_ids) - 1)
40 +
41 + # Ensure that the new sequence has at least one token
42 + # because we only use mqa scorer in the decoding stage.
43 + assert len(output_token_ids) >= 1
44 + new_seq_data_dict = {target_seq_id: new_seq_data}
45 +
46 + new_seq_group_metadata = SequenceGroupMetadata(
47 + request_id=seq_group_metadata.request_id,
48 + is_prompt=seq_group_metadata.is_prompt,
49 + seq_data=new_seq_data_dict,
50 + sampling_params=seq_group_metadata.sampling_params,
51 + block_tables={
52 + target_seq_id: seq_group_metadata.block_tables[seq_id],
53 + },
54 + lora_request=None,
55 + token_chunk_size=1,
56 + )
57 + target_seq_group_metadata_list.append(new_seq_group_metadata)
58 +
59 + target_sampler_output = self._scorer_worker.execute_model(
60 + execute_model_req=execute_model_req.clone(
61 + seq_group_metadata_list=target_seq_group_metadata_list))
62 +
63 + target_sampler_output = target_sampler_output[0]
64 +
65 + bs, k = proposals.proposal_token_ids.shape
66 + all_tokens = target_sampler_output.sampled_token_ids.reshape(bs, k + 1)
67 +
68 + all_probs = target_sampler_output.sampled_token_probs.reshape(
69 + bs, k + 1, self._vocab_size)
70 + all_logprobs = target_sampler_output.logprobs.reshape(
71 + bs, k + 1, self._vocab_size)
72 +
73 + hidden_states = None
74 + if target_sampler_output.hidden_states is not None:
75 + hidden_states = target_sampler_output.hidden_states.reshape(
76 + bs, (k + 1), -1)
77 + return SpeculativeScores(probs=all_probs,
78 + token_ids=all_tokens,
79 + logprobs=all_logprobs,
80 + hidden_states=hidden_states)
vllm/spec_decode/spec_decode_worker.py
+54 -7
@@ -1,6 +1,6 @@
1 from collections import defaultdict
2 from functools import cached_property
3 -from typing import Any, Dict, List, Optional, Set, Tuple
3 +from typing import Any, Dict, List, Optional, Set, Tuple, Type
4
5 import torch
6
@@ -24,6 +24,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
24 from vllm.spec_decode.medusa_worker import MedusaWorker
25 from vllm.spec_decode.metrics import AsyncMetricsCollector
26 from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
27 +from vllm.spec_decode.mqa_scorer import MQAScorer
28 from vllm.spec_decode.multi_step_worker import MultiStepWorker
29 from vllm.spec_decode.ngram_worker import NGramWorker
30 from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
@@ -70,6 +71,7 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
71 spec_decode_worker = SpecDecodeWorker.create_worker(
72 scorer_worker=target_worker,
73 draft_worker_kwargs=draft_worker_kwargs,
74 + disable_mqa_scorer=speculative_config.speculative_disable_mqa_scorer,
75 disable_by_batch_size=speculative_config.
76 speculative_disable_by_batch_size,
77 draft_token_acceptance_method=speculative_config.
@@ -116,6 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118 cls,
119 scorer_worker: Worker,
120 draft_worker_kwargs: Dict[str, Any],
121 + disable_mqa_scorer: bool,
122 disable_by_batch_size: Optional[int],
123 draft_token_acceptance_method: str,
124 typical_acceptance_sampler_posterior_threshold: float,
@@ -173,12 +176,43 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
176 typical_acceptance_sampler_posterior_threshold,
177 posterior_alpha=typical_acceptance_sampler_posterior_alpha,
178 )
176 - logger.info("Configuring SpecDecodeWorker with sampler=%s",
177 - type(spec_decode_sampler))
179 + logger.info(
180 + "[Speculative Decoding] Configuring"
181 + " SpecDecodeWorker with sampler=%s", type(spec_decode_sampler))
182 +
183 + if not disable_mqa_scorer:
184 + if scorer_worker.model_runner.attn_backend.get_name(
185 + ) != "flash-attn":
186 + disable_mqa_scorer = True
187 + logger.info(
188 + "[Speculative Decoding] Disabling MQA scorer as the "
189 + "MQA is only available with flash attn backend.")
190 +
191 + if ngram_prompt_lookup_max > 0:
192 + disable_mqa_scorer = True
193 + logger.info(
194 + "[Speculative Decoding] Disabling MQA scorer as the "
195 + "NGramWorker does not support MQA scorer.")
196 +
197 + if "model_config" in draft_worker_kwargs and \
198 + draft_worker_kwargs["model_config"].max_model_len < \
199 + scorer_worker.model_config.max_model_len:
200 + disable_mqa_scorer = True
201 + logger.info(
202 + "[Speculative Decoding] Disabling MQA scorer as the "
203 + "draft model max_model_len is smaller than the target "
204 + "model max_model_len.")
205 +
206 + if not scorer_worker.model_runner.model_config.enforce_eager:
207 + disable_mqa_scorer = True
208 + logger.info(
209 + "[Speculative Decoding] Disabling MQA scorer as the "
210 + "target model is not running in eager mode.")
211
212 return SpecDecodeWorker(
213 proposer_worker,
214 scorer_worker,
215 + disable_mqa_scorer=disable_mqa_scorer,
216 disable_logprobs=disable_logprobs,
217 disable_log_stats=disable_log_stats,
218 disable_by_batch_size=disable_by_batch_size,
@@ -190,6 +224,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
224 proposer_worker: ProposerWorkerBase,
225 scorer_worker: WorkerBase,
226 spec_decode_sampler: SpecDecodeBaseSampler,
227 + disable_mqa_scorer: bool = False,
228 disable_logprobs: bool = False,
229 disable_log_stats: bool = False,
230 metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -211,6 +246,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
246 types of sampler namely RejectionSampler and
247 TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
248 instance of RejectionSampler or TypicalAcceptanceSampler.
249 + disable_mqa_scorer: If set to True, disable the MQA scorer and use
250 + the BatchExpansionTop1Scorer instead.
251 disable_logprobs: If set to True, token log probabilities will
252 not be output in both the draft worker and the target worker.
253 If set to False, log probabilities will be output by both.
@@ -248,6 +285,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
285 self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
286 # Lazy initialization.
287 self.scorer: SpeculativeScorer
288 + self.disable_mqa_scorer = disable_mqa_scorer
289
290 # Hidden states from target model to pass to proposer
291 # in the subsequent step.
@@ -270,10 +308,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
308 self._metrics.init_gpu_tensors(self.rank)
309 self.spec_decode_sampler.init_gpu_tensors(self.rank)
310
273 - self.scorer = BatchExpansionTop1Scorer(
274 - scorer_worker=self.scorer_worker,
275 - device=self.device,
276 - vocab_size=self._vocab_size)
311 + scorer_cls: Type[SpeculativeScorer]
312 + if self.disable_mqa_scorer:
313 + scorer_cls = BatchExpansionTop1Scorer
314 + logger.info("[Speculative Decoding] Use batch "
315 + "expansion for scoring proposals.")
316 + else:
317 + scorer_cls = MQAScorer
318 + logger.info(
319 + "[Speculative Decoding] Use MQA scorer for scoring proposals.")
320 +
321 + self.scorer = scorer_cls(scorer_worker=self.scorer_worker,
322 + device=self.device,
323 + vocab_size=self._vocab_size)
324
325 self._configure_model_sampler_for_spec_decode()
326
vllm/worker/model_runner.py
+10 -27
@@ -468,43 +468,26 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
468
469 # Compute context length (the number of tokens that are
470 # already computed) and sequence length (total number of tokens).
471 +
472 seq_len = seq_data.get_len()
473 if inter_data.is_prompt:
474 context_len = seq_data.get_num_computed_tokens()
474 - else:
475 - # get_num_computed_tokens is incorrect for spec decoding.
476 - # So, we should have a special logic here.
477 - # TODO(sang): Fix it.
475 + seq_len = min(seq_len, context_len + token_chunk_size)
476 + elif self.runner.scheduler_config.is_multi_step or \
477 + self.runner.model_config.is_encoder_decoder_model:
478 context_len = seq_len - 1
479 - seq_len = min(seq_len, context_len + token_chunk_size)
479 + else:
480 + context_len = seq_data.get_num_computed_tokens()
481
482 # Compute tokens.
482 - if inter_data.is_prompt:
483 - tokens = seq_data.get_token_ids()
484 - if context_len != 0 or seq_len < len(tokens):
485 - tokens = tokens[context_len:seq_len]
486 - else:
487 - # Optimization. get_token_ids requires the entire copy of
488 - # tokens.
489 - tokens = seq_data.get_last_token_id()
483 + tokens = seq_data.get_token_ids()[context_len:seq_len]
484
485 inter_data.seq_lens[seq_idx] = seq_len
486 inter_data.orig_seq_lens[seq_idx] = seq_len
487 inter_data.context_lens[seq_idx] = context_len
494 -
495 - if isinstance(tokens, list):
496 - inter_data.input_tokens[seq_idx].extend(tokens)
497 - else:
498 - inter_data.input_tokens[seq_idx].append(tokens)
499 -
500 - if (seq_len - context_len) == 1:
501 - inter_data.input_positions[seq_idx].append(seq_len - 1)
502 - else:
503 - inter_data.input_positions[seq_idx].extend(
504 - range(context_len, seq_len))
505 -
506 - inter_data.query_lens[
507 - seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
488 + inter_data.input_tokens[seq_idx].extend(tokens)
489 + inter_data.input_positions[seq_idx].extend(range(context_len, seq_len))
490 + inter_data.query_lens[seq_idx] = seq_len - context_len
491
492 if seq_data.mrope_position_delta is not None:
493 if inter_data.mrope_input_positions is None: