A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 10:45 UTC
[Spec Decode] (1/2) Remove batch expansion (#8839)
Lily Liu committed
Oct 1, 2024 at 16:04 UTC
15702038642192002cd8973cf8948751b750fd07
29 files changed
+531
-99
.buildkite/test-pipeline.yaml
+1
-1
@@ -208,7 +208,7 @@ steps:
208
- tests/spec_decode
209
commands:
210
- pytest -v -s spec_decode/e2e/test_multistep_correctness.py
211
- - pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
211
+ - VLLM_ATTENTION_BACKEND=FLASH_ATTN pytest -v -s spec_decode --ignore=spec_decode/e2e/test_multistep_correctness.py
212
213
- label: LoRA Test %N # 15min each
214
mirror_hardwares: [amd]
tests/samplers/test_sampler.py
+1
-1
@@ -434,7 +434,7 @@ def test_sampler_min_tokens_penalty(seed: int, device: str):
434
sampling_metadata = SamplingMetadata.prepare(
435
seq_group_metadata_list,
436
seq_lens=seq_lens if seq_lens else None,
437
- query_lens=seq_lens if seq_lens else None,
437
+ query_lens=seq_lens if seq_lens else [1] * batch_size,
438
device=device,
439
pin_memory=is_pin_memory_available())
440
# the logits tensor is modified in-place by the sampler
tests/spec_decode/e2e/test_integration.py
+44
@@ -102,3 +102,47 @@ def test_speculative_model_quantization_config(vllm_runner, common_llm_kwargs,
102
max_output_len=32,
103
seed=seed,
104
temperature=0.0)
105
+
106
+
107
+@pytest.mark.parametrize(
108
+ "common_llm_kwargs",
109
+ [{
110
+ "model_name": MAIN_MODEL,
111
+
112
+ # Skip cuda graph recording for fast test.
113
+ "enforce_eager": True,
114
+
115
+ # Required for spec decode.
116
+ "use_v2_block_manager": True,
117
+ "speculative_model": "JackFram/llama-68m",
118
+ "num_speculative_tokens": 3,
119
+ }])
120
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
121
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
122
+@pytest.mark.parametrize("test_llm_kwargs",
123
+ [{
124
+ "speculative_disable_mqa_scorer": True,
125
+ }])
126
+@pytest.mark.parametrize("batch_size", [1, 5])
127
+@pytest.mark.parametrize(
128
+ "output_len",
129
+ [
130
+ # Use smaller output len for fast test.
131
+ 32,
132
+ ])
133
+@pytest.mark.parametrize("seed", [1])
134
+def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
135
+ baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
136
+ output_len: int, seed: int):
137
+ """Verify that ngram speculative decoding generates the same output
138
+ with batch expansion scorer and mqa scorer.
139
+ """
140
+ run_equality_correctness_test(vllm_runner,
141
+ common_llm_kwargs,
142
+ per_test_common_llm_kwargs,
143
+ baseline_llm_kwargs,
144
+ test_llm_kwargs,
145
+ batch_size,
146
+ max_output_len=output_len,
147
+ seed=seed,
148
+ temperature=0.0)
tests/spec_decode/e2e/test_medusa_correctness.py
+49
@@ -350,6 +350,55 @@ def test_medusa_disable_queue(vllm_runner, common_llm_kwargs,
350
temperature=0.0)
351
352
353
+@pytest.mark.parametrize(
354
+ "common_llm_kwargs",
355
+ [{
356
+ # Skip cuda graph recording for fast test.
357
+ "enforce_eager": True,
358
+
359
+ # Required for spec decode.
360
+ "use_v2_block_manager": True,
361
+
362
+ # Precision
363
+ "dtype": PRECISION,
364
+
365
+ # Main model
366
+ "model_name": MAIN_MODEL,
367
+ "speculative_model": SPEC_MODEL,
368
+ "num_speculative_tokens": MAX_SPEC_TOKENS,
369
+ "speculative_disable_by_batch_size": 4
370
+ }])
371
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
372
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
373
+@pytest.mark.parametrize("test_llm_kwargs",
374
+ [{
375
+ "speculative_disable_mqa_scorer": True,
376
+ }])
377
+@pytest.mark.parametrize("batch_size", [1, 5])
378
+@pytest.mark.parametrize(
379
+ "output_len",
380
+ [
381
+ # Use smaller output len for fast test.
382
+ 32,
383
+ ])
384
+@pytest.mark.parametrize("seed", [1])
385
+def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
386
+ baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
387
+ output_len: int, seed: int):
388
+ """Verify that speculative decoding generates the same output
389
+ with batch expansion scorer and mqa scorer.
390
+ """
391
+ run_equality_correctness_test(vllm_runner,
392
+ common_llm_kwargs,
393
+ per_test_common_llm_kwargs,
394
+ baseline_llm_kwargs,
395
+ test_llm_kwargs,
396
+ batch_size,
397
+ max_output_len=output_len,
398
+ seed=seed,
399
+ temperature=0.0)
400
+
401
+
402
if __name__ == "__main__":
403
import pytest
404
pytest.main([__file__])
tests/spec_decode/e2e/test_mlp_correctness.py
+43
@@ -460,3 +460,46 @@ def test_mlp_disable_queue(vllm_runner, common_llm_kwargs,
460
max_output_len=output_len,
461
seed=seed,
462
temperature=0.0)
463
+
464
+
465
+@pytest.mark.parametrize(
466
+ "common_llm_kwargs",
467
+ [{
468
+ "model_name": MAIN_MODEL,
469
+
470
+ # Skip cuda graph recording for fast test.
471
+ "enforce_eager": True,
472
+
473
+ # Required for spec decode.
474
+ "use_v2_block_manager": True,
475
+ "speculative_model": SPEC_MODEL,
476
+ }])
477
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
478
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
479
+@pytest.mark.parametrize("test_llm_kwargs",
480
+ [{
481
+ "speculative_disable_mqa_scorer": True,
482
+ }])
483
+@pytest.mark.parametrize("batch_size", [1, 5])
484
+@pytest.mark.parametrize(
485
+ "output_len",
486
+ [
487
+ # Use smaller output len for fast test.
488
+ 32,
489
+ ])
490
+@pytest.mark.parametrize("seed", [1])
491
+def test_mqa_scorer(vllm_runner, common_llm_kwargs, per_test_common_llm_kwargs,
492
+ baseline_llm_kwargs, test_llm_kwargs, batch_size: int,
493
+ output_len: int, seed: int):
494
+ """Verify that speculative decoding generates the same output
495
+ with batch expansion scorer and mqa scorer.
496
+ """
497
+ run_equality_correctness_test(vllm_runner,
498
+ common_llm_kwargs,
499
+ per_test_common_llm_kwargs,
500
+ baseline_llm_kwargs,
501
+ test_llm_kwargs,
502
+ batch_size,
503
+ max_output_len=output_len,
504
+ seed=seed,
505
+ temperature=0.0)
tests/spec_decode/e2e/test_ngram_correctness.py
+46
@@ -292,3 +292,49 @@ def test_ngram_disable_queue(vllm_runner, common_llm_kwargs,
292
max_output_len=output_len,
293
seed=seed,
294
temperature=0.0)
295
+
296
+
297
+@pytest.mark.parametrize(
298
+ "common_llm_kwargs",
299
+ [{
300
+ "model_name": "JackFram/llama-68m",
301
+
302
+ # Skip cuda graph recording for fast test.
303
+ "enforce_eager": True,
304
+
305
+ # Required for spec decode.
306
+ "use_v2_block_manager": True,
307
+ "speculative_model": "[ngram]",
308
+ "num_speculative_tokens": 5,
309
+ "ngram_prompt_lookup_max": 3,
310
+ }])
311
+@pytest.mark.parametrize("per_test_common_llm_kwargs", [{}])
312
+@pytest.mark.parametrize("baseline_llm_kwargs", [{}])
313
+@pytest.mark.parametrize("test_llm_kwargs",
314
+ [{
315
+ "speculative_disable_mqa_scorer": True,
316
+ }])
317
+@pytest.mark.parametrize("batch_size", [1, 5])
318
+@pytest.mark.parametrize(
319
+ "output_len",
320
+ [
321
+ # Use smaller output len for fast test.
322
+ 32,
323
+ ])
324
+@pytest.mark.parametrize("seed", [1])
325
+def test_ngram_scorer(vllm_runner, common_llm_kwargs,
326
+ per_test_common_llm_kwargs, baseline_llm_kwargs,
327
+ test_llm_kwargs, batch_size: int, output_len: int,
328
+ seed: int):
329
+ """Verify that ngram speculative decoding generates the same output
330
+ with batch expansion scorer and mqa scorer.
331
+ """
332
+ run_equality_correctness_test(vllm_runner,
333
+ common_llm_kwargs,
334
+ per_test_common_llm_kwargs,
335
+ baseline_llm_kwargs,
336
+ test_llm_kwargs,
337
+ batch_size,
338
+ max_output_len=output_len,
339
+ seed=seed,
340
+ temperature=0.0)
tests/spec_decode/test_multi_step_worker.py
-1
@@ -173,7 +173,6 @@ def test_same_output_for_multi_step():
173
block_size,
174
num_gpu_blocks,
175
seed,
176
- model_runner_cls=TP1DraftModelRunner,
176
)
177
178
worker = create_worker(
tests/spec_decode/test_scorer.py
new
+65
@@ -0,0 +1,65 @@
1
+import pytest
2
+import torch
3
+
4
+from vllm.sequence import ExecuteModelRequest
5
+from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
6
+from vllm.spec_decode.interfaces import SpeculativeProposals, SpeculativeScores
7
+from vllm.spec_decode.mqa_scorer import MQAScorer
8
+from vllm.worker.worker import Worker
9
+
10
+from .utils import create_batch, create_worker
11
+
12
+
13
+def create_proposal(batch_size: int, propose_len: int, vocab_size: int,
14
+ device: str) -> SpeculativeProposals:
15
+ proposal_probs = torch.rand((batch_size, propose_len, vocab_size),
16
+ device=device)
17
+ proposal_token_ids = torch.argmax(proposal_probs, dim=-1)
18
+ proposal_lens = torch.tensor([propose_len] * batch_size, device=device)
19
+ return SpeculativeProposals(proposal_token_ids, proposal_probs,
20
+ proposal_lens)
21
+
22
+
23
+def assert_score_equal(score1: SpeculativeScores,
24
+ score2: SpeculativeScores) -> None:
25
+ assert torch.allclose(score1.probs, score2.probs)
26
+ assert torch.allclose(score1.logprobs, score2.logprobs)
27
+ assert torch.equal(score1.token_ids, score2.token_ids)
28
+
29
+
30
+@pytest.mark.parametrize('model_name', ['facebook/opt-125m'])
31
+@pytest.mark.parametrize('batch_size', [1, 2, 4, 8, 16])
32
+@pytest.mark.parametrize('propose_len', [1, 3, 5])
33
+@pytest.mark.parametrize('device', ['cuda'])
34
+def test_scoroer(model_name: str, batch_size: int, propose_len: int,
35
+ device: str) -> None:
36
+ """
37
+ Compare the batch expansion scorer and mqa scorer return the same score
38
+ """
39
+ seed = 0
40
+ block_size = 32
41
+ num_gpu_blocks = 2048 // block_size
42
+ scorer_worker = create_worker(Worker, model_name, block_size,
43
+ num_gpu_blocks, seed)
44
+ scorer_worker.model_runner.model.sampler.include_gpu_probs_tensor = True
45
+ scorer_worker.model_runner.model.sampler.\
46
+ should_modify_greedy_probs_inplace = True
47
+
48
+ vocab_size = scorer_worker.vocab_size
49
+ proposals = create_proposal(batch_size, propose_len, vocab_size, device)
50
+ seq_group_metadatalist, _, _ = create_batch(batch_size,
51
+ propose_len,
52
+ block_size=block_size,
53
+ num_gpu_blocks=num_gpu_blocks)
54
+ requests = ExecuteModelRequest(seq_group_metadatalist,
55
+ num_lookahead_slots=propose_len)
56
+
57
+ batch_expansion_scorer = BatchExpansionTop1Scorer(scorer_worker, device,
58
+ vocab_size)
59
+ batch_expansion_score = batch_expansion_scorer.score_proposals(
60
+ requests, proposals)
61
+
62
+ mqa_scorer = MQAScorer(scorer_worker, device, vocab_size)
63
+ mqa_score = mqa_scorer.score_proposals(requests, proposals)
64
+
65
+ assert_score_equal(batch_expansion_score, mqa_score)
tests/spec_decode/test_spec_decode_worker.py
+5
-4
@@ -63,10 +63,10 @@ def test_correctly_calls_draft_model(k: int, batch_size: int,
63
@pytest.mark.parametrize("acceptance_sampler_method",
64
["rejection_sampler", "typical_acceptance_sampler"])
65
@torch.inference_mode()
66
-def test_correctly_calls_target_model(k: int, batch_size: int,
67
- acceptance_sampler_method: str):
66
+def test_batch_expansion_correctly_calls_target_model(
67
+ k: int, batch_size: int, acceptance_sampler_method: str):
68
"""Verify SpecDecodeWorker calls the target model with correct
69
- inputs. Everything else is mocked out.
69
+ inputs with batch expansion. Everything else is mocked out.
70
"""
71
draft_worker = mock_worker(cls=MultiStepWorker, use_spec=False)
72
target_worker = mock_worker(use_spec=False)
@@ -82,7 +82,8 @@ def test_correctly_calls_target_model(k: int, batch_size: int,
82
target_worker,
83
mock_spec_decode_sampler(acceptance_sampler_method),
84
disable_logprobs=False,
85
- metrics_collector=metrics_collector)
85
+ metrics_collector=metrics_collector,
86
+ disable_mqa_scorer=True)
87
worker.init_device()
88
89
vocab_size = 32_000
tests/spec_decode/utils.py
+16
-13
@@ -131,19 +131,22 @@ def create_seq_group_metadata_from_prompts(
131
for i, final_len in enumerate(final_prompt_lens)
132
}
133
134
- return [
135
- SequenceGroupMetadata(
136
- request_id=str(i),
137
- is_prompt=len(cont_token_ids) == 0,
138
- seq_data={
139
- i: SequenceData.from_seqs(prompt_token_ids[:],
140
- cont_token_ids[:]),
141
- },
142
- sampling_params=SamplingParams(temperature=0.0, ),
143
- block_tables={i: block_allocations[i][:]},
144
- ) for i, (prompt_token_ids,
145
- cont_token_ids) in enumerate(zip(prompts, continuations))
146
- ]
134
+ seq_grou_metadata_list = []
135
+ for i, (prompt_token_ids,
136
+ cont_token_ids) in enumerate(zip(prompts, continuations)):
137
+ data = SequenceData.from_seqs(prompt_token_ids, cont_token_ids)
138
+ data.update_num_computed_tokens(
139
+ len(prompt_token_ids) + len(cont_token_ids) - 1)
140
+ seq_data = {i: data}
141
+ seq_grou_metadata_list.append(
142
+ SequenceGroupMetadata(
143
+ request_id=str(i),
144
+ is_prompt=len(cont_token_ids) == 0,
145
+ seq_data=seq_data,
146
+ sampling_params=SamplingParams(temperature=0.0),
147
+ block_tables={i: block_allocations[i][:]},
148
+ ))
149
+ return seq_grou_metadata_list
150
151
152
def assert_logprobs_dict_allclose(
vllm/attention/backends/blocksparse_attn.py
+6
@@ -186,6 +186,12 @@ class BlocksparseFlashAttentionMetadata(AttentionMetadata):
186
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
187
use_cuda_graph: bool
188
189
+ # Number of query tokens for each request in the batch.
190
+ # Currently, we require that all requests have the same number of query
191
+ # tokens during the decoding phase. When speculavie decoding is enabled,
192
+ # decode_query_len might be greater than 1. In all other cases, it is 1.
193
+ decode_query_len: Optional[int] = None
194
+
195
_cached_prefill_metadata: Optional[
196
"BlocksparseFlashAttentionMetadata"] = None
197
_cached_decode_metadata: Optional[
vllm/attention/backends/flash_attn.py
+29
-7
@@ -245,8 +245,15 @@ class FlashAttentionMetadata(AttentionMetadata):
245
# |-------------------- seq_len ---------------------|
246
# |-- query_len ---|
247
248
- # Maximum query length in the batch. None for decoding.
248
+ # Maximum query length in the batch.
249
max_query_len: Optional[int]
250
+
251
+ # Number of query tokens for each request in the batch.
252
+ # Currently, we require that all requests have the same number of query
253
+ # tokens during the decoding phase. When speculavie decoding is enabled,
254
+ # decode_query_len might be greater than 1. In all other cases, it is 1.
255
+ decode_query_len: Optional[int]
256
+
257
# Maximum sequence length among prefill batch. 0 if there are decoding
258
# requests only.
259
max_prefill_seq_len: int
@@ -303,6 +310,7 @@ class FlashAttentionMetadata(AttentionMetadata):
310
slot_mapping=self.slot_mapping[:self.num_prefill_tokens],
311
seq_lens=self.seq_lens[:self.num_prefills],
312
seq_lens_tensor=self.seq_lens_tensor[:self.num_prefills],
313
+ decode_query_len=0,
314
max_query_len=self.max_query_len,
315
max_prefill_seq_len=self.max_prefill_seq_len,
316
max_decode_seq_len=0,
@@ -331,7 +339,8 @@ class FlashAttentionMetadata(AttentionMetadata):
339
slot_mapping=self.slot_mapping[self.num_prefill_tokens:],
340
seq_lens=None,
341
seq_lens_tensor=self.seq_lens_tensor[self.num_prefills:],
334
- max_query_len=None,
342
+ decode_query_len=self.decode_query_len,
343
+ max_query_len=self.max_query_len,
344
max_prefill_seq_len=0,
345
max_decode_seq_len=self.max_decode_seq_len,
346
query_start_loc=None,
@@ -461,9 +470,6 @@ class FlashAttentionMetadataBuilder(
470
self.num_prefill_tokens += token_len
471
self.prefill_seq_lens.append(seq_len)
472
else:
464
- assert query_len == 1, (
465
- "seq_len: {}, context_len: {}, query_len: {}".format(
466
- seq_len, context_len, query_len))
473
self.num_decode_tokens += query_len
474
self.curr_seq_lens.append(curr_seq_len)
475
@@ -518,6 +524,11 @@ class FlashAttentionMetadataBuilder(
524
use_captured_graph = cuda_graph_pad_size != -1
525
526
max_query_len = max(query_lens)
527
+ decode_query_lens = query_lens[self.num_prefills:]
528
+ if len(decode_query_lens) > 0:
529
+ decode_query_len = max(decode_query_lens)
530
+ else:
531
+ decode_query_len = 1
532
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
533
max_decode_seq_len = max(self.curr_seq_lens, default=0)
534
num_decode_tokens = self.num_decode_tokens
@@ -586,6 +597,7 @@ class FlashAttentionMetadataBuilder(
597
seq_lens=seq_lens,
598
seq_lens_tensor=seq_lens_tensor,
599
max_query_len=max_query_len,
600
+ decode_query_len=decode_query_len,
601
max_prefill_seq_len=max_prefill_seq_len,
602
max_decode_seq_len=max_decode_seq_len,
603
query_start_loc=query_start_loc,
@@ -786,8 +798,12 @@ class FlashAttentionImpl(AttentionImpl):
798
799
if decode_meta := attn_metadata.decode_metadata:
800
# Decoding run.
801
+ _, num_head, head_dim = decode_query.shape
802
+ decode_query = decode_query.reshape(-1,
803
+ decode_meta.decode_query_len,
804
+ num_head, head_dim)
805
decode_output = torch.ops.vllm.flash_attn_with_kvcache(
790
- decode_query.unsqueeze(1),
806
+ decode_query,
807
key_cache,
808
value_cache,
809
block_table=decode_meta.block_tables,
@@ -796,7 +812,7 @@ class FlashAttentionImpl(AttentionImpl):
812
causal=True,
813
alibi_slopes=self.alibi_slopes,
814
softcap=self.logits_soft_cap,
799
- ).squeeze(1)
815
+ )
816
817
if prefill_output is None:
818
assert decode_output is not None
@@ -804,5 +820,11 @@ class FlashAttentionImpl(AttentionImpl):
820
if decode_output is None:
821
assert prefill_output is not None
822
return prefill_output.view(num_prefill_tokens, hidden_size)
823
+
824
+ # Chunked prefill does not work with speculative decoding.
825
+ # Therefore, the query length for decode should be 1 in chunked prefill.
826
+ assert decode_meta is not None
827
+ assert decode_meta.decode_query_len == 1
828
+ decode_output = decode_output.squeeze(1)
829
output = torch.cat([prefill_output, decode_output], dim=0)
830
return output.view(num_tokens, hidden_size)
vllm/attention/backends/flashinfer.py
-2
@@ -595,7 +595,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
595
device = self.runner.device
596
use_captured_graph = cuda_graph_pad_size != -1
597
598
- max_query_len = max(query_lens)
598
max_prefill_seq_len = max(self.prefill_seq_lens, default=0)
599
num_decode_tokens = self.num_decode_tokens
600
@@ -634,7 +633,6 @@ class FlashInferMetadataBuilder(AttentionMetadataBuilder[FlashInferMetadata]):
633
dtype=torch.int,
634
device=device,
635
)
637
- assert max_query_len > 0, ("query_lens: {}".format(query_lens))
636
637
assert device is not None
638
seq_lens_tensor = async_tensor_h2d(seq_lens, torch.int, device,
vllm/attention/backends/rocm_flash_attn.py
+8
@@ -116,9 +116,17 @@ class ROCmFlashAttentionMetadata(AttentionMetadata, PagedAttentionMetadata):
116
# Cuda-graph is currently enabled for decoding only.
117
# TODO(woosuk): Move `use_cuda_graph` out since it's unrelated to attention.
118
use_cuda_graph: bool
119
+
120
# (batch_size,) A tensor of context lengths (tokens that are computed
121
# so far).
122
context_lens_tensor: Optional[torch.Tensor]
123
+
124
+ # Number of query tokens for each request in the batch.
125
+ # Currently, we require that all requests have the same number of query
126
+ # tokens during the decoding phase. When speculavie decoding is enabled,
127
+ # decode_query_len might be greater than 1. In all other cases, it is 1.
128
+ decode_query_len: Optional[int] = None
129
+
130
_cached_prefill_metadata: Optional["ROCmFlashAttentionMetadata"] = None
131
_cached_decode_metadata: Optional["ROCmFlashAttentionMetadata"] = None
132
vllm/attention/backends/utils.py
+2
-1
@@ -312,7 +312,8 @@ class CommonAttentionState(AttentionState):
312
slot_mapping=self._graph_slot_mapping[:batch_size],
313
seq_lens=None,
314
seq_lens_tensor=self._graph_seq_lens[:batch_size],
315
- max_query_len=None,
315
+ max_query_len=1,
316
+ decode_query_len=1,
317
max_prefill_seq_len=0,
318
max_decode_seq_len=self.runner.max_seq_len_to_capture,
319
query_start_loc=None,
vllm/attention/backends/xformers.py
+6
@@ -118,6 +118,12 @@ class XFormersMetadata(AttentionMetadata, PagedAttentionMetadata):
118
# Maximum query length in the batch. None for decoding.
119
max_query_len: Optional[int] = None
120
121
+ # Number of query tokens for each request in the batch.
122
+ # Currently, we require that all requests have the same number of query
123
+ # tokens during the decoding phase. When speculavie decoding is enabled,
124
+ # decode_query_len might be greater than 1. In all other cases, it is 1.
125
+ decode_query_len: Optional[int] = None
126
+
127
# (batch_size + 1,). The cumulative subquery lengths of the sequences in
128
# the batch, used to index into subquery. E.g., if the subquery length
129
# is [4, 6], it is [0, 4, 10].
vllm/config.py
+7
@@ -1116,6 +1116,7 @@ class SpeculativeConfig:
1116
speculative_model_quantization: Optional[str],
1117
speculative_draft_tensor_parallel_size: Optional[int],
1118
num_speculative_tokens: Optional[int],
1119
+ speculative_disable_mqa_scorer: Optional[bool],
1120
speculative_max_model_len: Optional[int],
1121
enable_chunked_prefill: bool,
1122
use_v2_block_manager: bool,
@@ -1150,6 +1151,9 @@ class SpeculativeConfig:
1151
num_speculative_tokens (Optional[int]): The number of speculative
1152
tokens, if provided. Will default to the number in the draft
1153
model config if present, otherwise is required.
1154
+ speculative_disable_mqa_scorer (Optional[bool]): Disable the MQA
1155
+ scorer for the speculative model and fall back to batch
1156
+ expansion for scoring.
1157
speculative_max_model_len (Optional[int]): The maximum model len of
1158
the speculative model. Used when testing the ability to skip
1159
speculation for some sequences.
@@ -1304,6 +1308,7 @@ class SpeculativeConfig:
1308
draft_model_config,
1309
draft_parallel_config,
1310
num_speculative_tokens,
1311
+ speculative_disable_mqa_scorer,
1312
speculative_disable_by_batch_size,
1313
ngram_prompt_lookup_max,
1314
ngram_prompt_lookup_min,
@@ -1400,6 +1405,7 @@ class SpeculativeConfig:
1405
draft_model_config: ModelConfig,
1406
draft_parallel_config: ParallelConfig,
1407
num_speculative_tokens: int,
1408
+ speculative_disable_mqa_scorer: Optional[bool],
1409
speculative_disable_by_batch_size: Optional[int],
1410
ngram_prompt_lookup_max: Optional[int],
1411
ngram_prompt_lookup_min: Optional[int],
@@ -1446,6 +1452,7 @@ class SpeculativeConfig:
1452
self.draft_model_config = draft_model_config
1453
self.draft_parallel_config = draft_parallel_config
1454
self.num_speculative_tokens = num_speculative_tokens
1455
+ self.speculative_disable_mqa_scorer = speculative_disable_mqa_scorer
1456
self.speculative_disable_by_batch_size = \
1457
speculative_disable_by_batch_size
1458
self.ngram_prompt_lookup_max = ngram_prompt_lookup_max or 0
vllm/engine/arg_utils.py
+8
@@ -162,6 +162,7 @@ class EngineArgs:
162
speculative_model_quantization: Optional[str] = None
163
speculative_draft_tensor_parallel_size: Optional[int] = None
164
num_speculative_tokens: Optional[int] = None
165
+ speculative_disable_mqa_scorer: Optional[bool] = False
166
speculative_max_model_len: Optional[int] = None
167
speculative_disable_by_batch_size: Optional[int] = None
168
ngram_prompt_lookup_max: Optional[int] = None
@@ -640,6 +641,12 @@ class EngineArgs:
641
default=EngineArgs.num_speculative_tokens,
642
help='The number of speculative tokens to sample from '
643
'the draft model in speculative decoding.')
644
+ parser.add_argument(
645
+ '--speculative-disable-mqa-scorer',
646
+ action='store_true',
647
+ help=
648
+ 'If set to True, the MQA scorer will be disabled in speculative '
649
+ ' and fall back to batch expansion')
650
parser.add_argument(
651
'--speculative-draft-tensor-parallel-size',
652
'-spec-draft-tp',
@@ -970,6 +977,7 @@ class EngineArgs:
977
speculative_draft_tensor_parallel_size = \
978
self.speculative_draft_tensor_parallel_size,
979
num_speculative_tokens=self.num_speculative_tokens,
980
+ speculative_disable_mqa_scorer=self.speculative_disable_mqa_scorer,
981
speculative_disable_by_batch_size=self.
982
speculative_disable_by_batch_size,
983
speculative_max_model_len=self.speculative_max_model_len,
vllm/engine/llm_engine.py
+14
-4
@@ -1110,6 +1110,8 @@ class LLMEngine:
1110
update_prefill_num_computed_tokens(seq_group, seq_group_meta,
1111
len(output),
1112
is_first_step_output)
1113
+ elif not is_async:
1114
+ seq_group.update_num_computed_tokens(1)
1115
1116
if outputs:
1117
for o in outputs:
@@ -1133,8 +1135,16 @@ class LLMEngine:
1135
else:
1136
self.output_processor.process_prompt_logprob(seq_group, output)
1137
if seq_group_meta.do_sample:
1136
- self.output_processor.process_outputs(
1138
+ output_token_num = self.output_processor.process_outputs(
1139
seq_group, output, is_async)
1140
+ if self.speculative_config:
1141
+ # We -1 here because we always
1142
+ # (w/o speculative decoding) add the number of
1143
+ # computed tokens by one in the decoding phase.
1144
+ # Therefore, we remove that one token that
1145
+ # is already added.
1146
+ seq_group.update_num_computed_tokens(output_token_num -
1147
+ 1)
1148
1149
if seq_group.is_finished():
1150
finished_now.append(i)
@@ -1251,11 +1261,12 @@ class LLMEngine:
1261
# decodes after the very first step. Therefore,
1262
# we skip the update to the num_computed_tokens
1263
# here.
1254
- pass
1264
+ seq_group.update_num_computed_tokens(1)
1265
else:
1266
seq_group.update_num_computed_tokens(
1267
seq_group_metadata.token_chunk_size)
1258
-
1268
+ else:
1269
+ seq_group.update_num_computed_tokens(1)
1270
if seq_group_metadata.do_sample:
1271
assert len(sequence_group_outputs.samples) == 1, (
1272
"Async output processor expects a single sample"
@@ -1266,7 +1277,6 @@ class LLMEngine:
1277
assert len(seq_group.seqs) == 1
1278
seq = seq_group.seqs[0]
1279
seq.append_token_id(sample.output_token, sample.logprobs)
1269
- seq_group.update_num_computed_tokens(1)
1280
1281
def step(self) -> List[Union[RequestOutput, EmbeddingRequestOutput]]:
1282
"""Performs one decoding iteration and returns newly generated results.
vllm/engine/output_processor/interfaces.py
+6
-2
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import Callable, List
2
+from typing import Callable, List, Optional
3
4
from vllm.config import SchedulerConfig
5
from vllm.core.scheduler import Scheduler
@@ -58,10 +58,14 @@ class SequenceGroupOutputProcessor(ABC):
58
@abstractmethod
59
def process_outputs(self, sequence_group: SequenceGroup,
60
outputs: List[SequenceGroupOutput],
61
- is_async: bool) -> None:
61
+ is_async: bool) -> Optional[int]:
62
"""Process new token ids for the sequence group. Handles logic such as
63
detokenization, stop checking, and freeing/forking sequences in the
64
scheduler.
65
+
66
+ Return the number of new tokens generated in the sequence group.
67
+ The returned value is optional because it is only used for
68
+ speculative decoding mqa scorer.
69
"""
70
pass
71
vllm/engine/output_processor/multi_step.py
+11
-8
@@ -1,5 +1,5 @@
1
import functools
2
-from typing import Callable, List
2
+from typing import Callable, List, Optional
3
4
from vllm.core.scheduler import Scheduler
5
from vllm.engine.output_processor.interfaces import (
@@ -69,7 +69,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
69
def process_outputs(self,
70
sequence_group: SequenceGroup,
71
outputs: List[SequenceGroupOutput],
72
- is_async: bool = False) -> None:
72
+ is_async: bool = False) -> Optional[int]:
73
"""Append new tokens in the outputs to sequences in the sequence group.
74
75
This only supports sequence groups of size 1. It supports greater than
@@ -84,6 +84,10 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
84
tokens from the previous step. If this is true, then
85
no tokens need to be appended since it is already done
86
externally (before the next schedule() call)
87
+
88
+ Returns:
89
+ The number of tokens appended to the sequence. This is optional
90
+ because only speculative decode uses this return value.
91
"""
92
# Sequences can be in RUNNING or FINISHED_ABORTED state
93
# once scheduled, as a sequence is moved to FINSIHED_ABORTED
@@ -106,6 +110,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
110
# was already appended, so we only need to do the rest of the
111
# postprocessor: Detokenization + stopping logic
112
self._process_decode_and_stop(seq, sequence_group.sampling_params)
113
+ return None
114
else:
115
# Standard multi-step case
116
@@ -121,8 +126,8 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
126
]
127
assert valid_samples
128
124
- self._process_seq_outputs(seq, valid_samples,
125
- sequence_group.sampling_params)
129
+ return self._process_seq_outputs(seq, valid_samples,
130
+ sequence_group.sampling_params)
131
132
def _process_decode_and_stop(self, seq: Sequence,
133
sampling_params: SamplingParams) -> None:
@@ -140,7 +145,7 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
145
146
def _process_seq_outputs(self, seq: Sequence,
147
valid_samples: List[SequenceOutput],
143
- sampling_params: SamplingParams) -> None:
148
+ sampling_params: SamplingParams) -> int:
149
output_token_ids = [sample.output_token for sample in valid_samples]
150
output_logprobs = [sample.logprobs for sample in valid_samples]
151
@@ -148,7 +153,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
153
remaining_tokens = sampling_params.max_tokens - (seq.get_output_len() +
154
len(output_token_ids))
155
if remaining_tokens < 0:
151
- valid_samples = valid_samples[:remaining_tokens]
156
output_token_ids = output_token_ids[:remaining_tokens]
157
158
# Truncate any tokens after EOS. This is required as spec decode
@@ -162,7 +166,6 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
166
for i in range(len(output_token_ids)):
167
if output_token_ids[i] == eos_token_id:
168
output_token_ids = output_token_ids[:i + 1]
165
- valid_samples = valid_samples[:i + 1]
169
break
170
171
# Incrementally append tokens to the sequence, as if we had only one new
@@ -173,9 +176,9 @@ class MultiStepOutputProcessor(SequenceGroupOutputProcessor):
176
token_id=output_token_id,
177
logprobs=output_logprob,
178
)
176
- seq.data.update_num_computed_tokens(1)
179
180
self._process_decode_and_stop(seq, sampling_params)
181
182
if seq.is_finished():
183
break
184
+ return len(output_token_ids)
vllm/model_executor/layers/sampler.py
+1
-1
@@ -912,7 +912,7 @@ def get_logprobs(
912
sampling_metadata: SamplingMetadata,
913
sample_results: SampleResultType,
914
) -> Tuple[List[Optional[PromptLogprobs]], List[SampleLogprobs]]:
915
- """Return sample lobprobs and prompt logprobs.
915
+ """Return sample logprobs and prompt logprobs.
916
917
The logic consists of 3 parts.
918
- Select indices to compute logprob from, ranks of token ids, and
vllm/model_executor/sampling_metadata.py
+12
-11
@@ -146,7 +146,7 @@ class SamplingMetadata:
146
def prepare(
147
seq_group_metadata_list: List[SequenceGroupMetadata],
148
seq_lens: List[int],
149
- query_lens: Optional[List[int]],
149
+ query_lens: List[int],
150
device: str,
151
pin_memory: bool,
152
generators: Optional[Dict[str, torch.Generator]] = None,
@@ -194,7 +194,7 @@ class SamplingMetadata:
194
def _prepare_seq_groups(
195
seq_group_metadata_list: List[SequenceGroupMetadata],
196
seq_lens: List[int],
197
- query_lens: Optional[List[int]],
197
+ query_lens: List[int],
198
device: str,
199
generators: Optional[Dict[str, torch.Generator]] = None,
200
cache: Optional[SamplingMetadataCache] = None,
@@ -284,7 +284,8 @@ def _prepare_seq_groups(
284
else:
285
# Decode
286
prompt_logprob_len = 0
287
- sample_len = len(seq_ids) if do_sample else 0
287
+ query_len = query_lens[i] if query_lens is not None else 1
288
+ sample_len = len(seq_ids) * query_len if do_sample else 0
289
290
if sampling_params.seed is not None and generators is not None:
291
generator = generators.get(seq_group_metadata.request_id)
@@ -440,14 +441,14 @@ class SamplingTensors:
441
442
if seq_group.do_sample:
443
sample_lens = len(seq_group.sample_indices)
443
- assert sample_lens == len(seq_ids)
444
- temperatures += [temperature] * len(seq_ids)
445
- top_ps += [top_p] * len(seq_ids)
446
- top_ks += [top_k] * len(seq_ids)
447
- min_ps += [min_p] * len(seq_ids)
448
- presence_penalties += [p] * len(seq_ids)
449
- frequency_penalties += [f] * len(seq_ids)
450
- repetition_penalties += [r] * len(seq_ids)
444
+ assert sample_lens >= len(seq_ids)
445
+ temperatures += [temperature] * sample_lens
446
+ top_ps += [top_p] * sample_lens
447
+ top_ks += [top_k] * sample_lens
448
+ min_ps += [min_p] * sample_lens
449
+ presence_penalties += [p] * sample_lens
450
+ frequency_penalties += [f] * sample_lens
451
+ repetition_penalties += [r] * sample_lens
452
453
if do_penalties:
454
for seq_group in sampling_metadata.seq_groups:
vllm/spec_decode/batch_expansion.py
-7
@@ -12,7 +12,6 @@ from vllm.sequence import (VLLM_INVALID_TOKEN_ID, VLLM_TOKEN_ID_ARRAY_TYPE,
12
from vllm.spec_decode.interfaces import (SpeculativeProposals,
13
SpeculativeScorer, SpeculativeScores)
14
from vllm.spec_decode.util import nvtx_range, split_batch_by_proposal_len
15
-from vllm.worker.worker_base import WorkerBase
15
16
SeqId = int
17
TargetSeqId = int
@@ -36,12 +35,6 @@ class BatchExpansionTop1Scorer(SpeculativeScorer):
35
of topk/tree.
36
"""
37
39
- def __init__(self, scorer_worker: WorkerBase, device: str,
40
- vocab_size: int):
41
- self._scorer_worker = scorer_worker
42
- self._device = device
43
- self._vocab_size = vocab_size
44
-
38
@nvtx_range("BatchExpansionTop1Scorer.score_proposals")
39
def score_proposals(
40
self,
vllm/spec_decode/draft_model_runner.py
-2
@@ -94,8 +94,6 @@ class TP1DraftModelRunner(ModelRunner):
94
assert seq_group.is_prompt is False # No prompt
95
assert seq_group.prompt_logprob_indices == [] # No prompt
96
assert seq_group.sample_indices == [i] # Simple
97
- assert seq_group.seq_len is None # Decode
98
- assert seq_group.query_len is None # Decode
97
98
def _gpu_advance_step(
99
self, model_input: ModelInputForGPUWithSamplingMetadata,
vllm/spec_decode/interfaces.py
+7
@@ -5,6 +5,7 @@ from typing import Optional, Set
5
import torch
6
7
from vllm.sequence import ExecuteModelRequest
8
+from vllm.worker.worker_base import WorkerBase
9
10
11
@dataclass
@@ -74,6 +75,12 @@ class SpeculativeProposer(ABC):
75
76
class SpeculativeScorer(ABC):
77
78
+ def __init__(self, scorer_worker: WorkerBase, device: str,
79
+ vocab_size: int):
80
+ self._scorer_worker = scorer_worker
81
+ self._device = device
82
+ self._vocab_size = vocab_size
83
+
84
@abstractmethod
85
def score_proposals(
86
self,
vllm/spec_decode/mqa_scorer.py
new
+80
@@ -0,0 +1,80 @@
1
+from vllm.sequence import (ExecuteModelRequest, SequenceData,
2
+ SequenceGroupMetadata, get_all_seq_ids)
3
+from vllm.spec_decode.interfaces import (SpeculativeProposals,
4
+ SpeculativeScorer, SpeculativeScores)
5
+
6
+SeqId = int
7
+TargetSeqId = int
8
+
9
+
10
+class MQAScorer(SpeculativeScorer):
11
+
12
+ def score_proposals(
13
+ self,
14
+ execute_model_req: ExecuteModelRequest,
15
+ proposals: SpeculativeProposals,
16
+ ) -> SpeculativeScores:
17
+ target_seq_group_metadata_list = []
18
+ target_seq_id_start = max(
19
+ get_all_seq_ids(execute_model_req.seq_group_metadata_list)) + 1
20
+ all_proposal_tokens = proposals.proposal_token_ids.tolist()
21
+ for i, seq_group_metadata in enumerate(
22
+ execute_model_req.seq_group_metadata_list):
23
+ seq_data_dict = seq_group_metadata.seq_data
24
+ assert len(seq_data_dict) == 1
25
+ seq_id = next(iter(seq_data_dict.keys()))
26
+
27
+ seq_data: SequenceData = seq_data_dict[seq_id]
28
+ prompt_token_ids = seq_data.get_prompt_token_ids()
29
+ output_token_ids = seq_data.get_output_token_ids()
30
+ proposal_token_ids = all_proposal_tokens[i]
31
+ new_output_token_ids = [*output_token_ids, *proposal_token_ids]
32
+
33
+ target_seq_id = target_seq_id_start + i
34
+ new_seq_data = SequenceData.from_seqs(
35
+ prompt_token_ids=prompt_token_ids,
36
+ output_token_ids=new_output_token_ids,
37
+ )
38
+ new_seq_data.update_num_computed_tokens(
39
+ len(prompt_token_ids) + len(output_token_ids) - 1)
40
+
41
+ # Ensure that the new sequence has at least one token
42
+ # because we only use mqa scorer in the decoding stage.
43
+ assert len(output_token_ids) >= 1
44
+ new_seq_data_dict = {target_seq_id: new_seq_data}
45
+
46
+ new_seq_group_metadata = SequenceGroupMetadata(
47
+ request_id=seq_group_metadata.request_id,
48
+ is_prompt=seq_group_metadata.is_prompt,
49
+ seq_data=new_seq_data_dict,
50
+ sampling_params=seq_group_metadata.sampling_params,
51
+ block_tables={
52
+ target_seq_id: seq_group_metadata.block_tables[seq_id],
53
+ },
54
+ lora_request=None,
55
+ token_chunk_size=1,
56
+ )
57
+ target_seq_group_metadata_list.append(new_seq_group_metadata)
58
+
59
+ target_sampler_output = self._scorer_worker.execute_model(
60
+ execute_model_req=execute_model_req.clone(
61
+ seq_group_metadata_list=target_seq_group_metadata_list))
62
+
63
+ target_sampler_output = target_sampler_output[0]
64
+
65
+ bs, k = proposals.proposal_token_ids.shape
66
+ all_tokens = target_sampler_output.sampled_token_ids.reshape(bs, k + 1)
67
+
68
+ all_probs = target_sampler_output.sampled_token_probs.reshape(
69
+ bs, k + 1, self._vocab_size)
70
+ all_logprobs = target_sampler_output.logprobs.reshape(
71
+ bs, k + 1, self._vocab_size)
72
+
73
+ hidden_states = None
74
+ if target_sampler_output.hidden_states is not None:
75
+ hidden_states = target_sampler_output.hidden_states.reshape(
76
+ bs, (k + 1), -1)
77
+ return SpeculativeScores(probs=all_probs,
78
+ token_ids=all_tokens,
79
+ logprobs=all_logprobs,
80
+ hidden_states=hidden_states)
vllm/spec_decode/spec_decode_worker.py
+54
-7
@@ -1,6 +1,6 @@
1
from collections import defaultdict
2
from functools import cached_property
3
-from typing import Any, Dict, List, Optional, Set, Tuple
3
+from typing import Any, Dict, List, Optional, Set, Tuple, Type
4
5
import torch
6
@@ -24,6 +24,7 @@ from vllm.spec_decode.interfaces import (SpeculativeProposals,
24
from vllm.spec_decode.medusa_worker import MedusaWorker
25
from vllm.spec_decode.metrics import AsyncMetricsCollector
26
from vllm.spec_decode.mlp_speculator_worker import MLPSpeculatorWorker
27
+from vllm.spec_decode.mqa_scorer import MQAScorer
28
from vllm.spec_decode.multi_step_worker import MultiStepWorker
29
from vllm.spec_decode.ngram_worker import NGramWorker
30
from vllm.spec_decode.proposer_worker_base import ProposerWorkerBase
@@ -70,6 +71,7 @@ def create_spec_worker(*args, **kwargs) -> "SpecDecodeWorker":
71
spec_decode_worker = SpecDecodeWorker.create_worker(
72
scorer_worker=target_worker,
73
draft_worker_kwargs=draft_worker_kwargs,
74
+ disable_mqa_scorer=speculative_config.speculative_disable_mqa_scorer,
75
disable_by_batch_size=speculative_config.
76
speculative_disable_by_batch_size,
77
draft_token_acceptance_method=speculative_config.
@@ -116,6 +118,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
118
cls,
119
scorer_worker: Worker,
120
draft_worker_kwargs: Dict[str, Any],
121
+ disable_mqa_scorer: bool,
122
disable_by_batch_size: Optional[int],
123
draft_token_acceptance_method: str,
124
typical_acceptance_sampler_posterior_threshold: float,
@@ -173,12 +176,43 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
176
typical_acceptance_sampler_posterior_threshold,
177
posterior_alpha=typical_acceptance_sampler_posterior_alpha,
178
)
176
- logger.info("Configuring SpecDecodeWorker with sampler=%s",
177
- type(spec_decode_sampler))
179
+ logger.info(
180
+ "[Speculative Decoding] Configuring"
181
+ " SpecDecodeWorker with sampler=%s", type(spec_decode_sampler))
182
+
183
+ if not disable_mqa_scorer:
184
+ if scorer_worker.model_runner.attn_backend.get_name(
185
+ ) != "flash-attn":
186
+ disable_mqa_scorer = True
187
+ logger.info(
188
+ "[Speculative Decoding] Disabling MQA scorer as the "
189
+ "MQA is only available with flash attn backend.")
190
+
191
+ if ngram_prompt_lookup_max > 0:
192
+ disable_mqa_scorer = True
193
+ logger.info(
194
+ "[Speculative Decoding] Disabling MQA scorer as the "
195
+ "NGramWorker does not support MQA scorer.")
196
+
197
+ if "model_config" in draft_worker_kwargs and \
198
+ draft_worker_kwargs["model_config"].max_model_len < \
199
+ scorer_worker.model_config.max_model_len:
200
+ disable_mqa_scorer = True
201
+ logger.info(
202
+ "[Speculative Decoding] Disabling MQA scorer as the "
203
+ "draft model max_model_len is smaller than the target "
204
+ "model max_model_len.")
205
+
206
+ if not scorer_worker.model_runner.model_config.enforce_eager:
207
+ disable_mqa_scorer = True
208
+ logger.info(
209
+ "[Speculative Decoding] Disabling MQA scorer as the "
210
+ "target model is not running in eager mode.")
211
212
return SpecDecodeWorker(
213
proposer_worker,
214
scorer_worker,
215
+ disable_mqa_scorer=disable_mqa_scorer,
216
disable_logprobs=disable_logprobs,
217
disable_log_stats=disable_log_stats,
218
disable_by_batch_size=disable_by_batch_size,
@@ -190,6 +224,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
224
proposer_worker: ProposerWorkerBase,
225
scorer_worker: WorkerBase,
226
spec_decode_sampler: SpecDecodeBaseSampler,
227
+ disable_mqa_scorer: bool = False,
228
disable_logprobs: bool = False,
229
disable_log_stats: bool = False,
230
metrics_collector: Optional[AsyncMetricsCollector] = None,
@@ -211,6 +246,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
246
types of sampler namely RejectionSampler and
247
TypicalAcceptanceSampler. 'spec_decode_sampler' is either an
248
instance of RejectionSampler or TypicalAcceptanceSampler.
249
+ disable_mqa_scorer: If set to True, disable the MQA scorer and use
250
+ the BatchExpansionTop1Scorer instead.
251
disable_logprobs: If set to True, token log probabilities will
252
not be output in both the draft worker and the target worker.
253
If set to False, log probabilities will be output by both.
@@ -248,6 +285,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
285
self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
286
# Lazy initialization.
287
self.scorer: SpeculativeScorer
288
+ self.disable_mqa_scorer = disable_mqa_scorer
289
290
# Hidden states from target model to pass to proposer
291
# in the subsequent step.
@@ -270,10 +308,19 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
308
self._metrics.init_gpu_tensors(self.rank)
309
self.spec_decode_sampler.init_gpu_tensors(self.rank)
310
273
- self.scorer = BatchExpansionTop1Scorer(
274
- scorer_worker=self.scorer_worker,
275
- device=self.device,
276
- vocab_size=self._vocab_size)
311
+ scorer_cls: Type[SpeculativeScorer]
312
+ if self.disable_mqa_scorer:
313
+ scorer_cls = BatchExpansionTop1Scorer
314
+ logger.info("[Speculative Decoding] Use batch "
315
+ "expansion for scoring proposals.")
316
+ else:
317
+ scorer_cls = MQAScorer
318
+ logger.info(
319
+ "[Speculative Decoding] Use MQA scorer for scoring proposals.")
320
+
321
+ self.scorer = scorer_cls(scorer_worker=self.scorer_worker,
322
+ device=self.device,
323
+ vocab_size=self._vocab_size)
324
325
self._configure_model_sampler_for_spec_decode()
326
vllm/worker/model_runner.py
+10
-27
@@ -468,43 +468,26 @@ class ModelInputForGPUBuilder(ModelRunnerInputBuilderBase[ModelInputForGPU]):
468
469
# Compute context length (the number of tokens that are
470
# already computed) and sequence length (total number of tokens).
471
+
472
seq_len = seq_data.get_len()
473
if inter_data.is_prompt:
474
context_len = seq_data.get_num_computed_tokens()
474
- else:
475
- # get_num_computed_tokens is incorrect for spec decoding.
476
- # So, we should have a special logic here.
477
- # TODO(sang): Fix it.
475
+ seq_len = min(seq_len, context_len + token_chunk_size)
476
+ elif self.runner.scheduler_config.is_multi_step or \
477
+ self.runner.model_config.is_encoder_decoder_model:
478
context_len = seq_len - 1
479
- seq_len = min(seq_len, context_len + token_chunk_size)
479
+ else:
480
+ context_len = seq_data.get_num_computed_tokens()
481
482
# Compute tokens.
482
- if inter_data.is_prompt:
483
- tokens = seq_data.get_token_ids()
484
- if context_len != 0 or seq_len < len(tokens):
485
- tokens = tokens[context_len:seq_len]
486
- else:
487
- # Optimization. get_token_ids requires the entire copy of
488
- # tokens.
489
- tokens = seq_data.get_last_token_id()
483
+ tokens = seq_data.get_token_ids()[context_len:seq_len]
484
485
inter_data.seq_lens[seq_idx] = seq_len
486
inter_data.orig_seq_lens[seq_idx] = seq_len
487
inter_data.context_lens[seq_idx] = context_len
494
-
495
- if isinstance(tokens, list):
496
- inter_data.input_tokens[seq_idx].extend(tokens)
497
- else:
498
- inter_data.input_tokens[seq_idx].append(tokens)
499
-
500
- if (seq_len - context_len) == 1:
501
- inter_data.input_positions[seq_idx].append(seq_len - 1)
502
- else:
503
- inter_data.input_positions[seq_idx].extend(
504
- range(context_len, seq_len))
505
-
506
- inter_data.query_lens[
507
- seq_idx] = seq_len - context_len if inter_data.is_prompt else 1
488
+ inter_data.input_tokens[seq_idx].extend(tokens)
489
+ inter_data.input_positions[seq_idx].extend(range(context_len, seq_len))
490
+ inter_data.query_lens[seq_idx] = seq_len - context_len
491
492
if seq_data.mrope_position_delta is not None:
493
if inter_data.mrope_input_positions is None: