A high-throughput and memory-efficient inference and serving engine for LLMs
Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git
Synced Nov 16, 22:00 UTC
[Speculative Decoding] Enabling bonus token in speculative decoding for KV cache based models (#5765)
sroy745 committed
Jul 10, 2024 at 16:02 UTC
ae151d73be479e9c0caa2fdfc30b17f073018ef3
14 files changed
+645
-80
tests/spec_decode/test_dynamic_spec_decode.py
+7
-4
@@ -70,14 +70,17 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
70
if queue_size < disable_by_batch_size:
71
# Should raise exception when executing the mocked draft model.
72
with pytest.raises(ValueError, match=exception_secret):
73
- proposer.get_spec_proposals(execute_model_req=ExecuteModelRequest(
74
- seq_group_metadata_list=seq_group_metadata_list,
75
- num_lookahead_slots=k), )
73
+ proposer.get_spec_proposals(
74
+ execute_model_req=ExecuteModelRequest(
75
+ seq_group_metadata_list=seq_group_metadata_list,
76
+ num_lookahead_slots=k),
77
+ seq_ids_with_bonus_token_in_last_step=set())
78
else:
79
# Should not execute the draft model because spec decode is disabled
80
# for all requests. Accordingly, the proposal length should be 0.
81
proposals = proposer.get_spec_proposals(
82
execute_model_req=ExecuteModelRequest(
83
seq_group_metadata_list=seq_group_metadata_list,
82
- num_lookahead_slots=k), )
84
+ num_lookahead_slots=k),
85
+ seq_ids_with_bonus_token_in_last_step=set())
86
assert proposals.proposal_lens.tolist() == [0] * batch_size
tests/spec_decode/test_multi_step_worker.py
+207
-5
@@ -118,7 +118,8 @@ def test_same_output_for_single_step():
118
actual_output, _ = multi_step_worker.sampler_output(
119
execute_model_req=ExecuteModelRequest(
120
seq_group_metadata_list=multi_step_seq_group),
121
- sample_len=num_steps)
121
+ sample_len=num_steps,
122
+ seq_ids_with_bonus_token_in_last_step=set())
123
assert len(actual_output) == num_steps
124
actual_output = actual_output[0]
125
@@ -210,7 +211,8 @@ def test_same_output_for_multi_step():
211
multi_step_output, _ = multi_step_worker.sampler_output(
212
execute_model_req=ExecuteModelRequest(
213
seq_group_metadata_list=seq_group_metadata_list),
213
- sample_len=num_steps)
214
+ sample_len=num_steps,
215
+ seq_ids_with_bonus_token_in_last_step=set())
216
217
# Run single-step repeatedly.
218
zero_kv_cache(worker.cache_engine)
@@ -277,6 +279,203 @@ def test_same_output_for_multi_step():
279
single_step_logprobs)
280
281
282
+@torch.inference_mode()
283
+def test_multi_step_with_batch_expansion_correct_output():
284
+ """
285
+ In this test we verify that the MultiStepWorker is able to handle bonus
286
+ tokens correctly. The test verifies that if a sequence has a
287
+ bonus token then the MultiStepWorker is able to expand the batch by adding
288
+ new sequences corresponding to the sequences with bonus tokens. The
289
+ expanded batch is then used for predicting the next tokens.
290
+ """
291
+ seed = 100
292
+ model_name = 'JackFram/llama-68m'
293
+
294
+ block_size = 16
295
+ num_gpu_blocks = 2048 // block_size
296
+ batch_size = 128
297
+ multi_step_worker = create_worker(
298
+ MultiStepWorker,
299
+ model_name,
300
+ block_size,
301
+ num_gpu_blocks,
302
+ seed,
303
+ model_runner_cls=TP1DraftModelRunner,
304
+ )
305
+ worker = create_worker(
306
+ Worker,
307
+ model_name,
308
+ block_size,
309
+ num_gpu_blocks,
310
+ seed,
311
+ )
312
+ random.seed(seed)
313
+ prompts = [[0] for _ in range(batch_size)]
314
+ num_steps = 2
315
+ final_prompt_lens = [(num_steps + 1) for prompt in prompts]
316
+ rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
317
+ multi_step_worker.execute_model = patch_execute_model_with_seeds(
318
+ multi_step_worker, rand_seeds)
319
+ worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
320
+ # Create the test continuations
321
+ continuations = [[random.randint(0, 1000)] for _ in prompts]
322
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
323
+ prompts,
324
+ num_gpu_blocks,
325
+ block_size,
326
+ continuations=continuations,
327
+ final_prompt_lens=final_prompt_lens)
328
+
329
+ # Run single-step twice to generate 2 tokens. This
330
+ # will simulate the bonus token case with the second token
331
+ # being the bonus token.
332
+ zero_kv_cache(worker.cache_engine)
333
+ single_step_output: List[SamplerOutput] = []
334
+ set_random_seed(seed)
335
+ for _ in range(num_steps):
336
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
337
+ prompts,
338
+ num_gpu_blocks,
339
+ block_size,
340
+ continuations=continuations,
341
+ final_prompt_lens=final_prompt_lens)
342
+ single_step_output.extend(
343
+ worker.execute_model(execute_model_req=ExecuteModelRequest(
344
+ seq_group_metadata_list=seq_group_metadata_list)))
345
+ # Append output tokens to new sequence data.
346
+ for i, seq_group_output in enumerate(single_step_output[-1]):
347
+ continuations[i].append(seq_group_output.samples[0].output_token)
348
+
349
+ # Create continuations for the MultiStepWorker. The continuations have
350
+ # 2 tokens in order to simulate the bonus token case.
351
+ multi_step_continuations = []
352
+ for continuation in continuations:
353
+ multi_step_continuations.append(continuation[:2])
354
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
355
+ prompts,
356
+ num_gpu_blocks,
357
+ block_size,
358
+ continuations=multi_step_continuations,
359
+ final_prompt_lens=final_prompt_lens)
360
+
361
+ # Run multi-step and verify that the third token prediction is accurate
362
+ # for all sequences.
363
+ zero_kv_cache(multi_step_worker.cache_engine)
364
+ all_seq_ids = {i for i in range(batch_size)}
365
+ multi_step_output, _ = multi_step_worker.sampler_output(
366
+ execute_model_req=ExecuteModelRequest(
367
+ seq_group_metadata_list=seq_group_metadata_list),
368
+ sample_len=1,
369
+ seq_ids_with_bonus_token_in_last_step=all_seq_ids)
370
+ for index, output in enumerate(multi_step_output[-1].outputs):
371
+ assert (continuations[index][-1] == output.samples[0].output_token)
372
+
373
+
374
+@torch.inference_mode()
375
+def test_multi_step_with_batch_expansion_incorrect_output():
376
+ """
377
+ Tests the MultiStepWorker's ability to handle batch expansion with bonus
378
+ tokens in a negative case scenario. This test provides the MultiStepWorker
379
+ with a batch containing sequences with bonus tokens but specifies the
380
+ sequence IDs with bonus tokens incorrectly. The test verifies that the
381
+ MultiStepWorker generates correct tokens for the sequences where the
382
+ sequence ID is specified correctly and incorrect tokens for those where
383
+ the sequence ID is specified incorrectly.
384
+ """
385
+ seed = 100
386
+ model_name = 'JackFram/llama-68m'
387
+
388
+ block_size = 16
389
+ num_gpu_blocks = 2048 // block_size
390
+ batch_size = 128
391
+ multi_step_worker = create_worker(
392
+ MultiStepWorker,
393
+ model_name,
394
+ block_size,
395
+ num_gpu_blocks,
396
+ seed,
397
+ model_runner_cls=TP1DraftModelRunner,
398
+ )
399
+ worker = create_worker(
400
+ Worker,
401
+ model_name,
402
+ block_size,
403
+ num_gpu_blocks,
404
+ seed,
405
+ )
406
+ random.seed(seed)
407
+ prompts = [[0] for _ in range(batch_size)]
408
+ num_steps = 2
409
+ final_prompt_lens = [(num_steps + 1) for prompt in prompts]
410
+ rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
411
+ multi_step_worker.execute_model = patch_execute_model_with_seeds(
412
+ multi_step_worker, rand_seeds)
413
+ worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
414
+ # Create the test continuations
415
+ continuations = [[random.randint(0, 1000)] for _ in prompts]
416
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
417
+ prompts,
418
+ num_gpu_blocks,
419
+ block_size,
420
+ continuations=continuations,
421
+ final_prompt_lens=final_prompt_lens)
422
+ # Run single-step twice to generate 2 tokens. This
423
+ # will simulate the bonus token case with the second token
424
+ # being the bonus token.
425
+ zero_kv_cache(worker.cache_engine)
426
+ single_step_output: List[SamplerOutput] = []
427
+ set_random_seed(seed)
428
+ for _ in range(num_steps):
429
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
430
+ prompts,
431
+ num_gpu_blocks,
432
+ block_size,
433
+ continuations=continuations,
434
+ final_prompt_lens=final_prompt_lens)
435
+ single_step_output.extend(
436
+ worker.execute_model(execute_model_req=ExecuteModelRequest(
437
+ seq_group_metadata_list=seq_group_metadata_list)))
438
+ # Append output tokens to new sequence data.
439
+ for i, seq_group_output in enumerate(single_step_output[-1]):
440
+ continuations[i].append(seq_group_output.samples[0].output_token)
441
+
442
+ # Create continuations for the MultiStepWorker. The continuations have
443
+ # 2 tokens in order to simulate the bonus token case.
444
+ multi_step_continuations = []
445
+ for continuation in continuations:
446
+ multi_step_continuations.append(continuation[:2])
447
+ seq_group_metadata_list = create_seq_group_metadata_from_prompts(
448
+ prompts,
449
+ num_gpu_blocks,
450
+ block_size,
451
+ continuations=multi_step_continuations,
452
+ final_prompt_lens=final_prompt_lens)
453
+
454
+ # Run multi-step. In this run INCORRECTLY specify that only the odd number
455
+ # sequences have bonus tokens. Verify that with this setting the third token
456
+ # prediction is accurate only for the odd numbered sequences. Also verify
457
+ # that the prediction might be wrong for some of the even numbered
458
+ # sequences.
459
+ zero_kv_cache(multi_step_worker.cache_engine)
460
+ set_random_seed(seed)
461
+ odd_seq_ids = {i for i in range(batch_size) if i % 2 != 0}
462
+ multi_step_output, _ = multi_step_worker.sampler_output(
463
+ execute_model_req=ExecuteModelRequest(
464
+ seq_group_metadata_list=seq_group_metadata_list),
465
+ sample_len=1,
466
+ seq_ids_with_bonus_token_in_last_step=odd_seq_ids)
467
+ num_mismatch = 0
468
+ for index, output in enumerate(multi_step_output[-1].outputs):
469
+ if (index % 2) != 0:
470
+ assert (continuations[index][-1] == output.samples[0].output_token)
471
+ elif (continuations[index][-1] != output.samples[0].output_token):
472
+ num_mismatch += 1
473
+ # The prediction is accurate for some of the sequences even without proper
474
+ # handling of the bonus tokens. Hence verify that the number of sequences
475
+ # for which there is a mismatch is > 0.
476
+ assert (num_mismatch > 0)
477
+
478
+
479
@torch.inference_mode()
480
def test_draft_proposals_full_speculation_len():
481
"""Verify Top1Proposer correctly handles case where all sequences
@@ -318,7 +517,8 @@ def test_draft_proposals_full_speculation_len():
517
proposals = proposer.get_spec_proposals(
518
execute_model_req=ExecuteModelRequest(
519
seq_group_metadata_list=seq_group_metadata_list,
321
- num_lookahead_slots=k), )
520
+ num_lookahead_slots=k),
521
+ seq_ids_with_bonus_token_in_last_step=set())
522
523
assert torch.is_tensor(proposals.proposal_token_ids)
524
assert torch.is_tensor(proposals.proposal_probs)
@@ -356,7 +556,8 @@ def test_draft_proposals_no_speculations():
556
proposals = proposer.get_spec_proposals(
557
execute_model_req=ExecuteModelRequest(
558
seq_group_metadata_list=seq_group_metadata_list,
359
- num_lookahead_slots=k), )
559
+ num_lookahead_slots=k),
560
+ seq_ids_with_bonus_token_in_last_step=set())
561
562
assert torch.is_tensor(proposals.proposal_token_ids)
563
assert torch.is_tensor(proposals.proposal_probs)
@@ -428,7 +629,8 @@ def test_draft_proposals_mixed_k():
629
proposals = proposer.get_spec_proposals(
630
execute_model_req=ExecuteModelRequest(
631
seq_group_metadata_list=seq_group_metadata_list,
431
- num_lookahead_slots=k), )
632
+ num_lookahead_slots=k),
633
+ seq_ids_with_bonus_token_in_last_step=set())
634
635
assert torch.is_tensor(proposals.proposal_token_ids)
636
assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+6
-3
@@ -53,7 +53,8 @@ def test_ngram_algo_correctness_for_single_no_match():
53
proposals = proposer.get_spec_proposals(
54
execute_model_req=ExecuteModelRequest(
55
seq_group_metadata_list=seq_group_metadata_list,
56
- num_lookahead_slots=proposal_len), )
56
+ num_lookahead_slots=proposal_len),
57
+ seq_ids_with_bonus_token_in_last_step=None)
58
59
assert torch.is_tensor(proposals.proposal_token_ids)
60
assert torch.is_tensor(proposals.proposal_probs)
@@ -121,7 +122,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
122
proposals = proposer.get_spec_proposals(
123
execute_model_req=ExecuteModelRequest(
124
seq_group_metadata_list=seq_group_metadata_list,
124
- num_lookahead_slots=proposal_len), )
125
+ num_lookahead_slots=proposal_len),
126
+ seq_ids_with_bonus_token_in_last_step=None)
127
128
assert torch.is_tensor(proposals.proposal_token_ids)
129
assert torch.is_tensor(proposals.proposal_probs)
@@ -193,7 +195,8 @@ def test_ngram_algo_correctness_for_batches_match_all():
195
proposals = proposer.get_spec_proposals(
196
execute_model_req=ExecuteModelRequest(
197
seq_group_metadata_list=seq_group_metadata_list,
196
- num_lookahead_slots=proposal_len), )
198
+ num_lookahead_slots=proposal_len),
199
+ seq_ids_with_bonus_token_in_last_step=None)
200
201
assert torch.is_tensor(proposals.proposal_token_ids)
202
assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_spec_decode_worker.py
+143
-4
@@ -1,6 +1,7 @@
1
import random
2
+from collections import defaultdict
3
from types import SimpleNamespace
3
-from typing import Dict, List
4
+from typing import Dict, List, Set
5
from unittest.mock import MagicMock
6
7
import pytest
@@ -377,8 +378,10 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
378
379
set_random_seed(1)
380
380
- worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
381
- metrics_collector)
381
+ worker = SpecDecodeWorker(draft_worker,
382
+ target_worker,
383
+ spec_decode_sampler,
384
+ metrics_collector=metrics_collector)
385
worker.init_device()
386
387
proposal_token_ids = torch.randint(low=0,
@@ -554,7 +557,6 @@ def test_init_device(acceptance_sampler_method: str):
557
558
worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
559
metrics_collector)
557
-
560
worker.init_device()
561
562
draft_worker.init_device.assert_called_once()
@@ -645,3 +647,140 @@ def test_split_num_cache_blocks_evenly(available_gpu_blocks: int,
647
assert (num_blocks * target_cache_block_size_bytes) + (
648
num_blocks * draft_kv_size_bytes) <= (available_gpu_blocks *
649
target_cache_block_size_bytes)
650
+
651
+
652
+@torch.inference_mode()
653
+def test_populate_seq_ids_with_bonus_tokens():
654
+ """
655
+ Verify that a call to _create_output_sampler_list correctly updates
656
+ seq_with_bonus_token_in_last_step.
657
+
658
+ seq_with_bonus_token_in_last_step is an internal data structure in
659
+ SpecDecodeWorker that tracks the sequence IDs which are assigned bonus
660
+ tokens by the target model in their last forward pass. This state is
661
+ maintained only for models relying on the KV cache, such as those using
662
+ the MultiStepWorker.
663
+ """
664
+ batch_size = 10
665
+ k = 5
666
+ vocab_size = 10000
667
+ num_sequences_with_bonus_tokens = 5
668
+ target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
669
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
670
+ target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
671
+ target_worker.device = 'cuda'
672
+
673
+ set_random_seed(1)
674
+ draft_worker = mock_worker(cls=MultiStepWorker)
675
+ draft_worker.device = 'cuda'
676
+ # The sequence_ids attached to each sequence in the batch.
677
+ # The sequence at index i has seq_id assigned_seq_ids[i]
678
+ assigned_seq_ids = list(range(batch_size))
679
+ seq_group_metadata_list, _, _ = create_batch(batch_size,
680
+ k,
681
+ seq_ids=assigned_seq_ids,
682
+ prev_output_token_len=10)
683
+ target_token_logprobs = torch.rand(batch_size, (k + 1),
684
+ vocab_size,
685
+ dtype=torch.float32,
686
+ device='cuda')
687
+ accepted_token_ids = torch.randint(low=0,
688
+ high=vocab_size,
689
+ size=(batch_size, (k + 1)),
690
+ dtype=torch.int64,
691
+ device='cuda')
692
+ expected_request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)
693
+ for seq_group_metadata in seq_group_metadata_list:
694
+ for seq_id in seq_group_metadata.seq_data:
695
+ expected_request_id_seq_ids_mapping[
696
+ seq_group_metadata.request_id].add(seq_id)
697
+ # Generate a random sample of sequence indexes with bonus tokens
698
+ seq_indexes_with_bonus_tokens = random.sample(
699
+ range(batch_size), num_sequences_with_bonus_tokens)
700
+ # Create a mask that is True for indices in seq_indexes_with_bonus_tokens
701
+ mask = torch.ones(batch_size, dtype=torch.bool, device='cuda')
702
+ mask[seq_indexes_with_bonus_tokens] = False
703
+ # Set the last token ID to -1 for all indices not in
704
+ # seq_indexes_with_bonus_tokens to indicate the lack of bonus token in
705
+ # those indices.
706
+ accepted_token_ids[mask, -1:] = -1
707
+ worker = SpecDecodeWorker(draft_worker,
708
+ target_worker,
709
+ mock_spec_decode_sampler("rejection_sampler"),
710
+ metrics_collector=metrics_collector)
711
+ # Initialize _seq_with_bonus_token_in_last_step with a set of sequence IDs.
712
+ # This set includes all sequence IDs in the batch as well as an additional
713
+ # `num_extra_sequence_ids` sequence IDs. Note that the sequence IDs are in
714
+ # the range [0, batch_size + num_extra_sequence_ids).
715
+ num_extra_sequence_ids = 10
716
+ worker._seq_with_bonus_token_in_last_step = set(
717
+ range(batch_size + num_extra_sequence_ids))
718
+ worker._create_output_sampler_list(
719
+ seq_group_metadata_list=seq_group_metadata_list,
720
+ accepted_token_ids=accepted_token_ids,
721
+ target_logprobs=target_token_logprobs,
722
+ k=k)
723
+ # Verify that _seq_with_bonus_token_in_last_step contains the following:
724
+ # 1. Sequence IDs that were already present in
725
+ # _seq_with_bonus_token_in_last_step but were not part of the current
726
+ # batch are retained.
727
+ # 2. Of the sequence IDs present in the current batch, only those with a
728
+ # bonus token are retained in _seq_with_bonus_token_in_last_step.
729
+ # Sequence IDs that are present in the current batch but do not have
730
+ # bonus tokens are removed from _seq_with_bonus_token_in_last_step.
731
+ expected_seq_ids_with_bonus_tokens = \
732
+ set([assigned_seq_ids[i] for i in seq_indexes_with_bonus_tokens])
733
+ additional_sequence_ids = \
734
+ set(range(batch_size, batch_size + num_extra_sequence_ids))
735
+ assert worker._seq_with_bonus_token_in_last_step == \
736
+ expected_seq_ids_with_bonus_tokens.union(additional_sequence_ids)
737
+ assert worker._request_id_seq_id_mapping == \
738
+ expected_request_id_seq_ids_mapping
739
+
740
+
741
+@torch.inference_mode()
742
+def test_handle_finished_requests():
743
+ """
744
+ Test to verify that finished request IDs are appropriately processed to
745
+ update the internal state of the SpecDecodeWorker.
746
+
747
+ This test initializes the SpecDecodeWorker with mock data, marks certain
748
+ requests as finished, and ensures that the corresponding sequence IDs are
749
+ correctly removed from the internal mappings.
750
+ """
751
+ batch_size = 32
752
+ k = 3
753
+ draft_worker = mock_worker(cls=MultiStepWorker)
754
+ target_worker = mock_worker()
755
+ metrics_collector = MagicMock(spec=AsyncMetricsCollector)
756
+ worker = SpecDecodeWorker(draft_worker, target_worker,
757
+ mock_spec_decode_sampler("rejection_sampler"),
758
+ metrics_collector)
759
+ # Initialize the request_id_seq_id_mapping mapping dict with a few fake
760
+ # request ids and corresponding sequence ids.
761
+ worker._request_id_seq_id_mapping = \
762
+ {'request-1': {1,2,3}, 'request-2': {4,5,6,7},
763
+ 'request-3': {8,9}, 'request-4': {10,11}}
764
+ # Initialize seq_with_bonus_token_in_last_step with a few fake
765
+ # sequence ids.
766
+ worker._seq_with_bonus_token_in_last_step = {1, 4, 5, 8, 9, 10}
767
+ exception_secret = 'artificial stop'
768
+ draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
769
+
770
+ seq_group_metadata_list, _, _ = create_batch(batch_size, k)
771
+ # Mark requests with ids request-1 and request-3 as finished.
772
+ execute_model_req = ExecuteModelRequest(
773
+ seq_group_metadata_list=seq_group_metadata_list,
774
+ num_lookahead_slots=k,
775
+ finished_requests_ids=['request-1', 'request-3'])
776
+
777
+ with pytest.raises(ValueError, match=exception_secret):
778
+ worker.execute_model(execute_model_req=execute_model_req)
779
+ # Verify that request-1 and request-3 are removed from
780
+ # request_id_seq_id_mapping
781
+ assert worker._request_id_seq_id_mapping == \
782
+ {'request-2': {4,5,6,7}, 'request-4': {10,11}}
783
+ # Verify that all sequence ids corresponding to 'request-1'
784
+ # and 'request-3' are removed from seq_with_bonus_token_in_last_step.
785
+ assert worker._seq_with_bonus_token_in_last_step == \
786
+ {4,5,10}
vllm/sequence.py
+17
-1
@@ -3,8 +3,9 @@ import copy
3
import enum
4
import math
5
from abc import ABC, abstractmethod
6
+from collections import defaultdict
7
from dataclasses import dataclass, field
7
-from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8
+from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
9
10
import torch
11
@@ -916,6 +917,21 @@ def get_all_seq_ids(
917
return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
918
919
920
+def get_all_seq_ids_and_request_ids(
921
+ seq_group_metadata_list: List[SequenceGroupMetadata]
922
+) -> Tuple[List[int], Dict[str, Set[int]]]:
923
+ """Given a list of SequenceGroupMetadata, create a list of all
924
+ sequence ids.
925
+ """
926
+ seq_ids: List[int] = []
927
+ request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)
928
+ for sg in seq_group_metadata_list:
929
+ for seq_id in sg.seq_data:
930
+ seq_ids.append(seq_id)
931
+ request_id_seq_ids_mapping[sg.request_id].add(seq_id)
932
+ return seq_ids, request_id_seq_ids_mapping
933
+
934
+
935
class HiddenStates:
936
"""Hidden states corresponding to in-progress sequences.
937
Used in speculative decoding to pass hidden states from
vllm/spec_decode/interfaces.py
+4
-1
@@ -1,6 +1,6 @@
1
from abc import ABC, abstractmethod
2
from dataclasses import dataclass
3
-from typing import Optional
3
+from typing import Optional, Set
4
5
import torch
6
@@ -62,6 +62,9 @@ class SpeculativeProposer(ABC):
62
def get_spec_proposals(
63
self,
64
execute_model_req: ExecuteModelRequest,
65
+ # If set, this contains all sequence IDs that were assigned
66
+ # bonus tokens in their last forward pass.
67
+ seq_ids_with_bonus_token_in_last_step: Set[int],
68
) -> SpeculativeProposals:
69
raise NotImplementedError
70
vllm/spec_decode/medusa_worker.py
+6
-2
@@ -1,5 +1,5 @@
1
import weakref
2
-from typing import List, Optional, Tuple
2
+from typing import List, Optional, Set, Tuple
3
4
import torch
5
@@ -40,6 +40,8 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
40
self,
41
execute_model_req: ExecuteModelRequest,
42
sample_len: int,
43
+ # Unused parameter.
44
+ seq_ids_with_bonus_token_in_last_step: Set[int],
45
) -> Tuple[List[SamplerOutput], bool]:
46
"""Run the model forward pass to generate sample_len future tokens.
47
Returns the list of sampler output, one per layer, along with indicator
@@ -97,12 +99,14 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
99
def get_spec_proposals(
100
self,
101
execute_model_req: ExecuteModelRequest,
102
+ seq_ids_with_bonus_token_in_last_step: Set[int],
103
) -> SpeculativeProposals:
104
"""Produce speculations given an input batch of sequences. The number of
105
speculative tokens per sequence is determined by max_proposal_len.
106
"""
107
105
- return self._proposer.get_spec_proposals(execute_model_req)
108
+ return self._proposer.get_spec_proposals(
109
+ execute_model_req, seq_ids_with_bonus_token_in_last_step)
110
111
def _raise_if_unsupported(
112
self,
vllm/spec_decode/mlp_speculator_worker.py
+4
-1
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Set, Tuple
2
3
import torch
4
@@ -20,6 +20,9 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
20
self,
21
execute_model_req: ExecuteModelRequest,
22
sample_len: int,
23
+ # Unused parameter. MLPSpeculatorWorker does not use the KV Cache and
24
+ # therefore does not need this parameter.
25
+ seq_ids_with_bonus_token_in_last_step: Set[int],
26
) -> Tuple[List[SamplerOutput], bool]:
27
"""Run the model forward pass to generate sample_len future tokens.
28
Returns the list of sampler output, one per layer, along with indicator
vllm/spec_decode/multi_step_worker.py
+170
-36
@@ -1,6 +1,6 @@
1
import copy
2
import weakref
3
-from typing import Dict, List, Tuple
3
+from typing import Dict, List, Set, Tuple
4
5
import torch
6
@@ -51,6 +51,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
51
self,
52
execute_model_req: ExecuteModelRequest,
53
sample_len: int,
54
+ seq_ids_with_bonus_token_in_last_step: Set[int],
55
) -> Tuple[List[SamplerOutput], bool]:
56
"""Run the model forward pass sample_len times. Returns the list of
57
sampler output, one per model forward pass, along with indicator of
@@ -60,44 +61,142 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
61
For multi step worker, this indicator shall be True.
62
"""
63
self._raise_if_unsupported(execute_model_req)
63
-
64
- # Shallow copy input data so modifications (such as appending tokens)
65
- # do not cause side-effects.
66
- copied_seq_group_metadata_list = self._shallow_copy_inputs(
67
- execute_model_req.seq_group_metadata_list)
68
- copied_execute_model_req = execute_model_req.clone(
69
- copied_seq_group_metadata_list)
70
-
64
+ # Expand the batch for sequences with a bonus token.
65
+ # Perform a forward pass on the expanded batch and filter the
66
+ # response to retain only the original sequences' responses.
67
+ expanded_request, indices_of_seq_with_bonus_tokens =\
68
+ self._expand_execute_model_request(
69
+ execute_model_req, seq_ids_with_bonus_token_in_last_step)
70
# Run model sample_len times.
71
model_outputs: List[SamplerOutput] = []
72
if isinstance(self.model_runner, TP1DraftModelRunner):
74
- copied_execute_model_req.num_steps = sample_len
73
+ expanded_request.num_steps = sample_len
74
model_outputs = self.execute_model(
76
- execute_model_req=copied_execute_model_req)
75
+ execute_model_req=expanded_request)
76
else:
77
# TODO: Remove this branch once DraftModelRunner supports TP>1.
78
for _ in range(sample_len):
79
model_output: List[SamplerOutput] = super().execute_model(
81
- execute_model_req=copied_execute_model_req)
80
+ execute_model_req=expanded_request)
81
assert (len(model_output) == 1
82
), "composing multistep workers not supported"
83
model_output = model_output[0]
84
86
- self._append_new_tokens(model_output,
87
- copied_seq_group_metadata_list)
85
+ self._append_new_tokens(
86
+ model_output, expanded_request.seq_group_metadata_list)
87
model_outputs.append(model_output)
88
90
- return model_outputs, True
89
+ filtered_model_outputs = self._filter_model_output(
90
+ model_outputs, indices_of_seq_with_bonus_tokens)
91
+ return filtered_model_outputs, True
92
+
93
+ @staticmethod
94
+ def _expand_execute_model_request(
95
+ execute_model_req: ExecuteModelRequest,
96
+ seq_with_bonus_token_in_last_step: set,
97
+ ) -> Tuple[ExecuteModelRequest, List[int]]:
98
+ """
99
+ Expands the execute model request based on sequences with bonus
100
+ tokens.
101
+
102
+ For each sequence with a bonus token, this method creates a new
103
+ sequence without the bonus token and adds it to the execute model
104
+ request. The original sequence groups are also retained. The indices
105
+ of the original sequence groups are returned for further processing.
106
+
107
+ Args:
108
+ execute_model_req (ExecuteModelRequest): The original execute
109
+ model request.
110
+ seq_with_bonus_token_in_last_step (set): Set of sequence IDs that
111
+ contain bonus tokens.
112
+
113
+ Returns:
114
+ Tuple[ExecuteModelRequest, List[int]]: The updated execute model
115
+ request with expanded sequences and a list of indices corresponding
116
+ to the original sequence groups.
117
+ """
118
+ updated_seq_group_metadata_list: List[SequenceGroupMetadata] = []
119
+ updated_execute_model_req = execute_model_req.clone(
120
+ updated_seq_group_metadata_list)
121
+ indices_of_original_sequence_groups = []
122
+ for seq_group in execute_model_req.seq_group_metadata_list:
123
+ seq_group_has_bonus_tokens = False
124
+ for seq_id, _ in seq_group.seq_data.items():
125
+ # Identify sequences with bonus tokens in the sequence group.
126
+ if seq_id in seq_with_bonus_token_in_last_step:
127
+ seq_group_has_bonus_tokens = True
128
+ break
129
+ if seq_group_has_bonus_tokens:
130
+ #Create new sequences without the last bonus token. These new
131
+ # sequence have the same sequence id as the original sequence.
132
+ # We create a new sequence group and add them there.
133
+ updated_seq_group_without_bonus_token = \
134
+ MultiStepWorker._copy_seq_metadata_excluding_last_token(
135
+ seq_group, seq_with_bonus_token_in_last_step)
136
+ updated_seq_group_metadata_list.append(
137
+ updated_seq_group_without_bonus_token)
138
+ # Add the original sequence group.
139
+ updated_seq_group_metadata_list.append(
140
+ MultiStepWorker._shallow_copy_seq_group_metadata(seq_group))
141
+ # Record the index of the original sequence group.
142
+ indices_of_original_sequence_groups.append(
143
+ len(updated_seq_group_metadata_list) - 1)
144
+
145
+ updated_execute_model_req.seq_group_metadata_list =\
146
+ updated_seq_group_metadata_list
147
+ return updated_execute_model_req, indices_of_original_sequence_groups
148
+
149
+ @staticmethod
150
+ def _filter_model_output(
151
+ expanded_batch_outputs: List[SamplerOutput],
152
+ output_indices_to_retain: List[int]) -> List[SamplerOutput]:
153
+ """
154
+ Filters the model output to include only the specified sequence
155
+ outputs. This method contracts the expanded batch output from the
156
+ model to retain the outputs of only those sequences indicated by the
157
+ provided indices.
158
+
159
+ Args:
160
+ expanded_batch_output (List[SamplerOutput]): The expanded output
161
+ batch from the model.
162
+ output_indices_to_retain (List[int]): Indices of the model outputs
163
+ to retain.
164
+
165
+ Returns:
166
+ List[SamplerOutput]: A list containing the filtered model
167
+ outputs for the specified indices.
168
+ """
169
+ return [
170
+ SamplerOutput(
171
+ outputs=[
172
+ expanded_batch_output.outputs[i]
173
+ for i in output_indices_to_retain
174
+ ],
175
+ sampled_token_probs=(
176
+ expanded_batch_output.
177
+ sampled_token_probs[output_indices_to_retain]
178
+ if expanded_batch_output.sampled_token_probs is not None
179
+ else None),
180
+ logprobs=(
181
+ expanded_batch_output.logprobs[output_indices_to_retain]
182
+ if expanded_batch_output.logprobs is not None else None),
183
+ sampled_token_ids=(expanded_batch_output.
184
+ sampled_token_ids[output_indices_to_retain]
185
+ if expanded_batch_output.sampled_token_ids
186
+ is not None else None))
187
+ for expanded_batch_output in expanded_batch_outputs
188
+ ]
189
190
def get_spec_proposals(
191
self,
192
execute_model_req: ExecuteModelRequest,
193
+ seq_ids_with_bonus_token_in_last_step: set,
194
) -> SpeculativeProposals:
195
"""Produce speculations given an input batch of sequences. The number of
196
speculative tokens per sequence is determined by max_proposal_len.
197
"""
99
-
100
- return self._proposer.get_spec_proposals(execute_model_req)
198
+ return self._proposer.get_spec_proposals(
199
+ execute_model_req, seq_ids_with_bonus_token_in_last_step)
200
201
@staticmethod
202
def _append_new_tokens(
@@ -123,9 +222,8 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
222
seq.update_num_computed_tokens(1)
223
224
@staticmethod
126
- def _shallow_copy_inputs(
127
- seq_group_metadata_list: List[SequenceGroupMetadata]
128
- ) -> List[SequenceGroupMetadata]:
225
+ def _shallow_copy_seq_group_metadata(
226
+ seq_group_metadata: SequenceGroupMetadata, ) -> SequenceGroupMetadata:
227
"""Copy input data structures to remove side-effects when input data
228
structures are shared with other modules.
229
@@ -133,26 +231,62 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
231
The alternative is deep-copying (or other form of deep copy); this has
232
performance downsides.
233
"""
136
-
137
- # Shallow-copy the list of SequenceGroupMetadata. This allows us to
234
+ # Shallow-copy the SequenceGroupMetadata. This allows us to
235
# append tokens and change is_prompt without external side-effects.
139
- new_seq_group_metadata_list: List[SequenceGroupMetadata] = []
236
+ # We must shallow-copy seq_group_metadata as is_prompt could change.
237
+ new_seq_group_metadata = copy.copy(seq_group_metadata)
238
141
- for old_seq_group_metadata in seq_group_metadata_list:
142
- # We must shallow-copy seq_group_metadata as is_prompt could change.
143
- seq_group_metadata = copy.copy(old_seq_group_metadata)
144
- new_seq_group_metadata_list.append(seq_group_metadata)
145
-
146
- # We must shallow-copy seq_data as we will append token ids
147
- new_seq_data: Dict[int, SequenceData] = {}
148
- for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
149
- new_seq_data[seq_id] = copy.copy(old_seq_data)
150
- new_seq_data[
151
- seq_id].output_token_ids = old_seq_data.output_token_ids[:]
239
+ # We must shallow-copy seq_data as we will append token ids
240
+ new_seq_data: Dict[int, SequenceData] = {}
241
+ for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
242
+ new_seq_data[seq_id] = copy.copy(old_seq_data)
243
+ new_seq_data[seq_id].output_token_ids =\
244
+ old_seq_data.output_token_ids[:]
245
153
- seq_group_metadata.seq_data = new_seq_data
246
+ new_seq_group_metadata.seq_data = new_seq_data
247
+ return new_seq_group_metadata
248
155
- return new_seq_group_metadata_list
249
+ @staticmethod
250
+ def _copy_seq_metadata_excluding_last_token(
251
+ seq_group_metadata: SequenceGroupMetadata,
252
+ seq_ids_to_copy: Set[int],
253
+ ) -> SequenceGroupMetadata:
254
+ """
255
+ Creates a shallow copy of the given SequenceGroupMetadata, retaining
256
+ only the sequence IDs specified in seq_ids_to_copy. For each of these
257
+ sequence IDs, all output_token_ids except the last one are copied.
258
+ Sequence IDs not in seq_ids_to_copy are excluded from the copy.
259
+
260
+ Parameters:
261
+ seq_group_metadata (SequenceGroupMetadata): The original sequence
262
+ group metadata.
263
+ seq_ids_to_copy (Set[int]): The set of sequence IDs to include in the
264
+ copy.
265
+
266
+ Returns:
267
+ SequenceGroupMetadata: A shallow copy of the sequence group metadata
268
+ with the specified modifications.
269
+ """
270
+ # Shallow-copy the SequenceGroupMetadata.
271
+ new_seq_group_metadata = copy.copy(seq_group_metadata)
272
+ # Shallow-copy seq_data and modify the output_token_ids.
273
+ new_seq_data: Dict[int, SequenceData] = {}
274
+ for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
275
+ if (seq_id in seq_ids_to_copy):
276
+ new_seq_data[seq_id] = copy.copy(old_seq_data)
277
+ # Copy all the output token ids except the last.
278
+ # Also reduce num_computed_tokens by 1 since we are not
279
+ # including the last output token.
280
+ # NOTE: num_computed_tokens is not directly used by the
281
+ # speculative decoding workers, as it is only relevant for
282
+ # chunked prefill, which is disabled for speculative decoding.
283
+ # However, to maintain consistency in num_computed_tokens,
284
+ # we update it here.
285
+ new_seq_data[seq_id].output_token_ids =\
286
+ old_seq_data.output_token_ids[:-1]
287
+ new_seq_data[seq_id].update_num_computed_tokens(-1)
288
+ new_seq_group_metadata.seq_data = new_seq_data
289
+ return new_seq_group_metadata
290
291
def _assert_enough_kv_space(
292
self, seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/spec_decode/ngram_worker.py
+9
-3
@@ -1,5 +1,5 @@
1
import weakref
2
-from typing import List, Optional, Tuple
2
+from typing import List, Optional, Set, Tuple
3
4
import torch
5
@@ -48,6 +48,9 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
48
self,
49
execute_model_req: ExecuteModelRequest,
50
sample_len: int,
51
+ # Unused parameter. NGramWorker does not use the KV Cache and
52
+ # therefore does not need this parameter.
53
+ seq_ids_with_bonus_token_in_last_step: Set[int],
54
) -> Tuple[Optional[List[Optional[SamplerOutput]]], bool]:
55
"""NGram match algo to pick proposal candidate. Returns the list of
56
sampler output, one per SequenceGroupMetadata.
@@ -133,12 +136,15 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
136
def get_spec_proposals(
137
self,
138
execute_model_req: ExecuteModelRequest,
139
+ # Unused parameter. NGramWorker does not use the KV Cache and
140
+ # therefore does not need this parameter.
141
+ seq_ids_with_bonus_token_in_last_step: Set[int],
142
) -> SpeculativeProposals:
143
"""Produce speculations given an input batch of sequences. The number of
144
speculative tokens per sequence is determined by max_proposal_len.
145
"""
140
-
141
- return self._proposer.get_spec_proposals(execute_model_req)
146
+ return self._proposer.get_spec_proposals(
147
+ execute_model_req, seq_ids_with_bonus_token_in_last_step)
148
149
def _raise_if_unsupported(
150
self,
vllm/spec_decode/proposer_worker_base.py
+8
-1
@@ -1,5 +1,5 @@
1
from abc import ABC, abstractmethod
2
-from typing import List, Optional, Tuple
2
+from typing import List, Optional, Set, Tuple
3
4
from vllm.sequence import ExecuteModelRequest, SamplerOutput
5
from vllm.spec_decode.interfaces import SpeculativeProposer
@@ -14,6 +14,13 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
14
self,
15
execute_model_req: ExecuteModelRequest,
16
sample_len: int,
17
+ # A set containing all sequence IDs that were assigned bonus tokens
18
+ # in their last forward pass. This set is used to backfill the KV cache
19
+ # with the key-value pairs of the penultimate token in the sequences.
20
+ # This parameter is only used by the MultiStepWorker, which relies on
21
+ # the KV cache for token generation. It is not used by workers that
22
+ # do not utilize the KV cache.
23
+ seq_ids_with_bonus_token_in_last_step: Set[int]
24
) -> Tuple[Optional[List[SamplerOutput]], bool]:
25
raise NotImplementedError
26
vllm/spec_decode/smaller_tp_proposer_worker.py
+8
-3
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Set, Tuple
2
3
import torch
4
@@ -110,13 +110,17 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
110
self,
111
execute_model_req: ExecuteModelRequest,
112
sample_len: int,
113
+ seq_ids_with_bonus_token_in_last_step: Set[int],
114
) -> Tuple[List[SamplerOutput], bool]:
115
# Do not check _is_dummy, as it's always called by get_spec_proposals
115
- return self._worker.sampler_output(execute_model_req, sample_len)
116
+ return self._worker.sampler_output(
117
+ execute_model_req, sample_len,
118
+ seq_ids_with_bonus_token_in_last_step)
119
120
def get_spec_proposals(
121
self,
122
execute_model_req: ExecuteModelRequest,
123
+ seq_ids_with_bonus_token_in_last_step: Set[int],
124
) -> SpeculativeProposals:
125
"""Produce speculations given an input batch of sequences. The number of
126
speculative tokens per sequence is determined by max_proposal_len.
@@ -125,7 +129,8 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
129
return SpeculativeProposals(None, None, None)
130
131
with self._patch_tensor_parallel_group():
128
- return self._worker.get_spec_proposals(execute_model_req)
132
+ return self._worker.get_spec_proposals(
133
+ execute_model_req, seq_ids_with_bonus_token_in_last_step)
134
135
def execute_model(
136
self,
vllm/spec_decode/spec_decode_worker.py
+52
-15
@@ -1,5 +1,6 @@
1
+from collections import defaultdict
2
from functools import cached_property
2
-from typing import Any, Dict, List, Optional, Tuple
3
+from typing import Any, Dict, List, Optional, Set, Tuple
4
5
import torch
6
@@ -13,7 +14,7 @@ from vllm.model_executor.layers.typical_acceptance_sampler import (
14
TypicalAcceptanceSampler)
15
from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16
HiddenStates, SamplerOutput, SequenceGroupMetadata,
16
- get_all_seq_ids)
17
+ get_all_seq_ids_and_request_ids)
18
from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
19
from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
20
from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -112,11 +113,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
113
draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
114
ngram_prompt_lookup_min = (
115
draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
115
-
116
- disable_bonus_tokens = True
117
-
116
if ngram_prompt_lookup_max > 0:
119
- disable_bonus_tokens = False
117
proposer_worker = NGramWorker(**draft_worker_kwargs)
118
proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
119
ngram_prompt_lookup_max)
@@ -128,11 +125,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
125
126
if draft_worker_kwargs[
127
"model_config"].hf_config.model_type == "mlp_speculator":
131
- disable_bonus_tokens = False
128
proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
129
elif draft_worker_kwargs[
130
"model_config"].hf_config.model_type == "medusa":
135
- disable_bonus_tokens = False
131
proposer_worker = MedusaWorker(**draft_worker_kwargs)
132
else:
133
if draft_tp == 1:
@@ -149,10 +144,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
144
spec_decode_sampler: SpecDecodeBaseSampler = None
145
if draft_token_acceptance_method == "rejection_sampler":
146
spec_decode_sampler = RejectionSampler(
152
- disable_bonus_tokens=disable_bonus_tokens, )
147
+ disable_bonus_tokens=False, )
148
elif draft_token_acceptance_method == "typical_acceptance_sampler":
149
spec_decode_sampler = TypicalAcceptanceSampler(
155
- disable_bonus_tokens=disable_bonus_tokens,
150
+ disable_bonus_tokens=False,
151
posterior_threshold=\
152
typical_acceptance_sampler_posterior_threshold,
153
posterior_alpha=typical_acceptance_sampler_posterior_alpha,
@@ -200,6 +195,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
195
self._metrics = AsyncMetricsCollector(
196
self.spec_decode_sampler
197
) if metrics_collector is None else metrics_collector
198
+ # Tracks the sequence IDs that received a bonus token ID in
199
+ # their last forward pass. Needed only if KV cache is being
200
+ # used for token generation such as in the case of MultiStepWorker.
201
+ self._seq_with_bonus_token_in_last_step: Set[int] = set()
202
+ # Tracks the currently active request ids and the sequence IDs
203
+ # corresponding to them
204
+ self._request_id_seq_id_mapping: Dict[str, Set[int]] = defaultdict(set)
205
+ # Tracks if the proposer worker uses the KV cache or not.
206
+
207
self.probs_dtype = self.spec_decode_sampler.probs_dtype
208
self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
209
# Lazy initiazliation.
@@ -307,6 +311,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
311
broadcast_tensor_dict({}, src=0)
312
return []
313
314
+ self._track_finished_requests(execute_model_req)
315
disable_all_speculation = self._should_disable_all_speculation(
316
execute_model_req)
317
num_lookahead_slots = execute_model_req.num_lookahead_slots
@@ -453,7 +458,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
458
self.previous_hidden_states = None
459
460
# Generate proposals using draft worker.
456
- proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
461
+ proposals = self.proposer_worker.get_spec_proposals(
462
+ execute_model_req, self._seq_with_bonus_token_in_last_step)
463
464
proposal_scores = self.scorer.score_proposals(
465
execute_model_req,
@@ -585,7 +591,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
591
592
# Get the sequence ids and num_logprobs (sampling parameter) in the
593
# batch.
588
- seq_ids = get_all_seq_ids(seq_group_metadata_list)
594
+ seq_ids, request_ids_seq_ids_mapping = get_all_seq_ids_and_request_ids(
595
+ seq_group_metadata_list)
596
+
597
num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
598
599
# Serialize all tensors to CPU Python lists.
@@ -608,7 +616,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
616
for sequence_index in range(batch_size):
617
# Each sequence may have a different num_logprobs; retrieve it.
618
num_logprobs = num_logprobs_per_seq[sequence_index]
611
-
619
step_output_token_ids.append(
620
create_sequence_group_output(
621
token_id=accepted_token_ids_by_step[step_index]
@@ -623,18 +630,48 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
630
topk_logprobs=topk_logprobs_by_step[step_index]
631
[sequence_index][:num_logprobs],
632
))
626
-
633
sampler_output_list.append(
634
SamplerOutput(outputs=step_output_token_ids))
635
636
+ # Populate the data structures needed to keep track of sequences with
637
+ # bonus tokens.
638
+ self._track_sequences_with_bonus_tokens(seq_ids,
639
+ request_ids_seq_ids_mapping,
640
+ accepted_token_ids_by_step)
641
maybe_rejsample_metrics = (
642
self._metrics.maybe_collect_rejsample_metrics(k))
643
if maybe_rejsample_metrics is not None:
644
sampler_output_list[
645
0].spec_decode_worker_metrics = maybe_rejsample_metrics
635
-
646
return sampler_output_list
647
648
+ def _track_finished_requests(self, execute_model_req: ExecuteModelRequest):
649
+ """
650
+ Removes the finished requests and their associated sequence ids from
651
+ internal book keeping data structures.
652
+ """
653
+ for finished_request in execute_model_req.finished_requests_ids:
654
+ for seq_id in self._request_id_seq_id_mapping[finished_request]:
655
+ self._seq_with_bonus_token_in_last_step.discard(seq_id)
656
+ del self._request_id_seq_id_mapping[finished_request]
657
+
658
+ def _track_sequences_with_bonus_tokens(
659
+ self, seq_ids: List[int],
660
+ request_ids_seq_ids_mapping: Dict[str, Set[int]],
661
+ accepted_token_ids_by_step: List[List[int]]):
662
+ """
663
+ Updates the internal data structures which keep track of sequences
664
+ which have been assigned bonus tokens in their last forward pass.
665
+ """
666
+ for seq_index, seq_id in enumerate(seq_ids):
667
+ last_token_id = accepted_token_ids_by_step[-1][seq_index]
668
+ if last_token_id == -1:
669
+ self._seq_with_bonus_token_in_last_step.discard(seq_id)
670
+ else:
671
+ self._seq_with_bonus_token_in_last_step.add(seq_id)
672
+ for request_id, sequences in request_ids_seq_ids_mapping.items():
673
+ self._request_id_seq_id_mapping[request_id].update(sequences)
674
+
675
@cached_property
676
def _vocab_size(self) -> int:
677
"""Get the vocab size of the model and make sure it's consistent between
vllm/spec_decode/top1_proposer.py
+4
-1
@@ -1,4 +1,4 @@
1
-from typing import List, Optional, Tuple
1
+from typing import List, Optional, Set, Tuple
2
3
import torch
4
@@ -42,6 +42,7 @@ class Top1Proposer(SpeculativeProposer):
42
def get_spec_proposals(
43
self,
44
execute_model_req: ExecuteModelRequest,
45
+ seq_ids_with_bonus_token_in_last_step: Set[int],
46
) -> SpeculativeProposals:
47
"""Get speculative proposals given the input batch.
48
@@ -76,6 +77,8 @@ class Top1Proposer(SpeculativeProposer):
77
maybe_sampler_output, transposed = self._worker.sampler_output(
78
execute_model_req=nonzero_execute_model_req,
79
sample_len=proposal_len,
80
+ seq_ids_with_bonus_token_in_last_step=\
81
+ seq_ids_with_bonus_token_in_last_step,
82
)
83
(
84
proposal_lens,