@tduncan / Nvidia-Optimization-Stack-VLLM Mirror · read-only / commits / ae151d73be
Fork 0

A high-throughput and memory-efficient inference and serving engine for LLMs

Mirrored from https://github.com/tduncan-lum/Nvidia-Optimization-Stack---VLLM.git

Synced Nov 16, 22:00 UTC

[Speculative Decoding] Enabling bonus token in speculative decoding for KV cache based models (#5765)

sroy745 committed Jul 10, 2024 at 16:02 UTC ae151d73be479e9c0caa2fdfc30b17f073018ef3
14 files changed +645 -80
tests/spec_decode/test_dynamic_spec_decode.py
+7 -4
@@ -70,14 +70,17 @@ def test_disable_spec_tokens(queue_size: int, batch_size: int, k: int,
70 if queue_size < disable_by_batch_size:
71 # Should raise exception when executing the mocked draft model.
72 with pytest.raises(ValueError, match=exception_secret):
73 - proposer.get_spec_proposals(execute_model_req=ExecuteModelRequest(
74 - seq_group_metadata_list=seq_group_metadata_list,
75 - num_lookahead_slots=k), )
73 + proposer.get_spec_proposals(
74 + execute_model_req=ExecuteModelRequest(
75 + seq_group_metadata_list=seq_group_metadata_list,
76 + num_lookahead_slots=k),
77 + seq_ids_with_bonus_token_in_last_step=set())
78 else:
79 # Should not execute the draft model because spec decode is disabled
80 # for all requests. Accordingly, the proposal length should be 0.
81 proposals = proposer.get_spec_proposals(
82 execute_model_req=ExecuteModelRequest(
83 seq_group_metadata_list=seq_group_metadata_list,
82 - num_lookahead_slots=k), )
84 + num_lookahead_slots=k),
85 + seq_ids_with_bonus_token_in_last_step=set())
86 assert proposals.proposal_lens.tolist() == [0] * batch_size
tests/spec_decode/test_multi_step_worker.py
+207 -5
@@ -118,7 +118,8 @@ def test_same_output_for_single_step():
118 actual_output, _ = multi_step_worker.sampler_output(
119 execute_model_req=ExecuteModelRequest(
120 seq_group_metadata_list=multi_step_seq_group),
121 - sample_len=num_steps)
121 + sample_len=num_steps,
122 + seq_ids_with_bonus_token_in_last_step=set())
123 assert len(actual_output) == num_steps
124 actual_output = actual_output[0]
125
@@ -210,7 +211,8 @@ def test_same_output_for_multi_step():
211 multi_step_output, _ = multi_step_worker.sampler_output(
212 execute_model_req=ExecuteModelRequest(
213 seq_group_metadata_list=seq_group_metadata_list),
213 - sample_len=num_steps)
214 + sample_len=num_steps,
215 + seq_ids_with_bonus_token_in_last_step=set())
216
217 # Run single-step repeatedly.
218 zero_kv_cache(worker.cache_engine)
@@ -277,6 +279,203 @@ def test_same_output_for_multi_step():
279 single_step_logprobs)
280
281
282 +@torch.inference_mode()
283 +def test_multi_step_with_batch_expansion_correct_output():
284 + """
285 + In this test we verify that the MultiStepWorker is able to handle bonus
286 + tokens correctly. The test verifies that if a sequence has a
287 + bonus token then the MultiStepWorker is able to expand the batch by adding
288 + new sequences corresponding to the sequences with bonus tokens. The
289 + expanded batch is then used for predicting the next tokens.
290 + """
291 + seed = 100
292 + model_name = 'JackFram/llama-68m'
293 +
294 + block_size = 16
295 + num_gpu_blocks = 2048 // block_size
296 + batch_size = 128
297 + multi_step_worker = create_worker(
298 + MultiStepWorker,
299 + model_name,
300 + block_size,
301 + num_gpu_blocks,
302 + seed,
303 + model_runner_cls=TP1DraftModelRunner,
304 + )
305 + worker = create_worker(
306 + Worker,
307 + model_name,
308 + block_size,
309 + num_gpu_blocks,
310 + seed,
311 + )
312 + random.seed(seed)
313 + prompts = [[0] for _ in range(batch_size)]
314 + num_steps = 2
315 + final_prompt_lens = [(num_steps + 1) for prompt in prompts]
316 + rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
317 + multi_step_worker.execute_model = patch_execute_model_with_seeds(
318 + multi_step_worker, rand_seeds)
319 + worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
320 + # Create the test continuations
321 + continuations = [[random.randint(0, 1000)] for _ in prompts]
322 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
323 + prompts,
324 + num_gpu_blocks,
325 + block_size,
326 + continuations=continuations,
327 + final_prompt_lens=final_prompt_lens)
328 +
329 + # Run single-step twice to generate 2 tokens. This
330 + # will simulate the bonus token case with the second token
331 + # being the bonus token.
332 + zero_kv_cache(worker.cache_engine)
333 + single_step_output: List[SamplerOutput] = []
334 + set_random_seed(seed)
335 + for _ in range(num_steps):
336 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
337 + prompts,
338 + num_gpu_blocks,
339 + block_size,
340 + continuations=continuations,
341 + final_prompt_lens=final_prompt_lens)
342 + single_step_output.extend(
343 + worker.execute_model(execute_model_req=ExecuteModelRequest(
344 + seq_group_metadata_list=seq_group_metadata_list)))
345 + # Append output tokens to new sequence data.
346 + for i, seq_group_output in enumerate(single_step_output[-1]):
347 + continuations[i].append(seq_group_output.samples[0].output_token)
348 +
349 + # Create continuations for the MultiStepWorker. The continuations have
350 + # 2 tokens in order to simulate the bonus token case.
351 + multi_step_continuations = []
352 + for continuation in continuations:
353 + multi_step_continuations.append(continuation[:2])
354 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
355 + prompts,
356 + num_gpu_blocks,
357 + block_size,
358 + continuations=multi_step_continuations,
359 + final_prompt_lens=final_prompt_lens)
360 +
361 + # Run multi-step and verify that the third token prediction is accurate
362 + # for all sequences.
363 + zero_kv_cache(multi_step_worker.cache_engine)
364 + all_seq_ids = {i for i in range(batch_size)}
365 + multi_step_output, _ = multi_step_worker.sampler_output(
366 + execute_model_req=ExecuteModelRequest(
367 + seq_group_metadata_list=seq_group_metadata_list),
368 + sample_len=1,
369 + seq_ids_with_bonus_token_in_last_step=all_seq_ids)
370 + for index, output in enumerate(multi_step_output[-1].outputs):
371 + assert (continuations[index][-1] == output.samples[0].output_token)
372 +
373 +
374 +@torch.inference_mode()
375 +def test_multi_step_with_batch_expansion_incorrect_output():
376 + """
377 + Tests the MultiStepWorker's ability to handle batch expansion with bonus
378 + tokens in a negative case scenario. This test provides the MultiStepWorker
379 + with a batch containing sequences with bonus tokens but specifies the
380 + sequence IDs with bonus tokens incorrectly. The test verifies that the
381 + MultiStepWorker generates correct tokens for the sequences where the
382 + sequence ID is specified correctly and incorrect tokens for those where
383 + the sequence ID is specified incorrectly.
384 + """
385 + seed = 100
386 + model_name = 'JackFram/llama-68m'
387 +
388 + block_size = 16
389 + num_gpu_blocks = 2048 // block_size
390 + batch_size = 128
391 + multi_step_worker = create_worker(
392 + MultiStepWorker,
393 + model_name,
394 + block_size,
395 + num_gpu_blocks,
396 + seed,
397 + model_runner_cls=TP1DraftModelRunner,
398 + )
399 + worker = create_worker(
400 + Worker,
401 + model_name,
402 + block_size,
403 + num_gpu_blocks,
404 + seed,
405 + )
406 + random.seed(seed)
407 + prompts = [[0] for _ in range(batch_size)]
408 + num_steps = 2
409 + final_prompt_lens = [(num_steps + 1) for prompt in prompts]
410 + rand_seeds = list(random.randint(0, 100) for _ in range(num_steps))
411 + multi_step_worker.execute_model = patch_execute_model_with_seeds(
412 + multi_step_worker, rand_seeds)
413 + worker.execute_model = patch_execute_model_with_seeds(worker, rand_seeds)
414 + # Create the test continuations
415 + continuations = [[random.randint(0, 1000)] for _ in prompts]
416 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
417 + prompts,
418 + num_gpu_blocks,
419 + block_size,
420 + continuations=continuations,
421 + final_prompt_lens=final_prompt_lens)
422 + # Run single-step twice to generate 2 tokens. This
423 + # will simulate the bonus token case with the second token
424 + # being the bonus token.
425 + zero_kv_cache(worker.cache_engine)
426 + single_step_output: List[SamplerOutput] = []
427 + set_random_seed(seed)
428 + for _ in range(num_steps):
429 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
430 + prompts,
431 + num_gpu_blocks,
432 + block_size,
433 + continuations=continuations,
434 + final_prompt_lens=final_prompt_lens)
435 + single_step_output.extend(
436 + worker.execute_model(execute_model_req=ExecuteModelRequest(
437 + seq_group_metadata_list=seq_group_metadata_list)))
438 + # Append output tokens to new sequence data.
439 + for i, seq_group_output in enumerate(single_step_output[-1]):
440 + continuations[i].append(seq_group_output.samples[0].output_token)
441 +
442 + # Create continuations for the MultiStepWorker. The continuations have
443 + # 2 tokens in order to simulate the bonus token case.
444 + multi_step_continuations = []
445 + for continuation in continuations:
446 + multi_step_continuations.append(continuation[:2])
447 + seq_group_metadata_list = create_seq_group_metadata_from_prompts(
448 + prompts,
449 + num_gpu_blocks,
450 + block_size,
451 + continuations=multi_step_continuations,
452 + final_prompt_lens=final_prompt_lens)
453 +
454 + # Run multi-step. In this run INCORRECTLY specify that only the odd number
455 + # sequences have bonus tokens. Verify that with this setting the third token
456 + # prediction is accurate only for the odd numbered sequences. Also verify
457 + # that the prediction might be wrong for some of the even numbered
458 + # sequences.
459 + zero_kv_cache(multi_step_worker.cache_engine)
460 + set_random_seed(seed)
461 + odd_seq_ids = {i for i in range(batch_size) if i % 2 != 0}
462 + multi_step_output, _ = multi_step_worker.sampler_output(
463 + execute_model_req=ExecuteModelRequest(
464 + seq_group_metadata_list=seq_group_metadata_list),
465 + sample_len=1,
466 + seq_ids_with_bonus_token_in_last_step=odd_seq_ids)
467 + num_mismatch = 0
468 + for index, output in enumerate(multi_step_output[-1].outputs):
469 + if (index % 2) != 0:
470 + assert (continuations[index][-1] == output.samples[0].output_token)
471 + elif (continuations[index][-1] != output.samples[0].output_token):
472 + num_mismatch += 1
473 + # The prediction is accurate for some of the sequences even without proper
474 + # handling of the bonus tokens. Hence verify that the number of sequences
475 + # for which there is a mismatch is > 0.
476 + assert (num_mismatch > 0)
477 +
478 +
479 @torch.inference_mode()
480 def test_draft_proposals_full_speculation_len():
481 """Verify Top1Proposer correctly handles case where all sequences
@@ -318,7 +517,8 @@ def test_draft_proposals_full_speculation_len():
517 proposals = proposer.get_spec_proposals(
518 execute_model_req=ExecuteModelRequest(
519 seq_group_metadata_list=seq_group_metadata_list,
321 - num_lookahead_slots=k), )
520 + num_lookahead_slots=k),
521 + seq_ids_with_bonus_token_in_last_step=set())
522
523 assert torch.is_tensor(proposals.proposal_token_ids)
524 assert torch.is_tensor(proposals.proposal_probs)
@@ -356,7 +556,8 @@ def test_draft_proposals_no_speculations():
556 proposals = proposer.get_spec_proposals(
557 execute_model_req=ExecuteModelRequest(
558 seq_group_metadata_list=seq_group_metadata_list,
359 - num_lookahead_slots=k), )
559 + num_lookahead_slots=k),
560 + seq_ids_with_bonus_token_in_last_step=set())
561
562 assert torch.is_tensor(proposals.proposal_token_ids)
563 assert torch.is_tensor(proposals.proposal_probs)
@@ -428,7 +629,8 @@ def test_draft_proposals_mixed_k():
629 proposals = proposer.get_spec_proposals(
630 execute_model_req=ExecuteModelRequest(
631 seq_group_metadata_list=seq_group_metadata_list,
431 - num_lookahead_slots=k), )
632 + num_lookahead_slots=k),
633 + seq_ids_with_bonus_token_in_last_step=set())
634
635 assert torch.is_tensor(proposals.proposal_token_ids)
636 assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_ngram_worker.py
+6 -3
@@ -53,7 +53,8 @@ def test_ngram_algo_correctness_for_single_no_match():
53 proposals = proposer.get_spec_proposals(
54 execute_model_req=ExecuteModelRequest(
55 seq_group_metadata_list=seq_group_metadata_list,
56 - num_lookahead_slots=proposal_len), )
56 + num_lookahead_slots=proposal_len),
57 + seq_ids_with_bonus_token_in_last_step=None)
58
59 assert torch.is_tensor(proposals.proposal_token_ids)
60 assert torch.is_tensor(proposals.proposal_probs)
@@ -121,7 +122,8 @@ def test_ngram_algo_correctness_for_batches_not_match_all():
122 proposals = proposer.get_spec_proposals(
123 execute_model_req=ExecuteModelRequest(
124 seq_group_metadata_list=seq_group_metadata_list,
124 - num_lookahead_slots=proposal_len), )
125 + num_lookahead_slots=proposal_len),
126 + seq_ids_with_bonus_token_in_last_step=None)
127
128 assert torch.is_tensor(proposals.proposal_token_ids)
129 assert torch.is_tensor(proposals.proposal_probs)
@@ -193,7 +195,8 @@ def test_ngram_algo_correctness_for_batches_match_all():
195 proposals = proposer.get_spec_proposals(
196 execute_model_req=ExecuteModelRequest(
197 seq_group_metadata_list=seq_group_metadata_list,
196 - num_lookahead_slots=proposal_len), )
198 + num_lookahead_slots=proposal_len),
199 + seq_ids_with_bonus_token_in_last_step=None)
200
201 assert torch.is_tensor(proposals.proposal_token_ids)
202 assert torch.is_tensor(proposals.proposal_probs)
tests/spec_decode/test_spec_decode_worker.py
+143 -4
@@ -1,6 +1,7 @@
1 import random
2 +from collections import defaultdict
3 from types import SimpleNamespace
3 -from typing import Dict, List
4 +from typing import Dict, List, Set
5 from unittest.mock import MagicMock
6
7 import pytest
@@ -377,8 +378,10 @@ def test_collects_metrics(k: int, batch_size: int, returns_metrics: bool,
378
379 set_random_seed(1)
380
380 - worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
381 - metrics_collector)
381 + worker = SpecDecodeWorker(draft_worker,
382 + target_worker,
383 + spec_decode_sampler,
384 + metrics_collector=metrics_collector)
385 worker.init_device()
386
387 proposal_token_ids = torch.randint(low=0,
@@ -554,7 +557,6 @@ def test_init_device(acceptance_sampler_method: str):
557
558 worker = SpecDecodeWorker(draft_worker, target_worker, spec_decode_sampler,
559 metrics_collector)
557 -
560 worker.init_device()
561
562 draft_worker.init_device.assert_called_once()
@@ -645,3 +647,140 @@ def test_split_num_cache_blocks_evenly(available_gpu_blocks: int,
647 assert (num_blocks * target_cache_block_size_bytes) + (
648 num_blocks * draft_kv_size_bytes) <= (available_gpu_blocks *
649 target_cache_block_size_bytes)
650 +
651 +
652 +@torch.inference_mode()
653 +def test_populate_seq_ids_with_bonus_tokens():
654 + """
655 + Verify that a call to _create_output_sampler_list correctly updates
656 + seq_with_bonus_token_in_last_step.
657 +
658 + seq_with_bonus_token_in_last_step is an internal data structure in
659 + SpecDecodeWorker that tracks the sequence IDs which are assigned bonus
660 + tokens by the target model in their last forward pass. This state is
661 + maintained only for models relying on the KV cache, such as those using
662 + the MultiStepWorker.
663 + """
664 + batch_size = 10
665 + k = 5
666 + vocab_size = 10000
667 + num_sequences_with_bonus_tokens = 5
668 + target_worker = mock_worker(vocab_size=vocab_size, use_spec=False)
669 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
670 + target_worker.execute_model.return_value = [MagicMock(spec=SamplerOutput)]
671 + target_worker.device = 'cuda'
672 +
673 + set_random_seed(1)
674 + draft_worker = mock_worker(cls=MultiStepWorker)
675 + draft_worker.device = 'cuda'
676 + # The sequence_ids attached to each sequence in the batch.
677 + # The sequence at index i has seq_id assigned_seq_ids[i]
678 + assigned_seq_ids = list(range(batch_size))
679 + seq_group_metadata_list, _, _ = create_batch(batch_size,
680 + k,
681 + seq_ids=assigned_seq_ids,
682 + prev_output_token_len=10)
683 + target_token_logprobs = torch.rand(batch_size, (k + 1),
684 + vocab_size,
685 + dtype=torch.float32,
686 + device='cuda')
687 + accepted_token_ids = torch.randint(low=0,
688 + high=vocab_size,
689 + size=(batch_size, (k + 1)),
690 + dtype=torch.int64,
691 + device='cuda')
692 + expected_request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)
693 + for seq_group_metadata in seq_group_metadata_list:
694 + for seq_id in seq_group_metadata.seq_data:
695 + expected_request_id_seq_ids_mapping[
696 + seq_group_metadata.request_id].add(seq_id)
697 + # Generate a random sample of sequence indexes with bonus tokens
698 + seq_indexes_with_bonus_tokens = random.sample(
699 + range(batch_size), num_sequences_with_bonus_tokens)
700 + # Create a mask that is True for indices in seq_indexes_with_bonus_tokens
701 + mask = torch.ones(batch_size, dtype=torch.bool, device='cuda')
702 + mask[seq_indexes_with_bonus_tokens] = False
703 + # Set the last token ID to -1 for all indices not in
704 + # seq_indexes_with_bonus_tokens to indicate the lack of bonus token in
705 + # those indices.
706 + accepted_token_ids[mask, -1:] = -1
707 + worker = SpecDecodeWorker(draft_worker,
708 + target_worker,
709 + mock_spec_decode_sampler("rejection_sampler"),
710 + metrics_collector=metrics_collector)
711 + # Initialize _seq_with_bonus_token_in_last_step with a set of sequence IDs.
712 + # This set includes all sequence IDs in the batch as well as an additional
713 + # `num_extra_sequence_ids` sequence IDs. Note that the sequence IDs are in
714 + # the range [0, batch_size + num_extra_sequence_ids).
715 + num_extra_sequence_ids = 10
716 + worker._seq_with_bonus_token_in_last_step = set(
717 + range(batch_size + num_extra_sequence_ids))
718 + worker._create_output_sampler_list(
719 + seq_group_metadata_list=seq_group_metadata_list,
720 + accepted_token_ids=accepted_token_ids,
721 + target_logprobs=target_token_logprobs,
722 + k=k)
723 + # Verify that _seq_with_bonus_token_in_last_step contains the following:
724 + # 1. Sequence IDs that were already present in
725 + # _seq_with_bonus_token_in_last_step but were not part of the current
726 + # batch are retained.
727 + # 2. Of the sequence IDs present in the current batch, only those with a
728 + # bonus token are retained in _seq_with_bonus_token_in_last_step.
729 + # Sequence IDs that are present in the current batch but do not have
730 + # bonus tokens are removed from _seq_with_bonus_token_in_last_step.
731 + expected_seq_ids_with_bonus_tokens = \
732 + set([assigned_seq_ids[i] for i in seq_indexes_with_bonus_tokens])
733 + additional_sequence_ids = \
734 + set(range(batch_size, batch_size + num_extra_sequence_ids))
735 + assert worker._seq_with_bonus_token_in_last_step == \
736 + expected_seq_ids_with_bonus_tokens.union(additional_sequence_ids)
737 + assert worker._request_id_seq_id_mapping == \
738 + expected_request_id_seq_ids_mapping
739 +
740 +
741 +@torch.inference_mode()
742 +def test_handle_finished_requests():
743 + """
744 + Test to verify that finished request IDs are appropriately processed to
745 + update the internal state of the SpecDecodeWorker.
746 +
747 + This test initializes the SpecDecodeWorker with mock data, marks certain
748 + requests as finished, and ensures that the corresponding sequence IDs are
749 + correctly removed from the internal mappings.
750 + """
751 + batch_size = 32
752 + k = 3
753 + draft_worker = mock_worker(cls=MultiStepWorker)
754 + target_worker = mock_worker()
755 + metrics_collector = MagicMock(spec=AsyncMetricsCollector)
756 + worker = SpecDecodeWorker(draft_worker, target_worker,
757 + mock_spec_decode_sampler("rejection_sampler"),
758 + metrics_collector)
759 + # Initialize the request_id_seq_id_mapping mapping dict with a few fake
760 + # request ids and corresponding sequence ids.
761 + worker._request_id_seq_id_mapping = \
762 + {'request-1': {1,2,3}, 'request-2': {4,5,6,7},
763 + 'request-3': {8,9}, 'request-4': {10,11}}
764 + # Initialize seq_with_bonus_token_in_last_step with a few fake
765 + # sequence ids.
766 + worker._seq_with_bonus_token_in_last_step = {1, 4, 5, 8, 9, 10}
767 + exception_secret = 'artificial stop'
768 + draft_worker.get_spec_proposals.side_effect = ValueError(exception_secret)
769 +
770 + seq_group_metadata_list, _, _ = create_batch(batch_size, k)
771 + # Mark requests with ids request-1 and request-3 as finished.
772 + execute_model_req = ExecuteModelRequest(
773 + seq_group_metadata_list=seq_group_metadata_list,
774 + num_lookahead_slots=k,
775 + finished_requests_ids=['request-1', 'request-3'])
776 +
777 + with pytest.raises(ValueError, match=exception_secret):
778 + worker.execute_model(execute_model_req=execute_model_req)
779 + # Verify that request-1 and request-3 are removed from
780 + # request_id_seq_id_mapping
781 + assert worker._request_id_seq_id_mapping == \
782 + {'request-2': {4,5,6,7}, 'request-4': {10,11}}
783 + # Verify that all sequence ids corresponding to 'request-1'
784 + # and 'request-3' are removed from seq_with_bonus_token_in_last_step.
785 + assert worker._seq_with_bonus_token_in_last_step == \
786 + {4,5,10}
vllm/sequence.py
+17 -1
@@ -3,8 +3,9 @@ import copy
3 import enum
4 import math
5 from abc import ABC, abstractmethod
6 +from collections import defaultdict
7 from dataclasses import dataclass, field
7 -from typing import TYPE_CHECKING, Dict, List, Optional, Tuple, Union
8 +from typing import TYPE_CHECKING, Dict, List, Optional, Set, Tuple, Union
9
10 import torch
11
@@ -916,6 +917,21 @@ def get_all_seq_ids(
917 return [seq_id for sg in seq_group_metadata_list for seq_id in sg.seq_data]
918
919
920 +def get_all_seq_ids_and_request_ids(
921 + seq_group_metadata_list: List[SequenceGroupMetadata]
922 +) -> Tuple[List[int], Dict[str, Set[int]]]:
923 + """Given a list of SequenceGroupMetadata, create a list of all
924 + sequence ids.
925 + """
926 + seq_ids: List[int] = []
927 + request_id_seq_ids_mapping: Dict[str, Set[int]] = defaultdict(set)
928 + for sg in seq_group_metadata_list:
929 + for seq_id in sg.seq_data:
930 + seq_ids.append(seq_id)
931 + request_id_seq_ids_mapping[sg.request_id].add(seq_id)
932 + return seq_ids, request_id_seq_ids_mapping
933 +
934 +
935 class HiddenStates:
936 """Hidden states corresponding to in-progress sequences.
937 Used in speculative decoding to pass hidden states from
vllm/spec_decode/interfaces.py
+4 -1
@@ -1,6 +1,6 @@
1 from abc import ABC, abstractmethod
2 from dataclasses import dataclass
3 -from typing import Optional
3 +from typing import Optional, Set
4
5 import torch
6
@@ -62,6 +62,9 @@ class SpeculativeProposer(ABC):
62 def get_spec_proposals(
63 self,
64 execute_model_req: ExecuteModelRequest,
65 + # If set, this contains all sequence IDs that were assigned
66 + # bonus tokens in their last forward pass.
67 + seq_ids_with_bonus_token_in_last_step: Set[int],
68 ) -> SpeculativeProposals:
69 raise NotImplementedError
70
vllm/spec_decode/medusa_worker.py
+6 -2
@@ -1,5 +1,5 @@
1 import weakref
2 -from typing import List, Optional, Tuple
2 +from typing import List, Optional, Set, Tuple
3
4 import torch
5
@@ -40,6 +40,8 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
40 self,
41 execute_model_req: ExecuteModelRequest,
42 sample_len: int,
43 + # Unused parameter.
44 + seq_ids_with_bonus_token_in_last_step: Set[int],
45 ) -> Tuple[List[SamplerOutput], bool]:
46 """Run the model forward pass to generate sample_len future tokens.
47 Returns the list of sampler output, one per layer, along with indicator
@@ -97,12 +99,14 @@ class MedusaWorker(NonLLMProposerWorkerBase, Worker):
99 def get_spec_proposals(
100 self,
101 execute_model_req: ExecuteModelRequest,
102 + seq_ids_with_bonus_token_in_last_step: Set[int],
103 ) -> SpeculativeProposals:
104 """Produce speculations given an input batch of sequences. The number of
105 speculative tokens per sequence is determined by max_proposal_len.
106 """
107
105 - return self._proposer.get_spec_proposals(execute_model_req)
108 + return self._proposer.get_spec_proposals(
109 + execute_model_req, seq_ids_with_bonus_token_in_last_step)
110
111 def _raise_if_unsupported(
112 self,
vllm/spec_decode/mlp_speculator_worker.py
+4 -1
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Set, Tuple
2
3 import torch
4
@@ -20,6 +20,9 @@ class MLPSpeculatorWorker(NonLLMProposerWorkerBase, MultiStepWorker):
20 self,
21 execute_model_req: ExecuteModelRequest,
22 sample_len: int,
23 + # Unused parameter. MLPSpeculatorWorker does not use the KV Cache and
24 + # therefore does not need this parameter.
25 + seq_ids_with_bonus_token_in_last_step: Set[int],
26 ) -> Tuple[List[SamplerOutput], bool]:
27 """Run the model forward pass to generate sample_len future tokens.
28 Returns the list of sampler output, one per layer, along with indicator
vllm/spec_decode/multi_step_worker.py
+170 -36
@@ -1,6 +1,6 @@
1 import copy
2 import weakref
3 -from typing import Dict, List, Tuple
3 +from typing import Dict, List, Set, Tuple
4
5 import torch
6
@@ -51,6 +51,7 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
51 self,
52 execute_model_req: ExecuteModelRequest,
53 sample_len: int,
54 + seq_ids_with_bonus_token_in_last_step: Set[int],
55 ) -> Tuple[List[SamplerOutput], bool]:
56 """Run the model forward pass sample_len times. Returns the list of
57 sampler output, one per model forward pass, along with indicator of
@@ -60,44 +61,142 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
61 For multi step worker, this indicator shall be True.
62 """
63 self._raise_if_unsupported(execute_model_req)
63 -
64 - # Shallow copy input data so modifications (such as appending tokens)
65 - # do not cause side-effects.
66 - copied_seq_group_metadata_list = self._shallow_copy_inputs(
67 - execute_model_req.seq_group_metadata_list)
68 - copied_execute_model_req = execute_model_req.clone(
69 - copied_seq_group_metadata_list)
70 -
64 + # Expand the batch for sequences with a bonus token.
65 + # Perform a forward pass on the expanded batch and filter the
66 + # response to retain only the original sequences' responses.
67 + expanded_request, indices_of_seq_with_bonus_tokens =\
68 + self._expand_execute_model_request(
69 + execute_model_req, seq_ids_with_bonus_token_in_last_step)
70 # Run model sample_len times.
71 model_outputs: List[SamplerOutput] = []
72 if isinstance(self.model_runner, TP1DraftModelRunner):
74 - copied_execute_model_req.num_steps = sample_len
73 + expanded_request.num_steps = sample_len
74 model_outputs = self.execute_model(
76 - execute_model_req=copied_execute_model_req)
75 + execute_model_req=expanded_request)
76 else:
77 # TODO: Remove this branch once DraftModelRunner supports TP>1.
78 for _ in range(sample_len):
79 model_output: List[SamplerOutput] = super().execute_model(
81 - execute_model_req=copied_execute_model_req)
80 + execute_model_req=expanded_request)
81 assert (len(model_output) == 1
82 ), "composing multistep workers not supported"
83 model_output = model_output[0]
84
86 - self._append_new_tokens(model_output,
87 - copied_seq_group_metadata_list)
85 + self._append_new_tokens(
86 + model_output, expanded_request.seq_group_metadata_list)
87 model_outputs.append(model_output)
88
90 - return model_outputs, True
89 + filtered_model_outputs = self._filter_model_output(
90 + model_outputs, indices_of_seq_with_bonus_tokens)
91 + return filtered_model_outputs, True
92 +
93 + @staticmethod
94 + def _expand_execute_model_request(
95 + execute_model_req: ExecuteModelRequest,
96 + seq_with_bonus_token_in_last_step: set,
97 + ) -> Tuple[ExecuteModelRequest, List[int]]:
98 + """
99 + Expands the execute model request based on sequences with bonus
100 + tokens.
101 +
102 + For each sequence with a bonus token, this method creates a new
103 + sequence without the bonus token and adds it to the execute model
104 + request. The original sequence groups are also retained. The indices
105 + of the original sequence groups are returned for further processing.
106 +
107 + Args:
108 + execute_model_req (ExecuteModelRequest): The original execute
109 + model request.
110 + seq_with_bonus_token_in_last_step (set): Set of sequence IDs that
111 + contain bonus tokens.
112 +
113 + Returns:
114 + Tuple[ExecuteModelRequest, List[int]]: The updated execute model
115 + request with expanded sequences and a list of indices corresponding
116 + to the original sequence groups.
117 + """
118 + updated_seq_group_metadata_list: List[SequenceGroupMetadata] = []
119 + updated_execute_model_req = execute_model_req.clone(
120 + updated_seq_group_metadata_list)
121 + indices_of_original_sequence_groups = []
122 + for seq_group in execute_model_req.seq_group_metadata_list:
123 + seq_group_has_bonus_tokens = False
124 + for seq_id, _ in seq_group.seq_data.items():
125 + # Identify sequences with bonus tokens in the sequence group.
126 + if seq_id in seq_with_bonus_token_in_last_step:
127 + seq_group_has_bonus_tokens = True
128 + break
129 + if seq_group_has_bonus_tokens:
130 + #Create new sequences without the last bonus token. These new
131 + # sequence have the same sequence id as the original sequence.
132 + # We create a new sequence group and add them there.
133 + updated_seq_group_without_bonus_token = \
134 + MultiStepWorker._copy_seq_metadata_excluding_last_token(
135 + seq_group, seq_with_bonus_token_in_last_step)
136 + updated_seq_group_metadata_list.append(
137 + updated_seq_group_without_bonus_token)
138 + # Add the original sequence group.
139 + updated_seq_group_metadata_list.append(
140 + MultiStepWorker._shallow_copy_seq_group_metadata(seq_group))
141 + # Record the index of the original sequence group.
142 + indices_of_original_sequence_groups.append(
143 + len(updated_seq_group_metadata_list) - 1)
144 +
145 + updated_execute_model_req.seq_group_metadata_list =\
146 + updated_seq_group_metadata_list
147 + return updated_execute_model_req, indices_of_original_sequence_groups
148 +
149 + @staticmethod
150 + def _filter_model_output(
151 + expanded_batch_outputs: List[SamplerOutput],
152 + output_indices_to_retain: List[int]) -> List[SamplerOutput]:
153 + """
154 + Filters the model output to include only the specified sequence
155 + outputs. This method contracts the expanded batch output from the
156 + model to retain the outputs of only those sequences indicated by the
157 + provided indices.
158 +
159 + Args:
160 + expanded_batch_output (List[SamplerOutput]): The expanded output
161 + batch from the model.
162 + output_indices_to_retain (List[int]): Indices of the model outputs
163 + to retain.
164 +
165 + Returns:
166 + List[SamplerOutput]: A list containing the filtered model
167 + outputs for the specified indices.
168 + """
169 + return [
170 + SamplerOutput(
171 + outputs=[
172 + expanded_batch_output.outputs[i]
173 + for i in output_indices_to_retain
174 + ],
175 + sampled_token_probs=(
176 + expanded_batch_output.
177 + sampled_token_probs[output_indices_to_retain]
178 + if expanded_batch_output.sampled_token_probs is not None
179 + else None),
180 + logprobs=(
181 + expanded_batch_output.logprobs[output_indices_to_retain]
182 + if expanded_batch_output.logprobs is not None else None),
183 + sampled_token_ids=(expanded_batch_output.
184 + sampled_token_ids[output_indices_to_retain]
185 + if expanded_batch_output.sampled_token_ids
186 + is not None else None))
187 + for expanded_batch_output in expanded_batch_outputs
188 + ]
189
190 def get_spec_proposals(
191 self,
192 execute_model_req: ExecuteModelRequest,
193 + seq_ids_with_bonus_token_in_last_step: set,
194 ) -> SpeculativeProposals:
195 """Produce speculations given an input batch of sequences. The number of
196 speculative tokens per sequence is determined by max_proposal_len.
197 """
99 -
100 - return self._proposer.get_spec_proposals(execute_model_req)
198 + return self._proposer.get_spec_proposals(
199 + execute_model_req, seq_ids_with_bonus_token_in_last_step)
200
201 @staticmethod
202 def _append_new_tokens(
@@ -123,9 +222,8 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
222 seq.update_num_computed_tokens(1)
223
224 @staticmethod
126 - def _shallow_copy_inputs(
127 - seq_group_metadata_list: List[SequenceGroupMetadata]
128 - ) -> List[SequenceGroupMetadata]:
225 + def _shallow_copy_seq_group_metadata(
226 + seq_group_metadata: SequenceGroupMetadata, ) -> SequenceGroupMetadata:
227 """Copy input data structures to remove side-effects when input data
228 structures are shared with other modules.
229
@@ -133,26 +231,62 @@ class MultiStepWorker(Worker, ProposerWorkerBase):
231 The alternative is deep-copying (or other form of deep copy); this has
232 performance downsides.
233 """
136 -
137 - # Shallow-copy the list of SequenceGroupMetadata. This allows us to
234 + # Shallow-copy the SequenceGroupMetadata. This allows us to
235 # append tokens and change is_prompt without external side-effects.
139 - new_seq_group_metadata_list: List[SequenceGroupMetadata] = []
236 + # We must shallow-copy seq_group_metadata as is_prompt could change.
237 + new_seq_group_metadata = copy.copy(seq_group_metadata)
238
141 - for old_seq_group_metadata in seq_group_metadata_list:
142 - # We must shallow-copy seq_group_metadata as is_prompt could change.
143 - seq_group_metadata = copy.copy(old_seq_group_metadata)
144 - new_seq_group_metadata_list.append(seq_group_metadata)
145 -
146 - # We must shallow-copy seq_data as we will append token ids
147 - new_seq_data: Dict[int, SequenceData] = {}
148 - for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
149 - new_seq_data[seq_id] = copy.copy(old_seq_data)
150 - new_seq_data[
151 - seq_id].output_token_ids = old_seq_data.output_token_ids[:]
239 + # We must shallow-copy seq_data as we will append token ids
240 + new_seq_data: Dict[int, SequenceData] = {}
241 + for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
242 + new_seq_data[seq_id] = copy.copy(old_seq_data)
243 + new_seq_data[seq_id].output_token_ids =\
244 + old_seq_data.output_token_ids[:]
245
153 - seq_group_metadata.seq_data = new_seq_data
246 + new_seq_group_metadata.seq_data = new_seq_data
247 + return new_seq_group_metadata
248
155 - return new_seq_group_metadata_list
249 + @staticmethod
250 + def _copy_seq_metadata_excluding_last_token(
251 + seq_group_metadata: SequenceGroupMetadata,
252 + seq_ids_to_copy: Set[int],
253 + ) -> SequenceGroupMetadata:
254 + """
255 + Creates a shallow copy of the given SequenceGroupMetadata, retaining
256 + only the sequence IDs specified in seq_ids_to_copy. For each of these
257 + sequence IDs, all output_token_ids except the last one are copied.
258 + Sequence IDs not in seq_ids_to_copy are excluded from the copy.
259 +
260 + Parameters:
261 + seq_group_metadata (SequenceGroupMetadata): The original sequence
262 + group metadata.
263 + seq_ids_to_copy (Set[int]): The set of sequence IDs to include in the
264 + copy.
265 +
266 + Returns:
267 + SequenceGroupMetadata: A shallow copy of the sequence group metadata
268 + with the specified modifications.
269 + """
270 + # Shallow-copy the SequenceGroupMetadata.
271 + new_seq_group_metadata = copy.copy(seq_group_metadata)
272 + # Shallow-copy seq_data and modify the output_token_ids.
273 + new_seq_data: Dict[int, SequenceData] = {}
274 + for seq_id, old_seq_data in seq_group_metadata.seq_data.items():
275 + if (seq_id in seq_ids_to_copy):
276 + new_seq_data[seq_id] = copy.copy(old_seq_data)
277 + # Copy all the output token ids except the last.
278 + # Also reduce num_computed_tokens by 1 since we are not
279 + # including the last output token.
280 + # NOTE: num_computed_tokens is not directly used by the
281 + # speculative decoding workers, as it is only relevant for
282 + # chunked prefill, which is disabled for speculative decoding.
283 + # However, to maintain consistency in num_computed_tokens,
284 + # we update it here.
285 + new_seq_data[seq_id].output_token_ids =\
286 + old_seq_data.output_token_ids[:-1]
287 + new_seq_data[seq_id].update_num_computed_tokens(-1)
288 + new_seq_group_metadata.seq_data = new_seq_data
289 + return new_seq_group_metadata
290
291 def _assert_enough_kv_space(
292 self, seq_group_metadata_list: List[SequenceGroupMetadata],
vllm/spec_decode/ngram_worker.py
+9 -3
@@ -1,5 +1,5 @@
1 import weakref
2 -from typing import List, Optional, Tuple
2 +from typing import List, Optional, Set, Tuple
3
4 import torch
5
@@ -48,6 +48,9 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
48 self,
49 execute_model_req: ExecuteModelRequest,
50 sample_len: int,
51 + # Unused parameter. NGramWorker does not use the KV Cache and
52 + # therefore does not need this parameter.
53 + seq_ids_with_bonus_token_in_last_step: Set[int],
54 ) -> Tuple[Optional[List[Optional[SamplerOutput]]], bool]:
55 """NGram match algo to pick proposal candidate. Returns the list of
56 sampler output, one per SequenceGroupMetadata.
@@ -133,12 +136,15 @@ class NGramWorker(NonLLMProposerWorkerBase, LoraNotSupportedWorkerBase):
136 def get_spec_proposals(
137 self,
138 execute_model_req: ExecuteModelRequest,
139 + # Unused parameter. NGramWorker does not use the KV Cache and
140 + # therefore does not need this parameter.
141 + seq_ids_with_bonus_token_in_last_step: Set[int],
142 ) -> SpeculativeProposals:
143 """Produce speculations given an input batch of sequences. The number of
144 speculative tokens per sequence is determined by max_proposal_len.
145 """
140 -
141 - return self._proposer.get_spec_proposals(execute_model_req)
146 + return self._proposer.get_spec_proposals(
147 + execute_model_req, seq_ids_with_bonus_token_in_last_step)
148
149 def _raise_if_unsupported(
150 self,
vllm/spec_decode/proposer_worker_base.py
+8 -1
@@ -1,5 +1,5 @@
1 from abc import ABC, abstractmethod
2 -from typing import List, Optional, Tuple
2 +from typing import List, Optional, Set, Tuple
3
4 from vllm.sequence import ExecuteModelRequest, SamplerOutput
5 from vllm.spec_decode.interfaces import SpeculativeProposer
@@ -14,6 +14,13 @@ class ProposerWorkerBase(LoraNotSupportedWorkerBase, SpeculativeProposer):
14 self,
15 execute_model_req: ExecuteModelRequest,
16 sample_len: int,
17 + # A set containing all sequence IDs that were assigned bonus tokens
18 + # in their last forward pass. This set is used to backfill the KV cache
19 + # with the key-value pairs of the penultimate token in the sequences.
20 + # This parameter is only used by the MultiStepWorker, which relies on
21 + # the KV cache for token generation. It is not used by workers that
22 + # do not utilize the KV cache.
23 + seq_ids_with_bonus_token_in_last_step: Set[int]
24 ) -> Tuple[Optional[List[SamplerOutput]], bool]:
25 raise NotImplementedError
26
vllm/spec_decode/smaller_tp_proposer_worker.py
+8 -3
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Set, Tuple
2
3 import torch
4
@@ -110,13 +110,17 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
110 self,
111 execute_model_req: ExecuteModelRequest,
112 sample_len: int,
113 + seq_ids_with_bonus_token_in_last_step: Set[int],
114 ) -> Tuple[List[SamplerOutput], bool]:
115 # Do not check _is_dummy, as it's always called by get_spec_proposals
115 - return self._worker.sampler_output(execute_model_req, sample_len)
116 + return self._worker.sampler_output(
117 + execute_model_req, sample_len,
118 + seq_ids_with_bonus_token_in_last_step)
119
120 def get_spec_proposals(
121 self,
122 execute_model_req: ExecuteModelRequest,
123 + seq_ids_with_bonus_token_in_last_step: Set[int],
124 ) -> SpeculativeProposals:
125 """Produce speculations given an input batch of sequences. The number of
126 speculative tokens per sequence is determined by max_proposal_len.
@@ -125,7 +129,8 @@ class SmallerTpProposerWorker(ProposerWorkerBase):
129 return SpeculativeProposals(None, None, None)
130
131 with self._patch_tensor_parallel_group():
128 - return self._worker.get_spec_proposals(execute_model_req)
132 + return self._worker.get_spec_proposals(
133 + execute_model_req, seq_ids_with_bonus_token_in_last_step)
134
135 def execute_model(
136 self,
vllm/spec_decode/spec_decode_worker.py
+52 -15
@@ -1,5 +1,6 @@
1 +from collections import defaultdict
2 from functools import cached_property
2 -from typing import Any, Dict, List, Optional, Tuple
3 +from typing import Any, Dict, List, Optional, Set, Tuple
4
5 import torch
6
@@ -13,7 +14,7 @@ from vllm.model_executor.layers.typical_acceptance_sampler import (
14 TypicalAcceptanceSampler)
15 from vllm.sequence import (CompletionSequenceGroupOutput, ExecuteModelRequest,
16 HiddenStates, SamplerOutput, SequenceGroupMetadata,
16 - get_all_seq_ids)
17 + get_all_seq_ids_and_request_ids)
18 from vllm.spec_decode.batch_expansion import BatchExpansionTop1Scorer
19 from vllm.spec_decode.draft_model_runner import TP1DraftModelRunner
20 from vllm.spec_decode.interfaces import (SpeculativeProposals,
@@ -112,11 +113,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
113 draft_worker_kwargs.pop("ngram_prompt_lookup_max"))
114 ngram_prompt_lookup_min = (
115 draft_worker_kwargs.pop("ngram_prompt_lookup_min"))
115 -
116 - disable_bonus_tokens = True
117 -
116 if ngram_prompt_lookup_max > 0:
119 - disable_bonus_tokens = False
117 proposer_worker = NGramWorker(**draft_worker_kwargs)
118 proposer_worker.set_ngram_window_size(ngram_prompt_lookup_min,
119 ngram_prompt_lookup_max)
@@ -128,11 +125,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
125
126 if draft_worker_kwargs[
127 "model_config"].hf_config.model_type == "mlp_speculator":
131 - disable_bonus_tokens = False
128 proposer_worker = MLPSpeculatorWorker(**draft_worker_kwargs)
129 elif draft_worker_kwargs[
130 "model_config"].hf_config.model_type == "medusa":
135 - disable_bonus_tokens = False
131 proposer_worker = MedusaWorker(**draft_worker_kwargs)
132 else:
133 if draft_tp == 1:
@@ -149,10 +144,10 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
144 spec_decode_sampler: SpecDecodeBaseSampler = None
145 if draft_token_acceptance_method == "rejection_sampler":
146 spec_decode_sampler = RejectionSampler(
152 - disable_bonus_tokens=disable_bonus_tokens, )
147 + disable_bonus_tokens=False, )
148 elif draft_token_acceptance_method == "typical_acceptance_sampler":
149 spec_decode_sampler = TypicalAcceptanceSampler(
155 - disable_bonus_tokens=disable_bonus_tokens,
150 + disable_bonus_tokens=False,
151 posterior_threshold=\
152 typical_acceptance_sampler_posterior_threshold,
153 posterior_alpha=typical_acceptance_sampler_posterior_alpha,
@@ -200,6 +195,15 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
195 self._metrics = AsyncMetricsCollector(
196 self.spec_decode_sampler
197 ) if metrics_collector is None else metrics_collector
198 + # Tracks the sequence IDs that received a bonus token ID in
199 + # their last forward pass. Needed only if KV cache is being
200 + # used for token generation such as in the case of MultiStepWorker.
201 + self._seq_with_bonus_token_in_last_step: Set[int] = set()
202 + # Tracks the currently active request ids and the sequence IDs
203 + # corresponding to them
204 + self._request_id_seq_id_mapping: Dict[str, Set[int]] = defaultdict(set)
205 + # Tracks if the proposer worker uses the KV cache or not.
206 +
207 self.probs_dtype = self.spec_decode_sampler.probs_dtype
208 self.token_id_dtype = self.spec_decode_sampler.token_id_dtype
209 # Lazy initiazliation.
@@ -307,6 +311,7 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
311 broadcast_tensor_dict({}, src=0)
312 return []
313
314 + self._track_finished_requests(execute_model_req)
315 disable_all_speculation = self._should_disable_all_speculation(
316 execute_model_req)
317 num_lookahead_slots = execute_model_req.num_lookahead_slots
@@ -453,7 +458,8 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
458 self.previous_hidden_states = None
459
460 # Generate proposals using draft worker.
456 - proposals = self.proposer_worker.get_spec_proposals(execute_model_req)
461 + proposals = self.proposer_worker.get_spec_proposals(
462 + execute_model_req, self._seq_with_bonus_token_in_last_step)
463
464 proposal_scores = self.scorer.score_proposals(
465 execute_model_req,
@@ -585,7 +591,9 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
591
592 # Get the sequence ids and num_logprobs (sampling parameter) in the
593 # batch.
588 - seq_ids = get_all_seq_ids(seq_group_metadata_list)
594 + seq_ids, request_ids_seq_ids_mapping = get_all_seq_ids_and_request_ids(
595 + seq_group_metadata_list)
596 +
597 num_logprobs_per_seq = get_all_num_logprobs(seq_group_metadata_list)
598
599 # Serialize all tensors to CPU Python lists.
@@ -608,7 +616,6 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
616 for sequence_index in range(batch_size):
617 # Each sequence may have a different num_logprobs; retrieve it.
618 num_logprobs = num_logprobs_per_seq[sequence_index]
611 -
619 step_output_token_ids.append(
620 create_sequence_group_output(
621 token_id=accepted_token_ids_by_step[step_index]
@@ -623,18 +630,48 @@ class SpecDecodeWorker(LoraNotSupportedWorkerBase):
630 topk_logprobs=topk_logprobs_by_step[step_index]
631 [sequence_index][:num_logprobs],
632 ))
626 -
633 sampler_output_list.append(
634 SamplerOutput(outputs=step_output_token_ids))
635
636 + # Populate the data structures needed to keep track of sequences with
637 + # bonus tokens.
638 + self._track_sequences_with_bonus_tokens(seq_ids,
639 + request_ids_seq_ids_mapping,
640 + accepted_token_ids_by_step)
641 maybe_rejsample_metrics = (
642 self._metrics.maybe_collect_rejsample_metrics(k))
643 if maybe_rejsample_metrics is not None:
644 sampler_output_list[
645 0].spec_decode_worker_metrics = maybe_rejsample_metrics
635 -
646 return sampler_output_list
647
648 + def _track_finished_requests(self, execute_model_req: ExecuteModelRequest):
649 + """
650 + Removes the finished requests and their associated sequence ids from
651 + internal book keeping data structures.
652 + """
653 + for finished_request in execute_model_req.finished_requests_ids:
654 + for seq_id in self._request_id_seq_id_mapping[finished_request]:
655 + self._seq_with_bonus_token_in_last_step.discard(seq_id)
656 + del self._request_id_seq_id_mapping[finished_request]
657 +
658 + def _track_sequences_with_bonus_tokens(
659 + self, seq_ids: List[int],
660 + request_ids_seq_ids_mapping: Dict[str, Set[int]],
661 + accepted_token_ids_by_step: List[List[int]]):
662 + """
663 + Updates the internal data structures which keep track of sequences
664 + which have been assigned bonus tokens in their last forward pass.
665 + """
666 + for seq_index, seq_id in enumerate(seq_ids):
667 + last_token_id = accepted_token_ids_by_step[-1][seq_index]
668 + if last_token_id == -1:
669 + self._seq_with_bonus_token_in_last_step.discard(seq_id)
670 + else:
671 + self._seq_with_bonus_token_in_last_step.add(seq_id)
672 + for request_id, sequences in request_ids_seq_ids_mapping.items():
673 + self._request_id_seq_id_mapping[request_id].update(sequences)
674 +
675 @cached_property
676 def _vocab_size(self) -> int:
677 """Get the vocab size of the model and make sure it's consistent between
vllm/spec_decode/top1_proposer.py
+4 -1
@@ -1,4 +1,4 @@
1 -from typing import List, Optional, Tuple
1 +from typing import List, Optional, Set, Tuple
2
3 import torch
4
@@ -42,6 +42,7 @@ class Top1Proposer(SpeculativeProposer):
42 def get_spec_proposals(
43 self,
44 execute_model_req: ExecuteModelRequest,
45 + seq_ids_with_bonus_token_in_last_step: Set[int],
46 ) -> SpeculativeProposals:
47 """Get speculative proposals given the input batch.
48
@@ -76,6 +77,8 @@ class Top1Proposer(SpeculativeProposer):
77 maybe_sampler_output, transposed = self._worker.sampler_output(
78 execute_model_req=nonzero_execute_model_req,
79 sample_len=proposal_len,
80 + seq_ids_with_bonus_token_in_last_step=\
81 + seq_ids_with_bonus_token_in_last_step,
82 )
83 (
84 proposal_lens,